libroot: make all areas executable for old binaries

* If at least one image is either B_HAIKU_ABI_GCC_2_ANCIENT or
   B_HAIKU_ABI_GCC_2_BEOS almost all areas are marked as executable.
 * B_EXECUTE_AREA and B_STACK_AREA are made public. The former is enforced since
   the introduction of DEP and apps need it to correctly set area protection.
   The latter is currently needed only to recognize stack areas and fix their
   protection in compatibility mode, but may also be useful if an app wants
   to use sigaltstack from POSIX API.

headers/os/kernel/OS.h

@@ -85,6 +85,11 @@ typedef struct area_info {
 /* area protection */
 #define B_READ_AREA				1
 #define B_WRITE_AREA			2
+#define B_EXECUTE_AREA			4
+#define B_STACK_AREA			8
+	// "stack" protection is not available on most platforms - it's used
+	// to only commit memory as needed, and have guard pages at the
+	// bottom of the stack.
 
 extern area_id		create_area(const char *name, void **startAddress,
 						uint32 addressSpec, size_t size, uint32 lock,

headers/private/libroot/libroot_private.h

@@ -18,6 +18,8 @@ struct real_time_data;
 extern "C" {
 #endif
 
+extern int __gCompatibilityMode;
+
 extern char _single_threaded;
 	/* This determines if a process runs single threaded or not */
 
@@ -31,7 +33,7 @@ status_t __flatten_process_args(const char* const* args, int32 argCount,
 			size_t* _flatSize);
 void _call_atexit_hooks_for_range(addr_t start, addr_t size);
 void __init_env(const struct user_space_program_args *args);
-void __init_heap(void);
+status_t __init_heap(void);
 void __init_heap_post_env(void);
 
 void __init_time(addr_t commPageTable);
@@ -42,6 +44,8 @@ void __init_pwd_backend(void);
 void __reinit_pwd_backend_after_fork(void);
 void* __arch_get_caller(void);
 
+void __set_stack_protection(void);
+
 
 #ifdef __cplusplus
 }

headers/private/runtime_loader/runtime_loader.h

@@ -52,6 +52,7 @@ struct rld_export {
 
 	const struct user_space_program_args *program_args;
 	const void* commpage_address;
+	int abi_version;
 };
 
 extern struct rld_export *__gRuntimeLoader;

headers/private/system/vm_defs.h

@@ -15,13 +15,6 @@
 // Note: the VM probably won't support all combinations - it will try
 // its best, but create_area() will fail if it has to.
 // Of course, the exact behaviour will be documented somewhere...
-#define B_EXECUTE_AREA			0x04
-#define B_STACK_AREA			0x08
-	// "stack" protection is not available on most platforms - it's used
-	// to only commit memory as needed, and have guard pages at the
-	// bottom of the stack.
-	// "execute" protection is currently ignored, but nevertheless, you
-	// should use it if you require to execute code in that area.
 
 #define B_KERNEL_EXECUTE_AREA	0x40
 #define B_KERNEL_STACK_AREA		0x80

src/system/libroot/libroot_init.c

@@ -7,6 +7,7 @@
 #include <pthread.h>
 #include <string.h>
 
+#include <OS.h>
 #include <image.h>
 
 #include <user_runtime.h>
@@ -30,6 +31,8 @@ char *__progname = NULL;
 int __libc_argc;
 char **__libc_argv;
 
+int __gCompatibilityMode;
+
 char _single_threaded = true;
 	// determines if I/O locking needed; needed for BeOS compatibility
 
@@ -47,6 +50,8 @@ initialize_before(image_id imageID)
 {
 	char *programPath = __gRuntimeLoader->program_args->args[0];
 	__gCommPageAddress = __gRuntimeLoader->commpage_address;
+	__gCompatibilityMode
+		= __gRuntimeLoader->abi_version < B_HAIKU_ABI_GCC_2_HAIKU;
 
 	if (programPath) {
 		if ((__progname = strrchr(programPath, '/')) == NULL)
@@ -71,6 +76,7 @@ initialize_before(image_id imageID)
 	__init_env(__gRuntimeLoader->program_args);
 	__init_heap_post_env();
 	__init_pwd_backend();
+	__set_stack_protection();
 }
 
 

src/system/libroot/os/area.c

@@ -5,6 +5,9 @@
 
 
 #include <OS.h>
+
+#include <libroot_private.h>
+
 #include "syscalls.h"
 
 
@@ -12,6 +15,8 @@ area_id
 create_area(const char *name, void **address, uint32 addressSpec, size_t size,
 	uint32 lock, uint32 protection)
 {
+	if (__gCompatibilityMode == 1)
+		protection |= B_EXECUTE_AREA;
 	return _kern_create_area(name, address, addressSpec, size, lock, protection);
 }
 
@@ -20,6 +25,8 @@ area_id
 clone_area(const char *name, void **address, uint32 addressSpec,
 	uint32 protection, area_id sourceArea)
 {
+	if (__gCompatibilityMode == 1)
+		protection |= B_EXECUTE_AREA;
 	return _kern_clone_area(name, address, addressSpec, protection, sourceArea);
 }
 
@@ -55,6 +62,8 @@ resize_area(area_id id, size_t newSize)
 status_t
 set_area_protection(area_id id, uint32 protection)
 {
+	if (__gCompatibilityMode == 1)
+		protection |= B_EXECUTE_AREA;
 	return _kern_set_area_protection(id, protection);
 }
 

src/system/libroot/os/thread.c

@@ -74,6 +74,23 @@ _thread_do_exit_work(void)
 }
 
 
+void
+__set_stack_protection(void)
+{
+	if (__gCompatibilityMode == 1) {
+		area_info info;
+		ssize_t cookie = 0;
+
+		while (get_next_area_info(B_CURRENT_TEAM, &cookie, &info) == B_OK) {
+			if ((info.protection & B_STACK_AREA) != 0) {
+				_kern_set_area_protection(info.area,
+					B_READ_AREA | B_WRITE_AREA | B_EXECUTE_AREA | B_STACK_AREA);
+			}
+		}
+	}
+}
+
+
 // #pragma mark -
 
 
@@ -99,8 +116,11 @@ spawn_thread(thread_func entry, const char *name, int32 priority, void *data)
 	id = _kern_spawn_thread(&attributes);
 	if (id < 0)
 		free(thread);
-	else
+	else {
 		thread->id = id;
+		__set_stack_protection();
+	}
+
 	return id;
 }
 

src/system/libroot/posix/malloc/arch-specific.cpp

@@ -24,6 +24,8 @@
 #include <Debug.h>
 #include <syscalls.h>
 
+#include <libroot_private.h>
+
 #include <stdlib.h>
 #include <unistd.h>
 
@@ -103,9 +105,12 @@ __init_heap(void)
 	if (status != B_OK)
 		sHeapBase = NULL;
 
+	uint32 protection = B_READ_AREA | B_WRITE_AREA;
+	if (__gCompatibilityMode == 1)
+		protection |= B_EXECUTE_AREA;
 	sHeapArea = create_area("heap", (void **)&sHeapBase,
 		status == B_OK ? B_EXACT_ADDRESS : B_RANDOMIZED_BASE_ADDRESS,
-		kInitialHeapSize, B_NO_LOCK, B_READ_AREA | B_WRITE_AREA);
+		kInitialHeapSize, B_NO_LOCK, protection);
 	if (sHeapArea < B_OK)
 		return sHeapArea;
 
@@ -164,6 +169,11 @@ hoardSbrk(long size)
 	// align size request
 	size = (size + hoardHeap::ALIGNMENT - 1) & ~(hoardHeap::ALIGNMENT - 1);
 
+	// choose correct protection flags
+	uint32 protection = B_READ_AREA | B_WRITE_AREA;
+	if (__gCompatibilityMode == 1)
+		protection |= B_EXECUTE_AREA;
+
 	hoardLock(sHeapLock);
 
 	// find chunk in free list
@@ -259,7 +269,7 @@ hoardSbrk(long size)
 			&& (addr_t)base + newHeapSize
 				<= (addr_t)sHeapBase + kHeapReservationSize) {
 			area = create_area("heap", &base, B_EXACT_ADDRESS, newHeapSize,
-				B_NO_LOCK, B_READ_AREA | B_WRITE_AREA);
+				B_NO_LOCK, protection);
 
 			if (area == B_NO_MEMORY) {
 				hoardUnlock(sHeapLock);
@@ -272,7 +282,7 @@ hoardSbrk(long size)
 		if (area < 0) {
 			base = (void*)(sFreeHeapBase + sHeapAreaSize);
 			area = create_area("heap", &base, B_RANDOMIZED_BASE_ADDRESS,
-				newHeapSize, B_NO_LOCK, B_READ_AREA | B_WRITE_AREA);
+				newHeapSize, B_NO_LOCK, protection);
 		}
 
 		if (area < 0) {

src/system/libroot/posix/pthread/pthread.cpp

@@ -15,6 +15,7 @@
 
 #include <syscall_utils.h>
 
+#include <libroot_private.h>
 #include <syscalls.h>
 #include <thread_defs.h>
 #include <tls.h>
@@ -158,6 +159,7 @@ pthread_create(pthread_t* _thread, const pthread_attr_t* attr,
 		return EAGAIN;
 	}
 
+	__set_stack_protection();
 	resume_thread(thread->id);
 	*_thread = thread;
 

src/system/runtime_loader/elf_load_image.cpp

@@ -536,6 +536,8 @@ load_image(char const* name, image_type type, const char* rpath,
 		#endif
 	}
 
+	set_abi_version(image->abi);
+
 	// init gcc version dependent image flags
 	// symbol resolution strategy
 	if (image->abi == B_HAIKU_ABI_GCC_2_ANCIENT)

src/system/runtime_loader/export.cpp

@@ -57,7 +57,12 @@ struct rld_export gRuntimeLoader = {
 
 	elf_reinit_after_fork,
 	NULL, // call_atexit_hooks_for_range
-	terminate_program
+	terminate_program,
+
+	// the following values will be set later
+	NULL,	// program_args
+	NULL,	// commpage_address
+	0		// ABI version
 };
 
 
@@ -67,3 +72,13 @@ rldexport_init(void)
 	gRuntimeLoader.program_args = gProgramArgs;
 	gRuntimeLoader.commpage_address = __gCommPageAddress;
 }
+
+
+void
+set_abi_version(int abi_version)
+{
+	if (gRuntimeLoader.abi_version == 0
+		|| gRuntimeLoader.abi_version > abi_version) {
+		gRuntimeLoader.abi_version = abi_version;
+	}
+}

src/system/runtime_loader/images.cpp

@@ -417,8 +417,10 @@ unmap_image(image_t* image)
 
 
 /*!	This function will change the protection of all read-only segments to really
-	be read-only.
+	be read-only (and executable).
 	The areas have to be read/write first, so that they can be relocated.
+	If at least one image is in compatibility mode then we allow execution of
+	all areas.
 */
 void
 remap_images()
@@ -426,14 +428,22 @@ remap_images()
 	for (image_t* image = sLoadedImages.head; image != NULL;
 			image = image->next) {
 		for (uint32 i = 0; i < image->num_regions; i++) {
-			if ((image->regions[i].flags & RFLAG_RW) == 0
+			// we only need to do this once, so we remember those we've already
-				&& (image->regions[i].flags & RFLAG_REMAPPED) == 0) {
+			// mapped
-				// we only need to do this once, so we remember those we've already mapped
+			if ((image->regions[i].flags & RFLAG_REMAPPED) != 0)
-				if (_kern_set_area_protection(image->regions[i].id,
+				continue;
-						B_READ_AREA | B_EXECUTE_AREA) == B_OK) {
+
-					image->regions[i].flags |= RFLAG_REMAPPED;
+			status_t result = B_OK;
-				}
+			if ((image->regions[i].flags & RFLAG_RW) == 0) {
+				result = _kern_set_area_protection(image->regions[i].id,
+						B_READ_AREA | B_EXECUTE_AREA);
+			} else if (image->abi < B_HAIKU_ABI_GCC_2_HAIKU) {
+				result = _kern_set_area_protection(image->regions[i].id,
+						B_READ_AREA | B_WRITE_AREA | B_EXECUTE_AREA);
 			}
+
+			if (result == B_OK)
+				image->regions[i].flags |= RFLAG_REMAPPED;
 		}
 	}
 }

src/system/runtime_loader/runtime_loader_private.h

@@ -81,6 +81,7 @@ int resolve_symbol(image_t* rootImage, image_t* image, elf_sym* sym,
 status_t elf_verify_header(void* header, size_t length);
 void rldelf_init(void);
 void rldexport_init(void);
+void set_abi_version(int abi_version);
 status_t elf_reinit_after_fork(void);
 
 status_t heap_init(void);