vm: implement B_RANDOMIZED_IMAGE_ADDRESS address specification

On some 64 bit architectures program and library images have to be mapped in
the lower 2 GB of the address space (due to instruction pointer relative
addressing). Address specification B_RANDOMIZED_IMAGE_ADDRESS ensures that
created area satisfies that requirement.

headers/os/kernel/OS.h

@@ -81,6 +81,7 @@ typedef struct area_info {
 /* B_ANY_KERNEL_BLOCK_ADDRESS		5 */
 #define B_RANDOMIZED_ANY_ADDRESS	6
 #define B_RANDOMIZED_BASE_ADDRESS	7
+#define B_RANDOMIZED_IMAGE_ADDRESS	8
 
 /* area protection */
 #define B_READ_AREA				1

src/system/kernel/vm/VMUserAddressSpace.cpp

@@ -36,6 +36,7 @@ const addr_t VMUserAddressSpace::kMaxInitialRandomize	= 0x20000000000ul;
 const addr_t VMUserAddressSpace::kMaxRandomize			=  0x800000ul;
 const addr_t VMUserAddressSpace::kMaxInitialRandomize	= 0x2000000ul;
 #endif
+const addr_t VMUserAddressSpace::kImageEndAddress		= 0x7ffffffful;
 
 
 /*!	Verifies that an area with the given aligned base and size fits into
@@ -69,6 +70,15 @@ log2(uint32_t v)
 }
 
 
+static inline bool
+is_randomized(uint32 addressSpec)
+{
+	return addressSpec == B_RANDOMIZED_ANY_ADDRESS
+		|| addressSpec == B_RANDOMIZED_BASE_ADDRESS
+		|| addressSpec == B_RANDOMIZED_IMAGE_ADDRESS;
+}
+
+
 VMUserAddressSpace::VMUserAddressSpace(team_id id, addr_t base, size_t size)
 	:
 	VMAddressSpace(id, base, size, "address space"),
@@ -183,6 +193,11 @@ VMUserAddressSpace::InsertArea(VMArea* _area, size_t size,
 			searchEnd = fEndAddress;
 			break;
 
+		case B_RANDOMIZED_IMAGE_ADDRESS:
+			searchBase = (addr_t)addressRestrictions->address;
+			searchEnd = min_c(fEndAddress, kImageEndAddress);
+			break;
+
 		default:
 			return B_BAD_VALUE;
 	}
@@ -561,7 +576,9 @@ VMUserAddressSpace::_InsertAreaSlot(addr_t start, addr_t size, addr_t end,
 
 	start = ROUNDUP(start, alignment);
 
-	if (addressSpec == B_RANDOMIZED_BASE_ADDRESS) {
+	if (addressSpec == B_RANDOMIZED_BASE_ADDRESS
+		|| addressSpec == B_RANDOMIZED_IMAGE_ADDRESS) {
+
 		originalStart = start;
 		start = _RandomizeAddress(start, end - size, alignment, true);
 	}
@@ -594,7 +611,7 @@ second_chance:
 				addr_t nextBase = next == NULL ? end : next->Base();
 				if (is_valid_spot(start, alignedBase, size, nextBase)) {
 
-					if (addressSpec == B_RANDOMIZED_ANY_ADDRESS) {
+					if (is_randomized(addressSpec)) {
 						alignedBase = _RandomizeAddress(alignedBase,
 							nextBase - size, alignment);
 					}
@@ -615,7 +632,7 @@ second_chance:
 				if (is_valid_spot(last->Base() + (last->Size() - 1),
 						alignedBase, size, next->Base())) {
 
-					if (addressSpec == B_RANDOMIZED_ANY_ADDRESS) {
+					if (is_randomized(addressSpec)) {
 						alignedBase = _RandomizeAddress(alignedBase,
 							next->Base() - size, alignment);
 					}
@@ -637,7 +654,7 @@ second_chance:
 			if (is_valid_spot(last->Base() + (last->Size() - 1), alignedBase,
 					size, end)) {
 
-				if (addressSpec == B_RANDOMIZED_ANY_ADDRESS) {
+				if (is_randomized(addressSpec)) {
 					alignedBase = _RandomizeAddress(alignedBase, end - size,
 						alignment);
 				}
@@ -677,7 +694,7 @@ second_chance:
 						&&  alignedBase == next->Base()
 						&& next->Size() >= size) {
 
-						if (addressSpec == B_RANDOMIZED_ANY_ADDRESS) {
+						if (is_randomized(addressSpec)) {
 							alignedBase = _RandomizeAddress(next->Base(),
 								next->Size() - size, alignment);
 						}
@@ -699,7 +716,7 @@ second_chance:
 						// reserved area, and the reserved area will be resized
 						// to make space
 
-						if (addressSpec == B_RANDOMIZED_ANY_ADDRESS) {
+						if (is_randomized(addressSpec)) {
 							addr_t alignedNextBase = ROUNDUP(next->Base(),
 								alignment);
 
@@ -731,6 +748,7 @@ second_chance:
 
 		case B_BASE_ADDRESS:
 		case B_RANDOMIZED_BASE_ADDRESS:
+		case B_RANDOMIZED_IMAGE_ADDRESS:
 		{
 			// find a hole big enough for a new area beginning with "start"
 			if (last == NULL) {
@@ -765,7 +783,7 @@ second_chance:
 					area->SetBase(start);
 				else {
 					start = lastEnd + 1;
-					if (addressSpec == B_RANDOMIZED_BASE_ADDRESS) {
+					if (is_randomized(addressSpec)) {
 						addr_t spaceEnd = end;
 						if (next != NULL)
 							spaceEnd = next->Base();
@@ -781,7 +799,7 @@ second_chance:
 
 			// we didn't find a free spot in the requested range, so we'll
 			// try again without any restrictions
-			if (addressSpec != B_RANDOMIZED_BASE_ADDRESS) {
+			if (!is_randomized(addressSpec)) {
 				start = fBase;
 				addressSpec = B_ANY_ADDRESS;
 			} else if (originalStart == 0) {

src/system/kernel/vm/VMUserAddressSpace.h

@@ -67,6 +67,7 @@ private:
 private:
 	static	const addr_t		kMaxRandomize;
 	static	const addr_t		kMaxInitialRandomize;
+	static	const addr_t		kImageEndAddress;
 
 			VMUserAreaList		fAreas;
 	mutable	VMUserArea*			fAreaHint;

src/system/kernel/vm/vm.cpp

@@ -1226,6 +1226,7 @@ vm_create_anonymous_area(team_id team, const char *name, addr_t size,
 		case B_ANY_KERNEL_BLOCK_ADDRESS:
 		case B_RANDOMIZED_ANY_ADDRESS:
 		case B_RANDOMIZED_BASE_ADDRESS:
+		case B_RANDOMIZED_IMAGE_ADDRESS:
 			break;
 
 		default:

src/system/runtime_loader/images.cpp

@@ -173,7 +173,7 @@ get_image_region_load_address(image_t* image, uint32 index, int32 lastDelta,
 		if (index == 0) {
 			// but only the first segment gets a free ride
 			loadAddress = RLD_PROGRAM_BASE;
-			addressSpecifier = B_RANDOMIZED_BASE_ADDRESS;
+			addressSpecifier = B_RANDOMIZED_IMAGE_ADDRESS;
 		} else {
 			loadAddress = image->regions[index].vmstart + lastDelta;
 			addressSpecifier = B_EXACT_ADDRESS;
@@ -298,7 +298,7 @@ map_image(int fd, char const* path, image_t* image, bool fixed)
 	addr_t loadAddress;
 	size_t reservedSize = 0;
 	size_t length = 0;
-	uint32 addressSpecifier = B_RANDOMIZED_ANY_ADDRESS;
+	uint32 addressSpecifier = B_RANDOMIZED_IMAGE_ADDRESS;
 
 	for (uint32 i = 0; i < image->num_regions; i++) {
 		// for BeOS compatibility: if we load an old BeOS executable, we