Handle 64-bit load addresses for ELF64 images in the bootloader.

The ELF loader now uses a new platform function, platform_allocate_elf_region,
which returns 2 addresses: the real load address and an address where the
region is mapped in the loader's address space. All of the ELF loading code
has been changed to access the load region through the mapped address rather
than the addresses contained in the ELF image. The ELF64 version of
platform_allocate_elf_region on x86 uses the existing MMU code, which maps
everything at 0x80000000, but returns the correct 64-bit address. The long
mode switch code will just set up the 64-bit address space with everything
remapped at the correct address.

headers/private/kernel/boot/elf.h

@@ -1,7 +1,7 @@
 /*
  * Copyright 2003-2004, Axel Dörfler, [email protected]. All rights reserved.
  * Copyright 2012, Alex Smith, [email protected].
- * Distributed under the terms of the OpenBeOS License.
+ * Distributed under the terms of the MIT License.
  */
 #ifndef KERNEL_BOOT_ELF_H
 #define KERNEL_BOOT_ELF_H
@@ -44,9 +44,10 @@ struct preloaded_image {
 } _PACKED;
 
 struct preloaded_elf32_image : public preloaded_image {
-	Elf32_Ehdr elf_header;
+	Elf32_Ehdr	elf_header;
 	elf32_region text_region;
 	elf32_region data_region;
+	uint32		mapped_delta;
 
 	FixedWidthPointer<Elf32_Sym> syms;
 	FixedWidthPointer<Elf32_Rel> rel;
@@ -61,9 +62,10 @@ struct preloaded_elf32_image : public preloaded_image {
 } _PACKED;
 
 struct preloaded_elf64_image : public preloaded_image {
-	Elf64_Ehdr elf_header;
+	Elf64_Ehdr	elf_header;
 	elf64_region text_region;
 	elf64_region data_region;
+	uint64		mapped_delta;
 
 	FixedWidthPointer<Elf64_Sym> syms;
 	FixedWidthPointer<Elf64_Rel> rel;
@@ -77,11 +79,23 @@ struct preloaded_elf64_image : public preloaded_image {
 	FixedWidthPointer<Elf64_Sym> debug_symbols;
 } _PACKED;
 
-
+#ifdef _BOOT_MODE
 extern status_t boot_elf_resolve_symbol(preloaded_elf32_image* image,
 	struct Elf32_Sym* symbol, Elf32_Addr* symbolAddress);
 extern status_t boot_elf_resolve_symbol(preloaded_elf64_image* image,
 	struct Elf64_Sym* symbol, Elf64_Addr* symbolAddress);
 
 
+// Helper method to set a relocation at the mapped address in the loader's
+// address space.
+template<typename ImageType, typename AddrType>
+inline void
+boot_elf_set_relocation(ImageType* image, AddrType resolveAddress,
+	AddrType finalAddress)
+{
+	AddrType* dest = (AddrType*)(addr_t)(resolveAddress + image->mapped_delta);
+	*dest = finalAddress;
+}
+#endif
+
 #endif	/* KERNEL_BOOT_ELF_H */

headers/private/kernel/boot/platform.h

@@ -72,6 +72,13 @@ extern size_t platform_get_user_input_text(Menu *menu, MenuItem *item,
 	char *buffer, size_t bufferSize);
 extern char* platform_debug_get_log_buffer(size_t* _size);
 
+/* ELF functions */
+
+extern status_t platform_allocate_elf_region(uint32 *_address, uint32 size,
+	uint8 protection, void **_mappedAddress);
+extern status_t platform_allocate_elf_region(uint64 *_address, uint64 size,
+	uint8 protection, void **_mappedAddress);
+
 #endif
 
 #endif	/* KERNEL_BOOT_PLATFORM_H */

src/system/boot/arch/x86_64/Jamfile

@@ -2,6 +2,10 @@ SubDir HAIKU_TOP src system boot arch x86_64 ;
 
 DEFINES += _BOOT_MODE ;
 
+local bootArchSources =
+	arch_elf64.cpp
+;
+
 local kernelArchSources =
 	arch_elf.cpp
 	cpuid.S
@@ -16,18 +20,18 @@ local librootOsArchSources =
 ;
 
 BootMergeObject boot_arch_$(TARGET_ARCH).o :
-	arch_elf64.cpp
+	$(bootArchSources)
 	$(kernelArchSources)
 	$(kernelLibArchSources)
 	$(librootOsArchSources)
 	: # additional flags
 ;
 
+SEARCH on [ FGristFiles $(bootArchSources) ]
+    = [ FDirName $(HAIKU_TOP) src system boot arch x86 ] ;
 SEARCH on [ FGristFiles $(kernelArchSources) ]
     = [ FDirName $(HAIKU_TOP) src system kernel arch x86 ] ;
 SEARCH on [ FGristFiles $(kernelLibArchSources) ]
     = [ FDirName $(HAIKU_TOP) src system kernel lib arch x86 ] ;
 SEARCH on [ FGristFiles $(librootOsArchSources) ]
     = [ FDirName $(HAIKU_TOP) src system libroot os arch x86 ] ;
-SEARCH on [ FGristFiles $(kernelArchSources) ]
-    = [ FDirName $(HAIKU_TOP) src system boot arch x86 ] ;

src/system/boot/loader/elf.cpp

@@ -38,6 +38,7 @@ class ELFLoader {
 private:
 	typedef typename Class::ImageType	ImageType;
 	typedef typename Class::RegionType	RegionType;
+	typedef typename Class::AddrType	AddrType;
 	typedef typename Class::EhdrType	EhdrType;
 	typedef typename Class::PhdrType	PhdrType;
 	typedef typename Class::ShdrType	ShdrType;
@@ -53,7 +54,7 @@ public:
 
 private:
 	static	status_t	_LoadSymbolTable(int fd, ImageType* image);
-	static	status_t	_ParseDynamicSection(ImageType* image);
+	static	status_t	_ParseDynamicSection(ImageType* image, AddrType delta);
 };
 
 
@@ -62,6 +63,7 @@ struct ELF32Class {
 
 	typedef preloaded_elf32_image	ImageType;
 	typedef elf32_region			RegionType;
+	typedef Elf32_Addr				AddrType;
 	typedef Elf32_Ehdr				EhdrType;
 	typedef Elf32_Phdr				PhdrType;
 	typedef Elf32_Shdr				ShdrType;
@@ -80,6 +82,7 @@ struct ELF64Class {
 
 	typedef preloaded_elf64_image	ImageType;
 	typedef elf64_region			RegionType;
+	typedef Elf64_Addr				AddrType;
 	typedef Elf64_Ehdr				EhdrType;
 	typedef Elf64_Phdr				PhdrType;
 	typedef Elf64_Shdr				ShdrType;
@@ -132,6 +135,7 @@ ELFLoader<Class>::Load(int fd, preloaded_image* _image)
 	size_t totalSize;
 	ssize_t length;
 	status_t status;
+	void* mappedRegion = NULL;
 
 	ImageType* image = static_cast<ImageType*>(_image);
 	EhdrType& elfHeader = image->elf_header;
@@ -196,8 +200,9 @@ ELFLoader<Class>::Load(int fd, preloaded_image* _image)
 			B_PAGE_SIZE);
 		region->delta = -region->start;
 
-		TRACE(("segment %ld: start = 0x%lx, size = %lu, delta = %lx\n", i,
-			region->start, region->size, region->delta));
+		TRACE(("segment %ld: start = 0x%llx, size = %llu, delta = %llx\n", i,
+			(uint64)region->start, (uint64)region->size,
+			(int64)(AddrType)region->delta));
 	}
 
 	// found both, text and data?
@@ -228,10 +233,10 @@ ELFLoader<Class>::Load(int fd, preloaded_image* _image)
 	}
 
 	// The kernel and the modules are relocatable, thus
-	// platform_allocate_region() can automatically allocate an address,
+	// platform_allocate_elf_region() can automatically allocate an address,
 	// but shall prefer the specified base address.
-	if (platform_allocate_region((void **)&firstRegion->start, totalSize,
-			B_READ_AREA | B_WRITE_AREA, false) < B_OK) {
+	if (platform_allocate_elf_region(&firstRegion->start, totalSize,
+			B_READ_AREA | B_WRITE_AREA, &mappedRegion) < B_OK) {
 		status = B_NO_MEMORY;
 		goto error1;
 	}
@@ -242,6 +247,16 @@ ELFLoader<Class>::Load(int fd, preloaded_image* _image)
 	image->data_region.delta += image->data_region.start;
 	image->text_region.delta += image->text_region.start;
 
+	TRACE(("text: start 0x%llx, size 0x%llx, delta 0x%llx\n",
+		(uint64)image->text_region.start, (uint64)image->text_region.size,
+		(int64)(AddrType)image->text_region.delta));
+	TRACE(("data: start 0x%llx, size 0x%llx, delta 0x%llx\n",
+		(uint64)image->data_region.start, (uint64)image->data_region.size,
+		(int64)(AddrType)image->data_region.delta));
+
+	// Calculate the delta from a real load address to the mapped address
+	image->mapped_delta = (AddrType)(addr_t)mappedRegion - firstRegion->start;
+
 	// load program data
 
 	for (int32 i = 0; i < elfHeader.e_phnum; i++) {
@@ -258,10 +273,14 @@ ELFLoader<Class>::Load(int fd, preloaded_image* _image)
 		else
 			continue;
 
-		TRACE(("load segment %d (%ld bytes)...\n", i, header.p_filesz));
+		// Calculate where to load the data to.
+		addr_t dest = region->start + image->mapped_delta;
+
+		TRACE(("load segment %ld (%llu bytes) mapped at 0x%lx...\n", i,
+			(uint64)header.p_filesz, dest));
 
 		length = read_pos(fd, header.p_offset,
-			(void*)(region->start + (header.p_vaddr % B_PAGE_SIZE)),
+			(void*)(dest + (header.p_vaddr % B_PAGE_SIZE)),
 			header.p_filesz);
 		if (length < (ssize_t)header.p_filesz) {
 			status = B_BAD_DATA;
@@ -274,7 +293,7 @@ ELFLoader<Class>::Load(int fd, preloaded_image* _image)
 
 		uint32 offset = (header.p_vaddr % B_PAGE_SIZE) + header.p_filesz;
 		if (offset < region->size)
-			memset((void*)(region->start + offset), 0, region->size - offset);
+			memset((void*)(dest + offset), 0, region->size - offset);
 	}
 
 	// offset dynamic section, and program entry addresses by the delta of the
@@ -294,8 +313,8 @@ ELFLoader<Class>::Load(int fd, preloaded_image* _image)
 	return B_OK;
 
 error2:
-	if (image->text_region.start != 0)
-		platform_free_region((void*)image->text_region.start, totalSize);
+	if (mappedRegion != NULL)
+		platform_free_region(mappedRegion, totalSize);
 error1:
 	free(programHeaders);
 	kernel_args_free(image);
@@ -310,13 +329,16 @@ ELFLoader<Class>::Relocate(preloaded_image* _image)
 {
 	ImageType* image = static_cast<ImageType*>(_image);
 
-	status_t status = _ParseDynamicSection(image);
+	// Pull information out of the dynamic section. First pass through we set
+	// the addresses we want in our address space.
+	status_t status = _ParseDynamicSection(image, image->mapped_delta);
 	if (status != B_OK)
 		return status;
 
 	// deal with the rels first
 	if (image->rel) {
-		TRACE(("total %i relocs\n", image->rel_len / (int)sizeof(RelType)));
+		TRACE(("total %i relocs\n",
+			(int)image->rel_len / (int)sizeof(RelType)));
 
 		status = boot_arch_elf_relocate_rel(image, image->rel, image->rel_len);
 		if (status < B_OK)
@@ -324,15 +346,18 @@ ELFLoader<Class>::Relocate(preloaded_image* _image)
 	}
 
 	if (image->pltrel) {
-		TRACE(("total %i plt-relocs\n",
-			image->pltrel_len / (int)sizeof(RelType)));
-
 		RelType* pltrel = image->pltrel;
 		if (image->pltrel_type == DT_REL) {
+			TRACE(("total %i plt-relocs\n",
+				(int)image->pltrel_len / (int)sizeof(RelType)));
+
 			status = boot_arch_elf_relocate_rel(image, pltrel,
 				image->pltrel_len);
 		} else {
-			status = boot_arch_elf_relocate_rela(image, (RelaType *)pltrel,
+			TRACE(("total %i plt-relocs\n",
+				(int)image->pltrel_len / (int)sizeof(RelaType)));
+
+			status = boot_arch_elf_relocate_rela(image, (RelaType*)pltrel,
 				image->pltrel_len);
 		}
 		if (status < B_OK)
@@ -341,13 +366,17 @@ ELFLoader<Class>::Relocate(preloaded_image* _image)
 
 	if (image->rela) {
 		TRACE(("total %i rela relocs\n",
-			image->rela_len / (int)sizeof(RelaType)));
+			(int)image->rela_len / (int)sizeof(RelaType)));
 		status = boot_arch_elf_relocate_rela(image, image->rela,
 			image->rela_len);
 		if (status < B_OK)
 			return status;
 	}
 
+	// Make a second pass through the dynamic section, storing the correct
+	// virtual addresses for the kernel.
+	_ParseDynamicSection(image, 0);
+
 	return B_OK;
 }
 
@@ -458,7 +487,7 @@ error1:
 
 template<typename Class>
 /*static*/ status_t
-ELFLoader<Class>::_ParseDynamicSection(ImageType* image)
+ELFLoader<Class>::_ParseDynamicSection(ImageType* image, AddrType delta)
 {
 	image->syms = 0;
 	image->rel = 0;
@@ -469,10 +498,12 @@ ELFLoader<Class>::_ParseDynamicSection(ImageType* image)
 	image->pltrel_len = 0;
 	image->pltrel_type = 0;
 
-	DynType* d = (DynType*)image->dynamic_section.start;
-	if (!d)
+	if(image->dynamic_section.start == 0)
 		return B_ERROR;
 
+	DynType* d = (DynType*)(addr_t)(image->dynamic_section.start
+		+ image->mapped_delta);
+
 	for (int i = 0; d[i].d_tag != DT_NULL; i++) {
 		switch (d[i].d_tag) {
 			case DT_HASH:
@@ -480,25 +511,25 @@ ELFLoader<Class>::_ParseDynamicSection(ImageType* image)
 				break;
 			case DT_SYMTAB:
 				image->syms = (SymType*)(d[i].d_un.d_ptr
-					+ image->text_region.delta);
+					+ image->text_region.delta + delta);
 				break;
 			case DT_REL:
 				image->rel = (RelType*)(d[i].d_un.d_ptr
-					+ image->text_region.delta);
+					+ image->text_region.delta + delta);
 				break;
 			case DT_RELSZ:
 				image->rel_len = d[i].d_un.d_val;
 				break;
 			case DT_RELA:
 				image->rela = (RelaType*)(d[i].d_un.d_ptr
-					+ image->text_region.delta);
+					+ image->text_region.delta + delta);
 				break;
 			case DT_RELASZ:
 				image->rela_len = d[i].d_un.d_val;
 				break;
 			case DT_JMPREL:
 				image->pltrel = (RelType*)(d[i].d_un.d_ptr
-					+ image->text_region.delta);
+					+ image->text_region.delta  + delta);
 				break;
 			case DT_PLTRELSZ:
 				image->pltrel_len = d[i].d_un.d_val;

src/system/boot/platform/amiga_m68k/mmu.cpp

@@ -662,6 +662,20 @@ platform_free_region(void *address, size_t size)
 }
 
 
+status_t
+platform_allocate_elf_region(uint32 *_address, uint32 size, uint8 protection,
+	void **_mappedAddress)
+{
+	void *address = mmu_allocate((void *)*_address, size);
+	if (address == NULL)
+		return B_NO_MEMORY;
+
+	*_address = (uint32)address;
+	*_mappedAddress = address;
+	return B_OK;
+}
+
+
 void
 platform_release_heap(struct stage2_args *args, void *base)
 {

src/system/boot/platform/atari_m68k/mmu.cpp

@@ -666,6 +666,20 @@ platform_free_region(void *address, size_t size)
 }
 
 
+status_t
+platform_allocate_elf_region(uint32 *_address, uint32 size, uint8 protection,
+	void **_mappedAddress)
+{
+	void *address = mmu_allocate((void *)*_address, size);
+	if (address == NULL)
+		return B_NO_MEMORY;
+
+	*_address = (uint32)address;
+	*_mappedAddress = address;
+	return B_OK;
+}
+
+
 void
 platform_release_heap(struct stage2_args *args, void *base)
 {

src/system/boot/platform/bios_ia32/mmu.cpp

@@ -769,6 +769,43 @@ platform_free_region(void *address, size_t size)
 }
 
 
+status_t
+platform_allocate_elf_region(uint32 *_address, uint32 size, uint8 protection,
+	void **_mappedAddress)
+{
+	void *address = mmu_allocate((void *)*_address, size);
+	if (address == NULL)
+		return B_NO_MEMORY;
+
+	*_address = (uint32)address;
+	*_mappedAddress = address;
+	return B_OK;
+}
+
+
+status_t
+platform_allocate_elf_region(uint64 *_address, uint64 size, uint8 protection,
+	void **_mappedAddress)
+{
+	// The 64-bit kernel is loaded to 0xFFFFFFFF80000000. You'll notice that
+	// the low 32 bits of this address are the same as the 32-bit KERNEL_BASE
+	// (0x80000000). Therefore, the way this function is implemented is to use
+	// mmu_allocate() and then set the upper 32 bits to all 1s. The long mode
+	// switch code will remap everything to the correct addresses.
+
+	void *address = mmu_allocate((void *)(addr_t)(*_address & 0xFFFFFFFF), size);
+	if (address == NULL)
+		return B_NO_MEMORY;
+
+	// This is the address that the ELF loading code will access the mapping
+	// through.
+	*_mappedAddress = address;
+
+	*_address = (uint64)(uint32)address | 0xFFFFFFFF00000000LL;
+	return B_OK;
+}
+
+
 void
 platform_release_heap(struct stage2_args *args, void *base)
 {

src/system/boot/platform/cfe/mmu.cpp

@@ -34,3 +34,21 @@ platform_free_region(void *address, size_t size)
 	return arch_mmu_free(address, size);
 }
 
+
+status_t
+platform_allocate_elf_region(uint32 *_address, uint32 size, uint8 protection,
+	void **_mappedAddress)
+{
+	if (size == 0)
+		return B_BAD_VALUE;
+
+	void *address = arch_mmu_allocate((void *)*_address, size, protection,
+		exactAddress);
+	if (address == NULL)
+		return B_NO_MEMORY;
+
+	*_address = (uint32)address;
+	*_mappedAddress = address;
+	return B_OK;
+}
+

src/system/boot/platform/openfirmware/mmu.cpp

@@ -34,3 +34,21 @@ platform_free_region(void *address, size_t size)
 	return arch_mmu_free(address, size);
 }
 
+
+status_t
+platform_allocate_elf_region(uint32 *_address, uint32 size, uint8 protection,
+	void **_mappedAddress)
+{
+	if (size == 0)
+		return B_BAD_VALUE;
+
+	void *address = arch_mmu_allocate((void *)*_address, size, protection,
+		exactAddress);
+	if (address == NULL)
+		return B_NO_MEMORY;
+
+	*_address = (uint32)address;
+	*_mappedAddress = address;
+	return B_OK;
+}
+

src/system/boot/platform/raspberrypi_arm/mmu.cpp

@@ -678,6 +678,20 @@ platform_free_region(void *address, size_t size)
 }
 
 
+status_t
+platform_allocate_elf_region(uint32 *_address, uint32 size, uint8 protection,
+	void **_mappedAddress)
+{
+	void *address = mmu_allocate((void *)*_address, size);
+	if (address == NULL)
+		return B_NO_MEMORY;
+
+	*_address = (uint32)address;
+	*_mappedAddress = address;
+	return B_OK;
+}
+
+
 void
 platform_release_heap(struct stage2_args *args, void *base)
 {

src/system/boot/platform/routerboard_mipsel/mmu.cpp

@@ -79,6 +79,15 @@ platform_free_region(void* address, size_t size)
 }
 
 
+status_t
+platform_allocate_elf_region(uint32 *_address, uint32 size, uint8 protection,
+	void **_mappedAddress)
+{
+#warning IMPLEMENT platform_allocate_elf_region
+	return B_ERROR;
+}
+
+
 void
 platform_release_heap(struct stage2_args* args, void* base)
 {

src/system/boot/platform/u-boot/mmu.cpp

@@ -681,6 +681,24 @@ platform_free_region(void *address, size_t size)
 }
 
 
+status_t
+platform_allocate_elf_region(uint32 *_address, uint32 size, uint8 protection,
+	void **_mappedAddress)
+{
+#ifdef __ARM__
+	void *address = mmu_allocate((void *)*_address, size);
+	if (address == NULL)
+		return B_NO_MEMORY;
+
+	*_address = (uint32)address;
+	*_mappedAddress = address;
+	return B_OK;
+#else
+	return B_ERROR;
+#endif
+}
+
+
 void
 platform_release_heap(struct stage2_args *args, void *base)
 {

src/system/kernel/arch/x86_64/arch_elf.cpp

@@ -136,31 +136,36 @@ boot_arch_elf_relocate_rela(preloaded_elf64_image* image, Elf64_Rela* rel,
 		}
 
 		// Address of the relocation.
-		Elf64_Addr* resolveAddr = (Elf64_Addr *)(image->text_region.delta
-			+ rel[i].r_offset);
+		Elf64_Addr relocAddr = image->text_region.delta + rel[i].r_offset;
 
-		// Perform the relocation.
+		// Calculate the relocation value.
+		Elf64_Addr relocValue;
 		switch(type) {
 			case R_X86_64_NONE:
-				break;
+				continue;
 			case R_X86_64_64:
-				*resolveAddr = symAddr + rel[i].r_addend;
+				relocValue = symAddr + rel[i].r_addend;
 				break;
 			case R_X86_64_PC32:
-				*resolveAddr = symAddr + rel[i].r_addend - rel[i].r_offset;
+				relocValue = symAddr + rel[i].r_addend - rel[i].r_offset;
 				break;
 			case R_X86_64_GLOB_DAT:
 			case R_X86_64_JUMP_SLOT:
-				*resolveAddr = symAddr + rel[i].r_addend;
+				relocValue = symAddr + rel[i].r_addend;
 				break;
 			case R_X86_64_RELATIVE:
-				*resolveAddr = image->text_region.delta + rel[i].r_addend;
+				relocValue = image->text_region.delta + rel[i].r_addend;
 				break;
 			default:
 				dprintf("arch_elf_relocate_rel: unhandled relocation type %d\n",
 					type);
 				return B_BAD_DATA;
 		}
+#ifdef _BOOT_MODE
+		boot_elf_set_relocation(image, relocAddr, relocValue);
+#else
+		*(Elf64_Addr *)relocAddr = relocValue;
+#endif
 	}
 
 	return B_OK;