runtime_loader: implement RTLD_GROUP

The RTLD_GROUP flag to dlopen comes from Solaris. It makes the symbol
resolution for dlopen not use the global symbols from the current team
(anything that's already loaded). The loaded object must be explicitly
linked against any symbol it needs to use (or it can use dlsym to search
symbols in the global scope explicitly).

This is also how symbol resolution worked in BeOS, meaning we already
have the code to do this, and just need to enable it.

This can be used in dosemu, where DOS-like executable are linked against
their own C library and should not use symbols from libroot.

Fixes #19674.

Change-Id: I8d127c7812a31e231edb1e44edf70b868c2670e7
Reviewed-on: https://review.haiku-os.org/c/haiku/+/9450
Reviewed-by: waddlesplash <[email protected]>

headers/posix/dlfcn.h

@@ -14,6 +14,7 @@
 #define RTLD_LOCAL	0	/* symbols are not available for relocating any other object */
 #define RTLD_GLOBAL	2	/* all symbols are available */
 #define RTLD_NOLOAD	4	/* do not load any new object */
+#define RTLD_GROUP	8	/* do not lookup symbols in the global symbol table */
 
 /* not-yet-POSIX extensions (dlsym() handles) */
 #define RTLD_DEFAULT	((void*)0)

src/system/runtime_loader/elf.cpp

@@ -677,6 +677,8 @@ load_library(char const *path, uint32 flags, bool addOn, void* caller,
 	if (image->find_undefined_symbol == NULL) {
 		if (addOn)
 			image->find_undefined_symbol = find_undefined_symbol_add_on;
+		else if (flags & RTLD_GROUP)
+			image->find_undefined_symbol = find_undefined_symbol_dependencies_only;
 		else
 			image->find_undefined_symbol = find_undefined_symbol_global;
 	}

src/system/runtime_loader/elf_load_image.cpp

@@ -657,7 +657,7 @@ load_image(char const* name, image_type type, const char* rpath, const char* run
 	// init gcc version dependent image flags
 	// symbol resolution strategy
 	if (image->abi == B_HAIKU_ABI_GCC_2_ANCIENT)
-		image->find_undefined_symbol = find_undefined_symbol_beos;
+		image->find_undefined_symbol = find_undefined_symbol_dependencies_only;
 
 	// init version infos
 	status = init_image_version_infos(image);

src/system/runtime_loader/elf_symbol_lookup.cpp

@@ -379,12 +379,18 @@ find_symbol_breadth_first(image_t* image, const SymbolLookupInfo& lookupInfo,
 
 
 elf_sym*
-find_undefined_symbol_beos(image_t* rootImage, image_t* image,
+find_undefined_symbol_dependencies_only(image_t* rootImage, image_t* image,
 	const SymbolLookupInfo& lookupInfo, image_t** foundInImage)
 {
 	// BeOS style symbol resolution: It is sufficient to check the image itself
 	// and its direct dependencies. The linker would have complained, if the
-	// symbol wasn't there. First we check whether the requesting symbol is
+	// symbol wasn't there.
+	//
+	// Also used for the RTLD_GROUP option in dlopen, which works similarly.
+	// Symbols must be defined by direct dependencies and existing symbols from
+	// the executable or previously loaded libraries cannot interfere.
+	//
+	// First we check whether the requesting symbol is
 	// defined already -- then we can simply return it, since, due to symbolic
 	// linking, that's the one we'd find anyway.
 	if (elf_sym* symbol = lookupInfo.requestingSymbol) {

src/system/runtime_loader/elf_symbol_lookup.h

@@ -127,7 +127,7 @@ status_t	find_symbol(image_t* image, const SymbolLookupInfo& lookupInfo,
 status_t	find_symbol_breadth_first(image_t* image,
 				const SymbolLookupInfo& lookupInfo, image_t** _foundInImage,
 				void** _location);
-elf_sym*	find_undefined_symbol_beos(image_t* rootImage, image_t* image,
+elf_sym*	find_undefined_symbol_dependencies_only(image_t* rootImage, image_t* image,
 				const SymbolLookupInfo& lookupInfo, image_t** foundInImage);
 elf_sym*	find_undefined_symbol_global(image_t* rootImage, image_t* image,
 				const SymbolLookupInfo& lookupInfo, image_t** foundInImage);