runtime_loader: Add support for RTLD_NOLOAD

Add support for using `RTLD_NOLOAD` with `dlopen` on Haiku.

While not specified in POSIX, this option is widely available (both
GNU/Linux and BSD) and is the only way to atomically check if a shared
object has been loaded.

Change-Id: I061332307d76c84f6a2abb6f92c4ed964bd90a1e
Reviewed-on: https://review.haiku-os.org/c/haiku/+/9161
Reviewed-by: waddlesplash <[email protected]>
Tested-by: Commit checker robot <[email protected]>
Reviewed-by: Jérôme Duval <[email protected]>

headers/posix/dlfcn.h

@@ -13,6 +13,7 @@
 #define RTLD_NOW	1	/* the file gets relocated at load time */
 #define RTLD_LOCAL	0	/* symbols are not available for relocating any other object */
 #define RTLD_GLOBAL	2	/* all symbols are available */
+#define RTLD_NOLOAD	4	/* do not load any new object */
 
 /* not-yet-POSIX extensions (dlsym() handles) */
 #define RTLD_DEFAULT	((void*)0)

src/system/runtime_loader/elf.cpp

@@ -651,6 +651,8 @@ load_library(char const *path, uint32 flags, bool addOn, void* caller,
 				path, image->id);
 			*_handle = image;
 			return image->id;
+		} else if ((flags & RTLD_NOLOAD) != 0) {
+			return B_NAME_NOT_FOUND;
 		}
 
 		// First of all, find the caller image.

src/tests/system/runtime_loader/test_suite/dlopen_noload1

@@ -0,0 +1,76 @@
+#!/bin/sh
+
+# program
+#
+# dlopen():
+# liba.so (no load)
+# liba.so
+# liba.so (no load)
+#
+# Expected: First attempt fails, next two attemps succeed,
+# returning the same handles.
+
+
+. ./test_setup
+
+
+# create liba.so
+cat > liba.c << EOI
+int a() { return 42; }
+EOI
+
+# build
+compile_lib -o liba.so liba.c
+
+
+# create program
+cat > program.c << EOI
+#include <dlfcn.h>
+#include <stdio.h>
+#include <stdlib.h>
+int
+main()
+{
+	void* liba_fail;
+	void* liba_1;
+	void* liba_2;
+	int (*a)();
+
+	liba_fail = dlopen("liba.so", RTLD_LOCAL | RTLD_LAZY | RTLD_NOLOAD);
+	if (liba_fail != NULL) {
+		fprintf(stderr, "dlopen() succeeded unexpectedly for non-loaded object");
+		exit(117);
+	}
+
+	liba_1 = dlopen("liba.so", RTLD_LOCAL | RTLD_LAZY);
+	if (liba_1 == NULL) {
+		fprintf(stderr, "Error opening liba.so: %s\n", dlerror());
+		exit(116);
+	}
+
+	liba_2 = dlopen("liba.so", RTLD_GLOBAL | RTLD_LAZY | RTLD_NOLOAD);
+	if (liba_2 == NULL) {
+		fprintf(stderr, "Error opening liba.so: %s\n", dlerror());
+		exit(115);
+	}
+
+	if (liba_1 != liba_2) {
+		fprintf(stderr, "dlopen() returned different handles for the same object");
+		exit(114);
+	}
+
+	a = (int (*)())dlsym(liba_2, "a");
+	if (a == NULL) {
+		fprintf(stderr, "Error getting symbol a: %s\n", dlerror());
+		exit(113);
+	}
+
+	return a();
+}
+EOI
+
+# build
+compile_program_dl -o program program.c
+
+# run
+test_run_ok ./program 42

src/tests/system/runtime_loader/test_suite/test_suite

@@ -7,6 +7,7 @@ for test in \
 	load_resolve_order2		\
 	load_resolve_order3		\
 	load_resolve_order4		\
+	dlopen_noload1			\
 	dlopen_resolve_basic1	\
 	dlopen_resolve_basic2	\
 	dlopen_resolve_basic3	\