bootloader: reduce stack usage

On Sparc Openboot, we get allocated a stack of only 8 kilobytes, and
each called function costs at least 176 bytes for the stack frame.

This means we need to be more careful than usual about stack usage. Move
some large-ish allocations off the stack by either making them static,
or allocated dynamically.

Add a compiler flag to error on functions which use too much stack. The
threshold is at 1023 bytes, because that's what allowed me to find the
two functions that were causing a stack overflow (open_from and
_ParseActivatedPackagesFile)

Change-Id: Ia0d13a9247e1a3fff4ce654bdffd6edb16e7cbc7
Reviewed-on: https://review.haiku-os.org/c/haiku/+/2371
Reviewed-by: waddlesplash <[email protected]>

build/jam/ArchitectureRules

@@ -538,6 +538,9 @@ rule KernelArchitectureSetup architecture
 			case riscv :
 				HAIKU_BOOT_$(bootTarget:U)_CCFLAGS += -mcmodel=medany -fno-omit-frame-pointer -fno-plt -fno-pic -fno-semantic-interposition ;
 				HAIKU_BOOT_$(bootTarget:U)_C++FLAGS += -mcmodel=medany -fno-omit-frame-pointer -fno-plt -fno-pic -fno-semantic-interposition ;
+			case openfirmware :
+				HAIKU_BOOT_$(bootTarget:U)_CCFLAGS += -fno-pic -fno-semantic-interposition -Wno-error=main -Wstack-usage=1023 ;
+				HAIKU_BOOT_$(bootTarget:U)_C++FLAGS += -fno-pic -fno-semantic-interposition -Wno-error=main -Wstack-usage=1023 ;
 			case * :
 				# all other bootloaders are non-PIC
 				HAIKU_BOOT_$(bootTarget:U)_CCFLAGS += -fno-pic -Wno-error=main ;

src/system/boot/loader/file_systems/tarfs/tarfs.cpp

@@ -741,10 +741,14 @@ status_t
 TarFS::Volume::_Inflate(boot::Partition* partition, void* cookie, off_t offset,
 	RegionDeleter& regionDeleter, size_t* inflatedBytes)
 {
-	char in[2048];
+	static const int kBufferSize = 2048;
+	char* in = (char*)malloc(kBufferSize);
+	if (in == NULL)
+		return B_NO_MEMORY;
+	MemoryDeleter deleter(in);
 	z_stream zStream = {
 		(Bytef*)in,		// next in
-		sizeof(in),		// avail in
+		kBufferSize,	// avail in
 		0,				// total in
 		NULL,			// next out
 		0,				// avail out
@@ -764,7 +768,7 @@ TarFS::Volume::_Inflate(boot::Partition* partition, void* cookie, off_t offset,
 	bool headerRead = false;
 
 	do {
-		ssize_t bytesRead = partition->ReadAt(cookie, offset, in, sizeof(in));
+		ssize_t bytesRead = partition->ReadAt(cookie, offset, in, kBufferSize);
 		if (bytesRead != (ssize_t)sizeof(in)) {
 			if (bytesRead <= 0) {
 				status = Z_STREAM_ERROR;

src/system/boot/loader/package_support.cpp

@@ -274,16 +274,23 @@ PackageVolumeInfo::_InitState(Directory* packagesDirectory, DIR* dir,
 	PackageVolumeState* state)
 {
 	// find the system package
-	char systemPackageName[B_FILE_NAME_LENGTH];
+	char* systemPackageName = (char*)malloc(B_FILE_NAME_LENGTH);
+	if (systemPackageName == NULL)
+		return B_NO_MEMORY;
+	char* packagePath = (char*)malloc(B_PATH_NAME_LENGTH);
+	if (packagePath == NULL) {
+		free(systemPackageName);
+		return B_NO_MEMORY;
+	}
+
 	status_t error = _ParseActivatedPackagesFile(packagesDirectory, state,
-		systemPackageName, sizeof(systemPackageName));
+		systemPackageName, B_FILE_NAME_LENGTH);
 	if (error == B_OK) {
 		// check, if package exists
 		for (PackageVolumeState* otherState = state; otherState != NULL;
 				otherState = fStates.GetPrevious(otherState)) {
-			char packagePath[B_PATH_NAME_LENGTH];
 			otherState->GetPackagePath(systemPackageName, packagePath,
-				sizeof(packagePath));
+				B_PATH_NAME_LENGTH);
 			struct stat st;
 			if (get_stat(packagesDirectory, packagePath, st) == B_OK
 				&& S_ISREG(st.st_mode)) {
@@ -310,6 +317,8 @@ PackageVolumeInfo::_InitState(Directory* packagesDirectory, DIR* dir,
 		}
 	}
 
+	free(packagePath);
+	free(systemPackageName);
 	if (state->SystemPackage() == NULL)
 		return B_ENTRY_NOT_FOUND;
 
@@ -322,28 +331,39 @@ PackageVolumeInfo::_ParseActivatedPackagesFile(Directory* packagesDirectory,
 	PackageVolumeState* state, char* packageName, size_t packageNameSize)
 {
 	// open the activated-packages file
-	char path[3 * B_FILE_NAME_LENGTH + 2];
-	snprintf(path, sizeof(path), "%s/%s/%s",
+	static const size_t kBufferSize = 3 * B_FILE_NAME_LENGTH + 2;
+	char* path = (char*)malloc(kBufferSize);
+	if (path == NULL)
+		return B_NO_MEMORY;
+	snprintf(path, kBufferSize, "%s/%s/%s",
 		kAdministrativeDirectory, state->Name() != NULL ? state->Name() : "",
 		kActivatedPackagesFile);
 	int fd = open_from(packagesDirectory, path, O_RDONLY);
-	if (fd < 0)
+	if (fd < 0) {
+		free(path);
 		return fd;
+	}
 	FileDescriptorCloser fdCloser(fd);
 
 	struct stat st;
-	if (fstat(fd, &st) != 0)
+	if (fstat(fd, &st) != 0) {
+		free(path);
 		return errno;
-	if (!S_ISREG(st.st_mode))
+	}
+	if (!S_ISREG(st.st_mode)) {
+		free(path);
 		return B_ENTRY_NOT_FOUND;
+	}
 
 	// read the file until we find the system package line
 	size_t remainingBytes = 0;
 	for (;;) {
 		ssize_t bytesRead = read(fd, path + remainingBytes,
-			sizeof(path) - remainingBytes - 1);
-		if (bytesRead <= 0)
+			kBufferSize - remainingBytes - 1);
+		if (bytesRead <= 0) {
+			free(path);
 			return B_ENTRY_NOT_FOUND;
+		}
 
 		remainingBytes += bytesRead;
 		path[remainingBytes] = '\0';
@@ -352,9 +372,11 @@ PackageVolumeInfo::_ParseActivatedPackagesFile(Directory* packagesDirectory,
 		while (char* lineEnd = strchr(line, '\n')) {
 			*lineEnd = '\0';
 			if (is_system_package(line)) {
-				return strlcpy(packageName, line, packageNameSize)
+				status_t result = strlcpy(packageName, line, packageNameSize)
 						< packageNameSize
 					?  B_OK : B_NAME_TOO_LONG;
+				free(path);
+				return result;
 			}
 
 			line = lineEnd + 1;
@@ -369,5 +391,6 @@ PackageVolumeInfo::_ParseActivatedPackagesFile(Directory* packagesDirectory,
 			remainingBytes = 0;
 	}
 
+	free(path);
 	return B_ENTRY_NOT_FOUND;
 }

src/system/boot/loader/vfs.cpp

@@ -250,20 +250,30 @@ Directory::Lookup(const char* name, bool traverseLinks)
 		return node;
 
 	// the node is a symbolic link, so we have to resolve the path
-	char linkPath[B_PATH_NAME_LENGTH];
-	status_t error = node->ReadLink(linkPath, sizeof(linkPath));
+	char* linkPath = (char*)malloc(B_PATH_NAME_LENGTH);
+	if (linkPath == NULL) {
+		node->Release();
+		return NULL;
+	}
+
+	status_t error = node->ReadLink(linkPath, B_PATH_NAME_LENGTH);
 
 	node->Release();
 		// we don't need this one anymore
 
-	if (error != B_OK)
+	if (error != B_OK) {
+		free(linkPath);
 		return NULL;
+	}
 
 	// let open_from() do the real work
 	int fd = open_from(this, linkPath, O_RDONLY);
-	if (fd < 0)
+	if (fd < 0) {
+		free(linkPath);
 		return NULL;
+	}
 
+	free(linkPath);
 	node = get_node_from(fd);
 	if (node != NULL)
 		node->Acquire();
@@ -1039,34 +1049,48 @@ open_from(Directory *directory, const char *name, int mode, mode_t permissions)
 		name++;
 	}
 
-	char path[B_PATH_NAME_LENGTH];
-	if (strlcpy(path, name, sizeof(path)) >= sizeof(path))
+	char* path = (char*)malloc(B_PATH_NAME_LENGTH);
+	if (path == NULL)
+		return B_NO_MEMORY;
+
+	if (strlcpy(path, name, B_PATH_NAME_LENGTH) >= B_PATH_NAME_LENGTH) {
+		free(path);
 		return B_NAME_TOO_LONG;
+	}
 
 	Node *node;
 	status_t error = get_node_for_path(directory, path, &node);
 	if (error != B_OK) {
-		if (error != B_ENTRY_NOT_FOUND)
+		if (error != B_ENTRY_NOT_FOUND) {
+			free(path);
 			return error;
+		}
 
-		if ((mode & O_CREAT) == 0)
+		if ((mode & O_CREAT) == 0) {
+			free(path);
 			return B_ENTRY_NOT_FOUND;
+		}
 
 		// try to resolve the parent directory
-		strlcpy(path, name, sizeof(path));
+		strlcpy(path, name, B_PATH_NAME_LENGTH);
 		if (char* lastSlash = strrchr(path, '/')) {
-			if (lastSlash[1] == '\0')
+			if (lastSlash[1] == '\0') {
+				free(path);
 				return B_ENTRY_NOT_FOUND;
+			}
 
 			*lastSlash = '\0';
 			name = lastSlash + 1;
 
 			// resolve the directory
-			if (get_node_for_path(directory, path, &node) != B_OK)
+			if (get_node_for_path(directory, path, &node) != B_OK) {
+				free(path);
 				return B_ENTRY_NOT_FOUND;
+			}
 
 			if (node->Type() != S_IFDIR) {
 				node->Release();
+				free(path);
 				return B_NOT_A_DIRECTORY;
 			}
 
@@ -1078,16 +1102,20 @@ open_from(Directory *directory, const char *name, int mode, mode_t permissions)
 		error = directory->CreateFile(name, permissions, &node);
 		directory->Release();
 
-		if (error != B_OK)
+		if (error != B_OK) {
+			free(path);
 			return error;
+		}
 	} else if ((mode & O_EXCL) != 0) {
 		node->Release();
+		free(path);
 		return B_FILE_EXISTS;
 	}
 
 	int fd = open_node(node, mode);
 
 	node->Release();
+	free(path);
 	return fd;
 }
 

src/system/boot/platform/generic/text_menu.cpp

@@ -159,7 +159,9 @@ print_item_at(int32 line, MenuItem *item, bool clearHelp = true)
 		if (length > width * 2)
 			width += 2 * kOffsetX - 1;
 
-		char buffer[width + 1];
+		char* buffer = (char*)malloc(width + 1);
+		if (buffer == NULL)
+			return;
 		buffer[width] = '\0';
 			// make sure the buffer is always terminated
 
@@ -195,6 +197,8 @@ print_item_at(int32 line, MenuItem *item, bool clearHelp = true)
 			print_centered(console_height() - kHelpLines + row, buffer);
 			row++;
 		}
+
+		free(buffer);
 	}
 }
 

src/system/boot/platform/openfirmware/arch/sparc/mmu.cpp

@@ -100,7 +100,7 @@ find_physical_memory_ranges(size_t &total)
 		return B_ERROR;
 	}
 
-	struct of_region<uint64, uint64> regions[64];
+	static struct of_region<uint64, uint64> regions[64];
 	int count = of_getprop(package, "reg", regions, sizeof(regions));
 	if (count == OF_FAILED)
 		count = of_getprop(sMemoryInstance, "reg", regions, sizeof(regions));
@@ -185,7 +185,7 @@ find_allocated_ranges(void **_exceptionHandlers)
 	// we have proper driver support for the target hardware).
 	intptr_t mmu = of_instance_to_package(sMmuInstance);
 
-	struct translation_map {
+	static struct translation_map {
 		void *PhysicalAddress() {
 			int64_t p = data;
 #if 0

src/system/boot/platform/openfirmware/start.cpp

@@ -100,7 +100,7 @@ platform_boot_options(void)
 extern "C" void
 start(void *openFirmwareEntry)
 {
-	char bootargs[512];
+	static char bootargs[512];
 
 	// stage2 args - might be set via the command line one day
 	stage2_args args;