From a959262cd0318f46265a279f3e5057329616cec8 Mon Sep 17 00:00:00 2001 From: Adrien Destugues Date: Sat, 14 Dec 2019 19:24:38 +0800 Subject: [PATCH] implement mlock(), munlock() Change-Id: I2f04b8986d2ed32bb4d30d238d668e21a1505778 Reviewed-on: https://review.haiku-os.org/c/haiku/+/1991 Reviewed-by: Adrien Destugues --- headers/posix/sys/mman.h | 3 + headers/private/kernel/thread_types.h | 7 + headers/private/kernel/vm/vm.h | 3 + headers/private/system/syscalls.h | 3 + src/system/kernel/vm/vm.cpp | 259 ++++++++++++++++++++++++- src/system/libroot/posix/sys/mman.cpp | 14 ++ src/tests/system/kernel/mlock_test.cpp | 36 ++++ 7 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 src/tests/system/kernel/mlock_test.cpp diff --git a/headers/posix/sys/mman.h b/headers/posix/sys/mman.h index 8661ad6c9f..9f345d9678 100644 --- a/headers/posix/sys/mman.h +++ b/headers/posix/sys/mman.h @@ -61,6 +61,9 @@ int msync(void* address, size_t length, int flags); int madvise(void* address, size_t length, int advice); int posix_madvise(void* address, size_t length, int advice); +int mlock(const void* address, size_t length); +int munlock(const void* address, size_t length); + int shm_open(const char* name, int openMode, mode_t permissions); int shm_unlink(const char* name); diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 2fa6b0090e..926baafcf1 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -26,6 +26,8 @@ #include #include +#include + enum additional_thread_state { THREAD_STATE_FREE_ON_RESCHED = 7, // free the thread structure upon reschedule @@ -64,6 +66,9 @@ struct select_info; struct user_thread; // defined in libroot/user_thread.h struct VMAddressSpace; struct xsi_sem_context; // defined in xsi_semaphore.cpp +struct LockedPages; + +typedef DoublyLinkedList LockedPagesList; namespace Scheduler { struct ThreadData; @@ -237,6 +242,8 @@ struct Team : TeamThreadIteratorEntry, KernelReferenceable, struct team_death_entry *death_entry; // protected by fLock struct list dead_threads; + LockedPagesList locked_pages_list; + // protected by the team's fLock team_dead_children dead_children; team_job_control_children stopped_children; diff --git a/headers/private/kernel/vm/vm.h b/headers/private/kernel/vm/vm.h index 715a75d39d..15e5618f7a 100644 --- a/headers/private/kernel/vm/vm.h +++ b/headers/private/kernel/vm/vm.h @@ -175,6 +175,9 @@ status_t _user_memory_advice(void* address, size_t size, uint32 advice); status_t _user_get_memory_properties(team_id teamID, const void *address, uint32 *_protected, uint32 *_lock); +status_t _user_mlock(const void* address, size_t size); +status_t _user_munlock(const void* address, size_t size); + area_id _user_area_for(void *address); area_id _user_find_area(const char *name); status_t _user_get_area_info(area_id area, area_info *info); diff --git a/headers/private/system/syscalls.h b/headers/private/system/syscalls.h index 125a6e3716..11e2e200ac 100644 --- a/headers/private/system/syscalls.h +++ b/headers/private/system/syscalls.h @@ -448,6 +448,9 @@ extern status_t _kern_memory_advice(void *address, size_t size, extern status_t _kern_get_memory_properties(team_id teamID, const void *address, uint32* _protected, uint32* _lock); +extern status_t _kern_mlock(const void* address, size_t size); +extern status_t _kern_munlock(const void* address, size_t size); + /* kernel port functions */ extern port_id _kern_create_port(int32 queue_length, const char *name); extern status_t _kern_close_port(port_id id); diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index d9f6c999f4..849e56a98c 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -5103,7 +5103,7 @@ fill_area_info(struct VMArea* area, area_info* info, size_t size) info->address = (void*)area->Base(); info->size = area->Size(); info->protection = area->protection; - info->lock = B_FULL_LOCK; + info->lock = area->wiring; info->team = area->address_space->ID(); info->copy_count = 0; info->in_count = 0; @@ -6882,6 +6882,263 @@ _user_get_memory_properties(team_id teamID, const void* address, } +// An ordered list of non-overlapping ranges to track mlock/munlock locking. +// It is allowed to call mlock/munlock in unbalanced ways (lock a range +// multiple times, unlock a part of it, lock several consecutive ranges and +// unlock them in one go, etc). However the low level lock_memory and +// unlock_memory calls require the locks/unlocks to be balanced (you lock a +// fixed range, and then unlock exactly the same range). This list allows to +// keep track of what was locked exactly so we can unlock the correct things. +struct LockedPages : DoublyLinkedListLinkImpl { + addr_t start; + addr_t end; + + status_t LockMemory() + { + return lock_memory((void*)start, end - start, 0); + } + + status_t UnlockMemory() + { + return unlock_memory((void*)start, end - start, 0); + } + + status_t Move(addr_t start, addr_t end) + { + status_t result = lock_memory((void*)start, end - start, 0); + if (result != B_OK) + return result; + + result = UnlockMemory(); + + if (result != B_OK) { + // What can we do if the unlock fails? + panic("Failed to unlock memory: %s", strerror(result)); + return result; + } + + this->start = start; + this->end = end; + + return B_OK; + } +}; + + +status_t +_user_mlock(const void* address, size_t size) { + // Maybe there's nothing to do, in which case, do nothing + if (size == 0) + return B_OK; + + // Make sure the address is multiple of B_PAGE_SIZE (POSIX allows us to + // reject the call otherwise) + if ((addr_t)address % B_PAGE_SIZE != 0) + return EINVAL; + + size = ROUNDUP(size, B_PAGE_SIZE); + + addr_t endAddress = (addr_t)address + size; + + // Pre-allocate a linked list element we may need (it's simpler to do it + // now than run out of memory in the midle of changing things) + LockedPages* newRange = new(std::nothrow) LockedPages(); + if (newRange == NULL) + return ENOMEM; + + // Get and lock the team + Team* team = thread_get_current_thread()->team; + TeamLocker teamLocker(team); + teamLocker.Lock(); + + status_t error = B_OK; + LockedPagesList* lockedPages = &team->locked_pages_list; + + // Locate the first locked range possibly overlapping ours + LockedPages* currentRange = lockedPages->Head(); + while (currentRange != NULL && currentRange->end <= (addr_t)address) + currentRange = lockedPages->GetNext(currentRange); + + if (currentRange == NULL || currentRange->start >= endAddress) { + // No existing range is overlapping with ours. We can just lock our + // range and stop here. + newRange->start = (addr_t)address; + newRange->end = endAddress; + error = newRange->LockMemory(); + if (error != B_OK) { + delete newRange; + return error; + } + lockedPages->InsertBefore(currentRange, newRange); + return B_OK; + } + + // We get here when there is at least one existing overlapping range. + + if (currentRange->start <= (addr_t)address) { + if (currentRange->end >= endAddress) { + // An existing range is already fully covering the pages we need to + // lock. Nothing to do then. + delete newRange; + return B_OK; + } else { + // An existing range covers the start of the area we want to lock. + // Advance our start address to avoid it. + address = (void*)currentRange->end; + + // Move on to the next range for the next step + currentRange = lockedPages->GetNext(currentRange); + } + } + + // First, lock the new range + newRange->start = (addr_t)address; + newRange->end = endAddress; + error = newRange->LockMemory(); + if (error != B_OK) { + delete newRange; + return error; + } + + // Unlock all ranges fully overlapping with the area we need to lock + while (currentRange != NULL && currentRange->end < endAddress) { + // The existing range is fully contained inside the new one we're + // trying to lock. Delete/unlock it, and replace it with a new one + // (this limits fragmentation of the range list, and is simpler to + // manage) + error = currentRange->UnlockMemory(); + if (error != B_OK) { + panic("Failed to unlock a memory range: %s", strerror(error)); + newRange->UnlockMemory(); + delete newRange; + return error; + } + LockedPages* temp = currentRange; + currentRange = lockedPages->GetNext(currentRange); + lockedPages->Remove(temp); + delete temp; + } + + if (currentRange != NULL) { + // One last range may cover the end of the area we're trying to lock + + if (currentRange->start == (addr_t)address) { + // In case two overlapping ranges (one at the start and the other + // at the end) already cover the area we're after, there's nothing + // more to do. So we destroy our new extra allocation + error = newRange->UnlockMemory(); + delete newRange; + return error; + } + + if (currentRange->start < endAddress) { + // Make sure the last range is not overlapping, by moving its start + error = currentRange->Move(endAddress, currentRange->end); + if (error != B_OK) { + panic("Failed to move a memory range: %s", strerror(error)); + newRange->UnlockMemory(); + delete newRange; + return error; + } + } + } + + // Finally, store the new range in the locked list + lockedPages->InsertBefore(currentRange, newRange); + return B_OK; +} + + +status_t +_user_munlock(const void* address, size_t size) { + // Maybe there's nothing to do, in which case, do nothing + if (size == 0) + return B_OK; + + // Make sure the address is multiple of B_PAGE_SIZE (POSIX allows us to + // reject the call otherwise) + if ((addr_t)address % B_PAGE_SIZE != 0) + return EINVAL; + + // Round size up to the next page + size = ROUNDUP(size, B_PAGE_SIZE); + + addr_t endAddress = (addr_t)address + size; + + // Get and lock the team + Team* team = thread_get_current_thread()->team; + TeamLocker teamLocker(team); + teamLocker.Lock(); + LockedPagesList* lockedPages = &team->locked_pages_list; + + status_t error = B_OK; + + // Locate the first locked range possibly overlapping ours + LockedPages* currentRange = lockedPages->Head(); + while (currentRange != NULL && currentRange->end <= (addr_t)address) + currentRange = lockedPages->GetNext(currentRange); + + if (currentRange == NULL || currentRange->start >= endAddress) { + // No range is intersecting, nothing to unlock + return B_OK; + } + + if (currentRange->start < (addr_t)address) { + if (currentRange->end > endAddress) { + // There is a range fully covering the area we want to unlock, + // and it extends on both sides. We need to split it in two + LockedPages* newRange = new(std::nothrow) LockedPages(); + if (newRange == NULL) + return ENOMEM; + + newRange->start = endAddress; + newRange->end = currentRange->end; + + error = newRange->LockMemory(); + if (error != B_OK) { + delete newRange; + return error; + } + + error = currentRange->Move(currentRange->start, (addr_t)address); + if (error != B_OK) { + delete newRange; + return error; + } + + lockedPages->InsertAfter(currentRange, newRange); + return B_OK; + } else { + // There is a range that overlaps and extends before the one we + // want to unlock, we need to shrink it + error = currentRange->Move(currentRange->start, (addr_t)address); + if (error != B_OK) + return error; + } + } + + while (currentRange != NULL && currentRange->end <= endAddress) { + // Unlock all fully overlapping ranges + error = currentRange->UnlockMemory(); + if (error != B_OK) + return error; + LockedPages* temp = currentRange; + currentRange = lockedPages->GetNext(currentRange); + lockedPages->Remove(temp); + delete temp; + } + + // Finally split the last partially overlapping range if any + if (currentRange != NULL && currentRange->start < endAddress) { + error = currentRange->Move(endAddress, currentRange->end); + if (error != B_OK) + return error; + } + + return B_OK; +} + + // #pragma mark -- compatibility diff --git a/src/system/libroot/posix/sys/mman.cpp b/src/system/libroot/posix/sys/mman.cpp index bb00ab00ef..0c5eb2fba4 100644 --- a/src/system/libroot/posix/sys/mman.cpp +++ b/src/system/libroot/posix/sys/mman.cpp @@ -194,6 +194,20 @@ posix_madvise(void* address, size_t length, int advice) } +int +mlock(const void* address, size_t length) +{ + RETURN_AND_SET_ERRNO(_kern_mlock(address, length)); +} + + +int +munlock(const void* address, size_t length) +{ + RETURN_AND_SET_ERRNO(_kern_munlock(address, length)); +} + + int shm_open(const char* name, int openMode, mode_t permissions) { diff --git a/src/tests/system/kernel/mlock_test.cpp b/src/tests/system/kernel/mlock_test.cpp new file mode 100644 index 0000000000..221c830bc2 --- /dev/null +++ b/src/tests/system/kernel/mlock_test.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include + +#define SPACE_SIZE (B_PAGE_SIZE * 9) + +int +main() +{ + void* space = NULL; + area_id area = create_area("mlock test area", &space, B_EXACT_ADDRESS, + SPACE_SIZE, B_NO_LOCK, B_READ_AREA | B_WRITE_AREA); + + /* --------- */ + int result = mlock(space + B_PAGE_SIZE, B_PAGE_SIZE * 7); + /* -xxxxxxx- */ + assert(result == 0); + result = munlock(space + 2 * B_PAGE_SIZE, B_PAGE_SIZE * 5); + /* -x-----x- */ + assert(result == 0); + result = mlock(space + 2 * B_PAGE_SIZE, B_PAGE_SIZE * 3); + /* -xxxx--x- */ + assert(result == 0); + result = mlock(space, B_PAGE_SIZE * 9); + /* xxxxxxxxx */ + assert(result == 0); + result = munlock(space + 4 * B_PAGE_SIZE, B_PAGE_SIZE * 5); + /* xxxx----- */ + assert(result == 0); + result = munlock(space, B_PAGE_SIZE * 9); + assert(result == 0); + + delete_area(area); + return 0; +}