Replaced the sReservedPages counter tracking the page reservations by

sUnreservedFreePages which tracks the difference between free/clear and
reserved pages. Access to it uses atomic operations which allows the three
page (un)reservation to avoid locking in most cases, thus reducing contention
of the "pages" lock.

In the -j8 Haiku image build that decreases the contention of the "pages"
lock to about one third of the previous value. As a positive side effect the
VMCache lock contention drops about the same factor. The total build speedup
is about 20%, the total kernel time drops about 20%.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@34888 a95241bf-73f2-0310-859d-f6bbb57e9c96

src/system/kernel/vm/vm_page.cpp

@@ -69,7 +69,7 @@ static VMPageQueue sActivePageQueue;
 static vm_page *sPages;
 static addr_t sPhysicalPageOffset;
 static size_t sNumPages;
-static size_t sReservedPages;
+static vint32 sUnreservedFreePages;
 static vint32 sPageDeficit;
 static size_t sModifiedTemporaryPages;
 
@@ -503,7 +503,7 @@ dump_page_stats(int argc, char **argv)
 	kprintf("wired: %lu\nmodified: %lu\nfree: %lu\nclear: %lu\n",
 		counter[PAGE_STATE_WIRED], counter[PAGE_STATE_MODIFIED],
 		counter[PAGE_STATE_FREE], counter[PAGE_STATE_CLEAR]);
-	kprintf("reserved pages: %lu\n", sReservedPages);
+	kprintf("unreserved free pages: %lu\n", sUnreservedFreePages);
 	kprintf("page deficit: %lu\n", sPageDeficit);
 	kprintf("mapped pages: %lu\n", gMappedPagesCount);
 
@@ -522,13 +522,6 @@ dump_page_stats(int argc, char **argv)
 }
 
 
-static inline size_t
-free_page_queue_count(void)
-{
-	return sFreePageQueue.Count() + sClearPageQueue.Count();
-}
-
-
 static status_t
 set_page_state_nolock(vm_page *page, int pageState)
 {
@@ -538,6 +531,8 @@ set_page_state_nolock(vm_page *page, int pageState)
 	VMPageQueue *fromQueue = NULL;
 	VMPageQueue *toQueue = NULL;
 
+	int32 freeCountDiff = 0;
+
 	switch (page->state) {
 		case PAGE_STATE_BUSY:
 		case PAGE_STATE_ACTIVE:
@@ -553,9 +548,11 @@ set_page_state_nolock(vm_page *page, int pageState)
 			break;
 		case PAGE_STATE_FREE:
 			fromQueue = &sFreePageQueue;
+			freeCountDiff = -1;
 			break;
 		case PAGE_STATE_CLEAR:
 			fromQueue = &sClearPageQueue;
+			freeCountDiff = -1;
 			break;
 		default:
 			panic("vm_page_set_state: vm_page %p in invalid state %d\n",
@@ -583,9 +580,11 @@ set_page_state_nolock(vm_page *page, int pageState)
 			break;
 		case PAGE_STATE_FREE:
 			toQueue = &sFreePageQueue;
+			freeCountDiff++;
 			break;
 		case PAGE_STATE_CLEAR:
 			toQueue = &sClearPageQueue;
+			freeCountDiff++;
 			break;
 		default:
 			panic("vm_page_set_state: invalid target state %d\n", pageState);
@@ -620,6 +619,9 @@ set_page_state_nolock(vm_page *page, int pageState)
 	page->state = pageState;
 	toQueue->MoveFrom(fromQueue, page);
 
+	if (freeCountDiff != 0)
+		atomic_add(&sUnreservedFreePages, freeCountDiff);
+
 	return B_OK;
 }
 
@@ -677,38 +679,36 @@ page_scrubber(void *unused)
 		if (sFreePageQueue.Count() == 0)
 			continue;
 
-		MutexLocker locker(sPageLock);
-
 		// Since we temporarily remove pages from the free pages reserve,
 		// we must make sure we don't cause a violation of the page
 		// reservation warranty. The following is usually stricter than
 		// necessary, because we don't have information on how many of the
 		// reserved pages have already been allocated.
-		int32 scrubCount = SCRUB_SIZE;
+		if (!vm_page_try_reserve_pages(SCRUB_SIZE))
-		uint32 freeCount = free_page_queue_count();
-		if (freeCount <= sReservedPages)
 			continue;
 
-		if ((uint32)scrubCount > freeCount - sReservedPages)
-			scrubCount = freeCount - sReservedPages;
-
 		// get some pages from the free queue
+		MutexLocker locker(sPageLock);
+
 		vm_page *page[SCRUB_SIZE];
-		for (int32 i = 0; i < scrubCount; i++) {
+		int32 scrubCount = 0;
+		for (int32 i = 0; i < SCRUB_SIZE; i++) {
 			page[i] = sFreePageQueue.RemoveHead();
-			if (page[i] == NULL) {
+			if (page[i] == NULL)
-				scrubCount = i;
 				break;
-			}
 
 			page[i]->state = PAGE_STATE_BUSY;
+			scrubCount++;
 		}
 
-		if (scrubCount == 0)
+		locker.Unlock();
+
+		if (scrubCount == 0) {
+			vm_page_unreserve_pages(SCRUB_SIZE);
 			continue;
+		}
 
 		T(ScrubbingPages(scrubCount));
-		locker.Unlock();
 
 		// clear them
 		for (int32 i = 0; i < scrubCount; i++)
@@ -722,6 +722,10 @@ page_scrubber(void *unused)
 			sClearPageQueue.Append(page[i]);
 		}
 
+		locker.Unlock();
+
+		vm_page_unreserve_pages(SCRUB_SIZE);
+
 		T(ScrubbedPages(scrubCount));
 	}
 
@@ -1442,6 +1446,8 @@ steal_pages(vm_page **pages, size_t count)
 				page->state = PAGE_STATE_FREE;
 				locker.Unlock();
 
+				atomic_add(&sUnreservedFreePages, 1);
+
 				T(StolenPage());
 
 				stolen++;
@@ -1454,7 +1460,7 @@ steal_pages(vm_page **pages, size_t count)
 
 		MutexLocker locker(sPageLock);
 
-		if (count == 0 || sReservedPages <= free_page_queue_count())
+		if (count == 0 || sUnreservedFreePages >= 0)
 			return stolen;
 
 		if (stolen && !tried && sInactivePageQueue.Count() > 0) {
@@ -1486,7 +1492,7 @@ steal_pages(vm_page **pages, size_t count)
 		locker.Lock();
 		sPageDeficit--;
 
-		if (sReservedPages <= free_page_queue_count())
+		if (sUnreservedFreePages >= 0)
 			return stolen;
 	}
 }
@@ -1714,6 +1720,8 @@ vm_page_init(kernel_args *args)
 		sFreePageQueue.Append(&sPages[i]);
 	}
 
+	atomic_add(&sUnreservedFreePages, sNumPages);
+
 	TRACE(("initialized table\n"));
 
 	// mark some of the page ranges inuse
@@ -1835,15 +1843,15 @@ vm_page_unreserve_pages(uint32 count)
 	if (count == 0)
 		return;
 
-	MutexLocker locker(sPageLock);
-	ASSERT(sReservedPages >= count);
-
 	T(UnreservePages(count));
 
-	sReservedPages -= count;
+	atomic_add(&sUnreservedFreePages, count);
 
+	if (sPageDeficit > 0) {
+		MutexLocker locker(sPageLock);
 		if (sPageDeficit > 0)
 			sFreePageCondition.NotifyAll();
+	}
 }
 
 
@@ -1859,16 +1867,17 @@ vm_page_reserve_pages(uint32 count)
 	if (count == 0)
 		return;
 
-	MutexLocker locker(sPageLock);
-
 	T(ReservePages(count));
 
-	sReservedPages += count;
+	int32 oldFreePages = atomic_add(&sUnreservedFreePages, -count);
-	size_t freePages = free_page_queue_count();
+	if (oldFreePages >= (int32)count)
-	if (sReservedPages <= freePages)
 		return;
 
-	count = sReservedPages - freePages;
+	MutexLocker locker(sPageLock);
+
+	if (oldFreePages > 0)
+		count -= oldFreePages;
+
 	locker.Unlock();
 
 	steal_pages(NULL, count + 1);
@@ -1883,16 +1892,20 @@ vm_page_try_reserve_pages(uint32 count)
 	if (count == 0)
 		return true;
 
-	MutexLocker locker(sPageLock);
-
 	T(ReservePages(count));
 
-	size_t freePages = free_page_queue_count();
+	while (true) {
-	if (sReservedPages + count > freePages)
+		int32 freePages = sUnreservedFreePages;
+		if (freePages < (int32)count)
 			return false;
 
-	sReservedPages += count;
+		if (atomic_test_and_set(&sUnreservedFreePages, freePages - count,
+				freePages) == freePages) {
 			return true;
+		}
+
+		// the count changed in the meantime -- retry
+	}
 }
 
 
@@ -1915,6 +1928,8 @@ vm_page_allocate_page(int pageState)
 			return NULL; // invalid
 	}
 
+	atomic_add(&sUnreservedFreePages, -1);
+
 	MutexLocker locker(sPageLock);
 
 	T(AllocatePage());
@@ -1959,13 +1974,12 @@ vm_page_allocate_page_run(int pageState, addr_t base, addr_t length)
 	vm_page *firstPage = NULL;
 	uint32 start = base >> PAGE_SHIFT;
 
-	MutexLocker locker(sPageLock);
+	if (!vm_page_try_reserve_pages(length))
-
+		return NULL;
-	if (free_page_queue_count() - sReservedPages < length) {
 		// TODO: add more tries, ie. free some inactive, ...
 		// no free space
-		return NULL;
+
-	}
+	MutexLocker locker(sPageLock);
 
 	for (;;) {
 		bool foundRun = true;
@@ -2001,6 +2015,8 @@ vm_page_allocate_page_run(int pageState, addr_t base, addr_t length)
 
 	locker.Unlock();
 
+	vm_page_unreserve_pages(length);
+
 	if (firstPage != NULL && pageState == PAGE_STATE_CLEAR) {
 		for (uint32 i = 0; i < length; i++) {
 			if (!sPages[start + i].is_cleared)
@@ -2015,14 +2031,6 @@ vm_page_allocate_page_run(int pageState, addr_t base, addr_t length)
 vm_page *
 vm_page_allocate_page_run_no_base(int pageState, addr_t count)
 {
-	MutexLocker locker(sPageLock);
-
-	if (free_page_queue_count() - sReservedPages < count) {
-		// TODO: add more tries, ie. free some inactive, ...
-		// no free space
-		return NULL;
-	}
-
 	VMPageQueue *queue;
 	VMPageQueue *otherQueue;
 	switch (pageState) {
@@ -2038,6 +2046,13 @@ vm_page_allocate_page_run_no_base(int pageState, addr_t count)
 			return NULL; // invalid
 	}
 
+	if (!vm_page_try_reserve_pages(count))
+		return NULL;
+		// TODO: add more tries, ie. free some inactive, ...
+		// no free space
+
+	MutexLocker locker(sPageLock);
+
 	vm_page *firstPage = NULL;
 	for (uint32 twice = 0; twice < 2; twice++) {
 		VMPageQueue::Iterator it = queue->GetIterator();
@@ -2056,6 +2071,8 @@ vm_page_allocate_page_run_no_base(int pageState, addr_t count)
 			}
 
 			if (foundRun) {
+				atomic_add(&sUnreservedFreePages, -count);
+
 				// pull the pages out of the appropriate queues
 				current = page;
 				for (uint32 i = 0; i < count; i++, current++) {
@@ -2079,6 +2096,8 @@ vm_page_allocate_page_run_no_base(int pageState, addr_t count)
 
 	locker.Unlock();
 
+	vm_page_unreserve_pages(count);
+
 	if (firstPage != NULL && pageState == PAGE_STATE_CLEAR) {
 		vm_page *current = firstPage;
 		for (uint32 i = 0; i < count; i++, current++) {
@@ -2206,24 +2225,16 @@ vm_page_num_available_pages(void)
 size_t
 vm_page_num_free_pages(void)
 {
-	size_t reservedPages = sReservedPages;
+	int32 count = sUnreservedFreePages + sInactivePageQueue.Count();
-	size_t count = free_page_queue_count() + sInactivePageQueue.Count();
+	return count > 0 ? count : 0;
-	if (reservedPages >= count)
-		return 0;
-
-	return count - reservedPages;
 }
 
 
 size_t
 vm_page_num_unused_pages(void)
 {
-	size_t reservedPages = sReservedPages;
+	int32 count = sUnreservedFreePages;
-	size_t count = free_page_queue_count();
+	return count > 0 ? count : 0;
-	if (reservedPages >= count)
-		return 0;
-
-	return count - reservedPages;
 }
 
 
@@ -2234,9 +2245,9 @@ vm_page_get_stats(system_info *info)
 	// of the reserved pages have already been allocated, but good citizens
 	// unreserve chunk-wise as they are allocating the pages, if they have
 	// reserved a larger quantity.
-	page_num_t reserved = sReservedPages;
+	int32 free = sUnreservedFreePages;
-	page_num_t free = free_page_queue_count();
+	if (free < 0)
-	free = free > reserved ? free - reserved : 0;
+		free = 0;
 
 	// The pages used for the block cache buffers. Those should not be counted
 	// as used but as cached pages.
@@ -2246,7 +2257,7 @@ vm_page_get_stats(system_info *info)
 
 	info->max_pages = sNumPages;
 	info->used_pages = gMappedPagesCount - blockCachePages;
-	info->cached_pages = sNumPages >= free + info->used_pages
+	info->cached_pages = sNumPages >= (uint32)free + info->used_pages
 		? sNumPages - free - info->used_pages : 0;
 	info->page_faults = vm_num_page_faults();