* VMCache::Write(): Added "uint32 flags" argument which is supposed to

be passed on to the IORequest. Most relevantly physical pages can now
  be written directly by passing B_PHYSICAL_IO_REQUEST.
* Added VMCache::WriteAsync() which is supposed to write pages
  asynchronously. The base class version version falls back to the
  synchronous Write(). Only VMVnodeCache implements WriteAsync() ATM,
  VMAnonymousCache (swap support) still has to be adjusted accordingly.
* write_page() doesn't need to map the page anymore as it can write the
  physical page directly.
* Modified the page writer to write pages asynchronously. This shouldn't
  have any noticeable effect yet. It will though as soon as the I/O
  scheduler reorders I/O operations.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@27056 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/private/kernel/vm_types.h

@@ -37,6 +37,7 @@
 
 #include <util/SplayTree.h>
 
+class AsyncIOCallback;
 struct vm_page_mapping;
 struct VMCache;
 typedef DoublyLinkedListLink<vm_page_mapping> vm_page_mapping_link;
@@ -224,7 +225,10 @@ public:
 	virtual	status_t	Read(off_t offset, const iovec *vecs, size_t count,
 							size_t *_numBytes);
 	virtual	status_t	Write(off_t offset, const iovec *vecs, size_t count,
-							size_t *_numBytes);
+							uint32 flags, size_t *_numBytes);
+	virtual	status_t	WriteAsync(off_t offset, const iovec* vecs,
+							size_t count, size_t numBytes, uint32 flags,
+							AsyncIOCallback* callback);
 
 	virtual	status_t	Fault(struct vm_address_space *aspace, off_t offset);
 

src/system/kernel/cache/vnode_store.cpp

@@ -73,9 +73,18 @@ VMVnodeCache::Read(off_t offset, const iovec *vecs, size_t count,
 
 status_t
 VMVnodeCache::Write(off_t offset, const iovec *vecs, size_t count,
-	size_t *_numBytes)
+	uint32 flags, size_t *_numBytes)
 {
-	return vfs_write_pages(fVnode, NULL, offset, vecs, count, 0, _numBytes);
+	return vfs_write_pages(fVnode, NULL, offset, vecs, count, flags, _numBytes);
+}
+
+
+status_t
+VMVnodeCache::WriteAsync(off_t offset, const iovec* vecs, size_t count,
+	size_t numBytes, uint32 flags, AsyncIOCallback* callback)
+{
+	return vfs_asynchronous_write_pages(fVnode, NULL, offset, vecs, count,
+		numBytes, flags, callback);
 }
 
 

src/system/kernel/cache/vnode_store.h

@@ -22,7 +22,10 @@ public:
 	virtual	status_t	Read(off_t offset, const iovec *vecs, size_t count,
 							size_t *_numBytes);
 	virtual	status_t	Write(off_t offset, const iovec *vecs, size_t count,
-							size_t *_numBytes);
+							uint32 flags, size_t *_numBytes);
+	virtual	status_t	WriteAsync(off_t offset, const iovec* vecs,
+							size_t count, size_t numBytes, uint32 flags,
+							AsyncIOCallback* callback);
 
 	virtual	status_t	Fault(struct vm_address_space *aspace, off_t offset);
 

src/system/kernel/vm/Jamfile

@@ -2,6 +2,8 @@ SubDir HAIKU_TOP src system kernel vm ;
 
 UsePrivateHeaders shared ;
 
+UseHeaders [ FDirName $(SUBDIR) $(DOTDOT) device_manager ] ;
+
 KernelMergeObject kernel_vm.o :
 	vm.cpp
 	vm_address_space.cpp

src/system/kernel/vm/VMAnonymousCache.cpp

@@ -376,7 +376,7 @@ VMAnonymousCache::Read(off_t offset, const iovec *vecs, size_t count,
 
 status_t
 VMAnonymousCache::Write(off_t offset, const iovec *vecs, size_t count,
-	size_t *_numBytes)
+	uint32 flags, size_t *_numBytes)
 {
 	off_t pageIndex = offset >> PAGE_SHIFT;
 
@@ -410,7 +410,7 @@ VMAnonymousCache::Write(off_t offset, const iovec *vecs, size_t count,
 		off_t pos = (swapAddr - swapFile->first_page) * B_PAGE_SIZE;
 
 		status_t status = vfs_write_pages(swapFile->vnode, NULL, pos, vecs + i,
-				n, 0, _numBytes);
+				n, flags, _numBytes);
 		if (status != B_OK) {
 			Lock();
 			fAllocatedSwapSize -= n * B_PAGE_SIZE;

src/system/kernel/vm/VMAnonymousCache.h

@@ -39,7 +39,7 @@ public:
 	virtual	status_t	Read(off_t offset, const iovec *vecs, size_t count,
 							size_t *_numBytes);
 	virtual	status_t	Write(off_t offset, const iovec *vecs, size_t count,
-							size_t *_numBytes);
+							uint32 flags, size_t *_numBytes);
 
 	virtual	status_t	Fault(struct vm_address_space *aspace, off_t offset);
 

src/system/kernel/vm/VMAnonymousNoSwapCache.cpp

@@ -105,7 +105,7 @@ VMAnonymousNoSwapCache::Read(off_t offset, const iovec *vecs, size_t count,
 
 status_t
 VMAnonymousNoSwapCache::Write(off_t offset, const iovec *vecs, size_t count,
-	size_t *_numBytes)
+	uint32 flags, size_t *_numBytes)
 {
 	// no place to write, this will cause the page daemon to skip this store
 	return B_ERROR;

src/system/kernel/vm/VMAnonymousNoSwapCache.h

@@ -25,7 +25,7 @@ public:
 	virtual	status_t	Read(off_t offset, const iovec *vecs, size_t count,
 							size_t *_numBytes);
 	virtual	status_t	Write(off_t offset, const iovec *vecs, size_t count,
-							size_t *_numBytes);
+							uint32 flags, size_t *_numBytes);
 
 	virtual	status_t	Fault(struct vm_address_space *aspace, off_t offset);
 

src/system/kernel/vm/VMDeviceCache.cpp

@@ -36,7 +36,7 @@ VMDeviceCache::Read(off_t offset, const iovec *vecs, size_t count,
 
 status_t
 VMDeviceCache::Write(off_t offset, const iovec *vecs, size_t count,
-	size_t *_numBytes)
+	uint32 flags, size_t *_numBytes)
 {
 	// no place to write, this will cause the page daemon to skip this store
 	return B_OK;

src/system/kernel/vm/VMDeviceCache.h

@@ -21,7 +21,7 @@ public:
 	virtual	status_t	Read(off_t offset, const iovec *vecs, size_t count,
 							size_t *_numBytes);
 	virtual	status_t	Write(off_t offset, const iovec *vecs, size_t count,
-							size_t *_numBytes);
+							uint32 flags, size_t *_numBytes);
 
 private:
 	addr_t	fBaseAddress;

src/system/kernel/vm/vm_cache.cpp

@@ -21,6 +21,7 @@
 #include <tracing.h>
 #include <util/khash.h>
 #include <util/AutoLock.h>
+#include <vfs.h>
 #include <vm.h>
 #include <vm_page.h>
 #include <vm_priv.h>
@@ -867,12 +868,28 @@ VMCache::Read(off_t offset, const iovec *vecs, size_t count, size_t *_numBytes)
 
 
 status_t
-VMCache::Write(off_t offset, const iovec *vecs, size_t count, size_t *_numBytes)
+VMCache::Write(off_t offset, const iovec *vecs, size_t count, uint32 flags,
+	size_t *_numBytes)
 {
 	return B_ERROR;
 }
 
 
+status_t
+VMCache::WriteAsync(off_t offset, const iovec* vecs, size_t count,
+	size_t numBytes, uint32 flags, AsyncIOCallback* callback)
+{
+	// Not supported, fall back to the synchronous hook.
+	size_t transferred = numBytes;
+	status_t error = Write(offset, vecs, count, flags, &transferred);
+
+	if (callback != NULL)
+		callback->IOFinished(error, transferred != numBytes, transferred);
+
+	return error;
+}
+
+
 status_t
 VMCache::Fault(struct vm_address_space *aspace, off_t offset)
 {

src/system/kernel/vm/vm_page.cpp

@@ -23,6 +23,7 @@
 #include <thread.h>
 #include <tracing.h>
 #include <util/AutoLock.h>
+#include <vfs.h>
 #include <vm.h>
 #include <vm_address_space.h>
 #include <vm_priv.h>
@@ -30,6 +31,7 @@
 #include <vm_cache.h>
 
 #include "PageCacheLocker.h"
+#include "io_requests.h"
 
 
 //#define TRACE_VM_PAGE
@@ -853,30 +855,27 @@ page_scrubber(void *unused)
 
 
 static status_t
-write_page(vm_page *page)
+write_page(vm_page *page, uint32 flags, AsyncIOCallback* callback)
 {
-	size_t length = B_PAGE_SIZE;
-	status_t status;
-	iovec vecs[1];
-
 	TRACE(("write_page(page = %p): offset = %Ld\n", page, (off_t)page->cache_offset << PAGE_SHIFT));
 
-	status = vm_get_physical_page(page->physical_page_number * B_PAGE_SIZE,
-		(addr_t *)&vecs[0].iov_base, PHYSICAL_PAGE_CAN_WAIT);
-	if (status < B_OK)
-		panic("could not map page!");
+	off_t offset = (off_t)page->cache_offset << PAGE_SHIFT;
+
+	iovec vecs[1];
+	vecs->iov_base = (void*)(addr_t)(page->physical_page_number * B_PAGE_SIZE);
 	vecs->iov_len = B_PAGE_SIZE;
 
-	status = page->cache->Write((off_t)page->cache_offset << PAGE_SHIFT,
-		vecs, 1, &length);
-
-	vm_put_physical_page((addr_t)vecs[0].iov_base);
-#if 0
-	if (status < B_OK) {
-		dprintf("write_page(page = %p): offset = %lx, status = %ld\n",
-			page, page->cache_offset, status);
+	if (callback != NULL) {
+		// asynchronous I/O
+		return page->cache->WriteAsync(offset, vecs, 1, B_PAGE_SIZE,
+			flags | B_PHYSICAL_IO_REQUEST, callback);
 	}
-#endif
+
+	// synchronous I/0
+	size_t length = B_PAGE_SIZE;
+	status_t status = page->cache->Write(offset, vecs, 1,
+		flags | B_PHYSICAL_IO_REQUEST, &length);
+
 	if (status == B_OK && length == 0)
 		status = B_ERROR;
 
@@ -947,6 +946,168 @@ next_modified_page(struct vm_page &marker)
 }
 
 
+class PageWriterCallback;
+
+
+class PageWriterRun {
+public:
+	status_t Init(uint32 maxPages);
+
+	void PrepareNextRun();
+	void AddPage(vm_page* page);
+	void Go();
+
+	void PageWritten(PageWriterCallback* callback, status_t status,
+		bool partialTransfer, size_t bytesTransferred);
+
+private:
+	uint32				fMaxPages;
+	uint32				fPageCount;
+	vint32				fPendingPages;
+	PageWriterCallback*	fCallbacks;
+	ConditionVariable	fAllFinishedCondition;
+};
+
+
+class PageWriterCallback : public AsyncIOCallback {
+public:
+	void SetTo(PageWriterRun* run, vm_page* page)
+	{
+		fRun = run;
+		fPage = page;
+		fCache = page->cache;
+		fStatus = B_OK;
+		fBusyCondition.Publish(page, "page");
+	}
+
+	vm_page* Page() const	{ return fPage; }
+	VMCache* Cache() const	{ return fCache; }
+	status_t Status() const	{ return fStatus; }
+
+	ConditionVariable& BusyCondition()	{ return fBusyCondition; }
+
+	virtual void IOFinished(status_t status, bool partialTransfer,
+		size_t bytesTransferred)
+	{
+		fStatus = status == B_OK && bytesTransferred == 0 ? B_ERROR : status;
+		fRun->PageWritten(this, status, partialTransfer, bytesTransferred);
+	}
+
+private:
+	PageWriterRun*		fRun;
+	vm_page*			fPage;
+	VMCache*			fCache;
+	status_t			fStatus;
+	ConditionVariable	fBusyCondition;
+};
+
+
+status_t
+PageWriterRun::Init(uint32 maxPages)
+{
+	fMaxPages = maxPages;
+	fPageCount = 0;
+	fPendingPages = 0;
+
+	fCallbacks = new(std::nothrow) PageWriterCallback[maxPages];
+	if (fCallbacks == NULL)
+		return B_NO_MEMORY;
+
+	return B_OK;
+}
+
+
+void
+PageWriterRun::PrepareNextRun()
+{
+	fPageCount = 0;
+	fPendingPages = 0;
+}
+
+
+void
+PageWriterRun::AddPage(vm_page* page)
+{
+	page->state = PAGE_STATE_BUSY;
+	page->busy_writing = true;
+
+	fCallbacks[fPageCount].SetTo(this, page);
+	fPageCount++;
+}
+
+
+void
+PageWriterRun::Go()
+{
+	fPendingPages = fPageCount;
+
+	fAllFinishedCondition.Init(this, "page writer wait for I/O");
+	ConditionVariableEntry waitEntry;
+	fAllFinishedCondition.Add(&waitEntry);
+
+	// schedule writes
+	for (uint32 i = 0; i < fPageCount; i++) {
+		PageWriterCallback& callback = fCallbacks[i];
+		write_page(callback.Page(), B_VIP_IO_REQUEST, &callback);
+	}
+
+	// wait until all pages have been written
+	waitEntry.Wait();
+
+	// mark pages depending on whether they could be written or not
+
+	for (uint32 i = 0; i < fPageCount; i++) {
+		PageWriterCallback& callback = fCallbacks[i];
+		vm_page* page = callback.Page();
+		vm_cache* cache = callback.Cache();
+		cache->Lock();
+
+		if (callback.Status() == B_OK) {
+			// put it into the active queue
+			InterruptsSpinLocker locker(sPageLock);
+			move_page_to_active_or_inactive_queue(page, true);
+			page->busy_writing = false;
+		} else {
+			// We don't have to put the PAGE_MODIFIED bit back, as it's
+			// still in the modified pages list.
+			{
+				InterruptsSpinLocker locker(sPageLock);
+				page->state = PAGE_STATE_MODIFIED;
+				enqueue_page(&sModifiedPageQueue, page);
+			}
+			if (!page->busy_writing) {
+				// someone has cleared the busy_writing flag which tells
+				// us our page has gone invalid
+				cache->RemovePage(page);
+			} else
+				page->busy_writing = false;
+		}
+
+		callback.BusyCondition().Unpublish();
+
+		cache->Unlock();
+	}
+
+	for (uint32 i = 0; i < fPageCount; i++) {
+		vm_cache* cache = fCallbacks[i].Cache();
+
+		// We release the cache references after all pages were made
+		// unbusy again - otherwise releasing a vnode could deadlock.
+		cache->ReleaseStoreRef();
+		cache->ReleaseRef();
+	}
+}
+
+
+void
+PageWriterRun::PageWritten(PageWriterCallback* callback, status_t status,
+	bool partialTransfer, size_t bytesTransferred)
+{
+	if (atomic_add(&fPendingPages, -1) == 1)
+		fAllFinishedCondition.NotifyAll();
+}
+
+
 /*!	The page writer continuously takes some pages from the modified
 	queue, writes them back, and moves them back to the active queue.
 	It runs in its own thread, and is only there to keep the number
@@ -956,6 +1117,16 @@ next_modified_page(struct vm_page &marker)
 status_t
 page_writer(void* /*unused*/)
 {
+	// TODO: once the I/O scheduler is there, we should write
+	// a lot more pages back.
+	const uint32 kNumPages = 32;
+
+	PageWriterRun run;
+	if (run.Init(kNumPages) != B_OK) {
+		panic("page writer: Failed to init PageWriterRun!");
+		return B_ERROR;
+	}
+
 	vm_page marker;
 	marker.type = PAGE_TYPE_DUMMY;
 	marker.cache = NULL;
@@ -972,16 +1143,9 @@ page_writer(void* /*unused*/)
 				// all 3 seconds when no one triggers us
 		}
 
-		const uint32 kNumPages = 32;
-		ConditionVariable busyConditions[kNumPages];
-		union {
-			vm_page *pages[kNumPages];
-			vm_cache *caches[kNumPages];
-		} u;
 		uint32 numPages = 0;
+		run.PrepareNextRun();
 
-		// TODO: once the I/O scheduler is there, we should write
-		// a lot more pages back.
 		// TODO: make this laptop friendly, too (ie. only start doing
 		// something if someone else did something or there is really
 		// enough to do).
@@ -1023,77 +1187,29 @@ page_writer(void* /*unused*/)
 
 			// state might have change while we were locking the cache
 			if (page->state != PAGE_STATE_MODIFIED) {
-				// release the cache reference first
+				// release the cache reference
+				locker.Unlock();
 				cache->ReleaseStoreRef();
 				continue;
 			}
 
 			remove_page_from_queue(&sModifiedPageQueue, page);
-			page->state = PAGE_STATE_BUSY;
-			page->busy_writing = true;
 
-			busyConditions[numPages].Publish(page, "page");
+			run.AddPage(page);
 
 			locker.Unlock();
 
 			//dprintf("write page %p, cache %p (%ld)\n", page, page->cache, page->cache->ref_count);
 			vm_clear_map_flags(page, PAGE_MODIFIED);
 			cache->AcquireRefLocked();
-			u.pages[numPages++] = page;
+			numPages++;
 		}
 
 		if (numPages == 0)
 			continue;
 
-		// write pages to disk
-
-		// TODO: put this as requests into the I/O scheduler
-		status_t writeStatus[kNumPages];
-		for (uint32 i = 0; i < numPages; i++) {
-			writeStatus[i] = write_page(u.pages[i]);
-		}
-
-		// mark pages depending on whether they could be written or not
-
-		for (uint32 i = 0; i < numPages; i++) {
-			vm_cache *cache = u.pages[i]->cache;
-			cache->Lock();
-
-			if (writeStatus[i] == B_OK) {
-				// put it into the active queue
-				InterruptsSpinLocker locker(sPageLock);
-				move_page_to_active_or_inactive_queue(u.pages[i], true);
-				u.pages[i]->busy_writing = false;
-			} else {
-				// We don't have to put the PAGE_MODIFIED bit back, as it's
-				// still in the modified pages list.
-				{
-					InterruptsSpinLocker locker(sPageLock);
-					u.pages[i]->state = PAGE_STATE_MODIFIED;
-					enqueue_page(&sModifiedPageQueue, u.pages[i]);
-				}
-				if (!u.pages[i]->busy_writing) {
-					// someone has cleared the busy_writing flag which tells
-					// us our page has gone invalid
-					cache->RemovePage(u.pages[i]);
-				} else
-					u.pages[i]->busy_writing = false;
-			}
-
-			busyConditions[i].Unpublish();
-
-			u.caches[i] = cache;
-			cache->Unlock();
-		}
-
-		for (uint32 i = 0; i < numPages; i++) {
-			vm_cache *cache = u.caches[i];
-
-			// We release the cache references after all pages were made
-			// unbusy again - otherwise releasing a vnode could deadlock.
-			cache->ReleaseStoreRef();
-			cache->ReleaseRef();
-		}
+		// write pages to disk and do all the cleanup
+		run.Go();
 	}
 
 	remove_page_marker(marker);
@@ -1336,7 +1452,7 @@ vm_page_write_modified_page_range(struct VMCache *cache, uint32 firstPage,
 		vm_clear_map_flags(page, PAGE_MODIFIED);
 
 		cache->Unlock();
-		status_t status = write_page(page);
+		status_t status = write_page(page, 0, NULL);
 		cache->Lock();
 
 		InterruptsSpinLocker locker(&sPageLock);