From 7f12cc54a729622cd04940ee9400958413d99b21 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Wed, 30 Jul 2008 23:34:10 +0000 Subject: [PATCH] * It is now supported that I/O operations and requests are only handled partially (e.g. due to hitting the end of file). The respective classes have grown new methods and attributes to deal with that. The "finished" callbacks have got additional parameters to indicate whether the transfer was only partial and how much has been transferred. Other callbacks and functions have a size_t* in/out parameter instead of a simple size_t, now. * vfs_{read,write}_pages() do now use the I/O request framework instead of the underlying FS's {read,write}_pages() hooks (those should be unused now). Furthermore they've got an additional "flags" parameter, which is passed to IORequest::Init(), i.e. it allows to specify that the given vecs refer to physical addresses. * The file cache's read_into_cache() reads directly into physical pages, now. * Fixed bug in DoIO::IO(): The offset was not adjusted, so that all pages were incorrectly transferred from/to the same location. * Fixed broken subrequest scheduling loop head in do_iterative_fd_io_iterate(). * Adjusted the test driver and implemented its io() hook. Using this driver I/O requests are passed all the way from the VFS/VM to the driver and through the I/O scheduler. It even seems to work. :-) * Added missing const to the iovec* parameter of the IORequest::Init() methods. * Disabled some debug output by default. Added new optional debug output. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@26692 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/os/drivers/fs_interface.h | 2 +- headers/private/fs_shell/fssh_fs_interface.h | 3 +- headers/private/kernel/vfs.h | 6 +- .../file_systems/bfs/kernel_interface.cpp | 3 +- src/system/kernel/cache/Jamfile | 2 + src/system/kernel/cache/file_cache.cpp | 60 +++---- src/system/kernel/cache/vnode_store.cpp | 4 +- .../kernel/device_manager/IOScheduler.cpp | 20 ++- .../kernel/device_manager/IOScheduler.h | 2 +- src/system/kernel/device_manager/devfs.cpp | 18 +- .../kernel/device_manager/dma_resources.cpp | 2 +- .../kernel/device_manager/io_requests.cpp | 72 ++++++-- .../kernel/device_manager/io_requests.h | 41 ++++- src/system/kernel/fs/vfs.cpp | 33 +++- src/system/kernel/fs/vfs_request_io.cpp | 162 ++++++++++++------ .../device_manager/dma_resource_test.cpp | 45 ++--- 16 files changed, 312 insertions(+), 163 deletions(-) diff --git a/headers/os/drivers/fs_interface.h b/headers/os/drivers/fs_interface.h index 724126f7ab..e81cea9341 100644 --- a/headers/os/drivers/fs_interface.h +++ b/headers/os/drivers/fs_interface.h @@ -299,7 +299,7 @@ typedef status_t (*iterative_io_get_vecs)(void *cookie, io_request* request, off_t offset, size_t size, struct file_io_vec *vecs, size_t *_count); typedef status_t (*iterative_io_finished)(void* cookie, io_request* request, - status_t status); + status_t status, bool partialTransfer, size_t bytesTransferred); extern status_t new_vnode(fs_volume *volume, ino_t vnodeID, void *privateNode, fs_vnode_ops *ops); diff --git a/headers/private/fs_shell/fssh_fs_interface.h b/headers/private/fs_shell/fssh_fs_interface.h index 9d25a09965..47231f5c71 100644 --- a/headers/private/fs_shell/fssh_fs_interface.h +++ b/headers/private/fs_shell/fssh_fs_interface.h @@ -334,7 +334,8 @@ typedef fssh_status_t (*fssh_iterative_io_get_vecs)(void *cookie, fssh_io_request* request, fssh_off_t offset, fssh_size_t size, struct fssh_file_io_vec *vecs, fssh_size_t *_count); typedef fssh_status_t (*fssh_iterative_io_finished)(void* cookie, - fssh_io_request* request, fssh_status_t status); + fssh_io_request* request, fssh_status_t status, + bool partialTransfer, fssh_size_t bytesTransferred); extern fssh_status_t fssh_new_vnode(fssh_fs_volume *volume, fssh_vnode_id vnodeID, void *privateNode, diff --git a/headers/private/kernel/vfs.h b/headers/private/kernel/vfs.h index db3e01690c..e8c621c700 100644 --- a/headers/private/kernel/vfs.h +++ b/headers/private/kernel/vfs.h @@ -94,13 +94,13 @@ void vfs_acquire_vnode(struct vnode *vnode); status_t vfs_get_cookie_from_fd(int fd, void **_cookie); bool vfs_can_page(struct vnode *vnode, void *cookie); status_t vfs_read_pages(struct vnode *vnode, void *cookie, off_t pos, - const iovec *vecs, size_t count, size_t *_numBytes); + const iovec *vecs, size_t count, uint32 flags, size_t *_numBytes); status_t vfs_write_pages(struct vnode *vnode, void *cookie, off_t pos, - const iovec *vecs, size_t count, size_t *_numBytes); + const iovec *vecs, size_t count, uint32 flags, size_t *_numBytes); status_t vfs_vnode_io(struct vnode* vnode, void* cookie, io_request* request); status_t vfs_synchronous_io(io_request* request, status_t (*doIO)(void* cookie, off_t offset, void* buffer, - size_t length), + size_t* length), void* cookie); status_t vfs_get_vnode_cache(struct vnode *vnode, struct VMCache **_cache, bool allocate); diff --git a/src/add-ons/kernel/file_systems/bfs/kernel_interface.cpp b/src/add-ons/kernel/file_systems/bfs/kernel_interface.cpp index 9e385b2bea..ebff7bd6eb 100644 --- a/src/add-ons/kernel/file_systems/bfs/kernel_interface.cpp +++ b/src/add-ons/kernel/file_systems/bfs/kernel_interface.cpp @@ -454,7 +454,8 @@ bfs_iterative_io_get_vecs(void* cookie, io_request *request, off_t offset, static status_t -bfs_iterative_io_finished(void *cookie, io_request *request, status_t status) +bfs_iterative_io_finished(void *cookie, io_request *request, status_t status, + bool partialTransfer, size_t bytesTransferred) { Inode *inode = (Inode*)cookie; diff --git a/src/system/kernel/cache/Jamfile b/src/system/kernel/cache/Jamfile index e1fc85fe96..264df9dcab 100644 --- a/src/system/kernel/cache/Jamfile +++ b/src/system/kernel/cache/Jamfile @@ -1,5 +1,7 @@ SubDir HAIKU_TOP src system kernel cache ; +UseHeaders [ FDirName $(SUBDIR) $(DOTDOT) device_manager ] ; + KernelMergeObject kernel_cache.o : block_cache.cpp file_cache.cpp diff --git a/src/system/kernel/cache/file_cache.cpp b/src/system/kernel/cache/file_cache.cpp index ac5c601f08..90b58cb3e1 100644 --- a/src/system/kernel/cache/file_cache.cpp +++ b/src/system/kernel/cache/file_cache.cpp @@ -24,6 +24,8 @@ #include #include +#include "io_requests.h" + //#define TRACE_FILE_CACHE #ifdef TRACE_FILE_CACHE @@ -196,12 +198,8 @@ read_into_cache(file_cache_ref *ref, void *cookie, off_t offset, cache->InsertPage(page, offset + pos); - addr_t virtualAddress; - if (vm_get_physical_page(page->physical_page_number * B_PAGE_SIZE, - &virtualAddress, PHYSICAL_PAGE_CAN_WAIT) < B_OK) - panic("could not get physical page"); - - add_to_iovec(vecs, vecCount, MAX_IO_VECS, virtualAddress, B_PAGE_SIZE); + add_to_iovec(vecs, vecCount, MAX_IO_VECS, + page->physical_page_number * B_PAGE_SIZE, B_PAGE_SIZE); // TODO: check if the array is large enough (currently panics)! } @@ -211,22 +209,12 @@ read_into_cache(file_cache_ref *ref, void *cookie, off_t offset, // read file into reserved pages status_t status = vfs_read_pages(ref->vnode, cookie, offset, vecs, - vecCount, &numBytes); + vecCount, B_PHYSICAL_IO_REQUEST, &numBytes); if (status < B_OK) { // reading failed, free allocated pages dprintf("file_cache: read pages failed: %s\n", strerror(status)); - for (int32 i = 0; i < vecCount; i++) { - addr_t base = (addr_t)vecs[i].iov_base; - size_t size = vecs[i].iov_len; - - for (size_t pos = 0; pos < size; - pos += B_PAGE_SIZE, base += B_PAGE_SIZE) { - vm_put_physical_page(base); - } - } - cache->Lock(); for (int32 i = 0; i < pageIndex; i++) { @@ -240,23 +228,24 @@ read_into_cache(file_cache_ref *ref, void *cookie, off_t offset, // copy the pages if needed and unmap them again - for (int32 i = 0; i < vecCount; i++) { - addr_t base = (addr_t)vecs[i].iov_base; - size_t size = vecs[i].iov_len; - - // copy to user buffer if necessary + for (int32 i = 0; i < pageIndex; i++) { if (useBuffer && bufferSize != 0) { - size_t bytes = min_c(bufferSize, size - pageOffset); + addr_t virtualAddress; + if (vm_get_physical_page( + pages[i]->physical_page_number * B_PAGE_SIZE, + &virtualAddress, PHYSICAL_PAGE_CAN_WAIT) < B_OK) { + panic("could not get physical page"); + } - user_memcpy((void *)buffer, (void *)(base + pageOffset), bytes); + size_t bytes = min_c(bufferSize, (size_t)B_PAGE_SIZE - pageOffset); + + user_memcpy((void*)buffer, (void*)(virtualAddress + pageOffset), + bytes); buffer += bytes; bufferSize -= bytes; pageOffset = 0; - } - for (size_t pos = 0; pos < size; pos += B_PAGE_SIZE, - base += B_PAGE_SIZE) { - vm_put_physical_page(base); + vm_put_physical_page(virtualAddress); } } @@ -294,7 +283,8 @@ read_from_file(file_cache_ref *ref, void *cookie, off_t offset, vm_page_unreserve_pages(lastReservedPages); status_t status = vfs_read_pages(ref->vnode, cookie, offset + pageOffset, - &vec, 1, &bufferSize); + &vec, 1, 0, &bufferSize); + if (status == B_OK) reserve_pages(ref, reservePages, false); @@ -360,7 +350,7 @@ write_to_cache(file_cache_ref *ref, void *cookie, off_t offset, iovec readVec = { vecs[0].iov_base, B_PAGE_SIZE }; size_t bytesRead = B_PAGE_SIZE; - status = vfs_read_pages(ref->vnode, cookie, offset, &readVec, 1, + status = vfs_read_pages(ref->vnode, cookie, offset, &readVec, 1, 0, &bytesRead); // ToDo: handle errors for real! if (status < B_OK) @@ -385,7 +375,7 @@ write_to_cache(file_cache_ref *ref, void *cookie, off_t offset, status = vfs_read_pages(ref->vnode, cookie, PAGE_ALIGN(offset + pageOffset + bufferSize) - B_PAGE_SIZE, - &readVec, 1, &bytesRead); + &readVec, 1, 0, &bytesRead); // ToDo: handle errors for real! if (status < B_OK) panic("vfs_read_pages() failed: %s!\n", strerror(status)); @@ -420,7 +410,7 @@ write_to_cache(file_cache_ref *ref, void *cookie, off_t offset, if (writeThrough) { // write cached pages back to the file if we were asked to do that status_t status = vfs_write_pages(ref->vnode, cookie, offset, vecs, - vecCount, &numBytes); + vecCount, 0, &numBytes); if (status < B_OK) { // ToDo: remove allocated pages, ...? panic("file_cache: remove allocated pages! write pages failed: %s\n", @@ -463,7 +453,7 @@ write_to_file(file_cache_ref *ref, void *cookie, off_t offset, int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer, size_t lastReservedPages, size_t reservePages) { - size_t chunkSize; + size_t chunkSize = 0; if (!useBuffer) { // we need to allocate a zero buffer // TODO: use smaller buffers if this fails @@ -491,7 +481,7 @@ write_to_file(file_cache_ref *ref, void *cookie, off_t offset, int32 pageOffset, chunkSize = bufferSize; status = vfs_write_pages(ref->vnode, cookie, offset + pageOffset, - &vec, 1, &chunkSize); + &vec, 1, 0, &chunkSize); if (status < B_OK) break; @@ -502,7 +492,7 @@ write_to_file(file_cache_ref *ref, void *cookie, off_t offset, int32 pageOffset, free((void*)buffer); } else { status = vfs_write_pages(ref->vnode, cookie, offset + pageOffset, - &vec, 1, &bufferSize); + &vec, 1, 0, &bufferSize); } if (status == B_OK) diff --git a/src/system/kernel/cache/vnode_store.cpp b/src/system/kernel/cache/vnode_store.cpp index 777ecdc270..b4c472e1c0 100644 --- a/src/system/kernel/cache/vnode_store.cpp +++ b/src/system/kernel/cache/vnode_store.cpp @@ -46,7 +46,7 @@ VMVnodeCache::Read(off_t offset, const iovec *vecs, size_t count, size_t bytesUntouched = *_numBytes; status_t status = vfs_read_pages(fVnode, NULL, offset, vecs, count, - _numBytes); + 0, _numBytes); bytesUntouched -= *_numBytes; @@ -75,7 +75,7 @@ status_t VMVnodeCache::Write(off_t offset, const iovec *vecs, size_t count, size_t *_numBytes) { - return vfs_write_pages(fVnode, NULL, offset, vecs, count, _numBytes); + return vfs_write_pages(fVnode, NULL, offset, vecs, count, 0, _numBytes); } diff --git a/src/system/kernel/device_manager/IOScheduler.cpp b/src/system/kernel/device_manager/IOScheduler.cpp index f45f8383b6..cd062b5066 100644 --- a/src/system/kernel/device_manager/IOScheduler.cpp +++ b/src/system/kernel/device_manager/IOScheduler.cpp @@ -138,7 +138,8 @@ IOScheduler::AbortRequest(IORequest* request, status_t status) void -IOScheduler::OperationCompleted(IOOperation* operation, status_t status) +IOScheduler::OperationCompleted(IOOperation* operation, status_t status, + size_t transferredBytes) { InterruptsSpinLocker _(fFinisherLock); @@ -148,6 +149,11 @@ IOScheduler::OperationCompleted(IOOperation* operation, status_t status) operation->SetStatus(status); + // set the bytes transferred (of the net data) + size_t partialBegin = operation->OriginalOffset() - operation->Offset(); + operation->SetTransferredBytes( + transferredBytes > partialBegin ? transferredBytes - partialBegin : 0); + fCompletedOperations.Add(operation); fFinishedOperationCondition.NotifyAll(); @@ -175,6 +181,7 @@ IOScheduler::_Finisher() if (!operation->Finish()) { TRACE(" operation: %p not finished yet\n", operation); + operation->SetTransferredBytes(0); // TODO: This must be done differently once the scheduler implements // an actual scheduling policy (other than no-op). fIOCallback(fIOCallbackData, operation); @@ -183,8 +190,15 @@ IOScheduler::_Finisher() // notify request and remove operation IORequest* request = operation->Parent(); - if (request != NULL) - request->OperationFinished(operation, operation->Status()); + if (request != NULL) { + size_t operationOffset = operation->OriginalOffset() + - request->Offset(); + request->OperationFinished(operation, operation->Status(), + operation->TransferredBytes() < operation->OriginalLength(), + operation->Status() == B_OK + ? operationOffset + operation->OriginalLength() + : operationOffset); + } // recycle the operation MutexLocker _(fLock); diff --git a/src/system/kernel/device_manager/IOScheduler.h b/src/system/kernel/device_manager/IOScheduler.h index 3e802b3798..0ecf5f978c 100644 --- a/src/system/kernel/device_manager/IOScheduler.h +++ b/src/system/kernel/device_manager/IOScheduler.h @@ -39,7 +39,7 @@ public: void AbortRequest(IORequest* request, status_t status = B_CANCELED); void OperationCompleted(IOOperation* operation, - status_t status); + status_t status, size_t transferredBytes); // called by the driver when the operation // has been completed successfully or failed // for some reason diff --git a/src/system/kernel/device_manager/devfs.cpp b/src/system/kernel/device_manager/devfs.cpp index d53d432f41..c970ceae66 100644 --- a/src/system/kernel/device_manager/devfs.cpp +++ b/src/system/kernel/device_manager/devfs.cpp @@ -756,26 +756,18 @@ get_device_name(struct devfs_vnode* vnode, char* buffer, size_t size) static status_t -device_read(void* _cookie, off_t offset, void* buffer, size_t length) +device_read(void* _cookie, off_t offset, void* buffer, size_t* length) { synchronous_io_cookie* cookie = (synchronous_io_cookie*)_cookie; - - size_t transferred = length; - status_t error = cookie->device->Read(cookie->cookie, offset, buffer, - &transferred); - return error == B_OK && transferred != length ? B_FILE_ERROR : error; + return cookie->device->Read(cookie->cookie, offset, buffer, length); } static status_t -device_write(void* _cookie, off_t offset, void* buffer, size_t length) +device_write(void* _cookie, off_t offset, void* buffer, size_t* length) { synchronous_io_cookie* cookie = (synchronous_io_cookie*)_cookie; - - size_t transferred = length; - status_t error = cookie->device->Write(cookie->cookie, offset, buffer, - &transferred); - return error == B_OK && transferred != length ? B_FILE_ERROR : error; + return cookie->device->Write(cookie->cookie, offset, buffer, length); } @@ -1753,6 +1745,8 @@ static status_t devfs_io(fs_volume *volume, fs_vnode *_vnode, void *_cookie, io_request *request) { + TRACE(("[%ld] devfs_io(request: %p)\n", find_thread(NULL), request)); + devfs_vnode* vnode = (devfs_vnode*)_vnode->private_node; devfs_cookie* cookie = (devfs_cookie*)_cookie; diff --git a/src/system/kernel/device_manager/dma_resources.cpp b/src/system/kernel/device_manager/dma_resources.cpp index 518bedb6d1..aea184e001 100644 --- a/src/system/kernel/device_manager/dma_resources.cpp +++ b/src/system/kernel/device_manager/dma_resources.cpp @@ -12,7 +12,7 @@ #include "io_requests.h" -#define TRACE_DMA_RESOURCE +//#define TRACE_DMA_RESOURCE #ifdef TRACE_DMA_RESOURCE # define TRACE(x...) dprintf(x) #else diff --git a/src/system/kernel/device_manager/io_requests.cpp b/src/system/kernel/device_manager/io_requests.cpp index 8215d2f620..9e837423c2 100644 --- a/src/system/kernel/device_manager/io_requests.cpp +++ b/src/system/kernel/device_manager/io_requests.cpp @@ -16,7 +16,7 @@ #include "dma_resources.h" -#define TRACE_IO_REQUEST +//#define TRACE_IO_REQUEST #ifdef TRACE_IO_REQUEST # define TRACE(x...) dprintf(x) #else @@ -237,6 +237,8 @@ IOOperation::Prepare(IORequest* request) fParent = request; + fTransferredBytes = 0; + // set initial phase fPhase = PHASE_DO_ALL; if (fParent->IsWrite()) { @@ -522,8 +524,8 @@ IORequest::Init(off_t offset, void* buffer, size_t length, bool write, status_t -IORequest::Init(off_t offset, size_t firstVecOffset, iovec* vecs, size_t count, - size_t length, bool write, uint32 flags) +IORequest::Init(off_t offset, size_t firstVecOffset, const iovec* vecs, + size_t count, size_t length, bool write, uint32 flags) { fBuffer = IOBuffer::Create(count); if (fBuffer == NULL) @@ -533,9 +535,12 @@ IORequest::Init(off_t offset, size_t firstVecOffset, iovec* vecs, size_t count, fOffset = offset; fLength = length; + fRelativeParentOffset = 0; + fTransferSize = 0; fFlags = flags; fTeam = team_get_current_team_id(); fIsWrite = write; + fPartialTransfer = 0; // these are for iteration fVecIndex = 0; @@ -596,6 +601,8 @@ IORequest::CreateSubRequest(off_t parentOffset, off_t offset, size_t length, return error; } + subRequest->fRelativeParentOffset = parentOffset - fOffset; + _subRequest = subRequest; subRequest->SetParent(this); @@ -671,20 +678,23 @@ IORequest::NotifyFinished() MutexLocker locker(fLock); - if (fStatus == B_OK && RemainingBytes() > 0) { + if (fStatus == B_OK && !fPartialTransfer && RemainingBytes() > 0) { // The request is not really done yet. If it has an iteration callback, // call it. if (fIterationCallback != NULL) { ResetStatus(); locker.Unlock(); - status_t error = fIterationCallback(fIterationCookie, this); - if (error == B_OK) + bool partialTransfer = false; + status_t error = fIterationCallback(fIterationCookie, this, + &partialTransfer); + if (error == B_OK && !partialTransfer) return; // Iteration failed, which means we're responsible for notifying the // requests finished. locker.Lock(); fStatus = error; + fPartialTransfer = true; } } @@ -695,6 +705,8 @@ IORequest::NotifyFinished() io_request_finished_callback finishedCallback = fFinishedCallback; void* finishedCookie = fFinishedCookie; status_t status = fStatus; + size_t lastTransferredOffset = fRelativeParentOffset + fTransferSize; + bool partialTransfer = status != B_OK || fPartialTransfer; // unblock waiters fFinishedCondition.NotifyAll(); @@ -702,12 +714,16 @@ IORequest::NotifyFinished() locker.Unlock(); // notify callback - if (finishedCallback != NULL) - finishedCallback(finishedCookie, this, status); + if (finishedCallback != NULL) { + finishedCallback(finishedCookie, this, status, partialTransfer, + lastTransferredOffset); + } // notify parent - if (parent != NULL) - parent->SubRequestFinished(this, status); + if (parent != NULL) { + parent->SubRequestFinished(this, status, partialTransfer, + lastTransferredOffset); + } } @@ -742,7 +758,8 @@ IORequest::SetStatusAndNotify(status_t status) void -IORequest::OperationFinished(IOOperation* operation, status_t status) +IORequest::OperationFinished(IOOperation* operation, status_t status, + bool partialTransfer, size_t transferEndOffset) { TRACE("IORequest::OperationFinished(%p, %#lx): request: %p\n", operation, status, this); @@ -752,6 +769,12 @@ IORequest::OperationFinished(IOOperation* operation, status_t status) fChildren.Remove(operation); operation->SetParent(NULL); + if (status != B_OK || partialTransfer) { + if (fTransferSize > transferEndOffset) + fTransferSize = transferEndOffset; + fPartialTransfer = true; + } + if (status != B_OK && fStatus == 1) fStatus = status; @@ -773,13 +796,20 @@ IORequest::OperationFinished(IOOperation* operation, status_t status) void -IORequest::SubRequestFinished(IORequest* request, status_t status) +IORequest::SubRequestFinished(IORequest* request, status_t status, + bool partialTransfer, size_t transferEndOffset) { - TRACE("IORequest::SubrequestFinished(%p, %#lx): request: %p\n", request, - status, this); + TRACE("IORequest::SubrequestFinished(%p, %#lx, %d, %lu): request: %p\n", + request, status, partialTransfer, transferEndOffset, this); MutexLocker locker(fLock); + if (status != B_OK || partialTransfer) { + if (fTransferSize > transferEndOffset) + fTransferSize = transferEndOffset; + fPartialTransfer = true; + } + if (status != B_OK && fStatus == 1) fStatus = status; @@ -798,12 +828,26 @@ IORequest::SubRequestFinished(IORequest* request, status_t status) } +void +IORequest::SetTransferredBytes(bool partialTransfer, size_t transferredBytes) +{ + TRACE("%p->IORequest::SetTransferredBytes(%d, %lu)\n", this, + partialTransfer, transferredBytes); + + MutexLocker _(fLock); + + fPartialTransfer = partialTransfer; + fTransferSize = transferredBytes; +} + + void IORequest::Advance(size_t bySize) { TRACE("IORequest::Advance(%lu): remaining: %lu -> %lu\n", bySize, fRemainingBytes, fRemainingBytes - bySize); fRemainingBytes -= bySize; + fTransferSize += bySize; iovec* vecs = fBuffer->Vecs(); while (vecs[fVecIndex].iov_len - fVecOffset <= bySize) { diff --git a/src/system/kernel/device_manager/io_requests.h b/src/system/kernel/device_manager/io_requests.h index 0c3787d200..5a3b482a63 100644 --- a/src/system/kernel/device_manager/io_requests.h +++ b/src/system/kernel/device_manager/io_requests.h @@ -125,6 +125,11 @@ public: size_t OriginalLength() const { return fOriginalLength; } + size_t TransferredBytes() const + { return fTransferredBytes; } + void SetTransferredBytes(size_t bytes) + { fTransferredBytes = bytes; } + iovec* Vecs() const; uint32 VecCount() const; @@ -159,6 +164,7 @@ protected: off_t fOriginalOffset; size_t fLength; size_t fOriginalLength; + size_t fTransferredBytes; size_t fBlockSize; uint16 fSavedVecIndex; uint16 fSavedVecLength; @@ -174,9 +180,10 @@ typedef DoublyLinkedList IOOperationList; typedef struct IORequest io_request; typedef status_t (*io_request_finished_callback)(void* data, - io_request* request, status_t status); + io_request* request, status_t status, bool partialTransfer, + size_t transferEndOffset); typedef status_t (*io_request_iterate_callback)(void* data, - io_request* request); + io_request* request, bool* _partialTransfer); struct IORequest : IORequestChunk, DoublyLinkedListLinkImpl { @@ -185,13 +192,14 @@ struct IORequest : IORequestChunk, DoublyLinkedListLinkImpl { status_t Init(off_t offset, void* buffer, size_t length, bool write, uint32 flags); - status_t Init(off_t offset, iovec* vecs, size_t count, - size_t length, bool write, uint32 flags) + status_t Init(off_t offset, const iovec* vecs, + size_t count, size_t length, bool write, + uint32 flags) { return Init(offset, 0, vecs, count, length, write, flags); } status_t Init(off_t offset, size_t firstVecOffset, - iovec* vecs, size_t count, size_t length, - bool write, uint32 flags); + const iovec* vecs, size_t count, + size_t length, bool write, uint32 flags); status_t CreateSubRequest(off_t parentOffset, off_t offset, size_t length, @@ -217,12 +225,20 @@ struct IORequest : IORequestChunk, DoublyLinkedListLinkImpl { void SetStatusAndNotify(status_t status); void OperationFinished(IOOperation* operation, - status_t status); + status_t status, bool partialTransfer, + size_t transferEndOffset); void SubRequestFinished(IORequest* request, - status_t status); + status_t status, bool partialTransfer, + size_t transferEndOffset); size_t RemainingBytes() const { return fRemainingBytes; } + size_t TransferredBytes() const + { return fTransferSize; } + bool IsPartialTransfer() const + { return fPartialTransfer; } + void SetTransferredBytes(bool partialTransfer, + size_t transferredBytes); bool IsWrite() const { return fIsWrite; } bool IsRead() const { return !fIsWrite; } @@ -265,11 +281,20 @@ private: IOBuffer* fBuffer; off_t fOffset; size_t fLength; + size_t fTransferSize; + // After all subrequests/operations have + // finished, number of contiguous bytes at + // the beginning of the request that have + // actually been transferred. + size_t fRelativeParentOffset; + // offset of this request relative to its + // parent IORequestChunkList fChildren; int32 fPendingChildren; uint32 fFlags; team_id fTeam; bool fIsWrite; + bool fPartialTransfer; io_request_finished_callback fFinishedCallback; void* fFinishedCookie; diff --git a/src/system/kernel/fs/vfs.cpp b/src/system/kernel/fs/vfs.cpp index d5321fb192..f34ad6338e 100644 --- a/src/system/kernel/fs/vfs.cpp +++ b/src/system/kernel/fs/vfs.cpp @@ -50,6 +50,7 @@ #include #include "fifo.h" +#include "io_requests.h" //#define TRACE_VFS @@ -3899,21 +3900,41 @@ vfs_can_page(struct vnode *vnode, void *cookie) extern "C" status_t vfs_read_pages(struct vnode *vnode, void *cookie, off_t pos, const iovec *vecs, - size_t count, size_t *_numBytes) + size_t count, uint32 flags, size_t *_numBytes) { - FUNCTION(("vfs_read_pages: vnode %p, vecs %p, pos %Ld\n", vnode, vecs, pos)); + FUNCTION(("vfs_read_pages: vnode %p, vecs %p, pos %Ld\n", vnode, vecs, + pos)); - return FS_CALL(vnode, read_pages, cookie, pos, vecs, count, _numBytes); + IORequest request; + status_t status = request.Init(pos, vecs, count, *_numBytes, false, flags); + if (status == B_OK) { + status = vfs_vnode_io(vnode, cookie, &request); + if (status == B_OK) + status = request.Wait(); + *_numBytes = request.TransferredBytes(); + } + + return status; } extern "C" status_t vfs_write_pages(struct vnode *vnode, void *cookie, off_t pos, const iovec *vecs, - size_t count, size_t *_numBytes) + size_t count, uint32 flags, size_t *_numBytes) { - FUNCTION(("vfs_write_pages: vnode %p, vecs %p, pos %Ld\n", vnode, vecs, pos)); + FUNCTION(("vfs_write_pages: vnode %p, vecs %p, pos %Ld\n", vnode, vecs, + pos)); - return FS_CALL(vnode, write_pages, cookie, pos, vecs, count, _numBytes); + IORequest request; + status_t status = request.Init(pos, vecs, count, *_numBytes, true, flags); + if (status == B_OK) { + status = vfs_vnode_io(vnode, cookie, &request); + if (status == B_OK) + status = request.Wait(); + *_numBytes = request.TransferredBytes(); + } + + return status; } diff --git a/src/system/kernel/fs/vfs_request_io.cpp b/src/system/kernel/fs/vfs_request_io.cpp index efc5562bc4..4834dacd79 100644 --- a/src/system/kernel/fs/vfs_request_io.cpp +++ b/src/system/kernel/fs/vfs_request_io.cpp @@ -5,7 +5,13 @@ // included by vfs.cpp -#include "io_requests.h" + +//#define TRACE_VFS_REQUEST_IO +#ifdef TRACE_VFS_REQUEST_IO +# define TRACE_RIO(x...) dprintf(x) +#else +# define TRACE_RIO(x...) do {} while (false) +#endif struct iterative_io_cookie { @@ -33,13 +39,14 @@ public: { } - status_t IO(off_t offset, void* _buffer, size_t length) + status_t IO(off_t offset, void* _buffer, size_t* _length) { if (!fIsPhysical) - return InternalIO(offset, _buffer, length); + return InternalIO(offset, _buffer, _length); // buffer points to physical address -- map pages addr_t buffer = (addr_t)_buffer; + size_t length = *_length; while (length > 0) { addr_t pageOffset = buffer % B_PAGE_SIZE; @@ -50,23 +57,29 @@ public: return error; size_t toTransfer = min_c(length, B_PAGE_SIZE - pageOffset); + size_t transferred = toTransfer; error = InternalIO(offset, (void*)(virtualAddress + pageOffset), - toTransfer); + &transferred); vm_put_physical_page(virtualAddress); if (error != B_OK) return error; - buffer += toTransfer; - length -= toTransfer; + offset += transferred; + buffer += transferred; + length -= transferred; + + if (transferred != toTransfer) + break; } + *_length -= length; return B_OK; } protected: - virtual status_t InternalIO(off_t offset, void* buffer, size_t length) = 0; + virtual status_t InternalIO(off_t offset, void* buffer, size_t* length) = 0; protected: bool fWrite; @@ -78,7 +91,7 @@ class CallbackIO : public DoIO { public: CallbackIO(bool write, bool isPhysical, status_t (*doIO)(void* cookie, off_t offset, void* buffer, - size_t length), + size_t* length), void* cookie) : DoIO(write, isPhysical), @@ -88,13 +101,13 @@ public: } protected: - virtual status_t InternalIO(off_t offset, void* buffer, size_t length) + virtual status_t InternalIO(off_t offset, void* buffer, size_t* length) { return fDoIO(fCookie, offset, buffer, length); } private: - status_t (*fDoIO)(void*, off_t, void*, size_t); + status_t (*fDoIO)(void*, off_t, void*, size_t*); void* fCookie; }; @@ -110,20 +123,12 @@ public: } protected: - virtual status_t InternalIO(off_t offset, void* buffer, size_t length) + virtual status_t InternalIO(off_t offset, void* buffer, size_t* length) { - size_t bytesTransferred = length; - status_t error; - if (fWrite) { - error = FS_CALL(fVnode, write, fCookie, offset, buffer, - &bytesTransferred); - } else { - error = FS_CALL(fVnode, read, fCookie, offset, buffer, - &bytesTransferred); - } + if (fWrite) + return FS_CALL(fVnode, write, fCookie, offset, buffer, length); - return error == B_OK && bytesTransferred != length - ? B_FILE_ERROR : error; + return FS_CALL(fVnode, read, fCookie, offset, buffer, length); } private: @@ -133,8 +138,12 @@ private: static status_t -do_iterative_fd_io_iterate(void* _cookie, io_request* request) +do_iterative_fd_io_iterate(void* _cookie, io_request* request, + bool* _partialTransfer) { + TRACE_RIO("[%ld] do_iterative_fd_io_iterate(request: %p)\n", + find_thread(NULL), request); + static const int32 kMaxSubRequests = 8; iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie; @@ -152,16 +161,23 @@ do_iterative_fd_io_iterate(void* _cookie, io_request* request) requestLength, vecs, &vecCount); if (error != B_OK) return error; - if (vecCount == 0) - return B_FILE_ERROR; + if (vecCount == 0) { + *_partialTransfer = true; + return B_OK; + } + TRACE_RIO("[%ld] got %lu file vecs\n", find_thread(NULL), vecCount); // create subrequests for the file vecs we've got int32 subRequestCount = 0; for (uint32 i = 0; i < vecCount && subRequestCount < kMaxSubRequests; i++) { off_t vecOffset = vecs[i].offset; - off_t vecLength = vecs[i].length; + off_t vecLength = min_c(vecs[i].length, requestLength); + TRACE_RIO("[%ld] vec %lu offset: %lld, length: %lld\n", + find_thread(NULL), i, vecOffset, vecLength); while (vecLength > 0 && subRequestCount < kMaxSubRequests) { + TRACE_RIO("[%ld] creating subrequest: offset: %lld, length: " + "%lld\n", find_thread(NULL), vecOffset, vecLength); IORequest* subRequest; error = request->CreateSubRequest(requestOffset, vecOffset, vecLength, subRequest); @@ -186,9 +202,14 @@ do_iterative_fd_io_iterate(void* _cookie, io_request* request) cookie->request_offset = requestOffset; // Schedule the subrequests. - for (IORequest* subRequest = request->FirstSubRequest(); - (subRequest = request->NextSubRequest(subRequest)) != NULL;) { + IORequest* nextSubRequest = request->FirstSubRequest(); + while (nextSubRequest != NULL) { + IORequest* subRequest = nextSubRequest; + nextSubRequest = request->NextSubRequest(subRequest); + if (error == B_OK) { + TRACE_RIO("[%ld] scheduling subrequest: %p\n", find_thread(NULL), + subRequest); error = FS_CALL(cookie->vnode, io, cookie->descriptor->cookie, subRequest); } else { @@ -205,18 +226,21 @@ do_iterative_fd_io_iterate(void* _cookie, io_request* request) static status_t -do_iterative_fd_io_finish(void* _cookie, io_request* request, status_t status) +do_iterative_fd_io_finish(void* _cookie, io_request* request, status_t status, + bool partialTransfer, size_t transferEndOffset) { iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie; - if (cookie->finished != NULL) - cookie->finished(cookie->cookie, request, status); + if (cookie->finished != NULL) { + cookie->finished(cookie->cookie, request, status, partialTransfer, + transferEndOffset); + } put_fd(cookie->descriptor); if (cookie->next_finished_callback != NULL) { cookie->next_finished_callback(cookie->next_finished_cookie, request, - status); + status, partialTransfer, transferEndOffset); } delete cookie; @@ -242,35 +266,39 @@ do_synchronous_iterative_vnode_io(struct vnode* vnode, void* openCookie, for (int32 i = 0; error == B_OK && length > 0 && i < vecCount; i++) { uint8* vecBase = (uint8*)vecs[i].iov_base; - size_t vecLength = vecs[i].iov_len; - if (vecLength > length) - vecLength = length; + size_t vecLength = min_c(vecs[i].iov_len, length); while (error == B_OK && vecLength > 0) { file_io_vec fileVecs[8]; uint32 fileVecCount = 8; error = getVecs(cookie, request, offset, vecLength, fileVecs, &fileVecCount); - if (error == B_OK && fileVecCount == 0) - error = B_FILE_ERROR; - if (error != B_OK) + if (error != B_OK || fileVecCount == 0) break; for (uint32 k = 0; k < fileVecCount; k++) { const file_io_vec& fileVec = fileVecs[i]; - error = io.IO(fileVec.offset, vecBase, fileVec.length); + size_t toTransfer = min_c(fileVec.length, length); + size_t transferred = toTransfer; + error = io.IO(fileVec.offset, vecBase, &transferred); if (error != B_OK) break; - offset += fileVec.length; - length -= fileVec.length; - vecBase += fileVec.length; - vecLength -= fileVec.length; + offset += transferred; + length -= transferred; + vecBase += transferred; + vecLength -= transferred; + + if (transferred != toTransfer) + break; } } } - finished(cookie, request, error); + bool partial = length > 0; + size_t bytesTransferred = request->Length() - length; + request->SetTransferredBytes(partial, bytesTransferred); + finished(cookie, request, error, partial, bytesTransferred); request->SetStatusAndNotify(error); return error; } @@ -279,6 +307,9 @@ do_synchronous_iterative_vnode_io(struct vnode* vnode, void* openCookie, static status_t synchronous_io(io_request* request, DoIO& io) { + TRACE_RIO("[%ld] synchronous_io(request: %p (offset: %lld, length: %lu))\n", + find_thread(NULL), request, request->Offset(), request->Length()); + IOBuffer* buffer = request->Buffer(); iovec* vecs = buffer->Vecs(); @@ -288,20 +319,29 @@ synchronous_io(io_request* request, DoIO& io) for (int32 i = 0; length > 0 && i < vecCount; i++) { void* vecBase = vecs[i].iov_base; - size_t vecLength = vecs[i].iov_len; - if (vecLength > length) - vecLength = length; + size_t vecLength = min_c(vecs[i].iov_len, length); - status_t error = io.IO(offset, vecBase, vecLength); + TRACE_RIO("[%ld] I/O: offset: %lld, vecBase: %p, length: %lu\n", + find_thread(NULL), offset, vecBase, vecLength); + + size_t transferred = vecLength; + status_t error = io.IO(offset, vecBase, &transferred); if (error != B_OK) { + TRACE_RIO("[%ld] I/O failed: %#lx\n", find_thread(NULL), error); request->SetStatusAndNotify(error); return error; } - offset += vecLength; - length -= vecLength; + offset += transferred; + length -= transferred; + + if (transferred != length) + break; } + TRACE_RIO("[%ld] synchronous_io() succeeded\n", find_thread(NULL)); + + request->SetTransferredBytes(length > 0, request->Length() - length); request->SetStatusAndNotify(B_OK); return B_OK; } @@ -326,7 +366,7 @@ vfs_vnode_io(struct vnode* vnode, void* cookie, io_request* request) status_t vfs_synchronous_io(io_request* request, - status_t (*doIO)(void* cookie, off_t offset, void* buffer, size_t length), + status_t (*doIO)(void* cookie, off_t offset, void* buffer, size_t* length), void* cookie) { IOBuffer* buffer = request->Buffer(); @@ -359,10 +399,14 @@ status_t do_iterative_fd_io(int fd, io_request* request, iterative_io_get_vecs getVecs, iterative_io_finished finished, void* cookie) { + TRACE_RIO("[%ld] do_iterative_fd_io(fd: %d, request: %p (offset: %lld, " + "length: %ld))\n", find_thread(NULL), fd, request, request->Offset(), + request->Length()); + struct vnode* vnode; file_descriptor* descriptor = get_fd_and_vnode(fd, &vnode, true); if (descriptor == NULL) { - finished(cookie, request, B_FILE_ERROR); + finished(cookie, request, B_FILE_ERROR, true, 0); request->SetStatusAndNotify(B_FILE_ERROR); return B_FILE_ERROR; } @@ -396,9 +440,17 @@ do_iterative_fd_io(int fd, io_request* request, iterative_io_get_vecs getVecs, request->SetFinishedCallback(&do_iterative_fd_io_finish, iterationCookie); request->SetIterationCallback(&do_iterative_fd_io_iterate, iterationCookie); - status_t error = do_iterative_fd_io_iterate(iterationCookie, request); - if (error != B_OK) { - finished(cookie, request, error); + bool partialTransfer = false; + status_t error = do_iterative_fd_io_iterate(iterationCookie, request, + &partialTransfer); + if (error != B_OK || partialTransfer) { + if (partialTransfer) { + request->SetTransferredBytes(partialTransfer, + request->TransferredBytes()); + } + + finished(cookie, request, error, request->IsPartialTransfer(), + request->TransferredBytes()); request->SetStatusAndNotify(error); return error; } diff --git a/src/tests/system/kernel/device_manager/dma_resource_test.cpp b/src/tests/system/kernel/device_manager/dma_resource_test.cpp index 3cb5b0bda9..fb871d575c 100644 --- a/src/tests/system/kernel/device_manager/dma_resource_test.cpp +++ b/src/tests/system/kernel/device_manager/dma_resource_test.cpp @@ -200,7 +200,7 @@ do_io(void* data, IOOperation* operation) } if (sIOScheduler != NULL) - sIOScheduler->OperationCompleted(operation, B_OK); + sIOScheduler->OperationCompleted(operation, B_OK, operation->Length()); return B_OK; } @@ -584,7 +584,10 @@ Test::Run(DMAResource& resource) } } - request.OperationFinished(&operation, operation.Status()); + request.OperationFinished(&operation, operation.Status(), + false, + operation.OriginalOffset() - operation.Parent()->Offset() + + operation.OriginalLength()); resultIndex++; } @@ -959,7 +962,7 @@ run_test() // #pragma mark - driver -float +static float dma_test_supports_device(device_node *parent) { const char* bus = NULL; @@ -971,7 +974,7 @@ dma_test_supports_device(device_node *parent) } -status_t +static status_t dma_test_register_device(device_node *parent) { device_attr attrs[] = { @@ -984,7 +987,7 @@ dma_test_register_device(device_node *parent) } -status_t +static status_t dma_test_init_driver(device_node *node, void **_driverCookie) { sAreaSize = 10 * 1024 * 1024; @@ -1000,14 +1003,14 @@ dma_test_init_driver(device_node *node, void **_driverCookie) } -void +static void dma_test_uninit_driver(void *driverCookie) { delete_area(sArea); } -status_t +static status_t dma_test_register_child_devices(void *driverCookie) { return sDeviceManager->publish_device((device_node*)driverCookie, @@ -1019,7 +1022,7 @@ dma_test_register_child_devices(void *driverCookie) // #pragma mark - device -status_t +static status_t dma_test_init_device(void *driverCookie, void **_deviceCookie) { const dma_restrictions restrictions = { @@ -1063,13 +1066,13 @@ dma_test_init_device(void *driverCookie, void **_deviceCookie) } -void +static void dma_test_uninit_device(void *deviceCookie) { } -status_t +static status_t dma_test_open(void *deviceCookie, const char *path, int openMode, void **_cookie) { @@ -1077,21 +1080,21 @@ dma_test_open(void *deviceCookie, const char *path, int openMode, } -status_t +static status_t dma_test_close(void *cookie) { return B_OK; } -status_t +static status_t dma_test_free(void *cookie) { return B_OK; } -status_t +static status_t dma_test_read(void *cookie, off_t pos, void *buffer, size_t *_length) { size_t length = *_length; @@ -1123,7 +1126,7 @@ dma_test_read(void *cookie, off_t pos, void *buffer, size_t *_length) } -status_t +static status_t dma_test_write(void *cookie, off_t pos, const void *buffer, size_t *_length) { size_t length = *_length; @@ -1157,14 +1160,16 @@ dma_test_write(void *cookie, off_t pos, const void *buffer, size_t *_length) } -status_t +static status_t dma_test_io(void *cookie, io_request *request) { - return B_BAD_VALUE; + dprintf("dma_test_io(%p)\n", request); + + return sIOScheduler->ScheduleRequest(request); } -status_t +static status_t dma_test_control(void *cookie, uint32 op, void *buffer, size_t length) { switch (op) { @@ -1219,7 +1224,7 @@ module_dependency module_dependencies[] = { }; -const static struct driver_module_info sDMATestDriverModule = { +static const struct driver_module_info sDMATestDriverModule = { { "drivers/disk/dma_resource_test/driver_v1", 0, @@ -1233,7 +1238,7 @@ const static struct driver_module_info sDMATestDriverModule = { dma_test_register_child_devices }; -const static struct device_module_info sDMATestDeviceModule = { +static const struct device_module_info sDMATestDeviceModule = { { "drivers/disk/dma_resource_test/device_v1", 0, @@ -1250,7 +1255,7 @@ const static struct device_module_info sDMATestDeviceModule = { dma_test_read, dma_test_write, - NULL, // io + dma_test_io, dma_test_control,