From aa4ba93e25c1c63730ba69e04d3d96c3253924fd Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Sun, 8 Mar 2009 17:30:37 +0000 Subject: [PATCH] * Renamed src/system/kernel/device_manager/io_requests.{h,cpp} to IORequest.{h,cpp}. * Introduced public header. Currently it only declares the single function BFS uses. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@29446 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/os/drivers/io_requests.h | 27 + .../drivers/disk/scsi/scsi_cd/scsi_cd.h | 2 +- .../drivers/disk/scsi/scsi_disk/scsi_disk.h | 2 +- .../file_systems/bfs/kernel_interface.cpp | 4 +- .../generic/scsi_periph/scsi_periph_int.h | 2 +- src/system/kernel/cache/file_cache.cpp | 2 +- src/system/kernel/cache/vnode_store.cpp | 2 +- .../kernel/device_manager/IORequest.cpp | 1405 +++++++++++++++++ .../{io_requests.h => IORequest.h} | 6 +- .../kernel/device_manager/IOScheduler.h | 2 +- src/system/kernel/device_manager/Jamfile | 1 + src/system/kernel/device_manager/devfs.cpp | 2 +- .../kernel/device_manager/device_manager.cpp | 2 +- .../kernel/device_manager/dma_resources.cpp | 2 +- .../kernel/device_manager/io_requests.cpp | 1400 +--------------- src/system/kernel/fs/vfs.cpp | 2 +- src/system/kernel/vm/VMAnonymousCache.cpp | 2 +- src/system/kernel/vm/vm.cpp | 2 +- src/system/kernel/vm/vm_page.cpp | 2 +- 19 files changed, 1456 insertions(+), 1413 deletions(-) create mode 100644 headers/os/drivers/io_requests.h create mode 100644 src/system/kernel/device_manager/IORequest.cpp rename src/system/kernel/device_manager/{io_requests.h => IORequest.h} (99%) diff --git a/headers/os/drivers/io_requests.h b/headers/os/drivers/io_requests.h new file mode 100644 index 0000000000..188af64908 --- /dev/null +++ b/headers/os/drivers/io_requests.h @@ -0,0 +1,27 @@ +/* + * Copyright 2009, Haiku Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + */ +#ifndef _IO_REQUESTS_H +#define _IO_REQUESTS_H + +/*! I/O request interface */ + + +#include + + +typedef struct IORequest io_request; + + +#ifdef __cplusplus +extern "C" { +#endif + +bool io_request_is_write(const io_request* request); + +#ifdef __cplusplus +} +#endif + +#endif /* _IO_REQUESTS_H */ diff --git a/src/add-ons/kernel/drivers/disk/scsi/scsi_cd/scsi_cd.h b/src/add-ons/kernel/drivers/disk/scsi/scsi_cd/scsi_cd.h index a8e19af81f..8671b46645 100644 --- a/src/add-ons/kernel/drivers/disk/scsi/scsi_cd/scsi_cd.h +++ b/src/add-ons/kernel/drivers/disk/scsi/scsi_cd/scsi_cd.h @@ -12,7 +12,7 @@ #include #include "dma_resources.h" -#include "io_requests.h" +#include "IORequest.h" #include "IOScheduler.h" diff --git a/src/add-ons/kernel/drivers/disk/scsi/scsi_disk/scsi_disk.h b/src/add-ons/kernel/drivers/disk/scsi/scsi_disk/scsi_disk.h index 20af9fd5c2..7a0f1dec67 100644 --- a/src/add-ons/kernel/drivers/disk/scsi/scsi_disk/scsi_disk.h +++ b/src/add-ons/kernel/drivers/disk/scsi/scsi_disk/scsi_disk.h @@ -12,7 +12,7 @@ #include #include "dma_resources.h" -#include "io_requests.h" +#include "IORequest.h" #include "IOScheduler.h" diff --git a/src/add-ons/kernel/file_systems/bfs/kernel_interface.cpp b/src/add-ons/kernel/file_systems/bfs/kernel_interface.cpp index 81682ee15b..8328db757c 100644 --- a/src/add-ons/kernel/file_systems/bfs/kernel_interface.cpp +++ b/src/add-ons/kernel/file_systems/bfs/kernel_interface.cpp @@ -18,7 +18,7 @@ // TODO: temporary solution as long as there is no public I/O requests API #ifndef BFS_SHELL -# include "io_requests.h" +# include #endif #define BFS_IO_SIZE 65536 @@ -458,7 +458,7 @@ bfs_io(fs_volume* _volume, fs_vnode* _node, void* _cookie, io_request* request) Inode* inode = (Inode*)_node->private_node; #ifndef BFS_SHELL - if (request->IsWrite() && volume->IsReadOnly()) + if (io_request_is_write(request) && volume->IsReadOnly()) return B_READ_ONLY_DEVICE; #endif diff --git a/src/add-ons/kernel/generic/scsi_periph/scsi_periph_int.h b/src/add-ons/kernel/generic/scsi_periph/scsi_periph_int.h index c4de9c1ac2..8c83aee933 100644 --- a/src/add-ons/kernel/generic/scsi_periph/scsi_periph_int.h +++ b/src/add-ons/kernel/generic/scsi_periph/scsi_periph_int.h @@ -12,7 +12,7 @@ #include #include -#include "io_requests.h" +#include "IORequest.h" #include "wrapper.h" diff --git a/src/system/kernel/cache/file_cache.cpp b/src/system/kernel/cache/file_cache.cpp index 659f47482a..b337b8579c 100644 --- a/src/system/kernel/cache/file_cache.cpp +++ b/src/system/kernel/cache/file_cache.cpp @@ -25,7 +25,7 @@ #include #include -#include "io_requests.h" +#include "IORequest.h" //#define TRACE_FILE_CACHE diff --git a/src/system/kernel/cache/vnode_store.cpp b/src/system/kernel/cache/vnode_store.cpp index db87caaf8a..ab6f1715f4 100644 --- a/src/system/kernel/cache/vnode_store.cpp +++ b/src/system/kernel/cache/vnode_store.cpp @@ -13,7 +13,7 @@ #include #include -#include "io_requests.h" +#include "IORequest.h" status_t diff --git a/src/system/kernel/device_manager/IORequest.cpp b/src/system/kernel/device_manager/IORequest.cpp new file mode 100644 index 0000000000..a5bd7dc110 --- /dev/null +++ b/src/system/kernel/device_manager/IORequest.cpp @@ -0,0 +1,1405 @@ +/* + * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2008, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + */ + +#include "IORequest.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "dma_resources.h" + + +//#define TRACE_IO_REQUEST +#ifdef TRACE_IO_REQUEST +# define TRACE(x...) dprintf(x) +#else +# define TRACE(x...) ; +#endif + + +#define VIP_HEAP_SIZE 1024 * 1024 + +// partial I/O operation phases +enum { + PHASE_READ_BEGIN = 0, + PHASE_READ_END = 1, + PHASE_DO_ALL = 2 +}; + +heap_allocator* sVIPHeap; + + +// #pragma mark - + + +IORequestChunk::IORequestChunk() + : + fParent(NULL), + fStatus(1) +{ +} + + +IORequestChunk::~IORequestChunk() +{ +} + + +void +IORequestChunk::operator delete(void* address, size_t size) +{ + io_request_free(address); +} + + +// #pragma mark - + + +IOBuffer* +IOBuffer::Create(uint32 count, bool vip) +{ + size_t size = sizeof(IOBuffer) + sizeof(iovec) * (count - 1); + IOBuffer* buffer + = (IOBuffer*)(vip ? vip_io_request_malloc(size) : malloc(size)); + if (buffer == NULL) + return NULL; + + buffer->fCapacity = count; + buffer->fVecCount = 0; + buffer->fUser = false; + buffer->fPhysical = false; + buffer->fVIP = vip; + buffer->fMemoryLocked = false; + + return buffer; +} + + +void +IOBuffer::Delete() +{ + if (this == NULL) + return; + + if (fVIP) + vip_io_request_free(this); + else + free(this); +} + + +void +IOBuffer::SetVecs(size_t firstVecOffset, const iovec* vecs, uint32 count, + size_t length, uint32 flags) +{ + memcpy(fVecs, vecs, sizeof(iovec) * count); + if (count > 0 && firstVecOffset > 0) { + fVecs[0].iov_base = (uint8*)fVecs[0].iov_base + firstVecOffset; + fVecs[0].iov_len -= firstVecOffset; + } + + fVecCount = count; + fLength = length; + fUser = IS_USER_ADDRESS(vecs[0].iov_base); + fPhysical = (flags & B_PHYSICAL_IO_REQUEST) != 0; +} + + +status_t +IOBuffer::LockMemory(team_id team, bool isWrite) +{ + if (fMemoryLocked) { + panic("memory already locked!"); + return B_BAD_VALUE; + } + + for (uint32 i = 0; i < fVecCount; i++) { + status_t status = lock_memory_etc(team, fVecs[i].iov_base, + fVecs[i].iov_len, isWrite ? 0 : B_READ_DEVICE); + if (status != B_OK) { + _UnlockMemory(team, i, isWrite); + return status; + } + } + + fMemoryLocked = true; + return B_OK; +} + + +void +IOBuffer::_UnlockMemory(team_id team, size_t count, bool isWrite) +{ + for (uint32 i = 0; i < count; i++) { + unlock_memory_etc(team, fVecs[i].iov_base, fVecs[i].iov_len, + isWrite ? 0 : B_READ_DEVICE); + } +} + + +void +IOBuffer::UnlockMemory(team_id team, bool isWrite) +{ + if (!fMemoryLocked) { + panic("memory not locked"); + return; + } + + _UnlockMemory(team, fVecCount, isWrite); + fMemoryLocked = false; +} + + +void +IOBuffer::Dump() const +{ + kprintf("IOBuffer at %p\n", this); + + kprintf(" origin: %s\n", fUser ? "user" : "kernel"); + kprintf(" kind: %s\n", fPhysical ? "physical" : "virtual"); + kprintf(" length: %lu\n", fLength); + kprintf(" capacity: %lu\n", fCapacity); + kprintf(" vecs: %lu\n", fVecCount); + + for (uint32 i = 0; i < fVecCount; i++) { + kprintf(" [%lu] %p, %lu\n", i, fVecs[i].iov_base, fVecs[i].iov_len); + } +} + + +// #pragma mark - + + +bool +IOOperation::Finish() +{ + TRACE("IOOperation::Finish()\n"); + if (fStatus == B_OK) { + if (fParent->IsWrite()) { + TRACE(" is write\n"); + if (fPhase == PHASE_READ_BEGIN) { + TRACE(" phase read begin\n"); + // repair phase adjusted vec + fDMABuffer->VecAt(fSavedVecIndex).iov_len = fSavedVecLength; + + // partial write: copy partial begin to bounce buffer + bool skipReadEndPhase; + status_t error = _CopyPartialBegin(true, skipReadEndPhase); + if (error == B_OK) { + // We're done with the first phase only (read in begin). + // Get ready for next phase... + fPhase = HasPartialEnd() && !skipReadEndPhase + ? PHASE_READ_END : PHASE_DO_ALL; + _PrepareVecs(); + ResetStatus(); + // TODO: Is there a race condition, if the request is + // aborted at the same time? + return false; + } + + SetStatus(error); + } else if (fPhase == PHASE_READ_END) { + TRACE(" phase read end\n"); + // repair phase adjusted vec + iovec& vec = fDMABuffer->VecAt(fSavedVecIndex); + vec.iov_base = (uint8*)vec.iov_base + + vec.iov_len - fSavedVecLength; + vec.iov_len = fSavedVecLength; + + // partial write: copy partial end to bounce buffer + status_t error = _CopyPartialEnd(true); + if (error == B_OK) { + // We're done with the second phase only (read in end). + // Get ready for next phase... + fPhase = PHASE_DO_ALL; + ResetStatus(); + // TODO: Is there a race condition, if the request is + // aborted at the same time? + return false; + } + + SetStatus(error); + } + } + } + + if (fParent->IsRead() && UsesBounceBuffer()) { + TRACE(" read with bounce buffer\n"); + // copy the bounce buffer segments to the final location + uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress(); + addr_t bounceBufferStart = fDMABuffer->PhysicalBounceBufferAddress(); + addr_t bounceBufferEnd = bounceBufferStart + + fDMABuffer->BounceBufferSize(); + + const iovec* vecs = fDMABuffer->Vecs(); + uint32 vecCount = fDMABuffer->VecCount(); + + status_t error = B_OK; + + off_t offset = fOffset; + off_t startOffset = fOriginalOffset; + off_t endOffset = fOriginalOffset + fOriginalLength; + + for (uint32 i = 0; error == B_OK && i < vecCount; i++) { + const iovec& vec = vecs[i]; + addr_t base = (addr_t)vec.iov_base; + size_t length = vec.iov_len; + + if (offset < startOffset) { + if (offset + length <= startOffset) { + offset += length; + continue; + } + + size_t diff = startOffset - offset; + base += diff; + length -= diff; + } + + if (offset + length > endOffset) { + if (offset >= endOffset) + break; + + length = endOffset - offset; + } + + if (base >= bounceBufferStart && base < bounceBufferEnd) { + error = fParent->CopyData( + bounceBuffer + (base - bounceBufferStart), offset, length); + } + + offset += length; + } + + if (error != B_OK) + SetStatus(error); + } + + return true; +} + + +/*! Note: SetPartial() must be called first! +*/ +status_t +IOOperation::Prepare(IORequest* request) +{ + if (fParent != NULL) + fParent->RemoveOperation(this); + + fParent = request; + + fTransferredBytes = 0; + + // set initial phase + fPhase = PHASE_DO_ALL; + if (fParent->IsWrite()) { + // Copy data to bounce buffer segments, save the partial begin/end vec, + // which will be copied after their respective read phase. + if (UsesBounceBuffer()) { + TRACE(" write with bounce buffer\n"); + uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress(); + addr_t bounceBufferStart + = fDMABuffer->PhysicalBounceBufferAddress(); + addr_t bounceBufferEnd = bounceBufferStart + + fDMABuffer->BounceBufferSize(); + + const iovec* vecs = fDMABuffer->Vecs(); + uint32 vecCount = fDMABuffer->VecCount(); + size_t vecOffset = 0; + uint32 i = 0; + + off_t offset = fOffset; + off_t endOffset = fOffset + fLength; + + if (HasPartialBegin()) { + // skip first block + size_t toSkip = fBlockSize; + while (toSkip > 0) { + if (vecs[i].iov_len <= toSkip) { + toSkip -= vecs[i].iov_len; + i++; + } else { + vecOffset = toSkip; + break; + } + } + + offset += fBlockSize; + } + + if (HasPartialEnd()) { + // skip last block + size_t toSkip = fBlockSize; + while (toSkip > 0) { + if (vecs[vecCount - 1].iov_len <= toSkip) { + toSkip -= vecs[vecCount - 1].iov_len; + vecCount--; + } else + break; + } + + endOffset -= fBlockSize; + } + + for (; i < vecCount; i++) { + const iovec& vec = vecs[i]; + addr_t base = (addr_t)vec.iov_base + vecOffset; + size_t length = vec.iov_len - vecOffset; + vecOffset = 0; + + if (base >= bounceBufferStart && base < bounceBufferEnd) { + if (offset + length > endOffset) + length = endOffset - offset; + status_t error = fParent->CopyData(offset, + bounceBuffer + (base - bounceBufferStart), length); + if (error != B_OK) + return error; + } + + offset += length; + } + } + + if (HasPartialBegin()) + fPhase = PHASE_READ_BEGIN; + else if (HasPartialEnd()) + fPhase = PHASE_READ_END; + + _PrepareVecs(); + } + + ResetStatus(); + + if (fParent != NULL) + fParent->AddOperation(this); + + return B_OK; +} + + +void +IOOperation::SetOriginalRange(off_t offset, size_t length) +{ + fOriginalOffset = fOffset = offset; + fOriginalLength = fLength = length; +} + + +void +IOOperation::SetRange(off_t offset, size_t length) +{ + fOffset = offset; + fLength = length; +} + + +off_t +IOOperation::Offset() const +{ + return fPhase == PHASE_READ_END ? fOffset + fLength - fBlockSize : fOffset; +} + + +size_t +IOOperation::Length() const +{ + return fPhase == PHASE_DO_ALL ? fLength : fBlockSize; +} + + +iovec* +IOOperation::Vecs() const +{ + switch (fPhase) { + case PHASE_READ_END: + return fDMABuffer->Vecs() + fSavedVecIndex; + case PHASE_READ_BEGIN: + case PHASE_DO_ALL: + default: + return fDMABuffer->Vecs(); + } +} + + +uint32 +IOOperation::VecCount() const +{ + switch (fPhase) { + case PHASE_READ_BEGIN: + return fSavedVecIndex + 1; + case PHASE_READ_END: + return fDMABuffer->VecCount() - fSavedVecIndex; + case PHASE_DO_ALL: + default: + return fDMABuffer->VecCount(); + } +} + + +void +IOOperation::SetPartial(bool partialBegin, bool partialEnd) +{ + TRACE("partial begin %d, end %d\n", partialBegin, partialEnd); + fPartialBegin = partialBegin; + fPartialEnd = partialEnd; +} + + +bool +IOOperation::IsWrite() const +{ + return fParent->IsWrite() && fPhase == PHASE_DO_ALL; +} + + +bool +IOOperation::IsRead() const +{ + return fParent->IsRead(); +} + + +void +IOOperation::_PrepareVecs() +{ + // we need to prepare the vecs for consumption by the drivers + if (fPhase == PHASE_READ_BEGIN) { + iovec* vecs = fDMABuffer->Vecs(); + uint32 vecCount = fDMABuffer->VecCount(); + size_t vecLength = fBlockSize; + for (uint32 i = 0; i < vecCount; i++) { + iovec& vec = vecs[i]; + if (vec.iov_len >= vecLength) { + fSavedVecIndex = i; + fSavedVecLength = vec.iov_len; + vec.iov_len = vecLength; + break; + } + vecLength -= vec.iov_len; + } + } else if (fPhase == PHASE_READ_END) { + iovec* vecs = fDMABuffer->Vecs(); + uint32 vecCount = fDMABuffer->VecCount(); + size_t vecLength = fBlockSize; + for (int32 i = vecCount - 1; i >= 0; i--) { + iovec& vec = vecs[i]; + if (vec.iov_len >= vecLength) { + fSavedVecIndex = i; + fSavedVecLength = vec.iov_len; + vec.iov_base = (uint8*)vec.iov_base + + vec.iov_len - vecLength; + vec.iov_len = vecLength; + break; + } + vecLength -= vec.iov_len; + } + } +} + + +status_t +IOOperation::_CopyPartialBegin(bool isWrite, bool& singleBlockOnly) +{ + size_t relativeOffset = OriginalOffset() - fOffset; + size_t length = fBlockSize - relativeOffset; + + singleBlockOnly = length >= OriginalLength(); + if (singleBlockOnly) + length = OriginalLength(); + + TRACE("_CopyPartialBegin(%s, single only %d)\n", + isWrite ? "write" : "read", singleBlockOnly); + + if (isWrite) { + return fParent->CopyData(OriginalOffset(), + (uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, length); + } else { + return fParent->CopyData( + (uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, + OriginalOffset(), length); + } +} + + +status_t +IOOperation::_CopyPartialEnd(bool isWrite) +{ + TRACE("_CopyPartialEnd(%s)\n", isWrite ? "write" : "read"); + + const iovec& lastVec = fDMABuffer->VecAt(fDMABuffer->VecCount() - 1); + off_t lastVecPos = fOffset + fLength - fBlockSize; + uint8* base = (uint8*)fDMABuffer->BounceBufferAddress() + + ((addr_t)lastVec.iov_base + lastVec.iov_len - fBlockSize + - fDMABuffer->PhysicalBounceBufferAddress()); + // NOTE: this won't work if we don't use the bounce buffer contiguously + // (because of boundary alignments). + size_t length = OriginalOffset() + OriginalLength() - lastVecPos; + + if (isWrite) + return fParent->CopyData(lastVecPos, base, length); + + return fParent->CopyData(base, lastVecPos, length); +} + + +void +IOOperation::Dump() const +{ + kprintf("io_operation at %p\n", this); + + kprintf(" parent: %p\n", fParent); + kprintf(" status: %s\n", strerror(fStatus)); + kprintf(" dma buffer: %p\n", fDMABuffer); + kprintf(" offset: %-8Ld (original: %Ld)\n", fOffset, + fOriginalOffset); + kprintf(" length: %-8lu (original: %lu)\n", fLength, + fOriginalLength); + kprintf(" transferred: %lu\n", fTransferredBytes); + kprintf(" block size: %lu\n", fBlockSize); + kprintf(" saved vec index: %u\n", fSavedVecIndex); + kprintf(" saved vec length: %u\n", fSavedVecLength); + kprintf(" r/w: %s\n", IsWrite() ? "write" : "read"); + kprintf(" phase: %s\n", fPhase == PHASE_READ_BEGIN + ? "read begin" : fPhase == PHASE_READ_END ? "read end" + : fPhase == PHASE_DO_ALL ? "do all" : "unknown"); + kprintf(" partial begin: %s\n", fPartialBegin ? "yes" : "no"); + kprintf(" partial end: %s\n", fPartialEnd ? "yes" : "no"); + kprintf(" bounce buffer: %s\n", fUsesBounceBuffer ? "yes" : "no"); + + set_debug_variable("_parent", (addr_t)fParent); + set_debug_variable("_buffer", (addr_t)fDMABuffer); +} + + +// #pragma mark - + + +IORequest::IORequest() + : + fFinishedCallback(NULL), + fFinishedCookie(NULL), + fIterationCallback(NULL), + fIterationCookie(NULL) +{ + mutex_init(&fLock, "I/O request lock"); + fFinishedCondition.Init(this, "I/O request finished"); +} + + +IORequest::~IORequest() +{ + mutex_lock(&fLock); + DeleteSubRequests(); + fBuffer->Delete(); + mutex_destroy(&fLock); +} + + +/* static */ IORequest* +IORequest::Create(bool vip) +{ + return vip ? new(vip_io_alloc) IORequest : new(std::nothrow) IORequest; +} + + +status_t +IORequest::Init(off_t offset, void* buffer, size_t length, bool write, + uint32 flags) +{ + iovec vec; + vec.iov_base = buffer; + vec.iov_len = length; + return Init(offset, &vec, 1, length, write, flags); +} + + +status_t +IORequest::Init(off_t offset, size_t firstVecOffset, const iovec* vecs, + size_t count, size_t length, bool write, uint32 flags) +{ + fBuffer = IOBuffer::Create(count, (flags & B_VIP_IO_REQUEST) != 0); + if (fBuffer == NULL) + return B_NO_MEMORY; + + fBuffer->SetVecs(firstVecOffset, vecs, count, length, flags); + + fOwner = NULL; + fOffset = offset; + fLength = length; + fRelativeParentOffset = 0; + fTransferSize = 0; + fFlags = flags; + struct thread* thread = thread_get_current_thread(); + fTeam = thread->team->id; + fThread = thread->id; + fIsWrite = write; + fPartialTransfer = 0; + + // these are for iteration + fVecIndex = 0; + fVecOffset = 0; + fRemainingBytes = length; + + fPendingChildren = 0; + + fStatus = 1; + + return B_OK; +} + + +status_t +IORequest::CreateSubRequest(off_t parentOffset, off_t offset, size_t length, + IORequest*& _subRequest) +{ + ASSERT(parentOffset >= fOffset && length <= fLength + && parentOffset - fOffset <= fLength - length); + + // find start vec + size_t vecOffset = parentOffset - fOffset; + iovec* vecs = fBuffer->Vecs(); + int32 vecCount = fBuffer->VecCount(); + int32 startVec = 0; + for (; startVec < vecCount; startVec++) { + const iovec& vec = vecs[startVec]; + if (vecOffset < vec.iov_len) + break; + + vecOffset -= vec.iov_len; + } + + // count vecs + size_t currentVecOffset = vecOffset; + int32 endVec = startVec; + size_t remainingLength = length; + for (; endVec < vecCount; endVec++) { + const iovec& vec = vecs[endVec]; + if (vec.iov_len - currentVecOffset >= remainingLength) + break; + + remainingLength -= vec.iov_len - currentVecOffset; + currentVecOffset = 0; + } + + // create subrequest + IORequest* subRequest = Create((fFlags & B_VIP_IO_REQUEST) != 0); + if (subRequest == NULL) + return B_NO_MEMORY; + + status_t error = subRequest->Init(offset, vecOffset, vecs + startVec, + endVec - startVec + 1, length, fIsWrite, fFlags & ~B_DELETE_IO_REQUEST); + if (error != B_OK) { + delete subRequest; + return error; + } + + subRequest->fRelativeParentOffset = parentOffset - fOffset; + subRequest->fTeam = fTeam; + subRequest->fThread = fThread; + + _subRequest = subRequest; + subRequest->SetParent(this); + + MutexLocker _(fLock); + + fChildren.Add(subRequest); + fPendingChildren++; + TRACE("IORequest::CreateSubRequest(): request: %p, subrequest: %p\n", this, + subRequest); + + return B_OK; +} + + +void +IORequest::DeleteSubRequests() +{ + while (IORequestChunk* chunk = fChildren.RemoveHead()) + delete chunk; +} + + +void +IORequest::SetFinishedCallback(io_request_finished_callback callback, + void* cookie) +{ + fFinishedCallback = callback; + fFinishedCookie = cookie; +} + + +void +IORequest::SetIterationCallback(io_request_iterate_callback callback, + void* cookie) +{ + fIterationCallback = callback; + fIterationCookie = cookie; +} + + +io_request_finished_callback +IORequest::FinishedCallback(void** _cookie) const +{ + if (_cookie != NULL) + *_cookie = fFinishedCookie; + return fFinishedCallback; +} + + +status_t +IORequest::Wait(uint32 flags, bigtime_t timeout) +{ + MutexLocker locker(fLock); + + if (IsFinished()) + return Status(); + + ConditionVariableEntry entry; + fFinishedCondition.Add(&entry); + + locker.Unlock(); + + status_t error = entry.Wait(flags, timeout); + if (error != B_OK) + return error; + + return Status(); +} + + +void +IORequest::NotifyFinished() +{ + TRACE("IORequest::NotifyFinished(): request: %p\n", this); + + MutexLocker locker(fLock); + + if (fStatus == B_OK && !fPartialTransfer && RemainingBytes() > 0) { + // The request is not really done yet. If it has an iteration callback, + // call it. + if (fIterationCallback != NULL) { + ResetStatus(); + locker.Unlock(); + bool partialTransfer = false; + status_t error = fIterationCallback(fIterationCookie, this, + &partialTransfer); + if (error == B_OK && !partialTransfer) + return; + + // Iteration failed, which means we're responsible for notifying the + // requests finished. + locker.Lock(); + fStatus = error; + fPartialTransfer = true; + } + } + + ASSERT(fPendingChildren == 0); + ASSERT(fChildren.IsEmpty() + || dynamic_cast(fChildren.Head()) == NULL); + + // unlock the memory + if (fBuffer->IsMemoryLocked()) + fBuffer->UnlockMemory(fTeam, fIsWrite); + + + // Cache the callbacks before we unblock waiters and unlock. Any of the + // following could delete this request, so we don't want to touch it + // once we have started telling others that it is done. + IORequest* parent = fParent; + io_request_finished_callback finishedCallback = fFinishedCallback; + void* finishedCookie = fFinishedCookie; + status_t status = fStatus; + size_t lastTransferredOffset = fRelativeParentOffset + fTransferSize; + bool partialTransfer = status != B_OK || fPartialTransfer; + bool deleteRequest = (fFlags & B_DELETE_IO_REQUEST) != 0; + + // unblock waiters + fFinishedCondition.NotifyAll(); + + locker.Unlock(); + + // notify callback + if (finishedCallback != NULL) { + finishedCallback(finishedCookie, this, status, partialTransfer, + lastTransferredOffset); + } + + // notify parent + if (parent != NULL) { + parent->SubRequestFinished(this, status, partialTransfer, + lastTransferredOffset); + } + + if (deleteRequest) + delete this; +} + + +/*! Returns whether this request or any of it's ancestors has a finished or + notification callback. Used to decide whether NotifyFinished() can be called + synchronously. +*/ +bool +IORequest::HasCallbacks() const +{ + if (fFinishedCallback != NULL || fIterationCallback != NULL) + return true; + + return fParent != NULL && fParent->HasCallbacks(); +} + + +void +IORequest::SetStatusAndNotify(status_t status) +{ + MutexLocker locker(fLock); + + if (fStatus != 1) + return; + + fStatus = status; + + locker.Unlock(); + + NotifyFinished(); +} + + +void +IORequest::OperationFinished(IOOperation* operation, status_t status, + bool partialTransfer, size_t transferEndOffset) +{ + TRACE("IORequest::OperationFinished(%p, %#lx): request: %p\n", operation, + status, this); + + MutexLocker locker(fLock); + + fChildren.Remove(operation); + operation->SetParent(NULL); + + if (status != B_OK || partialTransfer) { + if (fTransferSize > transferEndOffset) + fTransferSize = transferEndOffset; + fPartialTransfer = true; + } + + if (status != B_OK && fStatus == 1) + fStatus = status; + + if (--fPendingChildren > 0) + return; + + // last child finished + + // set status, if not done yet + if (fStatus == 1) + fStatus = B_OK; +} + + +void +IORequest::SubRequestFinished(IORequest* request, status_t status, + bool partialTransfer, size_t transferEndOffset) +{ + TRACE("IORequest::SubrequestFinished(%p, %#lx, %d, %lu): request: %p\n", + request, status, partialTransfer, transferEndOffset, this); + + MutexLocker locker(fLock); + + if (status != B_OK || partialTransfer) { + if (fTransferSize > transferEndOffset) + fTransferSize = transferEndOffset; + fPartialTransfer = true; + } + + if (status != B_OK && fStatus == 1) + fStatus = status; + + if (--fPendingChildren > 0) + return; + + // last child finished + + // set status, if not done yet + if (fStatus == 1) + fStatus = B_OK; + + locker.Unlock(); + + NotifyFinished(); +} + + +void +IORequest::SetUnfinished() +{ + MutexLocker _(fLock); + ResetStatus(); +} + + +void +IORequest::SetTransferredBytes(bool partialTransfer, size_t transferredBytes) +{ + TRACE("%p->IORequest::SetTransferredBytes(%d, %lu)\n", this, + partialTransfer, transferredBytes); + + MutexLocker _(fLock); + + fPartialTransfer = partialTransfer; + fTransferSize = transferredBytes; +} + + +void +IORequest::Advance(size_t bySize) +{ + TRACE("IORequest::Advance(%lu): remaining: %lu -> %lu\n", bySize, + fRemainingBytes, fRemainingBytes - bySize); + fRemainingBytes -= bySize; + fTransferSize += bySize; + + iovec* vecs = fBuffer->Vecs(); + uint32 vecCount = fBuffer->VecCount(); + while (fVecIndex < vecCount + && vecs[fVecIndex].iov_len - fVecOffset <= bySize) { + bySize -= vecs[fVecIndex].iov_len - fVecOffset; + fVecOffset = 0; + fVecIndex++; + } + + fVecOffset += bySize; +} + + +IORequest* +IORequest::FirstSubRequest() +{ + return dynamic_cast(fChildren.Head()); +} + + +IORequest* +IORequest::NextSubRequest(IORequest* previous) +{ + if (previous == NULL) + return NULL; + return dynamic_cast(fChildren.GetNext(previous)); +} + + +void +IORequest::AddOperation(IOOperation* operation) +{ + MutexLocker locker(fLock); + TRACE("IORequest::AddOperation(%p): request: %p\n", operation, this); + fChildren.Add(operation); + fPendingChildren++; +} + + +void +IORequest::RemoveOperation(IOOperation* operation) +{ + MutexLocker locker(fLock); + fChildren.Remove(operation); + operation->SetParent(NULL); +} + + +status_t +IORequest::CopyData(off_t offset, void* buffer, size_t size) +{ + return _CopyData(buffer, offset, size, true); +} + + +status_t +IORequest::CopyData(const void* buffer, off_t offset, size_t size) +{ + return _CopyData((void*)buffer, offset, size, false); +} + + +status_t +IORequest::_CopyData(void* _buffer, off_t offset, size_t size, bool copyIn) +{ + if (size == 0) + return B_OK; + + uint8* buffer = (uint8*)_buffer; + + if (offset < fOffset || offset + size > fOffset + fLength) { + panic("IORequest::_CopyData(): invalid range: (%lld, %lu)", offset, + size); + return B_BAD_VALUE; + } + + // If we can, we directly copy from/to the virtual buffer. The memory is + // locked in this case. + status_t (*copyFunction)(void*, void*, size_t, team_id, bool); + if (fBuffer->IsPhysical()) { + copyFunction = &IORequest::_CopyPhysical; + } else { + copyFunction = fBuffer->IsUser() + ? &IORequest::_CopyUser : &IORequest::_CopySimple; + } + + // skip bytes if requested + iovec* vecs = fBuffer->Vecs(); + size_t skipBytes = offset - fOffset; + size_t vecOffset = 0; + while (skipBytes > 0) { + if (vecs[0].iov_len > skipBytes) { + vecOffset = skipBytes; + break; + } + + skipBytes -= vecs[0].iov_len; + vecs++; + } + + // copy iovec-wise + while (size > 0) { + size_t toCopy = min_c(size, vecs[0].iov_len - vecOffset); + status_t error = copyFunction(buffer, + (uint8*)vecs[0].iov_base + vecOffset, toCopy, fTeam, copyIn); + if (error != B_OK) + return error; + + buffer += toCopy; + size -= toCopy; + vecs++; + vecOffset = 0; + } + + return B_OK; +} + + +/* static */ status_t +IORequest::_CopySimple(void* bounceBuffer, void* external, size_t size, + team_id team, bool copyIn) +{ + TRACE(" IORequest::_CopySimple(%p, %p, %lu, %d)\n", bounceBuffer, external, + size, copyIn); + if (copyIn) + memcpy(bounceBuffer, external, size); + else + memcpy(external, bounceBuffer, size); + return B_OK; +} + + +/* static */ status_t +IORequest::_CopyPhysical(void* bounceBuffer, void* external, size_t size, + team_id team, bool copyIn) +{ + if (copyIn) { + return vm_memcpy_from_physical(bounceBuffer, (addr_t)external, size, + false); + } + + return vm_memcpy_to_physical((addr_t)external, bounceBuffer, size, false); +} + + +/* static */ status_t +IORequest::_CopyUser(void* _bounceBuffer, void* _external, size_t size, + team_id team, bool copyIn) +{ + uint8* bounceBuffer = (uint8*)_bounceBuffer; + uint8* external = (uint8*)_external; + + while (size > 0) { + static const int32 kEntryCount = 8; + physical_entry entries[kEntryCount]; + + uint32 count = kEntryCount; + status_t error = get_memory_map_etc(team, external, size, entries, + &count); + if (error != B_OK && error != B_BUFFER_OVERFLOW) { + panic("IORequest::_CopyUser(): Failed to get physical memory for " + "user memory %p\n", external); + return B_BAD_ADDRESS; + } + + for (uint32 i = 0; i < count; i++) { + const physical_entry& entry = entries[i]; + error = _CopyPhysical(bounceBuffer, entry.address, + entry.size, team, copyIn); + if (error != B_OK) + return error; + + size -= entry.size; + bounceBuffer += entry.size; + external += entry.size; + } + } + + return B_OK; +} + + +void +IORequest::Dump() const +{ + kprintf("io_request at %p\n", this); + + kprintf(" owner: %p\n", fOwner); + kprintf(" parent: %p\n", fParent); + kprintf(" status: %s\n", strerror(fStatus)); + kprintf(" mutex: %p\n", &fLock); + kprintf(" IOBuffer: %p\n", fBuffer); + kprintf(" offset: %Ld\n", fOffset); + kprintf(" length: %lu\n", fLength); + kprintf(" transfer size: %lu\n", fTransferSize); + kprintf(" relative offset: %lu\n", fRelativeParentOffset); + kprintf(" pending children: %ld\n", fPendingChildren); + kprintf(" flags: %#lx\n", fFlags); + kprintf(" team: %ld\n", fTeam); + kprintf(" thread: %ld\n", fThread); + kprintf(" r/w: %s\n", fIsWrite ? "write" : "read"); + kprintf(" partial transfer: %s\n", fPartialTransfer ? "yes" : "no"); + kprintf(" finished cvar: %p\n", &fFinishedCondition); + kprintf(" iteration:\n"); + kprintf(" vec index: %lu\n", fVecIndex); + kprintf(" vec offset: %lu\n", fVecOffset); + kprintf(" remaining bytes: %lu\n", fRemainingBytes); + kprintf(" callbacks:\n"); + kprintf(" finished %p, cookie %p\n", fFinishedCallback, fFinishedCookie); + kprintf(" iteration %p, cookie %p\n", fIterationCallback, + fIterationCookie); + kprintf(" children:\n"); + + IORequestChunkList::ConstIterator iterator = fChildren.GetIterator(); + while (iterator.HasNext()) { + kprintf(" %p\n", iterator.Next()); + } + + set_debug_variable("_parent", (addr_t)fParent); + set_debug_variable("_mutex", (addr_t)&fLock); + set_debug_variable("_buffer", (addr_t)fBuffer); + set_debug_variable("_cvar", (addr_t)&fFinishedCondition); +} + + +// #pragma mark - allocator + + +#if KERNEL_HEAP_LEAK_CHECK +static addr_t +get_caller() +{ + // Find the first return address outside of the allocator code. Note, that + // this makes certain assumptions about how the code for the functions + // ends up in the kernel object. + addr_t returnAddresses[5]; + int32 depth = arch_debug_get_stack_trace(returnAddresses, 5, 0, 1, false); + + // find the first return address inside the VIP allocator + int32 i = 0; + for (i = 0; i < depth; i++) { + if (returnAddresses[i] >= (addr_t)&get_caller + && returnAddresses[i] < (addr_t)&vip_io_request_allocator_init) { + break; + } + } + + // now continue until we have the first one outside + for (; i < depth; i++) { + if (returnAddresses[i] < (addr_t)&get_caller + || returnAddresses[i] > (addr_t)&vip_io_request_allocator_init) { + return returnAddresses[i]; + } + } + + return 0; +} +#endif + + +void* +vip_io_request_malloc(size_t size) +{ + void* address = heap_memalign(sVIPHeap, 0, size); +#if KDEBUG + if (address == NULL) + panic("vip_io_request_malloc(): VIP heap %p out of memory", sVIPHeap); +#endif + return address; +} + + +void +vip_io_request_free(void* address) +{ + heap_free(sVIPHeap, address); +} + + +void +io_request_free(void* address) +{ + if (heap_free(sVIPHeap, address) != B_OK) + free(address); +} + + +void +vip_io_request_allocator_init() +{ + static const heap_class heapClass = { + "VIP I/O", /* name */ + 100, /* initial percentage */ + B_PAGE_SIZE / 8, /* max allocation size */ + B_PAGE_SIZE, /* page size */ + 8, /* min bin size */ + 4, /* bin alignment */ + 8, /* min count per page */ + 16 /* max waste per page */ + }; + + void* address = NULL; + area_id area = create_area("VIP I/O heap", &address, B_ANY_KERNEL_ADDRESS, + VIP_HEAP_SIZE, B_FULL_LOCK, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + if (area < B_OK) { + panic("vip_io_request_allocator_init(): couldn't allocate VIP I/O " + "heap area"); + return; + } + + sVIPHeap = heap_create_allocator("VIP I/O heap", (addr_t)address, + VIP_HEAP_SIZE, &heapClass); + if (sVIPHeap == NULL) { + panic("vip_io_request_allocator_init(): failed to create VIP I/O " + "heap\n"); + return; + } + +#if KERNEL_HEAP_LEAK_CHECK + heap_set_get_caller(sVIPHeap, &get_caller); +#endif + + dprintf("vip_io_request_allocator_init(): created VIP I/O heap: %p\n", + sVIPHeap); +} + + +// #pragma mark - + + +#if 0 + +/*! Creates an I/O request with the specified buffer and length. + + \param write write access if true, read access if false. + \param flags allows several flags to be specified: + \c B_USER_IO_REQUEST the buffer is assumed to be a userland buffer + and handled with special care. + \c B_ASYNC_IO_REQUEST the I/O request is to be fulfilled asynchronously. + \c B_PHYSICAL_IO_REQUEST the buffer specifies a physical rather than a + virtual address. + \param _request If successful, the location pointed to by this parameter + will contain a pointer to the created request. +*/ +status_t +create_io_request(void* buffer, size_t length, bool write, uint32 flags, + io_request** _request) +{ + return B_ERROR; +} + + +/*! Creates an I/O request from the specified I/O vector and length. + See above for more info. +*/ +status_t +create_io_request_vecs(iovec* vecs, size_t count, size_t length, bool write, + uint32 flags, io_request** _request) +{ + return B_ERROR; +} + + +/*! Prepares the I/O request by locking its memory, and, if \a virtualOnly + is \c false, will retrieve the physical pages. +*/ +status_t +prepare_io_request(io_request* request, bool virtualOnly) +{ + return B_ERROR; +} + + +/*! Prepares the I/O request by locking its memory, and mapping/moving the + pages as needed to fulfill the DMA restrictions. + If needed, a bounce buffer is used for DMA. +*/ +status_t +prepare_io_request_dma(io_request* request, dma_resource* dmaResource) +{ + return B_ERROR; +} + + +/*! Returns the buffers of the I/O request mapped into kernel memory. + This can be used by drivers to fill an I/O request manually. +*/ +status_t +map_io_request(io_request* request, iovec* vecs, size_t count) +{ + return B_ERROR; +} + + +/*! Get the memory map of the DMA buffer for this I/O request. + This can be used to retrieve the physical pages to feed the hardware's + DMA engine with. +*/ +status_t +get_io_request_memory_map(dma_buffer* buffer, io_request* request, iovec* vecs, + size_t count) +{ + return B_ERROR; +} + + +/*! Unmaps any previously mapped data, and will copy the data back from any + bounce buffers if necessary. +*/ +status_t +complete_io_request_dma(io_request* request, dma_resource* dmaResource) +{ + return B_ERROR; +} + + +/*! Unmaps any previously mapped data. +*/ +status_t +complete_io_request(io_request* request) +{ + return B_ERROR; +} + + +void +delete_io_request(io_request* request) +{ +} + +#endif // 0 + diff --git a/src/system/kernel/device_manager/io_requests.h b/src/system/kernel/device_manager/IORequest.h similarity index 99% rename from src/system/kernel/device_manager/io_requests.h rename to src/system/kernel/device_manager/IORequest.h index 70bd7b61f4..41f5ee42cd 100644 --- a/src/system/kernel/device_manager/io_requests.h +++ b/src/system/kernel/device_manager/IORequest.h @@ -3,8 +3,8 @@ * Copyright 2008, Axel Dörfler, axeld@pinc-software.de. * Distributed under the terms of the MIT License. */ -#ifndef IO_REQUESTS_H -#define IO_REQUESTS_H +#ifndef IO_REQUEST_H +#define IO_REQUEST_H #include @@ -371,4 +371,4 @@ operator new[](size_t size, const vip_io_alloc_t& vip_io_alloc) throw () } -#endif // IO_REQUESTS_H +#endif // IO_REQUEST_H diff --git a/src/system/kernel/device_manager/IOScheduler.h b/src/system/kernel/device_manager/IOScheduler.h index ec295fac88..96bc9eac58 100644 --- a/src/system/kernel/device_manager/IOScheduler.h +++ b/src/system/kernel/device_manager/IOScheduler.h @@ -14,7 +14,7 @@ #include #include "dma_resources.h" -#include "io_requests.h" +#include "IORequest.h" class IOCallback { diff --git a/src/system/kernel/device_manager/Jamfile b/src/system/kernel/device_manager/Jamfile index 8cfd1b1e1d..017345870f 100644 --- a/src/system/kernel/device_manager/Jamfile +++ b/src/system/kernel/device_manager/Jamfile @@ -16,6 +16,7 @@ KernelMergeObject kernel_device_manager.o : dma_resources.cpp io_requests.cpp + IORequest.cpp : $(TARGET_KERNEL_PIC_CCFLAGS) ; diff --git a/src/system/kernel/device_manager/devfs.cpp b/src/system/kernel/device_manager/devfs.cpp index f4b9a36012..54cdd0efb9 100644 --- a/src/system/kernel/device_manager/devfs.cpp +++ b/src/system/kernel/device_manager/devfs.cpp @@ -36,7 +36,7 @@ #include #include "BaseDevice.h" -#include "io_requests.h" +#include "IORequest.h" #include "legacy_drivers.h" diff --git a/src/system/kernel/device_manager/device_manager.cpp b/src/system/kernel/device_manager/device_manager.cpp index 9590e49807..2c7cf19d2f 100644 --- a/src/system/kernel/device_manager/device_manager.cpp +++ b/src/system/kernel/device_manager/device_manager.cpp @@ -30,7 +30,7 @@ #include "BaseDevice.h" #include "devfs_private.h" #include "id_generator.h" -#include "io_requests.h" +#include "IORequest.h" #include "io_resources.h" #include "IOScheduler.h" diff --git a/src/system/kernel/device_manager/dma_resources.cpp b/src/system/kernel/device_manager/dma_resources.cpp index 6b9125c791..df46e5c8f4 100644 --- a/src/system/kernel/device_manager/dma_resources.cpp +++ b/src/system/kernel/device_manager/dma_resources.cpp @@ -11,7 +11,7 @@ #include #include -#include "io_requests.h" +#include "IORequest.h" //#define TRACE_DMA_RESOURCE diff --git a/src/system/kernel/device_manager/io_requests.cpp b/src/system/kernel/device_manager/io_requests.cpp index b2a4296389..1372c7be20 100644 --- a/src/system/kernel/device_manager/io_requests.cpp +++ b/src/system/kernel/device_manager/io_requests.cpp @@ -1,1405 +1,15 @@ /* - * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. - * Copyright 2008, Axel Dörfler, axeld@pinc-software.de. + * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ -#include "io_requests.h" +#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "dma_resources.h" - - -//#define TRACE_IO_REQUEST -#ifdef TRACE_IO_REQUEST -# define TRACE(x...) dprintf(x) -#else -# define TRACE(x...) ; -#endif - - -#define VIP_HEAP_SIZE 1024 * 1024 - -// partial I/O operation phases -enum { - PHASE_READ_BEGIN = 0, - PHASE_READ_END = 1, - PHASE_DO_ALL = 2 -}; - -heap_allocator* sVIPHeap; - - -// #pragma mark - - - -IORequestChunk::IORequestChunk() - : - fParent(NULL), - fStatus(1) -{ -} - - -IORequestChunk::~IORequestChunk() -{ -} - - -void -IORequestChunk::operator delete(void* address, size_t size) -{ - io_request_free(address); -} - - -// #pragma mark - - - -IOBuffer* -IOBuffer::Create(uint32 count, bool vip) -{ - size_t size = sizeof(IOBuffer) + sizeof(iovec) * (count - 1); - IOBuffer* buffer - = (IOBuffer*)(vip ? vip_io_request_malloc(size) : malloc(size)); - if (buffer == NULL) - return NULL; - - buffer->fCapacity = count; - buffer->fVecCount = 0; - buffer->fUser = false; - buffer->fPhysical = false; - buffer->fVIP = vip; - buffer->fMemoryLocked = false; - - return buffer; -} - - -void -IOBuffer::Delete() -{ - if (this == NULL) - return; - - if (fVIP) - vip_io_request_free(this); - else - free(this); -} - - -void -IOBuffer::SetVecs(size_t firstVecOffset, const iovec* vecs, uint32 count, - size_t length, uint32 flags) -{ - memcpy(fVecs, vecs, sizeof(iovec) * count); - if (count > 0 && firstVecOffset > 0) { - fVecs[0].iov_base = (uint8*)fVecs[0].iov_base + firstVecOffset; - fVecs[0].iov_len -= firstVecOffset; - } - - fVecCount = count; - fLength = length; - fUser = IS_USER_ADDRESS(vecs[0].iov_base); - fPhysical = (flags & B_PHYSICAL_IO_REQUEST) != 0; -} - - -status_t -IOBuffer::LockMemory(team_id team, bool isWrite) -{ - if (fMemoryLocked) { - panic("memory already locked!"); - return B_BAD_VALUE; - } - - for (uint32 i = 0; i < fVecCount; i++) { - status_t status = lock_memory_etc(team, fVecs[i].iov_base, - fVecs[i].iov_len, isWrite ? 0 : B_READ_DEVICE); - if (status != B_OK) { - _UnlockMemory(team, i, isWrite); - return status; - } - } - - fMemoryLocked = true; - return B_OK; -} - - -void -IOBuffer::_UnlockMemory(team_id team, size_t count, bool isWrite) -{ - for (uint32 i = 0; i < count; i++) { - unlock_memory_etc(team, fVecs[i].iov_base, fVecs[i].iov_len, - isWrite ? 0 : B_READ_DEVICE); - } -} - - -void -IOBuffer::UnlockMemory(team_id team, bool isWrite) -{ - if (!fMemoryLocked) { - panic("memory not locked"); - return; - } - - _UnlockMemory(team, fVecCount, isWrite); - fMemoryLocked = false; -} - - -void -IOBuffer::Dump() const -{ - kprintf("IOBuffer at %p\n", this); - - kprintf(" origin: %s\n", fUser ? "user" : "kernel"); - kprintf(" kind: %s\n", fPhysical ? "physical" : "virtual"); - kprintf(" length: %lu\n", fLength); - kprintf(" capacity: %lu\n", fCapacity); - kprintf(" vecs: %lu\n", fVecCount); - - for (uint32 i = 0; i < fVecCount; i++) { - kprintf(" [%lu] %p, %lu\n", i, fVecs[i].iov_base, fVecs[i].iov_len); - } -} - - -// #pragma mark - +#include "IORequest.h" bool -IOOperation::Finish() +io_request_is_write(const io_request* request) { - TRACE("IOOperation::Finish()\n"); - if (fStatus == B_OK) { - if (fParent->IsWrite()) { - TRACE(" is write\n"); - if (fPhase == PHASE_READ_BEGIN) { - TRACE(" phase read begin\n"); - // repair phase adjusted vec - fDMABuffer->VecAt(fSavedVecIndex).iov_len = fSavedVecLength; - - // partial write: copy partial begin to bounce buffer - bool skipReadEndPhase; - status_t error = _CopyPartialBegin(true, skipReadEndPhase); - if (error == B_OK) { - // We're done with the first phase only (read in begin). - // Get ready for next phase... - fPhase = HasPartialEnd() && !skipReadEndPhase - ? PHASE_READ_END : PHASE_DO_ALL; - _PrepareVecs(); - ResetStatus(); - // TODO: Is there a race condition, if the request is - // aborted at the same time? - return false; - } - - SetStatus(error); - } else if (fPhase == PHASE_READ_END) { - TRACE(" phase read end\n"); - // repair phase adjusted vec - iovec& vec = fDMABuffer->VecAt(fSavedVecIndex); - vec.iov_base = (uint8*)vec.iov_base - + vec.iov_len - fSavedVecLength; - vec.iov_len = fSavedVecLength; - - // partial write: copy partial end to bounce buffer - status_t error = _CopyPartialEnd(true); - if (error == B_OK) { - // We're done with the second phase only (read in end). - // Get ready for next phase... - fPhase = PHASE_DO_ALL; - ResetStatus(); - // TODO: Is there a race condition, if the request is - // aborted at the same time? - return false; - } - - SetStatus(error); - } - } - } - - if (fParent->IsRead() && UsesBounceBuffer()) { - TRACE(" read with bounce buffer\n"); - // copy the bounce buffer segments to the final location - uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress(); - addr_t bounceBufferStart = fDMABuffer->PhysicalBounceBufferAddress(); - addr_t bounceBufferEnd = bounceBufferStart - + fDMABuffer->BounceBufferSize(); - - const iovec* vecs = fDMABuffer->Vecs(); - uint32 vecCount = fDMABuffer->VecCount(); - - status_t error = B_OK; - - off_t offset = fOffset; - off_t startOffset = fOriginalOffset; - off_t endOffset = fOriginalOffset + fOriginalLength; - - for (uint32 i = 0; error == B_OK && i < vecCount; i++) { - const iovec& vec = vecs[i]; - addr_t base = (addr_t)vec.iov_base; - size_t length = vec.iov_len; - - if (offset < startOffset) { - if (offset + length <= startOffset) { - offset += length; - continue; - } - - size_t diff = startOffset - offset; - base += diff; - length -= diff; - } - - if (offset + length > endOffset) { - if (offset >= endOffset) - break; - - length = endOffset - offset; - } - - if (base >= bounceBufferStart && base < bounceBufferEnd) { - error = fParent->CopyData( - bounceBuffer + (base - bounceBufferStart), offset, length); - } - - offset += length; - } - - if (error != B_OK) - SetStatus(error); - } - - return true; + return request->IsWrite(); } - - -/*! Note: SetPartial() must be called first! -*/ -status_t -IOOperation::Prepare(IORequest* request) -{ - if (fParent != NULL) - fParent->RemoveOperation(this); - - fParent = request; - - fTransferredBytes = 0; - - // set initial phase - fPhase = PHASE_DO_ALL; - if (fParent->IsWrite()) { - // Copy data to bounce buffer segments, save the partial begin/end vec, - // which will be copied after their respective read phase. - if (UsesBounceBuffer()) { - TRACE(" write with bounce buffer\n"); - uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress(); - addr_t bounceBufferStart - = fDMABuffer->PhysicalBounceBufferAddress(); - addr_t bounceBufferEnd = bounceBufferStart - + fDMABuffer->BounceBufferSize(); - - const iovec* vecs = fDMABuffer->Vecs(); - uint32 vecCount = fDMABuffer->VecCount(); - size_t vecOffset = 0; - uint32 i = 0; - - off_t offset = fOffset; - off_t endOffset = fOffset + fLength; - - if (HasPartialBegin()) { - // skip first block - size_t toSkip = fBlockSize; - while (toSkip > 0) { - if (vecs[i].iov_len <= toSkip) { - toSkip -= vecs[i].iov_len; - i++; - } else { - vecOffset = toSkip; - break; - } - } - - offset += fBlockSize; - } - - if (HasPartialEnd()) { - // skip last block - size_t toSkip = fBlockSize; - while (toSkip > 0) { - if (vecs[vecCount - 1].iov_len <= toSkip) { - toSkip -= vecs[vecCount - 1].iov_len; - vecCount--; - } else - break; - } - - endOffset -= fBlockSize; - } - - for (; i < vecCount; i++) { - const iovec& vec = vecs[i]; - addr_t base = (addr_t)vec.iov_base + vecOffset; - size_t length = vec.iov_len - vecOffset; - vecOffset = 0; - - if (base >= bounceBufferStart && base < bounceBufferEnd) { - if (offset + length > endOffset) - length = endOffset - offset; - status_t error = fParent->CopyData(offset, - bounceBuffer + (base - bounceBufferStart), length); - if (error != B_OK) - return error; - } - - offset += length; - } - } - - if (HasPartialBegin()) - fPhase = PHASE_READ_BEGIN; - else if (HasPartialEnd()) - fPhase = PHASE_READ_END; - - _PrepareVecs(); - } - - ResetStatus(); - - if (fParent != NULL) - fParent->AddOperation(this); - - return B_OK; -} - - -void -IOOperation::SetOriginalRange(off_t offset, size_t length) -{ - fOriginalOffset = fOffset = offset; - fOriginalLength = fLength = length; -} - - -void -IOOperation::SetRange(off_t offset, size_t length) -{ - fOffset = offset; - fLength = length; -} - - -off_t -IOOperation::Offset() const -{ - return fPhase == PHASE_READ_END ? fOffset + fLength - fBlockSize : fOffset; -} - - -size_t -IOOperation::Length() const -{ - return fPhase == PHASE_DO_ALL ? fLength : fBlockSize; -} - - -iovec* -IOOperation::Vecs() const -{ - switch (fPhase) { - case PHASE_READ_END: - return fDMABuffer->Vecs() + fSavedVecIndex; - case PHASE_READ_BEGIN: - case PHASE_DO_ALL: - default: - return fDMABuffer->Vecs(); - } -} - - -uint32 -IOOperation::VecCount() const -{ - switch (fPhase) { - case PHASE_READ_BEGIN: - return fSavedVecIndex + 1; - case PHASE_READ_END: - return fDMABuffer->VecCount() - fSavedVecIndex; - case PHASE_DO_ALL: - default: - return fDMABuffer->VecCount(); - } -} - - -void -IOOperation::SetPartial(bool partialBegin, bool partialEnd) -{ - TRACE("partial begin %d, end %d\n", partialBegin, partialEnd); - fPartialBegin = partialBegin; - fPartialEnd = partialEnd; -} - - -bool -IOOperation::IsWrite() const -{ - return fParent->IsWrite() && fPhase == PHASE_DO_ALL; -} - - -bool -IOOperation::IsRead() const -{ - return fParent->IsRead(); -} - - -void -IOOperation::_PrepareVecs() -{ - // we need to prepare the vecs for consumption by the drivers - if (fPhase == PHASE_READ_BEGIN) { - iovec* vecs = fDMABuffer->Vecs(); - uint32 vecCount = fDMABuffer->VecCount(); - size_t vecLength = fBlockSize; - for (uint32 i = 0; i < vecCount; i++) { - iovec& vec = vecs[i]; - if (vec.iov_len >= vecLength) { - fSavedVecIndex = i; - fSavedVecLength = vec.iov_len; - vec.iov_len = vecLength; - break; - } - vecLength -= vec.iov_len; - } - } else if (fPhase == PHASE_READ_END) { - iovec* vecs = fDMABuffer->Vecs(); - uint32 vecCount = fDMABuffer->VecCount(); - size_t vecLength = fBlockSize; - for (int32 i = vecCount - 1; i >= 0; i--) { - iovec& vec = vecs[i]; - if (vec.iov_len >= vecLength) { - fSavedVecIndex = i; - fSavedVecLength = vec.iov_len; - vec.iov_base = (uint8*)vec.iov_base - + vec.iov_len - vecLength; - vec.iov_len = vecLength; - break; - } - vecLength -= vec.iov_len; - } - } -} - - -status_t -IOOperation::_CopyPartialBegin(bool isWrite, bool& singleBlockOnly) -{ - size_t relativeOffset = OriginalOffset() - fOffset; - size_t length = fBlockSize - relativeOffset; - - singleBlockOnly = length >= OriginalLength(); - if (singleBlockOnly) - length = OriginalLength(); - - TRACE("_CopyPartialBegin(%s, single only %d)\n", - isWrite ? "write" : "read", singleBlockOnly); - - if (isWrite) { - return fParent->CopyData(OriginalOffset(), - (uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, length); - } else { - return fParent->CopyData( - (uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, - OriginalOffset(), length); - } -} - - -status_t -IOOperation::_CopyPartialEnd(bool isWrite) -{ - TRACE("_CopyPartialEnd(%s)\n", isWrite ? "write" : "read"); - - const iovec& lastVec = fDMABuffer->VecAt(fDMABuffer->VecCount() - 1); - off_t lastVecPos = fOffset + fLength - fBlockSize; - uint8* base = (uint8*)fDMABuffer->BounceBufferAddress() - + ((addr_t)lastVec.iov_base + lastVec.iov_len - fBlockSize - - fDMABuffer->PhysicalBounceBufferAddress()); - // NOTE: this won't work if we don't use the bounce buffer contiguously - // (because of boundary alignments). - size_t length = OriginalOffset() + OriginalLength() - lastVecPos; - - if (isWrite) - return fParent->CopyData(lastVecPos, base, length); - - return fParent->CopyData(base, lastVecPos, length); -} - - -void -IOOperation::Dump() const -{ - kprintf("io_operation at %p\n", this); - - kprintf(" parent: %p\n", fParent); - kprintf(" status: %s\n", strerror(fStatus)); - kprintf(" dma buffer: %p\n", fDMABuffer); - kprintf(" offset: %-8Ld (original: %Ld)\n", fOffset, - fOriginalOffset); - kprintf(" length: %-8lu (original: %lu)\n", fLength, - fOriginalLength); - kprintf(" transferred: %lu\n", fTransferredBytes); - kprintf(" block size: %lu\n", fBlockSize); - kprintf(" saved vec index: %u\n", fSavedVecIndex); - kprintf(" saved vec length: %u\n", fSavedVecLength); - kprintf(" r/w: %s\n", IsWrite() ? "write" : "read"); - kprintf(" phase: %s\n", fPhase == PHASE_READ_BEGIN - ? "read begin" : fPhase == PHASE_READ_END ? "read end" - : fPhase == PHASE_DO_ALL ? "do all" : "unknown"); - kprintf(" partial begin: %s\n", fPartialBegin ? "yes" : "no"); - kprintf(" partial end: %s\n", fPartialEnd ? "yes" : "no"); - kprintf(" bounce buffer: %s\n", fUsesBounceBuffer ? "yes" : "no"); - - set_debug_variable("_parent", (addr_t)fParent); - set_debug_variable("_buffer", (addr_t)fDMABuffer); -} - - -// #pragma mark - - - -IORequest::IORequest() - : - fFinishedCallback(NULL), - fFinishedCookie(NULL), - fIterationCallback(NULL), - fIterationCookie(NULL) -{ - mutex_init(&fLock, "I/O request lock"); - fFinishedCondition.Init(this, "I/O request finished"); -} - - -IORequest::~IORequest() -{ - mutex_lock(&fLock); - DeleteSubRequests(); - fBuffer->Delete(); - mutex_destroy(&fLock); -} - - -/* static */ IORequest* -IORequest::Create(bool vip) -{ - return vip ? new(vip_io_alloc) IORequest : new(std::nothrow) IORequest; -} - - -status_t -IORequest::Init(off_t offset, void* buffer, size_t length, bool write, - uint32 flags) -{ - iovec vec; - vec.iov_base = buffer; - vec.iov_len = length; - return Init(offset, &vec, 1, length, write, flags); -} - - -status_t -IORequest::Init(off_t offset, size_t firstVecOffset, const iovec* vecs, - size_t count, size_t length, bool write, uint32 flags) -{ - fBuffer = IOBuffer::Create(count, (flags & B_VIP_IO_REQUEST) != 0); - if (fBuffer == NULL) - return B_NO_MEMORY; - - fBuffer->SetVecs(firstVecOffset, vecs, count, length, flags); - - fOwner = NULL; - fOffset = offset; - fLength = length; - fRelativeParentOffset = 0; - fTransferSize = 0; - fFlags = flags; - struct thread* thread = thread_get_current_thread(); - fTeam = thread->team->id; - fThread = thread->id; - fIsWrite = write; - fPartialTransfer = 0; - - // these are for iteration - fVecIndex = 0; - fVecOffset = 0; - fRemainingBytes = length; - - fPendingChildren = 0; - - fStatus = 1; - - return B_OK; -} - - -status_t -IORequest::CreateSubRequest(off_t parentOffset, off_t offset, size_t length, - IORequest*& _subRequest) -{ - ASSERT(parentOffset >= fOffset && length <= fLength - && parentOffset - fOffset <= fLength - length); - - // find start vec - size_t vecOffset = parentOffset - fOffset; - iovec* vecs = fBuffer->Vecs(); - int32 vecCount = fBuffer->VecCount(); - int32 startVec = 0; - for (; startVec < vecCount; startVec++) { - const iovec& vec = vecs[startVec]; - if (vecOffset < vec.iov_len) - break; - - vecOffset -= vec.iov_len; - } - - // count vecs - size_t currentVecOffset = vecOffset; - int32 endVec = startVec; - size_t remainingLength = length; - for (; endVec < vecCount; endVec++) { - const iovec& vec = vecs[endVec]; - if (vec.iov_len - currentVecOffset >= remainingLength) - break; - - remainingLength -= vec.iov_len - currentVecOffset; - currentVecOffset = 0; - } - - // create subrequest - IORequest* subRequest = Create((fFlags & B_VIP_IO_REQUEST) != 0); - if (subRequest == NULL) - return B_NO_MEMORY; - - status_t error = subRequest->Init(offset, vecOffset, vecs + startVec, - endVec - startVec + 1, length, fIsWrite, fFlags & ~B_DELETE_IO_REQUEST); - if (error != B_OK) { - delete subRequest; - return error; - } - - subRequest->fRelativeParentOffset = parentOffset - fOffset; - subRequest->fTeam = fTeam; - subRequest->fThread = fThread; - - _subRequest = subRequest; - subRequest->SetParent(this); - - MutexLocker _(fLock); - - fChildren.Add(subRequest); - fPendingChildren++; - TRACE("IORequest::CreateSubRequest(): request: %p, subrequest: %p\n", this, - subRequest); - - return B_OK; -} - - -void -IORequest::DeleteSubRequests() -{ - while (IORequestChunk* chunk = fChildren.RemoveHead()) - delete chunk; -} - - -void -IORequest::SetFinishedCallback(io_request_finished_callback callback, - void* cookie) -{ - fFinishedCallback = callback; - fFinishedCookie = cookie; -} - - -void -IORequest::SetIterationCallback(io_request_iterate_callback callback, - void* cookie) -{ - fIterationCallback = callback; - fIterationCookie = cookie; -} - - -io_request_finished_callback -IORequest::FinishedCallback(void** _cookie) const -{ - if (_cookie != NULL) - *_cookie = fFinishedCookie; - return fFinishedCallback; -} - - -status_t -IORequest::Wait(uint32 flags, bigtime_t timeout) -{ - MutexLocker locker(fLock); - - if (IsFinished()) - return Status(); - - ConditionVariableEntry entry; - fFinishedCondition.Add(&entry); - - locker.Unlock(); - - status_t error = entry.Wait(flags, timeout); - if (error != B_OK) - return error; - - return Status(); -} - - -void -IORequest::NotifyFinished() -{ - TRACE("IORequest::NotifyFinished(): request: %p\n", this); - - MutexLocker locker(fLock); - - if (fStatus == B_OK && !fPartialTransfer && RemainingBytes() > 0) { - // The request is not really done yet. If it has an iteration callback, - // call it. - if (fIterationCallback != NULL) { - ResetStatus(); - locker.Unlock(); - bool partialTransfer = false; - status_t error = fIterationCallback(fIterationCookie, this, - &partialTransfer); - if (error == B_OK && !partialTransfer) - return; - - // Iteration failed, which means we're responsible for notifying the - // requests finished. - locker.Lock(); - fStatus = error; - fPartialTransfer = true; - } - } - - ASSERT(fPendingChildren == 0); - ASSERT(fChildren.IsEmpty() - || dynamic_cast(fChildren.Head()) == NULL); - - // unlock the memory - if (fBuffer->IsMemoryLocked()) - fBuffer->UnlockMemory(fTeam, fIsWrite); - - - // Cache the callbacks before we unblock waiters and unlock. Any of the - // following could delete this request, so we don't want to touch it - // once we have started telling others that it is done. - IORequest* parent = fParent; - io_request_finished_callback finishedCallback = fFinishedCallback; - void* finishedCookie = fFinishedCookie; - status_t status = fStatus; - size_t lastTransferredOffset = fRelativeParentOffset + fTransferSize; - bool partialTransfer = status != B_OK || fPartialTransfer; - bool deleteRequest = (fFlags & B_DELETE_IO_REQUEST) != 0; - - // unblock waiters - fFinishedCondition.NotifyAll(); - - locker.Unlock(); - - // notify callback - if (finishedCallback != NULL) { - finishedCallback(finishedCookie, this, status, partialTransfer, - lastTransferredOffset); - } - - // notify parent - if (parent != NULL) { - parent->SubRequestFinished(this, status, partialTransfer, - lastTransferredOffset); - } - - if (deleteRequest) - delete this; -} - - -/*! Returns whether this request or any of it's ancestors has a finished or - notification callback. Used to decide whether NotifyFinished() can be called - synchronously. -*/ -bool -IORequest::HasCallbacks() const -{ - if (fFinishedCallback != NULL || fIterationCallback != NULL) - return true; - - return fParent != NULL && fParent->HasCallbacks(); -} - - -void -IORequest::SetStatusAndNotify(status_t status) -{ - MutexLocker locker(fLock); - - if (fStatus != 1) - return; - - fStatus = status; - - locker.Unlock(); - - NotifyFinished(); -} - - -void -IORequest::OperationFinished(IOOperation* operation, status_t status, - bool partialTransfer, size_t transferEndOffset) -{ - TRACE("IORequest::OperationFinished(%p, %#lx): request: %p\n", operation, - status, this); - - MutexLocker locker(fLock); - - fChildren.Remove(operation); - operation->SetParent(NULL); - - if (status != B_OK || partialTransfer) { - if (fTransferSize > transferEndOffset) - fTransferSize = transferEndOffset; - fPartialTransfer = true; - } - - if (status != B_OK && fStatus == 1) - fStatus = status; - - if (--fPendingChildren > 0) - return; - - // last child finished - - // set status, if not done yet - if (fStatus == 1) - fStatus = B_OK; -} - - -void -IORequest::SubRequestFinished(IORequest* request, status_t status, - bool partialTransfer, size_t transferEndOffset) -{ - TRACE("IORequest::SubrequestFinished(%p, %#lx, %d, %lu): request: %p\n", - request, status, partialTransfer, transferEndOffset, this); - - MutexLocker locker(fLock); - - if (status != B_OK || partialTransfer) { - if (fTransferSize > transferEndOffset) - fTransferSize = transferEndOffset; - fPartialTransfer = true; - } - - if (status != B_OK && fStatus == 1) - fStatus = status; - - if (--fPendingChildren > 0) - return; - - // last child finished - - // set status, if not done yet - if (fStatus == 1) - fStatus = B_OK; - - locker.Unlock(); - - NotifyFinished(); -} - - -void -IORequest::SetUnfinished() -{ - MutexLocker _(fLock); - ResetStatus(); -} - - -void -IORequest::SetTransferredBytes(bool partialTransfer, size_t transferredBytes) -{ - TRACE("%p->IORequest::SetTransferredBytes(%d, %lu)\n", this, - partialTransfer, transferredBytes); - - MutexLocker _(fLock); - - fPartialTransfer = partialTransfer; - fTransferSize = transferredBytes; -} - - -void -IORequest::Advance(size_t bySize) -{ - TRACE("IORequest::Advance(%lu): remaining: %lu -> %lu\n", bySize, - fRemainingBytes, fRemainingBytes - bySize); - fRemainingBytes -= bySize; - fTransferSize += bySize; - - iovec* vecs = fBuffer->Vecs(); - uint32 vecCount = fBuffer->VecCount(); - while (fVecIndex < vecCount - && vecs[fVecIndex].iov_len - fVecOffset <= bySize) { - bySize -= vecs[fVecIndex].iov_len - fVecOffset; - fVecOffset = 0; - fVecIndex++; - } - - fVecOffset += bySize; -} - - -IORequest* -IORequest::FirstSubRequest() -{ - return dynamic_cast(fChildren.Head()); -} - - -IORequest* -IORequest::NextSubRequest(IORequest* previous) -{ - if (previous == NULL) - return NULL; - return dynamic_cast(fChildren.GetNext(previous)); -} - - -void -IORequest::AddOperation(IOOperation* operation) -{ - MutexLocker locker(fLock); - TRACE("IORequest::AddOperation(%p): request: %p\n", operation, this); - fChildren.Add(operation); - fPendingChildren++; -} - - -void -IORequest::RemoveOperation(IOOperation* operation) -{ - MutexLocker locker(fLock); - fChildren.Remove(operation); - operation->SetParent(NULL); -} - - -status_t -IORequest::CopyData(off_t offset, void* buffer, size_t size) -{ - return _CopyData(buffer, offset, size, true); -} - - -status_t -IORequest::CopyData(const void* buffer, off_t offset, size_t size) -{ - return _CopyData((void*)buffer, offset, size, false); -} - - -status_t -IORequest::_CopyData(void* _buffer, off_t offset, size_t size, bool copyIn) -{ - if (size == 0) - return B_OK; - - uint8* buffer = (uint8*)_buffer; - - if (offset < fOffset || offset + size > fOffset + fLength) { - panic("IORequest::_CopyData(): invalid range: (%lld, %lu)", offset, - size); - return B_BAD_VALUE; - } - - // If we can, we directly copy from/to the virtual buffer. The memory is - // locked in this case. - status_t (*copyFunction)(void*, void*, size_t, team_id, bool); - if (fBuffer->IsPhysical()) { - copyFunction = &IORequest::_CopyPhysical; - } else { - copyFunction = fBuffer->IsUser() - ? &IORequest::_CopyUser : &IORequest::_CopySimple; - } - - // skip bytes if requested - iovec* vecs = fBuffer->Vecs(); - size_t skipBytes = offset - fOffset; - size_t vecOffset = 0; - while (skipBytes > 0) { - if (vecs[0].iov_len > skipBytes) { - vecOffset = skipBytes; - break; - } - - skipBytes -= vecs[0].iov_len; - vecs++; - } - - // copy iovec-wise - while (size > 0) { - size_t toCopy = min_c(size, vecs[0].iov_len - vecOffset); - status_t error = copyFunction(buffer, - (uint8*)vecs[0].iov_base + vecOffset, toCopy, fTeam, copyIn); - if (error != B_OK) - return error; - - buffer += toCopy; - size -= toCopy; - vecs++; - vecOffset = 0; - } - - return B_OK; -} - - -/* static */ status_t -IORequest::_CopySimple(void* bounceBuffer, void* external, size_t size, - team_id team, bool copyIn) -{ - TRACE(" IORequest::_CopySimple(%p, %p, %lu, %d)\n", bounceBuffer, external, - size, copyIn); - if (copyIn) - memcpy(bounceBuffer, external, size); - else - memcpy(external, bounceBuffer, size); - return B_OK; -} - - -/* static */ status_t -IORequest::_CopyPhysical(void* bounceBuffer, void* external, size_t size, - team_id team, bool copyIn) -{ - if (copyIn) { - return vm_memcpy_from_physical(bounceBuffer, (addr_t)external, size, - false); - } - - return vm_memcpy_to_physical((addr_t)external, bounceBuffer, size, false); -} - - -/* static */ status_t -IORequest::_CopyUser(void* _bounceBuffer, void* _external, size_t size, - team_id team, bool copyIn) -{ - uint8* bounceBuffer = (uint8*)_bounceBuffer; - uint8* external = (uint8*)_external; - - while (size > 0) { - static const int32 kEntryCount = 8; - physical_entry entries[kEntryCount]; - - uint32 count = kEntryCount; - status_t error = get_memory_map_etc(team, external, size, entries, - &count); - if (error != B_OK && error != B_BUFFER_OVERFLOW) { - panic("IORequest::_CopyUser(): Failed to get physical memory for " - "user memory %p\n", external); - return B_BAD_ADDRESS; - } - - for (uint32 i = 0; i < count; i++) { - const physical_entry& entry = entries[i]; - error = _CopyPhysical(bounceBuffer, entry.address, - entry.size, team, copyIn); - if (error != B_OK) - return error; - - size -= entry.size; - bounceBuffer += entry.size; - external += entry.size; - } - } - - return B_OK; -} - - -void -IORequest::Dump() const -{ - kprintf("io_request at %p\n", this); - - kprintf(" owner: %p\n", fOwner); - kprintf(" parent: %p\n", fParent); - kprintf(" status: %s\n", strerror(fStatus)); - kprintf(" mutex: %p\n", &fLock); - kprintf(" IOBuffer: %p\n", fBuffer); - kprintf(" offset: %Ld\n", fOffset); - kprintf(" length: %lu\n", fLength); - kprintf(" transfer size: %lu\n", fTransferSize); - kprintf(" relative offset: %lu\n", fRelativeParentOffset); - kprintf(" pending children: %ld\n", fPendingChildren); - kprintf(" flags: %#lx\n", fFlags); - kprintf(" team: %ld\n", fTeam); - kprintf(" thread: %ld\n", fThread); - kprintf(" r/w: %s\n", fIsWrite ? "write" : "read"); - kprintf(" partial transfer: %s\n", fPartialTransfer ? "yes" : "no"); - kprintf(" finished cvar: %p\n", &fFinishedCondition); - kprintf(" iteration:\n"); - kprintf(" vec index: %lu\n", fVecIndex); - kprintf(" vec offset: %lu\n", fVecOffset); - kprintf(" remaining bytes: %lu\n", fRemainingBytes); - kprintf(" callbacks:\n"); - kprintf(" finished %p, cookie %p\n", fFinishedCallback, fFinishedCookie); - kprintf(" iteration %p, cookie %p\n", fIterationCallback, - fIterationCookie); - kprintf(" children:\n"); - - IORequestChunkList::ConstIterator iterator = fChildren.GetIterator(); - while (iterator.HasNext()) { - kprintf(" %p\n", iterator.Next()); - } - - set_debug_variable("_parent", (addr_t)fParent); - set_debug_variable("_mutex", (addr_t)&fLock); - set_debug_variable("_buffer", (addr_t)fBuffer); - set_debug_variable("_cvar", (addr_t)&fFinishedCondition); -} - - -// #pragma mark - allocator - - -#if KERNEL_HEAP_LEAK_CHECK -static addr_t -get_caller() -{ - // Find the first return address outside of the allocator code. Note, that - // this makes certain assumptions about how the code for the functions - // ends up in the kernel object. - addr_t returnAddresses[5]; - int32 depth = arch_debug_get_stack_trace(returnAddresses, 5, 0, 1, false); - - // find the first return address inside the VIP allocator - int32 i = 0; - for (i = 0; i < depth; i++) { - if (returnAddresses[i] >= (addr_t)&get_caller - && returnAddresses[i] < (addr_t)&vip_io_request_allocator_init) { - break; - } - } - - // now continue until we have the first one outside - for (; i < depth; i++) { - if (returnAddresses[i] < (addr_t)&get_caller - || returnAddresses[i] > (addr_t)&vip_io_request_allocator_init) { - return returnAddresses[i]; - } - } - - return 0; -} -#endif - - -void* -vip_io_request_malloc(size_t size) -{ - void* address = heap_memalign(sVIPHeap, 0, size); -#if KDEBUG - if (address == NULL) - panic("vip_io_request_malloc(): VIP heap %p out of memory", sVIPHeap); -#endif - return address; -} - - -void -vip_io_request_free(void* address) -{ - heap_free(sVIPHeap, address); -} - - -void -io_request_free(void* address) -{ - if (heap_free(sVIPHeap, address) != B_OK) - free(address); -} - - -void -vip_io_request_allocator_init() -{ - static const heap_class heapClass = { - "VIP I/O", /* name */ - 100, /* initial percentage */ - B_PAGE_SIZE / 8, /* max allocation size */ - B_PAGE_SIZE, /* page size */ - 8, /* min bin size */ - 4, /* bin alignment */ - 8, /* min count per page */ - 16 /* max waste per page */ - }; - - void* address = NULL; - area_id area = create_area("VIP I/O heap", &address, B_ANY_KERNEL_ADDRESS, - VIP_HEAP_SIZE, B_FULL_LOCK, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - if (area < B_OK) { - panic("vip_io_request_allocator_init(): couldn't allocate VIP I/O " - "heap area"); - return; - } - - sVIPHeap = heap_create_allocator("VIP I/O heap", (addr_t)address, - VIP_HEAP_SIZE, &heapClass); - if (sVIPHeap == NULL) { - panic("vip_io_request_allocator_init(): failed to create VIP I/O " - "heap\n"); - return; - } - -#if KERNEL_HEAP_LEAK_CHECK - heap_set_get_caller(sVIPHeap, &get_caller); -#endif - - dprintf("vip_io_request_allocator_init(): created VIP I/O heap: %p\n", - sVIPHeap); -} - - -// #pragma mark - - - -#if 0 - -/*! Creates an I/O request with the specified buffer and length. - - \param write write access if true, read access if false. - \param flags allows several flags to be specified: - \c B_USER_IO_REQUEST the buffer is assumed to be a userland buffer - and handled with special care. - \c B_ASYNC_IO_REQUEST the I/O request is to be fulfilled asynchronously. - \c B_PHYSICAL_IO_REQUEST the buffer specifies a physical rather than a - virtual address. - \param _request If successful, the location pointed to by this parameter - will contain a pointer to the created request. -*/ -status_t -create_io_request(void* buffer, size_t length, bool write, uint32 flags, - io_request** _request) -{ - return B_ERROR; -} - - -/*! Creates an I/O request from the specified I/O vector and length. - See above for more info. -*/ -status_t -create_io_request_vecs(iovec* vecs, size_t count, size_t length, bool write, - uint32 flags, io_request** _request) -{ - return B_ERROR; -} - - -/*! Prepares the I/O request by locking its memory, and, if \a virtualOnly - is \c false, will retrieve the physical pages. -*/ -status_t -prepare_io_request(io_request* request, bool virtualOnly) -{ - return B_ERROR; -} - - -/*! Prepares the I/O request by locking its memory, and mapping/moving the - pages as needed to fulfill the DMA restrictions. - If needed, a bounce buffer is used for DMA. -*/ -status_t -prepare_io_request_dma(io_request* request, dma_resource* dmaResource) -{ - return B_ERROR; -} - - -/*! Returns the buffers of the I/O request mapped into kernel memory. - This can be used by drivers to fill an I/O request manually. -*/ -status_t -map_io_request(io_request* request, iovec* vecs, size_t count) -{ - return B_ERROR; -} - - -/*! Get the memory map of the DMA buffer for this I/O request. - This can be used to retrieve the physical pages to feed the hardware's - DMA engine with. -*/ -status_t -get_io_request_memory_map(dma_buffer* buffer, io_request* request, iovec* vecs, - size_t count) -{ - return B_ERROR; -} - - -/*! Unmaps any previously mapped data, and will copy the data back from any - bounce buffers if necessary. -*/ -status_t -complete_io_request_dma(io_request* request, dma_resource* dmaResource) -{ - return B_ERROR; -} - - -/*! Unmaps any previously mapped data. -*/ -status_t -complete_io_request(io_request* request) -{ - return B_ERROR; -} - - -void -delete_io_request(io_request* request) -{ -} - -#endif // 0 - diff --git a/src/system/kernel/fs/vfs.cpp b/src/system/kernel/fs/vfs.cpp index 7255eda1e9..e21349881b 100644 --- a/src/system/kernel/fs/vfs.cpp +++ b/src/system/kernel/fs/vfs.cpp @@ -53,7 +53,7 @@ #include #include "fifo.h" -#include "io_requests.h" +#include "IORequest.h" //#define TRACE_VFS diff --git a/src/system/kernel/vm/VMAnonymousCache.cpp b/src/system/kernel/vm/VMAnonymousCache.cpp index c0d3c2c05e..5c99653454 100644 --- a/src/system/kernel/vm/VMAnonymousCache.cpp +++ b/src/system/kernel/vm/VMAnonymousCache.cpp @@ -38,7 +38,7 @@ #include #include -#include "io_requests.h" +#include "IORequest.h" #if ENABLE_SWAP_SUPPORT diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index 4849ae22d9..825297c5f9 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -46,7 +46,7 @@ #include #include "VMAnonymousCache.h" -#include "io_requests.h" +#include "IORequest.h" //#define TRACE_VM diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index 56f56bf8eb..78ddd14b9a 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -32,8 +32,8 @@ #include #include +#include "IORequest.h" #include "PageCacheLocker.h" -#include "io_requests.h" //#define TRACE_VM_PAGE