From 6969690afee7ab446c14e04011cf5cadedb09c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Axel=20D=C3=B6rfler?= Date: Fri, 18 Jul 2008 14:39:46 +0000 Subject: [PATCH] bonefish + axeld: * Moved the old I/O scheduler code into the device manager, and replaced its contents completely :-) * Implemented the DMA and I/O requests/scheduler framework - for now in C++ only. It's a work in progress and not used anywhere yet. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@26488 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- .../kernel/device_manager/IOScheduler.cpp | 219 +++++++++ .../kernel/device_manager/IOScheduler.h | 67 +++ src/system/kernel/device_manager/Jamfile | 4 + .../kernel/device_manager/dma_resources.cpp | 378 +++++++++++++++ .../kernel/device_manager/dma_resources.h | 94 ++++ .../kernel/device_manager/io_requests.cpp | 429 ++++++++++++++++++ .../kernel/device_manager/io_requests.h | 194 ++++++++ src/system/kernel/fs/IOScheduler.cpp | 115 ----- src/system/kernel/fs/IOScheduler.h | 56 --- src/system/kernel/fs/Jamfile | 1 - 10 files changed, 1385 insertions(+), 172 deletions(-) create mode 100644 src/system/kernel/device_manager/IOScheduler.cpp create mode 100644 src/system/kernel/device_manager/IOScheduler.h create mode 100644 src/system/kernel/device_manager/dma_resources.cpp create mode 100644 src/system/kernel/device_manager/dma_resources.h create mode 100644 src/system/kernel/device_manager/io_requests.cpp create mode 100644 src/system/kernel/device_manager/io_requests.h delete mode 100644 src/system/kernel/fs/IOScheduler.cpp delete mode 100644 src/system/kernel/fs/IOScheduler.h diff --git a/src/system/kernel/device_manager/IOScheduler.cpp b/src/system/kernel/device_manager/IOScheduler.cpp new file mode 100644 index 0000000000..39130b46a3 --- /dev/null +++ b/src/system/kernel/device_manager/IOScheduler.cpp @@ -0,0 +1,219 @@ +/* + * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2004-2008, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + */ + + +#include "IOScheduler.h" + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + + +IOScheduler::IOScheduler(DMAResource* resource) + : + fDMAResource(resource) +{ + mutex_init(&fLock, "I/O scheduler"); + B_INITIALIZE_SPINLOCK(&fFinisherLock); +} + + +IOScheduler::~IOScheduler() +{ + mutex_lock(&fLock); + mutex_destroy(&fLock); + + while (IOOperation* operation = fUnusedOperations.RemoveHead()) + delete operation; +} + + +status_t +IOScheduler::Init(const char* name) +{ + fNewRequestCondition.Init(this, "I/O new request"); + fFinishedOperationCondition.Init(this, "I/O finished operation"); + + size_t count = fDMAResource != NULL ? fDMAResource->BufferCount() : 16; + for (size_t i = 0; i < count; i++) { + IOOperation* operation = new(std::nothrow) IOOperation; + if (operation == NULL) + return B_NO_MEMORY; + + fUnusedOperations.Add(operation); + } + + // start thread for device + fThread = spawn_kernel_thread(&_SchedulerThread, name, B_NORMAL_PRIORITY, + (void *)this); + if (fThread < B_OK) + return fThread; + + resume_thread(fThread); + return B_OK; +} + + +status_t +IOScheduler::ScheduleRequest(IORequest* request) +{ + IOBuffer* buffer = request->Buffer(); + + // TODO: it would be nice to be able to lock the memory later, but we can't + // easily do it in the I/O scheduler without being able to asynchronously + // lock memory (via another thread or a dedicated call). + + if (buffer->IsVirtual()) { + status_t status = buffer->LockMemory(request->IsWrite()); + if (status != B_OK) + return status; + } + + MutexLocker _(fLock); + fUnscheduledRequests.Add(request); + + return B_OK; +} + + +void +IOScheduler::AbortRequest(IORequest* request, status_t status) +{ + // TODO:... +//B_CANCELED +} + + +void +IOScheduler::OperationCompleted(IOOperation* operation, status_t status) +{ + InterruptsLocker _; + SpinLocker locker(fFinisherLock); + + // finish operation only once + if (operation->Status() <= 0) + return; + + operation->SetStatus(status); + + fCompletedOperations.Add(operation); + locker.Unlock(); + + locker.SetTo(thread_spinlock, false); + thread_interrupt(thread_get_thread_struct_locked(fThread), false); +} + + +/*! Must not be called with the fLock held. */ +void +IOScheduler::_Finisher() +{ + while (true) { + InterruptsSpinLocker locker(fFinisherLock); + IOOperation* operation = fCompletedOperations.RemoveHead(); + if (operation == NULL) + return; + + locker.Unlock(); + + if (!operation->Finish()) { + // TODO: This must be done differently once the scheduler implements + // an actual scheduling policy (other than no-op). + fIOCallback(fIOCallbackData, operation); + } else { + MutexLocker _(fLock); + operation->Parent()->RemoveOperation(operation); + + fUnusedOperations.Add(operation); + } + } +} + + +IOOperation* +IOScheduler::_GetOperation() +{ + while (true) { + MutexLocker locker(fLock); + + IOOperation* operation = fUnusedOperations.RemoveHead(); + if (operation != NULL) + return operation; + + ConditionVariableEntry entry; + fFinishedOperationCondition.Add(&entry); + + locker.Unlock(); + + entry.Wait(); + _Finisher(); + } +} + + +status_t +IOScheduler::_Scheduler() +{ +// TODO: This is a no-op scheduler. Implement something useful! + while (true) { + MutexLocker locker(fLock); + IORequest* request = fUnscheduledRequests.RemoveHead(); + + if (request == NULL) { + ConditionVariableEntry entry; + fNewRequestCondition.Add(&entry); + locker.Unlock(); + + if (entry.Wait(B_CAN_INTERRUPT) != B_OK) + _Finisher(); + + continue; + } + + locker.Unlock(); + + if (fDMAResource != NULL) { + while (request->RemainingBytes() > 0) { + IOOperation* operation = _GetOperation(); + + status_t status = fDMAResource->TranslateNext(request, + operation); + if (status != B_OK) { + AbortRequest(request, status); + break; + } + + fIOCallback(fIOCallbackData, operation); + } + } else { +// TODO: If the device has block size restrictions, we might need to use a +// bounce buffer. + IOOperation* operation = _GetOperation(); + operation->SetRequest(request); + operation->SetOriginalRange(request->Offset(), request->Length()); + fIOCallback(fIOCallbackData, operation); + } + } + + return B_OK; +} + + +status_t +IOScheduler::_SchedulerThread(void *_self) +{ + IOScheduler *self = (IOScheduler *)_self; + return self->_Scheduler(); +} + diff --git a/src/system/kernel/device_manager/IOScheduler.h b/src/system/kernel/device_manager/IOScheduler.h new file mode 100644 index 0000000000..657dd874c1 --- /dev/null +++ b/src/system/kernel/device_manager/IOScheduler.h @@ -0,0 +1,67 @@ +/* + * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2004-2008, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + */ +#ifndef IO_SCHEDULER_H +#define IO_SCHEDULER_H + +#include + +#include +#include +#include + +#include "dma_resources.h" +#include "io_requests.h" + + +class IOCallback { +public: + virtual status_t DoIO(IOOperation* operation); +}; + +typedef status_t (*io_callback)(void* data, io_operation* operation); + + +class IOScheduler { +public: + IOScheduler(DMAResource* resource); + ~IOScheduler(); + + status_t Init(const char* name); + + void SetCallback(IOCallback& callback); + void SetCallback(io_callback callback, void* data); + + status_t ScheduleRequest(IORequest* request); + + void AbortRequest(IORequest* request, + status_t status = B_CANCELED); + void OperationCompleted(IOOperation* operation, + status_t status); + // called by the driver when the operation + // has been completed successfully or failed + // for some reason + +private: + void _Finisher(); + IOOperation* _GetOperation(); + status_t _Scheduler(); + static status_t _SchedulerThread(void* self); + +private: + DMAResource* fDMAResource; + spinlock fFinisherLock; + mutex fLock; + thread_id fThread; + io_callback fIOCallback; + void* fIOCallbackData; + IORequestList fUnscheduledRequests; + ConditionVariable fNewRequestCondition; + ConditionVariable fFinishedOperationCondition; + IOOperationList fUnusedOperations; + IOOperationList fCompletedOperations; +}; + +#endif // IO_SCHEDULER_H diff --git a/src/system/kernel/device_manager/Jamfile b/src/system/kernel/device_manager/Jamfile index 3fb122bbe3..8cfd1b1e1d 100644 --- a/src/system/kernel/device_manager/Jamfile +++ b/src/system/kernel/device_manager/Jamfile @@ -9,9 +9,13 @@ KernelMergeObject kernel_device_manager.o : devfs.cpp id_generator.cpp io_resources.cpp + IOScheduler.cpp legacy_drivers.cpp # probe.cpp settings.cpp + + dma_resources.cpp + io_requests.cpp : $(TARGET_KERNEL_PIC_CCFLAGS) ; diff --git a/src/system/kernel/device_manager/dma_resources.cpp b/src/system/kernel/device_manager/dma_resources.cpp new file mode 100644 index 0000000000..a18f781d4c --- /dev/null +++ b/src/system/kernel/device_manager/dma_resources.cpp @@ -0,0 +1,378 @@ +/* + * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2008, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + */ + +#include "dma_resources.h" + +#include +#include + +#include "io_requests.h" + + +const size_t kMaxBounceBufferSize = 4 * B_PAGE_SIZE; + + +DMABuffer* +DMABuffer::Create(size_t count, void* bounceBuffer, addr_t physicalBounceBuffer) +{ + DMABuffer* buffer = (DMABuffer*)malloc( + sizeof(DMABuffer) + sizeof(iovec) * (count - 1)); + if (buffer == NULL) + return NULL; + + buffer->fBounceBuffer = bounceBuffer; + buffer->fPhysicalBounceBuffer = physicalBounceBuffer; + buffer->fVecCount = count; + + return buffer; +} + + +void +DMABuffer::SetVecCount(uint32 count) +{ + fVecCount = count; +} + + +void +DMABuffer::AddVec(void* base, size_t size) +{ + iovec& vec = fVecs[fVecCount++]; + vec.iov_len = size; +} + + +void +DMABuffer::SetToBounceBuffer(size_t length) +{ + fVecs[0].iov_base = (void*)fPhysicalBounceBuffer; + fVecs[0].iov_len = length; + fVecCount = 1; +} + + +// #pragma mark - + + +DMAResource::DMAResource() +{ + mutex_init(&fLock, "dma resource"); +} + + +DMAResource::~DMAResource() +{ + mutex_destroy(&fLock); + free(fScratchVecs); +} + + +status_t +DMAResource::Init(const dma_restrictions& restrictions, size_t blockSize, + uint32 bufferCount) +{ + fRestrictions = restrictions; + fBlockSize = blockSize == 0 ? 1 : blockSize; + fBufferCount = bufferCount; + fBounceBufferSize = 0; + + if (fRestrictions.high_address == 0) + fRestrictions.high_address = ~(addr_t)0; + if (fRestrictions.max_segment_count == 0) + fRestrictions.max_segment_count = 16; + if (fRestrictions.alignment == 0) + fRestrictions.alignment = 1; + + if (_NeedsBoundsBuffers()) { +// TODO: Enforce that the bounce buffer size won't cross boundaries. + fBounceBufferSize = restrictions.max_segment_size; + if (fBounceBufferSize > kMaxBounceBufferSize) + fBounceBufferSize = max_c(kMaxBounceBufferSize, fBlockSize); + } + + fScratchVecs = (iovec*)malloc( + sizeof(iovec) * fRestrictions.max_segment_count); + if (fScratchVecs == NULL) + return B_NO_MEMORY; + + // TODO: create bounce buffers in as few areas as feasible + for (size_t i = 0; i < fBufferCount; i++) { + DMABuffer* buffer; + status_t error = CreateBuffer(fBounceBufferSize, &buffer); + if (error != B_OK) + return error; + + fDMABuffers.Add(buffer); + } + + return B_OK; +} + + +status_t +DMAResource::CreateBuffer(size_t size, DMABuffer** _buffer) +{ + void* bounceBuffer = NULL; + addr_t physicalBase = 0; + area_id area = -1; + + if (size != 0) { + if (fRestrictions.alignment > B_PAGE_SIZE + || fRestrictions.boundary > B_PAGE_SIZE) + panic("not yet implemented"); + + size = ROUNDUP(size, B_PAGE_SIZE); + + bounceBuffer = (void*)fRestrictions.low_address; +// TODO: We also need to enforce the boundary restrictions. + area = create_area("dma buffer", &bounceBuffer, size, + B_PHYSICAL_BASE_ADDRESS, B_CONTIGUOUS, + B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + if (area < B_OK) + return area; + + physical_entry entry; + if (get_memory_map(bounceBuffer, size, &entry, 1) != B_OK) { + panic("get_memory_map() failed."); + delete_area(area); + return B_ERROR; + } + + physicalBase = (addr_t)entry.address; + + if (fRestrictions.high_address < physicalBase + size) { + delete_area(area); + return B_NO_MEMORY; + } + } + + DMABuffer* buffer = DMABuffer::Create(fRestrictions.max_segment_count, + bounceBuffer, physicalBase); + if (buffer == NULL) { + delete_area(area); + return B_NO_MEMORY; + } + + *_buffer = buffer; + return B_OK; +} + + +status_t +DMAResource::TranslateNext(IORequest* request, IOOperation* operation) +{ + IOBuffer* buffer = request->Buffer(); + off_t offset = request->Offset(); + + MutexLocker locker(fLock); + + DMABuffer* dmaBuffer = fDMABuffers.RemoveHead(); + if (dmaBuffer == NULL) + return B_BUSY; + + iovec* vecs = NULL; + uint32 segmentCount = 0; + size_t totalLength = min_c(buffer->Length(), + fRestrictions.max_transfer_size); + + bool partialOperation = (offset & (fBlockSize - 1)) != 0; + bool needsBounceBuffer = partialOperation; + + if (buffer->IsVirtual()) { + // Unless we need the bounce buffer anyway, we have to translate the + // virtual addresses to physical addresses, so we can check the DMA + // restrictions. + if (!needsBounceBuffer) { + size_t transferLeft = totalLength; + vecs = fScratchVecs; + +// TODO: take iteration state of the IORequest into account! + for (uint32 i = 0; i < buffer->VecCount(); i++) { + iovec& vec = buffer->VecAt(i); + size_t size = vec.iov_len; + if (size > transferLeft) + size = transferLeft; + + addr_t base = (addr_t)vec.iov_base; + while (size > 0 && segmentCount + < fRestrictions.max_segment_count) { + physical_entry entry; + get_memory_map((void*)base, size, &entry, 1); + + vecs[segmentCount].iov_base = entry.address; + vecs[segmentCount].iov_len = entry.size; + + transferLeft -= entry.size; + segmentCount++; + } + + if (transferLeft == 0) + break; + } + + totalLength -= transferLeft; + } + } else { + // We do already have physical adresses. + locker.Unlock(); + vecs = buffer->Vecs(); + segmentCount = min_c(buffer->VecCount(), + fRestrictions.max_segment_count); + } + +// locker.Lock(); + + // check alignment, boundaries, etc. and set vecs in DMA buffer + + size_t dmaLength = 0; + iovec vec; + if (vecs != NULL) + vec = vecs[0]; + for (uint32 i = 0; i < segmentCount;) { + addr_t base = (addr_t)vec.iov_base; + size_t length = vec.iov_len; + + if ((base & (fRestrictions.alignment - 1)) != 0) { + needsBounceBuffer = true; + break; + } + + if (((base + length) & (fRestrictions.alignment - 1)) != 0) { + length = ((base + length) & ~(fRestrictions.alignment - 1)) - base; + if (length == 0) { + needsBounceBuffer = true; + break; + } + } + + if (fRestrictions.boundary > 0) { + addr_t baseBoundary = base / fRestrictions.boundary; + if (baseBoundary != (base + (length - 1)) / fRestrictions.boundary) + length = (baseBoundary + 1) * fRestrictions.boundary - base; + } + + dmaBuffer->AddVec((void*)base, length); + dmaLength += length; + + if ((vec.iov_len -= length) > 0) { + vec.iov_base = (void*)((addr_t)vec.iov_base + length); + } else { + if (++i < segmentCount) + vec = vecs[i]; + } + } + + if (dmaLength < fBlockSize) { + dmaLength = 0; + needsBounceBuffer = true; + partialOperation = true; + } else if ((dmaLength & (fBlockSize - 1)) != 0) { + size_t toCut = dmaLength & (fBlockSize - 1); + dmaLength -= toCut; + int32 dmaVecCount = dmaBuffer->VecCount(); + for (int32 i = dmaVecCount - 1 && toCut > 0; i >= 0; i--) { + iovec& vec = dmaBuffer->VecAt(i); + size_t length = vec.iov_len; + if (length <= toCut) { + dmaVecCount--; + toCut -= length; + } else { + vec.iov_len -= toCut; + break; + } + } + + dmaBuffer->SetVecCount(dmaVecCount); + } + + operation->SetOriginalRange(offset, dmaLength); + + if (needsBounceBuffer) { + // If the size of the buffer we could transfer is pathologically small, + // we always use the bounce buffer. + // TODO: Use a better heuristics than bounce buffer size / 2, Or even + // better attach the bounce buffer to the DMA buffer. + if (dmaLength < fBounceBufferSize / 2) { + if (partialOperation) { + off_t diff = offset & (fBlockSize - 1); + offset -= diff; + dmaLength += diff; + } + + addr_t base = (addr_t)vecs[0].iov_base; + size_t length = vecs[0].iov_len; + if ((base & (fRestrictions.alignment - 1)) != 0) { + addr_t diff = base - (base & ~(fRestrictions.alignment - 1)); + length += diff; + } + + dmaLength = max_c(totalLength, fBlockSize); + dmaLength = (dmaLength + fBlockSize - 1) & ~(fBlockSize - 1); + dmaLength = min_c(dmaLength, fBounceBufferSize); + dmaBuffer->SetToBounceBuffer(dmaLength); + + operation->SetRange(offset, dmaLength); + } else + needsBounceBuffer = false; + } + + operation->SetPartialOperation(partialOperation); + operation->SetRequest(request); + request->Advance(operation->OriginalLength()); + + return B_OK; +} + + +void +DMAResource::RecycleBuffer(DMABuffer* buffer) +{ + MutexLocker _(fLock); + fDMABuffers.Add(buffer); +} + + +bool +DMAResource::_NeedsBoundsBuffers() const +{ + return fRestrictions.alignment > 1 + || fRestrictions.low_address != 0 + || fRestrictions.high_address != ~(addr_t)0 + || fBlockSize > 1; +} + + + + +#if 0 + + +status_t +create_dma_resource(restrictions) +{ + // Restrictions are: transfer size, address space, alignment + // segment min/max size, num segments +} + + +void +delete_dma_resource(resource) +{ +} + + +dma_buffer_alloc(resource, size) +{ +} + + +dma_buffer_free(buffer) +{ +// Allocates or frees memory in that DMA buffer. +} + +#endif // 0 diff --git a/src/system/kernel/device_manager/dma_resources.h b/src/system/kernel/device_manager/dma_resources.h new file mode 100644 index 0000000000..5ca7e64d19 --- /dev/null +++ b/src/system/kernel/device_manager/dma_resources.h @@ -0,0 +1,94 @@ +/* + * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2008, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + */ +#ifndef DMA_RESOURCES_H +#define DMA_RESOURCES_H + +#include + +#include +#include + + +struct IOOperation; +struct IORequest; + + +struct dma_restrictions { + addr_t low_address; + addr_t high_address; + size_t alignment; + size_t boundary; + size_t max_transfer_size; + uint32 max_segment_count; + size_t max_segment_size; + uint32 flags; +}; + + +class DMABuffer : public DoublyLinkedListLinkImpl { +public: + static DMABuffer* Create(size_t count, void* bounceBuffer, + addr_t physicalBounceBuffer); + + iovec* Vecs() { return fVecs; } + iovec& VecAt(size_t index) { return fVecs[index]; } + uint32 VecCount() const { return fVecCount; } + void SetVecCount(uint32 count); + + void AddVec(void* base, size_t size); + + void* BounceBuffer() const { return fBounceBuffer; } + addr_t PhysicalBounceBuffer() const + { return fPhysicalBounceBuffer; } + + void SetToBounceBuffer(size_t length); + bool UsesBounceBuffer() const + { return fVecCount >= 1 + && (addr_t)fVecs[0].iov_base + == fPhysicalBounceBuffer; } + +private: + void* fBounceBuffer; + addr_t fPhysicalBounceBuffer; + uint32 fVecCount; + iovec fVecs[1]; +}; + + +typedef DoublyLinkedList DMABufferList; + + +class DMAResource { +public: + DMAResource(); + ~DMAResource(); + + status_t Init(const dma_restrictions& restrictions, + size_t blockSize, uint32 bufferCount); + + status_t CreateBuffer(DMABuffer** _buffer) + { return CreateBuffer(0, _buffer); } + status_t CreateBuffer(size_t size, DMABuffer** _buffer); + + status_t TranslateNext(IORequest* request, + IOOperation* operation); + void RecycleBuffer(DMABuffer* buffer); + + uint32 BufferCount() const { return fBufferCount; } + +private: + bool _NeedsBoundsBuffers() const; + + mutex fLock; + dma_restrictions fRestrictions; + size_t fBlockSize; + uint32 fBufferCount; + size_t fBounceBufferSize; + DMABufferList fDMABuffers; + iovec* fScratchVecs; +}; + +#endif // DMA_RESOURCES_H diff --git a/src/system/kernel/device_manager/io_requests.cpp b/src/system/kernel/device_manager/io_requests.cpp new file mode 100644 index 0000000000..0e2e02f424 --- /dev/null +++ b/src/system/kernel/device_manager/io_requests.cpp @@ -0,0 +1,429 @@ +/* + * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2008, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + */ + +#include "io_requests.h" + +#include + +#include + +#include "dma_resources.h" + + +IORequestChunk::~IORequestChunk() +{ +} + + +// #pragma mark - + + +status_t +IOBuffer::LockMemory(bool isWrite) +{ + for (uint32 i = 0; i < fCount; i++) { + status_t status = lock_memory(fVecs[i].iov_base, fVecs[i].iov_len, + isWrite ? 0 : B_READ_DEVICE); + if (status != B_OK) { + _UnlockMemory(i, isWrite); + return status; + } + } + + return B_OK; +} + + +void +IOBuffer::_UnlockMemory(size_t count, bool isWrite) +{ + for (uint32 i = 0; i < count; i++) { + unlock_memory(fVecs[i].iov_base, fVecs[i].iov_len, + isWrite ? 0 : B_READ_DEVICE); + } +} + + +void +IOBuffer::UnlockMemory(bool isWrite) +{ + _UnlockMemory(fCount, isWrite); +} + + +// #pragma mark - + + +bool +IOOperation::Finish() +{ + if (fStatus == B_OK) { + if (IsPartialOperation() && IsWrite()) { + // partial write: copy partial request to bounce buffer + status_t error = fParent->CopyData(OriginalOffset(), + (uint8*)fDMABuffer->BounceBuffer() + + (Offset() - OriginalOffset()), + OriginalLength()); + if (error == B_OK) { + // We're done with the first phase only (read-in block). Now + // do the actual write. + SetPartialOperation(false); + SetStatus(1); + // TODO: Is there a race condition, if the request is + // aborted at the same time? + return false; + } + + SetStatus(error); + } + } + + if (IsRead() && UsesBounceBuffer()) { + // copy the bounce buffer to the final location + status_t error = fParent->CopyData((uint8*)fDMABuffer->BounceBuffer() + + (Offset() - OriginalOffset()), OriginalOffset(), + OriginalLength()); + if (error != B_OK) + SetStatus(error); + } + + // notify parent request + if (fParent != NULL) + fParent->ChunkFinished(this, fStatus); + + return true; +} + + +void +IOOperation::SetRequest(IORequest* request) +{ + if (fParent != NULL) + fParent->RemoveOperation(this); + + fParent = request; + fStatus = 1; + + if (fParent != NULL) + fParent->AddOperation(this); +} + + +void +IOOperation::SetOriginalRange(off_t offset, size_t length) +{ + fOriginalOffset = fOffset = offset; + fOriginalLength = fLength = length; +} + + +void +IOOperation::SetRange(off_t offset, size_t length) +{ + fOffset = offset; + fLength = length; +} + + +void +IOOperation::SetPartialOperation(bool partialOperation) +{ + fIsPartitialOperation = partialOperation; +} + + +bool +IOOperation::IsWrite() const +{ + return fParent->IsWrite(); +} + + +bool +IOOperation::IsRead() const +{ + return fParent->IsRead(); +} + + +// #pragma mark - + + +void +IORequest::Advance(size_t bySize) +{ + fRemainingBytes -= bySize; + + iovec* vecs = fBuffer->Vecs(); + while (vecs[fVecIndex].iov_len - fVecOffset <= bySize) { + bySize -= vecs[fVecIndex].iov_len - fVecOffset; + fVecOffset = 0; + fVecIndex++; + } + + fVecOffset += bySize; +} + + +void +IORequest::AddOperation(IOOperation* operation) +{ + // TODO: locking? + fChildren.Add(operation); +} + + +void +IORequest::RemoveOperation(IOOperation* operation) +{ + // TODO: locking? + fChildren.Remove(operation); +} + + +status_t +IORequest::CopyData(off_t offset, void* buffer, size_t size) +{ + return _CopyData(buffer, offset, size, true); +} + + +status_t +IORequest::CopyData(const void* buffer, off_t offset, size_t size) +{ + return _CopyData((void*)buffer, offset, size, false); +} + + +status_t +IORequest::_CopyData(void* _buffer, off_t offset, size_t size, bool copyIn) +{ + uint8* buffer = (uint8*)_buffer; + + if (offset < fOffset || offset + size > fOffset + size) { + panic("IORequest::_CopyData(): invalid range: (%lld, %lu)", offset, + size); + return B_BAD_VALUE; + } + + // If we can, we directly copy from/to the virtual buffer. The memory is + // locked in this case. + status_t (*copyFunction)(void*, void*, size_t, bool); + if (fBuffer->IsPhysical()) { + copyFunction = &IORequest::_CopyPhysical; + } else { + copyFunction = fBuffer->IsUser() + ? &IORequest::_CopyUser : &IORequest::_CopySimple; + } + + // skip bytes if requested + iovec* vecs = fBuffer->Vecs(); + size_t skipBytes = offset - fOffset; + size_t vecOffset = 0; + while (skipBytes > 0) { + if (vecs[0].iov_len > skipBytes) { + vecOffset = skipBytes; + break; + } + + skipBytes -= vecs[0].iov_len; + vecs++; + } + + // copy iovec-wise + while (size > 0) { + size_t toCopy = min_c(size, vecs[0].iov_len - vecOffset); + status_t error = copyFunction(buffer, + (uint8*)vecs[0].iov_base + vecOffset, toCopy, copyIn); + if (error != B_OK) + return error; + + buffer += toCopy; + size -= toCopy; + vecs++; + vecOffset = 0; + } + + return B_OK; +} + + +/* static */ status_t +IORequest::_CopySimple(void* bounceBuffer, void* external, size_t size, + bool copyIn) +{ + if (copyIn) + memcpy(bounceBuffer, external, size); + else + memcpy(external, bounceBuffer, size); + return B_OK; +} + + +/* static */ status_t +IORequest::_CopyPhysical(void* _bounceBuffer, void* _external, size_t size, + bool copyIn) +{ + uint8* bounceBuffer = (uint8*)_bounceBuffer; + addr_t external = (addr_t)_external; + + while (size > 0) { + addr_t virtualAddress; + status_t error = vm_get_physical_page(external, &virtualAddress, 0); + if (error != B_OK) + return error; + + size_t toCopy = min_c(size, B_PAGE_SIZE); + _CopySimple(bounceBuffer, (void*)external, toCopy, copyIn); + + vm_put_physical_page(virtualAddress); + + size -= toCopy; + bounceBuffer += toCopy; + external += toCopy; + } + + return B_OK; +} + + +/* static */ status_t +IORequest::_CopyUser(void* _bounceBuffer, void* _external, size_t size, + bool copyIn) +{ + uint8* bounceBuffer = (uint8*)_bounceBuffer; + uint8* external = (uint8*)_external; + + while (size > 0) { + physical_entry entries[8]; + int32 count = get_memory_map(external, size, entries, 8); + if (count <= 0) { + panic("IORequest::_CopyUser(): Failed to get physical memory for " + "user memory %p\n", external); + return B_BAD_ADDRESS; + } + + for (int32 i = 0; i < count; i++) { + const physical_entry& entry = entries[i]; + status_t error = _CopyPhysical(bounceBuffer, entry.address, + entry.size, copyIn); + if (error != B_OK) + return error; + + size -= entry.size; + bounceBuffer += entry.size; + external += entry.size; + } + } + + return B_OK; +} + + +// #pragma mark - + + +#if 0 + +/*! Creates an I/O request with the specified buffer and length. + + \param write write access if true, read access if false. + \param flags allows several flags to be specified: + \c B_USER_IO_REQUEST the buffer is assumed to be a userland buffer + and handled with special care. + \c B_ASYNC_IO_REQUEST the I/O request is to be fulfilled asynchronously. + \c B_PHYSICAL_IO_REQUEST the buffer specifies a physical rather than a + virtual address. + \param _request If successful, the location pointed to by this parameter + will contain a pointer to the created request. +*/ +status_t +create_io_request(void* buffer, size_t length, bool write, uint32 flags, + io_request** _request) +{ + return B_ERROR; +} + + +/*! Creates an I/O request from the specified I/O vector and length. + See above for more info. +*/ +status_t +create_io_request_vecs(iovec* vecs, size_t count, size_t length, bool write, + uint32 flags, io_request** _request) +{ + return B_ERROR; +} + + +/*! Prepares the I/O request by locking its memory, and, if \a virtualOnly + is \c false, will retrieve the physical pages. +*/ +status_t +prepare_io_request(io_request* request, bool virtualOnly) +{ + return B_ERROR; +} + + +/*! Prepares the I/O request by locking its memory, and mapping/moving the + pages as needed to fulfill the DMA restrictions. + If needed, a bounce buffer is used for DMA. +*/ +status_t +prepare_io_request_dma(io_request* request, dma_resource* dmaResource) +{ + return B_ERROR; +} + + +/*! Returns the buffers of the I/O request mapped into kernel memory. + This can be used by drivers to fill an I/O request manually. +*/ +status_t +map_io_request(io_request* request, iovec* vecs, size_t count) +{ + return B_ERROR; +} + + +/*! Get the memory map of the DMA buffer for this I/O request. + This can be used to retrieve the physical pages to feed the hardware's + DMA engine with. +*/ +status_t +get_io_request_memory_map(dma_buffer* buffer, io_request* request, iovec* vecs, + size_t count) +{ + return B_ERROR; +} + + +/*! Unmaps any previously mapped data, and will copy the data back from any + bounce buffers if necessary. +*/ +status_t +complete_io_request_dma(io_request* request, dma_resource* dmaResource) +{ + return B_ERROR; +} + + +/*! Unmaps any previously mapped data. +*/ +status_t +complete_io_request(io_request* request) +{ + return B_ERROR; +} + + +void +delete_io_request(io_request* request) +{ +} + +#endif // 0 diff --git a/src/system/kernel/device_manager/io_requests.h b/src/system/kernel/device_manager/io_requests.h new file mode 100644 index 0000000000..49bc1b506d --- /dev/null +++ b/src/system/kernel/device_manager/io_requests.h @@ -0,0 +1,194 @@ +/* + * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2008, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + */ +#ifndef IO_REQUESTS_H +#define IO_REQUESTS_H + +#include + +#include + +#include + +#include "dma_resources.h" + + +#define IO_BUFFER_PHYSICAL 0x01 /* buffer points to physical memory */ +#define IO_BUFFER_USER 0x02 /* buffer points to user memory */ + + +struct DMABuffer; +struct IOOperation; + + +class IOBuffer : public DoublyLinkedListLinkImpl { +public: + static IOBuffer* Create(size_t count); + + bool IsVirtual() const { return !fPhysical; } + bool IsPhysical() const { return fPhysical; } + bool IsUser() const { return !fUser; } + + void SetPhysical(bool physical) + { fPhysical = physical; } + void SetUser(bool user) { fUser = user; } + void SetLength(size_t length) { fLength = length; } + + size_t Length() const { return fLength; } + + iovec* Vecs() { return fVecs; } + iovec& VecAt(size_t index) { return fVecs[index]; } + size_t VecCount() const { return fCount; } + size_t Capacity() const { return fCapacity; } + + status_t LockMemory(bool isWrite); + void UnlockMemory(bool isWrite); + +private: + ~IOBuffer(); + // not implemented + void _UnlockMemory(size_t count, bool isWrite); + + bool fUser; + bool fPhysical; + size_t fLength; + size_t fCount; + size_t fCapacity; + iovec fVecs[1]; +}; + + +class IORequest; + + +class IORequestChunk { +public: + virtual ~IORequestChunk(); + +// virtual status_t Wait(bigtime_t timeout = B_INFINITE_TIMEOUT); + + IORequest* Parent() const { return fParent; } + + status_t Status() const { return fStatus; } + void SetStatus(status_t status) + { fStatus = status; } + + DoublyLinkedListLink* + ListLink() { return &fListLink; } + +protected: + IORequest* fParent; + status_t fStatus; + +public: + DoublyLinkedListLink fListLink; +}; + +typedef DoublyLinkedList > + IORequestChunkList; + + +struct IOOperation : IORequestChunk, DoublyLinkedListLinkImpl { +public: + bool Finish(); + // returns true, if it can be recycled + + void SetRequest(IORequest* request); + void SetOriginalRange(off_t offset, size_t length); + // also sets range + void SetRange(off_t offset, size_t length); + + off_t Offset() const { return fOffset; } + size_t Length() const { return fLength; } + off_t OriginalOffset() const + { return fOriginalOffset; } + size_t OriginalLength() const + { return fOriginalLength; } + + void SetPartialOperation(bool partialOperation); + bool IsPartialOperation() const + { return fIsPartitialOperation; } + bool IsWrite() const; + bool IsRead() const; + + bool UsesBounceBuffer() const + { return fDMABuffer->UsesBounceBuffer(); } + +protected: + DMABuffer* fDMABuffer; + off_t fOffset; + size_t fLength; + off_t fOriginalOffset; + size_t fOriginalLength; + bool fIsPartitialOperation; + bool fUsesBoundsBuffer; +}; + +typedef IOOperation io_operation; +typedef DoublyLinkedList IOOperationList; + + +struct IORequest : IORequestChunk, DoublyLinkedListLinkImpl { + IORequest(); + virtual ~IORequest(); + + virtual void ChunkFinished(IORequestChunk* chunk, + status_t status); + + status_t Init(void* buffer, size_t length, bool write, + uint32 flags); + status_t Init(iovec* vecs, size_t count, size_t length, + bool write, uint32 flags); + + size_t RemainingBytes() const + { return fRemainingBytes; } + + bool IsWrite() const { return fIsWrite; } + bool IsRead() const { return !fIsWrite; } + + IOBuffer* Buffer() const { return fBuffer; } + off_t Offset() const { return fOffset; } + size_t Length() const { return fLength; } + + void Advance(size_t bySize); + + void AddOperation(IOOperation* operation); + void RemoveOperation(IOOperation* operation); + + status_t CopyData(off_t offset, void* buffer, + size_t size); + status_t CopyData(const void* buffer, off_t offset, + size_t size); + +private: + status_t _CopyData(void* buffer, off_t offset, + size_t size, bool copyIn); + static status_t _CopySimple(void* bounceBuffer, void* external, + size_t size, bool copyIn); + static status_t _CopyPhysical(void* bounceBuffer, + void* external, size_t size, bool copyIn); + static status_t _CopyUser(void* bounceBuffer, void* external, + size_t size, bool copyIn); + + IOBuffer* fBuffer; + off_t fOffset; + size_t fLength; + IORequestChunkList fChildren; + uint32 fFlags; + team_id fTeam; + bool fIsWrite; + + // these are for iteration + uint32 fVecIndex; + size_t fVecOffset; + size_t fRemainingBytes; +}; + + +typedef DoublyLinkedList IORequestList; + + +#endif // IO_REQUESTS_H diff --git a/src/system/kernel/fs/IOScheduler.cpp b/src/system/kernel/fs/IOScheduler.cpp deleted file mode 100644 index 5b58aa5a52..0000000000 --- a/src/system/kernel/fs/IOScheduler.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2004-2008, Axel Dörfler, axeld@pinc-software.de. All rights reserved. - * Distributed under the terms of the MIT License. - */ - - -#include "IOScheduler.h" - -#include - -#include -#include - -#include -#include -#include - - -IORequest::IORequest(void *_cookie, off_t _offset, void *_buffer, size_t _size, bool _writeMode) - : - cookie(_cookie), - virtual_address(addr_t(_buffer)), - offset(_offset), - size(_size), - write(_writeMode) -{ -} - - -IORequest::IORequest(void *_cookie, off_t _offset, const void *_buffer, size_t _size, bool _writeMode) - : - cookie(_cookie), - virtual_address(addr_t(const_cast(_buffer))), - offset(_offset), - size(_size), - write(_writeMode) -{ -} - - -// #pragma mark - - - -IOScheduler::IOScheduler(const char* name, device_module_info* module) - : - fModule(module) -{ - mutex_init(&fLock, "I/O scheduler queue"); - - // start thread for device - fThread = spawn_kernel_thread(&IOScheduler::scheduler, name, B_NORMAL_PRIORITY, (void *)this); -#if 0 - if (fThread >= B_OK) - resume_thread(fThread); -#endif -} - - -IOScheduler::~IOScheduler() -{ - kill_thread(fThread); - mutex_destroy(&fLock); -} - - -status_t -IOScheduler::InitCheck() const -{ - if (fThread < B_OK) - return fThread; - - return B_OK; -} - - -status_t -IOScheduler::Process(IORequest &request) -{ - // ToDo: put the request into the queue, and wait until it got processed by the scheduler - // ToDo: translate addresses into physical locations - // ToDo: connect to the DPC mechanism in the SCSI/IDE bus manager? - // ToDo: assume locked memory? - - if (request.write) - return fModule->write(request.cookie, request.offset, (const void *)request.virtual_address, &request.size); - - return fModule->read(request.cookie, request.offset, (void *)request.virtual_address, &request.size); -} - - -#if 0 -IOScheduler * -IOScheduler::GetScheduler() -{ - return NULL; -} -#endif - - -int32 -IOScheduler::Scheduler() -{ - // main loop - - return 0; -} - - -int32 -IOScheduler::scheduler(void *_self) -{ - IOScheduler *self = (IOScheduler *)_self; - return self->Scheduler(); -} - diff --git a/src/system/kernel/fs/IOScheduler.h b/src/system/kernel/fs/IOScheduler.h deleted file mode 100644 index b12c49e6b2..0000000000 --- a/src/system/kernel/fs/IOScheduler.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2004-2008, Axel Dörfler, axeld@pinc-software.de. All rights reserved. - * Distributed under the terms of the MIT License. - */ -#ifndef IO_SCHEDULER_H -#define IO_SCHEDULER_H - - -#include - -#include -#include - - -class IORequest : public DoublyLinkedListLinkImpl { - public: - IORequest(void *cookie, off_t offset, void *buffer, size_t size, bool write = false); - IORequest(void *cookie, off_t offset, const void *buffer, size_t size, bool write = true); - // ToDo: iovecs version? - - size_t Size() const { return size; } - - void *cookie; - addr_t physical_address; - addr_t virtual_address; - off_t offset; - size_t size; - bool write; - thread_id thread; -}; - - -class IOScheduler { - public: - IOScheduler(const char* name, device_module_info* module); - ~IOScheduler(); - - status_t InitCheck() const; - status_t Process(IORequest& request); - -#if 0 - static IOScheduler *GetScheduler(); -#endif - - private: - int32 Scheduler(); - static int32 scheduler(void*); - - private: - device_module_info* fModule; - mutex fLock; - thread_id fThread; - DoublyLinkedList fRequests; -}; - -#endif /* IO_SCHEDULER_H */ diff --git a/src/system/kernel/fs/Jamfile b/src/system/kernel/fs/Jamfile index f704d6a633..b392099443 100644 --- a/src/system/kernel/fs/Jamfile +++ b/src/system/kernel/fs/Jamfile @@ -8,7 +8,6 @@ UsePrivateHeaders net shared storage ; KernelMergeObject kernel_fs.o : fd.cpp fifo.cpp - IOScheduler.cpp KPath.cpp node_monitor.cpp rootfs.cpp