diff --git a/src/add-ons/kernel/drivers/disk/scsi/scsi_cd/scsi_cd.cpp b/src/add-ons/kernel/drivers/disk/scsi/scsi_cd/scsi_cd.cpp index f80cd8ee80..6e2ec9ff27 100644 --- a/src/add-ons/kernel/drivers/disk/scsi/scsi_cd/scsi_cd.cpp +++ b/src/add-ons/kernel/drivers/disk/scsi/scsi_cd/scsi_cd.cpp @@ -860,7 +860,8 @@ cd_set_capacity(cd_driver_info* info, uint64 capacity, uint32 blockSize) } // TODO: we need to replace the DMAResource in our IOScheduler - status_t status = info->dma_resource->Init(info->node, blockSize, 32); + status_t status = info->dma_resource->Init(info->node, blockSize, 1024, + 32); if (status != B_OK) panic("initializing DMAResource failed: %s", strerror(status)); diff --git a/src/add-ons/kernel/drivers/disk/scsi/scsi_disk/scsi_disk.cpp b/src/add-ons/kernel/drivers/disk/scsi/scsi_disk/scsi_disk.cpp index 6426812e5d..ea093e2e18 100644 --- a/src/add-ons/kernel/drivers/disk/scsi/scsi_disk/scsi_disk.cpp +++ b/src/add-ons/kernel/drivers/disk/scsi/scsi_disk/scsi_disk.cpp @@ -411,7 +411,8 @@ das_set_capacity(das_driver_info* info, uint64 capacity, uint32 blockSize) } // TODO: we need to replace the DMAResource in our IOScheduler - status_t status = info->dma_resource->Init(info->node, blockSize, 32); + status_t status = info->dma_resource->Init(info->node, blockSize, 1024, + 32); if (status != B_OK) panic("initializing DMAResource failed: %s", strerror(status)); diff --git a/src/system/kernel/device_manager/dma_resources.cpp b/src/system/kernel/device_manager/dma_resources.cpp index fc7e50a6f5..614e1e14de 100644 --- a/src/system/kernel/device_manager/dma_resources.cpp +++ b/src/system/kernel/device_manager/dma_resources.cpp @@ -28,17 +28,13 @@ const size_t kMaxBounceBufferSize = 4 * B_PAGE_SIZE; DMABuffer* -DMABuffer::Create(size_t count, void* bounceBuffer, addr_t physicalBounceBuffer, - size_t bounceBufferSize) +DMABuffer::Create(size_t count) { DMABuffer* buffer = (DMABuffer*)malloc( sizeof(DMABuffer) + sizeof(iovec) * (count - 1)); if (buffer == NULL) return NULL; - buffer->fBounceBuffer = bounceBuffer; - buffer->fPhysicalBounceBuffer = physicalBounceBuffer; - buffer->fBounceBufferSize = bounceBufferSize; buffer->fVecCount = count; return buffer; @@ -61,24 +57,15 @@ DMABuffer::AddVec(void* base, size_t size) } -void -DMABuffer::SetToBounceBuffer(size_t length) -{ - fVecs[0].iov_base = (void*)fPhysicalBounceBuffer; - fVecs[0].iov_len = length; - fVecCount = 1; -} - - bool DMABuffer::UsesBounceBufferAt(uint32 index) { - if (index >= fVecCount) + if (index >= fVecCount || fBounceBuffer == NULL) return false; - return (addr_t)fVecs[index].iov_base >= fPhysicalBounceBuffer + return (addr_t)fVecs[index].iov_base >= fBounceBuffer->physical_address && (addr_t)fVecs[index].iov_base - < fPhysicalBounceBuffer + fBounceBufferSize; + < fBounceBuffer->physical_address + fBounceBuffer->size; } @@ -87,9 +74,9 @@ DMABuffer::Dump() const { kprintf("DMABuffer at %p\n", this); - kprintf(" bounce buffer: %p (physical %#lx)\n", fBounceBuffer, - fPhysicalBounceBuffer); - kprintf(" bounce buffer size: %lu\n", fBounceBufferSize); + kprintf(" bounce buffer: %p (physical %#lx)\n", + fBounceBuffer->address, fBounceBuffer->physical_address); + kprintf(" bounce buffer size: %lu\n", fBounceBuffer->size); kprintf(" vecs: %lu\n", fVecCount); for (uint32 i = 0; i < fVecCount; i++) { @@ -111,11 +98,14 @@ DMAResource::~DMAResource() { mutex_destroy(&fLock); free(fScratchVecs); + +// TODO: Delete DMABuffers and BounceBuffers! } status_t -DMAResource::Init(device_node* node, size_t blockSize, uint32 bufferCount) +DMAResource::Init(device_node* node, size_t blockSize, uint32 bufferCount, + uint32 bounceBufferCount) { dma_restrictions restrictions; memset(&restrictions, 0, sizeof(dma_restrictions)); @@ -143,17 +133,18 @@ DMAResource::Init(device_node* node, size_t blockSize, uint32 bufferCount) B_DMA_MAX_SEGMENT_COUNT, &value, true) == B_OK) restrictions.max_segment_count = value; - return Init(restrictions, blockSize, bufferCount); + return Init(restrictions, blockSize, bufferCount, bounceBufferCount); } status_t DMAResource::Init(const dma_restrictions& restrictions, size_t blockSize, - uint32 bufferCount) + uint32 bufferCount, uint32 bounceBufferCount) { fRestrictions = restrictions; fBlockSize = blockSize == 0 ? 1 : blockSize; fBufferCount = bufferCount; + fBounceBufferCount = bounceBufferCount; fBounceBufferSize = 0; if (fRestrictions.high_address == 0) @@ -188,64 +179,85 @@ DMAResource::Init(const dma_restrictions& restrictions, size_t blockSize, if (fScratchVecs == NULL) return B_NO_MEMORY; - // TODO: create bounce buffers in as few areas as feasible for (size_t i = 0; i < fBufferCount; i++) { DMABuffer* buffer; - status_t error = CreateBuffer(fBounceBufferSize, &buffer); + status_t error = CreateBuffer(&buffer); if (error != B_OK) return error; fDMABuffers.Add(buffer); } + // TODO: create bounce buffers in as few areas as feasible + for (size_t i = 0; i < fBounceBufferCount; i++) { + DMABounceBuffer* buffer; + status_t error = CreateBounceBuffer(&buffer); + if (error != B_OK) + return error; + + fBounceBuffers.Add(buffer); + } + return B_OK; } status_t -DMAResource::CreateBuffer(size_t size, DMABuffer** _buffer) +DMAResource::CreateBuffer(DMABuffer** _buffer) +{ + DMABuffer* buffer = DMABuffer::Create(fRestrictions.max_segment_count); + if (buffer == NULL) + return B_NO_MEMORY; + + *_buffer = buffer; + return B_OK; +} + + +status_t +DMAResource::CreateBounceBuffer(DMABounceBuffer** _buffer) { void* bounceBuffer = NULL; addr_t physicalBase = 0; area_id area = -1; + size_t size = ROUNDUP(fBounceBufferSize, B_PAGE_SIZE); - if (size != 0) { - if (fRestrictions.alignment > B_PAGE_SIZE) - dprintf("dma buffer restrictions not yet implemented: alignment %lu\n", fRestrictions.alignment); - if (fRestrictions.boundary > B_PAGE_SIZE) - dprintf("dma buffer restrictions not yet implemented: boundary %lu\n", fRestrictions.boundary); + if (fRestrictions.alignment > B_PAGE_SIZE) + dprintf("dma buffer restrictions not yet implemented: alignment %lu\n", fRestrictions.alignment); + if (fRestrictions.boundary > B_PAGE_SIZE) + dprintf("dma buffer restrictions not yet implemented: boundary %lu\n", fRestrictions.boundary); - size = ROUNDUP(size, B_PAGE_SIZE); - - bounceBuffer = (void*)fRestrictions.low_address; + bounceBuffer = (void*)fRestrictions.low_address; // TODO: We also need to enforce the boundary restrictions. - area = create_area("dma buffer", &bounceBuffer, B_PHYSICAL_BASE_ADDRESS, - size, B_CONTIGUOUS, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - if (area < B_OK) - return area; + area = create_area("dma buffer", &bounceBuffer, B_PHYSICAL_BASE_ADDRESS, + size, B_CONTIGUOUS, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + if (area < B_OK) + return area; - physical_entry entry; - if (get_memory_map(bounceBuffer, size, &entry, 1) != B_OK) { - panic("get_memory_map() failed."); - delete_area(area); - return B_ERROR; - } - - physicalBase = (addr_t)entry.address; - - if (fRestrictions.high_address < physicalBase + size) { - delete_area(area); - return B_NO_MEMORY; - } + physical_entry entry; + if (get_memory_map(bounceBuffer, size, &entry, 1) != B_OK) { + panic("get_memory_map() failed."); + delete_area(area); + return B_ERROR; } - DMABuffer* buffer = DMABuffer::Create(fRestrictions.max_segment_count, - bounceBuffer, physicalBase, fBounceBufferSize); + physicalBase = (addr_t)entry.address; + + if (fRestrictions.high_address < physicalBase + size) { + delete_area(area); + return B_NO_MEMORY; + } + + DMABounceBuffer* buffer = new(std::nothrow) DMABounceBuffer; if (buffer == NULL) { delete_area(area); return B_NO_MEMORY; } + buffer->address = bounceBuffer; + buffer->physical_address = physicalBase; + buffer->size = size; + *_buffer = buffer; return B_OK; } @@ -458,8 +470,18 @@ DMAResource::TranslateNext(IORequest* request, IOOperation* operation) // check alignment, boundaries, etc. and set vecs in DMA buffer + // Fetch a bounce buffer we can use for the DMABuffer. + // TODO: We should do that lazily when needed! + DMABounceBuffer* bounceBuffer = NULL; + if (_NeedsBoundsBuffers()) { + bounceBuffer = fBounceBuffers.Head(); + if (bounceBuffer == NULL) + return B_BUSY; + } + dmaBuffer->SetBounceBuffer(bounceBuffer); + size_t dmaLength = 0; - addr_t physicalBounceBuffer = dmaBuffer->PhysicalBounceBuffer(); + addr_t physicalBounceBuffer = dmaBuffer->PhysicalBounceBufferAddress(); size_t bounceLeft = fBounceBufferSize; size_t transferLeft = totalLength; @@ -593,9 +615,11 @@ DMAResource::TranslateNext(IORequest* request, IOOperation* operation) // entirely. if (diff == 0 && offset + dmaLength > requestEnd) { const iovec& dmaVec = dmaBuffer->VecAt(dmaBuffer->VecCount() - 1); - ASSERT((addr_t)dmaVec.iov_base >= dmaBuffer->PhysicalBounceBuffer() + ASSERT((addr_t)dmaVec.iov_base + >= dmaBuffer->PhysicalBounceBufferAddress() && (addr_t)dmaVec.iov_base - < dmaBuffer->PhysicalBounceBuffer() + fBounceBufferSize); + < dmaBuffer->PhysicalBounceBufferAddress() + + fBounceBufferSize); // We can be certain that the last vec is a bounce buffer vec, // since otherwise the DMA buffer couldn't exceed the end of the // request data. @@ -647,7 +671,7 @@ DMAResource::TranslateNext(IORequest* request, IOOperation* operation) // and hit the max segment count. In this case we just use the // bounce buffer for as much as possible of the total length. dmaBuffer->SetVecCount(0); - addr_t base = dmaBuffer->PhysicalBounceBuffer(); + addr_t base = dmaBuffer->PhysicalBounceBufferAddress(); dmaLength = min_c(totalLength, fBounceBufferSize) & ~(max_c(fBlockSize, fRestrictions.alignment) - 1); _RestrictBoundaryAndSegmentSize(base, dmaLength); @@ -676,7 +700,14 @@ DMAResource::TranslateNext(IORequest* request, IOOperation* operation) min_c(offset + dmaLength, requestEnd) - originalOffset); operation->SetRange(offset, dmaLength); operation->SetPartial(partialBegin != 0, offset + dmaLength > requestEnd); + + // If we don't need the bounce buffer, we put it back, otherwise operation->SetUsesBounceBuffer(bounceLeft < fBounceBufferSize); + if (operation->UsesBounceBuffer()) + fBounceBuffers.RemoveHead(); + else + dmaBuffer->SetBounceBuffer(NULL); + status_t error = operation->Prepare(request); if (error != B_OK) @@ -696,6 +727,10 @@ DMAResource::RecycleBuffer(DMABuffer* buffer) MutexLocker _(fLock); fDMABuffers.Add(buffer); + if (buffer->BounceBuffer() != NULL) { + fBounceBuffers.Add(buffer->BounceBuffer()); + buffer->SetBounceBuffer(NULL); + } } diff --git a/src/system/kernel/device_manager/dma_resources.h b/src/system/kernel/device_manager/dma_resources.h index c5bc7b1dd3..2457929807 100644 --- a/src/system/kernel/device_manager/dma_resources.h +++ b/src/system/kernel/device_manager/dma_resources.h @@ -29,11 +29,18 @@ struct dma_restrictions { }; +struct DMABounceBuffer : public DoublyLinkedListLinkImpl { + void* address; + addr_t physical_address; + size_t size; +}; + +typedef DoublyLinkedList DMABounceBufferList; + + class DMABuffer : public DoublyLinkedListLinkImpl { public: - static DMABuffer* Create(size_t count, void* bounceBuffer, - addr_t physicalBounceBuffer, - size_t bounceBufferSize); + static DMABuffer* Create(size_t count); iovec* Vecs() { return fVecs; } iovec& VecAt(size_t index) { return fVecs[index]; } @@ -42,21 +49,27 @@ public: void AddVec(void* base, size_t size); - void* BounceBuffer() const { return fBounceBuffer; } - addr_t PhysicalBounceBuffer() const - { return fPhysicalBounceBuffer; } + void SetBounceBuffer(DMABounceBuffer* bounceBuffer) + { fBounceBuffer = bounceBuffer; } + DMABounceBuffer* BounceBuffer() const { return fBounceBuffer; } + + void* BounceBufferAddress() const + { return fBounceBuffer + ? fBounceBuffer->address : NULL; } + addr_t PhysicalBounceBufferAddress() const + { return fBounceBuffer + ? fBounceBuffer->physical_address + : 0; } size_t BounceBufferSize() const - { return fBounceBufferSize; } + { return fBounceBuffer + ? fBounceBuffer->size : 0; } bool UsesBounceBufferAt(uint32 index); - void SetToBounceBuffer(size_t length); void Dump() const; private: - void* fBounceBuffer; - addr_t fPhysicalBounceBuffer; - size_t fBounceBufferSize; + DMABounceBuffer* fBounceBuffer; uint32 fVecCount; iovec fVecs[1]; }; @@ -71,13 +84,14 @@ public: ~DMAResource(); status_t Init(const dma_restrictions& restrictions, - size_t blockSize, uint32 bufferCount); + size_t blockSize, uint32 bufferCount, + uint32 bounceBufferCount); status_t Init(device_node* node, size_t blockSize, - uint32 bufferCount); + uint32 bufferCount, + uint32 bounceBufferCount); - status_t CreateBuffer(DMABuffer** _buffer) - { return CreateBuffer(0, _buffer); } - status_t CreateBuffer(size_t size, DMABuffer** _buffer); + status_t CreateBuffer(DMABuffer** _buffer); + status_t CreateBounceBuffer(DMABounceBuffer** _buffer); status_t TranslateNext(IORequest* request, IOOperation* operation); @@ -101,8 +115,10 @@ private: dma_restrictions fRestrictions; size_t fBlockSize; uint32 fBufferCount; + uint32 fBounceBufferCount; size_t fBounceBufferSize; DMABufferList fDMABuffers; + DMABounceBufferList fBounceBuffers; iovec* fScratchVecs; }; diff --git a/src/system/kernel/device_manager/io_requests.cpp b/src/system/kernel/device_manager/io_requests.cpp index 64a5d1d398..9f0a817006 100644 --- a/src/system/kernel/device_manager/io_requests.cpp +++ b/src/system/kernel/device_manager/io_requests.cpp @@ -236,8 +236,8 @@ IOOperation::Finish() if (fParent->IsRead() && UsesBounceBuffer()) { TRACE(" read with bounce buffer\n"); // copy the bounce buffer segments to the final location - uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBuffer(); - addr_t bounceBufferStart = fDMABuffer->PhysicalBounceBuffer(); + uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress(); + addr_t bounceBufferStart = fDMABuffer->PhysicalBounceBufferAddress(); addr_t bounceBufferEnd = bounceBufferStart + fDMABuffer->BounceBufferSize(); @@ -308,8 +308,9 @@ IOOperation::Prepare(IORequest* request) // which will be copied after their respective read phase. if (UsesBounceBuffer()) { TRACE(" write with bounce buffer\n"); - uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBuffer(); - addr_t bounceBufferStart = fDMABuffer->PhysicalBounceBuffer(); + uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress(); + addr_t bounceBufferStart + = fDMABuffer->PhysicalBounceBufferAddress(); addr_t bounceBufferEnd = bounceBufferStart + fDMABuffer->BounceBufferSize(); @@ -522,10 +523,10 @@ IOOperation::_CopyPartialBegin(bool isWrite, bool& singleBlockOnly) if (isWrite) { return fParent->CopyData(OriginalOffset(), - (uint8*)fDMABuffer->BounceBuffer() + relativeOffset, length); + (uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, length); } else { return fParent->CopyData( - (uint8*)fDMABuffer->BounceBuffer() + relativeOffset, + (uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, OriginalOffset(), length); } } @@ -538,8 +539,9 @@ IOOperation::_CopyPartialEnd(bool isWrite) const iovec& lastVec = fDMABuffer->VecAt(fDMABuffer->VecCount() - 1); off_t lastVecPos = fOffset + fLength - fBlockSize; - uint8* base = (uint8*)fDMABuffer->BounceBuffer() + ((addr_t)lastVec.iov_base - + lastVec.iov_len - fBlockSize - fDMABuffer->PhysicalBounceBuffer()); + uint8* base = (uint8*)fDMABuffer->BounceBufferAddress() + + ((addr_t)lastVec.iov_base + lastVec.iov_len - fBlockSize + - fDMABuffer->PhysicalBounceBufferAddress()); // NOTE: this won't work if we don't use the bounce buffer contiguously // (because of boundary alignments). size_t length = OriginalOffset() + OriginalLength() - lastVecPos; diff --git a/src/tests/system/kernel/device_manager/dma_resource_test.cpp b/src/tests/system/kernel/device_manager/dma_resource_test.cpp index e79dcf75a2..65405497d7 100644 --- a/src/tests/system/kernel/device_manager/dma_resource_test.cpp +++ b/src/tests/system/kernel/device_manager/dma_resource_test.cpp @@ -16,8 +16,9 @@ #include "IOScheduler.h" -#define DMA_TEST_BLOCK_SIZE 512 -#define DMA_TEST_BUFFER_COUNT 10 +#define DMA_TEST_BLOCK_SIZE 512 +#define DMA_TEST_BUFFER_COUNT 10 +#define DMA_TEST_BOUNCE_BUFFER_COUNT 128 class TestSuite; @@ -120,7 +121,7 @@ public: restrictions.max_segment_count, restrictions.max_segment_size, restrictions.flags); - status_t status = fDMAResource.Init(restrictions, blockSize, 10); + status_t status = fDMAResource.Init(restrictions, blockSize, 10, 10); if (status != B_OK) panic("initializing DMA resource failed: %s\n", strerror(status)); } @@ -589,8 +590,8 @@ Test::Run(DMAResource& resource) operation.Offset(), operation.Length(), operation.OriginalOffset(), operation.OriginalLength()); dprintf(" DMABuffer %p, %lu vecs, bounce buffer: %p (%p) %s\n", buffer, - buffer->VecCount(), buffer->BounceBuffer(), - (void*)buffer->PhysicalBounceBuffer(), + buffer->VecCount(), buffer->BounceBufferAddress(), + (void*)buffer->PhysicalBounceBufferAddress(), operation.UsesBounceBuffer() ? "used" : "unused"); for (uint32 i = 0; i < buffer->VecCount(); i++) { dprintf(" [%lu] base %p, length %lu%s\n", i, @@ -616,7 +617,7 @@ Test::Run(DMAResource& resource) void* address; if (target.uses_bounce_buffer) { address = (void*)(target.address - + (addr_t)buffer->PhysicalBounceBuffer()); + + (addr_t)buffer->PhysicalBounceBufferAddress()); } else if (fSuite.IsContiguous() || target.address < B_PAGE_SIZE) { address = (void*)(target.address + fSuite.PhysicalDataBase()); } else { @@ -1183,7 +1184,7 @@ dma_test_init_device(void *driverCookie, void **_deviceCookie) return B_NO_MEMORY; status_t status = sDMAResource->Init(restrictions, DMA_TEST_BLOCK_SIZE, - DMA_TEST_BUFFER_COUNT); + DMA_TEST_BUFFER_COUNT, DMA_TEST_BOUNCE_BUFFER_COUNT); if (status != B_OK) { delete sDMAResource; return status;