i2p/haiku-beta6
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Essentially rewrote the IOCache to use a VMCache and unmapped pages for

Browse Source 
caching (similar to the file cache) instead of contiguous areas. This is
probably a little bit slower, but integrates better with the VM -- the
caching doesn't increase memory pressure and the least recently used pages
will automatically be recycled when needed.
There are still memory allocation issues on machines with little memory. The
USB stack apparently tries to allocate a rather big chunk of contiguous
memory, which fails when all not otherwise bound memory is used for caches,
since the VM functions for allocating contiguous memory consider only free
pages ATM.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@36458 a95241bf-73f2-0310-859d-f6bbb57e9c96
This commit is contained in:
Ingo Weinhold
2010-04-24 21:54:49 +00:00
parent 80c7dda631
commit f14480bc2c
2 changed files with 507 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/system/kernel/device_manager/IOCache.cpp
+480 -324
View File
@@ -12,8 +12,10 @@
#include <heap.h> #include <heap.h>
#include <low_resource_manager.h> #include <low_resource_manager.h>
#include <util/AutoLock.h> #include <util/AutoLock.h>
#include <util/DoublyLinkedList.h>
#include <vm/vm.h> #include <vm/vm.h>
#include <vm/VMAddressSpace.h>
#include <vm/VMCache.h>
#include <vm/VMTranslationMap.h>
//#define TRACE_IO_CACHE 1 //#define TRACE_IO_CACHE 1
@@ -24,10 +26,11 @@
#endif #endif
// size of the line table static inline bool
static const size_t kLineTableSize = 128; page_physical_number_less(const vm_page* a, const vm_page* b)
// TODO: The table should shrink/grow dynamically, but we can't allocate {
// memory without risking a deadlock. Use a resource resizer! return a->physical_page_number < b->physical_page_number;
}
struct IOCache::Operation : IOOperation { struct IOCache::Operation : IOOperation {
@@ -35,61 +38,18 @@ struct IOCache::Operation : IOOperation {
}; };
struct IOCache::LineHashDefinition {
typedef off_t KeyType;
typedef Line ValueType;
LineHashDefinition(uint32 sizeShift)
:
fSizeShift(sizeShift)
{
}
size_t HashKey(off_t key) const
{
return size_t(key >> fSizeShift);
}
size_t Hash(const Line* value) const
{
return HashKey(value->offset);
}
bool Compare(off_t key, const Line* value) const
{
return value->offset == key;
}
Line*& GetLink(Line* value) const
{
return value->hashNext;
}
private:
uint32 fSizeShift;
};
struct IOCache::LineTable : public BOpenHashTable<LineHashDefinition> {
LineTable(const LineHashDefinition& definition)
:
BOpenHashTable<LineHashDefinition>(definition)
{
}
};
IOCache::IOCache(DMAResource* resource, size_t cacheLineSize) IOCache::IOCache(DMAResource* resource, size_t cacheLineSize)
: :
fDeviceCapacity(0), fDeviceCapacity(0),
fLineSize(cacheLineSize), fLineSize(cacheLineSize),
fPagesPerLine(cacheLineSize / B_PAGE_SIZE),
fDMAResource(resource), fDMAResource(resource),
fIOCallback(NULL), fIOCallback(NULL),
fIOCallbackData(NULL), fIOCallbackData(NULL),
fLineTable(NULL), fArea(-1),
fUnusedLine(0), fCache(NULL),
fLineCount(0), fPages(NULL),
fLowMemoryHandlerRegistered(false) fVecs(NULL)
{ {
TRACE("%p->IOCache::IOCache(%p, %" B_PRIuSIZE ")\n", this, resource, TRACE("%p->IOCache::IOCache(%p, %" B_PRIuSIZE ")\n", this, resource,
cacheLineSize); cacheLineSize);
@@ -100,7 +60,6 @@ IOCache::IOCache(DMAResource* resource, size_t cacheLineSize)
"multiple of the page size.", cacheLineSize); "multiple of the page size.", cacheLineSize);
} }
mutex_init(&fLock, "I/O cache");
mutex_init(&fSerializationLock, "I/O cache request serialization"); mutex_init(&fSerializationLock, "I/O cache request serialization");
fLineSizeShift = 0; fLineSizeShift = 0;
@@ -113,18 +72,14 @@ IOCache::IOCache(DMAResource* resource, size_t cacheLineSize)
IOCache::~IOCache() IOCache::~IOCache()
{ {
if (fLowMemoryHandlerRegistered) { if (fArea >= 0) {
unregister_low_resource_handler(&_LowMemoryHandlerEntry, this); vm_page_unreserve_pages(&fMappingReservation);
// TODO: Avoid the race condition with the handler! delete_area(fArea);
} }
delete fLineTable; delete[] fPages;
delete[] fVecs;
while (Line* line = fUsedLines.RemoveHead())
_FreeLine(line);
_FreeLine(fUnusedLine);
mutex_destroy(&fLock);
mutex_destroy(&fSerializationLock); mutex_destroy(&fSerializationLock);
} }
@@ -134,34 +89,34 @@ IOCache::Init(const char* name)
{ {
TRACE("%p->IOCache::Init(\"%s\")\n", this, name); TRACE("%p->IOCache::Init(\"%s\")\n", this, name);
// create the line hash table // create the area for mapping cache lines
fLineTable = new(std::nothrow) LineTable(LineHashDefinition( fArea = vm_create_null_area(B_SYSTEM_TEAM, "I/O cache line", &fAreaBase,
fLineSizeShift)); B_ANY_KERNEL_ADDRESS, fLineSize, 0);
if (fLineTable == NULL) if (fArea < 0)
return fArea;
// reserve pages for mapping a complete cache line
VMAddressSpace* addressSpace = VMAddressSpace::Kernel();
VMTranslationMap* translationMap = addressSpace->TranslationMap();
size_t pagesNeeded = translationMap->MaxPagesNeededToMap((addr_t)fAreaBase,
(addr_t)fAreaBase + fLineSize - 1);
vm_page_reserve_pages(&fMappingReservation, pagesNeeded,
VM_PRIORITY_SYSTEM);
// get the area's cache
VMArea* area = VMAreaHash::Lookup(fArea);
if (area == NULL) {
panic("IOCache::Init(): Where's our area (id: %" B_PRId32 ")?!", fArea);
return B_ERROR;
}
fCache = area->cache;
// allocate arrays for pages and iovecs
fPages = new(std::nothrow) vm_page*[fPagesPerLine];
fVecs = new(std::nothrow) iovec[fPagesPerLine];
if (fPages == NULL || fVecs == NULL)
return B_NO_MEMORY; return B_NO_MEMORY;
status_t error = fLineTable->Init(kLineTableSize);
if (error != B_OK)
return error;
// create at least one cache line
MutexLocker locker(fLock);
fUnusedLine = _AllocateLine();
locker.Unlock();
if (fUnusedLine == NULL)
return B_NO_MEMORY;
// register low memory handler
error = register_low_resource_handler(&_LowMemoryHandlerEntry, this,
B_KERNEL_RESOURCE_PAGES | B_KERNEL_RESOURCE_MEMORY
| B_KERNEL_RESOURCE_ADDRESS_SPACE, 1);
// higher priority than the block cache, so we should be drained
// first
if (error != B_OK)
return error;
fLowMemoryHandlerRegistered = true;
return B_OK; return B_OK;
} }
@@ -184,14 +139,20 @@ IOCache::SetCallback(io_callback callback, void* data)
void void
IOCache::SetDeviceCapacity(off_t deviceCapacity) IOCache::SetDeviceCapacity(off_t deviceCapacity)
{ {
MutexLocker serializationLocker(fLock); TRACE("%p->IOCache::SetDeviceCapacity(%" B_PRIdOFF ")\n", this,
MutexLocker locker(fSerializationLock); deviceCapacity);
MutexLocker serializationLocker(fSerializationLock);
AutoLocker<VMCache> cacheLocker(fCache);
fDeviceCapacity = deviceCapacity; fDeviceCapacity = deviceCapacity;
// new media -- burn all cache lines // new media -- burn all cached data
while (Line* line = fUsedLines.Head()) while (vm_page* page = fCache->pages.Root()) {
_DiscardLine(line); DEBUG_PAGE_ACCESS_START(page);
fCache->RemovePage(page);
vm_page_free(NULL, page);
}
} }
@@ -271,11 +232,11 @@ IOCache::_DoRequest(IORequest* request, size_t& _bytesTransferred)
// intersection of request and cache line // intersection of request and cache line
off_t cacheLineEnd = std::min(lineOffset + fLineSize, fDeviceCapacity); off_t cacheLineEnd = std::min(lineOffset + fLineSize, fDeviceCapacity);
size_t requestLineLength size_t requestLineLength
= std::min(size_t(cacheLineEnd - offset), length); = std::min(cacheLineEnd - offset, (off_t)length);
// transfer the data of the cache line // transfer the data of the cache line
status_t error = _TransferRequestLine(request, lineOffset, offset, status_t error = _TransferRequestLine(request, lineOffset,
requestLineLength); cacheLineEnd - lineOffset, offset, requestLineLength);
if (error != B_OK) if (error != B_OK)
return error; return error;
@@ -290,88 +251,281 @@ IOCache::_DoRequest(IORequest* request, size_t& _bytesTransferred)
status_t status_t
IOCache::_TransferRequestLine(IORequest* request, off_t lineOffset, IOCache::_TransferRequestLine(IORequest* request, off_t lineOffset,
off_t requestOffset, size_t requestLength) size_t lineSize, off_t requestOffset, size_t requestLength)
{ {
TRACE("%p->IOCache::_TransferRequestLine(%p, %" B_PRIdOFF TRACE("%p->IOCache::_TransferRequestLine(%p, %" B_PRIdOFF
", %" B_PRIdOFF ", %" B_PRIuSIZE "\n", this, request, lineOffset, ", %" B_PRIdOFF ", %" B_PRIuSIZE ")\n", this, request, lineOffset,
requestOffset, requestLength); requestOffset, requestLength);
// check whether there are pages of the cache line and the mark them used
page_num_t firstPageOffset = lineOffset / B_PAGE_SIZE;
page_num_t linePageCount = (lineSize + B_PAGE_SIZE - 1) / B_PAGE_SIZE;
AutoLocker<VMCache> cacheLocker(fCache);
page_num_t firstMissing = 0;
page_num_t lastMissing = 0;
page_num_t missingPages = 0;
page_num_t pageOffset = firstPageOffset;
VMCachePagesTree::Iterator it = fCache->pages.GetIterator(pageOffset, true,
true);
while (pageOffset < firstPageOffset + linePageCount) {
vm_page* page = it.Next();
page_num_t currentPageOffset;
if (page == NULL
|| page->cache_offset >= firstPageOffset + linePageCount) {
page = NULL;
currentPageOffset = firstPageOffset + linePageCount;
} else
currentPageOffset = page->cache_offset;
if (pageOffset < currentPageOffset) {
// pages are missing
if (missingPages == 0)
firstMissing = pageOffset;
lastMissing = currentPageOffset - 1;
missingPages += currentPageOffset - pageOffset;
for (; pageOffset < currentPageOffset; pageOffset++)
fPages[pageOffset - firstPageOffset] = NULL;
}
if (page != NULL) {
fPages[pageOffset++ - firstPageOffset] = page;
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_UNUSED);
DEBUG_PAGE_ACCESS_END(page);
}
}
cacheLocker.Unlock();
bool isVIP = (request->Flags() & B_VIP_IO_REQUEST) != 0; bool isVIP = (request->Flags() & B_VIP_IO_REQUEST) != 0;
MutexLocker locker(fLock); if (missingPages > 0) {
// TODO: If this is a read request and the missing pages range doesn't intersect
// with the request, just satisfy the request and don't read anything at all.
// There are pages of the cache line missing. We have to allocate fresh
// ones.
// get the cache line // reserve
Line* line = _LookupLine(lineOffset); vm_page_reservation reservation;
if (line == NULL) { if (!vm_page_try_reserve_pages(&reservation, missingPages,
// line not cached yet -- create it VM_PRIORITY_SYSTEM)) {
TRACE("%p->IOCache::_TransferRequestLine(): line not cached yet\n", _DiscardPages(firstMissing - firstPageOffset, missingPages);
this);
line = _PrepareLine(lineOffset); // fall back to uncached transfer
return _TransferRequestLineUncached(request, lineOffset,
requestOffset, requestLength);
}
// in case of a read or partial write, read the line from disk // Allocate the missing pages and remove the already existing pages in
if (request->IsRead() || requestLength < line->size) { // the range from the cache. We're going to read/write the whole range
line->inUse = true; // anyway and this way we can sort it, possibly improving the physical
locker.Unlock(); // vecs.
// TODO: When memory is low, we should consider cannibalizing ourselves or
// simply transferring past the cache!
for (pageOffset = firstMissing; pageOffset <= lastMissing;
pageOffset++) {
page_num_t index = pageOffset - firstPageOffset;
if (fPages[index] == NULL) {
fPages[index] = vm_page_allocate_page( &reservation,
PAGE_STATE_UNUSED);
DEBUG_PAGE_ACCESS_END(fPages[index]);
} else {
cacheLocker.Lock();
fCache->RemovePage(fPages[index]);
cacheLocker.Unlock();
}
}
status_t error = _TransferLine(line, false, isVIP); missingPages = lastMissing - firstMissing + 1;
locker.Lock(); // sort the page array by physical page number
line->inUse = false; std::sort(fPages + firstMissing - firstPageOffset,
fPages + lastMissing - firstPageOffset + 1,
page_physical_number_less);
// add the pages to the cache
cacheLocker.Lock();
for (pageOffset = firstMissing; pageOffset <= lastMissing;
pageOffset++) {
page_num_t index = pageOffset - firstPageOffset;
fCache->InsertPage(fPages[index], (off_t)pageOffset * B_PAGE_SIZE);
}
cacheLocker.Unlock();
// Read in the missing pages, if this is a read request or a write
// request that doesn't cover the complete missing range.
if (request->IsRead()
|| requestOffset < (off_t)firstMissing * B_PAGE_SIZE
|| requestOffset + requestLength
> (lastMissing + 1) * B_PAGE_SIZE) {
status_t error = _TransferPages(firstMissing - firstPageOffset,
missingPages, false, isVIP);
if (error != B_OK) { if (error != B_OK) {
_DiscardLine(line); _DiscardPages(firstMissing - firstPageOffset, missingPages);
return error; return error;
} }
} }
} else {
TRACE("%p->IOCache::_TransferRequestLine(): line cached: %p\n", this,
line);
} }
// requeue the cache line -- it's most recently used now
fUsedLines.Remove(line);
fUsedLines.Add(line);
if (request->IsRead()) { if (request->IsRead()) {
// copy data from cache line to request // copy data to request
return request->CopyData(line->buffer + (requestOffset - lineOffset), status_t error = _CopyPages(request, requestOffset - lineOffset,
requestOffset, requestLength); requestOffset, requestLength, true);
_CachePages(0, linePageCount);
return error;
} else { } else {
// copy data from request to cache line // copy data from request
status_t error = request->CopyData(requestOffset, status_t error = _CopyPages(request, requestOffset - lineOffset,
line->buffer + (requestOffset - lineOffset), requestLength); requestOffset, requestLength, false);
if (error != B_OK) if (error != B_OK) {
_DiscardPages(0, linePageCount);
return error; return error;
}
// write the cache line to disk // write the pages to disk
line->inUse = true; page_num_t firstPage = (requestOffset - lineOffset) / B_PAGE_SIZE;
locker.Unlock(); page_num_t endPage = (requestOffset + requestLength - lineOffset
+ B_PAGE_SIZE - 1) / B_PAGE_SIZE;
error = _TransferPages(firstPage, endPage - firstPage, true, isVIP);
error = _TransferLine(line, true, isVIP); if (error != B_OK) {
// TODO: In case of a partial write, there's really no point in _DiscardPages(firstPage, endPage - firstPage);
// writing the complete cache line. return error;
}
locker.Lock(); _CachePages(0, linePageCount);
line->inUse = false;
if (error != B_OK)
_DiscardLine(line);
return error; return error;
} }
} }
status_t status_t
IOCache::_TransferLine(Line* line, bool isWrite, bool isVIP) IOCache::_TransferRequestLineUncached(IORequest* request, off_t lineOffset,
off_t requestOffset, size_t requestLength)
{ {
TRACE("%p->IOCache::_TransferLine(%p, write: %d, vip: %d)\n", this, line, TRACE("%p->IOCache::_TransferRequestLineUncached(%p, %" B_PRIdOFF
isWrite, isVIP); ", %" B_PRIdOFF ", %" B_PRIuSIZE ")\n", this, request, lineOffset,
requestOffset, requestLength);
// Advance the request to the interesting offset, so the DMAResource can
// provide us with fitting operations.
off_t actualRequestOffset
= request->Offset() + request->Length() - request->RemainingBytes();
if (actualRequestOffset > requestOffset) {
dprintf("IOCache::_TransferRequestLineUncached(): Request %p advanced "
"beyond current cache line (%" B_PRIdOFF " vs. %" B_PRIdOFF ")\n",
request, actualRequestOffset, requestOffset);
return B_BAD_VALUE;
}
if (actualRequestOffset < requestOffset)
request->Advance(requestOffset - actualRequestOffset);
size_t requestRemaining = request->RemainingBytes() - requestLength;
// Process single operations until the specified part of the request is
// finished or until an error occurs.
Operation operation;
operation.finishedCondition.Init(this, "I/O cache operation finished");
while (request->RemainingBytes() > requestRemaining
&& request->Status() > 0) {
status_t error = fDMAResource->TranslateNext(request, &operation,
request->RemainingBytes() - requestRemaining);
if (error != B_OK)
return error;
error = _DoOperation(operation);
request->OperationFinished(&operation, error, false,
error == B_OK ? operation.OriginalLength() : 0);
request->SetUnfinished();
// Keep the request in unfinished state. ScheduleRequest() will set
// the final status and notify.
if (fDMAResource != NULL)
fDMAResource->RecycleBuffer(operation.Buffer());
if (error != B_OK) {
TRACE("%p->IOCache::_TransferRequestLineUncached(): operation at "
"%" B_PRIdOFF " failed: %s\n", this, operation.Offset(),
strerror(error));
return error;
}
}
return B_OK;
}
status_t
IOCache::_DoOperation(Operation& operation)
{
TRACE("%p->IOCache::_DoOperation(%" B_PRIdOFF ", %" B_PRIuSIZE ")\n", this,
operation.Offset(), operation.Length());
while (true) {
ConditionVariableEntry waitEntry;
operation.finishedCondition.Add(&waitEntry);
status_t error = fIOCallback(fIOCallbackData, &operation);
if (error != B_OK) {
operation.finishedCondition.NotifyAll(false, error);
// removes the entry from the variable
return error;
}
// wait for the operation to finish
error = waitEntry.Wait();
if (error != B_OK)
return error;
if (operation.Finish())
return B_OK;
}
}
status_t
IOCache::_TransferPages(size_t firstPage, size_t pageCount, bool isWrite,
bool isVIP)
{
TRACE("%p->IOCache::_TransferPages(%" B_PRIuSIZE ", %" B_PRIuSIZE
", write: %d, vip: %d)\n", this, firstPage, pageCount, isWrite, isVIP);
off_t firstPageOffset = (off_t)fPages[firstPage]->cache_offset
* B_PAGE_SIZE;
size_t requestLength = std::min(
firstPageOffset + (off_t)pageCount * B_PAGE_SIZE, fDeviceCapacity)
- firstPageOffset;
// prepare the I/O vecs
size_t vecCount = 0;
size_t endPage = firstPage + pageCount;
addr_t vecsEndAddress = 0;
for (size_t i = firstPage; i < endPage; i++) {
addr_t pageAddress = fPages[i]->physical_page_number * B_PAGE_SIZE;
if (vecCount == 0 || pageAddress != vecsEndAddress) {
fVecs[vecCount].iov_base = (void*)pageAddress;
fVecs[vecCount++].iov_len = B_PAGE_SIZE;
vecsEndAddress = pageAddress + B_PAGE_SIZE;
} else {
// extend the previous vec
fVecs[vecCount - 1].iov_len += B_PAGE_SIZE;
vecsEndAddress += B_PAGE_SIZE;
}
}
// create a request for the transfer // create a request for the transfer
IORequest request; IORequest request;
status_t error = request.Init(line->offset, (void*)line->physicalBuffer, status_t error = request.Init(firstPageOffset, fVecs, vecCount,
line->size, isWrite, requestLength, isWrite,
B_PHYSICAL_IO_REQUEST | (isVIP ? B_VIP_IO_REQUEST : 0)); B_PHYSICAL_IO_REQUEST | (isVIP ? B_VIP_IO_REQUEST : 0));
if (error != B_OK) if (error != B_OK)
return error; return error;
@@ -382,7 +536,8 @@ IOCache::_TransferLine(Line* line, bool isWrite, bool isVIP)
operation.finishedCondition.Init(this, "I/O cache operation finished"); operation.finishedCondition.Init(this, "I/O cache operation finished");
while (request.RemainingBytes() > 0 && request.Status() > 0) { while (request.RemainingBytes() > 0 && request.Status() > 0) {
error = fDMAResource->TranslateNext(&request, &operation, line->size); error = fDMAResource->TranslateNext(&request, &operation,
requestLength);
if (error != B_OK) if (error != B_OK)
return error; return error;
@@ -404,200 +559,201 @@ IOCache::_TransferLine(Line* line, bool isWrite, bool isVIP)
} }
/*! Frees all pages in given range of the \c fPages array.
\c NULL entries in the range are OK. All non \c NULL entries must refer
to pages with \c PAGE_STATE_UNUSED. The pages may belong to \c fCache or
may not have a cache.
\c fCache must not be locked.
*/
void
IOCache::_DiscardPages(size_t firstPage, size_t pageCount)
{
TRACE("%p->IOCache::_DiscardPages(%" B_PRIuSIZE ", %" B_PRIuSIZE ")\n",
this, firstPage, pageCount);
AutoLocker<VMCache> cacheLocker(fCache);
for (size_t i = firstPage; i < firstPage + pageCount; i++) {
vm_page* page = fPages[i];
if (page == NULL)
continue;
DEBUG_PAGE_ACCESS_START(page);
ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED,
"page: %p @! page -m %p", page, page);
if (page->Cache() != NULL)
fCache->RemovePage(page);
vm_page_free(NULL, page);
}
}
/*! Marks all pages in the given range of the \c fPages array cached.
There must not be any \c NULL entries in the given array range. All pages
must belong to \c cache and have state \c PAGE_STATE_UNUSED.
\c fCache must not be locked.
*/
void
IOCache::_CachePages(size_t firstPage, size_t pageCount)
{
TRACE("%p->IOCache::_CachePages(%" B_PRIuSIZE ", %" B_PRIuSIZE ")\n",
this, firstPage, pageCount);
AutoLocker<VMCache> cacheLocker(fCache);
for (size_t i = firstPage; i < firstPage + pageCount; i++) {
vm_page* page = fPages[i];
ASSERT(page != NULL);
ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED
&& page->Cache() == fCache,
"page: %p @! page -m %p", page, page);
DEBUG_PAGE_ACCESS_START(page);
vm_page_set_state(page, PAGE_STATE_CACHED);
DEBUG_PAGE_ACCESS_END(page);
}
}
/*! Copies the contents of pages in \c fPages to \a request, or vice versa.
\param request The request.
\param pagesRelativeOffset The offset relative to \c fPages[0] where to
start copying.
\param requestOffset The request offset where to start copying.
\param requestLength The number of bytes to copy.
\param toRequest If \c true the copy directory is from \c fPages to
\a request, otherwise the other way around.
\return \c B_OK, if copying went fine, another error code otherwise.
*/
status_t status_t
IOCache::_DoOperation(Operation& operation) IOCache::_CopyPages(IORequest* request, size_t pagesRelativeOffset,
off_t requestOffset, size_t requestLength, bool toRequest)
{ {
TRACE("%p->IOCache::_DoOperation(%" B_PRIdOFF ", %" B_PRIuSIZE ")\n", this, TRACE("%p->IOCache::_CopyPages(%p, %" B_PRIuSIZE ", %" B_PRIdOFF
operation.Offset(), operation.Length()); ", %" B_PRIuSIZE ", %d)\n", this, request, pagesRelativeOffset,
requestOffset, requestLength, toRequest);
ConditionVariableEntry waitEntry; size_t firstPage = pagesRelativeOffset / B_PAGE_SIZE;
operation.finishedCondition.Add(&waitEntry); size_t endPage = (pagesRelativeOffset + requestLength + B_PAGE_SIZE - 1)
/ B_PAGE_SIZE;
status_t error = fIOCallback(fIOCallbackData, &operation); // map the pages
status_t error = _MapPages(firstPage, endPage);
// TODO: _MapPages() cannot fail, so the fallback is never needed. Test which
// method is faster (probably the active one)!
#if 0
if (error != B_OK) { if (error != B_OK) {
operation.finishedCondition.NotifyAll(false, error); // fallback to copying individual pages
// removes the entry from the variable size_t inPageOffset = pagesRelativeOffset % B_PAGE_SIZE;
return error; for (size_t i = firstPage; i < endPage; i++) {
// map the page
void* handle;
addr_t address;
error = vm_get_physical_page(
fPages[i]->physical_page_number * B_PAGE_SIZE, &address,
&handle);
if (error != B_OK)
return error;
// copy the page's data
size_t toCopy = std::min(B_PAGE_SIZE - inPageOffset, requestLength);
if (toRequest) {
error = request->CopyData((uint8*)(address + inPageOffset),
requestOffset, toCopy);
} else {
error = request->CopyData(requestOffset,
(uint8*)(address + inPageOffset), toCopy);
}
// unmap the page
vm_put_physical_page(address, handle);
if (error != B_OK)
return error;
inPageOffset = 0;
requestOffset += toCopy;
requestLength -= toCopy;
}
return B_OK;
} }
#endif // 0
// wait for the operation to finish // copy
return waitEntry.Wait(); if (toRequest) {
} error = request->CopyData((uint8*)fAreaBase + pagesRelativeOffset,
requestOffset, requestLength);
IOCache::Line*
IOCache::_LookupLine(off_t lineOffset)
{
ASSERT_LOCKED_MUTEX(&fLock);
return fLineTable->Lookup(lineOffset);
}
IOCache::Line*
IOCache::_PrepareLine(off_t lineOffset)
{
ASSERT_LOCKED_MUTEX(&fLock);
Line* line = NULL;
// if there is an unused line recycle it
if (fUnusedLine != NULL) {
line = fUnusedLine;
fUnusedLine = NULL;
} else if (!_MemoryIsLow()) {
// resources look fine -- allocate a new line
line = _AllocateLine();
}
if (line == NULL) {
// recycle the least recently used line
line = fUsedLines.RemoveHead();
fLineTable->RemoveUnchecked(line);
TRACE("%p->IOCache::_PrepareLine(%" B_PRIdOFF "): recycled line: %p\n",
this, lineOffset, line);
} else { } else {
TRACE("%p->IOCache::_PrepareLine(%" B_PRIdOFF error = request->CopyData(requestOffset,
"): allocated new line: %p\n", this, lineOffset, line); (uint8*)fAreaBase + pagesRelativeOffset, requestLength);
} }
line->offset = lineOffset; // unmap the pages
line->size = std::min((off_t)fLineSize, fDeviceCapacity - lineOffset); _UnmapPages(firstPage, endPage);
line->inUse = false;
fUsedLines.Add(line); return error;
fLineTable->InsertUnchecked(line);
return line;
} }
/*! Maps a range of pages in \c fPages into fArea.
If successful, it must be balanced by a call to _UnmapPages().
\param firstPage The \c fPages relative index of the first page to map.
\param endPage The \c fPages relative index of the page after the last page
to map.
\return \c B_OK, if mapping went fine, another error code otherwise.
*/
status_t
IOCache::_MapPages(size_t firstPage, size_t endPage)
{
VMTranslationMap* translationMap
= VMAddressSpace::Kernel()->TranslationMap();
translationMap->Lock();
for (size_t i = firstPage; i < endPage; i++) {
vm_page* page = fPages[i];
ASSERT_PRINT(page->State() == PAGE_STATE_UNUSED,
"page: %p @! page -m %p", page, page);
translationMap->Map((addr_t)fAreaBase + i * B_PAGE_SIZE,
page->physical_page_number * B_PAGE_SIZE,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, &fMappingReservation);
// NOTE: We don't increment gMappedPagesCount. Our pages have state
// PAGE_STATE_UNUSED anyway and we map them only for a short time.
}
translationMap->Unlock();
return B_OK;
}
/*! Unmaps a range of pages in \c fPages into fArea.
Must balance a call to _MapPages().
\param firstPage The \c fPages relative index of the first page to unmap.
\param endPage The \c fPages relative index of the page after the last page
to unmap.
*/
void void
IOCache::_DiscardLine(Line* line) IOCache::_UnmapPages(size_t firstPage, size_t endPage)
{ {
ASSERT_LOCKED_MUTEX(&fLock); VMTranslationMap* translationMap
= VMAddressSpace::Kernel()->TranslationMap();
fLineTable->RemoveUnchecked(line); translationMap->Lock();
fUsedLines.Remove(line);
if (fUsedLines.IsEmpty()) { translationMap->Unmap((addr_t)fAreaBase + firstPage * B_PAGE_SIZE,
// We keep the last cache line around, so I/O cannot fail due to a (addr_t)fAreaBase + endPage * B_PAGE_SIZE - 1);
// failing allocation.
TRACE("%p->IOCache::_DiscardLine(): parking line: %p\n", this, line); translationMap->Unlock();
fUnusedLine = line;
line->offset = -1;
line->inUse = false;
} else {
TRACE("%p->IOCache::_DiscardLine(): freeing line: %p\n", this, line);
_FreeLine(line);
}
}
IOCache::Line*
IOCache::_AllocateLine()
{
ASSERT_LOCKED_MUTEX(&fLock);
// create the line object
Line* line = new(malloc_flags(HEAP_DONT_WAIT_FOR_MEMORY)) Line;
if (line == NULL)
return NULL;
// create the buffer area
void* address;
line->area = vm_create_anonymous_area(B_SYSTEM_TEAM, "I/O cache line",
&address, B_ANY_KERNEL_ADDRESS, fLineSize, B_CONTIGUOUS,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0,
CREATE_AREA_DONT_WAIT | CREATE_AREA_DONT_CLEAR, true);
if (line->area < 0) {
delete line;
return NULL;
}
// get the physical address of the buffer
physical_entry entry;
get_memory_map(address, B_PAGE_SIZE, &entry, 1);
// init the line object
line->offset = -1;
line->buffer = (uint8*)address;
line->physicalBuffer = (addr_t)entry.address;
line->inUse = false;
fLineCount++;
return line;
}
void
IOCache::_FreeLine(Line* line)
{
ASSERT_LOCKED_MUTEX(&fLock);
if (line == NULL)
return;
fLineCount--;
delete_area(line->area);
delete line;
}
/*static*/ void
IOCache::_LowMemoryHandlerEntry(void* data, uint32 resources, int32 level)
{
((IOCache*)data)->_LowMemoryHandler(resources, level);
}
void
IOCache::_LowMemoryHandler(uint32 resources, int32 level)
{
TRACE("%p->IOCache::_LowMemoryHandler(): level: %ld\n", this, level);
MutexLocker locker(fLock);
// determine how many cache lines to keep
size_t linesToKeep = fLineCount;
switch (level) {
case B_NO_LOW_RESOURCE:
return;
case B_LOW_RESOURCE_NOTE:
linesToKeep = fLineCount / 2;
break;
case B_LOW_RESOURCE_WARNING:
linesToKeep = fLineCount / 4;
break;
case B_LOW_RESOURCE_CRITICAL:
linesToKeep = 1;
break;
}
if (linesToKeep < 1)
linesToKeep = 1;
// free lines until we reach our target
Line* line = fUsedLines.Head();
while (linesToKeep < fLineCount && line != NULL) {
Line* nextLine = fUsedLines.GetNext(line);
if (!line->inUse)
_DiscardLine(line);
line = nextLine;
}
}
bool
IOCache::_MemoryIsLow() const
{
return low_resource_state(B_KERNEL_RESOURCE_PAGES
| B_KERNEL_RESOURCE_MEMORY | B_KERNEL_RESOURCE_ADDRESS_SPACE)
!= B_NO_LOW_RESOURCE;
} }
src/system/kernel/device_manager/IOCache.h
+27 -32
View File
@@ -7,13 +7,17 @@
#include <lock.h> #include <lock.h>
#include <util/DoublyLinkedList.h> #include <vm/vm_page.h>
#include "dma_resources.h" #include "dma_resources.h"
#include "IOCallback.h" #include "IOCallback.h"
#include "IORequest.h" #include "IORequest.h"
struct VMCache;
struct vm_page;
class IOCache { class IOCache {
public: public:
IOCache(DMAResource* resource, IOCache(DMAResource* resource,
@@ -33,57 +37,48 @@ public:
status_t status, size_t transferredBytes); status_t status, size_t transferredBytes);
private: private:
struct Line : DoublyLinkedListLinkImpl<Line> {
Line* hashNext;
off_t offset;
off_t size;
uint8* buffer;
addr_t physicalBuffer;
area_id area;
bool inUse;
};
struct Operation; struct Operation;
struct LineHashDefinition;
struct LineTable;
typedef DoublyLinkedList<Line> LineList;
private: private:
status_t _DoRequest(IORequest* request, status_t _DoRequest(IORequest* request,
size_t& _bytesTransferred); size_t& _bytesTransferred);
status_t _TransferRequestLine(IORequest* request, status_t _TransferRequestLine(IORequest* request,
off_t lineOffset, size_t lineSize,
off_t requestOffset, size_t requestLength);
status_t _TransferRequestLineUncached(IORequest* request,
off_t lineOffset, off_t requestOffset, off_t lineOffset, off_t requestOffset,
size_t requestLength); size_t requestLength);
status_t _TransferLine(Line* line, bool isWrite,
bool isVIP);
status_t _DoOperation(Operation& operation); status_t _DoOperation(Operation& operation);
Line* _LookupLine(off_t lineOffset); status_t _TransferPages(size_t firstPage,
Line* _PrepareLine(off_t lineOffset); size_t pageCount, bool isWrite, bool isVIP);
void _DiscardLine(Line* line); void _DiscardPages(size_t firstPage,
Line* _AllocateLine(); size_t pageCount);
void _FreeLine(Line* line); void _CachePages(size_t firstPage, size_t pageCount);
static void _LowMemoryHandlerEntry(void* data, status_t _CopyPages(IORequest* request,
uint32 resources, int32 level); size_t pagesRelativeOffset,
void _LowMemoryHandler(uint32 resources, off_t requestOffset, size_t requestLength,
int32 level); bool toRequest);
inline bool _MemoryIsLow() const;
status_t _MapPages(size_t firstPage, size_t endPage);
void _UnmapPages(size_t firstPage, size_t endPage);
private: private:
mutex fLock;
mutex fSerializationLock; mutex fSerializationLock;
off_t fDeviceCapacity; off_t fDeviceCapacity;
size_t fLineSize; size_t fLineSize;
uint32 fLineSizeShift; uint32 fLineSizeShift;
size_t fPagesPerLine;
DMAResource* fDMAResource; DMAResource* fDMAResource;
io_callback fIOCallback; io_callback fIOCallback;
void* fIOCallbackData; void* fIOCallbackData;
LineTable* fLineTable; area_id fArea;
LineList fUsedLines; // LRU first void* fAreaBase;
Line* fUnusedLine; vm_page_reservation fMappingReservation;
size_t fLineCount; VMCache* fCache;
bool fLowMemoryHandlerRegistered; vm_page** fPages;
iovec* fVecs;
}; };