diff --git a/src/system/kernel/slab/MemoryManager.cpp b/src/system/kernel/slab/MemoryManager.cpp
index 5a48ed67ce..e0f83b8703 100644
--- a/src/system/kernel/slab/MemoryManager.cpp
+++ b/src/system/kernel/slab/MemoryManager.cpp
@@ -40,12 +40,14 @@ rw_lock MemoryManager::sAreaTableLock;
kernel_args* MemoryManager::sKernelArgs;
MemoryManager::AreaTable MemoryManager::sAreaTable;
MemoryManager::Area* MemoryManager::sFreeAreas;
+int MemoryManager::sFreeAreaCount;
MemoryManager::MetaChunkList MemoryManager::sFreeCompleteMetaChunks;
MemoryManager::MetaChunkList MemoryManager::sFreeShortMetaChunks;
MemoryManager::MetaChunkList MemoryManager::sPartialMetaChunksSmall;
MemoryManager::MetaChunkList MemoryManager::sPartialMetaChunksMedium;
MemoryManager::AllocationEntry* MemoryManager::sAllocationEntryCanWait;
MemoryManager::AllocationEntry* MemoryManager::sAllocationEntryDontWait;
+bool MemoryManager::sMaintenanceNeeded;
/*static*/ void
@@ -66,6 +68,8 @@ MemoryManager::Init(kernel_args* args)
// free it, that's not a problem, though.
sFreeAreas = NULL;
+ sFreeAreaCount = 0;
+ sMaintenanceNeeded = false;
}
@@ -96,6 +100,7 @@ MemoryManager::InitPostArea()
// Just "leak" all but the first of the free areas -- the VM will
// automatically free all unclaimed memory.
sFreeAreas->next = NULL;
+ sFreeAreaCount = 1;
Area* area = sFreeAreas;
_ConvertEarlyArea(area);
@@ -107,15 +112,31 @@ MemoryManager::InitPostArea()
_UnmapFreeChunksEarly(area);
}
+ sMaintenanceNeeded = true;
+ // might not be necessary, but doesn't harm
+
add_debugger_command_etc("slab_area", &_DumpArea,
"Dump information on a given slab area",
- "\n"
+ "[ -c ] \n"
"Dump information on a given slab area specified by its base "
- "address.\n", 0);
+ "address.\n"
+ "If \"-c\" is given, the chunks of all meta chunks area printed as "
+ "well.\n", 0);
add_debugger_command_etc("slab_areas", &_DumpAreas,
"List all slab areas",
"\n"
"Lists all slab areas.\n", 0);
+ add_debugger_command_etc("slab_meta_chunk", &_DumpMetaChunk,
+ "Dump information on a given slab meta chunk",
+ "\n"
+ "Dump information on a given slab meta chunk specified by its base "
+ "or object address.\n", 0);
+ add_debugger_command_etc("slab_meta_chunks", &_DumpMetaChunks,
+ "List all non-full slab meta chunks",
+ "[ -c ]\n"
+ "Lists all non-full slab meta chunks.\n"
+ "If \"-c\" is given, the chunks of all meta chunks area printed as "
+ "well.\n", 0);
}
@@ -179,8 +200,9 @@ MemoryManager::Free(void* pages, uint32 flags)
Chunk* chunk = &metaChunk->chunks[chunkIndex];
ASSERT(chunk->next != NULL);
- ASSERT(chunk->next < area->chunks
- || chunk->next >= area->chunks + SLAB_SMALL_CHUNKS_PER_AREA);
+ ASSERT(chunk->next < metaChunk->chunks
+ || chunk->next
+ >= metaChunk->chunks + SLAB_SMALL_CHUNKS_PER_META_CHUNK);
// and free it
MutexLocker locker(sLock);
@@ -223,6 +245,43 @@ MemoryManager::CacheForAddress(void* address)
}
+/*static*/ void
+MemoryManager::PerformMaintenance()
+{
+ MutexLocker locker(sLock);
+
+ while (sMaintenanceNeeded) {
+ sMaintenanceNeeded = false;
+
+ // We want to keep one or two areas as a reserve. This way we have at
+ // least one area to use in situations when we aren't allowed to
+ // allocate one and also avoid ping-pong effects.
+ if (sFreeAreaCount > 0 && sFreeAreaCount <= 2)
+ return;
+
+ if (sFreeAreaCount == 0) {
+ // try to allocate one
+ Area* area;
+ if (_AllocateArea(0, area) != B_OK)
+ return;
+
+ _push(sFreeAreas, area);
+ if (++sFreeAreaCount > 2)
+ sMaintenanceNeeded = true;
+ } else {
+ // free until we only have two free ones
+ while (sFreeAreaCount > 2) {
+ Area* area = _pop(sFreeAreas);
+ _FreeArea(area, true, 0);
+ }
+
+ if (sFreeAreaCount == 0)
+ sMaintenanceNeeded = true;
+ }
+ }
+}
+
+
/*static*/ status_t
MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags,
MetaChunk*& _metaChunk, Chunk*& _chunk)
@@ -243,8 +302,10 @@ MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags,
if (sFreeAreas != NULL) {
_AddArea(_pop(sFreeAreas));
+ sFreeAreaCount--;
+ _RequestMaintenance();
+
_GetChunk(metaChunkList, chunkSize, _metaChunk, _chunk);
-// TODO: If that was the last area, notify the cleanup/resizer thread!
return B_OK;
}
@@ -288,7 +349,7 @@ MemoryManager::_AllocateChunk(size_t chunkSize, uint32 flags,
allocationEntry->thread = find_thread(NULL);
Area* area;
- status_t error = _AllocateArea(chunkSize, flags, area);
+ status_t error = _AllocateArea(flags, area);
allocationEntry->condition.NotifyAll();
allocationEntry = NULL;
@@ -363,8 +424,16 @@ MemoryManager::_FreeChunk(Area* area, MetaChunk* metaChunk, Chunk* chunk,
// free the meta chunk, if it is unused now
ASSERT(metaChunk->usedChunkCount > 0);
if (--metaChunk->usedChunkCount == 0) {
+ // remove from partial meta chunk list
+ if (metaChunk->chunkSize == SLAB_CHUNK_SIZE_SMALL)
+ sPartialMetaChunksSmall.Remove(metaChunk);
+ else if (metaChunk->chunkSize == SLAB_CHUNK_SIZE_MEDIUM)
+ sPartialMetaChunksMedium.Remove(metaChunk);
+
+ // mark empty
metaChunk->chunkSize = 0;
+ // add to free list
if (metaChunk == area->metaChunks)
sFreeShortMetaChunks.Add(metaChunk, false);
else
@@ -416,12 +485,9 @@ MemoryManager::_AddArea(Area* area)
/*static*/ status_t
-MemoryManager::_AllocateArea(size_t chunkSize, uint32 flags, Area*& _area)
+MemoryManager::_AllocateArea(uint32 flags, Area*& _area)
{
- // TODO: Support reusing free areas!
-
- TRACE("MemoryManager::_AllocateArea(%" B_PRIuSIZE ", %#" B_PRIx32 ")\n",
- chunkSize, flags);
+ TRACE("MemoryManager::_AllocateArea(%#" B_PRIx32 ")\n", flags);
ASSERT((flags & CACHE_DONT_LOCK_KERNEL_SPACE) == 0);
@@ -487,8 +553,6 @@ MemoryManager::_AllocateArea(size_t chunkSize, uint32 flags, Area*& _area)
// meta chunk. They will be set in _PrepareMetaChunk().
metaChunk->chunkCount = 0;
metaChunk->usedChunkCount = 0;
- metaChunk->chunks = area->chunks
- + i * (SLAB_CHUNK_SIZE_LARGE / SLAB_CHUNK_SIZE_SMALL);
metaChunk->freeChunks = NULL;
}
@@ -521,17 +585,19 @@ MemoryManager::_FreeArea(Area* area, bool areaRemoved, uint32 flags)
writeLocker.Unlock();
}
- // We keep one free area as a reserve.
- if (sFreeAreas == NULL) {
- sFreeAreas = area;
+ // We want to keep one or two free areas as a reserve.
+ if (sFreeAreaCount <= 1) {
+ _push(sFreeAreas, area);
+ sFreeAreaCount++;
return;
}
if (area->vmArea == NULL || (flags & CACHE_DONT_LOCK_KERNEL_SPACE) != 0) {
// This is either early in the boot process or we aren't allowed to
// delete the area now.
-// TODO: Notify the cleanup/resizer thread!
_push(sFreeAreas, area);
+ sFreeAreaCount++;
+ _RequestMaintenance();
return;
}
@@ -736,30 +802,35 @@ MemoryManager::_ConvertEarlyArea(Area* area)
}
-/*static*/ int
-MemoryManager::_DumpArea(int argc, char** argv)
+/*static*/ void
+MemoryManager::_RequestMaintenance()
{
- if (argc != 2) {
- print_debugger_command_usage(argv[0]);
- return 0;
- }
+ if ((sFreeAreaCount > 0 && sFreeAreaCount <= 2) || sMaintenanceNeeded)
+ return;
- uint64 address;
- if (!evaluate_debug_expression(argv[1], &address, false))
- return 0;
+ sMaintenanceNeeded = true;
+ request_memory_manager_maintenance();
+}
- address = ROUNDDOWN(address, SLAB_AREA_SIZE);
- Area* area = (Area*)(addr_t)address;
+/*static*/ void
+MemoryManager::_PrintMetaChunkTableHeader(bool printChunks)
+{
+ if (printChunks)
+ kprintf("chunk base cache object size cache name\n");
+ else
+ kprintf("chunk base\n");
+}
- kprintf("chunk base cache object size cache name\n");
+/*static*/ void
+MemoryManager::_DumpMetaChunk(MetaChunk* metaChunk, bool printChunks,
+ bool printHeader)
+{
+ if (printHeader)
+ _PrintMetaChunkTableHeader(printChunks);
- for (uint32 k = 0; k < SLAB_META_CHUNKS_PER_AREA; k++) {
- MetaChunk* metaChunk = area->metaChunks + k;
- if (metaChunk->chunkSize == 0)
- continue;
-
- const char* type = "???";
+ const char* type = "empty";
+ if (metaChunk->chunkSize != 0) {
switch (metaChunk->chunkSize) {
case SLAB_CHUNK_SIZE_SMALL:
type = "small";
@@ -771,27 +842,132 @@ MemoryManager::_DumpArea(int argc, char** argv)
type = "large";
break;
}
+ }
- kprintf("-- %2" B_PRIu32 " %s ---------------------------------------"
- "--------------\n", k, type);
+ int metaChunkIndex = metaChunk - metaChunk->GetArea()->metaChunks;
+ kprintf("%5d %p --- %6s meta chunk", metaChunkIndex,
+ (void*)metaChunk->chunkBase, type);
+ if (metaChunk->chunkSize != 0) {
+ kprintf(": %4u/%4u used ----------------------------\n",
+ metaChunk->usedChunkCount, metaChunk->chunkCount);
+ } else
+ kprintf(" --------------------------------------------\n");
- for (uint32 i = 0; i < metaChunk->chunkCount; i++) {
- Chunk* chunk = metaChunk->chunks + i;
+ if (metaChunk->chunkSize == 0 || !printChunks)
+ return;
- // skip free chunks
- if (chunk->next == NULL)
- continue;
- if (chunk->next >= metaChunk->chunks
- && chunk->next < metaChunk->chunks + metaChunk->chunkCount) {
- continue;
- }
+ for (uint32 i = 0; i < metaChunk->chunkCount; i++) {
+ Chunk* chunk = metaChunk->chunks + i;
- ObjectCache* cache = chunk->cache;
- kprintf("%5" B_PRIu32 " %p %p %11" B_PRIuSIZE " %s\n", i,
- (void*)_ChunkAddress(metaChunk, chunk), cache,
- cache != NULL ? cache->object_size : 0,
- cache != NULL ? cache->name : "");
+ // skip free chunks
+ if (chunk->next == NULL)
+ continue;
+ if (chunk->next >= metaChunk->chunks
+ && chunk->next < metaChunk->chunks + metaChunk->chunkCount) {
+ continue;
}
+
+ ObjectCache* cache = chunk->cache;
+ kprintf("%5" B_PRIu32 " %p %p %11" B_PRIuSIZE " %s\n", i,
+ (void*)_ChunkAddress(metaChunk, chunk), cache,
+ cache != NULL ? cache->object_size : 0,
+ cache != NULL ? cache->name : "");
+ }
+}
+
+
+/*static*/ int
+MemoryManager::_DumpMetaChunk(int argc, char** argv)
+{
+ if (argc != 2) {
+ print_debugger_command_usage(argv[0]);
+ return 0;
+ }
+
+ uint64 address;
+ if (!evaluate_debug_expression(argv[1], &address, false))
+ return 0;
+
+ Area* area = (Area*)(addr_t)ROUNDDOWN(address, SLAB_AREA_SIZE);
+
+ MetaChunk* metaChunk;
+ if ((addr_t)address >= (addr_t)area->metaChunks
+ && (addr_t)address
+ < (addr_t)(area->metaChunks + SLAB_META_CHUNKS_PER_AREA)) {
+ metaChunk = (MetaChunk*)(addr_t)address;
+ } else {
+ metaChunk = area->metaChunks
+ + (address % SLAB_AREA_SIZE) / SLAB_CHUNK_SIZE_LARGE;
+ }
+
+ _DumpMetaChunk(metaChunk, true, true);
+
+ return 0;
+}
+
+
+/*static*/ void
+MemoryManager::_DumpMetaChunks(const char* name, MetaChunkList& metaChunkList,
+ bool printChunks)
+{
+ kprintf("%s:\n", name);
+
+ for (MetaChunkList::Iterator it = metaChunkList.GetIterator();
+ MetaChunk* metaChunk = it.Next();) {
+ _DumpMetaChunk(metaChunk, printChunks, false);
+ }
+}
+
+
+/*static*/ int
+MemoryManager::_DumpMetaChunks(int argc, char** argv)
+{
+ bool printChunks = argc > 1 && strcmp(argv[1], "-c") == 0;
+
+ _PrintMetaChunkTableHeader(printChunks);
+ _DumpMetaChunks("free complete", sFreeCompleteMetaChunks, printChunks);
+ _DumpMetaChunks("free short", sFreeShortMetaChunks, printChunks);
+ _DumpMetaChunks("partial small", sPartialMetaChunksSmall, printChunks);
+ _DumpMetaChunks("partial medium", sPartialMetaChunksMedium, printChunks);
+
+ return 0;
+}
+
+
+/*static*/ int
+MemoryManager::_DumpArea(int argc, char** argv)
+{
+ bool printChunks = false;
+
+ int argi = 1;
+ while (argi < argc) {
+ if (argv[argi][0] != '-')
+ break;
+ const char* arg = argv[argi++];
+ if (strcmp(arg, "-c") == 0) {
+ printChunks = true;
+ } else {
+ print_debugger_command_usage(argv[0]);
+ return 0;
+ }
+ }
+
+ if (argi + 1 != argc) {
+ print_debugger_command_usage(argv[0]);
+ return 0;
+ }
+
+ uint64 address;
+ if (!evaluate_debug_expression(argv[argi], &address, false))
+ return 0;
+
+ address = ROUNDDOWN(address, SLAB_AREA_SIZE);
+
+ Area* area = (Area*)(addr_t)address;
+
+ for (uint32 k = 0; k < SLAB_META_CHUNKS_PER_AREA; k++) {
+ MetaChunk* metaChunk = area->metaChunks + k;
+ _DumpMetaChunk(metaChunk, printChunks, k == 0);
}
return 0;
@@ -840,5 +1016,9 @@ MemoryManager::_DumpAreas(int argc, char** argv)
totalMedium, usedLarge, totalLarge);
}
+ kprintf("%d free areas:\n", sFreeAreaCount);
+ for (Area* area = sFreeAreas; area != NULL; area = area->next)
+ kprintf("%p %p\n", area, area->vmArea);
+
return 0;
}
diff --git a/src/system/kernel/slab/MemoryManager.h b/src/system/kernel/slab/MemoryManager.h
index 782f515d00..d389b6e2e3 100644
--- a/src/system/kernel/slab/MemoryManager.h
+++ b/src/system/kernel/slab/MemoryManager.h
@@ -28,6 +28,8 @@ struct VMArea;
#define SLAB_META_CHUNKS_PER_AREA (SLAB_AREA_SIZE / SLAB_CHUNK_SIZE_LARGE)
#define SLAB_SMALL_CHUNKS_PER_AREA (SLAB_AREA_SIZE / SLAB_CHUNK_SIZE_SMALL)
+#define SLAB_SMALL_CHUNKS_PER_META_CHUNK \
+ (SLAB_CHUNK_SIZE_LARGE / SLAB_CHUNK_SIZE_SMALL)
class MemoryManager {
@@ -42,6 +44,9 @@ public:
static size_t AcceptableChunkSize(size_t size);
static ObjectCache* CacheForAddress(void* address);
+ static bool MaintenanceNeeded();
+ static void PerformMaintenance();
+
private:
struct Area;
@@ -58,7 +63,7 @@ private:
size_t totalSize;
uint16 chunkCount;
uint16 usedChunkCount;
- Chunk* chunks;
+ Chunk chunks[SLAB_SMALL_CHUNKS_PER_META_CHUNK];
Chunk* freeChunks;
Area* GetArea() const;
@@ -73,7 +78,6 @@ private:
uint16 usedMetaChunkCount;
bool fullyMapped;
MetaChunk metaChunks[SLAB_META_CHUNKS_PER_AREA];
- Chunk chunks[SLAB_SMALL_CHUNKS_PER_AREA];
};
typedef DoublyLinkedList AreaList;
@@ -124,8 +128,7 @@ private:
size_t chunkSize);
static void _AddArea(Area* area);
- static status_t _AllocateArea(size_t chunkSize, uint32 flags,
- Area*& _area);
+ static status_t _AllocateArea(uint32 flags, Area*& _area);
static void _FreeArea(Area* area, bool areaRemoved,
uint32 flags);
@@ -139,11 +142,21 @@ private:
static void _UnmapFreeChunksEarly(Area* area);
static void _ConvertEarlyArea(Area* area);
+ static void _RequestMaintenance();
+
static uint32 _ChunkIndexForAddress(
const MetaChunk* metaChunk, addr_t address);
static addr_t _ChunkAddress(const MetaChunk* metaChunk,
const Chunk* chunk);
+ static void _PrintMetaChunkTableHeader(bool printChunks);
+ static void _DumpMetaChunk(MetaChunk* metaChunk,
+ bool printChunks, bool printHeader);
+ static int _DumpMetaChunk(int argc, char** argv);
+ static void _DumpMetaChunks(const char* name,
+ MetaChunkList& metaChunkList,
+ bool printChunks);
+ static int _DumpMetaChunks(int argc, char** argv);
static int _DumpArea(int argc, char** argv);
static int _DumpAreas(int argc, char** argv);
@@ -156,15 +169,24 @@ private:
static kernel_args* sKernelArgs;
static AreaTable sAreaTable;
static Area* sFreeAreas;
+ static int sFreeAreaCount;
static MetaChunkList sFreeCompleteMetaChunks;
static MetaChunkList sFreeShortMetaChunks;
static MetaChunkList sPartialMetaChunksSmall;
static MetaChunkList sPartialMetaChunksMedium;
static AllocationEntry* sAllocationEntryCanWait;
static AllocationEntry* sAllocationEntryDontWait;
+ static bool sMaintenanceNeeded;
};
+/*static*/ inline bool
+MemoryManager::MaintenanceNeeded()
+{
+ return sMaintenanceNeeded;
+}
+
+
/*static*/ inline uint32
MemoryManager::_ChunkIndexForAddress(const MetaChunk* metaChunk, addr_t address)
{
diff --git a/src/system/kernel/slab/Slab.cpp b/src/system/kernel/slab/Slab.cpp
index 18cf5d904c..b70148165e 100644
--- a/src/system/kernel/slab/Slab.cpp
+++ b/src/system/kernel/slab/Slab.cpp
@@ -266,6 +266,14 @@ slab_internal_free(void* buffer, uint32 flags)
}
+void
+request_memory_manager_maintenance()
+{
+ MutexLocker locker(sMaintenanceLock);
+ sMaintenanceCondition.NotifyAll();
+}
+
+
// #pragma mark -
@@ -479,6 +487,14 @@ object_cache_maintainer(void*)
// wait for the next request
while (sMaintenanceQueue.IsEmpty()) {
+ // perform memory manager maintenance, if needed
+ if (MemoryManager::MaintenanceNeeded()) {
+ locker.Unlock();
+ MemoryManager::PerformMaintenance();
+ locker.Lock();
+ continue;
+ }
+
ConditionVariableEntry entry;
sMaintenanceCondition.Add(&entry);
locker.Unlock();
diff --git a/src/system/kernel/slab/slab_private.h b/src/system/kernel/slab/slab_private.h
index 9ac556bb72..a4cb7b94fd 100644
--- a/src/system/kernel/slab/slab_private.h
+++ b/src/system/kernel/slab/slab_private.h
@@ -29,6 +29,8 @@ struct ObjectCache;
void* slab_internal_alloc(size_t size, uint32 flags);
void slab_internal_free(void *_buffer, uint32 flags);
+void request_memory_manager_maintenance();
+
void* block_alloc(size_t size, uint32 flags);
void* block_alloc_early(size_t size);
void block_free(void *block, uint32 flags);