IOScheduler: Rework IORequestOwner management.

* Make IORequestOwner an abstract (ish) class, and move details
   to the Scheduler implementations. This will allow other drivers
   (like NVMe) to use their own, separate IORequestOwner system.

 * Use an object_cache to allocate IOSchedulerSimple::RequestOwners.
   Previously, a single block large enough to store one for every
   thread at maximum thread count (4096) was allocated, meaning
   a few hundred KB per IOScheduler. In the case of low memory,
   a fallback IORequestOwner with a thread ID of -1 is used.

Tested with IDE and usb_disk drivers, seems to be working.

src/system/kernel/device_manager/IOScheduler.h

@@ -14,21 +14,12 @@
 #include "IORequest.h"
 
 
-struct IORequestOwner : DoublyLinkedListLinkImpl<IORequestOwner> {
+struct IORequestOwner {
 	team_id			team;
 	thread_id		thread;
-	int32			priority;
+	thread_id		priority;
-	IORequestList	requests;
-	IORequestList	completed_requests;
-	IOOperationList	operations;
-	IORequestOwner*	hash_link;
 
-			bool				IsActive() const
+	virtual void	Dump() const = 0;
-									{ return !requests.IsEmpty()
-										|| !completed_requests.IsEmpty()
-										|| !operations.IsEmpty(); }
-
-			void				Dump() const;
 };
 
 

src/system/kernel/device_manager/IOSchedulerSimple.cpp

@@ -17,6 +17,7 @@
 #include <lock.h>
 #include <thread_types.h>
 #include <thread.h>
+#include <slab/Slab.h>
 #include <util/AutoLock.h>
 
 #include "IOSchedulerRoster.h"
@@ -33,10 +34,29 @@
 // #pragma mark -
 
 
+static object_cache* sRequestOwnerCache;
+
+
+struct IOSchedulerSimple::RequestOwner
+		: IORequestOwner, DoublyLinkedListLinkImpl<RequestOwner> {
+	IORequestList	requests;
+	IORequestList	completed_requests;
+	IOOperationList	operations;
+	RequestOwner* hash_link;
+
+			bool				IsActive() const
+									{ return !requests.IsEmpty()
+										|| !completed_requests.IsEmpty()
+										|| !operations.IsEmpty(); }
+
+			void				Dump() const override;
+};
+
+
 void
-IORequestOwner::Dump() const
+IOSchedulerSimple::RequestOwner::Dump() const
 {
-	kprintf("IORequestOwner at %p\n", this);
+	kprintf("IOSchedulerSimple::RequestOwner at %p\n", this);
 	kprintf("  team:     %" B_PRId32 "\n", team);
 	kprintf("  thread:   %" B_PRId32 "\n", thread);
 	kprintf("  priority: %" B_PRId32 "\n", priority);
@@ -69,13 +89,13 @@ IORequestOwner::Dump() const
 
 struct IOSchedulerSimple::RequestOwnerHashDefinition {
 	typedef thread_id KeyType;
-	typedef IORequestOwner	ValueType;
+	typedef IOSchedulerSimple::RequestOwner ValueType;
 
 	size_t HashKey(thread_id key) const			{ return key; }
-	size_t Hash(const IORequestOwner* value) const	{ return value->thread; }
+	size_t Hash(const ValueType* value) const	{ return value->thread; }
-	bool Compare(thread_id key, const IORequestOwner* value) const
+	bool Compare(thread_id key, const ValueType* value) const
 		{ return value->thread == key; }
-	IORequestOwner*& GetLink(IORequestOwner* value) const
+	ValueType*& GetLink(ValueType* value) const
 		{ return value->hash_link; }
 };
 
@@ -90,7 +110,6 @@ IOSchedulerSimple::IOSchedulerSimple(DMAResource* resource)
 	fSchedulerThread(-1),
 	fRequestNotifierThread(-1),
 	fOperationArray(NULL),
-	fAllocatedRequestOwners(NULL),
 	fRequestOwners(NULL),
 	fBlockSize(0),
 	fPendingOperations(0),
@@ -103,6 +122,16 @@ IOSchedulerSimple::IOSchedulerSimple(DMAResource* resource)
 	fFinishedOperationCondition.Init(this, "I/O finished operation");
 	fFinishedRequestCondition.Init(this, "I/O finished request");
 
+	if (sRequestOwnerCache == NULL) {
+		// Borrow the SchedulerRoster lock to initialize.
+		IOSchedulerRoster::Default()->Lock();
+		if (sRequestOwnerCache == NULL) {
+			sRequestOwnerCache = create_object_cache("IOSchedulerSimpleRequestOwners",
+				sizeof(RequestOwner), 0);
+			object_cache_set_minimum_reserve(sRequestOwnerCache, smp_get_num_cpus());
+		}
+		IOSchedulerRoster::Default()->Unlock();
+	}
 }
 
 
@@ -135,8 +164,14 @@ IOSchedulerSimple::~IOSchedulerSimple()
 
 	delete[] fOperationArray;
 
+	RequestOwner* owner = fRequestOwners->Clear(true);
+	while (owner != NULL) {
+		RequestOwner* next = owner->hash_link;
+		object_cache_free(sRequestOwnerCache, owner, 0);
+		owner = next;
+	}
+
 	delete fRequestOwners;
-	delete[] fAllocatedRequestOwners;
 }
 
 
@@ -163,28 +198,20 @@ IOSchedulerSimple::Init(const char* name)
 	if (fBlockSize == 0)
 		fBlockSize = 512;
 
-	fAllocatedRequestOwnerCount = thread_max_threads();
-	fAllocatedRequestOwners
-		= new(std::nothrow) IORequestOwner[fAllocatedRequestOwnerCount];
-	if (fAllocatedRequestOwners == NULL)
-		return B_NO_MEMORY;
-
-	for (int32 i = 0; i < fAllocatedRequestOwnerCount; i++) {
-		IORequestOwner& owner = fAllocatedRequestOwners[i];
-		owner.team = -1;
-		owner.thread = -1;
-		owner.priority = B_IDLE_PRIORITY;
-		fUnusedRequestOwners.Add(&owner);
-	}
-
 	fRequestOwners = new(std::nothrow) RequestOwnerHashTable;
 	if (fRequestOwners == NULL)
 		return B_NO_MEMORY;
 
-	error = fRequestOwners->Init(fAllocatedRequestOwnerCount);
+	error = fRequestOwners->Init(count);
 	if (error != B_OK)
 		return error;
 
+	// Allocate a fallback RequestOwner, for use under low-memory conditions.
+	RequestOwner* fallbackOwner = _GetRequestOwner(-1, -1, true);
+	if (fallbackOwner == NULL)
+		return B_NO_MEMORY;
+	fallbackOwner->priority = B_LOWEST_ACTIVE_PRIORITY;
+
 	// TODO: Use a device speed dependent bandwidths!
 	fIterationBandwidth = fBlockSize * 8192;
 	fMinOwnerBandwidth = fBlockSize * 1024;
@@ -241,7 +268,7 @@ IOSchedulerSimple::ScheduleRequest(IORequest* request)
 
 	MutexLocker locker(fLock);
 
-	IORequestOwner* owner = _GetRequestOwner(request->TeamID(),
+	RequestOwner* owner = _GetRequestOwner(request->TeamID(),
 		request->ThreadID(), true);
 	if (owner == NULL) {
 		panic("IOSchedulerSimple: Out of request owners!\n");
@@ -256,9 +283,11 @@ IOSchedulerSimple::ScheduleRequest(IORequest* request)
 	request->SetOwner(owner);
 	owner->requests.Add(request);
 
+	if (owner->thread != -1) {
 		int32 priority = thread_get_io_priority(request->ThreadID());
 		if (priority >= 0)
 			owner->priority = priority;
+	}
 //dprintf("  request %p -> owner %p (thread %ld, active %d)\n", request, owner, owner->thread, wasActive);
 
 	if (!wasActive)
@@ -305,9 +334,9 @@ IOSchedulerSimple::Dump() const
 	kprintf("  DMA resource:   %p\n", fDMAResource);
 
 	kprintf("  active request owners:");
-	for (RequestOwnerList::ConstIterator it
+	for (RequestOwnerHashTable::Iterator it
-				= fActiveRequestOwners.GetIterator();
+				= fRequestOwners->GetIterator();
-			IORequestOwner* owner = it.Next();) {
+			RequestOwner* owner = it.Next();) {
 		kprintf(" %p", owner);
 	}
 	kprintf("\n");
@@ -338,7 +367,7 @@ IOSchedulerSimple::_Finisher()
 		if (!operationFinished) {
 			TRACE("  operation: %p not finished yet\n", operation);
 			MutexLocker _(fLock);
-			operation->Parent()->Owner()->operations.Add(operation);
+			((RequestOwner*)operation->Parent()->Owner())->operations.Add(operation);
 			fPendingOperations--;
 			continue;
 		}
@@ -364,14 +393,17 @@ IOSchedulerSimple::_Finisher()
 				request->SetUnfinished();
 			} else {
 				// Remove the request from the request owner.
-				IORequestOwner* owner = request->Owner();
+				RequestOwner* owner = (RequestOwner*)request->Owner();
 				owner->requests.TakeFrom(&owner->completed_requests);
 				owner->requests.Remove(request);
 				request->SetOwner(NULL);
 
 				if (!owner->IsActive()) {
 					fActiveRequestOwners.Remove(owner);
-					fUnusedRequestOwners.Add(owner);
+					if (owner->thread != -1) {
+						fRequestOwners->Remove(owner);
+						object_cache_free(sRequestOwnerCache, owner, 0);
+					}
 				}
 
 				if (request->HasCallbacks()) {
@@ -477,7 +509,7 @@ IOSchedulerSimple::_ComputeRequestOwnerBandwidth(int32 priority) const
 
 
 bool
-IOSchedulerSimple::_NextActiveRequestOwner(IORequestOwner*& owner,
+IOSchedulerSimple::_NextActiveRequestOwner(RequestOwner*& owner,
 	off_t& quantum)
 {
 	while (true) {
@@ -576,7 +608,7 @@ IOSchedulerSimple::_SortOperations(IOOperationList& operations,
 status_t
 IOSchedulerSimple::_Scheduler()
 {
-	IORequestOwner marker;
+	RequestOwner marker;
 	marker.thread = -1;
 	{
 		MutexLocker locker(fLock);
@@ -585,7 +617,7 @@ IOSchedulerSimple::_Scheduler()
 
 	off_t lastOffset = 0;
 
-	IORequestOwner* owner = NULL;
+	RequestOwner* owner = NULL;
 	off_t quantum = 0;
 
 	while (!fTerminating) {
@@ -781,34 +813,26 @@ IOSchedulerSimple::_RequestNotifierThread(void *_self)
 }
 
 
-IORequestOwner*
+IOSchedulerSimple::RequestOwner*
 IOSchedulerSimple::_GetRequestOwner(team_id team, thread_id thread,
 	bool allocate)
 {
 	// lookup in table
-	IORequestOwner* owner = fRequestOwners->Lookup(thread);
+	RequestOwner* owner = fRequestOwners->Lookup(thread);
-	if (owner != NULL && !owner->IsActive())
-		fUnusedRequestOwners.Remove(owner);
 	if (owner != NULL || !allocate)
 		return owner;
 
-	// not in table -- allocate an unused one
+	// not in table -- allocate a new one
-	RequestOwnerList existingOwners;
+	owner = new(sRequestOwnerCache, CACHE_DONT_WAIT_FOR_MEMORY) RequestOwner;
+	if (owner == NULL) {
+		// Use the fallback owner.
+		return fRequestOwners->Lookup(-1);
+	}
 
-	while ((owner = fUnusedRequestOwners.RemoveHead()) != NULL) {
-		if (owner->thread < 0 || !Thread::IsAlive(owner->thread)) {
-			if (owner->thread >= 0)
-				fRequestOwners->RemoveUnchecked(owner);
 	owner->team = team;
 	owner->thread = thread;
 	owner->priority = B_IDLE_PRIORITY;
 	fRequestOwners->InsertUnchecked(owner);
-			break;
-		}
 
-		existingOwners.Add(owner);
-	}
-
-	fUnusedRequestOwners.TakeFrom(&existingOwners);
 	return owner;
 }

src/system/kernel/device_manager/IOSchedulerSimple.h

@@ -38,7 +38,8 @@ public:
 	virtual	void				Dump() const;
 
 private:
-			typedef DoublyLinkedList<IORequestOwner> RequestOwnerList;
+			struct RequestOwner;
+			typedef DoublyLinkedList<RequestOwner> RequestOwnerList;
 
 			struct RequestOwnerHashDefinition;
 			struct RequestOwnerHashTable;
@@ -47,7 +48,7 @@ private:
 			bool				_FinisherWorkPending();
 			off_t				_ComputeRequestOwnerBandwidth(
 									int32 priority) const;
-			bool				_NextActiveRequestOwner(IORequestOwner*& owner,
+			bool				_NextActiveRequestOwner(RequestOwner*& owner,
 									off_t& quantum);
 			bool				_PrepareRequestOperations(IORequest* request,
 									IOOperationList& operations,
@@ -64,7 +65,7 @@ private:
 	static	status_t			_RequestNotifierThread(void* self);
 
 			void				_AddRequestOwner(IORequestOwner* owner);
-			IORequestOwner*		_GetRequestOwner(team_id team, thread_id thread,
+			RequestOwner*		_GetRequestOwner(team_id team, thread_id thread,
 									bool allocate);
 
 private:
@@ -80,10 +81,7 @@ private:
 			IOOperation**		fOperationArray;
 			IOOperationList		fUnusedOperations;
 			IOOperationList		fCompletedOperations;
-			IORequestOwner*		fAllocatedRequestOwners;
-			int32				fAllocatedRequestOwnerCount;
 			RequestOwnerList	fActiveRequestOwners;
-			RequestOwnerList	fUnusedRequestOwners;
 			RequestOwnerHashTable* fRequestOwners;
 			generic_size_t		fBlockSize;
 			int32				fPendingOperations;