kernel: add syscalls for thread affinity

scheduler modes adapted to take into account the thread affinity mask.
when no cpu in the affinity mask is enabled, the mask is ignored.

Change-Id: I577737441ab073941a4c5e06f94f7825cffdc2c4
Reviewed-on: https://review.haiku-os.org/c/haiku/+/7674
Reviewed-by: waddlesplash <[email protected]>
Tested-by: Commit checker robot <[email protected]>
This commit is contained in:
Jérôme Duval
2024-06-10 16:34:25 +00:00
committed by waddlesplash
parent 1dde4c4b94
commit f7a85eea15
15 changed files with 233 additions and 49 deletions
+1
View File
@@ -90,6 +90,7 @@ typedef struct CACHE_LINE_ALIGN cpu_ent {
extern cpu_ent gCPU[];
extern uint32 gCPUCacheLevelCount;
extern CPUSet gCPUEnabled;
#ifdef __cplusplus
+3
View File
@@ -160,6 +160,9 @@ thread_id _user_find_thread(const char *name);
status_t _user_get_thread_info(thread_id id, thread_info *info);
status_t _user_get_next_thread_info(team_id team, int32 *cookie, thread_info *info);
int _user_get_cpu();
status_t _user_get_thread_affinity(thread_id id, void* userMask, size_t size);
status_t _user_set_thread_affinity(thread_id id, const void* userMask, size_t size);
status_t _user_block_thread(uint32 flags, bigtime_t timeout);
status_t _user_unblock_thread(thread_id thread, status_t status);
+1
View File
@@ -449,6 +449,7 @@ struct Thread : TeamThreadIteratorEntry<thread_id>, KernelReferenceable {
int32 state; // protected by scheduler lock
struct cpu_ent *cpu; // protected by scheduler lock
struct cpu_ent *previous_cpu; // protected by scheduler lock
CPUSet cpumask;
int32 pinned_to_cpu; // only accessed by this thread or in the
// scheduler, when thread is not running
spinlock scheduler_lock;
+4 -4
View File
@@ -78,7 +78,7 @@ public:
Heap(int initialSize);
~Heap();
inline Element* PeekRoot() const;
inline Element* PeekRoot(int32 index = 0) const;
static const Key& GetKey(Element* element);
@@ -188,10 +188,10 @@ HEAP_CLASS_NAME::~Heap()
HEAP_TEMPLATE_LIST
Element*
HEAP_CLASS_NAME::PeekRoot() const
HEAP_CLASS_NAME::PeekRoot(int32 index) const
{
if (fLastElement > 0)
return fElements[0];
if (index < fLastElement)
return fElements[index];
return NULL;
}
+12 -14
View File
@@ -73,8 +73,8 @@ public:
MinMaxHeap(int initialSize);
~MinMaxHeap();
inline Element* PeekMinimum() const;
inline Element* PeekMaximum() const;
inline Element* PeekMinimum(int32 index = 0) const;
inline Element* PeekMaximum(int32 index = 0) const;
static const Key& GetKey(Element* element);
@@ -190,13 +190,12 @@ MIN_MAX_HEAP_CLASS_NAME::~MinMaxHeap()
MIN_MAX_HEAP_TEMPLATE_LIST
Element*
MIN_MAX_HEAP_CLASS_NAME::PeekMinimum() const
MIN_MAX_HEAP_CLASS_NAME::PeekMinimum(int32 index) const
{
if (fMinLastElement > 0)
return fMinElements[0];
else if (fMaxLastElement > 0) {
ASSERT(fMaxLastElement == 1);
return fMaxElements[0];
if (index < fMinLastElement)
return fMinElements[index];
else if (index - fMinLastElement < fMaxLastElement) {
return fMaxElements[fMaxLastElement - (index - fMinLastElement) - 1];
}
return NULL;
@@ -205,13 +204,12 @@ MIN_MAX_HEAP_CLASS_NAME::PeekMinimum() const
MIN_MAX_HEAP_TEMPLATE_LIST
Element*
MIN_MAX_HEAP_CLASS_NAME::PeekMaximum() const
MIN_MAX_HEAP_CLASS_NAME::PeekMaximum(int32 index) const
{
if (fMaxLastElement > 0)
return fMaxElements[0];
else if (fMinLastElement > 0) {
ASSERT(fMinLastElement == 1);
return fMinElements[0];
if (index < fMaxLastElement)
return fMaxElements[index];
else if (index - fMaxLastElement < fMinLastElement) {
return fMinElements[fMinLastElement - (index - fMaxLastElement) - 1];
}
return NULL;
+2
View File
@@ -197,6 +197,8 @@ extern status_t _kern_get_team_usage_info(team_id team, int32 who,
extern status_t _kern_get_extended_team_info(team_id teamID, uint32 flags,
void* buffer, size_t size, size_t* _sizeNeeded);
extern int _kern_get_cpu();
extern status_t _kern_get_thread_affinity(thread_id id, void* userMask, size_t size);
extern status_t _kern_set_thread_affinity(thread_id id, const void* userMask, size_t size);
extern status_t _kern_start_watching_system(int32 object, uint32 flags,
port_id port, int32 token);
+2
View File
@@ -28,6 +28,7 @@
/* global per-cpu structure */
cpu_ent gCPU[SMP_MAX_CPUS];
CPUSet gCPUEnabled;
uint32 gCPUCacheLevelCount;
static cpu_topology_node sCPUTopology;
@@ -149,6 +150,7 @@ cpu_preboot_init_percpu(kernel_args *args, int curr_cpu)
// we can use it for get_current_cpu
memset(&gCPU[curr_cpu], 0, sizeof(gCPU[curr_cpu]));
gCPU[curr_cpu].cpu_num = curr_cpu;
gCPUEnabled.SetBit(curr_cpu);
list_init(&gCPU[curr_cpu].irqs);
B_INITIALIZE_SPINLOCK(&gCPU[curr_cpu].irqs_lock);
+41 -10
View File
@@ -44,7 +44,7 @@ has_cache_expired(const ThreadData* threadData)
static CoreEntry*
choose_core(const ThreadData* /* threadData */)
choose_core(const ThreadData* threadData)
{
SCHEDULER_ENTER_FUNCTION();
@@ -55,16 +55,31 @@ choose_core(const ThreadData* /* threadData */)
package = PackageEntry::GetMostIdlePackage();
}
int32 index = 0;
CPUSet mask = threadData->GetCPUMask();
if (mask.IsEmpty()) {
// ignore when empty
mask.SetAll();
}
CoreEntry* core = NULL;
if (package != NULL)
core = package->GetIdleCore();
if (package != NULL) {
do {
core = package->GetIdleCore(index++);
} while (core != NULL && !core->CPUMask().Matches(mask));
}
if (core == NULL) {
ReadSpinLocker coreLocker(gCoreHeapsLock);
index = 0;
// no idle cores, use least occupied core
core = gCoreLoadHeap.PeekMinimum();
if (core == NULL)
core = gCoreHighLoadHeap.PeekMinimum();
do {
core = gCoreLoadHeap.PeekMinimum(index++);
} while (core != NULL && !core->CPUMask().Matches(mask));
if (core == NULL) {
index = 0;
do {
core = gCoreHighLoadHeap.PeekMinimum(index++);
} while (core != NULL && !core->CPUMask().Matches(mask));
}
}
ASSERT(core != NULL);
@@ -82,9 +97,25 @@ rebalance(const ThreadData* threadData)
// Get the least loaded core.
ReadSpinLocker coreLocker(gCoreHeapsLock);
CoreEntry* other = gCoreLoadHeap.PeekMinimum();
if (other == NULL)
other = gCoreHighLoadHeap.PeekMinimum();
CPUSet mask = threadData->GetCPUMask();
if (mask.IsEmpty()) {
// ignore when empty
mask.SetAll();
}
int32 index = 0;
CoreEntry* other;
do {
other = gCoreLoadHeap.PeekMinimum(index++);
if (other != NULL && other->CPUMask().IsEmpty())
panic("other->CPUMask().IsEmpty()\n");
} while (other != NULL && !other->CPUMask().Matches(mask));
if (other == NULL) {
index = 0;
do {
other = gCoreHighLoadHeap.PeekMinimum(index++);
} while (other != NULL && !other->CPUMask().Matches(mask));
}
coreLocker.Unlock();
ASSERT(other != NULL);
+36 -8
View File
@@ -77,7 +77,6 @@ choose_idle_core()
if (package != NULL)
return package->GetIdleCore();
return NULL;
}
@@ -89,22 +88,38 @@ choose_core(const ThreadData* threadData)
CoreEntry* core = NULL;
CPUSet mask = threadData->GetCPUMask();
if (mask.IsEmpty()) {
// ignore when empty
mask.SetAll();
}
// try to pack all threads on one core
core = choose_small_task_core();
if (!core->CPUMask().Matches(mask))
core = NULL;
if (core == NULL || core->GetLoad() + threadData->GetLoad() >= kHighLoad) {
ReadSpinLocker coreLocker(gCoreHeapsLock);
// run immediately on already woken core
core = gCoreLoadHeap.PeekMinimum();
int32 index = 0;
do {
core = gCoreLoadHeap.PeekMinimum(index++);
} while (core != NULL && !core->CPUMask().Matches(mask));
if (core == NULL) {
coreLocker.Unlock();
core = choose_idle_core();
if (!core->CPUMask().Matches(mask))
core = NULL;
if (core == NULL) {
coreLocker.Lock();
core = gCoreHighLoadHeap.PeekMinimum();
index = 0;
do {
core = gCoreHighLoadHeap.PeekMinimum(index++);
} while (core != NULL && !core->CPUMask().Matches(mask));
}
}
}
@@ -121,6 +136,11 @@ rebalance(const ThreadData* threadData)
ASSERT(!gSingleCore);
CPUSet mask = threadData->GetCPUMask();
if (mask.IsEmpty()) {
// ignore when empty
mask.SetAll();
}
CoreEntry* core = threadData->Core();
int32 coreLoad = core->GetLoad();
@@ -130,7 +150,7 @@ rebalance(const ThreadData* threadData)
sSmallTaskCore = NULL;
CoreEntry* smallTaskCore = choose_small_task_core();
if (threadLoad > coreLoad / 3)
if (threadLoad > coreLoad / 3 || !smallTaskCore->CPUMask().Matches(mask))
return core;
return coreLoad > kVeryHighLoad ? smallTaskCore : core;
}
@@ -139,9 +159,17 @@ rebalance(const ThreadData* threadData)
return core;
ReadSpinLocker coreLocker(gCoreHeapsLock);
CoreEntry* other = gCoreLoadHeap.PeekMaximum();
if (other == NULL)
other = gCoreHighLoadHeap.PeekMinimum();
CoreEntry* other;
int32 index = 0;
do {
other = gCoreLoadHeap.PeekMaximum(index++);
} while (other != NULL && !core->CPUMask().Matches(mask));
if (other == NULL) {
index = 0;
do {
other = gCoreHighLoadHeap.PeekMinimum(index++);
} while (other != NULL && !core->CPUMask().Matches(mask));
}
coreLocker.Unlock();
ASSERT(other != NULL);
@@ -154,7 +182,7 @@ rebalance(const ThreadData* threadData)
return core;
CoreEntry* smallTaskCore = choose_small_task_core();
if (smallTaskCore == NULL)
if (smallTaskCore == NULL || !smallTaskCore->CPUMask().Matches(mask))
return core;
return smallTaskCore->GetLoad() + threadLoad < kHighLoad
? smallTaskCore : core;
+22 -6
View File
@@ -117,7 +117,7 @@ enqueue(Thread* thread, bool newOne)
const bool rescheduleNeeded = threadData->ChooseCoreAndCPU(targetCore, targetCPU);
TRACE("enqueueing thread %ld with priority %ld on CPU %ld (core %ld)\n",
TRACE("enqueueing thread %" B_PRId32 " with priority %" B_PRId32 " on CPU %" B_PRId32 " (core %" B_PRId32 ")\n",
thread->id, threadPriority, targetCPU->ID(), targetCore->ID());
bool wasRunQueueEmpty = false;
@@ -153,7 +153,7 @@ scheduler_enqueue_in_run_queue(Thread *thread)
SchedulerModeLocker _;
TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id,
TRACE("enqueueing new thread %" B_PRId32 " with static priority %" B_PRId32 "\n", thread->id,
thread->priority);
ThreadData* threadData = thread->scheduler_data;
@@ -180,7 +180,7 @@ scheduler_set_thread_priority(Thread *thread, int32 priority)
ThreadData* threadData = thread->scheduler_data;
int32 oldPriority = thread->priority;
TRACE("changing thread %ld priority to %ld (old: %ld, effective: %ld)\n",
TRACE("changing thread %" B_PRId32 " priority to %" B_PRId32 " (old: %" B_PRId32 ", effective: %" B_PRId32 ")\n",
thread->id, priority, oldPriority, threadData->GetEffectivePriority());
thread->priority = priority;
@@ -333,7 +333,7 @@ reschedule(int32 nextState)
SchedulerModeLocker modeLocker;
TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU,
TRACE("reschedule(): cpu %" B_PRId32 ", current thread = %" B_PRId32 "\n", thisCPU,
oldThread->id);
oldThread->state = nextState;
@@ -348,7 +348,7 @@ reschedule(int32 nextState)
case B_THREAD_READY:
enqueueOldThread = true;
if (!oldThreadData->IsIdle()) {
if (!oldThreadData->IsIdle() && oldThreadData->GetCPUMask().GetBit(thisCPU)) {
oldThreadData->Continues();
if (oldThreadData->HasQuantumEnded(oldThread->cpu->preempted,
oldThread->has_yielded)) {
@@ -394,9 +394,21 @@ reschedule(int32 nextState)
} else
nextThreadData = oldThreadData;
} else {
CPUSet mask = oldThreadData->GetCPUMask();
if (mask.IsEmpty())
mask.SetAll();
bool oldThreadShouldMigrate = !mask.GetBit(thisCPU);
if (oldThreadShouldMigrate)
enqueueOldThread = false;
nextThreadData
= cpu->ChooseNextThread(enqueueOldThread ? oldThreadData : NULL,
putOldThreadAtBack);
if (oldThreadShouldMigrate) {
enqueue(oldThread, true);
// replace with the idle thread, if no other thread could be found
if (oldThreadData == nextThreadData)
nextThreadData = cpu->PeekIdleThread();
}
// update CPU heap
CoreCPUHeapLocker cpuLocker(core);
@@ -417,7 +429,7 @@ reschedule(int32 nextState)
acquire_spinlock(&nextThread->scheduler_lock);
}
TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU,
TRACE("reschedule(): cpu %" B_PRId32 ", next thread = %" B_PRId32 "\n", thisCPU,
nextThread->id);
T(ScheduleThread(nextThread, oldThread));
@@ -572,6 +584,10 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
}
gCPU[cpuID].disabled = !enabled;
if (enabled)
gCPUEnabled.SetBitAtomic(cpuID);
else
gCPUEnabled.ClearBitAtomic(cpuID);
if (!enabled) {
cpu->Stop();
@@ -233,7 +233,8 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack)
CoreRunQueueLocker coreLocker(fCore);
ThreadData* sharedThread = fCore->PeekThread();
ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL);
if (sharedThread == NULL && pinnedThread == NULL && oldThread == NULL)
return NULL;
int32 sharedPriority = -1;
if (sharedThread != NULL)
@@ -477,6 +478,7 @@ CoreEntry::AddCPU(CPUEntry* cpu)
fPackage->AddIdleCore(this);
}
fCPUSet.SetBit(cpu->ID());
fCPUHeap.Insert(cpu, B_IDLE_PRIORITY);
}
@@ -489,6 +491,7 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing)
ASSERT(fIdleCPUCount > 0);
fIdleCPUCount--;
fCPUSet.ClearBit(cpu->ID());
if (--fCPUCount == 0) {
// unassign threads
thread_map(CoreEntry::_UnassignThread, this);
+11 -3
View File
@@ -8,6 +8,7 @@
#include <OS.h>
#include <smp.h>
#include <thread.h>
#include <util/AutoLock.h>
#include <util/Heap.h>
@@ -131,6 +132,8 @@ public:
inline PackageEntry* Package() const { return fPackage; }
inline int32 CPUCount() const
{ return fCPUCount; }
inline const CPUSet& CPUMask() const
{ return fCPUSet; }
inline void LockCPUHeap();
inline void UnlockCPUHeap();
@@ -182,6 +185,7 @@ private:
PackageEntry* fPackage;
int32 fCPUCount;
CPUSet fCPUSet;
int32 fIdleCPUCount;
CPUPriorityHeap fCPUHeap;
spinlock fCPULock;
@@ -229,7 +233,7 @@ public:
inline void CoreGoesIdle(CoreEntry* core);
inline void CoreWakesUp(CoreEntry* core);
inline CoreEntry* GetIdleCore() const;
inline CoreEntry* GetIdleCore(int32 index = 0) const;
void AddIdleCore(CoreEntry* core);
void RemoveIdleCore(CoreEntry* core);
@@ -538,10 +542,14 @@ CoreEntry::GetCore(int32 cpu)
inline CoreEntry*
PackageEntry::GetIdleCore() const
PackageEntry::GetIdleCore(int32 index) const
{
SCHEDULER_ENTER_FUNCTION();
return fIdleCores.Last();
CoreEntry* element = fIdleCores.Last();
for (int32 i = 0; element != NULL && i < index; i++)
element = fIdleCores.GetPrevious(element);
return element;
}
@@ -59,10 +59,16 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const
int32 threadPriority = GetEffectivePriority();
if (fThread->previous_cpu != NULL) {
CPUSet mask = GetCPUMask();
if (mask.IsEmpty())
mask.SetAll();
ASSERT(mask.Matches(core->CPUMask()));
if (fThread->previous_cpu != NULL && !fThread->previous_cpu->disabled
&& mask.GetBit(fThread->previous_cpu->cpu_num)) {
CPUEntry* previousCPU
= CPUEntry::GetCPU(fThread->previous_cpu->cpu_num);
if (previousCPU->Core() == core && !fThread->previous_cpu->disabled) {
if (previousCPU->Core() == core) {
CoreCPUHeapLocker _(core);
if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) {
previousCPU->UpdatePriority(threadPriority);
@@ -73,7 +79,11 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const
}
CoreCPUHeapLocker _(core);
CPUEntry* cpu = core->CPUHeap()->PeekRoot();
int32 index = 0;
CPUEntry* cpu;
do {
cpu = core->CPUHeap()->PeekRoot(index++);
} while (cpu != NULL && !mask.GetBit(cpu->ID()));
ASSERT(cpu != NULL);
if (CPUPriorityHeap::GetKey(cpu) < threadPriority) {
@@ -156,12 +166,21 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU)
bool rescheduleNeeded = false;
if (targetCore != NULL && !targetCore->CPUMask().Matches(GetCPUMask()))
targetCore = NULL;
if (targetCPU != NULL && !GetCPUMask().GetBit(targetCPU->ID()))
targetCPU = NULL;
if (targetCore == NULL && targetCPU != NULL)
targetCore = targetCPU->Core();
else if (targetCore != NULL && targetCPU == NULL)
targetCPU = _ChooseCPU(targetCore, rescheduleNeeded);
else if (targetCore == NULL && targetCPU == NULL) {
targetCore = _ChooseCore();
CPUSet mask = GetCPUMask();
if (mask.IsEmpty())
mask.SetAll();
ASSERT(mask.Matches(targetCore->CPUMask()));
targetCPU = _ChooseCPU(targetCore, rescheduleNeeded);
}
@@ -39,6 +39,7 @@ public:
inline int32 GetPriority() const { return fThread->priority; }
inline Thread* GetThread() const { return fThread; }
inline CPUSet GetCPUMask() const { return fThread->cpumask.And(gCPUEnabled); }
inline bool IsRealTime() const;
inline bool IsIdle() const;
+71
View File
@@ -174,6 +174,7 @@ Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu)
io_priority(-1),
cpu(cpu),
previous_cpu(NULL),
cpumask(),
pinned_to_cpu(0),
sig_block_mask(0),
sigsuspend_original_unblocked_mask(0),
@@ -203,6 +204,7 @@ Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu)
{
id = threadID >= 0 ? threadID : allocate_thread_id();
visible = false;
cpumask.SetAll();
// init locks
char lockName[32];
@@ -1778,6 +1780,7 @@ _dump_thread_info(Thread *thread, bool shortInfo)
kprintf("(%d)\n", thread->cpu->cpu_num);
else
kprintf("\n");
kprintf("cpumask: %#" B_PRIx32 "\n", thread->cpumask.Bits(0));
kprintf("sig_pending: %#" B_PRIx64 " (blocked: %#" B_PRIx64
", before sigsuspend(): %#" B_PRIx64 ")\n",
(int64)thread->ThreadPendingSignals(),
@@ -2752,6 +2755,8 @@ thread_init(kernel_args *args)
thread->team = team_get_kernel_team();
thread->priority = B_IDLE_PRIORITY;
thread->state = B_THREAD_RUNNING;
thread->cpumask.SetAll();
sprintf(name, "idle thread %" B_PRIu32 " kstack", i + 1);
thread->kernel_stack_area = find_area(name);
@@ -3923,3 +3928,69 @@ _user_get_cpu()
Thread* thread = thread_get_current_thread();
return thread->cpu->cpu_num;
}
status_t
_user_get_thread_affinity(thread_id id, void* userMask, size_t size)
{
if (userMask == NULL || id < B_OK)
return B_BAD_VALUE;
if (!IS_USER_ADDRESS(userMask))
return B_BAD_ADDRESS;
CPUSet mask;
if (id == 0)
id = thread_get_current_thread_id();
// get the thread
Thread* thread = Thread::GetAndLock(id);
if (thread == NULL)
return B_BAD_THREAD_ID;
BReference<Thread> threadReference(thread, true);
ThreadLocker threadLocker(thread, true);
memcpy(&mask, &thread->cpumask, sizeof(mask));
if (user_memcpy(userMask, &mask, min_c(sizeof(mask), size)) < B_OK)
return B_BAD_ADDRESS;
return B_OK;
}
status_t
_user_set_thread_affinity(thread_id id, const void* userMask, size_t size)
{
if (userMask == NULL || id < B_OK || size < sizeof(CPUSet))
return B_BAD_VALUE;
if (!IS_USER_ADDRESS(userMask))
return B_BAD_ADDRESS;
CPUSet mask;
if (user_memcpy(&mask, userMask, min_c(sizeof(CPUSet), size)) < B_OK)
return B_BAD_ADDRESS;
CPUSet cpus;
cpus.SetAll();
for (int i = 0; i < smp_get_num_cpus(); i++)
cpus.ClearBit(i);
if (mask.Matches(cpus))
return B_BAD_VALUE;
if (id == 0)
id = thread_get_current_thread_id();
// get the thread
Thread* thread = Thread::GetAndLock(id);
if (thread == NULL)
return B_BAD_THREAD_ID;
BReference<Thread> threadReference(thread, true);
ThreadLocker threadLocker(thread, true);
memcpy(&thread->cpumask, &mask, sizeof(mask));
// check if running on masked cpu
if (!thread->cpumask.GetBit(thread->cpu->cpu_num))
thread_yield();
return B_OK;
}