scheduler: Move mode specific logic to separate files
This commit is contained in:
Pawel Dziepak
@@ -1,4 +1,5 @@
|
||||
/*
|
||||
* Copyright 2013, Paweł Dziepak, [email protected].
|
||||
* Copyright 2008-2011, Ingo Weinhold, [email protected].
|
||||
* Copyright 2005-2010, Axel Dörfler, [email protected].
|
||||
* Distributed under the terms of the MIT License.
|
||||
|
||||
@@ -6,12 +6,15 @@
|
||||
#define _KERNEL_LOAD_TRACKING_H
|
||||
|
||||
|
||||
#include <OS.h>
|
||||
|
||||
|
||||
const int32 kMaxLoad = 1000;
|
||||
const bigtime_t kLoadMeasureInterval = 50000;
|
||||
const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4;
|
||||
|
||||
|
||||
static int32
|
||||
static inline int32
|
||||
compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int32& load)
|
||||
{
|
||||
bigtime_t now = system_time();
|
||||
|
||||
@@ -62,6 +62,8 @@ KernelMergeObject kernel_core.o :
|
||||
user_mutex.cpp
|
||||
|
||||
# scheduler
|
||||
low_latency.cpp
|
||||
power_saving.cpp
|
||||
scheduler.cpp
|
||||
scheduler_tracing.cpp
|
||||
scheduling_analysis.cpp
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
* Copyright 2013, Paweł Dziepak, [email protected].
|
||||
* Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
|
||||
#include <util/AutoLock.h>
|
||||
|
||||
#include "scheduler_common.h"
|
||||
#include "scheduler_modes.h"
|
||||
|
||||
|
||||
using namespace Scheduler;
|
||||
|
||||
|
||||
static void
|
||||
switch_to_mode(void)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
static bool
|
||||
has_cache_expired(Thread* thread)
|
||||
{
|
||||
ASSERT(!gSingleCore);
|
||||
|
||||
if (thread_is_idle_thread(thread))
|
||||
return false;
|
||||
|
||||
scheduler_thread_data* schedulerThreadData = thread->scheduler_data;
|
||||
ASSERT(schedulerThreadData->previous_core >= 0);
|
||||
|
||||
CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core];
|
||||
return atomic_get64(&coreEntry->fActiveTime)
|
||||
- schedulerThreadData->went_sleep_active > kCacheExpire;
|
||||
}
|
||||
|
||||
|
||||
static int32
|
||||
choose_core(Thread* thread)
|
||||
{
|
||||
CoreEntry* entry;
|
||||
|
||||
if (gIdlePackageList->Last() != NULL) {
|
||||
// wake new package
|
||||
PackageEntry* package = gIdlePackageList->Last();
|
||||
entry = package->fIdleCores.Last();
|
||||
} else if (gPackageUsageHeap->PeekMaximum() != NULL) {
|
||||
// wake new core
|
||||
PackageEntry* package = gPackageUsageHeap->PeekMaximum();
|
||||
entry = package->fIdleCores.Last();
|
||||
} else {
|
||||
// no idle cores, use least occupied core
|
||||
entry = gCoreLoadHeap->PeekMinimum();
|
||||
if (entry == NULL)
|
||||
entry = gCoreHighLoadHeap->PeekMinimum();
|
||||
}
|
||||
|
||||
ASSERT(entry != NULL);
|
||||
return entry->fCoreID;
|
||||
}
|
||||
|
||||
|
||||
static bool
|
||||
should_rebalance(Thread* thread)
|
||||
{
|
||||
scheduler_thread_data* schedulerThreadData = thread->scheduler_data;
|
||||
ASSERT(schedulerThreadData->previous_core >= 0);
|
||||
|
||||
CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core];
|
||||
|
||||
// If the thread produces more than 50% of the load, leave it here. In
|
||||
// such situation it is better to move other threads away.
|
||||
if (schedulerThreadData->load >= coreEntry->fLoad / 2)
|
||||
return false;
|
||||
|
||||
// If there is high load on this core but this thread does not contribute
|
||||
// significantly consider giving it to someone less busy.
|
||||
if (coreEntry->fLoad > kHighLoad) {
|
||||
SpinLocker coreLocker(gCoreHeapsLock);
|
||||
|
||||
CoreEntry* other = gCoreLoadHeap->PeekMinimum();
|
||||
if (other != NULL && coreEntry->fLoad - other->fLoad >= kLoadDifference)
|
||||
return true;
|
||||
}
|
||||
|
||||
// No cpu bound threads - the situation is quite good. Make sure it
|
||||
// won't get much worse...
|
||||
SpinLocker coreLocker(gCoreHeapsLock);
|
||||
|
||||
CoreEntry* other = gCoreLoadHeap->PeekMinimum();
|
||||
if (other == NULL)
|
||||
other = gCoreHighLoadHeap->PeekMinimum();
|
||||
return coreEntry->fLoad - other->fLoad >= kLoadDifference * 2;
|
||||
}
|
||||
|
||||
|
||||
static void
|
||||
rebalance_irqs(bool idle)
|
||||
{
|
||||
if (idle)
|
||||
return;
|
||||
|
||||
cpu_ent* cpu = get_cpu_struct();
|
||||
SpinLocker locker(cpu->irqs_lock);
|
||||
|
||||
irq_assignment* chosen = NULL;
|
||||
irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs);
|
||||
|
||||
int32 totalLoad = 0;
|
||||
while (irq != NULL) {
|
||||
if (chosen == NULL || chosen->load < irq->load)
|
||||
chosen = irq;
|
||||
totalLoad += irq->load;
|
||||
irq = (irq_assignment*)list_get_next_item(&cpu->irqs, irq);
|
||||
}
|
||||
|
||||
locker.Unlock();
|
||||
|
||||
if (chosen == NULL || totalLoad < kLowLoad)
|
||||
return;
|
||||
|
||||
SpinLocker coreLocker(gCoreHeapsLock);
|
||||
CoreEntry* other = gCoreLoadHeap->PeekMinimum();
|
||||
if (other == NULL)
|
||||
other = gCoreHighLoadHeap->PeekMinimum();
|
||||
coreLocker.Unlock();
|
||||
|
||||
ASSERT(other != NULL);
|
||||
|
||||
int32 thigCore = gCPUToCore[smp_get_current_cpu()];
|
||||
if (other->fCoreID == thigCore)
|
||||
return;
|
||||
|
||||
if (other->fLoad + kLoadDifference >= gCoreEntries[thigCore].fLoad)
|
||||
return;
|
||||
|
||||
coreLocker.Lock();
|
||||
gCPUPriorityHeaps[other->fCoreID].PeekMinimum();
|
||||
}
|
||||
|
||||
|
||||
scheduler_mode_operations gSchedulerLowLatencyMode = {
|
||||
"low latency",
|
||||
|
||||
true,
|
||||
|
||||
switch_to_mode,
|
||||
has_cache_expired,
|
||||
choose_core,
|
||||
should_rebalance,
|
||||
rebalance_irqs,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
/*
|
||||
* Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org.
|
||||
* Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
|
||||
#include <util/AutoLock.h>
|
||||
|
||||
#include "scheduler_common.h"
|
||||
#include "scheduler_modes.h"
|
||||
|
||||
|
||||
using namespace Scheduler;
|
||||
|
||||
|
||||
static bigtime_t sDisableSmallTaskPacking;
|
||||
static int32 sSmallTaskCore;
|
||||
|
||||
|
||||
static bool
|
||||
has_cache_expired(Thread* thread)
|
||||
{
|
||||
ASSERT(!gSingleCore);
|
||||
|
||||
if (thread_is_idle_thread(thread))
|
||||
return false;
|
||||
|
||||
scheduler_thread_data* schedulerThreadData = thread->scheduler_data;
|
||||
ASSERT(schedulerThreadData->previous_core >= 0);
|
||||
|
||||
CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core];
|
||||
return system_time() - schedulerThreadData->went_sleep > kCacheExpire;
|
||||
}
|
||||
|
||||
|
||||
static inline bool
|
||||
is_small_task_packing_enabled(void)
|
||||
{
|
||||
if (sDisableSmallTaskPacking == -1)
|
||||
return false;
|
||||
return sDisableSmallTaskPacking < system_time();
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
disable_small_task_packing(void)
|
||||
{
|
||||
ASSERT(!gSingleCore);
|
||||
|
||||
ASSERT(is_small_task_packing_enabled());
|
||||
ASSERT(sSmallTaskCore == gCPUToCore[smp_get_current_cpu()]);
|
||||
|
||||
sDisableSmallTaskPacking = system_time() + kThreadQuantum * 100;
|
||||
sSmallTaskCore = -1;
|
||||
}
|
||||
|
||||
|
||||
static inline bool
|
||||
is_task_small(Thread* thread)
|
||||
{
|
||||
return thread->scheduler_data->load <= 200;
|
||||
}
|
||||
|
||||
|
||||
static void
|
||||
switch_to_mode(void)
|
||||
{
|
||||
sDisableSmallTaskPacking = -1;
|
||||
sSmallTaskCore = -1;
|
||||
}
|
||||
|
||||
|
||||
static int32
|
||||
choose_core(Thread* thread)
|
||||
{
|
||||
CoreEntry* entry;
|
||||
|
||||
if (is_small_task_packing_enabled() && is_task_small(thread)
|
||||
&& gCoreLoadHeap->PeekMaximum() != NULL) {
|
||||
// try to pack all threads on one core
|
||||
if (sSmallTaskCore < 0)
|
||||
sSmallTaskCore = gCoreLoadHeap->PeekMaximum()->fCoreID;
|
||||
entry = &gCoreEntries[sSmallTaskCore];
|
||||
} else if (gCoreLoadHeap->PeekMinimum() != NULL) {
|
||||
// run immediately on already woken core
|
||||
entry = gCoreLoadHeap->PeekMinimum();
|
||||
} else if (gPackageUsageHeap->PeekMinimum() != NULL) {
|
||||
// wake new core
|
||||
PackageEntry* package = gPackageUsageHeap->PeekMinimum();
|
||||
entry = package->fIdleCores.Last();
|
||||
} else if (gIdlePackageList->Last() != NULL) {
|
||||
// wake new package
|
||||
PackageEntry* package = gIdlePackageList->Last();
|
||||
entry = package->fIdleCores.Last();
|
||||
} else {
|
||||
// no idle cores, use least occupied core
|
||||
entry = gCoreLoadHeap->PeekMinimum();
|
||||
if (entry == NULL)
|
||||
entry = gCoreHighLoadHeap->PeekMinimum();
|
||||
}
|
||||
|
||||
ASSERT(entry != NULL);
|
||||
return entry->fCoreID;
|
||||
}
|
||||
|
||||
|
||||
static bool
|
||||
should_rebalance(Thread* thread)
|
||||
{
|
||||
ASSERT(!gSingleCore);
|
||||
|
||||
if (thread_is_idle_thread(thread))
|
||||
return false;
|
||||
|
||||
scheduler_thread_data* schedulerThreadData = thread->scheduler_data;
|
||||
ASSERT(schedulerThreadData->previous_core >= 0);
|
||||
|
||||
int32 core = schedulerThreadData->previous_core;
|
||||
CoreEntry* coreEntry = &gCoreEntries[core];
|
||||
|
||||
// If the thread produces more than 50% of the load, leave it here. In
|
||||
// such situation it is better to move other threads away.
|
||||
// Unless we are trying to pack small tasks here, in such case get rid
|
||||
// of CPU hungry thread and continue packing.
|
||||
if (schedulerThreadData->load >= coreEntry->fLoad / 2)
|
||||
return is_small_task_packing_enabled() && sSmallTaskCore == core;
|
||||
|
||||
// All cores try to give us small tasks, check whether we have enough.
|
||||
if (is_small_task_packing_enabled() && sSmallTaskCore == core) {
|
||||
if (coreEntry->fLoad > kHighLoad) {
|
||||
if (!is_task_small(thread))
|
||||
return true;
|
||||
} else if (coreEntry->fLoad > kVeryHighLoad)
|
||||
disable_small_task_packing();
|
||||
}
|
||||
|
||||
// Try small task packing.
|
||||
if (is_small_task_packing_enabled() && is_task_small(thread))
|
||||
return sSmallTaskCore != core;
|
||||
|
||||
// No cpu bound threads - the situation is quite good. Make sure it
|
||||
// won't get much worse...
|
||||
SpinLocker coreLocker(gCoreHeapsLock);
|
||||
|
||||
CoreEntry* other = gCoreLoadHeap->PeekMinimum();
|
||||
if (other == NULL)
|
||||
other = gCoreHighLoadHeap->PeekMinimum();
|
||||
return coreEntry->fLoad - other->fLoad >= kLoadDifference;
|
||||
}
|
||||
|
||||
|
||||
scheduler_mode_operations gSchedulerPowerSavingMode = {
|
||||
"power saving",
|
||||
|
||||
false,
|
||||
|
||||
switch_to_mode,
|
||||
has_cache_expired,
|
||||
choose_core,
|
||||
should_rebalance,
|
||||
NULL,
|
||||
};
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,5 @@
|
||||
/*
|
||||
* Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org.
|
||||
* Copyright 2011, Ingo Weinhold, ingo_weinhold@gmx.de.
|
||||
* Distributed under the terms of the MIT License.
|
||||
*/
|
||||
@@ -6,9 +7,167 @@
|
||||
#define KERNEL_SCHEDULER_COMMON_H
|
||||
|
||||
|
||||
#include <debug.h>
|
||||
#include <kscheduler.h>
|
||||
#include <load_tracking.h>
|
||||
#include <smp.h>
|
||||
#include <thread.h>
|
||||
#include <user_debugger.h>
|
||||
#include <util/MinMaxHeap.h>
|
||||
|
||||
#include "RunQueue.h"
|
||||
|
||||
|
||||
#define CACHE_LINE_ALIGN __attribute__((aligned(64)))
|
||||
|
||||
|
||||
//#define TRACE_SCHEDULER
|
||||
#ifdef TRACE_SCHEDULER
|
||||
# define TRACE(...) dprintf_no_syslog(__VA_ARGS__)
|
||||
#else
|
||||
# define TRACE(...) do { } while (false)
|
||||
#endif
|
||||
|
||||
|
||||
namespace Scheduler {
|
||||
|
||||
|
||||
const bigtime_t kThreadQuantum = 1000;
|
||||
const bigtime_t kMinThreadQuantum = 3000;
|
||||
const bigtime_t kMaxThreadQuantum = 10000;
|
||||
|
||||
const bigtime_t kMinimalWaitTime = kThreadQuantum / 4;
|
||||
|
||||
const bigtime_t kCacheExpire = 100000;
|
||||
|
||||
const int kTargetLoad = kMaxLoad * 55 / 100;
|
||||
const int kHighLoad = kMaxLoad * 70 / 100;
|
||||
const int kVeryHighLoad = (kMaxLoad + kHighLoad) / 2;
|
||||
const int kLoadDifference = kMaxLoad * 20 / 100;
|
||||
const int kLowLoad = kLoadDifference / 2;
|
||||
|
||||
extern bool gSingleCore;
|
||||
|
||||
// Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical
|
||||
// processors in the heap belong to. Since there are no cache affinity issues
|
||||
// at this level and the run queue is shared among all logical processors on
|
||||
// the core the only real concern is to make lower priority threads give way to
|
||||
// the higher priority threads.
|
||||
struct CPUEntry : public MinMaxHeapLinkImpl<CPUEntry, int32> {
|
||||
CPUEntry();
|
||||
|
||||
int32 fCPUNumber;
|
||||
|
||||
int32 fPriority;
|
||||
|
||||
bigtime_t fMeasureActiveTime;
|
||||
bigtime_t fMeasureTime;
|
||||
|
||||
int32 fLoad;
|
||||
} CACHE_LINE_ALIGN;
|
||||
typedef MinMaxHeap<CPUEntry, int32> CPUHeap CACHE_LINE_ALIGN;
|
||||
|
||||
extern CPUEntry* gCPUEntries;
|
||||
extern CPUHeap* gCPUPriorityHeaps;
|
||||
|
||||
struct CoreEntry : public MinMaxHeapLinkImpl<CoreEntry, int32>,
|
||||
DoublyLinkedListLinkImpl<CoreEntry> {
|
||||
CoreEntry();
|
||||
|
||||
int32 fCoreID;
|
||||
|
||||
spinlock fLock;
|
||||
|
||||
bigtime_t fStartedBottom;
|
||||
bigtime_t fReachedBottom;
|
||||
bigtime_t fStartedIdle;
|
||||
bigtime_t fReachedIdle;
|
||||
|
||||
bigtime_t fActiveTime;
|
||||
|
||||
int32 fLoad;
|
||||
} CACHE_LINE_ALIGN;
|
||||
typedef MinMaxHeap<CoreEntry, int32> CoreLoadHeap;
|
||||
|
||||
extern CoreEntry* gCoreEntries;
|
||||
extern CoreLoadHeap* gCoreLoadHeap;
|
||||
extern CoreLoadHeap* gCoreHighLoadHeap;
|
||||
extern spinlock gCoreHeapsLock;
|
||||
|
||||
// sPackageUsageHeap is used to decide which core should be woken up from the
|
||||
// idle state. When aiming for performance we should use as many packages as
|
||||
// possible with as little cores active in each package as possible (so that the
|
||||
// package can enter any boost mode if it has one and the active core have more
|
||||
// of the shared cache for themselves. If power saving is the main priority we
|
||||
// should keep active cores on as little packages as possible (so that other
|
||||
// packages can go to the deep state of sleep). The heap stores only packages
|
||||
// with at least one core active and one core idle. The packages with all cores
|
||||
// idle are stored in sPackageIdleList (in LIFO manner).
|
||||
struct PackageEntry : public MinMaxHeapLinkImpl<PackageEntry, int32>,
|
||||
DoublyLinkedListLinkImpl<PackageEntry> {
|
||||
PackageEntry();
|
||||
|
||||
int32 fPackageID;
|
||||
|
||||
DoublyLinkedList<CoreEntry> fIdleCores;
|
||||
int32 fIdleCoreCount;
|
||||
|
||||
int32 fCoreCount;
|
||||
} CACHE_LINE_ALIGN;
|
||||
typedef MinMaxHeap<PackageEntry, int32> PackageHeap;
|
||||
typedef DoublyLinkedList<PackageEntry> IdlePackageList;
|
||||
|
||||
extern PackageEntry* gPackageEntries;
|
||||
extern PackageHeap* gPackageUsageHeap;
|
||||
extern IdlePackageList* gIdlePackageList;
|
||||
extern spinlock gIdlePackageLock;
|
||||
|
||||
// The run queues. Holds the threads ready to run ordered by priority.
|
||||
// One queue per schedulable target per core. Additionally, each
|
||||
// logical processor has its sPinnedRunQueues used for scheduling
|
||||
// pinned threads.
|
||||
typedef RunQueue<Thread, THREAD_MAX_SET_PRIORITY> CACHE_LINE_ALIGN
|
||||
ThreadRunQueue;
|
||||
|
||||
extern ThreadRunQueue* gRunQueues;
|
||||
extern ThreadRunQueue* gPinnedRunQueues;
|
||||
extern int32 gRunQueueCount;
|
||||
|
||||
// Since CPU IDs used internally by the kernel bear no relation to the actual
|
||||
// CPU topology the following arrays are used to efficiently get the core
|
||||
// and the package that CPU in question belongs to.
|
||||
extern int32* gCPUToCore;
|
||||
extern int32* gCPUToPackage;
|
||||
|
||||
|
||||
} // namespace Scheduler
|
||||
|
||||
|
||||
struct scheduler_thread_data {
|
||||
inline scheduler_thread_data();
|
||||
void Init();
|
||||
|
||||
int32 priority_penalty;
|
||||
int32 additional_penalty;
|
||||
|
||||
bool lost_cpu;
|
||||
bool cpu_bound;
|
||||
|
||||
bigtime_t time_left;
|
||||
bigtime_t stolen_time;
|
||||
bigtime_t quantum_start;
|
||||
|
||||
bigtime_t measure_active_time;
|
||||
bigtime_t measure_time;
|
||||
int32 load;
|
||||
|
||||
bigtime_t went_sleep;
|
||||
bigtime_t went_sleep_active;
|
||||
|
||||
int32 previous_core;
|
||||
|
||||
bool enqueued;
|
||||
};
|
||||
|
||||
|
||||
/*! Switches the currently running thread.
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org.
|
||||
* Distributed under the terms of the MIT License.
|
||||
*/
|
||||
#ifndef KERNEL_SCHEDULER_MODES_H
|
||||
#define KERNEL_SCHEDULER_MODES_H
|
||||
|
||||
|
||||
#include <kscheduler.h>
|
||||
#include <thread_types.h>
|
||||
|
||||
|
||||
struct scheduler_mode_operations {
|
||||
const char* name;
|
||||
|
||||
bool avoid_boost;
|
||||
|
||||
void (*switch_to_mode)(void);
|
||||
bool (*has_cache_expired)(Thread* thread);
|
||||
int32 (*choose_core)(Thread* thread);
|
||||
bool (*should_rebalance)(Thread* thread);
|
||||
void (*rebalance_irqs)(bool idle);
|
||||
};
|
||||
|
||||
extern struct scheduler_mode_operations gSchedulerLowLatencyMode;
|
||||
extern struct scheduler_mode_operations gSchedulerPowerSavingMode;
|
||||
|
||||
#endif // KERNEL_SCHEDULER_MODES_H
|
||||
|
||||
Reference in New Issue
Block a user