From 3533b6597db4ad65493632da8a92c1f6ea3de149 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Thu, 10 Dec 2009 11:54:38 +0000 Subject: [PATCH] * Reintroduced the SMP_MSG_RESCHEDULE_IF_IDLE ICI message. This time implemented by means of an additional member in cpu_ent. * Removed thread::keep_scheduled and the related functions. The feature wasn't used yet and wouldn't have worked as implemented anyway. * Resurrected an older, SMP aware version of our simple scheduler and made it the default instead of the affine scheduler. The latter is in no state to be used yet. It causes enormous latencies (I've seen up to 0.1s) even when six or seven CPUs were idle at the same time, totally killing parallelism. That's also the reason why a -j8 build was slower than a -j2. This is no longer the case. On my machine the -j2 build takes about 10% less time now and the -j8 build saves another 20%. The latter is not particularly impressive (compared with Linux), but that seems to be due to lock contention. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@34615 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/private/kernel/cpu.h | 1 + headers/private/kernel/smp.h | 3 +- headers/private/kernel/thread.h | 16 +- headers/private/kernel/thread_types.h | 1 - src/system/kernel/Jamfile | 3 +- src/system/kernel/arch/x86/arch_int.cpp | 5 +- src/system/kernel/scheduler/scheduler.cpp | 6 + .../kernel/scheduler/scheduler_affine.cpp | 5 +- .../kernel/scheduler/scheduler_simple.cpp | 5 +- .../kernel/scheduler/scheduler_simple_smp.cpp | 452 ++++++++++++++++++ .../kernel/scheduler/scheduler_simple_smp.h | 12 + src/system/kernel/smp.cpp | 15 +- src/system/kernel/thread.cpp | 1 - 13 files changed, 498 insertions(+), 27 deletions(-) create mode 100644 src/system/kernel/scheduler/scheduler_simple_smp.cpp create mode 100644 src/system/kernel/scheduler/scheduler_simple_smp.h diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index b3e38b925c..8023699d19 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -43,6 +43,7 @@ typedef struct cpu_ent { jmp_buf fault_jump_buffer; bool invoke_scheduler; + bool invoke_scheduler_if_idle; bool disabled; // arch-specific stuff diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index a108bf651f..c6a69146d5 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -22,7 +22,8 @@ enum { SMP_MSG_GLOBAL_INVALIDATE_PAGES, SMP_MSG_CPU_HALT, SMP_MSG_CALL_FUNCTION, - SMP_MSG_RESCHEDULE + SMP_MSG_RESCHEDULE, + SMP_MSG_RESCHEDULE_IF_IDLE }; enum { diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index cfaca38611..f92f4c3ffb 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -197,7 +197,7 @@ thread_unblock_locked(struct thread* thread, status_t status) // wake up the thread, if it is sleeping if (thread->state == B_THREAD_WAITING) return scheduler_enqueue_in_run_queue(thread); - + return false; } @@ -229,18 +229,4 @@ thread_unpin_from_current_cpu(struct thread* thread) } -static inline void -thread_disable_scheduling(struct thread* thread) -{ - thread->keep_scheduled++; -} - - -static inline void -thread_enable_scheduling(struct thread* thread) -{ - thread->keep_scheduled--; -} - - #endif /* _THREAD_H */ diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index e672615278..5be78b1f2f 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -241,7 +241,6 @@ struct thread { struct cpu_ent *cpu; struct cpu_ent *previous_cpu; int32 pinned_to_cpu; - int32 keep_scheduled; sigset_t sig_pending; sigset_t sig_block_mask; diff --git a/src/system/kernel/Jamfile b/src/system/kernel/Jamfile index d6da3f3e1e..622286c868 100644 --- a/src/system/kernel/Jamfile +++ b/src/system/kernel/Jamfile @@ -55,8 +55,9 @@ KernelMergeObject kernel_core.o : # scheduler scheduler.cpp +# scheduler_affine.cpp scheduler_simple.cpp - scheduler_affine.cpp + scheduler_simple_smp.cpp scheduler_tracing.cpp scheduling_analysis.cpp diff --git a/src/system/kernel/arch/x86/arch_int.cpp b/src/system/kernel/arch/x86/arch_int.cpp index 2c18096814..f05d92c8c5 100644 --- a/src/system/kernel/arch/x86/arch_int.cpp +++ b/src/system/kernel/arch/x86/arch_int.cpp @@ -1010,7 +1010,10 @@ hardware_interrupt(struct iframe* frame) cpu_status state = disable_interrupts(); GRAB_THREAD_LOCK(); - scheduler_reschedule(); + if (ret == B_INVOKE_SCHEDULER || !thread->cpu->invoke_scheduler_if_idle + || thread->priority == B_IDLE_PRIORITY) { + scheduler_reschedule(); + } RELEASE_THREAD_LOCK(); restore_interrupts(state); diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index d5789d8acf..7babdf4b73 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -9,6 +9,7 @@ #include "scheduler_affine.h" #include "scheduler_simple.h" +#include "scheduler_simple_smp.h" struct scheduler_ops* gScheduler; @@ -46,8 +47,13 @@ scheduler_init(void) cpuCount != 1 ? "s" : ""); if (cpuCount > 1) { +#if 0 dprintf("scheduler_init: using affine scheduler\n"); scheduler_affine_init(); +#else + dprintf("scheduler_init: using simple SMP scheduler\n"); + scheduler_simple_smp_init(); +#endif } else { dprintf("scheduler_init: using simple scheduler\n"); scheduler_simple_init(); diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 3542bbe7fb..0643336040 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -346,11 +346,10 @@ context_switch(struct thread *fromThread, struct thread *toThread) static int32 reschedule_event(timer *unused) { - if (thread_get_current_thread()->keep_scheduled > 0) - return B_HANDLED_INTERRUPT; - // this function is called as a result of the timer event set by the // scheduler returning this causes a reschedule on the timer event + thread_get_current_thread()->cpu->invoke_scheduler = true; + thread_get_current_thread()->cpu->invoke_scheduler_if_idle = false; thread_get_current_thread()->cpu->preempted = 1; return B_INVOKE_SCHEDULER; } diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index a1c50a5945..e10a3422ba 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -178,11 +178,10 @@ context_switch(struct thread *fromThread, struct thread *toThread) static int32 reschedule_event(timer *unused) { - if (thread_get_current_thread()->keep_scheduled > 0) - return B_HANDLED_INTERRUPT; - // this function is called as a result of the timer event set by the // scheduler returning this causes a reschedule on the timer event + thread_get_current_thread()->cpu->invoke_scheduler = true; + thread_get_current_thread()->cpu->invoke_scheduler_if_idle = false; thread_get_current_thread()->cpu->preempted = 1; return B_INVOKE_SCHEDULER; } diff --git a/src/system/kernel/scheduler/scheduler_simple_smp.cpp b/src/system/kernel/scheduler/scheduler_simple_smp.cpp new file mode 100644 index 0000000000..84435beccd --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_simple_smp.cpp @@ -0,0 +1,452 @@ +/* + * Copyright 2008-2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. + * Copyright 2002, Angelo Mottola, a.mottola@libero.it. + * Distributed under the terms of the MIT License. + * + * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. + * Distributed under the terms of the NewOS License. + */ + +/*! The thread scheduler */ + + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "scheduler_tracing.h" + + +//#define TRACE_SCHEDULER +#ifdef TRACE_SCHEDULER +# define TRACE(x) dprintf x +#else +# define TRACE(x) ; +#endif + + +// The run queue. Holds the threads ready to run ordered by priority. +static struct thread *sRunQueue = NULL; +static cpu_mask_t sIdleCPUs = 0; + + +static int +_rand(void) +{ + static int next = 0; + + if (next == 0) + next = system_time(); + + next = next * 1103515245 + 12345; + return (next >> 16) & 0x7FFF; +} + + +static int +dump_run_queue(int argc, char **argv) +{ + struct thread *thread; + + thread = sRunQueue; + if (!thread) + kprintf("Run queue is empty!\n"); + else { + kprintf("thread id priority name\n"); + while (thread) { + kprintf("%p %-7ld %-8ld %s\n", thread, thread->id, + thread->priority, thread->name); + thread = thread->queue_next; + } + } + + return 0; +} + + +/*! Enqueues the thread into the run queue. + Note: thread lock must be held when entering this function +*/ +static bool +enqueue_in_run_queue(struct thread *thread) +{ + if (thread->state == B_THREAD_RUNNING) { + // The thread is currently running (on another CPU) and we cannot + // insert it into the run queue. Set the next state to ready so the + // thread is inserted into the run queue on the next reschedule. + thread->next_state = B_THREAD_READY; + return false; + } + + thread->state = thread->next_state = B_THREAD_READY; + + struct thread *curr, *prev; + for (curr = sRunQueue, prev = NULL; curr + && curr->priority >= thread->next_priority; + curr = curr->queue_next) { + if (prev) + prev = prev->queue_next; + else + prev = sRunQueue; + } + + T(EnqueueThread(thread, prev, curr)); + + thread->queue_next = curr; + if (prev) + prev->queue_next = thread; + else + sRunQueue = thread; + + thread->next_priority = thread->priority; + + if (thread->priority != B_IDLE_PRIORITY) { + int32 currentCPU = smp_get_current_cpu(); + if (sIdleCPUs != 0) { + if (thread->pinned_to_cpu > 0) { + // thread is pinned to a CPU -- notify it, if it is idle + int32 targetCPU = thread->previous_cpu->cpu_num; + if ((sIdleCPUs & (1 << targetCPU)) != 0) { + sIdleCPUs &= ~(1 << targetCPU); + smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE_IF_IDLE, 0, 0, + 0, NULL, SMP_MSG_FLAG_ASYNC); + } + } else { + // Thread is not pinned to any CPU -- take it ourselves, if we + // are idle, otherwise notify the next idle CPU. In either case + // we clear the idle bit of the chosen CPU, so that the + // enqueue_in_run_queue() won't try to bother the + // same CPU again, if invoked before it handled the interrupt. + cpu_mask_t idleCPUs = CLEAR_BIT(sIdleCPUs, currentCPU); + if ((sIdleCPUs & (1 << currentCPU)) != 0) { + sIdleCPUs = idleCPUs; + } else { + int32 targetCPU = 0; + for (; targetCPU < B_MAX_CPU_COUNT; targetCPU++) { + cpu_mask_t mask = 1 << targetCPU; + if ((idleCPUs & mask) != 0) { + sIdleCPUs &= ~mask; + break; + } + } + + smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE_IF_IDLE, 0, 0, + 0, NULL, SMP_MSG_FLAG_ASYNC); + } + } + } + } + + // notify listeners + NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, + thread); + return false; +} + + +/*! Sets the priority of a thread. + Note: thread lock must be held when entering this function +*/ +static void +set_thread_priority(struct thread *thread, int32 priority) +{ + if (priority == thread->priority) + return; + + if (thread->state != B_THREAD_READY) { + thread->priority = priority; + return; + } + + // The thread is in the run queue. We need to remove it and re-insert it at + // a new position. + + T(RemoveThread(thread)); + + // notify listeners + NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, + thread); + + // find thread in run queue + struct thread *item, *prev; + for (item = sRunQueue, prev = NULL; item && item != thread; + item = item->queue_next) { + if (prev) + prev = prev->queue_next; + else + prev = sRunQueue; + } + + ASSERT(item == thread); + + // remove the thread + if (prev) + prev->queue_next = item->queue_next; + else + sRunQueue = item->queue_next; + + // set priority and re-insert + thread->priority = thread->next_priority = priority; + enqueue_in_run_queue(thread); +} + + +static void +context_switch(struct thread *fromThread, struct thread *toThread) +{ + if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) + user_debug_thread_unscheduled(fromThread); + + toThread->previous_cpu = toThread->cpu = fromThread->cpu; + fromThread->cpu = NULL; + + arch_thread_set_current_thread(toThread); + arch_thread_context_switch(fromThread, toThread); + + // Looks weird, but is correct. fromThread had been unscheduled earlier, + // but is back now. The notification for a thread scheduled the first time + // happens in thread.cpp:thread_kthread_entry(). + if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) + user_debug_thread_scheduled(fromThread); +} + + +static int32 +reschedule_event(timer *unused) +{ + // this function is called as a result of the timer event set by the + // scheduler returning this causes a reschedule on the timer event + thread_get_current_thread()->cpu->invoke_scheduler = true; + thread_get_current_thread()->cpu->invoke_scheduler_if_idle = false; + thread_get_current_thread()->cpu->preempted = 1; + return B_INVOKE_SCHEDULER; +} + + +/*! Runs the scheduler. + Note: expects thread spinlock to be held +*/ +static void +reschedule(void) +{ + struct thread *oldThread = thread_get_current_thread(); + struct thread *nextThread, *prevThread; + + TRACE(("reschedule(): cpu %ld, cur_thread = %ld\n", smp_get_current_cpu(), thread_get_current_thread()->id)); + + oldThread->cpu->invoke_scheduler = false; + + oldThread->state = oldThread->next_state; + switch (oldThread->next_state) { + case B_THREAD_RUNNING: + case B_THREAD_READY: + TRACE(("enqueueing thread %ld into run q. pri = %ld\n", oldThread->id, oldThread->priority)); + enqueue_in_run_queue(oldThread); + break; + case B_THREAD_SUSPENDED: + TRACE(("reschedule(): suspending thread %ld\n", oldThread->id)); + break; + case THREAD_STATE_FREE_ON_RESCHED: + break; + default: + TRACE(("not enqueueing thread %ld into run q. next_state = %ld\n", oldThread->id, oldThread->next_state)); + break; + } + + nextThread = sRunQueue; + prevThread = NULL; + + if (oldThread->cpu->disabled) { + // CPU is disabled - service any threads we may have that are pinned, + // otherwise just select the idle thread + while (nextThread && nextThread->priority > B_IDLE_PRIORITY) { + if (nextThread->pinned_to_cpu > 0 && + nextThread->previous_cpu == oldThread->cpu) + break; + prevThread = nextThread; + nextThread = nextThread->queue_next; + } + } else { + while (nextThread) { + // select next thread from the run queue + while (nextThread && nextThread->priority > B_IDLE_PRIORITY) { +#if 0 + if (oldThread == nextThread && nextThread->was_yielded) { + // ignore threads that called thread_yield() once + nextThread->was_yielded = false; + prevThread = nextThread; + nextThread = nextThread->queue_next; + } +#endif + + // skip thread, if it doesn't want to run on this CPU + if (nextThread->pinned_to_cpu > 0 + && nextThread->previous_cpu != oldThread->cpu) { + prevThread = nextThread; + nextThread = nextThread->queue_next; + continue; + } + + // always extract real time threads + if (nextThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + break; + + // never skip last non-idle normal thread + if (nextThread->queue_next && nextThread->queue_next->priority == B_IDLE_PRIORITY) + break; + + // skip normal threads sometimes (roughly 20%) + if (_rand() > 0x1a00) + break; + + // skip until next lower priority + int32 priority = nextThread->priority; + do { + prevThread = nextThread; + nextThread = nextThread->queue_next; + } while (nextThread->queue_next != NULL + && priority == nextThread->queue_next->priority + && nextThread->queue_next->priority > B_IDLE_PRIORITY); + } + + if (nextThread->cpu + && nextThread->cpu->cpu_num != oldThread->cpu->cpu_num) { + panic("thread in run queue that's still running on another CPU!\n"); + // ToDo: remove this check completely when we're sure that this + // cannot happen anymore. + prevThread = nextThread; + nextThread = nextThread->queue_next; + continue; + } + + break; + } + } + + if (!nextThread) + panic("reschedule(): run queue is empty!\n"); + + // extract selected thread from the run queue + if (prevThread) + prevThread->queue_next = nextThread->queue_next; + else + sRunQueue = nextThread->queue_next; + + T(ScheduleThread(nextThread, oldThread)); + + // notify listeners + NotifySchedulerListeners(&SchedulerListener::ThreadScheduled, + oldThread, nextThread); + + nextThread->state = B_THREAD_RUNNING; + nextThread->next_state = B_THREAD_READY; + oldThread->was_yielded = false; + + // track kernel time (user time is tracked in thread_at_kernel_entry()) + bigtime_t now = system_time(); + oldThread->kernel_time += now - oldThread->last_time; + nextThread->last_time = now; + + // track CPU activity + if (!thread_is_idle_thread(oldThread)) { + oldThread->cpu->active_time += + (oldThread->kernel_time - oldThread->cpu->last_kernel_time) + + (oldThread->user_time - oldThread->cpu->last_user_time); + } + + if (!thread_is_idle_thread(nextThread)) { + oldThread->cpu->last_kernel_time = nextThread->kernel_time; + oldThread->cpu->last_user_time = nextThread->user_time; + } + + if (nextThread != oldThread || oldThread->cpu->preempted) { + bigtime_t quantum = 3000; // ToDo: calculate quantum! + timer *quantumTimer = &oldThread->cpu->quantum_timer; + + if (!oldThread->cpu->preempted) + cancel_timer(quantumTimer); + + oldThread->cpu->preempted = 0; + add_timer(quantumTimer, &reschedule_event, quantum, + B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_THREAD_LOCK); + + // update the idle bit for this CPU in the CPU mask + int32 cpuNum = smp_get_current_cpu(); + if (nextThread->priority == B_IDLE_PRIORITY) + sIdleCPUs = SET_BIT(sIdleCPUs, cpuNum); + else + sIdleCPUs = CLEAR_BIT(sIdleCPUs, cpuNum); + + if (nextThread != oldThread) + context_switch(oldThread, nextThread); + } +} + + +static void +on_thread_create(struct thread* thread) +{ + // do nothing +} + + +static void +on_thread_init(struct thread* thread) +{ + // do nothing +} + + +static void +on_thread_destroy(struct thread* thread) +{ + // do nothing +} + + +/*! This starts the scheduler. Must be run in the context of the initial idle + thread. Interrupts must be disabled and will be disabled when returning. +*/ +static void +start(void) +{ + GRAB_THREAD_LOCK(); + + reschedule(); + + RELEASE_THREAD_LOCK(); +} + + +static scheduler_ops kSimpleSMPOps = { + enqueue_in_run_queue, + reschedule, + set_thread_priority, + on_thread_create, + on_thread_init, + on_thread_destroy, + start +}; + + +// #pragma mark - + + +void +scheduler_simple_smp_init() +{ + gScheduler = &kSimpleSMPOps; + + add_debugger_command_etc("run_queue", &dump_run_queue, + "List threads in run queue", "\nLists threads in run queue", 0); +} diff --git a/src/system/kernel/scheduler/scheduler_simple_smp.h b/src/system/kernel/scheduler/scheduler_simple_smp.h new file mode 100644 index 0000000000..67b482494c --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_simple_smp.h @@ -0,0 +1,12 @@ +/* + * Copyright 2008-2009, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef KERNEL_SCHEDULER_SIMPLE_SMP_H +#define KERNEL_SCHEDULER_SIMPLE_SMP_H + + +void scheduler_simple_smp_init(); + + +#endif // KERNEL_SCHEDULER_SIMPLE_SMP_H diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index da7fd8b6f1..1cdb4432e5 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -702,8 +702,21 @@ process_pending_ici(int32 currentCPU) break; } case SMP_MSG_RESCHEDULE: - thread_get_current_thread()->cpu->invoke_scheduler = true; + { + cpu_ent* cpu = thread_get_current_thread()->cpu; + cpu->invoke_scheduler = true; + cpu->invoke_scheduler_if_idle = false; break; + } + case SMP_MSG_RESCHEDULE_IF_IDLE: + { + cpu_ent* cpu = thread_get_current_thread()->cpu; + if (!cpu->invoke_scheduler) { + cpu->invoke_scheduler = true; + cpu->invoke_scheduler_if_idle = true; + } + break; + } default: dprintf("smp_intercpu_int_handler: got unknown message %ld\n", msg->message); } diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 79eb25c482..8c3a7f06a3 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -254,7 +254,6 @@ create_thread_struct(struct thread *inthread, const char *name, thread->cpu = cpu; thread->previous_cpu = NULL; thread->pinned_to_cpu = 0; - thread->keep_scheduled = 0; thread->fault_handler = 0; thread->page_faults_allowed = 1; thread->kernel_stack_area = -1;