diff --git a/headers/private/kernel/timer.h b/headers/private/kernel/timer.h index 770de36ca5..2d76f66e58 100644 --- a/headers/private/kernel/timer.h +++ b/headers/private/kernel/timer.h @@ -15,13 +15,13 @@ extern "C" { struct kernel_args; +#define B_TIMER_ACQUIRE_THREAD_LOCK 0x8000 +#define B_TIMER_FLAGS B_TIMER_ACQUIRE_THREAD_LOCK + /* kernel functions */ status_t timer_init(struct kernel_args *); int32 timer_interrupt(void); -/* this one is only to be used by the scheduler */ -status_t _local_timer_cancel_event(int currentCPU, timer *event); - #ifdef __cplusplus } #endif diff --git a/src/system/kernel/Jamfile b/src/system/kernel/Jamfile index 11243fc18b..15d41e8fc6 100644 --- a/src/system/kernel/Jamfile +++ b/src/system/kernel/Jamfile @@ -42,7 +42,7 @@ KernelMergeObject kernel_core.o : syscalls.cpp team.cpp thread.cpp - timer.c + timer.cpp usergroup.cpp wait_for_objects.cpp diff --git a/src/system/kernel/scheduler.cpp b/src/system/kernel/scheduler.cpp index 91523d5108..4ed3bdaf6c 100644 --- a/src/system/kernel/scheduler.cpp +++ b/src/system/kernel/scheduler.cpp @@ -275,10 +275,11 @@ scheduler_reschedule(void) timer *quantumTimer = &oldThread->cpu->quantum_timer; if (!oldThread->cpu->preempted) - _local_timer_cancel_event(oldThread->cpu->cpu_num, quantumTimer); + cancel_timer(quantumTimer); oldThread->cpu->preempted = 0; - add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); + add_timer(quantumTimer, &reschedule_event, quantum, + B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_THREAD_LOCK); if (nextThread != oldThread) context_switch(oldThread, nextThread); diff --git a/src/system/kernel/timer.c b/src/system/kernel/timer.c deleted file mode 100644 index 7d5e9a2239..0000000000 --- a/src/system/kernel/timer.c +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright 2002-2008, Haiku. All rights reserved. - * Distributed under the terms of the MIT License. - * - * Copyright 2001, Travis Geiselbrecht. All rights reserved. - * Distributed under the terms of the NewOS License. - */ - -/*! Policy info for timers */ - - -#include - -#include -#include -#include -#include - - -static timer * volatile sEvents[B_MAX_CPU_COUNT] = { NULL, }; -static spinlock sTimerSpinlock[B_MAX_CPU_COUNT] = { 0, }; - - -//#define TRACE_TIMER -#ifdef TRACE_TIMER -# define TRACE(x) dprintf x -#else -# define TRACE(x) ; -#endif - -#if __INTEL__ -# define PAUSE() asm volatile ("pause;") -#else -# define PAUSE() -#endif - - -status_t -timer_init(kernel_args *args) -{ - TRACE(("timer_init: entry\n")); - - return arch_init_timer(args); -} - - -/*! NOTE: expects interrupts to be off */ -static void -add_event_to_list(timer *event, timer * volatile *list) -{ - timer *next; - timer *last = NULL; - - // stick it in the event list - for (next = *list; next; last = next, next = (timer *)next->next) { - if ((bigtime_t)next->schedule_time >= (bigtime_t)event->schedule_time) - break; - } - - if (last != NULL) { - event->next = last->next; - last->next = event; - } else { - event->next = next; - *list = event; - } -} - - -int32 -timer_interrupt() -{ - timer *event; - spinlock *spinlock; - int currentCPU = smp_get_current_cpu(); - int32 rc = B_HANDLED_INTERRUPT; - - TRACE(("timer_interrupt: time 0x%x 0x%x, cpu %d\n", system_time(), - smp_get_current_cpu())); - - spinlock = &sTimerSpinlock[currentCPU]; - - acquire_spinlock(spinlock); - -restart_scan: - event = sEvents[currentCPU]; - if (event != NULL && ((bigtime_t)event->schedule_time < system_time())) { - // this event needs to happen - int mode = event->flags; - - sEvents[currentCPU] = (timer *)event->next; - event->schedule_time = 0; - - release_spinlock(spinlock); - - TRACE(("timer_interrupt: calling hook %p for event %p\n", event->hook, - event)); - - // call the callback - // note: if the event is not periodic, it is ok - // to delete the event structure inside the callback - if (event->hook) - rc = event->hook(event); - - acquire_spinlock(spinlock); - - if (mode == B_PERIODIC_TIMER) { - // we need to adjust it and add it back to the list - bigtime_t scheduleTime = system_time() + event->period; - if (scheduleTime == 0) { - // if we wrapped around and happen to hit zero, set - // it to one, since zero represents not scheduled - scheduleTime = 1; - } - event->schedule_time = (int64)scheduleTime; - add_event_to_list(event, &sEvents[currentCPU]); - } - - goto restart_scan; // the list may have changed - } - - // setup the next hardware timer - if (sEvents[currentCPU] != NULL) { - arch_timer_set_hardware_timer( - (bigtime_t)sEvents[currentCPU]->schedule_time - system_time()); - } - - release_spinlock(spinlock); - - return rc; -} - - -status_t -add_timer(timer *event, timer_hook hook, bigtime_t period, int32 flags) -{ - bigtime_t scheduleTime; - bigtime_t currentTime = system_time(); - cpu_status state; - int currentCPU; - - if (event == NULL || hook == NULL || period < 0) - return B_BAD_VALUE; - - TRACE(("add_timer: event %p\n", event)); - - scheduleTime = period; - if (flags != B_ONE_SHOT_ABSOLUTE_TIMER) - scheduleTime += currentTime; - if (scheduleTime == 0) - scheduleTime = 1; - - event->schedule_time = (int64)scheduleTime; - event->period = period; - event->hook = hook; - event->flags = flags; - - state = disable_interrupts(); - currentCPU = smp_get_current_cpu(); - acquire_spinlock(&sTimerSpinlock[currentCPU]); - - add_event_to_list(event, &sEvents[currentCPU]); - event->cpu = currentCPU; - - // if we were stuck at the head of the list, set the hardware timer - if (event == sEvents[currentCPU]) - arch_timer_set_hardware_timer(scheduleTime - currentTime); - - release_spinlock(&sTimerSpinlock[currentCPU]); - restore_interrupts(state); - - return B_OK; -} - - -/*! This is a fast path to be called from reschedule() and from cancel_timer(). - Must always be invoked with interrupts disabled. -*/ -status_t -_local_timer_cancel_event(int cpu, timer *event) -{ - timer *last = NULL; - timer *current; - - acquire_spinlock(&sTimerSpinlock[cpu]); - current = sEvents[cpu]; - while (current != NULL) { - if (current == event) { - // we found it - if (current == sEvents[cpu]) - sEvents[cpu] = current->next; - else - last->next = current->next; - current->next = NULL; - // break out of the whole thing - break; - } - last = current; - current = current->next; - } - - if (sEvents[cpu] == NULL) - arch_timer_clear_hardware_timer(); - else { - arch_timer_set_hardware_timer( - (bigtime_t)sEvents[cpu]->schedule_time - system_time()); - } - - release_spinlock(&sTimerSpinlock[cpu]); - - return current == event ? B_OK : B_ERROR; -} - - -bool -cancel_timer(timer *event) -{ - int currentCPU = smp_get_current_cpu(); - cpu_status state; - - TRACE(("cancel_timer: event %p\n", event)); - - state = disable_interrupts(); - - // walk through all of the cpu's timer queues - // - // We start by peeking our own queue, aiming for - // a cheap match. If this fails, we start harassing - // other cpus. - - if (_local_timer_cancel_event(currentCPU, event) < 0) { - int numCPUs = smp_get_num_cpus(); - int cpu = 0; - timer *last = NULL; - timer *current; - - for (cpu = 0; cpu < numCPUs; cpu++) { - if (cpu == currentCPU) - continue; - - acquire_spinlock(&sTimerSpinlock[cpu]); - current = sEvents[cpu]; - while (current != NULL) { - if (current == event) { - // we found it - if (current == sEvents[cpu]) - sEvents[cpu] = current->next; - else - last->next = current->next; - current->next = NULL; - - // break out of the whole thing - - release_spinlock(&sTimerSpinlock[cpu]); - restore_interrupts(state); - return (bigtime_t)event->schedule_time < system_time(); - } - last = current; - current = current->next; - } - release_spinlock(&sTimerSpinlock[cpu]); - } - } - - restore_interrupts(state); - return false; -} - - -void -spin(bigtime_t microseconds) -{ - bigtime_t time = system_time(); - - while ((system_time() - time) < microseconds) { - PAUSE(); - } -} diff --git a/src/system/kernel/timer.cpp b/src/system/kernel/timer.cpp new file mode 100644 index 0000000000..87881a0423 --- /dev/null +++ b/src/system/kernel/timer.cpp @@ -0,0 +1,310 @@ +/* + * Copyright 2002-2008, Haiku. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Copyright 2001, Travis Geiselbrecht. All rights reserved. + * Distributed under the terms of the NewOS License. + */ + +/*! Policy info for timers */ + +#include + +#include + +#include +#include +#include +#include +#include + +struct per_cpu_timer_data { + spinlock lock; + timer* volatile events; + timer* volatile current_event; + vint32 current_event_in_progress; +}; + +static per_cpu_timer_data sPerCPU[B_MAX_CPU_COUNT]; + +/* +static timer * volatile sEvents[B_MAX_CPU_COUNT] = { NULL, }; +static timer * volatile sCurrentEvents[B_MAX_CPU_COUNT] = { NULL, }; +static spinlock sCurrentEventsCompletion[B_MAX_CPU_COUNT]; +static spinlock sTimerSpinlock[B_MAX_CPU_COUNT] = { 0, }; +*/ + + +//#define TRACE_TIMER +#ifdef TRACE_TIMER +# define TRACE(x) dprintf x +#else +# define TRACE(x) ; +#endif + +#if __INTEL__ +# define PAUSE() asm volatile ("pause;") +#else +# define PAUSE() +#endif + + +status_t +timer_init(kernel_args *args) +{ + TRACE(("timer_init: entry\n")); + + return arch_init_timer(args); +} + + +/*! NOTE: expects interrupts to be off */ +static void +add_event_to_list(timer *event, timer * volatile *list) +{ + timer *next; + timer *last = NULL; + + // stick it in the event list + for (next = *list; next; last = next, next = (timer *)next->next) { + if ((bigtime_t)next->schedule_time >= (bigtime_t)event->schedule_time) + break; + } + + if (last != NULL) { + event->next = last->next; + last->next = event; + } else { + event->next = next; + *list = event; + } +} + + +int32 +timer_interrupt() +{ + timer *event; + spinlock *spinlock; + per_cpu_timer_data& cpuData = sPerCPU[smp_get_current_cpu()]; + int32 rc = B_HANDLED_INTERRUPT; + + TRACE(("timer_interrupt: time 0x%x 0x%x, cpu %d\n", system_time(), + smp_get_current_cpu())); + + spinlock = &cpuData.lock; + + acquire_spinlock(spinlock); + + event = cpuData.events; + while (event != NULL && ((bigtime_t)event->schedule_time < system_time())) { + // this event needs to happen + int mode = event->flags; + + cpuData.events = (timer *)event->next; + cpuData.current_event = event; + cpuData.current_event_in_progress = 1; + event->schedule_time = 0; + + release_spinlock(spinlock); + + TRACE(("timer_interrupt: calling hook %p for event %p\n", event->hook, + event)); + + // call the callback + // note: if the event is not periodic, it is ok + // to delete the event structure inside the callback + if (event->hook) { + bool callHook = true; + + // we may need to acquire the thread spinlock + if ((mode & B_TIMER_ACQUIRE_THREAD_LOCK) != 0) { + GRAB_THREAD_LOCK(); + + // If the event has been cancelled in the meantime, we don't + // call the hook anymore. + if (cpuData.current_event == NULL) + callHook = false; + } + + if (callHook) + rc = event->hook(event); + + if ((mode & B_TIMER_ACQUIRE_THREAD_LOCK) != 0) + RELEASE_THREAD_LOCK(); + } + + cpuData.current_event_in_progress = 0; + + acquire_spinlock(spinlock); + + if ((mode & ~B_TIMER_FLAGS) == B_PERIODIC_TIMER + && cpuData.current_event != NULL) { + // we need to adjust it and add it back to the list + bigtime_t scheduleTime = system_time() + event->period; + if (scheduleTime == 0) { + // if we wrapped around and happen to hit zero, set + // it to one, since zero represents not scheduled + scheduleTime = 1; + } + event->schedule_time = (int64)scheduleTime; + add_event_to_list(event, &cpuData.events); + } + + cpuData.current_event = NULL; + + event = cpuData.events; + } + + // setup the next hardware timer + if (cpuData.events != NULL) { + arch_timer_set_hardware_timer( + (bigtime_t)cpuData.events->schedule_time - system_time()); + } + + release_spinlock(spinlock); + + return rc; +} + + +status_t +add_timer(timer *event, timer_hook hook, bigtime_t period, int32 flags) +{ + bigtime_t scheduleTime; + bigtime_t currentTime = system_time(); + cpu_status state; + + if (event == NULL || hook == NULL || period < 0) + return B_BAD_VALUE; + + TRACE(("add_timer: event %p\n", event)); + + scheduleTime = period; + if ((flags & ~B_TIMER_FLAGS) != B_ONE_SHOT_ABSOLUTE_TIMER) + scheduleTime += currentTime; + if (scheduleTime == 0) + scheduleTime = 1; + + event->schedule_time = (int64)scheduleTime; + event->period = period; + event->hook = hook; + event->flags = flags; + + state = disable_interrupts(); + int currentCPU = smp_get_current_cpu(); + per_cpu_timer_data& cpuData = sPerCPU[currentCPU]; + acquire_spinlock(&cpuData.lock); + + add_event_to_list(event, &cpuData.events); + event->cpu = currentCPU; + + // if we were stuck at the head of the list, set the hardware timer + if (event == cpuData.events) + arch_timer_set_hardware_timer(scheduleTime - currentTime); + + release_spinlock(&cpuData.lock); + restore_interrupts(state); + + return B_OK; +} + + +bool +cancel_timer(timer *event) +{ + TRACE(("cancel_timer: event %p\n", event)); + + InterruptsLocker _; + + // lock the right CPU spinlock + int cpu = event->cpu; + SpinLocker spinLocker; + while (true) { + if (cpu >= B_MAX_CPU_COUNT) + return false; + + spinLocker.SetTo(sPerCPU[cpu].lock, false); + if (cpu == event->cpu) + break; + + // cpu field changed while we were trying to lock + spinLocker.Unlock(); + cpu = event->cpu; + } + + per_cpu_timer_data& cpuData = sPerCPU[cpu]; + + timer *current = cpuData.events; + + if (event != cpuData.current_event) { + // The timer hook is not yet being executed. + timer *last = NULL; + + while (current != NULL) { + if (current == event) { + // we found it + if (current == cpuData.events) + cpuData.events = current->next; + else + last->next = current->next; + current->next = NULL; + // break out of the whole thing + break; + } + last = current; + current = current->next; + } + + // If not found, we assume this was a one-shot timer and has already + // fired. + if (current == NULL) + return true; + + // invalidate CPU field + event->cpu = 0xffff; + + // If on the current CPU, also reset the hardware timer. + if (cpu == smp_get_current_cpu()) { + if (cpuData.events == NULL) + arch_timer_clear_hardware_timer(); + else { + arch_timer_set_hardware_timer( + (bigtime_t)cpuData.events->schedule_time - system_time()); + } + } + + return false; + } else { + // The timer hook is currently being executed. We clear the current + // event so that timer_interrupt() will not reschedule periodic timers. + cpuData.current_event = NULL; + current = event; + + // Unless this is a kernel-private timer that also requires the thread + // lock to be held while calling the event hook, we'll have to wait + // for the hook to complete. When called from the timer hook we don't + // wait either, of course. + if ((event->flags & B_TIMER_ACQUIRE_THREAD_LOCK) == 0 + || cpu == smp_get_current_cpu()) { + spinLocker.Unlock(); + + while (cpuData.current_event_in_progress == 1) { + // spin + } + } + + return true; + } +} + + +void +spin(bigtime_t microseconds) +{ + bigtime_t time = system_time(); + + while ((system_time() - time) < microseconds) { + PAUSE(); + } +}