From 1c8de8581b66c14ea94bccd7ddcea99291955796 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Mon, 2 Jun 2008 02:04:12 +0000 Subject: [PATCH] * Added optional spinlock contention measurement feature. Enabled when B_DEBUG_SPINLOCK_CONTENTION is defined to 1. It typedefs spinlock to a structure (thus breaking BeOS binary compatibility), containing a counter which is incremented whenever a thread has to wait for the spinlock. * Added macros for spinlock initialization and access and changed code using spinlocks accordingly. This breaks compilation for BeOS -- the macros should be defined in the respective compatibility wrappers. * Added generic syscall to get the spinlock counters for the thread and the team spinlocks. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@25752 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- headers/os/drivers/KernelExport.h | 27 ++++- headers/private/kernel/smp.h | 1 + headers/private/system/spinlock_contention.h | 21 ++++ src/add-ons/kernel/bus_managers/ide/ide_sim.c | 2 +- .../pci/arch/x86/pci_controller.c | 2 +- .../kernel/bus_managers/ps2/packet_buffer.cpp | 2 +- .../kernel/bus_managers/scsi/scsi_lock.h | 2 +- .../kernel/busses/scsi/ahci/ahci_port.cpp | 2 +- src/add-ons/kernel/busses/usb/ehci.cpp | 2 +- src/add-ons/kernel/busses/usb/ohci.cpp | 2 +- src/add-ons/kernel/busses/usb/uhci.cpp | 2 +- .../kernel/drivers/audio/ac97/auich/util.c | 2 +- .../kernel/drivers/audio/ac97/auvia/util.c | 2 +- .../kernel/drivers/audio/emuxki/util.c | 2 +- .../kernel/drivers/graphics/radeon/irq.c | 2 +- .../kernel/drivers/network/rtl8169/device.c | 4 +- .../kernel/drivers/network/rtl8169/timer.c | 2 +- src/add-ons/kernel/generic/dpc/dpc.c | 2 +- src/add-ons/kernel/generic/mpu401/mpu401.c | 2 +- src/libs/compat/freebsd_network/taskqueue.c | 4 +- .../m68k/arch_vm_translation_map_impl.cpp | 2 +- .../kernel/arch/x86/arch_debug_console.c | 2 +- .../arch/x86/arch_vm_translation_map.cpp | 2 +- src/system/kernel/debug/debug.cpp | 2 +- src/system/kernel/debug/debug_commands.cpp | 2 +- src/system/kernel/debug/user_debugger.cpp | 2 +- src/system/kernel/int.c | 6 +- src/system/kernel/main.cpp | 1 + src/system/kernel/port.cpp | 2 +- src/system/kernel/sem.cpp | 2 +- src/system/kernel/smp.c | 109 ++++++++++++++++-- src/system/kernel/team.cpp | 2 +- src/system/kernel/thread.cpp | 2 +- 33 files changed, 179 insertions(+), 44 deletions(-) create mode 100644 headers/private/system/spinlock_contention.h diff --git a/headers/os/drivers/KernelExport.h b/headers/os/drivers/KernelExport.h index 2f419518ef..3a88928a65 100644 --- a/headers/os/drivers/KernelExport.h +++ b/headers/os/drivers/KernelExport.h @@ -13,7 +13,32 @@ /* interrupts and spinlocks */ typedef ulong cpu_status; -typedef vint32 spinlock; + +// WARNING: For Haiku debugging only! This changes the spinlock type in a +// binary incompatible way! +//#define B_DEBUG_SPINLOCK_CONTENTION 1 + +#if B_DEBUG_SPINLOCK_CONTENTION + typedef struct { + vint32 lock; + vint32 count_low; + vint32 count_high; + } spinlock; + +# define B_SPINLOCK_INITIALIZER { 0, 0, 0 } +# define B_INITIALIZE_SPINLOCK(spinlock) do { \ + (spinlock)->lock = 0; \ + (spinlock)->count_low = 0; \ + (spinlock)->count_high = 0; \ + } while (false) +# define B_SPINLOCK_IS_LOCKED(spinlock) ((spinlock)->lock > 0) +#else + typedef vint32 spinlock; + +# define B_SPINLOCK_INITIALIZER 0 +# define B_INITIALIZE_SPINLOCK(lock) do { *(lock) = 0; } while (false) +# define B_SPINLOCK_IS_LOCKED(lock) (*(lock) > 0) +#endif /* interrupt handling support for device drivers */ diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 80e6954fff..f11d0ba424 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -40,6 +40,7 @@ extern "C" { status_t smp_init(struct kernel_args *args); status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu); +status_t smp_init_post_generic_syscalls(void); bool smp_trap_non_boot_cpus(int32 cpu); void smp_wake_up_non_boot_cpus(void); void smp_cpu_rendezvous(volatile uint32 *var, int current_cpu); diff --git a/headers/private/system/spinlock_contention.h b/headers/private/system/spinlock_contention.h new file mode 100644 index 0000000000..0eb03dd040 --- /dev/null +++ b/headers/private/system/spinlock_contention.h @@ -0,0 +1,21 @@ +/* + * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef _SYSTEM_SPINLOCK_CONTENTION_H +#define _SYSTEM_SPINLOCK_CONTENTION_H + +#include + + +#define SPINLOCK_CONTENTION "spinlock contention" +#define GET_SPINLOCK_CONTENTION_INFO 0x01 + + +typedef struct spinlock_contention_info { + uint64 thread_spinlock_counter; + uint64 team_spinlock_counter; +} spinlock_contention_info; + + +#endif /* _SYSTEM_SPINLOCK_CONTENTION_H */ diff --git a/src/add-ons/kernel/bus_managers/ide/ide_sim.c b/src/add-ons/kernel/bus_managers/ide/ide_sim.c index 8ada3b5b20..3db1f5b491 100644 --- a/src/add-ons/kernel/bus_managers/ide/ide_sim.c +++ b/src/add-ons/kernel/bus_managers/ide/ide_sim.c @@ -521,7 +521,7 @@ ide_sim_init_bus(device_node *node, void **cookie) memset(bus, 0, sizeof(*bus)); bus->node = node; - bus->lock = 0; + B_INITIALIZE_SPINLOCK(&bus->lock); bus->num_running_reqs = 0; bus->active_qrequest = NULL; bus->disconnected = false; diff --git a/src/add-ons/kernel/bus_managers/pci/arch/x86/pci_controller.c b/src/add-ons/kernel/bus_managers/pci/arch/x86/pci_controller.c index cd1797d4b2..98b5063a89 100644 --- a/src/add-ons/kernel/bus_managers/pci/arch/x86/pci_controller.c +++ b/src/add-ons/kernel/bus_managers/pci/arch/x86/pci_controller.c @@ -32,7 +32,7 @@ restore_interrupts(cpu_status); \ } -spinlock sConfigLock = 0; +spinlock sConfigLock = B_SPINLOCK_INITIALIZER; static status_t pci_mech1_read_config(void *cookie, uint8 bus, uint8 device, uint8 function, diff --git a/src/add-ons/kernel/bus_managers/ps2/packet_buffer.cpp b/src/add-ons/kernel/bus_managers/ps2/packet_buffer.cpp index 6bf52e08d2..7911c1e0c7 100644 --- a/src/add-ons/kernel/bus_managers/ps2/packet_buffer.cpp +++ b/src/add-ons/kernel/bus_managers/ps2/packet_buffer.cpp @@ -40,7 +40,7 @@ create_packet_buffer(size_t size) free(buffer); return NULL; } - buffer->lock = 0; + B_INITIALIZE_SPINLOCK(&buffer->lock); return buffer; } diff --git a/src/add-ons/kernel/bus_managers/scsi/scsi_lock.h b/src/add-ons/kernel/bus_managers/scsi/scsi_lock.h index 7913673b25..8dd6776222 100644 --- a/src/add-ons/kernel/bus_managers/scsi/scsi_lock.h +++ b/src/add-ons/kernel/bus_managers/scsi/scsi_lock.h @@ -29,7 +29,7 @@ typedef struct spinlock_irq { static inline void spinlock_irq_init(spinlock_irq *lock) { - lock->lock = 0; + B_INITIALIZE_SPINLOCK(&lock->lock); } static inline void diff --git a/src/add-ons/kernel/busses/scsi/ahci/ahci_port.cpp b/src/add-ons/kernel/busses/scsi/ahci/ahci_port.cpp index b16562ae0f..5e2211f968 100644 --- a/src/add-ons/kernel/busses/scsi/ahci/ahci_port.cpp +++ b/src/add-ons/kernel/busses/scsi/ahci/ahci_port.cpp @@ -30,7 +30,6 @@ AHCIPort::AHCIPort(AHCIController *controller, int index) , fIndex(index) , fRegs(&controller->fRegs->port[index]) , fArea(-1) - , fSpinlock(0) , fCommandsActive(0) , fRequestSem(-1) , fResponseSem(-1) @@ -42,6 +41,7 @@ AHCIPort::AHCIPort(AHCIController *controller, int index) , fResetPort(false) , fError(false) { + B_INITIALIZE_SPINLOCK(&fSpinlock); fRequestSem = create_sem(1, "ahci request"); fResponseSem = create_sem(0, "ahci response"); } diff --git a/src/add-ons/kernel/busses/usb/ehci.cpp b/src/add-ons/kernel/busses/usb/ehci.cpp index bc4402a4f4..6822a5af2d 100644 --- a/src/add-ons/kernel/busses/usb/ehci.cpp +++ b/src/add-ons/kernel/busses/usb/ehci.cpp @@ -782,7 +782,7 @@ EHCI::InterruptHandler(void *data) int32 EHCI::Interrupt() { - static spinlock lock = 0; + static spinlock lock = B_SPINLOCK_INITIALIZER; acquire_spinlock(&lock); // check if any interrupt was generated diff --git a/src/add-ons/kernel/busses/usb/ohci.cpp b/src/add-ons/kernel/busses/usb/ohci.cpp index d16acfe986..e6f6f1bdb5 100644 --- a/src/add-ons/kernel/busses/usb/ohci.cpp +++ b/src/add-ons/kernel/busses/usb/ohci.cpp @@ -694,7 +694,7 @@ OHCI::_InterruptHandler(void *data) int32 OHCI::_Interrupt() { - static spinlock lock = 0; + static spinlock lock = B_SPINLOCK_INITIALIZER; acquire_spinlock(&lock); uint32 status = 0; diff --git a/src/add-ons/kernel/busses/usb/uhci.cpp b/src/add-ons/kernel/busses/usb/uhci.cpp index 89123887b0..2100689368 100644 --- a/src/add-ons/kernel/busses/usb/uhci.cpp +++ b/src/add-ons/kernel/busses/usb/uhci.cpp @@ -1531,7 +1531,7 @@ UHCI::InterruptHandler(void *data) int32 UHCI::Interrupt() { - static spinlock lock = 0; + static spinlock lock = B_SPINLOCK_INITIALIZER; acquire_spinlock(&lock); // Check if we really had an interrupt diff --git a/src/add-ons/kernel/drivers/audio/ac97/auich/util.c b/src/add-ons/kernel/drivers/audio/ac97/auich/util.c index 59fd55d6f8..8bde0fed20 100644 --- a/src/add-ons/kernel/drivers/audio/ac97/auich/util.c +++ b/src/add-ons/kernel/drivers/audio/ac97/auich/util.c @@ -34,7 +34,7 @@ #include "debug.h" #include "util.h" -spinlock slock = 0; +spinlock slock = B_SPINLOCK_INITIALIZER; uint32 round_to_pagesize(uint32 size); diff --git a/src/add-ons/kernel/drivers/audio/ac97/auvia/util.c b/src/add-ons/kernel/drivers/audio/ac97/auvia/util.c index d71521def3..fd8dade493 100644 --- a/src/add-ons/kernel/drivers/audio/ac97/auvia/util.c +++ b/src/add-ons/kernel/drivers/audio/ac97/auvia/util.c @@ -34,7 +34,7 @@ #include "debug.h" #include "util.h" -spinlock slock = 0; +spinlock slock = B_SPINLOCK_INITIALIZER; uint32 round_to_pagesize(uint32 size); diff --git a/src/add-ons/kernel/drivers/audio/emuxki/util.c b/src/add-ons/kernel/drivers/audio/emuxki/util.c index d71521def3..fd8dade493 100644 --- a/src/add-ons/kernel/drivers/audio/emuxki/util.c +++ b/src/add-ons/kernel/drivers/audio/emuxki/util.c @@ -34,7 +34,7 @@ #include "debug.h" #include "util.h" -spinlock slock = 0; +spinlock slock = B_SPINLOCK_INITIALIZER; uint32 round_to_pagesize(uint32 size); diff --git a/src/add-ons/kernel/drivers/graphics/radeon/irq.c b/src/add-ons/kernel/drivers/graphics/radeon/irq.c index 740e529305..d2b1465acc 100644 --- a/src/add-ons/kernel/drivers/graphics/radeon/irq.c +++ b/src/add-ons/kernel/drivers/graphics/radeon/irq.c @@ -237,7 +237,7 @@ Radeon_SetupIRQ(device_info *di, char *buffer) goto err3; } - di->cap_spinlock = 0; + B_INITIALIZE_SPINLOCK(&di->cap_spinlock); sprintf(buffer, "%04X_%04X_%02X%02X%02X DMA I", di->pcii.vendor_id, di->pcii.device_id, diff --git a/src/add-ons/kernel/drivers/network/rtl8169/device.c b/src/add-ons/kernel/drivers/network/rtl8169/device.c index e1f8dc9c85..4b947992ec 100644 --- a/src/add-ons/kernel/drivers/network/rtl8169/device.c +++ b/src/add-ons/kernel/drivers/network/rtl8169/device.c @@ -467,7 +467,7 @@ rtl8169_open(const char *name, uint32 flags, void** cookie) read_settings(device); device->rxBuf = (void **)malloc(sizeof(void *) * device->rxBufferCount); - device->rxSpinlock = 0; + B_INITIALIZE_SPINLOCK(&device->rxSpinlock); device->rxNextIndex = 0; device->rxIntIndex = 0; device->rxFree = device->rxBufferCount; @@ -475,7 +475,7 @@ rtl8169_open(const char *name, uint32 flags, void** cookie) set_sem_owner(device->rxReadySem, B_SYSTEM_TEAM); device->txBuf = (void **)malloc(sizeof(void *) * device->txBufferCount); - device->txSpinlock = 0; + B_INITIALIZE_SPINLOCK(&device->txSpinlock); device->txNextIndex = 0; device->txIntIndex = 0; device->txUsed = 0; diff --git a/src/add-ons/kernel/drivers/network/rtl8169/timer.c b/src/add-ons/kernel/drivers/network/rtl8169/timer.c index e7c3946de0..3618b3d70c 100644 --- a/src/add-ons/kernel/drivers/network/rtl8169/timer.c +++ b/src/add-ons/kernel/drivers/network/rtl8169/timer.c @@ -187,7 +187,7 @@ initialize_timer(void) { sTimerCount = 0; sTimerNextId = 1; - sTimerSpinlock = 0; + B_INITIALIZE_SPINLOCK(&sTimerSpinlock); sTimerThread = spawn_kernel_thread(timer_thread, "rtl8169 timer", 80, 0); sTimerSem = create_sem(0, "rtl8169 timer"); diff --git a/src/add-ons/kernel/generic/dpc/dpc.c b/src/add-ons/kernel/generic/dpc/dpc.c index 19fcde2477..bfa3dde3c7 100644 --- a/src/add-ons/kernel/generic/dpc/dpc.c +++ b/src/add-ons/kernel/generic/dpc/dpc.c @@ -91,7 +91,7 @@ new_dpc_queue(void **handle, const char *name, int32 priority) queue->head = queue->tail = 0; queue->size = DPC_QUEUE_SIZE; queue->count = 0; - queue->lock = 0; // Init the spinlock + B_INITIALIZE_SPINLOCK(&queue->lock); // Init the spinlock #ifdef __HAIKU__ snprintf(str, sizeof(str), "%.*s_wakeup_sem", diff --git a/src/add-ons/kernel/generic/mpu401/mpu401.c b/src/add-ons/kernel/generic/mpu401/mpu401.c index 91c5138ffc..300df0ffe7 100644 --- a/src/add-ons/kernel/generic/mpu401/mpu401.c +++ b/src/add-ons/kernel/generic/mpu401/mpu401.c @@ -480,7 +480,7 @@ _EXPORT generic_mpu401_module *modules[] = NULL }; -spinlock locked = 0; +spinlock locked = B_SPINLOCK_INITIALIZER; cpu_status lock(void) { diff --git a/src/libs/compat/freebsd_network/taskqueue.c b/src/libs/compat/freebsd_network/taskqueue.c index 0d39516e91..d998b98f98 100644 --- a/src/libs/compat/freebsd_network/taskqueue.c +++ b/src/libs/compat/freebsd_network/taskqueue.c @@ -20,7 +20,7 @@ struct taskqueue { taskqueue_enqueue_fn tq_enqueue; void *tq_arg; int tq_fast; - int32 tq_spinlock; + spinlock tq_spinlock; sem_id tq_sem; thread_id *tq_threads; thread_id tq_thread_storage; @@ -43,7 +43,7 @@ _taskqueue_create(const char *name, int mflags, int fast, tq->tq_fast = fast; if (fast) { - tq->tq_spinlock = 0; + B_INITIALIZE_SPINLOCK(&tq->tq_spinlock); } else { mutex_init_etc(&tq->tq_mutex, name, MUTEX_FLAG_CLONE_NAME); } diff --git a/src/system/kernel/arch/m68k/arch_vm_translation_map_impl.cpp b/src/system/kernel/arch/m68k/arch_vm_translation_map_impl.cpp index 912f923ff0..ff59eb52f4 100644 --- a/src/system/kernel/arch/m68k/arch_vm_translation_map_impl.cpp +++ b/src/system/kernel/arch/m68k/arch_vm_translation_map_impl.cpp @@ -1180,7 +1180,7 @@ arch_vm_translation_map_init(kernel_args *args) sQueryDesc.type = DT_INVALID; - tmap_list_lock = 0; + B_INITIALIZE_SPINLOCK(&tmap_list_lock); tmap_list = NULL; // allocate some space to hold physical page mapping info diff --git a/src/system/kernel/arch/x86/arch_debug_console.c b/src/system/kernel/arch/x86/arch_debug_console.c index 067736fdbe..3e35bb935e 100644 --- a/src/system/kernel/arch/x86/arch_debug_console.c +++ b/src/system/kernel/arch/x86/arch_debug_console.c @@ -99,7 +99,7 @@ static bool sKeyboardHandlerInstalled = false; static bool sBochsOutput = false; #endif -static spinlock sSerialOutputSpinlock = 0; +static spinlock sSerialOutputSpinlock = B_SPINLOCK_INITIALIZER; static void diff --git a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp index 5eaceca2d2..585dc04dca 100644 --- a/src/system/kernel/arch/x86/arch_vm_translation_map.cpp +++ b/src/system/kernel/arch/x86/arch_vm_translation_map.cpp @@ -852,7 +852,7 @@ arch_vm_translation_map_init(kernel_args *args) sKernelPhysicalPageDirectory = (page_directory_entry *)args->arch_args.phys_pgdir; sKernelVirtualPageDirectory = (page_directory_entry *)args->arch_args.vir_pgdir; - tmap_list_lock = 0; + B_INITIALIZE_SPINLOCK(&tmap_list_lock); tmap_list = NULL; // allocate some space to hold physical page mapping info diff --git a/src/system/kernel/debug/debug.cpp b/src/system/kernel/debug/debug.cpp index 0691dd1573..9a53a13ee0 100644 --- a/src/system/kernel/debug/debug.cpp +++ b/src/system/kernel/debug/debug.cpp @@ -54,7 +54,7 @@ static bool sBlueScreenEnabled = false; // must always be false on startup static bool sDebugScreenEnabled = false; static bool sBlueScreenOutput = true; -static spinlock sSpinlock = 0; +static spinlock sSpinlock = B_SPINLOCK_INITIALIZER; static int32 sDebuggerOnCPU = -1; static sem_id sSyslogNotify = -1; diff --git a/src/system/kernel/debug/debug_commands.cpp b/src/system/kernel/debug/debug_commands.cpp index a6bd7d211c..ea42af7f64 100644 --- a/src/system/kernel/debug/debug_commands.cpp +++ b/src/system/kernel/debug/debug_commands.cpp @@ -30,7 +30,7 @@ static const int32 kMaxInvokeCommandDepth = 5; static const int32 kOutputBufferSize = 1024; -static spinlock sSpinlock = 0; +static spinlock sSpinlock = B_SPINLOCK_INITIALIZER; static struct debugger_command *sCommands; diff --git a/src/system/kernel/debug/user_debugger.cpp b/src/system/kernel/debug/user_debugger.cpp index ba4316e745..af6fb00651 100644 --- a/src/system/kernel/debug/user_debugger.cpp +++ b/src/system/kernel/debug/user_debugger.cpp @@ -219,7 +219,7 @@ clear_team_debug_info(struct team_debug_info *info, bool initLock) info->debugger_write_lock = -1; if (initLock) - info->lock = 0; + B_INITIALIZE_SPINLOCK(&info->lock); } } diff --git a/src/system/kernel/int.c b/src/system/kernel/int.c index 63c82e8f45..8a609d55ab 100644 --- a/src/system/kernel/int.c +++ b/src/system/kernel/int.c @@ -62,7 +62,7 @@ dump_int_statistics(int argc, char **argv) for (i = 0; i < NUM_IO_VECTORS; i++) { struct io_handler *io; - if (sVectors[i].vector_lock == 0 + if (!B_SPINLOCK_IS_LOCKED(&sVectors[i].vector_lock) && sVectors[i].enable_count == 0 && sVectors[i].handled_count == 0 && sVectors[i].unhandled_count == 0 @@ -72,7 +72,7 @@ dump_int_statistics(int argc, char **argv) kprintf("int %3d, enabled %ld, handled %8lld, unhandled %8lld%s%s\n", i, sVectors[i].enable_count, sVectors[i].handled_count, sVectors[i].unhandled_count, - sVectors[i].vector_lock != 0 ? ", ACTIVE" : "", + B_SPINLOCK_IS_LOCKED(&sVectors[i].vector_lock) ? ", ACTIVE" : "", sVectors[i].handler_list.next == &sVectors[i].handler_list ? ", no handler" : ""); @@ -114,7 +114,7 @@ int_init_post_vm(kernel_args *args) /* initialize the vector list */ for (i = 0; i < NUM_IO_VECTORS; i++) { - sVectors[i].vector_lock = 0; /* initialize spinlock */ + B_INITIALIZE_SPINLOCK(&sVectors[i].vector_lock); sVectors[i].enable_count = 0; sVectors[i].no_lock_vector = false; #ifdef DEBUG_INT diff --git a/src/system/kernel/main.cpp b/src/system/kernel/main.cpp index 8bf022b309..957a3c1b80 100644 --- a/src/system/kernel/main.cpp +++ b/src/system/kernel/main.cpp @@ -160,6 +160,7 @@ _start(kernel_args *bootKernelArgs, int currentCPU) driver_settings_init_post_sem(&sKernelArgs); TRACE("init generic syscall\n"); generic_syscall_init(); + smp_init_post_generic_syscalls(); TRACE("init cbuf\n"); cbuf_init(); TRACE("init teams\n"); diff --git a/src/system/kernel/port.cpp b/src/system/kernel/port.cpp index 4db0f2eae4..2e372cc167 100644 --- a/src/system/kernel/port.cpp +++ b/src/system/kernel/port.cpp @@ -72,7 +72,7 @@ static bool sPortsActive = false; static port_id sNextPort = 1; static int32 sFirstFreeSlot = 1; -static spinlock sPortSpinlock = 0; +static spinlock sPortSpinlock = B_SPINLOCK_INITIALIZER; #define GRAB_PORT_LIST_LOCK() acquire_spinlock(&sPortSpinlock) #define RELEASE_PORT_LIST_LOCK() release_spinlock(&sPortSpinlock) diff --git a/src/system/kernel/sem.cpp b/src/system/kernel/sem.cpp index 732d81b647..00e61ce310 100644 --- a/src/system/kernel/sem.cpp +++ b/src/system/kernel/sem.cpp @@ -108,7 +108,7 @@ static bool sSemsActive = false; static struct sem_entry *sFreeSemsHead = NULL; static struct sem_entry *sFreeSemsTail = NULL; -static spinlock sem_spinlock = 0; +static spinlock sem_spinlock = B_SPINLOCK_INITIALIZER; #define GRAB_SEM_LIST_LOCK() acquire_spinlock(&sem_spinlock) #define RELEASE_SEM_LIST_LOCK() release_spinlock(&sem_spinlock) #define GRAB_SEM_LOCK(s) acquire_spinlock(&(s).lock) diff --git a/src/system/kernel/smp.c b/src/system/kernel/smp.c index 0fce443e5b..c26c1523e1 100644 --- a/src/system/kernel/smp.c +++ b/src/system/kernel/smp.c @@ -8,18 +8,22 @@ /* Functionality for symetrical multi-processors */ -#include -#include #include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + #define DEBUG_SPINLOCKS 1 //#define TRACE_SMP @@ -53,17 +57,17 @@ struct smp_msg { #define MAILBOX_LOCAL 1 #define MAILBOX_BCAST 2 -static spinlock boot_cpu_spin[SMP_MAX_CPUS] = { 0, }; +static spinlock boot_cpu_spin[SMP_MAX_CPUS] = { }; static struct smp_msg *free_msgs = NULL; static volatile int free_msg_count = 0; -static spinlock free_msg_spinlock = 0; +static spinlock free_msg_spinlock = B_SPINLOCK_INITIALIZER; static struct smp_msg *smp_msgs[SMP_MAX_CPUS] = { NULL, }; -static spinlock cpu_msg_spinlock[SMP_MAX_CPUS] = { 0, }; +static spinlock cpu_msg_spinlock[SMP_MAX_CPUS]; static struct smp_msg *smp_broadcast_msgs = NULL; -static spinlock broadcast_msg_spinlock = 0; +static spinlock broadcast_msg_spinlock = B_SPINLOCK_INITIALIZER; static bool sICIEnabled = false; static int32 sNumCPUs = 1; @@ -115,6 +119,10 @@ acquire_spinlock(spinlock *lock) int currentCPU = smp_get_current_cpu(); if (are_interrupts_enabled()) panic("acquire_spinlock: attempt to acquire lock %p with interrupts enabled\n", lock); +#if B_DEBUG_SPINLOCK_CONTENTION + while (atomic_add(&lock->lock, 1) != 0) + process_pending_ici(currentCPU); +#else while (1) { while (*lock != 0) { process_pending_ici(currentCPU); @@ -123,6 +131,7 @@ acquire_spinlock(spinlock *lock) if (atomic_set((int32 *)lock, 1) == 0) break; } +#endif } else { #if DEBUG_SPINLOCKS int32 oldValue; @@ -148,12 +157,17 @@ acquire_spinlock_nocheck(spinlock *lock) if (are_interrupts_enabled()) panic("acquire_spinlock_nocheck: attempt to acquire lock %p with interrupts enabled\n", lock); #endif +#if B_DEBUG_SPINLOCK_CONTENTION + while (atomic_add(&lock->lock, 1) != 0) { + } +#else while (1) { while(*lock != 0) PAUSE(); if (atomic_set((int32 *)lock, 1) == 0) break; } +#endif } else { #if DEBUG_SPINLOCKS if (are_interrupts_enabled()) @@ -171,8 +185,23 @@ release_spinlock(spinlock *lock) if (sNumCPUs > 1) { if (are_interrupts_enabled()) panic("release_spinlock: attempt to release lock %p with interrupts enabled\n", lock); +#if B_DEBUG_SPINLOCK_CONTENTION + { + int32 count = atomic_set(&lock->lock, 0) - 1; + if (count < 0) { + panic("release_spinlock: lock %p was already released\n", lock); + } else { + // add to the total count -- deal with carry manually + if ((uint32)atomic_add(&lock->count_low, count) + count + < (uint32)count) { + atomic_add(&lock->count_high, 1); + } + } + } +#else if (atomic_set((int32 *)lock, 0) != 1) panic("release_spinlock: lock %p was already released\n", lock); +#endif } else { #if DEBUG_SPINLOCKS if (are_interrupts_enabled()) @@ -405,6 +434,48 @@ process_pending_ici(int32 currentCPU) } +#if B_DEBUG_SPINLOCK_CONTENTION + +static uint64 +get_spinlock_counter(spinlock* lock) +{ + uint32 high; + uint32 low; + do { + high = (uint32)atomic_get(&lock->count_high); + low = (uint32)atomic_get(&lock->count_low); + } while (high != atomic_get(&lock->count_high)); + + return ((uint64)high << 32) | low; +} + + +static status_t +spinlock_contention_syscall(const char* subsystem, uint32 function, + void* buffer, size_t bufferSize) +{ + spinlock_contention_info info; + + if (function != GET_SPINLOCK_CONTENTION_INFO) + return B_BAD_VALUE; + + if (bufferSize < sizeof(spinlock_contention_info)) + return B_BAD_VALUE; + + info.thread_spinlock_counter = get_spinlock_counter(&thread_spinlock); + info.team_spinlock_counter = get_spinlock_counter(&team_spinlock); + + if (!IS_USER_ADDRESS(buffer) + || user_memcpy(buffer, &info, sizeof(info)) != B_OK) { + return B_BAD_ADDRESS; + } + + return B_OK; +} + +#endif // B_DEBUG_SPINLOCK_CONTENTION + + // #pragma mark - @@ -553,7 +624,11 @@ bool smp_trap_non_boot_cpus(int32 cpu) { if (cpu > 0) { +#if B_DEBUG_SPINLOCK_CONTENTION + boot_cpu_spin[cpu].lock = 1; +#else boot_cpu_spin[cpu] = 1; +#endif acquire_spinlock_nocheck(&boot_cpu_spin[cpu]); return false; } @@ -624,6 +699,18 @@ smp_per_cpu_init(kernel_args *args, int32 cpu) } +status_t +smp_init_post_generic_syscalls(void) +{ +#if B_DEBUG_SPINLOCK_CONTENTION + return register_generic_syscall(SPINLOCK_CONTENTION, + &spinlock_contention_syscall, 0, 0); +#else + return B_OK; +#endif +} + + void smp_set_num_cpus(int32 numCPUs) { diff --git a/src/system/kernel/team.cpp b/src/system/kernel/team.cpp index 5eb482d9d6..16335739b0 100644 --- a/src/system/kernel/team.cpp +++ b/src/system/kernel/team.cpp @@ -89,7 +89,7 @@ static struct team *sKernelTeam = NULL; static int32 sMaxTeams = 2048; static int32 sUsedTeams = 1; -spinlock team_spinlock = 0; +spinlock team_spinlock = B_SPINLOCK_INITIALIZER; // #pragma mark - Tracing diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 76882f6855..1035dd3a1e 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -60,7 +60,7 @@ struct thread_key { }; // global -spinlock thread_spinlock = 0; +spinlock thread_spinlock = B_SPINLOCK_INITIALIZER; // thread list static struct thread sIdleThreads[B_MAX_CPU_COUNT];