diff --git a/headers/private/kernel/boot/platform/efi/arch_smp.h b/headers/private/kernel/boot/platform/efi/arch_smp.h index 68bb22ff09..ea002c4ee6 100644 --- a/headers/private/kernel/boot/platform/efi/arch_smp.h +++ b/headers/private/kernel/boot/platform/efi/arch_smp.h @@ -1,5 +1,5 @@ /* - * Copyright 2013-2021 Haiku, Inc. All rights reserved. + * Copyright 2013-2022 Haiku, Inc. All rights reserved. * Distributed under the terms of the MIT License. */ #ifndef KERNEL_BOOT_PLATFORM_EFI_ARCH_SMP_H @@ -32,9 +32,9 @@ int arch_smp_get_current_cpu(void); void arch_smp_init_other_cpus(void); #ifdef __riscv platform_cpu_info* arch_smp_find_cpu(uint32 phandle); -void arch_smp_boot_other_cpus(uint64 satp, uint64 kernel_entry); +void arch_smp_boot_other_cpus(uint64 satp, uint64 kernelEntry, addr_t virtKernelArgs); #else -void arch_smp_boot_other_cpus(uint32 pml4, uint64 kernel_entry); +void arch_smp_boot_other_cpus(uint32 pml4, uint64 kernelEntry, addr_t virtKernelArgs); #endif void arch_smp_add_safemode_menus(Menu *menu); void arch_smp_init(void); diff --git a/src/system/boot/platform/efi/arch/arm/arch_smp.cpp b/src/system/boot/platform/efi/arch/arm/arch_smp.cpp index 0a989f4c52..9a40caeb8f 100644 --- a/src/system/boot/platform/efi/arch/arm/arch_smp.cpp +++ b/src/system/boot/platform/efi/arch/arm/arch_smp.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2019-2020, Haiku, Inc. All rights reserved. + * Copyright 2019-2022, Haiku, Inc. All rights reserved. * Released under the terms of the MIT License. */ @@ -61,7 +61,7 @@ arch_smp_init_other_cpus(void) void -arch_smp_boot_other_cpus(uint32 pml4, uint64 kernel_entry) +arch_smp_boot_other_cpus(uint32 pml4, uint64 kernelEntry, addr_t virtKernelArgs) { // One cpu for now. } diff --git a/src/system/boot/platform/efi/arch/arm/arch_start.cpp b/src/system/boot/platform/efi/arch/arm/arch_start.cpp index a585af9545..bfb7a06b70 100644 --- a/src/system/boot/platform/efi/arch/arm/arch_start.cpp +++ b/src/system/boot/platform/efi/arch/arm/arch_start.cpp @@ -219,7 +219,7 @@ arch_start_kernel(addr_t kernelEntry) // as there are some fixups happening to kernel_args even in the last minute memcpy(kernelArgs, &gKernelArgs, sizeof(struct kernel_args)); - //smp_boot_other_cpus(final_ttbr0, kernelEntry); + //smp_boot_other_cpus(final_ttbr0, kernelEntry, (addr_t)&gKernelArgs); // Enter the kernel! dprintf("enter_kernel(ttbr0: 0x%08x, kernelArgs: 0x%08x, " diff --git a/src/system/boot/platform/efi/arch/arm64/arch_smp.cpp b/src/system/boot/platform/efi/arch/arm64/arch_smp.cpp index fd4014c1e1..ca87faf8c5 100644 --- a/src/system/boot/platform/efi/arch/arm64/arch_smp.cpp +++ b/src/system/boot/platform/efi/arch/arm64/arch_smp.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2021, Haiku, Inc. All rights reserved. + * Copyright 2021-2022, Haiku, Inc. All rights reserved. * Released under the terms of the MIT License. */ @@ -50,7 +50,7 @@ arch_smp_init_other_cpus(void) void -arch_smp_boot_other_cpus(uint32 pml4, uint64 kernel_entry) +arch_smp_boot_other_cpus(uint32 pml4, uint64 kernelEntry, addr_t virtKernelArgs) { // One cpu for now. } diff --git a/src/system/boot/platform/efi/arch/arm64/arch_start.cpp b/src/system/boot/platform/efi/arch/arm64/arch_start.cpp index af5c19f065..307bdbd020 100644 --- a/src/system/boot/platform/efi/arch/arm64/arch_start.cpp +++ b/src/system/boot/platform/efi/arch/arm64/arch_start.cpp @@ -218,7 +218,7 @@ arch_start_kernel(addr_t kernelEntry) arch_cache_enable(); } - //smp_boot_other_cpus(final_pml4, kernelEntry); + //smp_boot_other_cpus(final_pml4, kernelEntry, (addr_t)&gKernelArgs); if (arch_mmu_read_access(kernelEntry) && arch_mmu_read_access(gKernelArgs.cpu_kstack[0].start)) { // Enter the kernel! diff --git a/src/system/boot/platform/efi/arch/riscv64/arch_smp.cpp b/src/system/boot/platform/efi/arch/riscv64/arch_smp.cpp index 71207f16e5..d46de71723 100644 --- a/src/system/boot/platform/efi/arch/riscv64/arch_smp.cpp +++ b/src/system/boot/platform/efi/arch/riscv64/arch_smp.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2019-2020, Haiku, Inc. All rights reserved. + * Copyright 2019-2022, Haiku, Inc. All rights reserved. * Released under the terms of the MIT License. */ @@ -200,7 +200,7 @@ arch_smp_init_other_cpus(void) void -arch_smp_boot_other_cpus(uint64 satp, uint64 kernel_entry) +arch_smp_boot_other_cpus(uint64 satp, uint64 kernel_entry, addr_t virtKernelArgs) { dprintf("arch_smp_boot_other_cpus(%p, %p)\n", (void*)satp, (void*)kernel_entry); diff --git a/src/system/boot/platform/efi/arch/riscv64/arch_start.cpp b/src/system/boot/platform/efi/arch/riscv64/arch_start.cpp index c93bebf225..fca8f260be 100644 --- a/src/system/boot/platform/efi/arch/riscv64/arch_start.cpp +++ b/src/system/boot/platform/efi/arch/riscv64/arch_start.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2019-2021 Haiku, Inc. All rights reserved. + * Copyright 2019-2022 Haiku, Inc. All rights reserved. * Released under the terms of the MIT License. */ @@ -147,7 +147,7 @@ arch_start_kernel(addr_t kernelEntry) FlushTlbAll(); dprintf("[POST] FlushTlbAll()\n"); - smp_boot_other_cpus(satp, kernelEntry); + smp_boot_other_cpus(satp, kernelEntry, (addr_t)&gKernelArgs); // Enter the kernel! dprintf("arch_enter_kernel(satp: %#" B_PRIxADDR ", kernelArgs: %#" B_PRIxADDR diff --git a/src/system/boot/platform/efi/arch/x86/Jamfile b/src/system/boot/platform/efi/arch/x86/Jamfile index 9f89c33904..db715bf1e4 100644 --- a/src/system/boot/platform/efi/arch/x86/Jamfile +++ b/src/system/boot/platform/efi/arch/x86/Jamfile @@ -18,9 +18,12 @@ for platform in [ MultiBootSubDirSetup efi ] { local arch_src = crt0-efi-$(TARGET_ARCH).S entry.S + smp_trampoline.S + support.S relocation_func.cpp arch_mmu.cpp arch_smp.cpp + arch_smp_32.cpp arch_start.cpp arch_timer.cpp ; diff --git a/src/system/boot/platform/efi/arch/x86/arch_smp.cpp b/src/system/boot/platform/efi/arch/x86/arch_smp.cpp index 73c5797ad0..fad379d67f 100644 --- a/src/system/boot/platform/efi/arch/x86/arch_smp.cpp +++ b/src/system/boot/platform/efi/arch/x86/arch_smp.cpp @@ -1,15 +1,182 @@ /* - * Copyright 2021 Haiku, Inc. All rights reserved. + * Copyright 2021-2022 Haiku, Inc. All rights reserved. * Released under the terms of the MIT License. + * + * Copyright 2008, Dustin Howett, dustin.howett@gmail.com. All rights reserved. + * Copyright 2004-2010, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + * + * Copyright 2001, Travis Geiselbrecht. All rights reserved. + * Distributed under the terms of the NewOS License. */ #include "arch_smp.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mmu.h" +#include "acpi.h" + + +#define NO_SMP 0 + +//#define TRACE_SMP +#ifdef TRACE_SMP +# define TRACE(x...) dprintf(x) +#else +# define TRACE(x...) ; +#endif + + +extern "C" void execute_n_instructions(int count); + +void copy_trampoline_code(uint64 trampolineCode, uint64 trampolineStack); +void prepare_trampoline_args(uint64 trampolineCode, uint64 trampolineStack, + uint32 pagedir, uint64 kernelEntry, addr_t virtKernelArgs, + uint32 currentCpu); +uint32 get_sentinel(uint64 trampolineStack); + + +static uint32 +apic_read(uint32 offset) +{ + return *(volatile uint32 *)((addr_t)gKernelArgs.arch_args.apic_phys + offset); +} + + +static void +apic_write(uint32 offset, uint32 data) +{ + *(volatile uint32 *)((addr_t)gKernelArgs.arch_args.apic_phys + offset) = data; +} + + +static status_t +acpi_do_smp_config(void) +{ + TRACE("smp: using ACPI to detect MP configuration\n"); + + // reset CPU count + gKernelArgs.num_cpus = 0; + + acpi_madt *madt = (acpi_madt *)acpi_find_table(ACPI_MADT_SIGNATURE); + + if (madt == NULL) { + TRACE("smp: Failed to find MADT!\n"); + return B_ERROR; + } + + gKernelArgs.arch_args.apic_phys = madt->local_apic_address; + TRACE("smp: local apic address is 0x%" B_PRIx32 "\n", madt->local_apic_address); + + acpi_apic *apic = (acpi_apic *)((uint8 *)madt + sizeof(acpi_madt)); + acpi_apic *end = (acpi_apic *)((uint8 *)madt + madt->header.length); + while (apic < end) { + switch (apic->type) { + case ACPI_MADT_LOCAL_APIC: + { + if (gKernelArgs.num_cpus == SMP_MAX_CPUS) { + TRACE("smp: already reached maximum CPUs (%d)\n", + SMP_MAX_CPUS); + break; + } + + acpi_local_apic *localApic = (acpi_local_apic *)apic; + TRACE("smp: found local APIC with id %u\n", + localApic->apic_id); + if ((localApic->flags & ACPI_LOCAL_APIC_ENABLED) == 0) { + TRACE("smp: APIC is disabled and will not be used\n"); + break; + } + + gKernelArgs.arch_args.cpu_apic_id[gKernelArgs.num_cpus] + = localApic->apic_id; + // TODO: how to find out? putting 0x10 in to indicate a local apic + gKernelArgs.arch_args.cpu_apic_version[gKernelArgs.num_cpus] + = 0x10; + gKernelArgs.num_cpus++; + break; + } + + case ACPI_MADT_IO_APIC: { + acpi_io_apic *ioApic = (acpi_io_apic *)apic; + TRACE("smp: found io APIC with id %" B_PRIu32 " and address 0x%" B_PRIx32 "\n", + ioApic->io_apic_id, ioApic->io_apic_address); + if (gKernelArgs.arch_args.ioapic_phys == 0) + gKernelArgs.arch_args.ioapic_phys = ioApic->io_apic_address; + break; + } + default: + break; + } + + apic = (acpi_apic *)((uint8 *)apic + apic->length); + } + + return gKernelArgs.num_cpus > 0 ? B_OK : B_ERROR; +} + + +static void +calculate_apic_timer_conversion_factor(void) +{ + int64 t1, t2; + uint32 config; + uint32 count; + + TRACE("calculating apic timer conversion factor\n"); + + // setup the timer + config = apic_read(APIC_LVT_TIMER); + config = (config & APIC_LVT_TIMER_MASK) + APIC_LVT_MASKED; + // timer masked, vector 0 + apic_write(APIC_LVT_TIMER, config); + + config = (apic_read(APIC_TIMER_DIVIDE_CONFIG) & ~0x0000000f); + apic_write(APIC_TIMER_DIVIDE_CONFIG, config | APIC_TIMER_DIVIDE_CONFIG_1); + // divide clock by one + + t1 = system_time(); + apic_write(APIC_INITIAL_TIMER_COUNT, 0xffffffff); // start the counter + + execute_n_instructions(128 * 20000); + + count = apic_read(APIC_CURRENT_TIMER_COUNT); + t2 = system_time(); + + count = 0xffffffff - count; + + gKernelArgs.arch_args.apic_time_cv_factor + = (uint32)((1000000.0/(t2 - t1)) * count); + + TRACE("APIC ticks/sec = %" B_PRId32 "\n", + gKernelArgs.arch_args.apic_time_cv_factor); +} + + +// #pragma mark - + int arch_smp_get_current_cpu(void) { + if (gKernelArgs.arch_args.apic == NULL) + return 0; + + uint8 apicID = apic_read(APIC_ID) >> 24; + for (uint32 i = 0; i < gKernelArgs.num_cpus; i++) { + if (gKernelArgs.arch_args.cpu_apic_id[i] == apicID) + return i; + } + return 0; } @@ -17,22 +184,220 @@ arch_smp_get_current_cpu(void) void arch_smp_init_other_cpus(void) { + if (get_safemode_boolean(B_SAFEMODE_DISABLE_SMP, false)) { + // SMP has been disabled! + TRACE("smp disabled per safemode setting\n"); + gKernelArgs.num_cpus = 1; + } + + if (get_safemode_boolean(B_SAFEMODE_DISABLE_APIC, false)) { + TRACE("local apic disabled per safemode setting, disabling smp\n"); + gKernelArgs.arch_args.apic_phys = 0; + gKernelArgs.num_cpus = 1; + } + + if (gKernelArgs.arch_args.apic_phys == 0) + return; + + TRACE("smp: found %" B_PRId32 " cpu%s\n", gKernelArgs.num_cpus, + gKernelArgs.num_cpus != 1 ? "s" : ""); + TRACE("smp: apic_phys = %lx\n", (addr_t)gKernelArgs.arch_args.apic_phys); + TRACE("smp: ioapic_phys = %lx\n", + (addr_t)gKernelArgs.arch_args.ioapic_phys); + + // map in the apic + gKernelArgs.arch_args.apic = (void *)mmu_map_physical_memory( + gKernelArgs.arch_args.apic_phys, B_PAGE_SIZE, kDefaultPageFlags); + + TRACE("smp: apic (mapped) = %lx\n", (addr_t)gKernelArgs.arch_args.apic.Pointer()); + + // calculate how fast the apic timer is + calculate_apic_timer_conversion_factor(); + + if (gKernelArgs.num_cpus < 2) + return; + + for (uint32 i = 1; i < gKernelArgs.num_cpus; i++) { + // create a final stack the trampoline code will put the ap processor on + void * stack = NULL; + const size_t size = KERNEL_STACK_SIZE + KERNEL_STACK_GUARD_PAGES * B_PAGE_SIZE; + if (platform_allocate_region(&stack, size, 0, false) != B_OK) { + panic("Unable to allocate AP stack"); + } + memset(stack, 0, size); + gKernelArgs.cpu_kstack[i].start = fix_address((uint64_t)stack); + gKernelArgs.cpu_kstack[i].size = size; + } } void -arch_smp_boot_other_cpus(uint32 pml4, uint64 kernel_entry) +arch_smp_boot_other_cpus(uint32 pagedir, uint64 kernelEntry, addr_t virtKernelArgs) { + TRACE("trampolining other cpus\n"); + + // allocate a stack and a code area for the smp trampoline + // (these have to be < 1M physical, 0xa0000-0xfffff is reserved by the BIOS) + uint64 trampolineCode = 0x9000; + uint64 trampolineStack = 0x8000; + + // copy the trampoline code over + copy_trampoline_code(trampolineCode, trampolineStack); + + // boot the cpus + TRACE("we have %" B_PRId32 " CPUs to boot...\n", gKernelArgs.num_cpus - 1); + for (uint32 i = 1; i < gKernelArgs.num_cpus; i++) { + TRACE("trampolining CPU %" B_PRId32 "\n", i); + uint32 config; + uint64 numStartups; + uint32 j; + + prepare_trampoline_args(trampolineCode, trampolineStack, + pagedir, kernelEntry, virtKernelArgs, i); + + /* clear apic errors */ + if (gKernelArgs.arch_args.cpu_apic_version[i] & 0xf0) { + apic_write(APIC_ERROR_STATUS, 0); + apic_read(APIC_ERROR_STATUS); + } + + /* send (aka assert) INIT IPI */ + config = (apic_read(APIC_INTR_COMMAND_2) & APIC_INTR_COMMAND_2_MASK) + | (gKernelArgs.arch_args.cpu_apic_id[i] << 24); + apic_write(APIC_INTR_COMMAND_2, config); /* set target pe */ + config = (apic_read(APIC_INTR_COMMAND_1) & 0xfff00000) + | APIC_TRIGGER_MODE_LEVEL | APIC_INTR_COMMAND_1_ASSERT + | APIC_DELIVERY_MODE_INIT; + apic_write(APIC_INTR_COMMAND_1, config); + + // wait for pending to end + while ((apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) != 0) + asm volatile ("pause;"); + + /* deassert INIT */ + config = (apic_read(APIC_INTR_COMMAND_2) & APIC_INTR_COMMAND_2_MASK) + | (gKernelArgs.arch_args.cpu_apic_id[i] << 24); + apic_write(APIC_INTR_COMMAND_2, config); + config = (apic_read(APIC_INTR_COMMAND_1) & 0xfff00000) + | APIC_TRIGGER_MODE_LEVEL | APIC_DELIVERY_MODE_INIT; + apic_write(APIC_INTR_COMMAND_1, config); + + // wait for pending to end + while ((apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) != 0) + asm volatile ("pause;"); + + /* wait 10ms */ + spin(10000); + /* is this a local apic or an 82489dx ? */ + numStartups = (gKernelArgs.arch_args.cpu_apic_version[i] & 0xf0) + ? 2 : 0; + for (j = 0; j < numStartups; j++) { + /* it's a local apic, so send STARTUP IPIs */ + apic_write(APIC_ERROR_STATUS, 0); + + /* set target pe */ + config = (apic_read(APIC_INTR_COMMAND_2) & APIC_INTR_COMMAND_2_MASK) + | (gKernelArgs.arch_args.cpu_apic_id[i] << 24); + apic_write(APIC_INTR_COMMAND_2, config); + + /* send the IPI */ + config = (apic_read(APIC_INTR_COMMAND_1) & 0xfff0f800) + | APIC_DELIVERY_MODE_STARTUP | (trampolineCode >> 12); + apic_write(APIC_INTR_COMMAND_1, config); + + /* wait */ + spin(200); + + while ((apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) != 0) + asm volatile ("pause;"); + } + + // Wait for the trampoline code to clear the final stack location. + // This serves as a notification for us that it has loaded the address + // and it is safe for us to overwrite it to trampoline the next CPU. + while (get_sentinel(trampolineStack) != 0) + spin(1000); + } + + TRACE("done trampolining\n"); } void arch_smp_add_safemode_menus(Menu *menu) { + MenuItem *item; + + if (gKernelArgs.arch_args.ioapic_phys != 0) { + menu->AddItem(item = new(nothrow) MenuItem("Disable IO-APIC")); + item->SetType(MENU_ITEM_MARKABLE); + item->SetData(B_SAFEMODE_DISABLE_IOAPIC); + item->SetHelpText("Disables using the IO APIC for interrupt routing, " + "forcing the use of the legacy PIC instead."); + } + + if (gKernelArgs.arch_args.apic_phys != 0) { + menu->AddItem(item = new(nothrow) MenuItem("Disable local APIC")); + item->SetType(MENU_ITEM_MARKABLE); + item->SetData(B_SAFEMODE_DISABLE_APIC); + item->SetHelpText("Disables using the local APIC, also disables SMP."); + + cpuid_info info; + if (get_current_cpuid(&info, 1, 0) == B_OK + && (info.regs.ecx & IA32_FEATURE_EXT_X2APIC) != 0) { + menu->AddItem(item = new(nothrow) MenuItem("Disable X2APIC")); + item->SetType(MENU_ITEM_MARKABLE); + item->SetData(B_SAFEMODE_DISABLE_X2APIC); + item->SetHelpText("Disables using X2APIC."); + } + + if (get_current_cpuid(&info, 7, 0) == B_OK + && ((info.regs.ebx & (IA32_FEATURE_SMEP + | IA32_FEATURE_SMAP)) != 0)) { + menu->AddItem(item = new(nothrow) MenuItem( + "Disable SMEP and SMAP")); + item->SetType(MENU_ITEM_MARKABLE); + item->SetData(B_SAFEMODE_DISABLE_SMEP_SMAP); + item->SetHelpText("Disables using SMEP and SMAP."); + } + } + + if (gKernelArgs.num_cpus < 2) + return; + + item = new(nothrow) MenuItem("Disable SMP"); + menu->AddItem(item); + item->SetData(B_SAFEMODE_DISABLE_SMP); + item->SetType(MENU_ITEM_MARKABLE); + item->SetHelpText("Disables all but one CPU core."); } void arch_smp_init(void) { + cpuid_info info; + if (get_current_cpuid(&info, 1, 0) != B_OK) + return; + + if ((info.eax_1.features & IA32_FEATURE_APIC) == 0) { + // Local APICs aren't present; As they form the basis for all inter CPU + // communication and therefore SMP, we don't need to go any further. + TRACE("no local APIC present, not attempting SMP init\n"); + return; + } + + // first try to find ACPI tables to get MP configuration as it handles + // physical as well as logical MP configurations as in multiple cpus, + // multiple cores or hyper threading. + if (acpi_do_smp_config() == B_OK) { + TRACE("smp init success\n"); + return; + } + + // Everything failed or we are not running an SMP system, reset anything + // that might have been set through an incomplete configuration attempt. + gKernelArgs.arch_args.apic_phys = 0; + gKernelArgs.arch_args.ioapic_phys = 0; + gKernelArgs.num_cpus = 1; } diff --git a/src/system/boot/platform/efi/arch/x86/arch_smp_32.cpp b/src/system/boot/platform/efi/arch/x86/arch_smp_32.cpp new file mode 100644 index 0000000000..ffd10f2d9e --- /dev/null +++ b/src/system/boot/platform/efi/arch/x86/arch_smp_32.cpp @@ -0,0 +1,104 @@ +/* + * Copyright 2021-2022 Haiku, Inc. All rights reserved. + * Released under the terms of the MIT License. + * + * Copyright 2008, Dustin Howett, dustin.howett@gmail.com. All rights reserved. + * Copyright 2004-2010, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + * + * Copyright 2001, Travis Geiselbrecht. All rights reserved. + * Distributed under the terms of the NewOS License. + */ + + +#include "arch_smp.h" + +#include + + +//#define TRACE_SMP +#ifdef TRACE_SMP +# define TRACE(x...) dprintf(x) +#else +# define TRACE(x...) ; +#endif + + +extern "C" void smp_trampoline(void); +extern "C" void smp_trampoline_args(void); +extern "C" void smp_trampoline_end(void); + +struct gdtr { + uint16 limit; + uint32 base; + unsigned char null[8]; + unsigned char code[8]; + unsigned char data[8]; +} __attribute__((packed)); + +// Arguments passed to the SMP trampoline. +struct trampoline_args { + uint32 trampoline; // Trampoline address + uint32 gdt; // GDTR + uint32 page_dir; // page directory + uint32 kernel_entry; // Kernel entry point + uint32 kernel_args; // Kernel arguments + uint32 current_cpu; // CPU number + uint32 stack_top; // Kernel stack + volatile uint32 sentinel; // Sentinel, AP sets to 0 when finished + + // smp_boot_other_cpus puts the GDTR here. + struct gdtr gdtr; +}; + + +void +copy_trampoline_code(uint64 trampolineCode, uint64 trampolineStack) +{ + TRACE("copying the trampoline code to %p from %p\n", (char*)trampolineCode, (const void*)&smp_trampoline); + TRACE("size of trampoline code = %" PRIu64 " bytes\n", (uint64)&smp_trampoline_end - (uint64)&smp_trampoline); + memcpy((char *)trampolineCode, (const void*)&smp_trampoline, + (uint64)&smp_trampoline_end - (uint64)&smp_trampoline); +} + + +void +prepare_trampoline_args(uint64 trampolineCode, uint64 trampolineStack, + uint32 pagedir, uint64 kernelEntry, addr_t virtKernelArgs, + uint32 currentCpu) +{ + trampoline_args* args = (trampoline_args *)trampolineStack; + args->trampoline = trampolineCode; + + args->gdt = (uint32) &args->gdtr; + args->gdtr.limit = 23; + args->gdtr.base = (uint32)(uint64)args->gdtr.null; + #define COPY_ARRAY(A, X0, X1, X2, X3, X4, X5, X6, X7) \ + { A[0] = X0; A[1] = X1; A[2] = X2; A[3] = X3; A[4] = X4; A[5] = X5; A[6] = X6; A[7] = X7; } + COPY_ARRAY(args->gdtr.null, 0, 0, 0, 0, 0, 0, 0, 0); + COPY_ARRAY(args->gdtr.code, 0xff, 0xff, 0, 0, 0, 0x9a, 0xcf, 0); + COPY_ARRAY(args->gdtr.data, 0xff, 0xff, 0, 0, 0, 0x92, 0xcf, 0); + #undef COPY_ARRAY + //TODO: final GDT + + args->page_dir = pagedir; + args->kernel_entry = kernelEntry; + args->kernel_args = virtKernelArgs; + args->current_cpu = currentCpu; + args->stack_top = gKernelArgs.cpu_kstack[currentCpu].start + + gKernelArgs.cpu_kstack[currentCpu].size; + args->sentinel = 1; + + // put the args in the right place + uint32 * args_ptr = + (uint32 *)(trampolineCode + (uint64)smp_trampoline_args - (uint64)smp_trampoline); + *args_ptr = (uint32)trampolineStack; +} + + +uint32 +get_sentinel(uint64 trampolineStack) +{ + trampoline_args* args = (trampoline_args *)trampolineStack; + return args->sentinel; +} diff --git a/src/system/boot/platform/efi/arch/x86/arch_start.cpp b/src/system/boot/platform/efi/arch/x86/arch_start.cpp index 0c722818af..fe303ef0e4 100644 --- a/src/system/boot/platform/efi/arch/x86/arch_start.cpp +++ b/src/system/boot/platform/efi/arch/x86/arch_start.cpp @@ -196,7 +196,7 @@ arch_start_kernel(addr_t kernelEntry) // as there are some fixups happening to kernel_args even in the last minute memcpy(kernelArgs, &gKernelArgs, sizeof(struct kernel_args)); - smp_boot_other_cpus(pageDirectory, kernelEntry); + smp_boot_other_cpus(pageDirectory, kernelEntry, virtKernelArgs); // Enter the kernel! dprintf("enter_kernel(pageDirectory: 0x%08x, kernelArgs: 0x%08x, " diff --git a/src/system/boot/platform/efi/arch/x86/smp_trampoline.S b/src/system/boot/platform/efi/arch/x86/smp_trampoline.S new file mode 100644 index 0000000000..c04ce23128 --- /dev/null +++ b/src/system/boot/platform/efi/arch/x86/smp_trampoline.S @@ -0,0 +1,96 @@ +/* + * Copyright 2021-2022 Haiku, Inc. All rights reserved. + * Released under the terms of the MIT License. + * + * Copyright 2001, Travis Geiselbrecht. All rights reserved. + * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. + * Distributed under the terms of the NewOS License. + */ + +// Relocatable. Before calling, smp_trampoline_args should point +// to a struct trampoline_args (see smp.cpp). This pointer should +// be 16-byte aligned, below 1MB and identity-mapped. +.globl smp_trampoline +.globl smp_trampoline_end +.globl smp_trampoline_args + +#include + +#include +#include "mmu.h" + +.code16 +smp_trampoline: + cli + +// movl 0xdeadbeef, esi + .byte 0x66 + .byte 0xbe +smp_trampoline_args: + .long 0xdeadbeef + + // Set up the stack pointer for 16-bit mode + movl %esi, %eax + shrl $4, %eax + movw %ax, %ss + xorw %sp, %sp + + popl %ebx // trampoline + + // Initialize GDT + popl %edx // gdt + lgdt (%edx) + + // Switch to protected mode + movl %cr0, %eax + orl $0x01, %eax + movl %eax, %cr0 + + pushl $KERNEL_CODE_SELECTOR + leal (trampoline_32 - smp_trampoline)(%ebx), %eax + pushl %eax +.code32 + .byte 0x66 + lret + +trampoline_32: + // Set data segments. + movw $KERNEL_DATA_SELECTOR, %ax + movw %ax, %ds + movw %ax, %es + movw %ax, %fs + movw %ax, %gs + movw %ax, %ss + + // Set up the stack pointer for 32-bit mode + movl %esi, %esp + addl $8, %esp + + // Point CR3 to the kernel's page dir. + popl %eax // page_dir + movl %eax, %cr3 + + popl %ebx // kernel_entry + popl %ecx // kernel_args + popl %edx // current_cpu + + // Set the stack pointer, write to the sentinel and clear the stack frame + popl %ebp // stack_top + movl $0, (%esp) // sentinel + movl %ebp, %esp + xorl %ebp, %ebp + + movl $0x80010021, %eax + movl %eax, %cr0 + + cli + cld + fninit + + pushl %edx // currentCpu + pushl %ecx // kernelArgs + pushl $0x0 // fake return address + pushl $KERNEL_CODE_SELECTOR + pushl %ebx // kernelEntry + lret +smp_trampoline_end: diff --git a/src/system/boot/platform/efi/arch/x86/support.S b/src/system/boot/platform/efi/arch/x86/support.S new file mode 100644 index 0000000000..4fe95030bc --- /dev/null +++ b/src/system/boot/platform/efi/arch/x86/support.S @@ -0,0 +1,35 @@ +/* + * Copyright 2021-2022 Haiku, Inc. All rights reserved. + * Released under the terms of the MIT License. + * + * Copyright 2001, Travis Geiselbrecht. All rights reserved. + * Distributed under the terms of the NewOS License. + */ + + +#define FUNCTION(x) .global x; .type x,@function; x + + +FUNCTION(execute_n_instructions): + movl 4(%esp), %ecx + shrl $4, %ecx +.again: + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + xorl %eax, %eax + loop .again + ret + diff --git a/src/system/boot/platform/efi/arch/x86_64/Jamfile b/src/system/boot/platform/efi/arch/x86_64/Jamfile index 0db1b8275c..4a1ba0bb79 100644 --- a/src/system/boot/platform/efi/arch/x86_64/Jamfile +++ b/src/system/boot/platform/efi/arch/x86_64/Jamfile @@ -18,11 +18,12 @@ for platform in [ MultiBootSubDirSetup efi ] { local arch_src = crt0-efi-$(TARGET_ARCH).S entry.S - smp_trampoline.S + long_smp_trampoline.S support.S relocation_func.cpp arch_start.cpp arch_smp.cpp + arch_smp_64.cpp arch_mmu.cpp arch_timer.cpp ; @@ -30,5 +31,8 @@ for platform in [ MultiBootSubDirSetup efi ] { BootMergeObject boot_platform_efi_x86_64.o : $(arch_src) ; + + SEARCH on [ FGristFiles arch_smp.cpp ] + = [ FDirName $(HAIKU_TOP) src system boot platform efi arch x86 ] ; } } diff --git a/src/system/boot/platform/efi/arch/x86_64/arch_smp.cpp b/src/system/boot/platform/efi/arch/x86_64/arch_smp.cpp deleted file mode 100644 index 5d79d8193f..0000000000 --- a/src/system/boot/platform/efi/arch/x86_64/arch_smp.cpp +++ /dev/null @@ -1,455 +0,0 @@ -/* - * Copyright 2008, Dustin Howett, dustin.howett@gmail.com. All rights reserved. - * Copyright 2004-2010, Axel Dörfler, axeld@pinc-software.de. - * Distributed under the terms of the MIT License. - * - * Copyright 2001, Travis Geiselbrecht. All rights reserved. - * Distributed under the terms of the NewOS License. -*/ - - -#include "arch_smp.h" - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "mmu.h" -#include "acpi.h" - - -#define NO_SMP 0 - -//#define TRACE_SMP -#ifdef TRACE_SMP -# define TRACE(x) dprintf x -#else -# define TRACE(x) ; -#endif - - -extern "C" void execute_n_instructions(int count); - -extern "C" void smp_trampoline(void); -extern "C" void smp_trampoline_args(void); -extern "C" void smp_trampoline_end(void); -extern "C" uint64 gLongGDTR; - -struct gdtr { - uint16 limit; - uint32 base; - unsigned char null[8]; - unsigned char code[8]; - unsigned char data[8]; -} __attribute__((packed)); - -// Arguments passed to the SMP trampoline. -struct trampoline_args { - uint32 trampoline; // Trampoline address - uint32 gdt32; // 32-bit GDTR - uint32 pml4; // 64-bit PML4 - uint32 gdt64; // 64-bit GDTR - uint64 kernel_entry; // Kernel entry point - uint64 kernel_args; // Kernel arguments - uint64 current_cpu; // CPU number - uint64 stack_top; // Kernel stack - volatile uint64 sentinel; // Sentinel, AP sets to 0 when finished - - // smp_boot_other_cpus puts the GDTR here. - struct gdtr gdtr; -}; - - -uint32 -apic_read(uint32 offset) -{ - return *(volatile uint32 *)((addr_t)gKernelArgs.arch_args.apic_phys + offset); -} - - -void -apic_write(uint32 offset, uint32 data) -{ - *(volatile uint32 *)((addr_t)gKernelArgs.arch_args.apic_phys + offset) = data; -} - - -static status_t -acpi_do_smp_config(void) -{ - TRACE(("smp: using ACPI to detect MP configuration\n")); - - // reset CPU count - gKernelArgs.num_cpus = 0; - - acpi_madt *madt = (acpi_madt *)acpi_find_table(ACPI_MADT_SIGNATURE); - - if (madt == NULL) { - TRACE(("smp: Failed to find MADT!\n")); - return B_ERROR; - } - - gKernelArgs.arch_args.apic_phys = madt->local_apic_address; - TRACE(("smp: local apic address is 0x%" B_PRIx32 "\n", madt->local_apic_address)); - - acpi_apic *apic = (acpi_apic *)((uint8 *)madt + sizeof(acpi_madt)); - acpi_apic *end = (acpi_apic *)((uint8 *)madt + madt->header.length); - while (apic < end) { - switch (apic->type) { - case ACPI_MADT_LOCAL_APIC: - { - if (gKernelArgs.num_cpus == SMP_MAX_CPUS) { - TRACE(("smp: already reached maximum CPUs (%d)\n", - SMP_MAX_CPUS)); - break; - } - - acpi_local_apic *localApic = (acpi_local_apic *)apic; - TRACE(("smp: found local APIC with id %u\n", - localApic->apic_id)); - if ((localApic->flags & ACPI_LOCAL_APIC_ENABLED) == 0) { - TRACE(("smp: APIC is disabled and will not be used\n")); - break; - } - - gKernelArgs.arch_args.cpu_apic_id[gKernelArgs.num_cpus] - = localApic->apic_id; - // TODO: how to find out? putting 0x10 in to indicate a local apic - gKernelArgs.arch_args.cpu_apic_version[gKernelArgs.num_cpus] - = 0x10; - gKernelArgs.num_cpus++; - break; - } - - case ACPI_MADT_IO_APIC: { - acpi_io_apic *ioApic = (acpi_io_apic *)apic; - TRACE(("smp: found io APIC with id %" B_PRIu32 " and address 0x%" B_PRIx32 "\n", - ioApic->io_apic_id, ioApic->io_apic_address)); - if (gKernelArgs.arch_args.ioapic_phys == 0) - gKernelArgs.arch_args.ioapic_phys = ioApic->io_apic_address; - break; - } - default: - break; - } - - apic = (acpi_apic *)((uint8 *)apic + apic->length); - } - - return gKernelArgs.num_cpus > 0 ? B_OK : B_ERROR; -} - - -static void -calculate_apic_timer_conversion_factor(void) -{ - int64 t1, t2; - uint32 config; - uint32 count; - - TRACE(("calculating apic timer conversion factor\n")); - - // setup the timer - config = apic_read(APIC_LVT_TIMER); - config = (config & APIC_LVT_TIMER_MASK) + APIC_LVT_MASKED; - // timer masked, vector 0 - apic_write(APIC_LVT_TIMER, config); - - config = (apic_read(APIC_TIMER_DIVIDE_CONFIG) & ~0x0000000f); - apic_write(APIC_TIMER_DIVIDE_CONFIG, config | APIC_TIMER_DIVIDE_CONFIG_1); - // divide clock by one - - t1 = system_time(); - apic_write(APIC_INITIAL_TIMER_COUNT, 0xffffffff); // start the counter - - execute_n_instructions(128 * 20000); - - count = apic_read(APIC_CURRENT_TIMER_COUNT); - t2 = system_time(); - - count = 0xffffffff - count; - - gKernelArgs.arch_args.apic_time_cv_factor - = (uint32)((1000000.0/(t2 - t1)) * count); - - TRACE(("APIC ticks/sec = %" B_PRId32 "\n", - gKernelArgs.arch_args.apic_time_cv_factor)); -} - - -// #pragma mark - - - -int -arch_smp_get_current_cpu(void) -{ - if (gKernelArgs.arch_args.apic == NULL) - return 0; - - uint8 apicID = apic_read(APIC_ID) >> 24; - for (uint32 i = 0; i < gKernelArgs.num_cpus; i++) { - if (gKernelArgs.arch_args.cpu_apic_id[i] == apicID) - return i; - } - - return 0; -} - - -void -arch_smp_init_other_cpus(void) -{ - if (get_safemode_boolean(B_SAFEMODE_DISABLE_SMP, false)) { - // SMP has been disabled! - TRACE(("smp disabled per safemode setting\n")); - gKernelArgs.num_cpus = 1; - } - - if (get_safemode_boolean(B_SAFEMODE_DISABLE_APIC, false)) { - TRACE(("local apic disabled per safemode setting, disabling smp\n")); - gKernelArgs.arch_args.apic_phys = 0; - gKernelArgs.num_cpus = 1; - } - - if (gKernelArgs.arch_args.apic_phys == 0) - return; - - TRACE(("smp: found %" B_PRId32 " cpu%s\n", gKernelArgs.num_cpus, - gKernelArgs.num_cpus != 1 ? "s" : "")); - TRACE(("smp: apic_phys = %lx\n", (addr_t)gKernelArgs.arch_args.apic_phys)); - TRACE(("smp: ioapic_phys = %lx\n", - (addr_t)gKernelArgs.arch_args.ioapic_phys)); - - // map in the apic - gKernelArgs.arch_args.apic = (void *)mmu_map_physical_memory( - gKernelArgs.arch_args.apic_phys, B_PAGE_SIZE, kDefaultPageFlags); - - TRACE(("smp: apic (mapped) = %lx\n", (addr_t)gKernelArgs.arch_args.apic.Pointer())); - - // calculate how fast the apic timer is - calculate_apic_timer_conversion_factor(); - - if (gKernelArgs.num_cpus < 2) - return; - - for (uint32 i = 1; i < gKernelArgs.num_cpus; i++) { - // create a final stack the trampoline code will put the ap processor on - void * stack = NULL; - const size_t size = KERNEL_STACK_SIZE + KERNEL_STACK_GUARD_PAGES * B_PAGE_SIZE; - if (platform_allocate_region(&stack, size, 0, false) != B_OK) { - panic("Unable to allocate AP stack"); - } - memset(stack, 0, size); - gKernelArgs.cpu_kstack[i].start = fix_address((uint64_t)stack); - gKernelArgs.cpu_kstack[i].size = size; - } -} - - -void -arch_smp_boot_other_cpus(uint32 pml4, uint64 kernel_entry) -{ - TRACE(("trampolining other cpus\n")); - - // allocate a stack and a code area for the smp trampoline - // (these have to be < 1M physical, 0xa0000-0xfffff is reserved by the BIOS) - uint64 trampolineCode = 0x9000; - uint64 trampolineStack = 0x8000; - - // copy the trampoline code over - TRACE(("copying the trampoline code to %p from %p\n", (char*)trampolineCode, (const void*)&smp_trampoline)); - TRACE(("size of trampoline code = %lu bytes\n", (uint64)&smp_trampoline_end - (uint64)&smp_trampoline)); - memcpy((char *)trampolineCode, (const void*)&smp_trampoline, - (uint64)&smp_trampoline_end - (uint64)&smp_trampoline); - - // boot the cpus - TRACE(("we have %d CPUs to boot...\n", gKernelArgs.num_cpus - 1)); - for (uint32 i = 1; i < gKernelArgs.num_cpus; i++) { - TRACE(("trampolining CPU %d\n", i)); - uint32 config; - uint64 numStartups; - uint32 j; - trampoline_args* args = (trampoline_args *)trampolineStack; - args->trampoline = trampolineCode; - - args->gdt32 = (uint64) &args->gdtr; - args->gdtr.limit = 23; - args->gdtr.base = (uint32)(uint64)args->gdtr.null; - #define COPY_ARRAY(A, X0, X1, X2, X3, X4, X5, X6, X7) \ - { A[0] = X0; A[1] = X1; A[2] = X2; A[3] = X3; A[4] = X4; A[5] = X5; A[6] = X6; A[7] = X7; } - COPY_ARRAY(args->gdtr.null, 0, 0, 0, 0, 0, 0, 0, 0); - COPY_ARRAY(args->gdtr.code, 0xff, 0xff, 0, 0, 0, 0x9a, 0xcf, 0); - COPY_ARRAY(args->gdtr.data, 0xff, 0xff, 0, 0, 0, 0x92, 0xcf, 0); - #undef COPY_ARRAY - args->gdt64 = (uint32_t)(uint64_t)&gLongGDTR; - - args->pml4 = pml4; - args->kernel_entry = kernel_entry; - args->kernel_args = (uint64)&gKernelArgs; - args->current_cpu = i; - args->stack_top = gKernelArgs.cpu_kstack[i].start + gKernelArgs.cpu_kstack[i].size; - args->sentinel = 1; - - // put the args in the right place - uint32 * args_ptr = - (uint32 *)(trampolineCode + (uint64)smp_trampoline_args - (uint64)smp_trampoline); - *args_ptr = (uint32)(uint64)args; - - /* clear apic errors */ - if (gKernelArgs.arch_args.cpu_apic_version[i] & 0xf0) { - apic_write(APIC_ERROR_STATUS, 0); - apic_read(APIC_ERROR_STATUS); - } - - /* send (aka assert) INIT IPI */ - config = (apic_read(APIC_INTR_COMMAND_2) & APIC_INTR_COMMAND_2_MASK) - | (gKernelArgs.arch_args.cpu_apic_id[i] << 24); - apic_write(APIC_INTR_COMMAND_2, config); /* set target pe */ - config = (apic_read(APIC_INTR_COMMAND_1) & 0xfff00000) - | APIC_TRIGGER_MODE_LEVEL | APIC_INTR_COMMAND_1_ASSERT - | APIC_DELIVERY_MODE_INIT; - apic_write(APIC_INTR_COMMAND_1, config); - - // wait for pending to end - while ((apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) != 0) - asm volatile ("pause;"); - - /* deassert INIT */ - config = (apic_read(APIC_INTR_COMMAND_2) & APIC_INTR_COMMAND_2_MASK) - | (gKernelArgs.arch_args.cpu_apic_id[i] << 24); - apic_write(APIC_INTR_COMMAND_2, config); - config = (apic_read(APIC_INTR_COMMAND_1) & 0xfff00000) - | APIC_TRIGGER_MODE_LEVEL | APIC_DELIVERY_MODE_INIT; - apic_write(APIC_INTR_COMMAND_1, config); - - // wait for pending to end - while ((apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) != 0) - asm volatile ("pause;"); - - /* wait 10ms */ - spin(10000); - /* is this a local apic or an 82489dx ? */ - numStartups = (gKernelArgs.arch_args.cpu_apic_version[i] & 0xf0) - ? 2 : 0; - for (j = 0; j < numStartups; j++) { - /* it's a local apic, so send STARTUP IPIs */ - apic_write(APIC_ERROR_STATUS, 0); - - /* set target pe */ - config = (apic_read(APIC_INTR_COMMAND_2) & APIC_INTR_COMMAND_2_MASK) - | (gKernelArgs.arch_args.cpu_apic_id[i] << 24); - apic_write(APIC_INTR_COMMAND_2, config); - - /* send the IPI */ - config = (apic_read(APIC_INTR_COMMAND_1) & 0xfff0f800) - | APIC_DELIVERY_MODE_STARTUP | (trampolineCode >> 12); - apic_write(APIC_INTR_COMMAND_1, config); - - /* wait */ - spin(200); - - while ((apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) != 0) - asm volatile ("pause;"); - } - - // Wait for the trampoline code to clear the final stack location. - // This serves as a notification for us that it has loaded the address - // and it is safe for us to overwrite it to trampoline the next CPU. - while (args->sentinel != 0) - spin(1000); - } - - TRACE(("done trampolining\n")); -} - - -void -arch_smp_add_safemode_menus(Menu *menu) -{ - MenuItem *item; - - if (gKernelArgs.arch_args.ioapic_phys != 0) { - menu->AddItem(item = new(nothrow) MenuItem("Disable IO-APIC")); - item->SetType(MENU_ITEM_MARKABLE); - item->SetData(B_SAFEMODE_DISABLE_IOAPIC); - item->SetHelpText("Disables using the IO APIC for interrupt routing, " - "forcing the use of the legacy PIC instead."); - } - - if (gKernelArgs.arch_args.apic_phys != 0) { - menu->AddItem(item = new(nothrow) MenuItem("Disable local APIC")); - item->SetType(MENU_ITEM_MARKABLE); - item->SetData(B_SAFEMODE_DISABLE_APIC); - item->SetHelpText("Disables using the local APIC, also disables SMP."); - - cpuid_info info; - if (get_current_cpuid(&info, 1, 0) == B_OK - && (info.regs.ecx & IA32_FEATURE_EXT_X2APIC) != 0) { - menu->AddItem(item = new(nothrow) MenuItem("Disable X2APIC")); - item->SetType(MENU_ITEM_MARKABLE); - item->SetData(B_SAFEMODE_DISABLE_X2APIC); - item->SetHelpText("Disables using X2APIC."); - } - - if (get_current_cpuid(&info, 7, 0) == B_OK - && ((info.regs.ebx & (IA32_FEATURE_SMEP - | IA32_FEATURE_SMAP)) != 0)) { - menu->AddItem(item = new(nothrow) MenuItem( - "Disable SMEP and SMAP")); - item->SetType(MENU_ITEM_MARKABLE); - item->SetData(B_SAFEMODE_DISABLE_SMEP_SMAP); - item->SetHelpText("Disables using SMEP and SMAP."); - } - } - - if (gKernelArgs.num_cpus < 2) - return; - - item = new(nothrow) MenuItem("Disable SMP"); - menu->AddItem(item); - item->SetData(B_SAFEMODE_DISABLE_SMP); - item->SetType(MENU_ITEM_MARKABLE); - item->SetHelpText("Disables all but one CPU core."); -} - - -void -arch_smp_init(void) -{ - cpuid_info info; - if (get_current_cpuid(&info, 1, 0) != B_OK) - return; - - if ((info.eax_1.features & IA32_FEATURE_APIC) == 0) { - // Local APICs aren't present; As they form the basis for all inter CPU - // communication and therefore SMP, we don't need to go any further. - TRACE(("no local APIC present, not attempting SMP init\n")); - return; - } - - // first try to find ACPI tables to get MP configuration as it handles - // physical as well as logical MP configurations as in multiple cpus, - // multiple cores or hyper threading. - if (acpi_do_smp_config() == B_OK) { - TRACE(("smp init success\n")); - return; - } - - // Everything failed or we are not running an SMP system, reset anything - // that might have been set through an incomplete configuration attempt. - gKernelArgs.arch_args.apic_phys = 0; - gKernelArgs.arch_args.ioapic_phys = 0; - gKernelArgs.num_cpus = 1; -} diff --git a/src/system/boot/platform/efi/arch/x86_64/arch_smp_64.cpp b/src/system/boot/platform/efi/arch/x86_64/arch_smp_64.cpp new file mode 100644 index 0000000000..d30c8a3d85 --- /dev/null +++ b/src/system/boot/platform/efi/arch/x86_64/arch_smp_64.cpp @@ -0,0 +1,106 @@ +/* + * Copyright 2021-2022 Haiku, Inc. All rights reserved. + * Released under the terms of the MIT License. + * + * Copyright 2008, Dustin Howett, dustin.howett@gmail.com. All rights reserved. + * Copyright 2004-2010, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + * + * Copyright 2001, Travis Geiselbrecht. All rights reserved. + * Distributed under the terms of the NewOS License. + */ + + +#include "arch_smp.h" + +#include + + +//#define TRACE_SMP +#ifdef TRACE_SMP +# define TRACE(x...) dprintf(x) +#else +# define TRACE(x...) ; +#endif + + +extern "C" void long_smp_trampoline(void); +extern "C" void long_smp_trampoline_args(void); +extern "C" void long_smp_trampoline_end(void); +extern "C" uint64 gLongGDTR; + +struct gdtr { + uint16 limit; + uint32 base; + unsigned char null[8]; + unsigned char code[8]; + unsigned char data[8]; +} __attribute__((packed)); + +// Arguments passed to the SMP trampoline. +struct long_trampoline_args { + uint32 trampoline; // Trampoline address + uint32 gdt32; // 32-bit GDTR + uint32 pml4; // 64-bit PML4 + uint32 gdt64; // 64-bit GDTR + uint64 kernel_entry; // Kernel entry point + uint64 kernel_args; // Kernel arguments + uint64 current_cpu; // CPU number + uint64 stack_top; // Kernel stack + volatile uint64 sentinel; // Sentinel, AP sets to 0 when finished + + // smp_boot_other_cpus puts the GDTR here. + struct gdtr gdtr; +}; + + +void +copy_trampoline_code(uint64 trampolineCode, uint64 trampolineStack) +{ + TRACE("copying the trampoline code to %p from %p\n", (char*)trampolineCode, (const void*)&long_smp_trampoline); + TRACE("size of trampoline code = %" PRIu64 " bytes\n", (uint64)&long_smp_trampoline_end - (uint64)&long_smp_trampoline); + memcpy((char *)trampolineCode, (const void*)&long_smp_trampoline, + (uint64)&long_smp_trampoline_end - (uint64)&long_smp_trampoline); +} + + +void +prepare_trampoline_args(uint64 trampolineCode, uint64 trampolineStack, + uint32 pagedir, uint64 kernelEntry, addr_t virtKernelArgs, + uint32 currentCpu) +{ + long_trampoline_args* args = (long_trampoline_args *)trampolineStack; + args->trampoline = trampolineCode; + + args->gdt32 = (uint64) &args->gdtr; + args->gdtr.limit = 23; + args->gdtr.base = (uint32)(uint64)args->gdtr.null; + #define COPY_ARRAY(A, X0, X1, X2, X3, X4, X5, X6, X7) \ + { A[0] = X0; A[1] = X1; A[2] = X2; A[3] = X3; A[4] = X4; A[5] = X5; A[6] = X6; A[7] = X7; } + COPY_ARRAY(args->gdtr.null, 0, 0, 0, 0, 0, 0, 0, 0); + COPY_ARRAY(args->gdtr.code, 0xff, 0xff, 0, 0, 0, 0x9a, 0xcf, 0); + COPY_ARRAY(args->gdtr.data, 0xff, 0xff, 0, 0, 0, 0x92, 0xcf, 0); + #undef COPY_ARRAY + args->gdt64 = (uint32_t)(uint64_t)&gLongGDTR; + + args->pml4 = pagedir; + args->kernel_entry = kernelEntry; + args->kernel_args = virtKernelArgs; + args->current_cpu = currentCpu; + args->stack_top = gKernelArgs.cpu_kstack[currentCpu].start + + gKernelArgs.cpu_kstack[currentCpu].size; + args->sentinel = 1; + + // put the args in the right place + uint32 * args_ptr = + (uint32 *)(trampolineCode + (uint64)long_smp_trampoline_args - (uint64)long_smp_trampoline); + *args_ptr = (uint32)trampolineStack; +} + + +uint32 +get_sentinel(uint64 trampolineStack) +{ + long_trampoline_args* args = (long_trampoline_args *)trampolineStack; + return args->sentinel; +} diff --git a/src/system/boot/platform/efi/arch/x86_64/arch_start.cpp b/src/system/boot/platform/efi/arch/x86_64/arch_start.cpp index 2cae36ee91..1564ca5baa 100644 --- a/src/system/boot/platform/efi/arch/x86_64/arch_start.cpp +++ b/src/system/boot/platform/efi/arch/x86_64/arch_start.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2014-2021 Haiku, Inc. All rights reserved. + * Copyright 2014-2022 Haiku, Inc. All rights reserved. * Copyright 2013-2014, Fredrik Holmqvist, fredrik.holmqvist@gmail.com. * Copyright 2014, Henry Harrington, henry.harrington@gmail.com. * All rights reserved. @@ -125,7 +125,7 @@ arch_start_kernel(addr_t kernelEntry) arch_mmu_post_efi_setup(memory_map_size, memory_map, descriptor_size, descriptor_version); - smp_boot_other_cpus(final_pml4, kernelEntry); + smp_boot_other_cpus(final_pml4, kernelEntry, (addr_t)&gKernelArgs); // Enter the kernel! arch_enter_kernel(final_pml4, kernelEntry, diff --git a/src/system/boot/platform/efi/arch/x86_64/smp_trampoline.S b/src/system/boot/platform/efi/arch/x86_64/long_smp_trampoline.S similarity index 82% rename from src/system/boot/platform/efi/arch/x86_64/smp_trampoline.S rename to src/system/boot/platform/efi/arch/x86_64/long_smp_trampoline.S index 54b5dfcf38..4016ca4adf 100644 --- a/src/system/boot/platform/efi/arch/x86_64/smp_trampoline.S +++ b/src/system/boot/platform/efi/arch/x86_64/long_smp_trampoline.S @@ -4,12 +4,12 @@ ** Distributed under the terms of the NewOS License. */ -// Relocatable. Before calling, smp_trampoline_args should point -// to a struct trampoline_args (see smp.cpp). This pointer should +// Relocatable. Before calling, long_smp_trampoline_args should point +// to a struct long_trampoline_args (see smp.cpp). This pointer should // be 16-byte aligned, below 1MB and identity-mapped. -.globl smp_trampoline -.globl smp_trampoline_end -.globl smp_trampoline_args +.globl long_smp_trampoline +.globl long_smp_trampoline_end +.globl long_smp_trampoline_args #include @@ -17,13 +17,13 @@ #include "mmu.h" .code16 -smp_trampoline: +long_smp_trampoline: cli // movl 0xdeadbeef, esi .byte 0x66 .byte 0xbe -smp_trampoline_args: +long_smp_trampoline_args: .long 0xdeadbeef // Load the trampoline args into ss. @@ -42,13 +42,13 @@ smp_trampoline_args: movl %eax,%cr0 pushl $8 - leal (trampoline_32 - smp_trampoline)(%ebx), %eax + leal (long_trampoline_32 - long_smp_trampoline)(%ebx), %eax pushl %eax .byte 0x66 .code32 lret -trampoline_32: +long_trampoline_32: mov $0x10, %ax mov %ax, %ds mov %ax, %es @@ -86,7 +86,7 @@ trampoline_32: // Jump into the 64-bit code segment. pushl $KERNEL_CODE_SELECTOR - leal (.Llmode - smp_trampoline)(%ebx), %eax + leal (.Llmode - long_smp_trampoline)(%ebx), %eax pushl %eax lret .align 8 @@ -126,4 +126,4 @@ trampoline_32: // Call the entry point. call *%rax -smp_trampoline_end: +long_smp_trampoline_end: diff --git a/src/system/boot/platform/efi/smp.cpp b/src/system/boot/platform/efi/smp.cpp index 827f674624..5fc15da588 100644 --- a/src/system/boot/platform/efi/smp.cpp +++ b/src/system/boot/platform/efi/smp.cpp @@ -48,12 +48,14 @@ smp_init_other_cpus(void) void -smp_boot_other_cpus(addr_t pageTable, addr_t kernelEntry) +smp_boot_other_cpus(addr_t pageTable, + addr_t kernelEntry, addr_t virtKernelArgs) { if (gKernelArgs.num_cpus < 2) return; - arch_smp_boot_other_cpus(pageTable, kernelEntry); + arch_smp_boot_other_cpus(pageTable, + kernelEntry, virtKernelArgs); } diff --git a/src/system/boot/platform/efi/smp.h b/src/system/boot/platform/efi/smp.h index 93281e5cb8..88fd4c63b4 100644 --- a/src/system/boot/platform/efi/smp.h +++ b/src/system/boot/platform/efi/smp.h @@ -19,7 +19,8 @@ extern "C" { extern void smp_init(void); extern void smp_init_other_cpus(void); -extern void smp_boot_other_cpus(addr_t pageTable, addr_t kernelEntry); +extern void smp_boot_other_cpus(addr_t pageTable, + addr_t kernelEntry, addr_t virtKernelArgs); extern int smp_get_current_cpu(void); #ifdef __cplusplus