x86_64: Implemented exception handling.

headers/private/kernel/arch/common_x86/cpu.h

@@ -225,6 +225,9 @@ enum x86_vendors {
 
 #define nop() __asm__ ("nop"::)
 
+#define read_cr2(value) \
+	__asm__("mov	%%cr2,%0" : "=r" (value))
+
 #define read_cr3(value) \
 	__asm__("mov	%%cr3,%0" : "=r" (value))
 

headers/private/kernel/arch/x86_64/arch_cpu.h

@@ -71,7 +71,7 @@ struct iframe {
 	unsigned long user_ss;
 } _PACKED;
 
-#define IFRAME_IS_USER(f)	((f)->cs == USER_CODE_SEG)
+#define IFRAME_IS_USER(f)	(((f)->cs & DPL_USER) == DPL_USER)
 
 
 typedef struct arch_cpu_info {
@@ -87,9 +87,8 @@ typedef struct arch_cpu_info {
 	int					model;
 	int					extended_model;
 
-	// TSS and double fault TSS for this CPU.
+	// TSS for this CPU.
 	struct tss			tss;
-	struct tss			double_fault_tss;
 } arch_cpu_info;
 
 

headers/private/kernel/arch/x86_64/arch_kernel_args.h

@@ -1,6 +1,14 @@
+/*
+ * Copyright 2012, Alex Smith, [email protected].
+ * Distributed under the terms of the MIT License.
+ */
 #ifndef _KERNEL_ARCH_X86_64_KERNEL_ARGS_H
 #define _KERNEL_ARCH_X86_64_KERNEL_ARGS_H
 
+
+// x86_64 kernel is loaded loaded by the x86 bootloader, kernel_args is
+// identical to x86.
 #include "../x86/arch_kernel_args.h"
 
+
 #endif	/* _KERNEL_ARCH_X86_64_KERNEL_ARGS_H */

headers/private/kernel/arch/x86_64/descriptors.h

@@ -59,13 +59,21 @@ struct interrupt_descriptor {
 	uint32 base0 : 16;
 	uint32 sel : 16;
 	uint32 ist : 3;
-	uint32 unused : 5;
+	uint32 unused1 : 5;
-	uint32 flags : 8;
+	uint32 type : 4;
+	uint32 unused2 : 1;
+	uint32 dpl : 2;
+	uint32 present : 1;
 	uint32 base1 : 16;
 	uint32 base2 : 32;
 	uint32 reserved : 32;
 } _PACKED;
 
+struct gdt_idt_descr {
+	uint16 limit;
+	addr_t base;
+} _PACKED;
+
 enum descriptor_privilege_levels {
 	DPL_KERNEL = 0,
 	DPL_USER = 3,
@@ -90,6 +98,11 @@ enum descriptor_types {
 	DT_CODE_DATA_SEGMENT = 1,
 };
 
+enum gate_types {
+	GATE_INTERRUPT = 14,
+	GATE_TRAP = 15,
+};
+
 
 static inline void
 clear_segment_descriptor(segment_descriptor* desc)
@@ -144,6 +157,24 @@ set_tss_descriptor(segment_descriptor* _desc, uint64 base, uint32 limit)
 }
 
 
+static inline void
+set_interrupt_descriptor(interrupt_descriptor* desc, uint64 addr, uint32 type,
+	uint16 seg, uint32 dpl, uint32 ist)
+{
+	desc->base0 = (addr & 0xFFFF);
+	desc->base1 = ((addr >> 16) & 0xFFFF);
+	desc->base2 = ((addr >> 32) & 0xFFFFFFFF);
+	desc->sel = seg;
+	desc->ist = ist;
+	desc->type = type;
+	desc->dpl = dpl;
+	desc->present = 1;
+	desc->unused1 = 0;
+	desc->unused2 = 0;
+	desc->reserved = 0;
+}
+
+
 #endif	/* _ASSEMBLER */
 
 #endif	/* _KERNEL_ARCH_X86_64_DESCRIPTORS_H */

src/system/boot/platform/bios_ia32/long.cpp

@@ -20,12 +20,6 @@
 #include "mmu.h"
 
 
-struct gdt_idt_descr {
-	uint16 limit;
-	addr_t base;
-} _PACKED;
-
-
 /*! Convert a 32-bit address to a 64-bit address. */
 static inline uint64
 fix_address(uint64 address)

src/system/kernel/arch/x86_64/Jamfile

@@ -13,6 +13,7 @@ KernelMergeObject kernel_arch_x86_64.o :
 	arch_debug_console.cpp
 	arch_elf.cpp
 	arch_int.cpp
+	arch_interrupts.S
 	arch_platform.cpp
 	arch_real_time_clock.cpp
 	arch_smp.cpp

src/system/kernel/arch/x86_64/arch_cpu.cpp

@@ -14,37 +14,35 @@
 
 
 status_t
-arch_cpu_preboot_init_percpu(kernel_args *args, int cpu)
+arch_cpu_preboot_init_percpu(kernel_args* args, int cpu)
 {
 	return B_OK;
 }
 
 
 status_t
-arch_cpu_init_percpu(kernel_args *args, int cpu)
+arch_cpu_init_percpu(kernel_args* args, int cpu)
 {
 	return B_OK;
 }
 
 
 status_t
-arch_cpu_init(kernel_args *args)
+arch_cpu_init(kernel_args* args)
-{
-	dprintf("not implemented...\n");
-	while (1);
-	return B_OK;
-}
-
-
-status_t
-arch_cpu_init_post_vm(kernel_args *args)
 {
 	return B_OK;
 }
 
 
 status_t
-arch_cpu_init_post_modules(kernel_args *args)
+arch_cpu_init_post_vm(kernel_args* args)
+{
+	return B_OK;
+}
+
+
+status_t
+arch_cpu_init_post_modules(kernel_args* args)
 {
 	return B_OK;
 }
@@ -79,8 +77,8 @@ arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages)
 
 
 ssize_t
-arch_cpu_user_strlcpy(char *to, const char *from, size_t size,
+arch_cpu_user_strlcpy(char* to, const char* from, size_t size,
-	addr_t *faultHandler)
+	addr_t* faultHandler)
 {
 	int fromLength = 0;
 	addr_t oldFaultHandler = *faultHandler;
@@ -99,6 +97,7 @@ arch_cpu_user_strlcpy(char *to, const char *from, size_t size,
 				break;
 		}
 	}
+
 	// count any leftover from chars
 	while (*from++ != '\0') {
 		fromLength++;
@@ -114,11 +113,11 @@ error:
 
 
 status_t
-arch_cpu_user_memcpy(void *to, const void *from, size_t size,
+arch_cpu_user_memcpy(void* to, const void* from, size_t size,
-	addr_t *faultHandler)
+	addr_t* faultHandler)
 {
-	char *d = (char *)to;
+	char* d = (char*)to;
-	const char *s = (const char *)from;
+	const char* s = (const char*)from;
 	addr_t oldFaultHandler = *faultHandler;
 
 	// this check is to trick the gcc4 compiler and have it keep the error label
@@ -141,9 +140,9 @@ error:
 
 
 status_t
-arch_cpu_user_memset(void *s, char c, size_t count, addr_t *faultHandler)
+arch_cpu_user_memset(void* s, char c, size_t count, addr_t* faultHandler)
 {
-	char *xs = (char *)s;
+	char* xs = (char*)s;
 	addr_t oldFaultHandler = *faultHandler;
 
 	// this check is to trick the gcc4 compiler and have it keep the error label
@@ -179,24 +178,22 @@ arch_cpu_idle(void)
 
 
 void
-arch_cpu_sync_icache(void *address, size_t length)
+arch_cpu_sync_icache(void* address, size_t length)
 {
-	// instruction cache is always consistent on x86
+	// Instruction cache is always consistent on x86.
 }
 
 
 void
 arch_cpu_memory_read_barrier(void)
 {
-	asm volatile ("lock;" : : : "memory");
+	asm volatile("lfence" : : : "memory");
-	asm volatile ("addl $0, 0(%%esp);" : : : "memory");
 }
 
 
 void
 arch_cpu_memory_write_barrier(void)
 {
-	asm volatile ("lock;" : : : "memory");
+	asm volatile("sfence" : : : "memory");
-	asm volatile ("addl $0, 0(%%esp);" : : : "memory");
 }
 

src/system/kernel/arch/x86_64/arch_int.cpp

@@ -6,7 +6,102 @@
 
 #include <int.h>
 
-#include <arch/int.h>
+#include <string.h>
+#include <stdio.h>
+
+#include <boot/kernel_args.h>
+#include <cpu.h>
+
+#include <arch/x86_64/descriptors.h>
+
+
+typedef void interrupt_handler_function(iframe* frame);
+
+static const char *kInterruptNames[] = {
+	/*  0 */ "Divide Error Exception",
+	/*  1 */ "Debug Exception",
+	/*  2 */ "NMI Interrupt",
+	/*  3 */ "Breakpoint Exception",
+	/*  4 */ "Overflow Exception",
+	/*  5 */ "BOUND Range Exceeded Exception",
+	/*  6 */ "Invalid Opcode Exception",
+	/*  7 */ "Device Not Available Exception",
+	/*  8 */ "Double Fault Exception",
+	/*  9 */ "Coprocessor Segment Overrun",
+	/* 10 */ "Invalid TSS Exception",
+	/* 11 */ "Segment Not Present",
+	/* 12 */ "Stack Fault Exception",
+	/* 13 */ "General Protection Exception",
+	/* 14 */ "Page-Fault Exception",
+	/* 15 */ "-",
+	/* 16 */ "x87 FPU Floating-Point Error",
+	/* 17 */ "Alignment Check Exception",
+	/* 18 */ "Machine-Check Exception",
+	/* 19 */ "SIMD Floating-Point Exception",
+};
+
+static const uint32 kInterruptHandlerTableSize = 256;
+
+static interrupt_descriptor* sIDT;
+interrupt_handler_function* gInterruptHandlerTable[kInterruptHandlerTableSize];
+
+extern uint8 isr_array[kInterruptHandlerTableSize][16];
+
+
+static const char*
+exception_name(unsigned long number, char* buffer, size_t bufferSize)
+{
+	if (number >= 0
+			&& number < (sizeof(kInterruptNames) / sizeof(kInterruptNames[0])))
+		return kInterruptNames[number];
+
+	snprintf(buffer, bufferSize, "exception %lu", number);
+	return buffer;
+}
+
+
+static void
+invalid_exception(iframe* frame)
+{
+	char name[32];
+	panic("unhandled trap 0x%lx (%s) at ip 0x%lx\n",
+		frame->vector, exception_name(frame->vector, name, sizeof(name)),
+		frame->rip);
+}
+
+
+static void
+fatal_exception(iframe *frame)
+{
+	char name[32];
+	panic("fatal exception 0x%lx (%s) at ip 0x%lx, error code 0x%lx\n",
+		frame->vector, exception_name(frame->vector, name, sizeof(name)),
+		frame->rip, frame->error_code);
+}
+
+
+static void
+unexpected_exception(iframe* frame)
+{
+	char name[32];
+	panic("fatal exception 0x%lx (%s) at ip 0x%lx, error code 0x%lx\n",
+		frame->vector, exception_name(frame->vector, name, sizeof(name)),
+		frame->rip, frame->error_code);
+}
+
+
+static void
+page_fault_exception(iframe* frame)
+{
+	unsigned long cr2;
+	read_cr2(cr2);
+
+	panic("page fault exception at ip 0x%lx on 0x%lx, error code 0x%lx\n",
+		frame->rip, cr2, frame->error_code);
+}
+
+
+// #pragma mark -
 
 
 void
@@ -31,14 +126,71 @@ arch_int_configure_io_interrupt(int irq, uint32 config)
 
 
 status_t
-arch_int_init(struct kernel_args *args)
+arch_int_init(kernel_args* args)
 {
+	// The loader allocates an empty IDT for us.
+	sIDT = (interrupt_descriptor*)args->arch_args.vir_idt;
+
+	// Fill out the IDT, pointing each entry to the corresponding entry in the
+	// ISR array created in arch_interrupts.S (see there to see how this works).
+	for(uint32 i = 0; i < kInterruptHandlerTableSize; i++) {
+		// x86_64 removes task gates, therefore we cannot use a separate TSS
+		// for the double fault exception. However, instead it adds a new stack
+		// switching mechanism, the IST. The IST is a table of stack addresses
+		// in the TSS. If the IST field of an interrupt descriptor is non-zero,
+		// the CPU will switch to the stack specified by that IST entry when
+		// handling that interrupt. So, we use IST entry 1 to store the double
+		// fault stack address (this is set up in arch_cpu.cpp).
+		uint32 ist = (i == 8) ? 1 : 0;
+
+		set_interrupt_descriptor(&sIDT[i], (addr_t)&isr_array[i],
+			GATE_INTERRUPT, KERNEL_CODE_SEG, DPL_KERNEL, ist);
+	}
+
+	interrupt_handler_function** table = gInterruptHandlerTable;
+
+	// Initialize the interrupt handler table.
+	for (uint32 i = 0; i < ARCH_INTERRUPT_BASE; i++)
+		table[i] = invalid_exception;
+	//for (uint32 i = ARCH_INTERRUPT_BASE; i < INTERRUPT_HANDLER_TABLE_SIZE; i++)
+	//	table[i] = hardware_interrupt;
+
+	table[0]  = unexpected_exception;	// Divide Error Exception (#DE)
+	//table[1]  = x86_handle_debug_exception; // Debug Exception (#DB)
+	table[1]  = unexpected_exception;
+	table[2]  = fatal_exception;		// NMI Interrupt
+	//table[3]  = x86_handle_breakpoint_exception; // Breakpoint Exception (#BP)
+	table[3]  = unexpected_exception;
+	table[4]  = unexpected_exception;	// Overflow Exception (#OF)
+	table[5]  = unexpected_exception;	// BOUND Range Exceeded Exception (#BR)
+	table[6]  = unexpected_exception;	// Invalid Opcode Exception (#UD)
+	table[7]  = fatal_exception;		// Device Not Available Exception (#NM)
+	table[8]  = fatal_exception;		 // Double Fault Exception (#DF)
+	table[9]  = fatal_exception;		// Coprocessor Segment Overrun
+	table[10] = fatal_exception;		// Invalid TSS Exception (#TS)
+	table[11] = fatal_exception;		// Segment Not Present (#NP)
+	table[12] = fatal_exception;		// Stack Fault Exception (#SS)
+	table[13] = unexpected_exception;	// General Protection Exception (#GP)
+	table[14] = page_fault_exception;	// Page-Fault Exception (#PF)
+	table[16] = unexpected_exception;	// x87 FPU Floating-Point Error (#MF)
+	table[17] = unexpected_exception;	// Alignment Check Exception (#AC)
+	table[18] = fatal_exception;		// Machine-Check Exception (#MC)
+	table[19] = unexpected_exception;	// SIMD Floating-Point Exception (#XF)
+
+	// Load the IDT.
+	gdt_idt_descr idtr = {
+		256 * sizeof(interrupt_descriptor) - 1,
+		(addr_t)sIDT
+	};
+	asm volatile("lidt %0" :: "m"(idtr));
+
+	panic("not implemented\n");
 	return B_OK;
 }
 
 
 status_t
-arch_int_init_post_vm(struct kernel_args *args)
+arch_int_init_post_vm(kernel_args* args)
 {
 	return B_OK;
 }
@@ -52,7 +204,7 @@ arch_int_init_io(kernel_args* args)
 
 
 status_t
-arch_int_init_post_device_manager(struct kernel_args *args)
+arch_int_init_post_device_manager(kernel_args* args)
 {
 	return B_OK;
 }

src/system/kernel/arch/x86_64/arch_interrupts.S

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include <asm_defs.h>
+
+#include <arch/x86_64/descriptors.h>
+#include <arch/x86_64/arch_cpu.h>
+
+#include "asm_offsets.h"
+
+
+// Push the remainder of the interrupt frame onto the stack.
+#define PUSH_IFRAME_BOTTOM(iframeType)	\
+	push	%rax;						\
+	push	%rbx;						\
+	push	%rcx;						\
+	push	%rdx;						\
+	push	%rdi;						\
+	push	%rsi;						\
+	push	%rbp;						\
+	push	%r8;						\
+	push	%r9;						\
+	push	%r10;						\
+	push	%r11;						\
+	push	%r12;						\
+	push	%r13;						\
+	push	%r14;						\
+	push	%r15;						\
+	push	$iframeType;
+
+// Restore the interrupt frame.
+#define RESTORE_IFRAME()				\
+	add		$8, %rsp;					\
+	pop		%r15;						\
+	pop		%r14;						\
+	pop		%r13;						\
+	pop		%r12;						\
+	pop		%r11;						\
+	pop		%r10;						\
+	pop		%r9;						\
+	pop		%r8;						\
+	pop		%rbp;						\
+	pop		%rsi;						\
+	pop		%rdi;						\
+	pop		%rdx;						\
+	pop		%rcx;						\
+	pop		%rbx;						\
+	pop		%rax;
+
+// The following code defines the interrupt service routines for all 256
+// interrupts. It creates a block of handlers, each 16 bytes, that the IDT
+// initialization code just loops through.
+
+// Interrupt with no error code, pushes a 0 error code.
+#define DEFINE_ISR(nr)					\
+	.align 16;							\
+	push	$0;							\
+	push	$nr;						\
+	jmp		int_bottom;
+
+// Interrupt with an error code.
+#define DEFINE_ISR_E(nr)				\
+	.align 16;							\
+	push	$nr;						\
+	jmp		int_bottom;
+
+// Array of interrupt service routines.
+.align 16
+SYMBOL(isr_array):
+	// Exceptions (0-19) and reserved interrupts (20-31).
+	DEFINE_ISR(0)
+	DEFINE_ISR(1)
+	DEFINE_ISR(2)
+	DEFINE_ISR(3)
+	DEFINE_ISR(4)
+	DEFINE_ISR(5)
+	DEFINE_ISR(6)
+	DEFINE_ISR(7)
+	DEFINE_ISR_E(8)
+	DEFINE_ISR(9)
+	DEFINE_ISR_E(10)
+	DEFINE_ISR_E(11)
+	DEFINE_ISR_E(12)
+	DEFINE_ISR_E(13)
+	DEFINE_ISR_E(14)
+	DEFINE_ISR(15)
+	DEFINE_ISR(16)
+	DEFINE_ISR_E(17)
+	DEFINE_ISR(18)
+	DEFINE_ISR(19)
+	DEFINE_ISR(20)
+	DEFINE_ISR(21)
+	DEFINE_ISR(22)
+	DEFINE_ISR(23)
+	DEFINE_ISR(24)
+	DEFINE_ISR(25)
+	DEFINE_ISR(26)
+	DEFINE_ISR(27)
+	DEFINE_ISR(28)
+	DEFINE_ISR(29)
+	DEFINE_ISR(30)
+	DEFINE_ISR(31)
+
+	// User-defined ISRs (32-255) - none take an error code.
+	.Lintr = 32
+	.rept 224
+		DEFINE_ISR(.Lintr)
+		.Lintr = .Lintr+1
+	.endr
+
+// Common interrupt handling code.
+FUNCTION(int_bottom):
+	// If coming from user-mode, need to load the kernel GS segment base.
+	testl	$3, 24(%rsp)
+	jz		1f
+	swapgs
+1:
+	// Push the rest of the interrupt frame to the stack.
+	PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER)
+
+	cld
+
+	// Frame pointer is the iframe.
+	movq	%rsp, %rbp
+
+	// Call the interrupt handler.
+	movq	%rsp, %rdi
+	movq	IFRAME_vector(%rsp), %rax
+	call	*gInterruptHandlerTable(, %rax, 8)
+
+	// Restore the saved registers.
+	RESTORE_IFRAME()
+
+	// Get rid of the error code and interrupt number, restore the previous
+	// GS base if returning to user-mode, and return.
+	addq	$16, %rsp
+	testl	$3, 8(%rsp)
+	jz		2f
+	swapgs
+2:	iretq

src/system/kernel/arch/x86_64/asm_offsets.cpp

@@ -11,7 +11,7 @@
 
 #include <computed_asm_macros.h>
 
-//#include <arch_cpu.h>
+#include <arch_cpu.h>
 //#include <cpu.h>
 //#include <ksignal.h>
 //#include <ksyscalls.h>
@@ -45,19 +45,8 @@ dummy()
 	//DEFINE_OFFSET_MACRO(THREAD, Thread, fault_handler);
 
 	// struct iframe
-	//DEFINE_SIZEOF_MACRO(IFRAME, iframe);
+	DEFINE_SIZEOF_MACRO(IFRAME, iframe);
-	//DEFINE_OFFSET_MACRO(IFRAME, iframe, cs);
+	DEFINE_OFFSET_MACRO(IFRAME, iframe, vector);
-	//DEFINE_OFFSET_MACRO(IFRAME, iframe, eax);
-	//DEFINE_OFFSET_MACRO(IFRAME, iframe, edx);
-	//DEFINE_OFFSET_MACRO(IFRAME, iframe, orig_eax);
-	//DEFINE_OFFSET_MACRO(IFRAME, iframe, vector);
-	//DEFINE_OFFSET_MACRO(IFRAME, iframe, eip);
-	//DEFINE_OFFSET_MACRO(IFRAME, iframe, flags);
-	//DEFINE_OFFSET_MACRO(IFRAME, iframe, user_esp);
-
-	// struct vm86_iframe
-	//DEFINE_SIZEOF_MACRO(VM86_IFRAME, vm86_iframe);
-	//DEFINE_OFFSET_MACRO(VM86_IFRAME, vm86_iframe, flags);
 
 	// struct syscall_info
 	//DEFINE_SIZEOF_MACRO(SYSCALL_INFO, syscall_info);