Debugger: add AVX support

- Unify storage of "FPU" registers between debugger and signal handler
  to use xsave format on both sides
- Handle YMM registers in Debugger (they are the same as XMM, but wider)

Tested:
- The system still boots with and without AVX
- The hello_avx test program can be debugged and the full value of YMM is visible

This changes the API of vregs in signal.h but not the ABI (structure are
declared differently but memory layout is the same). This changes the
API and ABI of arch_debugger.h for x86_64, but I don't think anything
outside Haiku uses it (did we ever have a 64bit compatible gdb?)

Change-Id: If93680ffa0339c19bab517876b4e029f5d66b240
Reviewed-on: https://review.haiku-os.org/c/haiku/+/3038
Reviewed-by: Rene Gollent <[email protected]>

headers/os/arch/x86_64/arch_debugger.h

@@ -6,38 +6,14 @@
 #define _ARCH_X86_64_DEBUGGER_H
 
 
-typedef struct x86_64_fp_register {
-	uint8		value[10];
-	uint8		reserved[6];
-} x86_64_fp_register;
+#if __x86_64__
 
 
-typedef struct x86_64_xmm_register {
-	uint8		value[16];
-} x86_64_xmm_register;
-
-
-typedef struct x86_64_extended_registers {
-	uint16					control;
-	uint16					status;
-	uint8					tag;
-	uint8					reserved1;
-	uint16					opcode;
-	uint64					instruction_pointer;
-	uint64					data_pointer;
-	uint32					mxcsr;
-	uint32					mxcsr_mask;
-	union {
-		x86_64_fp_register	fp_registers[8];	// st0-st7
-		x86_64_fp_register	mmx_registers[8];	// mm0-mm7
-	};
-	x86_64_xmm_register		xmm_registers[16];	// xmm0-xmm15
-	uint8					reserved2[96];		// 416 - 512
-} x86_64_extended_registers;
+#include <posix/arch/x86_64/signal.h>
 
 
 struct x86_64_debug_cpu_state {
-	x86_64_extended_registers	extended_registers;
+	struct savefpu	extended_registers;
 
 	uint64	gs;
 	uint64	fs;
@@ -68,4 +44,7 @@ struct x86_64_debug_cpu_state {
 } __attribute__((aligned(16)));
 
 
+#endif
+
+
 #endif	// _ARCH_X86_64_DEBUGGER_H

headers/posix/arch/x86_64/signal.h

@@ -12,64 +12,18 @@
 
 #if __x86_64__
 
-
-struct fp_stack {
-	unsigned char		st0[10];
-	unsigned char		_reserved_42_47[6];
-	unsigned char		st1[10];
-	unsigned char		_reserved_58_63[6];
-	unsigned char		st2[10];
-	unsigned char		_reserved_74_79[6];
-	unsigned char		st3[10];
-	unsigned char		_reserved_90_95[6];
-	unsigned char		st4[10];
-	unsigned char		_reserved_106_111[6];
-	unsigned char		st5[10];
-	unsigned char		_reserved_122_127[6];
-	unsigned char		st6[10];
-	unsigned char		_reserved_138_143[6];
-	unsigned char		st7[10];
-	unsigned char		_reserved_154_159[6];
+struct x86_64_fp_register {
+	unsigned char value[10];
+	unsigned char reserved[6];
 };
 
-struct mmx_regs {
-	unsigned char		mm0[10];
-	unsigned char		_reserved_42_47[6];
-	unsigned char		mm1[10];
-	unsigned char		_reserved_58_63[6];
-	unsigned char		mm2[10];
-	unsigned char		_reserved_74_79[6];
-	unsigned char		mm3[10];
-	unsigned char		_reserved_90_95[6];
-	unsigned char		mm4[10];
-	unsigned char		_reserved_106_111[6];
-	unsigned char		mm5[10];
-	unsigned char		_reserved_122_127[6];
-	unsigned char		mm6[10];
-	unsigned char		_reserved_138_143[6];
-	unsigned char		mm7[10];
-	unsigned char		_reserved_154_159[6];
+
+struct x86_64_xmm_register {
+	unsigned char value[16];
 };
 
-struct xmm_regs {
-	unsigned char		xmm0[16];
-	unsigned char		xmm1[16];
-	unsigned char		xmm2[16];
-	unsigned char		xmm3[16];
-	unsigned char		xmm4[16];
-	unsigned char		xmm5[16];
-	unsigned char		xmm6[16];
-	unsigned char		xmm7[16];
-	unsigned char		xmm8[16];
-	unsigned char		xmm9[16];
-	unsigned char		xmm10[16];
-	unsigned char		xmm11[16];
-	unsigned char		xmm12[16];
-	unsigned char		xmm13[16];
-	unsigned char		xmm14[16];
-	unsigned char		xmm15[16];
-};
 
+// The layout of this struct matches the one used by the FXSAVE instruction
 struct fpu_state {
 	unsigned short		control;
 	unsigned short		status;
@@ -81,26 +35,33 @@ struct fpu_state {
 	unsigned int		mscsr_mask;
 
 	union {
-		struct fp_stack	fp;
-		struct mmx_regs	mmx;
+		struct x86_64_fp_register fp[8];
+		struct x86_64_fp_register mmx[8];
 	};
 
-	struct xmm_regs		xmm;
+	struct x86_64_xmm_register		xmm[16];
 	unsigned char		_reserved_416_511[96];
 };
 
+
 struct xstate_hdr {
 	unsigned long		bv;
 	unsigned long		xcomp_bv;
 	unsigned char		_reserved[48];
 };
 
+
+// The layout of this struct matches the one used by the FXSAVE instruction on
+// an AVX CPU
 struct savefpu {
-	struct fpu_state	fp_fxsave;
-	struct xstate_hdr	fp_xstate;
-	unsigned long		fp_ymm[16][2];
+	struct fpu_state			fp_fxsave;
+	struct xstate_hdr			fp_xstate;
+	struct x86_64_xmm_register	fp_ymm[16];
+		// The high half of the YMM registers, to combine with the low half
+		// found in fp_fxsave.xmm
 };
 
+
 struct vregs {
 	unsigned long		rax;
 	unsigned long		rbx;