nvme: Import glue code to our PCI and VM APIs.

Largely simple and painless, thanks to libnvme's abstractions.

src/add-ons/kernel/drivers/disk/nvme/compat/libnvme_haiku.cpp

@@ -0,0 +1,200 @@
+/*
+ * Copyright 2019, Haiku, Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Augustin Cavalier <waddlesplash>
+ */
+
+#include <debug.h>
+#include <kernel/vm/vm.h>
+#include <PCI.h>
+
+extern "C" {
+#include "nvme.h"
+#include "nvme_log.h"
+#include "nvme_mem.h"
+#include "nvme_pci.h"
+}
+
+
+static pci_module_info* sPCIModule = NULL;
+
+
+// #pragma mark - memory
+
+
+int
+nvme_mem_init()
+{
+	/* nothing to do */
+	return 0;
+}
+
+
+void
+nvme_mem_cleanup()
+{
+	/* nothing to do */
+}
+
+
+void*
+nvme_mem_alloc_node(size_t size, size_t align, unsigned int node_id,
+	phys_addr_t* paddr)
+{
+	size = ROUNDUP(size, B_PAGE_SIZE);
+
+	virtual_address_restrictions virtualRestrictions = {};
+
+	physical_address_restrictions physicalRestrictions = {};
+	physicalRestrictions.alignment = align;
+
+	void* address;
+	area_id area = create_area_etc(B_SYSTEM_TEAM, "nvme physical buffer",
+		size, B_CONTIGUOUS, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA,
+		0, 0, &virtualRestrictions, &physicalRestrictions, &address);
+	if (area < 0)
+		return NULL;
+
+	if (paddr != NULL)
+		*paddr = nvme_mem_vtophys(address);
+	return address;
+}
+
+
+void*
+nvme_malloc_node(size_t size, size_t align, unsigned int node_id)
+{
+	return nvme_mem_alloc_node(size, align, node_id, NULL);
+}
+
+
+void
+nvme_free(void* addr)
+{
+	delete_area(area_for(addr));
+}
+
+
+phys_addr_t
+nvme_mem_vtophys(void* vaddr)
+{
+	physical_entry entry;
+	status_t status = get_memory_map((void*)vaddr, 1, &entry, 1);
+	if (status != B_OK) {
+		panic("nvme: get_memory_map failed for %p, error %08" B_PRIx32 "\n",
+			(void*)vaddr, status);
+		return NVME_VTOPHYS_ERROR;
+	}
+
+	return entry.address;
+}
+
+
+// #pragma mark - PCI
+
+
+int
+nvme_pci_init()
+{
+	status_t status = get_module(B_PCI_MODULE_NAME,
+		(module_info**)&sPCIModule);
+	return status;
+}
+
+
+int
+nvme_pcicfg_read32(struct pci_device* dev, uint32_t* value, uint32_t offset)
+{
+	*value = sPCIModule->read_pci_config(dev->bus, dev->dev, dev->func, offset,
+		sizeof(*value));
+	return 0;
+}
+
+
+int
+nvme_pcicfg_write32(struct pci_device* dev, uint32_t value, uint32_t offset)
+{
+	sPCIModule->write_pci_config(dev->bus, dev->dev, dev->func, offset,
+		sizeof(value), value);
+	return 0;
+}
+
+
+int
+nvme_pcicfg_map_bar(void* devhandle, unsigned int bar, bool read_only,
+	void** mapped_addr)
+{
+	struct pci_device* dev = (struct pci_device*)devhandle;
+	pci_info* info = (pci_info*)dev->pci_info;
+
+	uint32 addr = info->u.h0.base_registers[bar];
+	uint32 size = info->u.h0.base_register_sizes[bar];
+
+	area_id area = map_physical_memory("nvme mapped bar", (phys_addr_t)addr,
+		size, B_ANY_KERNEL_ADDRESS,
+		B_KERNEL_READ_AREA | (read_only ? 0 : B_KERNEL_WRITE_AREA),
+		mapped_addr);
+	if (area < B_OK)
+		return area;
+
+	return 0;
+}
+
+
+int
+nvme_pcicfg_map_bar_write_combine(void* devhandle, unsigned int bar,
+	void** mapped_addr)
+{
+	status_t status = nvme_pcicfg_map_bar(devhandle, bar, false, mapped_addr);
+	if (status != 0)
+		return status;
+
+	// Turn on write combining for the area
+	status = vm_set_area_memory_type(area_for(*mapped_addr),
+		nvme_mem_vtophys(*mapped_addr), B_MTR_WC);
+	if (status != 0)
+		nvme_pcicfg_unmap_bar(devhandle, bar, *mapped_addr);
+	return status;
+}
+
+
+int
+nvme_pcicfg_unmap_bar(void* devhandle, unsigned int bar, void* addr)
+{
+	return delete_area(area_for(addr));
+}
+
+
+void
+nvme_pcicfg_get_bar_addr_len(void* devhandle, unsigned int bar,
+	uint64_t* addr, uint64_t* size)
+{
+	struct pci_device* dev = (struct pci_device*)devhandle;
+	pci_info* info = (pci_info*)dev->pci_info;
+
+	*addr = info->u.h0.base_registers[bar];
+	*size = info->u.h0.base_register_sizes[bar];
+}
+
+
+// #pragma mark - logging
+
+
+void
+nvme_log(enum nvme_log_level level, const char *format, ...)
+{
+	va_list ap;
+
+	va_start(ap, format);
+	nvme_vlog(level, format, ap);
+	va_end(ap);
+}
+
+
+void
+nvme_vlog(enum nvme_log_level level, const char *format, va_list ap)
+{
+	dvprintf(format, ap);
+}

src/add-ons/kernel/drivers/disk/nvme/compat/nvme_arch.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2019, Haiku, Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Augustin Cavalier <waddlesplash>
+ */
+#ifndef __NVME_ARCH_H__
+#define __NVME_ARCH_H__
+
+#include <OS.h>
+
+
+#define NVME_ARCH_64 __HAIKU_ARCH_64_BIT
+#define NVME_MMIO_64BIT NVME_ARCH_64
+#define PAGE_SIZE B_PAGE_SIZE
+#define PAGE_SHIFT (12) // FIXME: values for other arches
+
+
+#ifndef asm
+#define asm __asm__
+#endif
+
+#define nvme_wmb()	__asm__ volatile("sfence" ::: "memory")
+
+
+typedef uint8 __u8;
+typedef uint32 __u32;
+typedef uint64 __u64;
+
+
+static inline __u32
+nvme_mmio_read_4(const volatile __u32 *addr)
+{
+	return *addr;
+}
+
+
+static inline void
+nvme_mmio_write_4(volatile __u32 *addr, __u32 val)
+{
+	*addr = val;
+}
+
+
+static inline __u64
+nvme_mmio_read_8(volatile __u64 *addr)
+{
+#ifdef NVME_MMIO_64BIT
+	return *addr;
+#else
+	volatile __u32 *addr32 = (volatile __u32 *)addr;
+	__u64 val;
+
+	/*
+	 * Read lower 4 bytes before upper 4 bytes.
+	 * This particular order is required by I/OAT.
+	 * If the other order is required, use a pair of
+	 * _nvme_mmio_read_4() calls.
+	 */
+	val = addr32[0];
+	val |= (__u64)addr32[1] << 32;
+
+	return val;
+#endif
+}
+
+
+static inline void
+nvme_mmio_write_8(volatile __u64 *addr, __u64 val)
+{
+
+#ifdef NVME_MMIO_64BIT
+	*addr = val;
+#else
+	volatile __u32 *addr32 = (volatile __u32 *)addr;
+
+	addr32[0] = (__u32)val;
+	addr32[1] = (__u32)(val >> 32);
+#endif
+}
+
+#endif /* __NVME_ARCH_H__ */

src/add-ons/kernel/drivers/disk/nvme/compat/nvme_atomic.h

@@ -0,0 +1,204 @@
+/*
+ * Copyright 2019, Haiku, Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Augustin Cavalier <waddlesplash>
+ */
+#ifndef __NVME_ATOMIC_H__
+#define __NVME_ATOMIC_H__
+
+#include <OS.h>
+
+
+/* 32-bit atomics */
+typedef int32 nvme_atomic_t;
+
+#define NVME_ATOMIC_INIT(val)	(val)
+
+static inline void
+nvme_atomic_init(nvme_atomic_t* v)
+{
+	*v = 0;
+}
+
+static inline void
+nvme_atomic_clear(nvme_atomic_t* v)
+{
+	*v = 0;
+}
+
+
+static inline int32
+nvme_atomic_read(const nvme_atomic_t* v)
+{
+	return atomic_get(v);
+}
+
+
+static inline void
+nvme_atomic_set(nvme_atomic_t* v, int32 new_value)
+{
+	atomic_set(v, new_value);
+}
+
+
+static inline void
+nvme_atomic_add(nvme_atomic_t* v, int32 inc)
+{
+	atomic_add(v, inc);
+}
+
+
+static inline void
+nvme_atomic_sub(nvme_atomic_t* v, int32 dec)
+{
+	atomic_add(v, -dec);
+}
+
+
+static inline void
+nvme_atomic_inc(nvme_atomic_t* v)
+{
+	nvme_atomic_add(v, 1);
+}
+
+
+static inline void
+nvme_atomic_dec(nvme_atomic_t* v)
+{
+	nvme_atomic_sub(v, 1);
+}
+
+
+static inline int32
+nvme_atomic_add_return(nvme_atomic_t* v, int32 inc)
+{
+	return atomic_add(v, inc);
+}
+
+
+static inline int32
+nvme_atomic_sub_return(nvme_atomic_t* v, int32 dec)
+{
+	return atomic_add(v, -dec);
+}
+
+
+static inline int
+nvme_atomic_inc_and_test(nvme_atomic_t *v)
+{
+	return nvme_atomic_add_return(v, 1) == 0;
+}
+
+
+static inline int
+nvme_atomic_dec_and_test(nvme_atomic64_t *v)
+{
+	return nvme_atomic_sub_return(v, 1) == 0;
+}
+
+
+static inline int
+nvme_atomic_test_and_set(nvme_atomic_t* v)
+{
+	return atomic_test_and_set(v, 1, 0);
+}
+
+
+/* 64-bit atomics */
+typedef int64 nvme_atomic64_t;
+
+#define NVME_ATOMIC64_INIT(val) (val)
+
+static inline void
+nvme_atomic64_init(nvme_atomic64_t *v)
+{
+	atomic_set64(v, 0);
+}
+
+static inline void
+nvme_atomic64_clear(nvme_atomic64_t *v)
+{
+	atomic_set64(v, 0);
+}
+
+
+static inline int64
+nvme_atomic64_read(nvme_atomic64_t *v)
+{
+	return atomic_get64(v);
+}
+
+
+static inline void
+nvme_atomic64_set(nvme_atomic64_t *v, int64_t new_value)
+{
+	atomic_set64(v, new_value);
+}
+
+
+static inline void
+nvme_atomic64_add(nvme_atomic64_t *v, int64_t inc)
+{
+	atomic_add64(v, inc);
+}
+
+
+static inline void
+nvme_atomic64_sub(nvme_atomic64_t *v, int64_t dec)
+{
+	nvme_atomic64_add(v, -dec);
+}
+
+
+static inline void
+nvme_atomic64_inc(nvme_atomic64_t *v)
+{
+	nvme_atomic64_add(v, 1);
+}
+
+
+static inline void
+nvme_atomic64_dec(nvme_atomic64_t *v)
+{
+	nvme_atomic64_add(v, -1);
+}
+
+
+static inline int64
+nvme_atomic64_add_return(nvme_atomic64_t *v, int64_t inc)
+{
+	return atomic_add64(v, inc);
+}
+
+
+static inline int64_t
+nvme_atomic64_sub_return(nvme_atomic64_t *v, int64_t dec)
+{
+	return nvme_atomic64_add_return(v, -dec);
+}
+
+
+static inline int
+nvme_atomic64_inc_and_test(nvme_atomic64_t *v)
+{
+	return nvme_atomic64_add_return(v, 1) == 0;
+}
+
+
+static inline int
+nvme_atomic64_dec_and_test(nvme_atomic64_t *v)
+{
+	return nvme_atomic64_sub_return(v, 1) == 0;
+}
+
+
+static inline int
+nvme_atomic64_test_and_set(nvme_atomic64_t *v)
+{
+	return atomic_test_and_set64(v, 1, 0);
+}
+
+
+#endif

src/add-ons/kernel/drivers/disk/nvme/compat/nvme_log.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2019, Haiku, Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Augustin Cavalier <waddlesplash>
+ */
+#ifndef __NVME_LOG_H__
+#define __NVME_LOG_H__
+
+#include <stdint.h>
+#include <stdarg.h>
+
+
+void nvme_log(enum nvme_log_level level, const char *format, ...)
+	__attribute__((format(printf, 2, 3)));
+void nvme_vlog(enum nvme_log_level level, const char *format, va_list ap)
+	__attribute__((format(printf,2,0)));
+
+
+#define nvme_emerg(format, args...)		\
+	nvme_log(NVME_LOG_EMERG,		\
+		"libnvme (FATAL): " format,	\
+		## args)
+
+#define nvme_alert(format, args...)		\
+	nvme_log(NVME_LOG_ALERT,		\
+		"libnvme (ALERT): " format,	\
+		## args)
+
+#define nvme_crit(format, args...)		\
+	nvme_log(NVME_LOG_CRIT,			\
+		"libnvme (CRITICAL): " format,	\
+		## args)
+
+#define nvme_err(format, args...)		\
+	nvme_log(NVME_LOG_ERR,			\
+		"libnvme (ERROR): " format,	\
+		## args)
+
+#define nvme_warning(format, args...)		\
+	nvme_log(NVME_LOG_WARNING,		\
+		"libnvme (WARNING): " format,	\
+		## args)
+
+#ifdef TRACE_NVME
+#define nvme_notice(format, args...)	\
+	nvme_log(NVME_LOG_NOTICE,	\
+		"libnvme: " format,	\
+		## args)
+
+#define nvme_info(format, args...)	\
+	nvme_log(NVME_LOG_INFO,		\
+		"libnvme: " format,	\
+		## args)
+
+#define nvme_debug(format, args...)	\
+	nvme_log(NVME_LOG_DEBUG,	\
+		"libnvme: " format,	\
+		## args)
+#else
+#define nvme_notice(format, args...)
+#define nvme_info(format, args...)
+#define nvme_debug(format, args...)
+#endif
+
+
+#endif /* __NVME_LOG_H__ */

src/add-ons/kernel/drivers/disk/nvme/compat/nvme_mem.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019, Haiku, Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Augustin Cavalier <waddlesplash>
+ */
+#ifndef __NVME_MEMORY_H_
+#define __NVME_MEMORY_H_
+
+#include <OS.h>
+
+
+int nvme_mem_init();
+void nvme_mem_cleanup();
+
+void* nvme_mem_alloc_node(size_t size, size_t align,
+	unsigned int node_id, phys_addr_t* paddr);
+void* nvme_malloc_node(size_t size, size_t align,
+	unsigned int node_id);
+
+phys_addr_t nvme_mem_vtophys(void* vaddr);
+
+
+#define NVME_VTOPHYS_ERROR	(~0ULL)
+
+#define nvme_node_max()		(1)
+#define NVME_NODE_MAX		(1)
+#define nvme_node_id()		(0)
+
+
+#endif /* __NVME_MEMORY_H_ */

src/add-ons/kernel/drivers/disk/nvme/compat/nvme_platform.h

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019, Haiku, Inc. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ *
+ * Authors:
+ *		Augustin Cavalier <waddlesplash>
+ */
+#ifndef __NVME_PLATFORM_H__
+#define __NVME_PLATFORM_H__
+
+#include <lock.h>
+
+
+#define pthread_mutex_t				mutex
+#define PTHREAD_MUTEX_INITIALIZER	MUTEX_INITIALIZER(__FILE__)
+#define pthread_mutex_init(mtx, attr) mutex_init(mtx, __FILE__)
+#define pthread_mutex_destroy		mutex_destroy
+#define pthread_mutex_lock			mutex_lock
+#define pthread_mutex_unlock		mutex_unlock
+
+
+#endif /* __NVME_PLATFORM_H__ */