From 4b88e723504583a68377602ba696d386d215766d Mon Sep 17 00:00:00 2001 From: Augustin Cavalier Date: Sun, 14 Apr 2019 15:02:47 -0400 Subject: [PATCH] nvme: Import glue code to our PCI and VM APIs. Largely simple and painless, thanks to libnvme's abstractions. --- .../disk/nvme/compat/libnvme_haiku.cpp | 200 +++++++++++++++++ .../drivers/disk/nvme/compat/nvme_arch.h | 83 +++++++ .../drivers/disk/nvme/compat/nvme_atomic.h | 204 ++++++++++++++++++ .../drivers/disk/nvme/compat/nvme_log.h | 68 ++++++ .../drivers/disk/nvme/compat/nvme_mem.h | 32 +++ .../drivers/disk/nvme/compat/nvme_platform.h | 22 ++ 6 files changed, 609 insertions(+) create mode 100644 src/add-ons/kernel/drivers/disk/nvme/compat/libnvme_haiku.cpp create mode 100644 src/add-ons/kernel/drivers/disk/nvme/compat/nvme_arch.h create mode 100644 src/add-ons/kernel/drivers/disk/nvme/compat/nvme_atomic.h create mode 100644 src/add-ons/kernel/drivers/disk/nvme/compat/nvme_log.h create mode 100644 src/add-ons/kernel/drivers/disk/nvme/compat/nvme_mem.h create mode 100644 src/add-ons/kernel/drivers/disk/nvme/compat/nvme_platform.h diff --git a/src/add-ons/kernel/drivers/disk/nvme/compat/libnvme_haiku.cpp b/src/add-ons/kernel/drivers/disk/nvme/compat/libnvme_haiku.cpp new file mode 100644 index 0000000000..501c872077 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/compat/libnvme_haiku.cpp @@ -0,0 +1,200 @@ +/* + * Copyright 2019, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Augustin Cavalier + */ + +#include +#include +#include + +extern "C" { +#include "nvme.h" +#include "nvme_log.h" +#include "nvme_mem.h" +#include "nvme_pci.h" +} + + +static pci_module_info* sPCIModule = NULL; + + +// #pragma mark - memory + + +int +nvme_mem_init() +{ + /* nothing to do */ + return 0; +} + + +void +nvme_mem_cleanup() +{ + /* nothing to do */ +} + + +void* +nvme_mem_alloc_node(size_t size, size_t align, unsigned int node_id, + phys_addr_t* paddr) +{ + size = ROUNDUP(size, B_PAGE_SIZE); + + virtual_address_restrictions virtualRestrictions = {}; + + physical_address_restrictions physicalRestrictions = {}; + physicalRestrictions.alignment = align; + + void* address; + area_id area = create_area_etc(B_SYSTEM_TEAM, "nvme physical buffer", + size, B_CONTIGUOUS, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, + 0, 0, &virtualRestrictions, &physicalRestrictions, &address); + if (area < 0) + return NULL; + + if (paddr != NULL) + *paddr = nvme_mem_vtophys(address); + return address; +} + + +void* +nvme_malloc_node(size_t size, size_t align, unsigned int node_id) +{ + return nvme_mem_alloc_node(size, align, node_id, NULL); +} + + +void +nvme_free(void* addr) +{ + delete_area(area_for(addr)); +} + + +phys_addr_t +nvme_mem_vtophys(void* vaddr) +{ + physical_entry entry; + status_t status = get_memory_map((void*)vaddr, 1, &entry, 1); + if (status != B_OK) { + panic("nvme: get_memory_map failed for %p, error %08" B_PRIx32 "\n", + (void*)vaddr, status); + return NVME_VTOPHYS_ERROR; + } + + return entry.address; +} + + +// #pragma mark - PCI + + +int +nvme_pci_init() +{ + status_t status = get_module(B_PCI_MODULE_NAME, + (module_info**)&sPCIModule); + return status; +} + + +int +nvme_pcicfg_read32(struct pci_device* dev, uint32_t* value, uint32_t offset) +{ + *value = sPCIModule->read_pci_config(dev->bus, dev->dev, dev->func, offset, + sizeof(*value)); + return 0; +} + + +int +nvme_pcicfg_write32(struct pci_device* dev, uint32_t value, uint32_t offset) +{ + sPCIModule->write_pci_config(dev->bus, dev->dev, dev->func, offset, + sizeof(value), value); + return 0; +} + + +int +nvme_pcicfg_map_bar(void* devhandle, unsigned int bar, bool read_only, + void** mapped_addr) +{ + struct pci_device* dev = (struct pci_device*)devhandle; + pci_info* info = (pci_info*)dev->pci_info; + + uint32 addr = info->u.h0.base_registers[bar]; + uint32 size = info->u.h0.base_register_sizes[bar]; + + area_id area = map_physical_memory("nvme mapped bar", (phys_addr_t)addr, + size, B_ANY_KERNEL_ADDRESS, + B_KERNEL_READ_AREA | (read_only ? 0 : B_KERNEL_WRITE_AREA), + mapped_addr); + if (area < B_OK) + return area; + + return 0; +} + + +int +nvme_pcicfg_map_bar_write_combine(void* devhandle, unsigned int bar, + void** mapped_addr) +{ + status_t status = nvme_pcicfg_map_bar(devhandle, bar, false, mapped_addr); + if (status != 0) + return status; + + // Turn on write combining for the area + status = vm_set_area_memory_type(area_for(*mapped_addr), + nvme_mem_vtophys(*mapped_addr), B_MTR_WC); + if (status != 0) + nvme_pcicfg_unmap_bar(devhandle, bar, *mapped_addr); + return status; +} + + +int +nvme_pcicfg_unmap_bar(void* devhandle, unsigned int bar, void* addr) +{ + return delete_area(area_for(addr)); +} + + +void +nvme_pcicfg_get_bar_addr_len(void* devhandle, unsigned int bar, + uint64_t* addr, uint64_t* size) +{ + struct pci_device* dev = (struct pci_device*)devhandle; + pci_info* info = (pci_info*)dev->pci_info; + + *addr = info->u.h0.base_registers[bar]; + *size = info->u.h0.base_register_sizes[bar]; +} + + +// #pragma mark - logging + + +void +nvme_log(enum nvme_log_level level, const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + nvme_vlog(level, format, ap); + va_end(ap); +} + + +void +nvme_vlog(enum nvme_log_level level, const char *format, va_list ap) +{ + dvprintf(format, ap); +} diff --git a/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_arch.h b/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_arch.h new file mode 100644 index 0000000000..d06d2c34e8 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_arch.h @@ -0,0 +1,83 @@ +/* + * Copyright 2019, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Augustin Cavalier + */ +#ifndef __NVME_ARCH_H__ +#define __NVME_ARCH_H__ + +#include + + +#define NVME_ARCH_64 __HAIKU_ARCH_64_BIT +#define NVME_MMIO_64BIT NVME_ARCH_64 +#define PAGE_SIZE B_PAGE_SIZE +#define PAGE_SHIFT (12) // FIXME: values for other arches + + +#ifndef asm +#define asm __asm__ +#endif + +#define nvme_wmb() __asm__ volatile("sfence" ::: "memory") + + +typedef uint8 __u8; +typedef uint32 __u32; +typedef uint64 __u64; + + +static inline __u32 +nvme_mmio_read_4(const volatile __u32 *addr) +{ + return *addr; +} + + +static inline void +nvme_mmio_write_4(volatile __u32 *addr, __u32 val) +{ + *addr = val; +} + + +static inline __u64 +nvme_mmio_read_8(volatile __u64 *addr) +{ +#ifdef NVME_MMIO_64BIT + return *addr; +#else + volatile __u32 *addr32 = (volatile __u32 *)addr; + __u64 val; + + /* + * Read lower 4 bytes before upper 4 bytes. + * This particular order is required by I/OAT. + * If the other order is required, use a pair of + * _nvme_mmio_read_4() calls. + */ + val = addr32[0]; + val |= (__u64)addr32[1] << 32; + + return val; +#endif +} + + +static inline void +nvme_mmio_write_8(volatile __u64 *addr, __u64 val) +{ + +#ifdef NVME_MMIO_64BIT + *addr = val; +#else + volatile __u32 *addr32 = (volatile __u32 *)addr; + + addr32[0] = (__u32)val; + addr32[1] = (__u32)(val >> 32); +#endif +} + +#endif /* __NVME_ARCH_H__ */ diff --git a/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_atomic.h b/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_atomic.h new file mode 100644 index 0000000000..a876d6aaad --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_atomic.h @@ -0,0 +1,204 @@ +/* + * Copyright 2019, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Augustin Cavalier + */ +#ifndef __NVME_ATOMIC_H__ +#define __NVME_ATOMIC_H__ + +#include + + +/* 32-bit atomics */ +typedef int32 nvme_atomic_t; + +#define NVME_ATOMIC_INIT(val) (val) + +static inline void +nvme_atomic_init(nvme_atomic_t* v) +{ + *v = 0; +} + +static inline void +nvme_atomic_clear(nvme_atomic_t* v) +{ + *v = 0; +} + + +static inline int32 +nvme_atomic_read(const nvme_atomic_t* v) +{ + return atomic_get(v); +} + + +static inline void +nvme_atomic_set(nvme_atomic_t* v, int32 new_value) +{ + atomic_set(v, new_value); +} + + +static inline void +nvme_atomic_add(nvme_atomic_t* v, int32 inc) +{ + atomic_add(v, inc); +} + + +static inline void +nvme_atomic_sub(nvme_atomic_t* v, int32 dec) +{ + atomic_add(v, -dec); +} + + +static inline void +nvme_atomic_inc(nvme_atomic_t* v) +{ + nvme_atomic_add(v, 1); +} + + +static inline void +nvme_atomic_dec(nvme_atomic_t* v) +{ + nvme_atomic_sub(v, 1); +} + + +static inline int32 +nvme_atomic_add_return(nvme_atomic_t* v, int32 inc) +{ + return atomic_add(v, inc); +} + + +static inline int32 +nvme_atomic_sub_return(nvme_atomic_t* v, int32 dec) +{ + return atomic_add(v, -dec); +} + + +static inline int +nvme_atomic_inc_and_test(nvme_atomic_t *v) +{ + return nvme_atomic_add_return(v, 1) == 0; +} + + +static inline int +nvme_atomic_dec_and_test(nvme_atomic64_t *v) +{ + return nvme_atomic_sub_return(v, 1) == 0; +} + + +static inline int +nvme_atomic_test_and_set(nvme_atomic_t* v) +{ + return atomic_test_and_set(v, 1, 0); +} + + +/* 64-bit atomics */ +typedef int64 nvme_atomic64_t; + +#define NVME_ATOMIC64_INIT(val) (val) + +static inline void +nvme_atomic64_init(nvme_atomic64_t *v) +{ + atomic_set64(v, 0); +} + +static inline void +nvme_atomic64_clear(nvme_atomic64_t *v) +{ + atomic_set64(v, 0); +} + + +static inline int64 +nvme_atomic64_read(nvme_atomic64_t *v) +{ + return atomic_get64(v); +} + + +static inline void +nvme_atomic64_set(nvme_atomic64_t *v, int64_t new_value) +{ + atomic_set64(v, new_value); +} + + +static inline void +nvme_atomic64_add(nvme_atomic64_t *v, int64_t inc) +{ + atomic_add64(v, inc); +} + + +static inline void +nvme_atomic64_sub(nvme_atomic64_t *v, int64_t dec) +{ + nvme_atomic64_add(v, -dec); +} + + +static inline void +nvme_atomic64_inc(nvme_atomic64_t *v) +{ + nvme_atomic64_add(v, 1); +} + + +static inline void +nvme_atomic64_dec(nvme_atomic64_t *v) +{ + nvme_atomic64_add(v, -1); +} + + +static inline int64 +nvme_atomic64_add_return(nvme_atomic64_t *v, int64_t inc) +{ + return atomic_add64(v, inc); +} + + +static inline int64_t +nvme_atomic64_sub_return(nvme_atomic64_t *v, int64_t dec) +{ + return nvme_atomic64_add_return(v, -dec); +} + + +static inline int +nvme_atomic64_inc_and_test(nvme_atomic64_t *v) +{ + return nvme_atomic64_add_return(v, 1) == 0; +} + + +static inline int +nvme_atomic64_dec_and_test(nvme_atomic64_t *v) +{ + return nvme_atomic64_sub_return(v, 1) == 0; +} + + +static inline int +nvme_atomic64_test_and_set(nvme_atomic64_t *v) +{ + return atomic_test_and_set64(v, 1, 0); +} + + +#endif diff --git a/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_log.h b/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_log.h new file mode 100644 index 0000000000..5b6c2bb13f --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_log.h @@ -0,0 +1,68 @@ +/* + * Copyright 2019, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Augustin Cavalier + */ +#ifndef __NVME_LOG_H__ +#define __NVME_LOG_H__ + +#include +#include + + +void nvme_log(enum nvme_log_level level, const char *format, ...) + __attribute__((format(printf, 2, 3))); +void nvme_vlog(enum nvme_log_level level, const char *format, va_list ap) + __attribute__((format(printf,2,0))); + + +#define nvme_emerg(format, args...) \ + nvme_log(NVME_LOG_EMERG, \ + "libnvme (FATAL): " format, \ + ## args) + +#define nvme_alert(format, args...) \ + nvme_log(NVME_LOG_ALERT, \ + "libnvme (ALERT): " format, \ + ## args) + +#define nvme_crit(format, args...) \ + nvme_log(NVME_LOG_CRIT, \ + "libnvme (CRITICAL): " format, \ + ## args) + +#define nvme_err(format, args...) \ + nvme_log(NVME_LOG_ERR, \ + "libnvme (ERROR): " format, \ + ## args) + +#define nvme_warning(format, args...) \ + nvme_log(NVME_LOG_WARNING, \ + "libnvme (WARNING): " format, \ + ## args) + +#ifdef TRACE_NVME +#define nvme_notice(format, args...) \ + nvme_log(NVME_LOG_NOTICE, \ + "libnvme: " format, \ + ## args) + +#define nvme_info(format, args...) \ + nvme_log(NVME_LOG_INFO, \ + "libnvme: " format, \ + ## args) + +#define nvme_debug(format, args...) \ + nvme_log(NVME_LOG_DEBUG, \ + "libnvme: " format, \ + ## args) +#else +#define nvme_notice(format, args...) +#define nvme_info(format, args...) +#define nvme_debug(format, args...) +#endif + + +#endif /* __NVME_LOG_H__ */ diff --git a/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_mem.h b/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_mem.h new file mode 100644 index 0000000000..c0ddf26365 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_mem.h @@ -0,0 +1,32 @@ +/* + * Copyright 2019, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Augustin Cavalier + */ +#ifndef __NVME_MEMORY_H_ +#define __NVME_MEMORY_H_ + +#include + + +int nvme_mem_init(); +void nvme_mem_cleanup(); + +void* nvme_mem_alloc_node(size_t size, size_t align, + unsigned int node_id, phys_addr_t* paddr); +void* nvme_malloc_node(size_t size, size_t align, + unsigned int node_id); + +phys_addr_t nvme_mem_vtophys(void* vaddr); + + +#define NVME_VTOPHYS_ERROR (~0ULL) + +#define nvme_node_max() (1) +#define NVME_NODE_MAX (1) +#define nvme_node_id() (0) + + +#endif /* __NVME_MEMORY_H_ */ diff --git a/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_platform.h b/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_platform.h new file mode 100644 index 0000000000..90418e2f8b --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/compat/nvme_platform.h @@ -0,0 +1,22 @@ +/* + * Copyright 2019, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Augustin Cavalier + */ +#ifndef __NVME_PLATFORM_H__ +#define __NVME_PLATFORM_H__ + +#include + + +#define pthread_mutex_t mutex +#define PTHREAD_MUTEX_INITIALIZER MUTEX_INITIALIZER(__FILE__) +#define pthread_mutex_init(mtx, attr) mutex_init(mtx, __FILE__) +#define pthread_mutex_destroy mutex_destroy +#define pthread_mutex_lock mutex_lock +#define pthread_mutex_unlock mutex_unlock + + +#endif /* __NVME_PLATFORM_H__ */