nvme: Import glue code to our PCI and VM APIs.
Largely simple and painless, thanks to libnvme's abstractions.
This commit is contained in:
Augustin Cavalier
@@ -0,0 +1,200 @@
|
||||
/*
|
||||
* Copyright 2019, Haiku, Inc. All rights reserved.
|
||||
* Distributed under the terms of the MIT License.
|
||||
*
|
||||
* Authors:
|
||||
* Augustin Cavalier <waddlesplash>
|
||||
*/
|
||||
|
||||
#include <debug.h>
|
||||
#include <kernel/vm/vm.h>
|
||||
#include <PCI.h>
|
||||
|
||||
extern "C" {
|
||||
#include "nvme.h"
|
||||
#include "nvme_log.h"
|
||||
#include "nvme_mem.h"
|
||||
#include "nvme_pci.h"
|
||||
}
|
||||
|
||||
|
||||
static pci_module_info* sPCIModule = NULL;
|
||||
|
||||
|
||||
// #pragma mark - memory
|
||||
|
||||
|
||||
int
|
||||
nvme_mem_init()
|
||||
{
|
||||
/* nothing to do */
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
nvme_mem_cleanup()
|
||||
{
|
||||
/* nothing to do */
|
||||
}
|
||||
|
||||
|
||||
void*
|
||||
nvme_mem_alloc_node(size_t size, size_t align, unsigned int node_id,
|
||||
phys_addr_t* paddr)
|
||||
{
|
||||
size = ROUNDUP(size, B_PAGE_SIZE);
|
||||
|
||||
virtual_address_restrictions virtualRestrictions = {};
|
||||
|
||||
physical_address_restrictions physicalRestrictions = {};
|
||||
physicalRestrictions.alignment = align;
|
||||
|
||||
void* address;
|
||||
area_id area = create_area_etc(B_SYSTEM_TEAM, "nvme physical buffer",
|
||||
size, B_CONTIGUOUS, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA,
|
||||
0, 0, &virtualRestrictions, &physicalRestrictions, &address);
|
||||
if (area < 0)
|
||||
return NULL;
|
||||
|
||||
if (paddr != NULL)
|
||||
*paddr = nvme_mem_vtophys(address);
|
||||
return address;
|
||||
}
|
||||
|
||||
|
||||
void*
|
||||
nvme_malloc_node(size_t size, size_t align, unsigned int node_id)
|
||||
{
|
||||
return nvme_mem_alloc_node(size, align, node_id, NULL);
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
nvme_free(void* addr)
|
||||
{
|
||||
delete_area(area_for(addr));
|
||||
}
|
||||
|
||||
|
||||
phys_addr_t
|
||||
nvme_mem_vtophys(void* vaddr)
|
||||
{
|
||||
physical_entry entry;
|
||||
status_t status = get_memory_map((void*)vaddr, 1, &entry, 1);
|
||||
if (status != B_OK) {
|
||||
panic("nvme: get_memory_map failed for %p, error %08" B_PRIx32 "\n",
|
||||
(void*)vaddr, status);
|
||||
return NVME_VTOPHYS_ERROR;
|
||||
}
|
||||
|
||||
return entry.address;
|
||||
}
|
||||
|
||||
|
||||
// #pragma mark - PCI
|
||||
|
||||
|
||||
int
|
||||
nvme_pci_init()
|
||||
{
|
||||
status_t status = get_module(B_PCI_MODULE_NAME,
|
||||
(module_info**)&sPCIModule);
|
||||
return status;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
nvme_pcicfg_read32(struct pci_device* dev, uint32_t* value, uint32_t offset)
|
||||
{
|
||||
*value = sPCIModule->read_pci_config(dev->bus, dev->dev, dev->func, offset,
|
||||
sizeof(*value));
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
nvme_pcicfg_write32(struct pci_device* dev, uint32_t value, uint32_t offset)
|
||||
{
|
||||
sPCIModule->write_pci_config(dev->bus, dev->dev, dev->func, offset,
|
||||
sizeof(value), value);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
nvme_pcicfg_map_bar(void* devhandle, unsigned int bar, bool read_only,
|
||||
void** mapped_addr)
|
||||
{
|
||||
struct pci_device* dev = (struct pci_device*)devhandle;
|
||||
pci_info* info = (pci_info*)dev->pci_info;
|
||||
|
||||
uint32 addr = info->u.h0.base_registers[bar];
|
||||
uint32 size = info->u.h0.base_register_sizes[bar];
|
||||
|
||||
area_id area = map_physical_memory("nvme mapped bar", (phys_addr_t)addr,
|
||||
size, B_ANY_KERNEL_ADDRESS,
|
||||
B_KERNEL_READ_AREA | (read_only ? 0 : B_KERNEL_WRITE_AREA),
|
||||
mapped_addr);
|
||||
if (area < B_OK)
|
||||
return area;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
nvme_pcicfg_map_bar_write_combine(void* devhandle, unsigned int bar,
|
||||
void** mapped_addr)
|
||||
{
|
||||
status_t status = nvme_pcicfg_map_bar(devhandle, bar, false, mapped_addr);
|
||||
if (status != 0)
|
||||
return status;
|
||||
|
||||
// Turn on write combining for the area
|
||||
status = vm_set_area_memory_type(area_for(*mapped_addr),
|
||||
nvme_mem_vtophys(*mapped_addr), B_MTR_WC);
|
||||
if (status != 0)
|
||||
nvme_pcicfg_unmap_bar(devhandle, bar, *mapped_addr);
|
||||
return status;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
nvme_pcicfg_unmap_bar(void* devhandle, unsigned int bar, void* addr)
|
||||
{
|
||||
return delete_area(area_for(addr));
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
nvme_pcicfg_get_bar_addr_len(void* devhandle, unsigned int bar,
|
||||
uint64_t* addr, uint64_t* size)
|
||||
{
|
||||
struct pci_device* dev = (struct pci_device*)devhandle;
|
||||
pci_info* info = (pci_info*)dev->pci_info;
|
||||
|
||||
*addr = info->u.h0.base_registers[bar];
|
||||
*size = info->u.h0.base_register_sizes[bar];
|
||||
}
|
||||
|
||||
|
||||
// #pragma mark - logging
|
||||
|
||||
|
||||
void
|
||||
nvme_log(enum nvme_log_level level, const char *format, ...)
|
||||
{
|
||||
va_list ap;
|
||||
|
||||
va_start(ap, format);
|
||||
nvme_vlog(level, format, ap);
|
||||
va_end(ap);
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
nvme_vlog(enum nvme_log_level level, const char *format, va_list ap)
|
||||
{
|
||||
dvprintf(format, ap);
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright 2019, Haiku, Inc. All rights reserved.
|
||||
* Distributed under the terms of the MIT License.
|
||||
*
|
||||
* Authors:
|
||||
* Augustin Cavalier <waddlesplash>
|
||||
*/
|
||||
#ifndef __NVME_ARCH_H__
|
||||
#define __NVME_ARCH_H__
|
||||
|
||||
#include <OS.h>
|
||||
|
||||
|
||||
#define NVME_ARCH_64 __HAIKU_ARCH_64_BIT
|
||||
#define NVME_MMIO_64BIT NVME_ARCH_64
|
||||
#define PAGE_SIZE B_PAGE_SIZE
|
||||
#define PAGE_SHIFT (12) // FIXME: values for other arches
|
||||
|
||||
|
||||
#ifndef asm
|
||||
#define asm __asm__
|
||||
#endif
|
||||
|
||||
#define nvme_wmb() __asm__ volatile("sfence" ::: "memory")
|
||||
|
||||
|
||||
typedef uint8 __u8;
|
||||
typedef uint32 __u32;
|
||||
typedef uint64 __u64;
|
||||
|
||||
|
||||
static inline __u32
|
||||
nvme_mmio_read_4(const volatile __u32 *addr)
|
||||
{
|
||||
return *addr;
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_mmio_write_4(volatile __u32 *addr, __u32 val)
|
||||
{
|
||||
*addr = val;
|
||||
}
|
||||
|
||||
|
||||
static inline __u64
|
||||
nvme_mmio_read_8(volatile __u64 *addr)
|
||||
{
|
||||
#ifdef NVME_MMIO_64BIT
|
||||
return *addr;
|
||||
#else
|
||||
volatile __u32 *addr32 = (volatile __u32 *)addr;
|
||||
__u64 val;
|
||||
|
||||
/*
|
||||
* Read lower 4 bytes before upper 4 bytes.
|
||||
* This particular order is required by I/OAT.
|
||||
* If the other order is required, use a pair of
|
||||
* _nvme_mmio_read_4() calls.
|
||||
*/
|
||||
val = addr32[0];
|
||||
val |= (__u64)addr32[1] << 32;
|
||||
|
||||
return val;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_mmio_write_8(volatile __u64 *addr, __u64 val)
|
||||
{
|
||||
|
||||
#ifdef NVME_MMIO_64BIT
|
||||
*addr = val;
|
||||
#else
|
||||
volatile __u32 *addr32 = (volatile __u32 *)addr;
|
||||
|
||||
addr32[0] = (__u32)val;
|
||||
addr32[1] = (__u32)(val >> 32);
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif /* __NVME_ARCH_H__ */
|
||||
@@ -0,0 +1,204 @@
|
||||
/*
|
||||
* Copyright 2019, Haiku, Inc. All rights reserved.
|
||||
* Distributed under the terms of the MIT License.
|
||||
*
|
||||
* Authors:
|
||||
* Augustin Cavalier <waddlesplash>
|
||||
*/
|
||||
#ifndef __NVME_ATOMIC_H__
|
||||
#define __NVME_ATOMIC_H__
|
||||
|
||||
#include <OS.h>
|
||||
|
||||
|
||||
/* 32-bit atomics */
|
||||
typedef int32 nvme_atomic_t;
|
||||
|
||||
#define NVME_ATOMIC_INIT(val) (val)
|
||||
|
||||
static inline void
|
||||
nvme_atomic_init(nvme_atomic_t* v)
|
||||
{
|
||||
*v = 0;
|
||||
}
|
||||
|
||||
static inline void
|
||||
nvme_atomic_clear(nvme_atomic_t* v)
|
||||
{
|
||||
*v = 0;
|
||||
}
|
||||
|
||||
|
||||
static inline int32
|
||||
nvme_atomic_read(const nvme_atomic_t* v)
|
||||
{
|
||||
return atomic_get(v);
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_atomic_set(nvme_atomic_t* v, int32 new_value)
|
||||
{
|
||||
atomic_set(v, new_value);
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_atomic_add(nvme_atomic_t* v, int32 inc)
|
||||
{
|
||||
atomic_add(v, inc);
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_atomic_sub(nvme_atomic_t* v, int32 dec)
|
||||
{
|
||||
atomic_add(v, -dec);
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_atomic_inc(nvme_atomic_t* v)
|
||||
{
|
||||
nvme_atomic_add(v, 1);
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_atomic_dec(nvme_atomic_t* v)
|
||||
{
|
||||
nvme_atomic_sub(v, 1);
|
||||
}
|
||||
|
||||
|
||||
static inline int32
|
||||
nvme_atomic_add_return(nvme_atomic_t* v, int32 inc)
|
||||
{
|
||||
return atomic_add(v, inc);
|
||||
}
|
||||
|
||||
|
||||
static inline int32
|
||||
nvme_atomic_sub_return(nvme_atomic_t* v, int32 dec)
|
||||
{
|
||||
return atomic_add(v, -dec);
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
nvme_atomic_inc_and_test(nvme_atomic_t *v)
|
||||
{
|
||||
return nvme_atomic_add_return(v, 1) == 0;
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
nvme_atomic_dec_and_test(nvme_atomic64_t *v)
|
||||
{
|
||||
return nvme_atomic_sub_return(v, 1) == 0;
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
nvme_atomic_test_and_set(nvme_atomic_t* v)
|
||||
{
|
||||
return atomic_test_and_set(v, 1, 0);
|
||||
}
|
||||
|
||||
|
||||
/* 64-bit atomics */
|
||||
typedef int64 nvme_atomic64_t;
|
||||
|
||||
#define NVME_ATOMIC64_INIT(val) (val)
|
||||
|
||||
static inline void
|
||||
nvme_atomic64_init(nvme_atomic64_t *v)
|
||||
{
|
||||
atomic_set64(v, 0);
|
||||
}
|
||||
|
||||
static inline void
|
||||
nvme_atomic64_clear(nvme_atomic64_t *v)
|
||||
{
|
||||
atomic_set64(v, 0);
|
||||
}
|
||||
|
||||
|
||||
static inline int64
|
||||
nvme_atomic64_read(nvme_atomic64_t *v)
|
||||
{
|
||||
return atomic_get64(v);
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_atomic64_set(nvme_atomic64_t *v, int64_t new_value)
|
||||
{
|
||||
atomic_set64(v, new_value);
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_atomic64_add(nvme_atomic64_t *v, int64_t inc)
|
||||
{
|
||||
atomic_add64(v, inc);
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_atomic64_sub(nvme_atomic64_t *v, int64_t dec)
|
||||
{
|
||||
nvme_atomic64_add(v, -dec);
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_atomic64_inc(nvme_atomic64_t *v)
|
||||
{
|
||||
nvme_atomic64_add(v, 1);
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
nvme_atomic64_dec(nvme_atomic64_t *v)
|
||||
{
|
||||
nvme_atomic64_add(v, -1);
|
||||
}
|
||||
|
||||
|
||||
static inline int64
|
||||
nvme_atomic64_add_return(nvme_atomic64_t *v, int64_t inc)
|
||||
{
|
||||
return atomic_add64(v, inc);
|
||||
}
|
||||
|
||||
|
||||
static inline int64_t
|
||||
nvme_atomic64_sub_return(nvme_atomic64_t *v, int64_t dec)
|
||||
{
|
||||
return nvme_atomic64_add_return(v, -dec);
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
nvme_atomic64_inc_and_test(nvme_atomic64_t *v)
|
||||
{
|
||||
return nvme_atomic64_add_return(v, 1) == 0;
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
nvme_atomic64_dec_and_test(nvme_atomic64_t *v)
|
||||
{
|
||||
return nvme_atomic64_sub_return(v, 1) == 0;
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
nvme_atomic64_test_and_set(nvme_atomic64_t *v)
|
||||
{
|
||||
return atomic_test_and_set64(v, 1, 0);
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 2019, Haiku, Inc. All rights reserved.
|
||||
* Distributed under the terms of the MIT License.
|
||||
*
|
||||
* Authors:
|
||||
* Augustin Cavalier <waddlesplash>
|
||||
*/
|
||||
#ifndef __NVME_LOG_H__
|
||||
#define __NVME_LOG_H__
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdarg.h>
|
||||
|
||||
|
||||
void nvme_log(enum nvme_log_level level, const char *format, ...)
|
||||
__attribute__((format(printf, 2, 3)));
|
||||
void nvme_vlog(enum nvme_log_level level, const char *format, va_list ap)
|
||||
__attribute__((format(printf,2,0)));
|
||||
|
||||
|
||||
#define nvme_emerg(format, args...) \
|
||||
nvme_log(NVME_LOG_EMERG, \
|
||||
"libnvme (FATAL): " format, \
|
||||
## args)
|
||||
|
||||
#define nvme_alert(format, args...) \
|
||||
nvme_log(NVME_LOG_ALERT, \
|
||||
"libnvme (ALERT): " format, \
|
||||
## args)
|
||||
|
||||
#define nvme_crit(format, args...) \
|
||||
nvme_log(NVME_LOG_CRIT, \
|
||||
"libnvme (CRITICAL): " format, \
|
||||
## args)
|
||||
|
||||
#define nvme_err(format, args...) \
|
||||
nvme_log(NVME_LOG_ERR, \
|
||||
"libnvme (ERROR): " format, \
|
||||
## args)
|
||||
|
||||
#define nvme_warning(format, args...) \
|
||||
nvme_log(NVME_LOG_WARNING, \
|
||||
"libnvme (WARNING): " format, \
|
||||
## args)
|
||||
|
||||
#ifdef TRACE_NVME
|
||||
#define nvme_notice(format, args...) \
|
||||
nvme_log(NVME_LOG_NOTICE, \
|
||||
"libnvme: " format, \
|
||||
## args)
|
||||
|
||||
#define nvme_info(format, args...) \
|
||||
nvme_log(NVME_LOG_INFO, \
|
||||
"libnvme: " format, \
|
||||
## args)
|
||||
|
||||
#define nvme_debug(format, args...) \
|
||||
nvme_log(NVME_LOG_DEBUG, \
|
||||
"libnvme: " format, \
|
||||
## args)
|
||||
#else
|
||||
#define nvme_notice(format, args...)
|
||||
#define nvme_info(format, args...)
|
||||
#define nvme_debug(format, args...)
|
||||
#endif
|
||||
|
||||
|
||||
#endif /* __NVME_LOG_H__ */
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Copyright 2019, Haiku, Inc. All rights reserved.
|
||||
* Distributed under the terms of the MIT License.
|
||||
*
|
||||
* Authors:
|
||||
* Augustin Cavalier <waddlesplash>
|
||||
*/
|
||||
#ifndef __NVME_MEMORY_H_
|
||||
#define __NVME_MEMORY_H_
|
||||
|
||||
#include <OS.h>
|
||||
|
||||
|
||||
int nvme_mem_init();
|
||||
void nvme_mem_cleanup();
|
||||
|
||||
void* nvme_mem_alloc_node(size_t size, size_t align,
|
||||
unsigned int node_id, phys_addr_t* paddr);
|
||||
void* nvme_malloc_node(size_t size, size_t align,
|
||||
unsigned int node_id);
|
||||
|
||||
phys_addr_t nvme_mem_vtophys(void* vaddr);
|
||||
|
||||
|
||||
#define NVME_VTOPHYS_ERROR (~0ULL)
|
||||
|
||||
#define nvme_node_max() (1)
|
||||
#define NVME_NODE_MAX (1)
|
||||
#define nvme_node_id() (0)
|
||||
|
||||
|
||||
#endif /* __NVME_MEMORY_H_ */
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright 2019, Haiku, Inc. All rights reserved.
|
||||
* Distributed under the terms of the MIT License.
|
||||
*
|
||||
* Authors:
|
||||
* Augustin Cavalier <waddlesplash>
|
||||
*/
|
||||
#ifndef __NVME_PLATFORM_H__
|
||||
#define __NVME_PLATFORM_H__
|
||||
|
||||
#include <lock.h>
|
||||
|
||||
|
||||
#define pthread_mutex_t mutex
|
||||
#define PTHREAD_MUTEX_INITIALIZER MUTEX_INITIALIZER(__FILE__)
|
||||
#define pthread_mutex_init(mtx, attr) mutex_init(mtx, __FILE__)
|
||||
#define pthread_mutex_destroy mutex_destroy
|
||||
#define pthread_mutex_lock mutex_lock
|
||||
#define pthread_mutex_unlock mutex_unlock
|
||||
|
||||
|
||||
#endif /* __NVME_PLATFORM_H__ */
|
||||
Reference in New Issue
Block a user