diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme.c b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme.c new file mode 100644 index 0000000000..3b09175102 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme.c @@ -0,0 +1,385 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +/* + * List of open controllers and its lock. + */ +LIST_HEAD(, nvme_ctrlr) ctrlr_head = LIST_HEAD_INITIALIZER(ctrlr_head); +static pthread_mutex_t ctrlr_lock = PTHREAD_MUTEX_INITIALIZER; + +/* + * Search for an open controller. + */ +static struct nvme_ctrlr *nvme_ctrlr_get(struct nvme_ctrlr *ctrlr, + bool remove) +{ + struct nvme_ctrlr *c; + + pthread_mutex_lock(&ctrlr_lock); + + LIST_FOREACH(c, &ctrlr_head, link) { + if (c == ctrlr) { + if (remove) + LIST_REMOVE(c, link); + goto out; + } + } + + ctrlr = NULL; + +out: + pthread_mutex_unlock(&ctrlr_lock); + + return ctrlr; +} + +#ifndef __HAIKU__ +/* + * Probe a pci device identified by its name. + * Name should be in the form: [0000:]00:00.0 + * Return NULL if failed + */ +static struct pci_device *nvme_pci_ctrlr_probe(const char *slot_name) +{ + char *domain = NULL, *bus = NULL, *dev = NULL, *func = NULL, *end = NULL; + char *pciid = strdup(slot_name); + struct pci_slot_match slot; + struct pci_device *pci_dev = NULL; + + if (!pciid) + return NULL; + + memset(&slot, 0, sizeof(struct pci_slot_match)); + + func = strrchr(pciid, '.'); + if (func) { + *func = '\0'; + func++; + } + + dev = strrchr(pciid, ':'); + if (dev) { + *dev = '\0'; + dev++; + } + + bus = strrchr(pciid, ':'); + if (!bus) { + domain = NULL; + bus = pciid; + } else { + domain = pciid; + *bus = '\0'; + bus++; + } + + if (!bus || !dev || !func) { + nvme_err("Malformed PCI device slot name %s\n", + slot_name); + goto out; + } + + if (domain) { + slot.domain = (uint32_t)strtoul(domain, &end, 16); + if ((end && *end) || (slot.domain > 0xffff)) { + nvme_err("Invalid domain number: 0x%X\n", slot.domain); + return NULL; + } + } else { + slot.domain = PCI_MATCH_ANY; + } + + slot.bus = (uint32_t)strtoul(bus, &end, 16); + if ((end && *end) || (slot.bus > 0xff)) { + nvme_err("Invalid bus number: 0x%X\n", slot.bus); + return NULL; + } + + slot.dev = strtoul(dev, &end, 16); + if ((end && *end) || (slot.dev > 0x1f)) { + nvme_err("Invalid device number: 0x%X\n", slot.dev); + return NULL; + } + + slot.func = strtoul(func, &end, 16); + if ((end && *end) || (slot.func > 7)) { + nvme_err("Invalid function number: 0x%X\n", slot.func); + return NULL; + } + + nvme_debug("PCI URL: domain 0x%X, bus 0x%X, dev 0x%X, func 0x%X\n", + slot.domain, slot.bus, slot.dev, slot.func); + + pci_dev = nvme_pci_device_probe(&slot); + if (pci_dev) { + slot.domain = pci_dev->domain; + if (slot.domain == PCI_MATCH_ANY) + slot.domain = 0; + nvme_info("Found NVMe controller %04x:%02x:%02x.%1u\n", + slot.domain, + slot.bus, + slot.dev, + slot.func); + } + +out: + free(pciid); + + return pci_dev; +} +#endif + +/* + * Open an NVMe controller. + */ +#ifdef __HAIKU__ +struct nvme_ctrlr *nvme_ctrlr_open(struct pci_device *pdev, + struct nvme_ctrlr_opts *opts) +#else +struct nvme_ctrlr *nvme_ctrlr_open(const char *url, + struct nvme_ctrlr_opts *opts) +#endif +{ + struct nvme_ctrlr *ctrlr; +#ifndef __HAIKU__ + char *slot; + + /* Check url */ + if (strncmp(url, "pci://", 6) != 0) { + nvme_err("Invalid URL %s\n", url); + return NULL; + } + + /* Probe PCI device */ + slot = (char *)url + 6; + pdev = nvme_pci_ctrlr_probe(slot); + if (!pdev) { + nvme_err("Device %s not found\n", url); + return NULL; + } +#endif + + pthread_mutex_lock(&ctrlr_lock); + + /* Verify that this controller is not already open */ + LIST_FOREACH(ctrlr, &ctrlr_head, link) { + if (nvme_pci_dev_cmp(ctrlr->pci_dev, pdev) == 0) { + nvme_err("Controller already open\n"); + ctrlr = NULL; + goto out; + } + } + + /* Attach the device */ + ctrlr = nvme_ctrlr_attach(pdev, opts); + if (!ctrlr) { + nvme_err("Attach failed\n"); + goto out; + } + + /* Add controller to the list */ + LIST_INSERT_HEAD(&ctrlr_head, ctrlr, link); + +out: + pthread_mutex_unlock(&ctrlr_lock); + + return ctrlr; + +} + +/* + * Close an open controller. + */ +int nvme_ctrlr_close(struct nvme_ctrlr *ctrlr) +{ + + /* + * Verify that this controller is open. + * If it is, remove it from the list. + */ + ctrlr = nvme_ctrlr_get(ctrlr, true); + if (!ctrlr) { + nvme_err("Invalid controller\n"); + return EINVAL; + } + + nvme_ctrlr_detach(ctrlr); + + return 0; +} + +/* + * Get controller information. + */ +int nvme_ctrlr_stat(struct nvme_ctrlr *ctrlr, struct nvme_ctrlr_stat *cstat) +{ + struct pci_device *pdev = ctrlr->pci_dev; + unsigned int i; + + /* Verify that this controller is open */ + ctrlr = nvme_ctrlr_get(ctrlr, false); + if (!ctrlr) { + nvme_err("Invalid controller\n"); + return EINVAL; + } + + pthread_mutex_lock(&ctrlr->lock); + + memset(cstat, 0, sizeof(struct nvme_ctrlr_stat)); + + /* Controller serial and model number */ + strncpy(cstat->sn, (char *)ctrlr->cdata.sn, + NVME_SERIAL_NUMBER_LENGTH - 1); + strncpy(cstat->mn, (char *)ctrlr->cdata.mn, + NVME_MODEL_NUMBER_LENGTH - 1); + + /* Remove heading and trailling spaces */ + nvme_str_trim(cstat->sn); + nvme_str_trim(cstat->mn); + + /* PCI device info */ + cstat->vendor_id = pdev->vendor_id; + cstat->device_id = pdev->device_id; + cstat->subvendor_id = pdev->subvendor_id; + cstat->subdevice_id = pdev->subdevice_id; +#ifndef __HAIKU__ + cstat->device_class = pdev->device_class; + cstat->revision = pdev->revision; + cstat->domain = pdev->domain; + cstat->bus = pdev->bus; + cstat->dev = pdev->dev; + cstat->func = pdev->func; +#endif + + /* Maximum transfer size */ + cstat->max_xfer_size = ctrlr->max_xfer_size; + + memcpy(&cstat->features, &ctrlr->feature_supported, + sizeof(ctrlr->feature_supported)); + memcpy(&cstat->log_pages, &ctrlr->log_page_supported, + sizeof(ctrlr->log_page_supported)); + + cstat->nr_ns = ctrlr->nr_ns; + for (i = 0; i < ctrlr->nr_ns; i++) { + cstat->ns_ids[i] = i + 1; + } + + /* Maximum io qpair possible */ + cstat->max_io_qpairs = ctrlr->max_io_queues; + + /* Constructed io qpairs */ + cstat->io_qpairs = ctrlr->io_queues; + + /* Enabled io qpairs */ + cstat->enabled_io_qpairs = ctrlr->enabled_io_qpairs; + + /* Max queue depth */ + cstat->max_qd = ctrlr->io_qpairs_max_entries; + + pthread_mutex_unlock(&ctrlr->lock); + + return 0; +} + +/* + * Get controller data + */ +int nvme_ctrlr_data(struct nvme_ctrlr *ctrlr, struct nvme_ctrlr_data *cdata, + struct nvme_register_data *rdata) +{ + union nvme_cap_register cap; + + /* Verify that this controller is open */ + ctrlr = nvme_ctrlr_get(ctrlr, false); + if (!ctrlr) { + nvme_err("Invalid controller\n"); + return EINVAL; + } + + pthread_mutex_lock(&ctrlr->lock); + + /* Controller data */ + if (cdata) + memcpy(cdata, &ctrlr->cdata, sizeof(struct nvme_ctrlr_data)); + + /* Read capabilities register */ + if (rdata) { + cap.raw = nvme_reg_mmio_read_8(ctrlr, cap.raw); + rdata->mqes = cap.bits.mqes; + } + + pthread_mutex_unlock(&ctrlr->lock); + + return 0; +} + +/* + * Get qpair information + */ +int nvme_qpair_stat(struct nvme_qpair *qpair, struct nvme_qpair_stat *qpstat) +{ + struct nvme_ctrlr *ctrlr = qpair->ctrlr; + + /* Verify that this controller is open */ + ctrlr = nvme_ctrlr_get(ctrlr, false); + if (!ctrlr) { + nvme_err("Invalid controller\n"); + return EINVAL; + } + + pthread_mutex_lock(&ctrlr->lock); + + qpstat->id = qpair->id; + qpstat->qd = qpair->entries; + qpstat->enabled = qpair->enabled; + qpstat->qprio = qpair->qprio; + + pthread_mutex_unlock(&ctrlr->lock); + + return 0; +} + +/* + * Close all open controllers on exit. + */ +void nvme_ctrlr_cleanup(void) +{ + struct nvme_ctrlr *ctrlr; + + while ((ctrlr = LIST_FIRST(&ctrlr_head))) { + LIST_REMOVE(ctrlr, link); + nvme_ctrlr_detach(ctrlr); + } +} diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme.h b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme.h new file mode 100644 index 0000000000..9e958ac161 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme.h @@ -0,0 +1,1249 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * NVMe driver public API + * + * @mainpage + * + * libnvme is a user space utility to provide control over NVMe, + * the host controller interface for drives based on PCI Express. + * + * \addtogroup libnvme + * @{ + */ + +#ifndef __LIBNVME_H__ +#define __LIBNVME_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#ifndef __HAIKU__ +#include +#endif +#include +#include +#include +#include + +/** + * Log levels. + */ +enum nvme_log_level { + + /** + * Disable all log messages. + */ + NVME_LOG_NONE = 0, + + /** + * System is unusable. + */ + NVME_LOG_EMERG, + + /** + * Action must be taken immediately. + */ + NVME_LOG_ALERT, + + /** + * Critical conditions. + */ + NVME_LOG_CRIT, + + /** + * Error conditions. + */ + NVME_LOG_ERR, + + /** + * Warning conditions. + */ + NVME_LOG_WARNING, + + /** + * Normal but significant condition. + */ + NVME_LOG_NOTICE, + + /** + * Informational messages. + */ + NVME_LOG_INFO, + + /** + * Debug-level messages. + */ + NVME_LOG_DEBUG, + +}; + +/** + * Log facilities. + */ +enum nvme_log_facility { + + /** + * Standard output log facility + */ + NVME_LOG_STDOUT = 0x00000001, + + /** + * Regular file output log facility + */ + NVME_LOG_FILE = 0x00000002, + + /** + * syslog service output log facility + */ + NVME_LOG_SYSLOG = 0x00000004, + +}; + +/** + * @brief Initialize libnvme + * + * @param level Library log level + * @param facility Facility code + * @param path File name for the NVME_LOG_FILE facility + * + * This function must always be called first before any other + * function provided by libnvme. The arguments allow setting the + * initial log level and log facility so that any problem during + * initialization can be caught. + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_lib_init(enum nvme_log_level level, + enum nvme_log_facility facility, const char *path); + +/** + * @brief Set the library log level + * + * @param level Library log level + */ +extern void nvme_set_log_level(enum nvme_log_level level); + +/** + * @brief Get the current log level + * + * @return The current library log level. + */ +extern enum nvme_log_level nvme_get_log_level(void); + +/** + * @brief Change the library log facility + * + * @param facility Facility code + * @param path File name for the NVME_LOG_FILE facility + * + * Set th library log facility. On failure, the facility is + * always automatically set to stdout. + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_set_log_facility(enum nvme_log_facility facility, + const char *path); + +/** + * @brief Get the current library log facility. + * + * @return The current library log facility. + */ +extern enum nvme_log_facility nvme_get_log_facility(void); + +/** + * @brief Opaque handle to a controller returned by nvme_ctrlr_open(). + */ +struct nvme_ctrlr; + +/** + * @brief Opaque handle to a namespace + */ +struct nvme_ns; + +/** + * @brief Opaque handle to an I/O queue pair + */ +struct nvme_qpair; + +/** + * @brief Capabilities register of a controller + */ +struct nvme_register_data { + + /** + * Maximum Queue Entries Supported indicates the maximum individual + * queue size that the controller supports. This is a 0’s based value, + * so 1 has to be added. + */ + unsigned int mqes; + +}; + +/** + * Length of the string for the serial number + */ +#define NVME_SERIAL_NUMBER_LENGTH NVME_SERIAL_NUMBER_CHARACTERS + 1 + +/** + * Length of the string for the model number + */ +#define NVME_MODEL_NUMBER_LENGTH NVME_MODEL_NUMBER_CHARACTERS + 1 + +/** + * @brief Controller information + */ +struct nvme_ctrlr_stat { + + /** + * PCI device vendor ID. + */ + unsigned short vendor_id; + + /** + * PCI device ID. + */ + unsigned short device_id; + + /** + * PCI device sub-vendor ID. + */ + unsigned short subvendor_id; + + /** + * PCI sub-device ID. + */ + unsigned short subdevice_id; + + /** + * PCI device class. + */ + unsigned int device_class; + + /** + * PCI device revision. + */ + unsigned char revision; + + /** + * PCI slot domain. + */ + unsigned int domain; + + /** + * PCI slot bus. + */ + unsigned int bus; + + /** + * PCI slot bus device number. + */ + unsigned int dev; + + /** + * PCI slot device function. + */ + unsigned int func; + + /** + * Serial number + */ + char sn[NVME_SERIAL_NUMBER_LENGTH]; + + /** + * Model number + */ + char mn[NVME_MODEL_NUMBER_LENGTH]; + + /** + * Maximum transfer size. + */ + size_t max_xfer_size; + + /** + * All the log pages supported. + */ + bool log_pages[256]; + + /** + * All the features supported. + */ + bool features[256]; + + /** + * Number of valid namespaces in the array of namespace IDs. + */ + unsigned int nr_ns; + + /** + * Array of valid namespace IDs of the controller. + * Namspeace IDs are integers between 1 and NVME_MAX_NS + */ + unsigned int ns_ids[NVME_MAX_NS]; + + /** + * Maximum number of I/O queue pairs + */ + unsigned int max_io_qpairs; + + /** + * Number of I/O queue pairs allocated + */ + unsigned int io_qpairs; + + /** + * Number of I/O queue pairs enabled + */ + unsigned int enabled_io_qpairs; + + /** + * IO qpairs maximum entries + */ + unsigned int max_qd; +}; + +/** + * @brief NVMe controller options + * + * Allow the user to request non-default options. + */ +struct nvme_ctrlr_opts { + + /** + * Number of I/O queues to initialize. + * (default: all possible I/O queues) + */ + unsigned int io_queues; + + /** + * Enable submission queue in controller memory buffer + * (default: false) + */ + bool use_cmb_sqs; + + /** + * Type of arbitration mechanism. + * (default: round-robin == NVME_CC_AMS_RR) + */ + enum nvme_cc_ams arb_mechanism; + +}; + +/** + * @brief Namespace command support flags + */ +enum nvme_ns_flags { + + /** + * The deallocate command is supported. + */ + NVME_NS_DEALLOCATE_SUPPORTED = 0x1, + + /** + * The flush command is supported. + */ + NVME_NS_FLUSH_SUPPORTED = 0x2, + + /** + * The reservation command is supported. + */ + NVME_NS_RESERVATION_SUPPORTED = 0x4, + + /** + * The write zeroes command is supported. + */ + NVME_NS_WRITE_ZEROES_SUPPORTED = 0x8, + + /** + * The end-to-end data protection is supported. + */ + NVME_NS_DPS_PI_SUPPORTED = 0x10, + + /** + * The extended lba format is supported, metadata is transferred as + * a contiguous part of the logical block that it is associated with. + */ + NVME_NS_EXTENDED_LBA_SUPPORTED = 0x20, + +}; + +/** + * @brief Namespace information + */ +struct nvme_ns_stat { + + /** + * Namespace ID. + */ + unsigned int id; + + /** + * Namespace command support flags. + */ + enum nvme_ns_flags flags; + + /** + * Namespace sector size in bytes. + */ + size_t sector_size; + + /** + * Namespace number of sectors. + */ + uint64_t sectors; + + /** + * Namespace metadata size in bytes. + */ + size_t md_size; + + /** + * Namespace priority information type. + */ + enum nvme_pi_type pi_type; + +}; + +/** + * @brief Queue pair information + */ +struct nvme_qpair_stat { + + /** + * Qpair ID + */ + unsigned int id; + + /** + * Qpair number of entries + */ + unsigned int qd; + + /** + * Qpair is enabled + */ + bool enabled; + + /** + * Qpair priority + */ + unsigned int qprio; +}; + +/** + * @brief Command completion callback function signature + * + * @param cmd_cb_arg Callback function input argument. + * @param cpl_status Contains the completion status. + */ +typedef void (*nvme_cmd_cb)(void *cmd_cb_arg, + const struct nvme_cpl *cpl_status); + +/** + * @brief Asynchronous error request completion callback + * + * @param aer_cb_arg AER context set by nvme_register_aer_callback() + * @param cpl_status Completion status of the asynchronous event request + */ +typedef void (*nvme_aer_cb)(void *aer_cb_arg, + const struct nvme_cpl *cpl_status); + +/** + * @brief Restart SGL walk to the specified offset callback + * + * @param cb_arg Value passed to nvme_readv/nvme_writev + * @param offset Offset in the SGL + */ +typedef void (*nvme_req_reset_sgl_cb)(void *cb_arg, uint32_t offset); + +/** + * @brief Get an SGL entry address and length and advance to the next entry + * + * @param cb_arg Value passed to readv/writev + * @param address Physical address of this segment + * @param length Length of this physical segment + * + * Fill out address and length with the current SGL entry and advance + * to the next entry for the next time the callback is invoked + */ +typedef int (*nvme_req_next_sge_cb)(void *cb_arg, + uint64_t *address, uint32_t *length); + +/** + * @brief Open an NVMe controller + * + * @param url PCI device URL + * @param opts controller options + * + * Obtain a handle for an NVMe controller specified as a PCI device URL, + * e.g. pci://[DDDD:]BB:DD.F. If called more than once for the same + * controller, NULL is returned. + * To stop using the the controller and release its associated resources, + * call nvme_ctrlr_close() with the handle returned by this function. + * + * @return A handle to the controller on success and NULL on failure. + */ +struct pci_device { + uint16_t vendor_id; + uint16_t device_id; + uint16_t subvendor_id; + uint16_t subdevice_id; + + uint16_t domain; + uint16_t bus; + uint16_t dev; + uint16_t func; + + void* pci_info; +}; + +extern struct nvme_ctrlr * nvme_ctrlr_open(struct pci_device *pdev, + struct nvme_ctrlr_opts *opts); + +/** + * @brief Close an open NVMe controller + * + * @param ctrlr Controller handle + * + * This function should be called while no other threads + * are actively using the controller. + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_ctrlr_close(struct nvme_ctrlr *ctrlr); + +/** + * @brief Get controller capabilities and features + * + * @param ctrlr Controller handle + * @param cstat Controller information + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_ctrlr_stat(struct nvme_ctrlr *ctrlr, + struct nvme_ctrlr_stat *cstat); + +/** + * @brief Get controller data and some data from the capabilities register + * + * @param ctrlr Controller handle + * @param cdata Controller data to fill + * @param rdata Capabilities register data to fill + * + * cdata and rdata are optional (NULL can be specified). + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_ctrlr_data(struct nvme_ctrlr *ctrlr, + struct nvme_ctrlr_data *cdata, + struct nvme_register_data *rdata); + +/** + * @brief Get a specific feature of a controller + * + * @param ctrlr Controller handle + * @param sel Feature selector + * @param feature Feature identifier + * @param cdw11 Command word 11 (command dependent) + * @param attributes Features attributes + * + * This function is thread safe and can be called at any point while + * the controller is attached. + * + * @return 0 on success and a negative error code on failure. + * + * See nvme_ctrlr_set_feature() + */ +extern int nvme_ctrlr_get_feature(struct nvme_ctrlr *ctrlr, + enum nvme_feat_sel sel, + enum nvme_feat feature, + uint32_t cdw11, uint32_t *attributes); + +/** + * @brief Set a specific feature of a controller + * + * @param ctrlr Controller handle + * @param save Save feature across power cycles + * @param feature Feature identifier + * @param cdw11 Command word 11 (feature dependent) + * @param cdw12 Command word 12 (feature dependent) + * @param attributes Features attributes + * + * This function is thread safe and can be called at any point while + * the controller is attached to the NVMe driver. + * + * @return 0 on success and a negative error code on failure. + * + * See nvme_ctrlr_get_feature() + */ +extern int nvme_ctrlr_set_feature(struct nvme_ctrlr *ctrlr, + bool save, enum nvme_feat feature, + uint32_t cdw11, uint32_t cdw12, + uint32_t *attributes); + +/** + * @brief Attach the specified namespace to controllers + * + * @param ctrlr Controller handle to use for command submission + * @param nsid Namespace ID of the namespaces to attach + * @param clist List of controllers as defined in the NVMe specification + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_ctrlr_attach_ns(struct nvme_ctrlr *ctrlr, unsigned int nsid, + struct nvme_ctrlr_list *clist); + +/** + * @brief Detach the specified namespace from controllers + * + * @param ctrlr Controller handle to use for command submission + * @param nsid Namespace ID of the namespaces to detach + * @param clist List of controllers as defined in the NVMe specification + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_ctrlr_detach_ns(struct nvme_ctrlr *ctrlr, unsigned int nsid, + struct nvme_ctrlr_list *clist); + +/** + * @brief Create a namespace + * + * @param ctrlr Controller handle + * @param nsdata namespace data + * + * @return Namespace ID (>= 1) on success and 0 on failure. + */ +extern unsigned int nvme_ctrlr_create_ns(struct nvme_ctrlr *ctrlr, + struct nvme_ns_data *nsdata); + +/** + * @brief Delete a namespace + * + * @param ctrlr Controller handle + * @param nsid ID of the namespace to delete + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_ctrlr_delete_ns(struct nvme_ctrlr *ctrlr, unsigned int nsid); + +/** + * @brief Format media + * + * @param ctrlr Controller handle + * @param nsid ID of the namespace to format + * @param format Format information + * + * This function requests a low-level format of the media. + * If nsid is NVME_GLOBAL_NS_TAG, all namspaces attached to the contoller + * are formatted. + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_ctrlr_format_ns(struct nvme_ctrlr *ctrlr, + unsigned int nsid, struct nvme_format *format); + +/** + * @brief Download a new firmware image + * + * @param ctrlr Controller handle + * @param fw Firmware data buffer + * @param size Firmware buffer size + * @param slot Firmware image slot to use + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_ctrlr_update_firmware(struct nvme_ctrlr *ctrlr, + void *fw, size_t size, int slot); + +/** + * @brief Get an I/O queue pair + * + * @param ctrlr Controller handle + * @param qprio I/O queue pair priority for weighted round robin arbitration + * @param qd I/O queue pair maximum submission queue depth + * + * A queue depth of 0 will result in the maximum hardware defined queue + * depth being used. The use of a queue pair is not thread safe. Applications + * must ensure mutual exclusion access to the queue pair during I/O processing. + * + * @return An I/O queue pair handle on success and NULL in case of failure. + */ +extern struct nvme_qpair * nvme_ioqp_get(struct nvme_ctrlr *ctrlr, + enum nvme_qprio qprio, + unsigned int qd); + +/** + * @brief Release an I/O queue pair + * + * @param qpair I/O queue pair handle + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_ioqp_release(struct nvme_qpair *qpair); + +/** + * @brief Get information on an I/O queue pair + * + * @param qpair I/O queue pair handle + * @param qpstat I/O queue pair information to fill + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_qpair_stat(struct nvme_qpair *qpair, + struct nvme_qpair_stat *qpstat); + +/** + * @brief Submit an NVMe command + * + * @param qpair I/O qpair handle + * @param cmd Command to submit + * @param buf Payload buffer + * @param len Payload buffer length + * @param cb_fn Callback function + * @param cb_arg Argument for the call back function + * + * This is a low level interface for submitting I/O commands directly. + * The validity of the command will not be checked. + * + * When constructing the nvme_command it is not necessary to fill out the PRP + * list/SGL or the CID. The driver will handle both of those for you. + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_ioqp_submit_cmd(struct nvme_qpair *qpair, + struct nvme_cmd *cmd, + void *buf, size_t len, + nvme_cmd_cb cb_fn, void *cb_arg); + +/** + * @brief Process I/O command completions + * + * @param qpair I/O queue pair handle + * @param max_completions Maximum number of completions to check + * + * This call is non-blocking, i.e. it only processes completions that are + * ready at the time of this function call. It does not wait for + * outstanding commands to complete. + * For each completed command, the request callback function will + * be called if specified as non-NULL when the request was submitted. + * This function may be called at any point after the command submission + * while the controller is open + * + * @return The number of completions processed (may be 0). + * + * @sa nvme_cmd_cb + */ +extern unsigned int nvme_ioqp_poll(struct nvme_qpair *qpair, + unsigned int max_completions); + +/** + * @brief Open a name space + * + * @param ctrlr Controller handle + * @param ns_id ID of the name space to open + * + * @return A namspace handle on success or NULL in case of failure. + */ +extern struct nvme_ns *nvme_ns_open(struct nvme_ctrlr *ctrlr, + unsigned int ns_id); + +/** + * @brief Close an open name space + * + * @param ns Namspace handle + * + * See nvme_ns_open() + */ +extern int nvme_ns_close(struct nvme_ns *ns); + +/** + * @brief Get information on a namespace + * + * @param ns Namespace handle + * @param ns_stat Namespace information + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_stat(struct nvme_ns *ns, + struct nvme_ns_stat *ns_stat); + +/** + * @brief Get namespace data + * + * @param ns Namespace handle + * @param nsdata Namespace data + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_data(struct nvme_ns *ns, + struct nvme_ns_data *nsdata); + +/** + * @brief Submit a write I/O + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param buffer Data buffer + * @param lba Starting LBA to read from + * @param lba_count Number of LBAs to read + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * @param io_flags I/O flags (NVME_IO_FLAGS_*) + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_write(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *buffer, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags); + +/** + * @brief Submit a scattered write I/O + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param lba Starting LBA to write to + * @param lba_count Number of LBAs to write + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * @param io_flags I/O flags (NVME_IO_FLAGS_*) + * @param reset_sgl_fn Reset scattered payload callback + * @param next_sge_fn Scattered payload iteration callback + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_writev(struct nvme_ns *ns, struct nvme_qpair *qpair, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags, + nvme_req_reset_sgl_cb reset_sgl_fn, + nvme_req_next_sge_cb next_sge_fn); + +/** + * @brief Submits a write I/O with metadata + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param payload Data buffer + * @param metadata Metadata payload + * @param lba Starting LBA to write to + * @param lba_count Number of LBAs to write + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * @param io_flags I/O flags (NVME_IO_FLAGS_*) + * @param apptag_mask Application tag mask + * @param apptag Application tag to use end-to-end protection information + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_write_with_md(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *payload, void *metadata, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags, + uint16_t apptag_mask, uint16_t apptag); + +/** + * @brief Submit a write zeroes I/O + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param lba Starting LBA to write to + * @param lba_count Number of LBAs to write + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * @param io_flags I/O flags (NVME_IO_FLAGS_*) + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_write_zeroes(struct nvme_ns *ns, struct nvme_qpair *qpair, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags); + +/** + * @brief Submit a read I/O + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param buffer Data buffer + * @param lba Starting LBA to read from + * @param lba_count Number of LBAs to read + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * @param io_flags I/O flags (NVME_IO_FLAGS_*) + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_read(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *buffer, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags); + +/** + * @brief Submit a scattered read I/O + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param lba Starting LBA to read from + * @param lba_count Number of LBAs to read + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * @param io_flags I/O flags (NVME_IO_FLAGS_*) + * @param reset_sgl_fn Reset scattered payload callback + * @param next_sge_fn Scattered payload iteration callback + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_readv(struct nvme_ns *ns, struct nvme_qpair *qpair, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags, + nvme_req_reset_sgl_cb reset_sgl_fn, + nvme_req_next_sge_cb next_sge_fn); + +/** + * @brief Submit a read I/O with metadata + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param buffer Data buffer + * @param metadata Metadata payload + * @param lba Starting LBA to read from + * @param lba_count Number of LBAs to read + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * @param io_flags I/O flags (NVME_IO_FLAGS_*) + * @param apptag_mask Application tag mask + * @param apptag Application tag to use end-to-end protection information + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_read_with_md(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *buffer, void *metadata, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags, + uint16_t apptag_mask, uint16_t apptag); + +/** + * @brief Submit a deallocate command + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param payload List of LBA ranges to deallocate + * @param num_ranges Number of ranges in the list + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * + * The number of LBA ranges must be at least 1 and at most + * NVME_DATASET_MANAGEMENT_MAX_RANGES. + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_deallocate(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *payload, uint16_t num_ranges, + nvme_cmd_cb cb_fn, void *cb_arg); + +/** + * @brief Submit a flush command + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_flush(struct nvme_ns *ns, struct nvme_qpair *qpair, + nvme_cmd_cb cb_fn, void *cb_arg); + +/** + * @brief Submit a reservation register command + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param payload Reservation register data buffer + * @param ignore_key Enable or not the current reservation key check + * @param action Registration action + * @param cptpl Persist Through Power Loss state + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_reservation_register(struct nvme_ns *ns, + struct nvme_qpair *qpair, + struct nvme_reservation_register_data *payload, + bool ignore_key, + enum nvme_reservation_register_action action, + enum nvme_reservation_register_cptpl cptpl, + nvme_cmd_cb cb_fn, void *cb_arg); + +/** + * @brief Submit a reservation release command + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param payload Current reservation key buffer + * @param ignore_key Enable or not the current reservation key check + * @param action Reservation release action + * @param type Reservation type + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_reservation_release(struct nvme_ns *ns, + struct nvme_qpair *qpair, + struct nvme_reservation_key_data *payload, + bool ignore_key, + enum nvme_reservation_release_action action, + enum nvme_reservation_type type, + nvme_cmd_cb cb_fn, void *cb_arg); + +/** + * @brief Submit a reservation acquire command + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param payload Reservation acquire data buffer + * @param ignore_key Enable or not the current reservation key check + * @param action Reservation acquire action + * @param type Reservation type + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_reservation_acquire(struct nvme_ns *ns, + struct nvme_qpair *qpair, + struct nvme_reservation_acquire_data *payload, + bool ignore_key, + enum nvme_reservation_acquire_action action, + enum nvme_reservation_type type, + nvme_cmd_cb cb_fn, void *cb_arg); + +/** + * @brief Submits a reservation report to a namespace + * + * @param ns Namespace handle + * @param qpair I/O queue pair handle + * @param payload Reservation status data buffer + * @param len Length in bytes of the reservation status data + * @param cb_fn Completion callback + * @param cb_arg Argument to pass to the completion callback + * + * The command is submitted to a qpair allocated by nvme_ctrlr_alloc_io_qpair(). + * The user must ensure that only one thread submits I/O on + * a given qpair at any given time. + * + * @return 0 on success and a negative error code in case of failure. + */ +extern int nvme_ns_reservation_report(struct nvme_ns *ns, + struct nvme_qpair *qpair, + void *payload, size_t len, + nvme_cmd_cb cb_fn, void *cb_arg); + +/** + * Any NUMA node. + */ +#define NVME_NODE_ID_ANY (~0U) + +/** + * @brief Allocate physically contiguous memory + * + * @param size Size (in bytes) to be allocated + * @param align Memory alignment constraint + * @param node_id The NUMA node to get memory from or NVME_NODE_ID_ANY + * + * This function allocates memory from the hugepage area of memory. The + * memory is not cleared. In NUMA systems, the memory allocated resides + * on the requested NUMA node if node_id is not NVME_NODE_ID_ANY. + * Otherwise, allocation will take preferrably on the node of the + * function call context, or any other node if that fails. + * + * @return The address of the allocated memory on success and NULL on failure. + */ +extern void *nvme_malloc_node(size_t size, size_t align, + unsigned int node_id); + +/** + * @brief Allocate zero'ed memory + * + * @param size Size (in bytes) to be allocated + * @param align Memory alignment constraint + * @param node_id The NUMA node to get memory from or NVME_NODE_ID_ANY + * + * See @nvme_malloc_node. + */ +static inline void *nvme_zmalloc_node(size_t size, size_t align, + unsigned int node_id) +{ + void *buf; + + buf = nvme_malloc_node(size, align, node_id); + if (buf) + memset(buf, 0, size); + + return buf; +} + +/** + * @brief Allocate zero'ed array memory + * + * @param num Size of the array + * @param size Size (in bytes) of the array elements + * @param align Memory alignment constraint + * @param node_id The NUMA node to get memory from or NVME_NODE_ID_ANY + * + * See @nvme_malloc_node. + */ +static inline void *nvme_calloc_node(size_t num, size_t size, + size_t align, unsigned int node_id) +{ + return nvme_zmalloc_node(size * num, align, node_id); +} + +/** + * @brief Allocate physically contiguous memory + * + * @param size Size (in bytes) to be allocated + * @param align Memory alignment constraint + * + * @return The address of the allocated memory on success and NULL on error + * + * See @nvme_malloc_node. + */ +static inline void *nvme_malloc(size_t size, size_t align) +{ + return nvme_malloc_node(size, align, NVME_NODE_ID_ANY); +} + +/** + * @brief Allocate zero'ed memory + * + * @param size Size (in bytes) to be allocated + * @param align Memory alignment constraint + * + * @return The address of the allocated memory on success and NULL on error + * + * See @nvme_zmalloc_node. + */ +static inline void *nvme_zmalloc(size_t size, size_t align) +{ + return nvme_zmalloc_node(size, align, NVME_NODE_ID_ANY); +} + +/** + * @brief Allocate zero'ed array memory + * + * @param num Size of the array + * @param size Size (in bytes) of the array elements + * @param align Memory alignment constraint + * + * See @nvme_calloc_node. + */ +static inline void *nvme_calloc(size_t num, size_t size, size_t align) +{ + return nvme_calloc_node(num, size, align, NVME_NODE_ID_ANY); +} + +/** + * @brief Free allocated memory + * + * @param addr Address of the memory to free + * + * Free the memory at the specified address. + * The address must be one that was returned by one of the + * allocation function nvme_malloc_node(), nvme_zmalloc_node() + * or nvme_calloc_node(). + * + * If the pointer is NULL, the function does nothing. + */ +extern void nvme_free(void *addr); + +/** + * Structure to hold memory statistics. + */ +struct nvme_mem_stats { + + /** + * Number of huge pages allocated. + */ + size_t nr_hugepages; + + /** + * Total bytes in memory pools. + */ + size_t total_bytes; + + /** + * Total free bytes in memory pools. + */ + size_t free_bytes; + +}; + +/** + * @brief Get memory usage information + * + * @param stats Memory usage inforamtion structure to fill + * @param node_id NUMA node ID or NVME_NVME_NODE_ID_ANY + * + * Return memory usage statistics for the specified + * NUMA node (CPU socket) or global memory usage if node_id + * is NVME_NODE_ID_ANY. + * + * @return 0 on success and a negative error code on failure. + */ +extern int nvme_memstat(struct nvme_mem_stats *stats, + unsigned int node_id); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif /* __LIBNVME_H__ */ diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_admin.c b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_admin.c new file mode 100644 index 0000000000..419d9704ed --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_admin.c @@ -0,0 +1,456 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in sourete and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of sourete code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +/* + * Allocate a request, set its command and submit it + * to the controller admin queue. + */ +static int nvme_admin_submit_cmd(struct nvme_ctrlr *ctrlr, + struct nvme_cmd *cmd, + void *buf, uint32_t len, + nvme_cmd_cb cb_fn, void *cb_arg) +{ + struct nvme_request *req; + + if (buf) + req = nvme_request_allocate_contig(&ctrlr->adminq, buf, len, + cb_fn, cb_arg); + else + req = nvme_request_allocate_null(&ctrlr->adminq, cb_fn, cb_arg); + if (!req) + return ENOMEM; + + memcpy(&req->cmd, cmd, sizeof(req->cmd)); + + return nvme_qpair_submit_request(&ctrlr->adminq, req); +} + +/* + * Poll the controller admin queue waiting for a + * command completion. + */ +static int nvme_admin_wait_cmd(struct nvme_ctrlr *ctrlr, + struct nvme_completion_poll_status *status) +{ + + /* Wait for completion and check result */ + while (status->done == false) + nvme_qpair_poll(&ctrlr->adminq, 0); + + if (nvme_cpl_is_error(&status->cpl)) { + nvme_notice("Admin command failed\n"); + return ENXIO; + } + + return 0; +} + +/* + * Execute an admin command. + */ +static int nvme_admin_exec_cmd(struct nvme_ctrlr *ctrlr, + struct nvme_cmd *cmd, + void *buf, uint32_t len) +{ + struct nvme_completion_poll_status status; + int ret; + + /* Submit the command */ + status.done = false; + ret = nvme_admin_submit_cmd(ctrlr, cmd, buf, len, + nvme_request_completion_poll_cb, + &status); + if (ret != 0) + return ret; + + /* Wait for the command completion and check result */ + return nvme_admin_wait_cmd(ctrlr, &status); +} + +/* + * Get a controller information. + */ +int nvme_admin_identify_ctrlr(struct nvme_ctrlr *ctrlr, + struct nvme_ctrlr_data *cdata) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_IDENTIFY; + cmd.cdw10 = NVME_IDENTIFY_CTRLR; + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, + cdata, sizeof(struct nvme_ctrlr_data)); +} + +/* + * Get a controller feature. + */ +int nvme_admin_get_feature(struct nvme_ctrlr *ctrlr, + enum nvme_feat_sel sel, + enum nvme_feat feature, + uint32_t cdw11, + uint32_t *attributes) +{ + struct nvme_completion_poll_status status; + struct nvme_cmd cmd; + int ret; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_GET_FEATURES; + cmd.cdw10 = (sel << 8) | feature; + cmd.cdw11 = cdw11; + + /* Submit the command */ + status.done = false; + ret = nvme_admin_submit_cmd(ctrlr, &cmd, NULL, 0, + nvme_request_completion_poll_cb, + &status); + if (ret == 0) { + /* Wait for the command completion and check result */ + ret = nvme_admin_wait_cmd(ctrlr, &status); + if (ret == 0 && attributes) + *attributes = status.cpl.cdw0; + } + + return ret; +} + +/* + * Set a feature. + */ +int nvme_admin_set_feature(struct nvme_ctrlr *ctrlr, + bool save, + enum nvme_feat feature, + uint32_t cdw11, + uint32_t cdw12, + uint32_t *attributes) +{ + struct nvme_completion_poll_status status; + struct nvme_cmd cmd; + int ret; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_SET_FEATURES; + cmd.cdw10 = feature; + if (save) + cmd.cdw10 |= (1 << 31); + cmd.cdw11 = cdw11; + cmd.cdw12 = cdw12; + + /* Submit the command */ + status.done = false; + ret = nvme_admin_submit_cmd(ctrlr, &cmd, NULL, 0, + nvme_request_completion_poll_cb, + &status); + if (ret == 0) { + /* Wait for the command completion and check result */ + ret = nvme_admin_wait_cmd(ctrlr, &status); + if (ret == 0 && attributes) + *attributes = status.cpl.cdw0; + } + + return ret; +} + +/* + * Create an I/O queue. + */ +int nvme_admin_create_ioq(struct nvme_ctrlr *ctrlr, + struct nvme_qpair *qpair, + enum nvme_io_queue_type io_qtype) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + switch(io_qtype) { + case NVME_IO_SUBMISSION_QUEUE: + cmd.opc = NVME_OPC_CREATE_IO_SQ; + cmd.cdw11 = (qpair->id << 16) | (qpair->qprio << 1) | 0x1; + cmd.dptr.prp.prp1 = qpair->cmd_bus_addr; + break; + case NVME_IO_COMPLETION_QUEUE: + cmd.opc = NVME_OPC_CREATE_IO_CQ; + cmd.cdw11 = 0x1; + cmd.dptr.prp.prp1 = qpair->cpl_bus_addr; + break; + default: + return EINVAL; + } + + cmd.cdw10 = ((qpair->entries - 1) << 16) | qpair->id; + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0); +} + +/* + * Delete an I/O queue. + */ +int nvme_admin_delete_ioq(struct nvme_ctrlr *ctrlr, + struct nvme_qpair *qpair, + enum nvme_io_queue_type io_qtype) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + switch(io_qtype) { + case NVME_IO_SUBMISSION_QUEUE: + cmd.opc = NVME_OPC_DELETE_IO_SQ; + break; + case NVME_IO_COMPLETION_QUEUE: + cmd.opc = NVME_OPC_DELETE_IO_CQ; + break; + default: + return EINVAL; + } + cmd.cdw10 = qpair->id; + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0); +} + +/* + * Get a namespace information. + */ +int nvme_admin_identify_ns(struct nvme_ctrlr *ctrlr, + uint16_t nsid, + struct nvme_ns_data *nsdata) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_IDENTIFY; + cmd.cdw10 = NVME_IDENTIFY_NS; + cmd.nsid = nsid; + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, + nsdata, sizeof(struct nvme_ns_data)); +} + +/* + * Attach a namespace. + */ +int nvme_admin_attach_ns(struct nvme_ctrlr *ctrlr, + uint32_t nsid, + struct nvme_ctrlr_list *clist) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_NS_ATTACHMENT; + cmd.nsid = nsid; + cmd.cdw10 = NVME_NS_CTRLR_ATTACH; + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, + clist, sizeof(struct nvme_ctrlr_list)); +} + +/* + * Detach a namespace. + */ +int nvme_admin_detach_ns(struct nvme_ctrlr *ctrlr, + uint32_t nsid, + struct nvme_ctrlr_list *clist) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_NS_ATTACHMENT; + cmd.nsid = nsid; + cmd.cdw10 = NVME_NS_CTRLR_DETACH; + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, + clist, sizeof(struct nvme_ctrlr_list)); +} + +/* + * Create a namespace. + */ +int nvme_admin_create_ns(struct nvme_ctrlr *ctrlr, + struct nvme_ns_data *nsdata, + unsigned int *nsid) +{ + struct nvme_completion_poll_status status; + struct nvme_cmd cmd; + int ret; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_NS_MANAGEMENT; + cmd.cdw10 = NVME_NS_MANAGEMENT_CREATE; + + /* Submit the command */ + status.done = false; + ret = nvme_admin_submit_cmd(ctrlr, &cmd, + nsdata, sizeof(struct nvme_ns_data), + nvme_request_completion_poll_cb, + &status); + if (ret == 0) + /* Wait for the command completion and check result */ + ret = nvme_admin_wait_cmd(ctrlr, &status); + + if (ret != 0) + return ret; + + *nsid = status.cpl.cdw0; + + return 0; +} + +/* + * Delete a namespace. + */ +int nvme_admin_delete_ns(struct nvme_ctrlr *ctrlr, + unsigned int nsid) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_NS_MANAGEMENT; + cmd.cdw10 = NVME_NS_MANAGEMENT_DELETE; + cmd.nsid = nsid; + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0); +} + +/* + * Format media. + * (entire device or just the specified namespace) + */ +int nvme_admin_format_nvm(struct nvme_ctrlr *ctrlr, + unsigned int nsid, + struct nvme_format *format) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_FORMAT_NVM; + cmd.nsid = nsid; + memcpy(&cmd.cdw10, format, sizeof(uint32_t)); + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0); +} + +/* + * Get a log page. + */ +int nvme_admin_get_log_page(struct nvme_ctrlr *ctrlr, + uint8_t log_page, + uint32_t nsid, + void *payload, + uint32_t payload_size) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_GET_LOG_PAGE; + cmd.nsid = nsid; + cmd.cdw10 = ((payload_size / sizeof(uint32_t)) - 1) << 16; + cmd.cdw10 |= log_page; + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, payload, payload_size); +} + +/* + * Abort an admin or an I/O command. + */ +int nvme_admin_abort_cmd(struct nvme_ctrlr *ctrlr, + uint16_t cid, uint16_t sqid) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_ABORT; + cmd.cdw10 = (cid << 16) | sqid; + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0); +} + +/* + * Validate a FW. + */ +int nvme_admin_fw_commit(struct nvme_ctrlr *ctrlr, + const struct nvme_fw_commit *fw_commit) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_FIRMWARE_COMMIT; + memcpy(&cmd.cdw10, fw_commit, sizeof(uint32_t)); + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0); +} + +/* + * Download to the device a firmware. + */ +int nvme_admin_fw_image_dl(struct nvme_ctrlr *ctrlr, + void *fw, uint32_t size, + uint32_t offset) +{ + struct nvme_cmd cmd; + + /* Setup the command */ + memset(&cmd, 0, sizeof(struct nvme_cmd)); + cmd.opc = NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD; + cmd.cdw10 = (size >> 2) - 1; + cmd.cdw11 = offset >> 2; + + /* Execute the command */ + return nvme_admin_exec_cmd(ctrlr, &cmd, fw, size); +} diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_common.c b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_common.c new file mode 100644 index 0000000000..97b8efd01b --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_common.c @@ -0,0 +1,289 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2012-2014 6WIND S.A. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_pci.h" +#include "nvme_common.h" +#include "nvme_mem.h" +#ifndef __HAIKU__ +#include "nvme_cpu.h" +#endif + +#include +#include +#include + +#if defined(NVME_ARCH_X86) +#include +#endif +#include +#include +#ifndef __HAIKU__ +#include +#include +#endif + +/* + * Trim whitespace from a string in place. + */ +void nvme_str_trim(char *s) +{ + char *p, *q; + + /* Remove header */ + p = s; + while (*p != '\0' && isspace(*p)) + p++; + + /* Remove tailer */ + q = p + strlen(p); + while (q - 1 >= p && isspace(*(q - 1))) { + q--; + *q = '\0'; + } + + /* if remove header, move */ + if (p != s) { + q = s; + while (*p != '\0') + *q++ = *p++; + *q = '\0'; + } +} + +/* + * Split string into tokens + */ +int nvme_str_split(char *string, int stringlen, + char **tokens, int maxtokens, char delim) +{ + int i, tok = 0; + int tokstart = 1; + + if (string == NULL || tokens == NULL) { + errno = EINVAL; + return -1; + } + + for (i = 0; i < stringlen; i++) { + if (string[i] == '\0' || tok >= maxtokens) + break; + if (tokstart) { + tokstart = 0; + tokens[tok++] = &string[i]; + } + if (string[i] == delim) { + string[i] = '\0'; + tokstart = 1; + } + } + + return tok; +} + +#ifndef __HAIKU__ +/* + * Parse a sysfs (or other) file containing one integer value + */ +int nvme_parse_sysfs_value(const char *filename, + unsigned long *val) +{ + FILE *f; + char buf[BUFSIZ]; + char *end = NULL; + + if ((f = fopen(filename, "r")) == NULL) { + nvme_err("%s(): cannot open sysfs value %s\n", + __func__, filename); + return -1; + } + + if (fgets(buf, sizeof(buf), f) == NULL) { + nvme_err("%s(): cannot read sysfs value %s\n", + __func__, filename); + fclose(f); + return -1; + } + *val = strtoul(buf, &end, 0); + if ((buf[0] == '\0') || (end == NULL) || (*end != '\n')) { + nvme_err("%s(): cannot parse sysfs value %s\n", + __func__, filename); + fclose(f); + return -1; + } + fclose(f); + return 0; +} + +/* + * Get a block device block size in Bytes. + */ +ssize_t nvme_dev_get_blocklen(int fd) +{ + uint32_t blocklen = 0; + + if (ioctl(fd, BLKSSZGET, &blocklen) < 0) { + nvme_err("iioctl BLKSSZGET failed %d (%s)\n", + errno, + strerror(errno)); + return -1; + } + + return blocklen; +} +#endif + +/* + * Get a file size in Bytes. + */ +uint64_t nvme_file_get_size(int fd) +{ + struct stat st; + + if (fstat(fd, &st) != 0) + return 0; + + if (S_ISLNK(st.st_mode)) + return 0; + + if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode)) { +#ifndef __HAIKU__ + uint64_t size; + if (ioctl(fd, BLKGETSIZE64, &size) == 0) + return size; + else +#endif + return 0; + } + + if (S_ISREG(st.st_mode)) + return st.st_size; + + /* Not REG, CHR or BLK */ + return 0; +} + +#ifndef __HAIKU__ +/* + * Dump the stack of the calling core. + */ +static void nvme_dump_stack(void) +{ +#define BACKTRACE_SIZE 256 + void *func[BACKTRACE_SIZE]; + char **symb = NULL; + int size; + + size = backtrace(func, BACKTRACE_SIZE); + symb = backtrace_symbols(func, size); + + if (symb == NULL) + return; + + while (size > 0) { + nvme_crit("%d: [%s]\n", size, symb[size - 1]); + size --; + } + + free(symb); +} +#endif + +#ifndef __HAIKU__ +void +/* + * call abort(), it will generate a coredump if enabled. + */ +void __nvme_panic(const char *funcname, const char *format, ...) +{ + va_list ap; + + nvme_crit("PANIC in %s():\n", funcname); + va_start(ap, format); + nvme_vlog(NVME_LOG_CRIT, format, ap); + va_end(ap); + nvme_dump_stack(); + abort(); +} +#endif + +/** + * Library initialization: must be run first by any application + * before calling any libnvme API. + */ +int nvme_lib_init(enum nvme_log_level level, + enum nvme_log_facility facility, const char *path) +{ + int ret; + +#ifndef __HAIKU__ + /* Set log level and facility first */ + nvme_set_log_level(level); + nvme_set_log_facility(facility, path); + + /* Gather CPU information */ + ret = nvme_cpu_init(); + if (ret != 0) { + nvme_crit("Failed to gather CPU information\n"); + goto out; + } +#endif + + /* PCI subsystem initialization (libpciaccess) */ + ret = nvme_pci_init(); + if (ret != 0) { + nvme_crit("PCI subsystem initialization failed\n"); + goto out; + } + + /* Initialize memory management */ + ret = nvme_mem_init(); + if (ret != 0) + nvme_crit("Memory management initialization failed\n"); + +out: + + return ret; +} + +/* + * Will be executed automatically last on termination of the user application. + */ +__attribute__((destructor)) void nvme_lib_exit(void) +{ + + nvme_ctrlr_cleanup(); + + nvme_mem_cleanup(); + +} diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_common.h b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_common.h new file mode 100644 index 0000000000..64c1ab7fa6 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_common.h @@ -0,0 +1,410 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __NVME_COMMON_H__ +#define __NVME_COMMON_H__ + +#define _GNU_SOURCE +#define _FILE_OFFSET_BITS 64 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __HAIKU__ +#define _BSD_SOURCE +#include +#else +#include +#include +#endif + +#include "libnvme/nvme.h" +#include "nvme_arch.h" +#include "nvme_log.h" + +/* + * Check if a branch is likely to be taken. + */ +#ifndef likely +#define likely(x) __builtin_expect((x),1) +#endif /* likely */ + +/* + * Check if a branch is unlikely to be taken. + */ +#ifndef unlikely +#define unlikely(x) __builtin_expect((x),0) +#endif /* unlikely */ + +#ifndef typeof +#define typeof __typeof__ +#endif + +/* + * Trim whitespace from a string in place. + */ +extern void nvme_str_trim(char *s); + +/* + * Split string into tokens + */ +extern int nvme_str_split(char *string, int stringlen, + char **tokens, int maxtokens, char delim); + +/* + * Converts a numeric string to the equivalent uint64_t value. + * As well as straight number conversion, also recognises the suffixes + * k, m and g for kilobytes, megabytes and gigabytes respectively. + * + * If a negative number is passed in, zero is returned. + * Zero is also returned in the case of an error with the + * strtoull call in the function. + */ +static inline size_t nvme_str2size(const char *str) +{ + unsigned long long size; + char *endptr; + + while (isspace((int)*str)) + str++; + if (*str == '-') + return 0; + + errno = 0; + size = strtoull(str, &endptr, 0); + if (errno) + return 0; + + /* Allow 1 space gap between number and unit */ + if (*endptr == ' ') + endptr++; + + switch (*endptr){ + case 'G': + case 'g': + size *= 1024; + /* Fall through */ + case 'M': + case 'm': + /* Fall through */ + size *= 1024; + case 'K': + case 'k': + /* Fall through */ + size *= 1024; + } + + return size; +} + +/* + * Function to read a single numeric value from a file on the filesystem. + * Used to read information from files on /sys + */ +extern int nvme_parse_sysfs_value(const char *filename, unsigned long *val); + +/* + * Get a file size in Bytes. + */ +extern uint64_t nvme_file_get_size(int fd); + +/* + * Get a block device block size in Bytes. + */ +extern ssize_t nvme_dev_get_blocklen(int fd); + +/* + * Get current time in nano seconds. + */ +static inline unsigned long long nvme_time_nsec(void) +{ +#ifdef __HAIKU__ + return (unsigned long long)system_time(); +#else + struct timespec ts; + + clock_gettime(CLOCK_REALTIME, &ts); + + return (unsigned long long) ts.tv_sec * 1000000000ULL + + (unsigned long long) ts.tv_nsec; +#endif +} + +/* + * Get current time in micro seconds. + */ +static inline unsigned long long nvme_time_usec(void) +{ + return nvme_time_nsec() / 1000; +} + +/* + * Get current time in milli seconds. + */ +static inline unsigned long long nvme_time_msec(void) +{ + return nvme_time_nsec() / 1000000; +} + +/* + * PAUSE instruction for tight loops (avoid busy waiting) + */ +#ifdef __SSE2__ +#include +static inline void nvme_pause(void) +{ + _mm_pause(); +} +#else +static inline void nvme_pause(void) {} +#endif + +#ifdef __HAIKU__ +static inline void +nvme_usleep(int usecs) +{ + snooze(usecs); +} + +static inline void +nvme_msleep(int msecs) +{ + snooze(msecs * 1000LL); +} +#else +/* + * Micro-seconds sleep. + */ +static inline void nvme_usleep(int usecs) +{ + struct timeval tv; + + tv.tv_sec = usecs / 1000000; + tv.tv_usec = usecs % 1000000; + select(0, NULL, NULL, NULL, &tv); +} + +/* + * Milli-seconds sleep. + */ +static inline void nvme_msleep(int msecs) +{ + struct timeval tv; + + tv.tv_sec = msecs / 1000; + tv.tv_usec = (msecs - tv.tv_sec * 1000) * 1000; + select(0, NULL, NULL, NULL, &tv); +} +#endif + +#ifndef __HAIKU__ +/* + * Provide notification of a critical non-recoverable error and stop. + * This function should not be called directly. Use nvme_panic() instead. + */ +extern void __nvme_panic(const char *funcname , const char *format, ...) +#ifdef __GNUC__ +#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 2)) + __attribute__((cold)) +#endif +#endif + __attribute__((noreturn)) + __attribute__((format(printf, 2, 3))); + +/* + * Provide notification of a critical non-recoverable error + * and terminate execution abnormally. + */ +#define nvme_panic(format, args...) \ + __nvme_panic(__FUNCTION__, format, ## args) +#else +#define nvme_panic panic +#endif + +/* + * Macro to evaluate a scalar expression and + * abort the program if the assertion is false. + */ +#define _nvme_assert_default(exp) \ + do { \ + if (unlikely(!(exp))) \ + nvme_panic("line %d, assert %s failed\n", \ + __LINE__, # exp); \ + } while (0) + +#define _nvme_assert_msg(exp, msg) \ + do { \ + if (unlikely(!(exp))) \ + nvme_panic("%s\n", msg); \ + } while (0) + +#define _NVME_GET_ASSERT_OVERLOAD(_1, _2, NAME, args...) NAME + +#define nvme_assert(args...) \ + _NVME_GET_ASSERT_OVERLOAD(args, \ + _nvme_assert_msg, \ + _nvme_assert_default) \ + (args) + +/* + * Macro to return the minimum of two numbers + */ +#define nvme_min(a, b) ({ \ + typeof (a) _a = (a); \ + typeof (b) _b = (b); \ + _a < _b ? _a : _b; \ + }) + +/* + * Macro to return the maximum of two numbers + */ +#define nvme_max(a, b) ({ \ + typeof (a) _a = (a); \ + typeof (b) _b = (b); \ + _a > _b ? _a : _b; \ + }) + +/* + * Returns true if n is a power of 2. + */ +static inline int nvme_is_pow2(__u64 v) +{ + return v && !(v & (v - 1)); +} + +/* + * Return the power of 2 immediately after v. + */ +static inline __u64 nvme_align_pow2(__u64 v) +{ + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + + return v + 1; +} + +/* + * Calculate log2 of a power of 2 size. + */ +static inline size_t nvme_log2(size_t size) +{ + size_t bits = 0; + + if (!nvme_is_pow2(size)) + return 0; + + while (size >>= 1) + bits++; + + return bits; +} + +/* + * Handle alignements. + */ +#define nvme_align_down(val, align) \ + ((val) & (~((typeof(val))((align) - 1)))) +#define nvme_align_up(val, align) \ + nvme_align_down((val) + (align) - 1, (align)) + +/* + * Test a bit value. + */ +static inline int test_bit(__u8 *bitmap, unsigned int bit) +{ + return bitmap[bit >> 3] & (1U << (bit & 0x7)); +} + +/* + * Set a bit. + */ +static inline void set_bit(__u8 *bitmap, unsigned int bit) +{ + bitmap[bit >> 3] |= 1U << (bit & 0x7); +} + +/* + * Clear a bit. + */ +static inline void clear_bit(__u8 *bitmap, unsigned int bit) +{ + bitmap[bit >> 3] &= ~(1U << (bit & 0x7)); +} + +/* + * Find the first zero bit in a bitmap of size nr_bits. + * If no zero bit is found, return -1. + */ +static inline int find_first_zero_bit(__u8 *bitmap, unsigned int nr_bits) +{ + __u64 *b = (__u64 *)bitmap; + unsigned int i, j, bit, count = (nr_bits + 63) >> 6; + + for(i = 0; i < count; i++) { + if (b[i] != ~0UL) + break; + } + + bit = i << 6; + for (j = bit; j < nr_bits; j++) { + if (!test_bit(bitmap, j)) + return j; + } + + return -1; +} + +/* + * Close all open controllers on exit. + * Defined in lib/nvme/nvme.c + */ +extern void nvme_ctrlr_cleanup(void); + +#endif /* __NVME_COMMON_H__ */ diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_ctrlr.c b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_ctrlr.c new file mode 100644 index 0000000000..81a31e48fd --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_ctrlr.c @@ -0,0 +1,1553 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in sourete and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of sourete code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +/* + * Host software shall wait a minimum of CAP.TO x 500 milleseconds for CSTS.RDY + * to be set to '1' after setting CC.EN to '1' from a previous value of '0'. + */ +static inline unsigned int +nvme_ctrlr_get_ready_to_in_ms(struct nvme_ctrlr *ctrlr) +{ + union nvme_cap_register cap; + +/* The TO unit in ms */ +#define NVME_READY_TIMEOUT_UNIT 500 + + cap.raw = nvme_reg_mmio_read_8(ctrlr, cap.raw); + + return (NVME_READY_TIMEOUT_UNIT * cap.bits.to); +} + +/* + * Create a queue pair. + */ +static int nvme_ctrlr_create_qpair(struct nvme_ctrlr *ctrlr, + struct nvme_qpair *qpair) +{ + int ret; + + /* Create the completion queue */ + ret = nvme_admin_create_ioq(ctrlr, qpair, NVME_IO_COMPLETION_QUEUE); + if (ret != 0) { + nvme_notice("Create completion queue %u failed\n", + qpair->id); + return ret; + } + + /* Create the submission queue */ + ret = nvme_admin_create_ioq(ctrlr, qpair, NVME_IO_SUBMISSION_QUEUE); + if (ret != 0) { + /* Attempt to delete the completion queue */ + nvme_notice("Create submission queue %u failed\n", + qpair->id); + nvme_admin_delete_ioq(ctrlr, qpair, NVME_IO_COMPLETION_QUEUE); + return ret; + } + + nvme_qpair_reset(qpair); + + return 0; +} + +/* + * Delete a queue pair. + */ +static int nvme_ctrlr_delete_qpair(struct nvme_ctrlr *ctrlr, + struct nvme_qpair *qpair) +{ + int ret; + + /* Delete the submission queue */ + ret = nvme_admin_delete_ioq(ctrlr, qpair, NVME_IO_SUBMISSION_QUEUE); + if (ret != 0) { + nvme_notice("Delete submission queue %u failed\n", + qpair->id); + return ret; + } + + /* Delete the completion queue */ + ret = nvme_admin_delete_ioq(ctrlr, qpair, NVME_IO_COMPLETION_QUEUE); + if (ret != 0) { + nvme_notice("Delete completion queue %u failed\n", + qpair->id); + return ret; + } + + return 0; +} + +/* + * Intel log page. + */ +static void +nvme_ctrlr_construct_intel_support_log_page_list(struct nvme_ctrlr *ctrlr, + struct nvme_intel_log_page_dir *log_page_dir) +{ + + if (ctrlr->cdata.vid != NVME_PCI_VID_INTEL || + log_page_dir == NULL) + return; + + ctrlr->log_page_supported[NVME_INTEL_LOG_PAGE_DIR] = true; + + if (log_page_dir->read_latency_log_len || + (ctrlr->quirks & NVME_INTEL_QUIRK_READ_LATENCY)) + ctrlr->log_page_supported[NVME_INTEL_LOG_READ_CMD_LATENCY] = true; + + if (log_page_dir->write_latency_log_len || + (ctrlr->quirks & NVME_INTEL_QUIRK_WRITE_LATENCY)) + ctrlr->log_page_supported[NVME_INTEL_LOG_WRITE_CMD_LATENCY] = true; + + if (log_page_dir->temperature_statistics_log_len) + ctrlr->log_page_supported[NVME_INTEL_LOG_TEMPERATURE] = true; + + if (log_page_dir->smart_log_len) + ctrlr->log_page_supported[NVME_INTEL_LOG_SMART] = true; + + if (log_page_dir->marketing_description_log_len) + ctrlr->log_page_supported[NVME_INTEL_MARKETING_DESCRIPTION] = true; +} + +/* + * Intel log page. + */ +static int nvme_ctrlr_set_intel_support_log_pages(struct nvme_ctrlr *ctrlr) +{ + struct nvme_intel_log_page_dir *log_page_dir; + int ret; + + log_page_dir = nvme_zmalloc(sizeof(struct nvme_intel_log_page_dir), 64); + if (!log_page_dir) { + nvme_err("Allocate log_page_directory failed\n"); + return ENOMEM; + } + + ret = nvme_admin_get_log_page(ctrlr, NVME_INTEL_LOG_PAGE_DIR, + NVME_GLOBAL_NS_TAG, + log_page_dir, + sizeof(struct nvme_intel_log_page_dir)); + if (ret != 0) + nvme_notice("Get NVME_INTEL_LOG_PAGE_DIR log page failed\n"); + else + nvme_ctrlr_construct_intel_support_log_page_list(ctrlr, + log_page_dir); + + nvme_free(log_page_dir); + + return ret; +} + +/* + * Initialize log page support directory. + */ +static void nvme_ctrlr_set_supported_log_pages(struct nvme_ctrlr *ctrlr) +{ + + memset(ctrlr->log_page_supported, 0, sizeof(ctrlr->log_page_supported)); + + /* Mandatory pages */ + ctrlr->log_page_supported[NVME_LOG_ERROR] = true; + ctrlr->log_page_supported[NVME_LOG_HEALTH_INFORMATION] = true; + ctrlr->log_page_supported[NVME_LOG_FIRMWARE_SLOT] = true; + + if (ctrlr->cdata.lpa.celp) + ctrlr->log_page_supported[NVME_LOG_COMMAND_EFFECTS_LOG] = true; + + if (ctrlr->cdata.vid == NVME_PCI_VID_INTEL) + nvme_ctrlr_set_intel_support_log_pages(ctrlr); +} + +/* + * Set Intel device features. + */ +static void nvme_ctrlr_set_intel_supported_features(struct nvme_ctrlr *ctrlr) +{ + bool *supported_feature = ctrlr->feature_supported; + + supported_feature[NVME_INTEL_FEAT_MAX_LBA] = true; + supported_feature[NVME_INTEL_FEAT_MAX_LBA] = true; + supported_feature[NVME_INTEL_FEAT_NATIVE_MAX_LBA] = true; + supported_feature[NVME_INTEL_FEAT_POWER_GOVERNOR_SETTING] = true; + supported_feature[NVME_INTEL_FEAT_SMBUS_ADDRESS] = true; + supported_feature[NVME_INTEL_FEAT_LED_PATTERN] = true; + supported_feature[NVME_INTEL_FEAT_RESET_TIMED_WORKLOAD_COUNTERS] = true; + supported_feature[NVME_INTEL_FEAT_LATENCY_TRACKING] = true; +} + +/* + * Set device features. + */ +static void nvme_ctrlr_set_supported_features(struct nvme_ctrlr *ctrlr) +{ + bool *supported_feature = ctrlr->feature_supported; + + memset(ctrlr->feature_supported, 0, sizeof(ctrlr->feature_supported)); + + /* Mandatory features */ + supported_feature[NVME_FEAT_ARBITRATION] = true; + supported_feature[NVME_FEAT_POWER_MANAGEMENT] = true; + supported_feature[NVME_FEAT_TEMPERATURE_THRESHOLD] = true; + supported_feature[NVME_FEAT_ERROR_RECOVERY] = true; + supported_feature[NVME_FEAT_NUMBER_OF_QUEUES] = true; + supported_feature[NVME_FEAT_INTERRUPT_COALESCING] = true; + supported_feature[NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION] = true; + supported_feature[NVME_FEAT_WRITE_ATOMICITY] = true; + supported_feature[NVME_FEAT_ASYNC_EVENT_CONFIGURATION] = true; + + /* Optional features */ + if (ctrlr->cdata.vwc.present) + supported_feature[NVME_FEAT_VOLATILE_WRITE_CACHE] = true; + if (ctrlr->cdata.apsta.supported) + supported_feature[NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION] + = true; + if (ctrlr->cdata.hmpre) + supported_feature[NVME_FEAT_HOST_MEM_BUFFER] = true; + if (ctrlr->cdata.vid == NVME_PCI_VID_INTEL) + nvme_ctrlr_set_intel_supported_features(ctrlr); +} + +/* + * Initialize I/O queue pairs. + */ +static int nvme_ctrlr_init_io_qpairs(struct nvme_ctrlr *ctrlr) +{ + struct nvme_qpair *qpair; + union nvme_cap_register cap; + uint32_t i; + + if (ctrlr->ioq != NULL) + /* + * io_qpairs were already constructed, so just return. + * This typically happens when the controller is + * initialized a second (or subsequent) time after a + * controller reset. + */ + return 0; + + /* + * NVMe spec sets a hard limit of 64K max entries, but + * devices may specify a smaller limit, so we need to check + * the MQES field in the capabilities register. + */ + cap.raw = nvme_reg_mmio_read_8(ctrlr, cap.raw); + ctrlr->io_qpairs_max_entries = + nvme_min(NVME_IO_ENTRIES, (unsigned int)cap.bits.mqes + 1); + + ctrlr->ioq = calloc(ctrlr->io_queues, sizeof(struct nvme_qpair)); + if (!ctrlr->ioq) + return ENOMEM; + + /* Keep queue pair ID 0 for the admin queue */ + for (i = 0; i < ctrlr->io_queues; i++) { + qpair = &ctrlr->ioq[i]; + qpair->id = i + 1; + TAILQ_INSERT_TAIL(&ctrlr->free_io_qpairs, qpair, tailq); + } + + return 0; +} + +/* + * Shutdown a controller. + */ +static void nvme_ctrlr_shutdown(struct nvme_ctrlr *ctrlr) +{ + union nvme_cc_register cc; + union nvme_csts_register csts; + int ms_waited = 0; + + cc.raw = nvme_reg_mmio_read_4(ctrlr, cc.raw); + cc.bits.shn = NVME_SHN_NORMAL; + nvme_reg_mmio_write_4(ctrlr, cc.raw, cc.raw); + + csts.raw = nvme_reg_mmio_read_4(ctrlr, csts.raw); + /* + * The NVMe spec does not define a timeout period for shutdown + * notification, so we just pick 5 seconds as a reasonable amount + * of time to wait before proceeding. + */ +#define NVME_CTRLR_SHUTDOWN_TIMEOUT 5000 + while (csts.bits.shst != NVME_SHST_COMPLETE) { + nvme_usleep(1000); + csts.raw = nvme_reg_mmio_read_4(ctrlr, csts.raw); + if (ms_waited++ >= NVME_CTRLR_SHUTDOWN_TIMEOUT) + break; + } + + if (csts.bits.shst != NVME_SHST_COMPLETE) + nvme_err("Controller did not shutdown within %d seconds\n", + NVME_CTRLR_SHUTDOWN_TIMEOUT / 1000); +} + +/* + * Enable a controller. + */ +static int nvme_ctrlr_enable(struct nvme_ctrlr *ctrlr) +{ + union nvme_cc_register cc; + union nvme_aqa_register aqa; + union nvme_cap_register cap; + + cc.raw = nvme_reg_mmio_read_4(ctrlr, cc.raw); + + if (cc.bits.en != 0) { + nvme_err("COntroller enable called with CC.EN = 1\n"); + return EINVAL; + } + + nvme_reg_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr); + nvme_reg_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr); + + aqa.raw = 0; + /* acqs and asqs are 0-based. */ + aqa.bits.acqs = ctrlr->adminq.entries - 1; + aqa.bits.asqs = ctrlr->adminq.entries - 1; + nvme_reg_mmio_write_4(ctrlr, aqa.raw, aqa.raw); + + cc.bits.en = 1; + cc.bits.css = 0; + cc.bits.shn = 0; + cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */ + cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */ + + /* Page size is 2 ^ (12 + mps). */ + cc.bits.mps = PAGE_SHIFT - 12; + + cap.raw = nvme_reg_mmio_read_8(ctrlr, cap.raw); + + switch (ctrlr->opts.arb_mechanism) { + case NVME_CC_AMS_RR: + break; + case NVME_CC_AMS_WRR: + if (NVME_CAP_AMS_WRR & cap.bits.ams) + break; + return EINVAL; + case NVME_CC_AMS_VS: + if (NVME_CAP_AMS_VS & cap.bits.ams) + break; + return EINVAL; + default: + return EINVAL; + } + + cc.bits.ams = ctrlr->opts.arb_mechanism; + + nvme_reg_mmio_write_4(ctrlr, cc.raw, cc.raw); + + return 0; +} + +/* + * Disable a controller. + */ +static inline void nvme_ctrlr_disable(struct nvme_ctrlr *ctrlr) +{ + union nvme_cc_register cc; + + cc.raw = nvme_reg_mmio_read_4(ctrlr, cc.raw); + cc.bits.en = 0; + + nvme_reg_mmio_write_4(ctrlr, cc.raw, cc.raw); +} + +/* + * Test if a controller is enabled. + */ +static inline int nvme_ctrlr_enabled(struct nvme_ctrlr *ctrlr) +{ + union nvme_cc_register cc; + + cc.raw = nvme_reg_mmio_read_4(ctrlr, cc.raw); + + return cc.bits.en; +} + +/* + * Test if a controller is ready. + */ +static inline int nvme_ctrlr_ready(struct nvme_ctrlr *ctrlr) +{ + union nvme_csts_register csts; + + csts.raw = nvme_reg_mmio_read_4(ctrlr, csts.raw); + + return csts.bits.rdy; +} + +/* + * Set a controller state. + */ +static void nvme_ctrlr_set_state(struct nvme_ctrlr *ctrlr, + enum nvme_ctrlr_state state, + uint64_t timeout_in_ms) +{ + ctrlr->state = state; + if (timeout_in_ms == NVME_TIMEOUT_INFINITE) + ctrlr->state_timeout_ms = NVME_TIMEOUT_INFINITE; + else + ctrlr->state_timeout_ms = nvme_time_msec() + timeout_in_ms; +} + +/* + * Get a controller data. + */ +static int nvme_ctrlr_identify(struct nvme_ctrlr *ctrlr) +{ + int ret; + + ret = nvme_admin_identify_ctrlr(ctrlr, &ctrlr->cdata); + if (ret != 0) { + nvme_notice("Identify controller failed\n"); + return ret; + } + + /* + * Use MDTS to ensure our default max_xfer_size doesn't + * exceed what the controller supports. + */ + if (ctrlr->cdata.mdts > 0) + ctrlr->max_xfer_size = nvme_min(ctrlr->max_xfer_size, + ctrlr->min_page_size + * (1 << (ctrlr->cdata.mdts))); + return 0; +} + +/* + * Set the number of I/O queue pairs. + */ +static int nvme_ctrlr_get_max_io_qpairs(struct nvme_ctrlr *ctrlr) +{ + unsigned int cdw0, cq_allocated, sq_allocated; + int ret; + + ret = nvme_admin_get_feature(ctrlr, NVME_FEAT_CURRENT, + NVME_FEAT_NUMBER_OF_QUEUES, + 0, &cdw0); + if (ret != 0) { + nvme_notice("Get feature NVME_FEAT_NUMBER_OF_QUEUES failed\n"); + return ret; + } + + /* + * Data in cdw0 is 0-based. + * Lower 16-bits indicate number of submission queues allocated. + * Upper 16-bits indicate number of completion queues allocated. + */ + sq_allocated = (cdw0 & 0xFFFF) + 1; + cq_allocated = (cdw0 >> 16) + 1; + + ctrlr->max_io_queues = nvme_min(sq_allocated, cq_allocated); + + return 0; +} + +/* + * Set the number of I/O queue pairs. + */ +static int nvme_ctrlr_set_num_qpairs(struct nvme_ctrlr *ctrlr) +{ + unsigned int num_queues, cdw0; + unsigned int cq_allocated, sq_allocated; + int ret; + + ret = nvme_ctrlr_get_max_io_qpairs(ctrlr); + if (ret != 0) { + nvme_notice("Failed to get the maximum of I/O qpairs\n"); + return ret; + } + + /* + * Format number of I/O queue: + * Remove 1 as it as be be 0-based, + * bits 31:16 represent the number of completion queues, + * bits 0:15 represent the number of submission queues + */ + num_queues = ((ctrlr->opts.io_queues - 1) << 16) | + (ctrlr->opts.io_queues - 1); + + /* + * Set the number of I/O queues. + * Note: The value allocated may be smaller or larger than the number + * of queues requested (see specifications). + */ + ret = nvme_admin_set_feature(ctrlr, false, NVME_FEAT_NUMBER_OF_QUEUES, + num_queues, 0, &cdw0); + if (ret != 0) { + nvme_notice("Set feature NVME_FEAT_NUMBER_OF_QUEUES failed\n"); + return ret; + } + + /* + * Data in cdw0 is 0-based. + * Lower 16-bits indicate number of submission queues allocated. + * Upper 16-bits indicate number of completion queues allocated. + */ + sq_allocated = (cdw0 & 0xFFFF) + 1; + cq_allocated = (cdw0 >> 16) + 1; + ctrlr->io_queues = nvme_min(sq_allocated, cq_allocated); + + /* + * Make sure the number of constructed qpair listed in free_io_qpairs + * will not be more than the requested one. + */ + ctrlr->io_queues = nvme_min(ctrlr->io_queues, ctrlr->opts.io_queues); + + return 0; +} + +static void nvme_ctrlr_destruct_namespaces(struct nvme_ctrlr *ctrlr) +{ + + if (ctrlr->ns) { + free(ctrlr->ns); + ctrlr->ns = NULL; + ctrlr->nr_ns = 0; + } + + if (ctrlr->nsdata) { + nvme_free(ctrlr->nsdata); + ctrlr->nsdata = NULL; + } +} + +static int nvme_ctrlr_construct_namespaces(struct nvme_ctrlr *ctrlr) +{ + unsigned int i, nr_ns = ctrlr->cdata.nn; + struct nvme_ns *ns = NULL; + + /* + * ctrlr->nr_ns may be 0 (startup) or a different number of + * namespaces (reset), so check if we need to reallocate. + */ + if (nr_ns != ctrlr->nr_ns) { + + nvme_ctrlr_destruct_namespaces(ctrlr); + + ctrlr->ns = calloc(nr_ns, sizeof(struct nvme_ns)); + if (!ctrlr->ns) + goto fail; + + nvme_debug("Allocate %u namespace data\n", nr_ns); + ctrlr->nsdata = nvme_calloc(nr_ns, sizeof(struct nvme_ns_data), + PAGE_SIZE); + if (!ctrlr->nsdata) + goto fail; + + ctrlr->nr_ns = nr_ns; + + } + + for (i = 0; i < nr_ns; i++) { + ns = &ctrlr->ns[i]; + if (nvme_ns_construct(ctrlr, ns, i + 1) != 0) + goto fail; + } + + return 0; + +fail: + nvme_ctrlr_destruct_namespaces(ctrlr); + + return -1; +} + +/* + * Forward declaration. + */ +static int nvme_ctrlr_construct_and_submit_aer(struct nvme_ctrlr *ctrlr, + struct nvme_async_event_request *aer); + +/* + * Async event completion callback. + */ +static void nvme_ctrlr_async_event_cb(void *arg, const struct nvme_cpl *cpl) +{ + struct nvme_async_event_request *aer = arg; + struct nvme_ctrlr *ctrlr = aer->ctrlr; + + if (cpl->status.sc == NVME_SC_ABORTED_SQ_DELETION) + /* + * This is simulated when controller is being shut down, to + * effectively abort outstanding asynchronous event requests + * and make sure all memory is freed. Do not repost the + * request in this case. + */ + return; + + if (ctrlr->aer_cb_fn != NULL) + ctrlr->aer_cb_fn(ctrlr->aer_cb_arg, cpl); + + /* + * Repost another asynchronous event request to replace + * the one that just completed. + */ + if (nvme_ctrlr_construct_and_submit_aer(ctrlr, aer)) + /* + * We can't do anything to recover from a failure here, + * so just print a warning message and leave the + * AER unsubmitted. + */ + nvme_err("Initialize AER failed\n"); +} + +/* + * Issue an async event request. + */ +static int nvme_ctrlr_construct_and_submit_aer(struct nvme_ctrlr *ctrlr, + struct nvme_async_event_request *aer) +{ + struct nvme_request *req; + + req = nvme_request_allocate_null(&ctrlr->adminq, + nvme_ctrlr_async_event_cb, aer); + if (req == NULL) + return -1; + + aer->ctrlr = ctrlr; + aer->req = req; + req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST; + + return nvme_qpair_submit_request(&ctrlr->adminq, req); +} + +/* + * Configure async event management. + */ +static int nvme_ctrlr_configure_aer(struct nvme_ctrlr *ctrlr) +{ + union nvme_critical_warning_state state; + struct nvme_async_event_request *aer; + unsigned int i; + int ret; + + state.raw = 0xFF; + state.bits.reserved = 0; + + ret = nvme_admin_set_feature(ctrlr, false, + NVME_FEAT_ASYNC_EVENT_CONFIGURATION, + state.raw, 0, NULL); + if (ret != 0) { + nvme_notice("Set feature ASYNC_EVENT_CONFIGURATION failed\n"); + return ret; + } + + /* aerl is a zero-based value, so we need to add 1 here. */ + ctrlr->num_aers = nvme_min(NVME_MAX_ASYNC_EVENTS, + (ctrlr->cdata.aerl + 1)); + + for (i = 0; i < ctrlr->num_aers; i++) { + aer = &ctrlr->aer[i]; + if (nvme_ctrlr_construct_and_submit_aer(ctrlr, aer)) { + nvme_notice("Construct AER failed\n"); + return -1; + } + } + + return 0; +} + +/* + * Start a controller. + */ +static int nvme_ctrlr_start(struct nvme_ctrlr *ctrlr) +{ + + nvme_qpair_reset(&ctrlr->adminq); + nvme_qpair_enable(&ctrlr->adminq); + + if (nvme_ctrlr_identify(ctrlr) != 0) + return -1; + + if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) + return -1; + + if (nvme_ctrlr_init_io_qpairs(ctrlr)) + return -1; + + if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) + return -1; + + if (nvme_ctrlr_configure_aer(ctrlr) != 0) + return -1; + + nvme_ctrlr_set_supported_log_pages(ctrlr); + nvme_ctrlr_set_supported_features(ctrlr); + + if (ctrlr->cdata.sgls.supported) + ctrlr->flags |= NVME_CTRLR_SGL_SUPPORTED; + + return 0; +} + +/* + * Memory map the controller side buffer. + */ +static void nvme_ctrlr_map_cmb(struct nvme_ctrlr *ctrlr) +{ + int ret; + void *addr; + uint32_t bir; + union nvme_cmbsz_register cmbsz; + union nvme_cmbloc_register cmbloc; + uint64_t size, unit_size, offset, bar_size, bar_phys_addr; + + cmbsz.raw = nvme_reg_mmio_read_4(ctrlr, cmbsz.raw); + cmbloc.raw = nvme_reg_mmio_read_4(ctrlr, cmbloc.raw); + if (!cmbsz.bits.sz) + goto out; + + /* Values 0 2 3 4 5 are valid for BAR */ + bir = cmbloc.bits.bir; + if (bir > 5 || bir == 1) + goto out; + + /* unit size for 4KB/64KB/1MB/16MB/256MB/4GB/64GB */ + unit_size = (uint64_t)1 << (12 + 4 * cmbsz.bits.szu); + + /* controller memory buffer size in Bytes */ + size = unit_size * cmbsz.bits.sz; + + /* controller memory buffer offset from BAR in Bytes */ + offset = unit_size * cmbloc.bits.ofst; + + nvme_pcicfg_get_bar_addr_len(ctrlr->pci_dev, bir, &bar_phys_addr, + &bar_size); + + if (offset > bar_size) + goto out; + + if (size > bar_size - offset) + goto out; + + ret = nvme_pcicfg_map_bar_write_combine(ctrlr->pci_dev, bir, &addr); + if ((ret != 0) || addr == NULL) + goto out; + + ctrlr->cmb_bar_virt_addr = addr; + ctrlr->cmb_bar_phys_addr = bar_phys_addr; + ctrlr->cmb_size = size; + ctrlr->cmb_current_offset = offset; + + if (!cmbsz.bits.sqs) + ctrlr->opts.use_cmb_sqs = false; + + return; + +out: + ctrlr->cmb_bar_virt_addr = NULL; + ctrlr->opts.use_cmb_sqs = false; + + return; +} + +/* + * Unmap the controller side buffer. + */ +static int nvme_ctrlr_unmap_cmb(struct nvme_ctrlr *ctrlr) +{ + union nvme_cmbloc_register cmbloc; + void *addr = ctrlr->cmb_bar_virt_addr; + int ret = 0; + + if (addr) { + cmbloc.raw = nvme_reg_mmio_read_4(ctrlr, cmbloc.raw); + ret = nvme_pcicfg_unmap_bar(ctrlr->pci_dev, cmbloc.bits.bir, + addr); + } + return ret; +} + +/* + * Map the controller PCI bars. + */ +static int nvme_ctrlr_map_bars(struct nvme_ctrlr *ctrlr) +{ + void *addr; + int ret; + + ret = nvme_pcicfg_map_bar(ctrlr->pci_dev, 0, 0, &addr); + if (ret != 0 || addr == NULL) { + nvme_err("Map PCI device bar failed %d (%s)\n", + ret, strerror(ret)); + return ret; + } + + nvme_debug("Controller BAR mapped at %p\n", addr); + + ctrlr->regs = (volatile struct nvme_registers *)addr; + nvme_ctrlr_map_cmb(ctrlr); + + return 0; +} + +/* + * Unmap the controller PCI bars. + */ +static int nvme_ctrlr_unmap_bars(struct nvme_ctrlr *ctrlr) +{ + void *addr = (void *)ctrlr->regs; + int ret; + + ret = nvme_ctrlr_unmap_cmb(ctrlr); + if (ret != 0) { + nvme_err("Unmap controller side buffer failed %d\n", ret); + return ret; + } + + if (addr) { + ret = nvme_pcicfg_unmap_bar(ctrlr->pci_dev, 0, addr); + if (ret != 0) { + nvme_err("Unmap PCI device bar failed %d\n", ret); + return ret; + } + } + + return 0; +} + +/* + * Set a controller in the failed state. + */ +static void nvme_ctrlr_fail(struct nvme_ctrlr *ctrlr) +{ + unsigned int i; + + ctrlr->failed = true; + + nvme_qpair_fail(&ctrlr->adminq); + if (ctrlr->ioq) + for (i = 0; i < ctrlr->io_queues; i++) + nvme_qpair_fail(&ctrlr->ioq[i]); +} + +/* + * This function will be called repeatedly during initialization + * until the controller is ready. + */ +static int nvme_ctrlr_init(struct nvme_ctrlr *ctrlr) +{ + unsigned int ready_timeout_in_ms = nvme_ctrlr_get_ready_to_in_ms(ctrlr); + int ret; + + /* + * Check if the current initialization step is done or has timed out. + */ + switch (ctrlr->state) { + + case NVME_CTRLR_STATE_INIT: + + /* Begin the hardware initialization by making + * sure the controller is disabled. */ + if (nvme_ctrlr_enabled(ctrlr)) { + /* + * Disable the controller to cause a reset. + */ + if (!nvme_ctrlr_ready(ctrlr)) { + /* Wait for the controller to be ready */ + nvme_ctrlr_set_state(ctrlr, + NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1, + ready_timeout_in_ms); + return 0; + } + + /* + * The controller is enabled and ready. + * It can be immediatly disabled + */ + nvme_ctrlr_disable(ctrlr); + nvme_ctrlr_set_state(ctrlr, + NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, + ready_timeout_in_ms); + + if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) + nvme_msleep(2000); + + return 0; + } + + if (nvme_ctrlr_ready(ctrlr)) { + /* + * Controller is in the process of shutting down. + * We need to wait for CSTS.RDY to become 0. + */ + nvme_ctrlr_set_state(ctrlr, + NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, + ready_timeout_in_ms); + return 0; + } + + /* + * Controller is currently disabled. + * We can jump straight to enabling it. + */ + ret = nvme_ctrlr_enable(ctrlr); + if (ret) + nvme_err("Enable controller failed\n"); + else + nvme_ctrlr_set_state(ctrlr, + NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, + ready_timeout_in_ms); + return ret; + + case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1: + + if (nvme_ctrlr_ready(ctrlr)) { + /* CC.EN = 1 && CSTS.RDY = 1, + * so we can disable the controller now. */ + nvme_ctrlr_disable(ctrlr); + nvme_ctrlr_set_state(ctrlr, + NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, + ready_timeout_in_ms); + return 0; + } + + break; + + case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0: + + if (!nvme_ctrlr_ready(ctrlr)) { + /* CC.EN = 0 && CSTS.RDY = 0, + * so we can enable the controller now. */ + ret = nvme_ctrlr_enable(ctrlr); + if (ret) + nvme_err("Enable controller failed\n"); + else + nvme_ctrlr_set_state(ctrlr, + NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, + ready_timeout_in_ms); + return ret; + } + break; + + case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1: + + if (nvme_ctrlr_ready(ctrlr)) { + if (ctrlr->quirks & NVME_QUIRK_DELAY_AFTER_RDY) + nvme_msleep(2000); + + ret = nvme_ctrlr_start(ctrlr); + if (ret) + nvme_err("Start controller failed\n"); + else + nvme_ctrlr_set_state(ctrlr, + NVME_CTRLR_STATE_READY, + NVME_TIMEOUT_INFINITE); + return ret; + } + break; + + default: + nvme_panic("Unhandled ctrlr state %d\n", ctrlr->state); + nvme_ctrlr_fail(ctrlr); + return -1; + } + + if ((ctrlr->state_timeout_ms != NVME_TIMEOUT_INFINITE) && + (nvme_time_msec() > ctrlr->state_timeout_ms)) { + nvme_err("Initialization timed out in state %d\n", + ctrlr->state); + nvme_ctrlr_fail(ctrlr); + return -1; + } + + return 0; +} + +/* + * Reset a controller. + */ +static int nvme_ctrlr_reset(struct nvme_ctrlr *ctrlr) +{ + struct nvme_qpair *qpair; + unsigned int i; + + if (ctrlr->resetting || ctrlr->failed) + /* + * Controller is already resetting or has failed. Return + * immediately since there is no need to kick off another + * reset in these cases. + */ + return 0; + + ctrlr->resetting = true; + + /* Disable all queues before disabling the controller hardware. */ + nvme_qpair_disable(&ctrlr->adminq); + for (i = 0; i < ctrlr->io_queues; i++) + nvme_qpair_disable(&ctrlr->ioq[i]); + + /* Set the state back to INIT to cause a full hardware reset. */ + nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, + NVME_TIMEOUT_INFINITE); + + while (ctrlr->state != NVME_CTRLR_STATE_READY) { + if (nvme_ctrlr_init(ctrlr) != 0) { + nvme_crit("Controller reset failed\n"); + nvme_ctrlr_fail(ctrlr); + goto out; + } + } + + /* Reinitialize qpairs */ + TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) { + if (nvme_ctrlr_create_qpair(ctrlr, qpair) != 0) + nvme_ctrlr_fail(ctrlr); + } + +out: + ctrlr->resetting = false; + + return ctrlr->failed ? -1 : 0; +} + +/* + * Set a controller options. + */ +static void nvme_ctrlr_set_opts(struct nvme_ctrlr *ctrlr, + struct nvme_ctrlr_opts *opts) +{ + if (opts) + memcpy(&ctrlr->opts, opts, sizeof(struct nvme_ctrlr_opts)); + else + memset(&ctrlr->opts, 0, sizeof(struct nvme_ctrlr_opts)); + + if (ctrlr->opts.io_queues == 0) + ctrlr->opts.io_queues = DEFAULT_MAX_IO_QUEUES; + + if (ctrlr->opts.io_queues > NVME_MAX_IO_QUEUES) { + nvme_info("Limiting requested I/O queues %u to %d\n", + ctrlr->opts.io_queues, NVME_MAX_IO_QUEUES); + ctrlr->opts.io_queues = NVME_MAX_IO_QUEUES; + } +} + +/* + * Attach a PCI controller. + */ +struct nvme_ctrlr * +nvme_ctrlr_attach(struct pci_device *pci_dev, + struct nvme_ctrlr_opts *opts) +{ + struct nvme_ctrlr *ctrlr; + union nvme_cap_register cap; + uint32_t cmd_reg; + int ret; + + /* Get a new controller handle */ + ctrlr = malloc(sizeof(struct nvme_ctrlr)); + if (!ctrlr) { + nvme_err("Allocate controller handle failed\n"); + return NULL; + } + + nvme_debug("New controller handle %p\n", ctrlr); + + /* Initialize the handle */ + memset(ctrlr, 0, sizeof(struct nvme_ctrlr)); + ctrlr->pci_dev = pci_dev; + ctrlr->resetting = false; + ctrlr->failed = false; + TAILQ_INIT(&ctrlr->free_io_qpairs); + TAILQ_INIT(&ctrlr->active_io_qpairs); + pthread_mutex_init(&ctrlr->lock, NULL); + ctrlr->quirks = nvme_ctrlr_get_quirks(pci_dev); + + nvme_ctrlr_set_state(ctrlr, + NVME_CTRLR_STATE_INIT, + NVME_TIMEOUT_INFINITE); + + ret = nvme_ctrlr_map_bars(ctrlr); + if (ret != 0) { + nvme_err("Map controller BAR failed\n"); + pthread_mutex_destroy(&ctrlr->lock); + free(ctrlr); + return NULL; + } + + /* Enable PCI busmaster and disable INTx */ + nvme_pcicfg_read32(pci_dev, &cmd_reg, 4); + cmd_reg |= 0x0404; + nvme_pcicfg_write32(pci_dev, cmd_reg, 4); + + /* + * Doorbell stride is 2 ^ (dstrd + 2), + * but we want multiples of 4, so drop the + 2. + */ + cap.raw = nvme_reg_mmio_read_8(ctrlr, cap.raw); + ctrlr->doorbell_stride_u32 = 1 << cap.bits.dstrd; + ctrlr->min_page_size = 1 << (12 + cap.bits.mpsmin); + + /* Set default transfer size */ + ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE; + + /* Create the admin queue pair */ + ret = nvme_qpair_construct(ctrlr, &ctrlr->adminq, 0, + NVME_ADMIN_ENTRIES, NVME_ADMIN_TRACKERS); + if (ret != 0) { + nvme_err("Initialize admin queue pair failed\n"); + goto err; + } + + /* Set options and then initialize */ + nvme_ctrlr_set_opts(ctrlr, opts); + do { + ret = nvme_ctrlr_init(ctrlr); + if (ret) + goto err; + } while (ctrlr->state != NVME_CTRLR_STATE_READY); + + return ctrlr; + +err: + nvme_ctrlr_detach(ctrlr); + + return NULL; +} + +/* + * Detach a PCI controller. + */ +void nvme_ctrlr_detach(struct nvme_ctrlr *ctrlr) +{ + struct nvme_qpair *qpair; + uint32_t i; + + while (!TAILQ_EMPTY(&ctrlr->active_io_qpairs)) { + qpair = TAILQ_FIRST(&ctrlr->active_io_qpairs); + nvme_ioqp_release(qpair); + } + + nvme_ctrlr_shutdown(ctrlr); + + nvme_ctrlr_destruct_namespaces(ctrlr); + if (ctrlr->ioq) { + for (i = 0; i < ctrlr->io_queues; i++) + nvme_qpair_destroy(&ctrlr->ioq[i]); + free(ctrlr->ioq); + } + + nvme_qpair_destroy(&ctrlr->adminq); + + nvme_ctrlr_unmap_bars(ctrlr); + + pthread_mutex_destroy(&ctrlr->lock); + free(ctrlr); +} + +/* + * Get a controller feature. + */ +int nvme_ctrlr_get_feature(struct nvme_ctrlr *ctrlr, + enum nvme_feat_sel sel, enum nvme_feat feature, + uint32_t cdw11, + uint32_t *attributes) +{ + int ret; + + pthread_mutex_lock(&ctrlr->lock); + + ret = nvme_admin_get_feature(ctrlr, sel, feature, cdw11, attributes); + if (ret != 0) + nvme_notice("Get feature 0x%08x failed\n", + (unsigned int) feature); + + pthread_mutex_unlock(&ctrlr->lock); + + return ret; +} + +/* + * Set a controller feature. + */ +int nvme_ctrlr_set_feature(struct nvme_ctrlr *ctrlr, + bool save, enum nvme_feat feature, + uint32_t cdw11, uint32_t cdw12, + uint32_t *attributes) +{ + int ret; + + pthread_mutex_lock(&ctrlr->lock); + + ret = nvme_admin_set_feature(ctrlr, save, feature, + cdw11, cdw12, attributes); + if (ret != 0) + nvme_notice("Set feature 0x%08x failed\n", + (unsigned int) feature); + + pthread_mutex_unlock(&ctrlr->lock); + + return ret; +} + +/* + * Attach a namespace. + */ +int nvme_ctrlr_attach_ns(struct nvme_ctrlr *ctrlr, unsigned int nsid, + struct nvme_ctrlr_list *clist) +{ + int ret; + + pthread_mutex_lock(&ctrlr->lock); + + ret = nvme_admin_attach_ns(ctrlr, nsid, clist); + if (ret) { + nvme_notice("Attach namespace %u failed\n", nsid); + goto out; + } + + ret = nvme_ctrlr_reset(ctrlr); + if (ret != 0) + nvme_notice("Reset controller failed\n"); + +out: + pthread_mutex_unlock(&ctrlr->lock); + + return ret; +} + +/* + * Detach a namespace. + */ +int nvme_ctrlr_detach_ns(struct nvme_ctrlr *ctrlr, unsigned int nsid, + struct nvme_ctrlr_list *clist) +{ + int ret; + + pthread_mutex_lock(&ctrlr->lock); + + ret = nvme_admin_detach_ns(ctrlr, nsid, clist); + if (ret != 0) { + nvme_notice("Detach namespace %u failed\n", nsid); + goto out; + } + + ret = nvme_ctrlr_reset(ctrlr); + if (ret) + nvme_notice("Reset controller failed\n"); + +out: + pthread_mutex_unlock(&ctrlr->lock); + + return ret; +} + +/* + * Create a namespace. + */ +unsigned int nvme_ctrlr_create_ns(struct nvme_ctrlr *ctrlr, + struct nvme_ns_data *nsdata) +{ + unsigned int nsid; + int ret; + + pthread_mutex_lock(&ctrlr->lock); + + ret = nvme_admin_create_ns(ctrlr, nsdata, &nsid); + if (ret != 0) { + nvme_notice("Create namespace failed\n"); + nsid = 0; + } + + pthread_mutex_unlock(&ctrlr->lock); + + return nsid; +} + +/* + * Delete a namespace. + */ +int nvme_ctrlr_delete_ns(struct nvme_ctrlr *ctrlr, unsigned int nsid) +{ + int ret; + + pthread_mutex_lock(&ctrlr->lock); + + ret = nvme_admin_delete_ns(ctrlr, nsid); + if (ret != 0) { + nvme_notice("Delete namespace %u failed\n", nsid); + goto out; + } + + ret = nvme_ctrlr_reset(ctrlr); + if (ret) + nvme_notice("Reset controller failed\n"); + +out: + pthread_mutex_unlock(&ctrlr->lock); + + return ret; +} + +/* + * Format NVM media. + */ +int nvme_ctrlr_format_ns(struct nvme_ctrlr *ctrlr, unsigned int nsid, + struct nvme_format *format) +{ + int ret; + + pthread_mutex_lock(&ctrlr->lock); + + ret = nvme_admin_format_nvm(ctrlr, nsid, format); + if (ret != 0) { + if (nsid == NVME_GLOBAL_NS_TAG) + nvme_notice("Format device failed\n"); + else + nvme_notice("Format namespace %u failed\n", nsid); + goto out; + } + + ret = nvme_ctrlr_reset(ctrlr); + if (ret) + nvme_notice("Reset controller failed\n"); + +out: + pthread_mutex_unlock(&ctrlr->lock); + + return ret; +} + +/* + * Update a device firmware. + */ +int nvme_ctrlr_update_firmware(struct nvme_ctrlr *ctrlr, + void *fw, size_t size, int slot) +{ + struct nvme_fw_commit fw_commit; + unsigned int size_remaining = size, offset = 0, transfer; + void *f = fw; + int ret; + + if (size & 0x3) { + nvme_err("Invalid firmware size\n"); + return EINVAL; + } + + pthread_mutex_lock(&ctrlr->lock); + + /* Download firmware */ + while (size_remaining > 0) { + + transfer = nvme_min(size_remaining, ctrlr->min_page_size); + + ret = nvme_admin_fw_image_dl(ctrlr, f, transfer, offset); + if (ret != 0) { + nvme_err("Download FW (%u B at %u) failed\n", + transfer, offset); + goto out; + } + + f += transfer; + offset += transfer; + size_remaining -= transfer; + + } + + /* Commit firmware */ + memset(&fw_commit, 0, sizeof(struct nvme_fw_commit)); + fw_commit.fs = slot; + fw_commit.ca = NVME_FW_COMMIT_REPLACE_IMG; + + ret = nvme_admin_fw_commit(ctrlr, &fw_commit); + if (ret != 0) { + nvme_err("Commit downloaded FW (%zu B) failed\n", + size); + goto out; + } + + ret = nvme_ctrlr_reset(ctrlr); + if (ret) + nvme_notice("Reset controller failed\n"); + +out: + pthread_mutex_unlock(&ctrlr->lock); + + return ret; +} + +/* + * Get an unused I/O queue pair. + */ +struct nvme_qpair *nvme_ioqp_get(struct nvme_ctrlr *ctrlr, + enum nvme_qprio qprio, unsigned int qd) +{ + struct nvme_qpair *qpair = NULL; + union nvme_cc_register cc; + uint32_t trackers; + int ret; + + cc.raw = nvme_reg_mmio_read_4(ctrlr, cc.raw); + + /* Only the low 2 bits (values 0, 1, 2, 3) of QPRIO are valid. */ + if ((qprio & 3) != qprio) + return NULL; + + /* + * Only value NVME_QPRIO_URGENT(0) is valid for the + * default round robin arbitration method. + */ + if ((cc.bits.ams == NVME_CC_AMS_RR) && (qprio != NVME_QPRIO_URGENT)) { + nvme_err("Invalid queue priority for default round " + "robin arbitration method\n"); + return NULL; + } + + /* I/O qpairs number of entries belong to [2, io_qpairs_max_entries] */ + if (qd == 1) { + nvme_err("Invalid queue depth\n"); + return NULL; + } + + if (qd == 0 || qd > ctrlr->io_qpairs_max_entries) + qd = ctrlr->io_qpairs_max_entries; + + /* + * No need to have more trackers than entries in the submit queue. + * Note also that for a queue size of N, we can only have (N-1) + * commands outstanding, hence the "-1" here. + */ + trackers = nvme_min(NVME_IO_TRACKERS, (qd - 1)); + + pthread_mutex_lock(&ctrlr->lock); + + /* Get the first available qpair structure */ + qpair = TAILQ_FIRST(&ctrlr->free_io_qpairs); + if (qpair == NULL) { + /* No free queue IDs */ + nvme_err("No free I/O queue pairs\n"); + goto out; + } + + /* Construct the qpair */ + ret = nvme_qpair_construct(ctrlr, qpair, qprio, qd, trackers); + if (ret != 0) { + nvme_qpair_destroy(qpair); + qpair = NULL; + goto out; + } + + /* + * At this point, qpair contains a preallocated submission + * and completion queue and a unique queue ID, but it is not + * yet created on the controller. + * Fill out the submission queue priority and send out the + * Create I/O Queue commands. + */ + if (nvme_ctrlr_create_qpair(ctrlr, qpair) != 0) { + nvme_err("Create queue pair on the controller failed\n"); + nvme_qpair_destroy(qpair); + qpair = NULL; + goto out; + } + + TAILQ_REMOVE(&ctrlr->free_io_qpairs, qpair, tailq); + TAILQ_INSERT_TAIL(&ctrlr->active_io_qpairs, qpair, tailq); + +out: + pthread_mutex_unlock(&ctrlr->lock); + + return qpair; +} + +/* + * Free an I/O queue pair. + */ +int nvme_ioqp_release(struct nvme_qpair *qpair) +{ + struct nvme_ctrlr *ctrlr; + int ret; + + if (qpair == NULL) + return 0; + + ctrlr = qpair->ctrlr; + + pthread_mutex_lock(&ctrlr->lock); + + /* Delete the I/O submission and completion queues */ + ret = nvme_ctrlr_delete_qpair(ctrlr, qpair); + if (ret != 0) { + nvme_notice("Delete queue pair %u failed\n", qpair->id); + } else { + TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq); + TAILQ_INSERT_HEAD(&ctrlr->free_io_qpairs, qpair, tailq); + } + + pthread_mutex_unlock(&ctrlr->lock); + + return ret; +} + +/* + * Submit an NVMe command using the specified I/O queue pair. + */ +int nvme_ioqp_submit_cmd(struct nvme_qpair *qpair, + struct nvme_cmd *cmd, + void *buf, size_t len, + nvme_cmd_cb cb_fn, void *cb_arg) +{ + struct nvme_ctrlr *ctrlr = qpair->ctrlr; + struct nvme_request *req; + int ret = ENOMEM; + + pthread_mutex_lock(&ctrlr->lock); + + req = nvme_request_allocate_contig(qpair, buf, len, cb_fn, cb_arg); + if (req) { + memcpy(&req->cmd, cmd, sizeof(req->cmd)); + ret = nvme_qpair_submit_request(qpair, req); + } + + pthread_mutex_unlock(&ctrlr->lock); + + return ret; +} + +/* + * Poll for completion of NVMe commands submitted to the + * specified I/O queue pair. + */ +unsigned int nvme_ioqp_poll(struct nvme_qpair *qpair, + unsigned int max_completions) +{ + struct nvme_ctrlr *ctrlr = qpair->ctrlr; + int ret; + + pthread_mutex_lock(&ctrlr->lock); + ret = nvme_qpair_poll(qpair, max_completions); + pthread_mutex_unlock(&ctrlr->lock); + + return ret; +} diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_intel.h b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_intel.h new file mode 100644 index 0000000000..e99d067bcd --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_intel.h @@ -0,0 +1,209 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Intel NVMe vendor-specific definitions + * See: http://www.intel.com/content/dam/www/public/us/en/documents/product-specifications/ssd-dc-p3700-spec.pdf + */ + +#ifndef __NVME_INTEL_H__ +#define __NVME_INTEL_H__ + +#include +#include + +enum nvme_intel_feat { + NVME_INTEL_FEAT_MAX_LBA = 0xC1, + NVME_INTEL_FEAT_NATIVE_MAX_LBA = 0xC2, + NVME_INTEL_FEAT_POWER_GOVERNOR_SETTING = 0xC6, + NVME_INTEL_FEAT_SMBUS_ADDRESS = 0xC8, + NVME_INTEL_FEAT_LED_PATTERN = 0xC9, + NVME_INTEL_FEAT_RESET_TIMED_WORKLOAD_COUNTERS = 0xD5, + NVME_INTEL_FEAT_LATENCY_TRACKING = 0xE2, +}; + +enum nvme_intel_set_max_lba_command_status_code { + NVME_INTEL_EXCEEDS_AVAILABLE_CAPACITY = 0xC0, + NVME_INTEL_SMALLER_THAN_MIN_LIMIT = 0xC1, + NVME_INTEL_SMALLER_THAN_NS_REQUIREMENTS = 0xC2, +}; + +enum nvme_intel_log_page { + NVME_INTEL_LOG_PAGE_DIR = 0xC0, + NVME_INTEL_LOG_READ_CMD_LATENCY = 0xC1, + NVME_INTEL_LOG_WRITE_CMD_LATENCY = 0xC2, + NVME_INTEL_LOG_TEMPERATURE = 0xC5, + NVME_INTEL_LOG_SMART = 0xCA, + NVME_INTEL_MARKETING_DESCRIPTION = 0xDD, +}; + +enum nvme_intel_smart_attribute_code { + NVME_INTEL_SMART_PROGRAM_FAIL_COUNT = 0xAB, + NVME_INTEL_SMART_ERASE_FAIL_COUNT = 0xAC, + NVME_INTEL_SMART_WEAR_LEVELING_COUNT = 0xAD, + NVME_INTEL_SMART_E2E_ERROR_COUNT = 0xB8, + NVME_INTEL_SMART_CRC_ERROR_COUNT = 0xC7, + NVME_INTEL_SMART_MEDIA_WEAR = 0xE2, + NVME_INTEL_SMART_HOST_READ_PERCENTAGE = 0xE3, + NVME_INTEL_SMART_TIMER = 0xE4, + NVME_INTEL_SMART_THERMAL_THROTTLE_STATUS = 0xEA, + NVME_INTEL_SMART_RETRY_BUFFER_OVERFLOW_COUNTER = 0xF0, + NVME_INTEL_SMART_PLL_LOCK_LOSS_COUNT = 0xF3, + NVME_INTEL_SMART_NAND_BYTES_WRITTEN = 0xF4, + NVME_INTEL_SMART_HOST_BYTES_WRITTEN = 0xF5, +}; + +struct nvme_intel_log_page_dir { + uint8_t version[2]; + uint8_t reserved[384]; + uint8_t read_latency_log_len; + uint8_t reserved2; + uint8_t write_latency_log_len; + uint8_t reserved3[5]; + uint8_t temperature_statistics_log_len; + uint8_t reserved4[9]; + uint8_t smart_log_len; + uint8_t reserved5[37]; + uint8_t marketing_description_log_len; + uint8_t reserved6[69]; +}; +nvme_static_assert(sizeof(struct nvme_intel_log_page_dir) == 512, + "Incorrect size"); + +struct nvme_intel_rw_latency_page { + uint16_t major_revison; + uint16_t minor_revison; + uint32_t buckets_32us[32]; + uint32_t buckets_1ms[31]; + uint32_t buckets_32ms[31]; +}; +nvme_static_assert(sizeof(struct nvme_intel_rw_latency_page) == 380, + "Incorrect size"); + +struct nvme_intel_temperature_page { + uint64_t current_temperature; + uint64_t shutdown_flag_last; + uint64_t shutdown_flag_life; + uint64_t highest_temperature; + uint64_t lowest_temperature; + uint64_t reserved[5]; + uint64_t specified_max_op_temperature; + uint64_t reserved2; + uint64_t specified_min_op_temperature; + uint64_t estimated_offset; +}; +nvme_static_assert(sizeof(struct nvme_intel_temperature_page) == 112, + "Incorrect size"); + +struct nvme_intel_smart_attribute { + uint8_t code; + uint8_t reserved[2]; + uint8_t normalized_value; + uint8_t reserved2; + uint8_t raw_value[6]; + uint8_t reserved3; +}; + +struct __attribute__((packed)) nvme_intel_smart_information_page { + struct nvme_intel_smart_attribute attributes[13]; +}; +nvme_static_assert(sizeof(struct nvme_intel_smart_information_page) == 156, + "Incorrect size"); + +union nvme_intel_feat_power_governor { + uint32_t raw; + struct { + /* Power governor setting: 00h = 25W 01h = 20W 02h = 10W */ + uint32_t power_governor_setting : 8; + uint32_t reserved : 24; + } bits; +}; +nvme_static_assert(sizeof(union nvme_intel_feat_power_governor) == 4, + "Incorrect size"); + +union nvme_intel_feat_smbus_address { + uint32_t raw; + struct { + uint32_t reserved : 1; + uint32_t smbus_controller_address : 8; + uint32_t reserved2 : 23; + } bits; +}; +nvme_static_assert(sizeof(union nvme_intel_feat_smbus_address) == 4, + "Incorrect size"); + +union nvme_intel_feat_led_pattern { + uint32_t raw; + struct { + uint32_t feature_options : 24; + uint32_t value : 8; + } bits; +}; +nvme_static_assert(sizeof(union nvme_intel_feat_led_pattern) == 4, + "Incorrect size"); + +union nvme_intel_feat_reset_timed_workload_counters { + uint32_t raw; + struct { + /* + * Write Usage: 00 = NOP, 1 = Reset E2, E3,E4 counters; + * Read Usage: Not Supported + */ + uint32_t reset : 1; + uint32_t reserved : 31; + } bits; +}; +nvme_static_assert(sizeof(union nvme_intel_feat_reset_timed_workload_counters) == 4, + "Incorrect size"); + +union nvme_intel_feat_latency_tracking { + uint32_t raw; + struct { + /* + * Write Usage: + * 00h = Disable Latency Tracking (Default) + * 01h = Enable Latency Tracking + */ + uint32_t enable : 32; + } bits; +}; +nvme_static_assert(sizeof(union nvme_intel_feat_latency_tracking) == 4, + "Incorrect size"); + +struct nvme_intel_marketing_description_page { + uint8_t marketing_product[512]; +}; +nvme_static_assert(sizeof(struct nvme_intel_marketing_description_page) == 512, + "Incorrect size"); + +#endif /* __NVME_INTEL_H__ */ diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_internal.h b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_internal.h new file mode 100644 index 0000000000..3388102353 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_internal.h @@ -0,0 +1,738 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __NVME_INTERNAL_H__ +#define __NVME_INTERNAL_H__ + +#include "nvme_common.h" +#include "nvme_pci.h" +#include "nvme_intel.h" +#include "nvme_mem.h" + +#ifndef __HAIKU__ +#include +#include /* PAGE_SIZE */ +#else +#include "nvme_platform.h" +#endif + +/* + * List functions. + */ +#define LIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = LIST_FIRST((head)); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +/* + * Tail queue functions. + */ +#define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = TAILQ_FIRST((head)); \ + (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define INTEL_DC_P3X00_DEVID 0x09538086 + +#define NVME_TIMEOUT_INFINITE UINT64_MAX + +/* + * Some Intel devices support vendor-unique read latency log page even + * though the log page directory says otherwise. + */ +#define NVME_INTEL_QUIRK_READ_LATENCY 0x1 + +/* + * Some Intel devices support vendor-unique write latency log page even + * though the log page directory says otherwise. + */ +#define NVME_INTEL_QUIRK_WRITE_LATENCY 0x2 + +/* + * Some controllers need a delay before starting to check the device + * readiness, which is done by reading the controller status register rdy bit. + */ +#define NVME_QUIRK_DELAY_BEFORE_CHK_RDY 0x4 + +/* + * Some controllers need a delay once the controller status register rdy bit + * switches from 0 to 1. + */ +#define NVME_QUIRK_DELAY_AFTER_RDY 0x8 + +/* + * Queues may consist of a contiguous block of physical + * memory or optionally a non-contiguous set of physical + * memory pages (defined by a Physical Region Pages List) + */ +#define NVME_MAX_PRP_LIST_ENTRIES (506) + +/* + * For commands requiring more than 2 PRP entries, one PRP will be + * embedded in the command (prp1), and the rest of the PRP entries + * will be in a list pointed to by the command (prp2). This means + * that real max number of PRP entries we support is 506+1, which + * results in a max xfer size of 506*PAGE_SIZE. + */ +#define NVME_MAX_XFER_SIZE NVME_MAX_PRP_LIST_ENTRIES * PAGE_SIZE + +#define NVME_ADMIN_TRACKERS (16) +#define NVME_ADMIN_ENTRIES (128) + +/* + * NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion + * queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we + * will allow outstanding on an I/O qpair at any time. The only advantage in + * having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping + * the contents of the submission and completion queues, it will show a longer + * history of data. + */ +#define NVME_IO_ENTRIES (1024U) +#define NVME_IO_TRACKERS (128U) +#define NVME_IO_ENTRIES_VS_TRACKERS_RATIO (NVME_IO_ENTRIES / NVME_IO_TRACKERS) + +/* + * NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL + * segment. + */ +#define NVME_MAX_SGL_DESCRIPTORS (253) + +/* + * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES + * for each controller. + */ + +#define NVME_MAX_ASYNC_EVENTS (8) + +/* + * NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this + * define specifies the maximum number of queues this driver will actually + * try to configure, if available. + */ +#define DEFAULT_MAX_IO_QUEUES (1024) + +/* + * Maximum of times a failed command can be retried. + */ +#define NVME_MAX_RETRY_COUNT (3) + +/* + * I/O queue type. + */ +enum nvme_io_queue_type { + + NVME_IO_QTYPE_INVALID = 0, + NVME_IO_SUBMISSION_QUEUE, + NVME_IO_COMPLETION_QUEUE, +}; + +enum nvme_payload_type { + + NVME_PAYLOAD_TYPE_INVALID = 0, + + /* + * nvme_request::u.payload.contig_buffer is valid for this request. + */ + NVME_PAYLOAD_TYPE_CONTIG, + + /* + * nvme_request::u.sgl is valid for this request + */ + NVME_PAYLOAD_TYPE_SGL, +}; + +/* + * Controller support flags. + */ +enum nvme_ctrlr_flags { + + /* + * The SGL is supported. + */ + NVME_CTRLR_SGL_SUPPORTED = 0x1, + +}; + +/* + * Descriptor for a request data payload. + * + * This struct is arranged so that it fits nicely in struct nvme_request. + */ +struct __attribute__((packed)) nvme_payload { + + union { + /* + * Virtual memory address of a single + * physically contiguous buffer + */ + void *contig; + + /* + * Call back functions for retrieving physical + * addresses for scattered payloads. + */ + struct { + nvme_req_reset_sgl_cb reset_sgl_fn; + nvme_req_next_sge_cb next_sge_fn; + void *cb_arg; + } sgl; + } u; + + /* + * Virtual memory address of a single physically + * contiguous metadata buffer + */ + void *md; + + /* + * Payload type. + */ + uint8_t type; + +}; + +struct nvme_request { + + /* + * NVMe command: must be aligned on 64B. + */ + struct nvme_cmd cmd; + + /* + * Data payload for this request's command. + */ + struct nvme_payload payload; + + uint8_t retries; + + /* + * Number of child requests still outstanding for this + * request which was split into multiple child requests. + */ + uint8_t child_reqs; + uint32_t payload_size; + + /* + * Offset in bytes from the beginning of payload for this request. + * This is used for I/O commands that are split into multiple requests. + */ + uint32_t payload_offset; + uint32_t md_offset; + + nvme_cmd_cb cb_fn; + void *cb_arg; + + /* + * The following members should not be reordered with members + * above. These members are only needed when splitting + * requests which is done rarely, and the driver is careful + * to not touch the following fields until a split operation is + * needed, to avoid touching an extra cacheline. + */ + + /* + * Points to the outstanding child requests for a parent request. + * Only valid if a request was split into multiple child + * requests, and is not initialized for non-split requests. + */ + TAILQ_HEAD(, nvme_request) children; + + /* + * Linked-list pointers for a child request in its parent's list. + */ + TAILQ_ENTRY(nvme_request) child_tailq; + + /* + * For queueing in qpair queued_req or free_req. + */ + struct nvme_qpair *qpair; + STAILQ_ENTRY(nvme_request) stailq; + + /* + * Points to a parent request if part of a split request, + * NULL otherwise. + */ + struct nvme_request *parent; + + /* + * Completion status for a parent request. Initialized to all 0's + * (SUCCESS) before child requests are submitted. If a child + * request completes with error, the error status is copied here, + * to ensure that the parent request is also completed with error + * status once all child requests are completed. + */ + struct nvme_cpl parent_status; + +} __attribute__((aligned(64))); + +struct nvme_completion_poll_status { + struct nvme_cpl cpl; + bool done; +}; + +struct nvme_async_event_request { + struct nvme_ctrlr *ctrlr; + struct nvme_request *req; + struct nvme_cpl cpl; +}; + +struct nvme_tracker { + + LIST_ENTRY(nvme_tracker) list; + + struct nvme_request *req; + uint16_t cid; + + uint16_t rsvd1: 15; + uint16_t active: 1; + + uint32_t rsvd2; + + uint64_t prp_sgl_bus_addr; + + union { + uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES]; + struct nvme_sgl_descriptor sgl[NVME_MAX_SGL_DESCRIPTORS]; + } u; + + uint64_t rsvd3; +}; + +/* + * struct nvme_tracker must be exactly 4K so that the prp[] array does not + * cross a page boundery and so that there is no padding required to meet + * alignment requirements. + */ +nvme_static_assert(sizeof(struct nvme_tracker) == 4096, + "nvme_tracker is not 4K"); +nvme_static_assert((offsetof(struct nvme_tracker, u.sgl) & 7) == 0, + "SGL must be Qword aligned"); + +struct nvme_qpair { + + volatile uint32_t *sq_tdbl; + volatile uint32_t *cq_hdbl; + + /* + * Submission queue + */ + struct nvme_cmd *cmd; + + /* + * Completion queue + */ + struct nvme_cpl *cpl; + + LIST_HEAD(, nvme_tracker) free_tr; + LIST_HEAD(, nvme_tracker) outstanding_tr; + + /* + * Array of trackers indexed by command ID. + */ + uint16_t trackers; + struct nvme_tracker *tr; + + struct nvme_request *reqs; + unsigned int num_reqs; + STAILQ_HEAD(, nvme_request) free_req; + STAILQ_HEAD(, nvme_request) queued_req; + + uint16_t id; + + uint16_t entries; + uint16_t sq_tail; + uint16_t cq_head; + + uint8_t phase; + + bool enabled; + bool sq_in_cmb; + + /* + * Fields below this point should not be touched on the + * normal I/O happy path. + */ + + uint8_t qprio; + + struct nvme_ctrlr *ctrlr; + + /* List entry for nvme_ctrlr::free_io_qpairs and active_io_qpairs */ + TAILQ_ENTRY(nvme_qpair) tailq; + + phys_addr_t cmd_bus_addr; + phys_addr_t cpl_bus_addr; +}; + +struct nvme_ns { + + struct nvme_ctrlr *ctrlr; + + uint32_t stripe_size; + uint32_t sector_size; + + uint32_t md_size; + uint32_t pi_type; + + uint32_t sectors_per_max_io; + uint32_t sectors_per_stripe; + + uint16_t id; + uint16_t flags; + + int open_count; + +}; + +/* + * State of struct nvme_ctrlr (in particular, during initialization). + */ +enum nvme_ctrlr_state { + + /* + * Controller has not been initialized yet. + */ + NVME_CTRLR_STATE_INIT = 0, + + /* + * Waiting for CSTS.RDY to transition from 0 to 1 + * so that CC.EN may be set to 0. + */ + NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1, + + /* + * Waiting for CSTS.RDY to transition from 1 to 0 + * so that CC.EN may be set to 1. + */ + NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, + + /* + * Waiting for CSTS.RDY to transition from 0 to 1 + * after enabling the controller. + */ + NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, + + /* + * Controller initialization has completed and + * the controller is ready. + */ + NVME_CTRLR_STATE_READY +}; + +/* + * One of these per allocated PCI device. + */ +struct nvme_ctrlr { + + /* + * NVMe MMIO register space. + */ + volatile struct nvme_registers *regs; + + /* + * Array of I/O queue pairs. + */ + struct nvme_qpair *ioq; + + /* + * Size of the array of I/O queue pairs. + */ + unsigned int io_queues; + + /* + * Maximum I/O queue pairs. + */ + unsigned int max_io_queues; + + /* + * Number of I/O queue pairs enabled + */ + unsigned int enabled_io_qpairs; + + /* + * Maximum entries for I/O qpairs + */ + unsigned int io_qpairs_max_entries; + + /* + * Array of namespace IDs. + */ + unsigned int nr_ns; + struct nvme_ns *ns; + + /* + * Controller state. + */ + bool resetting; + bool failed; + + /* + * Controller support flags. + */ + uint64_t flags; + + /* + * Cold data (not accessed in normal I/O path) is after this point. + */ + enum nvme_ctrlr_state state; + uint64_t state_timeout_ms; + + /* + * All the log pages supported. + */ + bool log_page_supported[256]; + + /* + * All the features supported. + */ + bool feature_supported[256]; + + /* + * Associated PCI device information. + */ + struct pci_device *pci_dev; + + /* + * Maximum i/o size in bytes. + */ + uint32_t max_xfer_size; + + /* + * Minimum page size supported by this controller in bytes. + */ + uint32_t min_page_size; + + /* + * Stride in uint32_t units between doorbell registers + * (1 = 4 bytes, 2 = 8 bytes, ...). + */ + uint32_t doorbell_stride_u32; + + uint32_t num_aers; + struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS]; + nvme_aer_cb aer_cb_fn; + void *aer_cb_arg; + + /* + * Guards access to the controller itself, including admin queues. + */ + pthread_mutex_t lock; + + + /* + * Admin queue pair. + */ + struct nvme_qpair adminq; + + /* + * Identify Controller data. + */ + struct nvme_ctrlr_data cdata; + + /* + * Array of Identify Namespace data. + * Stored separately from ns since nsdata should + * not normally be accessed during I/O. + */ + struct nvme_ns_data *nsdata; + + TAILQ_HEAD(, nvme_qpair) free_io_qpairs; + TAILQ_HEAD(, nvme_qpair) active_io_qpairs; + + /* + * Controller option set on open. + */ + struct nvme_ctrlr_opts opts; + + /* + * BAR mapping address which contains controller memory buffer. + */ + void *cmb_bar_virt_addr; + + /* + * BAR physical address which contains controller memory buffer. + */ + uint64_t cmb_bar_phys_addr; + + /* + * Controller memory buffer size in Bytes. + */ + uint64_t cmb_size; + + /* + * Current offset of controller memory buffer. + */ + uint64_t cmb_current_offset; + + /* + * Quirks flags. + */ + unsigned int quirks; + + /* + * For controller list. + */ + LIST_ENTRY(nvme_ctrlr) link; + +} __attribute__((aligned(PAGE_SIZE))); + +/* + * Admin functions. + */ +extern int nvme_admin_identify_ctrlr(struct nvme_ctrlr *ctrlr, + struct nvme_ctrlr_data *cdata); + +extern int nvme_admin_get_feature(struct nvme_ctrlr *ctrlr, + enum nvme_feat_sel sel, + enum nvme_feat feature, + uint32_t cdw11, uint32_t *attributes); + +extern int nvme_admin_set_feature(struct nvme_ctrlr *ctrlr, + bool save, + enum nvme_feat feature, + uint32_t cdw11, uint32_t cdw12, + uint32_t *attributes); + +extern int nvme_admin_format_nvm(struct nvme_ctrlr *ctrlr, + unsigned int nsid, + struct nvme_format *format); + +extern int nvme_admin_get_log_page(struct nvme_ctrlr *ctrlr, + uint8_t log_page, uint32_t nsid, + void *payload, uint32_t payload_size); + +extern int nvme_admin_abort_cmd(struct nvme_ctrlr *ctrlr, + uint16_t cid, uint16_t sqid); + +extern int nvme_admin_create_ioq(struct nvme_ctrlr *ctrlr, + struct nvme_qpair *io_que, + enum nvme_io_queue_type io_qtype); + +extern int nvme_admin_delete_ioq(struct nvme_ctrlr *ctrlr, + struct nvme_qpair *qpair, + enum nvme_io_queue_type io_qtype); + +extern int nvme_admin_identify_ns(struct nvme_ctrlr *ctrlr, + uint16_t nsid, + struct nvme_ns_data *nsdata); + +extern int nvme_admin_attach_ns(struct nvme_ctrlr *ctrlr, + uint32_t nsid, + struct nvme_ctrlr_list *clist); + +extern int nvme_admin_detach_ns(struct nvme_ctrlr *ctrlr, + uint32_t nsid, + struct nvme_ctrlr_list *clist); + +extern int nvme_admin_create_ns(struct nvme_ctrlr *ctrlr, + struct nvme_ns_data *nsdata, + unsigned int *nsid); + +extern int nvme_admin_delete_ns(struct nvme_ctrlr *ctrlr, + unsigned int nsid); + +extern int nvme_admin_fw_commit(struct nvme_ctrlr *ctrlr, + const struct nvme_fw_commit *fw_commit); + +extern int nvme_admin_fw_image_dl(struct nvme_ctrlr *ctrlr, + void *fw, uint32_t size, uint32_t offset); + +extern void nvme_request_completion_poll_cb(void *arg, + const struct nvme_cpl *cpl); + +extern struct nvme_ctrlr *nvme_ctrlr_attach(struct pci_device *pci_dev, + struct nvme_ctrlr_opts *opts); + +extern void nvme_ctrlr_detach(struct nvme_ctrlr *ctrlr); + +extern int nvme_qpair_construct(struct nvme_ctrlr *ctrlr, + struct nvme_qpair *qpair, enum nvme_qprio qprio, + uint16_t entries, uint16_t trackers); + +extern void nvme_qpair_destroy(struct nvme_qpair *qpair); +extern void nvme_qpair_enable(struct nvme_qpair *qpair); +extern void nvme_qpair_disable(struct nvme_qpair *qpair); +extern int nvme_qpair_submit_request(struct nvme_qpair *qpair, + struct nvme_request *req); +extern void nvme_qpair_reset(struct nvme_qpair *qpair); +extern void nvme_qpair_fail(struct nvme_qpair *qpair); + +extern unsigned int nvme_qpair_poll(struct nvme_qpair *qpair, + unsigned int max_completions); + +extern int nvme_request_pool_construct(struct nvme_qpair *qpair); + +extern void nvme_request_pool_destroy(struct nvme_qpair *qpair); + +extern struct nvme_request *nvme_request_allocate(struct nvme_qpair *qpair, + const struct nvme_payload *payload, uint32_t payload_size, + nvme_cmd_cb cb_fn, void *cb_arg); + +extern struct nvme_request *nvme_request_allocate_null(struct nvme_qpair *qpair, + nvme_cmd_cb cb_fn, + void *cb_arg); + +extern struct nvme_request * +nvme_request_allocate_contig(struct nvme_qpair *qpair, + void *buffer, uint32_t payload_size, + nvme_cmd_cb cb_fn, void *cb_arg); + +extern void nvme_request_free(struct nvme_request *req); + +extern void nvme_request_add_child(struct nvme_request *parent, + struct nvme_request *child); + +extern void nvme_request_remove_child(struct nvme_request *parent, + struct nvme_request *child); + +extern unsigned int nvme_ctrlr_get_quirks(struct pci_device *pdev); + +extern int nvme_ns_construct(struct nvme_ctrlr *ctrlr, + struct nvme_ns *ns, unsigned int id); + +/* + * Registers mmio access. + */ +#define nvme_reg_mmio_read_4(sc, reg) \ + nvme_mmio_read_4((__u32 *)&(sc)->regs->reg) + +#define nvme_reg_mmio_read_8(sc, reg) \ + nvme_mmio_read_8((__u64 *)&(sc)->regs->reg) + +#define nvme_reg_mmio_write_4(sc, reg, val) \ + nvme_mmio_write_4((__u32 *)&(sc)->regs->reg, val) + +#define nvme_reg_mmio_write_8(sc, reg, val) \ + nvme_mmio_write_8((__u64 *)&(sc)->regs->reg, val) + +#endif /* __NVME_INTERNAL_H__ */ diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_ns.c b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_ns.c new file mode 100644 index 0000000000..b9b50e33d7 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_ns.c @@ -0,0 +1,702 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +static inline struct nvme_ns_data *nvme_ns_get_data(struct nvme_ns *ns) +{ + return &ns->ctrlr->nsdata[ns->id - 1]; +} + +static int nvme_ns_identify_update(struct nvme_ns *ns) +{ + struct nvme_ctrlr *ctrlr = ns->ctrlr; + struct nvme_ns_data *nsdata = nvme_ns_get_data(ns); + uint32_t sector_size; + int ret; + + ret = nvme_admin_identify_ns(ctrlr, ns->id, nsdata); + if (ret != 0) { + nvme_err("nvme_identify_namespace failed\n"); + return ret; + } + + sector_size = 1 << nsdata->lbaf[nsdata->flbas.format].lbads; + + ns->sector_size = sector_size; + ns->sectors_per_max_io = ctrlr->max_xfer_size / sector_size; + ns->sectors_per_stripe = ns->stripe_size / sector_size; + + ns->flags = 0x0000; + + if (ctrlr->cdata.oncs.dsm) + ns->flags |= NVME_NS_DEALLOCATE_SUPPORTED; + + if (ctrlr->cdata.vwc.present) + ns->flags |= NVME_NS_FLUSH_SUPPORTED; + + if (ctrlr->cdata.oncs.write_zeroes) + ns->flags |= NVME_NS_WRITE_ZEROES_SUPPORTED; + + if (nsdata->nsrescap.raw) + ns->flags |= NVME_NS_RESERVATION_SUPPORTED; + + ns->md_size = nsdata->lbaf[nsdata->flbas.format].ms; + ns->pi_type = NVME_FMT_NVM_PROTECTION_DISABLE; + + if (nsdata->lbaf[nsdata->flbas.format].ms && nsdata->dps.pit) { + ns->flags |= NVME_NS_DPS_PI_SUPPORTED; + ns->pi_type = nsdata->dps.pit; + if (nsdata->flbas.extended) + ns->flags |= NVME_NS_EXTENDED_LBA_SUPPORTED; + } + + return 0; +} + +/* + * Initialize a namespace. + */ +int nvme_ns_construct(struct nvme_ctrlr *ctrlr, struct nvme_ns *ns, + unsigned int id) +{ + uint32_t pci_devid; + + ns->ctrlr = ctrlr; + ns->id = id; + ns->stripe_size = 0; + + nvme_pcicfg_read32(ctrlr->pci_dev, &pci_devid, 0); + if (pci_devid == INTEL_DC_P3X00_DEVID && ctrlr->cdata.vs[3] != 0) + ns->stripe_size = (1 << ctrlr->cdata.vs[3]) + * ctrlr->min_page_size; + + return nvme_ns_identify_update(ns); +} + +/* + * Open a namespace. + */ +struct nvme_ns *nvme_ns_open(struct nvme_ctrlr *ctrlr, unsigned int ns_id) +{ + struct nvme_ns *ns = NULL; + + pthread_mutex_lock(&ctrlr->lock); + + if (ns_id >= 1 && ns_id <= ctrlr->nr_ns) { + ns = &ctrlr->ns[ns_id - 1]; + ns->open_count++; + } + + pthread_mutex_unlock(&ctrlr->lock); + + return ns; +} + +/* + * Get the controller of an open name space and lock it, + * making sure in the process that the ns handle is valid. + */ +static struct nvme_ctrlr *nvme_ns_ctrlr_lock(struct nvme_ns *ns) +{ + struct nvme_ctrlr *ctrlr; + + if (!ns) + return NULL; + + ctrlr = ns->ctrlr; + if (ns->id < 1 || + ns->id > ctrlr->nr_ns || + ns != &ctrlr->ns[ns->id - 1]) + return NULL; + + pthread_mutex_lock(&ctrlr->lock); + + /* + * Between the check and lock, the ns may have gone away. + * So check again, and make sure that the name space is open. + */ + if (ns->id > ctrlr->nr_ns || + ns != &ctrlr->ns[ns->id - 1] || + ns->open_count == 0) { + pthread_mutex_unlock(&ctrlr->lock); + return NULL; + } + + return ctrlr; +} + +/* + * Close an open namespace. + */ +int nvme_ns_close(struct nvme_ns *ns) +{ + struct nvme_ctrlr *ctrlr; + + ctrlr = nvme_ns_ctrlr_lock(ns); + if (!ctrlr) { + nvme_err("Invalid name space handle\n"); + return EINVAL; + } + + ns->open_count--; + + pthread_mutex_unlock(&ctrlr->lock); + + return 0; +} + +/* + * Get namespace information + */ +int nvme_ns_stat(struct nvme_ns *ns, struct nvme_ns_stat *ns_stat) +{ + struct nvme_ctrlr *ctrlr; + + ctrlr = nvme_ns_ctrlr_lock(ns); + if (!ctrlr) { + nvme_err("Invalid name space handle\n"); + return EINVAL; + } + + ns_stat->id = ns->id; + ns_stat->sector_size = ns->sector_size; + ns_stat->sectors = nvme_ns_get_data(ns)->nsze; + ns_stat->flags = ns->flags; + ns_stat->pi_type = ns->pi_type; + ns_stat->md_size = ns->md_size; + + pthread_mutex_unlock(&ctrlr->lock); + + return 0; +} + +/* + * Get namespace data + */ +int nvme_ns_data(struct nvme_ns *ns, struct nvme_ns_data *nsdata) +{ + struct nvme_ctrlr *ctrlr; + + ctrlr = nvme_ns_ctrlr_lock(ns); + if (!ctrlr) { + nvme_err("Invalid name space handle\n"); + return EINVAL; + } + + memcpy(nsdata, nvme_ns_get_data(ns), sizeof(struct nvme_ns_data)); + + pthread_mutex_unlock(&ctrlr->lock); + + return 0; +} + +static struct nvme_request *_nvme_ns_rw(struct nvme_ns *ns, + struct nvme_qpair *qpair, + const struct nvme_payload *payload, uint64_t lba, + uint32_t lba_count, nvme_cmd_cb cb_fn, + void *cb_arg, uint32_t opc, uint32_t io_flags, + uint16_t apptag_mask, uint16_t apptag); + +static struct nvme_request * +_nvme_ns_split_request(struct nvme_ns *ns, + struct nvme_qpair *qpair, + const struct nvme_payload *payload, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + uint32_t opc, + uint32_t io_flags, + struct nvme_request *req, + uint32_t sectors_per_max_io, + uint32_t sector_mask, + uint16_t apptag_mask, + uint16_t apptag) +{ + uint32_t sector_size = ns->sector_size; + uint32_t md_size = ns->md_size; + uint32_t remaining_lba_count = lba_count; + uint32_t offset = 0; + uint32_t md_offset = 0; + struct nvme_request *child, *tmp; + + if (ns->flags & NVME_NS_DPS_PI_SUPPORTED) { + /* for extended LBA only */ + if ((ns->flags & NVME_NS_EXTENDED_LBA_SUPPORTED) + && !(io_flags & NVME_IO_FLAGS_PRACT)) + sector_size += ns->md_size; + } + + while (remaining_lba_count > 0) { + + lba_count = sectors_per_max_io - (lba & sector_mask); + lba_count = nvme_min(remaining_lba_count, lba_count); + + child = _nvme_ns_rw(ns, qpair, payload, lba, lba_count, cb_fn, + cb_arg, opc, io_flags, apptag_mask, apptag); + if (child == NULL) { + if (req->child_reqs) { + /* free all child nvme_request */ + TAILQ_FOREACH_SAFE(child, &req->children, + child_tailq, tmp) { + nvme_request_remove_child(req, child); + nvme_request_free(child); + } + } + return NULL; + } + + child->payload_offset = offset; + + /* for separate metadata buffer only */ + if (payload->md) + child->md_offset = md_offset; + + nvme_request_add_child(req, child); + + remaining_lba_count -= lba_count; + lba += lba_count; + offset += lba_count * sector_size; + md_offset += lba_count * md_size; + + } + + return req; +} + +static struct nvme_request *_nvme_ns_rw(struct nvme_ns *ns, + struct nvme_qpair *qpair, + const struct nvme_payload *payload, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + uint32_t opc, + uint32_t io_flags, + uint16_t apptag_mask, + uint16_t apptag) +{ + struct nvme_request *req; + struct nvme_cmd *cmd; + uint64_t *tmp_lba; + uint32_t sector_size; + uint32_t sectors_per_max_io; + uint32_t sectors_per_stripe; + + /* The bottom 16 bits must be empty */ + if (io_flags & 0xFFFF) + return NULL; + + sector_size = ns->sector_size; + sectors_per_max_io = ns->sectors_per_max_io; + sectors_per_stripe = ns->sectors_per_stripe; + + if (ns->flags & NVME_NS_DPS_PI_SUPPORTED) + /* for extended LBA only */ + if ((ns->flags & NVME_NS_EXTENDED_LBA_SUPPORTED) && + !(io_flags & NVME_IO_FLAGS_PRACT)) + sector_size += ns->md_size; + + req = nvme_request_allocate(qpair, payload, + lba_count * sector_size, cb_fn, cb_arg); + if (req == NULL) + return NULL; + + /* + * Intel DC P3*00 NVMe controllers benefit from driver-assisted striping. + * If this controller defines a stripe boundary and this I/O spans + * a stripe boundary, split the request into multiple requests and + * submit each separately to hardware. + */ + if (sectors_per_stripe > 0 && + (((lba & (sectors_per_stripe - 1)) + lba_count) > sectors_per_stripe)) + return _nvme_ns_split_request(ns, qpair, payload, lba, + lba_count, cb_fn, cb_arg, opc, + io_flags, req, sectors_per_stripe, + sectors_per_stripe - 1, + apptag_mask, apptag); + + if (lba_count > sectors_per_max_io) + return _nvme_ns_split_request(ns, qpair, payload, lba, + lba_count, cb_fn, cb_arg, opc, + io_flags, req, sectors_per_max_io, + 0, apptag_mask, apptag); + + cmd = &req->cmd; + cmd->opc = opc; + cmd->nsid = ns->id; + + tmp_lba = (uint64_t *)&cmd->cdw10; + *tmp_lba = lba; + + if (ns->flags & NVME_NS_DPS_PI_SUPPORTED) { + switch (ns->pi_type) { + case NVME_FMT_NVM_PROTECTION_TYPE1: + case NVME_FMT_NVM_PROTECTION_TYPE2: + cmd->cdw14 = (uint32_t)lba; + break; + } + } + + cmd->cdw12 = lba_count - 1; + cmd->cdw12 |= io_flags; + + cmd->cdw15 = apptag_mask; + cmd->cdw15 = (cmd->cdw15 << 16 | apptag); + + return req; +} + +int nvme_ns_read(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *buffer, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags) +{ + struct nvme_request *req; + struct nvme_payload payload; + + payload.type = NVME_PAYLOAD_TYPE_CONTIG; + payload.u.contig = buffer; + payload.md = NULL; + + req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg, + NVME_OPC_READ, io_flags, 0, 0); + if (req != NULL) + return nvme_qpair_submit_request(qpair, req); + + return ENOMEM; +} + +int nvme_ns_read_with_md(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *buffer, void *metadata, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags, + uint16_t apptag_mask, uint16_t apptag) +{ + struct nvme_request *req; + struct nvme_payload payload; + + payload.type = NVME_PAYLOAD_TYPE_CONTIG; + payload.u.contig = buffer; + payload.md = metadata; + + req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg, + NVME_OPC_READ, io_flags, apptag_mask, apptag); + if (req != NULL) + return nvme_qpair_submit_request(qpair, req); + + return ENOMEM; +} + +int nvme_ns_readv(struct nvme_ns *ns, struct nvme_qpair *qpair, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags, + nvme_req_reset_sgl_cb reset_sgl_fn, + nvme_req_next_sge_cb next_sge_fn) +{ + struct nvme_request *req; + struct nvme_payload payload; + + if (reset_sgl_fn == NULL || next_sge_fn == NULL) + return EINVAL; + + payload.type = NVME_PAYLOAD_TYPE_SGL; + payload.md = NULL; + payload.u.sgl.reset_sgl_fn = reset_sgl_fn; + payload.u.sgl.next_sge_fn = next_sge_fn; + payload.u.sgl.cb_arg = cb_arg; + + req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg, + NVME_OPC_READ, io_flags, 0, 0); + if (req != NULL) + return nvme_qpair_submit_request(qpair, req); + + return ENOMEM; +} + +int nvme_ns_write(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *buffer, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags) +{ + struct nvme_request *req; + struct nvme_payload payload; + + payload.type = NVME_PAYLOAD_TYPE_CONTIG; + payload.u.contig = buffer; + payload.md = NULL; + + req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg, + NVME_OPC_WRITE, io_flags, 0, 0); + if (req != NULL) + return nvme_qpair_submit_request(qpair, req); + + return ENOMEM; +} + +int nvme_ns_write_with_md(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *buffer, void *metadata, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags, + uint16_t apptag_mask, uint16_t apptag) +{ + struct nvme_request *req; + struct nvme_payload payload; + + payload.type = NVME_PAYLOAD_TYPE_CONTIG; + payload.u.contig = buffer; + payload.md = metadata; + + req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg, + NVME_OPC_WRITE, io_flags, apptag_mask, apptag); + if (req != NULL) + return nvme_qpair_submit_request(qpair, req); + + return ENOMEM; +} + +int nvme_ns_writev(struct nvme_ns *ns, struct nvme_qpair *qpair, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags, + nvme_req_reset_sgl_cb reset_sgl_fn, + nvme_req_next_sge_cb next_sge_fn) +{ + struct nvme_request *req; + struct nvme_payload payload; + + if (reset_sgl_fn == NULL || next_sge_fn == NULL) + return EINVAL; + + payload.type = NVME_PAYLOAD_TYPE_SGL; + payload.md = NULL; + payload.u.sgl.reset_sgl_fn = reset_sgl_fn; + payload.u.sgl.next_sge_fn = next_sge_fn; + payload.u.sgl.cb_arg = cb_arg; + + req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg, + NVME_OPC_WRITE, io_flags, 0, 0); + if (req != NULL) + return nvme_qpair_submit_request(qpair, req); + + return ENOMEM; +} + +int nvme_ns_write_zeroes(struct nvme_ns *ns, struct nvme_qpair *qpair, + uint64_t lba, uint32_t lba_count, + nvme_cmd_cb cb_fn, void *cb_arg, + unsigned int io_flags) +{ + struct nvme_request *req; + struct nvme_cmd *cmd; + uint64_t *tmp_lba; + + if (lba_count == 0) + return EINVAL; + + req = nvme_request_allocate_null(qpair, cb_fn, cb_arg); + if (req == NULL) + return ENOMEM; + + cmd = &req->cmd; + cmd->opc = NVME_OPC_WRITE_ZEROES; + cmd->nsid = ns->id; + + tmp_lba = (uint64_t *)&cmd->cdw10; + *tmp_lba = lba; + cmd->cdw12 = lba_count - 1; + cmd->cdw12 |= io_flags; + + return nvme_qpair_submit_request(qpair, req); +} + +int nvme_ns_deallocate(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *payload, uint16_t ranges, + nvme_cmd_cb cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_cmd *cmd; + + if (ranges == 0 || ranges > NVME_DATASET_MANAGEMENT_MAX_RANGES) + return EINVAL; + + req = nvme_request_allocate_contig(qpair, payload, + ranges * sizeof(struct nvme_dsm_range), + cb_fn, cb_arg); + if (req == NULL) + return ENOMEM; + + cmd = &req->cmd; + cmd->opc = NVME_OPC_DATASET_MANAGEMENT; + cmd->nsid = ns->id; + + /* TODO: create a delete command data structure */ + cmd->cdw10 = ranges - 1; + cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE; + + return nvme_qpair_submit_request(qpair, req); +} + +int nvme_ns_flush(struct nvme_ns *ns, struct nvme_qpair *qpair, + nvme_cmd_cb cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_cmd *cmd; + + req = nvme_request_allocate_null(qpair, cb_fn, cb_arg); + if (req == NULL) + return ENOMEM; + + cmd = &req->cmd; + cmd->opc = NVME_OPC_FLUSH; + cmd->nsid = ns->id; + + return nvme_qpair_submit_request(qpair, req); +} + +int nvme_ns_reservation_register(struct nvme_ns *ns, struct nvme_qpair *qpair, + struct nvme_reservation_register_data *payload, + bool ignore_key, + enum nvme_reservation_register_action action, + enum nvme_reservation_register_cptpl cptpl, + nvme_cmd_cb cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_cmd *cmd; + + req = nvme_request_allocate_contig(qpair, payload, + sizeof(struct nvme_reservation_register_data), + cb_fn, cb_arg); + if (req == NULL) + return ENOMEM; + + cmd = &req->cmd; + cmd->opc = NVME_OPC_RESERVATION_REGISTER; + cmd->nsid = ns->id; + + /* Bits 0-2 */ + cmd->cdw10 = action; + /* Bit 3 */ + cmd->cdw10 |= ignore_key ? 1 << 3 : 0; + /* Bits 30-31 */ + cmd->cdw10 |= (uint32_t)cptpl << 30; + + return nvme_qpair_submit_request(qpair, req); +} + +int nvme_ns_reservation_release(struct nvme_ns *ns, struct nvme_qpair *qpair, + struct nvme_reservation_key_data *payload, + bool ignore_key, + enum nvme_reservation_release_action action, + enum nvme_reservation_type type, + nvme_cmd_cb cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_cmd *cmd; + + req = nvme_request_allocate_contig(qpair, payload, + sizeof(struct nvme_reservation_key_data), + cb_fn, cb_arg); + if (req == NULL) + return ENOMEM; + + cmd = &req->cmd; + cmd->opc = NVME_OPC_RESERVATION_RELEASE; + cmd->nsid = ns->id; + + /* Bits 0-2 */ + cmd->cdw10 = action; + /* Bit 3 */ + cmd->cdw10 |= ignore_key ? 1 << 3 : 0; + /* Bits 8-15 */ + cmd->cdw10 |= (uint32_t)type << 8; + + return nvme_qpair_submit_request(qpair, req); +} + +int nvme_ns_reservation_acquire(struct nvme_ns *ns, struct nvme_qpair *qpair, + struct nvme_reservation_acquire_data *payload, + bool ignore_key, + enum nvme_reservation_acquire_action action, + enum nvme_reservation_type type, + nvme_cmd_cb cb_fn, void *cb_arg) +{ + struct nvme_request *req; + struct nvme_cmd *cmd; + + req = nvme_request_allocate_contig(qpair, payload, + sizeof(struct nvme_reservation_acquire_data), + cb_fn, cb_arg); + if (req == NULL) + return ENOMEM; + + cmd = &req->cmd; + cmd->opc = NVME_OPC_RESERVATION_ACQUIRE; + cmd->nsid = ns->id; + + /* Bits 0-2 */ + cmd->cdw10 = action; + /* Bit 3 */ + cmd->cdw10 |= ignore_key ? 1 << 3 : 0; + /* Bits 8-15 */ + cmd->cdw10 |= (uint32_t)type << 8; + + return nvme_qpair_submit_request(qpair, req); +} + +int nvme_ns_reservation_report(struct nvme_ns *ns, struct nvme_qpair *qpair, + void *payload, size_t len, + nvme_cmd_cb cb_fn, void *cb_arg) +{ + uint32_t num_dwords; + struct nvme_request *req; + struct nvme_cmd *cmd; + + if (len % 4) + return EINVAL; + num_dwords = len / 4; + + req = nvme_request_allocate_contig(qpair, payload, len, cb_fn, cb_arg); + if (req == NULL) + return ENOMEM; + + cmd = &req->cmd; + cmd->opc = NVME_OPC_RESERVATION_REPORT; + cmd->nsid = ns->id; + + cmd->cdw10 = num_dwords; + + return nvme_qpair_submit_request(qpair, req); +} diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_pci.h b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_pci.h new file mode 100644 index 0000000000..0507aa6463 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_pci.h @@ -0,0 +1,298 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __NVME_PCI_H__ +#define __NVME_PCI_H__ + +#include "nvme_common.h" + +#ifndef __HAIKU__ +#include +#endif + +#define NVME_PCI_PATH_MAX 256 +#define NVME_PCI_CFG_SIZE 256 +#define NVME_PCI_EXT_CAP_ID_SN 0x03 + +#define NVME_PCI_ANY_ID 0xffff +#define NVME_PCI_VID_INTEL 0x8086 +#define NVME_PCI_VID_MEMBLAZE 0x1c5f + +/* + * PCI class code for NVMe devices. + * + * Base class code 01h: mass storage + * Subclass code 08h: non-volatile memory + * Programming interface 02h: NVM Express + */ +#define NVME_PCI_CLASS 0x010802 + +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB0 0x3c20 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB1 0x3c21 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB2 0x3c22 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB3 0x3c23 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB4 0x3c24 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB5 0x3c25 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB6 0x3c26 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB7 0x3c27 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB8 0x3c2e +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB9 0x3c2f + +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB0 0x0e20 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB1 0x0e21 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB2 0x0e22 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB3 0x0e23 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB4 0x0e24 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB5 0x0e25 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB6 0x0e26 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB7 0x0e27 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB8 0x0e2e +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB9 0x0e2f + +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW0 0x2f20 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW1 0x2f21 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW2 0x2f22 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW3 0x2f23 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW4 0x2f24 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW5 0x2f25 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW6 0x2f26 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW7 0x2f27 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW8 0x2f2e +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW9 0x2f2f + +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BWD0 0x0C50 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BWD1 0x0C51 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BWD2 0x0C52 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BWD3 0x0C53 + +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDXDE0 0x6f50 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDXDE1 0x6f51 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDXDE2 0x6f52 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDXDE3 0x6f53 + +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX0 0x6f20 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX1 0x6f21 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX2 0x6f22 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX3 0x6f23 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX4 0x6f24 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX5 0x6f25 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX6 0x6f26 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX7 0x6f27 +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX8 0x6f2e +#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX9 0x6f2f + +struct pci_slot_match; + +struct pci_id { + uint16_t vendor_id; + uint16_t device_id; + uint16_t subvendor_id; + uint16_t subdevice_id; +}; + +/* + * Initialize PCI subsystem. + */ +extern int nvme_pci_init(void); + +/* + * Search a PCI device and grab it if found. + */ +extern struct pci_device * +nvme_pci_device_probe(const struct pci_slot_match *slot); + +/* + * Reset a PCI device. + */ +extern int nvme_pci_device_reset(struct pci_device *dev); + +/* + * Get a device serial number. + */ +extern int nvme_pci_device_get_serial_number(struct pci_device *dev, + char *sn, size_t len); + +/* + * Compare two devices. + * Return 0 if the devices are the same, 1 otherwise. + */ +static inline int nvme_pci_dev_cmp(struct pci_device *pci_dev1, + struct pci_device *pci_dev2) +{ + if (pci_dev1 == pci_dev2) + return 0; + + if (pci_dev1->domain == pci_dev2->domain && + pci_dev1->bus == pci_dev2->bus && + pci_dev1->dev == pci_dev2->dev && + pci_dev1->func == pci_dev2->func) + return 0; + + return 1; +} + +/* + * Get a device PCI ID. + */ +static inline void nvme_pci_get_pci_id(struct pci_device *pci_dev, + struct pci_id *pci_id) +{ + pci_id->vendor_id = pci_dev->vendor_id; + pci_id->device_id = pci_dev->device_id; + pci_id->subvendor_id = pci_dev->subvendor_id; + pci_id->subdevice_id = pci_dev->subdevice_id; +} + +#ifdef __HAIKU__ +int nvme_pcicfg_read8(struct pci_device *dev, uint8_t *value, uint32_t offset); +int nvme_pcicfg_write8(struct pci_device *dev, uint8_t value, uint32_t offset); +int nvme_pcicfg_read16(struct pci_device *dev, uint16_t *value, uint32_t offset); +int nvme_pcicfg_write16(struct pci_device *dev, uint16_t value, uint32_t offset); +int nvme_pcicfg_read32(struct pci_device *dev, uint32_t *value, uint32_t offset); +int nvme_pcicfg_write32(struct pci_device *dev, uint32_t value, uint32_t offset); +int nvme_pcicfg_map_bar(void *devhandle, unsigned int bar, bool read_only, + void **mapped_addr); +int nvme_pcicfg_map_bar_write_combine(void *devhandle, unsigned int bar, + void **mapped_addr); +int nvme_pcicfg_unmap_bar(void *devhandle, unsigned int bar, void *addr); +void nvme_pcicfg_get_bar_addr_len(void *devhandle, unsigned int bar, + uint64_t *addr, uint64_t *size); +#else +/* + * Read a device config register. + */ +static inline int nvme_pcicfg_read8(struct pci_device *dev, + uint8_t *value, uint32_t offset) +{ + return pci_device_cfg_read_u8(dev, value, offset); +} + +/* + * Write a device config register. + */ +static inline int nvme_pcicfg_write8(struct pci_device *dev, + uint8_t value, uint32_t offset) +{ + return pci_device_cfg_write_u8(dev, value, offset); +} + +/* + * Read a device config register. + */ +static inline int nvme_pcicfg_read16(struct pci_device *dev, + uint16_t *value, uint32_t offset) +{ + return pci_device_cfg_read_u16(dev, value, offset); +} + +/* + * Write a device config register. + */ +static inline int nvme_pcicfg_write16(struct pci_device *dev, + uint16_t value, uint32_t offset) +{ + return pci_device_cfg_write_u16(dev, value, offset); +} + +/* + * Read a device config register. + */ +static inline int nvme_pcicfg_read32(struct pci_device *dev, + uint32_t *value, uint32_t offset) +{ + return pci_device_cfg_read_u32(dev, value, offset); +} + +/* + * Write a device config register. + */ +static inline int nvme_pcicfg_write32(struct pci_device *dev, + uint32_t value, uint32_t offset) +{ + return pci_device_cfg_write_u32(dev, value, offset); +} + +/* + * Map a device PCI BAR. + */ +static inline int nvme_pcicfg_map_bar(void *devhandle, unsigned int bar, + bool read_only, void **mapped_addr) +{ + struct pci_device *dev = devhandle; + uint32_t flags = (read_only ? 0 : PCI_DEV_MAP_FLAG_WRITABLE); + + return pci_device_map_range(dev, dev->regions[bar].base_addr, + dev->regions[bar].size, flags, mapped_addr); +} + +/* + * Map a device PCI BAR (write combine). + */ +static inline int nvme_pcicfg_map_bar_write_combine(void *devhandle, + unsigned int bar, + void **mapped_addr) +{ + struct pci_device *dev = devhandle; + uint32_t flags = PCI_DEV_MAP_FLAG_WRITABLE | + PCI_DEV_MAP_FLAG_WRITE_COMBINE; + + return pci_device_map_range(dev, dev->regions[bar].base_addr, + dev->regions[bar].size, flags, mapped_addr); +} + +/* + * Unmap a device PCI BAR. + */ +static inline int nvme_pcicfg_unmap_bar(void *devhandle, unsigned int bar, + void *addr) +{ + struct pci_device *dev = devhandle; + + return pci_device_unmap_range(dev, addr, dev->regions[bar].size); +} + +/* + * Get a device PCI BAR address and length. + */ +static inline void nvme_pcicfg_get_bar_addr_len(void *devhandle, + unsigned int bar, + uint64_t *addr, uint64_t *size) +{ + struct pci_device *dev = devhandle; + + *addr = (uint64_t)dev->regions[bar].base_addr; + *size = (uint64_t)dev->regions[bar].size; +} +#endif + +#endif /* __NVME_PCI_H__ */ diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_qpair.c b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_qpair.c new file mode 100644 index 0000000000..cb407824ad --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_qpair.c @@ -0,0 +1,1186 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +struct nvme_qpair_string { + uint16_t value; + const char *str; +}; + +static const struct nvme_qpair_string admin_opcode[] = { + { NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" }, + { NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" }, + { NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" }, + { NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" }, + { NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" }, + { NVME_OPC_IDENTIFY, "IDENTIFY" }, + { NVME_OPC_ABORT, "ABORT" }, + { NVME_OPC_SET_FEATURES, "SET FEATURES" }, + { NVME_OPC_GET_FEATURES, "GET FEATURES" }, + { NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" }, + { NVME_OPC_NS_MANAGEMENT, "NAMESPACE MANAGEMENT" }, + { NVME_OPC_FIRMWARE_COMMIT, "FIRMWARE COMMIT" }, + { NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" }, + { NVME_OPC_NS_ATTACHMENT, "NAMESPACE ATTACHMENT" }, + { NVME_OPC_FORMAT_NVM, "FORMAT NVM" }, + { NVME_OPC_SECURITY_SEND, "SECURITY SEND" }, + { NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" }, + { 0xFFFF, "ADMIN COMMAND" } +}; + +static const struct nvme_qpair_string io_opcode[] = { + { NVME_OPC_FLUSH, "FLUSH" }, + { NVME_OPC_WRITE, "WRITE" }, + { NVME_OPC_READ, "READ" }, + { NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" }, + { NVME_OPC_COMPARE, "COMPARE" }, + { NVME_OPC_WRITE_ZEROES, "WRITE ZEROES" }, + { NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" }, + { NVME_OPC_RESERVATION_REGISTER, "RESERVATION REGISTER" }, + { NVME_OPC_RESERVATION_REPORT, "RESERVATION REPORT" }, + { NVME_OPC_RESERVATION_ACQUIRE, "RESERVATION ACQUIRE" }, + { NVME_OPC_RESERVATION_RELEASE, "RESERVATION RELEASE" }, + { 0xFFFF, "IO COMMAND" } +}; + +static const struct nvme_qpair_string generic_status[] = { + { NVME_SC_SUCCESS, "SUCCESS" }, + { NVME_SC_INVALID_OPCODE, "INVALID OPCODE" }, + { NVME_SC_INVALID_FIELD, "INVALID FIELD" }, + { NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" }, + { NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" }, + { NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" }, + { NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" }, + { NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" }, + { NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" }, + { NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" }, + { NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" }, + { NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" }, + { NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" }, + { NVME_SC_INVALID_SGL_SEG_DESCRIPTOR, "INVALID SGL SEGMENT DESCRIPTOR" }, + { NVME_SC_INVALID_NUM_SGL_DESCIRPTORS, "INVALID NUMBER OF SGL DESCRIPTORS" }, + { NVME_SC_DATA_SGL_LENGTH_INVALID, "DATA SGL LENGTH INVALID" }, + { NVME_SC_METADATA_SGL_LENGTH_INVALID, "METADATA SGL LENGTH INVALID" }, + { NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID, "SGL DESCRIPTOR TYPE INVALID" }, + { NVME_SC_INVALID_CONTROLLER_MEM_BUF, "INVALID CONTROLLER MEMORY BUFFER" }, + { NVME_SC_INVALID_PRP_OFFSET, "INVALID PRP OFFSET" }, + { NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED, "ATOMIC WRITE UNIT EXCEEDED" }, + { NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" }, + { NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" }, + { NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" }, + { NVME_SC_RESERVATION_CONFLICT, "RESERVATION CONFLICT" }, + { NVME_SC_FORMAT_IN_PROGRESS, "FORMAT IN PROGRESS" }, + { 0xFFFF, "GENERIC" } +}; + +static const struct nvme_qpair_string command_specific_status[] = { + { NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" }, + { NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" }, + { NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" }, + { NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" }, + { NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED,"ASYNC LIMIT EXCEEDED" }, + { NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" }, + { NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" }, + { NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" }, + { NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" }, + { NVME_SC_INVALID_FORMAT, "INVALID FORMAT" }, + { NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET,"FIRMWARE REQUIRES CONVENTIONAL RESET" }, + { NVME_SC_INVALID_QUEUE_DELETION, "INVALID QUEUE DELETION" }, + { NVME_SC_FEATURE_ID_NOT_SAVEABLE, "FEATURE ID NOT SAVEABLE" }, + { NVME_SC_FEATURE_NOT_CHANGEABLE, "FEATURE NOT CHANGEABLE" }, + { NVME_SC_FEATURE_NOT_NAMESPACE_SPECIFIC,"FEATURE NOT NAMESPACE SPECIFIC" }, + { NVME_SC_FIRMWARE_REQ_NVM_RESET, "FIRMWARE REQUIRES NVM RESET" }, + { NVME_SC_FIRMWARE_REQ_RESET, "FIRMWARE REQUIRES RESET" }, + { NVME_SC_FIRMWARE_REQ_MAX_TIME_VIOLATION,"FIRMWARE REQUIRES MAX TIME VIOLATION" }, + { NVME_SC_FIRMWARE_ACTIVATION_PROHIBITED,"FIRMWARE ACTIVATION PROHIBITED" }, + { NVME_SC_OVERLAPPING_RANGE, "OVERLAPPING RANGE" }, + { NVME_SC_NAMESPACE_INSUFFICIENT_CAPACITY,"NAMESPACE INSUFFICIENT CAPACITY" }, + { NVME_SC_NAMESPACE_ID_UNAVAILABLE, "NAMESPACE ID UNAVAILABLE" }, + { NVME_SC_NAMESPACE_ALREADY_ATTACHED, "NAMESPACE ALREADY ATTACHED" }, + { NVME_SC_NAMESPACE_IS_PRIVATE, "NAMESPACE IS PRIVATE" }, + { NVME_SC_NAMESPACE_NOT_ATTACHED, "NAMESPACE NOT ATTACHED" }, + { NVME_SC_THINPROVISIONING_NOT_SUPPORTED,"THINPROVISIONING NOT SUPPORTED" }, + { NVME_SC_CONTROLLER_LIST_INVALID, "CONTROLLER LIST INVALID" }, + { NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" }, + { NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" }, + { NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" }, + { 0xFFFF, "COMMAND SPECIFIC" } +}; + +static const struct nvme_qpair_string media_error_status[] = { + { NVME_SC_WRITE_FAULTS, "WRITE FAULTS" }, + { NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" }, + { NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" }, + { NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" }, + { NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" }, + { NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" }, + { NVME_SC_ACCESS_DENIED, "ACCESS DENIED" }, + { NVME_SC_DEALLOCATED_OR_UNWRITTEN_BLOCK, "DEALLOCATED OR UNWRITTEN BLOCK" }, + { 0xFFFF, "MEDIA ERROR" } +}; + +static inline bool nvme_qpair_is_admin_queue(struct nvme_qpair *qpair) +{ + return qpair->id == 0; +} + +static inline bool nvme_qpair_is_io_queue(struct nvme_qpair *qpair) +{ + return qpair->id != 0; +} + +static const char*nvme_qpair_get_string(const struct nvme_qpair_string *strings, + uint16_t value) +{ + const struct nvme_qpair_string *entry; + + entry = strings; + + while (entry->value != 0xFFFF) { + if (entry->value == value) + return entry->str; + entry++; + } + return entry->str; +} + +static void nvme_qpair_admin_qpair_print_command(struct nvme_qpair *qpair, + struct nvme_cmd *cmd) +{ + nvme_info("%s (%02x) sqid:%d cid:%d nsid:%x cdw10:%08x cdw11:%08x\n", + nvme_qpair_get_string(admin_opcode, cmd->opc), cmd->opc, + qpair->id, cmd->cid, + cmd->nsid, cmd->cdw10, cmd->cdw11); +} + +static void nvme_qpair_io_qpair_print_command(struct nvme_qpair *qpair, + struct nvme_cmd *cmd) +{ + nvme_assert(qpair != NULL, "print_command: qpair == NULL\n"); + nvme_assert(cmd != NULL, "print_command: cmd == NULL\n"); + + switch ((int)cmd->opc) { + case NVME_OPC_WRITE: + case NVME_OPC_READ: + case NVME_OPC_WRITE_UNCORRECTABLE: + case NVME_OPC_COMPARE: + nvme_info("%s sqid:%d cid:%d nsid:%d lba:%llu len:%d\n", + nvme_qpair_get_string(io_opcode, cmd->opc), + qpair->id, cmd->cid, cmd->nsid, + ((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10, + (cmd->cdw12 & 0xFFFF) + 1); + break; + case NVME_OPC_FLUSH: + case NVME_OPC_DATASET_MANAGEMENT: + nvme_info("%s sqid:%d cid:%d nsid:%d\n", + nvme_qpair_get_string(io_opcode, cmd->opc), + qpair->id, cmd->cid, cmd->nsid); + break; + default: + nvme_info("%s (%02x) sqid:%d cid:%d nsid:%d\n", + nvme_qpair_get_string(io_opcode, cmd->opc), + cmd->opc, qpair->id, cmd->cid, cmd->nsid); + break; + } +} + +static void nvme_qpair_print_command(struct nvme_qpair *qpair, + struct nvme_cmd *cmd) +{ + nvme_assert(qpair != NULL, "qpair can not be NULL"); + nvme_assert(cmd != NULL, "cmd can not be NULL"); + + if (nvme_qpair_is_admin_queue(qpair)) + return nvme_qpair_admin_qpair_print_command(qpair, cmd); + + return nvme_qpair_io_qpair_print_command(qpair, cmd); +} + +static const char *get_status_string(uint16_t sct, uint16_t sc) +{ + const struct nvme_qpair_string *entry; + + switch (sct) { + case NVME_SCT_GENERIC: + entry = generic_status; + break; + case NVME_SCT_COMMAND_SPECIFIC: + entry = command_specific_status; + break; + case NVME_SCT_MEDIA_ERROR: + entry = media_error_status; + break; + case NVME_SCT_VENDOR_SPECIFIC: + return "VENDOR SPECIFIC"; + default: + return "RESERVED"; + } + + return nvme_qpair_get_string(entry, sc); +} + +static void nvme_qpair_print_completion(struct nvme_qpair *qpair, + struct nvme_cpl *cpl) +{ + nvme_info("Cpl: %s (%02x/%02x) sqid:%d cid:%d " + "cdw0:%x sqhd:%04x p:%x m:%x dnr:%x\n", + get_status_string(cpl->status.sct, cpl->status.sc), + cpl->status.sct, + cpl->status.sc, + cpl->sqid, + cpl->cid, + cpl->cdw0, + cpl->sqhd, + cpl->status.p, + cpl->status.m, + cpl->status.dnr); +} + +static bool nvme_qpair_completion_retry(const struct nvme_cpl *cpl) +{ + /* + * TODO: spec is not clear how commands that are aborted due + * to TLER will be marked. So for now, it seems + * NAMESPACE_NOT_READY is the only case where we should + * look at the DNR bit. + */ + switch ((int)cpl->status.sct) { + case NVME_SCT_GENERIC: + switch ((int)cpl->status.sc) { + case NVME_SC_NAMESPACE_NOT_READY: + case NVME_SC_FORMAT_IN_PROGRESS: + if (cpl->status.dnr) + return false; + return true; + case NVME_SC_INVALID_OPCODE: + case NVME_SC_INVALID_FIELD: + case NVME_SC_COMMAND_ID_CONFLICT: + case NVME_SC_DATA_TRANSFER_ERROR: + case NVME_SC_ABORTED_POWER_LOSS: + case NVME_SC_INTERNAL_DEVICE_ERROR: + case NVME_SC_ABORTED_BY_REQUEST: + case NVME_SC_ABORTED_SQ_DELETION: + case NVME_SC_ABORTED_FAILED_FUSED: + case NVME_SC_ABORTED_MISSING_FUSED: + case NVME_SC_INVALID_NAMESPACE_OR_FORMAT: + case NVME_SC_COMMAND_SEQUENCE_ERROR: + case NVME_SC_LBA_OUT_OF_RANGE: + case NVME_SC_CAPACITY_EXCEEDED: + default: + return false; + } + case NVME_SCT_COMMAND_SPECIFIC: + case NVME_SCT_MEDIA_ERROR: + case NVME_SCT_VENDOR_SPECIFIC: + default: + return false; + } +} + +static void nvme_qpair_construct_tracker(struct nvme_tracker *tr, + uint16_t cid, uint64_t phys_addr) +{ + tr->prp_sgl_bus_addr = phys_addr + offsetof(struct nvme_tracker, u.prp); + tr->cid = cid; + tr->active = false; +} + +static inline void nvme_qpair_copy_command(struct nvme_cmd *dst, + const struct nvme_cmd *src) +{ + /* dst and src are known to be non-overlapping and 64-byte aligned. */ +#if defined(__AVX__) + __m256i *d256 = (__m256i *)dst; + const __m256i *s256 = (const __m256i *)src; + + _mm256_store_si256(&d256[0], _mm256_load_si256(&s256[0])); + _mm256_store_si256(&d256[1], _mm256_load_si256(&s256[1])); +#elif defined(__SSE2__) + __m128i *d128 = (__m128i *)dst; + const __m128i *s128 = (const __m128i *)src; + + _mm_store_si128(&d128[0], _mm_load_si128(&s128[0])); + _mm_store_si128(&d128[1], _mm_load_si128(&s128[1])); + _mm_store_si128(&d128[2], _mm_load_si128(&s128[2])); + _mm_store_si128(&d128[3], _mm_load_si128(&s128[3])); +#else + *dst = *src; +#endif +} + +static void nvme_qpair_submit_tracker(struct nvme_qpair *qpair, + struct nvme_tracker *tr) +{ + struct nvme_request *req = tr->req; + + /* + * Set the tracker active and copy its command + * to the submission queue. + */ + nvme_debug("qpair %d: Submit command, tail %d, cid %d / %d\n", + qpair->id, + (int)qpair->sq_tail, + (int)tr->cid, + (int)tr->req->cmd.cid); + + qpair->tr[tr->cid].active = true; + nvme_qpair_copy_command(&qpair->cmd[qpair->sq_tail], &req->cmd); + + if (++qpair->sq_tail == qpair->entries) + qpair->sq_tail = 0; + + nvme_wmb(); + nvme_mmio_write_4(qpair->sq_tdbl, qpair->sq_tail); +} + +static void nvme_qpair_complete_tracker(struct nvme_qpair *qpair, + struct nvme_tracker *tr, + struct nvme_cpl *cpl, + bool print_on_error) +{ + struct nvme_request *req = tr->req; + bool retry, error; + + if (!req) { + nvme_crit("tracker has no request\n"); + qpair->tr[cpl->cid].active = false; + goto done; + } + + error = nvme_cpl_is_error(cpl); + retry = error && nvme_qpair_completion_retry(cpl) && + (req->retries < NVME_MAX_RETRY_COUNT); + if (error && print_on_error) { + nvme_qpair_print_command(qpair, &req->cmd); + nvme_qpair_print_completion(qpair, cpl); + } + + qpair->tr[cpl->cid].active = false; + + if (cpl->cid != req->cmd.cid) + nvme_crit("cpl and command CID mismatch (%d / %d)\n", + (int)cpl->cid, (int)req->cmd.cid); + + if (retry) { + req->retries++; + nvme_qpair_submit_tracker(qpair, tr); + return; + } + + if (req->cb_fn) + req->cb_fn(req->cb_arg, cpl); + + nvme_request_free(req); + +done: + tr->req = NULL; + + LIST_REMOVE(tr, list); + LIST_INSERT_HEAD(&qpair->free_tr, tr, list); + + /* + * If the controller is in the middle of a reset, don't + * try to submit queued requests here - let the reset logic + * handle that instead. + */ + if (!STAILQ_EMPTY(&qpair->queued_req) && + !qpair->ctrlr->resetting) { + req = STAILQ_FIRST(&qpair->queued_req); + STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); + nvme_qpair_submit_request(qpair, req); + } +} + +static void nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair, + struct nvme_tracker *tr, + uint32_t sct, + uint32_t sc, + uint32_t dnr, + bool print_on_error) +{ + struct nvme_cpl cpl; + + memset(&cpl, 0, sizeof(cpl)); + cpl.sqid = qpair->id; + cpl.cid = tr->cid; + cpl.status.sct = sct; + cpl.status.sc = sc; + cpl.status.dnr = dnr; + + nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error); +} + +static void nvme_qpair_manual_complete_request(struct nvme_qpair *qpair, + struct nvme_request *req, + uint32_t sct, uint32_t sc, + bool print_on_error) +{ + struct nvme_cpl cpl; + bool error; + + memset(&cpl, 0, sizeof(cpl)); + cpl.sqid = qpair->id; + cpl.status.sct = sct; + cpl.status.sc = sc; + + error = nvme_cpl_is_error(&cpl); + + if (error && print_on_error) { + nvme_qpair_print_command(qpair, &req->cmd); + nvme_qpair_print_completion(qpair, &cpl); + } + + if (req->cb_fn) + req->cb_fn(req->cb_arg, &cpl); + + nvme_request_free(req); +} + +static void nvme_qpair_abort_aers(struct nvme_qpair *qpair) +{ + struct nvme_tracker *tr; + + tr = LIST_FIRST(&qpair->outstanding_tr); + while (tr != NULL) { + nvme_assert(tr->req != NULL, + "tr->req == NULL in abort_aers\n"); + if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) { + nvme_qpair_manual_complete_tracker(qpair, tr, + NVME_SCT_GENERIC, + NVME_SC_ABORTED_SQ_DELETION, + 0, false); + tr = LIST_FIRST(&qpair->outstanding_tr); + continue; + } + tr = LIST_NEXT(tr, list); + } +} + +static inline void _nvme_qpair_admin_qpair_destroy(struct nvme_qpair *qpair) +{ + nvme_qpair_abort_aers(qpair); +} + +static inline void _nvme_qpair_req_bad_phys(struct nvme_qpair *qpair, + struct nvme_tracker *tr) +{ + /* + * Bad vtophys translation, so abort this request + * and return immediately, without retry. + */ + nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, + NVME_SC_INVALID_FIELD, + 1, true); +} + +/* + * Build PRP list describing physically contiguous payload buffer. + */ +static int _nvme_qpair_build_contig_request(struct nvme_qpair *qpair, + struct nvme_request *req, + struct nvme_tracker *tr) +{ + uint64_t phys_addr; + void *seg_addr; + uint32_t nseg, cur_nseg, modulo, unaligned; + void *md_payload; + void *payload = req->payload.u.contig + req->payload_offset; + + phys_addr = nvme_mem_vtophys(payload); + if (phys_addr == NVME_VTOPHYS_ERROR) { + _nvme_qpair_req_bad_phys(qpair, tr); + return -1; + } + nseg = req->payload_size >> PAGE_SHIFT; + modulo = req->payload_size & (PAGE_SIZE - 1); + unaligned = phys_addr & (PAGE_SIZE - 1); + if (modulo || unaligned) + nseg += 1 + ((modulo + unaligned - 1) >> PAGE_SHIFT); + + if (req->payload.md) { + md_payload = req->payload.md + req->md_offset; + tr->req->cmd.mptr = nvme_mem_vtophys(md_payload); + if (tr->req->cmd.mptr == NVME_VTOPHYS_ERROR) { + _nvme_qpair_req_bad_phys(qpair, tr); + return -1; + } + } + + tr->req->cmd.psdt = NVME_PSDT_PRP; + tr->req->cmd.dptr.prp.prp1 = phys_addr; + if (nseg == 2) { + seg_addr = payload + PAGE_SIZE - unaligned; + tr->req->cmd.dptr.prp.prp2 = nvme_mem_vtophys(seg_addr); + } else if (nseg > 2) { + cur_nseg = 1; + tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr; + while (cur_nseg < nseg) { + seg_addr = payload + cur_nseg * PAGE_SIZE - unaligned; + phys_addr = nvme_mem_vtophys(seg_addr); + if (phys_addr == NVME_VTOPHYS_ERROR) { + _nvme_qpair_req_bad_phys(qpair, tr); + return -1; + } + tr->u.prp[cur_nseg - 1] = phys_addr; + cur_nseg++; + } + } + + return 0; +} + +/* + * Build SGL list describing scattered payload buffer. + */ +static int _nvme_qpair_build_hw_sgl_request(struct nvme_qpair *qpair, + struct nvme_request *req, + struct nvme_tracker *tr) +{ + struct nvme_sgl_descriptor *sgl; + uint64_t phys_addr; + uint32_t remaining_transfer_len, length, nseg = 0; + int ret; + + /* + * Build scattered payloads. + */ + nvme_assert(req->payload_size != 0, + "cannot build SGL for zero-length transfer\n"); + nvme_assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL, + "sgl payload type required\n"); + nvme_assert(req->payload.u.sgl.reset_sgl_fn != NULL, + "sgl reset callback required\n"); + nvme_assert(req->payload.u.sgl.next_sge_fn != NULL, + "sgl callback required\n"); + req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg, + req->payload_offset); + + sgl = tr->u.sgl; + req->cmd.psdt = NVME_PSDT_SGL_MPTR_SGL; + req->cmd.dptr.sgl1.unkeyed.subtype = 0; + + remaining_transfer_len = req->payload_size; + + while (remaining_transfer_len > 0) { + + if (nseg >= NVME_MAX_SGL_DESCRIPTORS) { + _nvme_qpair_req_bad_phys(qpair, tr); + return -1; + } + + ret = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg, + &phys_addr, &length); + if (ret != 0) { + _nvme_qpair_req_bad_phys(qpair, tr); + return ret; + } + + length = nvme_min(remaining_transfer_len, length); + remaining_transfer_len -= length; + + sgl->unkeyed.type = NVME_SGL_TYPE_DATA_BLOCK; + sgl->unkeyed.length = length; + sgl->address = phys_addr; + sgl->unkeyed.subtype = 0; + + sgl++; + nseg++; + + } + + if (nseg == 1) { + /* + * The whole transfer can be described by a single Scatter + * Gather List descriptor. Use the special case described + * by the spec where SGL1's type is Data Block. + * This means the SGL in the tracker is not used at all, + * so copy the first (and only) SGL element into SGL1. + */ + req->cmd.dptr.sgl1.unkeyed.type = NVME_SGL_TYPE_DATA_BLOCK; + req->cmd.dptr.sgl1.address = tr->u.sgl[0].address; + req->cmd.dptr.sgl1.unkeyed.length = tr->u.sgl[0].unkeyed.length; + } else { + /* For now we only support 1 SGL segment in NVMe controller */ + req->cmd.dptr.sgl1.unkeyed.type = NVME_SGL_TYPE_LAST_SEGMENT; + req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr; + req->cmd.dptr.sgl1.unkeyed.length = + nseg * sizeof(struct nvme_sgl_descriptor); + } + + return 0; +} + +/* + * Build Physical Region Page list describing scattered payload buffer. + */ +static int _nvme_qpair_build_prps_sgl_request(struct nvme_qpair *qpair, + struct nvme_request *req, + struct nvme_tracker *tr) +{ + uint64_t phys_addr, prp2 = 0; + uint32_t data_transferred, remaining_transfer_len, length; + uint32_t nseg, cur_nseg, total_nseg = 0, last_nseg = 0; + uint32_t modulo, unaligned, sge_count = 0; + int ret; + + /* + * Build scattered payloads. + */ + nvme_assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL, + "sgl payload type required\n"); + nvme_assert(req->payload.u.sgl.reset_sgl_fn != NULL, + "sgl reset callback required\n"); + req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg, + req->payload_offset); + + remaining_transfer_len = req->payload_size; + + while (remaining_transfer_len > 0) { + + nvme_assert(req->payload.u.sgl.next_sge_fn != NULL, + "sgl callback required\n"); + + ret = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg, + &phys_addr, &length); + if (ret != 0) { + _nvme_qpair_req_bad_phys(qpair, tr); + return -1; + } + + data_transferred = nvme_min(remaining_transfer_len, length); + + nseg = data_transferred >> PAGE_SHIFT; + modulo = data_transferred & (PAGE_SIZE - 1); + unaligned = phys_addr & (PAGE_SIZE - 1); + if (modulo || unaligned) + nseg += 1 + ((modulo + unaligned - 1) >> PAGE_SHIFT); + + if (total_nseg == 0) { + req->cmd.psdt = NVME_PSDT_PRP; + req->cmd.dptr.prp.prp1 = phys_addr; + } + + total_nseg += nseg; + sge_count++; + remaining_transfer_len -= data_transferred; + + if (total_nseg == 2) { + if (sge_count == 1) + tr->req->cmd.dptr.prp.prp2 = phys_addr + + PAGE_SIZE - unaligned; + else if (sge_count == 2) + tr->req->cmd.dptr.prp.prp2 = phys_addr; + /* save prp2 value */ + prp2 = tr->req->cmd.dptr.prp.prp2; + } else if (total_nseg > 2) { + if (sge_count == 1) + cur_nseg = 1; + else + cur_nseg = 0; + + tr->req->cmd.dptr.prp.prp2 = + (uint64_t)tr->prp_sgl_bus_addr; + + while (cur_nseg < nseg) { + if (prp2) { + tr->u.prp[0] = prp2; + tr->u.prp[last_nseg + 1] = phys_addr + + cur_nseg * PAGE_SIZE - unaligned; + } else { + tr->u.prp[last_nseg] = phys_addr + + cur_nseg * PAGE_SIZE - unaligned; + } + last_nseg++; + cur_nseg++; + + /* physical address and length check */ + if (remaining_transfer_len || + (!remaining_transfer_len && + (cur_nseg < nseg))) { + if ((length & (PAGE_SIZE - 1)) || + unaligned) { + _nvme_qpair_req_bad_phys(qpair, + tr); + return -1; + } + } + } + } + } + + return 0; +} + +static void _nvme_qpair_admin_qpair_enable(struct nvme_qpair *qpair) +{ + struct nvme_tracker *tr, *tr_temp; + + /* + * Manually abort each outstanding admin command. Do not retry + * admin commands found here, since they will be left over from + * a controller reset and its likely the context in which the + * command was issued no longer applies. + */ + LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, tr_temp) { + nvme_info("Aborting outstanding admin command\n"); + nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, + NVME_SC_ABORTED_BY_REQUEST, + 1 /* do not retry */, true); + } + + qpair->enabled = true; +} + +static void _nvme_qpair_io_qpair_enable(struct nvme_qpair *qpair) +{ + struct nvme_tracker *tr, *temp; + struct nvme_request *req; + + qpair->enabled = true; + + qpair->ctrlr->enabled_io_qpairs++; + + /* Manually abort each queued I/O. */ + while (!STAILQ_EMPTY(&qpair->queued_req)) { + req = STAILQ_FIRST(&qpair->queued_req); + STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); + nvme_info("Aborting queued I/O command\n"); + nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC, + NVME_SC_ABORTED_BY_REQUEST, + true); + } + + /* Manually abort each outstanding I/O. */ + LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, temp) { + nvme_info("Aborting outstanding I/O command\n"); + nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, + NVME_SC_ABORTED_BY_REQUEST, + 0, true); + } +} + +static inline void _nvme_qpair_admin_qpair_disable(struct nvme_qpair *qpair) +{ + qpair->enabled = false; + nvme_qpair_abort_aers(qpair); +} + +static inline void _nvme_qpair_io_qpair_disable(struct nvme_qpair *qpair) +{ + qpair->enabled = false; + + qpair->ctrlr->enabled_io_qpairs--; +} + +/* + * Reserve room for the submission queue + * in the controller memory buffer + */ +static int nvme_ctrlr_reserve_sq_in_cmb(struct nvme_ctrlr *ctrlr, + uint16_t entries, + uint64_t aligned, uint64_t *offset) +{ + uint64_t round_offset; + const uint64_t length = entries * sizeof(struct nvme_cmd); + + round_offset = ctrlr->cmb_current_offset; + round_offset = (round_offset + (aligned - 1)) & ~(aligned - 1); + + if (round_offset + length > ctrlr->cmb_size) + return -1; + + *offset = round_offset; + ctrlr->cmb_current_offset = round_offset + length; + + return 0; +} + +/* + * Initialize a queue pair on the host side. + */ +int nvme_qpair_construct(struct nvme_ctrlr *ctrlr, struct nvme_qpair *qpair, + enum nvme_qprio qprio, + uint16_t entries, uint16_t trackers) +{ + volatile uint32_t *doorbell_base; + struct nvme_tracker *tr; + uint64_t offset; + unsigned long phys_addr = 0; + uint16_t i; + int ret; + + nvme_assert(entries != 0, "Invalid number of entries\n"); + nvme_assert(trackers != 0, "Invalid trackers\n"); + + qpair->entries = entries; + qpair->trackers = trackers; + qpair->qprio = qprio; + qpair->sq_in_cmb = false; + qpair->ctrlr = ctrlr; + + if (ctrlr->opts.use_cmb_sqs) { + /* + * Reserve room for the submission queue in ctrlr + * memory buffer. + */ + ret = nvme_ctrlr_reserve_sq_in_cmb(ctrlr, entries, + PAGE_SIZE, + &offset); + if (ret == 0) { + + qpair->cmd = ctrlr->cmb_bar_virt_addr + offset; + qpair->cmd_bus_addr = ctrlr->cmb_bar_phys_addr + offset; + qpair->sq_in_cmb = true; + + nvme_debug("Allocated qpair %d cmd in cmb at %p / 0x%llx\n", + qpair->id, + qpair->cmd, qpair->cmd_bus_addr); + + } + } + + if (qpair->sq_in_cmb == false) { + + qpair->cmd = + nvme_mem_alloc_node(sizeof(struct nvme_cmd) * entries, + PAGE_SIZE, NVME_NODE_ID_ANY, + (unsigned long *) &qpair->cmd_bus_addr); + if (!qpair->cmd) { + nvme_err("Allocate qpair commands failed\n"); + goto fail; + } + memset(qpair->cmd, 0, sizeof(struct nvme_cmd) * entries); + + nvme_debug("Allocated qpair %d cmd %p / 0x%llx\n", + qpair->id, + qpair->cmd, qpair->cmd_bus_addr); + } + + qpair->cpl = nvme_mem_alloc_node(sizeof(struct nvme_cpl) * entries, + PAGE_SIZE, NVME_NODE_ID_ANY, + (unsigned long *) &qpair->cpl_bus_addr); + if (!qpair->cpl) { + nvme_err("Allocate qpair completions failed\n"); + goto fail; + } + memset(qpair->cpl, 0, sizeof(struct nvme_cpl) * entries); + + nvme_debug("Allocated qpair %d cpl at %p / 0x%llx\n", + qpair->id, + qpair->cpl, + qpair->cpl_bus_addr); + + doorbell_base = &ctrlr->regs->doorbell[0].sq_tdbl; + qpair->sq_tdbl = doorbell_base + + (2 * qpair->id + 0) * ctrlr->doorbell_stride_u32; + qpair->cq_hdbl = doorbell_base + + (2 * qpair->id + 1) * ctrlr->doorbell_stride_u32; + + LIST_INIT(&qpair->free_tr); + LIST_INIT(&qpair->outstanding_tr); + STAILQ_INIT(&qpair->free_req); + STAILQ_INIT(&qpair->queued_req); + + /* Request pool */ + if (nvme_request_pool_construct(qpair)) { + nvme_err("Create request pool failed\n"); + goto fail; + } + + /* + * Reserve space for all of the trackers in a single allocation. + * struct nvme_tracker must be padded so that its size is already + * a power of 2. This ensures the PRP list embedded in the nvme_tracker + * object will not span a 4KB boundary, while allowing access to + * trackers in tr[] via normal array indexing. + */ + qpair->tr = nvme_mem_alloc_node(sizeof(struct nvme_tracker) * trackers, + sizeof(struct nvme_tracker), + NVME_NODE_ID_ANY, &phys_addr); + if (!qpair->tr) { + nvme_err("Allocate request trackers failed\n"); + goto fail; + } + memset(qpair->tr, 0, sizeof(struct nvme_tracker) * trackers); + + nvme_debug("Allocated qpair %d trackers at %p / 0x%lx\n", + qpair->id, qpair->tr, phys_addr); + + for (i = 0; i < trackers; i++) { + tr = &qpair->tr[i]; + nvme_qpair_construct_tracker(tr, i, phys_addr); + LIST_INSERT_HEAD(&qpair->free_tr, tr, list); + phys_addr += sizeof(struct nvme_tracker); + } + + nvme_qpair_reset(qpair); + + return 0; + +fail: + nvme_qpair_destroy(qpair); + + return -1; +} + +void nvme_qpair_destroy(struct nvme_qpair *qpair) +{ + if (nvme_qpair_is_admin_queue(qpair)) + _nvme_qpair_admin_qpair_destroy(qpair); + + if (qpair->cmd && !qpair->sq_in_cmb) { + nvme_free(qpair->cmd); + qpair->cmd = NULL; + } + if (qpair->cpl) { + nvme_free(qpair->cpl); + qpair->cpl = NULL; + } + if (qpair->tr) { + nvme_free(qpair->tr); + qpair->tr = NULL; + } + nvme_request_pool_destroy(qpair); + +} + +bool nvme_qpair_enabled(struct nvme_qpair *qpair) +{ + if (!qpair->enabled && !qpair->ctrlr->resetting) + nvme_qpair_enable(qpair); + + return qpair->enabled; +} + +int nvme_qpair_submit_request(struct nvme_qpair *qpair, + struct nvme_request *req) +{ + struct nvme_tracker *tr; + struct nvme_request *child_req, *tmp; + struct nvme_ctrlr *ctrlr = qpair->ctrlr; + bool child_req_failed = false; + int ret = 0; + + if (ctrlr->failed) { + nvme_request_free(req); + return ENXIO; + } + + nvme_qpair_enabled(qpair); + + if (req->child_reqs) { + + /* + * This is a splitted (parent) request. Submit all of the + * children but not the parent request itself, since the + * parent is the original unsplit request. + */ + TAILQ_FOREACH_SAFE(child_req, &req->children, child_tailq, tmp) { + if (!child_req_failed) { + ret = nvme_qpair_submit_request(qpair, child_req); + if (ret != 0) + child_req_failed = true; + } else { + /* free remaining child_reqs since + * one child_req fails */ + nvme_request_remove_child(req, child_req); + nvme_request_free(child_req); + } + } + + return ret; + } + + tr = LIST_FIRST(&qpair->free_tr); + if (tr == NULL || !qpair->enabled) { + /* + * No tracker is available, or the qpair is disabled due + * to an in-progress controller-level reset. + * + * Put the request on the qpair's request queue to be + * processed when a tracker frees up via a command + * completion or when the controller reset is completed. + */ + STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); + return 0; + } + + /* remove tr from free_tr */ + LIST_REMOVE(tr, list); + LIST_INSERT_HEAD(&qpair->outstanding_tr, tr, list); + tr->req = req; + req->cmd.cid = tr->cid; + + if (req->payload_size == 0) { + /* Null payload - leave PRP fields zeroed */ + ret = 0; + } else if (req->payload.type == NVME_PAYLOAD_TYPE_CONTIG) { + ret = _nvme_qpair_build_contig_request(qpair, req, tr); + } else if (req->payload.type == NVME_PAYLOAD_TYPE_SGL) { + if (ctrlr->flags & NVME_CTRLR_SGL_SUPPORTED) + ret = _nvme_qpair_build_hw_sgl_request(qpair, req, tr); + else + ret = _nvme_qpair_build_prps_sgl_request(qpair, req, tr); + } else { + nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, + NVME_SC_INVALID_FIELD, + 1 /* do not retry */, true); + ret = -EINVAL; + } + + if (ret == 0) + nvme_qpair_submit_tracker(qpair, tr); + + return ret; +} + +unsigned int nvme_qpair_poll(struct nvme_qpair *qpair, + unsigned int max_completions) +{ + struct nvme_tracker *tr; + struct nvme_cpl *cpl; + uint32_t num_completions = 0; + + if (!nvme_qpair_enabled(qpair)) + /* + * qpair is not enabled, likely because a controller reset is + * is in progress. Ignore the interrupt - any I/O that was + * associated with this interrupt will get retried when the + * reset is complete. + */ + return 0; + + if ((max_completions == 0) || + (max_completions > (qpair->entries - 1U))) + /* + * max_completions == 0 means unlimited, but complete at most + * one queue depth batch of I/O at a time so that the completion + * queue doorbells don't wrap around. + */ + max_completions = qpair->entries - 1; + + while (1) { + + cpl = &qpair->cpl[qpair->cq_head]; + if (cpl->status.p != qpair->phase) + break; + + tr = &qpair->tr[cpl->cid]; + if (tr->active) { + nvme_qpair_complete_tracker(qpair, tr, cpl, true); + } else { + nvme_info("cpl does not map to outstanding cmd\n"); + nvme_qpair_print_completion(qpair, cpl); + nvme_panic("received completion for unknown cmd\n"); + } + + if (++qpair->cq_head == qpair->entries) { + qpair->cq_head = 0; + qpair->phase = !qpair->phase; + } + + if (++num_completions == max_completions) + break; + } + + if (num_completions > 0) + nvme_mmio_write_4(qpair->cq_hdbl, qpair->cq_head); + + return num_completions; +} + +void nvme_qpair_reset(struct nvme_qpair *qpair) +{ + qpair->sq_tail = qpair->cq_head = 0; + + /* + * First time through the completion queue, HW will set phase + * bit on completions to 1. So set this to 1 here, indicating + * we're looking for a 1 to know which entries have completed. + * we'll toggle the bit each time when the completion queue rolls over. + */ + qpair->phase = 1; + + memset(qpair->cmd, 0, qpair->entries * sizeof(struct nvme_cmd)); + memset(qpair->cpl, 0, qpair->entries * sizeof(struct nvme_cpl)); +} + +void nvme_qpair_enable(struct nvme_qpair *qpair) +{ + if (nvme_qpair_is_io_queue(qpair)) + _nvme_qpair_io_qpair_enable(qpair); + else + _nvme_qpair_admin_qpair_enable(qpair); +} + +void nvme_qpair_disable(struct nvme_qpair *qpair) +{ + if (nvme_qpair_is_io_queue(qpair)) + _nvme_qpair_io_qpair_disable(qpair); + else + _nvme_qpair_admin_qpair_disable(qpair); +} + +void nvme_qpair_fail(struct nvme_qpair *qpair) +{ + struct nvme_tracker *tr; + struct nvme_request *req; + + while (!STAILQ_EMPTY(&qpair->queued_req)) { + + nvme_notice("Failing queued I/O command\n"); + req = STAILQ_FIRST(&qpair->queued_req); + STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); + nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC, + NVME_SC_ABORTED_BY_REQUEST, + true); + + } + + /* Manually abort each outstanding I/O. */ + while (!LIST_EMPTY(&qpair->outstanding_tr)) { + + /* + * Do not remove the tracker. The abort_tracker path + * will do that for us. + */ + nvme_notice("Failing outstanding I/O command\n"); + tr = LIST_FIRST(&qpair->outstanding_tr); + nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, + NVME_SC_ABORTED_BY_REQUEST, + 1, true); + + } +} + diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_quirks.c b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_quirks.c new file mode 100644 index 0000000000..4689370d85 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_quirks.c @@ -0,0 +1,111 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" +#include "nvme_pci.h" + +struct nvme_quirk { + struct pci_id id; + unsigned int flags; +}; + +static const struct nvme_quirk nvme_quirks[] = { + { + { NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x3702 }, + NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY + }, + { + { NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x3703 }, + NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY + }, + { + { NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x3704 }, + NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY + }, + { + { NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x3705 }, + NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY + }, + { + { NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x3709 }, + NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY + }, + { + { NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x370a }, + NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY + }, + { + { NVME_PCI_VID_MEMBLAZE, 0x0540, NVME_PCI_ANY_ID, NVME_PCI_ANY_ID }, + NVME_QUIRK_DELAY_BEFORE_CHK_RDY + }, + { + { NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x370d }, + NVME_QUIRK_DELAY_AFTER_RDY + }, + { + { 0x0000, 0x0000, 0x0000, 0x0000 }, + 0 + } +}; + +/* + * Compare each field. NVME_PCI_ANY_ID in s1 matches everything. + */ +static bool nvme_quirks_pci_id_match(const struct pci_id *id, + struct pci_device *pdev) +{ + if ((id->vendor_id == NVME_PCI_ANY_ID || + id->vendor_id == pdev->vendor_id) && + (id->device_id == NVME_PCI_ANY_ID || + id->device_id == pdev->device_id) && + (id->subvendor_id == NVME_PCI_ANY_ID || + id->subvendor_id == pdev->subvendor_id) && + (id->subdevice_id == NVME_PCI_ANY_ID || + id->subdevice_id == pdev->subdevice_id)) + return true; + + return false; +} + +unsigned int nvme_ctrlr_get_quirks(struct pci_device *pdev) +{ + const struct nvme_quirk *quirk = nvme_quirks; + + while (quirk->id.vendor_id) { + if (nvme_quirks_pci_id_match(&quirk->id, pdev)) + return quirk->flags; + quirk++; + } + + return 0; +} diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_request.c b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_request.c new file mode 100644 index 0000000000..9ca24452d4 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_request.c @@ -0,0 +1,211 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "nvme_internal.h" + +/* + * Allocate a request descriptor from the queue pair free list. + */ +static struct nvme_request *nvme_alloc_request(struct nvme_qpair *qpair) +{ + struct nvme_request *req; + + req = STAILQ_FIRST(&qpair->free_req); + if (req) { + STAILQ_REMOVE_HEAD(&qpair->free_req, stailq); + memset(&req->cmd, 0, sizeof(struct nvme_cmd)); + } + + return req; +} + +static void nvme_request_cb_complete_child(void *child_arg, + const struct nvme_cpl *cpl) +{ + struct nvme_request *child = child_arg; + struct nvme_request *parent = child->parent; + + nvme_request_remove_child(parent, child); + + if (nvme_cpl_is_error(cpl)) + memcpy(&parent->parent_status, cpl, sizeof(*cpl)); + + if (parent->child_reqs == 0) { + if (parent->cb_fn) + parent->cb_fn(parent->cb_arg, &parent->parent_status); + nvme_request_free(parent); + } +} + +void nvme_request_completion_poll_cb(void *arg, const struct nvme_cpl *cpl) +{ + struct nvme_completion_poll_status *status = arg; + + memcpy(&status->cpl, cpl, sizeof(*cpl)); + status->done = true; +} + +int nvme_request_pool_construct(struct nvme_qpair *qpair) +{ + struct nvme_request *req; + unsigned int i; + + qpair->num_reqs = qpair->trackers * NVME_IO_ENTRIES_VS_TRACKERS_RATIO; + qpair->reqs = calloc(qpair->num_reqs, sizeof(struct nvme_request)); + if (!qpair->reqs) { + nvme_err("QPair %d: allocate %u requests failed\n", + (int)qpair->id, qpair->num_reqs); + return ENOMEM; + } + + nvme_info("QPair %d: %d requests in pool\n", + (int)qpair->id, + (int)qpair->num_reqs); + + for(i = 0; i < qpair->num_reqs; i++) { + req = &qpair->reqs[i]; + req->qpair = qpair; + STAILQ_INSERT_TAIL(&qpair->free_req, req, stailq); + req++; + } + + return 0; +} + +void nvme_request_pool_destroy(struct nvme_qpair *qpair) +{ + struct nvme_request *req; + unsigned int n = 0; + + while ((req = STAILQ_FIRST(&qpair->free_req))) { + STAILQ_REMOVE_HEAD(&qpair->free_req, stailq); + n++; + } + + if (n != qpair->num_reqs) + nvme_err("QPair %d: Freed %d/%d requests\n", + (int)qpair->id, n, (int)qpair->num_reqs); + + free(qpair->reqs); +} + +struct nvme_request *nvme_request_allocate(struct nvme_qpair *qpair, + const struct nvme_payload *payload, + uint32_t payload_size, + nvme_cmd_cb cb_fn, + void *cb_arg) +{ + struct nvme_request *req; + + req = nvme_alloc_request(qpair); + if (req == NULL) + return NULL; + + /* + * Only memset up to (but not including) the children TAILQ_ENTRY. + * Children, and following members, are only used as part of I/O + * splitting so we avoid memsetting them until it is actually needed. + * They will be initialized in nvme_request_add_child() + * if the request is split. + */ + memset(req, 0, offsetof(struct nvme_request, children)); + req->cb_fn = cb_fn; + req->cb_arg = cb_arg; + req->payload = *payload; + req->payload_size = payload_size; + + return req; +} + +struct nvme_request *nvme_request_allocate_contig(struct nvme_qpair *qpair, + void *buffer, + uint32_t payload_size, + nvme_cmd_cb cb_fn, + void *cb_arg) +{ + struct nvme_payload payload; + + payload.type = NVME_PAYLOAD_TYPE_CONTIG; + payload.u.contig = buffer; + payload.md = NULL; + + return nvme_request_allocate(qpair, &payload, payload_size, + cb_fn, cb_arg); +} + +struct nvme_request *nvme_request_allocate_null(struct nvme_qpair *qpair, + nvme_cmd_cb cb_fn, void *cb_arg) +{ + return nvme_request_allocate_contig(qpair, NULL, 0, cb_fn, cb_arg); +} + +void nvme_request_free(struct nvme_request *req) +{ + struct nvme_qpair *qpair = req->qpair; + + nvme_assert(req->child_reqs == 0, "Number of child request not 0\n"); + + STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq); +} + +void nvme_request_add_child(struct nvme_request *parent, + struct nvme_request *child) +{ + if (parent->child_reqs == 0) { + /* + * Defer initialization of the children TAILQ since it falls + * on a separate cacheline. This ensures we do not touch this + * cacheline except on request splitting cases, which are + * relatively rare. + */ + TAILQ_INIT(&parent->children); + parent->parent = NULL; + memset(&parent->parent_status, 0, sizeof(struct nvme_cpl)); + } + + parent->child_reqs++; + TAILQ_INSERT_TAIL(&parent->children, child, child_tailq); + child->parent = parent; + child->cb_fn = nvme_request_cb_complete_child; + child->cb_arg = child; +} + +void nvme_request_remove_child(struct nvme_request *parent, + struct nvme_request *child) +{ + nvme_assert(child->parent == parent, "child->parent != parent\n"); + nvme_assert(parent->child_reqs != 0, "child_reqs is 0\n"); + + parent->child_reqs--; + TAILQ_REMOVE(&parent->children, child, child_tailq); +} diff --git a/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_spec.h b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_spec.h new file mode 100644 index 0000000000..86a7915565 --- /dev/null +++ b/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_spec.h @@ -0,0 +1,2138 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. All rights reserved. + * Copyright (c) 2017, Western Digital Corporation or its affiliates. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __LIBNVME_SPEC_H__ +#define __LIBNVME_SPEC_H__ + +#include +#include + +/* + * Use to mark a command to apply to all namespaces, + * or to retrieve global log pages. + */ +#define NVME_GLOBAL_NS_TAG ((uint32_t)0xFFFFFFFF) + +#define NVME_MAX_IO_QUEUES (65535) + +#define NVME_ADMIN_QUEUE_MIN_ENTRIES 2 +#define NVME_ADMIN_QUEUE_MAX_ENTRIES 4096 + +#define NVME_IO_QUEUE_MIN_ENTRIES 2 +#define NVME_IO_QUEUE_MAX_ENTRIES 65536 + +#define NVME_MAX_NS 1024 + +/* + * Number of characters in the serial number. + */ +#define NVME_SERIAL_NUMBER_CHARACTERS 20 + +/* + * Number of characters in the model number. + */ +#define NVME_MODEL_NUMBER_CHARACTERS 40 + +/* + * Indicates the maximum number of range sets that may be specified + * in the dataset mangement command. + */ +#define NVME_DATASET_MANAGEMENT_MAX_RANGES 256 + +/* + * Compile time assert check. + */ +#ifdef static_assert +#define nvme_static_assert(cond, msg) static_assert(cond, msg) +#else +#define nvme_static_assert(cond, msg) +#endif + +union nvme_cap_register { + + uint64_t raw; + + struct { + /* Maximum queue entries supported */ + uint32_t mqes : 16; + + /* Contiguous queues required */ + uint32_t cqr : 1; + + /* Arbitration mechanism supported */ + uint32_t ams : 2; + + uint32_t reserved1 : 5; + + /* + * Worst case time in 500 millisecond units + * to wait for the controller to become ready + * (CSTS.RDY set to '1') after a power-on or reset + */ + uint32_t to : 8; + + /* Doorbell stride */ + uint32_t dstrd : 4; + + /* NVM subsystem reset supported */ + uint32_t nssrs : 1; + + /* Command sets supported */ + uint32_t css_nvm : 1; + + uint32_t css_reserved : 3; + uint32_t reserved2 : 7; + + /* Memory page size minimum */ + uint32_t mpsmin : 4; + + /* Memory page size maximum */ + uint32_t mpsmax : 4; + + uint32_t reserved3 : 8; + } bits; + +}; +nvme_static_assert(sizeof(union nvme_cap_register) == 8, "Incorrect size"); + +union nvme_cc_register { + + uint32_t raw; + + struct { + /* Enable */ + uint32_t en : 1; + + uint32_t reserved1 : 3; + + /* I/O command set selected */ + uint32_t css : 3; + + /* Memory page size */ + uint32_t mps : 4; + + /* Arbitration mechanism selected */ + uint32_t ams : 3; + + /* Shutdown notification */ + uint32_t shn : 2; + + /* I/O submission queue entry size */ + uint32_t iosqes : 4; + + /* I/O completion queue entry size */ + uint32_t iocqes : 4; + + uint32_t reserved2 : 8; + } bits; + +}; +nvme_static_assert(sizeof(union nvme_cc_register) == 4, "Incorrect size"); + +enum nvme_shn_value { + NVME_SHN_NORMAL = 0x1, + NVME_SHN_ABRUPT = 0x2, +}; + +union nvme_csts_register { + + uint32_t raw; + + struct { + /* Ready */ + uint32_t rdy : 1; + + /* Controller fatal status */ + uint32_t cfs : 1; + + /* Shutdown status */ + uint32_t shst : 2; + + uint32_t reserved1 : 28; + } bits; +}; +nvme_static_assert(sizeof(union nvme_csts_register) == 4, "Incorrect size"); + +enum nvme_shst_value { + NVME_SHST_NORMAL = 0x0, + NVME_SHST_OCCURRING = 0x1, + NVME_SHST_COMPLETE = 0x2, +}; + +union nvme_aqa_register { + + uint32_t raw; + + struct { + /* Admin submission queue size */ + uint32_t asqs : 12; + + uint32_t reserved1 : 4; + + /* Admin completion queue size */ + uint32_t acqs : 12; + + uint32_t reserved2 : 4; + } bits; + +}; +nvme_static_assert(sizeof(union nvme_aqa_register) == 4, "Incorrect size"); + +union nvme_vs_register { + + uint32_t raw; + + struct { + /* Indicates the tertiary version */ + uint32_t ter : 8; + + /* Indicates the minor version */ + uint32_t mnr : 8; + + /* Indicates the major version */ + uint32_t mjr : 16; + } bits; + +}; +nvme_static_assert(sizeof(union nvme_vs_register) == 4, "Incorrect size"); + +/* + * Generate raw version. + */ +#define NVME_VERSION(mjr, mnr, ter) \ + (((uint32_t)(mjr) << 16) | \ + ((uint32_t)(mnr) << 8) | \ + (uint32_t)(ter)) + +/* + * Test that the shifts are correct + */ +nvme_static_assert(NVME_VERSION(1, 0, 0) == 0x00010000, "version macro error"); +nvme_static_assert(NVME_VERSION(1, 2, 1) == 0x00010201, "version macro error"); + +union nvme_cmbloc_register { + + uint32_t raw; + + struct { + /* + * Indicator of BAR which contains controller + * memory buffer(CMB). + */ + uint32_t bir : 3; + + uint32_t reserved1 : 9; + + /* Offset of CMB in multiples of the size unit */ + uint32_t ofst : 20; + } bits; +}; +nvme_static_assert(sizeof(union nvme_cmbloc_register) == 4, "Incorrect size"); + +union nvme_cmbsz_register { + + uint32_t raw; + + struct { + /* Support submission queues in CMB */ + uint32_t sqs : 1; + + /* Support completion queues in CMB */ + uint32_t cqs : 1; + + /* Support PRP and SGLs lists in CMB */ + uint32_t lists : 1; + + /* Support read data and metadata in CMB */ + uint32_t rds : 1; + + /* Support write data and metadata in CMB */ + uint32_t wds : 1; + + uint32_t reserved1 : 3; + + /* Indicates the granularity of the size unit */ + uint32_t szu : 4; + + /* Size of CMB in multiples of the size unit */ + uint32_t sz : 20; + } bits; +}; +nvme_static_assert(sizeof(union nvme_cmbsz_register) == 4, "Incorrect size"); + +struct nvme_registers { + + /* Controller capabilities */ + union nvme_cap_register cap; + + /* Version of NVMe specification */ + union nvme_vs_register vs; + + /* Interrupt mask set */ + uint32_t intms; + + /* Interrupt mask clear */ + uint32_t intmc; + + /* Controller configuration */ + union nvme_cc_register cc; + + uint32_t reserved1; + + /* Controller status */ + union nvme_csts_register csts; + + /* NVM subsystem reset */ + uint32_t nssr; + + /* Admin queue attributes */ + union nvme_aqa_register aqa; + + /* Admin submission queue base addr */ + uint64_t asq; + + /* Admin completion queue base addr */ + uint64_t acq; + + /* Controller memory buffer location */ + union nvme_cmbloc_register cmbloc; + + /* Controller memory buffer size */ + union nvme_cmbsz_register cmbsz; + + uint32_t reserved3[0x3f0]; + + struct { + /* Submission queue tail doorbell */ + uint32_t sq_tdbl; + + /* Completion queue head doorbell */ + uint32_t cq_hdbl; + } doorbell[1]; +}; + +/* + * Return the offset of a field in a structure. + */ +#ifndef offsetof +#define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER) +#endif + +/* + * NVMe controller register space offsets. + */ +nvme_static_assert(0x00 == offsetof(struct nvme_registers, cap), + "Incorrect register offset"); +nvme_static_assert(0x08 == offsetof(struct nvme_registers, vs), + "Incorrect register offset"); +nvme_static_assert(0x0C == offsetof(struct nvme_registers, intms), + "Incorrect register offset"); +nvme_static_assert(0x10 == offsetof(struct nvme_registers, intmc), + "Incorrect register offset"); +nvme_static_assert(0x14 == offsetof(struct nvme_registers, cc), + "Incorrect register offset"); +nvme_static_assert(0x1C == offsetof(struct nvme_registers, csts), + "Incorrect register offset"); +nvme_static_assert(0x20 == offsetof(struct nvme_registers, nssr), + "Incorrect register offset"); +nvme_static_assert(0x24 == offsetof(struct nvme_registers, aqa), + "Incorrect register offset"); +nvme_static_assert(0x28 == offsetof(struct nvme_registers, asq), + "Incorrect register offset"); +nvme_static_assert(0x30 == offsetof(struct nvme_registers, acq), + "Incorrect register offset"); +nvme_static_assert(0x38 == offsetof(struct nvme_registers, cmbloc), + "Incorrect register offset"); +nvme_static_assert(0x3C == offsetof(struct nvme_registers, cmbsz), + "Incorrect register offset"); + +enum nvme_sgl_descriptor_type { + + NVME_SGL_TYPE_DATA_BLOCK = 0x0, + NVME_SGL_TYPE_BIT_BUCKET = 0x1, + NVME_SGL_TYPE_SEGMENT = 0x2, + NVME_SGL_TYPE_LAST_SEGMENT = 0x3, + NVME_SGL_TYPE_KEYED_DATA_BLOCK = 0x4, + + /* 0x5 - 0xE reserved */ + + NVME_SGL_TYPE_VENDOR_SPECIFIC = 0xF + +}; + +enum nvme_sgl_descriptor_subtype { + NVME_SGL_SUBTYPE_ADDRESS = 0x0, + NVME_SGL_SUBTYPE_OFFSET = 0x1, +}; + +/* + * Scatter Gather List descriptor + */ +struct __attribute__((packed)) nvme_sgl_descriptor { + + uint64_t address; + + union { + struct { + uint8_t reserved[7]; + uint8_t subtype : 4; + uint8_t type : 4; + } generic; + + struct { + uint32_t length; + uint8_t reserved[3]; + uint8_t subtype : 4; + uint8_t type : 4; + } unkeyed; + + struct { + uint64_t length : 24; + uint64_t key : 32; + uint64_t subtype : 4; + uint64_t type : 4; + } keyed; + }; + +}; +nvme_static_assert(sizeof(struct nvme_sgl_descriptor) == 16, + "Incorrect size"); + +enum nvme_psdt_value { + NVME_PSDT_PRP = 0x0, + NVME_PSDT_SGL_MPTR_CONTIG = 0x1, + NVME_PSDT_SGL_MPTR_SGL = 0x2, + NVME_PSDT_RESERVED = 0x3 +}; + +/* + * Submission queue priority values for Create I/O Submission Queue Command. + * Only valid for weighted round robin arbitration method. + */ +enum nvme_qprio { + NVME_QPRIO_URGENT = 0x0, + NVME_QPRIO_HIGH = 0x1, + NVME_QPRIO_MEDIUM = 0x2, + NVME_QPRIO_LOW = 0x3 +}; + +/* + * Optional Arbitration Mechanism Supported by the controller. + * Two bits for CAP.AMS (18:17) field are set to '1' when the controller + * supports. There is no bit for AMS_RR where all controllers support and + * set to 0x0 by default. + */ +enum nvme_cap_ams { + + /* + * Weighted round robin + */ + NVME_CAP_AMS_WRR = 0x1, + + /* + * Vendor specific. + */ + NVME_CAP_AMS_VS = 0x2, + +}; + +/* + * Arbitration Mechanism Selected to the controller. + * Value 0x2 to 0x6 is reserved. + */ +enum nvme_cc_ams { + + /* + * Default round robin. + */ + NVME_CC_AMS_RR = 0x0, + + /* + * Weighted round robin. + */ + NVME_CC_AMS_WRR = 0x1, + + /* + * Vendor specific. + */ + NVME_CC_AMS_VS = 0x7, + +}; + +struct nvme_cmd { + + /* dword 0 */ + uint16_t opc : 8; /* opcode */ + uint16_t fuse : 2; /* fused operation */ + uint16_t rsvd1 : 4; + uint16_t psdt : 2; + uint16_t cid; /* command identifier */ + + /* dword 1 */ + uint32_t nsid; /* namespace identifier */ + + /* dword 2-3 */ + uint32_t rsvd2; + uint32_t rsvd3; + + /* dword 4-5 */ + uint64_t mptr; /* metadata pointer */ + + /* dword 6-9: data pointer */ + union { + struct { + uint64_t prp1; /* prp entry 1 */ + uint64_t prp2; /* prp entry 2 */ + } prp; + + struct nvme_sgl_descriptor sgl1; + } dptr; + + /* dword 10-15 */ + uint32_t cdw10; /* command-specific */ + uint32_t cdw11; /* command-specific */ + uint32_t cdw12; /* command-specific */ + uint32_t cdw13; /* command-specific */ + uint32_t cdw14; /* command-specific */ + uint32_t cdw15; /* command-specific */ + +}; +nvme_static_assert(sizeof(struct nvme_cmd) == 64, "Incorrect size"); + +struct nvme_status { + uint16_t p : 1; /* phase tag */ + uint16_t sc : 8; /* status code */ + uint16_t sct : 3; /* status code type */ + uint16_t rsvd2 : 2; + uint16_t m : 1; /* more */ + uint16_t dnr : 1; /* do not retry */ +}; +nvme_static_assert(sizeof(struct nvme_status) == 2, "Incorrect size"); + +/* + * Completion queue entry + */ +struct nvme_cpl { + + /* dword 0 */ + uint32_t cdw0; /* command-specific */ + + /* dword 1 */ + uint32_t rsvd1; + + /* dword 2 */ + uint16_t sqhd; /* submission queue head pointer */ + uint16_t sqid; /* submission queue identifier */ + + /* dword 3 */ + uint16_t cid; /* command identifier */ + + struct nvme_status status; + +}; +nvme_static_assert(sizeof(struct nvme_cpl) == 16, "Incorrect size"); + +/* + * Dataset Management range + */ +struct nvme_dsm_range { + uint32_t attributes; + uint32_t length; + uint64_t starting_lba; +}; +nvme_static_assert(sizeof(struct nvme_dsm_range) == 16, "Incorrect size"); + +/* + * Status code types + */ +enum nvme_status_code_type { + + NVME_SCT_GENERIC = 0x0, + NVME_SCT_COMMAND_SPECIFIC = 0x1, + NVME_SCT_MEDIA_ERROR = 0x2, + + /* 0x3-0x6 - reserved */ + NVME_SCT_VENDOR_SPECIFIC = 0x7, + +}; + +/* + * Generic command status codes + */ +enum nvme_generic_command_status_code { + + NVME_SC_SUCCESS = 0x00, + NVME_SC_INVALID_OPCODE = 0x01, + NVME_SC_INVALID_FIELD = 0x02, + NVME_SC_COMMAND_ID_CONFLICT = 0x03, + NVME_SC_DATA_TRANSFER_ERROR = 0x04, + NVME_SC_ABORTED_POWER_LOSS = 0x05, + NVME_SC_INTERNAL_DEVICE_ERROR = 0x06, + NVME_SC_ABORTED_BY_REQUEST = 0x07, + NVME_SC_ABORTED_SQ_DELETION = 0x08, + NVME_SC_ABORTED_FAILED_FUSED = 0x09, + NVME_SC_ABORTED_MISSING_FUSED = 0x0a, + NVME_SC_INVALID_NAMESPACE_OR_FORMAT = 0x0b, + NVME_SC_COMMAND_SEQUENCE_ERROR = 0x0c, + NVME_SC_INVALID_SGL_SEG_DESCRIPTOR = 0x0d, + NVME_SC_INVALID_NUM_SGL_DESCIRPTORS = 0x0e, + NVME_SC_DATA_SGL_LENGTH_INVALID = 0x0f, + NVME_SC_METADATA_SGL_LENGTH_INVALID = 0x10, + NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID = 0x11, + NVME_SC_INVALID_CONTROLLER_MEM_BUF = 0x12, + NVME_SC_INVALID_PRP_OFFSET = 0x13, + NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED = 0x14, + NVME_SC_INVALID_SGL_OFFSET = 0x16, + NVME_SC_INVALID_SGL_SUBTYPE = 0x17, + NVME_SC_HOSTID_INCONSISTENT_FORMAT = 0x18, + NVME_SC_KEEP_ALIVE_EXPIRED = 0x19, + NVME_SC_KEEP_ALIVE_INVALID = 0x1a, + + NVME_SC_LBA_OUT_OF_RANGE = 0x80, + NVME_SC_CAPACITY_EXCEEDED = 0x81, + NVME_SC_NAMESPACE_NOT_READY = 0x82, + NVME_SC_RESERVATION_CONFLICT = 0x83, + NVME_SC_FORMAT_IN_PROGRESS = 0x84, + +}; + +/* + * Command specific status codes + */ +enum nvme_command_specific_status_code { + + NVME_SC_COMPLETION_QUEUE_INVALID = 0x00, + NVME_SC_INVALID_QUEUE_IDENTIFIER = 0x01, + NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED = 0x02, + NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED = 0x03, + + /* 0x04 - reserved */ + + NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED = 0x05, + NVME_SC_INVALID_FIRMWARE_SLOT = 0x06, + NVME_SC_INVALID_FIRMWARE_IMAGE = 0x07, + NVME_SC_INVALID_INTERRUPT_VECTOR = 0x08, + NVME_SC_INVALID_LOG_PAGE = 0x09, + NVME_SC_INVALID_FORMAT = 0x0a, + NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET = 0x0b, + NVME_SC_INVALID_QUEUE_DELETION = 0x0c, + NVME_SC_FEATURE_ID_NOT_SAVEABLE = 0x0d, + NVME_SC_FEATURE_NOT_CHANGEABLE = 0x0e, + NVME_SC_FEATURE_NOT_NAMESPACE_SPECIFIC = 0x0f, + NVME_SC_FIRMWARE_REQ_NVM_RESET = 0x10, + NVME_SC_FIRMWARE_REQ_RESET = 0x11, + NVME_SC_FIRMWARE_REQ_MAX_TIME_VIOLATION = 0x12, + NVME_SC_FIRMWARE_ACTIVATION_PROHIBITED = 0x13, + NVME_SC_OVERLAPPING_RANGE = 0x14, + NVME_SC_NAMESPACE_INSUFFICIENT_CAPACITY = 0x15, + NVME_SC_NAMESPACE_ID_UNAVAILABLE = 0x16, + + /* 0x17 - reserved */ + + NVME_SC_NAMESPACE_ALREADY_ATTACHED = 0x18, + NVME_SC_NAMESPACE_IS_PRIVATE = 0x19, + NVME_SC_NAMESPACE_NOT_ATTACHED = 0x1a, + NVME_SC_THINPROVISIONING_NOT_SUPPORTED = 0x1b, + NVME_SC_CONTROLLER_LIST_INVALID = 0x1c, + + NVME_SC_CONFLICTING_ATTRIBUTES = 0x80, + NVME_SC_INVALID_PROTECTION_INFO = 0x81, + NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE = 0x82, + +}; + +/* + * Media error status codes + */ +enum nvme_media_error_status_code { + NVME_SC_WRITE_FAULTS = 0x80, + NVME_SC_UNRECOVERED_READ_ERROR = 0x81, + NVME_SC_GUARD_CHECK_ERROR = 0x82, + NVME_SC_APPLICATION_TAG_CHECK_ERROR = 0x83, + NVME_SC_REFERENCE_TAG_CHECK_ERROR = 0x84, + NVME_SC_COMPARE_FAILURE = 0x85, + NVME_SC_ACCESS_DENIED = 0x86, + NVME_SC_DEALLOCATED_OR_UNWRITTEN_BLOCK = 0x87, +}; + +/* + * Admin opcodes + */ +enum nvme_admin_opcode { + + NVME_OPC_DELETE_IO_SQ = 0x00, + NVME_OPC_CREATE_IO_SQ = 0x01, + NVME_OPC_GET_LOG_PAGE = 0x02, + + /* 0x03 - reserved */ + + NVME_OPC_DELETE_IO_CQ = 0x04, + NVME_OPC_CREATE_IO_CQ = 0x05, + NVME_OPC_IDENTIFY = 0x06, + + /* 0x07 - reserved */ + + NVME_OPC_ABORT = 0x08, + NVME_OPC_SET_FEATURES = 0x09, + NVME_OPC_GET_FEATURES = 0x0a, + + /* 0x0b - reserved */ + + NVME_OPC_ASYNC_EVENT_REQUEST = 0x0c, + NVME_OPC_NS_MANAGEMENT = 0x0d, + + /* 0x0e-0x0f - reserved */ + + NVME_OPC_FIRMWARE_COMMIT = 0x10, + NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD = 0x11, + + NVME_OPC_NS_ATTACHMENT = 0x15, + + NVME_OPC_KEEP_ALIVE = 0x18, + + NVME_OPC_FORMAT_NVM = 0x80, + NVME_OPC_SECURITY_SEND = 0x81, + NVME_OPC_SECURITY_RECEIVE = 0x82, + +}; + +/* + * NVM command set opcodes + */ +enum nvme_nvm_opcode { + + NVME_OPC_FLUSH = 0x00, + NVME_OPC_WRITE = 0x01, + NVME_OPC_READ = 0x02, + + /* 0x03 - reserved */ + + NVME_OPC_WRITE_UNCORRECTABLE = 0x04, + NVME_OPC_COMPARE = 0x05, + + /* 0x06-0x07 - reserved */ + + NVME_OPC_WRITE_ZEROES = 0x08, + NVME_OPC_DATASET_MANAGEMENT = 0x09, + + NVME_OPC_RESERVATION_REGISTER = 0x0d, + NVME_OPC_RESERVATION_REPORT = 0x0e, + + NVME_OPC_RESERVATION_ACQUIRE = 0x11, + NVME_OPC_RESERVATION_RELEASE = 0x15, + +}; + +/* + * Data transfer (bits 1:0) of an NVMe opcode. + */ +enum nvme_data_transfer { + + /* + * Opcode does not transfer data. + */ + NVME_DATA_NONE = 0, + + /* + * Opcode transfers data from host to controller (e.g. Write). + */ + NVME_DATA_HOST_TO_CONTROLLER = 1, + + /* + * Opcode transfers data from controller to host (e.g. Read). + */ + NVME_DATA_CONTROLLER_TO_HOST = 2, + + /* + * Opcode transfers data both directions. + */ + NVME_DATA_BIDIRECTIONAL = 3 + +}; + +/* + * Extract the Data Transfer bits from an NVMe opcode. + * + * This determines whether a command requires a data buffer and + * which direction (host to controller or controller to host) it is + * transferred. + */ +static inline enum nvme_data_transfer nvme_opc_get_data_transfer(uint8_t opc) +{ + return (enum nvme_data_transfer)(opc & 3); +} + +/* + * Features. + */ +enum nvme_feat { + + /* 0x00 - reserved */ + + NVME_FEAT_ARBITRATION = 0x01, + NVME_FEAT_POWER_MANAGEMENT = 0x02, + NVME_FEAT_LBA_RANGE_TYPE = 0x03, + NVME_FEAT_TEMPERATURE_THRESHOLD = 0x04, + NVME_FEAT_ERROR_RECOVERY = 0x05, + NVME_FEAT_VOLATILE_WRITE_CACHE = 0x06, + NVME_FEAT_NUMBER_OF_QUEUES = 0x07, + NVME_FEAT_INTERRUPT_COALESCING = 0x08, + NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION = 0x09, + NVME_FEAT_WRITE_ATOMICITY = 0x0A, + NVME_FEAT_ASYNC_EVENT_CONFIGURATION = 0x0B, + NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION = 0x0C, + NVME_FEAT_HOST_MEM_BUFFER = 0x0D, + NVME_FEAT_KEEP_ALIVE_TIMER = 0x0F, + + /* 0x0C-0x7F - reserved */ + + NVME_FEAT_SOFTWARE_PROGRESS_MARKER = 0x80, + + /* 0x81-0xBF - command set specific */ + + NVME_FEAT_HOST_IDENTIFIER = 0x81, + NVME_FEAT_HOST_RESERVE_MASK = 0x82, + NVME_FEAT_HOST_RESERVE_PERSIST = 0x83, + + /* 0xC0-0xFF - vendor specific */ + +}; + +/* + * Get features selection. + */ +enum nvme_feat_sel { + NVME_FEAT_CURRENT = 0x0, + NVME_FEAT_DEFAULT = 0x1, + NVME_FEAT_SAVED = 0x2, + NVME_FEAT_SUPPORTED = 0x3, +}; + +enum nvme_dsm_attribute { + NVME_DSM_ATTR_INTEGRAL_READ = 0x1, + NVME_DSM_ATTR_INTEGRAL_WRITE = 0x2, + NVME_DSM_ATTR_DEALLOCATE = 0x4, +}; + +struct nvme_power_state { + + /* + * bits 15:00: maximum power. + */ + uint16_t mp; + + uint8_t reserved1; + + /* + * bit 24: max power scale. + */ + uint8_t mps : 1; + + /* + * bit 25: non-operational state. + */ + uint8_t nops : 1; + uint8_t reserved2 : 6; + + /* + * bits 63:32: entry latency in microseconds. + */ + uint32_t enlat; + + /* + * bits 95:64: exit latency in microseconds. + */ + uint32_t exlat; + + /* + * bits 100:96: relative read throughput. + */ + uint8_t rrt : 5; + uint8_t reserved3 : 3; + + /* + * bits 108:104: relative read latency. + */ + uint8_t rrl : 5; + uint8_t reserved4 : 3; + + /* + * bits 116:112: relative write throughput. + */ + uint8_t rwt : 5; + uint8_t reserved5 : 3; + + /* + * bits 124:120: relative write latency. + */ + uint8_t rwl : 5; + uint8_t reserved6 : 3; + + uint8_t reserved7[16]; + +}; +nvme_static_assert(sizeof(struct nvme_power_state) == 32, "Incorrect size"); + +/* + * Identify command CNS value + */ +enum nvme_identify_cns { + + /* + * Identify namespace indicated in CDW1.NSID. + */ + NVME_IDENTIFY_NS = 0x00, + + /* + * Identify controller. + */ + NVME_IDENTIFY_CTRLR = 0x01, + + /* + * List active NSIDs greater than CDW1.NSID. + */ + NVME_IDENTIFY_ACTIVE_NS_LIST = 0x02, + + /* + * List allocated NSIDs greater than CDW1.NSID. + */ + NVME_IDENTIFY_ALLOCATED_NS_LIST = 0x10, + + /* + * Identify namespace if CDW1.NSID is allocated. + */ + NVME_IDENTIFY_NS_ALLOCATED = 0x11, + + /* + * Get list of controllers starting at CDW10.CNTID + * that are attached to CDW1.NSID. + */ + NVME_IDENTIFY_NS_ATTACHED_CTRLR_LIST = 0x12, + + /* + * Get list of controllers starting at CDW10.CNTID. + */ + NVME_IDENTIFY_CTRLR_LIST = 0x13, +}; + +/* + * NVMe over Fabrics controller model. + */ +enum nvmf_ctrlr_model { + + /* + * NVM subsystem uses dynamic controller model. + */ + NVMF_CTRLR_MODEL_DYNAMIC = 0, + + /* + * NVM subsystem uses static controller model. + */ + NVMF_CTRLR_MODEL_STATIC = 1, + +}; + +struct __attribute__((packed)) nvme_ctrlr_data { + + /* Bytes 0-255: controller capabilities and features */ + + /* + * PCI vendor id. + */ + uint16_t vid; + + /* + * PCI subsystem vendor id. + */ + uint16_t ssvid; + + /* + * Serial number. + */ + int8_t sn[NVME_SERIAL_NUMBER_CHARACTERS]; + + /* + * Model number. + */ + int8_t mn[NVME_MODEL_NUMBER_CHARACTERS]; + + /* + * Firmware revision. + */ + uint8_t fr[8]; + + /* + * Recommended arbitration burst. + */ + uint8_t rab; + + /* + * IEEE oui identifier. + */ + uint8_t ieee[3]; + + /* + * Controller multi-path I/O and namespace sharing capabilities. + */ + struct { + uint8_t multi_port : 1; + uint8_t multi_host : 1; + uint8_t sr_iov : 1; + uint8_t reserved : 5; + } cmic; + + /* + * Maximum data transfer size. + */ + uint8_t mdts; + + /* + * Controller ID. + */ + uint16_t cntlid; + + /* + * Version. + */ + union nvme_vs_register ver; + + /* + * RTD3 resume latency. + */ + uint32_t rtd3r; + + /* + * RTD3 entry latency. + */ + uint32_t rtd3e; + + /* + * Optional asynchronous events supported. + */ + uint32_t oaes; + + /* + * Controller attributes. + */ + struct { + uint32_t host_id_exhid_supported: 1; + uint32_t reserved: 31; + } ctratt; + + uint8_t reserved1[156]; + + /* + * Bytes 256-511: admin command set attributes. + */ + + /* + * Optional admin command support. + */ + struct { + /* + * Supports security send/receive commands. + */ + uint16_t security : 1; + + /* + * Supports format nvm command. + */ + uint16_t format : 1; + + /* + * Supports firmware activate/download commands. + */ + uint16_t firmware : 1; + + /* + * Supports ns manage/ns attach commands. + */ + uint16_t ns_manage : 1; + + uint16_t oacs_rsvd : 12; + } oacs; + + /* + * Abort command limit. + */ + uint8_t acl; + + /* + * Asynchronous event request limit. + */ + uint8_t aerl; + + /* + * Firmware updates. + */ + struct { + /* + * First slot is read-only. + */ + uint8_t slot1_ro : 1; + + /* + * Number of firmware slots. + */ + uint8_t num_slots : 3; + + /* + * Support activation without reset. + */ + uint8_t activation_without_reset : 1; + + uint8_t frmw_rsvd : 3; + } frmw; + + /* + * Log page attributes. + */ + struct { + /* + * Per namespace smart/health log page. + */ + uint8_t ns_smart : 1; + /* + * Command effects log page. + */ + uint8_t celp : 1; + /* + * Extended data for get log page. + */ + uint8_t edlp : 1; + uint8_t lpa_rsvd : 5; + } lpa; + + /* + * Error log page entries. + */ + uint8_t elpe; + + /* + * Number of power states supported. + */ + uint8_t npss; + + /* + * Admin vendor specific command configuration. + */ + struct { + /* + * Admin vendor specific commands use disk format. + */ + uint8_t spec_format : 1; + uint8_t avscc_rsvd : 7; + } avscc; + + /* + * Autonomous power state transition attributes. + */ + struct { + uint8_t supported : 1; + uint8_t apsta_rsvd : 7; + } apsta; + + /* + * Warning composite temperature threshold. + */ + uint16_t wctemp; + + /* + * Critical composite temperature threshold. + */ + uint16_t cctemp; + + /* + * Maximum time for firmware activation. + */ + uint16_t mtfa; + + /* + * Host memory buffer preferred size. + */ + uint32_t hmpre; + + /* + * Host memory buffer minimum size. + */ + uint32_t hmmin; + + /* + * Total NVM capacity. + */ + uint64_t tnvmcap[2]; + + /* + * Unallocated NVM capacity. + */ + uint64_t unvmcap[2]; + + /* + * Replay protected memory block support. + */ + struct { + uint8_t num_rpmb_units : 3; + uint8_t auth_method : 3; + uint8_t reserved1 : 2; + + uint8_t reserved2; + + uint8_t total_size; + uint8_t access_size; + } rpmbs; + + uint8_t reserved2[4]; + + uint16_t kas; + + uint8_t reserved3[190]; + + /* + * Bytes 512-703: nvm command set attributes. + */ + + /* + * Submission queue entry size. + */ + struct { + uint8_t min : 4; + uint8_t max : 4; + } sqes; + + /* + * Completion queue entry size. + */ + struct { + uint8_t min : 4; + uint8_t max : 4; + } cqes; + + uint16_t maxcmd; + + /* + * Number of namespaces. + */ + uint32_t nn; + + /* + * Optional nvm command support. + */ + struct { + uint16_t compare : 1; + uint16_t write_unc : 1; + uint16_t dsm : 1; + uint16_t write_zeroes : 1; + uint16_t set_features_save : 1; + uint16_t reservations : 1; + uint16_t reserved : 10; + } oncs; + + /* + * Fused operation support. + */ + uint16_t fuses; + + /* + * Format nvm attributes. + */ + struct { + uint8_t format_all_ns : 1; + uint8_t erase_all_ns : 1; + uint8_t crypto_erase_supported : 1; + uint8_t reserved : 5; + } fna; + + /* + * Volatile write cache. + */ + struct { + uint8_t present : 1; + uint8_t reserved : 7; + } vwc; + + /* + * Atomic write unit normal. + */ + uint16_t awun; + + /* + * Atomic write unit power fail. + */ + uint16_t awupf; + + /* + * NVM vendor specific command configuration. + */ + uint8_t nvscc; + + uint8_t reserved531; + + /* + * Atomic compare & write unit. + */ + uint16_t acwu; + + uint16_t reserved534; + + /* + * SGL support. + */ + struct { + uint32_t supported : 1; + uint32_t reserved0 : 1; + uint32_t keyed_sgl : 1; + uint32_t reserved1 : 13; + uint32_t bit_bucket_descriptor : 1; + uint32_t metadata_pointer : 1; + uint32_t oversized_sgl : 1; + uint32_t metadata_address : 1; + uint32_t sgl_offset : 1; + uint32_t reserved2 : 11; + } sgls; + + uint8_t reserved4[228]; + + uint8_t subnqn[256]; + + uint8_t reserved5[768]; + + /* + * NVMe over Fabrics-specific fields. + */ + struct { + /* + * I/O queue command capsule supported size (16-byte units). + */ + uint32_t ioccsz; + + /* + * I/O queue response capsule supported size (16-byte units). + */ + uint32_t iorcsz; + + /* + * In-capsule data offset (16-byte units). + */ + uint16_t icdoff; + + /* + * Controller attributes: model. + */ + struct { + uint8_t ctrlr_model : 1; + uint8_t reserved : 7; + } ctrattr; + + /* + * Maximum SGL block descriptors (0 = no limit). + */ + uint8_t msdbd; + + uint8_t reserved[244]; + } nvmf_specific; + + /* + * Bytes 2048-3071: power state descriptors. + */ + struct nvme_power_state psd[32]; + + /* + * Bytes 3072-4095: vendor specific. + */ + uint8_t vs[1024]; + +}; +nvme_static_assert(sizeof(struct nvme_ctrlr_data) == 4096, "Incorrect size"); + +struct nvme_ns_data { + + /* + * Namespace size (number of sectors). + */ + uint64_t nsze; + + /* + * Namespace capacity. + */ + uint64_t ncap; + + /* + * Namespace utilization. + */ + uint64_t nuse; + + /* + * Namespace features. + */ + struct { + /* + * Thin provisioning. + */ + uint8_t thin_prov : 1; + uint8_t reserved1 : 7; + } nsfeat; + + /* + * Number of lba formats. + */ + uint8_t nlbaf; + + /* + * Formatted lba size. + */ + struct { + uint8_t format : 4; + uint8_t extended : 1; + uint8_t reserved2 : 3; + } flbas; + + /* + * Metadata capabilities. + */ + struct { + /* + * Metadata can be transferred as part of data prp list. + */ + uint8_t extended : 1; + + /* + * Metadata can be transferred with separate metadata pointer. + */ + uint8_t pointer : 1; + + uint8_t reserved3 : 6; + } mc; + + /* + * End-to-end data protection capabilities. + */ + struct { + /* + * Protection information type 1. + */ + uint8_t pit1 : 1; + + /* + * Protection information type 2. + */ + uint8_t pit2 : 1; + + /* + * Protection information type 3. + */ + uint8_t pit3 : 1; + + /* + * First eight bytes of metadata. + */ + uint8_t md_start : 1; + + /* + * Last eight bytes of metadata. + */ + uint8_t md_end : 1; + } dpc; + + /* + * End-to-end data protection type settings. + */ + struct { + /* + * Protection information type. + */ + uint8_t pit : 3; + + /* + * 1 == protection info transferred at start of metadata. + * 0 == protection info transferred at end of metadata. + */ + uint8_t md_start : 1; + + uint8_t reserved4 : 4; + } dps; + + /* + * Namespace multi-path I/O and namespace sharing capabilities. + */ + struct { + uint8_t can_share : 1; + uint8_t reserved : 7; + } nmic; + + /* + * Reservation capabilities. + */ + union { + struct { + /* + * Supports persist through power loss. + */ + uint8_t persist : 1; + + /* + * Supports write exclusive. + */ + uint8_t write_exclusive : 1; + + /* + * Supports exclusive access. + */ + uint8_t exclusive_access : 1; + + /* + * Supports write exclusive - registrants only. + */ + uint8_t write_exclusive_reg_only : 1; + + /* + * Supports exclusive access - registrants only. + */ + uint8_t exclusive_access_reg_only : 1; + + /* + * Supports write exclusive - all registrants. + */ + uint8_t write_exclusive_all_reg : 1; + + /* + * Supports exclusive access - all registrants. + */ + uint8_t exclusive_access_all_reg : 1; + + uint8_t reserved : 1; + } rescap; + uint8_t raw; + } nsrescap; + + /* + * Format progress indicator. + */ + struct { + uint8_t percentage_remaining : 7; + uint8_t fpi_supported : 1; + } fpi; + + uint8_t reserved33; + + /* + * Namespace atomic write unit normal. + */ + uint16_t nawun; + + /* + * Namespace atomic write unit power fail. + */ + uint16_t nawupf; + + /* + * Namespace atomic compare & write unit. + */ + uint16_t nacwu; + + /* + * Namespace atomic boundary size normal. + */ + uint16_t nabsn; + + /* + * Namespace atomic boundary offset. + */ + uint16_t nabo; + + /* + * Namespace atomic boundary size power fail. + */ + uint16_t nabspf; + + uint16_t reserved46; + + /* + * NVM capacity. + */ + uint64_t nvmcap[2]; + + uint8_t reserved64[40]; + + /* + * Namespace globally unique identifier. + */ + uint8_t nguid[16]; + + /* + * IEEE extended unique identifier. + */ + uint64_t eui64; + + /* + * LBA format support. + */ + struct { + /* + * Metadata size. + */ + uint32_t ms : 16; + + /* + * LBA data size. + */ + uint32_t lbads : 8; + + /* + * Relative performance. + */ + uint32_t rp : 2; + + uint32_t reserved6 : 6; + } lbaf[16]; + + uint8_t reserved6[192]; + + uint8_t vendor_specific[3712]; +}; +nvme_static_assert(sizeof(struct nvme_ns_data) == 4096, "Incorrect size"); + +/* + * Reservation Type Encoding + */ +enum nvme_reservation_type { + + /* 0x00 - reserved */ + + /* + * Write Exclusive Reservation. + */ + NVME_RESERVE_WRITE_EXCLUSIVE = 0x1, + + /* + * Exclusive Access Reservation. + */ + NVME_RESERVE_EXCLUSIVE_ACCESS = 0x2, + + /* + * Write Exclusive - Registrants Only Reservation. + */ + NVME_RESERVE_WRITE_EXCLUSIVE_REG_ONLY = 0x3, + + /* + * Exclusive Access - Registrants Only Reservation. + */ + NVME_RESERVE_EXCLUSIVE_ACCESS_REG_ONLY = 0x4, + + /* + * Write Exclusive - All Registrants Reservation. + */ + NVME_RESERVE_WRITE_EXCLUSIVE_ALL_REGS = 0x5, + + /* + * Exclusive Access - All Registrants Reservation. + */ + NVME_RESERVE_EXCLUSIVE_ACCESS_ALL_REGS = 0x6, + + /* 0x7-0xFF - Reserved */ +}; + +struct nvme_reservation_acquire_data { + + /* + * Current reservation key. + */ + uint64_t crkey; + + /* + * Preempt reservation key. + */ + uint64_t prkey; + +}; +nvme_static_assert(sizeof(struct nvme_reservation_acquire_data) == 16, + "Incorrect size"); + +/* + * Reservation Acquire action + */ +enum nvme_reservation_acquire_action { + NVME_RESERVE_ACQUIRE = 0x0, + NVME_RESERVE_PREEMPT = 0x1, + NVME_RESERVE_PREEMPT_ABORT = 0x2, +}; + +struct __attribute__((packed)) nvme_reservation_status_data { + + /* + * Reservation action generation counter. + */ + uint32_t generation; + + /* + * Reservation type. + */ + uint8_t type; + + /* + * Number of registered controllers. + */ + uint16_t nr_regctl; + uint16_t reserved1; + + /* + * Persist through power loss state. + */ + uint8_t ptpl_state; + uint8_t reserved[14]; + +}; +nvme_static_assert(sizeof(struct nvme_reservation_status_data) == 24, + "Incorrect size"); + +struct __attribute__((packed)) nvme_reservation_ctrlr_data { + + uint16_t ctrlr_id; + + /* + * Reservation status. + */ + struct { + uint8_t status : 1; + uint8_t reserved1 : 7; + } rcsts; + uint8_t reserved2[5]; + + /* + * Host identifier. + */ + uint64_t host_id; + + /* + * Reservation key. + */ + uint64_t key; +}; +nvme_static_assert(sizeof(struct nvme_reservation_ctrlr_data) == 24, + "Incorrect size"); + +/* + * Change persist through power loss state for + * Reservation Register command + */ +enum nvme_reservation_register_cptpl { + NVME_RESERVE_PTPL_NO_CHANGES = 0x0, + NVME_RESERVE_PTPL_CLEAR_POWER_ON = 0x2, + NVME_RESERVE_PTPL_PERSIST_POWER_LOSS = 0x3, +}; + +/* + * Registration action for Reservation Register command + */ +enum nvme_reservation_register_action { + NVME_RESERVE_REGISTER_KEY = 0x0, + NVME_RESERVE_UNREGISTER_KEY = 0x1, + NVME_RESERVE_REPLACE_KEY = 0x2, +}; + +struct nvme_reservation_register_data { + + /* + * Current reservation key. + */ + uint64_t crkey; + + /* + * New reservation key. + */ + uint64_t nrkey; + +}; +nvme_static_assert(sizeof(struct nvme_reservation_register_data) == 16, + "Incorrect size"); + +struct nvme_reservation_key_data { + + /* + * Current reservation key. + */ + uint64_t crkey; + +}; +nvme_static_assert(sizeof(struct nvme_reservation_key_data) == 8, + "Incorrect size"); + +/* + * Reservation Release action + */ +enum nvme_reservation_release_action { + NVME_RESERVE_RELEASE = 0x0, + NVME_RESERVE_CLEAR = 0x1, +}; + +/* + * Log page identifiers for NVME_OPC_GET_LOG_PAGE + */ +enum nvme_log_page { + + /* 0x00 - reserved */ + + /* + * Error information (mandatory). + */ + NVME_LOG_ERROR = 0x01, + + /* + * SMART / health information (mandatory). + */ + NVME_LOG_HEALTH_INFORMATION = 0x02, + + /* + * Firmware slot information (mandatory). + */ + NVME_LOG_FIRMWARE_SLOT = 0x03, + + /* + * Changed namespace list (optional). + */ + NVME_LOG_CHANGED_NS_LIST = 0x04, + + /* + * Command effects log (optional). + */ + NVME_LOG_COMMAND_EFFECTS_LOG = 0x05, + + /* 0x06-0x6F - reserved */ + + /* + * Discovery(refer to the NVMe over Fabrics specification). + */ + NVME_LOG_DISCOVERY = 0x70, + + /* 0x71-0x7f - reserved for NVMe over Fabrics */ + + /* + * Reservation notification (optional). + */ + NVME_LOG_RESERVATION_NOTIFICATION = 0x80, + + /* 0x81-0xBF - I/O command set specific */ + + /* 0xC0-0xFF - vendor specific */ +}; + +/* + * Error information log page (\ref NVME_LOG_ERROR) + */ +struct nvme_error_information_entry { + uint64_t error_count; + uint16_t sqid; + uint16_t cid; + struct nvme_status status; + uint16_t error_location; + uint64_t lba; + uint32_t nsid; + uint8_t vendor_specific; + uint8_t reserved[35]; +}; +nvme_static_assert(sizeof(struct nvme_error_information_entry) == 64, + "Incorrect size"); + +union nvme_critical_warning_state { + + uint8_t raw; + + struct { + uint8_t available_spare : 1; + uint8_t temperature : 1; + uint8_t device_reliability : 1; + uint8_t read_only : 1; + uint8_t volatile_memory_backup : 1; + uint8_t reserved : 3; + } bits; + +}; +nvme_static_assert(sizeof(union nvme_critical_warning_state) == 1, + "Incorrect size"); + +/* + * SMART / health information page (\ref NVME_LOG_HEALTH_INFORMATION) + */ +struct __attribute__((packed)) nvme_health_information_page { + + union nvme_critical_warning_state critical_warning; + + uint16_t temperature; + uint8_t available_spare; + uint8_t available_spare_threshold; + uint8_t percentage_used; + + uint8_t reserved[26]; + + /* + * Note that the following are 128-bit values, but are + * defined as an array of 2 64-bit values. + */ + + /* + * Data Units Read is always in 512-byte units. + */ + uint64_t data_units_read[2]; + + /* + * Data Units Written is always in 512-byte units. + */ + uint64_t data_units_written[2]; + + /* + * For NVM command set, this includes Compare commands. + */ + uint64_t host_read_commands[2]; + uint64_t host_write_commands[2]; + + /* + * Controller Busy Time is reported in minutes. + */ + uint64_t controller_busy_time[2]; + uint64_t power_cycles[2]; + uint64_t power_on_hours[2]; + uint64_t unsafe_shutdowns[2]; + uint64_t media_errors[2]; + uint64_t num_error_info_log_entries[2]; + + uint8_t reserved2[320]; +}; +nvme_static_assert(sizeof(struct nvme_health_information_page) == 512, + "Incorrect size"); + +/* + * Firmware slot information page (\ref NVME_LOG_FIRMWARE_SLOT) + */ +struct nvme_firmware_page { + + struct { + /* + * Slot for current FW. + */ + uint8_t slot : 3; + uint8_t reserved : 5; + } afi; + + uint8_t reserved[7]; + + /* + * Revisions for 7 slots. + */ + uint64_t revision[7]; + + uint8_t reserved2[448]; + +}; +nvme_static_assert(sizeof(struct nvme_firmware_page) == 512, + "Incorrect size"); + +/* + * Namespace attachment Type Encoding + */ +enum nvme_ns_attach_type { + + /* + * Controller attach. + */ + NVME_NS_CTRLR_ATTACH = 0x0, + + /* + * Controller detach. + */ + NVME_NS_CTRLR_DETACH = 0x1, + + /* 0x2-0xF - Reserved */ + +}; + +/* + * Namespace management Type Encoding + */ +enum nvme_ns_management_type { + + /* + * Create. + */ + NVME_NS_MANAGEMENT_CREATE = 0x0, + + /* + * Delete. + */ + NVME_NS_MANAGEMENT_DELETE = 0x1, + + /* 0x2-0xF - Reserved */ + +}; + +struct nvme_ns_list { + uint32_t ns_list[NVME_MAX_NS]; +}; +nvme_static_assert(sizeof(struct nvme_ns_list) == 4096, "Incorrect size"); + +struct nvme_ctrlr_list { + uint16_t ctrlr_count; + uint16_t ctrlr_list[2047]; +}; +nvme_static_assert(sizeof(struct nvme_ctrlr_list) == 4096, "Incorrect size"); + +enum nvme_secure_erase_setting { + NVME_FMT_NVM_SES_NO_SECURE_ERASE = 0x0, + NVME_FMT_NVM_SES_USER_DATA_ERASE = 0x1, + NVME_FMT_NVM_SES_CRYPTO_ERASE = 0x2, +}; + +enum nvme_pi_location { + NVME_FMT_NVM_PROTECTION_AT_TAIL = 0x0, + NVME_FMT_NVM_PROTECTION_AT_HEAD = 0x1, +}; + +enum nvme_pi_type { + NVME_FMT_NVM_PROTECTION_DISABLE = 0x0, + NVME_FMT_NVM_PROTECTION_TYPE1 = 0x1, + NVME_FMT_NVM_PROTECTION_TYPE2 = 0x2, + NVME_FMT_NVM_PROTECTION_TYPE3 = 0x3, +}; + +enum nvme_metadata_setting { + NVME_FMT_NVM_METADATA_TRANSFER_AS_BUFFER = 0x0, + NVME_FMT_NVM_METADATA_TRANSFER_AS_LBA = 0x1, +}; + +struct nvme_format { + uint32_t lbaf : 4; + uint32_t ms : 1; + uint32_t pi : 3; + uint32_t pil : 1; + uint32_t ses : 3; + uint32_t reserved : 20; +}; +nvme_static_assert(sizeof(struct nvme_format) == 4, "Incorrect size"); + +struct nvme_protection_info { + uint16_t guard; + uint16_t app_tag; + uint32_t ref_tag; +}; +nvme_static_assert(sizeof(struct nvme_protection_info) == 8, "Incorrect size"); + +/* + * Parameters for NVME_OPC_FIRMWARE_COMMIT cdw10: commit action. + */ +enum nvme_fw_commit_action { + + /* + * Downloaded image replaces the image specified by + * the Firmware Slot field. This image is not activated. + */ + NVME_FW_COMMIT_REPLACE_IMG = 0x0, + + /* + * Downloaded image replaces the image specified by + * the Firmware Slot field. This image is activated at the next reset. + */ + NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG = 0x1, + + /* + * The image specified by the Firmware Slot field is + * activated at the next reset. + */ + NVME_FW_COMMIT_ENABLE_IMG = 0x2, + + /* + * The image specified by the Firmware Slot field is + * requested to be activated immediately without reset. + */ + NVME_FW_COMMIT_RUN_IMG = 0x3, + +}; + +/* + * Parameters for NVME_OPC_FIRMWARE_COMMIT cdw10. + */ +struct nvme_fw_commit { + + /* + * Firmware Slot. Specifies the firmware slot that shall be used for the + * Commit Action. The controller shall choose the firmware slot (slot 1 - 7) + * to use for the operation if the value specified is 0h. + */ + uint32_t fs : 3; + + /* + * Commit Action. Specifies the action that is taken on the image downloaded + * with the Firmware Image Download command or on a previously downloaded and + * placed image. + */ + uint32_t ca : 3; + + uint32_t reserved : 26; + +}; +nvme_static_assert(sizeof(struct nvme_fw_commit) == 4, "Incorrect size"); + +#define nvme_cpl_is_error(cpl) \ + ((cpl)->status.sc != 0 || (cpl)->status.sct != 0) + +/* + * Enable protection information checking of the + * Logical Block Reference Tag field. + */ +#define NVME_IO_FLAGS_PRCHK_REFTAG (1U << 26) + +/* + * Enable protection information checking of the + * Application Tag field. + */ +#define NVME_IO_FLAGS_PRCHK_APPTAG (1U << 27) + +/* + * Enable protection information checking of the Guard field. + */ +#define NVME_IO_FLAGS_PRCHK_GUARD (1U << 28) + +/* + * Strip or insert (when set) the protection information. + */ +#define NVME_IO_FLAGS_PRACT (1U << 29) + +/* + * Bypass device cache. + */ +#define NVME_IO_FLAGS_FORCE_UNIT_ACCESS (1U << 30) + +/* + * Limit retries on error. + */ +#define NVME_IO_FLAGS_LIMITED_RETRY (1U << 31) + +#endif /* define __LIBNVME_SPEC_H__ */ +