libnvme: Import.
Minor modifications for Haiku included, and for the most part marked as such.
This commit is contained in:
Augustin Cavalier
@@ -0,0 +1,385 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation. All rights reserved.
|
||||
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "nvme_internal.h"
|
||||
|
||||
/*
|
||||
* List of open controllers and its lock.
|
||||
*/
|
||||
LIST_HEAD(, nvme_ctrlr) ctrlr_head = LIST_HEAD_INITIALIZER(ctrlr_head);
|
||||
static pthread_mutex_t ctrlr_lock = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
/*
|
||||
* Search for an open controller.
|
||||
*/
|
||||
static struct nvme_ctrlr *nvme_ctrlr_get(struct nvme_ctrlr *ctrlr,
|
||||
bool remove)
|
||||
{
|
||||
struct nvme_ctrlr *c;
|
||||
|
||||
pthread_mutex_lock(&ctrlr_lock);
|
||||
|
||||
LIST_FOREACH(c, &ctrlr_head, link) {
|
||||
if (c == ctrlr) {
|
||||
if (remove)
|
||||
LIST_REMOVE(c, link);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
ctrlr = NULL;
|
||||
|
||||
out:
|
||||
pthread_mutex_unlock(&ctrlr_lock);
|
||||
|
||||
return ctrlr;
|
||||
}
|
||||
|
||||
#ifndef __HAIKU__
|
||||
/*
|
||||
* Probe a pci device identified by its name.
|
||||
* Name should be in the form: [0000:]00:00.0
|
||||
* Return NULL if failed
|
||||
*/
|
||||
static struct pci_device *nvme_pci_ctrlr_probe(const char *slot_name)
|
||||
{
|
||||
char *domain = NULL, *bus = NULL, *dev = NULL, *func = NULL, *end = NULL;
|
||||
char *pciid = strdup(slot_name);
|
||||
struct pci_slot_match slot;
|
||||
struct pci_device *pci_dev = NULL;
|
||||
|
||||
if (!pciid)
|
||||
return NULL;
|
||||
|
||||
memset(&slot, 0, sizeof(struct pci_slot_match));
|
||||
|
||||
func = strrchr(pciid, '.');
|
||||
if (func) {
|
||||
*func = '\0';
|
||||
func++;
|
||||
}
|
||||
|
||||
dev = strrchr(pciid, ':');
|
||||
if (dev) {
|
||||
*dev = '\0';
|
||||
dev++;
|
||||
}
|
||||
|
||||
bus = strrchr(pciid, ':');
|
||||
if (!bus) {
|
||||
domain = NULL;
|
||||
bus = pciid;
|
||||
} else {
|
||||
domain = pciid;
|
||||
*bus = '\0';
|
||||
bus++;
|
||||
}
|
||||
|
||||
if (!bus || !dev || !func) {
|
||||
nvme_err("Malformed PCI device slot name %s\n",
|
||||
slot_name);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (domain) {
|
||||
slot.domain = (uint32_t)strtoul(domain, &end, 16);
|
||||
if ((end && *end) || (slot.domain > 0xffff)) {
|
||||
nvme_err("Invalid domain number: 0x%X\n", slot.domain);
|
||||
return NULL;
|
||||
}
|
||||
} else {
|
||||
slot.domain = PCI_MATCH_ANY;
|
||||
}
|
||||
|
||||
slot.bus = (uint32_t)strtoul(bus, &end, 16);
|
||||
if ((end && *end) || (slot.bus > 0xff)) {
|
||||
nvme_err("Invalid bus number: 0x%X\n", slot.bus);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
slot.dev = strtoul(dev, &end, 16);
|
||||
if ((end && *end) || (slot.dev > 0x1f)) {
|
||||
nvme_err("Invalid device number: 0x%X\n", slot.dev);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
slot.func = strtoul(func, &end, 16);
|
||||
if ((end && *end) || (slot.func > 7)) {
|
||||
nvme_err("Invalid function number: 0x%X\n", slot.func);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
nvme_debug("PCI URL: domain 0x%X, bus 0x%X, dev 0x%X, func 0x%X\n",
|
||||
slot.domain, slot.bus, slot.dev, slot.func);
|
||||
|
||||
pci_dev = nvme_pci_device_probe(&slot);
|
||||
if (pci_dev) {
|
||||
slot.domain = pci_dev->domain;
|
||||
if (slot.domain == PCI_MATCH_ANY)
|
||||
slot.domain = 0;
|
||||
nvme_info("Found NVMe controller %04x:%02x:%02x.%1u\n",
|
||||
slot.domain,
|
||||
slot.bus,
|
||||
slot.dev,
|
||||
slot.func);
|
||||
}
|
||||
|
||||
out:
|
||||
free(pciid);
|
||||
|
||||
return pci_dev;
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Open an NVMe controller.
|
||||
*/
|
||||
#ifdef __HAIKU__
|
||||
struct nvme_ctrlr *nvme_ctrlr_open(struct pci_device *pdev,
|
||||
struct nvme_ctrlr_opts *opts)
|
||||
#else
|
||||
struct nvme_ctrlr *nvme_ctrlr_open(const char *url,
|
||||
struct nvme_ctrlr_opts *opts)
|
||||
#endif
|
||||
{
|
||||
struct nvme_ctrlr *ctrlr;
|
||||
#ifndef __HAIKU__
|
||||
char *slot;
|
||||
|
||||
/* Check url */
|
||||
if (strncmp(url, "pci://", 6) != 0) {
|
||||
nvme_err("Invalid URL %s\n", url);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Probe PCI device */
|
||||
slot = (char *)url + 6;
|
||||
pdev = nvme_pci_ctrlr_probe(slot);
|
||||
if (!pdev) {
|
||||
nvme_err("Device %s not found\n", url);
|
||||
return NULL;
|
||||
}
|
||||
#endif
|
||||
|
||||
pthread_mutex_lock(&ctrlr_lock);
|
||||
|
||||
/* Verify that this controller is not already open */
|
||||
LIST_FOREACH(ctrlr, &ctrlr_head, link) {
|
||||
if (nvme_pci_dev_cmp(ctrlr->pci_dev, pdev) == 0) {
|
||||
nvme_err("Controller already open\n");
|
||||
ctrlr = NULL;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
/* Attach the device */
|
||||
ctrlr = nvme_ctrlr_attach(pdev, opts);
|
||||
if (!ctrlr) {
|
||||
nvme_err("Attach failed\n");
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* Add controller to the list */
|
||||
LIST_INSERT_HEAD(&ctrlr_head, ctrlr, link);
|
||||
|
||||
out:
|
||||
pthread_mutex_unlock(&ctrlr_lock);
|
||||
|
||||
return ctrlr;
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* Close an open controller.
|
||||
*/
|
||||
int nvme_ctrlr_close(struct nvme_ctrlr *ctrlr)
|
||||
{
|
||||
|
||||
/*
|
||||
* Verify that this controller is open.
|
||||
* If it is, remove it from the list.
|
||||
*/
|
||||
ctrlr = nvme_ctrlr_get(ctrlr, true);
|
||||
if (!ctrlr) {
|
||||
nvme_err("Invalid controller\n");
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
nvme_ctrlr_detach(ctrlr);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get controller information.
|
||||
*/
|
||||
int nvme_ctrlr_stat(struct nvme_ctrlr *ctrlr, struct nvme_ctrlr_stat *cstat)
|
||||
{
|
||||
struct pci_device *pdev = ctrlr->pci_dev;
|
||||
unsigned int i;
|
||||
|
||||
/* Verify that this controller is open */
|
||||
ctrlr = nvme_ctrlr_get(ctrlr, false);
|
||||
if (!ctrlr) {
|
||||
nvme_err("Invalid controller\n");
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&ctrlr->lock);
|
||||
|
||||
memset(cstat, 0, sizeof(struct nvme_ctrlr_stat));
|
||||
|
||||
/* Controller serial and model number */
|
||||
strncpy(cstat->sn, (char *)ctrlr->cdata.sn,
|
||||
NVME_SERIAL_NUMBER_LENGTH - 1);
|
||||
strncpy(cstat->mn, (char *)ctrlr->cdata.mn,
|
||||
NVME_MODEL_NUMBER_LENGTH - 1);
|
||||
|
||||
/* Remove heading and trailling spaces */
|
||||
nvme_str_trim(cstat->sn);
|
||||
nvme_str_trim(cstat->mn);
|
||||
|
||||
/* PCI device info */
|
||||
cstat->vendor_id = pdev->vendor_id;
|
||||
cstat->device_id = pdev->device_id;
|
||||
cstat->subvendor_id = pdev->subvendor_id;
|
||||
cstat->subdevice_id = pdev->subdevice_id;
|
||||
#ifndef __HAIKU__
|
||||
cstat->device_class = pdev->device_class;
|
||||
cstat->revision = pdev->revision;
|
||||
cstat->domain = pdev->domain;
|
||||
cstat->bus = pdev->bus;
|
||||
cstat->dev = pdev->dev;
|
||||
cstat->func = pdev->func;
|
||||
#endif
|
||||
|
||||
/* Maximum transfer size */
|
||||
cstat->max_xfer_size = ctrlr->max_xfer_size;
|
||||
|
||||
memcpy(&cstat->features, &ctrlr->feature_supported,
|
||||
sizeof(ctrlr->feature_supported));
|
||||
memcpy(&cstat->log_pages, &ctrlr->log_page_supported,
|
||||
sizeof(ctrlr->log_page_supported));
|
||||
|
||||
cstat->nr_ns = ctrlr->nr_ns;
|
||||
for (i = 0; i < ctrlr->nr_ns; i++) {
|
||||
cstat->ns_ids[i] = i + 1;
|
||||
}
|
||||
|
||||
/* Maximum io qpair possible */
|
||||
cstat->max_io_qpairs = ctrlr->max_io_queues;
|
||||
|
||||
/* Constructed io qpairs */
|
||||
cstat->io_qpairs = ctrlr->io_queues;
|
||||
|
||||
/* Enabled io qpairs */
|
||||
cstat->enabled_io_qpairs = ctrlr->enabled_io_qpairs;
|
||||
|
||||
/* Max queue depth */
|
||||
cstat->max_qd = ctrlr->io_qpairs_max_entries;
|
||||
|
||||
pthread_mutex_unlock(&ctrlr->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get controller data
|
||||
*/
|
||||
int nvme_ctrlr_data(struct nvme_ctrlr *ctrlr, struct nvme_ctrlr_data *cdata,
|
||||
struct nvme_register_data *rdata)
|
||||
{
|
||||
union nvme_cap_register cap;
|
||||
|
||||
/* Verify that this controller is open */
|
||||
ctrlr = nvme_ctrlr_get(ctrlr, false);
|
||||
if (!ctrlr) {
|
||||
nvme_err("Invalid controller\n");
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&ctrlr->lock);
|
||||
|
||||
/* Controller data */
|
||||
if (cdata)
|
||||
memcpy(cdata, &ctrlr->cdata, sizeof(struct nvme_ctrlr_data));
|
||||
|
||||
/* Read capabilities register */
|
||||
if (rdata) {
|
||||
cap.raw = nvme_reg_mmio_read_8(ctrlr, cap.raw);
|
||||
rdata->mqes = cap.bits.mqes;
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&ctrlr->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get qpair information
|
||||
*/
|
||||
int nvme_qpair_stat(struct nvme_qpair *qpair, struct nvme_qpair_stat *qpstat)
|
||||
{
|
||||
struct nvme_ctrlr *ctrlr = qpair->ctrlr;
|
||||
|
||||
/* Verify that this controller is open */
|
||||
ctrlr = nvme_ctrlr_get(ctrlr, false);
|
||||
if (!ctrlr) {
|
||||
nvme_err("Invalid controller\n");
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&ctrlr->lock);
|
||||
|
||||
qpstat->id = qpair->id;
|
||||
qpstat->qd = qpair->entries;
|
||||
qpstat->enabled = qpair->enabled;
|
||||
qpstat->qprio = qpair->qprio;
|
||||
|
||||
pthread_mutex_unlock(&ctrlr->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Close all open controllers on exit.
|
||||
*/
|
||||
void nvme_ctrlr_cleanup(void)
|
||||
{
|
||||
struct nvme_ctrlr *ctrlr;
|
||||
|
||||
while ((ctrlr = LIST_FIRST(&ctrlr_head))) {
|
||||
LIST_REMOVE(ctrlr, link);
|
||||
nvme_ctrlr_detach(ctrlr);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,456 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation. All rights reserved.
|
||||
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* Redistribution and use in sourete and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of sourete code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "nvme_internal.h"
|
||||
|
||||
/*
|
||||
* Allocate a request, set its command and submit it
|
||||
* to the controller admin queue.
|
||||
*/
|
||||
static int nvme_admin_submit_cmd(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_cmd *cmd,
|
||||
void *buf, uint32_t len,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
|
||||
if (buf)
|
||||
req = nvme_request_allocate_contig(&ctrlr->adminq, buf, len,
|
||||
cb_fn, cb_arg);
|
||||
else
|
||||
req = nvme_request_allocate_null(&ctrlr->adminq, cb_fn, cb_arg);
|
||||
if (!req)
|
||||
return ENOMEM;
|
||||
|
||||
memcpy(&req->cmd, cmd, sizeof(req->cmd));
|
||||
|
||||
return nvme_qpair_submit_request(&ctrlr->adminq, req);
|
||||
}
|
||||
|
||||
/*
|
||||
* Poll the controller admin queue waiting for a
|
||||
* command completion.
|
||||
*/
|
||||
static int nvme_admin_wait_cmd(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_completion_poll_status *status)
|
||||
{
|
||||
|
||||
/* Wait for completion and check result */
|
||||
while (status->done == false)
|
||||
nvme_qpair_poll(&ctrlr->adminq, 0);
|
||||
|
||||
if (nvme_cpl_is_error(&status->cpl)) {
|
||||
nvme_notice("Admin command failed\n");
|
||||
return ENXIO;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Execute an admin command.
|
||||
*/
|
||||
static int nvme_admin_exec_cmd(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_cmd *cmd,
|
||||
void *buf, uint32_t len)
|
||||
{
|
||||
struct nvme_completion_poll_status status;
|
||||
int ret;
|
||||
|
||||
/* Submit the command */
|
||||
status.done = false;
|
||||
ret = nvme_admin_submit_cmd(ctrlr, cmd, buf, len,
|
||||
nvme_request_completion_poll_cb,
|
||||
&status);
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
|
||||
/* Wait for the command completion and check result */
|
||||
return nvme_admin_wait_cmd(ctrlr, &status);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get a controller information.
|
||||
*/
|
||||
int nvme_admin_identify_ctrlr(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_ctrlr_data *cdata)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_IDENTIFY;
|
||||
cmd.cdw10 = NVME_IDENTIFY_CTRLR;
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd,
|
||||
cdata, sizeof(struct nvme_ctrlr_data));
|
||||
}
|
||||
|
||||
/*
|
||||
* Get a controller feature.
|
||||
*/
|
||||
int nvme_admin_get_feature(struct nvme_ctrlr *ctrlr,
|
||||
enum nvme_feat_sel sel,
|
||||
enum nvme_feat feature,
|
||||
uint32_t cdw11,
|
||||
uint32_t *attributes)
|
||||
{
|
||||
struct nvme_completion_poll_status status;
|
||||
struct nvme_cmd cmd;
|
||||
int ret;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_GET_FEATURES;
|
||||
cmd.cdw10 = (sel << 8) | feature;
|
||||
cmd.cdw11 = cdw11;
|
||||
|
||||
/* Submit the command */
|
||||
status.done = false;
|
||||
ret = nvme_admin_submit_cmd(ctrlr, &cmd, NULL, 0,
|
||||
nvme_request_completion_poll_cb,
|
||||
&status);
|
||||
if (ret == 0) {
|
||||
/* Wait for the command completion and check result */
|
||||
ret = nvme_admin_wait_cmd(ctrlr, &status);
|
||||
if (ret == 0 && attributes)
|
||||
*attributes = status.cpl.cdw0;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Set a feature.
|
||||
*/
|
||||
int nvme_admin_set_feature(struct nvme_ctrlr *ctrlr,
|
||||
bool save,
|
||||
enum nvme_feat feature,
|
||||
uint32_t cdw11,
|
||||
uint32_t cdw12,
|
||||
uint32_t *attributes)
|
||||
{
|
||||
struct nvme_completion_poll_status status;
|
||||
struct nvme_cmd cmd;
|
||||
int ret;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_SET_FEATURES;
|
||||
cmd.cdw10 = feature;
|
||||
if (save)
|
||||
cmd.cdw10 |= (1 << 31);
|
||||
cmd.cdw11 = cdw11;
|
||||
cmd.cdw12 = cdw12;
|
||||
|
||||
/* Submit the command */
|
||||
status.done = false;
|
||||
ret = nvme_admin_submit_cmd(ctrlr, &cmd, NULL, 0,
|
||||
nvme_request_completion_poll_cb,
|
||||
&status);
|
||||
if (ret == 0) {
|
||||
/* Wait for the command completion and check result */
|
||||
ret = nvme_admin_wait_cmd(ctrlr, &status);
|
||||
if (ret == 0 && attributes)
|
||||
*attributes = status.cpl.cdw0;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Create an I/O queue.
|
||||
*/
|
||||
int nvme_admin_create_ioq(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_qpair *qpair,
|
||||
enum nvme_io_queue_type io_qtype)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
switch(io_qtype) {
|
||||
case NVME_IO_SUBMISSION_QUEUE:
|
||||
cmd.opc = NVME_OPC_CREATE_IO_SQ;
|
||||
cmd.cdw11 = (qpair->id << 16) | (qpair->qprio << 1) | 0x1;
|
||||
cmd.dptr.prp.prp1 = qpair->cmd_bus_addr;
|
||||
break;
|
||||
case NVME_IO_COMPLETION_QUEUE:
|
||||
cmd.opc = NVME_OPC_CREATE_IO_CQ;
|
||||
cmd.cdw11 = 0x1;
|
||||
cmd.dptr.prp.prp1 = qpair->cpl_bus_addr;
|
||||
break;
|
||||
default:
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
cmd.cdw10 = ((qpair->entries - 1) << 16) | qpair->id;
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Delete an I/O queue.
|
||||
*/
|
||||
int nvme_admin_delete_ioq(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_qpair *qpair,
|
||||
enum nvme_io_queue_type io_qtype)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
switch(io_qtype) {
|
||||
case NVME_IO_SUBMISSION_QUEUE:
|
||||
cmd.opc = NVME_OPC_DELETE_IO_SQ;
|
||||
break;
|
||||
case NVME_IO_COMPLETION_QUEUE:
|
||||
cmd.opc = NVME_OPC_DELETE_IO_CQ;
|
||||
break;
|
||||
default:
|
||||
return EINVAL;
|
||||
}
|
||||
cmd.cdw10 = qpair->id;
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get a namespace information.
|
||||
*/
|
||||
int nvme_admin_identify_ns(struct nvme_ctrlr *ctrlr,
|
||||
uint16_t nsid,
|
||||
struct nvme_ns_data *nsdata)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_IDENTIFY;
|
||||
cmd.cdw10 = NVME_IDENTIFY_NS;
|
||||
cmd.nsid = nsid;
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd,
|
||||
nsdata, sizeof(struct nvme_ns_data));
|
||||
}
|
||||
|
||||
/*
|
||||
* Attach a namespace.
|
||||
*/
|
||||
int nvme_admin_attach_ns(struct nvme_ctrlr *ctrlr,
|
||||
uint32_t nsid,
|
||||
struct nvme_ctrlr_list *clist)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_NS_ATTACHMENT;
|
||||
cmd.nsid = nsid;
|
||||
cmd.cdw10 = NVME_NS_CTRLR_ATTACH;
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd,
|
||||
clist, sizeof(struct nvme_ctrlr_list));
|
||||
}
|
||||
|
||||
/*
|
||||
* Detach a namespace.
|
||||
*/
|
||||
int nvme_admin_detach_ns(struct nvme_ctrlr *ctrlr,
|
||||
uint32_t nsid,
|
||||
struct nvme_ctrlr_list *clist)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_NS_ATTACHMENT;
|
||||
cmd.nsid = nsid;
|
||||
cmd.cdw10 = NVME_NS_CTRLR_DETACH;
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd,
|
||||
clist, sizeof(struct nvme_ctrlr_list));
|
||||
}
|
||||
|
||||
/*
|
||||
* Create a namespace.
|
||||
*/
|
||||
int nvme_admin_create_ns(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_ns_data *nsdata,
|
||||
unsigned int *nsid)
|
||||
{
|
||||
struct nvme_completion_poll_status status;
|
||||
struct nvme_cmd cmd;
|
||||
int ret;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_NS_MANAGEMENT;
|
||||
cmd.cdw10 = NVME_NS_MANAGEMENT_CREATE;
|
||||
|
||||
/* Submit the command */
|
||||
status.done = false;
|
||||
ret = nvme_admin_submit_cmd(ctrlr, &cmd,
|
||||
nsdata, sizeof(struct nvme_ns_data),
|
||||
nvme_request_completion_poll_cb,
|
||||
&status);
|
||||
if (ret == 0)
|
||||
/* Wait for the command completion and check result */
|
||||
ret = nvme_admin_wait_cmd(ctrlr, &status);
|
||||
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
|
||||
*nsid = status.cpl.cdw0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Delete a namespace.
|
||||
*/
|
||||
int nvme_admin_delete_ns(struct nvme_ctrlr *ctrlr,
|
||||
unsigned int nsid)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_NS_MANAGEMENT;
|
||||
cmd.cdw10 = NVME_NS_MANAGEMENT_DELETE;
|
||||
cmd.nsid = nsid;
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Format media.
|
||||
* (entire device or just the specified namespace)
|
||||
*/
|
||||
int nvme_admin_format_nvm(struct nvme_ctrlr *ctrlr,
|
||||
unsigned int nsid,
|
||||
struct nvme_format *format)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_FORMAT_NVM;
|
||||
cmd.nsid = nsid;
|
||||
memcpy(&cmd.cdw10, format, sizeof(uint32_t));
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get a log page.
|
||||
*/
|
||||
int nvme_admin_get_log_page(struct nvme_ctrlr *ctrlr,
|
||||
uint8_t log_page,
|
||||
uint32_t nsid,
|
||||
void *payload,
|
||||
uint32_t payload_size)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_GET_LOG_PAGE;
|
||||
cmd.nsid = nsid;
|
||||
cmd.cdw10 = ((payload_size / sizeof(uint32_t)) - 1) << 16;
|
||||
cmd.cdw10 |= log_page;
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd, payload, payload_size);
|
||||
}
|
||||
|
||||
/*
|
||||
* Abort an admin or an I/O command.
|
||||
*/
|
||||
int nvme_admin_abort_cmd(struct nvme_ctrlr *ctrlr,
|
||||
uint16_t cid, uint16_t sqid)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_ABORT;
|
||||
cmd.cdw10 = (cid << 16) | sqid;
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Validate a FW.
|
||||
*/
|
||||
int nvme_admin_fw_commit(struct nvme_ctrlr *ctrlr,
|
||||
const struct nvme_fw_commit *fw_commit)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_FIRMWARE_COMMIT;
|
||||
memcpy(&cmd.cdw10, fw_commit, sizeof(uint32_t));
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd, NULL, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Download to the device a firmware.
|
||||
*/
|
||||
int nvme_admin_fw_image_dl(struct nvme_ctrlr *ctrlr,
|
||||
void *fw, uint32_t size,
|
||||
uint32_t offset)
|
||||
{
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/* Setup the command */
|
||||
memset(&cmd, 0, sizeof(struct nvme_cmd));
|
||||
cmd.opc = NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD;
|
||||
cmd.cdw10 = (size >> 2) - 1;
|
||||
cmd.cdw11 = offset >> 2;
|
||||
|
||||
/* Execute the command */
|
||||
return nvme_admin_exec_cmd(ctrlr, &cmd, fw, size);
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
|
||||
* Copyright(c) 2012-2014 6WIND S.A. All rights reserved.
|
||||
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "nvme_pci.h"
|
||||
#include "nvme_common.h"
|
||||
#include "nvme_mem.h"
|
||||
#ifndef __HAIKU__
|
||||
#include "nvme_cpu.h"
|
||||
#endif
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <sys/file.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#if defined(NVME_ARCH_X86)
|
||||
#include <sys/io.h>
|
||||
#endif
|
||||
#include <sys/ioctl.h>
|
||||
#include <sys/types.h>
|
||||
#ifndef __HAIKU__
|
||||
#include <linux/fs.h>
|
||||
#include <execinfo.h>
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Trim whitespace from a string in place.
|
||||
*/
|
||||
void nvme_str_trim(char *s)
|
||||
{
|
||||
char *p, *q;
|
||||
|
||||
/* Remove header */
|
||||
p = s;
|
||||
while (*p != '\0' && isspace(*p))
|
||||
p++;
|
||||
|
||||
/* Remove tailer */
|
||||
q = p + strlen(p);
|
||||
while (q - 1 >= p && isspace(*(q - 1))) {
|
||||
q--;
|
||||
*q = '\0';
|
||||
}
|
||||
|
||||
/* if remove header, move */
|
||||
if (p != s) {
|
||||
q = s;
|
||||
while (*p != '\0')
|
||||
*q++ = *p++;
|
||||
*q = '\0';
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Split string into tokens
|
||||
*/
|
||||
int nvme_str_split(char *string, int stringlen,
|
||||
char **tokens, int maxtokens, char delim)
|
||||
{
|
||||
int i, tok = 0;
|
||||
int tokstart = 1;
|
||||
|
||||
if (string == NULL || tokens == NULL) {
|
||||
errno = EINVAL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
for (i = 0; i < stringlen; i++) {
|
||||
if (string[i] == '\0' || tok >= maxtokens)
|
||||
break;
|
||||
if (tokstart) {
|
||||
tokstart = 0;
|
||||
tokens[tok++] = &string[i];
|
||||
}
|
||||
if (string[i] == delim) {
|
||||
string[i] = '\0';
|
||||
tokstart = 1;
|
||||
}
|
||||
}
|
||||
|
||||
return tok;
|
||||
}
|
||||
|
||||
#ifndef __HAIKU__
|
||||
/*
|
||||
* Parse a sysfs (or other) file containing one integer value
|
||||
*/
|
||||
int nvme_parse_sysfs_value(const char *filename,
|
||||
unsigned long *val)
|
||||
{
|
||||
FILE *f;
|
||||
char buf[BUFSIZ];
|
||||
char *end = NULL;
|
||||
|
||||
if ((f = fopen(filename, "r")) == NULL) {
|
||||
nvme_err("%s(): cannot open sysfs value %s\n",
|
||||
__func__, filename);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (fgets(buf, sizeof(buf), f) == NULL) {
|
||||
nvme_err("%s(): cannot read sysfs value %s\n",
|
||||
__func__, filename);
|
||||
fclose(f);
|
||||
return -1;
|
||||
}
|
||||
*val = strtoul(buf, &end, 0);
|
||||
if ((buf[0] == '\0') || (end == NULL) || (*end != '\n')) {
|
||||
nvme_err("%s(): cannot parse sysfs value %s\n",
|
||||
__func__, filename);
|
||||
fclose(f);
|
||||
return -1;
|
||||
}
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get a block device block size in Bytes.
|
||||
*/
|
||||
ssize_t nvme_dev_get_blocklen(int fd)
|
||||
{
|
||||
uint32_t blocklen = 0;
|
||||
|
||||
if (ioctl(fd, BLKSSZGET, &blocklen) < 0) {
|
||||
nvme_err("iioctl BLKSSZGET failed %d (%s)\n",
|
||||
errno,
|
||||
strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
|
||||
return blocklen;
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Get a file size in Bytes.
|
||||
*/
|
||||
uint64_t nvme_file_get_size(int fd)
|
||||
{
|
||||
struct stat st;
|
||||
|
||||
if (fstat(fd, &st) != 0)
|
||||
return 0;
|
||||
|
||||
if (S_ISLNK(st.st_mode))
|
||||
return 0;
|
||||
|
||||
if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode)) {
|
||||
#ifndef __HAIKU__
|
||||
uint64_t size;
|
||||
if (ioctl(fd, BLKGETSIZE64, &size) == 0)
|
||||
return size;
|
||||
else
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (S_ISREG(st.st_mode))
|
||||
return st.st_size;
|
||||
|
||||
/* Not REG, CHR or BLK */
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifndef __HAIKU__
|
||||
/*
|
||||
* Dump the stack of the calling core.
|
||||
*/
|
||||
static void nvme_dump_stack(void)
|
||||
{
|
||||
#define BACKTRACE_SIZE 256
|
||||
void *func[BACKTRACE_SIZE];
|
||||
char **symb = NULL;
|
||||
int size;
|
||||
|
||||
size = backtrace(func, BACKTRACE_SIZE);
|
||||
symb = backtrace_symbols(func, size);
|
||||
|
||||
if (symb == NULL)
|
||||
return;
|
||||
|
||||
while (size > 0) {
|
||||
nvme_crit("%d: [%s]\n", size, symb[size - 1]);
|
||||
size --;
|
||||
}
|
||||
|
||||
free(symb);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef __HAIKU__
|
||||
void
|
||||
/*
|
||||
* call abort(), it will generate a coredump if enabled.
|
||||
*/
|
||||
void __nvme_panic(const char *funcname, const char *format, ...)
|
||||
{
|
||||
va_list ap;
|
||||
|
||||
nvme_crit("PANIC in %s():\n", funcname);
|
||||
va_start(ap, format);
|
||||
nvme_vlog(NVME_LOG_CRIT, format, ap);
|
||||
va_end(ap);
|
||||
nvme_dump_stack();
|
||||
abort();
|
||||
}
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Library initialization: must be run first by any application
|
||||
* before calling any libnvme API.
|
||||
*/
|
||||
int nvme_lib_init(enum nvme_log_level level,
|
||||
enum nvme_log_facility facility, const char *path)
|
||||
{
|
||||
int ret;
|
||||
|
||||
#ifndef __HAIKU__
|
||||
/* Set log level and facility first */
|
||||
nvme_set_log_level(level);
|
||||
nvme_set_log_facility(facility, path);
|
||||
|
||||
/* Gather CPU information */
|
||||
ret = nvme_cpu_init();
|
||||
if (ret != 0) {
|
||||
nvme_crit("Failed to gather CPU information\n");
|
||||
goto out;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* PCI subsystem initialization (libpciaccess) */
|
||||
ret = nvme_pci_init();
|
||||
if (ret != 0) {
|
||||
nvme_crit("PCI subsystem initialization failed\n");
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* Initialize memory management */
|
||||
ret = nvme_mem_init();
|
||||
if (ret != 0)
|
||||
nvme_crit("Memory management initialization failed\n");
|
||||
|
||||
out:
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Will be executed automatically last on termination of the user application.
|
||||
*/
|
||||
__attribute__((destructor)) void nvme_lib_exit(void)
|
||||
{
|
||||
|
||||
nvme_ctrlr_cleanup();
|
||||
|
||||
nvme_mem_cleanup();
|
||||
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
|
||||
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __NVME_COMMON_H__
|
||||
#define __NVME_COMMON_H__
|
||||
|
||||
#define _GNU_SOURCE
|
||||
#define _FILE_OFFSET_BITS 64
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdarg.h>
|
||||
#include <string.h>
|
||||
#include <ctype.h>
|
||||
#include <errno.h>
|
||||
#include <inttypes.h>
|
||||
#include <unistd.h>
|
||||
#include <limits.h>
|
||||
#include <time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/cdefs.h>
|
||||
#ifdef __HAIKU__
|
||||
#define _BSD_SOURCE
|
||||
#include <compatibility/bsd/sys/queue.h>
|
||||
#else
|
||||
#include <sys/queue.h>
|
||||
#include <linux/types.h>
|
||||
#endif
|
||||
|
||||
#include "libnvme/nvme.h"
|
||||
#include "nvme_arch.h"
|
||||
#include "nvme_log.h"
|
||||
|
||||
/*
|
||||
* Check if a branch is likely to be taken.
|
||||
*/
|
||||
#ifndef likely
|
||||
#define likely(x) __builtin_expect((x),1)
|
||||
#endif /* likely */
|
||||
|
||||
/*
|
||||
* Check if a branch is unlikely to be taken.
|
||||
*/
|
||||
#ifndef unlikely
|
||||
#define unlikely(x) __builtin_expect((x),0)
|
||||
#endif /* unlikely */
|
||||
|
||||
#ifndef typeof
|
||||
#define typeof __typeof__
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Trim whitespace from a string in place.
|
||||
*/
|
||||
extern void nvme_str_trim(char *s);
|
||||
|
||||
/*
|
||||
* Split string into tokens
|
||||
*/
|
||||
extern int nvme_str_split(char *string, int stringlen,
|
||||
char **tokens, int maxtokens, char delim);
|
||||
|
||||
/*
|
||||
* Converts a numeric string to the equivalent uint64_t value.
|
||||
* As well as straight number conversion, also recognises the suffixes
|
||||
* k, m and g for kilobytes, megabytes and gigabytes respectively.
|
||||
*
|
||||
* If a negative number is passed in, zero is returned.
|
||||
* Zero is also returned in the case of an error with the
|
||||
* strtoull call in the function.
|
||||
*/
|
||||
static inline size_t nvme_str2size(const char *str)
|
||||
{
|
||||
unsigned long long size;
|
||||
char *endptr;
|
||||
|
||||
while (isspace((int)*str))
|
||||
str++;
|
||||
if (*str == '-')
|
||||
return 0;
|
||||
|
||||
errno = 0;
|
||||
size = strtoull(str, &endptr, 0);
|
||||
if (errno)
|
||||
return 0;
|
||||
|
||||
/* Allow 1 space gap between number and unit */
|
||||
if (*endptr == ' ')
|
||||
endptr++;
|
||||
|
||||
switch (*endptr){
|
||||
case 'G':
|
||||
case 'g':
|
||||
size *= 1024;
|
||||
/* Fall through */
|
||||
case 'M':
|
||||
case 'm':
|
||||
/* Fall through */
|
||||
size *= 1024;
|
||||
case 'K':
|
||||
case 'k':
|
||||
/* Fall through */
|
||||
size *= 1024;
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
/*
|
||||
* Function to read a single numeric value from a file on the filesystem.
|
||||
* Used to read information from files on /sys
|
||||
*/
|
||||
extern int nvme_parse_sysfs_value(const char *filename, unsigned long *val);
|
||||
|
||||
/*
|
||||
* Get a file size in Bytes.
|
||||
*/
|
||||
extern uint64_t nvme_file_get_size(int fd);
|
||||
|
||||
/*
|
||||
* Get a block device block size in Bytes.
|
||||
*/
|
||||
extern ssize_t nvme_dev_get_blocklen(int fd);
|
||||
|
||||
/*
|
||||
* Get current time in nano seconds.
|
||||
*/
|
||||
static inline unsigned long long nvme_time_nsec(void)
|
||||
{
|
||||
#ifdef __HAIKU__
|
||||
return (unsigned long long)system_time();
|
||||
#else
|
||||
struct timespec ts;
|
||||
|
||||
clock_gettime(CLOCK_REALTIME, &ts);
|
||||
|
||||
return (unsigned long long) ts.tv_sec * 1000000000ULL
|
||||
+ (unsigned long long) ts.tv_nsec;
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Get current time in micro seconds.
|
||||
*/
|
||||
static inline unsigned long long nvme_time_usec(void)
|
||||
{
|
||||
return nvme_time_nsec() / 1000;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get current time in milli seconds.
|
||||
*/
|
||||
static inline unsigned long long nvme_time_msec(void)
|
||||
{
|
||||
return nvme_time_nsec() / 1000000;
|
||||
}
|
||||
|
||||
/*
|
||||
* PAUSE instruction for tight loops (avoid busy waiting)
|
||||
*/
|
||||
#ifdef __SSE2__
|
||||
#include <emmintrin.h>
|
||||
static inline void nvme_pause(void)
|
||||
{
|
||||
_mm_pause();
|
||||
}
|
||||
#else
|
||||
static inline void nvme_pause(void) {}
|
||||
#endif
|
||||
|
||||
#ifdef __HAIKU__
|
||||
static inline void
|
||||
nvme_usleep(int usecs)
|
||||
{
|
||||
snooze(usecs);
|
||||
}
|
||||
|
||||
static inline void
|
||||
nvme_msleep(int msecs)
|
||||
{
|
||||
snooze(msecs * 1000LL);
|
||||
}
|
||||
#else
|
||||
/*
|
||||
* Micro-seconds sleep.
|
||||
*/
|
||||
static inline void nvme_usleep(int usecs)
|
||||
{
|
||||
struct timeval tv;
|
||||
|
||||
tv.tv_sec = usecs / 1000000;
|
||||
tv.tv_usec = usecs % 1000000;
|
||||
select(0, NULL, NULL, NULL, &tv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Milli-seconds sleep.
|
||||
*/
|
||||
static inline void nvme_msleep(int msecs)
|
||||
{
|
||||
struct timeval tv;
|
||||
|
||||
tv.tv_sec = msecs / 1000;
|
||||
tv.tv_usec = (msecs - tv.tv_sec * 1000) * 1000;
|
||||
select(0, NULL, NULL, NULL, &tv);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef __HAIKU__
|
||||
/*
|
||||
* Provide notification of a critical non-recoverable error and stop.
|
||||
* This function should not be called directly. Use nvme_panic() instead.
|
||||
*/
|
||||
extern void __nvme_panic(const char *funcname , const char *format, ...)
|
||||
#ifdef __GNUC__
|
||||
#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 2))
|
||||
__attribute__((cold))
|
||||
#endif
|
||||
#endif
|
||||
__attribute__((noreturn))
|
||||
__attribute__((format(printf, 2, 3)));
|
||||
|
||||
/*
|
||||
* Provide notification of a critical non-recoverable error
|
||||
* and terminate execution abnormally.
|
||||
*/
|
||||
#define nvme_panic(format, args...) \
|
||||
__nvme_panic(__FUNCTION__, format, ## args)
|
||||
#else
|
||||
#define nvme_panic panic
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Macro to evaluate a scalar expression and
|
||||
* abort the program if the assertion is false.
|
||||
*/
|
||||
#define _nvme_assert_default(exp) \
|
||||
do { \
|
||||
if (unlikely(!(exp))) \
|
||||
nvme_panic("line %d, assert %s failed\n", \
|
||||
__LINE__, # exp); \
|
||||
} while (0)
|
||||
|
||||
#define _nvme_assert_msg(exp, msg) \
|
||||
do { \
|
||||
if (unlikely(!(exp))) \
|
||||
nvme_panic("%s\n", msg); \
|
||||
} while (0)
|
||||
|
||||
#define _NVME_GET_ASSERT_OVERLOAD(_1, _2, NAME, args...) NAME
|
||||
|
||||
#define nvme_assert(args...) \
|
||||
_NVME_GET_ASSERT_OVERLOAD(args, \
|
||||
_nvme_assert_msg, \
|
||||
_nvme_assert_default) \
|
||||
(args)
|
||||
|
||||
/*
|
||||
* Macro to return the minimum of two numbers
|
||||
*/
|
||||
#define nvme_min(a, b) ({ \
|
||||
typeof (a) _a = (a); \
|
||||
typeof (b) _b = (b); \
|
||||
_a < _b ? _a : _b; \
|
||||
})
|
||||
|
||||
/*
|
||||
* Macro to return the maximum of two numbers
|
||||
*/
|
||||
#define nvme_max(a, b) ({ \
|
||||
typeof (a) _a = (a); \
|
||||
typeof (b) _b = (b); \
|
||||
_a > _b ? _a : _b; \
|
||||
})
|
||||
|
||||
/*
|
||||
* Returns true if n is a power of 2.
|
||||
*/
|
||||
static inline int nvme_is_pow2(__u64 v)
|
||||
{
|
||||
return v && !(v & (v - 1));
|
||||
}
|
||||
|
||||
/*
|
||||
* Return the power of 2 immediately after v.
|
||||
*/
|
||||
static inline __u64 nvme_align_pow2(__u64 v)
|
||||
{
|
||||
v--;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
v |= v >> 32;
|
||||
|
||||
return v + 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Calculate log2 of a power of 2 size.
|
||||
*/
|
||||
static inline size_t nvme_log2(size_t size)
|
||||
{
|
||||
size_t bits = 0;
|
||||
|
||||
if (!nvme_is_pow2(size))
|
||||
return 0;
|
||||
|
||||
while (size >>= 1)
|
||||
bits++;
|
||||
|
||||
return bits;
|
||||
}
|
||||
|
||||
/*
|
||||
* Handle alignements.
|
||||
*/
|
||||
#define nvme_align_down(val, align) \
|
||||
((val) & (~((typeof(val))((align) - 1))))
|
||||
#define nvme_align_up(val, align) \
|
||||
nvme_align_down((val) + (align) - 1, (align))
|
||||
|
||||
/*
|
||||
* Test a bit value.
|
||||
*/
|
||||
static inline int test_bit(__u8 *bitmap, unsigned int bit)
|
||||
{
|
||||
return bitmap[bit >> 3] & (1U << (bit & 0x7));
|
||||
}
|
||||
|
||||
/*
|
||||
* Set a bit.
|
||||
*/
|
||||
static inline void set_bit(__u8 *bitmap, unsigned int bit)
|
||||
{
|
||||
bitmap[bit >> 3] |= 1U << (bit & 0x7);
|
||||
}
|
||||
|
||||
/*
|
||||
* Clear a bit.
|
||||
*/
|
||||
static inline void clear_bit(__u8 *bitmap, unsigned int bit)
|
||||
{
|
||||
bitmap[bit >> 3] &= ~(1U << (bit & 0x7));
|
||||
}
|
||||
|
||||
/*
|
||||
* Find the first zero bit in a bitmap of size nr_bits.
|
||||
* If no zero bit is found, return -1.
|
||||
*/
|
||||
static inline int find_first_zero_bit(__u8 *bitmap, unsigned int nr_bits)
|
||||
{
|
||||
__u64 *b = (__u64 *)bitmap;
|
||||
unsigned int i, j, bit, count = (nr_bits + 63) >> 6;
|
||||
|
||||
for(i = 0; i < count; i++) {
|
||||
if (b[i] != ~0UL)
|
||||
break;
|
||||
}
|
||||
|
||||
bit = i << 6;
|
||||
for (j = bit; j < nr_bits; j++) {
|
||||
if (!test_bit(bitmap, j))
|
||||
return j;
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Close all open controllers on exit.
|
||||
* Defined in lib/nvme/nvme.c
|
||||
*/
|
||||
extern void nvme_ctrlr_cleanup(void);
|
||||
|
||||
#endif /* __NVME_COMMON_H__ */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,209 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation. All rights reserved.
|
||||
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Intel NVMe vendor-specific definitions
|
||||
* See: http://www.intel.com/content/dam/www/public/us/en/documents/product-specifications/ssd-dc-p3700-spec.pdf
|
||||
*/
|
||||
|
||||
#ifndef __NVME_INTEL_H__
|
||||
#define __NVME_INTEL_H__
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
|
||||
enum nvme_intel_feat {
|
||||
NVME_INTEL_FEAT_MAX_LBA = 0xC1,
|
||||
NVME_INTEL_FEAT_NATIVE_MAX_LBA = 0xC2,
|
||||
NVME_INTEL_FEAT_POWER_GOVERNOR_SETTING = 0xC6,
|
||||
NVME_INTEL_FEAT_SMBUS_ADDRESS = 0xC8,
|
||||
NVME_INTEL_FEAT_LED_PATTERN = 0xC9,
|
||||
NVME_INTEL_FEAT_RESET_TIMED_WORKLOAD_COUNTERS = 0xD5,
|
||||
NVME_INTEL_FEAT_LATENCY_TRACKING = 0xE2,
|
||||
};
|
||||
|
||||
enum nvme_intel_set_max_lba_command_status_code {
|
||||
NVME_INTEL_EXCEEDS_AVAILABLE_CAPACITY = 0xC0,
|
||||
NVME_INTEL_SMALLER_THAN_MIN_LIMIT = 0xC1,
|
||||
NVME_INTEL_SMALLER_THAN_NS_REQUIREMENTS = 0xC2,
|
||||
};
|
||||
|
||||
enum nvme_intel_log_page {
|
||||
NVME_INTEL_LOG_PAGE_DIR = 0xC0,
|
||||
NVME_INTEL_LOG_READ_CMD_LATENCY = 0xC1,
|
||||
NVME_INTEL_LOG_WRITE_CMD_LATENCY = 0xC2,
|
||||
NVME_INTEL_LOG_TEMPERATURE = 0xC5,
|
||||
NVME_INTEL_LOG_SMART = 0xCA,
|
||||
NVME_INTEL_MARKETING_DESCRIPTION = 0xDD,
|
||||
};
|
||||
|
||||
enum nvme_intel_smart_attribute_code {
|
||||
NVME_INTEL_SMART_PROGRAM_FAIL_COUNT = 0xAB,
|
||||
NVME_INTEL_SMART_ERASE_FAIL_COUNT = 0xAC,
|
||||
NVME_INTEL_SMART_WEAR_LEVELING_COUNT = 0xAD,
|
||||
NVME_INTEL_SMART_E2E_ERROR_COUNT = 0xB8,
|
||||
NVME_INTEL_SMART_CRC_ERROR_COUNT = 0xC7,
|
||||
NVME_INTEL_SMART_MEDIA_WEAR = 0xE2,
|
||||
NVME_INTEL_SMART_HOST_READ_PERCENTAGE = 0xE3,
|
||||
NVME_INTEL_SMART_TIMER = 0xE4,
|
||||
NVME_INTEL_SMART_THERMAL_THROTTLE_STATUS = 0xEA,
|
||||
NVME_INTEL_SMART_RETRY_BUFFER_OVERFLOW_COUNTER = 0xF0,
|
||||
NVME_INTEL_SMART_PLL_LOCK_LOSS_COUNT = 0xF3,
|
||||
NVME_INTEL_SMART_NAND_BYTES_WRITTEN = 0xF4,
|
||||
NVME_INTEL_SMART_HOST_BYTES_WRITTEN = 0xF5,
|
||||
};
|
||||
|
||||
struct nvme_intel_log_page_dir {
|
||||
uint8_t version[2];
|
||||
uint8_t reserved[384];
|
||||
uint8_t read_latency_log_len;
|
||||
uint8_t reserved2;
|
||||
uint8_t write_latency_log_len;
|
||||
uint8_t reserved3[5];
|
||||
uint8_t temperature_statistics_log_len;
|
||||
uint8_t reserved4[9];
|
||||
uint8_t smart_log_len;
|
||||
uint8_t reserved5[37];
|
||||
uint8_t marketing_description_log_len;
|
||||
uint8_t reserved6[69];
|
||||
};
|
||||
nvme_static_assert(sizeof(struct nvme_intel_log_page_dir) == 512,
|
||||
"Incorrect size");
|
||||
|
||||
struct nvme_intel_rw_latency_page {
|
||||
uint16_t major_revison;
|
||||
uint16_t minor_revison;
|
||||
uint32_t buckets_32us[32];
|
||||
uint32_t buckets_1ms[31];
|
||||
uint32_t buckets_32ms[31];
|
||||
};
|
||||
nvme_static_assert(sizeof(struct nvme_intel_rw_latency_page) == 380,
|
||||
"Incorrect size");
|
||||
|
||||
struct nvme_intel_temperature_page {
|
||||
uint64_t current_temperature;
|
||||
uint64_t shutdown_flag_last;
|
||||
uint64_t shutdown_flag_life;
|
||||
uint64_t highest_temperature;
|
||||
uint64_t lowest_temperature;
|
||||
uint64_t reserved[5];
|
||||
uint64_t specified_max_op_temperature;
|
||||
uint64_t reserved2;
|
||||
uint64_t specified_min_op_temperature;
|
||||
uint64_t estimated_offset;
|
||||
};
|
||||
nvme_static_assert(sizeof(struct nvme_intel_temperature_page) == 112,
|
||||
"Incorrect size");
|
||||
|
||||
struct nvme_intel_smart_attribute {
|
||||
uint8_t code;
|
||||
uint8_t reserved[2];
|
||||
uint8_t normalized_value;
|
||||
uint8_t reserved2;
|
||||
uint8_t raw_value[6];
|
||||
uint8_t reserved3;
|
||||
};
|
||||
|
||||
struct __attribute__((packed)) nvme_intel_smart_information_page {
|
||||
struct nvme_intel_smart_attribute attributes[13];
|
||||
};
|
||||
nvme_static_assert(sizeof(struct nvme_intel_smart_information_page) == 156,
|
||||
"Incorrect size");
|
||||
|
||||
union nvme_intel_feat_power_governor {
|
||||
uint32_t raw;
|
||||
struct {
|
||||
/* Power governor setting: 00h = 25W 01h = 20W 02h = 10W */
|
||||
uint32_t power_governor_setting : 8;
|
||||
uint32_t reserved : 24;
|
||||
} bits;
|
||||
};
|
||||
nvme_static_assert(sizeof(union nvme_intel_feat_power_governor) == 4,
|
||||
"Incorrect size");
|
||||
|
||||
union nvme_intel_feat_smbus_address {
|
||||
uint32_t raw;
|
||||
struct {
|
||||
uint32_t reserved : 1;
|
||||
uint32_t smbus_controller_address : 8;
|
||||
uint32_t reserved2 : 23;
|
||||
} bits;
|
||||
};
|
||||
nvme_static_assert(sizeof(union nvme_intel_feat_smbus_address) == 4,
|
||||
"Incorrect size");
|
||||
|
||||
union nvme_intel_feat_led_pattern {
|
||||
uint32_t raw;
|
||||
struct {
|
||||
uint32_t feature_options : 24;
|
||||
uint32_t value : 8;
|
||||
} bits;
|
||||
};
|
||||
nvme_static_assert(sizeof(union nvme_intel_feat_led_pattern) == 4,
|
||||
"Incorrect size");
|
||||
|
||||
union nvme_intel_feat_reset_timed_workload_counters {
|
||||
uint32_t raw;
|
||||
struct {
|
||||
/*
|
||||
* Write Usage: 00 = NOP, 1 = Reset E2, E3,E4 counters;
|
||||
* Read Usage: Not Supported
|
||||
*/
|
||||
uint32_t reset : 1;
|
||||
uint32_t reserved : 31;
|
||||
} bits;
|
||||
};
|
||||
nvme_static_assert(sizeof(union nvme_intel_feat_reset_timed_workload_counters) == 4,
|
||||
"Incorrect size");
|
||||
|
||||
union nvme_intel_feat_latency_tracking {
|
||||
uint32_t raw;
|
||||
struct {
|
||||
/*
|
||||
* Write Usage:
|
||||
* 00h = Disable Latency Tracking (Default)
|
||||
* 01h = Enable Latency Tracking
|
||||
*/
|
||||
uint32_t enable : 32;
|
||||
} bits;
|
||||
};
|
||||
nvme_static_assert(sizeof(union nvme_intel_feat_latency_tracking) == 4,
|
||||
"Incorrect size");
|
||||
|
||||
struct nvme_intel_marketing_description_page {
|
||||
uint8_t marketing_product[512];
|
||||
};
|
||||
nvme_static_assert(sizeof(struct nvme_intel_marketing_description_page) == 512,
|
||||
"Incorrect size");
|
||||
|
||||
#endif /* __NVME_INTEL_H__ */
|
||||
@@ -0,0 +1,738 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation. All rights reserved.
|
||||
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __NVME_INTERNAL_H__
|
||||
#define __NVME_INTERNAL_H__
|
||||
|
||||
#include "nvme_common.h"
|
||||
#include "nvme_pci.h"
|
||||
#include "nvme_intel.h"
|
||||
#include "nvme_mem.h"
|
||||
|
||||
#ifndef __HAIKU__
|
||||
#include <pthread.h>
|
||||
#include <sys/user.h> /* PAGE_SIZE */
|
||||
#else
|
||||
#include "nvme_platform.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* List functions.
|
||||
*/
|
||||
#define LIST_FOREACH_SAFE(var, head, field, tvar) \
|
||||
for ((var) = LIST_FIRST((head)); \
|
||||
(var) && ((tvar) = LIST_NEXT((var), field), 1); \
|
||||
(var) = (tvar))
|
||||
|
||||
/*
|
||||
* Tail queue functions.
|
||||
*/
|
||||
#define TAILQ_FOREACH_SAFE(var, head, field, tvar) \
|
||||
for ((var) = TAILQ_FIRST((head)); \
|
||||
(var) && ((tvar) = TAILQ_NEXT((var), field), 1); \
|
||||
(var) = (tvar))
|
||||
|
||||
#define INTEL_DC_P3X00_DEVID 0x09538086
|
||||
|
||||
#define NVME_TIMEOUT_INFINITE UINT64_MAX
|
||||
|
||||
/*
|
||||
* Some Intel devices support vendor-unique read latency log page even
|
||||
* though the log page directory says otherwise.
|
||||
*/
|
||||
#define NVME_INTEL_QUIRK_READ_LATENCY 0x1
|
||||
|
||||
/*
|
||||
* Some Intel devices support vendor-unique write latency log page even
|
||||
* though the log page directory says otherwise.
|
||||
*/
|
||||
#define NVME_INTEL_QUIRK_WRITE_LATENCY 0x2
|
||||
|
||||
/*
|
||||
* Some controllers need a delay before starting to check the device
|
||||
* readiness, which is done by reading the controller status register rdy bit.
|
||||
*/
|
||||
#define NVME_QUIRK_DELAY_BEFORE_CHK_RDY 0x4
|
||||
|
||||
/*
|
||||
* Some controllers need a delay once the controller status register rdy bit
|
||||
* switches from 0 to 1.
|
||||
*/
|
||||
#define NVME_QUIRK_DELAY_AFTER_RDY 0x8
|
||||
|
||||
/*
|
||||
* Queues may consist of a contiguous block of physical
|
||||
* memory or optionally a non-contiguous set of physical
|
||||
* memory pages (defined by a Physical Region Pages List)
|
||||
*/
|
||||
#define NVME_MAX_PRP_LIST_ENTRIES (506)
|
||||
|
||||
/*
|
||||
* For commands requiring more than 2 PRP entries, one PRP will be
|
||||
* embedded in the command (prp1), and the rest of the PRP entries
|
||||
* will be in a list pointed to by the command (prp2). This means
|
||||
* that real max number of PRP entries we support is 506+1, which
|
||||
* results in a max xfer size of 506*PAGE_SIZE.
|
||||
*/
|
||||
#define NVME_MAX_XFER_SIZE NVME_MAX_PRP_LIST_ENTRIES * PAGE_SIZE
|
||||
|
||||
#define NVME_ADMIN_TRACKERS (16)
|
||||
#define NVME_ADMIN_ENTRIES (128)
|
||||
|
||||
/*
|
||||
* NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion
|
||||
* queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we
|
||||
* will allow outstanding on an I/O qpair at any time. The only advantage in
|
||||
* having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping
|
||||
* the contents of the submission and completion queues, it will show a longer
|
||||
* history of data.
|
||||
*/
|
||||
#define NVME_IO_ENTRIES (1024U)
|
||||
#define NVME_IO_TRACKERS (128U)
|
||||
#define NVME_IO_ENTRIES_VS_TRACKERS_RATIO (NVME_IO_ENTRIES / NVME_IO_TRACKERS)
|
||||
|
||||
/*
|
||||
* NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL
|
||||
* segment.
|
||||
*/
|
||||
#define NVME_MAX_SGL_DESCRIPTORS (253)
|
||||
|
||||
/*
|
||||
* NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES
|
||||
* for each controller.
|
||||
*/
|
||||
|
||||
#define NVME_MAX_ASYNC_EVENTS (8)
|
||||
|
||||
/*
|
||||
* NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this
|
||||
* define specifies the maximum number of queues this driver will actually
|
||||
* try to configure, if available.
|
||||
*/
|
||||
#define DEFAULT_MAX_IO_QUEUES (1024)
|
||||
|
||||
/*
|
||||
* Maximum of times a failed command can be retried.
|
||||
*/
|
||||
#define NVME_MAX_RETRY_COUNT (3)
|
||||
|
||||
/*
|
||||
* I/O queue type.
|
||||
*/
|
||||
enum nvme_io_queue_type {
|
||||
|
||||
NVME_IO_QTYPE_INVALID = 0,
|
||||
NVME_IO_SUBMISSION_QUEUE,
|
||||
NVME_IO_COMPLETION_QUEUE,
|
||||
};
|
||||
|
||||
enum nvme_payload_type {
|
||||
|
||||
NVME_PAYLOAD_TYPE_INVALID = 0,
|
||||
|
||||
/*
|
||||
* nvme_request::u.payload.contig_buffer is valid for this request.
|
||||
*/
|
||||
NVME_PAYLOAD_TYPE_CONTIG,
|
||||
|
||||
/*
|
||||
* nvme_request::u.sgl is valid for this request
|
||||
*/
|
||||
NVME_PAYLOAD_TYPE_SGL,
|
||||
};
|
||||
|
||||
/*
|
||||
* Controller support flags.
|
||||
*/
|
||||
enum nvme_ctrlr_flags {
|
||||
|
||||
/*
|
||||
* The SGL is supported.
|
||||
*/
|
||||
NVME_CTRLR_SGL_SUPPORTED = 0x1,
|
||||
|
||||
};
|
||||
|
||||
/*
|
||||
* Descriptor for a request data payload.
|
||||
*
|
||||
* This struct is arranged so that it fits nicely in struct nvme_request.
|
||||
*/
|
||||
struct __attribute__((packed)) nvme_payload {
|
||||
|
||||
union {
|
||||
/*
|
||||
* Virtual memory address of a single
|
||||
* physically contiguous buffer
|
||||
*/
|
||||
void *contig;
|
||||
|
||||
/*
|
||||
* Call back functions for retrieving physical
|
||||
* addresses for scattered payloads.
|
||||
*/
|
||||
struct {
|
||||
nvme_req_reset_sgl_cb reset_sgl_fn;
|
||||
nvme_req_next_sge_cb next_sge_fn;
|
||||
void *cb_arg;
|
||||
} sgl;
|
||||
} u;
|
||||
|
||||
/*
|
||||
* Virtual memory address of a single physically
|
||||
* contiguous metadata buffer
|
||||
*/
|
||||
void *md;
|
||||
|
||||
/*
|
||||
* Payload type.
|
||||
*/
|
||||
uint8_t type;
|
||||
|
||||
};
|
||||
|
||||
struct nvme_request {
|
||||
|
||||
/*
|
||||
* NVMe command: must be aligned on 64B.
|
||||
*/
|
||||
struct nvme_cmd cmd;
|
||||
|
||||
/*
|
||||
* Data payload for this request's command.
|
||||
*/
|
||||
struct nvme_payload payload;
|
||||
|
||||
uint8_t retries;
|
||||
|
||||
/*
|
||||
* Number of child requests still outstanding for this
|
||||
* request which was split into multiple child requests.
|
||||
*/
|
||||
uint8_t child_reqs;
|
||||
uint32_t payload_size;
|
||||
|
||||
/*
|
||||
* Offset in bytes from the beginning of payload for this request.
|
||||
* This is used for I/O commands that are split into multiple requests.
|
||||
*/
|
||||
uint32_t payload_offset;
|
||||
uint32_t md_offset;
|
||||
|
||||
nvme_cmd_cb cb_fn;
|
||||
void *cb_arg;
|
||||
|
||||
/*
|
||||
* The following members should not be reordered with members
|
||||
* above. These members are only needed when splitting
|
||||
* requests which is done rarely, and the driver is careful
|
||||
* to not touch the following fields until a split operation is
|
||||
* needed, to avoid touching an extra cacheline.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Points to the outstanding child requests for a parent request.
|
||||
* Only valid if a request was split into multiple child
|
||||
* requests, and is not initialized for non-split requests.
|
||||
*/
|
||||
TAILQ_HEAD(, nvme_request) children;
|
||||
|
||||
/*
|
||||
* Linked-list pointers for a child request in its parent's list.
|
||||
*/
|
||||
TAILQ_ENTRY(nvme_request) child_tailq;
|
||||
|
||||
/*
|
||||
* For queueing in qpair queued_req or free_req.
|
||||
*/
|
||||
struct nvme_qpair *qpair;
|
||||
STAILQ_ENTRY(nvme_request) stailq;
|
||||
|
||||
/*
|
||||
* Points to a parent request if part of a split request,
|
||||
* NULL otherwise.
|
||||
*/
|
||||
struct nvme_request *parent;
|
||||
|
||||
/*
|
||||
* Completion status for a parent request. Initialized to all 0's
|
||||
* (SUCCESS) before child requests are submitted. If a child
|
||||
* request completes with error, the error status is copied here,
|
||||
* to ensure that the parent request is also completed with error
|
||||
* status once all child requests are completed.
|
||||
*/
|
||||
struct nvme_cpl parent_status;
|
||||
|
||||
} __attribute__((aligned(64)));
|
||||
|
||||
struct nvme_completion_poll_status {
|
||||
struct nvme_cpl cpl;
|
||||
bool done;
|
||||
};
|
||||
|
||||
struct nvme_async_event_request {
|
||||
struct nvme_ctrlr *ctrlr;
|
||||
struct nvme_request *req;
|
||||
struct nvme_cpl cpl;
|
||||
};
|
||||
|
||||
struct nvme_tracker {
|
||||
|
||||
LIST_ENTRY(nvme_tracker) list;
|
||||
|
||||
struct nvme_request *req;
|
||||
uint16_t cid;
|
||||
|
||||
uint16_t rsvd1: 15;
|
||||
uint16_t active: 1;
|
||||
|
||||
uint32_t rsvd2;
|
||||
|
||||
uint64_t prp_sgl_bus_addr;
|
||||
|
||||
union {
|
||||
uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES];
|
||||
struct nvme_sgl_descriptor sgl[NVME_MAX_SGL_DESCRIPTORS];
|
||||
} u;
|
||||
|
||||
uint64_t rsvd3;
|
||||
};
|
||||
|
||||
/*
|
||||
* struct nvme_tracker must be exactly 4K so that the prp[] array does not
|
||||
* cross a page boundery and so that there is no padding required to meet
|
||||
* alignment requirements.
|
||||
*/
|
||||
nvme_static_assert(sizeof(struct nvme_tracker) == 4096,
|
||||
"nvme_tracker is not 4K");
|
||||
nvme_static_assert((offsetof(struct nvme_tracker, u.sgl) & 7) == 0,
|
||||
"SGL must be Qword aligned");
|
||||
|
||||
struct nvme_qpair {
|
||||
|
||||
volatile uint32_t *sq_tdbl;
|
||||
volatile uint32_t *cq_hdbl;
|
||||
|
||||
/*
|
||||
* Submission queue
|
||||
*/
|
||||
struct nvme_cmd *cmd;
|
||||
|
||||
/*
|
||||
* Completion queue
|
||||
*/
|
||||
struct nvme_cpl *cpl;
|
||||
|
||||
LIST_HEAD(, nvme_tracker) free_tr;
|
||||
LIST_HEAD(, nvme_tracker) outstanding_tr;
|
||||
|
||||
/*
|
||||
* Array of trackers indexed by command ID.
|
||||
*/
|
||||
uint16_t trackers;
|
||||
struct nvme_tracker *tr;
|
||||
|
||||
struct nvme_request *reqs;
|
||||
unsigned int num_reqs;
|
||||
STAILQ_HEAD(, nvme_request) free_req;
|
||||
STAILQ_HEAD(, nvme_request) queued_req;
|
||||
|
||||
uint16_t id;
|
||||
|
||||
uint16_t entries;
|
||||
uint16_t sq_tail;
|
||||
uint16_t cq_head;
|
||||
|
||||
uint8_t phase;
|
||||
|
||||
bool enabled;
|
||||
bool sq_in_cmb;
|
||||
|
||||
/*
|
||||
* Fields below this point should not be touched on the
|
||||
* normal I/O happy path.
|
||||
*/
|
||||
|
||||
uint8_t qprio;
|
||||
|
||||
struct nvme_ctrlr *ctrlr;
|
||||
|
||||
/* List entry for nvme_ctrlr::free_io_qpairs and active_io_qpairs */
|
||||
TAILQ_ENTRY(nvme_qpair) tailq;
|
||||
|
||||
phys_addr_t cmd_bus_addr;
|
||||
phys_addr_t cpl_bus_addr;
|
||||
};
|
||||
|
||||
struct nvme_ns {
|
||||
|
||||
struct nvme_ctrlr *ctrlr;
|
||||
|
||||
uint32_t stripe_size;
|
||||
uint32_t sector_size;
|
||||
|
||||
uint32_t md_size;
|
||||
uint32_t pi_type;
|
||||
|
||||
uint32_t sectors_per_max_io;
|
||||
uint32_t sectors_per_stripe;
|
||||
|
||||
uint16_t id;
|
||||
uint16_t flags;
|
||||
|
||||
int open_count;
|
||||
|
||||
};
|
||||
|
||||
/*
|
||||
* State of struct nvme_ctrlr (in particular, during initialization).
|
||||
*/
|
||||
enum nvme_ctrlr_state {
|
||||
|
||||
/*
|
||||
* Controller has not been initialized yet.
|
||||
*/
|
||||
NVME_CTRLR_STATE_INIT = 0,
|
||||
|
||||
/*
|
||||
* Waiting for CSTS.RDY to transition from 0 to 1
|
||||
* so that CC.EN may be set to 0.
|
||||
*/
|
||||
NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1,
|
||||
|
||||
/*
|
||||
* Waiting for CSTS.RDY to transition from 1 to 0
|
||||
* so that CC.EN may be set to 1.
|
||||
*/
|
||||
NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,
|
||||
|
||||
/*
|
||||
* Waiting for CSTS.RDY to transition from 0 to 1
|
||||
* after enabling the controller.
|
||||
*/
|
||||
NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,
|
||||
|
||||
/*
|
||||
* Controller initialization has completed and
|
||||
* the controller is ready.
|
||||
*/
|
||||
NVME_CTRLR_STATE_READY
|
||||
};
|
||||
|
||||
/*
|
||||
* One of these per allocated PCI device.
|
||||
*/
|
||||
struct nvme_ctrlr {
|
||||
|
||||
/*
|
||||
* NVMe MMIO register space.
|
||||
*/
|
||||
volatile struct nvme_registers *regs;
|
||||
|
||||
/*
|
||||
* Array of I/O queue pairs.
|
||||
*/
|
||||
struct nvme_qpair *ioq;
|
||||
|
||||
/*
|
||||
* Size of the array of I/O queue pairs.
|
||||
*/
|
||||
unsigned int io_queues;
|
||||
|
||||
/*
|
||||
* Maximum I/O queue pairs.
|
||||
*/
|
||||
unsigned int max_io_queues;
|
||||
|
||||
/*
|
||||
* Number of I/O queue pairs enabled
|
||||
*/
|
||||
unsigned int enabled_io_qpairs;
|
||||
|
||||
/*
|
||||
* Maximum entries for I/O qpairs
|
||||
*/
|
||||
unsigned int io_qpairs_max_entries;
|
||||
|
||||
/*
|
||||
* Array of namespace IDs.
|
||||
*/
|
||||
unsigned int nr_ns;
|
||||
struct nvme_ns *ns;
|
||||
|
||||
/*
|
||||
* Controller state.
|
||||
*/
|
||||
bool resetting;
|
||||
bool failed;
|
||||
|
||||
/*
|
||||
* Controller support flags.
|
||||
*/
|
||||
uint64_t flags;
|
||||
|
||||
/*
|
||||
* Cold data (not accessed in normal I/O path) is after this point.
|
||||
*/
|
||||
enum nvme_ctrlr_state state;
|
||||
uint64_t state_timeout_ms;
|
||||
|
||||
/*
|
||||
* All the log pages supported.
|
||||
*/
|
||||
bool log_page_supported[256];
|
||||
|
||||
/*
|
||||
* All the features supported.
|
||||
*/
|
||||
bool feature_supported[256];
|
||||
|
||||
/*
|
||||
* Associated PCI device information.
|
||||
*/
|
||||
struct pci_device *pci_dev;
|
||||
|
||||
/*
|
||||
* Maximum i/o size in bytes.
|
||||
*/
|
||||
uint32_t max_xfer_size;
|
||||
|
||||
/*
|
||||
* Minimum page size supported by this controller in bytes.
|
||||
*/
|
||||
uint32_t min_page_size;
|
||||
|
||||
/*
|
||||
* Stride in uint32_t units between doorbell registers
|
||||
* (1 = 4 bytes, 2 = 8 bytes, ...).
|
||||
*/
|
||||
uint32_t doorbell_stride_u32;
|
||||
|
||||
uint32_t num_aers;
|
||||
struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS];
|
||||
nvme_aer_cb aer_cb_fn;
|
||||
void *aer_cb_arg;
|
||||
|
||||
/*
|
||||
* Guards access to the controller itself, including admin queues.
|
||||
*/
|
||||
pthread_mutex_t lock;
|
||||
|
||||
|
||||
/*
|
||||
* Admin queue pair.
|
||||
*/
|
||||
struct nvme_qpair adminq;
|
||||
|
||||
/*
|
||||
* Identify Controller data.
|
||||
*/
|
||||
struct nvme_ctrlr_data cdata;
|
||||
|
||||
/*
|
||||
* Array of Identify Namespace data.
|
||||
* Stored separately from ns since nsdata should
|
||||
* not normally be accessed during I/O.
|
||||
*/
|
||||
struct nvme_ns_data *nsdata;
|
||||
|
||||
TAILQ_HEAD(, nvme_qpair) free_io_qpairs;
|
||||
TAILQ_HEAD(, nvme_qpair) active_io_qpairs;
|
||||
|
||||
/*
|
||||
* Controller option set on open.
|
||||
*/
|
||||
struct nvme_ctrlr_opts opts;
|
||||
|
||||
/*
|
||||
* BAR mapping address which contains controller memory buffer.
|
||||
*/
|
||||
void *cmb_bar_virt_addr;
|
||||
|
||||
/*
|
||||
* BAR physical address which contains controller memory buffer.
|
||||
*/
|
||||
uint64_t cmb_bar_phys_addr;
|
||||
|
||||
/*
|
||||
* Controller memory buffer size in Bytes.
|
||||
*/
|
||||
uint64_t cmb_size;
|
||||
|
||||
/*
|
||||
* Current offset of controller memory buffer.
|
||||
*/
|
||||
uint64_t cmb_current_offset;
|
||||
|
||||
/*
|
||||
* Quirks flags.
|
||||
*/
|
||||
unsigned int quirks;
|
||||
|
||||
/*
|
||||
* For controller list.
|
||||
*/
|
||||
LIST_ENTRY(nvme_ctrlr) link;
|
||||
|
||||
} __attribute__((aligned(PAGE_SIZE)));
|
||||
|
||||
/*
|
||||
* Admin functions.
|
||||
*/
|
||||
extern int nvme_admin_identify_ctrlr(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_ctrlr_data *cdata);
|
||||
|
||||
extern int nvme_admin_get_feature(struct nvme_ctrlr *ctrlr,
|
||||
enum nvme_feat_sel sel,
|
||||
enum nvme_feat feature,
|
||||
uint32_t cdw11, uint32_t *attributes);
|
||||
|
||||
extern int nvme_admin_set_feature(struct nvme_ctrlr *ctrlr,
|
||||
bool save,
|
||||
enum nvme_feat feature,
|
||||
uint32_t cdw11, uint32_t cdw12,
|
||||
uint32_t *attributes);
|
||||
|
||||
extern int nvme_admin_format_nvm(struct nvme_ctrlr *ctrlr,
|
||||
unsigned int nsid,
|
||||
struct nvme_format *format);
|
||||
|
||||
extern int nvme_admin_get_log_page(struct nvme_ctrlr *ctrlr,
|
||||
uint8_t log_page, uint32_t nsid,
|
||||
void *payload, uint32_t payload_size);
|
||||
|
||||
extern int nvme_admin_abort_cmd(struct nvme_ctrlr *ctrlr,
|
||||
uint16_t cid, uint16_t sqid);
|
||||
|
||||
extern int nvme_admin_create_ioq(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_qpair *io_que,
|
||||
enum nvme_io_queue_type io_qtype);
|
||||
|
||||
extern int nvme_admin_delete_ioq(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_qpair *qpair,
|
||||
enum nvme_io_queue_type io_qtype);
|
||||
|
||||
extern int nvme_admin_identify_ns(struct nvme_ctrlr *ctrlr,
|
||||
uint16_t nsid,
|
||||
struct nvme_ns_data *nsdata);
|
||||
|
||||
extern int nvme_admin_attach_ns(struct nvme_ctrlr *ctrlr,
|
||||
uint32_t nsid,
|
||||
struct nvme_ctrlr_list *clist);
|
||||
|
||||
extern int nvme_admin_detach_ns(struct nvme_ctrlr *ctrlr,
|
||||
uint32_t nsid,
|
||||
struct nvme_ctrlr_list *clist);
|
||||
|
||||
extern int nvme_admin_create_ns(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_ns_data *nsdata,
|
||||
unsigned int *nsid);
|
||||
|
||||
extern int nvme_admin_delete_ns(struct nvme_ctrlr *ctrlr,
|
||||
unsigned int nsid);
|
||||
|
||||
extern int nvme_admin_fw_commit(struct nvme_ctrlr *ctrlr,
|
||||
const struct nvme_fw_commit *fw_commit);
|
||||
|
||||
extern int nvme_admin_fw_image_dl(struct nvme_ctrlr *ctrlr,
|
||||
void *fw, uint32_t size, uint32_t offset);
|
||||
|
||||
extern void nvme_request_completion_poll_cb(void *arg,
|
||||
const struct nvme_cpl *cpl);
|
||||
|
||||
extern struct nvme_ctrlr *nvme_ctrlr_attach(struct pci_device *pci_dev,
|
||||
struct nvme_ctrlr_opts *opts);
|
||||
|
||||
extern void nvme_ctrlr_detach(struct nvme_ctrlr *ctrlr);
|
||||
|
||||
extern int nvme_qpair_construct(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_qpair *qpair, enum nvme_qprio qprio,
|
||||
uint16_t entries, uint16_t trackers);
|
||||
|
||||
extern void nvme_qpair_destroy(struct nvme_qpair *qpair);
|
||||
extern void nvme_qpair_enable(struct nvme_qpair *qpair);
|
||||
extern void nvme_qpair_disable(struct nvme_qpair *qpair);
|
||||
extern int nvme_qpair_submit_request(struct nvme_qpair *qpair,
|
||||
struct nvme_request *req);
|
||||
extern void nvme_qpair_reset(struct nvme_qpair *qpair);
|
||||
extern void nvme_qpair_fail(struct nvme_qpair *qpair);
|
||||
|
||||
extern unsigned int nvme_qpair_poll(struct nvme_qpair *qpair,
|
||||
unsigned int max_completions);
|
||||
|
||||
extern int nvme_request_pool_construct(struct nvme_qpair *qpair);
|
||||
|
||||
extern void nvme_request_pool_destroy(struct nvme_qpair *qpair);
|
||||
|
||||
extern struct nvme_request *nvme_request_allocate(struct nvme_qpair *qpair,
|
||||
const struct nvme_payload *payload, uint32_t payload_size,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg);
|
||||
|
||||
extern struct nvme_request *nvme_request_allocate_null(struct nvme_qpair *qpair,
|
||||
nvme_cmd_cb cb_fn,
|
||||
void *cb_arg);
|
||||
|
||||
extern struct nvme_request *
|
||||
nvme_request_allocate_contig(struct nvme_qpair *qpair,
|
||||
void *buffer, uint32_t payload_size,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg);
|
||||
|
||||
extern void nvme_request_free(struct nvme_request *req);
|
||||
|
||||
extern void nvme_request_add_child(struct nvme_request *parent,
|
||||
struct nvme_request *child);
|
||||
|
||||
extern void nvme_request_remove_child(struct nvme_request *parent,
|
||||
struct nvme_request *child);
|
||||
|
||||
extern unsigned int nvme_ctrlr_get_quirks(struct pci_device *pdev);
|
||||
|
||||
extern int nvme_ns_construct(struct nvme_ctrlr *ctrlr,
|
||||
struct nvme_ns *ns, unsigned int id);
|
||||
|
||||
/*
|
||||
* Registers mmio access.
|
||||
*/
|
||||
#define nvme_reg_mmio_read_4(sc, reg) \
|
||||
nvme_mmio_read_4((__u32 *)&(sc)->regs->reg)
|
||||
|
||||
#define nvme_reg_mmio_read_8(sc, reg) \
|
||||
nvme_mmio_read_8((__u64 *)&(sc)->regs->reg)
|
||||
|
||||
#define nvme_reg_mmio_write_4(sc, reg, val) \
|
||||
nvme_mmio_write_4((__u32 *)&(sc)->regs->reg, val)
|
||||
|
||||
#define nvme_reg_mmio_write_8(sc, reg, val) \
|
||||
nvme_mmio_write_8((__u64 *)&(sc)->regs->reg, val)
|
||||
|
||||
#endif /* __NVME_INTERNAL_H__ */
|
||||
@@ -0,0 +1,702 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation. All rights reserved.
|
||||
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "nvme_internal.h"
|
||||
|
||||
static inline struct nvme_ns_data *nvme_ns_get_data(struct nvme_ns *ns)
|
||||
{
|
||||
return &ns->ctrlr->nsdata[ns->id - 1];
|
||||
}
|
||||
|
||||
static int nvme_ns_identify_update(struct nvme_ns *ns)
|
||||
{
|
||||
struct nvme_ctrlr *ctrlr = ns->ctrlr;
|
||||
struct nvme_ns_data *nsdata = nvme_ns_get_data(ns);
|
||||
uint32_t sector_size;
|
||||
int ret;
|
||||
|
||||
ret = nvme_admin_identify_ns(ctrlr, ns->id, nsdata);
|
||||
if (ret != 0) {
|
||||
nvme_err("nvme_identify_namespace failed\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
sector_size = 1 << nsdata->lbaf[nsdata->flbas.format].lbads;
|
||||
|
||||
ns->sector_size = sector_size;
|
||||
ns->sectors_per_max_io = ctrlr->max_xfer_size / sector_size;
|
||||
ns->sectors_per_stripe = ns->stripe_size / sector_size;
|
||||
|
||||
ns->flags = 0x0000;
|
||||
|
||||
if (ctrlr->cdata.oncs.dsm)
|
||||
ns->flags |= NVME_NS_DEALLOCATE_SUPPORTED;
|
||||
|
||||
if (ctrlr->cdata.vwc.present)
|
||||
ns->flags |= NVME_NS_FLUSH_SUPPORTED;
|
||||
|
||||
if (ctrlr->cdata.oncs.write_zeroes)
|
||||
ns->flags |= NVME_NS_WRITE_ZEROES_SUPPORTED;
|
||||
|
||||
if (nsdata->nsrescap.raw)
|
||||
ns->flags |= NVME_NS_RESERVATION_SUPPORTED;
|
||||
|
||||
ns->md_size = nsdata->lbaf[nsdata->flbas.format].ms;
|
||||
ns->pi_type = NVME_FMT_NVM_PROTECTION_DISABLE;
|
||||
|
||||
if (nsdata->lbaf[nsdata->flbas.format].ms && nsdata->dps.pit) {
|
||||
ns->flags |= NVME_NS_DPS_PI_SUPPORTED;
|
||||
ns->pi_type = nsdata->dps.pit;
|
||||
if (nsdata->flbas.extended)
|
||||
ns->flags |= NVME_NS_EXTENDED_LBA_SUPPORTED;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Initialize a namespace.
|
||||
*/
|
||||
int nvme_ns_construct(struct nvme_ctrlr *ctrlr, struct nvme_ns *ns,
|
||||
unsigned int id)
|
||||
{
|
||||
uint32_t pci_devid;
|
||||
|
||||
ns->ctrlr = ctrlr;
|
||||
ns->id = id;
|
||||
ns->stripe_size = 0;
|
||||
|
||||
nvme_pcicfg_read32(ctrlr->pci_dev, &pci_devid, 0);
|
||||
if (pci_devid == INTEL_DC_P3X00_DEVID && ctrlr->cdata.vs[3] != 0)
|
||||
ns->stripe_size = (1 << ctrlr->cdata.vs[3])
|
||||
* ctrlr->min_page_size;
|
||||
|
||||
return nvme_ns_identify_update(ns);
|
||||
}
|
||||
|
||||
/*
|
||||
* Open a namespace.
|
||||
*/
|
||||
struct nvme_ns *nvme_ns_open(struct nvme_ctrlr *ctrlr, unsigned int ns_id)
|
||||
{
|
||||
struct nvme_ns *ns = NULL;
|
||||
|
||||
pthread_mutex_lock(&ctrlr->lock);
|
||||
|
||||
if (ns_id >= 1 && ns_id <= ctrlr->nr_ns) {
|
||||
ns = &ctrlr->ns[ns_id - 1];
|
||||
ns->open_count++;
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&ctrlr->lock);
|
||||
|
||||
return ns;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the controller of an open name space and lock it,
|
||||
* making sure in the process that the ns handle is valid.
|
||||
*/
|
||||
static struct nvme_ctrlr *nvme_ns_ctrlr_lock(struct nvme_ns *ns)
|
||||
{
|
||||
struct nvme_ctrlr *ctrlr;
|
||||
|
||||
if (!ns)
|
||||
return NULL;
|
||||
|
||||
ctrlr = ns->ctrlr;
|
||||
if (ns->id < 1 ||
|
||||
ns->id > ctrlr->nr_ns ||
|
||||
ns != &ctrlr->ns[ns->id - 1])
|
||||
return NULL;
|
||||
|
||||
pthread_mutex_lock(&ctrlr->lock);
|
||||
|
||||
/*
|
||||
* Between the check and lock, the ns may have gone away.
|
||||
* So check again, and make sure that the name space is open.
|
||||
*/
|
||||
if (ns->id > ctrlr->nr_ns ||
|
||||
ns != &ctrlr->ns[ns->id - 1] ||
|
||||
ns->open_count == 0) {
|
||||
pthread_mutex_unlock(&ctrlr->lock);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return ctrlr;
|
||||
}
|
||||
|
||||
/*
|
||||
* Close an open namespace.
|
||||
*/
|
||||
int nvme_ns_close(struct nvme_ns *ns)
|
||||
{
|
||||
struct nvme_ctrlr *ctrlr;
|
||||
|
||||
ctrlr = nvme_ns_ctrlr_lock(ns);
|
||||
if (!ctrlr) {
|
||||
nvme_err("Invalid name space handle\n");
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
ns->open_count--;
|
||||
|
||||
pthread_mutex_unlock(&ctrlr->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get namespace information
|
||||
*/
|
||||
int nvme_ns_stat(struct nvme_ns *ns, struct nvme_ns_stat *ns_stat)
|
||||
{
|
||||
struct nvme_ctrlr *ctrlr;
|
||||
|
||||
ctrlr = nvme_ns_ctrlr_lock(ns);
|
||||
if (!ctrlr) {
|
||||
nvme_err("Invalid name space handle\n");
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
ns_stat->id = ns->id;
|
||||
ns_stat->sector_size = ns->sector_size;
|
||||
ns_stat->sectors = nvme_ns_get_data(ns)->nsze;
|
||||
ns_stat->flags = ns->flags;
|
||||
ns_stat->pi_type = ns->pi_type;
|
||||
ns_stat->md_size = ns->md_size;
|
||||
|
||||
pthread_mutex_unlock(&ctrlr->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get namespace data
|
||||
*/
|
||||
int nvme_ns_data(struct nvme_ns *ns, struct nvme_ns_data *nsdata)
|
||||
{
|
||||
struct nvme_ctrlr *ctrlr;
|
||||
|
||||
ctrlr = nvme_ns_ctrlr_lock(ns);
|
||||
if (!ctrlr) {
|
||||
nvme_err("Invalid name space handle\n");
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
memcpy(nsdata, nvme_ns_get_data(ns), sizeof(struct nvme_ns_data));
|
||||
|
||||
pthread_mutex_unlock(&ctrlr->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct nvme_request *_nvme_ns_rw(struct nvme_ns *ns,
|
||||
struct nvme_qpair *qpair,
|
||||
const struct nvme_payload *payload, uint64_t lba,
|
||||
uint32_t lba_count, nvme_cmd_cb cb_fn,
|
||||
void *cb_arg, uint32_t opc, uint32_t io_flags,
|
||||
uint16_t apptag_mask, uint16_t apptag);
|
||||
|
||||
static struct nvme_request *
|
||||
_nvme_ns_split_request(struct nvme_ns *ns,
|
||||
struct nvme_qpair *qpair,
|
||||
const struct nvme_payload *payload,
|
||||
uint64_t lba, uint32_t lba_count,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg,
|
||||
uint32_t opc,
|
||||
uint32_t io_flags,
|
||||
struct nvme_request *req,
|
||||
uint32_t sectors_per_max_io,
|
||||
uint32_t sector_mask,
|
||||
uint16_t apptag_mask,
|
||||
uint16_t apptag)
|
||||
{
|
||||
uint32_t sector_size = ns->sector_size;
|
||||
uint32_t md_size = ns->md_size;
|
||||
uint32_t remaining_lba_count = lba_count;
|
||||
uint32_t offset = 0;
|
||||
uint32_t md_offset = 0;
|
||||
struct nvme_request *child, *tmp;
|
||||
|
||||
if (ns->flags & NVME_NS_DPS_PI_SUPPORTED) {
|
||||
/* for extended LBA only */
|
||||
if ((ns->flags & NVME_NS_EXTENDED_LBA_SUPPORTED)
|
||||
&& !(io_flags & NVME_IO_FLAGS_PRACT))
|
||||
sector_size += ns->md_size;
|
||||
}
|
||||
|
||||
while (remaining_lba_count > 0) {
|
||||
|
||||
lba_count = sectors_per_max_io - (lba & sector_mask);
|
||||
lba_count = nvme_min(remaining_lba_count, lba_count);
|
||||
|
||||
child = _nvme_ns_rw(ns, qpair, payload, lba, lba_count, cb_fn,
|
||||
cb_arg, opc, io_flags, apptag_mask, apptag);
|
||||
if (child == NULL) {
|
||||
if (req->child_reqs) {
|
||||
/* free all child nvme_request */
|
||||
TAILQ_FOREACH_SAFE(child, &req->children,
|
||||
child_tailq, tmp) {
|
||||
nvme_request_remove_child(req, child);
|
||||
nvme_request_free(child);
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
child->payload_offset = offset;
|
||||
|
||||
/* for separate metadata buffer only */
|
||||
if (payload->md)
|
||||
child->md_offset = md_offset;
|
||||
|
||||
nvme_request_add_child(req, child);
|
||||
|
||||
remaining_lba_count -= lba_count;
|
||||
lba += lba_count;
|
||||
offset += lba_count * sector_size;
|
||||
md_offset += lba_count * md_size;
|
||||
|
||||
}
|
||||
|
||||
return req;
|
||||
}
|
||||
|
||||
static struct nvme_request *_nvme_ns_rw(struct nvme_ns *ns,
|
||||
struct nvme_qpair *qpair,
|
||||
const struct nvme_payload *payload,
|
||||
uint64_t lba, uint32_t lba_count,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg,
|
||||
uint32_t opc,
|
||||
uint32_t io_flags,
|
||||
uint16_t apptag_mask,
|
||||
uint16_t apptag)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_cmd *cmd;
|
||||
uint64_t *tmp_lba;
|
||||
uint32_t sector_size;
|
||||
uint32_t sectors_per_max_io;
|
||||
uint32_t sectors_per_stripe;
|
||||
|
||||
/* The bottom 16 bits must be empty */
|
||||
if (io_flags & 0xFFFF)
|
||||
return NULL;
|
||||
|
||||
sector_size = ns->sector_size;
|
||||
sectors_per_max_io = ns->sectors_per_max_io;
|
||||
sectors_per_stripe = ns->sectors_per_stripe;
|
||||
|
||||
if (ns->flags & NVME_NS_DPS_PI_SUPPORTED)
|
||||
/* for extended LBA only */
|
||||
if ((ns->flags & NVME_NS_EXTENDED_LBA_SUPPORTED) &&
|
||||
!(io_flags & NVME_IO_FLAGS_PRACT))
|
||||
sector_size += ns->md_size;
|
||||
|
||||
req = nvme_request_allocate(qpair, payload,
|
||||
lba_count * sector_size, cb_fn, cb_arg);
|
||||
if (req == NULL)
|
||||
return NULL;
|
||||
|
||||
/*
|
||||
* Intel DC P3*00 NVMe controllers benefit from driver-assisted striping.
|
||||
* If this controller defines a stripe boundary and this I/O spans
|
||||
* a stripe boundary, split the request into multiple requests and
|
||||
* submit each separately to hardware.
|
||||
*/
|
||||
if (sectors_per_stripe > 0 &&
|
||||
(((lba & (sectors_per_stripe - 1)) + lba_count) > sectors_per_stripe))
|
||||
return _nvme_ns_split_request(ns, qpair, payload, lba,
|
||||
lba_count, cb_fn, cb_arg, opc,
|
||||
io_flags, req, sectors_per_stripe,
|
||||
sectors_per_stripe - 1,
|
||||
apptag_mask, apptag);
|
||||
|
||||
if (lba_count > sectors_per_max_io)
|
||||
return _nvme_ns_split_request(ns, qpair, payload, lba,
|
||||
lba_count, cb_fn, cb_arg, opc,
|
||||
io_flags, req, sectors_per_max_io,
|
||||
0, apptag_mask, apptag);
|
||||
|
||||
cmd = &req->cmd;
|
||||
cmd->opc = opc;
|
||||
cmd->nsid = ns->id;
|
||||
|
||||
tmp_lba = (uint64_t *)&cmd->cdw10;
|
||||
*tmp_lba = lba;
|
||||
|
||||
if (ns->flags & NVME_NS_DPS_PI_SUPPORTED) {
|
||||
switch (ns->pi_type) {
|
||||
case NVME_FMT_NVM_PROTECTION_TYPE1:
|
||||
case NVME_FMT_NVM_PROTECTION_TYPE2:
|
||||
cmd->cdw14 = (uint32_t)lba;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
cmd->cdw12 = lba_count - 1;
|
||||
cmd->cdw12 |= io_flags;
|
||||
|
||||
cmd->cdw15 = apptag_mask;
|
||||
cmd->cdw15 = (cmd->cdw15 << 16 | apptag);
|
||||
|
||||
return req;
|
||||
}
|
||||
|
||||
int nvme_ns_read(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
void *buffer,
|
||||
uint64_t lba, uint32_t lba_count,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg,
|
||||
unsigned int io_flags)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_payload payload;
|
||||
|
||||
payload.type = NVME_PAYLOAD_TYPE_CONTIG;
|
||||
payload.u.contig = buffer;
|
||||
payload.md = NULL;
|
||||
|
||||
req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg,
|
||||
NVME_OPC_READ, io_flags, 0, 0);
|
||||
if (req != NULL)
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
int nvme_ns_read_with_md(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
void *buffer, void *metadata,
|
||||
uint64_t lba, uint32_t lba_count,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg,
|
||||
unsigned int io_flags,
|
||||
uint16_t apptag_mask, uint16_t apptag)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_payload payload;
|
||||
|
||||
payload.type = NVME_PAYLOAD_TYPE_CONTIG;
|
||||
payload.u.contig = buffer;
|
||||
payload.md = metadata;
|
||||
|
||||
req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg,
|
||||
NVME_OPC_READ, io_flags, apptag_mask, apptag);
|
||||
if (req != NULL)
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
int nvme_ns_readv(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
uint64_t lba, uint32_t lba_count,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg,
|
||||
unsigned int io_flags,
|
||||
nvme_req_reset_sgl_cb reset_sgl_fn,
|
||||
nvme_req_next_sge_cb next_sge_fn)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_payload payload;
|
||||
|
||||
if (reset_sgl_fn == NULL || next_sge_fn == NULL)
|
||||
return EINVAL;
|
||||
|
||||
payload.type = NVME_PAYLOAD_TYPE_SGL;
|
||||
payload.md = NULL;
|
||||
payload.u.sgl.reset_sgl_fn = reset_sgl_fn;
|
||||
payload.u.sgl.next_sge_fn = next_sge_fn;
|
||||
payload.u.sgl.cb_arg = cb_arg;
|
||||
|
||||
req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg,
|
||||
NVME_OPC_READ, io_flags, 0, 0);
|
||||
if (req != NULL)
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
int nvme_ns_write(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
void *buffer,
|
||||
uint64_t lba, uint32_t lba_count,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg,
|
||||
unsigned int io_flags)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_payload payload;
|
||||
|
||||
payload.type = NVME_PAYLOAD_TYPE_CONTIG;
|
||||
payload.u.contig = buffer;
|
||||
payload.md = NULL;
|
||||
|
||||
req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg,
|
||||
NVME_OPC_WRITE, io_flags, 0, 0);
|
||||
if (req != NULL)
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
int nvme_ns_write_with_md(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
void *buffer, void *metadata,
|
||||
uint64_t lba, uint32_t lba_count,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg,
|
||||
unsigned int io_flags,
|
||||
uint16_t apptag_mask, uint16_t apptag)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_payload payload;
|
||||
|
||||
payload.type = NVME_PAYLOAD_TYPE_CONTIG;
|
||||
payload.u.contig = buffer;
|
||||
payload.md = metadata;
|
||||
|
||||
req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg,
|
||||
NVME_OPC_WRITE, io_flags, apptag_mask, apptag);
|
||||
if (req != NULL)
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
int nvme_ns_writev(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
uint64_t lba, uint32_t lba_count,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg,
|
||||
unsigned int io_flags,
|
||||
nvme_req_reset_sgl_cb reset_sgl_fn,
|
||||
nvme_req_next_sge_cb next_sge_fn)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_payload payload;
|
||||
|
||||
if (reset_sgl_fn == NULL || next_sge_fn == NULL)
|
||||
return EINVAL;
|
||||
|
||||
payload.type = NVME_PAYLOAD_TYPE_SGL;
|
||||
payload.md = NULL;
|
||||
payload.u.sgl.reset_sgl_fn = reset_sgl_fn;
|
||||
payload.u.sgl.next_sge_fn = next_sge_fn;
|
||||
payload.u.sgl.cb_arg = cb_arg;
|
||||
|
||||
req = _nvme_ns_rw(ns, qpair, &payload, lba, lba_count, cb_fn, cb_arg,
|
||||
NVME_OPC_WRITE, io_flags, 0, 0);
|
||||
if (req != NULL)
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
int nvme_ns_write_zeroes(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
uint64_t lba, uint32_t lba_count,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg,
|
||||
unsigned int io_flags)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_cmd *cmd;
|
||||
uint64_t *tmp_lba;
|
||||
|
||||
if (lba_count == 0)
|
||||
return EINVAL;
|
||||
|
||||
req = nvme_request_allocate_null(qpair, cb_fn, cb_arg);
|
||||
if (req == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
cmd = &req->cmd;
|
||||
cmd->opc = NVME_OPC_WRITE_ZEROES;
|
||||
cmd->nsid = ns->id;
|
||||
|
||||
tmp_lba = (uint64_t *)&cmd->cdw10;
|
||||
*tmp_lba = lba;
|
||||
cmd->cdw12 = lba_count - 1;
|
||||
cmd->cdw12 |= io_flags;
|
||||
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
}
|
||||
|
||||
int nvme_ns_deallocate(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
void *payload, uint16_t ranges,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_cmd *cmd;
|
||||
|
||||
if (ranges == 0 || ranges > NVME_DATASET_MANAGEMENT_MAX_RANGES)
|
||||
return EINVAL;
|
||||
|
||||
req = nvme_request_allocate_contig(qpair, payload,
|
||||
ranges * sizeof(struct nvme_dsm_range),
|
||||
cb_fn, cb_arg);
|
||||
if (req == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
cmd = &req->cmd;
|
||||
cmd->opc = NVME_OPC_DATASET_MANAGEMENT;
|
||||
cmd->nsid = ns->id;
|
||||
|
||||
/* TODO: create a delete command data structure */
|
||||
cmd->cdw10 = ranges - 1;
|
||||
cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE;
|
||||
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
}
|
||||
|
||||
int nvme_ns_flush(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_cmd *cmd;
|
||||
|
||||
req = nvme_request_allocate_null(qpair, cb_fn, cb_arg);
|
||||
if (req == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
cmd = &req->cmd;
|
||||
cmd->opc = NVME_OPC_FLUSH;
|
||||
cmd->nsid = ns->id;
|
||||
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
}
|
||||
|
||||
int nvme_ns_reservation_register(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
struct nvme_reservation_register_data *payload,
|
||||
bool ignore_key,
|
||||
enum nvme_reservation_register_action action,
|
||||
enum nvme_reservation_register_cptpl cptpl,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_cmd *cmd;
|
||||
|
||||
req = nvme_request_allocate_contig(qpair, payload,
|
||||
sizeof(struct nvme_reservation_register_data),
|
||||
cb_fn, cb_arg);
|
||||
if (req == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
cmd = &req->cmd;
|
||||
cmd->opc = NVME_OPC_RESERVATION_REGISTER;
|
||||
cmd->nsid = ns->id;
|
||||
|
||||
/* Bits 0-2 */
|
||||
cmd->cdw10 = action;
|
||||
/* Bit 3 */
|
||||
cmd->cdw10 |= ignore_key ? 1 << 3 : 0;
|
||||
/* Bits 30-31 */
|
||||
cmd->cdw10 |= (uint32_t)cptpl << 30;
|
||||
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
}
|
||||
|
||||
int nvme_ns_reservation_release(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
struct nvme_reservation_key_data *payload,
|
||||
bool ignore_key,
|
||||
enum nvme_reservation_release_action action,
|
||||
enum nvme_reservation_type type,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_cmd *cmd;
|
||||
|
||||
req = nvme_request_allocate_contig(qpair, payload,
|
||||
sizeof(struct nvme_reservation_key_data),
|
||||
cb_fn, cb_arg);
|
||||
if (req == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
cmd = &req->cmd;
|
||||
cmd->opc = NVME_OPC_RESERVATION_RELEASE;
|
||||
cmd->nsid = ns->id;
|
||||
|
||||
/* Bits 0-2 */
|
||||
cmd->cdw10 = action;
|
||||
/* Bit 3 */
|
||||
cmd->cdw10 |= ignore_key ? 1 << 3 : 0;
|
||||
/* Bits 8-15 */
|
||||
cmd->cdw10 |= (uint32_t)type << 8;
|
||||
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
}
|
||||
|
||||
int nvme_ns_reservation_acquire(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
struct nvme_reservation_acquire_data *payload,
|
||||
bool ignore_key,
|
||||
enum nvme_reservation_acquire_action action,
|
||||
enum nvme_reservation_type type,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
struct nvme_cmd *cmd;
|
||||
|
||||
req = nvme_request_allocate_contig(qpair, payload,
|
||||
sizeof(struct nvme_reservation_acquire_data),
|
||||
cb_fn, cb_arg);
|
||||
if (req == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
cmd = &req->cmd;
|
||||
cmd->opc = NVME_OPC_RESERVATION_ACQUIRE;
|
||||
cmd->nsid = ns->id;
|
||||
|
||||
/* Bits 0-2 */
|
||||
cmd->cdw10 = action;
|
||||
/* Bit 3 */
|
||||
cmd->cdw10 |= ignore_key ? 1 << 3 : 0;
|
||||
/* Bits 8-15 */
|
||||
cmd->cdw10 |= (uint32_t)type << 8;
|
||||
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
}
|
||||
|
||||
int nvme_ns_reservation_report(struct nvme_ns *ns, struct nvme_qpair *qpair,
|
||||
void *payload, size_t len,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg)
|
||||
{
|
||||
uint32_t num_dwords;
|
||||
struct nvme_request *req;
|
||||
struct nvme_cmd *cmd;
|
||||
|
||||
if (len % 4)
|
||||
return EINVAL;
|
||||
num_dwords = len / 4;
|
||||
|
||||
req = nvme_request_allocate_contig(qpair, payload, len, cb_fn, cb_arg);
|
||||
if (req == NULL)
|
||||
return ENOMEM;
|
||||
|
||||
cmd = &req->cmd;
|
||||
cmd->opc = NVME_OPC_RESERVATION_REPORT;
|
||||
cmd->nsid = ns->id;
|
||||
|
||||
cmd->cdw10 = num_dwords;
|
||||
|
||||
return nvme_qpair_submit_request(qpair, req);
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation. All rights reserved.
|
||||
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __NVME_PCI_H__
|
||||
#define __NVME_PCI_H__
|
||||
|
||||
#include "nvme_common.h"
|
||||
|
||||
#ifndef __HAIKU__
|
||||
#include <pciaccess.h>
|
||||
#endif
|
||||
|
||||
#define NVME_PCI_PATH_MAX 256
|
||||
#define NVME_PCI_CFG_SIZE 256
|
||||
#define NVME_PCI_EXT_CAP_ID_SN 0x03
|
||||
|
||||
#define NVME_PCI_ANY_ID 0xffff
|
||||
#define NVME_PCI_VID_INTEL 0x8086
|
||||
#define NVME_PCI_VID_MEMBLAZE 0x1c5f
|
||||
|
||||
/*
|
||||
* PCI class code for NVMe devices.
|
||||
*
|
||||
* Base class code 01h: mass storage
|
||||
* Subclass code 08h: non-volatile memory
|
||||
* Programming interface 02h: NVM Express
|
||||
*/
|
||||
#define NVME_PCI_CLASS 0x010802
|
||||
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB0 0x3c20
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB1 0x3c21
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB2 0x3c22
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB3 0x3c23
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB4 0x3c24
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB5 0x3c25
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB6 0x3c26
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB7 0x3c27
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB8 0x3c2e
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_SNB9 0x3c2f
|
||||
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB0 0x0e20
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB1 0x0e21
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB2 0x0e22
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB3 0x0e23
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB4 0x0e24
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB5 0x0e25
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB6 0x0e26
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB7 0x0e27
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB8 0x0e2e
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_IVB9 0x0e2f
|
||||
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW0 0x2f20
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW1 0x2f21
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW2 0x2f22
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW3 0x2f23
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW4 0x2f24
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW5 0x2f25
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW6 0x2f26
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW7 0x2f27
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW8 0x2f2e
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_HSW9 0x2f2f
|
||||
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BWD0 0x0C50
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BWD1 0x0C51
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BWD2 0x0C52
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BWD3 0x0C53
|
||||
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDXDE0 0x6f50
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDXDE1 0x6f51
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDXDE2 0x6f52
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDXDE3 0x6f53
|
||||
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX0 0x6f20
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX1 0x6f21
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX2 0x6f22
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX3 0x6f23
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX4 0x6f24
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX5 0x6f25
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX6 0x6f26
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX7 0x6f27
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX8 0x6f2e
|
||||
#define NVME_PCI_DEVICE_ID_INTEL_IOAT_BDX9 0x6f2f
|
||||
|
||||
struct pci_slot_match;
|
||||
|
||||
struct pci_id {
|
||||
uint16_t vendor_id;
|
||||
uint16_t device_id;
|
||||
uint16_t subvendor_id;
|
||||
uint16_t subdevice_id;
|
||||
};
|
||||
|
||||
/*
|
||||
* Initialize PCI subsystem.
|
||||
*/
|
||||
extern int nvme_pci_init(void);
|
||||
|
||||
/*
|
||||
* Search a PCI device and grab it if found.
|
||||
*/
|
||||
extern struct pci_device *
|
||||
nvme_pci_device_probe(const struct pci_slot_match *slot);
|
||||
|
||||
/*
|
||||
* Reset a PCI device.
|
||||
*/
|
||||
extern int nvme_pci_device_reset(struct pci_device *dev);
|
||||
|
||||
/*
|
||||
* Get a device serial number.
|
||||
*/
|
||||
extern int nvme_pci_device_get_serial_number(struct pci_device *dev,
|
||||
char *sn, size_t len);
|
||||
|
||||
/*
|
||||
* Compare two devices.
|
||||
* Return 0 if the devices are the same, 1 otherwise.
|
||||
*/
|
||||
static inline int nvme_pci_dev_cmp(struct pci_device *pci_dev1,
|
||||
struct pci_device *pci_dev2)
|
||||
{
|
||||
if (pci_dev1 == pci_dev2)
|
||||
return 0;
|
||||
|
||||
if (pci_dev1->domain == pci_dev2->domain &&
|
||||
pci_dev1->bus == pci_dev2->bus &&
|
||||
pci_dev1->dev == pci_dev2->dev &&
|
||||
pci_dev1->func == pci_dev2->func)
|
||||
return 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get a device PCI ID.
|
||||
*/
|
||||
static inline void nvme_pci_get_pci_id(struct pci_device *pci_dev,
|
||||
struct pci_id *pci_id)
|
||||
{
|
||||
pci_id->vendor_id = pci_dev->vendor_id;
|
||||
pci_id->device_id = pci_dev->device_id;
|
||||
pci_id->subvendor_id = pci_dev->subvendor_id;
|
||||
pci_id->subdevice_id = pci_dev->subdevice_id;
|
||||
}
|
||||
|
||||
#ifdef __HAIKU__
|
||||
int nvme_pcicfg_read8(struct pci_device *dev, uint8_t *value, uint32_t offset);
|
||||
int nvme_pcicfg_write8(struct pci_device *dev, uint8_t value, uint32_t offset);
|
||||
int nvme_pcicfg_read16(struct pci_device *dev, uint16_t *value, uint32_t offset);
|
||||
int nvme_pcicfg_write16(struct pci_device *dev, uint16_t value, uint32_t offset);
|
||||
int nvme_pcicfg_read32(struct pci_device *dev, uint32_t *value, uint32_t offset);
|
||||
int nvme_pcicfg_write32(struct pci_device *dev, uint32_t value, uint32_t offset);
|
||||
int nvme_pcicfg_map_bar(void *devhandle, unsigned int bar, bool read_only,
|
||||
void **mapped_addr);
|
||||
int nvme_pcicfg_map_bar_write_combine(void *devhandle, unsigned int bar,
|
||||
void **mapped_addr);
|
||||
int nvme_pcicfg_unmap_bar(void *devhandle, unsigned int bar, void *addr);
|
||||
void nvme_pcicfg_get_bar_addr_len(void *devhandle, unsigned int bar,
|
||||
uint64_t *addr, uint64_t *size);
|
||||
#else
|
||||
/*
|
||||
* Read a device config register.
|
||||
*/
|
||||
static inline int nvme_pcicfg_read8(struct pci_device *dev,
|
||||
uint8_t *value, uint32_t offset)
|
||||
{
|
||||
return pci_device_cfg_read_u8(dev, value, offset);
|
||||
}
|
||||
|
||||
/*
|
||||
* Write a device config register.
|
||||
*/
|
||||
static inline int nvme_pcicfg_write8(struct pci_device *dev,
|
||||
uint8_t value, uint32_t offset)
|
||||
{
|
||||
return pci_device_cfg_write_u8(dev, value, offset);
|
||||
}
|
||||
|
||||
/*
|
||||
* Read a device config register.
|
||||
*/
|
||||
static inline int nvme_pcicfg_read16(struct pci_device *dev,
|
||||
uint16_t *value, uint32_t offset)
|
||||
{
|
||||
return pci_device_cfg_read_u16(dev, value, offset);
|
||||
}
|
||||
|
||||
/*
|
||||
* Write a device config register.
|
||||
*/
|
||||
static inline int nvme_pcicfg_write16(struct pci_device *dev,
|
||||
uint16_t value, uint32_t offset)
|
||||
{
|
||||
return pci_device_cfg_write_u16(dev, value, offset);
|
||||
}
|
||||
|
||||
/*
|
||||
* Read a device config register.
|
||||
*/
|
||||
static inline int nvme_pcicfg_read32(struct pci_device *dev,
|
||||
uint32_t *value, uint32_t offset)
|
||||
{
|
||||
return pci_device_cfg_read_u32(dev, value, offset);
|
||||
}
|
||||
|
||||
/*
|
||||
* Write a device config register.
|
||||
*/
|
||||
static inline int nvme_pcicfg_write32(struct pci_device *dev,
|
||||
uint32_t value, uint32_t offset)
|
||||
{
|
||||
return pci_device_cfg_write_u32(dev, value, offset);
|
||||
}
|
||||
|
||||
/*
|
||||
* Map a device PCI BAR.
|
||||
*/
|
||||
static inline int nvme_pcicfg_map_bar(void *devhandle, unsigned int bar,
|
||||
bool read_only, void **mapped_addr)
|
||||
{
|
||||
struct pci_device *dev = devhandle;
|
||||
uint32_t flags = (read_only ? 0 : PCI_DEV_MAP_FLAG_WRITABLE);
|
||||
|
||||
return pci_device_map_range(dev, dev->regions[bar].base_addr,
|
||||
dev->regions[bar].size, flags, mapped_addr);
|
||||
}
|
||||
|
||||
/*
|
||||
* Map a device PCI BAR (write combine).
|
||||
*/
|
||||
static inline int nvme_pcicfg_map_bar_write_combine(void *devhandle,
|
||||
unsigned int bar,
|
||||
void **mapped_addr)
|
||||
{
|
||||
struct pci_device *dev = devhandle;
|
||||
uint32_t flags = PCI_DEV_MAP_FLAG_WRITABLE |
|
||||
PCI_DEV_MAP_FLAG_WRITE_COMBINE;
|
||||
|
||||
return pci_device_map_range(dev, dev->regions[bar].base_addr,
|
||||
dev->regions[bar].size, flags, mapped_addr);
|
||||
}
|
||||
|
||||
/*
|
||||
* Unmap a device PCI BAR.
|
||||
*/
|
||||
static inline int nvme_pcicfg_unmap_bar(void *devhandle, unsigned int bar,
|
||||
void *addr)
|
||||
{
|
||||
struct pci_device *dev = devhandle;
|
||||
|
||||
return pci_device_unmap_range(dev, addr, dev->regions[bar].size);
|
||||
}
|
||||
|
||||
/*
|
||||
* Get a device PCI BAR address and length.
|
||||
*/
|
||||
static inline void nvme_pcicfg_get_bar_addr_len(void *devhandle,
|
||||
unsigned int bar,
|
||||
uint64_t *addr, uint64_t *size)
|
||||
{
|
||||
struct pci_device *dev = devhandle;
|
||||
|
||||
*addr = (uint64_t)dev->regions[bar].base_addr;
|
||||
*size = (uint64_t)dev->regions[bar].size;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* __NVME_PCI_H__ */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,111 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation. All rights reserved.
|
||||
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "nvme_internal.h"
|
||||
#include "nvme_pci.h"
|
||||
|
||||
struct nvme_quirk {
|
||||
struct pci_id id;
|
||||
unsigned int flags;
|
||||
};
|
||||
|
||||
static const struct nvme_quirk nvme_quirks[] = {
|
||||
{
|
||||
{ NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x3702 },
|
||||
NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY
|
||||
},
|
||||
{
|
||||
{ NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x3703 },
|
||||
NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY
|
||||
},
|
||||
{
|
||||
{ NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x3704 },
|
||||
NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY
|
||||
},
|
||||
{
|
||||
{ NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x3705 },
|
||||
NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY
|
||||
},
|
||||
{
|
||||
{ NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x3709 },
|
||||
NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY
|
||||
},
|
||||
{
|
||||
{ NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x370a },
|
||||
NVME_INTEL_QUIRK_READ_LATENCY | NVME_INTEL_QUIRK_WRITE_LATENCY
|
||||
},
|
||||
{
|
||||
{ NVME_PCI_VID_MEMBLAZE, 0x0540, NVME_PCI_ANY_ID, NVME_PCI_ANY_ID },
|
||||
NVME_QUIRK_DELAY_BEFORE_CHK_RDY
|
||||
},
|
||||
{
|
||||
{ NVME_PCI_VID_INTEL, 0x0953, NVME_PCI_VID_INTEL, 0x370d },
|
||||
NVME_QUIRK_DELAY_AFTER_RDY
|
||||
},
|
||||
{
|
||||
{ 0x0000, 0x0000, 0x0000, 0x0000 },
|
||||
0
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
* Compare each field. NVME_PCI_ANY_ID in s1 matches everything.
|
||||
*/
|
||||
static bool nvme_quirks_pci_id_match(const struct pci_id *id,
|
||||
struct pci_device *pdev)
|
||||
{
|
||||
if ((id->vendor_id == NVME_PCI_ANY_ID ||
|
||||
id->vendor_id == pdev->vendor_id) &&
|
||||
(id->device_id == NVME_PCI_ANY_ID ||
|
||||
id->device_id == pdev->device_id) &&
|
||||
(id->subvendor_id == NVME_PCI_ANY_ID ||
|
||||
id->subvendor_id == pdev->subvendor_id) &&
|
||||
(id->subdevice_id == NVME_PCI_ANY_ID ||
|
||||
id->subdevice_id == pdev->subdevice_id))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
unsigned int nvme_ctrlr_get_quirks(struct pci_device *pdev)
|
||||
{
|
||||
const struct nvme_quirk *quirk = nvme_quirks;
|
||||
|
||||
while (quirk->id.vendor_id) {
|
||||
if (nvme_quirks_pci_id_match(&quirk->id, pdev))
|
||||
return quirk->flags;
|
||||
quirk++;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation. All rights reserved.
|
||||
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "nvme_internal.h"
|
||||
|
||||
/*
|
||||
* Allocate a request descriptor from the queue pair free list.
|
||||
*/
|
||||
static struct nvme_request *nvme_alloc_request(struct nvme_qpair *qpair)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
|
||||
req = STAILQ_FIRST(&qpair->free_req);
|
||||
if (req) {
|
||||
STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
|
||||
memset(&req->cmd, 0, sizeof(struct nvme_cmd));
|
||||
}
|
||||
|
||||
return req;
|
||||
}
|
||||
|
||||
static void nvme_request_cb_complete_child(void *child_arg,
|
||||
const struct nvme_cpl *cpl)
|
||||
{
|
||||
struct nvme_request *child = child_arg;
|
||||
struct nvme_request *parent = child->parent;
|
||||
|
||||
nvme_request_remove_child(parent, child);
|
||||
|
||||
if (nvme_cpl_is_error(cpl))
|
||||
memcpy(&parent->parent_status, cpl, sizeof(*cpl));
|
||||
|
||||
if (parent->child_reqs == 0) {
|
||||
if (parent->cb_fn)
|
||||
parent->cb_fn(parent->cb_arg, &parent->parent_status);
|
||||
nvme_request_free(parent);
|
||||
}
|
||||
}
|
||||
|
||||
void nvme_request_completion_poll_cb(void *arg, const struct nvme_cpl *cpl)
|
||||
{
|
||||
struct nvme_completion_poll_status *status = arg;
|
||||
|
||||
memcpy(&status->cpl, cpl, sizeof(*cpl));
|
||||
status->done = true;
|
||||
}
|
||||
|
||||
int nvme_request_pool_construct(struct nvme_qpair *qpair)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
unsigned int i;
|
||||
|
||||
qpair->num_reqs = qpair->trackers * NVME_IO_ENTRIES_VS_TRACKERS_RATIO;
|
||||
qpair->reqs = calloc(qpair->num_reqs, sizeof(struct nvme_request));
|
||||
if (!qpair->reqs) {
|
||||
nvme_err("QPair %d: allocate %u requests failed\n",
|
||||
(int)qpair->id, qpair->num_reqs);
|
||||
return ENOMEM;
|
||||
}
|
||||
|
||||
nvme_info("QPair %d: %d requests in pool\n",
|
||||
(int)qpair->id,
|
||||
(int)qpair->num_reqs);
|
||||
|
||||
for(i = 0; i < qpair->num_reqs; i++) {
|
||||
req = &qpair->reqs[i];
|
||||
req->qpair = qpair;
|
||||
STAILQ_INSERT_TAIL(&qpair->free_req, req, stailq);
|
||||
req++;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void nvme_request_pool_destroy(struct nvme_qpair *qpair)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
unsigned int n = 0;
|
||||
|
||||
while ((req = STAILQ_FIRST(&qpair->free_req))) {
|
||||
STAILQ_REMOVE_HEAD(&qpair->free_req, stailq);
|
||||
n++;
|
||||
}
|
||||
|
||||
if (n != qpair->num_reqs)
|
||||
nvme_err("QPair %d: Freed %d/%d requests\n",
|
||||
(int)qpair->id, n, (int)qpair->num_reqs);
|
||||
|
||||
free(qpair->reqs);
|
||||
}
|
||||
|
||||
struct nvme_request *nvme_request_allocate(struct nvme_qpair *qpair,
|
||||
const struct nvme_payload *payload,
|
||||
uint32_t payload_size,
|
||||
nvme_cmd_cb cb_fn,
|
||||
void *cb_arg)
|
||||
{
|
||||
struct nvme_request *req;
|
||||
|
||||
req = nvme_alloc_request(qpair);
|
||||
if (req == NULL)
|
||||
return NULL;
|
||||
|
||||
/*
|
||||
* Only memset up to (but not including) the children TAILQ_ENTRY.
|
||||
* Children, and following members, are only used as part of I/O
|
||||
* splitting so we avoid memsetting them until it is actually needed.
|
||||
* They will be initialized in nvme_request_add_child()
|
||||
* if the request is split.
|
||||
*/
|
||||
memset(req, 0, offsetof(struct nvme_request, children));
|
||||
req->cb_fn = cb_fn;
|
||||
req->cb_arg = cb_arg;
|
||||
req->payload = *payload;
|
||||
req->payload_size = payload_size;
|
||||
|
||||
return req;
|
||||
}
|
||||
|
||||
struct nvme_request *nvme_request_allocate_contig(struct nvme_qpair *qpair,
|
||||
void *buffer,
|
||||
uint32_t payload_size,
|
||||
nvme_cmd_cb cb_fn,
|
||||
void *cb_arg)
|
||||
{
|
||||
struct nvme_payload payload;
|
||||
|
||||
payload.type = NVME_PAYLOAD_TYPE_CONTIG;
|
||||
payload.u.contig = buffer;
|
||||
payload.md = NULL;
|
||||
|
||||
return nvme_request_allocate(qpair, &payload, payload_size,
|
||||
cb_fn, cb_arg);
|
||||
}
|
||||
|
||||
struct nvme_request *nvme_request_allocate_null(struct nvme_qpair *qpair,
|
||||
nvme_cmd_cb cb_fn, void *cb_arg)
|
||||
{
|
||||
return nvme_request_allocate_contig(qpair, NULL, 0, cb_fn, cb_arg);
|
||||
}
|
||||
|
||||
void nvme_request_free(struct nvme_request *req)
|
||||
{
|
||||
struct nvme_qpair *qpair = req->qpair;
|
||||
|
||||
nvme_assert(req->child_reqs == 0, "Number of child request not 0\n");
|
||||
|
||||
STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
|
||||
}
|
||||
|
||||
void nvme_request_add_child(struct nvme_request *parent,
|
||||
struct nvme_request *child)
|
||||
{
|
||||
if (parent->child_reqs == 0) {
|
||||
/*
|
||||
* Defer initialization of the children TAILQ since it falls
|
||||
* on a separate cacheline. This ensures we do not touch this
|
||||
* cacheline except on request splitting cases, which are
|
||||
* relatively rare.
|
||||
*/
|
||||
TAILQ_INIT(&parent->children);
|
||||
parent->parent = NULL;
|
||||
memset(&parent->parent_status, 0, sizeof(struct nvme_cpl));
|
||||
}
|
||||
|
||||
parent->child_reqs++;
|
||||
TAILQ_INSERT_TAIL(&parent->children, child, child_tailq);
|
||||
child->parent = parent;
|
||||
child->cb_fn = nvme_request_cb_complete_child;
|
||||
child->cb_arg = child;
|
||||
}
|
||||
|
||||
void nvme_request_remove_child(struct nvme_request *parent,
|
||||
struct nvme_request *child)
|
||||
{
|
||||
nvme_assert(child->parent == parent, "child->parent != parent\n");
|
||||
nvme_assert(parent->child_reqs != 0, "child_reqs is 0\n");
|
||||
|
||||
parent->child_reqs--;
|
||||
TAILQ_REMOVE(&parent->children, child, child_tailq);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user