drivers/disk/nvme/nvme.h

/*
 * Copyright (c) 2022 Intel Corporation
 * SPDX-License-Identifier: Apache-2.0
 *
 * Derived from FreeBSD original driver made by Jim Harris
 * with contributions from Alexander Motin and Wojciech Macek
 */

#ifndef ZEPHYR_DRIVERS_DISK_NVME_NHME_H_
#define ZEPHYR_DRIVERS_DISK_NVME_NVME_H_

#include "nvme_helpers.h"
#include "nvme_cmd.h"
#include "nvme_namespace.h"

struct nvme_registers {
	uint32_t	cap_lo; /* controller capabilities */
	uint32_t	cap_hi;
	uint32_t	vs;	/* version */
	uint32_t	intms;	/* interrupt mask set */
	uint32_t	intmc;	/* interrupt mask clear */
	uint32_t	cc;	/* controller configuration */
	uint32_t	reserved1;
	uint32_t	csts;	/* controller status */
	uint32_t	nssr;	/* NVM Subsystem Reset */
	uint32_t	aqa;	/* admin queue attributes */
	uint64_t	asq;	/* admin submission queue base addr */
	uint64_t	acq;	/* admin completion queue base addr */
	uint32_t	cmbloc;	/* Controller Memory Buffer Location */
	uint32_t	cmbsz;	/* Controller Memory Buffer Size */
	uint32_t	bpinfo;	/* Boot Partition Information */
	uint32_t	bprsel;	/* Boot Partition Read Select */
	uint64_t	bpmbl;	/* Boot Partition Memory Buffer Location */
	uint64_t	cmbmsc;	/* Controller Memory Buffer Memory Space Control */
	uint32_t	cmbsts;	/* Controller Memory Buffer Status */
	uint8_t		reserved3[3492]; /* 5Ch - DFFh */
	uint32_t	pmrcap;	/* Persistent Memory Capabilities */
	uint32_t	pmrctl;	/* Persistent Memory Region Control */
	uint32_t	pmrsts;	/* Persistent Memory Region Status */
	uint32_t	pmrebs;	/* Persistent Memory Region Elasticity Buffer Size */
	uint32_t	pmrswtp; /* Persistent Memory Region Sustained Write Throughput */
	uint32_t	pmrmsc_lo; /* Persistent Memory Region Controller Memory Space Control */
	uint32_t	pmrmsc_hi;
	uint8_t		reserved4[484]; /* E1Ch - FFFh */
	struct {
		uint32_t	sq_tdbl; /* submission queue tail doorbell */
		uint32_t	cq_hdbl; /* completion queue head doorbell */
	} doorbell[1];
};

struct nvme_power_state {
	/** Maximum Power */
	uint16_t	mp;
	uint8_t		ps_rsvd1;

	/** Max Power Scale, Non-Operational State */
	uint8_t		mps_nops;

	/** Entry Latency */
	uint32_t	enlat;

	/** Exit Latency */
	uint32_t	exlat;

	/** Relative Read Throughput */
	uint8_t		rrt;

	/** Relative Read Latency */
	uint8_t		rrl;

	/** Relative Write Throughput */
	uint8_t		rwt;

	/** Relative Write Latency */
	uint8_t		rwl;

	/** Idle Power */
	uint16_t	idlp;

	/** Idle Power Scale */
	uint8_t		ips;
	uint8_t		ps_rsvd8;

	/** Active Power */
	uint16_t	actp;

	/** Active Power Workload, Active Power Scale */
	uint8_t		apw_aps;

	uint8_t		ps_rsvd10[9];
} __packed;

#define NVME_SERIAL_NUMBER_LENGTH	20
#define NVME_MODEL_NUMBER_LENGTH	40
#define NVME_FIRMWARE_REVISION_LENGTH	8

struct nvme_controller_data {
	/* bytes 0-255: controller capabilities and features */

	/** pci vendor id */
	uint16_t		vid;

	/** pci subsystem vendor id */
	uint16_t		ssvid;

	/** serial number */
	uint8_t			sn[NVME_SERIAL_NUMBER_LENGTH];

	/** model number */
	uint8_t			mn[NVME_MODEL_NUMBER_LENGTH];

	/** firmware revision */
	uint8_t			fr[NVME_FIRMWARE_REVISION_LENGTH];

	/** recommended arbitration burst */
	uint8_t			rab;

	/** ieee oui identifier */
	uint8_t			ieee[3];

	/** multi-interface capabilities */
	uint8_t			mic;

	/** maximum data transfer size */
	uint8_t			mdts;

	/** Controller ID */
	uint16_t		ctrlr_id;

	/** Version */
	uint32_t		ver;

	/** RTD3 Resume Latency */
	uint32_t		rtd3r;

	/** RTD3 Enter Latency */
	uint32_t		rtd3e;

	/** Optional Asynchronous Events Supported */
	uint32_t		oaes;	/* bitfield really */

	/** Controller Attributes */
	uint32_t		ctratt;	/* bitfield really */

	/** Read Recovery Levels Supported */
	uint16_t		rrls;

	uint8_t			reserved1[9];

	/** Controller Type */
	uint8_t			cntrltype;

	/** FRU Globally Unique Identifier */
	uint8_t			fguid[16];

	/** Command Retry Delay Time 1 */
	uint16_t		crdt1;

	/** Command Retry Delay Time 2 */
	uint16_t		crdt2;

	/** Command Retry Delay Time 3 */
	uint16_t		crdt3;

	uint8_t			reserved2[122];

	/* bytes 256-511: admin command set attributes */

	/** optional admin command support */
	uint16_t		oacs;

	/** abort command limit */
	uint8_t			acl;

	/** asynchronous event request limit */
	uint8_t			aerl;

	/** firmware updates */
	uint8_t			frmw;

	/** log page attributes */
	uint8_t			lpa;

	/** error log page entries */
	uint8_t			elpe;

	/** number of power states supported */
	uint8_t			npss;

	/** admin vendor specific command configuration */
	uint8_t			avscc;

	/** Autonomous Power State Transition Attributes */
	uint8_t			apsta;

	/** Warning Composite Temperature Threshold */
	uint16_t		wctemp;

	/** Critical Composite Temperature Threshold */
	uint16_t		cctemp;

	/** Maximum Time for Firmware Activation */
	uint16_t		mtfa;

	/** Host Memory Buffer Preferred Size */
	uint32_t		hmpre;

	/** Host Memory Buffer Minimum Size */
	uint32_t		hmmin;

	/** Name space capabilities  */
	struct {
		/* if nsmgmt, report tnvmcap and unvmcap */
		uint8_t    tnvmcap[16];
		uint8_t    unvmcap[16];
	} __packed untncap;

	/** Replay Protected Memory Block Support */
	uint32_t		rpmbs; /* Really a bitfield */

	/** Extended Device Self-test Time */
	uint16_t		edstt;

	/** Device Self-test Options */
	uint8_t			dsto; /* Really a bitfield */

	/** Firmware Update Granularity */
	uint8_t			fwug;

	/** Keep Alive Support */
	uint16_t		kas;

	/** Host Controlled Thermal Management Attributes */
	uint16_t		hctma; /* Really a bitfield */

	/** Minimum Thermal Management Temperature */
	uint16_t		mntmt;

	/** Maximum Thermal Management Temperature */
	uint16_t		mxtmt;

	/** Sanitize Capabilities */
	uint32_t		sanicap; /* Really a bitfield */

	/** Host Memory Buffer Minimum Descriptor Entry Size */
	uint32_t		hmminds;

	/** Host Memory Maximum Descriptors Entries */
	uint16_t		hmmaxd;

	/** NVM Set Identifier Maximum */
	uint16_t		nsetidmax;

	/** Endurance Group Identifier Maximum */
	uint16_t		endgidmax;

	/** ANA Transition Time */
	uint8_t			anatt;

	/** Asymmetric Namespace Access Capabilities */
	uint8_t			anacap;

	/** ANA Group Identifier Maximum */
	uint32_t		anagrpmax;

	/** Number of ANA Group Identifiers */
	uint32_t		nanagrpid;

	/** Persistent Event Log Size */
	uint32_t		pels;

	uint8_t			reserved3[156];
	/* bytes 512-703: nvm command set attributes */

	/** submission queue entry size */
	uint8_t			sqes;

	/** completion queue entry size */
	uint8_t			cqes;

	/** Maximum Outstanding Commands */
	uint16_t		maxcmd;

	/** number of namespaces */
	uint32_t		nn;

	/** optional nvm command support */
	uint16_t		oncs;

	/** fused operation support */
	uint16_t		fuses;

	/** format nvm attributes */
	uint8_t			fna;

	/** volatile write cache */
	uint8_t			vwc;

	/** Atomic Write Unit Normal */
	uint16_t		awun;

	/** Atomic Write Unit Power Fail */
	uint16_t		awupf;

	/** NVM Vendor Specific Command Configuration */
	uint8_t			nvscc;

	/** Namespace Write Protection Capabilities */
	uint8_t			nwpc;

	/** Atomic Compare & Write Unit */
	uint16_t		acwu;
	uint16_t		reserved6;

	/** SGL Support */
	uint32_t		sgls;

	/** Maximum Number of Allowed Namespaces */
	uint32_t		mnan;

	/* bytes 540-767: Reserved */
	uint8_t			reserved7[224];

	/** NVM Subsystem NVMe Qualified Name */
	uint8_t			subnqn[256];

	/* bytes 1024-1791: Reserved */
	uint8_t			reserved8[768];

	/* bytes 1792-2047: NVMe over Fabrics specification */
	uint8_t			reserved9[256];

	/* bytes 2048-3071: power state descriptors */
	struct nvme_power_state power_state[32];

	/* bytes 3072-4095: vendor specific */
	uint8_t			vs[1024];
} __packed __aligned(4);

static inline
void nvme_controller_data_swapbytes(struct nvme_controller_data *s)
{
#if _BYTE_ORDER != _LITTLE_ENDIAN
	s->vid = sys_le16_to_cpu(s->vid);
	s->ssvid = sys_le16_to_cpu(s->ssvid);
	s->ctrlr_id = sys_le16_to_cpu(s->ctrlr_id);
	s->ver = sys_le32_to_cpu(s->ver);
	s->rtd3r = sys_le32_to_cpu(s->rtd3r);
	s->rtd3e = sys_le32_to_cpu(s->rtd3e);
	s->oaes = sys_le32_to_cpu(s->oaes);
	s->ctratt = sys_le32_to_cpu(s->ctratt);
	s->rrls = sys_le16_to_cpu(s->rrls);
	s->crdt1 = sys_le16_to_cpu(s->crdt1);
	s->crdt2 = sys_le16_to_cpu(s->crdt2);
	s->crdt3 = sys_le16_to_cpu(s->crdt3);
	s->oacs = sys_le16_to_cpu(s->oacs);
	s->wctemp = sys_le16_to_cpu(s->wctemp);
	s->cctemp = sys_le16_to_cpu(s->cctemp);
	s->mtfa = sys_le16_to_cpu(s->mtfa);
	s->hmpre = sys_le32_to_cpu(s->hmpre);
	s->hmmin = sys_le32_to_cpu(s->hmmin);
	s->rpmbs = sys_le32_to_cpu(s->rpmbs);
	s->edstt = sys_le16_to_cpu(s->edstt);
	s->kas = sys_le16_to_cpu(s->kas);
	s->hctma = sys_le16_to_cpu(s->hctma);
	s->mntmt = sys_le16_to_cpu(s->mntmt);
	s->mxtmt = sys_le16_to_cpu(s->mxtmt);
	s->sanicap = sys_le32_to_cpu(s->sanicap);
	s->hmminds = sys_le32_to_cpu(s->hmminds);
	s->hmmaxd = sys_le16_to_cpu(s->hmmaxd);
	s->nsetidmax = sys_le16_to_cpu(s->nsetidmax);
	s->endgidmax = sys_le16_to_cpu(s->endgidmax);
	s->anagrpmax = sys_le32_to_cpu(s->anagrpmax);
	s->nanagrpid = sys_le32_to_cpu(s->nanagrpid);
	s->pels = sys_le32_to_cpu(s->pels);
	s->maxcmd = sys_le16_to_cpu(s->maxcmd);
	s->nn = sys_le32_to_cpu(s->nn);
	s->oncs = sys_le16_to_cpu(s->oncs);
	s->fuses = sys_le16_to_cpu(s->fuses);
	s->awun = sys_le16_to_cpu(s->awun);
	s->awupf = sys_le16_to_cpu(s->awupf);
	s->acwu = sys_le16_to_cpu(s->acwu);
	s->sgls = sys_le32_to_cpu(s->sgls);
	s->mnan = sys_le32_to_cpu(s->mnan);
#else
	ARG_UNUSED(s);
#endif
}

#include <zephyr/device.h>
#include <zephyr/drivers/pcie/pcie.h>
#include <zephyr/drivers/pcie/msi.h>

#define NVME_PCIE_BAR_IDX 0

#define NVME_REQUEST_AMOUNT (CONFIG_NVME_ADMIN_ENTRIES +	\
			     CONFIG_NVME_IO_ENTRIES)

/* admin queue + io queue(s) */
#define NVME_PCIE_MSIX_VECTORS 1 + CONFIG_NVME_IO_QUEUES

#define NVME_QUEUE_ALLOCATE(name, n_entries)				\
	static struct nvme_command cmd_##name[n_entries] __aligned(0x1000); \
	static struct nvme_completion cpl_##name[n_entries] __aligned(0x1000); \
									\
	static struct nvme_cmd_qpair name = {				\
		.num_entries = n_entries,				\
		.cmd = cmd_##name,					\
		.cpl = cpl_##name,					\
	}

#define NVME_ADMINQ_ALLOCATE(n, n_entries)		\
	NVME_QUEUE_ALLOCATE(admin_##n, n_entries)
#define NVME_IOQ_ALLOCATE(n, n_entries)			\
	NVME_QUEUE_ALLOCATE(io_##n, n_entries)

struct nvme_controller_config {
	struct pcie_dev *pcie;
};

struct nvme_controller {
	DEVICE_MMIO_RAM;

	const struct device *dev;

	struct k_mutex lock;

	uint32_t id;

	msi_vector_t vectors[NVME_PCIE_MSIX_VECTORS];

	struct nvme_controller_data cdata;

	uint32_t num_io_queues;
	struct nvme_cmd_qpair *adminq;
	struct nvme_cmd_qpair *ioq;

	uint32_t ready_timeout_in_ms;

	/** LO and HI capacity mask */
	uint32_t cap_lo;
	uint32_t cap_hi;

	/** Page size and log2(page_size) - 12 that we're currently using */
	uint32_t page_size;
	uint32_t mps;

	/** doorbell stride */
	uint32_t dstrd;

	/** maximum i/o size in bytes */
	uint32_t max_xfer_size;

	struct nvme_namespace ns[CONFIG_NVME_MAX_NAMESPACES];
};

static inline
bool nvme_controller_has_dataset_mgmt(struct nvme_controller *ctrlr)
{
	/* Assumes cd was byte swapped by nvme_controller_data_swapbytes() */
	return ((ctrlr->cdata.oncs >> NVME_CTRLR_DATA_ONCS_DSM_SHIFT) &
		NVME_CTRLR_DATA_ONCS_DSM_MASK);
}

static inline void nvme_lock(const struct device *dev)
{
	struct nvme_controller *nvme_ctrlr = dev->data;

	k_mutex_lock(&nvme_ctrlr->lock, K_FOREVER);
}

static inline void nvme_unlock(const struct device *dev)
{
	struct nvme_controller *nvme_ctrlr = dev->data;

	k_mutex_unlock(&nvme_ctrlr->lock);
}

#endif /* ZEPHYR_DRIVERS_DISK_NVME_NHME_H_ */