blob: ba1450d6d8ecff0c40f40019c5aa72f850fa1146 [file] [log] [blame]
/*
* Copyright 2020 Broadcom
*
* SPDX-License-Identifier: Apache-2.0
*/
#define DT_DRV_COMPAT brcm_iproc_pax_dma_v1
#include <zephyr/arch/cpu.h>
#include <zephyr/cache.h>
#include <errno.h>
#include <zephyr/init.h>
#include <zephyr/kernel.h>
#include <zephyr/linker/sections.h>
#include <soc.h>
#include <string.h>
#include <zephyr/toolchain.h>
#include <zephyr/types.h>
#include <zephyr/drivers/dma.h>
#include <zephyr/drivers/pcie/endpoint/pcie_ep.h>
#include "dma_iproc_pax_v1.h"
#define LOG_LEVEL CONFIG_DMA_LOG_LEVEL
#include <zephyr/logging/log.h>
#include <zephyr/irq.h>
LOG_MODULE_REGISTER(dma_iproc_pax);
/* Driver runtime data for PAX DMA and RM */
static struct dma_iproc_pax_data pax_dma_data;
static inline uint32_t reset_pkt_id(struct dma_iproc_pax_ring_data *ring)
{
return ring->pkt_id = 0x0;
}
/**
* @brief Opaque/packet id allocator, range 0 to 31
*/
static inline uint32_t alloc_pkt_id(struct dma_iproc_pax_ring_data *ring)
{
ring->pkt_id = (ring->pkt_id + 1) % 32;
return ring->pkt_id;
}
static inline uint32_t curr_pkt_id(struct dma_iproc_pax_ring_data *ring)
{
return ring->pkt_id;
}
static inline uint32_t curr_toggle_val(struct dma_iproc_pax_ring_data *ring)
{
return ring->curr.toggle;
}
/**
* @brief Populate header descriptor
*/
static inline void rm_write_header_desc(void *desc, uint32_t toggle,
uint32_t opq, uint32_t bdcount)
{
struct rm_header *r = (struct rm_header *)desc;
r->opq = opq;
/* DMA descriptor count init value */
r->bdcount = bdcount;
r->prot = 0x0;
/* No packet extension, start and end set to '1' */
r->start = 1;
r->end = 1;
r->toggle = toggle;
/* RM header type */
r->type = PAX_DMA_TYPE_RM_HEADER;
}
/**
* @brief Fill RM header descriptor for next transfer
* with invalid toggle
*/
static inline void rm_write_header_next_desc(void *desc,
struct dma_iproc_pax_ring_data *r,
uint32_t opq, uint32_t bdcount)
{
/* Toggle bit is invalid until next payload configured */
rm_write_header_desc(desc, (r->curr.toggle == 0) ? 1 : 0, opq, bdcount);
}
static inline void rm_header_set_bd_count(void *desc, uint32_t bdcount)
{
struct rm_header *r = (struct rm_header *)desc;
/* DMA descriptor count */
r->bdcount = bdcount;
}
static inline void rm_header_set_toggle(void *desc, uint32_t toggle)
{
struct rm_header *r = (struct rm_header *)desc;
r->toggle = toggle;
}
/**
* @brief Populate dma header descriptor
*/
static inline void rm_write_dma_header_desc(void *desc,
struct dma_iproc_pax_payload *pl)
{
struct dma_header_desc *hdr = (struct dma_header_desc *)desc;
hdr->length = pl->xfer_sz;
hdr->opcode = pl->direction;
/* DMA header type */
hdr->type = PAX_DMA_TYPE_DMA_DESC;
}
/**
* @brief Populate axi address descriptor
*/
static inline void rm_write_axi_addr_desc(void *desc,
struct dma_iproc_pax_payload *pl)
{
struct axi_addr_desc *axi = (struct axi_addr_desc *)desc;
axi->axi_addr = pl->axi_addr;
axi->type = PAX_DMA_TYPE_DMA_DESC;
}
/**
* @brief Populate pci address descriptor
*/
static inline void rm_write_pci_addr_desc(void *desc,
struct dma_iproc_pax_payload *pl)
{
struct pci_addr_desc *pci = (struct pci_addr_desc *)desc;
pci->pcie_addr = pl->pci_addr >> PAX_DMA_PCI_ADDR_ALIGNMT_SHIFT;
pci->type = PAX_DMA_TYPE_DMA_DESC;
}
/**
* @brief Return's pointer to the descriptor memory to be written next,
* skip next pointer descriptor address.
*/
static void *next_desc_addr(struct dma_iproc_pax_ring_data *ring)
{
struct next_ptr_desc *nxt;
uintptr_t curr;
curr = (uintptr_t)ring->curr.write_ptr + PAX_DMA_RM_DESC_BDWIDTH;
/* if hit next table ptr, skip to next location, flip toggle */
nxt = (struct next_ptr_desc *)curr;
if (nxt->type == PAX_DMA_TYPE_NEXT_PTR) {
LOG_DBG("hit next_ptr@0x%lx:T%d, next_table@0x%lx\n",
curr, nxt->toggle, (uintptr_t)nxt->addr);
uintptr_t last = (uintptr_t)ring->bd +
PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS;
ring->curr.toggle = (ring->curr.toggle == 0) ? 1 : 0;
/* move to next addr, wrap around if hits end */
curr += PAX_DMA_RM_DESC_BDWIDTH;
if (curr == last) {
curr = (uintptr_t)ring->bd;
LOG_DBG("hit end of desc:0x%lx, wrap to 0x%lx\n",
last, curr);
}
}
ring->curr.write_ptr = (void *)curr;
return (void *)curr;
}
/**
* @brief Populate next ptr descriptor
*/
static void rm_write_next_table_desc(void *desc, void *next_ptr,
uint32_t toggle)
{
struct next_ptr_desc *nxt = (struct next_ptr_desc *)desc;
nxt->addr = (uintptr_t)next_ptr;
nxt->type = PAX_DMA_TYPE_NEXT_PTR;
nxt->toggle = toggle;
}
static void prepare_ring(struct dma_iproc_pax_ring_data *ring)
{
uintptr_t curr, next, last;
uint32_t toggle;
int buff_count = PAX_DMA_NUM_BD_BUFFS;
/* zero out descriptor area */
memset(ring->bd, 0x0, PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS);
memset(ring->cmpl, 0x0, PAX_DMA_RM_CMPL_RING_SIZE);
/* opaque/packet id value */
rm_write_header_desc(ring->bd, 0x0, reset_pkt_id(ring),
PAX_DMA_RM_DESC_BDCOUNT);
/* start with first buffer, valid toggle is 0x1 */
toggle = 0x1;
curr = (uintptr_t)ring->bd;
next = curr + PAX_DMA_RM_DESC_RING_SIZE;
last = curr + PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS;
do {
/* Place next_table desc as last BD entry on each buffer */
rm_write_next_table_desc(PAX_DMA_NEXT_TBL_ADDR((void *)curr),
(void *)next, toggle);
/* valid toggle flips for each buffer */
toggle = toggle ? 0x0 : 0x1;
curr += PAX_DMA_RM_DESC_RING_SIZE;
next += PAX_DMA_RM_DESC_RING_SIZE;
/* last entry, chain back to first buffer */
if (next == last) {
next = (uintptr_t)ring->bd;
}
} while (--buff_count);
dma_mb();
/* start programming from first RM header */
ring->curr.write_ptr = ring->bd;
/* valid toggle starts with 1 after reset */
ring->curr.toggle = 1;
/* completion read offset */
ring->curr.cmpl_rd_offs = 0;
/* init sync data for the ring */
ring->curr.sync_data.signature = PAX_DMA_WRITE_SYNC_SIGNATURE;
ring->curr.sync_data.ring = ring->idx;
/* pkt id for active dma xfer */
ring->curr.sync_data.opaque = 0x0;
/* pkt count for active dma xfer */
ring->curr.sync_data.total_pkts = 0x0;
}
static int init_rm(struct dma_iproc_pax_data *pd)
{
int ret = -ETIMEDOUT, timeout = 1000;
k_mutex_lock(&pd->dma_lock, K_FOREVER);
/* Wait for Ring Manager ready */
do {
LOG_DBG("Waiting for RM HW init\n");
if ((sys_read32(RM_COMM_REG(pd, RM_COMM_MAIN_HW_INIT_DONE)) &
RM_COMM_MAIN_HW_INIT_DONE_MASK)) {
ret = 0;
break;
}
k_sleep(K_MSEC(1));
} while (--timeout);
k_mutex_unlock(&pd->dma_lock);
if (!timeout) {
LOG_WRN("RM HW Init timedout!\n");
} else {
LOG_INF("PAX DMA RM HW Init Done\n");
}
return ret;
}
static void rm_cfg_start(struct dma_iproc_pax_data *pd)
{
uint32_t val;
k_mutex_lock(&pd->dma_lock, K_FOREVER);
/* set config done 0, enable toggle mode */
val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
val &= ~RM_COMM_CONTROL_CONFIG_DONE;
sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
val &= ~(RM_COMM_CONTROL_MODE_MASK << RM_COMM_CONTROL_MODE_SHIFT);
val |= (RM_COMM_CONTROL_MODE_TOGGLE << RM_COMM_CONTROL_MODE_SHIFT);
sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
/* Disable MSI */
sys_write32(RM_COMM_MSI_DISABLE_VAL,
RM_COMM_REG(pd, RM_COMM_MSI_DISABLE));
/* Enable Line interrupt */
val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
val |= RM_COMM_CONTROL_LINE_INTR_EN;
sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
/* Enable AE_TIMEOUT */
sys_write32(RM_COMM_AE_TIMEOUT_VAL, RM_COMM_REG(pd,
RM_COMM_AE_TIMEOUT));
val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
val |= RM_COMM_CONTROL_AE_TIMEOUT_EN;
sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
/* AE (Acceleration Engine) grouping to group '0' */
val = sys_read32(RM_COMM_REG(pd, RM_AE0_AE_CONTROL));
val &= ~RM_AE_CTRL_AE_GROUP_MASK;
sys_write32(val, RM_COMM_REG(pd, RM_AE0_AE_CONTROL));
val |= RM_AE_CONTROL_ACTIVE;
sys_write32(val, RM_COMM_REG(pd, RM_AE0_AE_CONTROL));
/* AXI read/write channel enable */
val = sys_read32(RM_COMM_REG(pd, RM_COMM_AXI_CONTROL));
val |= (RM_COMM_AXI_CONTROL_RD_CH_EN | RM_COMM_AXI_CONTROL_WR_CH_EN);
sys_write32(val, RM_COMM_REG(pd, RM_COMM_AXI_CONTROL));
/* Tune RM control programming for 4 rings */
sys_write32(RM_COMM_TIMER_CONTROL0_VAL,
RM_COMM_REG(pd, RM_COMM_TIMER_CONTROL_0));
sys_write32(RM_COMM_TIMER_CONTROL1_VAL,
RM_COMM_REG(pd, RM_COMM_TIMER_CONTROL_1));
sys_write32(RM_COMM_RM_BURST_LENGTH,
RM_COMM_REG(pd, RM_COMM_RM_BURST_LENGTH));
/* Set Sequence max count to the max supported value */
val = sys_read32(RM_COMM_REG(pd, RM_COMM_MASK_SEQUENCE_MAX_COUNT));
val = (val | RING_MASK_SEQ_MAX_COUNT_MASK);
sys_write32(val, RM_COMM_REG(pd, RM_COMM_MASK_SEQUENCE_MAX_COUNT));
k_mutex_unlock(&pd->dma_lock);
}
static void rm_ring_clear_stats(struct dma_iproc_pax_data *pd,
enum ring_idx idx)
{
/* Read ring Tx, Rx, and Outstanding counts to clear */
sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_RECV_LS));
sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_RECV_MS));
sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_TRANS_LS));
sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_TRANS_MS));
sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_OUTSTAND));
}
static void rm_cfg_finish(struct dma_iproc_pax_data *pd)
{
uint32_t val;
k_mutex_lock(&pd->dma_lock, K_FOREVER);
/* set Ring config done */
val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
val |= RM_COMM_CONTROL_CONFIG_DONE;
sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
k_mutex_unlock(&pd->dma_lock);
}
/* Activate/Deactivate rings */
static inline void set_ring_active(struct dma_iproc_pax_data *pd,
enum ring_idx idx,
bool active)
{
uint32_t val;
val = sys_read32(RM_RING_REG(pd, idx, RING_CONTROL));
if (active) {
val |= RING_CONTROL_ACTIVE;
} else {
val &= ~RING_CONTROL_ACTIVE;
}
sys_write32(val, RM_RING_REG(pd, idx, RING_CONTROL));
}
static int init_ring(struct dma_iproc_pax_data *pd, enum ring_idx idx)
{
uint32_t val;
uintptr_t desc = (uintptr_t)pd->ring[idx].bd;
uintptr_t cmpl = (uintptr_t)pd->ring[idx].cmpl;
int timeout = 5000, ret = 0;
k_mutex_lock(&pd->dma_lock, K_FOREVER);
/* Read cmpl write ptr incase previous dma stopped */
sys_read32(RM_RING_REG(pd, idx, RING_CMPL_WRITE_PTR));
/* Inactivate ring */
sys_write32(0x0, RM_RING_REG(pd, idx, RING_CONTROL));
/* Flush ring before loading new descriptor */
sys_write32(RING_CONTROL_FLUSH, RM_RING_REG(pd, idx, RING_CONTROL));
do {
if (sys_read32(RM_RING_REG(pd, idx, RING_FLUSH_DONE)) &
RING_FLUSH_DONE_MASK) {
break;
}
k_busy_wait(1);
} while (--timeout);
if (!timeout) {
LOG_WRN("Ring %d flush timedout!\n", idx);
ret = -ETIMEDOUT;
goto err;
}
/* clear ring after flush */
sys_write32(0x0, RM_RING_REG(pd, idx, RING_CONTROL));
/* ring group id set to '0' */
val = sys_read32(RM_COMM_REG(pd, RM_COMM_CTRL_REG(idx)));
val &= ~RING_COMM_CTRL_AE_GROUP_MASK;
sys_write32(val, RM_COMM_REG(pd, RM_COMM_CTRL_REG(idx)));
/* DDR update control, set timeout value */
val = RING_DDR_CONTROL_COUNT(RING_DDR_CONTROL_COUNT_VAL) |
RING_DDR_CONTROL_TIMER(RING_DDR_CONTROL_TIMER_VAL) |
RING_DDR_CONTROL_ENABLE;
sys_write32(val, RM_RING_REG(pd, idx, RING_CMPL_WR_PTR_DDR_CONTROL));
val = (uint32_t)((uintptr_t)desc >> PAX_DMA_RING_BD_ALIGN_ORDER);
sys_write32(val, RM_RING_REG(pd, idx, RING_BD_START_ADDR));
val = (uint32_t)((uintptr_t)cmpl >> PAX_DMA_RING_CMPL_ALIGN_ORDER);
sys_write32(val, RM_RING_REG(pd, idx, RING_CMPL_START_ADDR));
val = sys_read32(RM_RING_REG(pd, idx, RING_BD_READ_PTR));
/* keep ring inactive after init to avoid BD poll */
set_ring_active(pd, idx, false);
rm_ring_clear_stats(pd, idx);
err:
k_mutex_unlock(&pd->dma_lock);
return ret;
}
static int poll_on_write_sync(const struct device *dev,
struct dma_iproc_pax_ring_data *ring)
{
const struct dma_iproc_pax_cfg *cfg = dev->config;
struct dma_iproc_pax_write_sync_data sync_rd, *recv, *sent;
uint64_t pci_addr;
uint32_t *pci32, *axi32;
uint32_t zero_init = 0, timeout = PAX_DMA_MAX_SYNC_WAIT;
int ret;
recv = &sync_rd;
sent = &(ring->curr.sync_data);
/* form host pci sync address */
pci32 = (uint32_t *)&pci_addr;
pci32[0] = ring->sync_pci.addr_lo;
pci32[1] = ring->sync_pci.addr_hi;
axi32 = (uint32_t *)&sync_rd;
do {
ret = pcie_ep_xfer_data_memcpy(cfg->pcie_dev, pci_addr,
(uintptr_t *)axi32, 4,
PCIE_OB_LOWMEM, HOST_TO_DEVICE);
if (memcmp((void *)recv, (void *)sent, 4) == 0) {
/* clear the sync word */
ret = pcie_ep_xfer_data_memcpy(cfg->pcie_dev, pci_addr,
(uintptr_t *)&zero_init,
4, PCIE_OB_LOWMEM,
DEVICE_TO_HOST);
dma_mb();
ret = 0;
break;
}
k_busy_wait(1);
} while (--timeout);
if (!timeout) {
LOG_DBG("[ring %d]: not recvd write sync!\n", ring->idx);
ret = -ETIMEDOUT;
}
return ret;
}
static int process_cmpl_event(const struct device *dev,
enum ring_idx idx, uint32_t pl_len)
{
struct dma_iproc_pax_data *pd = dev->data;
uint32_t wr_offs, rd_offs, ret = DMA_STATUS_COMPLETE;
struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);
struct cmpl_pkt *c;
uint32_t is_outstanding;
/* cmpl read offset, unprocessed cmpl location */
rd_offs = ring->curr.cmpl_rd_offs;
wr_offs = sys_read32(RM_RING_REG(pd, idx,
RING_CMPL_WRITE_PTR));
/* Update read ptr to "processed" */
ring->curr.cmpl_rd_offs = wr_offs;
/*
* Ensure consistency of completion descriptor
* The completion desc is updated by RM via AXI stream
* CPU need to ensure the memory operations are completed
* before reading cmpl area, by a "dsb"
* If Dcache enabled, need to invalidate the cachelines to
* read updated cmpl desc. The cache API also issues dsb.
*/
dma_mb();
/* Decode cmpl pkt id to verify */
c = (struct cmpl_pkt *)((uintptr_t)ring->cmpl +
PAX_DMA_CMPL_DESC_SIZE * PAX_DMA_CURR_CMPL_IDX(wr_offs));
LOG_DBG("RING%d WR_PTR:%d opq:%d, rm_status:%x dma_status:%x\n",
idx, wr_offs, c->opq, c->rm_status, c->dma_status);
is_outstanding = sys_read32(RM_RING_REG(pd, idx,
RING_NUM_REQ_OUTSTAND));
if ((ring->curr.opq != c->opq) && (is_outstanding != 0)) {
LOG_ERR("RING%d: pkt id should be %d, rcvd %d outst=%d\n",
idx, ring->curr.opq, c->opq, is_outstanding);
ret = -EIO;
}
/* check for completion AE timeout */
if (c->rm_status == RM_COMPLETION_AE_TIMEOUT) {
LOG_ERR("RING%d WR_PTR:%d rm_status:%x AE Timeout!\n",
idx, wr_offs, c->rm_status);
/* TBD: Issue full card reset to restore operations */
LOG_ERR("Needs Card Reset to recover!\n");
ret = -ETIMEDOUT;
}
if (ring->dma_callback) {
ring->dma_callback(dev, ring->callback_arg, idx, ret);
}
return ret;
}
#ifdef CONFIG_DMA_IPROC_PAX_POLL_MODE
static int peek_ring_cmpl(const struct device *dev,
enum ring_idx idx, uint32_t pl_len)
{
struct dma_iproc_pax_data *pd = dev->data;
uint32_t wr_offs, rd_offs, timeout = PAX_DMA_MAX_POLL_WAIT;
struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);
/* cmpl read offset, unprocessed cmpl location */
rd_offs = ring->curr.cmpl_rd_offs;
/* poll write_ptr until cmpl received for all buffers */
do {
wr_offs = sys_read32(RM_RING_REG(pd, idx,
RING_CMPL_WRITE_PTR));
if (PAX_DMA_GET_CMPL_COUNT(wr_offs, rd_offs) >= pl_len)
break;
k_busy_wait(1);
} while (--timeout);
if (timeout == 0) {
LOG_ERR("RING%d timeout, rcvd %d, expected %d!\n",
idx, PAX_DMA_GET_CMPL_COUNT(wr_offs, rd_offs), pl_len);
/* More debug info on current dma instance */
LOG_ERR("WR_PTR:%x RD_PTR%x\n", wr_offs, rd_offs);
return -ETIMEDOUT;
}
return process_cmpl_event(dev, idx, pl_len);
}
#else
static void rm_isr(const struct device *dev)
{
uint32_t status, err_stat, idx;
struct dma_iproc_pax_data *pd = dev->data;
/* read and clear interrupt status */
status = sys_read32(RM_COMM_REG(pd, RM_COMM_MSI_INTR_INTERRUPT_STATUS));
sys_write32(status, RM_COMM_REG(pd,
RM_COMM_MSI_INTERRUPT_STATUS_CLEAR));
/* read and clear DME/AE error interrupts */
err_stat = sys_read32(RM_COMM_REG(pd,
RM_COMM_DME_INTERRUPT_STATUS_MASK));
sys_write32(err_stat,
RM_COMM_REG(pd, RM_COMM_DME_INTERRUPT_STATUS_CLEAR));
err_stat =
sys_read32(RM_COMM_REG(pd,
RM_COMM_AE_INTERFACE_GROUP_0_INTERRUPT_MASK));
sys_write32(err_stat,
RM_COMM_REG(pd,
RM_COMM_AE_INTERFACE_GROUP_0_INTERRUPT_CLEAR));
/* alert waiting thread to process, for each completed ring */
for (idx = PAX_DMA_RING0; idx < PAX_DMA_RINGS_MAX; idx++) {
if (status & (0x1 << idx)) {
k_sem_give(&pd->ring[idx].alert);
}
}
}
#endif
static int dma_iproc_pax_init(const struct device *dev)
{
const struct dma_iproc_pax_cfg *cfg = dev->config;
struct dma_iproc_pax_data *pd = dev->data;
int r;
uintptr_t mem_aligned;
if (!device_is_ready(cfg->pcie_dev)) {
LOG_ERR("PCIe device not ready");
return -ENODEV;
}
pd->dma_base = cfg->dma_base;
pd->rm_comm_base = cfg->rm_comm_base;
pd->used_rings = (cfg->use_rings < PAX_DMA_RINGS_MAX) ?
cfg->use_rings : PAX_DMA_RINGS_MAX;
LOG_DBG("dma base:0x%x, rm comm base:0x%x, needed rings %d\n",
pd->dma_base, pd->rm_comm_base, pd->used_rings);
/* dma/rm access lock */
k_mutex_init(&pd->dma_lock);
/* Ring Manager H/W init */
if (init_rm(pd)) {
return -ETIMEDOUT;
}
/* common rm config */
rm_cfg_start(pd);
/* individual ring config */
for (r = 0; r < pd->used_rings; r++) {
/* per-ring mutex lock */
k_mutex_init(&pd->ring[r].lock);
/* Init alerts */
k_sem_init(&pd->ring[r].alert, 0, 1);
pd->ring[r].idx = r;
pd->ring[r].ring_base = cfg->rm_base +
PAX_DMA_RING_ADDR_OFFSET(r);
LOG_DBG("RING%d,VERSION:0x%x\n", pd->ring[r].idx,
sys_read32(RM_RING_REG(pd, r, RING_VER)));
/* Allocate for 2 BD buffers + cmpl buffer + payload struct */
pd->ring[r].ring_mem = (void *)((uintptr_t)cfg->bd_memory_base +
r *
PAX_DMA_PER_RING_ALLOC_SIZE);
if (!pd->ring[r].ring_mem) {
LOG_ERR("RING%d failed to alloc desc memory!\n", r);
return -ENOMEM;
}
/* Find 8K aligned address within allocated region */
mem_aligned = ((uintptr_t)pd->ring[r].ring_mem +
PAX_DMA_RING_ALIGN - 1) &
~(PAX_DMA_RING_ALIGN - 1);
pd->ring[r].cmpl = (void *)mem_aligned;
pd->ring[r].bd = (void *)(mem_aligned +
PAX_DMA_RM_CMPL_RING_SIZE);
pd->ring[r].payload = (void *)((uintptr_t)pd->ring[r].bd +
PAX_DMA_RM_DESC_RING_SIZE *
PAX_DMA_NUM_BD_BUFFS);
LOG_DBG("Ring%d,allocated Mem:0x%p Size %d\n",
pd->ring[r].idx,
pd->ring[r].ring_mem,
PAX_DMA_PER_RING_ALLOC_SIZE);
LOG_DBG("Ring%d,BD:0x%p, CMPL:0x%p, PL:0x%p\n",
pd->ring[r].idx,
pd->ring[r].bd,
pd->ring[r].cmpl,
pd->ring[r].payload);
/* Prepare ring desc table */
prepare_ring(&(pd->ring[r]));
/* initialize ring */
init_ring(pd, r);
}
/* set ring config done */
rm_cfg_finish(pd);
#ifndef CONFIG_DMA_IPROC_PAX_POLL_MODE
/* Register and enable RM interrupt */
IRQ_CONNECT(DT_INST_IRQN(0),
DT_INST_IRQ(0, priority),
rm_isr,
DEVICE_DT_INST_GET(0),
0);
irq_enable(DT_INST_IRQN(0));
#else
LOG_INF("%s PAX DMA rings in poll mode!\n", dev->name);
#endif
LOG_INF("%s RM setup %d rings\n", dev->name, pd->used_rings);
return 0;
}
#ifdef CONFIG_DMA_IPROC_PAX_POLL_MODE
static void set_pkt_count(const struct device *dev,
enum ring_idx idx,
uint32_t pl_len)
{
/* Nothing needs to be programmed here in poll mode */
}
static int wait_for_pkt_completion(const struct device *dev,
enum ring_idx idx,
uint32_t pl_len)
{
/* poll for completion */
return peek_ring_cmpl(dev, idx, pl_len + 1);
}
#else
static void set_pkt_count(const struct device *dev,
enum ring_idx idx,
uint32_t pl_len)
{
struct dma_iproc_pax_data *pd = dev->data;
uint32_t val;
/* program packet count for interrupt assertion */
val = sys_read32(RM_RING_REG(pd, idx,
RING_CMPL_WR_PTR_DDR_CONTROL));
val &= ~RING_DDR_CONTROL_COUNT_MASK;
val |= RING_DDR_CONTROL_COUNT(pl_len);
sys_write32(val, RM_RING_REG(pd, idx,
RING_CMPL_WR_PTR_DDR_CONTROL));
}
static int wait_for_pkt_completion(const struct device *dev,
enum ring_idx idx,
uint32_t pl_len)
{
struct dma_iproc_pax_data *pd = dev->data;
struct dma_iproc_pax_ring_data *ring;
ring = &(pd->ring[idx]);
/* wait for sg dma completion alert */
if (k_sem_take(&ring->alert, K_MSEC(PAX_DMA_TIMEOUT)) != 0) {
LOG_ERR("PAX DMA [ring %d] Timeout!\n", idx);
return -ETIMEDOUT;
}
return process_cmpl_event(dev, idx, pl_len);
}
#endif
static int dma_iproc_pax_do_xfer(const struct device *dev,
enum ring_idx idx,
struct dma_iproc_pax_payload *pl,
uint32_t pl_len)
{
struct dma_iproc_pax_data *pd = dev->data;
const struct dma_iproc_pax_cfg *cfg = dev->config;
int ret = 0, cnt;
struct dma_iproc_pax_ring_data *ring;
void *hdr;
uint32_t toggle_bit;
struct dma_iproc_pax_payload sync_pl;
struct dma_iproc_pax_addr64 sync;
ring = &(pd->ring[idx]);
pl = ring->payload;
/*
* Host sync buffer isn't ready at zephyr/driver init-time
* Read the host address location once at first DMA write
* on that ring.
*/
if ((ring->sync_pci.addr_lo == 0x0) &&
(ring->sync_pci.addr_hi == 0x0)) {
/* populate sync data location */
LOG_DBG("sync addr loc 0x%x\n", cfg->scr_addr_loc);
sync.addr_lo = sys_read32(cfg->scr_addr_loc
+ 4);
sync.addr_hi = sys_read32(cfg->scr_addr_loc);
ring->sync_pci.addr_lo = sync.addr_lo + idx * 4;
ring->sync_pci.addr_hi = sync.addr_hi;
LOG_DBG("ring:%d,sync addr:0x%x.0x%x\n", idx,
ring->sync_pci.addr_hi,
ring->sync_pci.addr_lo);
}
/* account extra sync packet */
ring->curr.sync_data.opaque = ring->curr.opq;
ring->curr.sync_data.total_pkts = pl_len;
memcpy((void *)&ring->sync_loc,
(void *)&(ring->curr.sync_data), 4);
sync_pl.pci_addr = ring->sync_pci.addr_lo |
(uint64_t)ring->sync_pci.addr_hi << 32;
sync_pl.axi_addr = (uintptr_t)&ring->sync_loc;
sync_pl.xfer_sz = 4; /* 4-bytes */
sync_pl.direction = CARD_TO_HOST;
/* Get descriptor write pointer for first header */
hdr = (void *)ring->curr.write_ptr;
/* current toggle bit */
toggle_bit = ring->curr.toggle;
/* current opq value for cmpl check */
ring->curr.opq = curr_pkt_id(ring);
/* DMA desc count for first payload */
rm_header_set_bd_count(hdr, PAX_DMA_RM_DESC_BDCOUNT);
/* Form dma descriptors for total sg payload */
for (cnt = 0; cnt < pl_len; cnt++) {
rm_write_dma_header_desc(next_desc_addr(ring), pl + cnt);
rm_write_axi_addr_desc(next_desc_addr(ring), pl + cnt);
rm_write_pci_addr_desc(next_desc_addr(ring), pl + cnt);
/* Toggle may flip, program updated toggle value */
rm_write_header_desc(next_desc_addr(ring),
curr_toggle_val(ring),
curr_pkt_id(ring),
PAX_DMA_RM_DESC_BDCOUNT);
}
/* Append write sync payload descriptors */
rm_write_dma_header_desc(next_desc_addr(ring), &sync_pl);
rm_write_axi_addr_desc(next_desc_addr(ring), &sync_pl);
rm_write_pci_addr_desc(next_desc_addr(ring), &sync_pl);
/* RM header for next transfer, RM wait on (invalid) toggle bit */
rm_write_header_next_desc(next_desc_addr(ring), ring, alloc_pkt_id(ring),
PAX_DMA_RM_DESC_BDCOUNT);
set_pkt_count(dev, idx, pl_len + 1);
/* Ensure memory write before toggle flip */
dma_mb();
/* set toggle to valid in first header */
rm_header_set_toggle(hdr, toggle_bit);
/* activate the ring */
set_ring_active(pd, idx, true);
ret = wait_for_pkt_completion(dev, idx, pl_len + 1);
if (ret) {
goto err_ret;
}
ret = poll_on_write_sync(dev, ring);
k_mutex_lock(&ring->lock, K_FOREVER);
ring->ring_active = 0;
k_mutex_unlock(&ring->lock);
err_ret:
ring->ring_active = 0;
/* deactivate the ring until next active transfer */
set_ring_active(pd, idx, false);
return ret;
}
static int dma_iproc_pax_configure(const struct device *dev, uint32_t channel,
struct dma_config *cfg)
{
struct dma_iproc_pax_data *pd = dev->data;
struct dma_iproc_pax_ring_data *ring;
uint32_t xfer_sz;
int ret = 0;
#ifdef CONFIG_DMA_IPROC_PAX_DEBUG
uint32_t *pci_addr32;
uint32_t *axi_addr32;
#endif
if (channel >= PAX_DMA_RINGS_MAX) {
LOG_ERR("Invalid ring/channel %d\n", channel);
return -EINVAL;
}
ring = &(pd->ring[channel]);
k_mutex_lock(&ring->lock, K_FOREVER);
if (cfg->block_count > 1) {
/* Scatter/gather list handling is not supported */
ret = -ENOTSUP;
goto err;
}
if (ring->ring_active) {
ret = -EBUSY;
goto err;
}
ring->ring_active = 1;
if (cfg->channel_direction == MEMORY_TO_PERIPHERAL) {
#ifdef CONFIG_DMA_IPROC_PAX_DEBUG
axi_addr32 = (uint32_t *)&cfg->head_block->source_address;
pci_addr32 = (uint32_t *)&cfg->head_block->dest_address;
#endif
ring->payload->direction = CARD_TO_HOST;
ring->payload->pci_addr = cfg->head_block->dest_address;
ring->payload->axi_addr = cfg->head_block->source_address;
} else if (cfg->channel_direction == PERIPHERAL_TO_MEMORY) {
#ifdef CONFIG_DMA_IPROC_PAX_DEBUG
axi_addr32 = (uint32_t *)&cfg->head_block->dest_address;
pci_addr32 = (uint32_t *)&cfg->head_block->source_address;
#endif
ring->payload->direction = HOST_TO_CARD;
ring->payload->pci_addr = cfg->head_block->source_address;
ring->payload->axi_addr = cfg->head_block->dest_address;
} else {
ring->ring_active = 0;
ret = -ENOTSUP;
goto err;
}
xfer_sz = cfg->head_block->block_size;
#ifdef CONFIG_DMA_IPROC_PAX_DEBUG
if (xfer_sz > PAX_DMA_MAX_SIZE) {
LOG_ERR("Unsupported size: %d\n", xfer_size);
ring->ring_active = 0;
ret = -EINVAL;
goto err;
}
if (xfer_sz % PAX_DMA_MIN_SIZE) {
LOG_ERR("Unaligned size 0x%x\n", xfer_size);
ring->ring_active = 0;
ret = -EINVAL;
goto err;
}
if (pci_addr32[0] % PAX_DMA_ADDR_ALIGN) {
LOG_ERR("Unaligned Host addr: 0x%x.0x%x\n",
pci_addr32[1], pci_addr32[0]);
ring->ring_active = 0;
ret = -EINVAL;
goto err;
}
if (axi_addr32[0] % PAX_DMA_ADDR_ALIGN) {
LOG_ERR("Unaligned Card addr: 0x%x.0x%x\n",
axi_addr32[1], axi_addr32[0]);
ring->ring_active = 0;
ret = -EINVAL;
goto err;
}
#endif
ring->payload->xfer_sz = xfer_sz;
ring->dma_callback = cfg->dma_callback;
ring->callback_arg = cfg->user_data;
err:
k_mutex_unlock(&ring->lock);
return ret;
}
static int dma_iproc_pax_transfer_start(const struct device *dev,
uint32_t channel)
{
int ret = 0;
struct dma_iproc_pax_data *pd = dev->data;
struct dma_iproc_pax_ring_data *ring;
if (channel >= PAX_DMA_RINGS_MAX) {
LOG_ERR("Invalid ring %d\n", channel);
return -EINVAL;
}
ring = &(pd->ring[channel]);
/* do dma transfer of single buffer */
ret = dma_iproc_pax_do_xfer(dev, channel, ring->payload, 1);
return ret;
}
static int dma_iproc_pax_transfer_stop(const struct device *dev,
uint32_t channel)
{
return 0;
}
static const struct dma_driver_api pax_dma_driver_api = {
.config = dma_iproc_pax_configure,
.start = dma_iproc_pax_transfer_start,
.stop = dma_iproc_pax_transfer_stop,
};
static const struct dma_iproc_pax_cfg pax_dma_cfg = {
.dma_base = DT_INST_REG_ADDR_BY_NAME(0, dme_regs),
.rm_base = DT_INST_REG_ADDR_BY_NAME(0, rm_ring_regs),
.rm_comm_base = DT_INST_REG_ADDR_BY_NAME(0, rm_comm_regs),
.use_rings = DT_INST_PROP(0, dma_channels),
.bd_memory_base = (void *)DT_INST_PROP_BY_IDX(0, bd_memory, 0),
.scr_addr_loc = DT_INST_PROP(0, scr_addr_loc),
.pcie_dev = DEVICE_DT_GET(DT_INST_PHANDLE(0, pcie_ep)),
};
DEVICE_DT_INST_DEFINE(0,
&dma_iproc_pax_init,
NULL,
&pax_dma_data,
&pax_dma_cfg,
POST_KERNEL,
CONFIG_DMA_INIT_PRIORITY,
&pax_dma_driver_api);