drivers/interrupt_controller: Adding QI support in VT-D
Looks like it's mandatory to invalidate the Interrupt Entry Cache in
VT-D and the only way to do so is to enable Queued Interface.
Signed-off-by: Tomasz Bursztyka <tomasz.bursztyka@linux.intel.com>
diff --git a/drivers/interrupt_controller/intc_intel_vtd.c b/drivers/interrupt_controller/intc_intel_vtd.c
index 3cce54d..61303f9 100644
--- a/drivers/interrupt_controller/intc_intel_vtd.c
+++ b/drivers/interrupt_controller/intc_intel_vtd.c
@@ -26,6 +26,11 @@
#include "intc_intel_vtd.h"
+static inline void vtd_pause_cpu(void)
+{
+ __asm__ volatile("pause" ::: "memory");
+}
+
static void vtd_write_reg32(const struct device *dev,
uint16_t reg, uint32_t value)
{
@@ -69,6 +74,94 @@
}
}
+static void vtd_qi_init(const struct device *dev)
+{
+ struct vtd_ictl_data *data = dev->data;
+ uint64_t value;
+
+ vtd_write_reg64(dev, VTD_IQT_REG, 0);
+ data->qi_tail = 0;
+
+ value = VTD_IQA_REG_GEN_CONTENT((uintptr_t)data->qi,
+ VTD_IQA_WIDTH_128_BIT, QI_SIZE);
+ vtd_write_reg64(dev, VTD_IQA_REG, value);
+
+ vtd_send_cmd(dev, VTD_GCMD_QIE, VTD_GSTS_QIES);
+}
+
+static inline void vtd_qi_tail_inc(const struct device *dev)
+{
+ struct vtd_ictl_data *data = dev->data;
+
+ data->qi_tail += sizeof(struct qi_descriptor);
+ data->qi_tail %= (QI_NUM * sizeof(struct qi_descriptor));
+}
+
+static int vtd_qi_send(const struct device *dev,
+ struct qi_descriptor *descriptor)
+{
+ struct vtd_ictl_data *data = dev->data;
+ union qi_wait_descriptor wait_desc = { 0 };
+ struct qi_descriptor *desc;
+ uint32_t wait_status;
+
+ desc = (struct qi_descriptor *)((uintptr_t)data->qi + data->qi_tail);
+
+ desc->low = descriptor->low;
+ desc->high = descriptor->high;
+
+ vtd_qi_tail_inc(dev);
+
+ desc++;
+
+ wait_status = QI_WAIT_STATUS_INCOMPLETE;
+
+ wait_desc.wait.type = QI_TYPE_WAIT;
+ wait_desc.wait.status_write = 1;
+ wait_desc.wait.status_data = QI_WAIT_STATUS_COMPLETE;
+ wait_desc.wait.address = ((uintptr_t)&wait_status) >> 2;
+
+ desc->low = wait_desc.desc.low;
+ desc->high = wait_desc.desc.high;
+
+ vtd_qi_tail_inc(dev);
+
+ vtd_write_reg64(dev, VTD_IQT_REG, data->qi_tail);
+
+ while (wait_status != QI_WAIT_STATUS_COMPLETE) {
+ if (vtd_read_reg32(dev, VTD_FSTS_REG) & VTD_FSTS_IQE) {
+ return -EIO;
+ }
+
+ vtd_pause_cpu();
+ }
+
+ return 0;
+}
+
+static int vtd_global_iec_invalidate(const struct device *dev)
+{
+ union qi_iec_descriptor iec_desc = { 0 };
+
+ iec_desc.iec.type = QI_TYPE_IEC;
+ iec_desc.iec.granularity = 0; /* Global Invalidation requested */
+
+ return vtd_qi_send(dev, &iec_desc.desc);
+}
+
+static int vtd_index_iec_invalidate(const struct device *dev, uint8_t irte_idx)
+{
+ union qi_iec_descriptor iec_desc = { 0 };
+
+ iec_desc.iec.type = QI_TYPE_IEC;
+ iec_desc.iec.granularity = 1; /* Index based invalidation requested */
+
+ iec_desc.iec.interrupt_index = irte_idx;
+ iec_desc.iec.index_mask = 0;
+
+ return vtd_qi_send(dev, &iec_desc.desc);
+}
+
static void fault_status_description(uint32_t status)
{
if (status & VTD_FSTS_PFO) {
@@ -233,6 +326,8 @@
data->irte[irte_idx].low = irte.low;
data->irte[irte_idx].high = irte.high;
+ vtd_index_iec_invalidate(dev, irte_idx);
+
return 0;
}
@@ -318,11 +413,14 @@
unsigned int key = irq_lock();
uint64_t eime = 0;
uint64_t value;
+ int ret = 0;
DEVICE_MMIO_MAP(dev, K_MEM_CACHE_NONE);
vtd_fault_event_init(dev);
+ vtd_qi_init(dev);
+
if (IS_ENABLED(CONFIG_X2APIC)) {
eime = VTD_IRTA_EIME;
}
@@ -332,6 +430,12 @@
vtd_write_reg64(dev, VTD_IRTA_REG, value);
+ if (vtd_global_iec_invalidate(dev) != 0) {
+ printk("Could not perform IEC invalidation\n");
+ ret = -EIO;
+ goto out;
+ }
+
if (!IS_ENABLED(CONFIG_X2APIC) &&
IS_ENABLED(CONFIG_INTEL_VTD_ICTL_XAPIC_PASSTHROUGH)) {
vtd_send_cmd(dev, VTD_GCMD_CFI, VTD_GSTS_CFIS);
@@ -343,9 +447,10 @@
printk("Intel VT-D up and running (status 0x%x)\n",
vtd_read_reg32(dev, VTD_GSTS_REG));
+out:
irq_unlock(key);
- return 0;
+ return ret;
}
static const struct vtd_driver_api vtd_api = {
diff --git a/drivers/interrupt_controller/intc_intel_vtd.h b/drivers/interrupt_controller/intc_intel_vtd.h
index 2789dda..22caea6 100644
--- a/drivers/interrupt_controller/intc_intel_vtd.h
+++ b/drivers/interrupt_controller/intc_intel_vtd.h
@@ -52,8 +52,59 @@
#define IRTE_NUM 256
#define IRTA_SIZE 7 /* size = 2^(X+1) where IRTA_SIZE is X 2^8 = 256 */
+#define QI_NUM 256 /* Which is the minimal number we can set for the queue */
+#define QI_SIZE 0 /* size = 2^(X+8) where QI_SIZE is X: 2^8 = 256 */
+#define QI_WIDTH 128
+
+struct qi_descriptor {
+ uint64_t low;
+ uint64_t high;
+};
+
+#define QI_TYPE_IEC 0x4UL
+
+union qi_iec_descriptor {
+ struct qi_descriptor desc;
+
+ struct iec_bits {
+ uint64_t type : 4;
+ uint64_t granularity : 1;
+ uint64_t _reserved_0 : 4;
+ uint64_t zero : 3;
+ uint64_t _reserved_1 : 15;
+ uint64_t index_mask : 5;
+ uint64_t interrupt_index: 16;
+ uint64_t _reserved_2 : 16;
+ uint64_t reserved;
+ } iec __packed;
+};
+
+#define QI_TYPE_WAIT 0x5UL
+
+union qi_wait_descriptor {
+ struct qi_descriptor desc;
+
+ struct wait_bits {
+ uint64_t type : 4;
+ uint64_t interrupt_flag : 1;
+ uint64_t status_write : 1;
+ uint64_t fence_flag : 1;
+ uint64_t page_req_drain : 1;
+ uint64_t _reserved_0 : 1;
+ uint64_t zero : 3;
+ uint64_t _reserved_1 : 20;
+ uint64_t status_data : 32;
+ uint64_t reserved : 2;
+ uint64_t address : 62;
+ } wait __packed;
+};
+
+#define QI_WAIT_STATUS_INCOMPLETE 0x0UL
+#define QI_WAIT_STATUS_COMPLETE 0x1UL
+
struct vtd_ictl_data {
struct vtd_irte irte[IRTE_NUM];
+ struct qi_descriptor qi[QI_NUM] __aligned(0x1000);
int irqs[IRTE_NUM];
int vectors[IRTE_NUM];
bool msi[IRTE_NUM];
@@ -61,6 +112,7 @@
unsigned int fault_irq;
uintptr_t fault_record_reg;
uint16_t fault_record_num;
+ uint16_t qi_tail;
uint8_t fault_vector;
};
diff --git a/include/arch/x86/intel_vtd.h b/include/arch/x86/intel_vtd.h
index 2f6ebbd..2fe005c 100644
--- a/include/arch/x86/intel_vtd.h
+++ b/include/arch/x86/intel_vtd.h
@@ -140,6 +140,7 @@
#define VTD_FSTS_IQE BIT(4)
#define VTD_FSTS_ICE BIT(5)
#define VTD_FSTS_ITE BIT(6)
+
#define VTD_FSTS_FRI_POS 8
#define VTD_FSTS_FRI_MASK (0xF << VTD_FSTS_FRI_POS)
#define VTD_FSTS_FRI(status) \
@@ -182,6 +183,21 @@
#define VTD_FRCD_FI_IR(fault) \
((fault & VTD_FRCD_FI_IR_MASK) >> VTD_FRCD_FI_IR_POS)
+/* Invalidation Queue Address register details */
+#define VTD_IQA_SIZE_MASK 0x7
+#define VTD_IQA_WIDTH_128_BIT 0
+#define VTD_IQA_WIDTH_256_BIT BIT(11)
+#define VTD_IQA_REG_GEN_CONTENT(addr, width, size) \
+ ((uint64_t)0 | (addr) | (width) | (size & VTD_IQA_SIZE_MASK))
+
+/* Invalidation Queue Head register details */
+#define VTD_IQH_QH_POS_128 4
+#define VTD_IQH_QH_MASK ((uint64_t)0xEF << VTD_IQH_QH_POS_128)
+
+/* Invalidation Queue Tail register details */
+#define VTD_IQT_QT_POS_128 4
+#define VTD_IQT_QT_MASK ((uint64_t)0xEF << VTD_IQT_QT_POS_128)
+
#endif /* _ASMLANGUAGE */
#endif /* ZEPHYR_INCLUDE_ARCH_X86_INTEL_VTD_H */
