121 lines
4.5 KiB
Diff
121 lines
4.5 KiB
Diff
diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
|
|
--- a/drivers/pci/controller/vmd.c
|
|
+++ b/drivers/pci/controller/vmd.c
|
|
@@ -92,6 +92,22 @@ enum vmd_features {
|
|
* referred to as MEMBAR2 or MSI-X BAR.
|
|
*/
|
|
VMD_FEAT_USE_BIOS_INFO = (1 << 6),
|
|
+
|
|
+ /*
|
|
+ * Meteor Lake VMD (device ID 0x7d0b) is affected by erratum MTL016:
|
|
+ * the VMD may signal its MSI before the posted writes that carry the
|
|
+ * child device's DMA data have landed in memory. The demuxed handler
|
|
+ * then runs against a not-yet-coherent completion queue and misses the
|
|
+ * completion entirely, so the I/O is only recovered when the block
|
|
+ * layer timeout fires and polls the queue ("timeout, completion
|
|
+ * polled"). Intel's documented workaround is to issue a dummy read
|
|
+ * to the MSI initiator (the child device) before handling the
|
|
+ * interrupt: the read completion cannot pass the device's earlier
|
|
+ * posted writes, so it pulls them into memory per PCIe ordering
|
|
+ * rules. A read that terminates at the VMD itself does not order
|
|
+ * against the child's writes and is not sufficient (measured).
|
|
+ */
|
|
+ VMD_FEAT_INTERRUPT_QUIRK = (1 << 7),
|
|
};
|
|
|
|
#define VMD_BIOS_PM_QUIRK_LTR 0x1003 /* 3145728 ns */
|
|
@@ -114,6 +130,9 @@ static DEFINE_RAW_SPINLOCK(list_lock);
|
|
* @irq: back pointer to parent.
|
|
* @enabled: true if driver enabled IRQ
|
|
* @virq: the virtual IRQ value provided to the requesting driver.
|
|
+ * @flush_addr: config space address of the initiating device, read before
|
|
+ * demuxing to flush its posted writes (MTL016); NULL if the
|
|
+ * VMD is not affected.
|
|
*
|
|
* Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to
|
|
* a VMD IRQ using this structure.
|
|
@@ -123,8 +142,14 @@ struct vmd_irq {
|
|
struct vmd_irq_list *irq;
|
|
bool enabled;
|
|
unsigned int virq;
|
|
+ void __iomem *flush_addr;
|
|
};
|
|
|
|
+struct vmd_dev;
|
|
+
|
|
+static void __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,
|
|
+ unsigned int devfn, int reg, int len);
|
|
+
|
|
/**
|
|
* struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector
|
|
* @irq_list: the list of irq's the VMD one demuxes to.
|
|
@@ -132,12 +157,15 @@ struct vmd_irq {
|
|
* @count: number of child IRQs assigned to this vector; used to track
|
|
* sharing.
|
|
* @virq: The underlying VMD Linux interrupt number
|
|
+ * @vmd: back pointer to the owning VMD device; serializes the MTL016
|
|
+ * flush read against other config space access.
|
|
*/
|
|
struct vmd_irq_list {
|
|
struct list_head irq_list;
|
|
struct srcu_struct srcu;
|
|
unsigned int count;
|
|
unsigned int virq;
|
|
+ struct vmd_dev *vmd;
|
|
};
|
|
|
|
struct vmd_dev {
|
|
@@ -295,6 +323,13 @@ static int vmd_msi_alloc(struct irq_domain *domain, unsigned int virq,
|
|
INIT_LIST_HEAD(&vmdirq->node);
|
|
vmdirq->irq = vmd_next_irq(vmd, desc);
|
|
vmdirq->virq = virq + i;
|
|
+ if (vmd->features & VMD_FEAT_INTERRUPT_QUIRK) {
|
|
+ struct pci_dev *pdev = msi_desc_to_pci_dev(desc);
|
|
+
|
|
+ vmdirq->flush_addr = vmd_cfg_addr(vmd, pdev->bus,
|
|
+ pdev->devfn,
|
|
+ PCI_VENDOR_ID, 2);
|
|
+ }
|
|
|
|
irq_domain_set_info(domain, virq + i, vmdirq->irq->virq,
|
|
&vmd_msi_controller, vmdirq,
|
|
@@ -753,8 +788,20 @@ static irqreturn_t vmd_irq(int irq, void *data)
|
|
int idx;
|
|
|
|
idx = srcu_read_lock(&irqs->srcu);
|
|
- list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node)
|
|
+ list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node) {
|
|
+ /*
|
|
+ * MTL016: the MSI may have outrun the initiating device's
|
|
+ * posted writes (e.g. its NVMe completion entry). A read
|
|
+ * that completes at the initiator flushes them, so the
|
|
+ * demuxed handler observes a coherent completion queue.
|
|
+ * The value is discarded; only the ordering matters.
|
|
+ */
|
|
+ if (vmdirq->flush_addr) {
|
|
+ guard(raw_spinlock)(&irqs->vmd->cfg_lock);
|
|
+ readw(vmdirq->flush_addr);
|
|
+ }
|
|
generic_handle_irq(vmdirq->virq);
|
|
+ }
|
|
srcu_read_unlock(&irqs->srcu, idx);
|
|
|
|
return IRQ_HANDLED;
|
|
@@ -785,6 +832,7 @@ static int vmd_alloc_irqs(struct vmd_dev *vmd)
|
|
return err;
|
|
|
|
INIT_LIST_HEAD(&vmd->irqs[i].irq_list);
|
|
+ vmd->irqs[i].vmd = vmd;
|
|
vmd->irqs[i].virq = pci_irq_vector(dev, i);
|
|
err = devm_request_irq(&dev->dev, vmd->irqs[i].virq,
|
|
vmd_irq, IRQF_NO_THREAD,
|
|
@@ -1239,7 +1287,7 @@ static const struct pci_device_id vmd_ids[] = {
|
|
{PCI_VDEVICE(INTEL, 0xa77f),
|
|
.driver_data = VMD_FEATS_CLIENT,},
|
|
{PCI_VDEVICE(INTEL, 0x7d0b),
|
|
- .driver_data = VMD_FEATS_CLIENT,},
|
|
+ .driver_data = VMD_FEATS_CLIENT | VMD_FEAT_INTERRUPT_QUIRK,},
|
|
{PCI_VDEVICE(INTEL, 0xad0b),
|
|
.driver_data = VMD_FEATS_CLIENT,},
|
|
{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B),
|