diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -92,6 +92,22 @@ enum vmd_features { * referred to as MEMBAR2 or MSI-X BAR. */ VMD_FEAT_USE_BIOS_INFO = (1 << 6), + + /* + * Meteor Lake VMD (device ID 0x7d0b) is affected by erratum MTL016: + * the VMD may signal its MSI before the posted writes that carry the + * child device's DMA data have landed in memory. The demuxed handler + * then runs against a not-yet-coherent completion queue and misses the + * completion entirely, so the I/O is only recovered when the block + * layer timeout fires and polls the queue ("timeout, completion + * polled"). Intel's documented workaround is to issue a dummy read + * to the MSI initiator (the child device) before handling the + * interrupt: the read completion cannot pass the device's earlier + * posted writes, so it pulls them into memory per PCIe ordering + * rules. A read that terminates at the VMD itself does not order + * against the child's writes and is not sufficient (measured). + */ + VMD_FEAT_INTERRUPT_QUIRK = (1 << 7), }; #define VMD_BIOS_PM_QUIRK_LTR 0x1003 /* 3145728 ns */ @@ -114,6 +130,9 @@ static DEFINE_RAW_SPINLOCK(list_lock); * @irq: back pointer to parent. * @enabled: true if driver enabled IRQ * @virq: the virtual IRQ value provided to the requesting driver. + * @flush_addr: config space address of the initiating device, read before + * demuxing to flush its posted writes (MTL016); NULL if the + * VMD is not affected. * * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to * a VMD IRQ using this structure. @@ -123,8 +142,14 @@ struct vmd_irq { struct vmd_irq_list *irq; bool enabled; unsigned int virq; + void __iomem *flush_addr; }; +struct vmd_dev; + +static void __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus, + unsigned int devfn, int reg, int len); + /** * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector * @irq_list: the list of irq's the VMD one demuxes to. @@ -132,12 +157,15 @@ struct vmd_irq { * @count: number of child IRQs assigned to this vector; used to track * sharing. * @virq: The underlying VMD Linux interrupt number + * @vmd: back pointer to the owning VMD device; serializes the MTL016 + * flush read against other config space access. */ struct vmd_irq_list { struct list_head irq_list; struct srcu_struct srcu; unsigned int count; unsigned int virq; + struct vmd_dev *vmd; }; struct vmd_dev { @@ -295,6 +323,13 @@ static int vmd_msi_alloc(struct irq_domain *domain, unsigned int virq, INIT_LIST_HEAD(&vmdirq->node); vmdirq->irq = vmd_next_irq(vmd, desc); vmdirq->virq = virq + i; + if (vmd->features & VMD_FEAT_INTERRUPT_QUIRK) { + struct pci_dev *pdev = msi_desc_to_pci_dev(desc); + + vmdirq->flush_addr = vmd_cfg_addr(vmd, pdev->bus, + pdev->devfn, + PCI_VENDOR_ID, 2); + } irq_domain_set_info(domain, virq + i, vmdirq->irq->virq, &vmd_msi_controller, vmdirq, @@ -753,8 +788,20 @@ static irqreturn_t vmd_irq(int irq, void *data) int idx; idx = srcu_read_lock(&irqs->srcu); - list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node) + list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node) { + /* + * MTL016: the MSI may have outrun the initiating device's + * posted writes (e.g. its NVMe completion entry). A read + * that completes at the initiator flushes them, so the + * demuxed handler observes a coherent completion queue. + * The value is discarded; only the ordering matters. + */ + if (vmdirq->flush_addr) { + guard(raw_spinlock)(&irqs->vmd->cfg_lock); + readw(vmdirq->flush_addr); + } generic_handle_irq(vmdirq->virq); + } srcu_read_unlock(&irqs->srcu, idx); return IRQ_HANDLED; @@ -785,6 +832,7 @@ static int vmd_alloc_irqs(struct vmd_dev *vmd) return err; INIT_LIST_HEAD(&vmd->irqs[i].irq_list); + vmd->irqs[i].vmd = vmd; vmd->irqs[i].virq = pci_irq_vector(dev, i); err = devm_request_irq(&dev->dev, vmd->irqs[i].virq, vmd_irq, IRQF_NO_THREAD, @@ -1239,7 +1287,7 @@ static const struct pci_device_id vmd_ids[] = { {PCI_VDEVICE(INTEL, 0xa77f), .driver_data = VMD_FEATS_CLIENT,}, {PCI_VDEVICE(INTEL, 0x7d0b), - .driver_data = VMD_FEATS_CLIENT,}, + .driver_data = VMD_FEATS_CLIENT | VMD_FEAT_INTERRUPT_QUIRK,}, {PCI_VDEVICE(INTEL, 0xad0b), .driver_data = VMD_FEATS_CLIENT,}, {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B),