[PATCH 27/40] nitro_enclaves: Dispatch multiplexed interrupts to per-queue bindings

From: Alexander Graf

Date: Wed Nov 11 2026 - 18:15:34 EST


Nitro Sandboxes provide a way for user space to receive
per-device-virtqueue events. These are multiplexed through a
nitro_enclaves device built-in interrupt multiplexer. Implement logic to
support this multiplexer and distribute the respective interrupt events.

Signed-off-by: Alexander Graf <graf@xxxxxxxxxx>
---
drivers/virt/nitro_enclaves/ne_pci_dev.c | 179 ++++++++++++++++++++++-
drivers/virt/nitro_enclaves/ne_pci_dev.h | 47 ++++++
2 files changed, 225 insertions(+), 1 deletion(-)

diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.c b/drivers/virt/nitro_enclaves/ne_pci_dev.c
index 0fafd42162f3a..de96d68906bbf 100644
--- a/drivers/virt/nitro_enclaves/ne_pci_dev.c
+++ b/drivers/virt/nitro_enclaves/ne_pci_dev.c
@@ -7,6 +7,7 @@
* DOC: Nitro Enclaves (NE) PCI device driver.
*/

+#include <linux/bitops.h>
#include <linux/cma.h>
#include <linux/delay.h>
#include <linux/device.h>
@@ -27,6 +28,7 @@
#include <linux/sysfs.h>
#include <linux/types.h>
#include <linux/wait.h>
+#include <linux/xarray.h>

#include "ne_misc_dev.h"
#include "ne_pci_dev.h"
@@ -551,6 +553,129 @@ static void ne_teardown_msix(struct pci_dev *pdev)
pci_free_irq_vectors(pdev);
}

+/**
+ * ne_mux_binding_rcu_free() - RCU callback freeing one enclave queue binding.
+ * @head: RCU head of the binding to free.
+ *
+ * Context: Softirq or process context.
+ */
+static void ne_mux_binding_rcu_free(struct rcu_head *head)
+{
+ struct ne_mux_binding *binding = container_of(head,
+ struct ne_mux_binding,
+ rcu);
+
+ eventfd_ctx_put(binding->ctx);
+
+ kfree(binding);
+}
+
+void ne_mux_binding_free_rcu(struct ne_mux_binding *binding)
+{
+ call_rcu(&binding->rcu, ne_mux_binding_rcu_free);
+}
+
+/**
+ * ne_mux_scan() - Deliver the bits set in the bitmap of a pooled vector that it
+ * has a binding for.
+ * @mux_vec: Multiplexer state of the pooled vector.
+ *
+ * A bit leaves the bitmap by the same atomic read-modify-write that decides to
+ * deliver it, so a bit this vector has no binding for stays set. It is left set
+ * on purpose: a queue can be kicked before user space has bound an eventfd to
+ * it, and a bit taken out of the bitmap on the strength of a binding that is not
+ * there yet is a kick nobody can ever be told about. The bit waits instead, and
+ * the bind of that queue takes it and delivers it.
+ *
+ * So a set bit is consumed at most once, and exactly once if a scan finds a
+ * binding for it or a bind for its index runs. A bit whose index is never bound
+ * is consumed never, and stays set until the page is reclaimed when the device
+ * unbinds. A bit whose bind is still coming looks the same to the driver, so it
+ * holds both.
+ *
+ * The bound only ever rises, so a scan reads every word a binding has reached
+ * into and not only the words the bindings of the moment need.
+ *
+ * Context: Interrupt context. Called with the RCU read-side held.
+ * Return:
+ * * true if the scan found no bit it could deliver, false if it found at least
+ * one.
+ */
+static bool ne_mux_scan(struct ne_mux_vec *mux_vec)
+{
+ struct ne_pci_dev *ne_pci_dev = mux_vec->ne_pci_dev;
+ u32 bits_per_word = BITS_PER_TYPE(*mux_vec->bitmap);
+ u32 nr_words = READ_ONCE(mux_vec->max_bit) / bits_per_word + 1;
+ bool empty = true;
+ u32 i = 0;
+
+ for (i = 0; i < nr_words; i++) {
+ u64 word = READ_ONCE(mux_vec->bitmap[i]);
+
+ while (word) {
+ u32 bit = i * bits_per_word + __ffs64(word);
+ unsigned long notify_index = bit * ne_pci_dev->mux_nr_vecs +
+ mux_vec->vec_rel;
+ struct ne_mux_binding *binding = NULL;
+
+ word &= word - 1;
+
+ binding = xa_load(&ne_pci_dev->notify_xa, notify_index);
+ if (!binding)
+ continue;
+
+ empty = false;
+
+ if (test_and_clear_bit(bit, (unsigned long *)mux_vec->bitmap))
+ eventfd_signal(binding->ctx);
+ }
+ }
+
+ return empty;
+}
+
+/**
+ * ne_mux_handler() - Interrupt handler for one pooled MSI-X vector.
+ * @irq: Interrupt line the pooled vector is on.
+ * @data: Multiplexer state of the pooled vector.
+ *
+ * Context: Interrupt context.
+ * Return:
+ * * IRQ_HANDLED on interrupt handled.
+ */
+static irqreturn_t ne_mux_handler(int irq, void *data)
+{
+ struct ne_mux_vec *mux_vec = data;
+ u32 pass = 0;
+
+ /*
+ * xa_load() drops its own RCU read-side before it returns, so take one
+ * here to keep a binding alive for as long as it is dereferenced.
+ */
+ rcu_read_lock();
+
+ /*
+ * The device sets a bit and then raises the interrupt, and one interrupt
+ * can stand for several bits, so a scan can miss a bit set while it runs
+ * or one past the bound it read. The device raises for every bit it sets
+ * whatever the bitmap holds, so such a bit has an interrupt of its own
+ * still to come and reaches its eventfd on that one. Scanning again,
+ * reading the bound again, delivers it on this interrupt instead, so stop
+ * on the first scan that delivers nothing. Stopping on an empty bitmap
+ * instead would wait for a read a bit no bind is coming for never gives.
+ * NE_MUX_MAX_PASSES caps the scans either way, so a device that goes on
+ * setting bits cannot hold the handler.
+ */
+ do {
+ if (ne_mux_scan(mux_vec))
+ break;
+ } while (++pass < NE_MUX_MAX_PASSES);
+
+ rcu_read_unlock();
+
+ return IRQ_HANDLED;
+}
+
/**
* ne_mux_teardown() - Take the interrupt multiplexer bitmaps back from the
* device.
@@ -561,6 +686,8 @@ static void ne_teardown_msix(struct pci_dev *pdev)
static void ne_mux_teardown(struct pci_dev *pdev)
{
struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
+ struct ne_mux_binding *binding = NULL;
+ unsigned long notify_index = 0;
u32 i = 0;

if (!ne_pci_dev->mux)
@@ -568,13 +695,40 @@ static void ne_mux_teardown(struct pci_dev *pdev)

iowrite32(NE_MUX_SHUTDOWN_ON, ne_pci_dev->iomem_base + NE_MUX_SHUTDOWN);

- for (i = 0; i < ne_pci_dev->mux_nr_vecs; i++)
+ for (i = 0; i < ne_pci_dev->mux_nr_vecs; i++) {
+ if (!ne_pci_dev->mux[i].bitmap)
+ continue;
+
+ if (ne_pci_dev->mux[i].irq_requested)
+ free_irq(pci_irq_vector(pdev,
+ ne_pci_dev->mux_vec_base + i),
+ &ne_pci_dev->mux[i]);
+
free_page((unsigned long)ne_pci_dev->mux[i].bitmap);
+ }

kfree(ne_pci_dev->mux);

ne_pci_dev->mux = NULL;
ne_pci_dev->mux_nr_vecs = 0;
+
+ xa_for_each(&ne_pci_dev->notify_xa, notify_index, binding) {
+ xa_erase(&ne_pci_dev->notify_xa, notify_index);
+
+ eventfd_ctx_put(binding->ctx);
+
+ kfree(binding);
+ }
+
+ /*
+ * Freeing the handlers above quiesced the readers that hold no lock,
+ * and the loop above freed every binding that outlived them. Wait for
+ * the callbacks a replaced binding already queued, so that none of
+ * them runs once the module text is gone.
+ */
+ rcu_barrier();
+
+ xa_destroy(&ne_pci_dev->notify_xa);
}

/**
@@ -631,8 +785,11 @@ static int ne_mux_setup(struct pci_dev *pdev)
ne_pci_dev->mux_nr_vecs = nr_vecs;
ne_pci_dev->mux_vec_base = vec_base;

+ xa_init(&ne_pci_dev->notify_xa);
+
for (i = 0; i < nr_vecs; i++) {
u16 donate_err = 0;
+ int irq = 0;

ne_pci_dev->mux[i].bitmap = (u64 *)get_zeroed_page(GFP_KERNEL);
if (!ne_pci_dev->mux[i].bitmap) {
@@ -663,6 +820,26 @@ static int ne_mux_setup(struct pci_dev *pdev)

goto teardown_mux;
}
+
+ ne_pci_dev->mux[i].ne_pci_dev = ne_pci_dev;
+ ne_pci_dev->mux[i].vec_rel = i;
+
+ irq = pci_irq_vector(pdev, vec_base + i);
+ if (irq < 0) {
+ rc = irq;
+
+ goto teardown_mux;
+ }
+
+ rc = request_irq(irq, ne_mux_handler, 0, "enclave_mux",
+ &ne_pci_dev->mux[i]);
+ if (rc < 0) {
+ dev_err(&pdev->dev, "Error in request irq mux [rc=%d]\n", rc);
+
+ goto teardown_mux;
+ }
+
+ ne_pci_dev->mux[i].irq_requested = true;
}

return 0;
diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.h b/drivers/virt/nitro_enclaves/ne_pci_dev.h
index 2cef40c627f9f..0dbce83cb06bd 100644
--- a/drivers/virt/nitro_enclaves/ne_pci_dev.h
+++ b/drivers/virt/nitro_enclaves/ne_pci_dev.h
@@ -9,12 +9,15 @@
#include <linux/atomic.h>
#include <linux/bits.h>
#include <linux/build_bug.h>
+#include <linux/eventfd.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
+#include <linux/rcupdate.h>
#include <linux/stddef.h>
#include <linux/wait.h>
+#include <linux/xarray.h>

/**
* DOC: Nitro Enclaves (NE) PCI device
@@ -201,6 +204,12 @@
static_assert(NE_MUX_MAX_BITS / BITS_PER_TYPE(u64) * sizeof(u64) <= PAGE_SIZE,
"the bitmap of a pooled vector has to fit its donated page");

+/**
+ * NE_MUX_MAX_PASSES - Largest number of times the handler of a pooled vector
+ * scans its bitmap in one interrupt.
+ */
+#define NE_MUX_MAX_PASSES (4)
+
/**
* enum ne_pci_dev_cmd_type - Device command types.
* @INVALID_CMD: Invalid command.
@@ -648,9 +657,35 @@ struct ne_slot_pool_run {
* @bitmap: Page-sized bitmap in which the device sets a bit
* for every enclave queue that signalled through
* this vector.
+ * @max_bit: Highest bitmap bit an enclave queue has been
+ * bound to, bounding the scan of @bitmap. It only
+ * rises, so a scan reads the words of bindings that
+ * are gone as well.
+ * @ne_pci_dev: NE PCI device this vector belongs to.
+ * @vec_rel: Index of this vector within the pool.
+ * @irq_requested: Whether the interrupt handler of this vector is
+ * installed, which is not implied by @bitmap: a
+ * vector whose bitmap the device already owns may
+ * have no handler yet.
*/
struct ne_mux_vec {
u64 *bitmap;
+ u32 max_bit;
+ struct ne_pci_dev *ne_pci_dev;
+ u32 vec_rel;
+ bool irq_requested;
+};
+
+/**
+ * struct ne_mux_binding - Eventfd bound to one enclave queue.
+ * @ctx: Eventfd signalled when the queue is kicked.
+ * @rcu: Head used to free the binding after a grace
+ * period, so that an interrupt handler which
+ * already loaded it can still dereference it.
+ */
+struct ne_mux_binding {
+ struct eventfd_ctx *ctx;
+ struct rcu_head rcu;
};

/**
@@ -672,6 +707,8 @@ struct ne_mux_vec {
* capability, and with it the multiplexer and slot
* pool registers.
* @iomem_base : MMIO region of the PCI device.
+ * @notify_xa: Enclave queue bindings, keyed by the notify index of
+ * the queue.
* @mux: Per-pooled-vector interrupt multiplexer state.
* @mux_nr_vecs: Number of MSI-X vectors in the interrupt
* multiplexer pool.
@@ -706,6 +743,7 @@ struct ne_pci_dev {
struct workqueue_struct *event_wq;
bool has_vendor_cap;
void __iomem *iomem_base;
+ struct xarray notify_xa;
struct ne_mux_vec *mux;
u32 mux_nr_vecs;
u32 mux_vec_base;
@@ -750,6 +788,15 @@ int ne_do_request(struct pci_dev *pdev, enum ne_pci_dev_cmd_type cmd_type,
struct ne_pci_dev_cmd_reply *cmd_reply,
size_t cmd_reply_size);

+/**
+ * ne_mux_binding_free_rcu() - Free an enclave queue binding after a grace
+ * period.
+ * @binding: Binding that has been removed from the queue bindings xarray.
+ *
+ * Context: Process context.
+ */
+void ne_mux_binding_free_rcu(struct ne_mux_binding *binding);
+
/* Nitro Enclaves (NE) PCI device driver */
extern struct pci_driver ne_pci_driver;

--
2.47.1