[PATCH 30/40] nitro_enclaves: Count multiplexed interrupts under debugfs

From: Alexander Graf

Date: Wed Nov 11 2026 - 18:22:48 EST


To debug event delivery of Nitro Sandboxes, count what the multiplexer
handler and a bind do, and report it under debugfs:

/sys/kernel/debug/nitro_enclaves/mux_vectors
/sys/kernel/debug/nitro_enclaves/notifications

Signed-off-by: Alexander Graf <graf@xxxxxxxxxx>
---
drivers/virt/nitro_enclaves/ne_misc_dev.c | 9 ++-
drivers/virt/nitro_enclaves/ne_pci_dev.c | 84 +++++++++++++++++++++++
drivers/virt/nitro_enclaves/ne_pci_dev.h | 19 +++++
3 files changed, 110 insertions(+), 2 deletions(-)

diff --git a/drivers/virt/nitro_enclaves/ne_misc_dev.c b/drivers/virt/nitro_enclaves/ne_misc_dev.c
index d47fc8b514bcc..80ca23cd18251 100644
--- a/drivers/virt/nitro_enclaves/ne_misc_dev.c
+++ b/drivers/virt/nitro_enclaves/ne_misc_dev.c
@@ -2075,10 +2075,15 @@ static int ne_set_notify_eventfd(struct ne_enclave *ne_enclave,
* a bit with an atomic read-modify-write on the same word, so exactly
* one of the two consumes it. The bitmap words are the width the device
* writes them in, which is the width the bit numbering of the handler
- * is in as well.
+ * is in as well. Count the bit as delivered here, so that a bit this path
+ * takes is counted where it is taken the way one the handler takes is.
*/
- if (test_and_clear_bit(bit, (unsigned long *)ne_pci_dev->mux[vec_rel].bitmap))
+ if (test_and_clear_bit(bit, (unsigned long *)ne_pci_dev->mux[vec_rel].bitmap)) {
+ atomic64_inc(&ne_pci_dev->mux[vec_rel].nr_delivered);
+ atomic64_inc(&binding->nr_signals);
+
eventfd_signal(binding->ctx);
+ }

mutex_unlock(&ne_enclave->enclave_info_mutex);

diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.c b/drivers/virt/nitro_enclaves/ne_pci_dev.c
index cedf828240214..4678799998f4a 100644
--- a/drivers/virt/nitro_enclaves/ne_pci_dev.c
+++ b/drivers/virt/nitro_enclaves/ne_pci_dev.c
@@ -9,6 +9,7 @@

#include <linux/bitops.h>
#include <linux/cma.h>
+#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/err.h>
@@ -23,6 +24,7 @@
#include <linux/nitro_enclaves.h>
#include <linux/nodemask.h>
#include <linux/pci.h>
+#include <linux/seq_file.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <linux/sysfs.h>
@@ -649,6 +651,9 @@ static bool ne_mux_scan(struct ne_mux_vec *mux_vec)
if (!binding)
continue;

+ atomic64_inc(&mux_vec->nr_delivered);
+ atomic64_inc(&binding->nr_signals);
+
eventfd_signal(binding->ctx);
}
}
@@ -670,6 +675,8 @@ static irqreturn_t ne_mux_handler(int irq, void *data)
struct ne_mux_vec *mux_vec = data;
u32 pass = 0;

+ atomic64_inc(&mux_vec->nr_interrupts);
+
/*
* xa_load() drops its own RCU read-side before it returns, so take one
* here to keep a binding alive for as long as it is dereferenced.
@@ -698,6 +705,66 @@ static irqreturn_t ne_mux_handler(int irq, void *data)
return IRQ_HANDLED;
}

+/*
+ * The multiplexer's debugfs files, under /sys/kernel/debug/nitro_enclaves/.
+ *
+ * mux_vectors one row per pooled MSI-X vector
+ * notifications one row per bound enclave notification
+ *
+ * A row is rendered without serialising against the handler or against a bind,
+ * so it holds a snapshot of each counter and not one of the pool.
+ */
+
+static int ne_mux_vectors_show(struct seq_file *s, void *unused)
+{
+ struct ne_pci_dev *ne_pci_dev = s->private;
+ u32 i = 0;
+
+ seq_puts(s, "vector msix interrupts delivered max_bit\n");
+
+ for (i = 0; i < ne_pci_dev->mux_nr_vecs; i++) {
+ struct ne_mux_vec *mux_vec = &ne_pci_dev->mux[i];
+
+ seq_printf(s, "%6u %4u %10lld %9lld %7u\n",
+ i, ne_pci_dev->mux_vec_base + i,
+ atomic64_read(&mux_vec->nr_interrupts),
+ atomic64_read(&mux_vec->nr_delivered),
+ READ_ONCE(mux_vec->max_bit));
+ }
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(ne_mux_vectors);
+
+static int ne_notifications_show(struct seq_file *s, void *unused)
+{
+ struct ne_pci_dev *ne_pci_dev = s->private;
+ struct ne_mux_binding *binding = NULL;
+ unsigned long notify_index = 0;
+
+ seq_puts(s, "index vector bit signals\n");
+
+ /*
+ * A replaced or erased binding is freed after a grace period by
+ * ne_mux_binding_free_rcu(), so hold the RCU read side across the walk
+ * the way ne_mux_handler() does. seq_printf() renders into the
+ * seq_file buffer and does not sleep.
+ */
+ rcu_read_lock();
+
+ xa_for_each(&ne_pci_dev->notify_xa, notify_index, binding)
+ seq_printf(s, "%5lu %6lu %3lu %7lld\n",
+ notify_index,
+ notify_index % ne_pci_dev->mux_nr_vecs,
+ notify_index / ne_pci_dev->mux_nr_vecs,
+ atomic64_read(&binding->nr_signals));
+
+ rcu_read_unlock();
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(ne_notifications);
+
/**
* ne_mux_teardown() - Take the interrupt multiplexer bitmaps back from the
* device.
@@ -712,6 +779,16 @@ static void ne_mux_teardown(struct pci_dev *pdev)
unsigned long notify_index = 0;
u32 i = 0;

+ /*
+ * The walk in ne_notifications_show() is kept safe by the RCU read
+ * side against the deferred free, and not against the loop below,
+ * which frees the surviving bindings outright. Take the files away
+ * first: debugfs_remove_recursive() waits for a reader that is already
+ * in one.
+ */
+ debugfs_remove_recursive(ne_pci_dev->debugfs_dir);
+ ne_pci_dev->debugfs_dir = NULL;
+
if (!ne_pci_dev->mux)
return;

@@ -864,6 +941,13 @@ static int ne_mux_setup(struct pci_dev *pdev)
ne_pci_dev->mux[i].irq_requested = true;
}

+ ne_pci_dev->debugfs_dir = debugfs_create_dir("nitro_enclaves", NULL);
+
+ debugfs_create_file("mux_vectors", 0400, ne_pci_dev->debugfs_dir,
+ ne_pci_dev, &ne_mux_vectors_fops);
+ debugfs_create_file("notifications", 0400, ne_pci_dev->debugfs_dir,
+ ne_pci_dev, &ne_notifications_fops);
+
return 0;

teardown_mux:
diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.h b/drivers/virt/nitro_enclaves/ne_pci_dev.h
index 18aafe8111a67..71aaa8f557679 100644
--- a/drivers/virt/nitro_enclaves/ne_pci_dev.h
+++ b/drivers/virt/nitro_enclaves/ne_pci_dev.h
@@ -669,6 +669,11 @@ struct ne_slot_pool_run {
* bound to, bounding the scan of @bitmap. It only
* rises, so a scan reads the words of bindings that
* are gone as well.
+ * @nr_interrupts: Interrupts this vector's handler has taken.
+ * @nr_delivered: Bits it has taken out of @bitmap and delivered
+ * to an eventfd. An interrupt can deliver several
+ * bits, and one that finds no bit it can deliver
+ * delivers none.
* @ne_pci_dev: NE PCI device this vector belongs to.
* @vec_rel: Index of this vector within the pool.
* @irq_requested: Whether the interrupt handler of this vector is
@@ -679,6 +684,8 @@ struct ne_slot_pool_run {
struct ne_mux_vec {
u64 *bitmap;
u32 max_bit;
+ atomic64_t nr_interrupts;
+ atomic64_t nr_delivered;
struct ne_pci_dev *ne_pci_dev;
u32 vec_rel;
bool irq_requested;
@@ -691,10 +698,13 @@ struct ne_mux_vec {
* @rcu: Head used to free the binding after a grace
* period, so that an interrupt handler which
* already loaded it can still dereference it.
+ * @nr_signals: Times @ctx has been signalled for this
+ * notification.
*/
struct ne_mux_binding {
struct eventfd_ctx *ctx;
struct rcu_head rcu;
+ atomic64_t nr_signals;
};

/**
@@ -706,6 +716,14 @@ struct ne_mux_binding {
* the device after that.
* @cmd_reply_wait_q: Wait queue for handling command reply from the
* PCI device.
+ * @debugfs_dir: Directory holding the multiplexer's debugfs
+ * files. NULL until the multiplexer pool is set up
+ * and once it is torn down. debugfs_create_dir()
+ * answers with an error pointer rather than NULL
+ * where debugfs is not built in or the directory
+ * could not be made, and the calls that take it
+ * turn such a pointer down themselves, so it is
+ * kept as it came.
* @enclaves_list: List of the enclaves managed by the PCI device.
* @enclaves_list_mutex: Mutex for accessing the list of enclaves.
* @event_wq: Work queue for handling out-of-band events
@@ -747,6 +765,7 @@ struct ne_pci_dev {
atomic_t cmd_reply_avail;
bool failed;
wait_queue_head_t cmd_reply_wait_q;
+ struct dentry *debugfs_dir;
struct list_head enclaves_list;
struct mutex enclaves_list_mutex;
struct workqueue_struct *event_wq;
--
2.47.1