[PATCH] xen/xenbus: don't deliver user replies under msgbuffer_mutex

From: Benjamin Leggett

Date: Wed Nov 18 2026 - 18:35:48 EST


Found this while testing, and wrote a little reproducer. Seems like the
guest kernel's xenbus thread is stuck waiting for a lock, so no xenstore
replies reach the guest once this happens.

The lock involved, from what I can tell, is u->msgbuffer_mutex in
drivers/xen/xenbus/xenbus_dev_frontend.c:

- xenbus_file_write() takes the lock. For XS_WATCH/XS_UNWATCH it keeps
the lock while it waits for the xenstored reply
(unregister_xenbus_watch() -> xs_talkv).

- xenbus_dev_queue_reply() takes the same lock. The xenbus thread
calls it to deliver the reply of an earlier request on the same fd.

So if a fd still has a request waiting for its reply when it gets an
(UN)WATCH, the (UN)WATCH holds the lock and waits for the xenbus thread.
The xenbus thread waits for the lock, and at this point the whole guest
xenstore connection stops.

sysrq: Show Blocked State
task:xenbus state:D stack:14672 pid:42
__mutex_lock.constprop.0+0x3a2/0x9e0
xenbus_dev_queue_reply+0x49/0x330
xenbus_thread+0x221/0xa40
kthread+0xff/0x250
task:xenbus-unwatch- state:D stack:12760 pid:1353
xs_talkv+0x27d/0x3a0
unregister_xenbus_watch+0x171/0x320
xenbus_file_write+0x491/0x670
vfs_write+0x109/0x460
ksys_write+0xc8/0xf0

This is reproducible by opening /dev/xen/xenbus, writing an XS_WATCH for
any path, then on the same fd write an XS_READ for that path followed by
an XS_UNWATCH, without reading the READ reply first.

Fix is the following:
- xenbus_dev_queue_reply() needed the lock only for the transactions
list.
- That list now has its own mutex, taken in xenbus_write_transaction()
and xenbus_dev_queue_reply().
- xenbus_worker() keeps walking the list without a lock, because it
runs after the last reference is gone.

On an unpatched guest kernel the first XS_UNWATCH never returns, and the
guest's watches stop firing. With this patch, a guest kernel built with
CONFIG_PROVE_LOCKING=y completes 100000 such rounds without a stall, and
lockdep stays enabled with the new lock ordered between the existing two:

/proc/lockdep_stats:
debug_locks: 1
/proc/lockdep:
&u->msgbuffer_mutex
-> &u->reply_mutex
-> &u->transactions_mutex
&u->transactions_mutex
-> &u->reply_mutex

Fixes: fd8aa9095a95 ("xen: optimize xenbus driver for multiple concurrent xenstore accesses")
Cc: stable@xxxxxxxxxxxxxxx
Signed-off-by: Benjamin Leggett <benjamin@xxxxxxxx>
---
drivers/xen/xenbus/xenbus_dev_frontend.c | 46 ++++++++++++++++++++------------
1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/drivers/xen/xenbus/xenbus_dev_frontend.c b/drivers/xen/xenbus/xenbus_dev_frontend.c
index 61db6932a9d2..92ddc37311cf 100644
--- a/drivers/xen/xenbus/xenbus_dev_frontend.c
+++ b/drivers/xen/xenbus/xenbus_dev_frontend.c
@@ -89,8 +89,12 @@ struct xenbus_file_priv {
/*
* msgbuffer_mutex is held while partial requests are built up
* and complete requests are acted on. It therefore protects
- * the "transactions" and "watches" lists, and the partial
- * request length and buffer.
+ * the "watches" list, and the partial request length and buffer.
+ *
+ * transactions_mutex protects the "transactions" list. It nests
+ * inside msgbuffer_mutex, and xenbus_thread takes it alone to
+ * deliver a reply, so a (UN)WATCH that holds msgbuffer_mutex
+ * while it waits on xenbus_thread cannot stall that delivery.
*
* reply_mutex protects the reply being built up to return to
* usermode. It nests inside msgbuffer_mutex but may be held
@@ -99,6 +103,7 @@ struct xenbus_file_priv {
struct mutex msgbuffer_mutex;

/* In-progress transactions */
+ struct mutex transactions_mutex;
struct list_head transactions;

/* Active watches. */
@@ -369,7 +374,7 @@ void xenbus_dev_queue_reply(struct xb_req_data *req)

xs_request_exit(req);

- mutex_lock(&u->msgbuffer_mutex);
+ mutex_lock(&u->transactions_mutex);

if (req->type == XS_TRANSACTION_START) {
trans = xenbus_get_transaction(u, 0);
@@ -391,7 +396,7 @@ void xenbus_dev_queue_reply(struct xb_req_data *req)
kfree(trans);
}

- mutex_unlock(&u->msgbuffer_mutex);
+ mutex_unlock(&u->transactions_mutex);

mutex_lock(&u->reply_mutex);
rc = queue_reply(&staging_q, &req->msg, sizeof(req->msg));
@@ -413,7 +418,7 @@ void xenbus_dev_queue_reply(struct xb_req_data *req)
return;

out:
- mutex_unlock(&u->msgbuffer_mutex);
+ mutex_unlock(&u->transactions_mutex);
}

static int xenbus_command_reply(struct xenbus_file_priv *u,
@@ -453,6 +458,8 @@ static int xenbus_write_transaction(unsigned msg_type,
char body[];
} *msg = (void *)u->u.buffer;

+ mutex_lock(&u->transactions_mutex);
+
if (msg_type == XS_TRANSACTION_START) {
trans = kzalloc_obj(*trans);
if (!trans) {
@@ -462,24 +469,27 @@ static int xenbus_write_transaction(unsigned msg_type,
trans->generation_id = xb_dev_generation_id;
list_add(&trans->list, &u->transactions);
} else if (msg->hdr.tx_id != 0 &&
- !xenbus_get_transaction(u, msg->hdr.tx_id))
- return xenbus_command_reply(u, XS_ERROR, "ENOENT");
- else if (msg_type == XS_TRANSACTION_END &&
- !(msg->hdr.len == 2 &&
- (!strcmp(msg->body, "T") || !strcmp(msg->body, "F"))))
- return xenbus_command_reply(u, XS_ERROR, "EINVAL");
- else if (msg_type == XS_TRANSACTION_END) {
+ !xenbus_get_transaction(u, msg->hdr.tx_id)) {
+ rc = xenbus_command_reply(u, XS_ERROR, "ENOENT");
+ goto out;
+ } else if (msg_type == XS_TRANSACTION_END &&
+ !(msg->hdr.len == 2 &&
+ (!strcmp(msg->body, "T") || !strcmp(msg->body, "F")))) {
+ rc = xenbus_command_reply(u, XS_ERROR, "EINVAL");
+ goto out;
+ } else if (msg_type == XS_TRANSACTION_END) {
trans = xenbus_get_transaction(u, msg->hdr.tx_id);
if (trans && trans->generation_id != xb_dev_generation_id) {
list_del(&trans->list);
kfree(trans);
if (!strcmp(msg->body, "T"))
- return xenbus_command_reply(u, XS_ERROR,
- "EAGAIN");
+ rc = xenbus_command_reply(u, XS_ERROR,
+ "EAGAIN");
else
- return xenbus_command_reply(u,
- XS_TRANSACTION_END,
- "OK");
+ rc = xenbus_command_reply(u,
+ XS_TRANSACTION_END,
+ "OK");
+ goto out;
}
}

@@ -490,6 +500,7 @@ static int xenbus_write_transaction(unsigned msg_type,
}

out:
+ mutex_unlock(&u->transactions_mutex);
return rc;
}

@@ -669,6 +680,7 @@ static int xenbus_file_open(struct inode *inode, struct file *filp)

mutex_init(&u->reply_mutex);
mutex_init(&u->msgbuffer_mutex);
+ mutex_init(&u->transactions_mutex);

filp->private_data = u;


---
base-commit: 2b66f2cc753243a6adb3e9a5c038b0fc59ba7ec9
change-id: 20261118-bleggett-xenbus-transactions-mutex-ca2d20625f10

Best regards,
--
Benjamin Leggett <benjamin@xxxxxxxx>