fs/ceph: debugfs mdsmap_show() can dereference a freed MDS map after a concurrent map update (UAF)
From: 徐晗翔
Date: Tue Nov 17 2026 - 07:47:48 EST
The Ceph client keeps the MDS map in mdsc->mdsmap. ceph_mdsc_handle_mdsmap()
replaces that pointer and immediately frees the previous struct ceph_mdsmap
while holding mdsc->mutex, and struct ceph_mdsmap has neither a reference
count nor an rcu_head. debugfs mdsmap_show() loads mdsc->mdsmap and
dereferences it (epoch/root/max_mds/... and the m_info[] array) without
taking mdsc->mutex, so a read of the debugfs file that overlaps any MDS map
update can touch freed memory. I reproduced this on 7.3 as 10 KASAN
slab-use-after-free reports in mdsmap_show(), with the free stack in
ceph_mdsc_handle_mdsmap(). The debugfs file is 0400/root-only, but MDS map
updates are routine cluster events (MDS restart, standby join, laggy set
changes).
The detailed description:
* Test environment / affected version
The tree under test is a Linux 7.3.0 tree (Makefile: VERSION 7, PATCHLEVEL 3,
SUBLEVEL 0) at /home/xuhanxiang/project/race/linux; it is not a git
checkout, so file hashes are given instead of a commit:
fs/ceph/debugfs.c 203ab6dbe128c3aab0a513d86f2f18e8
fs/ceph/mds_client.c 8678459253e5b1452667946bf4457c5a
fs/ceph/mdsmap.c 5cb2b54f0d3007961c36e68da1f6318c
fs/ceph/mdsmap.h 8effd15a2d508dc1f69b44b47a5ce3da
fs/ceph/super.c d06bb506183e26374582ee5d82322b8c
All line numbers below are from this tree. The writer, the free path and
the production reader are unmodified; the only source change used for
testing is the test-only instrumentation described below.
Kernel: 7.3.0 #2 PREEMPT(lazy), built with gcc 13.3.0, KASAN enabled:
CONFIG_KASAN=y
CONFIG_CEPH_FS=y
CONFIG_CEPH_LIB=y
CONFIG_DEBUG_FS=y
CONFIG_CEPH_RACE_VERIFY=y (test harness only)
Tested in a QEMU/KVM guest (4 vCPU, 4G), busybox initramfs, serial console.
* Mechanism
ceph_mdsc_handle_mdsmap() publishes a new map and frees the old one in one
locked step (mds_client.c:7043-7072):
mutex_lock(&mdsc->mutex);
if (mdsc->mdsmap && epoch <= mdsc->mdsmap->m_epoch) {
mutex_unlock(&mdsc->mutex);
return;
}
newmap = ceph_mdsmap_decode(mdsc, &p, end, ceph_msgr2(...));
...
if (mdsc->mdsmap) {
oldmap = mdsc->mdsmap;
mdsc->mdsmap = newmap;
check_new_map(mdsc, newmap, oldmap);
ceph_mdsmap_destroy(oldmap);
} else {
mdsc->mdsmap = newmap;
}
...
mutex_unlock(&mdsc->mutex);
ceph_mdsmap_destroy() (mdsmap.c:443-455) synchronously frees
m_info[].export_targets, m_info, m_data_pg_pools, m_fs_name and the struct
itself. struct ceph_mdsmap (mdsmap.h:24-49) has no refcount and no
rcu_head, so there is no lifetime extension for readers.
mdsmap_show() (debugfs.c:58-80) snapshots the pointer and walks it with no
mdsc->mutex:
if (!fsc->mdsc || !fsc->mdsc->mdsmap)
return 0;
mdsmap = fsc->mdsc->mdsmap;
seq_printf(s, "epoch %d\n", mdsmap->m_epoch);
seq_printf(s, "root %d\n", mdsmap->m_root);
seq_printf(s, "max_mds %d\n", mdsmap->m_max_mds);
seq_printf(s, "session_timeout %d\n", mdsmap->m_session_timeout);
seq_printf(s, "session_autoclose %d\n", mdsmap->m_session_autoclose);
for (i = 0; i < mdsmap->possible_max_rank; i++) {
struct ceph_entity_addr *addr = &mdsmap->m_info[i].addr;
int state = mdsmap->m_info[i].state;
seq_printf(s, "\tmds%d\t%s\t(%s)\n", i,
ceph_pr_addr(addr),
ceph_mds_state_name(state));
}
The same file already shows how it should be done: mdsc_show()
(debugfs.c:85-156) prints under mutex_lock(&mdsc->mutex) at line 93 and
mutex_unlock() at line 156.
The window: T1 (cat) executes line 66 and gets pointer M0; T2 (mon work)
executes the swap at 7058-7059, calls check_new_map() (which itself drops and
reacquires mdsc->mutex) and frees M0 at 7061; T1 resumes at line 67 and reads
M0->m_epoch, then M0->m_info[] and possibly a wild possible_max_rank.
* Trigger model
- A cephfs mount must exist (debugfs file is per-client, created by
ceph_fs_debugfs_init() under the client's debugfs dir) and debugfs must be
mounted; the file mode is 0400, so the reader needs root (or equivalent
capability in the debugfs mount's userns).
- A concurrent MDS map update is needed. Any of: MDS daemon restart /
failover, a standby taking over, a rank being removed, laggy set changes,
`ceph mds fail`, `ceph fs set max_mds` etc. The map epoch increments and
handle_mdsmap() runs in mon connection work context.
- The two operations need to overlap between the pointer snapshot and the
first dereference (a few instructions in production; a stall was used to
make it deterministic in the reproducer).
* Reproducer (attached: ceph_mdsmap_race_harness.c + instrumentation.patch)
and results on 7.3
The reproducer is a test-only in-tree harness, not a proposed fix:
- ceph_mdsmap_race_harness.c/.h build a minimal ceph_fs_client around a
real struct ceph_mds_client (real ceph_mdsc_init()), encode a minimal MDS
map (mdsmap_v=1) including the 24-byte message header (ceph_fsid + epoch
+ maplen), and publish/retire maps through the real
ceph_mdsc_handle_mdsmap() - the exact production publish/free path.
- instrumentation.patch adds one stall hook right after the pointer
snapshot in mdsmap_show() (debugfs.c:66), a debugfs file that creates the
real mdsmap_fops file, and the Kconfig/Makefile lines. The stall only
widens the "pointer snapshotted, not yet dereferenced" window; it does
not change locking or the publish/free order.
Build and run (inside the guest, with debugfs mounted):
make -C <tree> O=<build-dir> -j$(nproc) bzImage # KASAN kernel
# boot: QEMU -kernel bzImage -initrd initramfs -append "plan=mr01 ..."
# plan mr01:
# setup (publish epoch 2) -> arm -> background
# "cat /sys/kernel/debug/cephrace/mdsmap"
# wait until the show hook has taken the pointer -> trigger
# (publish epoch 3 and free epoch 2) -> release
Result (plan-mr01.log): 10 KASAN reports, all with the same free stack:
BUG: KASAN: slab-use-after-free in mdsmap_show+0x66/0x190
Read of size 4 at addr ff11000100c18900 by task cat/117
...
Allocated by task 115:
ceph_mdsmap_decode+0xe1/0x13c0
ceph_mdsc_handle_mdsmap+0x288/0x3c0
Freed by task 115:
ceph_mdsc_handle_mdsmap+0x2c7/0x3c0
and further reports at mdsmap_show+0x83/0xa1/0xbf/0xdd/0xf9/0x116/0x136
(each field and each m_info[] dereference) and ceph_pr_addr+0x6f
(m_info[i].addr). Control run mr01ctl (release the reader before publishing
the new map) gives KASAN count 0.
* Suggested direction
Take mdsc->mutex around the pointer snapshot and the walk in mdsmap_show();
mdsc_show() already prints under mdsc->mutex (debugfs.c:93/156), so this is
consistent with the existing debugfs locking. Longer term, mdsc->mdsmap
should be published with RCU (or protected by a kref) so readers can use the
snapshot outside the lock; the same change would fix the other sites.
Report-first; no patch attached. Happy to test any proposal.
Please keep me in Cc; I am not subscribed to the lists.
Regards,
Hanxiang Xu
xuhx@xxxxxxxxxxx
Attachment:
ceph_mdsmap_race_harness.h
Description: Binary data
/*
* fs/ceph mdsmap lifetime race verification harness (TEST ONLY, not upstream).
*
* This file is compiled into ceph.o only when CONFIG_CEPH_RACE_VERIFY=y.
* It builds a fake (but minimally consistent) ceph client / fs_client /
* mds_client, publishes MDS maps into mdsc->mdsmap through the *real*
* ceph_mdsc_handle_mdsmap() writer, and lets readers (the real debugfs
* mdsmap_show() and line-faithful replicas of the audited read sites)
* dereference a map while a concurrent map update publishes a new map and
* ceph_mdsmap_destroy()s the old one.
*
* Only the test scaffolding and one stall hook are added; the writer, the
* free path, the debugfs show callback and ceph_mdsmap_is_cluster_available()
* are the unmodified production code under test.
*/
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/debugfs.h>
#include <linux/fs.h>
#include <linux/kthread.h>
#include <linux/delay.h>
#include <linux/completion.h>
#include <linux/atomic.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/uaccess.h>
#include <linux/ceph/decode.h>
#include <linux/ceph/libceph.h>
#include "super.h"
#include "mds_client.h"
#include "mdsmap.h"
#include "race_verify.h"
static struct ceph_options rv_copts; /* con_modes[0] == UNKNOWN -> msgr1 */
static struct ceph_client *rv_client;
static struct ceph_fs_client *rv_fsc;
static struct ceph_mount_options *rv_mopts;
static struct ceph_mds_client *rv_mdsc;
static struct dentry *rv_dir;
static u32 rv_cur_epoch;
static bool rv_pause_enabled;
static atomic_t rv_loaded = ATOMIC_INIT(0);
static atomic_t rv_released = ATOMIC_INIT(0);
static DECLARE_COMPLETION(rv_go);
#define RV_BUF_SZ 4096
#define RV_NMDS 1
/* ------------------------------------------------------------------ */
/* MDS map wire encoder: mdsmap_v=1 (legacy), one ACTIVE mds rank. */
/* ------------------------------------------------------------------ */
#define RV_MSG_HDR_SZ 24 /* ceph_fsid (16) + epoch (u32) + maplen (u32) */
static int rv_build_map(u8 *buf, size_t bufsz, u32 epoch, u32 npools, u32 in_n)
{
struct ceph_fsid fsid;
void *p = buf + RV_MSG_HDR_SZ;
void *end = buf + bufsz;
u32 maplen;
u32 i;
int j;
if (bufsz <= RV_MSG_HDR_SZ)
return -EINVAL;
ceph_encode_8(&p, 1); /* mdsmap_v */
ceph_encode_8(&p, 0); /* compat version byte */
ceph_encode_32(&p, epoch); /* m_epoch */
ceph_encode_32(&p, 0); /* m_client_epoch */
ceph_encode_32(&p, 0); /* m_last_failure */
ceph_encode_32(&p, 1); /* m_root */
ceph_encode_32(&p, 60); /* m_session_timeout */
ceph_encode_32(&p, 300); /* m_session_autoclose */
ceph_encode_64(&p, 1ULL << 40); /* m_max_file_size */
ceph_encode_32(&p, RV_NMDS); /* m_max_mds */
ceph_encode_32(&p, RV_NMDS); /* m_num_active_mds */
for (i = 0; i < RV_NMDS; i++) {
ceph_encode_64(&p, i + 1); /* global_id */
ceph_encode_8(&p, 1); /* info_v (no addrvec, no exports) */
ceph_encode_64(&p, 0); /* skipped field */
ceph_encode_32(&p, 0); /* namelen */
ceph_encode_32(&p, i); /* mds rank */
ceph_encode_32(&p, 0); /* inc */
ceph_encode_32(&p, CEPH_MDS_STATE_ACTIVE);
ceph_encode_64(&p, 0); /* state_seq */
/* legacy entity_addr: 1 marker byte + 3 type bytes + nonce */
ceph_encode_8(&p, CEPH_ENTITY_ADDR_TYPE_NONE);
ceph_encode_8(&p, 0);
ceph_encode_8(&p, 0);
ceph_encode_8(&p, 0);
ceph_encode_32(&p, 0); /* nonce */
for (j = 0; j < 16; j++) /* sockaddr_storage (128B) */
ceph_encode_64(&p, 0);
ceph_encode_64(&p, 0); /* laggy_since (ceph_timespec) */
ceph_encode_32(&p, 0); /* skipped field */
ceph_encode_32(&p, 0); /* namelen */
}
ceph_encode_32(&p, npools); /* m_num_data_pg_pools */
for (i = 0; i < npools; i++)
ceph_encode_64(&p, 1); /* data pool id */
ceph_encode_64(&p, 0); /* m_cas_pg_pool */
/* tail for mdsmap_ev == 1 */
ceph_encode_32(&p, 1); /* metadata_pool */
ceph_encode_64(&p, 0); /* created (ceph_timespec) */
ceph_encode_64(&p, 0); /* modified (ceph_timespec) */
ceph_encode_32(&p, 0); /* tableserver */
ceph_encode_32(&p, in_n); /* in set: in_n entries */
for (i = 0; i < in_n; i++)
ceph_encode_32(&p, i); /* rank i is in */
ceph_encode_32(&p, 0); /* inc map: 0 entries */
ceph_encode_32(&p, 0); /* up map: 0 entries */
ceph_encode_32(&p, 0); /* failed set: 0 entries */
ceph_encode_32(&p, 0); /* stopped set: 0 entries */
if (p > end)
return -EOVERFLOW;
maplen = p - (void *)(buf + RV_MSG_HDR_SZ);
/* message header: ceph_fsid + map epoch + map length */
memset(&fsid, 0, sizeof(fsid));
fsid.fsid[0] = 0xce;
fsid.fsid[15] = 0x71;
p = buf;
ceph_encode_copy(&p, &fsid, sizeof(fsid));
ceph_encode_32(&p, epoch);
ceph_encode_32(&p, maplen);
if (p != buf + RV_MSG_HDR_SZ)
return -EINVAL;
return (buf + RV_MSG_HDR_SZ + maplen) - buf;
}
static int rv_deliver_map_ex(u32 epoch, u32 npools, u32 in_n)
{
struct ceph_msg msg = { };
u8 buf[RV_BUF_SZ];
int len;
len = rv_build_map(buf, sizeof(buf), epoch, npools, in_n);
if (len < 0) {
pr_err("CEPHV: map encode failed %d\n", len);
return len;
}
msg.front.iov_base = buf;
msg.front.iov_len = len;
ceph_mdsc_handle_mdsmap(rv_mdsc, &msg);
rv_cur_epoch = epoch;
return 0;
}
static int rv_deliver_map(u32 epoch, u32 npools)
{
return rv_deliver_map_ex(epoch, npools, RV_NMDS);
}
/* ------------------------------------------------------------------ */
/* Setup: fake client / fs_client / mds_client, then first map epoch 2 */
/* ------------------------------------------------------------------ */
static int rv_setup(void)
{
int ret;
if (rv_mdsc)
return -EALREADY;
rv_client = kzalloc(sizeof(*rv_client), GFP_KERNEL);
rv_fsc = kzalloc(sizeof(*rv_fsc), GFP_KERNEL);
rv_mopts = kzalloc(sizeof(*rv_mopts), GFP_KERNEL);
if (!rv_client || !rv_fsc || !rv_mopts)
return -ENOMEM;
rv_client->options = &rv_copts;
mutex_init(&rv_client->monc.mutex);
rv_client->monc.client = rv_client;
rv_fsc->client = rv_client;
rv_fsc->mount_options = rv_mopts;
INIT_LIST_HEAD(&rv_fsc->metric_wakeup);
ret = ceph_mdsc_init(rv_fsc);
if (ret) {
pr_err("CEPHV: ceph_mdsc_init failed %d\n", ret);
return ret;
}
rv_mdsc = rv_fsc->mdsc;
/* Neutralize the delayed_work timer before the first map delivery,
* so the work item early-returns instead of running cap machinery.
*/
rv_mdsc->stopping = CEPH_MDSC_STOPPING_FLUSHED;
ret = rv_deliver_map(2, 1);
if (ret)
return ret;
flush_delayed_work(&rv_mdsc->delayed_work);
/* The real debugfs show callback, using the real mdsmap_fops. */
ceph_race_create_mdsmap_file(rv_fsc, rv_dir);
pr_info("CEPHV: setup done, epoch=%u stopping=%d\n",
rv_cur_epoch, (int)rv_mdsc->stopping);
return 0;
}
/* ------------------------------------------------------------------ */
/* Stall hook called from the real mdsmap_show() after it snapshots */
/* fsc->mdsc->mdsmap (debugfs.c:66 in the unmodified tree). */
/* ------------------------------------------------------------------ */
void ceph_race_verify_pause(struct ceph_mds_client *mdsc,
struct ceph_mdsmap *mdsmap)
{
if (!READ_ONCE(rv_pause_enabled) || !rv_mdsc)
return;
atomic_inc(&rv_loaded);
wait_for_completion(&rv_go);
atomic_set(&rv_loaded, 0);
}
/* ------------------------------------------------------------------ */
/* Kernel-thread readers: faithful replicas of the audited read sites */
/* ------------------------------------------------------------------ */
enum rv_reader_kind {
RV_R_STATFS = 1, /* super.c:68-69 (ceph_statfs) */
RV_R_TOCTOU, /* super.c:69 (double load) */
RV_R_XATTR, /* xattr.c:1237 (__ceph_setxattr) */
RV_R_RENEW, /* mds_client.c:2130/2179 */
RV_R_PEER, /* mds_client.c:5227 */
};
static int rv_reader_fn(void *arg)
{
long kind = (long)arg;
struct ceph_mdsmap *m, *m2;
u64 val = 0;
u32 u32v = 0;
int st;
m = READ_ONCE(rv_mdsc->mdsmap);
atomic_inc(&rv_loaded);
wait_for_completion(&rv_go);
switch (kind) {
case RV_R_STATFS:
u32v = m->m_num_data_pg_pools;
if (u32v == 1) {
u64 *p = m->m_data_pg_pools;
val = p[0];
}
break;
case RV_R_TOCTOU:
u32v = m->m_num_data_pg_pools;
m2 = READ_ONCE(rv_mdsc->mdsmap);
if (u32v == 1) {
u64 *p = m2->m_data_pg_pools;
val = p[0];
}
break;
case RV_R_XATTR:
val = m->m_max_xattr_size;
break;
case RV_R_RENEW:
u32v = m->m_session_timeout;
break;
case RV_R_PEER:
st = ceph_mdsmap_get_state(m, 0);
{
struct ceph_entity_addr addr;
struct ceph_entity_addr *a =
ceph_mdsmap_get_addr(m, 0);
if (a)
memcpy(&addr, a, sizeof(addr));
pr_info("CEPHV: peer replica state=%d addr=%p\n",
st, a);
}
return 0;
default:
return -EINVAL;
}
pr_info("CEPHV: reader kind=%ld val=%llu u32=%u\n", kind,
(unsigned long long)val, u32v);
return 0;
}
/* Real production reader: ceph_mdsmap_is_cluster_available() on a
* previously snapshotted map (the ceph_get_tree() out_splat pattern).
*/
static int rv_reader3_fn(void *arg)
{
struct ceph_mdsmap *m = READ_ONCE(rv_mdsc->mdsmap);
bool avail;
atomic_inc(&rv_loaded);
wait_for_completion(&rv_go);
avail = ceph_mdsmap_is_cluster_available(m);
pr_info("CEPHV: reader3 avail=%d\n", avail);
return 0;
}
static int rv_spawn(int (*fn)(void *), long kind, const char *name,
struct task_struct **out)
{
struct task_struct *t = kthread_run(fn, (void *)kind, "%s", name);
if (IS_ERR(t))
return PTR_ERR(t);
if (out)
*out = t;
return 0;
}
static void rv_release(void)
{
atomic_set(&rv_released, 1);
complete_all(&rv_go);
}
/* ------------------------------------------------------------------ */
/* control file */
/* ------------------------------------------------------------------ */
static ssize_t rv_ctl_write(struct file *f, const char __user *ubuf,
size_t len, loff_t *off)
{
struct task_struct *t;
char buf[32];
int ret = 0;
if (len >= sizeof(buf))
return -EINVAL;
if (copy_from_user(buf, ubuf, len))
return -EFAULT;
buf[len] = 0;
strim(buf);
if (!strcmp(buf, "setup")) {
ret = rv_setup();
} else if (!strcmp(buf, "arm")) {
rv_pause_enabled = true;
reinit_completion(&rv_go);
atomic_set(&rv_loaded, 0);
atomic_set(&rv_released, 0);
pr_info("CEPHV: armed (pause hook enabled)\n");
} else if (!strcmp(buf, "trigger")) {
ret = rv_deliver_map(rv_cur_epoch + 1, 1);
pr_info("CEPHV: trigger epoch=%u pools=1\n", rv_cur_epoch);
} else if (!strcmp(buf, "trigger0")) {
ret = rv_deliver_map_ex(rv_cur_epoch + 1, 0, 0);
pr_info("CEPHV: trigger epoch=%u pools=0\n", rv_cur_epoch);
} else if (!strcmp(buf, "release")) {
rv_release();
pr_info("CEPHV: released\n");
} else if (!strcmp(buf, "reader3")) {
ret = rv_spawn(rv_reader3_fn, 0, "cephrv3", &t);
} else if (!strcmp(buf, "rep")) {
ret = rv_spawn(rv_reader_fn, RV_R_STATFS, "cephrv-s", NULL);
if (!ret)
ret = rv_spawn(rv_reader_fn, RV_R_TOCTOU, "cephrv-t", NULL);
if (!ret)
ret = rv_spawn(rv_reader_fn, RV_R_XATTR, "cephrv-x", NULL);
if (!ret)
ret = rv_spawn(rv_reader_fn, RV_R_RENEW, "cephrv-r", NULL);
if (!ret)
ret = rv_spawn(rv_reader_fn, RV_R_PEER, "cephrv-p", NULL);
} else if (!strcmp(buf, "rep0")) {
ret = rv_spawn(rv_reader_fn, RV_R_TOCTOU, "cephrv-t0", NULL);
} else if (!strcmp(buf, "noop")) {
ret = 0;
} else {
ret = -EINVAL;
}
if (ret)
return ret;
return len;
}
static const struct file_operations rv_ctl_fops = {
.write = rv_ctl_write,
};
static int rv_state_show(struct seq_file *s, void *unused)
{
seq_printf(s, "mdsc=%d epoch=%u loaded=%d released=%d pause=%d stopping=%d\n",
rv_mdsc ? 1 : 0, rv_cur_epoch,
atomic_read(&rv_loaded), atomic_read(&rv_released),
rv_pause_enabled ? 1 : 0,
rv_mdsc ? (int)rv_mdsc->stopping : -1);
return 0;
}
DEFINE_SHOW_ATTRIBUTE(rv_state);
static int __init rv_init(void)
{
rv_dir = debugfs_create_dir("cephrace", NULL);
if (IS_ERR_OR_NULL(rv_dir))
return 0;
debugfs_create_file("control", 0200, rv_dir, NULL, &rv_ctl_fops);
debugfs_create_file("state", 0400, rv_dir, NULL, &rv_state_fops);
pr_info("CEPHV: harness loaded\n");
return 0;
}
late_initcall(rv_init);
Attachment:
instrumentation.patch
Description: Binary data