[PATCH v2 6/7] ceph: request a full mdsmap when an MDS stays unreachable
From: Ionut Nechita
Date: Fri Nov 20 2026 - 02:35:47 EST
The client subscribes to incremental mdsmap updates starting at
current_epoch + 1. If an MDS restarts at a different address while
the monitor session is also down (e.g. during a rolling upgrade in a
containerized deployment), the client can miss that update and keep
reconnecting to the stale address indefinitely. This was observed
with the mdsmap stuck at epoch 53 while the cluster was at epoch 68,
the session HUNG and monc reporting "have mdsmap 53 want 54+".
Nothing currently reacts to this: mds_con_ops has no .fault callback,
and a session going HUNG doesn't touch the subscription either.
Add a .fault callback that asks for a full mdsmap (start epoch 0) on
every MDS_CON_FAIL_REFRESH_MDSMAP consecutive faults, resetting the
count on any session message, and do the same when a session is
marked HUNG, which covers a connection that went to STANDBY and thus
never faults.
Signed-off-by: Ionut Nechita <ionut.nechita@xxxxxxxxxxxxx>
---
fs/ceph/mds_client.c | 48 ++++++++++++++++++++++++++++++++++++++++++++
fs/ceph/mds_client.h | 2 ++
2 files changed, 50 insertions(+)
diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index 4460f614050e4..3c7ecf3147e6f 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -72,6 +72,13 @@ static void ceph_mdsc_reset_workfn(struct work_struct *work);
static const struct ceph_connection_operations mds_con_ops;
+/*
+ * Consecutive MDS connection faults after which we ask the monitor for
+ * a full mdsmap. With the messenger backoff (250ms doubling up to 15s)
+ * ten faults take about a minute, comparable to mount_timeout.
+ */
+#define MDS_CON_FAIL_REFRESH_MDSMAP 10
+
static void ceph_metric_bind_session(struct ceph_mds_client *mdsc,
struct ceph_mds_session *session)
{
@@ -4651,6 +4658,9 @@ static void handle_session(struct ceph_mds_session *session,
ceph_session_op_name(op), session,
ceph_session_state_name(session->s_state), seq);
+ /* the connection works, see mds_fault() */
+ session->s_con_failures = 0;
+
if (session->s_state == CEPH_MDS_SESSION_HUNG) {
session->s_state = CEPH_MDS_SESSION_OPEN;
pr_info_client(cl, "mds%d came back\n", session->s_mds);
@@ -6285,6 +6295,18 @@ static void maybe_recover_session(struct ceph_mds_client *mdsc)
ceph_force_reconnect(fsc->sb);
}
+/*
+ * Subscribe to the mdsmap from epoch 0 so that the monitor sends the
+ * current full map rather than waiting for an incremental update.
+ */
+static void request_full_mdsmap(struct ceph_mds_client *mdsc)
+{
+ struct ceph_mon_client *monc = &mdsc->fsc->client->monc;
+
+ if (ceph_monc_want_map(monc, CEPH_SUB_MDSMAP, 0, true))
+ ceph_monc_renew_subs(monc);
+}
+
bool check_session_state(struct ceph_mds_session *s)
{
struct ceph_client *cl = s->s_mdsc->fsc->client;
@@ -6294,6 +6316,12 @@ bool check_session_state(struct ceph_mds_session *s)
if (s->s_ttl && time_after(jiffies, s->s_ttl)) {
s->s_state = CEPH_MDS_SESSION_HUNG;
pr_info_client(cl, "mds%d hung\n", s->s_mds);
+ /*
+ * The MDS may have come back at a different
+ * address while the connection sat in STANDBY,
+ * which doesn't fault.
+ */
+ request_full_mdsmap(s->s_mdsc);
}
break;
case CEPH_MDS_SESSION_CLOSING:
@@ -7525,12 +7553,32 @@ static int mds_check_message_signature(struct ceph_msg *msg)
return ceph_auth_check_message_signature(auth, msg);
}
+/*
+ * If the MDS keeps failing to connect, our mdsmap may be stale (e.g. the
+ * MDS restarted at a different address and the incremental update was
+ * missed while the monitor session was down). Periodically ask for a
+ * full map.
+ */
+static void mds_fault(struct ceph_connection *con)
+{
+ struct ceph_mds_session *s = con->private;
+
+ if (++s->s_con_failures < MDS_CON_FAIL_REFRESH_MDSMAP)
+ return;
+
+ s->s_con_failures = 0;
+ doutc(s->s_mdsc->fsc->client,
+ "mds%d keeps faulting, requesting full mdsmap\n", s->s_mds);
+ request_full_mdsmap(s->s_mdsc);
+}
+
static const struct ceph_connection_operations mds_con_ops = {
.get = mds_get_con,
.put = mds_put_con,
.alloc_msg = mds_alloc_msg,
.dispatch = mds_dispatch,
.peer_reset = mds_peer_reset,
+ .fault = mds_fault,
.get_authorizer = mds_get_authorizer,
.add_authorizer_challenge = mds_add_authorizer_challenge,
.verify_authorizer_reply = mds_verify_authorizer_reply,
diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h
index 31abbb3b3bdb3..46e13446afcc7 100644
--- a/fs/ceph/mds_client.h
+++ b/fs/ceph/mds_client.h
@@ -301,6 +301,8 @@ struct ceph_mds_session {
struct list_head s_unsafe; /* unsafe requests */
struct xarray s_delegated_inos;
atomic_t s_num_deleg_inos;
+
+ u8 s_con_failures; /* consecutive connection faults */
};
/*
--
2.56.0