[BUG] xfrm: ESP SA in route optimization mode hits WARN_ON_ONCE in xfrm_state_mtu() and xfrm_input()

From: Kyuwon Cho

Date: Tue Nov 17 2026 - 11:05:55 EST


Hi Steffen, Herbert,

While testing v7.4-rc3 with an LLM-guided test generator that we are
developing as a research prototype, we found that XFRM_MSG_NEWSA accepts
an ESP state with mode XFRM_MODE_ROUTEOPTIMIZATION, and that one packet
through it hits two WARN_ON_ONCE()s. An unprivileged user can do this
from its own user and network namespace. Sending the packet:

WARNING: net/xfrm/xfrm_state.c:3264 at xfrm_state_mtu+0x239/0x3d0, CPU#1: test/107
CPU: 1 UID: 65534 PID: 107 Comm: test Not tainted 7.4.0-rc3 #1 PREEMPT(lazy)
RIP: 0010:xfrm_state_mtu+0x239/0x3d0
Call Trace:
<TASK>
xfrm_bundle_create+0x1560/0x2340
xfrm_resolve_and_create_bundle+0x158/0x2d0
xfrm_lookup_with_ifid+0x483/0x18e0
xfrm_lookup_route+0x1e/0x180
ip6_dst_lookup_flow+0x196/0x1e0
udpv6_sendmsg+0x163b/0x2d70
__sys_sendto+0x2b1/0x3c0
__x64_sys_sendto+0xdb/0x1b0
do_syscall_64+0x116/0x630
entry_SYSCALL_64_after_hwframe+0x77/0x7f
</TASK>

and receiving it:

WARNING: net/xfrm/xfrm_input.c:449 at xfrm_input+0x2176/0x5f50, CPU#1: test/107
CPU: 1 UID: 65534 PID: 107 Comm: test Tainted: G W 7.4.0-rc3 #1 PREEMPT(lazy)
RIP: 0010:xfrm_input+0x2176/0x5f50
Call Trace:
<IRQ>
xfrm6_esp_rcv+0x83/0x1c0
ip6_protocol_deliver_rcu+0xc71/0x13d0
ip6_input_finish+0x192/0x3f0
ip6_input+0xcb/0x220
__netif_receive_skb_one_core+0x114/0x1b0
process_backlog+0x3c6/0x12c0
__napi_poll.constprop.0+0x9c/0x5c0
net_rx_action+0x48e/0xe30
handle_softirqs+0x22d/0x980
do_softirq+0x43/0xa0
</IRQ>

verify_newsa_info() accepts XFRM_MODE_ROUTEOPTIMIZATION for any
protocol; only IP-TFS is limited to ESP. esp6_init_state() treats every
mode other than BEET and TUNNEL as transport:

switch (x->props.mode) {
case XFRM_MODE_BEET:
...
default:
case XFRM_MODE_TRANSPORT:
break;

so the state is created. The ESP paths then reach code that assumes
route optimization never comes with ESP. xfrm_state_mtu() has no case
for it and falls to:

default:
if (x->mode_cbs && x->mode_cbs->get_inner_mtu)
return x->mode_cbs->get_inner_mtu(x, mtu);

WARN_ON_ONCE(1);

and xfrm_inner_mode_input() has

case XFRM_MODE_ROUTEOPTIMIZATION:
WARN_ON_ONCE(1);
break;

The packet is dropped on input, so the effect is the warnings, which
panic a machine with panic_on_warn. Both WARN_ON_ONCE()s date from the
2019 xfrm mode cleanup (c2d305e51038, c7b37c769d2a).

Accepting XFRM_MODE_ROUTEOPTIMIZATION only for the Mobile IPv6 types
(IPPROTO_ROUTING and IPPROTO_DSTOPTS) in verify_newsa_info(), as it
already limits IP-TFS to ESP, or rejecting it in esp6_init_state()
(route optimization exists only for IPv6 states), should avoid it. We
have not tested either change.

The reproducer runs as an unprivileged user (uid 65534). It creates a
user and network namespace and brings up lo. It then adds an IPv6 ESP
state ::1 -> ::1 in mode XFRM_MODE_ROUTEOPTIMIZATION with
rfc4106(gcm(aes)), and an outbound policy for UDP to ::1 with a matching
template. Last, it sends one UDP datagram. On an x86_64 QEMU guest with
2 vCPUs and KASAN (gcc 13.3, config attached), both warnings fire in
every run (9 of 9).

Checked against v7.4-rc3 (aebb72744764). The code involved is the same
in net-next (6ff4c5942a30), ipsec (f0e68014f4ac) and ipsec-next
(7b6ab4dbc156). We found no earlier report on syzbot or in a web search.

The reproducer (repro.c) and the kernel config are attached.

Thanks,
Kyuwon Cho
// WARNING in xfrm_state_mtu: an unprivileged user installs an ESP SA in
// route optimization mode and sends one UDP packet through it.
#define _GNU_SOURCE
#include <arpa/inet.h>
#include <fcntl.h>
#include <linux/netlink.h>
#include <linux/xfrm.h>
#include <net/if.h>
#include <netinet/in.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <unistd.h>

#define PORT 9000

static int fd;

static void put_attr(char *buf, size_t *len, unsigned short type, const void *data, size_t dlen)
{
struct nlattr *a = (struct nlattr *)(buf + *len);

a->nla_type = type;
a->nla_len = NLA_HDRLEN + dlen;
memcpy((char *)a + NLA_HDRLEN, data, dlen);
*len += NLA_ALIGN(a->nla_len);
}

/* Send one xfrm request, with body and attributes, and return its error. */
static int req(unsigned short type, const void *body, size_t blen, const char *attrs, size_t alen)
{
char msg[4096] = {0}, rb[4096];
struct nlmsghdr *nh = (struct nlmsghdr *)msg;

memcpy(NLMSG_DATA(nh), body, blen);
memcpy((char *)NLMSG_DATA(nh) + NLMSG_ALIGN(blen), attrs, alen);
nh->nlmsg_len = NLMSG_LENGTH(NLMSG_ALIGN(blen) + alen);
nh->nlmsg_type = type;
nh->nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
if (send(fd, msg, nh->nlmsg_len, 0) < 0 || recv(fd, rb, sizeof(rb), 0) < 0)
return -1;
return ((struct nlmsgerr *)NLMSG_DATA((struct nlmsghdr *)rb))->error;
}

static void write_file(const char *path, const char *s)
{
int f = open(path, O_WRONLY);

if (f < 0 || write(f, s, strlen(s)) < 0)
exit(1);
close(f);
}

int main(void)
{
struct { struct xfrm_algo_aead a; char key[20]; } aead = {0};
struct xfrm_userpolicy_info pol = {0};
struct xfrm_usersa_info sa = {0};
struct xfrm_user_tmpl tmpl = {0};
struct sockaddr_in6 dst = {0};
struct ifreq ifr = {0};
char attrs[512], map[32];
size_t al = 0;
int s;

snprintf(map, sizeof(map), "0 %d 1", getuid());
if (unshare(CLONE_NEWUSER | CLONE_NEWNET))
return 1;
write_file("/proc/self/setgroups", "deny");
write_file("/proc/self/uid_map", map);
snprintf(map, sizeof(map), "0 %d 1", getgid());
write_file("/proc/self/gid_map", map);

s = socket(AF_INET, SOCK_DGRAM, 0);
strcpy(ifr.ifr_name, "lo");
ifr.ifr_flags = IFF_UP;
ioctl(s, SIOCSIFFLAGS, &ifr);
close(s);
fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_XFRM);

/* ESP SA ::1 -> ::1 with XFRM_MODE_ROUTEOPTIMIZATION. */
inet_pton(AF_INET6, "::1", &sa.id.daddr);
inet_pton(AF_INET6, "::1", &sa.saddr);
sa.id.spi = htonl(0x100);
sa.id.proto = IPPROTO_ESP;
sa.family = AF_INET6;
sa.mode = XFRM_MODE_ROUTEOPTIMIZATION;
sa.reqid = 1;
sa.lft.soft_byte_limit = sa.lft.hard_byte_limit = XFRM_INF;
sa.lft.soft_packet_limit = sa.lft.hard_packet_limit = XFRM_INF;
strcpy(aead.a.alg_name, "rfc4106(gcm(aes))");
aead.a.alg_key_len = 160;
aead.a.alg_icv_len = 128;
put_attr(attrs, &al, XFRMA_ALG_AEAD, &aead, sizeof(aead));
printf("newsa: %d\n", req(XFRM_MSG_NEWSA, &sa, sizeof(sa), attrs, al));

/* Outbound policy for UDP to ::1 port PORT, using that SA. */
pol.sel.family = AF_INET6;
inet_pton(AF_INET6, "::1", &pol.sel.daddr);
pol.sel.prefixlen_d = 128;
pol.sel.proto = IPPROTO_UDP;
pol.sel.dport = htons(PORT);
pol.sel.dport_mask = 0xffff;
pol.dir = XFRM_POLICY_OUT;
pol.action = XFRM_POLICY_ALLOW;
pol.lft = sa.lft;
tmpl.id = sa.id;
tmpl.family = AF_INET6;
tmpl.saddr = sa.saddr;
tmpl.reqid = 1;
tmpl.mode = XFRM_MODE_ROUTEOPTIMIZATION;
tmpl.aalgos = tmpl.ealgos = tmpl.calgos = ~0;
al = 0;
put_attr(attrs, &al, XFRMA_TMPL, &tmpl, sizeof(tmpl));
printf("newpolicy: %d\n", req(XFRM_MSG_NEWPOLICY, &pol, sizeof(pol), attrs, al));

s = socket(AF_INET6, SOCK_DGRAM, 0);
dst.sin6_family = AF_INET6;
dst.sin6_port = htons(PORT);
inet_pton(AF_INET6, "::1", &dst.sin6_addr);
printf("sendto: %zd\n", sendto(s, "x", 1, 0, (struct sockaddr *)&dst, sizeof(dst)));
return 0;
}

Attachment: config.gz
Description: application/gzip