Re: [PATCH v2] ftrace: Finish removing subops after a hash update failure
From: Weiming Shi
Date: Sun Nov 22 2026 - 10:57:19 EST
Hi,
Here is the exact reproducer and the commands used for the crash in the
commit message.
The tested kernel was commit 4405c18c95d2d0f99f6c581ef0ee8591a1aa6ccd
with:
```
CONFIG_SMP=y
CONFIG_DEBUG_INFO=y
CONFIG_DEBUG_FS=y
CONFIG_DEBUG_FS_ALLOW_ALL=y
CONFIG_KASAN=y
CONFIG_KASAN_GENERIC=y
CONFIG_KASAN_INLINE=y
CONFIG_FUNCTION_TRACER=y
CONFIG_FUNCTION_GRAPH_TRACER=y
CONFIG_DYNAMIC_FTRACE=y
CONFIG_FPROBE=y
CONFIG_FAULT_INJECTION=y
CONFIG_FAILSLAB=y
CONFIG_FAULT_INJECTION_DEBUG_FS=y
CONFIG_FAULT_INJECTION_STACKTRACE_FILTER=y
CONFIG_FTRACE_STARTUP_TEST=y
CONFIG_EVENT_TRACE_STARTUP_TEST=y
```
KASAN and DEBUG_INFO produce and decode the report. FAILSLAB and its
stacktrace filter make the allocation failure deterministic. The reproducer
does not need a race or any kernel source modification. It must run as root
because it mounts tracefs and debugfs and configures failslab.
The test kernel also had the ftrace startup tests enabled. The `ftrace=nop`
command-line option below skips those boot-time tests; it does not disable
ftrace, and the reproducer selects the function_graph tracer itself. The
option is not required when the startup tests are disabled in the kernel
configuration.
Host command:
```
qemu-system-x86_64 \
-accel kvm -cpu host -smp 4 -m 4096 \
-no-reboot -nographic \
-kernel arch/x86/boot/bzImage \
-initrd initramfs.cpio.gz \
-append "console=ttyS0 panic=1 oops=panic panic_on_warn=0 nokaslr net.ifnames=0 ftrace=nop"
```
The tested initramfs runs `/poc` from `/init`. To build and run it in an
existing guest instead:
```
gcc -O2 -Wall -Wextra -o poc poc.c
sudo ./poc
```
The reproducer does the following:
1. It enables the function_graph tracer with `tcp_v4_connect` as a nonempty
filter.
2. It creates and enables a return fprobe on `udp_sendmsg`.
3. It scopes failslab to allocations below `ftrace_shutdown_subops()` and
fails one allocation while disabling the fprobe. This makes
`rebuild_hashes()` return -ENOMEM.
4. It updates the remaining graph filter, verifies that the fprobe operation
is stuck with -EBUSY, and disables the function_graph tracer.
5. It enables the fprobe again. The stale SUBOP state sends the filter update
through an empty manager list, and `ops_equal()` dereferences an
uninitialized hash pointer.
With the command above, the failure was reproduced on the first run. The
relevant output was:
```
[*] removing subops B (fprobe) with rebuild_hashes() forced to -ENOMEM
FAULT_INJECTION: forcing a failure.
[*] updating the remaining graph filter after the failed teardown
[*] probing the bricked state (expect EBUSY + WARNs)
[-] write(/tmp/poctrace/events/fprobes/poc/enable, "1"): Device or resource busy
[!] CONFIRMED: fprobe_graph_ops is bricked (-EBUSY)
[*] removing subops A -> graph_ops->subop_list is now empty
[*] re-registering the fprobe: expect a wild deref in ops_equal()/ftrace_hash_empty()
Oops: general protection fault, probably for non-canonical address 0xdffffc0000000802: 0000 [#1] SMP KASAN NOPTI
KASAN: probably user-memory-access in range [0x0000000000004010-0x0000000000004017]
RIP: 0010:ops_equal+0x41/0x600
Call Trace:
ftrace_update_ops+0x64/0x140
ftrace_hash_move_and_update_subops.isra.0+0x180/0x360
ftrace_hash_move_and_update_ops+0x271/0x350
ftrace_set_hash+0x38e/0x860
ftrace_set_filter_ips+0x88/0x270
register_fprobe_ips+0x395/0x14d0
register_fprobe+0x204/0x260
enable_trace_fprobe+0x1ef/0x4b0
fprobe_register+0xa3/0xd0
```
Complete PoC:
```c
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <stdarg.h>
#define TR "/tmp/poctrace"
#define FS "/sys/kernel/debug/failslab"
static int kmsg_fd = -1;
static void say(const char *fmt, ...)
{
char buf[512];
va_list ap;
int n;
va_start(ap, fmt);
n = vsnprintf(buf, sizeof(buf) - 1, fmt, ap);
va_end(ap);
if (n < 0)
return;
if (kmsg_fd >= 0)
write(kmsg_fd, buf, n);
else {
fwrite(buf, 1, n, stdout);
fflush(stdout);
}
}
static int wr(const char *path, const char *val, int flags)
{
int fd, n, e;
fd = open(path, O_WRONLY | flags);
if (fd < 0) {
say("[-] open(%s): %s\n", path, strerror(errno));
return -1;
}
n = write(fd, val, strlen(val));
e = errno;
close(fd);
if (n < 0) {
say("[-] write(%s, \"%s\"): %s\n", path, val, strerror(e));
return -e;
}
return 0;
}
static int w(const char *path, const char *val) { return wr(path, val, O_TRUNC); }
static int wa(const char *path, const char *val) { return wr(path, val, O_APPEND); }
static int wl(const char *path, unsigned long v)
{
char buf[64];
snprintf(buf, sizeof(buf), "%lu", v);
return w(path, buf);
}
static int wx(const char *path, unsigned long v)
{
char buf[64];
snprintf(buf, sizeof(buf), "0x%lx", v);
return w(path, buf);
}
static int sym_range(const char *name, unsigned long *start, unsigned long *end)
{
FILE *f = fopen("/proc/kallsyms", "r");
char line[512], nm[256], type;
unsigned long addr;
int found = 0;
if (!f) {
say("[-] /proc/kallsyms: %s\n", strerror(errno));
return -1;
}
*start = *end = 0;
while (fgets(line, sizeof(line), f)) {
if (sscanf(line, "%lx %c %255s", &addr, &type, nm) != 3)
continue;
if (found) {
if (addr > *start) {
*end = addr;
break;
}
continue;
}
if (!strcmp(nm, name)) {
*start = addr;
found = 1;
}
}
fclose(f);
if (!found)
return -1;
if (!*end)
*end = *start + 0x400;
return 0;
}
int main(void)
{
unsigned long s = 0, e = 0;
setvbuf(stdout, NULL, _IONBF, 0);
kmsg_fd = open("/dev/kmsg", O_WRONLY);
w("/proc/sys/kernel/printk_devkmsg", "on\n");
w("/proc/sys/kernel/printk", "8 4 1 7");
mkdir("/sys/kernel/debug", 0755);
if (mount("none", "/sys/kernel/debug", "debugfs", 0, NULL) && errno != EBUSY)
say("[-] mount debugfs: %s\n", strerror(errno));
mkdir("/tmp", 0755);
mkdir(TR, 0755);
if (mount("none", TR, "tracefs", 0, NULL) && errno != EBUSY) {
say("[-] mount tracefs: %s\n", strerror(errno));
return 1;
}
if (sym_range("ftrace_shutdown_subops", &s, &e)) {
say("[-] ftrace_shutdown_subops not in kallsyms\n");
return 1;
}
say("[*] ftrace_shutdown_subops = [0x%lx, 0x%lx)\n", s, e);
w(TR "/current_tracer", "nop");
w(TR "/set_ftrace_filter", "tcp_v4_connect");
if (w(TR "/current_tracer", "function_graph")) {
say("[-] cannot enable function_graph\n");
return 1;
}
wa(TR "/dynamic_events", "f:poc udp_sendmsg%return");
if (w(TR "/events/fprobes/poc/enable", "1")) {
say("[-] cannot enable the fprobe event\n");
return 1;
}
w(FS "/times", "0");
wl(FS "/probability", 0);
wl(FS "/interval", 1);
w(FS "/space", "0");
wl(FS "/verbose", 2);
w(FS "/task-filter", "N");
w(FS "/ignore-gfp-wait", "N");
wl(FS "/stacktrace-depth", 32);
wx(FS "/require-start", s);
wx(FS "/require-end", e);
wl(FS "/probability", 100);
w(FS "/times", "1");
say("[*] removing subops B (fprobe) with rebuild_hashes() forced to -ENOMEM\n");
w(TR "/events/fprobes/poc/enable", "0");
wl(FS "/probability", 0);
w(FS "/times", "0");
wx(FS "/require-start", 0);
wx(FS "/require-end", ~0UL);
say("[*] updating the remaining graph filter after the failed teardown\n");
if (w(TR "/set_ftrace_filter", "udp_sendmsg")) {
say("[-] cannot update the remaining graph filter\n");
return 1;
}
say("[*] probing the bricked state (expect EBUSY + WARNs)\n");
{
int r = w(TR "/events/fprobes/poc/enable", "1");
if (r == -EBUSY)
say("[!] CONFIRMED: fprobe_graph_ops is bricked (-EBUSY)\n");
else
say("[?] re-enable returned %d (expected -EBUSY=-16)\n", r);
w(TR "/events/fprobes/poc/enable", "0");
}
say("[*] removing subops A -> graph_ops->subop_list is now empty\n");
w(TR "/current_tracer", "nop");
say("[*] re-registering the fprobe: expect a wild deref in "
"ops_equal()/ftrace_hash_empty()\n");
w(TR "/events/fprobes/poc/enable", "1");
say("[*] survived - no crash this round\n");
w(TR "/events/fprobes/poc/enable", "0");
wa(TR "/dynamic_events", "-:fprobes/poc");
return 0;
}
```
Thanks,
Weiming