[PATCH] delayacct: Record blkio start on the correct task

[Jie Du]

__block_task() calls delayacct_blkio_start(), which stamps
current->delays->blkio_start. That is only correct when the task being
blocked is current, as it is when __schedule() blocks prev.

With DELAY_DEQUEUE, a task that blocks while not eligible stays on the
runqueue as sched_delayed. If it gets picked before it is woken,
pick_next_entity() dequeues it and __block_task() runs on behalf of
whatever task is scheduling on that CPU. Proxy execution can also block a
task that is not current. In these cases blkio_start is written to
current, and p keeps whatever blkio_start it had before: the start of an
older sleep, or 0 for a thread that never blocked on I/O. When the I/O
completes, ttwu() calls delayacct_blkio_end(p), which charges p with
local_clock() - blkio_start. For a new thread that is the whole time
since boot. current's own blkio_start, if it is sleeping on I/O itself,
is moved forward at the same time.

This shows up as freshly created threads reporting days of block I/O
delay through taskstats and field 42 (delayacct_blkio_ticks) of
/proc/<pid>/stat, in steps of exactly the system uptime. Anything summing
those numbers per cgroup or per host reports nonsense.

Pass the task down to delayacct_blkio_start(), like commit c96f5471ce7d
("delayacct: Account blkio completion on the correct task") did for the
completion side. __block_task() still sets blkio_start before it clears
p->on_rq, so a concurrent ttwu() cannot reach delayacct_blkio_end(p)
before the start is recorded.

Tested with a small reproducer that keeps one CPU busy with two spinning
threads and keeps creating short-lived threads that spin for 2ms and then
do O_DIRECT reads; a thread is counted as bad when its
delayacct_blkio_ticks is larger than its age. Under QEMU (TCG, 2 vCPUs),
4 rounds of 60s each, bad threads:

                      v7.4-rc1           v7.4-rc1 + this patch
  DELAY_DEQUEUE       20 of 14718        0 of 19383
  NO_DELAY_DEQUEUE     0 of 15766        0 of 20020

Each of the 20 was charged roughly the uptime at that point.

On a 6.6-based distribution kernel that carries the EEVDF backport the
same reproducer gives about 690 bad threads out of 2000 in 20s with
DELAY_DEQUEUE and none with NO_DELAY_DEQUEUE.

The problem was investigated, and this patch and its reproducer were
written, with help from an AI coding assistant (Claude). I have tested
it as described above.

Fixes: 152e11f6df29 ("sched/fair: Implement delayed dequeue")
Closes: https://lore.kernel.org/all/tencent_6CFB6E9B26A308DC2800D4059DF93FEC9706@xxxxxx/
Cc: stable@xxxxxxxxxxxxxxx
Signed-off-by: Jie Du <806144828@xxxxxx>
---
Reproducer used above (gcc -O2 -pthread; run as
./blkio-repro <file or block device of at least 64MB> [seconds]):

// gcc -O2 -pthread -o blkio-repro blkio-repro.c
// ./blkio-repro <file or block device, >= 64MB> [seconds]
#define _GNU_SOURCE
#include <fcntl.h>
#include <pthread.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

static int fd;
static volatile int stop;
static unsigned long nr, bad;

static double now(void)
{
	struct timespec ts;

	clock_gettime(CLOCK_MONOTONIC, &ts);
	return ts.tv_sec + ts.tv_nsec / 1e9;
}

/* delayacct_blkio_ticks, field 42 of /proc/thread-self/stat */
static unsigned long long blkio_ticks(void)
{
	char buf[1024], *p;
	FILE *f = fopen("/proc/thread-self/stat", "r");
	int i;

	if (!f || !fgets(buf, sizeof(buf), f))
		exit(1);
	fclose(f);
	p = strrchr(buf, ')') + 2;
	for (i = 3; i < 42; i++)
		p = strchr(p, ' ') + 1;
	return strtoull(p, NULL, 10);
}

static void *hog(void *arg)
{
	while (!stop)
		;
	return NULL;
}

/* burn 2ms so we are not eligible, then block on synchronous reads */
static void *worker(void *arg)
{
	unsigned int seed = (unsigned long)arg;
	double born = now();
	void *buf;
	int i;

	if (posix_memalign(&buf, 4096, 4096))
		exit(1);
	while (now() - born < 0.002)
		;
	for (i = 0; i < 8; i++)
		if (pread(fd, buf, 4096, (rand_r(&seed) % 16384) * 4096L) != 4096)
			exit(1);
	if (blkio_ticks() > (now() - born) * sysconf(_SC_CLK_TCK) + 1)
		bad++;
	nr++;
	free(buf);
	return NULL;
}

int main(int argc, char **argv)
{
	double end = now() + (argc > 2 ? atoi(argv[2]) : 30);
	pthread_t h[2], t;
	unsigned long n = 0;
	cpu_set_t set;

	fd = open(argv[1], O_RDONLY | O_DIRECT);
	if (fd < 0)
		return 1;
	/* everything on one CPU */
	CPU_ZERO(&set);
	CPU_SET(0, &set);
	sched_setaffinity(0, sizeof(set), &set);
	pthread_create(&h[0], NULL, hog, NULL);
	pthread_create(&h[1], NULL, hog, NULL);
	while (now() < end) {
		pthread_create(&t, NULL, worker, (void *)++n);
		pthread_join(t, NULL);
	}
	stop = 1;
	printf("threads %lu, blkio delay longer than thread age: %lu\n", nr, bad);
	return 0;
}

 include/linux/delayacct.h | 10 +++++-----
 kernel/delayacct.c        |  8 ++++++--
 kernel/sched/sched.h      |  2 +-
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/include/linux/delayacct.h b/include/linux/delayacct.h
index ecb06f16d2..4bb01a1f86 100644
--- a/include/linux/delayacct.h
+++ b/include/linux/delayacct.h
@@ -92,7 +92,7 @@ extern void delayacct_init(void);
 
 extern void __delayacct_tsk_init(struct task_struct *);
 extern void __delayacct_tsk_exit(struct task_struct *);
-extern void __delayacct_blkio_start(void);
+void __delayacct_blkio_start(struct task_struct *p);
 extern void __delayacct_blkio_end(struct task_struct *);
 extern int delayacct_add_tsk(struct taskstats *, struct task_struct *);
 extern __u64 __delayacct_blkio_ticks(struct task_struct *);
@@ -126,13 +126,13 @@ static inline void delayacct_tsk_free(struct task_struct *tsk)
 	tsk->delays = NULL;
 }
 
-static inline void delayacct_blkio_start(void)
+static inline void delayacct_blkio_start(struct task_struct *p)
 {
 	if (!static_branch_unlikely(&delayacct_key))
 		return;
 
-	if (current->delays)
-		__delayacct_blkio_start();
+	if (p->delays)
+		__delayacct_blkio_start(p);
 }
 
 static inline void delayacct_blkio_end(struct task_struct *p)
@@ -257,7 +257,7 @@ static inline void delayacct_tsk_init(struct task_struct *tsk)
 {}
 static inline void delayacct_tsk_free(struct task_struct *tsk)
 {}
-static inline void delayacct_blkio_start(void)
+static inline void delayacct_blkio_start(struct task_struct *p)
 {}
 static inline void delayacct_blkio_end(struct task_struct *p)
 {}
diff --git a/kernel/delayacct.c b/kernel/delayacct.c
index 479e860aff..1d735d58e1 100644
--- a/kernel/delayacct.c
+++ b/kernel/delayacct.c
@@ -126,9 +126,13 @@ static void delayacct_end(raw_spinlock_t *lock, u64 *start, u64 *total, u32 *cou
 	}
 }
 
-void __delayacct_blkio_start(void)
+/*
+ * @p is not necessarily current: a sched_delayed task can be blocked by
+ * another task picking it, see __block_task().
+ */
+void __delayacct_blkio_start(struct task_struct *p)
 {
-	current->delays->blkio_start = local_clock();
+	p->delays->blkio_start = local_clock();
 }
 
 /*
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 6c7c563352..4ca7eac523 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3126,7 +3126,7 @@ static inline void __block_task(struct rq *rq, struct task_struct *p)
 
 	if (p->in_iowait) {
 		atomic_inc(&rq->nr_iowait);
-		delayacct_blkio_start();
+		delayacct_blkio_start(p);
 	}
 
 	ASSERT_EXCLUSIVE_WRITER(p->on_rq);

base-commit: 6628e7d4c6b25c8d18f1345e3a20d21829aae7e4