Re: [PATCH v2] tools/workqueue/wq_dump.py: Add busy worker inspection and BH pool states

[Aaron Tomlin]

On Sun, Sep 13, 2026 at 06:01:41AM -1000, Tejun Heo wrote:
> Hello, Aaron.
> 
> On Fri, Sep 11, 2026 at 05:06:25PM -0400, Aaron Tomlin wrote:
> > +        for bkt in pool.busy_hash:
> > +            for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
> 
> worker->hentry shares storage with the idle-list entry, so a worker going
> idle can send this iterator into the circular idle list. Leaving idle can
> leave a self-link with WORKER_IDLE clear. The per-worker retries do not
> bound either case.
> 
> Could you break out of the bucket on an idle worker, a repeated address, or
> a traversal read fault? Warn that the pool's busy-worker output is
> incomplete and ask the user to retry, then continue with the other pools.
> The exception handler needs to cover iterator advancement too.
> 
> Thanks.
> 
> -- 
> tejun

Hi Tejun,

Understood, we need to address the lockless race conditions on a live
system without holding pool->lock. How about the following?

 - Guard lockless busy_hash traversal against circular idle list
   diversions, self-links, and traversal read faults: check for
   WORKER_IDLE, track visited worker addresses per bucket, cover iterator
   advancement with (drgn.FaultError, LookupError) exception handling, and
   warn if a pool's busy worker dump was incomplete

diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py
index 4a3281bc5605..32be5c69a61b 100644
--- a/tools/workqueue/wq_dump.py
+++ b/tools/workqueue/wq_dump.py
@@ -137,6 +137,7 @@ POOL_BH                 = prog['POOL_BH']
 POOL_BH_DRAINING        = prog['POOL_BH_DRAINING']
 POOL_DISASSOCIATED      = prog['POOL_DISASSOCIATED']
 HIGHPRI_NICE_LEVEL      = prog['HIGHPRI_NICE_LEVEL']
+WORKER_IDLE             = prog['WORKER_IDLE']
 
 WQ_NAME_LEN             = prog['WQ_NAME_LEN'].value_()
 cpumask_str_len         = len(cpumask_str(wq_unbound_cpumask))
@@ -203,38 +204,55 @@ for pi, pool in idr_for_each(worker_pool_idr):
     print('')
 
     if args.busy:
+        incomplete = False
         for bkt in pool.busy_hash:
-            for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
-                for _ in range(3):
-                    try:
-                        pwq = worker.current_pwq
-                        func = worker.current_func.value_()
-                        if not pwq.value_() or not func:
+            if incomplete:
+                break
+            seen = set()
+            try:
+                for worker in hlist_for_each_entry('struct worker', bkt.address_of_(), 'hentry'):
+                    addr = worker.value_()
+                    if addr in seen or (worker.flags & WORKER_IDLE):
+                        incomplete = True
+                        break
+                    seen.add(addr)
+
+                    for _ in range(3):
+                        try:
+                            pwq = worker.current_pwq
+                            func = worker.current_func.value_()
+                            if not pwq.value_() or not func:
+                                continue
+
+                            wq_name = pwq.wq.name.string_().decode()
+                            fn_name = prog.symbol(func).name
+
+                            dur_str = ''
+                            start = worker_current_start(worker)
+                            if 'jiffies' in prog and start:
+                                jiffies = prog['jiffies'].value_()
+                                dur_s = ((jiffies - start) & jiffies_mask) // hz
+                                dur_str = f' for {dur_s}s'
+
+                            if pool.flags & POOL_BH:
+                                w_id = 'bh' if pool.attrs.nice != HIGHPRI_NICE_LEVEL else 'bh-hi'
+                            elif worker.task.value_():
+                                w_id = f'PID {worker.task.pid.value_():<6} ({worker.task.comm.string_().decode()})'
+                            else:
+                                w_id = f'worker[{worker.id.value_()}]'
+
+                            desc = worker.desc.string_().decode()
+                            desc_str = f' desc="{desc}"' if desc and desc != wq_name else ''
+                            print(f'    busy: {w_id}: {wq_name}:{fn_name}{dur_str}{desc_str}')
+                            break
+                        except (drgn.FaultError, LookupError):
                             continue
+            except (drgn.FaultError, LookupError):
+                incomplete = True
+                break
 
-                        wq_name = pwq.wq.name.string_().decode()
-                        fn_name = prog.symbol(func).name
-
-                        dur_str = ''
-                        start = worker_current_start(worker)
-                        if 'jiffies' in prog and start:
-                            jiffies = prog['jiffies'].value_()
-                            dur_s = ((jiffies - start) & jiffies_mask) // hz
-                            dur_str = f' for {dur_s}s'
-
-                        if pool.flags & POOL_BH:
-                            w_id = 'bh' if pool.attrs.nice != HIGHPRI_NICE_LEVEL else 'bh-hi'
-                        elif worker.task.value_():
-                            w_id = f'PID {worker.task.pid.value_():<6} ({worker.task.comm.string_().decode()})'
-                        else:
-                            w_id = f'worker[{worker.id.value_()}]'
-
-                        desc = worker.desc.string_().decode()
-                        desc_str = f' desc="{desc}"' if desc and desc != wq_name else ''
-                        print(f'    busy: {w_id}: {wq_name}:{fn_name}{dur_str}{desc_str}')
-                        break
-                    except (drgn.FaultError, LookupError):
-                        continue
+        if incomplete:
+            print(f'    warning: pool[{pi:02}] busy worker dump incomplete, please retry')
 
 print('')
 print('Workqueue CPU -> pool')


Kind regards,
-- 
Aaron Tomlin