[PATCH v4 04/40] watch_queue: optimize post_one_notification() by using find_and_clear_bit()

From: Yury Norov
Date: Thu Jun 20 2024 - 14:00:15 EST


post_one_notification() searches for a set bit in wqueue->notes_bitmap,
and after some housekeeping work clears it, firing a BUG() if someone
else cleared the bit in-between.

We can allocate the bit atomically with an atomic find_and_clear_bit(),
and remove the BUG() possibility entirely.

Signed-off-by: Yury Norov <yury.norov@xxxxxxxxx>
---
kernel/watch_queue.c | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/kernel/watch_queue.c b/kernel/watch_queue.c
index 03b90d7d2175..387ee88af71d 100644
--- a/kernel/watch_queue.c
+++ b/kernel/watch_queue.c
@@ -9,6 +9,7 @@

#define pr_fmt(fmt) "watchq: " fmt
#include <linux/module.h>
+#include <linux/find_atomic.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/slab.h>
@@ -112,7 +113,7 @@ static bool post_one_notification(struct watch_queue *wqueue,
if (pipe_full(head, tail, pipe->ring_size))
goto lost;

- note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes);
+ note = find_and_clear_bit(wqueue->notes_bitmap, wqueue->nr_notes);
if (note >= wqueue->nr_notes)
goto lost;

@@ -133,10 +134,6 @@ static bool post_one_notification(struct watch_queue *wqueue,
buf->flags = PIPE_BUF_FLAG_WHOLE;
smp_store_release(&pipe->head, head + 1); /* vs pipe_read() */

- if (!test_and_clear_bit(note, wqueue->notes_bitmap)) {
- spin_unlock_irq(&pipe->rd_wait.lock);
- BUG();
- }
wake_up_interruptible_sync_poll_locked(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
done = true;

--
2.43.0