[PATCH 4/6] workqueue: account nr_active by the backing pool

[Breno Leitao]

Use is_pool_cpu_specific() instead of wq_node_nr_active() to check for
per workqueues.

Signed-off-by: Breno Leitao <leitao@xxxxxxxxxx>
---
 kernel/workqueue.c | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 513193be00a7d..d17fdcaaf1685 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -1782,13 +1782,16 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
 {
 	struct workqueue_struct *wq = pwq->wq;
 	struct worker_pool *pool = pwq->pool;
-	struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);
+	struct wq_node_nr_active *nna;
 	bool obtained = false;
 
 	lockdep_assert_held(&pool->lock);
 
-	if (!nna) {
-		/* BH or per-cpu workqueue, pwq->nr_active is sufficient */
+	/*
+	 * A concurrency-managed per-cpu pool accounts nr_active per pwq, so
+	 * pwq->nr_active against wq->max_active is sufficient.
+	 */
+	if (is_pool_cpu_specific(pool)) {
 		obtained = pwq->nr_active < READ_ONCE(wq->max_active);
 		goto out;
 	}
@@ -1796,6 +1799,10 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
 	if (unlikely(pwq->plugged))
 		return false;
 
+	nna = wq_node_nr_active(wq, pool->node);
+	if (WARN_ON_ONCE(!nna))
+		return false;
+
 	/*
 	 * Unbound workqueue uses per-node shared nr_active $nna. If @pwq is
 	 * already waiting on $nna, pwq_dec_nr_active() will maintain the
@@ -2013,7 +2020,7 @@ static void node_activate_pending_pwq(struct wq_node_nr_active *nna,
 static void pwq_dec_nr_active(struct pool_workqueue *pwq)
 {
 	struct worker_pool *pool = pwq->pool;
-	struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);
+	struct wq_node_nr_active *nna;
 
 	lockdep_assert_held(&pool->lock);
 
@@ -2024,14 +2031,18 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq)
 	pwq->nr_active--;
 
 	/*
-	 * For a percpu workqueue, it's simple. Just need to kick the first
+	 * A concurrency-managed per-cpu pool only needs to kick the first
 	 * inactive work item on @pwq itself.
 	 */
-	if (!nna) {
+	if (is_pool_cpu_specific(pool)) {
 		pwq_activate_first_inactive(pwq, false);
 		return;
 	}
 
+	nna = wq_node_nr_active(pwq->wq, pool->node);
+	if (WARN_ON_ONCE(!nna))
+		return;
+
 	/*
 	 * If @pwq is for an unbound workqueue, it's more complicated because
 	 * multiple pwqs and pools may be sharing the nr_active count. When a

-- 
2.53.0-Meta