[PATCH 4/4] crypto: iaa - keep each CPU on one WQ for decompression

[Vinicius Costa Gomes]

Decompressions round-robin over the package's WQs like compressions
do. Decompressions are latency-sensitive, and do better staying on a
warm engine.

Have each CPU pick one WQ of its package's table the first time it
decompresses, spread over the table by a per-table counter, and stay on
it.

The CPU caches the index, not the WQ, so it cannot outlive the table it
came from; rebalance makes every CPU pick again.

Signed-off-by: Vinicius Costa Gomes <vinicius.gomes@xxxxxxxxx>
---
 drivers/crypto/intel/iaa/iaa_crypto_main.c | 40 +++++++++++++++++++++++++-----
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c
index 2ce545e8df42..4f2c99c099df 100644
--- a/drivers/crypto/intel/iaa/iaa_crypto_main.c
+++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c
@@ -35,12 +35,15 @@ static unsigned int nr_iaa;
 struct iaa_pkg_table {
 	struct rcu_head	rcu;
 	int		n_wqs;
+	atomic_t	decomp_next;
 	struct idxd_wq	*wqs[];
 };
 
 /* Indexed by logical package id, the count is fixed at boot */
 static struct iaa_pkg_table __rcu **iaa_pkg_tables;
 static DEFINE_PER_CPU(unsigned int, iaa_wq_idx);
+/* The WQ of its package's table a CPU decompresses on, -1 until it picks */
+static DEFINE_PER_CPU(int, iaa_decomp_wq_idx) = -1;
 
 static int iaa_cpu_pkg(int cpu)
 {
@@ -49,16 +52,35 @@ static int iaa_cpu_pkg(int cpu)
 	return (pkg >= 0 && pkg < topology_max_packages()) ? pkg : 0;
 }
 
-/* Round-robins over the WQs of the caller's package */
-static struct idxd_wq *iaa_next_wq(void)
+/* Pick the idx of the WQ on first usage on each CPU */
+static unsigned int iaa_decomp_wq_pick(struct iaa_pkg_table *table)
+{
+	int idx = this_cpu_read(iaa_decomp_wq_idx);
+
+	/* An index left over from a wider table just means picking again */
+	if (unlikely(idx < 0 || idx >= table->n_wqs)) {
+		idx = (unsigned int)atomic_inc_return(&table->decomp_next) %
+		      table->n_wqs;
+		this_cpu_write(iaa_decomp_wq_idx, idx);
+	}
+
+	return idx;
+}
+
+/* Compression round-robins over the package's WQs, decompression keeps one */
+static struct idxd_wq *iaa_next_wq(bool decomp)
 {
 	struct iaa_pkg_table *table;
+	unsigned int idx;
 
 	table = rcu_dereference(iaa_pkg_tables[iaa_cpu_pkg(raw_smp_processor_id())]);
 	if (!table)
 		return NULL;
 
-	return table->wqs[this_cpu_inc_return(iaa_wq_idx) % table->n_wqs];
+	idx = decomp ? iaa_decomp_wq_pick(table) :
+		       this_cpu_inc_return(iaa_wq_idx) % table->n_wqs;
+
+	return table->wqs[idx];
 }
 
 LIST_HEAD(iaa_devices);
@@ -779,7 +801,7 @@ static int rebalance_pkg_tables(void)
 	struct iaa_pkg_table **tables, *table, *first = NULL;
 	struct iaa_device *iaa_device;
 	struct iaa_wq *iaa_wq;
-	int pkg, n_wqs = 0;
+	int cpu, pkg, n_wqs = 0;
 
 	if (nr_iaa == 0) {
 		publish_pkg_tables(NULL);
@@ -820,6 +842,10 @@ static int rebalance_pkg_tables(void)
 	publish_pkg_tables(tables);
 	kfree(tables);
 
+	/* Have every CPU pick its decompression WQ again, from the new table */
+	for_each_possible_cpu(cpu)
+		WRITE_ONCE(per_cpu(iaa_decomp_wq_idx, cpu), -1);
+
 	return 0;
 err:
 	for (pkg = 0; pkg < topology_max_packages(); pkg++)
@@ -1412,6 +1438,7 @@ static int iaa_comp_acompress(struct acomp_req *req)
 	struct iaa_compression_ctx *compression_ctx;
 	struct crypto_tfm *tfm = req->base.tfm;
 	dma_addr_t src_addr, dst_addr;
+	bool decomp = false;
 	int ret = 0;
 	struct iaa_wq *iaa_wq;
 	struct idxd_wq *wq;
@@ -1439,7 +1466,7 @@ static int iaa_comp_acompress(struct acomp_req *req)
 
 	/* The table, and its WQs until iaa_wq_get() returns, need RCU */
 	rcu_read_lock();
-	wq = iaa_next_wq();
+	wq = iaa_next_wq(decomp);
 	ret = wq ? iaa_wq_get(wq) : -ENODEV;
 	rcu_read_unlock();
 	if (ret) {
@@ -1512,6 +1539,7 @@ static int iaa_comp_adecompress(struct acomp_req *req)
 	struct crypto_tfm *tfm = req->base.tfm;
 	dma_addr_t src_addr, dst_addr;
 	bool use_bounce_src = false;
+	bool decomp = true;
 	int ret = 0;
 	struct iaa_wq *iaa_wq;
 	struct device *dev;
@@ -1544,7 +1572,7 @@ static int iaa_comp_adecompress(struct acomp_req *req)
 
 	/* The table, and its WQs until iaa_wq_get() returns, need RCU */
 	rcu_read_lock();
-	wq = iaa_next_wq();
+	wq = iaa_next_wq(decomp);
 	ret = wq ? iaa_wq_get(wq) : -ENODEV;
 	rcu_read_unlock();
 	if (ret) {

-- 
2.56.0