[PATCH v2 28/30] crypto: sa2ul - route requests through crypto_engine

From: Manorit Chawdhry

Date: Wed Nov 04 2026 - 00:48:30 EST


Use crypto_engine to queue skcipher, ahash and AEAD requests and
process them in a sleepable context. Keep the cipher and AEAD operation
direction in each request context until submission.

Allocate and start the engine during probe, and release it on teardown.

Signed-off-by: Manorit Chawdhry <m-chawdhry@xxxxxx>
---
drivers/crypto/Kconfig | 1 +
drivers/crypto/sa2ul.c | 646 ++++++++++++++++++++++++++++++++-----------------
drivers/crypto/sa2ul.h | 28 +++
3 files changed, 454 insertions(+), 221 deletions(-)

diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 0189dfdcbbe1..c9596f0c9087 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -816,6 +816,7 @@ config CRYPTO_DEV_SA2UL
select CRYPTO_ALGAPI
select CRYPTO_AUTHENC
select CRYPTO_DES
+ select CRYPTO_ENGINE
select CRYPTO_SHA1
select CRYPTO_SHA256
select CRYPTO_SHA512
diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c
index 19caaa41ba03..5716b44afaeb 100644
--- a/drivers/crypto/sa2ul.c
+++ b/drivers/crypto/sa2ul.c
@@ -12,6 +12,7 @@
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/dmapool.h>
+#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/mempool.h>
#include <linux/module.h>
@@ -22,6 +23,7 @@
#include <linux/pm_runtime.h>

#include <crypto/aes.h>
+#include <crypto/algapi.h>
#include <crypto/authenc.h>
#include <crypto/utils.h>
#include <crypto/des.h>
@@ -31,6 +33,7 @@
#include <crypto/scatterwalk.h>
#include <crypto/sha1.h>
#include <crypto/sha2.h>
+#include <crypto/engine.h>

#include "sa2ul.h"

@@ -143,9 +146,9 @@ struct algo_data {
struct sa_alg_tmpl {
u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */
union {
- struct skcipher_alg skcipher;
- struct ahash_alg ahash;
- struct aead_alg aead;
+ struct skcipher_engine_alg skcipher;
+ struct ahash_engine_alg ahash;
+ struct aead_engine_alg aead;
} alg;
bool registered;
};
@@ -169,6 +172,7 @@ struct sa_mapped_sg {
* struct sa_rx_data: RX Packet miscellaneous data place holder
* @req: crypto request data pointer
* @ddev: pointer to the DMA device
+ * @pdata: owning SA device
* @tx_in: dma_async_tx_descriptor pointer for rx channel
* @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
* @enc: Flag indicating either encryption or decryption
@@ -178,6 +182,7 @@ struct sa_mapped_sg {
struct sa_rx_data {
void *req;
struct device *ddev;
+ struct sa_crypto_data *pdata;
struct dma_async_tx_descriptor *tx_in;
struct sa_mapped_sg mapped_sg[2];
u8 enc;
@@ -955,6 +960,8 @@ static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
{
int i;

+ rxd->pdata->rxd = NULL;
+
for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];

@@ -968,10 +975,29 @@ static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
kfree(rxd);
}

+static struct crypto_async_request *sa_cancel_dma(struct sa_crypto_data *data)
+{
+ struct crypto_async_request *req = NULL;
+
+ dmaengine_terminate_sync(data->dma_tx);
+ dmaengine_terminate_sync(data->dma_rx1);
+ dmaengine_terminate_sync(data->dma_rx2);
+
+ /* Termination does not invoke callbacks for the discarded descriptors. */
+ if (data->rxd) {
+ req = data->rxd->req;
+ sa_sync_from_device(data->rxd);
+ sa_free_sa_rx_data(data->rxd);
+ }
+
+ return req;
+}
+
static void sa_aes_dma_in_callback(void *data,
const struct dmaengine_result *dma_result)
{
struct sa_rx_data *rxd = data;
+ struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
struct skcipher_request *req;
u8 *result;
__be32 *mdptr;
@@ -994,7 +1020,7 @@ static void sa_aes_dma_in_callback(void *data,

sa_free_sa_rx_data(rxd);

- skcipher_request_complete(req, err);
+ crypto_finalize_skcipher_request(pdata->engine, req, err);
}

static int
@@ -1041,6 +1067,7 @@ static int sa_run(struct sa_req *req)
rxd = kzalloc_obj(*rxd, gfp_flags);
if (!rxd)
return -ENOMEM;
+ rxd->pdata = pdata;

if (req->src != req->dst) {
diff_dst = true;
@@ -1218,6 +1245,7 @@ static int sa_run(struct sa_req *req)
dmaengine_submit(tx_out);
dmaengine_submit(rxd->tx_in);

+ pdata->rxd = rxd;
dma_async_issue_pending(dma_rx);
dma_async_issue_pending(pdata->dma_tx);

@@ -1229,12 +1257,77 @@ static int sa_run(struct sa_req *req)
return ret;
}

-static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc)
+static void sa_complete_req(struct crypto_engine *engine,
+ struct crypto_async_request *req, int err)
+{
+ struct skcipher_request *skreq;
+
+ local_bh_disable();
+ switch (crypto_tfm_alg_type(req->tfm)) {
+ case CRYPTO_ALG_TYPE_SKCIPHER:
+ skreq = container_of(req, struct skcipher_request, base);
+ crypto_finalize_skcipher_request(engine, skreq, err);
+ break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ crypto_finalize_hash_request(engine,
+ container_of(req, struct ahash_request, base),
+ err);
+ break;
+ case CRYPTO_ALG_TYPE_AEAD:
+ crypto_finalize_aead_request(engine,
+ container_of(req, struct aead_request, base),
+ err);
+ break;
+ }
+ local_bh_enable();
+}
+
+static int sa_run_one(struct sa_req *req)
+{
+ struct sa_crypto_data *data = req->ctx->dev_data;
+ int ret;
+
+ mutex_lock(&data->dma_lock);
+ ret = READ_ONCE(data->stopping) ? -ESHUTDOWN : sa_run(req);
+ mutex_unlock(&data->dma_lock);
+ if (ret == -EINPROGRESS)
+ return 0;
+
+ /* Finalize setup failures here so the engine can dequeue another request. */
+ sa_complete_req(data->engine, req->base, ret);
+
+ /* The request was completed above; do not let the engine complete it again. */
+ return 0;
+}
+
+static void sa_stop_engine(struct sa_crypto_data *data)
+{
+ struct crypto_async_request *req;
+ unsigned long flags;
+
+ spin_lock_irqsave(&data->request_lock, flags);
+ WRITE_ONCE(data->stopping, true);
+ spin_unlock_irqrestore(&data->request_lock, flags);
+
+ /* Wait for setup, then stop DMA and synchronize its completion callbacks. */
+ mutex_lock(&data->dma_lock);
+ req = sa_cancel_dma(data);
+ mutex_unlock(&data->dma_lock);
+ if (req)
+ sa_complete_req(data->engine, req, -ESHUTDOWN);
+
+ /* Queued requests now fail in sa_run_one() without submitting DMA. */
+ crypto_engine_exit(data->engine);
+}
+
+static int sa_cipher_run(struct skcipher_request *req, int enc)
{
struct sa_tfm_ctx *ctx =
crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
struct crypto_alg *alg = req->base.tfm->__crt_alg;
- struct sa_req sa_req = { 0 };
+ struct sa_cipher_req_ctx *rctx = skcipher_request_ctx(req);
+ unsigned long flags;
+ int ret;

if (!req->cryptlen)
return 0;
@@ -1258,34 +1351,51 @@ static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc)
return ret;
}

+ rctx->enc = enc;
+
+ spin_lock_irqsave(&ctx->dev_data->request_lock, flags);
+ ret = ctx->dev_data->stopping ? -ESHUTDOWN :
+ crypto_transfer_skcipher_request_to_engine(ctx->dev_data->engine, req);
+ spin_unlock_irqrestore(&ctx->dev_data->request_lock, flags);
+ return ret;
+}
+
+static int sa_cipher_do_one_req(struct crypto_engine *engine, void *areq)
+{
+ struct skcipher_request *req = container_of(areq, struct skcipher_request, base);
+ struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+ struct sa_cipher_req_ctx *rctx = skcipher_request_ctx(req);
+ struct sa_req sa_req = { 0 };
+
sa_req.size = req->cryptlen;
sa_req.enc_size = req->cryptlen;
sa_req.src = req->src;
sa_req.dst = req->dst;
- sa_req.enc_iv = iv;
+ sa_req.enc_iv = req->iv;
sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER;
- sa_req.enc = enc;
+ sa_req.enc = rctx->enc;
sa_req.callback = sa_aes_dma_in_callback;
sa_req.base = &req->base;
sa_req.ctx = ctx;

- return sa_run(&sa_req);
+ return sa_run_one(&sa_req);
}

static int sa_encrypt(struct skcipher_request *req)
{
- return sa_cipher_run(req, req->iv, 1);
+ return sa_cipher_run(req, 1);
}

static int sa_decrypt(struct skcipher_request *req)
{
- return sa_cipher_run(req, req->iv, 0);
+ return sa_cipher_run(req, 0);
}

static void sa_sha_dma_in_callback(void *data,
const struct dmaengine_result *dma_result)
{
struct sa_rx_data *rxd = data;
+ struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
struct ahash_request *req;
struct crypto_ahash *tfm;
unsigned int authsize;
@@ -1311,7 +1421,7 @@ static void sa_sha_dma_in_callback(void *data,
out:
sa_free_sa_rx_data(rxd);

- ahash_request_complete(req, err);
+ crypto_finalize_hash_request(pdata->engine, req, err);
}

static int zero_message_process(struct ahash_request *req)
@@ -1340,8 +1450,9 @@ static int sa_sha_run(struct ahash_request *req)
{
struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
- struct sa_req sa_req = { 0 };
size_t auth_len;
+ unsigned long flags;
+ int ret;

auth_len = req->nbytes;

@@ -1363,8 +1474,21 @@ static int sa_sha_run(struct ahash_request *req)
return ret;
}

- sa_req.size = auth_len;
- sa_req.auth_size = auth_len;
+ spin_lock_irqsave(&ctx->dev_data->request_lock, flags);
+ ret = ctx->dev_data->stopping ? -ESHUTDOWN :
+ crypto_transfer_hash_request_to_engine(ctx->dev_data->engine, req);
+ spin_unlock_irqrestore(&ctx->dev_data->request_lock, flags);
+ return ret;
+}
+
+static int sa_sha_do_one_req(struct crypto_engine *engine, void *areq)
+{
+ struct ahash_request *req = container_of(areq, struct ahash_request, base);
+ struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+ struct sa_req sa_req = { 0 };
+
+ sa_req.size = req->nbytes;
+ sa_req.auth_size = req->nbytes;
sa_req.src = req->src;
sa_req.dst = req->src;
sa_req.enc = true;
@@ -1373,7 +1497,7 @@ static int sa_sha_run(struct ahash_request *req)
sa_req.ctx = ctx;
sa_req.base = &req->base;

- return sa_run(&sa_req);
+ return sa_run_one(&sa_req);
}

static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad)
@@ -1595,6 +1719,7 @@ static void sa_aead_dma_in_callback(void *data,
const struct dmaengine_result *dma_result)
{
struct sa_rx_data *rxd = data;
+ struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
struct aead_request *req;
struct crypto_aead *tfm;
unsigned int start;
@@ -1631,7 +1756,7 @@ static void sa_aead_dma_in_callback(void *data,
out:
sa_free_sa_rx_data(rxd);

- aead_request_complete(req, err);
+ crypto_finalize_aead_request(pdata->engine, req, err);
}

static int sa_aead_init_tfm(struct crypto_aead *tfm, const char *fallback)
@@ -1800,12 +1925,14 @@ static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc,
return sa_aead_setkey(authenc, key, keylen, &ad);
}

-static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
+static int sa_aead_run(struct aead_request *req, int enc)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
- struct sa_req sa_req = { 0 };
+ struct sa_aead_req_ctx *rctx = aead_request_ctx(req);
size_t auth_size, enc_size;
+ unsigned long flags;
+ int ret;

enc_size = req->cryptlen;
auth_size = req->assoclen + req->cryptlen;
@@ -1833,271 +1960,330 @@ static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
return ret;
}

+ rctx->enc = enc;
+
+ spin_lock_irqsave(&ctx->dev_data->request_lock, flags);
+ ret = ctx->dev_data->stopping ? -ESHUTDOWN :
+ crypto_transfer_aead_request_to_engine(ctx->dev_data->engine, req);
+ spin_unlock_irqrestore(&ctx->dev_data->request_lock, flags);
+ return ret;
+}
+
+static int sa_aead_do_one_req(struct crypto_engine *engine, void *areq)
+{
+ struct aead_request *req = container_of(areq, struct aead_request, base);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
+ struct sa_aead_req_ctx *rctx = aead_request_ctx(req);
+ struct sa_req sa_req = { 0 };
+ size_t auth_size, enc_size;
+
+ enc_size = req->cryptlen;
+ auth_size = req->assoclen + req->cryptlen;
+
+ if (!rctx->enc) {
+ enc_size -= crypto_aead_authsize(tfm);
+ auth_size -= crypto_aead_authsize(tfm);
+ }
+
sa_req.enc_offset = req->assoclen;
sa_req.enc_size = enc_size;
sa_req.auth_size = auth_size;
sa_req.size = auth_size;
- sa_req.enc_iv = iv;
+ sa_req.enc_iv = req->iv;
sa_req.type = CRYPTO_ALG_TYPE_AEAD;
- sa_req.enc = enc;
+ sa_req.enc = rctx->enc;
sa_req.callback = sa_aead_dma_in_callback;
sa_req.base = &req->base;
sa_req.ctx = ctx;
sa_req.src = req->src;
sa_req.dst = req->dst;

- return sa_run(&sa_req);
+ return sa_run_one(&sa_req);
}

/* AEAD algorithm encrypt interface function */
static int sa_aead_encrypt(struct aead_request *req)
{
- return sa_aead_run(req, req->iv, 1);
+ return sa_aead_run(req, 1);
}

/* AEAD algorithm decrypt interface function */
static int sa_aead_decrypt(struct aead_request *req)
{
- return sa_aead_run(req, req->iv, 0);
+ return sa_aead_run(req, 0);
}

static struct sa_alg_tmpl sa_algs[] = {
[SA_ALG_CBC_AES] = {
.type = CRYPTO_ALG_TYPE_SKCIPHER,
.alg.skcipher = {
- .base.cra_name = "cbc(aes)",
- .base.cra_driver_name = "cbc-aes-sa2ul",
- .base.cra_priority = 30000,
- .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
- .base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
- .base.cra_module = THIS_MODULE,
- .init = sa_cipher_init_tfm,
- .exit = sa_cipher_exit_tfm,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = sa_aes_cbc_setkey,
- .encrypt = sa_encrypt,
- .decrypt = sa_decrypt,
+ .base = {
+ .base.cra_name = "cbc(aes)",
+ .base.cra_driver_name = "cbc-aes-sa2ul",
+ .base.cra_priority = 30000,
+ .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
+ .base.cra_reqsize = sizeof(struct sa_cipher_req_ctx),
+ .base.cra_module = THIS_MODULE,
+ .init = sa_cipher_init_tfm,
+ .exit = sa_cipher_exit_tfm,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = sa_aes_cbc_setkey,
+ .encrypt = sa_encrypt,
+ .decrypt = sa_decrypt,
+ },
+ .op.do_one_request = sa_cipher_do_one_req,
}
},
[SA_ALG_EBC_AES] = {
.type = CRYPTO_ALG_TYPE_SKCIPHER,
.alg.skcipher = {
- .base.cra_name = "ecb(aes)",
- .base.cra_driver_name = "ecb-aes-sa2ul",
- .base.cra_priority = 30000,
- .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
- .base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
- .base.cra_module = THIS_MODULE,
- .init = sa_cipher_init_tfm,
- .exit = sa_cipher_exit_tfm,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .setkey = sa_aes_ecb_setkey,
- .encrypt = sa_encrypt,
- .decrypt = sa_decrypt,
+ .base = {
+ .base.cra_name = "ecb(aes)",
+ .base.cra_driver_name = "ecb-aes-sa2ul",
+ .base.cra_priority = 30000,
+ .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
+ .base.cra_reqsize = sizeof(struct sa_cipher_req_ctx),
+ .base.cra_module = THIS_MODULE,
+ .init = sa_cipher_init_tfm,
+ .exit = sa_cipher_exit_tfm,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = sa_aes_ecb_setkey,
+ .encrypt = sa_encrypt,
+ .decrypt = sa_decrypt,
+ },
+ .op.do_one_request = sa_cipher_do_one_req,
}
},
[SA_ALG_CBC_DES3] = {
.type = CRYPTO_ALG_TYPE_SKCIPHER,
.alg.skcipher = {
- .base.cra_name = "cbc(des3_ede)",
- .base.cra_driver_name = "cbc-des3-sa2ul",
- .base.cra_priority = 30000,
- .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
- .base.cra_blocksize = DES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
- .base.cra_module = THIS_MODULE,
- .init = sa_cipher_init_tfm,
- .exit = sa_cipher_exit_tfm,
- .min_keysize = 3 * DES_KEY_SIZE,
- .max_keysize = 3 * DES_KEY_SIZE,
- .ivsize = DES_BLOCK_SIZE,
- .setkey = sa_3des_cbc_setkey,
- .encrypt = sa_encrypt,
- .decrypt = sa_decrypt,
+ .base = {
+ .base.cra_name = "cbc(des3_ede)",
+ .base.cra_driver_name = "cbc-des3-sa2ul",
+ .base.cra_priority = 30000,
+ .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
+ .base.cra_reqsize = sizeof(struct sa_cipher_req_ctx),
+ .base.cra_module = THIS_MODULE,
+ .init = sa_cipher_init_tfm,
+ .exit = sa_cipher_exit_tfm,
+ .min_keysize = 3 * DES_KEY_SIZE,
+ .max_keysize = 3 * DES_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ .setkey = sa_3des_cbc_setkey,
+ .encrypt = sa_encrypt,
+ .decrypt = sa_decrypt,
+ },
+ .op.do_one_request = sa_cipher_do_one_req,
}
},
[SA_ALG_ECB_DES3] = {
.type = CRYPTO_ALG_TYPE_SKCIPHER,
.alg.skcipher = {
- .base.cra_name = "ecb(des3_ede)",
- .base.cra_driver_name = "ecb-des3-sa2ul",
- .base.cra_priority = 30000,
- .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
- .base.cra_blocksize = DES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
- .base.cra_module = THIS_MODULE,
- .init = sa_cipher_init_tfm,
- .exit = sa_cipher_exit_tfm,
- .min_keysize = 3 * DES_KEY_SIZE,
- .max_keysize = 3 * DES_KEY_SIZE,
- .setkey = sa_3des_ecb_setkey,
- .encrypt = sa_encrypt,
- .decrypt = sa_decrypt,
+ .base = {
+ .base.cra_name = "ecb(des3_ede)",
+ .base.cra_driver_name = "ecb-des3-sa2ul",
+ .base.cra_priority = 30000,
+ .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = DES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
+ .base.cra_reqsize = sizeof(struct sa_cipher_req_ctx),
+ .base.cra_module = THIS_MODULE,
+ .init = sa_cipher_init_tfm,
+ .exit = sa_cipher_exit_tfm,
+ .min_keysize = 3 * DES_KEY_SIZE,
+ .max_keysize = 3 * DES_KEY_SIZE,
+ .setkey = sa_3des_ecb_setkey,
+ .encrypt = sa_encrypt,
+ .decrypt = sa_decrypt,
+ },
+ .op.do_one_request = sa_cipher_do_one_req,
}
},
[SA_ALG_SHA1] = {
.type = CRYPTO_ALG_TYPE_AHASH,
.alg.ahash = {
- .halg.base = {
- .cra_name = "sha1",
- .cra_driver_name = "sha1-sa2ul",
- .cra_priority = 400,
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_NEED_FALLBACK,
- .cra_blocksize = SHA1_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct sa_tfm_ctx),
- .cra_reqsize = sizeof(struct sa_sha_req_ctx) +
- MAX_SYNC_HASH_REQSIZE,
- .cra_module = THIS_MODULE,
+ .base = {
+ .halg.base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "sha1-sa2ul",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sa_tfm_ctx),
+ .cra_reqsize = sizeof(struct sa_sha_req_ctx) +
+ MAX_SYNC_HASH_REQSIZE,
+ .cra_module = THIS_MODULE,
+ },
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct sa_sha_req_ctx) +
+ sizeof(struct sha1_state),
+ .init_tfm = sa_sha1_init_tfm,
+ .exit_tfm = sa_sha_exit_tfm,
+ .init = sa_sha_init,
+ .update = sa_sha_update,
+ .final = sa_sha_final,
+ .finup = sa_sha_finup,
+ .digest = sa_sha_run,
+ .export = sa_sha_export,
+ .import = sa_sha_import,
},
- .halg.digestsize = SHA1_DIGEST_SIZE,
- .halg.statesize = sizeof(struct sa_sha_req_ctx) +
- sizeof(struct sha1_state),
- .init_tfm = sa_sha1_init_tfm,
- .exit_tfm = sa_sha_exit_tfm,
- .init = sa_sha_init,
- .update = sa_sha_update,
- .final = sa_sha_final,
- .finup = sa_sha_finup,
- .digest = sa_sha_run,
- .export = sa_sha_export,
- .import = sa_sha_import,
+ .op.do_one_request = sa_sha_do_one_req,
},
},
[SA_ALG_SHA256] = {
.type = CRYPTO_ALG_TYPE_AHASH,
.alg.ahash = {
- .halg.base = {
- .cra_name = "sha256",
- .cra_driver_name = "sha256-sa2ul",
- .cra_priority = 400,
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_NEED_FALLBACK,
- .cra_blocksize = SHA256_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct sa_tfm_ctx),
- .cra_reqsize = sizeof(struct sa_sha_req_ctx) +
- MAX_SYNC_HASH_REQSIZE,
- .cra_module = THIS_MODULE,
+ .base = {
+ .halg.base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-sa2ul",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sa_tfm_ctx),
+ .cra_reqsize = sizeof(struct sa_sha_req_ctx) +
+ MAX_SYNC_HASH_REQSIZE,
+ .cra_module = THIS_MODULE,
+ },
+ .halg.digestsize = SHA256_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct sa_sha_req_ctx) +
+ sizeof(struct sha256_state),
+ .init_tfm = sa_sha256_init_tfm,
+ .exit_tfm = sa_sha_exit_tfm,
+ .init = sa_sha_init,
+ .update = sa_sha_update,
+ .final = sa_sha_final,
+ .finup = sa_sha_finup,
+ .digest = sa_sha_run,
+ .export = sa_sha_export,
+ .import = sa_sha_import,
},
- .halg.digestsize = SHA256_DIGEST_SIZE,
- .halg.statesize = sizeof(struct sa_sha_req_ctx) +
- sizeof(struct sha256_state),
- .init_tfm = sa_sha256_init_tfm,
- .exit_tfm = sa_sha_exit_tfm,
- .init = sa_sha_init,
- .update = sa_sha_update,
- .final = sa_sha_final,
- .finup = sa_sha_finup,
- .digest = sa_sha_run,
- .export = sa_sha_export,
- .import = sa_sha_import,
+ .op.do_one_request = sa_sha_do_one_req,
},
},
[SA_ALG_SHA512] = {
.type = CRYPTO_ALG_TYPE_AHASH,
.alg.ahash = {
- .halg.base = {
- .cra_name = "sha512",
- .cra_driver_name = "sha512-sa2ul",
- .cra_priority = 400,
- .cra_flags = CRYPTO_ALG_TYPE_AHASH |
- CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_NEED_FALLBACK,
- .cra_blocksize = SHA512_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct sa_tfm_ctx),
- .cra_reqsize = sizeof(struct sa_sha_req_ctx) +
- MAX_SYNC_HASH_REQSIZE,
- .cra_module = THIS_MODULE,
+ .base = {
+ .halg.base = {
+ .cra_name = "sha512",
+ .cra_driver_name = "sha512-sa2ul",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sa_tfm_ctx),
+ .cra_reqsize = sizeof(struct sa_sha_req_ctx) +
+ MAX_SYNC_HASH_REQSIZE,
+ .cra_module = THIS_MODULE,
+ },
+ .halg.digestsize = SHA512_DIGEST_SIZE,
+ .halg.statesize = sizeof(struct sa_sha_req_ctx) +
+ sizeof(struct sha512_state),
+ .init_tfm = sa_sha512_init_tfm,
+ .exit_tfm = sa_sha_exit_tfm,
+ .init = sa_sha_init,
+ .update = sa_sha_update,
+ .final = sa_sha_final,
+ .finup = sa_sha_finup,
+ .digest = sa_sha_run,
+ .export = sa_sha_export,
+ .import = sa_sha_import,
},
- .halg.digestsize = SHA512_DIGEST_SIZE,
- .halg.statesize = sizeof(struct sa_sha_req_ctx) +
- sizeof(struct sha512_state),
- .init_tfm = sa_sha512_init_tfm,
- .exit_tfm = sa_sha_exit_tfm,
- .init = sa_sha_init,
- .update = sa_sha_update,
- .final = sa_sha_final,
- .finup = sa_sha_finup,
- .digest = sa_sha_run,
- .export = sa_sha_export,
- .import = sa_sha_import,
+ .op.do_one_request = sa_sha_do_one_req,
},
},
[SA_ALG_AUTHENC_SHA1_AES] = {
.type = CRYPTO_ALG_TYPE_AEAD,
.alg.aead = {
.base = {
- .cra_name = "authenc(hmac(sha1),cbc(aes))",
- .cra_driver_name =
- "authenc(hmac(sha1),cbc(aes))-sa2ul",
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_flags = CRYPTO_ALG_TYPE_AEAD |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
- .cra_ctxsize = sizeof(struct sa_tfm_ctx),
- .cra_module = THIS_MODULE,
- .cra_priority = 3000,
+ .base = {
+ .cra_name = "authenc(hmac(sha1),cbc(aes))",
+ .cra_driver_name =
+ "authenc(hmac(sha1),cbc(aes))-sa2ul",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_ctxsize = sizeof(struct sa_tfm_ctx),
+ .cra_reqsize = sizeof(struct sa_aead_req_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_priority = 3000,
+ },
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+
+ .init = sa_aead_sha1_init_tfm,
+ .exit = sa_aead_exit_tfm,
+ .setkey = sa_aead_cbc_sha1_setkey,
+ .setauthsize = sa_aead_setauthsize,
+ .encrypt = sa_aead_encrypt,
+ .decrypt = sa_aead_decrypt,
},
- .ivsize = AES_BLOCK_SIZE,
- .maxauthsize = SHA1_DIGEST_SIZE,
-
- .init = sa_aead_sha1_init_tfm,
- .exit = sa_aead_exit_tfm,
- .setkey = sa_aead_cbc_sha1_setkey,
- .setauthsize = sa_aead_setauthsize,
- .encrypt = sa_aead_encrypt,
- .decrypt = sa_aead_decrypt,
+ .op.do_one_request = sa_aead_do_one_req,
},
},
[SA_ALG_AUTHENC_SHA256_AES] = {
.type = CRYPTO_ALG_TYPE_AEAD,
.alg.aead = {
.base = {
- .cra_name = "authenc(hmac(sha256),cbc(aes))",
- .cra_driver_name =
- "authenc(hmac(sha256),cbc(aes))-sa2ul",
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_flags = CRYPTO_ALG_TYPE_AEAD |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
- .cra_ctxsize = sizeof(struct sa_tfm_ctx),
- .cra_module = THIS_MODULE,
- .cra_alignmask = 0,
- .cra_priority = 3000,
+ .base = {
+ .cra_name = "authenc(hmac(sha256),cbc(aes))",
+ .cra_driver_name =
+ "authenc(hmac(sha256),cbc(aes))-sa2ul",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_ctxsize = sizeof(struct sa_tfm_ctx),
+ .cra_reqsize = sizeof(struct sa_aead_req_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_alignmask = 0,
+ .cra_priority = 3000,
+ },
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA256_DIGEST_SIZE,
+
+ .init = sa_aead_sha256_init_tfm,
+ .exit = sa_aead_exit_tfm,
+ .setkey = sa_aead_cbc_sha256_setkey,
+ .setauthsize = sa_aead_setauthsize,
+ .encrypt = sa_aead_encrypt,
+ .decrypt = sa_aead_decrypt,
},
- .ivsize = AES_BLOCK_SIZE,
- .maxauthsize = SHA256_DIGEST_SIZE,
-
- .init = sa_aead_sha256_init_tfm,
- .exit = sa_aead_exit_tfm,
- .setkey = sa_aead_cbc_sha256_setkey,
- .setauthsize = sa_aead_setauthsize,
- .encrypt = sa_aead_encrypt,
- .decrypt = sa_aead_decrypt,
+ .op.do_one_request = sa_aead_do_one_req,
},
},
};
@@ -2118,14 +2304,14 @@ static void sa_register_algos(struct sa_crypto_data *dev_data)

type = sa_algs[i].type;
if (type == CRYPTO_ALG_TYPE_SKCIPHER) {
- alg_name = sa_algs[i].alg.skcipher.base.cra_name;
- err = crypto_register_skcipher(&sa_algs[i].alg.skcipher);
+ alg_name = sa_algs[i].alg.skcipher.base.base.cra_name;
+ err = crypto_engine_register_skcipher(&sa_algs[i].alg.skcipher);
} else if (type == CRYPTO_ALG_TYPE_AHASH) {
- alg_name = sa_algs[i].alg.ahash.halg.base.cra_name;
- err = crypto_register_ahash(&sa_algs[i].alg.ahash);
+ alg_name = sa_algs[i].alg.ahash.base.halg.base.cra_name;
+ err = crypto_engine_register_ahash(&sa_algs[i].alg.ahash);
} else if (type == CRYPTO_ALG_TYPE_AEAD) {
- alg_name = sa_algs[i].alg.aead.base.cra_name;
- err = crypto_register_aead(&sa_algs[i].alg.aead);
+ alg_name = sa_algs[i].alg.aead.base.base.cra_name;
+ err = crypto_engine_register_aead(&sa_algs[i].alg.aead);
} else {
dev_err(dev,
"un-supported crypto algorithm (%d)",
@@ -2151,11 +2337,11 @@ static void sa_unregister_algos(const struct device *dev)
if (!sa_algs[i].registered)
continue;
if (type == CRYPTO_ALG_TYPE_SKCIPHER)
- crypto_unregister_skcipher(&sa_algs[i].alg.skcipher);
+ crypto_engine_unregister_skcipher(&sa_algs[i].alg.skcipher);
else if (type == CRYPTO_ALG_TYPE_AHASH)
- crypto_unregister_ahash(&sa_algs[i].alg.ahash);
+ crypto_engine_unregister_ahash(&sa_algs[i].alg.ahash);
else if (type == CRYPTO_ALG_TYPE_AEAD)
- crypto_unregister_aead(&sa_algs[i].alg.aead);
+ crypto_engine_unregister_aead(&sa_algs[i].alg.aead);

sa_algs[i].registered = false;
}
@@ -2331,6 +2517,18 @@ static int sa_ul_probe(struct platform_device *pdev)
if (ret)
goto destroy_dma_pool;

+ spin_lock_init(&dev_data->request_lock);
+ mutex_init(&dev_data->dma_lock);
+ dev_data->engine = crypto_engine_alloc_init(dev, 0);
+ if (!dev_data->engine) {
+ ret = -ENOMEM;
+ goto release_dma;
+ }
+
+ ret = crypto_engine_start(dev_data->engine);
+ if (ret)
+ goto free_engine;
+
spin_lock_init(&dev_data->scid_lock);

val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
@@ -2345,15 +2543,19 @@ static int sa_ul_probe(struct platform_device *pdev)

ret = of_platform_populate(node, NULL, NULL, dev);
if (ret)
- goto release_dma;
+ goto unregister_algos;

device_for_each_child(dev, dev, sa_link_child);

return 0;

-release_dma:
+unregister_algos:
sa_unregister_algos(dev);

+free_engine:
+ sa_stop_engine(dev_data);
+
+release_dma:
dma_release_channel(dev_data->dma_rx2);
dma_release_channel(dev_data->dma_rx1);
dma_release_channel(dev_data->dma_tx);
@@ -2376,6 +2578,8 @@ static void sa_ul_remove(struct platform_device *pdev)

sa_unregister_algos(&pdev->dev);

+ sa_stop_engine(dev_data);
+
dma_release_channel(dev_data->dma_rx2);
dma_release_channel(dev_data->dma_rx1);
dma_release_channel(dev_data->dma_tx);
diff --git a/drivers/crypto/sa2ul.h b/drivers/crypto/sa2ul.h
index 5152c497858f..510a9fc4ab62 100644
--- a/drivers/crypto/sa2ul.h
+++ b/drivers/crypto/sa2ul.h
@@ -15,6 +15,7 @@
#include <crypto/aes.h>
#include <crypto/sha1.h>
#include <crypto/sha2.h>
+#include <linux/mutex.h>

#define SA_ENGINE_STATUS 0x0008
#define SA_ENGINE_ENABLE_CONTROL 0x1000
@@ -146,6 +147,7 @@ struct sa_tfm_ctx;
#define SA_UNSAFE_DATA_SZ_MAX 255

struct sa_match_data;
+struct sa_rx_data;

/**
* struct sa_crypto_data - Crypto driver instance data
@@ -159,6 +161,11 @@ struct sa_match_data;
* @dma_rx1: Pointer to DMA rx channel for sizes < 256 Bytes
* @dma_rx2: Pointer to DMA rx channel for sizes > 256 Bytes
* @dma_tx: Pointer to DMA TX channel
+ * @engine: crypto_engine instance for this device
+ * @rxd: DMA state for the active engine request
+ * @request_lock: protects request admission against teardown
+ * @dma_lock: serializes DMA setup against cancellation
+ * @stopping: reject hardware requests during teardown
*/
struct sa_crypto_data {
const struct sa_match_data *match_data;
@@ -173,6 +180,11 @@ struct sa_crypto_data {
struct dma_chan *dma_rx1;
struct dma_chan *dma_rx2;
struct dma_chan *dma_tx;
+ struct crypto_engine *engine;
+ struct sa_rx_data *rxd;
+ spinlock_t request_lock; /* request admission */
+ struct mutex dma_lock; /* DMA setup and cancellation */
+ bool stopping;
};

/**
@@ -256,6 +268,22 @@ struct sa_sha_req_ctx {
struct ahash_request fallback_req;
};

+/**
+ * struct sa_cipher_req_ctx: Structure used for skcipher request
+ * @enc: True if this request is an encrypt operation
+ */
+struct sa_cipher_req_ctx {
+ bool enc;
+};
+
+/**
+ * struct sa_aead_req_ctx: Structure used for aead request
+ * @enc: True if this request is an encrypt operation
+ */
+struct sa_aead_req_ctx {
+ bool enc;
+};
+
/**
* struct sa_eng_info: Security accelerator engine info
* @eng_id: Engine ID

--
2.43.0