[PATCH 4/5] crypto: caam/qi - avoid allocating memory at crypto request runtime

From: meenakshi . aggarwal
Date: Tue May 23 2023 - 11:35:23 EST

From: Meenakshi Aggarwal <meenakshi.aggarwal@xxxxxxx>

Remove CRYPTO_ALG_ALLOCATES_MEMORY flag and allocate the memory
needed by the driver, to fulfil a request, within the crypto
request object.
The extra size needed for base extended descriptor, hw
descriptor commands and link tables is computed in frontend
driver (caamalg_qi) initialization and saved in reqsize field
that indicates how much memory could be needed per request.

CRYPTO_ALG_ALLOCATES_MEMORY flag is limited only to
dm-crypt use-cases, which seems to be 4 entries maximum.
Therefore in reqsize we allocate memory for maximum 4 entries
for src and 4 for dst, aligned.
If the driver needs more than the 4 entries maximum, the memory
is dynamically allocated, at runtime.

Signed-off-by: Iuliana Prodan <iuliana.prodan@xxxxxxx>
Signed-off-by: Meenakshi Aggarwal <meenakshi.aggarwal@xxxxxxx>
---
drivers/crypto/caam/caamalg_qi.c | 131 +++++++++++++++++++++----------
1 file changed, 89 insertions(+), 42 deletions(-)

diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index 743ce50c14f2..d7078f37ef9f 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -793,6 +793,7 @@ static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
* @dst_nents: number of segments in output scatterlist
* @iv_dma: dma address of iv for checking continuity and link table
* @qm_sg_bytes: length of dma mapped h/w link table
+ * @free: stored to determine if aead_edesc needs to be freed
* @qm_sg_dma: bus physical mapped address of h/w link table
* @assoclen: associated data length, in CAAM endianness
* @assoclen_dma: bus physical mapped address of req->assoclen
@@ -804,6 +805,7 @@ struct aead_edesc {
int dst_nents;
dma_addr_t iv_dma;
int qm_sg_bytes;
+ bool free;
dma_addr_t qm_sg_dma;
unsigned int assoclen;
dma_addr_t assoclen_dma;
@@ -817,6 +819,7 @@ struct aead_edesc {
* @dst_nents: number of segments in output scatterlist
* @iv_dma: dma address of iv for checking continuity and link table
* @qm_sg_bytes: length of dma mapped h/w link table
+ * @free: stored to determine if skcipher_edesc needs to be freed
* @qm_sg_dma: bus physical mapped address of h/w link table
* @drv_req: driver-specific request structure
* @sgt: the h/w link table, followed by IV
@@ -826,6 +829,7 @@ struct skcipher_edesc {
int dst_nents;
dma_addr_t iv_dma;
int qm_sg_bytes;
+ bool free;
dma_addr_t qm_sg_dma;
struct caam_drv_req drv_req;
struct qm_sg_entry sgt[];
@@ -932,7 +936,8 @@ static void aead_done(struct caam_drv_req *drv_req, u32 status)
aead_unmap(qidev, edesc, aead_req);

aead_request_complete(aead_req, ecode);
- qi_cache_free(edesc);
+ if (edesc->free)
+ qi_cache_free(edesc);
}

/*
@@ -954,7 +959,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dma_addr_t qm_sg_dma, iv_dma = 0;
int ivsize = 0;
unsigned int authsize = ctx->authsize;
- int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes;
+ int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes, edesc_size = 0;
int in_len, out_len;
struct qm_sg_entry *sg_table, *fd_sgt;
struct caam_drv_ctx *drv_ctx;
@@ -963,13 +968,6 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (IS_ERR(drv_ctx))
return (struct aead_edesc *)drv_ctx;

- /* allocate space for base edesc and hw desc commands, link tables */
- edesc = qi_cache_alloc(flags);
- if (unlikely(!edesc)) {
- dev_err(qidev, "could not allocate extended descriptor\n");
- return ERR_PTR(-ENOMEM);
- }
-
if (likely(req->src == req->dst)) {
src_len = req->assoclen + req->cryptlen +
(encrypt ? authsize : 0);
@@ -978,7 +976,6 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (unlikely(src_nents < 0)) {
dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
src_len);
- qi_cache_free(edesc);
return ERR_PTR(src_nents);
}

@@ -986,7 +983,6 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
DMA_BIDIRECTIONAL);
if (unlikely(!mapped_src_nents)) {
dev_err(qidev, "unable to map source\n");
- qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
} else {
@@ -997,7 +993,6 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (unlikely(src_nents < 0)) {
dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
src_len);
- qi_cache_free(edesc);
return ERR_PTR(src_nents);
}

@@ -1005,7 +1000,6 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
if (unlikely(dst_nents < 0)) {
dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
dst_len);
- qi_cache_free(edesc);
return ERR_PTR(dst_nents);
}

@@ -1014,7 +1008,6 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
src_nents, DMA_TO_DEVICE);
if (unlikely(!mapped_src_nents)) {
dev_err(qidev, "unable to map source\n");
- qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
} else {
@@ -1029,7 +1022,6 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dev_err(qidev, "unable to map destination\n");
dma_unmap_sg(qidev, req->src, src_nents,
DMA_TO_DEVICE);
- qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
} else {
@@ -1061,18 +1053,35 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
else
qm_sg_ents = pad_sg_nents(qm_sg_ents);

- sg_table = &edesc->sgt[0];
qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
- if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
- CAAM_QI_MEMCACHE_SIZE)) {
+
+ /* Check if there's enough space for edesc saved in req */
+ edesc_size = offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize;
+ if (unlikely(edesc_size > CAAM_QI_MEMCACHE_SIZE)) {
dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
0, DMA_NONE, 0, 0);
- qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
+ } else if (edesc_size > crypto_aead_reqsize(aead)) {
+ /* allocate space for base edesc, link tables and IV */
+ edesc = qi_cache_alloc(flags);
+ if (unlikely(!edesc)) {
+ dev_err(qidev, "could not allocate extended descriptor\n");
+ caam_unmap(qidev, req->src, req->dst, src_nents,
+ dst_nents, 0, 0, DMA_NONE, 0, 0);
+ return ERR_PTR(-ENOMEM);
+ }
+ edesc->free = true;
+ } else {
+ /* get address for base edesc, link tables and IV */
+ edesc = (struct aead_edesc *)((u8 *)aead_request_ctx(req));
+ /* clear memory */
+ memset(edesc, 0, sizeof(*edesc));
}

+ sg_table = &edesc->sgt[0];
+
if (ivsize) {
u8 *iv = (u8 *)(sg_table + qm_sg_ents);

@@ -1084,7 +1093,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dev_err(qidev, "unable to map IV\n");
caam_unmap(qidev, req->src, req->dst, src_nents,
dst_nents, 0, 0, DMA_NONE, 0, 0);
- qi_cache_free(edesc);
+ if (edesc->free)
+ qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}
}
@@ -1103,7 +1113,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dev_err(qidev, "unable to map assoclen\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
- qi_cache_free(edesc);
+ if (edesc->free)
+ qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}

@@ -1125,7 +1136,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
- qi_cache_free(edesc);
+ if (edesc->free)
+ qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}

@@ -1179,7 +1191,8 @@ static inline int aead_crypt(struct aead_request *req, bool encrypt)
ret = -EINPROGRESS;
} else {
aead_unmap(ctx->qidev, edesc, req);
- qi_cache_free(edesc);
+ if (edesc->free)
+ qi_cache_free(edesc);
}

return ret;
@@ -1247,7 +1260,8 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
if (!ecode)
memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize);

- qi_cache_free(edesc);
+ if (edesc->free)
+ qi_cache_free(edesc);
skcipher_request_complete(req, ecode);
}

@@ -1333,21 +1347,28 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
len = ALIGN(len, dma_get_cache_alignment());
len += ivsize;

+ /* Check if there's enough space for edesc saved in req */
if (unlikely(len > CAAM_QI_MEMCACHE_SIZE)) {
dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
qm_sg_ents, ivsize);
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
0, DMA_NONE, 0, 0);
return ERR_PTR(-ENOMEM);
- }
-
- /* allocate space for base edesc, link tables and IV */
- edesc = qi_cache_alloc(flags);
- if (unlikely(!edesc)) {
- dev_err(qidev, "could not allocate extended descriptor\n");
- caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
- 0, DMA_NONE, 0, 0);
- return ERR_PTR(-ENOMEM);
+ } else if (len > crypto_skcipher_reqsize(skcipher)) {
+ /* allocate space for base edesc, link tables and IV */
+ edesc = qi_cache_alloc(flags);
+ if (unlikely(!edesc)) {
+ dev_err(qidev, "could not allocate extended descriptor\n");
+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
+ 0, DMA_NONE, 0, 0);
+ return ERR_PTR(-ENOMEM);
+ }
+ edesc->free = true;
+ } else {
+ /* get address for base edesc, link tables and IV */
+ edesc = (struct skcipher_edesc *)((u8 *)skcipher_request_ctx(req));
+ /* clear memory */
+ memset(edesc, 0, sizeof(*edesc));
}

edesc->src_nents = src_nents;
@@ -1367,7 +1388,8 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
dev_err(qidev, "unable to map IV\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
0, DMA_NONE, 0, 0);
- qi_cache_free(edesc);
+ if (edesc->free)
+ qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}

@@ -1388,7 +1410,8 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
dev_err(qidev, "unable to map S/G table\n");
caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
- qi_cache_free(edesc);
+ if (edesc->free)
+ qi_cache_free(edesc);
return ERR_PTR(-ENOMEM);
}

@@ -1462,7 +1485,8 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
ret = -EINPROGRESS;
} else {
skcipher_unmap(ctx->qidev, edesc, req);
- qi_cache_free(edesc);
+ if (edesc->free)
+ qi_cache_free(edesc);
}

return ret;
@@ -2509,7 +2533,16 @@ static int caam_cra_init(struct crypto_skcipher *tfm)
container_of(alg, typeof(*caam_alg), skcipher);
struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
- int ret = 0;
+ int ret = 0, extra_reqsize = 0;
+
+ /* Compute extra space needed for base edesc, link tables and IV */
+ extra_reqsize = sizeof(struct skcipher_edesc) +
+ /* link tables for src and dst:
+ * 4 entries max + 1 for IV, aligned = 8
+ */
+ 16 * sizeof(struct qm_sg_entry);
+ extra_reqsize += ALIGN(extra_reqsize, dma_get_cache_alignment());
+ extra_reqsize += AES_BLOCK_SIZE; /* ivsize */

if (alg_aai == OP_ALG_AAI_XTS) {
const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
@@ -2525,7 +2558,10 @@ static int caam_cra_init(struct crypto_skcipher *tfm)

ctx->fallback = fallback;
crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) +
- crypto_skcipher_reqsize(fallback));
+ crypto_skcipher_reqsize(fallback) +
+ extra_reqsize);
+ } else {
+ crypto_skcipher_set_reqsize(tfm, extra_reqsize);
}

ret = caam_init_common(ctx, &caam_alg->caam, false);
@@ -2541,6 +2577,19 @@ static int caam_aead_init(struct crypto_aead *tfm)
struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
aead);
struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm);
+ int extra_reqsize = 0;
+
+ /* Compute extra space needed for base edesc, link tables and IV */
+ extra_reqsize = sizeof(struct aead_edesc) +
+ /* link tables for src and dst:
+ * 4 entries max + 1 for IV, aligned = 8
+ */
+ (16 * sizeof(struct qm_sg_entry)) +
+ AES_BLOCK_SIZE; /* ivsize */
+ /*
+ * Set the size for the space needed for base edesc, link tables, IV
+ */
+ crypto_aead_set_reqsize(tfm, extra_reqsize);

return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);
}
@@ -2596,8 +2645,7 @@ static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
alg->base.cra_module = THIS_MODULE;
alg->base.cra_priority = CAAM_CRA_PRIORITY;
alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
- alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
- CRYPTO_ALG_KERN_DRIVER_ONLY);
+ alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY);

alg->init = caam_cra_init;
alg->exit = caam_cra_exit;
@@ -2610,8 +2658,7 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
alg->base.cra_module = THIS_MODULE;
alg->base.cra_priority = CAAM_CRA_PRIORITY;
alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
- alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
- CRYPTO_ALG_KERN_DRIVER_ONLY;
+ alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;

alg->init = caam_aead_init;
alg->exit = caam_aead_exit;
--
2.25.1