[PATCH 19/20] lib/crypto: riscv/aes-ctr: Migrate optimized code into library
From: Eric Biggers
Date: Mon Sep 21 2026 - 01:20:20 EST
Instead of exposing the riscv-optimized AES-CTR code via a
riscv-specific crypto_skcipher algorithm, just implement the AES-CTR
library functions. This is simpler, it makes the AES-CTR library
functions be riscv-optimized, and it also fixes the longstanding issue
where the riscv-optimized AES-CTR code was disabled by default. AES-CTR
support still remains available through crypto_skcipher via
crypto/aes.c, but individual architectures no longer need to handle it.
To match what the library expects, update the assembly functions to
operate on struct aes_enckey rather than struct crypto_aes_ctx, and
adjust the argument order.
Bump up the priority of the corresponding library-based algorithm on
riscv now that it no longer has to be lower than arch/riscv/crypto/.
Also re-enable the library-based "ccm(aes)" and "gcm(aes)".
Signed-off-by: Eric Biggers <ebiggers@xxxxxxxxxx>
---
arch/riscv/crypto/Kconfig | 5 +-
arch/riscv/crypto/Makefile | 2 +-
arch/riscv/crypto/aes-riscv64-glue.c | 114 +-----------------
crypto/aes.c | 5 +-
lib/crypto/Makefile | 4 +
.../crypto/riscv}/aes-riscv64-zvkned-zvkb.S | 23 ++--
lib/crypto/riscv/aes.h | 69 ++++++++++-
7 files changed, 91 insertions(+), 131 deletions(-)
rename {arch/riscv/crypto => lib/crypto/riscv}/aes-riscv64-zvkned-zvkb.S (93%)
diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 0a3f87ad384e..0733d4894401 100644
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -3,18 +3,17 @@
menu "Accelerated Cryptographic Algorithms for CPU (riscv)"
config CRYPTO_AES_RISCV64
- tristate "Ciphers: AES, modes: CTR, XTS"
+ tristate "Ciphers: AES, modes: XTS"
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
select CRYPTO_LIB_AES
select CRYPTO_SKCIPHER
help
- Length-preserving ciphers: AES with CTR, XTS
+ Length-preserving ciphers: AES with XTS
Architecture: riscv64 using:
- Zvkned vector crypto extension
- Zvbb vector extension (XTS)
- - Zvkb vector crypto extension (CTR)
- Zvkg vector crypto extension (XTS)
config CRYPTO_SM4_RISCV64
diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile
index d8b85afa6d0b..08904603fc94 100644
--- a/arch/riscv/crypto/Makefile
+++ b/arch/riscv/crypto/Makefile
@@ -2,7 +2,7 @@
obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o
aes-riscv64-y := aes-riscv64-glue.o \
- aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o
+ aes-riscv64-zvkned-zvbb-zvkg.o
obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o
sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o
diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c
index 97f5369d7e71..a7dcceb77c49 100644
--- a/arch/riscv/crypto/aes-riscv64-glue.c
+++ b/arch/riscv/crypto/aes-riscv64-glue.c
@@ -22,10 +22,6 @@
#include <linux/minmax.h>
#include <linux/module.h>
-asmlinkage void aes_ctr32_crypt_zvkned_zvkb(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len,
- u8 iv[AES_BLOCK_SIZE]);
-
asmlinkage void aes_xts_encrypt_zvkned_zvbb_zvkg(
const struct crypto_aes_ctx *key,
const u8 *in, u8 *out, size_t len,
@@ -62,75 +58,6 @@ static int riscv64_aes_setkey(struct crypto_aes_ctx *ctx,
return aes_expandkey(ctx, key, keylen);
}
-static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm,
- const u8 *key, unsigned int keylen)
-{
- struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
-
- return riscv64_aes_setkey(ctx, key, keylen);
-}
-
-/* AES-CTR */
-
-static int riscv64_aes_ctr_crypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- unsigned int nbytes, p1_nbytes;
- struct skcipher_walk walk;
- u32 ctr32, nblocks;
- int err;
-
- /* Get the low 32-bit word of the 128-bit big endian counter. */
- ctr32 = get_unaligned_be32(req->iv + 12);
-
- err = skcipher_walk_virt(&walk, req, false);
- while ((nbytes = walk.nbytes) != 0) {
- if (nbytes < walk.total) {
- /* Not the end yet, so keep the length block-aligned. */
- nbytes = round_down(nbytes, AES_BLOCK_SIZE);
- nblocks = nbytes / AES_BLOCK_SIZE;
- } else {
- /* It's the end, so include any final partial block. */
- nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
- }
- ctr32 += nblocks;
-
- kernel_vector_begin();
- if (ctr32 >= nblocks) {
- /* The low 32-bit word of the counter won't overflow. */
- aes_ctr32_crypt_zvkned_zvkb(ctx, walk.src.virt.addr,
- walk.dst.virt.addr, nbytes,
- req->iv);
- } else {
- /*
- * The low 32-bit word of the counter will overflow.
- * The assembly doesn't handle this case, so split the
- * operation into two at the point where the overflow
- * will occur. After the first part, add the carry bit.
- */
- p1_nbytes = min(nbytes, (nblocks - ctr32) * AES_BLOCK_SIZE);
- aes_ctr32_crypt_zvkned_zvkb(ctx, walk.src.virt.addr,
- walk.dst.virt.addr,
- p1_nbytes, req->iv);
- crypto_inc(req->iv, 12);
-
- if (ctr32) {
- aes_ctr32_crypt_zvkned_zvkb(
- ctx,
- walk.src.virt.addr + p1_nbytes,
- walk.dst.virt.addr + p1_nbytes,
- nbytes - p1_nbytes, req->iv);
- }
- }
- kernel_vector_end();
-
- err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
- }
-
- return err;
-}
-
/* AES-XTS */
struct riscv64_aes_xts_ctx {
@@ -251,25 +178,6 @@ static int riscv64_aes_xts_decrypt(struct skcipher_request *req)
/* Algorithm definitions */
-static struct skcipher_alg riscv64_zvkned_zvkb_aes_skcipher_alg = {
- .setkey = riscv64_aes_setkey_skcipher,
- .encrypt = riscv64_aes_ctr_crypt,
- .decrypt = riscv64_aes_ctr_crypt,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .chunksize = AES_BLOCK_SIZE,
- .walksize = 4 * AES_BLOCK_SIZE, /* matches LMUL=4 */
- .base = {
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_priority = 300,
- .cra_name = "ctr(aes)",
- .cra_driver_name = "ctr-aes-riscv64-zvkned-zvkb",
- .cra_module = THIS_MODULE,
- },
-};
-
static struct skcipher_alg riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg = {
.setkey = riscv64_aes_xts_setkey,
.encrypt = riscv64_aes_xts_encrypt,
@@ -302,43 +210,27 @@ static int __init riscv64_aes_mod_init(void)
if (riscv_isa_extension_available(NULL, ZVKNED) &&
riscv_vector_vlen() >= 128) {
- if (riscv_isa_extension_available(NULL, ZVKB)) {
- err = crypto_register_skcipher(
- &riscv64_zvkned_zvkb_aes_skcipher_alg);
- if (err)
- return err;
- }
-
if (riscv64_aes_xts_supported()) {
err = crypto_register_skcipher(
&riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
if (err)
- goto unregister_zvkned_zvkb_skcipher_alg;
+ return err;
}
}
return err;
-
-unregister_zvkned_zvkb_skcipher_alg:
- if (riscv_isa_extension_available(NULL, ZVKB))
- crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg);
- return err;
}
static void __exit riscv64_aes_mod_exit(void)
{
- if (riscv64_aes_xts_supported())
- crypto_unregister_skcipher(&riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
- if (riscv_isa_extension_available(NULL, ZVKB))
- crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg);
+ crypto_unregister_skcipher(&riscv64_zvkned_zvbb_zvkg_aes_skcipher_alg);
}
module_init(riscv64_aes_mod_init);
module_exit(riscv64_aes_mod_exit);
-MODULE_DESCRIPTION("AES-CTR/XTS (RISC-V accelerated)");
+MODULE_DESCRIPTION("AES-XTS (RISC-V accelerated)");
MODULE_AUTHOR("Jerry Shih <jerry.shih@xxxxxxxxxx>");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("aes");
-MODULE_ALIAS_CRYPTO("ctr(aes)");
MODULE_ALIAS_CRYPTO("xts(aes)");
diff --git a/crypto/aes.c b/crypto/aes.c
index e951f0e1fe5a..9990e5034d34 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -670,7 +670,8 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ctr(aes)",
.base.cra_driver_name = "ctr-aes-lib",
- .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
+ .base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_X86)) ? 300 : 110,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct aes_enckey),
.base.cra_module = THIS_MODULE,
@@ -1002,7 +1003,6 @@ static struct aead_alg aead_algs[] = {
!(IS_ENABLED(CONFIG_ARM) || \
IS_ENABLED(CONFIG_ARM64) || \
IS_ENABLED(CONFIG_POWERPC) || \
- IS_ENABLED(CONFIG_RISCV) || \
IS_ENABLED(CONFIG_S390) || \
IS_ENABLED(CONFIG_SPARC))
{
@@ -1045,7 +1045,6 @@ static struct aead_alg aead_algs[] = {
!(IS_ENABLED(CONFIG_ARM) || \
IS_ENABLED(CONFIG_ARM64) || \
IS_ENABLED(CONFIG_POWERPC) || \
- IS_ENABLED(CONFIG_RISCV) || \
IS_ENABLED(CONFIG_S390) || \
IS_ENABLED(CONFIG_SPARC))
{
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 02d89a226377..ff34aeda37ba 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -51,6 +51,10 @@ endif # !CONFIG_SPE
endif # CONFIG_PPC
libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned.o
+ifneq ($(CONFIG_CRYPTO_LIB_AES_CTR),)
+libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned-zvkb.o
+endif
+
libaes-$(CONFIG_SPARC) += sparc/aes_asm.o
libaes-$(CONFIG_X86) += x86/aes-aesni.o
diff --git a/arch/riscv/crypto/aes-riscv64-zvkned-zvkb.S b/lib/crypto/riscv/aes-riscv64-zvkned-zvkb.S
similarity index 93%
rename from arch/riscv/crypto/aes-riscv64-zvkned-zvkb.S
rename to lib/crypto/riscv/aes-riscv64-zvkned-zvkb.S
index 9962d4500587..93747d4cb5f3 100644
--- a/arch/riscv/crypto/aes-riscv64-zvkned-zvkb.S
+++ b/lib/crypto/riscv/aes-riscv64-zvkned-zvkb.S
@@ -49,11 +49,11 @@
#include "aes-macros.S"
-#define KEYP a0
-#define INP a1
-#define OUTP a2
-#define LEN a3
-#define IVP a4
+#define DST a0
+#define SRC a1
+#define LEN a2
+#define IVP a3
+#define KEYP a4
#define LEN32 a5
#define VL_E32 a6
@@ -110,13 +110,13 @@
// XOR the data with the keystream.
vsetvli t0, LEN, e8, m4, ta, ma
- vle8.v v20, (INP)
+ vle8.v v20, (SRC)
vxor.vv v20, v20, v24
- vse8.v v20, (OUTP)
+ vse8.v v20, (DST)
// Advance the pointers and update the remaining length.
- add INP, INP, t0
- add OUTP, OUTP, t0
+ add SRC, SRC, t0
+ add DST, DST, t0
sub LEN, LEN, t0
sub LEN32, LEN32, VL_E32
srli VL_BLOCKS, VL_E32, 2
@@ -133,9 +133,8 @@
ret
.endm
-// void aes_ctr32_crypt_zvkned_zvkb(const struct crypto_aes_ctx *key,
-// const u8 *in, u8 *out, size_t len,
-// u8 iv[16]);
+// void aes_ctr32_crypt_zvkned_zvkb(u8 *dst, const u8 *src, u32 len, u8 iv[16],
+// const struct aes_enckey *key);
SYM_FUNC_START(aes_ctr32_crypt_zvkned_zvkb)
aes_begin KEYP, 128f, 192f
aes_ctr32_crypt 256
diff --git a/lib/crypto/riscv/aes.h b/lib/crypto/riscv/aes.h
index e02f9343d67d..2c4d1e58c703 100644
--- a/lib/crypto/riscv/aes.h
+++ b/lib/crypto/riscv/aes.h
@@ -9,6 +9,7 @@
#include <asm/vector.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned_zvkb);
/* The assembly code assumes the following offsets. */
static_assert(offsetof(struct aes_enckey, len) == 0);
@@ -159,10 +160,76 @@ static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
}
#endif /* CONFIG_CRYPTO_LIB_AES_CBC */
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
+void aes_ctr32_crypt_zvkned_zvkb(u8 *dst, const u8 *src, u32 len,
+ u8 iv[16], const struct aes_enckey *key);
+
+static void aes_ctr_riscv(u8 *dst, const u8 *src, u32 len,
+ u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key)
+{
+ u32 ctr32 = get_unaligned_be32(&ctr[12]);
+ u32 part1_len;
+ u32 nblocks;
+
+ nblocks = DIV_ROUND_UP(len, AES_BLOCK_SIZE);
+ ctr32 += nblocks;
+
+ if (likely(ctr32 >= nblocks)) {
+ /* The low 32 bits of the counter won't overflow. */
+ aes_ctr32_crypt_zvkned_zvkb(dst, src, len, ctr, key);
+ } else {
+ /*
+ * The low 32 bits of the counter will overflow. The
+ * assembly doesn't handle this case, so split the
+ * operation into two at the point where the overflow
+ * will occur. After the first part, add the carry bit.
+ */
+ part1_len = min(len, (nblocks - ctr32) * AES_BLOCK_SIZE);
+ aes_ctr32_crypt_zvkned_zvkb(dst, src, part1_len, ctr, key);
+ for (int i = AES_BLOCK_SIZE - 5; i >= 0; i--) {
+ if (++ctr[i] != 0)
+ break;
+ }
+ if (part1_len < len)
+ aes_ctr32_crypt_zvkned_zvkb(dst + part1_len,
+ src + part1_len,
+ len - part1_len, ctr, key);
+ }
+}
+
+#define aes_ctr_arch aes_ctr_arch
+static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
+ u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key)
+{
+ if (!static_branch_likely(&have_zvkned_zvkb) ||
+ unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ while (len) {
+ /*
+ * Process at most a 32-bit len at a time, so that each step
+ * needs to handle at most 1 carry bit out of the low 32-bit
+ * word of the counter.
+ */
+ u32 n = min(len, round_down(U32_MAX, AES_BLOCK_SIZE));
+
+ aes_ctr_riscv(dst, src, n, ctr, key);
+ dst += n;
+ src += n;
+ len -= n;
+ }
+ kernel_vector_end();
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CTR */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
if (riscv_isa_extension_available(NULL, ZVKNED) &&
- riscv_vector_vlen() >= 128)
+ riscv_vector_vlen() >= 128) {
static_branch_enable(&have_zvkned);
+ if (riscv_isa_extension_available(NULL, ZVKB))
+ static_branch_enable(&have_zvkned_zvkb);
+ }
}
--
2.55.0