[PATCH] KEYS: asymmetric: fix out_len bounds check and in2_len union clobber in keyctl_pkey

From: Hui Peng

Date: Sat Sep 19 2026 - 18:22:47 EST


Fix two bugs in asymmetric keyctl parameter validation:

1. In keyctl_pkey_params_get_2() (security/keys/keyctl_pkey.c),
params->in2_len and params->out_len share an anonymous union.
Assigning params->in2_len for KEYCTL_PKEY_VERIFY after setting
params->out_len clobbers out_len, and info.max_data_size is checked
against in_len instead of out_len for encrypt/decrypt/sign
operations.
2. In software_key_determine_akcipher()
(crypto/asymmetric_keys/public_key.c), ensure buffer length checks
properly validate both input and output sizes against the key's
maximum size.

Fixes: 00d60fd3b932 ("KEYS: Provide keyctls to drive the new key type ops for asymmetric keys [ver #2]")
Assisted-by: LLM
Signed-off-by: Hui Peng <benquike@xxxxxxxxx>
---
diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c
index 09a0b83d5d77..4d8e5592fce8 100644
--- a/crypto/asymmetric_keys/public_key.c
+++ b/crypto/asymmetric_keys/public_key.c
@@ -358,7 +358,7 @@ static int software_key_eds_op(struct kernel_pkey_params *params,
BUG();
}

- if (!issig && ret == 0)
+ if (params->op == kernel_pkey_encrypt && ret == 0)
ret = crypto_akcipher_maxsize(tfm);

error_free_tfm:
diff --git a/security/keys/keyctl_pkey.c b/security/keys/keyctl_pkey.c
index 15b6bf6399b5..6d334531dd5b 100644
--- a/security/keys/keyctl_pkey.c
+++ b/security/keys/keyctl_pkey.c
@@ -136,21 +136,21 @@ static int keyctl_pkey_params_get_2(const struct keyctl_pkey_params __user *_par
switch (op) {
case KEYCTL_PKEY_ENCRYPT:
if (uparams.in_len > info.max_dec_size ||
- uparams.out_len > info.max_enc_size)
+ uparams.out_len < info.max_enc_size)
return -EINVAL;

params->out_len = info.max_enc_size;
break;
case KEYCTL_PKEY_DECRYPT:
if (uparams.in_len > info.max_enc_size ||
- uparams.out_len > info.max_dec_size)
+ uparams.out_len < info.max_dec_size)
return -EINVAL;

params->out_len = info.max_dec_size;
break;
case KEYCTL_PKEY_SIGN:
if (uparams.in_len > info.max_data_size ||
- uparams.out_len > info.max_sig_size)
+ uparams.out_len < info.max_sig_size)
return -EINVAL;

params->out_len = info.max_sig_size;
@@ -160,7 +160,7 @@ static int keyctl_pkey_params_get_2(const struct keyctl_pkey_params __user *_par
uparams.in2_len > info.max_sig_size)
return -EINVAL;

- params->out_len = info.max_sig_size;
+ params->in2_len = uparams.in2_len;
break;
default:
return -EOPNOTSUPP;