[PATCH v2 02/11] staging: media: cedrus: reject too large H.264/HEVC SPS

From: Jernej Skrabec

Date: Mon Nov 09 2026 - 12:54:26 EST


The H.264 decoder takes the picture size from the SPS, and the HEVC
decoder the position of the slices. An SPS beyond CEDRUS_MAX_WIDTH/HEIGHT,
or beyond the capture buffers, makes the hardware overrun memory and
hang. On H616, the JCT-VC PICSIZE_A..D streams (1056x8440, 8440x1056,
528x4216 and 4216x528) leave the hardware wedged, so that every
following job times out until reboot.

Reject an SPS beyond the hardware limits in try_ctrl, and move the
limits to cedrus_video.h for that.

Whether the picture fits the buffers can only be checked when a job
runs: the SPS can be set before the buffers are allocated, while the
OUTPUT format they are sized from can still change, and a request
doesn't have to carry an SPS at all. Check it in the setup callbacks,
so that such a job fails with an error buffer instead.

Fixes: 6eb9b758e307 ("media: cedrus: Add H264 decoding support")
Fixes: 86caab29da78 ("media: cedrus: Add HEVC/H.265 decoding support")
Signed-off-by: Jernej Skrabec <jernej.skrabec@xxxxxxxxx>
---
drivers/staging/media/sunxi/cedrus/cedrus.c | 16 +++++++++
drivers/staging/media/sunxi/cedrus/cedrus.h | 34 +++++++++++++++++++
.../staging/media/sunxi/cedrus/cedrus_h264.c | 6 ++++
.../staging/media/sunxi/cedrus/cedrus_h265.c | 4 +++
.../staging/media/sunxi/cedrus/cedrus_video.c | 5 ---
.../staging/media/sunxi/cedrus/cedrus_video.h | 5 +++
6 files changed, 65 insertions(+), 5 deletions(-)

diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c
index c3331b4f3f02..197213911bf7 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus.c
@@ -28,6 +28,15 @@
#include "cedrus_dec.h"
#include "cedrus_hw.h"

+/*
+ * Reject picture sizes beyond the hardware limits. Whether the picture fits
+ * the buffers is checked when a job is set up, see cedrus_check_pic_size().
+ */
+static bool cedrus_pic_size_supported(u32 width, u32 height)
+{
+ return width <= CEDRUS_MAX_WIDTH && height <= CEDRUS_MAX_HEIGHT;
+}
+
static int cedrus_try_ctrl(struct v4l2_ctrl *ctrl)
{
if (ctrl->id == V4L2_CID_STATELESS_H264_SPS) {
@@ -42,6 +51,9 @@ static int cedrus_try_ctrl(struct v4l2_ctrl *ctrl)
if (sps->bit_depth_luma_minus8 != 0)
/* Only 8-bit is supported */
return -EINVAL;
+ if (!cedrus_pic_size_supported(cedrus_h264_sps_width(sps),
+ cedrus_h264_sps_height(sps)))
+ return -EINVAL;
} else if (ctrl->id == V4L2_CID_STATELESS_HEVC_SPS) {
const struct v4l2_ctrl_hevc_sps *sps = ctrl->p_new.p_hevc_sps;
struct cedrus_ctx *ctx = container_of(ctrl->handler, struct cedrus_ctx, hdl);
@@ -52,6 +64,10 @@ static int cedrus_try_ctrl(struct v4l2_ctrl *ctrl)
/* Only 4:2:0 is supported */
return -EINVAL;

+ if (!cedrus_pic_size_supported(sps->pic_width_in_luma_samples,
+ sps->pic_height_in_luma_samples))
+ return -EINVAL;
+
bit_depth = max(sps->bit_depth_luma_minus8,
sps->bit_depth_chroma_minus8) + 8;

diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.h b/drivers/staging/media/sunxi/cedrus/cedrus.h
index 54f867949fba..c2318521d7e5 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus.h
+++ b/drivers/staging/media/sunxi/cedrus/cedrus.h
@@ -274,6 +274,40 @@ cedrus_is_capable(struct cedrus_ctx *ctx, unsigned int capabilities)
return (ctx->dev->capabilities & capabilities) == capabilities;
}

+/*
+ * The decoders take the picture size, or the position of the slices, from
+ * the bitstream headers. Check that the picture fits the buffers, which are
+ * sized from the OUTPUT format and padded to at least 16 pixels in both
+ * directions, so that the hardware doesn't overrun them and hang.
+ */
+static inline int cedrus_check_pic_size(struct cedrus_ctx *ctx,
+ unsigned int width, unsigned int height)
+{
+ if (width > ALIGN(ctx->src_fmt.width, 16) ||
+ height > ALIGN(ctx->src_fmt.height, 16))
+ return -EINVAL;
+
+ return 0;
+}
+
+static inline unsigned int
+cedrus_h264_sps_width(const struct v4l2_ctrl_h264_sps *sps)
+{
+ return (sps->pic_width_in_mbs_minus1 + 1) * 16;
+}
+
+static inline unsigned int
+cedrus_h264_sps_height(const struct v4l2_ctrl_h264_sps *sps)
+{
+ unsigned int height = (sps->pic_height_in_map_units_minus1 + 1) * 16;
+
+ /* Map units are macroblock pairs, unless only frames are coded. */
+ if (!(sps->flags & V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY))
+ height *= 2;
+
+ return height;
+}
+
void *cedrus_find_control_data(struct cedrus_ctx *ctx, u32 id);
u32 cedrus_get_num_of_controls(struct cedrus_ctx *ctx, u32 id);

diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
index 892193c12b2f..e8229c60d3f5 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
@@ -518,9 +518,15 @@ static void cedrus_h264_irq_disable(struct cedrus_ctx *ctx)

static int cedrus_h264_setup(struct cedrus_ctx *ctx, struct cedrus_run *run)
{
+ const struct v4l2_ctrl_h264_sps *sps = run->h264.sps;
struct cedrus_dev *dev = ctx->dev;
int ret;

+ ret = cedrus_check_pic_size(ctx, cedrus_h264_sps_width(sps),
+ cedrus_h264_sps_height(sps));
+ if (ret)
+ return ret;
+
cedrus_engine_enable(ctx);

cedrus_write(dev, VE_H264_SDROT_CTRL, 0);
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h265.c b/drivers/staging/media/sunxi/cedrus/cedrus_h265.c
index 6c643862ab73..aa772a8d71c8 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_h265.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_h265.c
@@ -445,6 +445,10 @@ static int cedrus_h265_setup(struct cedrus_ctx *ctx, struct cedrus_run *run)
cedrus_buf = vb2_to_cedrus_buffer(&run->dst->vb2_buf);
slice_bytes = vb2_get_plane_payload(&run->src->vb2_buf, 0);

+ if (cedrus_check_pic_size(ctx, sps->pic_width_in_luma_samples,
+ sps->pic_height_in_luma_samples))
+ return -EINVAL;
+
/*
* If entry points offsets are present, we should get them
* exactly the right amount.
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_video.c b/drivers/staging/media/sunxi/cedrus/cedrus_video.c
index adc7f255d7cf..2521315abdb7 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_video.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_video.c
@@ -29,11 +29,6 @@
#define CEDRUS_DECODE_SRC BIT(0)
#define CEDRUS_DECODE_DST BIT(1)

-#define CEDRUS_MIN_WIDTH 16U
-#define CEDRUS_MIN_HEIGHT 16U
-#define CEDRUS_MAX_WIDTH 4096U
-#define CEDRUS_MAX_HEIGHT 2304U
-
static struct cedrus_format cedrus_formats[] = {
{
.pixelformat = V4L2_PIX_FMT_MPEG2_SLICE,
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_video.h b/drivers/staging/media/sunxi/cedrus/cedrus_video.h
index 8e1afc16a6a1..367e52158f01 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_video.h
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_video.h
@@ -22,6 +22,11 @@ struct cedrus_format {
unsigned int capabilities;
};

+#define CEDRUS_MIN_WIDTH 16U
+#define CEDRUS_MIN_HEIGHT 16U
+#define CEDRUS_MAX_WIDTH 4096U
+#define CEDRUS_MAX_HEIGHT 2304U
+
extern const struct v4l2_ioctl_ops cedrus_ioctl_ops;

int cedrus_queue_init(void *priv, struct vb2_queue *src_vq,
--
2.43.0