[PATCH 3/3] media: venus: report the frame size of the codec being enumerated
From: Dmitry Baryshkov
Date: Wed Aug 12 2026 - 16:07:44 EST
VIDIOC_ENUM_FRAMESIZES takes the pixel format to describe from
userspace, but both vdec and venc answer it out of frame_width_min() and
friends, which resolve the capabilities through inst->hfi_codec, the
codec the instance is currently set to. Enumerating any format other
than that one therefore returns the limits of an unrelated codec: on a
freshly opened decoder, which starts out as H.264, asking about MPEG-2
on MSM8996 reports the H.264 limits where the firmware describes MPEG-2
as 16x16 to 1920x1920.
Resolve the capabilities through the format being enumerated instead,
falling back to the codec in use for the raw formats, which have no
codec of their own. Both drivers now share one helper, alongside the
pixel format to codec mapping that venus_helper_check_codec() already
carried.
Assisted-by: Claude:claude-opus-5
Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@xxxxxxxxxxxxxxxx>
---
drivers/media/platform/qcom/venus/helpers.c | 75 +++++++++++++++++---------
drivers/media/platform/qcom/venus/helpers.h | 2 +
drivers/media/platform/qcom/venus/hfi_parser.h | 12 +++--
drivers/media/platform/qcom/venus/vdec.c | 7 +--
drivers/media/platform/qcom/venus/venc.c | 7 +--
5 files changed, 62 insertions(+), 41 deletions(-)
diff --git a/drivers/media/platform/qcom/venus/helpers.c b/drivers/media/platform/qcom/venus/helpers.c
index 59eee3dd9e06..17415d0cdf8e 100644
--- a/drivers/media/platform/qcom/venus/helpers.c
+++ b/drivers/media/platform/qcom/venus/helpers.c
@@ -38,47 +38,43 @@ struct intbuf {
u32 dpb_out_tag;
};
-bool venus_helper_check_codec(struct venus_inst *inst, u32 v4l2_pixfmt)
+static u32 venus_pixfmt_to_hfi_codec(u32 v4l2_pixfmt)
{
- struct venus_core *core = inst->core;
- u32 session_type = inst->session_type;
- u32 codec;
-
switch (v4l2_pixfmt) {
case V4L2_PIX_FMT_H264:
- codec = HFI_VIDEO_CODEC_H264;
- break;
+ return HFI_VIDEO_CODEC_H264;
case V4L2_PIX_FMT_H263:
- codec = HFI_VIDEO_CODEC_H263;
- break;
+ return HFI_VIDEO_CODEC_H263;
case V4L2_PIX_FMT_MPEG1:
- codec = HFI_VIDEO_CODEC_MPEG1;
- break;
+ return HFI_VIDEO_CODEC_MPEG1;
case V4L2_PIX_FMT_MPEG2:
- codec = HFI_VIDEO_CODEC_MPEG2;
- break;
+ return HFI_VIDEO_CODEC_MPEG2;
case V4L2_PIX_FMT_MPEG4:
- codec = HFI_VIDEO_CODEC_MPEG4;
- break;
+ return HFI_VIDEO_CODEC_MPEG4;
case V4L2_PIX_FMT_VC1_ANNEX_G:
case V4L2_PIX_FMT_VC1_ANNEX_L:
- codec = HFI_VIDEO_CODEC_VC1;
- break;
+ return HFI_VIDEO_CODEC_VC1;
case V4L2_PIX_FMT_VP8:
- codec = HFI_VIDEO_CODEC_VP8;
- break;
+ return HFI_VIDEO_CODEC_VP8;
case V4L2_PIX_FMT_VP9:
- codec = HFI_VIDEO_CODEC_VP9;
- break;
+ return HFI_VIDEO_CODEC_VP9;
case V4L2_PIX_FMT_XVID:
- codec = HFI_VIDEO_CODEC_DIVX;
- break;
+ return HFI_VIDEO_CODEC_DIVX;
case V4L2_PIX_FMT_HEVC:
- codec = HFI_VIDEO_CODEC_HEVC;
- break;
+ return HFI_VIDEO_CODEC_HEVC;
default:
- return false;
+ return 0;
}
+}
+
+bool venus_helper_check_codec(struct venus_inst *inst, u32 v4l2_pixfmt)
+{
+ struct venus_core *core = inst->core;
+ u32 session_type = inst->session_type;
+ u32 codec = venus_pixfmt_to_hfi_codec(v4l2_pixfmt);
+
+ if (!codec)
+ return false;
if (session_type == VIDC_SESSION_TYPE_ENC && core->enc_codecs & codec)
return true;
@@ -90,6 +86,33 @@ bool venus_helper_check_codec(struct venus_inst *inst, u32 v4l2_pixfmt)
}
EXPORT_SYMBOL_GPL(venus_helper_check_codec);
+/*
+ * Report the frame size of the codec being enumerated, which is not
+ * necessarily the one the instance is currently set to. A raw format maps to
+ * no codec of its own and is described by the coded format in use.
+ */
+void venus_helper_get_frame_sizes(struct venus_inst *inst, u32 v4l2_pixfmt,
+ struct v4l2_frmsize_stepwise *fsize)
+{
+ struct venus_core *core = inst->core;
+ u32 dom = inst->session_type;
+ u32 codec = venus_pixfmt_to_hfi_codec(v4l2_pixfmt) ?: inst->hfi_codec;
+
+ fsize->min_width = get_codec_cap(core, codec, dom,
+ HFI_CAPABILITY_FRAME_WIDTH, WHICH_CAP_MIN);
+ fsize->max_width = get_codec_cap(core, codec, dom,
+ HFI_CAPABILITY_FRAME_WIDTH, WHICH_CAP_MAX);
+ fsize->step_width = get_codec_cap(core, codec, dom,
+ HFI_CAPABILITY_FRAME_WIDTH, WHICH_CAP_STEP);
+ fsize->min_height = get_codec_cap(core, codec, dom,
+ HFI_CAPABILITY_FRAME_HEIGHT, WHICH_CAP_MIN);
+ fsize->max_height = get_codec_cap(core, codec, dom,
+ HFI_CAPABILITY_FRAME_HEIGHT, WHICH_CAP_MAX);
+ fsize->step_height = get_codec_cap(core, codec, dom,
+ HFI_CAPABILITY_FRAME_HEIGHT, WHICH_CAP_STEP);
+}
+EXPORT_SYMBOL_GPL(venus_helper_get_frame_sizes);
+
static void free_dpb_buf(struct venus_inst *inst, struct intbuf *buf)
{
ida_free(&inst->dpb_ids, buf->dpb_out_tag);
diff --git a/drivers/media/platform/qcom/venus/helpers.h b/drivers/media/platform/qcom/venus/helpers.h
index 358e4f39c9c0..80d623db106c 100644
--- a/drivers/media/platform/qcom/venus/helpers.h
+++ b/drivers/media/platform/qcom/venus/helpers.h
@@ -12,6 +12,8 @@ struct venus_inst;
struct venus_core;
bool venus_helper_check_codec(struct venus_inst *inst, u32 v4l2_pixfmt);
+void venus_helper_get_frame_sizes(struct venus_inst *inst, u32 v4l2_pixfmt,
+ struct v4l2_frmsize_stepwise *fsize);
struct vb2_v4l2_buffer *venus_helper_find_buf(struct venus_inst *inst,
unsigned int type, u32 idx);
void venus_helper_change_dpb_owner(struct venus_inst *inst,
diff --git a/drivers/media/platform/qcom/venus/hfi_parser.h b/drivers/media/platform/qcom/venus/hfi_parser.h
index 5751d0140700..bcd11e7a9c1c 100644
--- a/drivers/media/platform/qcom/venus/hfi_parser.h
+++ b/drivers/media/platform/qcom/venus/hfi_parser.h
@@ -12,14 +12,14 @@ u32 hfi_parser(struct venus_core *core, struct venus_inst *inst,
#define WHICH_CAP_MAX 1
#define WHICH_CAP_STEP 2
-static inline u32 get_cap(struct venus_inst *inst, u32 type, u32 which)
+static inline u32 get_codec_cap(struct venus_core *core, u32 codec, u32 domain,
+ u32 type, u32 which)
{
- struct venus_core *core = inst->core;
struct hfi_capability *cap = NULL;
struct hfi_plat_caps *caps;
unsigned int i;
- caps = venus_caps_by_codec(core, inst->hfi_codec, inst->session_type);
+ caps = venus_caps_by_codec(core, codec, domain);
if (!caps)
return 0;
@@ -47,6 +47,12 @@ static inline u32 get_cap(struct venus_inst *inst, u32 type, u32 which)
return 0;
}
+static inline u32 get_cap(struct venus_inst *inst, u32 type, u32 which)
+{
+ return get_codec_cap(inst->core, inst->hfi_codec, inst->session_type,
+ type, which);
+}
+
static inline u32 cap_min(struct venus_inst *inst, u32 type)
{
return get_cap(inst, type, WHICH_CAP_MIN);
diff --git a/drivers/media/platform/qcom/venus/vdec.c b/drivers/media/platform/qcom/venus/vdec.c
index 6a43ea191da1..0f62931eea4c 100644
--- a/drivers/media/platform/qcom/venus/vdec.c
+++ b/drivers/media/platform/qcom/venus/vdec.c
@@ -512,12 +512,7 @@ static int vdec_enum_framesizes(struct file *file, void *fh,
fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
- fsize->stepwise.min_width = frame_width_min(inst);
- fsize->stepwise.max_width = frame_width_max(inst);
- fsize->stepwise.step_width = frame_width_step(inst);
- fsize->stepwise.min_height = frame_height_min(inst);
- fsize->stepwise.max_height = frame_height_max(inst);
- fsize->stepwise.step_height = frame_height_step(inst);
+ venus_helper_get_frame_sizes(inst, fsize->pixel_format, &fsize->stepwise);
return 0;
}
diff --git a/drivers/media/platform/qcom/venus/venc.c b/drivers/media/platform/qcom/venus/venc.c
index 79acf7c1ec9a..e0f7b9817ccf 100644
--- a/drivers/media/platform/qcom/venus/venc.c
+++ b/drivers/media/platform/qcom/venus/venc.c
@@ -456,12 +456,7 @@ static int venc_enum_framesizes(struct file *file, void *fh,
if (fsize->index)
return -EINVAL;
- fsize->stepwise.min_width = frame_width_min(inst);
- fsize->stepwise.max_width = frame_width_max(inst);
- fsize->stepwise.step_width = frame_width_step(inst);
- fsize->stepwise.min_height = frame_height_min(inst);
- fsize->stepwise.max_height = frame_height_max(inst);
- fsize->stepwise.step_height = frame_height_step(inst);
+ venus_helper_get_frame_sizes(inst, fsize->pixel_format, &fsize->stepwise);
return 0;
}
--
2.47.3