[RFC PATCH 07/28] media: mali-vpu: implement TRY_FMT and S_FMT ioctls

From: Charvi Mehta

Date: Thu Nov 19 2026 - 10:42:44 EST


Implement VIDIOC_TRY_FMT and VIDIOC_S_FMT for the decoder OUTPUT and
CAPTURE queues.

Move format lookup and buffer layout calculation into shared helpers so
session initialization and format negotiation use the same sizing rules.

Signed-off-by: Charvi Mehta <charvi.mehta@xxxxxxx>
---
drivers/media/platform/arm/mali-vpu/Makefile | 3 +-
.../platform/arm/mali-vpu/mali-vpu-formats.c | 81 +++++++++++
.../platform/arm/mali-vpu/mali-vpu-formats.h | 10 ++
.../platform/arm/mali-vpu/mali-vpu-session.c | 24 +---
.../platform/arm/mali-vpu/mali-vpu-vidioc.c | 132 +++++++++++++++---
5 files changed, 209 insertions(+), 41 deletions(-)
create mode 100644 drivers/media/platform/arm/mali-vpu/mali-vpu-formats.c

diff --git a/drivers/media/platform/arm/mali-vpu/Makefile b/drivers/media/platform/arm/mali-vpu/Makefile
index 94ddde02fc528..1dc62621d9dc0 100644
--- a/drivers/media/platform/arm/mali-vpu/Makefile
+++ b/drivers/media/platform/arm/mali-vpu/Makefile
@@ -6,6 +6,7 @@ mali-vpu-objs := mali-vpu-probe.o \
mali-vpu-session.o \
mali-vpu-dec.o \
mali-vpu-buffer.o \
- mali-vpu-vb2.o
+ mali-vpu-vb2.o \
+ mali-vpu-formats.o

obj-$(CONFIG_VIDEO_MALI_VPU) += mali-vpu.o
diff --git a/drivers/media/platform/arm/mali-vpu/mali-vpu-formats.c b/drivers/media/platform/arm/mali-vpu/mali-vpu-formats.c
new file mode 100644
index 0000000000000..d268435432371
--- /dev/null
+++ b/drivers/media/platform/arm/mali-vpu/mali-vpu-formats.c
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 ARM Limited.
+ */
+
+#include <linux/errno.h>
+#include <linux/minmax.h>
+#include <linux/mm.h>
+#include <linux/overflow.h>
+#include <linux/string.h>
+#include <media/v4l2-common.h>
+
+#include "mali-vpu-formats.h"
+#include "mali-vpu-session.h"
+
+const struct mali_vpu_format_desc *
+mali_vpu_find_format(struct mali_vpu_session *sess, u32 pixfmt,
+ enum v4l2_buf_type type)
+{
+ const struct mali_vpu_format_list *fmt_list;
+ u32 i;
+
+ switch (type) {
+ case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+ fmt_list = &sess->port_src.supported_fmts;
+ break;
+ case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
+ fmt_list = &sess->port_dst.supported_fmts;
+ break;
+ default:
+ return NULL;
+ }
+
+ for (i = 0; i < fmt_list->num_fmts; i++) {
+ if (fmt_list->fmts[i].fourcc == pixfmt)
+ return &fmt_list->fmts[i];
+ }
+
+ return NULL;
+}
+
+int mali_vpu_fill_pixmp_fmt(const struct mali_vpu_format_desc *fmt,
+ u32 width, u32 height,
+ struct v4l2_pix_format_mplane *pix_mp)
+{
+ u32 sizeimage;
+ int ret;
+
+ memset(pix_mp->reserved, 0, sizeof(pix_mp->reserved));
+ memset(pix_mp->plane_fmt, 0, sizeof(pix_mp->plane_fmt));
+ pix_mp->flags = 0;
+
+ if (fmt->flags & V4L2_FMT_FLAG_COMPRESSED) {
+ pix_mp->width = width;
+ pix_mp->height = height;
+ pix_mp->pixelformat = fmt->fourcc;
+ pix_mp->num_planes = 1;
+ pix_mp->plane_fmt[0].bytesperline = 0;
+
+ /*
+ * Use one byte per pixel as the default compressed sizeimage,
+ * with a minimum of one page, aligned to a page boundary.
+ */
+ if (check_mul_overflow(width, height, &sizeimage))
+ return -EOVERFLOW;
+
+ sizeimage = max_t(u32, PAGE_SIZE, sizeimage);
+
+ if (sizeimage > U32_MAX - (PAGE_SIZE - 1))
+ return -EOVERFLOW;
+
+ pix_mp->plane_fmt[0].sizeimage = PAGE_ALIGN(sizeimage);
+
+ } else {
+ ret = v4l2_fill_pixfmt_mp(pix_mp, fmt->fourcc, width, height);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
diff --git a/drivers/media/platform/arm/mali-vpu/mali-vpu-formats.h b/drivers/media/platform/arm/mali-vpu/mali-vpu-formats.h
index 13031809fdb4d..5703142b93b46 100644
--- a/drivers/media/platform/arm/mali-vpu/mali-vpu-formats.h
+++ b/drivers/media/platform/arm/mali-vpu/mali-vpu-formats.h
@@ -7,6 +7,9 @@
#define __MALI_VPU_FORMATS_H__

#include <linux/types.h>
+#include <linux/videodev2.h>
+
+struct mali_vpu_session;

enum mali_vpu_codec_fmt {
MALI_VPU_FORMAT_CODEC_H264,
@@ -24,4 +27,11 @@ struct mali_vpu_format_list {
u32 num_fmts;
};

+const struct mali_vpu_format_desc *mali_vpu_find_format(struct mali_vpu_session *sess,
+ u32 pixfmt,
+ enum v4l2_buf_type type);
+int mali_vpu_fill_pixmp_fmt(const struct mali_vpu_format_desc *fmt,
+ u32 width, u32 height,
+ struct v4l2_pix_format_mplane *pix_mp);
+
#endif /* __MALI_VPU_FORMATS_H__ */
diff --git a/drivers/media/platform/arm/mali-vpu/mali-vpu-session.c b/drivers/media/platform/arm/mali-vpu/mali-vpu-session.c
index 95050296f9904..6867f51287f48 100644
--- a/drivers/media/platform/arm/mali-vpu/mali-vpu-session.c
+++ b/drivers/media/platform/arm/mali-vpu/mali-vpu-session.c
@@ -5,14 +5,12 @@

#include <linux/err.h>
#include <linux/errno.h>
-#include <linux/minmax.h>
-#include <linux/mm.h>
#include <linux/string.h>
-#include <media/v4l2-common.h>
#include <media/v4l2-mem2mem.h>

#include "mali-vpu-session.h"
#include "mali-vpu-dec.h"
+#include "mali-vpu-formats.h"
#include "mali-vpu-variant.h"
#include "mali-vpu-vb2.h"

@@ -32,23 +30,9 @@ static int mali_vpu_set_port_fmt(struct mali_vpu_port *port,
memset(&port->curr_fmt, 0, sizeof(port->curr_fmt));
port->curr_fmt.type = type;

- if (fmt_default->flags & V4L2_FMT_FLAG_COMPRESSED) {
- pix_mp->pixelformat = fmt_default->fourcc;
- pix_mp->num_planes = 1;
- pix_mp->width = width;
- pix_mp->height = height;
- pix_mp->plane_fmt[0].bytesperline = 0;
-
- /*
- * Use one byte per pixel as the default compressed sizeimage,
- * with a minimum of one page, aligned to a page boundary.
- */
- pix_mp->plane_fmt[0].sizeimage = PAGE_ALIGN(max_t(u32, PAGE_SIZE, width * height));
- } else {
- ret = v4l2_fill_pixfmt_mp(pix_mp, fmt_default->fourcc, width, height);
- if (ret)
- return ret;
- }
+ ret = mali_vpu_fill_pixmp_fmt(fmt_default, width, height, pix_mp);
+ if (ret)
+ return ret;

pix_mp->field = V4L2_FIELD_NONE;

diff --git a/drivers/media/platform/arm/mali-vpu/mali-vpu-vidioc.c b/drivers/media/platform/arm/mali-vpu/mali-vpu-vidioc.c
index b993255b8c817..dd142527a1a0c 100644
--- a/drivers/media/platform/arm/mali-vpu/mali-vpu-vidioc.c
+++ b/drivers/media/platform/arm/mali-vpu/mali-vpu-vidioc.c
@@ -7,6 +7,7 @@
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/string.h>
+#include <media/v4l2-common.h>
#include <media/v4l2-dev.h>
#include <media/v4l2-fh.h>
#include <media/v4l2-ioctl.h>
@@ -168,32 +169,119 @@ static int mali_vpu_g_fmt(struct file *filp, void *fh, struct v4l2_format *fmt)
return ret;
}

-static const struct mali_vpu_format_desc *mali_vpu_find_format(struct mali_vpu_session *sess,
- u32 pixfmt, u32 type)
+static int mali_vpu_try_format_common(struct mali_vpu_session *sess, struct v4l2_format *fmt)
{
- const struct mali_vpu_format_desc *fmt_list;
- u32 num_fmts;
- u32 i;
+ const struct mali_vpu_framesize_caps *fmsz = &sess->vpu_dev->variant->frmsz_caps;
+ struct v4l2_pix_format_mplane *pix_mp = &fmt->fmt.pix_mp;
+ const struct mali_vpu_format_desc *vpu_fmt;
+ const struct v4l2_format *port_fmt, *port_fmt_dec;
+ struct vb2_queue *src_q;
+ u32 w, h;
+ int ret = 0;

- switch (type) {
- case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
- fmt_list = sess->port_src.supported_fmts.fmts;
- num_fmts = sess->port_src.supported_fmts.num_fmts;
- break;
- case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
- fmt_list = sess->port_dst.supported_fmts.fmts;
- num_fmts = sess->port_dst.supported_fmts.num_fmts;
- break;
- default:
- return NULL;
+ vpu_fmt = mali_vpu_find_format(sess, pix_mp->pixelformat, fmt->type);
+ if (!vpu_fmt) {
+ port_fmt = mali_vpu_get_port_fmt(sess, fmt->type);
+ if (!port_fmt)
+ return -EINVAL;
+
+ *pix_mp = port_fmt->fmt.pix_mp;
+ return 0;
+ }
+
+ /* Requested format found, validate other fields */
+ w = pix_mp->width ? pix_mp->width : fmsz->width_default;
+ h = pix_mp->height ? pix_mp->height : fmsz->height_default;
+
+ v4l2_apply_frmsize_constraints(&w, &h, &fmsz->stepwise);
+
+ /* Decode CAPTURE queue: lock geometry to input if streaming */
+ if (sess->role == MALI_VPU_DECODER && fmt->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
+ src_q = v4l2_m2m_get_src_vq(sess->m2m_ctx);
+ if (vb2_is_streaming(src_q)) {
+ port_fmt_dec = &sess->port_src.curr_fmt;
+ w = port_fmt_dec->fmt.pix_mp.width;
+ h = port_fmt_dec->fmt.pix_mp.height;
+ }
+ }
+
+ ret = mali_vpu_fill_pixmp_fmt(vpu_fmt, w, h, pix_mp);
+ if (ret)
+ return ret;
+
+ if (pix_mp->field == V4L2_FIELD_ANY)
+ pix_mp->field = V4L2_FIELD_NONE;
+
+ return 0;
+}
+
+static int mali_vpu_try_fmt(struct file *filp, void *fh, struct v4l2_format *fmt)
+{
+ struct mali_vpu_session *sess = mali_vpu_get_session(filp);
+ int ret;
+
+ mutex_lock(&sess->vq_lock);
+
+ ret = mali_vpu_try_format_common(sess, fmt);
+
+ mutex_unlock(&sess->vq_lock);
+ return ret;
+}
+
+static int mali_vpu_s_fmt(struct file *filp, void *fh, struct v4l2_format *fmt)
+{
+ struct mali_vpu_session *sess = mali_vpu_get_session(filp);
+ struct vb2_queue *vq;
+ struct v4l2_pix_format_mplane *pixmp_src = &sess->port_src.curr_fmt.fmt.pix_mp;
+ struct v4l2_pix_format_mplane *pixmp_dst = &sess->port_dst.curr_fmt.fmt.pix_mp;
+ int ret = 0;
+
+ mutex_lock(&sess->vq_lock);
+
+ vq = v4l2_m2m_get_vq(sess->m2m_ctx, fmt->type);
+ if (vb2_is_busy(vq)) {
+ ret = -EBUSY;
+ goto mutex_unlock_return;
}

- for (i = 0; i < num_fmts; i++) {
- if (fmt_list[i].fourcc == pixfmt)
- return &fmt_list[i];
+ ret = mali_vpu_try_format_common(sess, fmt);
+ if (ret)
+ goto mutex_unlock_return;
+
+ /* OUTPUT provides initial CAPTURE geometry */
+ if (V4L2_TYPE_IS_OUTPUT(fmt->type)) {
+ const struct mali_vpu_format_desc *cap_fmt;
+ struct v4l2_pix_format_mplane *cap;
+ struct v4l2_pix_format_mplane *fmt_mp = &fmt->fmt.pix_mp;
+
+ cap = &sess->port_dst.curr_fmt.fmt.pix_mp;
+ cap_fmt = mali_vpu_find_format(sess, cap->pixelformat,
+ V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
+ if (!cap_fmt) {
+ ret = -EINVAL;
+ goto mutex_unlock_return;
+ }
+
+ ret = mali_vpu_fill_pixmp_fmt(cap_fmt, fmt_mp->width, fmt_mp->height, cap);
+ if (ret)
+ goto mutex_unlock_return;
}

- return NULL;
+ if (V4L2_TYPE_IS_OUTPUT(fmt->type))
+ sess->port_src.curr_fmt = *fmt;
+ else
+ sess->port_dst.curr_fmt = *fmt;
+
+ /* Keep peer queue colorimetry aligned with input */
+ if (V4L2_TYPE_IS_OUTPUT(fmt->type)) {
+ pixmp_dst->colorspace = pixmp_src->colorspace;
+ pixmp_dst->ycbcr_enc = pixmp_src->ycbcr_enc;
+ pixmp_dst->quantization = pixmp_src->quantization;
+ pixmp_dst->xfer_func = pixmp_src->xfer_func;
+ }
+mutex_unlock_return:
+ mutex_unlock(&sess->vq_lock);
+ return ret;
}

static int mali_vpu_validate_format(struct mali_vpu_session *sess, u32 pixfmt)
@@ -236,6 +324,10 @@ const struct v4l2_ioctl_ops mali_vpu_ioctl_ops_dec = {
.vidioc_enum_fmt_vid_cap = mali_vpu_enum_fmt,
.vidioc_g_fmt_vid_out_mplane = mali_vpu_g_fmt,
.vidioc_g_fmt_vid_cap_mplane = mali_vpu_g_fmt,
+ .vidioc_s_fmt_vid_out_mplane = mali_vpu_s_fmt,
+ .vidioc_s_fmt_vid_cap_mplane = mali_vpu_s_fmt,
+ .vidioc_try_fmt_vid_out_mplane = mali_vpu_try_fmt,
+ .vidioc_try_fmt_vid_cap_mplane = mali_vpu_try_fmt,
.vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs,
.vidioc_querybuf = v4l2_m2m_ioctl_querybuf,
.vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs,
--
2.43.0