Re: [PATCH v4 11/11] media: rkvdec: h264: Support High 10 and 4:2:2 profiles

From: Jonas Karlman
Date: Thu Nov 09 2023 - 13:25:24 EST

Next message: Boqun Feng: "Re: Fw: [isocpp-parallel] OOTA fix (via fake branch-after-load) discussion"
Previous message: Konstantin Khorenko: "[PATCH 0/1] KVM: x86/vPMU: Speed up vmexit for AMD Zen 4 CPUs"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

On 2023-11-08 03:20, Nicolas Dufresne wrote:
> Le dimanche 05 novembre 2023 à 16:55 +0000, Jonas Karlman a écrit :
>> Add support and enable decoding of H264 High 10 and 4:2:2 profiles.
>>
>> Decoded CAPTURE buffer width is aligned to 64 pixels to accommodate HW
>> requirement of 10-bit format buffers, fixes decoding of:
>>
>> - Hi422FR13_SONY_A
>> - Hi422FR14_SONY_A
>> - Hi422FR15_SONY_A
>> - Hi422FR6_SONY_A
>> - Hi422FR7_SONY_A
>> - Hi422FR8_SONY_A
>> - Hi422FR9_SONY_A
>> - Hi422FREXT18_SONY_A
>>
>> The get_image_fmt() ops is implemented to select an image format
>> required for the provided SPS control.
>>
>> Signed-off-by: Jonas Karlman <jonas@xxxxxxxxx>
>> ---
>> v4:
>> - Change to use get_image_fmt() ops
>>
>> v3:
>> - Add get_fmt_opaque ops, the expected pixelformat is used as opaque
>> - Add new valid_fmt ops that validate pixelformat matches opaque
>> - Update H264_PROFILE control max value
>>
>> drivers/staging/media/rkvdec/rkvdec-h264.c | 37 ++++++++++++++++------
>> drivers/staging/media/rkvdec/rkvdec.c | 33 +++++++++++++++----
>> drivers/staging/media/rkvdec/rkvdec.h | 3 ++
>> 3 files changed, 57 insertions(+), 16 deletions(-)
>>
>> diff --git a/drivers/staging/media/rkvdec/rkvdec-h264.c b/drivers/staging/media/rkvdec/rkvdec-h264.c
>> index 815d5359ddd5..baac6d012ddd 100644
>> --- a/drivers/staging/media/rkvdec/rkvdec-h264.c
>> +++ b/drivers/staging/media/rkvdec/rkvdec-h264.c
>> @@ -1027,24 +1027,42 @@ static int rkvdec_h264_adjust_fmt(struct rkvdec_ctx *ctx,
>> return 0;
>> }
>>
>> +static enum rkvdec_image_fmt rkvdec_h264_get_image_fmt(struct rkvdec_ctx *ctx,
>> + struct v4l2_ctrl *ctrl)
>> +{
>> + const struct v4l2_ctrl_h264_sps *sps = ctrl->p_new.p_h264_sps;
>> +
>> + if (ctrl->id != V4L2_CID_STATELESS_H264_SPS)
>> + return RKVDEC_IMG_FMT_ANY;
>> +
>> + if (sps->bit_depth_luma_minus8 == 0) {
>> + if (sps->chroma_format_idc == 2)
>> + return RKVDEC_IMG_FMT_422_8BIT;
>> + else
>> + return RKVDEC_IMG_FMT_420_8BIT;
>> + } else if (sps->bit_depth_luma_minus8 == 2) {
>> + if (sps->chroma_format_idc == 2)
>> + return RKVDEC_IMG_FMT_422_10BIT;
>> + else
>> + return RKVDEC_IMG_FMT_420_10BIT;
>> + }
>> +
>> + return RKVDEC_IMG_FMT_ANY;
>> +}
>> +
>> static int rkvdec_h264_validate_sps(struct rkvdec_ctx *ctx,
>> const struct v4l2_ctrl_h264_sps *sps)
>> {
>> unsigned int width, height;
>>
>> - /*
>> - * TODO: The hardware supports 10-bit and 4:2:2 profiles,
>> - * but it's currently broken in the driver.
>> - * Reject them for now, until it's fixed.
>> - */
>> - if (sps->chroma_format_idc > 1)
>> - /* Only 4:0:0 and 4:2:0 are supported */
>> + if (sps->chroma_format_idc > 2)
>> + /* Only 4:0:0, 4:2:0 and 4:2:2 are supported */
>> return -EINVAL;
>> if (sps->bit_depth_luma_minus8 != sps->bit_depth_chroma_minus8)
>> /* Luma and chroma bit depth mismatch */
>> return -EINVAL;
>> - if (sps->bit_depth_luma_minus8 != 0)
>> - /* Only 8-bit is supported */
>> + if (sps->bit_depth_luma_minus8 != 0 && sps->bit_depth_luma_minus8 != 2)
>> + /* Only 8-bit and 10-bit is supported */
>> return -EINVAL;
>>
>> width = (sps->pic_width_in_mbs_minus1 + 1) * 16;
>> @@ -1175,4 +1193,5 @@ const struct rkvdec_coded_fmt_ops rkvdec_h264_fmt_ops = {
>> .stop = rkvdec_h264_stop,
>> .run = rkvdec_h264_run,
>> .try_ctrl = rkvdec_h264_try_ctrl,
>> + .get_image_fmt = rkvdec_h264_get_image_fmt,
>> };
>> diff --git a/drivers/staging/media/rkvdec/rkvdec.c b/drivers/staging/media/rkvdec/rkvdec.c
>> index 225aa1f0ac48..eb59605ccf28 100644
>> --- a/drivers/staging/media/rkvdec/rkvdec.c
>> +++ b/drivers/staging/media/rkvdec/rkvdec.c
>> @@ -73,7 +73,7 @@ static void rkvdec_fill_decoded_pixfmt(struct rkvdec_ctx *ctx,
>> struct v4l2_pix_format_mplane *pix_mp)
>> {
>> v4l2_fill_pixfmt_mp(pix_mp, pix_mp->pixelformat,
>> - pix_mp->width, pix_mp->height);
>> + ALIGN(pix_mp->width, 64), pix_mp->height);
>
> If you align regardless if its 8/10bit (which I don't really mind, it
> does not cost much and helps if you have a Mali GPU), please do in
> rkvdec_coded_fmts[].frmsize.step_width.
>
> Otherwise you'll endup creating a config per bit depth, and probably
> won't need any of the rk format stuff, since you could just update the
> config, and enumerate from there. I don't mind your method though, but
> lets not hardcode alignment where it shouldn't be.

Agree, will change to use step_width/min_width = 64 in v5, not sure why
I changed from using step_width in v2 to begin with:

"
Changes in v2:
- Only align decoded buffer instead of using frmsize step_width
"

https://lore.kernel.org/linux-media/20200706215430.22859-12-jonas@xxxxxxxxx/

>
>> pix_mp->plane_fmt[0].sizeimage += 128 *
>> DIV_ROUND_UP(pix_mp->width, 16) *
>> DIV_ROUND_UP(pix_mp->height, 16);
>> @@ -193,7 +193,7 @@ static const struct rkvdec_ctrl_desc rkvdec_h264_ctrl_descs[] = {
>> {
>> .cfg.id = V4L2_CID_MPEG_VIDEO_H264_PROFILE,
>> .cfg.min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
>
> Do we want to keep this small lie ? Baseline is not supported as we
> don't support FMO and ASO. That being said, in framework like
> GStreamer, we try to decode anyway, cause we know we don't have a
> software fallback anyway.

Found a note to change this to H264_PROFILE_CONSTRAINED_BASELINE from
the old v2 series. Will include such change in v5.

https://patchwork.linuxtv.org/project/linux-media/patch/20200706215430.22859-13-jonas@xxxxxxxxx/#119406

>
>> - .cfg.max = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH,
>> + .cfg.max = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_422,
>
> Should include V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_10_INTRA and
> V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_422_INTRA too ....
>
>> .cfg.menu_skip_mask =
>> BIT(V4L2_MPEG_VIDEO_H264_PROFILE_EXTENDED),
>
> Which requires substracting
> V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_444_PREDICTIVE too.

Will update in v5, thanks.

Regards,
Jonas

>
>> .cfg.def = V4L2_MPEG_VIDEO_H264_PROFILE_MAIN,
>> @@ -210,11 +210,23 @@ static const struct rkvdec_ctrls rkvdec_h264_ctrls = {
>> .num_ctrls = ARRAY_SIZE(rkvdec_h264_ctrl_descs),
>> };
>>
>> -static const struct rkvdec_decoded_fmt_desc rkvdec_h264_vp9_decoded_fmts[] = {
>> +static const struct rkvdec_decoded_fmt_desc rkvdec_h264_decoded_fmts[] = {
>> {
>> .fourcc = V4L2_PIX_FMT_NV12,
>> .image_fmt = RKVDEC_IMG_FMT_420_8BIT,
>> },
>> + {
>> + .fourcc = V4L2_PIX_FMT_NV15,
>> + .image_fmt = RKVDEC_IMG_FMT_420_10BIT,
>> + },
>> + {
>> + .fourcc = V4L2_PIX_FMT_NV16,
>> + .image_fmt = RKVDEC_IMG_FMT_422_8BIT,
>> + },
>> + {
>> + .fourcc = V4L2_PIX_FMT_NV20,
>> + .image_fmt = RKVDEC_IMG_FMT_422_10BIT,
>> + },
>> };
>>
>> static const struct rkvdec_ctrl_desc rkvdec_vp9_ctrl_descs[] = {
>> @@ -237,6 +249,13 @@ static const struct rkvdec_ctrls rkvdec_vp9_ctrls = {
>> .num_ctrls = ARRAY_SIZE(rkvdec_vp9_ctrl_descs),
>> };
>>
>> +static const struct rkvdec_decoded_fmt_desc rkvdec_vp9_decoded_fmts[] = {
>> + {
>> + .fourcc = V4L2_PIX_FMT_NV12,
>> + .image_fmt = RKVDEC_IMG_FMT_420_8BIT,
>> + },
>> +};
>> +
>> static const struct rkvdec_coded_fmt_desc rkvdec_coded_fmts[] = {
>> {
>> .fourcc = V4L2_PIX_FMT_H264_SLICE,
>> @@ -250,8 +269,8 @@ static const struct rkvdec_coded_fmt_desc rkvdec_coded_fmts[] = {
>> },
>> .ctrls = &rkvdec_h264_ctrls,
>> .ops = &rkvdec_h264_fmt_ops,
>> - .num_decoded_fmts = ARRAY_SIZE(rkvdec_h264_vp9_decoded_fmts),
>> - .decoded_fmts = rkvdec_h264_vp9_decoded_fmts,
>> + .num_decoded_fmts = ARRAY_SIZE(rkvdec_h264_decoded_fmts),
>> + .decoded_fmts = rkvdec_h264_decoded_fmts,
>> .subsystem_flags = VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF,
>> },
>> {
>> @@ -266,8 +285,8 @@ static const struct rkvdec_coded_fmt_desc rkvdec_coded_fmts[] = {
>> },
>> .ctrls = &rkvdec_vp9_ctrls,
>> .ops = &rkvdec_vp9_fmt_ops,
>> - .num_decoded_fmts = ARRAY_SIZE(rkvdec_h264_vp9_decoded_fmts),
>> - .decoded_fmts = rkvdec_h264_vp9_decoded_fmts,
>> + .num_decoded_fmts = ARRAY_SIZE(rkvdec_vp9_decoded_fmts),
>> + .decoded_fmts = rkvdec_vp9_decoded_fmts,
>> }
>> };
>>
>> diff --git a/drivers/staging/media/rkvdec/rkvdec.h b/drivers/staging/media/rkvdec/rkvdec.h
>> index e466a2753ccf..9a9f4fced7a1 100644
>> --- a/drivers/staging/media/rkvdec/rkvdec.h
>> +++ b/drivers/staging/media/rkvdec/rkvdec.h
>> @@ -80,6 +80,9 @@ struct rkvdec_coded_fmt_ops {
>> enum rkvdec_image_fmt {
>> RKVDEC_IMG_FMT_ANY = 0,
>> RKVDEC_IMG_FMT_420_8BIT,
>> + RKVDEC_IMG_FMT_420_10BIT,
>> + RKVDEC_IMG_FMT_422_8BIT,
>> + RKVDEC_IMG_FMT_422_10BIT,
>> };
>>
>> struct rkvdec_decoded_fmt_desc {
>

Next message: Boqun Feng: "Re: Fw: [isocpp-parallel] OOTA fix (via fake branch-after-load) discussion"
Previous message: Konstantin Khorenko: "[PATCH 0/1] KVM: x86/vPMU: Speed up vmexit for AMD Zen 4 CPUs"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]