Re: [PATCH] drm/etnaviv: Keep busy jobs alive in the hangcheck

[Lucas Stach]

Hi Christian,

Am Montag, dem 02.11.2026 um 12:41 +0100 schrieb Christian Gmeiner:
> From: Christian Gmeiner <cgmeiner@xxxxxxxxxx>
> 
> A compute dispatch consists of a single FE command. The FE DMA address
> does not change while the kernel is running, so the hangcheck kills any
> dispatch that takes longer than about one second. Long-running OpenCL
> kernels never finish.
> 
> Read VIVS_HI_IDLE_STATE at each timeout tick. If any unit other than FE
> and MC is busy, the job is still executing, so give it more time.
> If all units are idle for two consecutive ticks, the FE is waiting for
> something that will never come, and the job is killed as before.
> 
> A job that stays busy without FE progress for longer than the new
> hang_max_ms module parameter (default 30 seconds) is killed anyway.
> This also catches kernels that loop forever. Setting it to 0 restores
> the old behaviour.
> 
This will regress recovery time for a lot of actual GPU hang scenarios.
Most of the real GPU hangs I've seen lead to a stall somewhere in the
engines, which keeps them from going idle, so various busy flags are
set when real GPU hangs occur.

Do we have any way to cover the CL cases in a less invasive way by
checking for some kind of forward progress on the shader core?
Something like a current instruction pointer or retired instruction
performance counter we can read from the hangcheck handler? This will
still not cover the infinite loop case, but it would allow to recover
from actual GPU hangs faster than the 30s timeout.

Regards,
Lucas

> Signed-off-by: Christian Gmeiner <cgmeiner@xxxxxxxxxx>
> ---
>  drivers/gpu/drm/etnaviv/etnaviv_gpu.h   |  3 +++
>  drivers/gpu/drm/etnaviv/etnaviv_sched.c | 39 ++++++++++++++++++++++++++++++++-
>  2 files changed, 41 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> index e89d591d7404..f871ab7899fd 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> @@ -147,6 +147,9 @@ struct etnaviv_gpu {
>  	u32 hangcheck_dma_addr;
>  	u32 hangcheck_primid;
>  	u32 hangcheck_fence;
> +	bool hangcheck_idle;
> +	bool hangcheck_busy;
> +	unsigned long hangcheck_busy_since;
>  
>  	void __iomem *mmio;
>  	int irq;
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> index 3cc50d697c89..cef1ed33ff91 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
> @@ -17,6 +17,9 @@ static int etnaviv_job_hang_limit = 0;
>  module_param_named(job_hang_limit, etnaviv_job_hang_limit, int , 0444);
>  static int etnaviv_hw_jobs_limit = 4;
>  module_param_named(hw_job_limit, etnaviv_hw_jobs_limit, int , 0444);
> +static unsigned int etnaviv_hang_max_ms = 30000;
> +module_param_named(hang_max_ms, etnaviv_hang_max_ms, uint, 0644);
> +MODULE_PARM_DESC(hang_max_ms, "Time a busy job may run without front-end progress (ms, 0 = no extension)");
>  
>  static struct dma_fence *etnaviv_sched_run_job(struct drm_sched_job *sched_job)
>  {
> @@ -36,7 +39,8 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job
>  {
>  	struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job);
>  	struct etnaviv_gpu *gpu = submit->gpu;
> -	u32 dma_addr, primid = 0;
> +	u32 dma_addr, primid = 0, mask;
> +	bool idle, was_idle;
>  	int change;
>  
>  	/*
> @@ -62,6 +66,12 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job
>  		primid = gpu_read(gpu, VIVS_MC_PROFILE_FE_READ);
>  		mutex_unlock(&gpu->lock);
>  	}
> +
> +	mask = gpu->idle_mask & ~(VIVS_HI_IDLE_STATE_FE | VIVS_HI_IDLE_STATE_MC);
> +	idle = (gpu_read(gpu, VIVS_HI_IDLE_STATE) & mask) == mask;
> +	was_idle = gpu->hangcheck_idle;
> +	gpu->hangcheck_idle = idle;
> +
>  	if (gpu->state == ETNA_GPU_STATE_RUNNING &&
>  	    (gpu->completed_fence != gpu->hangcheck_fence ||
>  	     change < 0 || change > 16 ||
> @@ -70,9 +80,36 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job
>  		gpu->hangcheck_dma_addr = dma_addr;
>  		gpu->hangcheck_primid = primid;
>  		gpu->hangcheck_fence = gpu->completed_fence;
> +		gpu->hangcheck_busy = false;
>  		return DRM_GPU_SCHED_STAT_NO_HANG;
>  	}
>  
> +	/*
> +	 * A compute dispatch is a single FE command, so the FE does not move
> +	 * while a long kernel runs. Keep the job alive as long as some unit
> +	 * other than FE and MC is busy, up to hang_max_ms. All of them idle at
> +	 * two ticks in a row means the FE waits for something that never
> +	 * comes.
> +	 */
> +	if (gpu->state == ETNA_GPU_STATE_RUNNING && !(idle && was_idle)) {
> +		unsigned long max = msecs_to_jiffies(READ_ONCE(etnaviv_hang_max_ms));
> +
> +		if (!gpu->hangcheck_busy) {
> +			gpu->hangcheck_busy = true;
> +			gpu->hangcheck_busy_since = jiffies;
> +		}
> +
> +		if (time_before(jiffies, gpu->hangcheck_busy_since + max))
> +			return DRM_GPU_SCHED_STAT_NO_HANG;
> +
> +		if (max)
> +			dev_err(gpu->dev, "job busy for more than %u ms without FE progress\n",
> +				jiffies_to_msecs(max));
> +	}
> +
> +	gpu->hangcheck_idle = false;
> +	gpu->hangcheck_busy = false;
> +
>  	/* block scheduler */
>  	drm_sched_stop(&gpu->sched, sched_job);
>  
> 
> ---
> base-commit: 6628e7d4c6b25c8d18f1345e3a20d21829aae7e4
> change-id: 20261102-etnaviv-compute-hangcheck-cb8d1e7af68f
> 
> Best regards,