[PATCH v2 2/4] drm/msm/adreno: add a timestamp offset for counters that restart

[Dmitry Baryshkov]

Before a750, the always-on counter behind MSM_PARAM_TIMESTAMP, which the
GPU also writes for its own timestamps, restarts whenever the GPU powers
up, so timestamps go backwards across a runtime suspend. The counter
can't be written back, and adding an offset to MSM_PARAM_TIMESTAMP alone
breaks its match with the timestamps the GPU writes.

Keep an offset that makes the counter continuous, advancing it by the
CLOCK_MONOTONIC time the counter was stopped, and publish it in the
vdso page so that shaders can add it to the timestamps the GPU writes.
Report the counter with the offset added as the new
MSM_PARAM_TIMESTAMP_MONOTONIC, read under gpu->lock so that recovery
can't move the offset in between. MSM_PARAM_TIMESTAMP stays raw for the
existing userspace. On GPUs where the counter doesn't restart the offset
stays zero, so userspace can take the same path everywhere.

Nothing calls the save and restore yet, so the offset stays zero.

Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@xxxxxxxxxxxxxxxx>
---
 drivers/gpu/drm/msm/adreno/adreno_gpu.c | 63 +++++++++++++++++++++++++++++++++
 drivers/gpu/drm/msm/adreno/adreno_gpu.h | 10 ++++++
 include/uapi/drm/msm_drm.h              | 16 +++++++++
 3 files changed, 89 insertions(+)

diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index 0adf56fb9563..f13e54e75dc1 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -226,6 +226,50 @@ adreno_iommu_create_vm(struct msm_gpu *gpu,
 	return vm;
 }
 
+/* The always-on counters tick at 19.2 MHz */
+static u64 adreno_ns_to_ticks(u64 ns)
+{
+	return mul_u64_u32_div(ns, 12, 625);
+}
+
+/* For a GPU whose always-on counter restarts when it powers up */
+void adreno_timestamp_init(struct msm_gpu *gpu)
+{
+	to_adreno_gpu(gpu)->timestamp_restarts = true;
+}
+
+/* Call with the GPU powered, before anything that may restart the counter */
+void adreno_timestamp_save(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+
+	if (!adreno_gpu->timestamp_restarts || adreno_gpu->timestamp_stopped)
+		return;
+
+	adreno_gpu->timestamp_saved = adreno_gpu->funcs->get_timestamp(gpu);
+	adreno_gpu->timestamp_saved_ns = ktime_get_ns();
+	adreno_gpu->timestamp_stopped = true;
+}
+
+/* Call with the GPU powered, before it runs anything */
+void adreno_timestamp_restore(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	u64 now, gap;
+
+	if (!adreno_gpu->timestamp_stopped)
+		return;
+
+	now = adreno_gpu->funcs->get_timestamp(gpu);
+	gap = adreno_ns_to_ticks(ktime_get_ns() - adreno_gpu->timestamp_saved_ns);
+	adreno_gpu->timestamp_offset += adreno_gpu->timestamp_saved + gap - now;
+
+	gpu->vdso->timestamp_offset = adreno_gpu->timestamp_offset;
+	wmb();
+
+	adreno_gpu->timestamp_stopped = false;
+}
+
 u64 adreno_private_vm_size(struct msm_gpu *gpu)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
@@ -404,6 +448,25 @@ int adreno_get_param(struct msm_gpu *gpu, struct msm_context *ctx,
 			return 0;
 		}
 		return -EINVAL;
+	case MSM_PARAM_TIMESTAMP_MONOTONIC:
+		if (adreno_gpu->funcs->get_timestamp) {
+			int ret;
+
+			/* recovery moves the offset under gpu->lock */
+			ret = mutex_lock_interruptible(&gpu->lock);
+			if (ret)
+				return ret;
+
+			pm_runtime_get_sync(&gpu->pdev->dev);
+			*value = adreno_gpu->timestamp_offset +
+				 adreno_gpu->funcs->get_timestamp(gpu);
+			pm_runtime_put_autosuspend(&gpu->pdev->dev);
+
+			mutex_unlock(&gpu->lock);
+
+			return 0;
+		}
+		return -EINVAL;
 	case MSM_PARAM_VDSO_IOVA:
 		*value = gpu->vdso_iova;
 		return 0;
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index 4b2156825bdf..51ef57b536ed 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -209,6 +209,13 @@ struct adreno_gpu {
 	uint16_t speedbin;
 	const struct adreno_gpu_funcs *funcs;
 
+	/* offset that keeps a restarting always-on counter continuous: */
+	bool timestamp_restarts;
+	u64 timestamp_offset;
+	u64 timestamp_saved;
+	u64 timestamp_saved_ns;
+	bool timestamp_stopped;
+
 	struct completion fault_coredump_done;
 
 	/* interesting register offsets to dump: */
@@ -641,6 +648,9 @@ static inline int adreno_is_a850_family(struct adreno_gpu *gpu)
 /* Put vm_start above 32b to catch issues with not setting xyz_BASE_HI */
 #define ADRENO_VM_START 0x100000000ULL
 u64 adreno_private_vm_size(struct msm_gpu *gpu);
+void adreno_timestamp_init(struct msm_gpu *gpu);
+void adreno_timestamp_save(struct msm_gpu *gpu);
+void adreno_timestamp_restore(struct msm_gpu *gpu);
 int adreno_get_param(struct msm_gpu *gpu, struct msm_context *ctx,
 		     uint32_t param, uint64_t *value, uint32_t *len);
 int adreno_set_param(struct msm_gpu *gpu, struct msm_context *ctx,
diff --git a/include/uapi/drm/msm_drm.h b/include/uapi/drm/msm_drm.h
index f8bf48fdc339..2fd185f0ffcc 100644
--- a/include/uapi/drm/msm_drm.h
+++ b/include/uapi/drm/msm_drm.h
@@ -120,6 +120,16 @@ struct drm_msm_timespec {
 #define MSM_PARAM_AQE	     0x17  /* RO */
 #define MSM_PARAM_VDSO_IOVA  0x18  /* RO: GPU address of struct drm_msm_vdso */
 
+/* On GPUs before A750 the always-on counter behind MSM_PARAM_TIMESTAMP
+ * restarts whenever the GPU powers down. The GPU writes the same value as a
+ * timestamp.  MSM_PARAM_TIMESTAMP_MONOTONIC returns the counter with
+ * drm_msm_vdso::timestamp_offset added, resulting in a monotonic timestamp
+ * that most uAPIs require. Shaders can add the same to the timestamps
+ * the GPU writes when required, ending up with the stable counter values.
+ */
+
+#define MSM_PARAM_TIMESTAMP_MONOTONIC 0x19  /* RO */
+
 /* For backwards compat.  The original support for preemption was based on
  * a single ring per priority level so # of priority levels equals the #
  * of rings.  With drm/scheduler providing additional levels of priority,
@@ -134,6 +144,12 @@ struct drm_msm_timespec {
  */
 struct drm_msm_vdso {
 	__u64 flags;
+	/*
+	 * Value added to produce MSM_PARAM_TIMESTAMP_MONOTONIC.
+	 * The offset doesn't change while the GPU executes a submit, and stays
+	 * zero where the counter doesn't restart.
+	 */
+	__u64 timestamp_offset;
 };
 
 struct drm_msm_param {

-- 
2.47.3