[PATCH v2 30/33] drm/xe: Add packed copy-step IOVA mapping for defrag

[Matthew Brost]

Add machinery to map only changed pages during defrag when the
destination TT is IOVA-mapped. Only the non-borrowed (freshly
allocated) pages need copying; borrowed pages are aliases from
the old TT.

Add xe_tt_iova_copy_init() to diff new vs old TT folios and
record changed-page runs. Add xe_tt_map_iova_copy() to link
those changed pages packed contiguously at the front of the
new TT's IOVA reservation, synced and ready for GPU blits.

No users yet; wired up in later patches.

Cc: Carlos Santa <carlos.santa@xxxxxxxxx>
Cc: Ryan Neph <ryanneph@xxxxxxxxxx>
Cc: Christian Koenig <christian.koenig@xxxxxxx>
Cc: Huang Rui <ray.huang@xxxxxxx>
Cc: Matthew Auld <matthew.auld@xxxxxxxxx>
Cc: Maarten Lankhorst <maarten.lankhorst@xxxxxxxxxxxxxxx>
Cc: Maxime Ripard <mripard@xxxxxxxxxx>
Cc: Thomas Zimmermann <tzimmermann@xxxxxxx>
Cc: David Airlie <airlied@xxxxxxxxx>
Cc: Simona Vetter <simona@xxxxxxxx>
Cc: dri-devel@xxxxxxxxxxxxxxxxxxxxx
Cc: linux-kernel@xxxxxxxxxxxxxxx
Cc: Thomas Hellström <thomas.hellstrom@xxxxxxxxxxxxxxx>
Assisted-by: GitHub_Copilot:claude-opus-4.8
Signed-off-by: Matthew Brost <matthew.brost@xxxxxxxxx>
---
 drivers/gpu/drm/xe/xe_bo.c | 364 +++++++++++++++++++++++++++++++++++++
 1 file changed, 364 insertions(+)

diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index ec034518e2ea..16d9e135cba2 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -531,6 +531,31 @@ static void xe_tt_unaccount_iova(struct xe_device *xe,
 		}
 	}
 }
+
+/*
+ * xe_tt_account_iova_pages - account a tt's pages from a full folio walk
+ * @xe: the xe device
+ * @xe_tt: the xe_ttm_tt to account (its @iova_dma_pages must start zeroed)
+ *
+ * Walk @xe_tt's pages one folio at a time (the same head-page order walk used
+ * by xe_tt_map_iova()) and account each into xe->mem.dma_mapped_pages[] and
+ * @xe_tt->iova_dma_pages[]. Used to (re)derive accounting that exactly matches
+ * the pages a tt is actually mapped with.
+ */
+__maybe_unused
+static void xe_tt_account_iova_pages(struct xe_device *xe,
+				     struct xe_ttm_tt *xe_tt)
+{
+	struct ttm_tt *tt = &xe_tt->ttm;
+	unsigned long i = 0;
+
+	while (i < tt->num_pages) {
+		unsigned int order = ttm_pool_page_order_nodma(tt->pages[i]);
+
+		xe_tt_account_iova_add(xe, xe_tt, order);
+		i += 1UL << order;
+	}
+}
 #else
 static void xe_tt_account_iova_add(struct xe_device *xe,
 				   struct xe_ttm_tt *xe_tt, unsigned int order)
@@ -541,6 +566,12 @@ static void xe_tt_unaccount_iova(struct xe_device *xe,
 				 struct xe_ttm_tt *xe_tt)
 {
 }
+
+__maybe_unused
+static void xe_tt_account_iova_pages(struct xe_device *xe,
+				     struct xe_ttm_tt *xe_tt)
+{
+}
 #endif
 
 static int xe_tt_map_sg(struct xe_device *xe, struct ttm_tt *tt)
@@ -714,6 +745,339 @@ void xe_res_first_tt(struct ttm_tt *tt, u64 start, u64 size,
 		xe_res_first_sg(xe_tt->sg, start, size, cur);
 }
 
+/**
+ * struct xe_tt_iova_run - A contiguous run of changed pages in a defrag move
+ * @start: First page offset (natural ttm_tt index) of the run.
+ * @count: Number of pages in the run.
+ *
+ * Runs are folio aligned: defrag borrows whole beneficial-order chunks, so a
+ * folio is either entirely borrowed (unchanged) or entirely freshly allocated
+ * (changed), never split across the boundary.
+ */
+struct xe_tt_iova_run {
+	pgoff_t start;
+	pgoff_t count;
+};
+
+/**
+ * struct xe_tt_iova_link - A snapshot folio link descriptor for defrag relink
+ * @phys: Physical address of the folio's head page.
+ * @offset: Natural byte offset into the reservation to link the folio at.
+ * @len: Byte length of the folio (PAGE_SIZE << order).
+ *
+ * Captured at copy-init time while the destination tt's page array is valid and
+ * stable under the BO lock. The post-copy finalize job runs asynchronously,
+ * after TTM may have mutated (or torn down) the destination tt page array, so
+ * it relinks purely from these snapshots and never dereferences that array.
+ */
+struct xe_tt_iova_link {
+	phys_addr_t phys;
+	size_t offset;
+	size_t len;
+};
+
+/**
+ * struct xe_tt_iova_copy - Changed-page subset of an IOVA defrag move
+ * @runs: Array of changed-page runs, in ascending natural tt offset.
+ * @nr_runs: Number of entries in @runs.
+ * @links: Per-folio link snapshots covering the changed runs, in ascending
+ *         natural offset. Drives the asynchronous finalize relink so it never
+ *         touches the destination tt's (possibly mutated) page array.
+ * @nr_links: Number of entries in @links.
+ * @nr_changed: Total number of changed pages (sum of the run counts). This is
+ *              also the size, in pages, of the packed copy-step IOVA mapping
+ *              produced by xe_tt_map_iova_copy().
+ */
+struct xe_tt_iova_copy {
+	struct xe_tt_iova_run *runs;
+	unsigned int nr_runs;
+	struct xe_tt_iova_link *links;
+	unsigned int nr_links;
+	pgoff_t nr_changed;
+};
+
+/*
+ * xe_tt_iova_walk_changed - Walk a defrag move's changed-page runs.
+ *
+ * Iterate @new_tt one folio at a time (folio order read from TTM via
+ * ttm_pool_page_order_nodma()) and compare each folio's head page against
+ * @old_tt at the same natural offset. Pages the new backing borrowed from the
+ * old tt alias the same struct page and are unchanged; freshly allocated pages
+ * differ. Coalesce contiguous changed folios into runs and, for each run,
+ * invoke @fn. Returns the total number of changed pages.
+ */
+__maybe_unused
+static pgoff_t xe_tt_iova_walk_changed(struct ttm_tt *new_tt,
+				       struct ttm_tt *old_tt,
+				       void (*fn)(pgoff_t start, pgoff_t count,
+						  void *arg),
+				       void *arg)
+{
+	pgoff_t i = 0, n = new_tt->num_pages;
+	pgoff_t run_start = 0, nr_changed = 0;
+	bool in_run = false;
+
+	while (i < n) {
+		unsigned int order = ttm_pool_page_order_nodma(new_tt->pages[i]);
+		pgoff_t nr = 1UL << order;
+		bool changed = new_tt->pages[i] != old_tt->pages[i];
+
+		if (changed) {
+			if (!in_run) {
+				run_start = i;
+				in_run = true;
+			}
+			nr_changed += nr;
+		} else if (in_run) {
+			if (fn)
+				fn(run_start, i - run_start, arg);
+			in_run = false;
+		}
+
+		i += nr;
+	}
+
+	if (in_run && fn)
+		fn(run_start, n - run_start, arg);
+
+	return nr_changed;
+}
+
+struct xe_tt_iova_run_collect {
+	struct ttm_tt *new_tt;
+	struct xe_tt_iova_run *runs;
+	unsigned int nr_runs;
+	unsigned int runs_cap;
+	struct xe_tt_iova_link *links;
+	unsigned int nr_links;
+	unsigned int links_cap;
+	int err;
+};
+
+/* Geometrically grow @arr (element size @elem) to hold at least @need slots. */
+__maybe_unused
+static int xe_tt_iova_grow(void **arr, unsigned int *cap, unsigned int need,
+			   size_t elem)
+{
+	unsigned int new_cap;
+	void *p;
+
+	if (need <= *cap)
+		return 0;
+
+	new_cap = *cap ? *cap * 2 : 64;
+	if (new_cap < need)
+		new_cap = need;
+
+	p = krealloc_array(*arr, new_cap, elem, GFP_KERNEL);
+	if (!p)
+		return -ENOMEM;
+
+	*arr = p;
+	*cap = new_cap;
+	return 0;
+}
+
+__maybe_unused
+static void xe_tt_iova_store_run(pgoff_t start, pgoff_t count, void *arg)
+{
+	struct xe_tt_iova_run_collect *c = arg;
+	pgoff_t i = start, end = start + count;
+
+	if (c->err)
+		return;
+
+	if (xe_tt_iova_grow((void **)&c->runs, &c->runs_cap, c->nr_runs + 1,
+			    sizeof(*c->runs))) {
+		c->err = -ENOMEM;
+		return;
+	}
+	c->runs[c->nr_runs].start = start;
+	c->runs[c->nr_runs].count = count;
+	c->nr_runs++;
+
+	while (i < end) {
+		struct page *page = c->new_tt->pages[i];
+		unsigned int order = ttm_pool_page_order_nodma(page);
+
+		if (xe_tt_iova_grow((void **)&c->links, &c->links_cap,
+				    c->nr_links + 1, sizeof(*c->links))) {
+			c->err = -ENOMEM;
+			return;
+		}
+		c->links[c->nr_links].phys = page_to_phys(page);
+		c->links[c->nr_links].offset = (size_t)i << PAGE_SHIFT;
+		c->links[c->nr_links].len = PAGE_SIZE << order;
+		c->nr_links++;
+
+		i += 1UL << order;
+	}
+}
+
+/**
+ * xe_tt_iova_copy_init() - Compute the changed-page subset of a defrag move
+ * @new_tt: The freshly (re)allocated destination tt.
+ * @old_tt: The stashed source tt (bo->defrag_old_tt).
+ * @copy: Output description of the changed pages.
+ *
+ * Diff @new_tt against @old_tt and record the runs of changed (non-borrowed)
+ * pages into @copy, along with a per-folio snapshot of their physical
+ * addresses, natural offsets and lengths. The result drives both the packed
+ * copy-step IOVA mapping (xe_tt_map_iova_copy()) and the post-copy finalize
+ * that relinks the new backing into the IOVA reservation. The finalize runs
+ * asynchronously, after TTM may have mutated @new_tt->pages, so it consumes the
+ * snapshot rather than re-reading the page array. @new_tt and @old_tt must
+ * describe the same BO (equal num_pages) and both be populated.
+ *
+ * The caller owns @copy and must release it with xe_tt_iova_copy_fini().
+ *
+ * Return: 0 on success, -ENOMEM on allocation failure.
+ */
+__maybe_unused
+static int xe_tt_iova_copy_init(struct ttm_tt *new_tt, struct ttm_tt *old_tt,
+				struct xe_tt_iova_copy *copy)
+{
+	struct xe_tt_iova_run_collect c = { .new_tt = new_tt };
+
+	memset(copy, 0, sizeof(*copy));
+
+	/*
+	 * Single pass: collect the changed runs and per-folio link snapshots
+	 * directly, growing the arrays on demand. Changed runs are few (the
+	 * borrowed majority is skipped), so the geometric reallocs are rare and
+	 * this avoids a second full O(num_pages) walk just to size the arrays.
+	 */
+	copy->nr_changed = xe_tt_iova_walk_changed(new_tt, old_tt,
+						   xe_tt_iova_store_run, &c);
+	if (c.err) {
+		kfree(c.runs);
+		kfree(c.links);
+		return c.err;
+	}
+	if (!c.nr_runs)
+		return 0;
+
+	copy->runs = c.runs;
+	copy->nr_runs = c.nr_runs;
+	copy->links = c.links;
+	copy->nr_links = c.nr_links;
+
+	return 0;
+}
+
+/**
+ * xe_tt_iova_copy_fini() - Release a struct xe_tt_iova_copy
+ * @copy: The copy description to release.
+ */
+__maybe_unused
+static void xe_tt_iova_copy_fini(struct xe_tt_iova_copy *copy)
+{
+	kfree(copy->runs);
+	kfree(copy->links);
+	copy->runs = NULL;
+	copy->links = NULL;
+	copy->nr_runs = 0;
+	copy->nr_links = 0;
+	copy->nr_changed = 0;
+}
+
+/**
+ * xe_tt_map_iova_copy() - DMA map the changed-page subset for a defrag copy
+ * @xe: The xe device.
+ * @new_tt: The destination tt (must be IOVA eligible, use_iova == true).
+ * @copy: The changed-page description from xe_tt_iova_copy_init().
+ *
+ * Link only the changed pages of @new_tt into its IOVA reservation, packed
+ * contiguously at the front of the range ([0, @copy->nr_changed) pages). This
+ * provides a compact, contiguous destination for the GPU defrag copy without
+ * mapping the (majority) borrowed pages, and lets the whole temporary mapping
+ * later be torn down with a single dma_iova_destroy().
+ *
+ * The reservation's remaining tail is left unlinked. This does not establish
+ * the tt's final mapping (@iova_linked stays false); the post-copy finalize is
+ * responsible for that.
+ *
+ * Return: 0 on success, negative error code on failure (any partial linkage is
+ * unwound).
+ */
+__maybe_unused
+static int xe_tt_map_iova_copy(struct xe_device *xe, struct ttm_tt *new_tt,
+			       const struct xe_tt_iova_copy *copy)
+{
+	struct xe_ttm_tt *xe_tt = container_of(new_tt, struct xe_ttm_tt, ttm);
+	struct device *dev = xe->drm.dev;
+	size_t mapped = 0;
+	unsigned int r;
+	int ret;
+
+	xe_assert(xe, xe_tt->use_iova);
+	xe_assert(xe, !xe_tt->iova_linked);
+
+	for (r = 0; r < copy->nr_runs; r++) {
+		pgoff_t i = copy->runs[r].start;
+		pgoff_t end = i + copy->runs[r].count;
+
+		while (i < end) {
+			struct page *page = new_tt->pages[i];
+			unsigned int order = ttm_pool_page_order_nodma(page);
+			size_t len = PAGE_SIZE << order;
+
+			ret = dma_iova_link(dev, &xe_tt->iova,
+					    page_to_phys(page), mapped, len,
+					    DMA_BIDIRECTIONAL,
+					    DMA_ATTR_SKIP_CPU_SYNC);
+			if (ret)
+				goto err_unlink;
+
+			mapped += len;
+			i += 1UL << order;
+		}
+	}
+
+	ret = dma_iova_sync(dev, &xe_tt->iova, 0, mapped);
+	if (ret)
+		goto err_unlink;
+
+	return 0;
+
+err_unlink:
+	if (mapped)
+		dma_iova_unlink(dev, &xe_tt->iova, 0, mapped,
+				DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
+	return ret;
+}
+
+/*
+ * xe_tt_iova_transfer - Transfer a full IOVA reservation old_tt -> new_tt.
+ *
+ * Move the (fully linked) IOVA reservation from the old tt to the new tt,
+ * leaving the old tt without a reservation so its destroy path does not double
+ * free. The new tt keeps its existing reservation snapshotted in the finalize
+ * job for teardown. Called at the defrag finalize job's point of no failure.
+ *
+ * Also rebalance the per-order DMA-page accounting: drop the old tt's
+ * accounting and account the pages the new tt actually ends up mapped with.
+ * The finalize relink leaves the new tt's mapping equal to new_tt->pages
+ * (borrowed pages alias the old tt; changed runs use the new pages, which may
+ * have a different order distribution), so deriving the accounting from a full
+ * folio walk of the new tt keeps xe->mem.dma_mapped_pages[] exact while the new
+ * mapping lives, rather than carrying the old tt's stale distribution.
+ */
+__maybe_unused
+static void xe_tt_iova_transfer(struct xe_device *xe,
+				struct xe_ttm_tt *new_tt,
+				struct xe_ttm_tt *old_tt)
+{
+	new_tt->iova = old_tt->iova;
+	new_tt->iova_linked = true;
+
+	xe_tt_unaccount_iova(xe, old_tt);
+	xe_tt_account_iova_pages(xe, new_tt);
+
+	old_tt->iova_linked = false;
+	old_tt->use_iova = false;
+}
+
 /*
  * Account ttm pages against the device shrinker's shrinkable and
  * purgeable counts.
-- 
2.34.1