Re: [PATCH v3 15/33] drm/xe: Track BOs backed at a sub-optimal page order
From: Niranjana Vishwanathapura
Date: Fri Nov 06 2026 - 23:52:48 EST
On Fri, Jul 10, 2026 at 07:56:01PM -0700, Matthew Brost wrote:
Add per-device tracking of BOs whose backing TT pages were allocated
below the pool's beneficial order, as reported by the
TTM_TT_FLAG_BENEFICIAL_ORDER_FAILED tt page flag (read via
ttm_tt_is_beneficial_order_failed()). These BOs are future candidates for
a defragmentation pass that reallocates their pages at the beneficial
order.
Add a mem.defrag sub-struct (lock, list, atomic count) to struct
xe_device and a defrag_link to struct xe_bo, initialised at BO create
and device init. Membership is updated from xe_bo_move() via
xe_bo_defrag_update(): a BO is added when it is a populated, needed,
non-pinned ttm_bo_type_device object flagged
TTM_TT_FLAG_BENEFICIAL_ORDER_FAILED, and removed otherwise. BOs are also
removed on pin, on purge, moved to dontneed, and at destroy.
Only ttm_bo_type_device BOs are tracked.
Cc: Carlos Santa <carlos.santa@xxxxxxxxx>
Cc: Ryan Neph <ryanneph@xxxxxxxxxx>
Cc: Christian Koenig <christian.koenig@xxxxxxx>
Cc: Huang Rui <ray.huang@xxxxxxx>
Cc: Matthew Auld <matthew.auld@xxxxxxxxx>
Cc: Maarten Lankhorst <maarten.lankhorst@xxxxxxxxxxxxxxx>
Cc: Maxime Ripard <mripard@xxxxxxxxxx>
Cc: Thomas Zimmermann <tzimmermann@xxxxxxx>
Cc: David Airlie <airlied@xxxxxxxxx>
Cc: Simona Vetter <simona@xxxxxxxx>
Cc: dri-devel@xxxxxxxxxxxxxxxxxxxxx
Cc: linux-kernel@xxxxxxxxxxxxxxx
Cc: Thomas Hellström <thomas.hellstrom@xxxxxxxxxxxxxxx>
Assisted-by: GitHub_Copilot:claude-opus-4.8
Signed-off-by: Matthew Brost <matthew.brost@xxxxxxxxx>
---
drivers/gpu/drm/xe/xe_bo.c | 115 +++++++++++++++++++++++++++
drivers/gpu/drm/xe/xe_bo.h | 1 +
drivers/gpu/drm/xe/xe_bo_types.h | 6 ++
drivers/gpu/drm/xe/xe_device.c | 2 +
drivers/gpu/drm/xe/xe_device_types.h | 19 +++++
5 files changed, 143 insertions(+)
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 4ee48cdc02ac..ff324caab6da 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -926,6 +926,9 @@ static void xe_bo_set_purgeable_shrinker(struct xe_bo *bo,
}
}
+static void xe_bo_defrag_update(struct xe_bo *bo);
+static void xe_bo_defrag_remove(struct xe_bo *bo);
+
/**
* xe_bo_set_purgeable_state() - Set BO purgeable state with validation
* @bo: Buffer object
@@ -954,6 +957,7 @@ void xe_bo_set_purgeable_state(struct xe_bo *bo,
bo->purgeable.state = new_state;
xe_bo_set_purgeable_shrinker(bo, new_state);
+ xe_bo_defrag_update(bo);
}
/**
@@ -1003,9 +1007,100 @@ static int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operatio
/* Commit the state transition only once invalidation was queued */
xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_PURGED);
+ /*
+ * The backing store is gone, so the BO no longer needs defrag. Drop it
+ * from the defrag list here so the count stays accurate even for purge
+ * paths that don't pass through xe_bo_move() (e.g. swap_notify).
+ */
+ xe_bo_defrag_remove(bo);
+
return 0;
}
+/**
+ * xe_bo_defrag_init_early() - Initialize the device defrag BO tracking
+ * @xe: The xe device
+ *
+ * Initialize the list, lock and count used to track BOs whose backing TT
+ * pages were allocated at a sub-optimal order.
+ */
+void xe_bo_defrag_init_early(struct xe_device *xe)
+{
+ spin_lock_init(&xe->mem.defrag.lock);
+ INIT_LIST_HEAD(&xe->mem.defrag.list);
+ atomic_set(&xe->mem.defrag.count, 0);
+}
+
+static bool xe_bo_needs_defrag(struct xe_bo *bo)
+{
+ struct ttm_buffer_object *ttm_bo = &bo->ttm;
+ struct ttm_tt *tt = ttm_bo->ttm;
+
+ xe_bo_assert_held(bo);
+
+ return ttm_bo->type == ttm_bo_type_device && tt &&
+ !xe_bo_madv_is_dontneed(bo) &&
+ ttm_tt_is_populated(tt) &&
+ ttm_tt_is_beneficial_order_failed(tt) &&
+ ttm_bo->resource && ttm_bo->resource->mem_type == XE_PL_TT &&
+ !xe_bo_is_pinned(bo);
+}
+
+static void xe_bo_defrag_add(struct xe_bo *bo)
+{
+ struct xe_device *xe = xe_bo_device(bo);
+
+ xe_bo_assert_held(bo);
+ xe_assert(xe, xe_bo_needs_defrag(bo));
+
+ scoped_guard(spinlock, &xe->mem.defrag.lock) {
+ if (list_empty(&bo->defrag_link)) {
+ list_add_tail(&bo->defrag_link, &xe->mem.defrag.list);
+ atomic_inc(&xe->mem.defrag.count);
+ }
+ }
+}
+
+static void __xe_bo_defrag_remove(struct xe_bo *bo)
+{
+ struct xe_device *xe = xe_bo_device(bo);
+
+ guard(spinlock)(&xe->mem.defrag.lock);
+ if (!list_empty(&bo->defrag_link)) {
+ list_del_init(&bo->defrag_link);
+ atomic_dec(&xe->mem.defrag.count);
+ }
+}
Assisted-by: LLM
NIT...probably worth adding a comment saying this version
can be called without holding bo dma-resv lock?
+
+static void xe_bo_defrag_remove(struct xe_bo *bo)
+{
+ xe_bo_assert_held(bo);
+
+ if (list_empty(&bo->defrag_link))
+ return;
+
+ __xe_bo_defrag_remove(bo);
+}
+
+/**
+ * xe_bo_defrag_update() - Update defrag list membership for a BO
+ * @bo: The buffer object
+ *
+ * Add @bo to the device defrag list when it is a ttm_bo_type_device BO resident
+ * in XE_PL_TT with a populated TT whose pages were allocated at a sub-optimal
+ * order (ttm_tt_is_beneficial_order_failed()) and it isn't pinned. Otherwise
+ * ensure it is removed from the list.
+ */
+static void xe_bo_defrag_update(struct xe_bo *bo)
+{
+ xe_bo_assert_held(bo);
+
+ if (xe_bo_needs_defrag(bo))
+ xe_bo_defrag_add(bo);
+ else
+ xe_bo_defrag_remove(bo);
+}
+
static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
struct ttm_operation_ctx *ctx,
struct ttm_resource *new_mem,
@@ -1231,6 +1326,8 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
xe_tt_unmap_sg(xe, ttm_bo->ttm);
}
+ xe_bo_defrag_update(bo);
+
return ret;
}
@@ -1267,6 +1364,15 @@ static long xe_bo_shrink_purge(struct ttm_operation_ctx *ctx,
if (lret > 0) {
xe_ttm_tt_account_subtract(xe, bo->ttm);
update_global_total_pages(bo->bdev, -(long)tt->num_pages);
+
+ /*
+ * The pages are gone, so a tracked BO no longer needs defrag.
+ * This shrinker purge path frees pages without going through
+ * xe_bo_move(), so drop any stale defrag list entry here. Ghost
+ * bos are never tracked, so guard for real xe bos.
+ */
+ if (xe_bo_is_xe_bo(bo))
+ xe_bo_defrag_remove(ttm_to_xe_bo(bo));
}
return lret;
@@ -1909,6 +2015,8 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)
list_del(&bo->vram_userfault_link);
mutex_unlock(&xe->mem_access.vram_userfault.lock);
+ __xe_bo_defrag_remove(bo);
+
kfree(bo);
}
@@ -2414,6 +2522,7 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo,
bo->ttm.base.funcs = &xe_gem_object_funcs;
bo->ttm.priority = XE_BO_PRIORITY_NORMAL;
INIT_LIST_HEAD(&bo->pinned_link);
+ INIT_LIST_HEAD(&bo->defrag_link);
#ifdef CONFIG_PROC_FS
INIT_LIST_HEAD(&bo->client_link);
#endif
@@ -3020,6 +3129,8 @@ int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec)
if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
xe_ttm_tt_account_subtract(xe, bo->ttm.ttm);
+ xe_bo_defrag_remove(bo);
+
/*
* FIXME: If we always use the reserve / unreserve functions for locking
* we do not need this.
@@ -3078,6 +3189,8 @@ int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec)
if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
xe_ttm_tt_account_subtract(xe, bo->ttm.ttm);
+ xe_bo_defrag_remove(bo);
+
/*
* FIXME: If we always use the reserve / unreserve functions for locking
* we do not need this.
@@ -3113,6 +3226,7 @@ void xe_bo_unpin_external(struct xe_bo *bo)
ttm_bo_unpin(&bo->ttm);
if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
xe_ttm_tt_account_add(xe, bo->ttm.ttm);
+ xe_bo_defrag_update(bo);
/*
* FIXME: If we always use the reserve / unreserve functions for locking
@@ -3145,6 +3259,7 @@ void xe_bo_unpin(struct xe_bo *bo)
ttm_bo_unpin(&bo->ttm);
if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
xe_ttm_tt_account_add(xe, bo->ttm.ttm);
+ xe_bo_defrag_update(bo);
}
Looks like list update needs to be called from 2 other places,
1. ttm_bo_populate() getting called from CPU fault path and from TTM fault
never calls xe_bo_defrag_update().
2. In xe_bo_shrink() if ttm_bo_shrink() returns >0 value. This will inflate
the defrag.count value.
/**
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index 6340317f7d2e..4186cb51af87 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -224,6 +224,7 @@ int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec);
int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec);
void xe_bo_unpin_external(struct xe_bo *bo);
void xe_bo_unpin(struct xe_bo *bo);
+void xe_bo_defrag_init_early(struct xe_device *xe);
int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict,
struct drm_exec *exec);
diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h
index fcc63ae3f455..9eea9ab380e5 100644
--- a/drivers/gpu/drm/xe/xe_bo_types.h
+++ b/drivers/gpu/drm/xe/xe_bo_types.h
@@ -54,6 +54,12 @@ struct xe_bo {
struct ttm_bo_kmap_obj kmap;
/** @pinned_link: link to present / evicted list of pinned BO */
struct list_head pinned_link;
+ /**
+ * @defrag_link: link into @xe_device.mem.defrag.list for BOs whose
+ * backing TT pages were allocated at a sub-optimal order. Protected by
+ * @xe_device.mem.defrag.lock.
+ */
+ struct list_head defrag_link;
#ifdef CONFIG_PROC_FS
/**
* @client: @xe_drm_client which created the bo
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index 97cc2d185e45..bb93425a5c70 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -607,6 +607,8 @@ int xe_device_init_early(struct xe_device *xe)
if (err)
return err;
+ xe_bo_defrag_init_early(xe);
+
xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
WQ_MEM_RECLAIM);
xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 95e7cea91cbf..9e6244d8f41b 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -296,6 +296,25 @@ struct xe_device {
*/
atomic_long_t dma_mapped_pages_svm[NR_PAGE_ORDERS];
#endif
+ /**
+ * @mem.defrag: Tracking of BOs whose backing TT pages were
+ * allocated at a sub-optimal (smaller than beneficial) order.
+ *
+ * Such BOs are candidates for a future defragmentation pass that
+ * tries to reallocate their pages at the device's beneficial
+ * order. Only ttm_bo_type_device BOs are tracked.
+ */
+ struct {
+ /** @mem.defrag.lock: Protects @mem.defrag.list. */
+ spinlock_t lock;
+ /** @mem.defrag.list: List of struct xe_bo defrag_link. */
+ struct list_head list;
+ /**
+ * @mem.defrag.count: Number of BOs currently on
+ * @mem.defrag.list.
+ */
+ atomic_t count;
NIT...Not sure if we need it to be atomic. Count is only used in patch 16 and
usage is approximation anyhow (doesn't guarantee anything as list lenght can
be changing).
Niranjana
+ } defrag;
} mem;
/** @sriov: device level virtualization data */
--
2.34.1