Re: [PATCH v3 15/33] drm/xe: Track BOs backed at a sub-optimal page order
From: Matthew Brost
Date: Fri Nov 20 2026 - 01:03:16 EST
On Fri, Nov 06, 2026 at 08:49:18PM -0800, Niranjana Vishwanathapura wrote:
> On Fri, Jul 10, 2026 at 07:56:01PM -0700, Matthew Brost wrote:
> > Add per-device tracking of BOs whose backing TT pages were allocated
> > below the pool's beneficial order, as reported by the
> > TTM_TT_FLAG_BENEFICIAL_ORDER_FAILED tt page flag (read via
> > ttm_tt_is_beneficial_order_failed()). These BOs are future candidates for
> > a defragmentation pass that reallocates their pages at the beneficial
> > order.
> >
> > Add a mem.defrag sub-struct (lock, list, atomic count) to struct
> > xe_device and a defrag_link to struct xe_bo, initialised at BO create
> > and device init. Membership is updated from xe_bo_move() via
> > xe_bo_defrag_update(): a BO is added when it is a populated, needed,
> > non-pinned ttm_bo_type_device object flagged
> > TTM_TT_FLAG_BENEFICIAL_ORDER_FAILED, and removed otherwise. BOs are also
> > removed on pin, on purge, moved to dontneed, and at destroy.
> >
> > Only ttm_bo_type_device BOs are tracked.
> >
> > Cc: Carlos Santa <carlos.santa@xxxxxxxxx>
> > Cc: Ryan Neph <ryanneph@xxxxxxxxxx>
> > Cc: Christian Koenig <christian.koenig@xxxxxxx>
> > Cc: Huang Rui <ray.huang@xxxxxxx>
> > Cc: Matthew Auld <matthew.auld@xxxxxxxxx>
> > Cc: Maarten Lankhorst <maarten.lankhorst@xxxxxxxxxxxxxxx>
> > Cc: Maxime Ripard <mripard@xxxxxxxxxx>
> > Cc: Thomas Zimmermann <tzimmermann@xxxxxxx>
> > Cc: David Airlie <airlied@xxxxxxxxx>
> > Cc: Simona Vetter <simona@xxxxxxxx>
> > Cc: dri-devel@xxxxxxxxxxxxxxxxxxxxx
> > Cc: linux-kernel@xxxxxxxxxxxxxxx
> > Cc: Thomas Hellström <thomas.hellstrom@xxxxxxxxxxxxxxx>
> > Assisted-by: GitHub_Copilot:claude-opus-4.8
> > Signed-off-by: Matthew Brost <matthew.brost@xxxxxxxxx>
> > ---
> > drivers/gpu/drm/xe/xe_bo.c | 115 +++++++++++++++++++++++++++
> > drivers/gpu/drm/xe/xe_bo.h | 1 +
> > drivers/gpu/drm/xe/xe_bo_types.h | 6 ++
> > drivers/gpu/drm/xe/xe_device.c | 2 +
> > drivers/gpu/drm/xe/xe_device_types.h | 19 +++++
> > 5 files changed, 143 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
> > index 4ee48cdc02ac..ff324caab6da 100644
> > --- a/drivers/gpu/drm/xe/xe_bo.c
> > +++ b/drivers/gpu/drm/xe/xe_bo.c
> > @@ -926,6 +926,9 @@ static void xe_bo_set_purgeable_shrinker(struct xe_bo *bo,
> > }
> > }
> >
> > +static void xe_bo_defrag_update(struct xe_bo *bo);
> > +static void xe_bo_defrag_remove(struct xe_bo *bo);
> > +
> > /**
> > * xe_bo_set_purgeable_state() - Set BO purgeable state with validation
> > * @bo: Buffer object
> > @@ -954,6 +957,7 @@ void xe_bo_set_purgeable_state(struct xe_bo *bo,
> >
> > bo->purgeable.state = new_state;
> > xe_bo_set_purgeable_shrinker(bo, new_state);
> > + xe_bo_defrag_update(bo);
> > }
> >
> > /**
> > @@ -1003,9 +1007,100 @@ static int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operatio
> > /* Commit the state transition only once invalidation was queued */
> > xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_PURGED);
> >
> > + /*
> > + * The backing store is gone, so the BO no longer needs defrag. Drop it
> > + * from the defrag list here so the count stays accurate even for purge
> > + * paths that don't pass through xe_bo_move() (e.g. swap_notify).
> > + */
> > + xe_bo_defrag_remove(bo);
> > +
> > return 0;
> > }
> >
> > +/**
> > + * xe_bo_defrag_init_early() - Initialize the device defrag BO tracking
> > + * @xe: The xe device
> > + *
> > + * Initialize the list, lock and count used to track BOs whose backing TT
> > + * pages were allocated at a sub-optimal order.
> > + */
> > +void xe_bo_defrag_init_early(struct xe_device *xe)
> > +{
> > + spin_lock_init(&xe->mem.defrag.lock);
> > + INIT_LIST_HEAD(&xe->mem.defrag.list);
> > + atomic_set(&xe->mem.defrag.count, 0);
> > +}
> > +
> > +static bool xe_bo_needs_defrag(struct xe_bo *bo)
> > +{
> > + struct ttm_buffer_object *ttm_bo = &bo->ttm;
> > + struct ttm_tt *tt = ttm_bo->ttm;
> > +
> > + xe_bo_assert_held(bo);
> > +
> > + return ttm_bo->type == ttm_bo_type_device && tt &&
> > + !xe_bo_madv_is_dontneed(bo) &&
> > + ttm_tt_is_populated(tt) &&
> > + ttm_tt_is_beneficial_order_failed(tt) &&
> > + ttm_bo->resource && ttm_bo->resource->mem_type == XE_PL_TT &&
> > + !xe_bo_is_pinned(bo);
> > +}
> > +
> > +static void xe_bo_defrag_add(struct xe_bo *bo)
> > +{
> > + struct xe_device *xe = xe_bo_device(bo);
> > +
> > + xe_bo_assert_held(bo);
> > + xe_assert(xe, xe_bo_needs_defrag(bo));
> > +
> > + scoped_guard(spinlock, &xe->mem.defrag.lock) {
> > + if (list_empty(&bo->defrag_link)) {
> > + list_add_tail(&bo->defrag_link, &xe->mem.defrag.list);
> > + atomic_inc(&xe->mem.defrag.count);
> > + }
> > + }
> > +}
> > +
> > +static void __xe_bo_defrag_remove(struct xe_bo *bo)
> > +{
> > + struct xe_device *xe = xe_bo_device(bo);
> > +
> > + guard(spinlock)(&xe->mem.defrag.lock);
> > + if (!list_empty(&bo->defrag_link)) {
> > + list_del_init(&bo->defrag_link);
> > + atomic_dec(&xe->mem.defrag.count);
> > + }
> > +}
>
> Assisted-by: LLM
>
Yes.
> NIT...probably worth adding a comment saying this version
> can be called without holding bo dma-resv lock?
>
It should be self documenting with lockdep but can add this.
> > +
> > +static void xe_bo_defrag_remove(struct xe_bo *bo)
> > +{
> > + xe_bo_assert_held(bo);
> > +
> > + if (list_empty(&bo->defrag_link))
> > + return;
> > +
> > + __xe_bo_defrag_remove(bo);
> > +}
> > +
> > +/**
> > + * xe_bo_defrag_update() - Update defrag list membership for a BO
> > + * @bo: The buffer object
> > + *
> > + * Add @bo to the device defrag list when it is a ttm_bo_type_device BO resident
> > + * in XE_PL_TT with a populated TT whose pages were allocated at a sub-optimal
> > + * order (ttm_tt_is_beneficial_order_failed()) and it isn't pinned. Otherwise
> > + * ensure it is removed from the list.
> > + */
> > +static void xe_bo_defrag_update(struct xe_bo *bo)
> > +{
> > + xe_bo_assert_held(bo);
> > +
> > + if (xe_bo_needs_defrag(bo))
> > + xe_bo_defrag_add(bo);
> > + else
> > + xe_bo_defrag_remove(bo);
> > +}
> > +
> > static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
> > struct ttm_operation_ctx *ctx,
> > struct ttm_resource *new_mem,
> > @@ -1231,6 +1326,8 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
> > xe_tt_unmap_sg(xe, ttm_bo->ttm);
> > }
> >
> > + xe_bo_defrag_update(bo);
> > +
> > return ret;
> > }
> >
> > @@ -1267,6 +1364,15 @@ static long xe_bo_shrink_purge(struct ttm_operation_ctx *ctx,
> > if (lret > 0) {
> > xe_ttm_tt_account_subtract(xe, bo->ttm);
> > update_global_total_pages(bo->bdev, -(long)tt->num_pages);
> > +
> > + /*
> > + * The pages are gone, so a tracked BO no longer needs defrag.
> > + * This shrinker purge path frees pages without going through
> > + * xe_bo_move(), so drop any stale defrag list entry here. Ghost
> > + * bos are never tracked, so guard for real xe bos.
> > + */
> > + if (xe_bo_is_xe_bo(bo))
> > + xe_bo_defrag_remove(ttm_to_xe_bo(bo));
> > }
> >
> > return lret;
> > @@ -1909,6 +2015,8 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)
> > list_del(&bo->vram_userfault_link);
> > mutex_unlock(&xe->mem_access.vram_userfault.lock);
> >
> > + __xe_bo_defrag_remove(bo);
> > +
> > kfree(bo);
> > }
> >
> > @@ -2414,6 +2522,7 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo,
> > bo->ttm.base.funcs = &xe_gem_object_funcs;
> > bo->ttm.priority = XE_BO_PRIORITY_NORMAL;
> > INIT_LIST_HEAD(&bo->pinned_link);
> > + INIT_LIST_HEAD(&bo->defrag_link);
> > #ifdef CONFIG_PROC_FS
> > INIT_LIST_HEAD(&bo->client_link);
> > #endif
> > @@ -3020,6 +3129,8 @@ int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec)
> > if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
> > xe_ttm_tt_account_subtract(xe, bo->ttm.ttm);
> >
> > + xe_bo_defrag_remove(bo);
> > +
> > /*
> > * FIXME: If we always use the reserve / unreserve functions for locking
> > * we do not need this.
> > @@ -3078,6 +3189,8 @@ int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec)
> > if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
> > xe_ttm_tt_account_subtract(xe, bo->ttm.ttm);
> >
> > + xe_bo_defrag_remove(bo);
> > +
> > /*
> > * FIXME: If we always use the reserve / unreserve functions for locking
> > * we do not need this.
> > @@ -3113,6 +3226,7 @@ void xe_bo_unpin_external(struct xe_bo *bo)
> > ttm_bo_unpin(&bo->ttm);
> > if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
> > xe_ttm_tt_account_add(xe, bo->ttm.ttm);
> > + xe_bo_defrag_update(bo);
> >
> > /*
> > * FIXME: If we always use the reserve / unreserve functions for locking
> > @@ -3145,6 +3259,7 @@ void xe_bo_unpin(struct xe_bo *bo)
> > ttm_bo_unpin(&bo->ttm);
> > if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
> > xe_ttm_tt_account_add(xe, bo->ttm.ttm);
> > + xe_bo_defrag_update(bo);
> > }
>
> Looks like list update needs to be called from 2 other places,
>
> 1. ttm_bo_populate() getting called from CPU fault path and from TTM fault
> never calls xe_bo_defrag_update().
Yes. We also need to strip out reclaim flags in CPU fault path as this a
foreground op.
> 2. In xe_bo_shrink() if ttm_bo_shrink() returns >0 value. This will inflate
> the defrag.count value.
>
This routes through ttm_bo_validate which ends up in xe_bo_move so I
think this case is covered.
> >
> > /**
> > diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
> > index 6340317f7d2e..4186cb51af87 100644
> > --- a/drivers/gpu/drm/xe/xe_bo.h
> > +++ b/drivers/gpu/drm/xe/xe_bo.h
> > @@ -224,6 +224,7 @@ int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec);
> > int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec);
> > void xe_bo_unpin_external(struct xe_bo *bo);
> > void xe_bo_unpin(struct xe_bo *bo);
> > +void xe_bo_defrag_init_early(struct xe_device *xe);
> > int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict,
> > struct drm_exec *exec);
> >
> > diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h
> > index fcc63ae3f455..9eea9ab380e5 100644
> > --- a/drivers/gpu/drm/xe/xe_bo_types.h
> > +++ b/drivers/gpu/drm/xe/xe_bo_types.h
> > @@ -54,6 +54,12 @@ struct xe_bo {
> > struct ttm_bo_kmap_obj kmap;
> > /** @pinned_link: link to present / evicted list of pinned BO */
> > struct list_head pinned_link;
> > + /**
> > + * @defrag_link: link into @xe_device.mem.defrag.list for BOs whose
> > + * backing TT pages were allocated at a sub-optimal order. Protected by
> > + * @xe_device.mem.defrag.lock.
> > + */
> > + struct list_head defrag_link;
> > #ifdef CONFIG_PROC_FS
> > /**
> > * @client: @xe_drm_client which created the bo
> > diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
> > index 97cc2d185e45..bb93425a5c70 100644
> > --- a/drivers/gpu/drm/xe/xe_device.c
> > +++ b/drivers/gpu/drm/xe/xe_device.c
> > @@ -607,6 +607,8 @@ int xe_device_init_early(struct xe_device *xe)
> > if (err)
> > return err;
> >
> > + xe_bo_defrag_init_early(xe);
> > +
> > xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
> > WQ_MEM_RECLAIM);
> > xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
> > diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
> > index 95e7cea91cbf..9e6244d8f41b 100644
> > --- a/drivers/gpu/drm/xe/xe_device_types.h
> > +++ b/drivers/gpu/drm/xe/xe_device_types.h
> > @@ -296,6 +296,25 @@ struct xe_device {
> > */
> > atomic_long_t dma_mapped_pages_svm[NR_PAGE_ORDERS];
> > #endif
> > + /**
> > + * @mem.defrag: Tracking of BOs whose backing TT pages were
> > + * allocated at a sub-optimal (smaller than beneficial) order.
> > + *
> > + * Such BOs are candidates for a future defragmentation pass that
> > + * tries to reallocate their pages at the device's beneficial
> > + * order. Only ttm_bo_type_device BOs are tracked.
> > + */
> > + struct {
> > + /** @mem.defrag.lock: Protects @mem.defrag.list. */
> > + spinlock_t lock;
> > + /** @mem.defrag.list: List of struct xe_bo defrag_link. */
> > + struct list_head list;
> > + /**
> > + * @mem.defrag.count: Number of BOs currently on
> > + * @mem.defrag.list.
> > + */
> > + atomic_t count;
>
> NIT...Not sure if we need it to be atomic. Count is only used in patch 16 and
> usage is approximation anyhow (doesn't guarantee anything as list lenght can
> be changing).
Right - incr / decr are protected by the spin lock along with the list.
I think count might actually be able to be dropped as target has slighly
moved - foreground just never does reclaim, thus
XE_BO_DEFRAG_RECLAIM_BACKOFF_THRESHOLD / count isn't needed.
Matt
>
> Niranjana
>
> > + } defrag;
> > } mem;
> >
> > /** @sriov: device level virtualization data */
> > --
> > 2.34.1
> >