[PATCH v2 33/33] drm/amdgpu: Preallocate system BO pages outside the reservation lock
From: Matthew Brost
Date: Fri Jul 10 2026 - 18:03:10 EST
Populating a GTT (system) buffer object under the reservation lock can
stall in reclaim and compaction while trying to satisfy beneficial-order
allocations, holding the lock for the duration.
Mirror the Xe out-of-lock preallocation for pure system BOs: when a
gem_create request targets AMDGPU_GEM_DOMAIN_GTT only, fill the full page
backing up front via ttm_pool_prealloc_fill_full() before taking the
reservation lock. The populate under the lock then simply installs these
pages instead of reclaiming and compacting in the critical section. The
fill is best-effort - a short fill falls back to the normal in-lock
allocation for the remaining pages, and any leftover pages are released
by ttm_pool_prealloc_fini() on all exit paths.
Unlike Xe, amdgpu has no background defragmenter, so the higher-order
reclaim backoff is left disabled (normal reclaim). If the pool uses
dma-alloc (swiotlb), the fill bails and the feature is a silent no-op.
Factor the tt pool selection into amdgpu_ttm_tt_pool_id() and
amdgpu_ttm_pool() so the preallocation targets the exact pool the
populate will consume, and thread an optional prealloc bag through
amdgpu_gem_object_create() and amdgpu_bo_param into the populate ctx.
Cc: amd-gfx@xxxxxxxxxxxxxxxxxxxxx
Cc: Alex Deucher <alexander.deucher@xxxxxxx>
Cc: Carlos Santa <carlos.santa@xxxxxxxxx>
Cc: Ryan Neph <ryanneph@xxxxxxxxxx>
Cc: Christian Koenig <christian.koenig@xxxxxxx>
Cc: Huang Rui <ray.huang@xxxxxxx>
Cc: Matthew Auld <matthew.auld@xxxxxxxxx>
Cc: Maarten Lankhorst <maarten.lankhorst@xxxxxxxxxxxxxxx>
Cc: Maxime Ripard <mripard@xxxxxxxxxx>
Cc: Thomas Zimmermann <tzimmermann@xxxxxxx>
Cc: David Airlie <airlied@xxxxxxxxx>
Cc: Simona Vetter <simona@xxxxxxxx>
Cc: dri-devel@xxxxxxxxxxxxxxxxxxxxx
Cc: linux-kernel@xxxxxxxxxxxxxxx
Cc: Thomas Hellström <thomas.hellstrom@xxxxxxxxxxxxxxx>
Assisted-by: GitHub_Copilot:claude-opus-4.8
Signed-off-by: Matthew Brost <matthew.brost@xxxxxxxxx>
---
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 4 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 2 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 45 +++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h | 5 ++-
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 3 +-
drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 4 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 39 +++++++++++-----
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 3 ++
8 files changed, 80 insertions(+), 25 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 20831dbebc31..940f58848a97 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -345,7 +345,7 @@ create_dmamap_sg_bo(struct amdgpu_device *adev,
ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1,
AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags,
- ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0);
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0, NULL);
amdgpu_bo_unreserve(mem->bo);
@@ -1811,7 +1811,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
domain_string(alloc_domain), xcp_id);
ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain, alloc_flags,
- bo_type, NULL, &gobj, xcp_id + 1);
+ bo_type, NULL, &gobj, xcp_id + 1, NULL);
if (ret) {
pr_debug("Failed to create BO on domain %s. ret %d\n",
domain_string(alloc_domain), ret);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index b33c300e26e2..51510e831129 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -435,7 +435,7 @@ amdgpu_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf)
ret = amdgpu_gem_object_create(adev, dma_buf->size, PAGE_SIZE,
AMDGPU_GEM_DOMAIN_CPU, flags,
- ttm_bo_type_sg, resv, &gobj, 0);
+ ttm_bo_type_sg, resv, &gobj, 0, NULL);
if (ret)
goto error;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 6a0699746fbc..bef8b5bc8b89 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -39,6 +39,7 @@
#include <drm/drm_exec.h>
#include <drm/drm_gem_ttm_helper.h>
#include <drm/ttm/ttm_tt.h>
+#include <drm/ttm/ttm_pool.h>
#include <drm/drm_syncobj.h>
#include "amdgpu.h"
@@ -168,7 +169,8 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
int alignment, u32 initial_domain,
u64 flags, enum ttm_bo_type type,
struct dma_resv *resv,
- struct drm_gem_object **obj, int8_t xcp_id_plus1)
+ struct drm_gem_object **obj, int8_t xcp_id_plus1,
+ struct ttm_pool_prealloc *prealloc)
{
struct amdgpu_bo *bo;
struct amdgpu_bo_user *ubo;
@@ -188,6 +190,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
bp.domain = initial_domain;
bp.bo_ptr_size = sizeof(struct amdgpu_bo);
bp.xcp_id_plus1 = xcp_id_plus1;
+ bp.prealloc = prealloc;
r = amdgpu_bo_create_user(adev, &bp, &ubo);
if (r)
@@ -412,6 +415,8 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
struct dma_resv *resv = NULL;
struct drm_gem_object *gobj;
uint32_t handle, initial_domain;
+ struct ttm_pool_prealloc prealloc = {};
+ struct ttm_pool *prealloc_pool = NULL;
int r;
/* reject invalid gem flags */
@@ -443,10 +448,29 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
}
+ /*
+ * For system-only (pure GTT) BOs, preallocate the whole page backing
+ * up front, outside the reservation lock. Populate under the lock then
+ * just installs these pages instead of reclaiming/compacting in the
+ * critical section. Best-effort: a short fill falls back to the normal
+ * in-lock allocation for the missing pages.
+ */
+ if (args->in.domains == AMDGPU_GEM_DOMAIN_GTT) {
+ int32_t xcp_id = adev->gmc.mem_partitions ? fpriv->xcp_id : 0;
+ int32_t pool_id = amdgpu_ttm_tt_pool_id(adev, xcp_id);
+ enum ttm_caching caching =
+ (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) ?
+ ttm_write_combined : ttm_cached;
+
+ prealloc_pool = amdgpu_ttm_pool(adev, pool_id);
+ ttm_pool_prealloc_fill_full(prealloc_pool, caching, &prealloc,
+ PFN_UP(size), false);
+ }
+
if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
r = amdgpu_bo_reserve(vm->root.bo, false);
if (r)
- return r;
+ goto out_prealloc;
resv = vm->root.bo->tbo.base.resv;
}
@@ -455,7 +479,8 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
retry:
r = amdgpu_gem_object_create(adev, size, args->in.alignment,
initial_domain,
- flags, ttm_bo_type_device, resv, &gobj, fpriv->xcp_id + 1);
+ flags, ttm_bo_type_device, resv, &gobj,
+ fpriv->xcp_id + 1, prealloc_pool ? &prealloc : NULL);
if (r && r != -ERESTARTSYS) {
if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
@@ -479,17 +504,21 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
amdgpu_bo_unreserve(vm->root.bo);
}
if (r)
- return r;
+ goto out_prealloc;
r = drm_gem_handle_create(filp, gobj, &handle);
/* drop reference from allocate - handle holds it now */
drm_gem_object_put(gobj);
if (r)
- return r;
+ goto out_prealloc;
memset(args, 0, sizeof(*args));
args->out.handle = handle;
- return 0;
+
+out_prealloc:
+ if (prealloc_pool)
+ ttm_pool_prealloc_fini(prealloc_pool, &prealloc);
+ return r;
}
int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
@@ -528,7 +557,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
/* create a gem object to contain this object in */
r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
- 0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
+ 0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1, NULL);
if (r)
return r;
@@ -1298,7 +1327,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
domain = amdgpu_bo_get_preferred_domain(adev,
amdgpu_display_supported_domains(adev, flags));
r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
- ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
+ ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1, NULL);
if (r)
return -ENOMEM;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
index b558336bc4c6..706aeca011f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
@@ -35,6 +35,8 @@
extern const struct drm_gem_object_funcs amdgpu_gem_object_funcs;
+struct ttm_pool_prealloc;
+
unsigned long amdgpu_gem_timeout(uint64_t timeout_ns);
/*
@@ -45,7 +47,8 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
int alignment, u32 initial_domain,
u64 flags, enum ttm_bo_type type,
struct dma_resv *resv,
- struct drm_gem_object **obj, int8_t xcp_id_plus1);
+ struct drm_gem_object **obj, int8_t xcp_id_plus1,
+ struct ttm_pool_prealloc *prealloc);
int amdgpu_mode_dumb_create(struct drm_file *file_priv,
struct drm_device *dev,
struct drm_mode_create_dumb *args);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index f98bfba59a2c..18c4cf3f35a5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -632,7 +632,8 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
/* We opt to avoid OOM on system pages allocations */
.gfp_retry_mayfail = true,
.allow_res_evict = bp->type != ttm_bo_type_kernel,
- .resv = bp->resv
+ .resv = bp->resv,
+ .prealloc = bp->prealloc,
};
struct amdgpu_bo *bo;
unsigned long page_align, size = bp->size;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index ff11a0903499..11f1d403f152 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -45,6 +45,8 @@
#define to_amdgpu_bo_user(abo) container_of((abo), struct amdgpu_bo_user, bo)
#define to_amdgpu_bo_vm(abo) container_of((abo), struct amdgpu_bo_vm, bo)
+struct ttm_pool_prealloc;
+
struct amdgpu_bo_param {
unsigned long size;
int byte_align;
@@ -58,6 +60,8 @@ struct amdgpu_bo_param {
void (*destroy)(struct ttm_buffer_object *bo);
/* xcp partition number plus 1, 0 means any partition */
int8_t xcp_id_plus1;
+ /* optional out-of-lock preallocated backing (system/GTT only) */
+ struct ttm_pool_prealloc *prealloc;
};
/* bo virtual addresses in a vm */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index b10b0878df37..c9cd4714f515 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1173,6 +1173,30 @@ void amdgpu_ttm_mmio_remap_free_sgt(struct device *dev,
kfree(sgt);
}
+/*
+ * amdgpu_ttm_tt_pool_id - compute the ttm pool id backing a given xcp
+ *
+ * Mirrors the mapping used when creating the gtt ttm_tt, so callers that only
+ * have an xcp id (e.g. an out-of-lock preallocation before the bo exists) pick
+ * the same pool the populate will use.
+ */
+int32_t amdgpu_ttm_tt_pool_id(struct amdgpu_device *adev, int32_t xcp_id)
+{
+ if (adev->gmc.mem_partitions && xcp_id >= 0)
+ return KFD_XCP_MEM_ID(adev, xcp_id);
+
+ return xcp_id;
+}
+
+/* amdgpu_ttm_pool - select the ttm pool for a given pool id */
+struct ttm_pool *amdgpu_ttm_pool(struct amdgpu_device *adev, int32_t pool_id)
+{
+ if (adev->mman.ttm_pools && pool_id >= 0)
+ return &adev->mman.ttm_pools[pool_id];
+
+ return &adev->mman.bdev.pool;
+}
+
/**
* amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
*
@@ -1194,10 +1218,7 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
return NULL;
gtt->gobj = &bo->base;
- if (adev->gmc.mem_partitions && abo->xcp_id >= 0)
- gtt->pool_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
- else
- gtt->pool_id = abo->xcp_id;
+ gtt->pool_id = amdgpu_ttm_tt_pool_id(adev, abo->xcp_id);
if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
caching = ttm_write_combined;
@@ -1239,10 +1260,7 @@ static int amdgpu_ttm_tt_populate(struct ttm_device *bdev,
if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL)
return 0;
- if (adev->mman.ttm_pools && gtt->pool_id >= 0)
- pool = &adev->mman.ttm_pools[gtt->pool_id];
- else
- pool = &adev->mman.bdev.pool;
+ pool = amdgpu_ttm_pool(adev, gtt->pool_id);
ret = ttm_pool_alloc(pool, ttm, ctx);
if (ret)
return ret;
@@ -1284,10 +1302,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_device *bdev,
adev = amdgpu_ttm_adev(bdev);
- if (adev->mman.ttm_pools && gtt->pool_id >= 0)
- pool = &adev->mman.ttm_pools[gtt->pool_id];
- else
- pool = &adev->mman.bdev.pool;
+ pool = amdgpu_ttm_pool(adev, gtt->pool_id);
return ttm_pool_free(pool, ttm);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index ff9e2e346609..f90844937b73 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -207,6 +207,9 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
void amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type);
+int32_t amdgpu_ttm_tt_pool_id(struct amdgpu_device *adev, int32_t xcp_id);
+struct ttm_pool *amdgpu_ttm_pool(struct amdgpu_device *adev, int32_t pool_id);
+
#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,
struct amdgpu_hmm_range *range);
--
2.34.1