[PATCH v2] mm/migrate_device: add class-based migration trace events
From: Balbir Singh
Date: Fri Aug 28 2026 - 00:08:48 EST
Device-private migrations currently require ad-hoc instrumentation to
understand selection, unmap, copy, and finalization behavior.
Add reusable ftrace event classes for migrate_vma ranges, device
batches, and individual folios. Emit start, done traces with
ranges, flags, page counts, success/failure summaries, PFNs,
and migrate flags.
Enabling all traces can make the migration traces verbose, when used
they can also help debug issues with device migration.
Signed-off-by: Balbir Singh <balbirs@xxxxxxxxxx>
Suggested-by: Steven Rostedt <rostedt@xxxxxxxxxxx>
Assisted-by: Codex:gpt-5.6-sol
Link: https://lore.kernel.org/linux-mm/20251110161954.47d88433@xxxxxxxxxxxxxxxxxx/
Cc: Steven Rostedt <rostedt@xxxxxxxxxxx>
CC: Masami Hiramatsu <mhiramat@xxxxxxxxxx>
CC: Mathieu Desnoyers <mathieu.desnoyers@xxxxxxxxxxxx>
CC: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
CC: David Hildenbrand <david@xxxxxxxxxx>
CC: Zi Yan <ziy@xxxxxxxxxx>
CC: Matthew Brost <matthew.brost@xxxxxxxxx>
CC: Joshua Hahn <joshua.hahnjy@xxxxxxxxx>
CC: Rakie Kim <rakie.kim@xxxxxx>
CC: Byungchul Park <byungchul@xxxxxx>
CC: Gregory Price <gourry@xxxxxxxxxx>
CC: Ying Huang <ying.huang@xxxxxxxxxxxxxxxxx>
CC: Alistair Popple <apopple@xxxxxxxxxx>
---
include/trace/events/migrate_device.h | 324 ++++++++++++++++++++++++++
mm/migrate_device.c | 79 ++++++-
2 files changed, 391 insertions(+), 12 deletions(-)
create mode 100644 include/trace/events/migrate_device.h
diff --git a/include/trace/events/migrate_device.h b/include/trace/events/migrate_device.h
new file mode 100644
index 000000000000..46582c919dc2
--- /dev/null
+++ b/include/trace/events/migrate_device.h
@@ -0,0 +1,324 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM migrate_device
+
+#if !defined(_TRACE_MIGRATE_DEVICE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_MIGRATE_DEVICE_H
+
+#include <linux/migrate.h>
+#include <linux/tracepoint.h>
+
+/*
+ * Define enums for tracing information.
+ */
+#ifndef __MIGRATE_DEVICE_DECLARE_TRACE_ENUMS_ONCE_ONLY
+#define __MIGRATE_DEVICE_DECLARE_TRACE_ENUMS_ONCE_ONLY
+
+/*
+ * Why a page table walk callback classified the range the way it did.
+ * Several call sites collect the same kind of range, so the reason is
+ * what distinguishes them in a trace.
+ */
+enum migrate_vma_walk_reason {
+ MIGRATE_VMA_WALK_HOLE,
+ MIGRATE_VMA_WALK_HOLE_COMPOUND,
+ MIGRATE_VMA_WALK_HUGE_PMD,
+ MIGRATE_VMA_WALK_SKIP_NOT_ANON,
+ MIGRATE_VMA_WALK_SKIP_NOT_SELECTED,
+ MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH,
+ MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED,
+ MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL,
+ MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED,
+};
+
+#endif /* __MIGRATE_DEVICE_DECLARE_TRACE_ENUMS_ONCE_ONLY */
+
+TRACE_DEFINE_ENUM(MIGRATE_VMA_SELECT_SYSTEM);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_SELECT_DEVICE_PRIVATE);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_SELECT_DEVICE_COHERENT);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_SELECT_COMPOUND);
+
+TRACE_DEFINE_ENUM(MIGRATE_VMA_WALK_HOLE);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_WALK_HOLE_COMPOUND);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_WALK_HUGE_PMD);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_WALK_SKIP_NOT_ANON);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_WALK_SKIP_NOT_SELECTED);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL);
+TRACE_DEFINE_ENUM(MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED);
+
+#define show_walk_reason(reason) \
+ __print_symbolic(reason, \
+ { MIGRATE_VMA_WALK_HOLE, "HOLE" }, \
+ { MIGRATE_VMA_WALK_HOLE_COMPOUND, "HOLE_COMPOUND" }, \
+ { MIGRATE_VMA_WALK_HUGE_PMD, "HUGE_PMD" }, \
+ { MIGRATE_VMA_WALK_SKIP_NOT_ANON, "NOT_ANON" }, \
+ { MIGRATE_VMA_WALK_SKIP_NOT_SELECTED, "NOT_SELECTED" }, \
+ { MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH, "OWNER_MISMATCH" }, \
+ { MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED, "LOCK_CONTENDED" }, \
+ { MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL, "COMPOUND_TAIL" }, \
+ { MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED, "SPLIT_FAILED" })
+
+#define show_migrate_vma_flags(flags) \
+ __print_flags(flags, "|", \
+ { MIGRATE_VMA_SELECT_SYSTEM, "SYSTEM" }, \
+ { MIGRATE_VMA_SELECT_DEVICE_PRIVATE, "DEVICE_PRIVATE" }, \
+ { MIGRATE_VMA_SELECT_DEVICE_COHERENT, "DEVICE_COHERENT" }, \
+ { MIGRATE_VMA_SELECT_COMPOUND, "COMPOUND" })
+
+DECLARE_EVENT_CLASS(migrate_vma_range_class,
+
+ TP_PROTO(const struct migrate_vma *migrate),
+
+ TP_ARGS(migrate),
+
+ TP_STRUCT__entry(
+ __field(void *, vma)
+ __field(void *, mm)
+ __field(void *, src)
+ __field(void *, dst)
+ __field(void *, pgmap_owner)
+ __field(void *, fault_page)
+ __field(unsigned long, start)
+ __field(unsigned long, end)
+ __field(unsigned long, npages)
+ __field(unsigned long, cpages)
+ __field(unsigned long, flags)
+ ),
+
+ TP_fast_assign(
+ __entry->vma = migrate->vma;
+ __entry->mm = migrate->vma ? migrate->vma->vm_mm : NULL;
+ __entry->src = migrate->src;
+ __entry->dst = migrate->dst;
+ __entry->pgmap_owner = migrate->pgmap_owner;
+ __entry->fault_page = migrate->fault_page;
+ __entry->start = migrate->start;
+ __entry->end = migrate->end;
+ __entry->npages = migrate->npages;
+ __entry->cpages = migrate->cpages;
+ __entry->flags = migrate->flags;
+ ),
+
+ TP_printk("mm=%p vma=%p range=%#lx-%#lx bytes=%lu npages=%lu cpages=%lu flags=%s src=%p dst=%p owner=%p fault_page=%p",
+ __entry->mm, __entry->vma, __entry->start, __entry->end,
+ __entry->end - __entry->start, __entry->npages,
+ __entry->cpages,
+ show_migrate_vma_flags(__entry->flags),
+ __entry->src, __entry->dst, __entry->pgmap_owner,
+ __entry->fault_page)
+);
+
+DEFINE_EVENT(migrate_vma_range_class, migrate_vma_setup_start,
+ TP_PROTO(const struct migrate_vma *migrate),
+ TP_ARGS(migrate));
+
+DEFINE_EVENT(migrate_vma_range_class, migrate_vma_setup_done,
+ TP_PROTO(const struct migrate_vma *migrate),
+ TP_ARGS(migrate));
+
+DEFINE_EVENT(migrate_vma_range_class, migrate_vma_collect_start,
+ TP_PROTO(const struct migrate_vma *migrate),
+ TP_ARGS(migrate));
+
+DEFINE_EVENT(migrate_vma_range_class, migrate_vma_collect_done,
+ TP_PROTO(const struct migrate_vma *migrate),
+ TP_ARGS(migrate));
+
+DEFINE_EVENT(migrate_vma_range_class, migrate_vma_pages_start,
+ TP_PROTO(const struct migrate_vma *migrate),
+ TP_ARGS(migrate));
+
+DEFINE_EVENT(migrate_vma_range_class, migrate_vma_pages_done,
+ TP_PROTO(const struct migrate_vma *migrate),
+ TP_ARGS(migrate));
+
+DEFINE_EVENT(migrate_vma_range_class, migrate_vma_finalize_start,
+ TP_PROTO(const struct migrate_vma *migrate),
+ TP_ARGS(migrate));
+
+DEFINE_EVENT(migrate_vma_range_class, migrate_vma_finalize_done,
+ TP_PROTO(const struct migrate_vma *migrate),
+ TP_ARGS(migrate));
+
+/*
+ * Page table walk callbacks act on a sub-range of the migration and can be
+ * reached from several call sites, so they report the range they were handed
+ * and why, in addition to the migrate-wide state.
+ */
+DECLARE_EVENT_CLASS(migrate_vma_walk_class,
+
+ TP_PROTO(const struct migrate_vma *migrate, unsigned long start,
+ unsigned long end, enum migrate_vma_walk_reason reason),
+
+ TP_ARGS(migrate, start, end, reason),
+
+ TP_STRUCT__entry(
+ __field(void *, vma)
+ __field(void *, mm)
+ __field(void *, src)
+ __field(void *, dst)
+ __field(void *, pgmap_owner)
+ __field(void *, fault_page)
+ __field(unsigned long, walk_start)
+ __field(unsigned long, walk_end)
+ __field(unsigned long, start)
+ __field(unsigned long, end)
+ __field(unsigned long, npages)
+ __field(unsigned long, cpages)
+ __field(unsigned long, flags)
+ __field(unsigned int, reason)
+ ),
+
+ TP_fast_assign(
+ __entry->vma = migrate->vma;
+ __entry->mm = migrate->vma ? migrate->vma->vm_mm : NULL;
+ __entry->src = migrate->src;
+ __entry->dst = migrate->dst;
+ __entry->pgmap_owner = migrate->pgmap_owner;
+ __entry->fault_page = migrate->fault_page;
+ __entry->walk_start = start;
+ __entry->walk_end = end;
+ __entry->start = migrate->start;
+ __entry->end = migrate->end;
+ __entry->npages = migrate->npages;
+ __entry->cpages = migrate->cpages;
+ __entry->flags = migrate->flags;
+ __entry->reason = reason;
+ ),
+
+ TP_printk("mm=%p vma=%p walk=%#lx-%#lx bytes=%lu reason=%s range=%#lx-%#lx npages=%lu cpages=%lu flags=%s src=%p dst=%p owner=%p fault_page=%p",
+ __entry->mm, __entry->vma,
+ __entry->walk_start, __entry->walk_end,
+ __entry->walk_end - __entry->walk_start,
+ show_walk_reason(__entry->reason),
+ __entry->start, __entry->end, __entry->npages,
+ __entry->cpages,
+ show_migrate_vma_flags(__entry->flags),
+ __entry->src, __entry->dst, __entry->pgmap_owner,
+ __entry->fault_page)
+);
+
+DEFINE_EVENT(migrate_vma_walk_class, migrate_vma_collect_skip,
+ TP_PROTO(const struct migrate_vma *migrate, unsigned long start,
+ unsigned long end, enum migrate_vma_walk_reason reason),
+ TP_ARGS(migrate, start, end, reason));
+
+DEFINE_EVENT(migrate_vma_walk_class, migrate_vma_collect_hole,
+ TP_PROTO(const struct migrate_vma *migrate, unsigned long start,
+ unsigned long end, enum migrate_vma_walk_reason reason),
+ TP_ARGS(migrate, start, end, reason));
+
+DEFINE_EVENT(migrate_vma_walk_class, migrate_vma_collect_huge_pmd,
+ TP_PROTO(const struct migrate_vma *migrate, unsigned long start,
+ unsigned long end, enum migrate_vma_walk_reason reason),
+ TP_ARGS(migrate, start, end, reason));
+
+DECLARE_EVENT_CLASS(migrate_device_batch_class,
+
+ TP_PROTO(const unsigned long *src, const unsigned long *dst,
+ unsigned long npages),
+
+ TP_ARGS(src, dst, npages),
+
+ TP_STRUCT__entry(
+ __field(void *, src)
+ __field(void *, dst)
+ __field(unsigned long, src_head)
+ __field(unsigned long, dst_head)
+ __field(unsigned long, npages)
+ ),
+
+ TP_fast_assign(
+ __entry->src = (void *)src;
+ __entry->dst = (void *)dst;
+ __entry->src_head = npages && src ? src[0] : 0;
+ __entry->dst_head = npages && dst ? dst[0] : 0;
+ __entry->npages = npages;
+ ),
+
+ TP_printk("src=%p dst=%p npages=%lu src_head=%#lx dst_head=%#lx",
+ __entry->src, __entry->dst, __entry->npages,
+ __entry->src_head, __entry->dst_head)
+);
+
+DEFINE_EVENT(migrate_device_batch_class, migrate_device_unmap_done,
+ TP_PROTO(const unsigned long *src, const unsigned long *dst,
+ unsigned long npages),
+ TP_ARGS(src, dst, npages));
+
+DEFINE_EVENT(migrate_device_batch_class, migrate_device_pages_start,
+ TP_PROTO(const unsigned long *src, const unsigned long *dst,
+ unsigned long npages),
+ TP_ARGS(src, dst, npages));
+
+DEFINE_EVENT(migrate_device_batch_class, migrate_device_pages_done,
+ TP_PROTO(const unsigned long *src, const unsigned long *dst,
+ unsigned long npages),
+ TP_ARGS(src, dst, npages));
+
+DEFINE_EVENT(migrate_device_batch_class, migrate_device_finalize_start,
+ TP_PROTO(const unsigned long *src, const unsigned long *dst,
+ unsigned long npages),
+ TP_ARGS(src, dst, npages));
+
+DEFINE_EVENT(migrate_device_batch_class, migrate_device_finalize_done,
+ TP_PROTO(const unsigned long *src, const unsigned long *dst,
+ unsigned long npages),
+ TP_ARGS(src, dst, npages));
+
+DECLARE_EVENT_CLASS(migrate_device_folio_class,
+
+ TP_PROTO(unsigned long index, unsigned long nr_pages,
+ unsigned long src, unsigned long dst),
+
+ TP_ARGS(index, nr_pages, src, dst),
+
+ TP_STRUCT__entry(
+ __field(unsigned long, index)
+ __field(unsigned long, nr_pages)
+ __field(unsigned long, src)
+ __field(unsigned long, dst)
+ __field(unsigned long, src_pfn)
+ __field(unsigned long, dst_pfn)
+ ),
+
+ TP_fast_assign(
+ __entry->index = index;
+ __entry->nr_pages = nr_pages;
+ __entry->src = src;
+ __entry->dst = dst;
+ __entry->src_pfn = src >> MIGRATE_PFN_SHIFT;
+ __entry->dst_pfn = dst >> MIGRATE_PFN_SHIFT;
+ ),
+
+ TP_printk("index=%lu nr_pages=%lu src=%#lx src_pfn=%#lx src_flags=%s dst=%#lx dst_pfn=%#lx dst_flags=%s",
+ __entry->index, __entry->nr_pages, __entry->src,
+ __entry->src_pfn,
+ __print_flags(__entry->src & ((1UL << MIGRATE_PFN_SHIFT) - 1),
+ "|",
+ { MIGRATE_PFN_VALID, "VALID" },
+ { MIGRATE_PFN_MIGRATE, "MIGRATE" },
+ { MIGRATE_PFN_WRITE, "WRITE" },
+ { MIGRATE_PFN_COMPOUND, "COMPOUND" }),
+ __entry->dst, __entry->dst_pfn,
+ __print_flags(__entry->dst & ((1UL << MIGRATE_PFN_SHIFT) - 1),
+ "|",
+ { MIGRATE_PFN_VALID, "VALID" },
+ { MIGRATE_PFN_MIGRATE, "MIGRATE" },
+ { MIGRATE_PFN_WRITE, "WRITE" },
+ { MIGRATE_PFN_COMPOUND, "COMPOUND" }))
+);
+
+DEFINE_EVENT(migrate_device_folio_class, migrate_device_folio_finalize,
+ TP_PROTO(unsigned long index, unsigned long nr_pages,
+ unsigned long src, unsigned long dst),
+ TP_ARGS(index, nr_pages, src, dst));
+
+#endif /* _TRACE_MIGRATE_DEVICE_H */
+
+#include <trace/define_trace.h>
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 009bfa8b212d..cf2a78d2e18e 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -16,11 +16,16 @@
#include <linux/leafops.h>
#include <linux/pgalloc.h>
#include <asm/tlbflush.h>
+
+#define CREATE_TRACE_POINTS
+#include <trace/events/migrate_device.h>
+
#include "internal.h"
static int migrate_vma_collect_skip(unsigned long start,
unsigned long end,
- struct mm_walk *walk)
+ struct mm_walk *walk,
+ enum migrate_vma_walk_reason reason)
{
struct migrate_vma *migrate = walk->private;
unsigned long addr;
@@ -30,6 +35,7 @@ static int migrate_vma_collect_skip(unsigned long start,
migrate->src[migrate->npages++] = 0;
}
+ trace_migrate_vma_collect_skip(migrate, start, end, reason);
return 0;
}
@@ -43,7 +49,8 @@ static int migrate_vma_collect_hole(unsigned long start,
/* Only allow populating anonymous memory. */
if (!vma_is_anonymous(walk->vma))
- return migrate_vma_collect_skip(start, end, walk);
+ return migrate_vma_collect_skip(start, end, walk,
+ MIGRATE_VMA_WALK_SKIP_NOT_ANON);
if (thp_migration_supported() &&
(migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) &&
@@ -55,11 +62,15 @@ static int migrate_vma_collect_hole(unsigned long start,
migrate->npages++;
migrate->cpages++;
+ trace_migrate_vma_collect_hole(migrate, start, end,
+ MIGRATE_VMA_WALK_HOLE_COMPOUND);
+
/*
* Collect the remaining entries as holes, in case we
* need to split later
*/
- return migrate_vma_collect_skip(start + PAGE_SIZE, end, walk);
+ return migrate_vma_collect_skip(start + PAGE_SIZE, end, walk,
+ MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL);
}
for (addr = start; addr < end; addr += PAGE_SIZE) {
@@ -69,6 +80,8 @@ static int migrate_vma_collect_hole(unsigned long start,
migrate->cpages++;
}
+ trace_migrate_vma_collect_hole(migrate, start, end,
+ MIGRATE_VMA_WALK_HOLE);
return 0;
}
@@ -154,7 +167,8 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,
if (pmd_trans_huge(*pmdp)) {
if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) {
spin_unlock(ptl);
- return migrate_vma_collect_skip(start, end, walk);
+ return migrate_vma_collect_skip(start, end, walk,
+ MIGRATE_VMA_WALK_SKIP_NOT_SELECTED);
}
folio = pmd_folio(*pmdp);
@@ -170,13 +184,15 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,
if (!softleaf_is_device_private(entry) ||
!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE)) {
spin_unlock(ptl);
- return migrate_vma_collect_skip(start, end, walk);
+ return migrate_vma_collect_skip(start, end, walk,
+ MIGRATE_VMA_WALK_SKIP_NOT_SELECTED);
}
folio = softleaf_to_folio(entry);
if (folio->pgmap->owner != migrate->pgmap_owner) {
spin_unlock(ptl);
- return migrate_vma_collect_skip(start, end, walk);
+ return migrate_vma_collect_skip(start, end, walk,
+ MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH);
}
if (softleaf_is_device_private_write(entry))
@@ -190,7 +206,8 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,
if (folio != fault_folio && unlikely(!folio_trylock(folio))) {
spin_unlock(ptl);
folio_put(folio);
- return migrate_vma_collect_skip(start, end, walk);
+ return migrate_vma_collect_skip(start, end, walk,
+ MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED);
}
if (thp_migration_supported() &&
@@ -220,8 +237,12 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,
migrate->dst[migrate->npages] = 0;
goto fallback;
}
- migrate_vma_collect_skip(start + PAGE_SIZE, end, walk);
+ migrate_vma_collect_skip(start + PAGE_SIZE, end, walk,
+ MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL);
spin_unlock(ptl);
+
+ trace_migrate_vma_collect_huge_pmd(migrate, start, end,
+ MIGRATE_VMA_WALK_HUGE_PMD);
return 0;
}
@@ -234,7 +255,8 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,
folio_unlock(folio);
folio_put(folio);
if (ret)
- return migrate_vma_collect_skip(start, end, walk);
+ return migrate_vma_collect_skip(start, end, walk,
+ MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED);
if (pmd_none(pmdp_get_lockless(pmdp)))
return migrate_vma_collect_hole(start, end, -1, walk);
@@ -323,7 +345,9 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
if (unmapped)
flush_tlb_range(walk->vma, start, end);
- return migrate_vma_collect_skip(addr, end, walk);
+ return migrate_vma_collect_skip(addr,
+ end, walk,
+ MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED);
}
goto again;
@@ -369,7 +393,9 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
if (unmapped)
flush_tlb_range(walk->vma, start, end);
- return migrate_vma_collect_skip(addr, end, walk);
+ return migrate_vma_collect_skip(addr,
+ end, walk,
+ MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED);
}
goto again;
@@ -514,6 +540,8 @@ static void migrate_vma_collect(struct migrate_vma *migrate)
{
struct mmu_notifier_range range;
+ trace_migrate_vma_collect_start(migrate);
+
/*
* Note that the pgmap_owner is passed to the mmu notifier callback so
* that the registered device driver can skip invalidating device
@@ -529,6 +557,7 @@ static void migrate_vma_collect(struct migrate_vma *migrate)
mmu_notifier_invalidate_range_end(&range);
migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
+ trace_migrate_vma_collect_done(migrate);
}
/*
@@ -653,6 +682,7 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
restore--;
}
+ trace_migrate_device_unmap_done(src_pfns, NULL, npages);
return unmapped;
}
@@ -763,11 +793,13 @@ int migrate_vma_setup(struct migrate_vma *args)
memset(args->src, 0, sizeof(*args->src) * nr_pages);
args->cpages = 0;
args->npages = 0;
+ trace_migrate_vma_setup_start(args);
migrate_vma_collect(args);
if (args->cpages)
migrate_vma_unmap(args);
+ trace_migrate_vma_setup_done(args);
/*
* At this point pages are locked and unmapped, and thus they have
@@ -1115,7 +1147,9 @@ static void __migrate_device_pages(unsigned long *src_pfns,
bool notified = false;
unsigned long addr;
- for (i = 0; i < npages; ) {
+ trace_migrate_device_pages_start(src_pfns, dst_pfns, npages);
+
+ for (i = 0; i < npages;) {
struct page *newpage = migrate_pfn_to_page(dst_pfns[i]);
struct page *page = migrate_pfn_to_page(src_pfns[i]);
struct address_space *mapping;
@@ -1257,6 +1291,8 @@ static void __migrate_device_pages(unsigned long *src_pfns,
if (notified)
mmu_notifier_invalidate_range_end(&range);
+
+ trace_migrate_device_pages_done(src_pfns, dst_pfns, npages);
}
/**
@@ -1285,7 +1321,9 @@ EXPORT_SYMBOL(migrate_device_pages);
*/
void migrate_vma_pages(struct migrate_vma *migrate)
{
+ trace_migrate_vma_pages_start(migrate);
__migrate_device_pages(migrate->src, migrate->dst, migrate->npages, migrate);
+ trace_migrate_vma_pages_done(migrate);
}
EXPORT_SYMBOL(migrate_vma_pages);
@@ -1298,11 +1336,24 @@ static void __migrate_device_finalize(unsigned long *src_pfns,
page_folio(fault_page) : NULL;
unsigned long i;
+ trace_migrate_device_finalize_start(src_pfns, dst_pfns, npages);
+
for (i = 0; i < npages; i++) {
struct folio *dst = NULL, *src = NULL;
struct page *newpage = migrate_pfn_to_page(dst_pfns[i]);
struct page *page = migrate_pfn_to_page(src_pfns[i]);
+ if (trace_migrate_device_folio_finalize_enabled() &&
+ (page || newpage)) {
+ unsigned long nr_pages = page ?
+ folio_nr_pages(page_folio(page)) :
+ folio_nr_pages(page_folio(newpage));
+
+ trace_migrate_device_folio_finalize(i, nr_pages,
+ src_pfns[i],
+ dst_pfns[i]);
+ }
+
if (newpage)
dst = page_folio(newpage);
@@ -1339,6 +1390,8 @@ static void __migrate_device_finalize(unsigned long *src_pfns,
folio_put(dst);
}
}
+
+ trace_migrate_device_finalize_done(src_pfns, dst_pfns, npages);
}
/*
@@ -1371,8 +1424,10 @@ EXPORT_SYMBOL(migrate_device_finalize);
*/
void migrate_vma_finalize(struct migrate_vma *migrate)
{
+ trace_migrate_vma_finalize_start(migrate);
__migrate_device_finalize(migrate->src, migrate->dst, migrate->npages,
migrate->fault_page);
+ trace_migrate_vma_finalize_done(migrate);
}
EXPORT_SYMBOL(migrate_vma_finalize);
--
2.55.0