[PATCH] mm: khugepaged: simplify scanning progress
From: Vernon Yang
Date: Thu Feb 26 2026 - 05:24:21 EST
Placing "progress" inside "struct collapse_control" makes the overall
code simpler, there also coincidentally has a 4-bytes hole, as shown
below:
struct collapse_control {
bool is_khugepaged; /* 0 1 */
/* XXX 3 bytes hole, try to pack */
u32 node_load[64]; /* 4 256 */
/* XXX 4 bytes hole, try to pack */
/* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */
nodemask_t alloc_nmask; /* 264 8 */
/* size: 272, cachelines: 5, members: 3 */
/* sum members: 265, holes: 2, sum holes: 7 */
/* last cacheline: 16 bytes */
};
No function changes.
Signed-off-by: Vernon Yang <yanglincheng@xxxxxxxxxx>
---
mm/khugepaged.c | 78 ++++++++++++++++++++++---------------------------
1 file changed, 35 insertions(+), 43 deletions(-)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 7c1642fbe394..13b0fe50dfc5 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -70,8 +70,8 @@ static struct task_struct *khugepaged_thread __read_mostly;
static DEFINE_MUTEX(khugepaged_mutex);
/*
- * default scan 8*HPAGE_PMD_NR ptes, pmd_mapped, no_pte_table or vmas
- * every 10 second.
+ * default scan 8*HPAGE_PMD_NR ptes, pte_mapped_hugepage, pmd_mapped,
+ * no_pte_table or vmas every 10 second.
*/
static unsigned int khugepaged_pages_to_scan __read_mostly;
static unsigned int khugepaged_pages_collapsed;
@@ -104,6 +104,9 @@ struct collapse_control {
/* Num pages scanned per node */
u32 node_load[MAX_NUMNODES];
+ /* Num pages scanned (see khugepaged_pages_to_scan) */
+ unsigned int progress;
+
/* nodemask for allocation fallback */
nodemask_t alloc_nmask;
};
@@ -1246,8 +1249,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
static enum scan_result hpage_collapse_scan_pmd(struct mm_struct *mm,
struct vm_area_struct *vma, unsigned long start_addr,
- bool *mmap_locked, unsigned int *cur_progress,
- struct collapse_control *cc)
+ bool *mmap_locked, struct collapse_control *cc)
{
pmd_t *pmd;
pte_t *pte, *_pte;
@@ -1263,8 +1265,7 @@ static enum scan_result hpage_collapse_scan_pmd(struct mm_struct *mm,
result = find_pmd_or_thp_or_none(mm, start_addr, &pmd);
if (result != SCAN_SUCCEED) {
- if (cur_progress)
- *cur_progress = 1;
+ cc->progress++;
goto out;
}
@@ -1272,16 +1273,14 @@ static enum scan_result hpage_collapse_scan_pmd(struct mm_struct *mm,
nodes_clear(cc->alloc_nmask);
pte = pte_offset_map_lock(mm, pmd, start_addr, &ptl);
if (!pte) {
- if (cur_progress)
- *cur_progress = 1;
+ cc->progress++;
result = SCAN_NO_PTE_TABLE;
goto out;
}
for (addr = start_addr, _pte = pte; _pte < pte + HPAGE_PMD_NR;
_pte++, addr += PAGE_SIZE) {
- if (cur_progress)
- *cur_progress += 1;
+ cc->progress++;
pte_t pteval = ptep_get(_pte);
if (pte_none_or_zero(pteval)) {
@@ -2314,7 +2313,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
static enum scan_result hpage_collapse_scan_file(struct mm_struct *mm,
unsigned long addr, struct file *file, pgoff_t start,
- unsigned int *cur_progress, struct collapse_control *cc)
+ struct collapse_control *cc)
{
struct folio *folio = NULL;
struct address_space *mapping = file->f_mapping;
@@ -2404,12 +2403,10 @@ static enum scan_result hpage_collapse_scan_file(struct mm_struct *mm,
}
}
rcu_read_unlock();
- if (cur_progress) {
- if (result == SCAN_PTE_MAPPED_HUGEPAGE)
- *cur_progress = 1;
- else
- *cur_progress = HPAGE_PMD_NR;
- }
+ if (result == SCAN_PTE_MAPPED_HUGEPAGE)
+ cc->progress++;
+ else
+ cc->progress += HPAGE_PMD_NR;
if (result == SCAN_SUCCEED) {
if (cc->is_khugepaged &&
@@ -2425,8 +2422,8 @@ static enum scan_result hpage_collapse_scan_file(struct mm_struct *mm,
return result;
}
-static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result *result,
- struct collapse_control *cc)
+static void khugepaged_scan_mm_slot(unsigned int progress_max,
+ enum scan_result *result, struct collapse_control *cc)
__releases(&khugepaged_mm_lock)
__acquires(&khugepaged_mm_lock)
{
@@ -2434,9 +2431,8 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result
struct mm_slot *slot;
struct mm_struct *mm;
struct vm_area_struct *vma;
- int progress = 0;
+ unsigned int progress_prev = cc->progress;
- VM_BUG_ON(!pages);
lockdep_assert_held(&khugepaged_mm_lock);
*result = SCAN_FAIL;
@@ -2459,7 +2455,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result
if (unlikely(!mmap_read_trylock(mm)))
goto breakouterloop_mmap_lock;
- progress++;
+ cc->progress++;
if (unlikely(hpage_collapse_test_exit_or_disable(mm)))
goto breakouterloop;
@@ -2469,17 +2465,17 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result
cond_resched();
if (unlikely(hpage_collapse_test_exit_or_disable(mm))) {
- progress++;
+ cc->progress++;
break;
}
if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_KHUGEPAGED, PMD_ORDER)) {
- progress++;
+ cc->progress++;
continue;
}
hstart = round_up(vma->vm_start, HPAGE_PMD_SIZE);
hend = round_down(vma->vm_end, HPAGE_PMD_SIZE);
if (khugepaged_scan.address > hend) {
- progress++;
+ cc->progress++;
continue;
}
if (khugepaged_scan.address < hstart)
@@ -2488,7 +2484,6 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result
while (khugepaged_scan.address < hend) {
bool mmap_locked = true;
- unsigned int cur_progress = 0;
cond_resched();
if (unlikely(hpage_collapse_test_exit_or_disable(mm)))
@@ -2505,8 +2500,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result
mmap_read_unlock(mm);
mmap_locked = false;
*result = hpage_collapse_scan_file(mm,
- khugepaged_scan.address, file, pgoff,
- &cur_progress, cc);
+ khugepaged_scan.address, file, pgoff, cc);
fput(file);
if (*result == SCAN_PTE_MAPPED_HUGEPAGE) {
mmap_read_lock(mm);
@@ -2520,8 +2514,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result
}
} else {
*result = hpage_collapse_scan_pmd(mm, vma,
- khugepaged_scan.address, &mmap_locked,
- &cur_progress, cc);
+ khugepaged_scan.address, &mmap_locked, cc);
}
if (*result == SCAN_SUCCEED)
@@ -2529,7 +2522,6 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result
/* move to next address */
khugepaged_scan.address += HPAGE_PMD_SIZE;
- progress += cur_progress;
if (!mmap_locked)
/*
* We released mmap_lock so break loop. Note
@@ -2539,7 +2531,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result
* correct result back to caller.
*/
goto breakouterloop_mmap_lock;
- if (progress >= pages)
+ if (cc->progress >= progress_max)
goto breakouterloop;
}
}
@@ -2570,9 +2562,8 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, enum scan_result
collect_mm_slot(slot);
}
- trace_mm_khugepaged_scan(mm, progress, khugepaged_scan.mm_slot == NULL);
-
- return progress;
+ trace_mm_khugepaged_scan(mm, cc->progress - progress_prev,
+ khugepaged_scan.mm_slot == NULL);
}
static int khugepaged_has_work(void)
@@ -2588,13 +2579,14 @@ static int khugepaged_wait_event(void)
static void khugepaged_do_scan(struct collapse_control *cc)
{
- unsigned int progress = 0, pass_through_head = 0;
- unsigned int pages = READ_ONCE(khugepaged_pages_to_scan);
+ const unsigned int progress_max = READ_ONCE(khugepaged_pages_to_scan);
+ unsigned int pass_through_head = 0;
bool wait = true;
enum scan_result result = SCAN_SUCCEED;
lru_add_drain_all();
+ cc->progress = 0;
while (true) {
cond_resched();
@@ -2606,13 +2598,12 @@ static void khugepaged_do_scan(struct collapse_control *cc)
pass_through_head++;
if (khugepaged_has_work() &&
pass_through_head < 2)
- progress += khugepaged_scan_mm_slot(pages - progress,
- &result, cc);
+ khugepaged_scan_mm_slot(progress_max, &result, cc);
else
- progress = pages;
+ cc->progress = progress_max;
spin_unlock(&khugepaged_mm_lock);
- if (progress >= pages)
+ if (cc->progress >= progress_max)
break;
if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) {
@@ -2818,6 +2809,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
if (!cc)
return -ENOMEM;
cc->is_khugepaged = false;
+ cc->progress = 0;
mmgrab(mm);
lru_add_drain_all();
@@ -2852,7 +2844,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
mmap_locked = false;
*lock_dropped = true;
result = hpage_collapse_scan_file(mm, addr, file, pgoff,
- NULL, cc);
+ cc);
if (result == SCAN_PAGE_DIRTY_OR_WRITEBACK && !triggered_wb &&
mapping_can_writeback(file->f_mapping)) {
@@ -2867,7 +2859,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
fput(file);
} else {
result = hpage_collapse_scan_pmd(mm, vma, addr,
- &mmap_locked, NULL, cc);
+ &mmap_locked, cc);
}
if (!mmap_locked)
*lock_dropped = true;
--
2.51.0