[PATCH 18/25] mm/fbatch: remove lru_add_drain() and _all() calls from various
From: Hugh Dickins
Date: Mon Aug 24 2026 - 10:44:32 EST
Splitting into little patches gets tedious: now that folios on per-cpu
fbatches no longer hold an extra reference (and have the lru flag set),
most calls to lru_add_drain(), and more importantly lru_add_drain_all(),
should be removed.
Remove lru_add_drain() from compact_zone().
Remove lru_add_drain_all() from compact_nodes() and compact_store().
Keep lru_add_drain_cpu_zone() in compact_zone(): it was always a bit of
a hack, a cheap way to get a local_lock() around pcp drain_pages_zone().
Remove lru_add_drain() and lru_add_drain_all() from generic_fadvise(
POSIX_FADV_DONTNEED), but still retry once on failure; remove outdated
comment line from mapping_try_invalidate().
Remove every lru_add_drain() and lru_add_drain_all() from khugepaged.c.
Remove lru_add_drain_all() from KSM's scan_get_next_rmap_item() restart.
Remove lru_add_drain() from wp_can_reuse_anon_folio() and do_swap_page().
Remove lru_add_drain(), lru_add_drain_all() from migrate_device_unmap().
Keep lru_add_drain()s in mm/gup.c populate_vma_page_range() and
faultin_page_range(): good housekeeping after a bulk operation.
Keep lru_add_drain()s in mm/madvise.c: the ones after a bulk op probably
do want to "Push any new pages onto the LRU now", and update the stats;
the ones before a bulk op may be trying to stabilize initial conditions,
or to minimize draining under ptlock.
Keep lru_add_drain_all() in memcg-v1's mem_cgroup_force_empty():
it probably does want to push onto the LRU and update the stats.
Keep lru_add_drain() after bulk op in mlock_vma_pages_range(), partly
for "Unevictable kB" accuracy; but remove the lru_add_drain() before it.
Keep lru_add_drain()s in swap_cluster_readahead(), swap_vma_readahead():
they do want to "Push any new pages onto the LRU now" (akpm 2.5.46).
Keep lru_add_drain()s and lru_add_drain_all() throughout mm/vmscan.c:
it works on LRUs, so it does need folios to be on an actual LRU.
Signed-off-by: Hugh Dickins <hughd@xxxxxxxxxx>
---
mm/compaction.c | 12 +-----------
mm/fadvise.c | 17 +----------------
mm/khugepaged.c | 11 -----------
mm/ksm.c | 12 ------------
mm/memory.c | 14 ++------------
mm/migrate_device.c | 9 ---------
mm/mlock.c | 1 -
mm/truncate.c | 1 -
8 files changed, 4 insertions(+), 73 deletions(-)
diff --git a/mm/compaction.c b/mm/compaction.c
index 9e045a90ba21..8ed375b08f3e 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -2652,9 +2652,6 @@ compact_zone(struct compact_control *cc, struct capture_control *capc)
trace_mm_compaction_begin(cc, start_pfn, end_pfn, sync);
- /* lru_add_drain_all could be expensive with involving other CPUs */
- lru_add_drain();
-
while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
int err;
unsigned long iteration_start_pfn = cc->migrate_pfn;
@@ -2969,9 +2966,6 @@ static int compact_nodes(void)
{
int ret, nid;
- /* Flush pending updates to the LRU lists */
- lru_add_drain_all();
-
for_each_online_node(nid) {
ret = compact_node(NODE_DATA(nid), false);
if (ret)
@@ -3036,12 +3030,8 @@ static ssize_t compact_store(struct device *dev,
{
int nid = dev->id;
- if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
- /* Flush pending updates to the LRU lists */
- lru_add_drain_all();
-
+ if (nid >= 0 && nid < nr_node_ids && node_online(nid))
compact_node(NODE_DATA(nid), false);
- }
return count;
}
diff --git a/mm/fadvise.c b/mm/fadvise.c
index b63fe21416ff..f788a5020384 100644
--- a/mm/fadvise.c
+++ b/mm/fadvise.c
@@ -143,27 +143,12 @@ int generic_fadvise(struct file *file, loff_t offset, loff_t len, int advice)
if (end_index >= start_index) {
unsigned long nr_failed = 0;
- /*
- * It's common to FADV_DONTNEED right after
- * the read or write that instantiates the
- * pages, in which case there will be some
- * sitting on the local LRU cache. Try to
- * avoid the expensive remote drain and the
- * second cache tree walk below by flushing
- * them out right away.
- */
- lru_add_drain();
-
mapping_try_invalidate(mapping, start_index, end_index,
&nr_failed);
-
/*
- * The failures may be due to the folio being
- * in the LRU cache of a remote CPU. Drain all
- * caches and try again.
+ * Retry if any failures, in case they were transient.
*/
if (nr_failed) {
- lru_add_drain_all();
invalidate_mapping_pages(mapping, start_index,
end_index);
}
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index b237f6e7662a..904a45c11fca 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1225,10 +1225,6 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm,
if (pte)
pte_unmap(pte);
- /* Drain LRU cache to remove extra pin on the swapped in pages */
- if (swapped_in)
- lru_add_drain();
-
result = SCAN_SUCCEED;
out:
trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, result,
@@ -2325,8 +2321,6 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
result = SCAN_FAIL;
goto xa_unlocked;
}
- /* drain lru cache to help folio_isolate_lru() */
- lru_add_drain();
} else if (folio_trylock(folio)) {
folio_get(folio);
xas_unlock_irq(&xas);
@@ -2340,8 +2334,6 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
page_cache_sync_readahead(mapping, &file->f_ra,
file, index,
end - index);
- /* drain lru cache to help folio_isolate_lru() */
- lru_add_drain();
folio = filemap_lock_folio(mapping, index);
if (IS_ERR(folio)) {
result = SCAN_FAIL;
@@ -2981,8 +2973,6 @@ static void khugepaged_do_scan(struct collapse_control *cc)
bool wait = true;
enum scan_result result = SCAN_SUCCEED;
- lru_add_drain_all();
-
cc->progress = 0;
while (true) {
cond_resched();
@@ -3214,7 +3204,6 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
cc->progress = 0;
mmgrab(mm);
- lru_add_drain_all();
for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) {
enum scan_result result = SCAN_FAIL;
diff --git a/mm/ksm.c b/mm/ksm.c
index b4142746777e..5bae928620ad 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -2626,18 +2626,6 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page)
advisor_start_scan();
trace_ksm_start_scan(ksm_scan.seqnr, ksm_rmap_items);
- /*
- * A number of pages can hang around indefinitely in per-cpu
- * LRU cache, raised page count preventing write_protect_page
- * from merging them. Though it doesn't really matter much,
- * it is puzzling to see some stuck in pages_volatile until
- * other activity jostles them out, and they also prevented
- * LTP's KSM test from succeeding deterministically; so drain
- * them here (here rather than on entry to ksm_do_scan(),
- * so we don't IPI too often when pages_to_scan is set low).
- */
- lru_add_drain_all();
-
/*
* Whereas stale stable_nodes on the stable_tree itself
* get pruned in the regular course of stable_tree_search(),
diff --git a/mm/memory.c b/mm/memory.c
index 8da0f945141b..1478bee69e30 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4251,9 +4251,6 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio,
static bool wp_can_reuse_anon_folio(struct folio *folio,
struct vm_area_struct *vma)
{
- const bool maybe_in_lru_cache = !folio_test_lru(folio);
- const bool in_swapcache = folio_test_swapcache(folio);
-
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio))
return __wp_can_reuse_large_anon_folio(folio, vma);
@@ -4264,16 +4261,9 @@ static bool wp_can_reuse_anon_folio(struct folio *folio,
*
* KSM doesn't necessarily raise the folio refcount.
*/
- if (folio_test_ksm(folio) ||
- folio_ref_count(folio) > 1 + maybe_in_lru_cache + in_swapcache)
+ if (folio_test_ksm(folio))
return false;
- if (maybe_in_lru_cache)
- /*
- * We cannot easily detect+handle references from
- * remote LRU caches or references to LRU folios.
- */
- lru_add_drain();
- if (folio_ref_count(folio) > 1 + in_swapcache)
+ if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio))
return false;
if (!folio_trylock(folio))
return false;
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 18d097c38853..d3f865904c48 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -575,11 +575,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
struct folio *fault_folio = fault_page ?
page_folio(fault_page) : NULL;
unsigned long i, restore = 0;
- bool allow_drain = true;
unsigned long unmapped = 0;
- lru_add_drain();
-
for (i = 0; i < npages; ) {
struct page *page = migrate_pfn_to_page(src_pfns[i]);
struct folio *folio;
@@ -600,12 +597,6 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns,
/* ZONE_DEVICE folios are not on LRU */
if (!folio_is_zone_device(folio)) {
- if (!folio_test_lru(folio) && allow_drain) {
- /* Drain CPU's lru cache */
- lru_add_drain_all();
- allow_drain = false;
- }
-
if (!folio_isolate_lru(folio)) {
src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
restore++;
diff --git a/mm/mlock.c b/mm/mlock.c
index 97134eff6b56..971430e6251e 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -424,7 +424,6 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma,
vma_start_write(vma);
vma_flags_reset_once(vma, new_vma_flags);
- lru_add_drain();
walk_page_range_vma(vma, start, end, &mlock_walk_ops, NULL);
lru_add_drain();
diff --git a/mm/truncate.c b/mm/truncate.c
index 4151f7a167e3..7ea5d513f76a 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -571,7 +571,6 @@ unsigned long mapping_try_invalidate(struct address_space *mapping,
*/
if (!ret) {
deactivate_file_folio(folio);
- /* Likely in the lru cache of a remote CPU */
if (nr_failed)
(*nr_failed)++;
}
--
2.51.0