[PATCH] riscv: mm: Flush vmemmap TLB after sparse_init()

[Yaxing Guo]

Commit 4267739cabb8 ("arch, mm: consolidate initialization of SPARSE
memory model") moved sparse_init() from RISC-V arch setup code into
free_area_init(), but left the RISC-V VMEMMAP TLB flush in
misc_mem_init(). RISC-V now flushes the VMEMMAP range before
sparse_init() populates the vmemmap page tables.

RISC-V supports caching non-present entries in the TLB. If a hart cached
a non-present translation for the VMEMMAP range, installing the vmemmap
page-table entries is not guaranteed to be observed until an sfence.vma
invalidates the stale entry. The first access to the corresponding
struct page during memmap initialization can then hit the old non-present
entry.

Call an arch hook at the end of sparse_init(), after the boot-time
vmemmap population has completed. On RISC-V, use flush_cache_vmap_early()
for the hook. Unlike flush_cache_vmap(), which can defer RISC-V
invalidation through the new_vmalloc lazy fault path,
flush_cache_vmap_early() maps to local_flush_tlb_kernel_range() and
performs the local sfence.vma needed before the boot hart accesses the
new mappings.

Observed boot failure:

  Unable to handle kernel paging request at virtual address ffffffd8002200038
  Current swapper pgtable: 4K pagesize, 48-bit VAs, pgdp=0000000083006000
  [ffff8d8002200038] pgd=0000000081a74401, p4d=0000000081a74401, pud=0000000081a74401, pmd=000000007fe800e7
  Oops [#1]
  CPU: 0 UID: 0 PID: 0 Comm: swapper Not tainted 7.1.9-g3846b7652eb2-dirty #18 PREEMPTLAZY

  epc : __init_single_page+0x2a/0x128
  ra  : __init_single_page+0x26/0x128
  status: 0000000200000100 badaddr: ffffffd8002200038 cause: 000000000000000f
  [<ffffffff81621e04>] __init_single_page+0x2a/0x128
  [<ffffffff8162286c>] memmap_init_range+0x218/0x284
  [<ffffffff81624e68>] mm_core_init_early+0xe2c/0xf6e
  [<ffffffff81601188>] start_kernel+0x7a/0x612
  Code: 0385 3c23 fac4 8ab6 1097 feea 80e7 3580 8513 0304 (bc23) 0204
  Kernel panic - not syncing: Attempted to kill the idle task!

Fixes: 4267739cabb8 ("arch, mm: consolidate initialization of SPARSE memory model")
Link: https://lore.kernel.org/linux-riscv/7B0CB4ED-C190-4F49-97EE-E1F14F08DED2@xxxxxxxxx/
Co-developed-by: Zhizun Wang <anzoso@xxxxxxxxxxx>
Signed-off-by: Zhizun Wang <anzoso@xxxxxxxxxxx>
Signed-off-by: Yaxing Guo <guoyaxing@xxxxxxxxxx>
---
 arch/riscv/mm/init.c | 13 +++++++++----
 include/linux/mm.h   |  1 +
 mm/sparse.c          |  5 +++++
 3 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index decd7df40fa4..200a058e80ca 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -19,6 +19,7 @@
 #include <linux/set_memory.h>
 #include <linux/dma-map-ops.h>
 #include <linux/crash_dump.h>
+#include <linux/cacheflush.h>
 #include <linux/hugetlb.h>
 #include <linux/kfence.h>
 #include <linux/execmem.h>
@@ -1325,14 +1326,18 @@ void __init misc_mem_init(void)
 {
 	early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT);
 	arch_numa_init();
-#ifdef CONFIG_SPARSEMEM_VMEMMAP
-	/* The entire VMEMMAP region has been populated. Flush TLB for this region */
-	local_flush_tlb_kernel_range(VMEMMAP_START, VMEMMAP_END);
-#endif
 	arch_reserve_crashkernel();
 	memblock_dump_all();
 }
 
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+void __init arch_post_sparse_init(void)
+{
+	/* The entire VMEMMAP region has been populated by sparse_init(). */
+	flush_cache_vmap_early(VMEMMAP_START, VMEMMAP_END);
+}
+#endif
+
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 void __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
 			       unsigned long addr, unsigned long next)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index af23453e9dbd..5a75b69296cd 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -46,6 +46,7 @@ struct pt_regs;
 struct folio_batch;
 
 void arch_mm_preinit(void);
+void arch_post_sparse_init(void);
 void mm_core_init_early(void);
 void mm_core_init(void);
 void init_mm_internals(void);
diff --git a/mm/sparse.c b/mm/sparse.c
index effdac6b0ab1..ecf1b2510fb3 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -309,6 +309,10 @@ void __weak __meminit vmemmap_populate_print_last(void)
 {
 }
 
+void __weak __init arch_post_sparse_init(void)
+{
+}
+
 static void *sparse_usagebuf __meminitdata;
 static void *sparse_usagebuf_end __meminitdata;
 
@@ -446,4 +450,5 @@ void __init sparse_init(void)
 	/* cover the last node */
 	sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
 	vmemmap_populate_print_last();
+	arch_post_sparse_init();
 }
-- 
2.43.0