[PATCH v2 1/6] riscv: Split early and final KASAN population functions
From: Alexandre Ghiti
Date: Mon Jan 23 2023 - 05:11:45 EST
This is a preliminary work that allows to make the code more
understandable.
Signed-off-by: Alexandre Ghiti <alexghiti@xxxxxxxxxxxx>
---
arch/riscv/mm/kasan_init.c | 187 +++++++++++++++++++++++--------------
1 file changed, 117 insertions(+), 70 deletions(-)
diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c
index e1226709490f..9a5211ca8368 100644
--- a/arch/riscv/mm/kasan_init.c
+++ b/arch/riscv/mm/kasan_init.c
@@ -95,23 +95,13 @@ static void __init kasan_populate_pmd(pud_t *pud, unsigned long vaddr, unsigned
}
static void __init kasan_populate_pud(pgd_t *pgd,
- unsigned long vaddr, unsigned long end,
- bool early)
+ unsigned long vaddr, unsigned long end)
{
phys_addr_t phys_addr;
pud_t *pudp, *base_pud;
unsigned long next;
- if (early) {
- /*
- * We can't use pgd_page_vaddr here as it would return a linear
- * mapping address but it is not mapped yet, but when populating
- * early_pg_dir, we need the physical address and when populating
- * swapper_pg_dir, we need the kernel virtual address so use
- * pt_ops facility.
- */
- base_pud = pt_ops.get_pud_virt(pfn_to_phys(_pgd_pfn(*pgd)));
- } else if (pgd_none(*pgd)) {
+ if (pgd_none(*pgd)) {
base_pud = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE);
memcpy(base_pud, (void *)kasan_early_shadow_pud,
sizeof(pud_t) * PTRS_PER_PUD);
@@ -130,16 +120,10 @@ static void __init kasan_populate_pud(pgd_t *pgd,
next = pud_addr_end(vaddr, end);
if (pud_none(*pudp) && IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE) {
- if (early) {
- phys_addr = __pa(((uintptr_t)kasan_early_shadow_pmd));
- set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_TABLE));
+ phys_addr = memblock_phys_alloc(PUD_SIZE, PUD_SIZE);
+ if (phys_addr) {
+ set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL));
continue;
- } else {
- phys_addr = memblock_phys_alloc(PUD_SIZE, PUD_SIZE);
- if (phys_addr) {
- set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL));
- continue;
- }
}
}
@@ -152,35 +136,22 @@ static void __init kasan_populate_pud(pgd_t *pgd,
* it entirely, memblock could allocate a page at a physical address
* where KASAN is not populated yet and then we'd get a page fault.
*/
- if (!early)
- set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pud)), PAGE_TABLE));
+ set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pud)), PAGE_TABLE));
}
static void __init kasan_populate_p4d(pgd_t *pgd,
- unsigned long vaddr, unsigned long end,
- bool early)
+ unsigned long vaddr, unsigned long end)
{
phys_addr_t phys_addr;
p4d_t *p4dp, *base_p4d;
unsigned long next;
- if (early) {
- /*
- * We can't use pgd_page_vaddr here as it would return a linear
- * mapping address but it is not mapped yet, but when populating
- * early_pg_dir, we need the physical address and when populating
- * swapper_pg_dir, we need the kernel virtual address so use
- * pt_ops facility.
- */
- base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgd)));
- } else {
- base_p4d = (p4d_t *)pgd_page_vaddr(*pgd);
- if (base_p4d == lm_alias(kasan_early_shadow_p4d)) {
- base_p4d = memblock_alloc(PTRS_PER_PUD * sizeof(p4d_t), PAGE_SIZE);
- memcpy(base_p4d, (void *)kasan_early_shadow_p4d,
- sizeof(p4d_t) * PTRS_PER_P4D);
- }
- }
+ base_p4d = (p4d_t *)pgd_page_vaddr(*pgd);
+ if (base_p4d == lm_alias(kasan_early_shadow_p4d)) {
+ base_p4d = memblock_alloc(PTRS_PER_PUD * sizeof(p4d_t), PAGE_SIZE);
+ memcpy(base_p4d, (void *)kasan_early_shadow_p4d,
+ sizeof(p4d_t) * PTRS_PER_P4D);
+ }
p4dp = base_p4d + p4d_index(vaddr);
@@ -188,20 +159,14 @@ static void __init kasan_populate_p4d(pgd_t *pgd,
next = p4d_addr_end(vaddr, end);
if (p4d_none(*p4dp) && IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) {
- if (early) {
- phys_addr = __pa(((uintptr_t)kasan_early_shadow_pud));
- set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_TABLE));
+ phys_addr = memblock_phys_alloc(P4D_SIZE, P4D_SIZE);
+ if (phys_addr) {
+ set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL));
continue;
- } else {
- phys_addr = memblock_phys_alloc(P4D_SIZE, P4D_SIZE);
- if (phys_addr) {
- set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL));
- continue;
- }
}
}
- kasan_populate_pud((pgd_t *)p4dp, vaddr, next, early);
+ kasan_populate_pud((pgd_t *)p4dp, vaddr, next);
} while (p4dp++, vaddr = next, vaddr != end);
/*
@@ -210,8 +175,7 @@ static void __init kasan_populate_p4d(pgd_t *pgd,
* it entirely, memblock could allocate a page at a physical address
* where KASAN is not populated yet and then we'd get a page fault.
*/
- if (!early)
- set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_p4d)), PAGE_TABLE));
+ set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_p4d)), PAGE_TABLE));
}
#define kasan_early_shadow_pgd_next (pgtable_l5_enabled ? \
@@ -219,16 +183,15 @@ static void __init kasan_populate_p4d(pgd_t *pgd,
(pgtable_l4_enabled ? \
(uintptr_t)kasan_early_shadow_pud : \
(uintptr_t)kasan_early_shadow_pmd))
-#define kasan_populate_pgd_next(pgdp, vaddr, next, early) \
+#define kasan_populate_pgd_next(pgdp, vaddr, next) \
(pgtable_l5_enabled ? \
- kasan_populate_p4d(pgdp, vaddr, next, early) : \
+ kasan_populate_p4d(pgdp, vaddr, next) : \
(pgtable_l4_enabled ? \
- kasan_populate_pud(pgdp, vaddr, next, early) : \
+ kasan_populate_pud(pgdp, vaddr, next) : \
kasan_populate_pmd((pud_t *)pgdp, vaddr, next)))
static void __init kasan_populate_pgd(pgd_t *pgdp,
- unsigned long vaddr, unsigned long end,
- bool early)
+ unsigned long vaddr, unsigned long end)
{
phys_addr_t phys_addr;
unsigned long next;
@@ -237,11 +200,7 @@ static void __init kasan_populate_pgd(pgd_t *pgdp,
next = pgd_addr_end(vaddr, end);
if (IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) {
- if (early) {
- phys_addr = __pa((uintptr_t)kasan_early_shadow_pgd_next);
- set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_TABLE));
- continue;
- } else if (pgd_page_vaddr(*pgdp) ==
+ if (pgd_page_vaddr(*pgdp) ==
(unsigned long)lm_alias(kasan_early_shadow_pgd_next)) {
/*
* pgdp can't be none since kasan_early_init
@@ -258,7 +217,95 @@ static void __init kasan_populate_pgd(pgd_t *pgdp,
}
}
- kasan_populate_pgd_next(pgdp, vaddr, next, early);
+ kasan_populate_pgd_next(pgdp, vaddr, next);
+ } while (pgdp++, vaddr = next, vaddr != end);
+}
+
+static void __init kasan_early_populate_pud(p4d_t *p4dp,
+ unsigned long vaddr,
+ unsigned long end)
+{
+ pud_t *pudp, *base_pud;
+ phys_addr_t phys_addr;
+ unsigned long next;
+
+ if (!pgtable_l4_enabled) {
+ pudp = (pud_t *)p4dp;
+ } else {
+ base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(*p4dp)));
+ pudp = base_pud + pud_index(vaddr);
+ }
+
+ do {
+ next = pud_addr_end(vaddr, end);
+
+ if (pud_none(*pudp) && IS_ALIGNED(vaddr, PUD_SIZE) &&
+ (next - vaddr) >= PUD_SIZE) {
+ phys_addr = __pa((uintptr_t)kasan_early_shadow_pmd);
+ set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_TABLE));
+ continue;
+ }
+
+ BUG();
+ } while (pudp++, vaddr = next, vaddr != end);
+}
+
+static void __init kasan_early_populate_p4d(pgd_t *pgdp,
+ unsigned long vaddr,
+ unsigned long end)
+{
+ p4d_t *p4dp, *base_p4d;
+ phys_addr_t phys_addr;
+ unsigned long next;
+
+ /*
+ * We can't use pgd_page_vaddr here as it would return a linear
+ * mapping address but it is not mapped yet, but when populating
+ * early_pg_dir, we need the physical address and when populating
+ * swapper_pg_dir, we need the kernel virtual address so use
+ * pt_ops facility.
+ * Note that this test is then completely equivalent to
+ * p4dp = p4d_offset(pgdp, vaddr)
+ */
+ if (!pgtable_l5_enabled) {
+ p4dp = (p4d_t *)pgdp;
+ } else {
+ base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgdp)));
+ p4dp = base_p4d + p4d_index(vaddr);
+ }
+
+ do {
+ next = p4d_addr_end(vaddr, end);
+
+ if (p4d_none(*p4dp) && IS_ALIGNED(vaddr, P4D_SIZE) &&
+ (next - vaddr) >= P4D_SIZE) {
+ phys_addr = __pa((uintptr_t)kasan_early_shadow_pud);
+ set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_TABLE));
+ continue;
+ }
+
+ kasan_early_populate_pud(p4dp, vaddr, next);
+ } while (p4dp++, vaddr = next, vaddr != end);
+}
+
+static void __init kasan_early_populate_pgd(pgd_t *pgdp,
+ unsigned long vaddr,
+ unsigned long end)
+{
+ phys_addr_t phys_addr;
+ unsigned long next;
+
+ do {
+ next = pgd_addr_end(vaddr, end);
+
+ if (pgd_none(*pgdp) && IS_ALIGNED(vaddr, PGDIR_SIZE) &&
+ (next - vaddr) >= PGDIR_SIZE) {
+ phys_addr = __pa((uintptr_t)kasan_early_shadow_p4d);
+ set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_TABLE));
+ continue;
+ }
+
+ kasan_early_populate_p4d(pgdp, vaddr, next);
} while (pgdp++, vaddr = next, vaddr != end);
}
@@ -295,16 +342,16 @@ asmlinkage void __init kasan_early_init(void)
PAGE_TABLE));
}
- kasan_populate_pgd(early_pg_dir + pgd_index(KASAN_SHADOW_START),
- KASAN_SHADOW_START, KASAN_SHADOW_END, true);
+ kasan_early_populate_pgd(early_pg_dir + pgd_index(KASAN_SHADOW_START),
+ KASAN_SHADOW_START, KASAN_SHADOW_END);
local_flush_tlb_all();
}
void __init kasan_swapper_init(void)
{
- kasan_populate_pgd(pgd_offset_k(KASAN_SHADOW_START),
- KASAN_SHADOW_START, KASAN_SHADOW_END, true);
+ kasan_early_populate_pgd(pgd_offset_k(KASAN_SHADOW_START),
+ KASAN_SHADOW_START, KASAN_SHADOW_END);
local_flush_tlb_all();
}
@@ -314,7 +361,7 @@ static void __init kasan_populate(void *start, void *end)
unsigned long vaddr = (unsigned long)start & PAGE_MASK;
unsigned long vend = PAGE_ALIGN((unsigned long)end);
- kasan_populate_pgd(pgd_offset_k(vaddr), vaddr, vend, false);
+ kasan_populate_pgd(pgd_offset_k(vaddr), vaddr, vend);
local_flush_tlb_all();
memset(start, KASAN_SHADOW_INIT, end - start);
--
2.37.2