[PATCH] mm: optimize pmdp_get() and friends for folded pagetable levels

[David Hildenbrand (Arm)]

Using pmdp_get() and friends in common code on a kernel config with
folded page tables is suboptimal: pdmp_get() and friends defaults to a
READ_ONCE(), forcing the compiler to actually read that value even though
it will not actually be used afterwards.

This was recently reported by Christophe Leroy [1].

Once we realize that the output of pdmp_get() on these kernel configs is
entirely ignored, as pmd_present()==1 and pmd_leaf()==0 are just
hard-coded, we can just make it return some dummy value.

pmd_offset() is expected to be called with the pudp afterwards, simply
performing a typecast of the pudp pointer to a pmdp pointer.

Let's introduce a pmd_offset_lockless() that does exactly the same:
perform a typecast.

[1] https://lore.kernel.org/all/0019d675-ce3d-4a5c-89ed-f126c45145c9@xxxxxxxxxx/

Signed-off-by: David Hildenbrand (Arm) <david@xxxxxxxxxx>
---
 include/asm-generic/pgtable-nop4d.h |  8 ++++++++
 include/asm-generic/pgtable-nopmd.h | 19 +++++++++++++++++++
 include/asm-generic/pgtable-nopud.h |  8 ++++++++
 3 files changed, 35 insertions(+)

diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h
index 03b7dae47dd4..cfe87036d61c 100644
--- a/include/asm-generic/pgtable-nop4d.h
+++ b/include/asm-generic/pgtable-nop4d.h
@@ -32,6 +32,14 @@ static inline void pgd_clear(pgd_t *pgd)	{ }
  */
 #define set_pgd(pgdptr, pgdval)	set_p4d((p4d_t *)(pgdptr), (p4d_t) { pgdval })
 
+static inline pgd_t pgdp_get(pgd_t *p4dp)
+{
+	pgd_t dummy = { 0 };
+
+	return dummy;
+}
+#define pgdp_get pgdp_get
+
 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
 {
 	return (p4d_t *)pgd;
diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h
index 8ffd64e7a24c..e87a42407563 100644
--- a/include/asm-generic/pgtable-nopmd.h
+++ b/include/asm-generic/pgtable-nopmd.h
@@ -43,12 +43,31 @@ static inline void pud_clear(pud_t *pud)	{ }
  */
 #define set_pud(pudptr, pudval)			set_pmd((pmd_t *)(pudptr), (pmd_t) { pudval })
 
+static inline pud_t pudp_get(pud_t *pudp)
+{
+	pud_t dummy = { 0 };
+
+	/*
+	 * Given that pud_present()==1 and pud_leaf==0, page table walking code
+	 * treats this like a page table and calls pmd_offset() /
+	 * pmd_offset_lockless() with pudp, ignoring the returned value.
+	 */
+	return dummy;
+}
+#define pudp_get pudp_get
+
 static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address)
 {
 	return (pmd_t *)pud;
 }
 #define pmd_offset pmd_offset
 
+static inline pmd_t * pmd_offset_lockless(pud_t *pud, puf_t pud, unsigned long address)
+{
+	return (pmd_t *)pud;
+}
+#define pmd_offset_lockless pmd_offset_lockless
+
 #define pmd_val(x)				(pud_val((x).pud))
 #define __pmd(x)				((pmd_t) { __pud(x) } )
 
diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h
index eb70c6d7ceff..2cacd9571b2f 100644
--- a/include/asm-generic/pgtable-nopud.h
+++ b/include/asm-generic/pgtable-nopud.h
@@ -39,6 +39,14 @@ static inline void p4d_clear(p4d_t *p4d)	{ }
  */
 #define set_p4d(p4dptr, p4dval)	set_pud((pud_t *)(p4dptr), (pud_t) { p4dval })
 
+static inline p4d_t p4dp_get(p4d_t *p4dp)
+{
+	p4d_t dummy = { 0 };
+
+	return dummy;
+}
+#define p4dp_get p4dp_get
+
 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
 {
 	return (pud_t *)p4d;
-- 
2.43.0


-- 
Cheers,

David