[PATCH 10/12] mm: enable MMU_GATHER_RCU_TABLE_FREE for sparc32
From: Lorenzo Stoakes (ARM)
Date: Tue Sep 01 2026 - 07:12:05 EST
Careful handling is required for sparc32 which implements page tables as
part of a shared backing page.
To support this, a custom __tlb_remove_table() function is required, as
specified by CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE.
This allows __pte_free_tlb() and __pmd_free_tlb() to specify which page
table level is being freed, which is transmitted to __tlb_remove_table()
through setting the lowest bit of the page table to 1 for a PMD and 0 for a
PTE (the page tables are 256-byte aligned so this is safe to do).
Next, since the page table freeing is done via RCU callback, and thus might
be executed in softirq context, update the spin locks to IRQ save/restore.
Then, in __tlb_remove_table(), figure out whether to free a PMD page table
via free_pmd_fast() or a PTE via the newly introduced __pte_free()
function, using the lower bit encoded in __pte_free_tlb() or
__pmd_free_tlb() to determine which to call.
__pte_free() is identical to preexisting pte_free(), except that it
optionally allows a NULL mm pointer to be provided, in which case there is
no mm whose mm->page_table_lock can be taken.
This lock doesn't appear to have been doing quite as much as it intended,
as backing pages can contain page tables for multiple mm's, which are not
serialised by it.
But more importantly - the reference count increment in pte_alloc_one() and
decrement in __pte_free() are atomic with full ordering, so it simply isn't
possible for there to be a meaningful race here.
Note that the specification of CONFIG_HAVE_ARCH_TLB_REMOVE_TABLE disables
CONFIG_PT_RECLAIM for sparc32, which mirrors sparc64.
This forms part of an overall effort to switch every architecture to this
mode, and with it complete, means every architecture now supports
CONFIG_MMU_GATHER_RCU_TABLE_FREE.
Signed-off-by: Lorenzo Stoakes (ARM) <ljs@xxxxxxxxxx>
---
arch/sparc/Kconfig | 2 ++
arch/sparc/include/asm/pgalloc_32.h | 7 +++++--
arch/sparc/lib/bitext.c | 14 +++++++-------
arch/sparc/mm/srmmu.c | 26 +++++++++++++++++++++++---
4 files changed, 37 insertions(+), 12 deletions(-)
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 8d42ebc6d302..79c09d6ee466 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -64,6 +64,8 @@ config SPARC32
select HAVE_UID16
select HAVE_PAGE_SIZE_4KB
select LOCK_MM_AND_FIND_VMA
+ select MMU_GATHER_RCU_TABLE_FREE
+ select HAVE_ARCH_TLB_REMOVE_TABLE
select OLD_SIGACTION
select ZONE_DMA
diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h
index 4f73e87b22a3..36010852ba0c 100644
--- a/arch/sparc/include/asm/pgalloc_32.h
+++ b/arch/sparc/include/asm/pgalloc_32.h
@@ -48,7 +48,9 @@ static inline void free_pmd_fast(pmd_t * pmd)
}
#define pmd_free(mm, pmd) free_pmd_fast(pmd)
-#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd)
+
+#define __pmd_free_tlb(tlb, pmd, addr) \
+ tlb_remove_table((tlb), (void *)((unsigned long)(pmd) | 1UL))
#define pmd_populate(mm, pmd, pte) pmd_set(pmd, pte)
@@ -72,6 +74,7 @@ static inline void free_pte_fast(pte_t *pte)
#define pte_free_kernel(mm, pte) free_pte_fast(pte)
void pte_free(struct mm_struct * mm, pgtable_t pte);
-#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte)
+void __tlb_remove_table(void *table);
+#define __pte_free_tlb(tlb, pte, addr) tlb_remove_table((tlb), (void *)(pte))
#endif /* _SPARC_PGALLOC_H */
diff --git a/arch/sparc/lib/bitext.c b/arch/sparc/lib/bitext.c
index 32a5c1d9459c..c309e27973ce 100644
--- a/arch/sparc/lib/bitext.c
+++ b/arch/sparc/lib/bitext.c
@@ -22,8 +22,6 @@
* @align: requested alignment
*
* Returns offset in the map or -1 if out of space.
- *
- * Not safe to call from an interrupt (uses spin_lock).
*/
int bit_map_string_get(struct bit_map *t, int len, int align)
{
@@ -31,6 +29,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align)
int off_new;
int align1;
int i, color;
+ unsigned long flags;
if (t->num_colors) {
/* align is overloaded to be the page color */
@@ -50,7 +49,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align)
BUG();
color &= align1;
- spin_lock(&t->lock);
+ spin_lock_irqsave(&t->lock, flags);
if (len < t->last_size)
offset = t->first_free;
else
@@ -64,7 +63,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align)
if (offset >= t->size)
offset = 0;
if (count + len > t->size) {
- spin_unlock(&t->lock);
+ spin_unlock_irqrestore(&t->lock, flags);
/* P3 */ printk(KERN_ERR
"bitmap out: size %d used %d off %d len %d align %d count %d\n",
t->size, t->used, offset, len, align, count);
@@ -90,7 +89,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align)
t->last_off = 0;
t->used += len;
t->last_size = len;
- spin_unlock(&t->lock);
+ spin_unlock_irqrestore(&t->lock, flags);
return offset;
}
}
@@ -103,10 +102,11 @@ int bit_map_string_get(struct bit_map *t, int len, int align)
void bit_map_clear(struct bit_map *t, int offset, int len)
{
int i;
+ unsigned long flags;
if (t->used < len)
BUG(); /* Much too late to do any good, but alas... */
- spin_lock(&t->lock);
+ spin_lock_irqsave(&t->lock, flags);
for (i = 0; i < len; i++) {
if (test_bit(offset + i, t->map) == 0)
BUG();
@@ -115,7 +115,7 @@ void bit_map_clear(struct bit_map *t, int offset, int len)
if (offset < t->first_free)
t->first_free = offset;
t->used -= len;
- spin_unlock(&t->lock);
+ spin_unlock_irqrestore(&t->lock, flags);
}
void bit_map_init(struct bit_map *t, unsigned long *map, int size)
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index 9a74902ad181..2a2c7bd21011 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -359,19 +359,39 @@ pgtable_t pte_alloc_one(struct mm_struct *mm)
return ptep;
}
-void pte_free(struct mm_struct *mm, pgtable_t ptep)
+static void __pte_free(struct mm_struct *mm, pgtable_t ptep)
{
+ const bool process_context = mm;
struct page *page;
page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT);
- spin_lock(&mm->page_table_lock);
+ if (process_context)
+ spin_lock(&mm->page_table_lock);
if (page_ref_dec_return(page) == 1)
pagetable_dtor(page_ptdesc(page));
- spin_unlock(&mm->page_table_lock);
+ if (process_context)
+ spin_unlock(&mm->page_table_lock);
srmmu_free_nocache(ptep, SRMMU_PTE_TABLE_SIZE);
}
+void pte_free(struct mm_struct *mm, pgtable_t ptep)
+{
+ __pte_free(mm, ptep);
+}
+
+void __tlb_remove_table(void *table)
+{
+ const unsigned long encoded = (unsigned long)table;
+ const unsigned long addr = encoded & ~1UL;
+ const bool is_pmd = encoded & 1;
+
+ if (is_pmd)
+ free_pmd_fast((pmd_t *)addr);
+ else /* Called from softirq context, no mm. */
+ __pte_free(NULL, (pgtable_t)addr);
+}
+
/* context handling - a dynamically sized pool is used */
#define NO_CONTEXT -1
--
2.55.0