[PATCH v6 3/3] mm: implement page refcount locking via dedicated bit

[Ilya Gladyshev]

The current page refcount implementation uses a single counter value
(zero) as dead. So, to prevent incrementing a dead refcount in
folio_try_get(), it fundamentally requires a CAS loop.

This CAS loop can act as a serialization point and can become a
significant bottleneck during high-frequency file read operations
[1][2].

This patch reallocates the refcount value range:

(1) refcount < 0 means dead refcount (uninit / frozen)
(2) refcount = 0 allowed only as a temporary state (see below)
(3) refcount > 0 is a regular reference count

In other words, refcount is now split into "dead bit" and a 31-bit
counter.

Refcount decrement now works as follows:
1. Counter decrement
2. If it is now zero, try to put it deep inside the dead zone
   (CAS to INT_MIN). Or you can view it as "set up frozen bit and reset
   counter".
3. This CAS can fail only if someone grabbed a reference in-between --
   that's okay, this page is their problem now.

The size of the dead zone allows performing an optimistic increment
inside page_ref_add_unless_frozen(), replacing the previous read + CAS
loop with a single RMW operation. This reduces cache line bouncing and
improves scalability, especially in NUMA scenarios.

[1]: https://lore.kernel.org/all/20251017141536.577466-1-kirill@xxxxxxxxxxxxx/
[2]: https://lore.kernel.org/all/CAHk-=wj00-nGmXEkxY=-=Z_qP6kiGUziSFvxHJ9N-cLWry5zpA@xxxxxxxxxxxxxx/

Reviewed-by: Artem Kuzin <artem.kuzin@xxxxxxxxxx>
Co-developed-by: Ivan Gorbunov <ivgorbunov@xxxxxx>
Signed-off-by: Ivan Gorbunov <ivgorbunov@xxxxxx>
Signed-off-by: Ilya Gladyshev <ilya.gladyshev@xxxxxxxxx>
Acked-by: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxx>
---
 include/linux/page-flags.h | 13 +++++++++++++
 include/linux/page_ref.h   | 30 +++++++++++++++++++++++++-----
 2 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 7a863572adce..b19721e0e7ca 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -196,6 +196,19 @@ enum pageflags {
 
 #define PAGEFLAGS_MASK		((1UL << NR_PAGEFLAGS) - 1)
 
+/* Most significant bit in page refcount */
+#define PAGEREF_FROZEN_BIT BIT(31)
+
+/* Page reference counter can be in 3 logical states,
+ * which are described below with their value representation
+ *        state              |         value
+ * (1)  safe with  owners    |   1...INT_MAX
+ * (2)  safe with no owners  |         0
+ * (3)  frozen               |  INT_MIN....-1
+ *
+ * State (2) can only temporarily occur inside dec_and_test.
+ */
+
 #ifndef __GENERATING_BOUNDS_H
 
 /*
diff --git a/include/linux/page_ref.h b/include/linux/page_ref.h
index 82ff3a99297a..cc7a9d7db504 100644
--- a/include/linux/page_ref.h
+++ b/include/linux/page_ref.h
@@ -64,7 +64,7 @@ static inline void __page_ref_unfreeze(struct page *page, int v)
 
 static inline bool __page_count_is_frozen(int count)
 {
-	return count == 0;
+	return count & PAGEREF_FROZEN_BIT;
 }
 
 static inline bool page_is_frozen(const struct page *page)
@@ -79,7 +79,12 @@ static inline bool folio_is_frozen(const struct folio *folio)
 
 static inline int page_ref_count(const struct page *page)
 {
-	return atomic_read(&page->_refcount);
+	int val = atomic_read(&page->_refcount);
+
+	if (unlikely(val & PAGEREF_FROZEN_BIT))
+		return 0;
+
+	return val;
 }
 
 /**
@@ -150,7 +155,7 @@ static inline void init_page_count(struct page *page)
 
 static inline void set_page_count_frozen(struct page *page)
 {
-	set_page_count(page, 0);
+	set_page_count(page, PAGEREF_FROZEN_BIT);
 }
 
 static inline void page_ref_add(struct page *page, int nr)
@@ -220,6 +225,9 @@ static inline int page_ref_sub_and_test(struct page *page, int nr)
 	VM_WARN_ON_ONCE_PAGE(page_is_frozen(page), page);
 	int ret = atomic_sub_and_test(nr, &page->_refcount);
 
+	if (ret)
+		ret = !atomic_cmpxchg_relaxed(&page->_refcount, 0, PAGEREF_FROZEN_BIT);
+
 	if (page_ref_tracepoint_active(page_ref_mod_and_test))
 		__page_ref_mod_and_test(page, -nr, ret);
 	return ret;
@@ -250,6 +258,9 @@ static inline int page_ref_dec_and_test(struct page *page)
 	VM_WARN_ON_ONCE_PAGE(page_is_frozen(page), page);
 	int ret = atomic_dec_and_test(&page->_refcount);
 
+	if (ret)
+		ret = !atomic_cmpxchg_relaxed(&page->_refcount, 0, PAGEREF_FROZEN_BIT);
+
 	if (page_ref_tracepoint_active(page_ref_mod_and_test))
 		__page_ref_mod_and_test(page, -1, ret);
 	return ret;
@@ -275,9 +286,18 @@ static inline int folio_ref_dec_return(struct folio *folio)
 	return page_ref_dec_return(&folio->page);
 }
 
+#define _PAGEREF_FROZEN_LIMIT	((1 << 30) | PAGEREF_FROZEN_BIT)
+
 static inline bool page_ref_add_unless_frozen(struct page *page, int nr)
 {
-	bool ret = atomic_add_unless(&page->_refcount, nr, 0);
+	int val = atomic_add_return(nr, &page->_refcount);
+	bool ret = !(val & PAGEREF_FROZEN_BIT);
+
+	/* Undo atomic_add() if counter is locked and scary big */
+	while (unlikely((unsigned int)val >= _PAGEREF_FROZEN_LIMIT)) {
+		if (atomic_try_cmpxchg_relaxed(&page->_refcount, &val, PAGEREF_FROZEN_BIT))
+			break;
+	}
 
 	if (page_ref_tracepoint_active(page_ref_mod_unless))
 		__page_ref_mod_unless(page, nr, ret);
@@ -312,7 +332,7 @@ static inline bool folio_ref_try_add(struct folio *folio, int count)
 
 static inline int page_ref_freeze(struct page *page, int count)
 {
-	int ret = likely(atomic_cmpxchg(&page->_refcount, count, 0) == count);
+	int ret = likely(atomic_cmpxchg(&page->_refcount, count, PAGEREF_FROZEN_BIT) == count);
 
 	if (page_ref_tracepoint_active(page_ref_freeze))
 		__page_ref_freeze(page, count, ret);
-- 
2.55.0