[RFC PATCH 6/6] f2fs: give referenced node cache entries a shared second chance

[Wenjie Qi]

The shrinker already gives a referenced cache entry a second chance by
clearing REFERENCED and moving it to the queue tail. The compression worker
only skips such entries, so it can revisit the same referenced prefix on
every pass.

Use the same clear-and-move rule in both paths. The worker records the
original raw-list tail so entries moved during a pass are not visited again
in that pass.

If an access races with compression after candidate collection, discard
the temporary compressed object and give the raw entry the same second
chance.

Signed-off-by: Wenjie Qi <qiwenjie@xxxxxxxxxx>
---
 fs/f2fs/cache.c               | 15 ++++++++++++---
 fs/f2fs/cache.h               |  3 +++
 fs/f2fs/debug.c               |  4 ++--
 fs/f2fs/node_cache_compress.c | 36 ++++++++++++++++++++++++++---------
 fs/f2fs/node_cache_compress.h |  1 +
 5 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/fs/f2fs/cache.c b/fs/f2fs/cache.c
index b3158f4ab45c..2b0a6eccd3af 100644
--- a/fs/f2fs/cache.c
+++ b/fs/f2fs/cache.c
@@ -60,6 +60,17 @@ void f2fs_cache_update_tag(struct f2fs_cached_block *entry,
 	spin_unlock_irqrestore(&cache->tree_lock, flags);
 }
 
+bool f2fs_cache_clear_referenced_and_move(struct f2fs_cached_block_list *cache,
+					  struct list_head *head,
+					  struct f2fs_cached_block *entry)
+{
+	lockdep_assert_held(&cache->list_lock);
+	if (!f2fs_cache_test_and_clear_referenced(entry))
+		return false;
+	list_move_tail(&entry->list, head);
+	return true;
+}
+
 bool f2fs_mark_cache_dirty(struct f2fs_cached_block *entry)
 {
 	struct f2fs_cached_block_list *cache = entry->cache;
@@ -594,10 +605,8 @@ f2fs_shrink_cache_list(struct f2fs_cached_block_list *cache,
 		scanned++;
 
 		/* If accessed, give it a second chance to rotate to tail */
-		if (f2fs_cache_test_and_clear_referenced(entry)) {
-			list_move_tail(&entry->list, head);
+		if (f2fs_cache_clear_referenced_and_move(cache, head, entry))
 			continue;
-		}
 
 		if (f2fs_cache_test_dirty(entry) ||
 		    f2fs_cache_test_writeback(entry) ||
diff --git a/fs/f2fs/cache.h b/fs/f2fs/cache.h
index 3c1fdc7894b7..8acbbbaf897d 100644
--- a/fs/f2fs/cache.h
+++ b/fs/f2fs/cache.h
@@ -232,6 +232,9 @@ void f2fs_cache_wait_writeback_cond(struct f2fs_cached_block *entry,
 void f2fs_cache_wait_writeback(struct f2fs_cached_block *entry);
 void f2fs_cache_update_tag(struct f2fs_cached_block *entry,
 		unsigned int clear_from, unsigned int set_to);
+bool f2fs_cache_clear_referenced_and_move(struct f2fs_cached_block_list *cache,
+					  struct list_head *head,
+					  struct f2fs_cached_block *entry);
 struct f2fs_cached_block *f2fs_grab_cache(struct f2fs_cached_block_list *cache,
 				unsigned long index, int flags);
 void f2fs_truncate_locked_cache(struct f2fs_cached_block *entry,
diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c
index 984fed727a48..8625dccc2131 100644
--- a/fs/f2fs/debug.c
+++ b/fs/f2fs/debug.c
@@ -728,9 +728,9 @@ static int stat_show(struct seq_file *s, void *v)
 				nc.payload[0], nc.payload[1], nc.payload[2],
 				nc.slots[0], nc.slots[1], nc.slots[2]);
 			seq_printf(s,
-				   "NCWorker: cycles=%llu visited=%llu candidates=%llu attempts=%llu converted=%llu candidate_refs=%llu transient=%llu alloc_fail=%llu\n",
+				   "NCWorker: cycles=%llu visited=%llu candidates=%llu attempts=%llu converted=%llu referenced_moved=%llu candidate_refs=%llu transient=%llu alloc_fail=%llu\n",
 				nc.cycles, nc.visited, nc.candidates, nc.attempts,
-				nc.converted, nc.candidate_refs,
+				nc.converted, nc.referenced_moved, nc.candidate_refs,
 				nc.transient_extra_bytes, nc.allocation_failures);
 			seq_printf(s,
 				   "NCRestore: compressed_to_raw=%llu restored_hits=%llu restore_failures=%llu\n",
diff --git a/fs/f2fs/node_cache_compress.c b/fs/f2fs/node_cache_compress.c
index 6e474fea7241..cf13110b2961 100644
--- a/fs/f2fs/node_cache_compress.c
+++ b/fs/f2fs/node_cache_compress.c
@@ -147,6 +147,7 @@ struct f2fs_nc_ctx {
 	atomic64_t candidate_count;
 	atomic64_t attempts;
 	atomic64_t converted;
+	atomic64_t referenced_moved;
 	atomic64_t candidate_refs;
 	atomic64_t transient_extra_bytes;
 	atomic64_t allocation_failures;
@@ -574,7 +575,6 @@ static bool f2fs_nc_candidate(struct f2fs_cached_block *entry,
 		!f2fs_cache_test_dirty(entry) &&
 		!f2fs_cache_test_writeback(entry) &&
 		!f2fs_cache_test_locked(entry) &&
-		!f2fs_cache_test_referenced(entry) &&
 		!f2fs_cache_test_incompressible(entry) &&
 		atomic_read(&entry->refcount) == refs;
 }
@@ -601,6 +601,7 @@ f2fs_nc_compress(struct f2fs_cached_block *entry,
 	u32 crc;
 	int bucket;
 	int len;
+	bool referenced;
 
 	if (!cache || !IS_NODE_CACHE(cache) || !cfg ||
 	    f2fs_cache_test_compressed(entry))
@@ -654,9 +655,14 @@ f2fs_nc_compress(struct f2fs_cached_block *entry,
 
 	spin_lock(&cache->list_lock);
 	spin_lock_irqsave(&cache->tree_lock, flags);
+	referenced = f2fs_cache_test_referenced(entry);
+	/* A concurrent access wins; rotate the entry and discard this result. */
+	if (referenced && entry->cache == cache &&
+	    f2fs_cache_clear_referenced_and_move(cache, &cache->lru_list, entry))
+		atomic64_inc(&ctx->referenced_moved);
 	if (entry->cache != cache || entry->data != raw ||
 	    atomic_read(&entry->refcount) != 2 || f2fs_cp_error(sbi) ||
-	    f2fs_cache_test_referenced(entry) ||
+	    referenced ||
 	    !f2fs_cache_test_uptodate(entry) || f2fs_cache_test_dirty(entry) ||
 	    f2fs_cache_test_writeback(entry) ||
 	    f2fs_cache_test_incompressible(entry)) {
@@ -693,7 +699,7 @@ void f2fs_nc_run(struct f2fs_sb_info *sbi)
 	struct f2fs_nc_ctx *ctx = sbi->node_compress;
 	struct f2fs_cached_block_list *cache = NODE_CACHE(sbi);
 	struct f2fs_nc_config cfg;
-	struct f2fs_cached_block *entry;
+	struct f2fs_cached_block *entry, *next, *scan_tail;
 	unsigned long flags;
 	u64 now_ms;
 	unsigned long raw_population;
@@ -714,15 +720,26 @@ void f2fs_nc_run(struct f2fs_sb_info *sbi)
 		goto out;
 
 	spin_lock(&cache->list_lock);
-	list_for_each_entry(entry, &cache->lru_list, list) {
+	/* Do not revisit entries rotated to the tail during this pass. */
+	scan_tail = list_empty(&cache->lru_list) ? NULL :
+		list_last_entry(&cache->lru_list, struct f2fs_cached_block, list);
+	list_for_each_entry_safe(entry, next, &cache->lru_list, list) {
 		if (visited == goal || count == F2FS_NC_MAX_CANDIDATES)
 			break;
 		visited++;
-		if (!f2fs_nc_candidate(entry, cache, 1))
-			continue;
-		f2fs_cache_get(entry);
-		ctx->candidates[count++] = entry;
-		atomic64_inc(&ctx->candidate_refs);
+		if (f2fs_nc_candidate(entry, cache, 1)) {
+			if (f2fs_cache_clear_referenced_and_move(cache,
+								 &cache->lru_list,
+								 entry)) {
+				atomic64_inc(&ctx->referenced_moved);
+			} else {
+				f2fs_cache_get(entry);
+				ctx->candidates[count++] = entry;
+				atomic64_inc(&ctx->candidate_refs);
+			}
+		}
+		if (entry == scan_tail)
+			break;
 	}
 	spin_unlock(&cache->list_lock);
 	atomic64_add(visited, &ctx->visited);
@@ -890,6 +907,7 @@ void f2fs_nc_get_stats(struct f2fs_sb_info *sbi,
 	stats->candidates = atomic64_read(&ctx->candidate_count);
 	stats->attempts = atomic64_read(&ctx->attempts);
 	stats->converted = atomic64_read(&ctx->converted);
+	stats->referenced_moved = atomic64_read(&ctx->referenced_moved);
 	stats->candidate_refs = atomic64_read(&ctx->candidate_refs);
 	stats->transient_extra_bytes =
 		atomic64_read(&ctx->transient_extra_bytes);
diff --git a/fs/f2fs/node_cache_compress.h b/fs/f2fs/node_cache_compress.h
index efe2eceef352..542093f2eca7 100644
--- a/fs/f2fs/node_cache_compress.h
+++ b/fs/f2fs/node_cache_compress.h
@@ -31,6 +31,7 @@ struct f2fs_nc_stats {
 	u64 candidates;
 	u64 attempts;
 	u64 converted;
+	u64 referenced_moved;
 	u64 candidate_refs;		/* Worker-held references right now. */
 	u64 transient_extra_bytes;	/* Temporary raw/compressed overlap. */
 	u64 allocation_failures;
-- 
2.43.0