[RFC PATCH v2 03/13] mm: implement __anon_node_prepare for ANON_VMA_FRACTAL

[tao]

Add anon_node APIs corresponding to existing anon_vma operations:

- anon_node_clone() corresponds to anon_vma_clone()
- __anon_node_prepare() corresponds to __anon_vma_prepare()
- anon_node_fork_with_prev() corresponds to anon_vma_fork()
- unlink_anon_nodes() corresponds to unlink_anon_vmas()

Implement __anon_node_prepare() first to keep the patch small.

Signed-off-by: tao <tao.wangtao@xxxxxxxxx>
---
 mm/internal.h | 11 ++++++++++
 mm/rmap.c     | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)

diff --git a/mm/internal.h b/mm/internal.h
index c1b3961b9d3e..710baf02575b 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -314,6 +314,17 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma);
 int  __anon_vma_prepare(struct vm_area_struct *vma);
 void unlink_anon_vmas(struct vm_area_struct *vma);
 
+#ifdef CONFIG_ANON_VMA_FRACTAL
+int anon_node_clone(struct vm_area_struct *vma, struct vm_area_struct *src,
+	enum vma_operation operation);
+int anon_node_fork_with_prev(struct vm_area_struct *vma,
+	struct vm_area_struct *pvma, struct vm_area_struct *vma_prev,
+	struct vm_area_struct *pvma_prev);
+int __anon_node_prepare(struct vm_area_struct *vma);
+void unlink_anon_nodes(struct vm_area_struct *vma);
+
+#endif
+
 static inline int anon_vma_prepare(struct vm_area_struct *vma)
 {
 	if (likely(vma->anon_vma))
diff --git a/mm/rmap.c b/mm/rmap.c
index fa9d82271005..b98ba6a83272 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -645,6 +645,64 @@ static inline struct anon_node *anon_node_alloc(struct vm_area_struct *vma,
 	return anon_nod;
 }
 
+static bool try_track_rmap(struct anon_node *anon_nod, unsigned long rmap_base)
+{
+	const unsigned long rbc_max = rmap_base + ANON_RMAP_BASE_COUNT_MAX;
+
+	return rmap_base == anon_node_rmap_base(anon_nod) &&
+		atomic_long_add_unless(&anon_nod->rbc, 1, rbc_max);
+}
+
+static inline bool try_untrack_rmap(struct anon_node *anon_nod,
+		unsigned long rmap_base, bool *test_empty)
+{
+	const unsigned long rbc_max = rmap_base + ANON_RMAP_BASE_COUNT_MAX;
+	unsigned long rbc = atomic_long_read(&anon_nod->rbc);
+
+	if (rbc > rmap_base && rbc <= rbc_max) {
+		rbc = atomic_long_dec_return(&anon_nod->rbc);
+		*test_empty = (rbc == rmap_base);
+		return true;
+	}
+	return false;
+}
+
+/* attach an anon_node to the VMA. */
+int __anon_node_prepare(struct vm_area_struct *vma)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	unsigned long rmap_base = vma_rmap_base(vma);
+	struct anon_node *anon_nod;
+	struct anon_vma *anon_old = NULL, *anon_new;
+	bool test_empty = false;
+
+	mmap_assert_locked(mm);
+	might_sleep();
+
+	anon_nod = NULL; /* Fix find_mergeable_anon_vma(vma) later. */
+	if (anon_nod) {
+		anon_new = (struct anon_vma *)anon_nod;
+		if (try_track_rmap(anon_nod, rmap_base) &&
+		    !try_cmpxchg(&vma->anon_vma, &anon_old, anon_new))
+			try_untrack_rmap(anon_nod, rmap_base, &test_empty);
+		if (vma->anon_vma)
+			return 0;
+	}
+
+	anon_nod = anon_node_alloc(vma, NULL, false);
+	if (unlikely(!anon_nod))
+		return -ENOMEM;
+
+	anon_new = (struct anon_vma *)anon_nod;
+	/* maybe prepared by another thread */
+	if (!try_cmpxchg(&vma->anon_vma, &anon_old, anon_new)) {
+		try_untrack_rmap(anon_nod, rmap_base, &test_empty);
+		put_anon_node(anon_nod);
+	}
+
+	return 0;
+}
+
 #endif
 
 /*
-- 
2.17.1