[PATCH] mm: explicitly enable an expand-only merge mode for brk()

[Lorenzo Stoakes]

Try to do less work on brk() to improve perf.
---
 mm/mmap.c |  1 +
 mm/vma.c  | 25 ++++++++++++++++---------
 mm/vma.h  | 11 +++++++++++
 3 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/mm/mmap.c b/mm/mmap.c
index 02f7b45c3076..c2c68ef45a3b 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1740,6 +1740,7 @@ static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma,
 	if (vma && vma->vm_end == addr) {
 		VMG_STATE(vmg, mm, vmi, addr, addr + len, flags, PHYS_PFN(addr));

+		vmg.mode = VMA_MERGE_MODE_EXPAND_ONLY;
 		vmg.prev = vma;
 		vma_iter_next_range(vmi);

diff --git a/mm/vma.c b/mm/vma.c
index 749c4881fd60..f525a0750c41 100644
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -561,6 +561,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg)
 	unsigned long end = vmg->end;
 	pgoff_t pgoff = vmg->pgoff;
 	pgoff_t pglen = PHYS_PFN(end - start);
+	bool expand_only = vmg_mode_expand_only(vmg);
 	bool can_merge_left, can_merge_right;

 	mmap_assert_write_locked(vmg->mm);
@@ -575,7 +576,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg)
 		return NULL;

 	can_merge_left = can_vma_merge_left(vmg);
-	can_merge_right = can_vma_merge_right(vmg, can_merge_left);
+	can_merge_right = !expand_only && can_vma_merge_right(vmg, can_merge_left);

 	/* If we can merge with the next VMA, adjust vmg accordingly. */
 	if (can_merge_right) {
@@ -603,13 +604,18 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg)
 		return vmg->vma;
 	}

-	/* If expansion failed, reset state. Allows us to retry merge later. */
-	vmg->vma = NULL;
-	vmg->start = start;
-	vmg->end = end;
-	vmg->pgoff = pgoff;
-	if (vmg->vma == prev)
-		vma_iter_set(vmg->vmi, start);
+	/*
+	 * Unless in expand only case and expansion failed, reset state.
+	 * Allows us to retry merge later.
+	 */
+	if (!expand_only) {
+		vmg->vma = NULL;
+		vmg->start = start;
+		vmg->end = end;
+		vmg->pgoff = pgoff;
+		if (vmg->vma == prev)
+			vma_iter_set(vmg->vmi, start);
+	}

 	return NULL;
 }
@@ -641,7 +647,8 @@ int vma_expand(struct vma_merge_struct *vmg)
 	mmap_assert_write_locked(vmg->mm);

 	vma_start_write(vma);
-	if (next && (vma != next) && (vmg->end == next->vm_end)) {
+	if (!vmg_mode_expand_only(vmg) && next &&
+	    (vma != next) && (vmg->end == next->vm_end)) {
 		int ret;

 		remove_next = true;
diff --git a/mm/vma.h b/mm/vma.h
index 82354fe5edd0..14224b36a979 100644
--- a/mm/vma.h
+++ b/mm/vma.h
@@ -52,6 +52,11 @@ struct vma_munmap_struct {
 	unsigned long data_vm;
 };

+enum vma_merge_mode {
+	VMA_MERGE_MODE_NORMAL,
+	VMA_MERGE_MODE_EXPAND_ONLY,
+};
+
 enum vma_merge_state {
 	VMA_MERGE_START,
 	VMA_MERGE_ERROR_NOMEM,
@@ -75,9 +80,15 @@ struct vma_merge_struct {
 	struct mempolicy *policy;
 	struct vm_userfaultfd_ctx uffd_ctx;
 	struct anon_vma_name *anon_name;
+	enum vma_merge_mode mode;
 	enum vma_merge_state state;
 };

+static inline bool vmg_mode_expand_only(struct vma_merge_struct *vmg)
+{
+	return vmg->mode == VMA_MERGE_MODE_EXPAND_ONLY;
+}
+
 static inline bool vmg_nomem(struct vma_merge_struct *vmg)
 {
 	return vmg->state == VMA_MERGE_ERROR_NOMEM;
--
2.46.2