[RFC PATCH v2 12/12] x86/mm/tlb: Reverting the removal of flush_tlb_info from stack

[Nadav Amit]

Partially revert 3db6d5a5eca ("x86/mm/tlb: Remove 'struct
flush_tlb_info' from the stack").

Now that we copy flush_tlb_info and inline it with the IPI information,
we can put it back onto the stack. This simplifies the code and should be
slightly more robust. The stack is also a bit more likely to be cached
than a global variable.

Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: Andy Lutomirski <luto@xxxxxxxxxx>
Cc: Borislav Petkov <bp@xxxxxxxxx>
Cc: Brian Gerst <brgerst@xxxxxxxxx>
Cc: Dave Hansen <dave.hansen@xxxxxxxxx>
Cc: H. Peter Anvin <hpa@xxxxxxxxx>
Cc: Rik van Riel <riel@xxxxxxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Signed-off-by: Nadav Amit <namit@xxxxxxxxxx>
---
 arch/x86/mm/tlb.c | 46 +++++++++++++++++++++++++---------------------
 1 file changed, 25 insertions(+), 21 deletions(-)

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index fd7e90adbe43..81170fd6b1dd 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -827,15 +827,17 @@ static inline void put_flush_tlb_info(void)
 static void flush_tlb_on_cpus(const cpumask_t *cpumask,
 			      const struct flush_tlb_info *info)
 {
-	int this_cpu = smp_processor_id();
 	bool flush_others = false;
+	int this_cpu;
+
+	this_cpu = get_cpu();
 
 	if (cpumask_any_but(cpumask, this_cpu) < nr_cpu_ids)
 		flush_others = true;
 
 	if (static_branch_likely(&flush_tlb_multi_enabled) && flush_others) {
 		flush_tlb_multi(cpumask, info);
-		return;
+		goto out;
 	}
 
 	if (cpumask_test_cpu(this_cpu, cpumask)) {
@@ -847,27 +849,33 @@ static void flush_tlb_on_cpus(const cpumask_t *cpumask,
 
 	if (flush_others)
 		flush_tlb_others(cpumask, info);
+
+out:
+	put_cpu();
 }
 
 void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
 				unsigned long end, unsigned int stride_shift,
 				bool freed_tables)
 {
-	struct flush_tlb_info *info;
-	u64 new_tlb_gen;
+	struct flush_tlb_info info = {
+		.mm		= mm,
+		.stride_shift	= stride_shift,
+		.freed_tables	= freed_tables,
+	};
 
 	/* Should we flush just the requested range? */
 	if ((end == TLB_FLUSH_ALL) ||
 	    ((end - start) >> stride_shift) > tlb_single_page_flush_ceiling) {
-		start = 0;
-		end = TLB_FLUSH_ALL;
+		info.start = 0;
+		info.end = TLB_FLUSH_ALL;
+	} else {
+		info.start = start;
+		info.end = end;
 	}
 
 	/* This is also a barrier that synchronizes with switch_mm(). */
-	new_tlb_gen = inc_mm_tlb_gen(mm);
-
-	info = get_flush_tlb_info(mm, start, end, stride_shift, freed_tables,
-				  new_tlb_gen);
+	info.new_tlb_gen = inc_mm_tlb_gen(mm);
 
 	/*
 	 * Assert that mm_cpumask() corresponds with the loaded mm. We got one
@@ -878,9 +886,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
 			(mm == this_cpu_read(cpu_tlbstate.loaded_mm)) ||
 			mm == &init_mm);
 
-	flush_tlb_on_cpus(mm_cpumask(mm), info);
-
-	put_flush_tlb_info();
+	flush_tlb_on_cpus(mm_cpumask(mm), &info);
 }
 
 
@@ -913,18 +919,18 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 	    (end - start) > tlb_single_page_flush_ceiling << PAGE_SHIFT) {
 		on_each_cpu(do_flush_tlb_all, NULL, 1);
 	} else {
-		struct flush_tlb_info *info;
-
-		info = get_flush_tlb_info(NULL, start, end, 0, false, 0);
+		struct flush_tlb_info info = {
+			.mm		= NULL,
+			.start		= start,
+			.end		= end,
+		};
 
 		/*
 		 * We have to wait for the remote shootdown to be done since it
 		 * is kernel space.
 		 */
 		__on_each_cpu_mask(cpu_online_mask, do_kernel_range_flush,
-				   info, sizeof(*info), 1);
-
-		put_flush_tlb_info();
+				   &info, sizeof(info), 1);
 	}
 }
 
@@ -942,9 +948,7 @@ static const struct flush_tlb_info full_flush_tlb_info = {
 
 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
 {
-	preempt_disable();
 	flush_tlb_on_cpus(&batch->cpumask, &full_flush_tlb_info);
-	preempt_enable();
 
 	cpumask_clear(&batch->cpumask);
 }
-- 
2.20.1