[RFC PATCH 4/9] arm64/kprobes: Make the single-step machinery noinstr

[Hongyan Xia]

From: Hongyan Xia <hongyan.xia@xxxxxxxxxxxxx>

Convert the core of the arm64 kprobe single-step flow to noinstr:
save_previous_kprobe(), restore_previous_kprobe(), set_current_kprobe(),
kprobes_save_local_irqflag(), kprobes_restore_local_irqflag(),
setup_singlestep(), reenter_kprobe() and post_kprobe_handler().

These functions only touch per-cpu state, pt_regs and DAIF, except for:

- kprobes_inc_nmissed_count(), which is generic and instrumentable; add
  an arm64-local wrapper that calls it bounded by
  instrumentation_begin()/end();
- the instruction simulation path in setup_singlestep(), whose decode
  handlers are instrumentable; bound arch_simulate_insn() with an
  instrumentation window;
- the unrecoverable-reentry pr_warn()/dump_kprobe()/BUG() path and the
  default WARN_ON(), both bounded with instrumentation windows.

The __kprobes attribute (notrace + .kprobes.text) is replaced by
noinstr, which is a strict superset for these functions.

Signed-off-by: Hongyan Xia <hongyan.xia@xxxxxxxxxxxxx>
---
 arch/arm64/kernel/probes/kprobes.c | 39 ++++++++++++++++++------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c
index 4e0efad5caf2..1b12341b2af3 100644
--- a/arch/arm64/kernel/probes/kprobes.c
+++ b/arch/arm64/kernel/probes/kprobes.c
@@ -14,6 +14,7 @@
 #include <linux/extable.h>
 #include <linux/kasan.h>
 #include <linux/kernel.h>
+#include <linux/instrumentation.h>
 #include <linux/kprobes.h>
 #include <linux/sched/debug.h>
 #include <linux/set_memory.h>
@@ -39,7 +40,7 @@
 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
 
-static void __kprobes
+static void noinstr
 post_kprobe_handler(struct kprobe *, struct kprobe_ctlblk *, struct pt_regs *);
 
 void *alloc_insn_page(void)
@@ -170,7 +171,7 @@ void __kprobes arch_remove_kprobe(struct kprobe *p)
 	}
 }
 
-static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
+static void noinstr save_previous_kprobe(struct kprobe_ctlblk *kcb)
 {
 	kcb->prev_kprobe.kp = kprobe_running();
 	kcb->prev_kprobe.status = kcb->kprobe_status;
@@ -184,7 +185,7 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
 	kcb->prev_kprobe.saved_irqflag = kcb->saved_irqflag;
 }
 
-static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
+static void noinstr restore_previous_kprobe(struct kprobe_ctlblk *kcb)
 {
 	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
 	kcb->kprobe_status = kcb->prev_kprobe.status;
@@ -197,7 +198,7 @@ static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
 	kcb->saved_irqflag = kcb->prev_kprobe.saved_irqflag;
 }
 
-static void __kprobes set_current_kprobe(struct kprobe *p)
+static void noinstr set_current_kprobe(struct kprobe *p)
 {
 	__this_cpu_write(current_kprobe, p);
 }
@@ -207,23 +208,23 @@ static void __kprobes set_current_kprobe(struct kprobe *p)
  * simple and avoid nesting exceptions. Interrupts do have to be disabled since
  * the kprobe state is per-CPU and doesn't get migrated.
  */
-static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb,
-						struct pt_regs *regs)
+static void noinstr kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb,
+					       struct pt_regs *regs)
 {
 	kcb->saved_irqflag = regs->pstate & DAIF_MASK;
 	regs->pstate |= DAIF_MASK;
 }
 
-static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb,
-						struct pt_regs *regs)
+static void noinstr kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb,
+						  struct pt_regs *regs)
 {
 	regs->pstate &= ~DAIF_MASK;
 	regs->pstate |= kcb->saved_irqflag;
 }
 
-static void __kprobes setup_singlestep(struct kprobe *p,
-				       struct pt_regs *regs,
-				       struct kprobe_ctlblk *kcb, int reenter)
+static void noinstr setup_singlestep(struct kprobe *p,
+				     struct pt_regs *regs,
+				     struct kprobe_ctlblk *kcb, int reenter)
 {
 	unsigned long slot;
 
@@ -244,13 +245,15 @@ static void __kprobes setup_singlestep(struct kprobe *p,
 		instruction_pointer_set(regs, slot);
 	} else {
 		/* insn simulation */
+		instrumentation_begin();
 		arch_simulate_insn(p, regs);
+		instrumentation_end();
 	}
 }
 
-static int __kprobes reenter_kprobe(struct kprobe *p,
-				    struct pt_regs *regs,
-				    struct kprobe_ctlblk *kcb)
+static int noinstr reenter_kprobe(struct kprobe *p,
+				  struct pt_regs *regs,
+				  struct kprobe_ctlblk *kcb)
 {
 	switch (kcb->kprobe_status) {
 	case KPROBE_HIT_SSDONE:
@@ -262,23 +265,29 @@ static int __kprobes reenter_kprobe(struct kprobe *p,
 		 * recoverable one-level reentry, so handle it in the same way as
 		 * reentry from KPROBE_HIT_ACTIVE or KPROBE_HIT_SSDONE.
 		 */
+		instrumentation_begin();
 		kprobes_inc_nmissed_count(p);
+		instrumentation_end();
 		setup_singlestep(p, regs, kcb, 1);
 		break;
 	case KPROBE_REENTER:
+		instrumentation_begin();
 		pr_warn("Failed to recover from reentered kprobes.\n");
 		dump_kprobe(p);
 		BUG();
+		instrumentation_end();
 		break;
 	default:
+		instrumentation_begin();
 		WARN_ON(1);
+		instrumentation_end();
 		return 0;
 	}
 
 	return 1;
 }
 
-static void __kprobes
+static void noinstr
 post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, struct pt_regs *regs)
 {
 	/* return addr restore if non-branching insn */
-- 
2.47.3