[PATCH] arm64: fpsimd: Added API to manage fpsimd state inside kernel

From: Wooyeon Kim
Date: Fri Jun 05 2020 - 03:32:50 EST

From: Wooki Min <wooki.min@xxxxxxxxxxx>

This is an patch to use FPSIMD register in Kernel space.
It need to manage to use FPSIMD register without damaging it
of the user task.
Following items have been implemented and added.

1. Using FPSIMD in ISR (in_interrupt)
It can used __efi_fpsimd_begin/__efi_fpsimd_end
which is already implemented.
Save fpsimd state before entering ISR,
and restore fpsimd state after ISR ends.
For use in external kernel module,
it is declared as EXPORT_SYMBOL.

2. User task -> Function in kernel
Add fpsimd_get/fpsimd_put API to save/restore
FPSIMD in use by the user.
In this case, depth variable is used to set fpsimd usage
depth between User and Kernel space.
* fpsimd_get: Save the FPSIMD of the user task.
* fpsimd_put: Restore the FPSIMD of the user task.

EX> fpsimd_get();
API in kernel space();
fpsimd_put();

3. Add kernel task FPSIMD save/restore in "fpsimd_thread_switch"
It checks the depth value in current task structure and
does save/restore action for FP/SIMD register used by kernel
context.

Signed-off-by: Wooki Min <wooki.min@xxxxxxxxxxx>
Signed-off-by: Jeongtae Park <jtp.park@xxxxxxxxxxx>
Signed-off-by: Sanghoon Lee <shoon114.lee@xxxxxxxxxxx>
Signed-off-by: Wooyeon Kim <wooy88.kim@xxxxxxxxxxx>
---
arch/arm64/include/asm/fpsimd.h | 6 +++
arch/arm64/include/asm/processor.h | 13 +++++
arch/arm64/include/uapi/asm/ptrace.h | 8 ++++
arch/arm64/kernel/fpsimd.c | 71 +++++++++++++++++++++++++---
4 files changed, 92 insertions(+), 6 deletions(-)

diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 59f10dd13f12..462c434fc57c 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -167,6 +167,12 @@ static inline void sve_setup(void) { }
extern void __efi_fpsimd_begin(void);
extern void __efi_fpsimd_end(void);

+void fpsimd_set_task_using(struct task_struct *t);
+void fpsimd_clr_task_using(struct task_struct *t);
+
+void fpsimd_get(void);
+void fpsimd_put(void);
+
#endif

#endif
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 240fe5e5b720..265669456bcb 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -139,6 +139,19 @@ struct thread_struct {
unsigned long tp2_value;
struct user_fpsimd_state fpsimd_state;
} uw;
+ struct fpsimd_kernel_state fpsimd_kernel_state;
+
+ /*
+ * indicate the depth of using FP/SIMD registers in kernel mode.
+ * above kernel state should be preserved at first time
+ * before FP/SIMD registers be used by other tasks
+ * and the state should be restored before they be used by own.
+ *
+ * a kernel thread which uses FP/SIMD registers have to
+ * set this depth and it could utilize for a tasks executes
+ * some NEON instructions without preemption disable.
+ */
+ atomic_t fpsimd_kernel_depth;

unsigned int fpsimd_cpu;
void *sve_state; /* SVE registers, if any */
diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h
index 42cbe34d95ce..0327e719721e 100644
--- a/arch/arm64/include/uapi/asm/ptrace.h
+++ b/arch/arm64/include/uapi/asm/ptrace.h
@@ -105,6 +105,14 @@ struct user_hwdebug_state {
} dbg_regs[16];
};

+/* Kernel structure for floating point */
+struct fpsimd_kernel_state {
+ __uint128_t vregs[32];
+ __u32 fpsr;
+ __u32 fpcr;
+ unsigned int cpu;
+};
+
/* SVE/FP/SIMD state (NT_ARM_SVE) */

struct user_sve_header {
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 35cb5e66c504..07597423fcfc 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -269,9 +269,6 @@ static void sve_free(struct task_struct *task)
*/
static void task_fpsimd_load(void)
{
- WARN_ON(!system_supports_fpsimd());
- WARN_ON(!have_cpu_fpsimd_context());
-
if (system_supports_sve() && test_thread_flag(TIF_SVE))
sve_load_state(sve_pffr(&current->thread),
&current->thread.uw.fpsimd_state.fpsr,
@@ -290,9 +287,6 @@ static void fpsimd_save(void)
this_cpu_ptr(&fpsimd_last_state);
/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */

- WARN_ON(!system_supports_fpsimd());
- WARN_ON(!have_cpu_fpsimd_context());
-
if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
if (WARN_ON(sve_get_vl() != last->sve_vl)) {
@@ -982,6 +976,10 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
void fpsimd_thread_switch(struct task_struct *next)
{
bool wrong_task, wrong_cpu;
+ struct fpsimd_kernel_state *cur_kst
+ = &current->thread.fpsimd_kernel_state;
+ struct fpsimd_kernel_state *nxt_kst
+ = &next->thread.fpsimd_kernel_state;

if (!system_supports_fpsimd())
return;
@@ -991,6 +989,16 @@ void fpsimd_thread_switch(struct task_struct *next)
/* Save unsaved fpsimd state, if any: */
fpsimd_save();

+ if (atomic_read(&current->thread.fpsimd_kernel_depth))
+ fpsimd_save_state((struct user_fpsimd_state *)cur_kst);
+
+ if (atomic_read(&next->thread.fpsimd_kernel_depth)) {
+ fpsimd_load_state((struct user_fpsimd_state *)nxt_kst);
+ this_cpu_write(fpsimd_last_state.st,
+ (struct user_fpsimd_state *)nxt_kst);
+ nxt_kst->cpu = smp_processor_id();
+ }
+
/*
* Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
* state. For kernel threads, FPSIMD registers are never loaded
@@ -1233,6 +1241,55 @@ void fpsimd_save_and_flush_cpu_state(void)
__put_cpu_fpsimd_context();
}

+void fpsimd_set_task_using(struct task_struct *t)
+{
+ atomic_set(&t->thread.fpsimd_kernel_depth, 1);
+}
+EXPORT_SYMBOL(fpsimd_set_task_using);
+
+void fpsimd_clr_task_using(struct task_struct *t)
+{
+ atomic_set(&t->thread.fpsimd_kernel_depth, 0);
+}
+EXPORT_SYMBOL(fpsimd_clr_task_using);
+
+void fpsimd_get(void)
+{
+ if (in_interrupt())
+ return;
+
+ if (atomic_inc_return(&current->thread.fpsimd_kernel_depth) == 1) {
+ preempt_disable();
+ if (current->mm) {
+ fpsimd_save();
+ fpsimd_flush_task_state(current);
+ }
+ fpsimd_flush_cpu_state();
+ preempt_enable();
+ }
+}
+EXPORT_SYMBOL(fpsimd_get);
+
+void fpsimd_put(void)
+{
+ if (in_interrupt())
+ return;
+
+ WARN_ON(atomic_dec_return(
+ &current->thread.fpsimd_kernel_depth) < 0);
+
+ if (atomic_read(&current->thread.fpsimd_kernel_depth) == 0) {
+ preempt_disable();
+ if (current->mm && test_thread_flag(TIF_FOREIGN_FPSTATE)) {
+ task_fpsimd_load();
+ fpsimd_bind_task_to_cpu();
+ clear_thread_flag(TIF_FOREIGN_FPSTATE);
+ }
+ preempt_enable();
+ }
+}
+EXPORT_SYMBOL(fpsimd_put);
+
#ifdef CONFIG_KERNEL_MODE_NEON

/*
@@ -1338,6 +1395,7 @@ void __efi_fpsimd_begin(void)
__this_cpu_write(efi_fpsimd_state_used, true);
}
}
+EXPORT_SYMBOL(__efi_fpsimd_begin);

/*
* __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
@@ -1364,6 +1422,7 @@ void __efi_fpsimd_end(void)
}
}
}
+EXPORT_SYMBOL(__efi_fpsimd_end);

#endif /* CONFIG_EFI */

--
2.27.0.rc0