Re: [PATCH v3 03/15] arm64: kvm: Add build rules for separate nVHE object files

[Marc Zyngier]

Hi David,

On 2020-06-18 13:25, David Brazdil wrote:
> Add new folder arch/arm64/kvm/hyp/nvhe and a Makefile for building code 
> that
> runs in EL2 under nVHE KVM.
>
> Compile each source file into a `.hyp.tmp.o` object first, then prefix 
> all
> its symbols with "__kvm_nvhe_" using `objcopy` and produce a `.hyp.o`.
> Suffixes were chosen so that it would be possible for VHE and nVHE to 
> share
> some source files, but compiled with different CFLAGS. nVHE build rules 
> add
> -D__KVM_NVHE_HYPERVISOR__.
>
> The nVHE ELF symbol prefix is added to kallsyms.c as ignored. EL2-only 
> symbols
> will never appear in EL1 stack traces.
>
> Signed-off-by: David Brazdil <dbrazdil@xxxxxxxxxx>
> ---
>  arch/arm64/kernel/image-vars.h   | 12 +++++++++++
>  arch/arm64/kvm/hyp/Makefile      |  2 +-
>  arch/arm64/kvm/hyp/nvhe/Makefile | 35 ++++++++++++++++++++++++++++++++
>  scripts/kallsyms.c               |  1 +
>  4 files changed, 49 insertions(+), 1 deletion(-)
>  create mode 100644 arch/arm64/kvm/hyp/nvhe/Makefile
>
> diff --git a/arch/arm64/kernel/image-vars.h 
> b/arch/arm64/kernel/image-vars.h
> index be0a63ffed23..f32b406e90c0 100644
> --- a/arch/arm64/kernel/image-vars.h
> +++ b/arch/arm64/kernel/image-vars.h
> @@ -51,4 +51,16 @@ __efistub__ctype		= _ctype;
>
>  #endif
>
> +#ifdef CONFIG_KVM
> +
> +/*
> + * KVM nVHE code has its own symbol namespace prefixed by __kvm_nvhe_, 
> to
> + * isolate it from the kernel proper. The following symbols are 
> legally
> + * accessed by it, therefore provide aliases to make them linkable.
> + * Do not include symbols which may not be safely accessed under 
> hypervisor
> + * memory mappings.
> + */
> +
> +#endif /* CONFIG_KVM */
> +
>  #endif /* __ARM64_KERNEL_IMAGE_VARS_H */
> diff --git a/arch/arm64/kvm/hyp/Makefile b/arch/arm64/kvm/hyp/Makefile
> index 5d8357ddc234..5f4f217532e0 100644
> --- a/arch/arm64/kvm/hyp/Makefile
> +++ b/arch/arm64/kvm/hyp/Makefile
> @@ -6,7 +6,7 @@
>  ccflags-y += -fno-stack-protector -DDISABLE_BRANCH_PROFILING \
>  		$(DISABLE_STACKLEAK_PLUGIN)
>
> -obj-$(CONFIG_KVM) += hyp.o
> +obj-$(CONFIG_KVM) += hyp.o nvhe/
>  obj-$(CONFIG_KVM_INDIRECT_VECTORS) += smccc_wa.o
>
>  hyp-y := vgic-v3-sr.o timer-sr.o aarch32.o vgic-v2-cpuif-proxy.o 
> sysreg-sr.o \
> diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile 
> b/arch/arm64/kvm/hyp/nvhe/Makefile
> new file mode 100644
> index 000000000000..7d64235dba62
> --- /dev/null
> +++ b/arch/arm64/kvm/hyp/nvhe/Makefile
> @@ -0,0 +1,35 @@
> +# SPDX-License-Identifier: GPL-2.0
> +#
> +# Makefile for Kernel-based Virtual Machine module, HYP/nVHE part
> +#
> +
> +asflags-y := -D__KVM_NVHE_HYPERVISOR__
> +ccflags-y := -D__KVM_NVHE_HYPERVISOR__ -fno-stack-protector \
> +	     -DDISABLE_BRANCH_PROFILING $(DISABLE_STACKLEAK_PLUGIN)
> +
> +obj-y :=
> +
> +obj-y := $(patsubst %.o,%.hyp.o,$(obj-y))
> +extra-y := $(patsubst %.hyp.o,%.hyp.tmp.o,$(obj-y))
> +
> +$(obj)/%.hyp.tmp.o: $(src)/%.c FORCE
> +	$(call if_changed_rule,cc_o_c)
> +$(obj)/%.hyp.tmp.o: $(src)/%.S FORCE
> +	$(call if_changed_rule,as_o_S)
> +$(obj)/%.hyp.o: $(obj)/%.hyp.tmp.o FORCE
> +	$(call if_changed,hypcopy)
> +
> +quiet_cmd_hypcopy = HYPCOPY $@
> +      cmd_hypcopy = $(OBJCOPY) --prefix-symbols=__kvm_nvhe_ $< $@
> +
> +# KVM nVHE code is run at a different exception code with a different 
> map, so
> +# compiler instrumentation that inserts callbacks or checks into the 
> code may
> +# cause crashes. Just disable it.
> +GCOV_PROFILE	:= n
> +KASAN_SANITIZE	:= n
> +UBSAN_SANITIZE	:= n
> +KCOV_INSTRUMENT	:= n
> +
> +# Skip objtool checking for this directory because nVHE code is 
> compiled with
> +# non-standard build rules.
> +OBJECT_FILES_NON_STANDARD := y
> diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c
> index 6dc3078649fa..0096cd965332 100644
> --- a/scripts/kallsyms.c
> +++ b/scripts/kallsyms.c
> @@ -109,6 +109,7 @@ static bool is_ignored_symbol(const char *name, 
> char type)
>  		".LASANPC",		/* s390 kasan local symbols */
>  		"__crc_",		/* modversions */
>  		"__efistub_",		/* arm64 EFI stub namespace */
> +		"__kvm_nvhe_",		/* arm64 non-VHE KVM namespace */
>  		NULL
>  	};

I guess that one of the first use of this __KVM_NVHE_HYPERVISOR__
flag could be the has_vhe() predicate: if you're running the nVHE
code, you are *guaranteed* not to use VHE at all.

Something like:

diff --git a/arch/arm64/include/asm/virt.h 
b/arch/arm64/include/asm/virt.h
index 5051b388c654..b2cb8fce43dd 100644
--- a/arch/arm64/include/asm/virt.h
+++ b/arch/arm64/include/asm/virt.h
@@ -85,10 +85,8 @@ static inline bool is_kernel_in_hyp_mode(void)

 static __always_inline bool has_vhe(void)
 {
-	if (cpus_have_final_cap(ARM64_HAS_VIRT_HOST_EXTN))
-		return true;
-
-	return false;
+	return (__is_defined(__KVM_NVHE_HYPERVISOR__) &&
+		cpus_have_final_cap(ARM64_HAS_VIRT_HOST_EXTN));
 }

 #endif /* __ASSEMBLY__ */

Thanks,

        M.
--
Jazz is not dead. It just smells funny...