Re: [RFC PATCH v2 3/3] selftests/x86: Augment SGX selftest to test new __vdso_sgx_enter_enclave() and its callback interface

[Xing, Cedric]

On 7/11/2019 8:25 PM, Jarkko Sakkinen wrote:
> On Tue, Apr 23, 2019 at 11:26:23PM -0700, Cedric Xing wrote:
> > This patch augments SGX selftest with two new tests.
> >
> > The first test exercises the newly added callback interface, by marking the
> > whole enclave range as PROT_READ, then calling mprotect() upon #PFs to add
> > necessary PTE permissions per PFEC (#PF Error Code) until the enclave finishes.
> > This test also serves as an example to demonstrate the callback interface.
> >
> > The second test single-steps through __vdso_sgx_enter_enclave() to make sure
> > the call stack can be unwound at every instruction within that vDSO API. Its
> > purpose is to validate the hand-crafted CFI directives in the assembly.
> >
> > Signed-off-by: Cedric Xing <cedric.xing@xxxxxxxxx>
> > ---
> >   tools/testing/selftests/x86/sgx/Makefile   |   6 +-
> >   tools/testing/selftests/x86/sgx/main.c     | 323 ++++++++++++++++++---
> >   tools/testing/selftests/x86/sgx/sgx_call.S |  40 ++-
> >   3 files changed, 322 insertions(+), 47 deletions(-)
> >
> > diff --git a/tools/testing/selftests/x86/sgx/Makefile b/tools/testing/selftests/x86/sgx/Makefile
> > index 3af15d7c8644..31f937e220c4 100644
> > --- a/tools/testing/selftests/x86/sgx/Makefile
> > +++ b/tools/testing/selftests/x86/sgx/Makefile
> > @@ -14,16 +14,16 @@ TEST_CUSTOM_PROGS := $(OUTPUT)/test_sgx
> >   all_64: $(TEST_CUSTOM_PROGS)
> >   
> >   $(TEST_CUSTOM_PROGS): main.c sgx_call.S $(OUTPUT)/encl_piggy.o
> > -	$(CC) $(HOST_CFLAGS) -o $@ $^
> > +	$(CC) $(HOST_CFLAGS) -o $@ $^ -lunwind -ldl -Wl,--defsym,__image_base=0 -pie
> >   
> >   $(OUTPUT)/encl_piggy.o: encl_piggy.S $(OUTPUT)/encl.bin $(OUTPUT)/encl.ss
> >   	$(CC) $(HOST_CFLAGS) -I$(OUTPUT) -c $< -o $@
> >   
> >   $(OUTPUT)/encl.bin: $(OUTPUT)/encl.elf
> > -	objcopy --remove-section=.got.plt -O binary $< $@
> > +	objcopy -O binary $< $@
> >   
> >   $(OUTPUT)/encl.elf: encl.lds encl.c encl_bootstrap.S
> > -	$(CC) $(ENCL_CFLAGS) -T $^ -o $@
> > +	$(CC) $(ENCL_CFLAGS) -T $^ -o $@ -Wl,--build-id=none
> >   
> >   $(OUTPUT)/encl.ss: $(OUTPUT)/sgxsign signing_key.pem $(OUTPUT)/encl.bin
> >   	$^ $@
> > diff --git a/tools/testing/selftests/x86/sgx/main.c b/tools/testing/selftests/x86/sgx/main.c
> > index e2265f841fb0..d3e53c71306d 100644
> > --- a/tools/testing/selftests/x86/sgx/main.c
> > +++ b/tools/testing/selftests/x86/sgx/main.c
> > @@ -1,6 +1,7 @@
> >   // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
> >   // Copyright(c) 2016-18 Intel Corporation.
> >   
> > +#define _GNU_SOURCE
> >   #include <elf.h>
> >   #include <fcntl.h>
> >   #include <stdbool.h>
> > @@ -9,16 +10,31 @@
> >   #include <stdlib.h>
> >   #include <string.h>
> >   #include <unistd.h>
> > +#include <errno.h>
> >   #include <sys/ioctl.h>
> >   #include <sys/mman.h>
> >   #include <sys/stat.h>
> > -#include <sys/time.h>
> > +#include <sys/auxv.h>
> > +#include <signal.h>
> > +#include <sys/ucontext.h>
> > +
> > +#define UNW_LOCAL_ONLY
> > +#include <libunwind.h>
> > +
> >   #include "encl_piggy.h"
> >   #include "defines.h"
> >   #include "../../../../../arch/x86/kernel/cpu/sgx/arch.h"
> >   #include "../../../../../arch/x86/include/uapi/asm/sgx.h"
> >   
> > -static const uint64_t MAGIC = 0x1122334455667788ULL;
> > +#define _Q(x)	__Q(x)
> > +#define __Q(x)	#x
> > +#define ERRLN	"Line " _Q(__LINE__)
> > +
> > +#define X86_EFLAGS_TF	(1ul << 8)
> > +
> > +extern char __image_base[];
> > +size_t eenter;
> > +static size_t vdso_base;
> >   
> >   struct vdso_symtab {
> >   	Elf64_Sym *elf_symtab;
> > @@ -26,20 +42,11 @@ struct vdso_symtab {
> >   	Elf64_Word *elf_hashtab;
> >   };
> >   
> > -static void *vdso_get_base_addr(char *envp[])
> > +static void vdso_init(void)
> >   {
> > -	Elf64_auxv_t *auxv;
> > -	int i;
> > -
> > -	for (i = 0; envp[i]; i++);
> > -	auxv = (Elf64_auxv_t *)&envp[i + 1];
> > -
> > -	for (i = 0; auxv[i].a_type != AT_NULL; i++) {
> > -		if (auxv[i].a_type == AT_SYSINFO_EHDR)
> > -			return (void *)auxv[i].a_un.a_val;
> > -	}
> > -
> > -	return NULL;
> > +	vdso_base = getauxval(AT_SYSINFO_EHDR);
> > +	if (!vdso_base)
> > +		exit(1);
> >   }
>
> The clean up makes sense but should be a separate patch i.e. one
> logical change per patch. Right now the patch does other mods
> than the ones explcitly stated in the commit message.
>
> I'd suggest open coding vdso_init() to the call site in that
> patch.
>
> Please try to always minimize for diff's.
>
> >   
> >   static Elf64_Dyn *vdso_get_dyntab(void *addr)
> > @@ -66,8 +73,9 @@ static void *vdso_get_dyn(void *addr, Elf64_Dyn *dyntab, Elf64_Sxword tag)
> >   	return NULL;
> >   }
> >   
> > -static bool vdso_get_symtab(void *addr, struct vdso_symtab *symtab)
> > +static bool vdso_get_symtab(struct vdso_symtab *symtab)
> >   {
> > +	void *addr = (void *)vdso_base;
> >   	Elf64_Dyn *dyntab = vdso_get_dyntab(addr);
> >   
> >   	symtab->elf_symtab = vdso_get_dyn(addr, dyntab, DT_SYMTAB);
> > @@ -138,7 +146,7 @@ static bool encl_create(int dev_fd, unsigned long bin_size,
> >   	base = mmap(NULL, secs->size, PROT_READ | PROT_WRITE | PROT_EXEC,
> >   		    MAP_SHARED, dev_fd, 0);
> >   	if (base == MAP_FAILED) {
> > -		perror("mmap");
> > +		perror(ERRLN);
> >   		return false;
> >   	}
> >   
> > @@ -224,35 +232,271 @@ static bool encl_load(struct sgx_secs *secs, unsigned long bin_size)
> >   	return false;
> >   }
> >   
> > -void sgx_call(void *rdi, void *rsi, void *tcs,
> > -	      struct sgx_enclave_exception *exception,
> > -	      void *eenter);
> > +int sgx_call(void *rdi, void *rsi, long rdx, void *rcx, void *r8, void *r9,
> > +	     void *tcs, struct sgx_enclave_exinfo *ei, void *cb);
> > +
> > +static void show_enclave_exinfo(const struct sgx_enclave_exinfo *exinfop,
> > +				const char *header)
> > +{
> > +	static const char * const enclu_leaves[] = {
> > +		"EREPORT",
> > +		"EGETKEY",
> > +		"EENTER",
> > +		"ERESUME",
> > +		"EEXIT"
> > +	};
> > +	static const char * const exception_names[] = {
> > +		"#DE",
> > +		"#DB",
> > +		"NMI",
> > +		"#BP",
> > +		"#OF",
> > +		"#BR",
> > +		"#UD",
> > +		"#NM",
> > +		"#DF",
> > +		"CSO",
> > +		"#TS",
> > +		"#NP",
> > +		"#SS",
> > +		"#GP",
> > +		"#PF",
> > +		"Unknown",
> > +		"#MF",
> > +		"#AC",
> > +		"#MC",
> > +		"#XM",
> > +		"#VE",
> > +		"Unknown",
> > +		"Unknown",
> > +		"Unknown",
> > +		"Unknown",
> > +		"Unknown",
> > +		"Unknown",
> > +		"Unknown",
> > +		"Unknown",
> > +		"Unknown",
> > +		"Unknown",
> > +		"Unknown"
> > +	};
> > +
> > +	printf("%s: leaf:%s(%d)", header,
> > +		enclu_leaves[exinfop->leaf], exinfop->leaf);
> > +	if (exinfop->leaf != 4)
> > +		printf(" trap:%s(%d) ec:%d addr:0x%llx\n",
> > +			exception_names[exinfop->trapnr], exinfop->trapnr,
> > +			exinfop->error_code, exinfop->address);
> > +	else
> > +		printf("\n");
> > +}
> > +
> > +static const uint64_t MAGIC = 0x1122334455667788ULL;
> >   
> > -int main(int argc, char *argv[], char *envp[])
> > +static void test1(struct sgx_secs *secs)
>
> test1, test2 and test3 are not too descriptive names. Every patch should
> make the code base cleaner, not messier.

I don't think it that important, as there are many test## occurrences in 
existing selftests. Anyway, I've added comments to those test functions 
to brief what is done. Hope you'll find them helpful.

> /Jarkko