RE: [PATCH v19,RESEND 24/27] x86/vdso: Add __vdso_sgx_enter_enclave() to wrap SGX enclave transitions

[Xing, Cedric]

Hi Andy,

> From: linux-sgx-owner@xxxxxxxxxxxxxxx [mailto:linux-sgx-
> owner@xxxxxxxxxxxxxxx] On Behalf Of Andy Lutomirski
> 
> I suppose the real question is: are there a significant number of
> users who will want to run enclaves created using an old SDK on Linux?
>  And will there actually be support for doing this in the software
> stack?

To your first question, I cannot share information of Intel customers or speak for them. But in general, people would like to stay with an old enclave usually because of: 1) attestation, because MRENCLAVE will change after rebuild; and/or 2) the need to support a mix of older and newer Linux kernels. So I'd say it'll be commonly desired, especially when this vDSO API is still "new" (so not available on every platform).

To your second question, Intel will support all "legacy" enclaves built with older SGX SDKs on newer kernels. And that's why we are so eager to find a migration path. I can't speak for other companies, but guess backward compatibility is always desirable.

> 
> If the answer to both questions is yes, then it seems like it could be
> reasonable to support it in the vDSO.  But I still think it should
> probably be a different vDSO entry point so that the normal case
> doesn't become more complicated.

I'll support whatever you think is more appropriate.

At the end, I'd like to give out the full version of my proposal, with your feedbacks (i.e. stack unwinder and Spectre variant 2) addressed. I'm a bit concerned by retpoline, which won't work (or be needed) when CET comes online. Are you looking to change it again then?

Here's the summary of the changes:
 - Added CFI directives for proper unwinding.
 - Defined sgx_ex_callback - the callback function on enclave exit/exception.
 - Aligned stack properly before calling sgx_ex_callback (per x86_64 ABI).
 - Used retpoline in place of indirect call.
 - The block starting at label "4:" captures all the code necessary to support sgx_ex_call. It has grown longer due to retpoline.

/**
 * __vdso_sgx_enter_enclave() - Enter an SGX enclave
 *
 * %eax:        ENCLU leaf, must be either EENTER or ERESUME
 * 0x08(%rsp):  TCS
 * 0x10(%rsp):  Optional pointer to 'struct sgx_enclave_exception'
 * 0x18(%rsp):  Optional function pointer to 'sgx_ex_callback', whose
 *              definition will be given below. Note that this function, if
 *              present, shall follow x86_64 ABI.
 * return:      0 (zero) on success, or a negative error code on failure.
 *
 * Note that __vdso_sgx_enter_enclave() is not compatible with x86_64 ABI.
 * All registers except RBP must be treated as volatile from the caller's
 * perspective, including but not limited to GPRs, EFLAGS.DF, MXCSR, FCW, etc...
 * Enclave may decrement RSP, but must not increment it - i.e. existing content
 * of the stack shall be preserved.
 *
 * sgx_ex_callback - A callback function to be invoked by
 * __vdso_sgx_enter_enclave() upon exception or after the enclave exits.
 *
 * typedef int (*sgx_ex_callback)(long rdi, long rsi, long rdx,
 *      struct sgx_enclave_exception *ex_info, long r8, long r9,
 *      long rsp, void *tcs);
 *
 * Note that sgx_ex_callback shall be x86_64 ABI compliant.
 *
 * Note that other GPRs (except %rax, %rbx and %rcx) are also passed through to
 * sgx_ex_callback, even though accessing them requires assembly code.
 */
__vdso_sgx_enter_enclave:
        /* prolog */
        .cfi_startproc
        push    %rbp
        .cfi_adjust_cfa_offset  8
        .cfi_rel_offset         %rbp, 0
        mov     %rsp, %rbp
        .cfi_def_cfa_register   %rbp

1:      /* EENTER <= leaf <= ERESUME */
        cmp     $0x2, %eax
        jb      5f
        cmp     $0x3, %eax
        ja      5f

        /* Load TCS and AEP */
        mov     0x10(%rbp), %rbx
        lea     2f(%rip), %rcx

2:      enclu

        /* EEXIT path */
        mov     0x18(%rbp), %rcx
        jrcxz   3f
        mov     %eax, EX_LEAF(%rcx)
        /* normalize return value */
3:      xor     %eax, %eax

4:      /* call sgx_ex_callback if supplied */
        cmpq    $0, 0x20(%rbp)
        jz      6f
        /* align stack per x86_64 ABI */
        mov     %rsp, %rbx
        and     $-0x10, %rsp
        /* parameters */
        push    0x10(%rbp)
        push    %rbx
        /* call *0x20(%rbp) using retpoline */
        mov     0x20(%rbp), %rax
        call    41f
        /* stack cleanup */
        mov     %rbx, %rsp
        jmp     1b
41:     call    43f
42:     pause
        lfence
        jmp     42b
43:     mov     %rax, (%rsp)
        ret

5:      /* bad leaf */
        cmp     $0, %eax
        jle     6f
        mov     $(-EINVAL), %eax

6:      /* epilog */
        leave
        .cfi_def_cfa            %rsp, 8
        ret
        .cfi_endproc

.pushsection    .fixup, "ax"
7:      mov     0x18(%rbp), %rcx
        jrcxz   8f
        /* fill in ex_info */
        mov     %eax, EX_LEAF(%rcx)
        mov     %di, EX_TRAPNR(%rcx)
        mov     %si, EX_ERROR_CODE(%rcx)
        mov     %rdx, EX_ADDRESS(%rcx)
8:      mov     $(-EFAULT), %eax
        jmp     4b
.popsection

_ASM_VDSO_EXTABLE_HANDLE(2b, 7b)


-Cedric