Re: about seg fault inside rseq critical section

[Mathieu Desnoyers]

----- On Apr 10, 2021, at 6:49 PM, Mingyi Liu mingyiliu@xxxxxxxxxx wrote:

> Hi Mathieu ,

> Thanks for your reply. From my current understanding of rseq, when flags
> enabled, any signal generated/happened inside the critical section would
> trigger the user-defined abort handler.

No, that's not how rseq works.

If a signal handler is executed on top of a rseq critical section, it will be allowed
to run on top of the userspace thread, but the instruction pointer where the signal
handler will return is modified by the kernel when it detects that it delivers a signal
on top of a rseq critical section so the signal handler will return to the abort ip.

See rseq_signal_deliver() for details.

Thanks,

Mathieu


> However, I tried to cause a segmentation fault like the following but it didn't
> go to the abort handler and was terminated by OS immediately.

> sigsegv_case.c

>> #define _GNU_SOURCE
>> #include <linux/rseq.h>
>> #include <stdio.h>
>> #include <stdlib.h>
>> #include <syscall.h>
>> #include <stdint.h>
>> #include <unistd.h>
>> #include <sys/syscall.h>
>> #include <signal.h>

>> static __thread volatile struct rseq __rseq_abi;

>> #define RSEQ_SIG 0x53053053

>> static int sys_rseq ( volatile struct rseq * rseq_abi , uint32_t rseq_len ,
>> int flags , uint32_t sig ) {
>> return syscall (__NR_rseq, rseq_abi, rseq_len, flags, sig);
>> }

>> static void register_thread ( void ) {
>> int rc;
>> rc = sys_rseq (&__rseq_abi, sizeof ( struct rseq), 0 , RSEQ_SIG);
>> if (rc) {
>> fprintf (stderr, "Failed to register rseq \n " );
>> exit ( 1 );
>> }
>> }

>> #define RSEQ_ACCESS_ONCE( x ) ( * (__volatile__ __typeof__ (x) * ) & (x))

>> int do_test ( void * addr , int cpu ) {

>> __asm__ __volatile__ goto (

>> ".pushsection __rseq_table, \" aw \"\n\t "
>> ".balign 32 \n\t "
>> "cs_obj: \n\t "
>> ".int 0, 0 \n\t "
>> /* start_ip, post_commit_offset, abort_ip */
>> ".quad 1f, 2f - 1f, 4f \n\t "
>> ".popsection \n\t "
>> "1: \n\t "
>> "leaq cs_obj(%%rip), %%rax \n\t "
>> "movq %%rax, % [rseq_cs] \n\t "
>> "cmpl % [cpu_id], % [current_cpu_id] \n\t "
>> "jnz 4f \n\t "

>> /* enable seg fault */
>> "jmp * % [_addr] \n\t " // seg falut
>> /********************/

>> "jmp % l[committed] \n\t "

>> "2: \n\t "
>> /* Disassembler-friendly signature: nopl <sig>(%rip). */
>> ".byte 0x0f, 0x1f, 0x05 \n\t "
>> ".long 0x53053053 \n\t " /* RSEQ_SIG */
>> "4: \n\t "
>> "jmp % l[aborted] \n\t "
>> : /* no outputs */
>> : [cpu_id] "r" (cpu),
>> [current_cpu_id] "m" ( __rseq_abi . cpu_id ),
>> [rseq_cs] "m" ( __rseq_abi . rseq_cs ),
>> [_addr] "r" (addr)
>> : "memory" , "cc" , "rax"
>> : aborted, committed

>> );

>> committed:
>> printf ( "committed \n " );
>> return 0 ;

>> aborted:
>> printf ( "aborted \n " );
>> return - 1 ;

>> }

>> int main ( int argc , char ** argv ) {
>> int cpu, ret;

>> register_thread ();
>> cpu = RSEQ_ACCESS_ONCE ( __rseq_abi . cpu_id_start );

>> printf ( "ret = %d \n " , do_test (( void *) 0xdeadbeef , cpu));
>> return 0 ;
>> }

> Also, I've tried SIGINT inside the critical section as below, but the program
> didn't go to the abort handler, either.

> sigint_case.c

>> #define _GNU_SOURCE
>> #include <linux/rseq.h>
>> #include <stdio.h>
>> #include <stdlib.h>
>> #include <syscall.h>
>> #include <stdint.h>
>> #include <unistd.h>
>> #include <sys/syscall.h>
>> #include <signal.h>

>> static __thread volatile struct rseq __rseq_abi;

>> #define RSEQ_SIG 0x53053053

>> static int sys_rseq ( volatile struct rseq * rseq_abi , uint32_t rseq_len ,
>> int flags , uint32_t sig ) {
>> return syscall (__NR_rseq, rseq_abi, rseq_len, flags, sig);
>> }

>> static void register_thread ( void ) {
>> int rc;
>> rc = sys_rseq (&__rseq_abi, sizeof ( struct rseq), 0 , RSEQ_SIG);
>> if (rc) {
>> fprintf (stderr, "Failed to register rseq \n " );
>> exit ( 1 );
>> }
>> }

>> #define RSEQ_ACCESS_ONCE( x ) ( * (__volatile__ __typeof__ (x) * ) & (x))

>> int do_test ( int cpu ) {

>> __asm__ __volatile__ goto (

>> ".pushsection __rseq_table, \" aw \"\n\t "
>> ".balign 32 \n\t "
>> "cs_obj: \n\t "
>> ".int 0, 0 \n\t "
>> /* start_ip, post_commit_offset, abort_ip */
>> ".quad 1f, 2f - 1f, 4f \n\t "
>> ".popsection \n\t "
>> "1: \n\t "
>> "leaq cs_obj(%%rip), %%rax \n\t "
>> "movq %%rax, % [rseq_cs] \n\t "
>> "cmpl % [cpu_id], % [current_cpu_id] \n\t "
>> "jnz 4f \n\t "
>> /* enable signal testing */
>> "movq $5, %%rdi \n\t "
>> "call sleep@plt \n\t "
>> /*************************/

>> "jmp % l[committed] \n\t "

>> "2: \n\t "
>> /* Disassembler-friendly signature: nopl <sig>(%rip). */
>> ".byte 0x0f, 0x1f, 0x05 \n\t "
>> ".long 0x53053053 \n\t " /* RSEQ_SIG */
>> "4: \n\t "
>> "jmp % l[aborted] \n\t "
>> : /* no outputs */
>> : [cpu_id] "r" (cpu),
>> [current_cpu_id] "m" ( __rseq_abi . cpu_id ),
>> [rseq_cs] "m" ( __rseq_abi . rseq_cs )
>> : "memory" , "cc" , "rax"
>> : aborted, committed

>> );

>> committed:
>> printf ( "committed \n " );
>> return 0 ;

>> aborted:
>> printf ( "aborted \n " );
>> return - 1 ;

>> }

>> void signal_callback_handler ( int signum ) {
>> printf ( "Caught signal %d \n " , signum);
>> }

>> int main ( int argc , char ** argv ) {
>> signal (SIGINT, signal_callback_handler);
>> int cpu, ret;

>> register_thread ();
>> cpu = RSEQ_ACCESS_ONCE ( __rseq_abi . cpu_id_start );

>> printf ( "ret = %d \n " , do_test (cpu));

>> return 0 ;
>> }

> As the screenshot is shown, the program executed the signal handler instead of
> rseq abort handler after interrupted the program.

> I am confused about how rseq deal with signal delivery as I assumed abort
> handler will be triggered anyway when flags permitted. Could you please explain
> such two cases or could you please share any references (code, article, etc)
> here?

> Thanks in advance!

> My virtual machine environment:

>> $ cat /etc/os-release
>> NAME="Ubuntu"
>> VERSION="20.04.2 LTS (Focal Fossa)"
>> ...

>> $ uname -r
>> 5.4.0-66-generic

> Best regards,
> Mingyi

> From: Mathieu Desnoyers <mathieu.desnoyers@xxxxxxxxxxxx>
> Sent: Wednesday, April 7, 2021 9:25 AM
> To: Liu, Mingyi <mingyiliu@xxxxxxxxxx>
> Cc: linux-kernel <linux-kernel@xxxxxxxxxxxxxxx>; Peter Zijlstra
> <peterz@xxxxxxxxxxxxx>; paulmck <paulmck@xxxxxxxxxx>; Boqun Feng
> <boqun.feng@xxxxxxxxx>
> Subject: Re: about seg fault inside rseq critical section
> (re-sent in plain-text for lkml)

> ----- On Apr 6, 2021, at 6:24 PM, Mingyi Liu mingyiliu@xxxxxxxxxx wrote:

> > Hi Mathieu,

> > I notice that the program will be terminated with segmentation fault when it is
> > even seg faulted inside the rseq critical section. In Linux kernel rseq.c, I
> > haven't found comment or code regarding this. Could you share any references on
> > why it doesn't execute user defined abort handler but uses the OS handler
> > instead?

> > Thanks in advance!

> Hi Mingyi,

> Please let me add the other rseq maintainers and LKML in CC. I'm a bit stretched
> on time
> here, so maybe they will have time to answer before I do.

> Meanwhile, if you could provide details about your architecture, kernel .config,
> and a
> small reproducer program, it would help bootstrapping the discussion.

> Thanks,

> Mathieu

> > Best,

> > Mingyi

> --
> Mathieu Desnoyers
> EfficiOS Inc.
> [ http://www.efficios.com/ | http://www.efficios.com ]

-- 
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com