Re: [RFC PATCH 4/5] rseq: register and unregister operations via prctl

[Dmitry Vyukov]

On Fri, 28 Aug 2026 at 17:34, <odion@xxxxxxxxxxxx> wrote:
>
> From: Olivier Dion <odion@xxxxxxxxxxxx>
>
> Implement the PR_RSEQ_OP prctl. Registration splices a user-provided
> pristine node into the head of the circular operation list, and
> unregistration unsplices it after validating it is properly linked
> between its neighbours; the kernel owns the node's next/prev links for
> the lifetime of the registration. The 0<->1 nr_ops transition reflects
> the enabled state into rseq_event::rseq_op and the user visible flag via
> rseq_op_update_enabled().
>
> Initialize the operation list sentinel as an empty self-pointing circular
> list on rseq registration, and reset nr_ops / rseq_op state there.
>
> Apply pending operations on the slow path and on signal delivery, and
> factor the common "clear error and force SIGSEGV" fixup into
> force_fault().
>
> Signed-off-by: Olivier Dion <odion@xxxxxxxxxxxx>
> ---
>  kernel/rseq.c | 220 +++++++++++++++++++++++++++++++++++++++++++++++---
>  kernel/sys.c  |   5 ++
>  2 files changed, 212 insertions(+), 13 deletions(-)
>
> diff --git a/kernel/rseq.c b/kernel/rseq.c
> index e75e3a5e312c..a277c11dae99 100644
> --- a/kernel/rseq.c
> +++ b/kernel/rseq.c
> @@ -253,17 +253,18 @@ static bool rseq_handle_cs(struct task_struct *t, struct pt_regs *regs)
>  static void rseq_slowpath_update_usr(struct pt_regs *regs)
>  {
>         /*
> -        * Preserve has_rseq and user_irq state. The generic entry code clears
> -        * user_irq on the way out, the non-generic entry architectures are not
> -        * setting user_irq.
> +        * Preserve has_rseq, rseq_op and user_irq state. The generic entry
> +        * code clears user_irq on the way out, the non-generic entry
> +        * architectures are not setting user_irq.
>          */
>         const struct rseq_event evt_mask = {
>                 .has_rseq       = RSEQ_HAS_RSEQ_VERSION_MASK,
> +               .rseq_op        = true,
>                 .user_irq       = true,
>         };
>         struct task_struct *t = current;
>         struct rseq_ids ids;
> -       bool event;
> +       bool event, should_fault = false;
>
>         if (unlikely(t->flags & PF_EXITING))
>                 return;
> @@ -300,7 +301,12 @@ static void rseq_slowpath_update_usr(struct pt_regs *regs)
>
>         ids.node_id = cpu_to_node(ids.cpu_id);
>
> -       if (unlikely(!rseq_update_usr(t, regs, &ids))) {
> +       if (unlikely(!rseq_update_usr(t, regs, &ids)))
> +               should_fault = true;
> +       else if (t->rseq.event.rseq_op && unlikely(!rseq_apply_ops(t)))
> +               should_fault = true;
> +
> +       if (should_fault) {
>                 /*
>                  * Clear the errors just in case this might survive magically, but
>                  * leave the rest intact.
> @@ -329,8 +335,20 @@ void __rseq_handle_slowpath(struct pt_regs *regs)
>         rseq_slowpath_update_usr(regs);
>  }
>
> +static inline void force_fault(int sig)
> +{
> +       /*
> +        * Clear the errors just in case this might survive magically, but leave
> +        * the rest intact.
> +        */
> +       current->rseq.event.error = 0;
> +       force_sigsegv(sig);
> +}
> +
>  void __rseq_signal_deliver(int sig, struct pt_regs *regs)
>  {
> +       bool should_fault = false;
> +
>         rseq_stat_inc(rseq_stats.signal);
>
>         /*
> @@ -339,14 +357,13 @@ void __rseq_signal_deliver(int sig, struct pt_regs *regs)
>          * the interrupted context as after this point the instruction
>          * pointer in @regs points to the signal handler.
>          */
> -       if (unlikely(!rseq_handle_cs(current, regs))) {
> -               /*
> -                * Clear the errors just in case this might survive
> -                * magically, but leave the rest intact.
> -                */
> -               current->rseq.event.error = 0;
> -               force_sigsegv(sig);
> -       }
> +       if (unlikely(!rseq_handle_cs(current, regs)))
> +               should_fault = true;
> +       else if (current->rseq.event.rseq_op && unlikely(!rseq_apply_ops(current)))

Just to make sure: membarrier delivered to a task will also trigger
rseq_apply_ops, right?

> +               should_fault = true;
> +
> +       if (should_fault)
> +               force_fault(sig);
>
>         /*
>          * In legacy mode, force the update of IDs before returning to user
> @@ -436,6 +453,8 @@ static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u3
>                                 rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_ENABLED;
>                 }
>         }
> +       if (version > 1)
> +               rseqfl |= RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE;
>
>         scoped_user_write_access(rseq, efault) {
>                 /*
> @@ -458,8 +477,16 @@ static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u3
>                  * registrations.
>                  */
>                 if (version > 1) {
> +                       u64 sentinel = (u64)&rseq->rseq_op_list;
> +
>                         if (IS_ENABLED(CONFIG_RSEQ_SLICE_EXTENSION))
>                                 unsafe_put_user(0U, &rseq->slice_ctrl.all, efault);
> +                       /*
> +                        * Initialize the rseq operation list sentinel as an
> +                        * empty circular doubly-linked list pointing to itself.
> +                        */
> +                       unsafe_put_user(sentinel, &rseq->rseq_op_list.next, efault);
> +                       unsafe_put_user(sentinel, &rseq->rseq_op_list.prev, efault);

Is there always enough space to write these? Can this corrupt program memory?


>                 }
>         }
>
> @@ -474,6 +501,12 @@ static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u3
>  #ifdef CONFIG_RSEQ_SLICE_EXTENSION
>         current->rseq.slice.state.enabled = !!(rseqfl & RSEQ_CS_FLAG_SLICE_EXT_ENABLED);
>  #endif
> +       /*
> +        * A fresh registration starts with no operations, so operation
> +        * processing is disabled until the first one is registered.
> +        */
> +       current->rseq.event.rseq_op = false;
> +       current->rseq.nr_ops = 0;
>
>         /*
>          * Ensure the cpu_id_start and cpu_id fields are updated before
> @@ -887,3 +920,164 @@ device_initcall(rseq_slice_init);
>  #else
>  static void rseq_slice_ext_init(struct dentry *root_dir) { }
>  #endif /* CONFIG_RSEQ_SLICE_EXTENSION */
> +
> +/*
> + * Reflect the operation enabled state into the user visible flags field.
> + * Called on the 0<->1 transition of nr_ops. The kill path is taken on fault
> + * because losing this update leaves user space and kernel state inconsistent.
> + */
> +static int rseq_op_update_enabled(struct task_struct *t, bool enable)
> +{
> +       struct rseq __user *rseq = t->rseq.usrptr;
> +       u32 rflags;
> +
> +       if (get_user(rflags, &rseq->flags))
> +               return -EFAULT;
> +
> +       rflags &= ~RSEQ_CS_FLAG_RSEQ_OP_ENABLED;
> +       rflags |= RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE;
> +       if (enable)
> +               rflags |= RSEQ_CS_FLAG_RSEQ_OP_ENABLED;
> +
> +       if (put_user(rflags, &rseq->flags))
> +               return -EFAULT;
> +
> +       t->rseq.event.rseq_op = enable;
> +       return 0;
> +}
> +
> +/*
> + * Register @node at the head of the circular doubly-linked operation list
> + * anchored by the kernel owned sentinel in struct rseq.
> + */
> +static int rseq_op_register(struct task_struct *t, struct rseq_op_node __user *node)
> +{
> +       struct rseq_op_node __user *sentinel = &t->rseq.usrptr->rseq_op_list;
> +       struct rseq_op_node __user *first;
> +       u64 next, prev, first_addr;
> +       u8 type, i;
> +
> +       if (t->rseq.nr_ops >= RSEQ_OP_LIST_LIMIT)
> +               return -ENOSPC;
> +
> +       if (!IS_ALIGNED((unsigned long)node, __alignof__(struct rseq_op_node)))
> +               return -EINVAL;
> +       if (!access_ok(node, sizeof(*node)))
> +               return -EFAULT;
> +
> +       /*
> +        * The node links are owned by the kernel. User space must present a
> +        * pristine node: next, prev and the reserved bytes all zeroed, and a
> +        * known operation type.
> +        */
> +       if (get_user(next, &node->next) || get_user(prev, &node->prev) ||
> +           get_user(type, &node->type))
> +               return -EFAULT;
> +       if (next || prev)
> +               return -EINVAL;
> +       if (type >= RSEQ_OP_NR)
> +               return -EINVAL;
> +       for (i = 1; i < sizeof(node->reserved); i++) {
> +               u8 r;
> +
> +               if (get_user(r, &node->reserved[i]))
> +                       return -EFAULT;
> +               if (r)
> +                       return -EINVAL;
> +       }
> +
> +       /* Splice the node in right after the sentinel. */
> +       if (get_user(first_addr, &sentinel->next))
> +               goto die;
> +       first = (struct rseq_op_node __user *)first_addr;
> +
> +       if (put_user((u64)(unsigned long)first, &node->next) ||
> +           put_user((u64)(unsigned long)sentinel, &node->prev) ||
> +           put_user((u64)(unsigned long)node, &first->prev) ||
> +           put_user((u64)(unsigned long)node, &sentinel->next))
> +               goto die;
> +
> +       t->rseq.nr_ops += 1;
> +
> +       if (t->rseq.nr_ops == 1)
> +               return rseq_op_update_enabled(t, true) ? -EFAULT : 0;
> +       return 0;
> +die:
> +       force_sig(SIGSEGV);
> +       return -EFAULT;
> +}
> +
> +/*
> + * Unregister @node from the operation list. The node links are validated
> + * against its neighbours to reject bogus or double unregistration.
> + */
> +static int rseq_op_unregister(struct task_struct *t, struct rseq_op_node __user *node)
> +{
> +       struct rseq_op_node __user *prev, *next;
> +       u64 prev_addr, next_addr, tmp;
> +
> +       if (!t->rseq.nr_ops)
> +               return -ENOENT;
> +
> +       if (!IS_ALIGNED((unsigned long)node, __alignof__(struct rseq_op_node)))
> +               return -EINVAL;
> +       if (!access_ok(node, sizeof(*node)))
> +               return -EFAULT;
> +
> +       if (get_user(next_addr, &node->next) || get_user(prev_addr, &node->prev))
> +               return -EFAULT;
> +       prev = (struct rseq_op_node __user *)prev_addr;
> +       next = (struct rseq_op_node __user *)next_addr;
> +
> +       /* A registered node always has both links set. */
> +       if (!prev || !next)
> +               return -EINVAL;
> +
> +       /* Verify the node is properly linked between its neighbours. */
> +       if (get_user(tmp, &prev->next))
> +               goto die;
> +       if (tmp != (u64)(unsigned long)node)
> +               return -EINVAL;
> +       if (get_user(tmp, &next->prev))
> +               goto die;
> +       if (tmp != (u64)(unsigned long)node)
> +               return -EINVAL;
> +
> +       /* Unsplice and clear the node links so it can be reused. */
> +       if (put_user(next_addr, &prev->next) ||
> +           put_user(prev_addr, &next->prev) ||
> +           put_user(0ULL, &node->next) ||
> +           put_user(0ULL, &node->prev))
> +               goto die;
> +
> +       t->rseq.nr_ops -= 1;
> +
> +       if (t->rseq.nr_ops == 0)
> +               return rseq_op_update_enabled(t, false) ? -EFAULT : 0;
> +       return 0;
> +die:
> +       force_sig(SIGSEGV);
> +       return -EFAULT;
> +}
> +
> +int rseq_op_prctl(unsigned long arg2, unsigned long arg3)
> +{
> +       struct rseq_op_node __user *node = (struct rseq_op_node __user *)arg3;
> +       struct task_struct *t = current;
> +
> +       if (!t->rseq.usrptr)
> +               return -ENXIO;
> +       if (!rseq_v2(t))
> +               return -ENOTSUPP;
> +       if (!node)
> +               return -EINVAL;
> +
> +       switch (arg2) {
> +       case PR_RSEQ_OP_REGISTER:
> +               return rseq_op_register(t, node);
> +       case PR_RSEQ_OP_UNREGISTER:
> +               return rseq_op_unregister(t, node);
> +       default:
> +               return -EINVAL;
> +       }
> +}
> diff --git a/kernel/sys.c b/kernel/sys.c
> index df69bd71de03..494c91b03c3f 100644
> --- a/kernel/sys.c
> +++ b/kernel/sys.c
> @@ -2889,6 +2889,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
>                         return -EINVAL;
>                 error = rseq_slice_extension_prctl(arg2, arg3);
>                 break;
> +       case PR_RSEQ_OP:
> +               if (arg4 || arg5)
> +                       return -EINVAL;
> +               error = rseq_op_prctl(arg2, arg3);
> +               break;
>         case PR_GET_CFI:
>                 if (arg2 != PR_CFI_BRANCH_LANDING_PADS)
>                         return -EINVAL;
> --
> 2.54.0
>