Re: [PATCH v4 2/5] LoongArch: Use la.pcrel instead of la.abs for exception handlers

[Youling Tang]

On 02/10/2023 05:09 PM, Huacai Chen wrote:
> Hi, Youling and Ruoyao,
>
> Thank you very much for implementing the per-node exceptions. But I
> want to know if the per-node solution is really worthy for a PIE
> kernel. So, could you please test the performance? Maybe we can reduce
> the complexity if we give up the per-node solution.

I will test performance on NUMA machines based on v2 and v3 patch sets.

Youling.
> Huacai
>
> On Fri, Feb 10, 2023 at 4:47 PM Youling Tang <tangyouling@xxxxxxxxxxx> wrote:
> > From: Xi Ruoyao <xry111@xxxxxxxxxxx>
> >
> > It's needed to build the kernel as a PIE, or the linker will complain.
> >
> > For the consideration about performance, we copy the exception handlers
> > to a dedicated 64 KB area for each CPU.  So, the PC-relative offset
> > calculated at link time will be incorrect and we need to relocate the
> > exception handlers after copying them.
> >
> > For the simplicity, we don't use the ELF R_LARCH_* relocations, but code
> > an relocation entry as simply (offset_in_the_handler, symbol_addr).  For
> > each exception handler, we also code the number of relocation entries.
> > Then we can use the relocation information to fix up the handlers after
> > copying them.
> >
> > Signed-off-by: Xi Ruoyao <xry111@xxxxxxxxxxx>
> > ---
> >  arch/loongarch/include/asm/inst.h       |   1 +
> >  arch/loongarch/include/asm/setup.h      |   6 +-
> >  arch/loongarch/include/asm/stackframe.h |   3 +-
> >  arch/loongarch/kernel/genex.S           |  40 +++++-
> >  arch/loongarch/kernel/traps.c           | 158 ++++++++++++++++++++----
> >  arch/loongarch/mm/tlb.c                 |  23 ++--
> >  arch/loongarch/mm/tlbex.S               |  69 +++++++++--
> >  7 files changed, 255 insertions(+), 45 deletions(-)
> >
> > diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h
> > index 7eedd83fd0d7..426054518a3d 100644
> > --- a/arch/loongarch/include/asm/inst.h
> > +++ b/arch/loongarch/include/asm/inst.h
> > @@ -32,6 +32,7 @@ enum reg1i20_op {
> >         lu12iw_op       = 0x0a,
> >         lu32id_op       = 0x0b,
> >         pcaddi_op       = 0x0c,
> > +       pcalau12i_op    = 0x0d,
> >         pcaddu12i_op    = 0x0e,
> >         pcaddu18i_op    = 0x0f,
> >  };
> > diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h
> > index 72ead58039f3..f0a2b34365f1 100644
> > --- a/arch/loongarch/include/asm/setup.h
> > +++ b/arch/loongarch/include/asm/setup.h
> > @@ -11,6 +11,9 @@
> >
> >  #define VECSIZE 0x200
> >
> > +struct handler_reloc;
> > +
> > +extern struct handler_reloc *eentry_reloc[];
> >  extern unsigned long eentry;
> >  extern unsigned long tlbrentry;
> >  extern char init_command_line[COMMAND_LINE_SIZE];
> > @@ -18,7 +21,8 @@ extern void tlb_init(int cpu);
> >  extern void cpu_cache_init(void);
> >  extern void cache_error_setup(void);
> >  extern void per_cpu_trap_init(int cpu);
> > -extern void set_handler(unsigned long offset, void *addr, unsigned long len);
> > +extern void set_handler(unsigned long exccode, void *addr);
> >  extern void set_merr_handler(unsigned long offset, void *addr, unsigned long len);
> > +extern void reloc_handler(unsigned long handler, struct handler_reloc *rel);
> >
> >  #endif /* __SETUP_H */
> > diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h
> > index 7deb043ce387..bbec1e56b61b 100644
> > --- a/arch/loongarch/include/asm/stackframe.h
> > +++ b/arch/loongarch/include/asm/stackframe.h
> > @@ -77,7 +77,8 @@
> >   * new value in sp.
> >   */
> >         .macro  get_saved_sp docfi=0
> > -       la.abs    t1, kernelsp
> > +       /* The label is used for generating reloc tables for handlers */
> > +514:   la.pcrel  t1, t0, kernelsp
> >  #ifdef CONFIG_SMP
> >         csrrd     t0, PERCPU_BASE_KS
> >         LONG_ADD  t1, t1, t0
> > diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S
> > index 7e5c293ed89f..005a10fe5a50 100644
> > --- a/arch/loongarch/kernel/genex.S
> > +++ b/arch/loongarch/kernel/genex.S
> > @@ -34,7 +34,7 @@ SYM_FUNC_END(__arch_cpu_idle)
> >  SYM_FUNC_START(handle_vint)
> >         BACKUP_T0T1
> >         SAVE_ALL
> > -       la.abs  t1, __arch_cpu_idle
> > +0:     la.pcrel t1, t2, __arch_cpu_idle
> >         LONG_L  t0, sp, PT_ERA
> >         /* 32 byte rollback region */
> >         ori     t0, t0, 0x1f
> > @@ -43,11 +43,25 @@ SYM_FUNC_START(handle_vint)
> >         LONG_S  t0, sp, PT_ERA
> >  1:     move    a0, sp
> >         move    a1, sp
> > -       la.abs  t0, do_vint
> > +2:     la.pcrel t0, t2, do_vint
> >         jirl    ra, t0, 0
> >         RESTORE_ALL_AND_RET
> >  SYM_FUNC_END(handle_vint)
> >
> > +SYM_DATA_START(rel_handle_vint)
> > +LONG   3
> > +
> > +LONG   514b - handle_vint
> > +LONG   kernelsp
> > +
> > +LONG   0b - handle_vint
> > +LONG   __arch_cpu_idle
> > +
> > +LONG   2b - handle_vint
> > +LONG   do_vint
> > +
> > +SYM_DATA_END(rel_handle_vint)
> > +
> >  SYM_FUNC_START(except_vec_cex)
> >         b       cache_parity_error
> >  SYM_FUNC_END(except_vec_cex)
> > @@ -72,12 +86,24 @@ SYM_FUNC_END(except_vec_cex)
> >         SAVE_ALL
> >         build_prep_\prep
> >         move    a0, sp
> > -       la.abs  t0, do_\handler
> > +       667:
> > +       la.pcrel t0, t1, do_\handler
> >         jirl    ra, t0, 0
> >         668:
> >         RESTORE_ALL_AND_RET
> >         SYM_FUNC_END(handle_\exception)
> >         SYM_DATA(unwind_hint_\exception, .word 668b - 666b)
> > +
> > +       SYM_DATA_START(rel_handle_\exception)
> > +       LONG    2
> > +
> > +       LONG    514b - 666b
> > +       LONG    kernelsp
> > +
> > +       LONG    667b - 666b
> > +       LONG    do_\handler
> > +
> > +       SYM_DATA_END(rel_handle_\exception)
> >         .endm
> >
> >         BUILD_HANDLER ade ade badv
> > @@ -93,6 +119,12 @@ SYM_FUNC_END(except_vec_cex)
> >         BUILD_HANDLER reserved reserved none    /* others */
> >
> >  SYM_FUNC_START(handle_sys)
> > -       la.abs  t0, handle_syscall
> > +       la.pcrel t0, t1, handle_syscall
> >         jr      t0
> >  SYM_FUNC_END(handle_sys)
> > +
> > +SYM_DATA_START(rel_handle_sys)
> > +LONG   1
> > +LONG   0
> > +LONG   handle_syscall
> > +SYM_DATA_END(rel_handle_sys)
> > diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c
> > index c38a146a973b..7e073854f493 100644
> > --- a/arch/loongarch/kernel/traps.c
> > +++ b/arch/loongarch/kernel/traps.c
> > @@ -62,6 +62,127 @@ extern asmlinkage void handle_reserved(void);
> >  extern asmlinkage void handle_watch(void);
> >  extern asmlinkage void handle_vint(void);
> >
> > +struct handler_reloc_entry {
> > +       unsigned long offset;
> > +       unsigned long sym;
> > +};
> > +
> > +struct handler_reloc {
> > +       unsigned long cnt;
> > +       struct handler_reloc_entry entries[];
> > +};
> > +
> > +extern struct handler_reloc rel_handle_tlb_load;
> > +extern struct handler_reloc rel_handle_tlb_store;
> > +extern struct handler_reloc rel_handle_tlb_modify;
> > +extern struct handler_reloc rel_handle_tlb_protect;
> > +extern struct handler_reloc rel_handle_ade;
> > +extern struct handler_reloc rel_handle_ale;
> > +extern struct handler_reloc rel_handle_sys;
> > +extern struct handler_reloc rel_handle_bp;
> > +extern struct handler_reloc rel_handle_ri;
> > +extern struct handler_reloc rel_handle_fpu;
> > +extern struct handler_reloc rel_handle_lsx;
> > +extern struct handler_reloc rel_handle_lasx;
> > +extern struct handler_reloc rel_handle_fpe;
> > +extern struct handler_reloc rel_handle_lbt;
> > +extern struct handler_reloc rel_handle_watch;
> > +extern struct handler_reloc rel_handle_reserved;
> > +extern struct handler_reloc rel_handle_vint;
> > +
> > +struct handler_reloc *eentry_reloc[128] = {
> > +       [0] = NULL, /* merr handler */
> > +       [EXCCODE_TLBL] = &rel_handle_tlb_load,
> > +       [EXCCODE_TLBS] = &rel_handle_tlb_store,
> > +       [EXCCODE_TLBI] = &rel_handle_tlb_load,
> > +       [EXCCODE_TLBM] = &rel_handle_tlb_modify,
> > +       [EXCCODE_TLBNR] = &rel_handle_tlb_protect,
> > +       [EXCCODE_TLBNX] = &rel_handle_tlb_protect,
> > +       [EXCCODE_TLBPE] = &rel_handle_tlb_protect,
> > +       [EXCCODE_ADE] = &rel_handle_ade,
> > +       [EXCCODE_ALE] = &rel_handle_ale,
> > +       [EXCCODE_SYS] = &rel_handle_sys,
> > +       [EXCCODE_BP] = &rel_handle_bp,
> > +       [EXCCODE_INE] = &rel_handle_ri,
> > +       [EXCCODE_IPE] = &rel_handle_ri,
> > +       [EXCCODE_FPDIS] = &rel_handle_fpu,
> > +       [EXCCODE_LSXDIS] = &rel_handle_lsx,
> > +       [EXCCODE_LASXDIS] = &rel_handle_lasx,
> > +       [EXCCODE_FPE] = &rel_handle_fpe,
> > +       [EXCCODE_BTDIS] = &rel_handle_lbt,
> > +       [EXCCODE_WATCH] = &rel_handle_watch,
> > +       [(EXCCODE_WATCH + 1) ... (EXCCODE_INT_START - 1)] = &rel_handle_reserved,
> > +       [EXCCODE_INT_START ... (EXCCODE_INT_END - 1)] = &rel_handle_vint,
> > +};
> > +
> > +void reloc_handler(unsigned long handler, struct handler_reloc *rel)
> > +{
> > +       if (!rel)
> > +               return;
> > +
> > +       for (unsigned long i = 0; i < rel->cnt; i++) {
> > +               unsigned long pc = handler + rel->entries[i].offset;
> > +               union loongarch_instruction *insn =
> > +                       (union loongarch_instruction *)pc;
> > +               u32 imm[4];
> > +               unsigned long v = rel->entries[i].sym;
> > +
> > +               /* GNU as >= 2.40 uses pcalau12i for la.pcrel, but GNU ld <= 2.39
> > +                * uses pcaddu12i.
> > +                */
> > +               if (insn->reg1i20_format.opcode == pcalau12i_op) {
> > +                       /* Use s32 deliberately for sign extension. */
> > +                       s32 offset_hi20 = ((v + 0x800) & ~0xfff) -
> > +                                         (pc & ~0xfff);
> > +                       unsigned long anchor = (pc & ~0xfff) + offset_hi20;
> > +                       unsigned long offset_rem = v - anchor;
> > +
> > +                       imm[0] = (offset_hi20 >> 12) & 0xfffff;
> > +                       imm[1] = v & 0xfff;
> > +                       imm[2] = (offset_rem >> 32) & 0xfffff;
> > +                       imm[3] = offset_rem >> 52;
> > +               } else if (insn->reg1i20_format.opcode == pcaddu12i_op) {
> > +                       /* Use s32 deliberately for sign extension. */
> > +                       s32 offset_lo = v - pc;
> > +                       unsigned long offset_hi = v - pc - offset_lo;
> > +
> > +                       imm[0] = (offset_lo >> 12) & 0xfffff;
> > +                       imm[1] = offset_lo & 0xfff;
> > +                       imm[2] = (offset_hi >> 32) & 0xfffff;
> > +                       imm[3] = offset_hi >> 52;
> > +               } else
> > +                       panic("Cannot fixup la.pcrel for exception handler at %lu: unexpected instruction %d!",
> > +                             pc, insn->word);
> > +
> > +               insn[0].reg1i20_format.immediate = imm[0];
> > +               insn[1].reg2i12_format.immediate = imm[1];
> > +               insn[2].reg1i20_format.immediate = imm[2];
> > +               insn[3].reg2i12_format.immediate = imm[3];
> > +       }
> > +}
> > +
> > +/* Install CPU exception handler */
> > +static void do_set_handler(unsigned long exccode, void *addr,
> > +                          struct handler_reloc *rel)
> > +{
> > +       unsigned long dest_addr = eentry + exccode * VECSIZE;
> > +
> > +       memcpy((void *)dest_addr, addr, VECSIZE);
> > +       reloc_handler(dest_addr, rel);
> > +       local_flush_icache_range(dest_addr, dest_addr + VECSIZE);
> > +}
> > +
> > +/* Install CPU exception handler, with the reloc table from eentry_reloc */
> > +void set_handler(unsigned long exccode, void *addr)
> > +{
> > +       do_set_handler(exccode, addr, eentry_reloc[exccode]);
> > +}
> > +
> > +static void set_handler_reserved(unsigned long exccode)
> > +{
> > +       do_set_handler(exccode, handle_reserved, &rel_handle_reserved);
> > +}
> > +
> >  static void show_backtrace(struct task_struct *task, const struct pt_regs *regs,
> >                            const char *loglvl, bool user)
> >  {
> > @@ -704,19 +825,12 @@ void per_cpu_trap_init(int cpu)
> >         /* Initialise exception handlers */
> >         if (cpu == 0)
> >                 for (i = 0; i < 64; i++)
> > -                       set_handler(i * VECSIZE, handle_reserved, VECSIZE);
> > +                       set_handler_reserved(i);
> >
> >         tlb_init(cpu);
> >         cpu_cache_init();
> >  }
> >
> > -/* Install CPU exception handler */
> > -void set_handler(unsigned long offset, void *addr, unsigned long size)
> > -{
> > -       memcpy((void *)(eentry + offset), addr, size);
> > -       local_flush_icache_range(eentry + offset, eentry + offset + size);
> > -}
> > -
> >  static const char panic_null_cerr[] =
> >         "Trying to set NULL cache error exception handler\n";
> >
> > @@ -741,20 +855,20 @@ void __init trap_init(void)
> >
> >         /* Set interrupt vector handler */
> >         for (i = EXCCODE_INT_START; i < EXCCODE_INT_END; i++)
> > -               set_handler(i * VECSIZE, handle_vint, VECSIZE);
> > -
> > -       set_handler(EXCCODE_ADE * VECSIZE, handle_ade, VECSIZE);
> > -       set_handler(EXCCODE_ALE * VECSIZE, handle_ale, VECSIZE);
> > -       set_handler(EXCCODE_SYS * VECSIZE, handle_sys, VECSIZE);
> > -       set_handler(EXCCODE_BP * VECSIZE, handle_bp, VECSIZE);
> > -       set_handler(EXCCODE_INE * VECSIZE, handle_ri, VECSIZE);
> > -       set_handler(EXCCODE_IPE * VECSIZE, handle_ri, VECSIZE);
> > -       set_handler(EXCCODE_FPDIS * VECSIZE, handle_fpu, VECSIZE);
> > -       set_handler(EXCCODE_LSXDIS * VECSIZE, handle_lsx, VECSIZE);
> > -       set_handler(EXCCODE_LASXDIS * VECSIZE, handle_lasx, VECSIZE);
> > -       set_handler(EXCCODE_FPE * VECSIZE, handle_fpe, VECSIZE);
> > -       set_handler(EXCCODE_BTDIS * VECSIZE, handle_lbt, VECSIZE);
> > -       set_handler(EXCCODE_WATCH * VECSIZE, handle_watch, VECSIZE);
> > +               set_handler(i, handle_vint);
> > +
> > +       set_handler(EXCCODE_ADE, handle_ade);
> > +       set_handler(EXCCODE_ALE, handle_ale);
> > +       set_handler(EXCCODE_SYS, handle_sys);
> > +       set_handler(EXCCODE_BP, handle_bp);
> > +       set_handler(EXCCODE_INE, handle_ri);
> > +       set_handler(EXCCODE_IPE, handle_ri);
> > +       set_handler(EXCCODE_FPDIS, handle_fpu);
> > +       set_handler(EXCCODE_LSXDIS, handle_lsx);
> > +       set_handler(EXCCODE_LASXDIS, handle_lasx);
> > +       set_handler(EXCCODE_FPE, handle_fpe);
> > +       set_handler(EXCCODE_BTDIS, handle_lbt);
> > +       set_handler(EXCCODE_WATCH, handle_watch);
> >
> >         cache_error_setup();
> >
> > diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c
> > index 8bad6b0cff59..6f70aab7202a 100644
> > --- a/arch/loongarch/mm/tlb.c
> > +++ b/arch/loongarch/mm/tlb.c
> > @@ -253,7 +253,6 @@ static void output_pgtable_bits_defines(void)
> >  #ifdef CONFIG_NUMA
> >  unsigned long pcpu_handlers[NR_CPUS];
> >  #endif
> > -extern long exception_handlers[VECSIZE * 128 / sizeof(long)];
> >
> >  void setup_tlb_handler(int cpu)
> >  {
> > @@ -264,19 +263,20 @@ void setup_tlb_handler(int cpu)
> >         if (cpu == 0) {
> >                 memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
> >                 local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
> > -               set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
> > -               set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
> > -               set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
> > -               set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
> > -               set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
> > -               set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
> > -               set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
> > +               set_handler(EXCCODE_TLBI, handle_tlb_load);
> > +               set_handler(EXCCODE_TLBL, handle_tlb_load);
> > +               set_handler(EXCCODE_TLBS, handle_tlb_store);
> > +               set_handler(EXCCODE_TLBM, handle_tlb_modify);
> > +               set_handler(EXCCODE_TLBNR, handle_tlb_protect);
> > +               set_handler(EXCCODE_TLBNX, handle_tlb_protect);
> > +               set_handler(EXCCODE_TLBPE, handle_tlb_protect);
> >         }
> >  #ifdef CONFIG_NUMA
> >         else {
> >                 void *addr;
> > +               unsigned long addr_ul;
> >                 struct page *page;
> > -               const int vec_sz = sizeof(exception_handlers);
> > +               const int vec_sz = VECSIZE * 128;
> >
> >                 if (pcpu_handlers[cpu])
> >                         return;
> > @@ -286,8 +286,11 @@ void setup_tlb_handler(int cpu)
> >                         return;
> >
> >                 addr = page_address(page);
> > +               addr_ul = (unsigned long)addr;
> >                 pcpu_handlers[cpu] = (unsigned long)addr;
> > -               memcpy((void *)addr, (void *)eentry, vec_sz);
> > +               memcpy(addr, (void *)eentry, vec_sz);
> > +               for (unsigned long i = 0; i < 128; i++)
> > +                       reloc_handler(addr_ul + i * VECSIZE, eentry_reloc[i]);
> >                 local_flush_icache_range((unsigned long)addr, (unsigned long)addr + vec_sz);
> >                 csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_EENTRY);
> >                 csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_MERRENTRY);
> > diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S
> > index 3dd2a9615cd9..044c2190771a 100644
> > --- a/arch/loongarch/mm/tlbex.S
> > +++ b/arch/loongarch/mm/tlbex.S
> > @@ -39,11 +39,21 @@ SYM_FUNC_START(handle_tlb_protect)
> >         move            a1, zero
> >         csrrd           a2, LOONGARCH_CSR_BADV
> >         REG_S           a2, sp, PT_BVADDR
> > -       la.abs          t0, do_page_fault
> > +1:     la.pcrel        t0, t1, do_page_fault
> >         jirl            ra, t0, 0
> >         RESTORE_ALL_AND_RET
> >  SYM_FUNC_END(handle_tlb_protect)
> >
> > +SYM_DATA_START(rel_handle_tlb_protect)
> > +       LONG    2
> > +
> > +       LONG    514b - handle_tlb_protect
> > +       LONG    kernelsp
> > +
> > +       LONG    1b - handle_tlb_protect
> > +       LONG    do_page_fault
> > +SYM_DATA_END(rel_handle_tlb_protect)
> > +
> >  SYM_FUNC_START(handle_tlb_load)
> >         csrwr           t0, EXCEPTION_KS0
> >         csrwr           t1, EXCEPTION_KS1
> > @@ -115,7 +125,8 @@ smp_pgtable_change_load:
> >
> >  #ifdef CONFIG_64BIT
> >  vmalloc_load:
> > -       la.abs          t1, swapper_pg_dir
> > +       /* The first insn of vmalloc_done_load overwrites ra */
> > +1:     la.pcrel        t1, ra, swapper_pg_dir
> >         b               vmalloc_done_load
> >  #endif
> >
> > @@ -186,10 +197,24 @@ tlb_huge_update_load:
> >  nopage_tlb_load:
> >         dbar            0
> >         csrrd           ra, EXCEPTION_KS2
> > -       la.abs          t0, tlb_do_page_fault_0
> > +2:     la.pcrel        t0, t1, tlb_do_page_fault_0
> >         jr              t0
> >  SYM_FUNC_END(handle_tlb_load)
> >
> > +SYM_DATA_START(rel_handle_tlb_load)
> > +#ifdef CONFIG_64BIT
> > +       LONG    2
> > +
> > +       LONG    1b - handle_tlb_load
> > +       LONG    swapper_pg_dir
> > +#else
> > +       LONG    1
> > +#endif
> > +
> > +       LONG    2b - handle_tlb_load
> > +       LONG    tlb_do_page_fault_0
> > +SYM_DATA_END(rel_handle_tlb_load)
> > +
> >  SYM_FUNC_START(handle_tlb_store)
> >         csrwr           t0, EXCEPTION_KS0
> >         csrwr           t1, EXCEPTION_KS1
> > @@ -262,7 +287,8 @@ smp_pgtable_change_store:
> >
> >  #ifdef CONFIG_64BIT
> >  vmalloc_store:
> > -       la.abs          t1, swapper_pg_dir
> > +       /* The first insn of vmalloc_done_store overwrites ra */
> > +1:     la.pcrel        t1, ra, swapper_pg_dir
> >         b               vmalloc_done_store
> >  #endif
> >
> > @@ -335,10 +361,24 @@ tlb_huge_update_store:
> >  nopage_tlb_store:
> >         dbar            0
> >         csrrd           ra, EXCEPTION_KS2
> > -       la.abs          t0, tlb_do_page_fault_1
> > +2:     la.pcrel        t0, t1, tlb_do_page_fault_1
> >         jr              t0
> >  SYM_FUNC_END(handle_tlb_store)
> >
> > +SYM_DATA_START(rel_handle_tlb_store)
> > +#ifdef CONFIG_64BIT
> > +       LONG    2
> > +
> > +       LONG    1b - handle_tlb_store
> > +       LONG    swapper_pg_dir
> > +#else
> > +       LONG    1
> > +#endif
> > +
> > +       LONG    2b - handle_tlb_store
> > +       LONG    tlb_do_page_fault_1
> > +SYM_DATA_END(rel_handle_tlb_store)
> > +
> >  SYM_FUNC_START(handle_tlb_modify)
> >         csrwr           t0, EXCEPTION_KS0
> >         csrwr           t1, EXCEPTION_KS1
> > @@ -410,7 +450,8 @@ smp_pgtable_change_modify:
> >
> >  #ifdef CONFIG_64BIT
> >  vmalloc_modify:
> > -       la.abs          t1, swapper_pg_dir
> > +       /* The first insn of vmalloc_done_modify overwrites ra */
> > +1:     la.pcrel        t1, ra, swapper_pg_dir
> >         b               vmalloc_done_modify
> >  #endif
> >
> > @@ -482,10 +523,24 @@ tlb_huge_update_modify:
> >  nopage_tlb_modify:
> >         dbar            0
> >         csrrd           ra, EXCEPTION_KS2
> > -       la.abs          t0, tlb_do_page_fault_1
> > +2:     la.pcrel        t0, t1, tlb_do_page_fault_1
> >         jr              t0
> >  SYM_FUNC_END(handle_tlb_modify)
> >
> > +SYM_DATA_START(rel_handle_tlb_modify)
> > +#ifdef CONFIG_64BIT
> > +       LONG    2
> > +
> > +       LONG    1b - handle_tlb_modify
> > +       LONG    swapper_pg_dir
> > +#else
> > +       LONG    1
> > +#endif
> > +
> > +       LONG    2b - handle_tlb_modify
> > +       LONG    tlb_do_page_fault_1
> > +SYM_DATA_END(rel_handle_tlb_modify)
> > +
> >  SYM_FUNC_START(handle_tlb_refill)
> >         csrwr           t0, LOONGARCH_CSR_TLBRSAVE
> >         csrrd           t0, LOONGARCH_CSR_PGD
> > --
> > 2.37.3