Re: [PATCH] perf llvm: Fix compile bpf failed to cope with latest llvm

[Nick Desaulniers]

On Thu, Apr 14, 2022 at 6:42 AM Yang Jihong <yangjihong1@xxxxxxxxxx> wrote:
>
> Inline assembly used by asm/sysreg.h is incompatible with latest llvm,

With "latest" llvm makes it sound like LLVM has regressed. Has it? Or
have the headers changed in a way where now inline asm from a
different target (x86) than what's being targeted (BPF)? Perhaps
fixing that is simpler?

Clang will validate inline asm before LLVM drops unreferenced static
inline functions; this was a headache getting i386 building with
clang, but not insurmountable.

> If bpf C program include "linux/ptrace.h" (which is common), compile will fail:
>
>  # perf --debug verbose record -e bpf-output/no-inherit,name=evt/ \
>                               -e ./perf_bpf_test.c/map:channel.event=evt/ ls
>  [SNIP]
>  /lib/modules/5.17/build/./arch/x86/include/asm/arch_hweight.h:20:7: error: invalid output constraint '=a' in asm
>                           : "="REG_OUT (res)
>                            ^
>  /lib/modules/5.17/build/./arch/x86/include/asm/arch_hweight.h:48:7: error: invalid output constraint '=a' in asm
>                           : "="REG_OUT (res)
>                            ^
>  In file included from /root/projects/perf-bpf/perf_bpf_test.c:2:
>  In file included from /lib/modules/5.17/build/./include/linux/ptrace.h:6:
>  In file included from /lib/modules/5.17/build/./include/linux/sched.h:12:
>  In file included from /lib/modules/5.17/build/./arch/x86/include/asm/current.h:6:
>  In file included from /lib/modules/5.17/build/./arch/x86/include/asm/percpu.h:27:
>  In file included from /lib/modules/5.17/build/./include/linux/kernel.h:25:
>  In file included from /lib/modules/5.17/build/./include/linux/math.h:6:
>  /lib/modules/5.17.0/build/./arch/x86/include/asm/div64.h:85:28: error: invalid output constraint '=a' in asm
>          asm ("mulq %2; divq %3" : "=a" (q)
>  [SNIP]
>  # cat /root/projects/perf-bpf/perf_bpf_test.c
>  /************************ BEGIN **************************/
>  #include <uapi/linux/bpf.h>
>  #include <linux/ptrace.h>
>  #include <linux/types.h>
>  #define SEC(NAME) __attribute__((section(NAME), used))
>
>  struct bpf_map_def {
>          unsigned int type;
>          unsigned int key_size;
>          unsigned int value_size;
>          unsigned int max_entries;
>  };
>
>  static int (*perf_event_output)(void *, struct bpf_map_def *, int, void *,
>      unsigned long) = (void *)BPF_FUNC_perf_event_output;
>
>  struct bpf_map_def SEC("maps") channel = {
>          .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,
>          .key_size = sizeof(int),
>          .value_size = sizeof(__u32),
>          .max_entries = __NR_CPUS__,
>  };
>
>  #define MIN_BYTES 1024
>
>  SEC("func=vfs_read")
>  int bpf_myprog(struct pt_regs *ctx)
>  {
>          long bytes = ctx->dx;
>          if (bytes >= MIN_BYTES) {
>                  perf_event_output(ctx, &channel, BPF_F_CURRENT_CPU,
>                                    &bytes, sizeof(bytes));
>          }
>
>          return 0;
>  }
>
> char _license[] SEC("license") = "GPL";
> int _version SEC("version") = LINUX_VERSION_CODE;
> /************************* END ***************************/
>
> Compilation command is modified to be the same as that in samples/bpf/Makefile,
> use clang | opt | llvm-dis | llc chain of commands to solve the problem.
> see the comment in samples/bpf/Makefile.
>
> Modifications:
> 1. Change "clang --target bpf" to chain of commands "clang | opt | llvm-dis | llc"

This will drop the --target flag. That will mess up the default target
triple, and can affect command line option feature detection.

> 2. Delete "CLANG_EMIT_LLVM" and "LLVM_OPTIONS_PIPE" macros from the compilation
>    command because the two macros are not defined and the content is empty.
> 3. Add options llvm.llvm-opt-path, llvm.llvm-dis-path, and llvm.llc-path to
>    perf config for user-defined settings, which are similar to llvm.clang-path.
> 4. Add "-Wno-address-of-packed-member" to resolve the compilation warning of
>    "/lib/modules/5.17/build/./arch/x86/include/asm/processor.h:552:17: \
>     warning: taking address of packed member 'sp0' of class or structure \
>     'x86_hw_tss' may result in an unaligned pointer value [-Waddress-of-packed-member]
>         this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0);
>                        ^~~~~~~~~~~~~~~~~~~~~~
>    /lib/modules/5.17/build/./include/linux/percpu-defs.h:508:68: note: \
>    expanded from macro 'this_cpu_write' \
>     #define this_cpu_write(pcp, val)        __pcpu_size_call(this_cpu_write_, pcp, val)",
>    which is similar to that of sample/bpf/Makefile.
>
> Signed-off-by: Yang Jihong <yangjihong1@xxxxxxxxxx>
> ---
>  tools/perf/Documentation/perf-config.txt | 16 ++++++++++-
>  tools/perf/util/llvm-utils.c             | 35 ++++++++++++++++++------
>  tools/perf/util/llvm-utils.h             |  4 +++
>  3 files changed, 45 insertions(+), 10 deletions(-)
>
> diff --git a/tools/perf/Documentation/perf-config.txt b/tools/perf/Documentation/perf-config.txt
> index 0420e71698ee..48f12bd6ca9a 100644
> --- a/tools/perf/Documentation/perf-config.txt
> +++ b/tools/perf/Documentation/perf-config.txt
> @@ -655,6 +655,15 @@ llvm.*::
>         llvm.clang-path::
>                 Path to clang. If omit, search it from $PATH.
>
> +       llvm.llvm-opt-path::
> +               Path to llvm opt. If omit, search it from $PATH.
> +
> +       llvm.llvm-dis-path::
> +               Path to llvm-dis. If omit, search it from $PATH.
> +
> +       llvm.llc-path::
> +               Path to llc. If omit, search it from $PATH.
> +
>         llvm.clang-bpf-cmd-template::
>                 Cmdline template. Below lines show its default value. Environment
>                 variable is used to pass options.
> @@ -662,8 +671,13 @@ llvm.*::
>                 "-DLINUX_VERSION_CODE=$LINUX_VERSION_CODE "     \
>                 "$CLANG_OPTIONS $PERF_BPF_INC_OPTIONS $KERNEL_INC_OPTIONS " \
>                 "-Wno-unused-value -Wno-pointer-sign "          \
> +               "-Wno-address-of-packed-member "                \
>                 "-working-directory $WORKING_DIR "              \
> -               "-c \"$CLANG_SOURCE\" -target bpf $CLANG_EMIT_LLVM -O2 -o - $LLVM_OPTIONS_PIPE"
> +               "-c \"$CLANG_SOURCE\" "                               \
> +               "-O2 -emit-llvm -Xclang -disable-llvm-passes -o - | " \
> +               "$LLVM_OPT_EXEC -O2 -mtriple=bpf-pc-linux | "         \
> +               "$LLVM_DIS_EXEC | "                                   \
> +               "$LLC_EXEC -march=bpf -filetype=obj -o -"
>
>         llvm.clang-opt::
>                 Options passed to clang.
> diff --git a/tools/perf/util/llvm-utils.c b/tools/perf/util/llvm-utils.c
> index 96c8ef60f4f8..c939681dfafb 100644
> --- a/tools/perf/util/llvm-utils.c
> +++ b/tools/perf/util/llvm-utils.c
> @@ -24,11 +24,18 @@
>                 "-DLINUX_VERSION_CODE=$LINUX_VERSION_CODE "     \
>                 "$CLANG_OPTIONS $PERF_BPF_INC_OPTIONS $KERNEL_INC_OPTIONS " \
>                 "-Wno-unused-value -Wno-pointer-sign "          \
> +               "-Wno-address-of-packed-member "                \
>                 "-working-directory $WORKING_DIR "              \
> -               "-c \"$CLANG_SOURCE\" -target bpf $CLANG_EMIT_LLVM -O2 -o - $LLVM_OPTIONS_PIPE"
> +               "-c \"$CLANG_SOURCE\" "                               \
> +               "-O2 -emit-llvm -Xclang -disable-llvm-passes -o - | " \
> +               "$LLVM_OPT_EXEC -O2 -mtriple=bpf-pc-linux | "         \
> +               "$LLVM_DIS_EXEC | "                                   \
> +               "$LLC_EXEC -march=bpf -filetype=obj -o -"
>
>  struct llvm_param llvm_param = {
>         .clang_path = "clang",
> +       .llvm_opt_path = "opt",
> +       .llvm_dis_path = "llvm-dis",
>         .llc_path = "llc",
>         .clang_bpf_cmd_template = CLANG_BPF_CMD_DEFAULT_TEMPLATE,
>         .clang_opt = NULL,
> @@ -48,6 +55,12 @@ int perf_llvm_config(const char *var, const char *value)
>
>         if (!strcmp(var, "clang-path"))
>                 llvm_param.clang_path = strdup(value);
> +       else if (!strcmp(var, "llvm-opt-path"))
> +               llvm_param.llvm_opt_path = strdup(value);
> +       else if (!strcmp(var, "llvm-dis-path"))
> +               llvm_param.llvm_dis_path = strdup(value);
> +       else if (!strcmp(var, "llc-path"))
> +               llvm_param.llc_path = strdup(value);
>         else if (!strcmp(var, "clang-bpf-cmd-template"))
>                 llvm_param.clang_bpf_cmd_template = strdup(value);
>         else if (!strcmp(var, "clang-opt"))
> @@ -456,6 +469,7 @@ int llvm__compile_bpf(const char *path, void **p_obj_buf,
>         char linux_version_code_str[64];
>         const char *clang_opt = llvm_param.clang_opt;
>         char clang_path[PATH_MAX], llc_path[PATH_MAX], abspath[PATH_MAX], nr_cpus_avail_str[64];
> +       char llvm_opt_path[PATH_MAX], llvm_dis_path[PATH_MAX];
>         char serr[STRERR_BUFSIZE];
>         char *kbuild_dir = NULL, *kbuild_include_opts = NULL,
>              *perf_bpf_include_opts = NULL;
> @@ -475,9 +489,10 @@ int llvm__compile_bpf(const char *path, void **p_obj_buf,
>         if (!template)
>                 template = CLANG_BPF_CMD_DEFAULT_TEMPLATE;
>
> -       err = search_program_and_warn(llvm_param.clang_path,
> -                            "clang", clang_path);
> -       if (err)
> +       if (search_program_and_warn(llvm_param.clang_path, "clang", clang_path) ||
> +           search_program_and_warn(llvm_param.llvm_opt_path, "opt", llvm_opt_path) ||
> +           search_program_and_warn(llvm_param.llvm_dis_path, "llvm-dis", llvm_dis_path) ||
> +           search_program_and_warn(llvm_param.llc_path, "llc", llc_path))
>                 return -ENOENT;
>
>         /*
> @@ -495,21 +510,23 @@ int llvm__compile_bpf(const char *path, void **p_obj_buf,
>
>         snprintf(linux_version_code_str, sizeof(linux_version_code_str),
>                  "0x%x", kernel_version);
> -       if (asprintf(&perf_bpf_include_opts, "-I%s/bpf", perf_include_dir) < 0)
> +       if (asprintf(&perf_bpf_include_opts, "-I%s/bpf", perf_include_dir) < 0) {
> +               err = -ENOMEM;
>                 goto errout;
> +       }
> +
>         force_set_env("NR_CPUS", nr_cpus_avail_str);
>         force_set_env("LINUX_VERSION_CODE", linux_version_code_str);
>         force_set_env("CLANG_EXEC", clang_path);
> +       force_set_env("LLVM_OPT_EXEC", llvm_opt_path);
> +       force_set_env("LLVM_DIS_EXEC", llvm_dis_path);
> +       force_set_env("LLC_EXEC", llc_path);
>         force_set_env("CLANG_OPTIONS", clang_opt);
>         force_set_env("KERNEL_INC_OPTIONS", kbuild_include_opts);
>         force_set_env("PERF_BPF_INC_OPTIONS", perf_bpf_include_opts);
>         force_set_env("WORKING_DIR", kbuild_dir ? : ".");
>
>         if (opts) {
> -               err = search_program_and_warn(llvm_param.llc_path, "llc", llc_path);
> -               if (err)
> -                       goto errout;
> -
>                 err = -ENOMEM;
>                 if (asprintf(&pipe_template, "%s -emit-llvm | %s -march=bpf %s -filetype=obj -o -",
>                               template, llc_path, opts) < 0) {
> diff --git a/tools/perf/util/llvm-utils.h b/tools/perf/util/llvm-utils.h
> index 7878a0e3fa98..e276d10f85b4 100644
> --- a/tools/perf/util/llvm-utils.h
> +++ b/tools/perf/util/llvm-utils.h
> @@ -11,6 +11,10 @@
>  struct llvm_param {
>         /* Path of clang executable */
>         const char *clang_path;
> +       /* Path of llvm opt executable */
> +       const char *llvm_opt_path;
> +       /* Path of llvm-dis executable */
> +       const char *llvm_dis_path;
>         /* Path of llc executable */
>         const char *llc_path;
>         /*
> --
> 2.30.GIT
>


-- 
Thanks,
~Nick Desaulniers