[PATCH bpf-next 06/16] libbpf: Relocate arena percpu access instructions

[Xu Kuohai]

From: Xu Kuohai <xukuohai@xxxxxxxxxx>

Introduce a pseudo instruction for arena percpu, and relocate the normal
arena percpu access instruction to it.

The pseudo instruction is encoded as a BPF_PSEUDO_ARENA_PERCPU ldimm64,
and carries the offset of the value in cpu 0's chunk. At run time, the JIT
translates the offset to the value's address in the running cpu's replica.

Signed-off-by: Xu Kuohai <xukuohai@xxxxxxxxxx>
---
 include/uapi/linux/bpf.h       | 13 +++++++++++++
 tools/include/uapi/linux/bpf.h | 13 +++++++++++++
 tools/lib/bpf/libbpf.c         | 35 +++++++++++++++++++++++++++++++++-
 3 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 6f70db7515dd..9e5fa52c9bc2 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1418,6 +1418,19 @@ enum {
  */
 #define BPF_PSEUDO_FUNC		4
 
+/* insn[0].src_reg:  BPF_PSEUDO_ARENA_PERCPU
+ * insn[0].imm:      map fd
+ * insn[1].imm:      offset of the value in cpu 0's chunk, including the chunk's
+ *                   offset in the arena and the value's offset in the chunk
+ * insn[0].off:      0
+ * insn[1].off:      0
+ * ldimm64 rewrite:  address of the value in cpu 0's chunk. The JIT adds
+ *                   smp_processor_id() * chunk_sz to step it into the
+ *                   running cpu's replica
+ * verifier type:    PTR_TO_ARENA
+ */
+#define BPF_PSEUDO_ARENA_PERCPU	7
+
 /* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative
  * offset to another bpf function
  */
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 246f52361372..6b17dcfc36b4 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1418,6 +1418,19 @@ enum {
  */
 #define BPF_PSEUDO_FUNC		4
 
+/* insn[0].src_reg:  BPF_PSEUDO_ARENA_PERCPU
+ * insn[0].imm:      map fd
+ * insn[1].imm:      offset of the value in cpu 0's chunk, including the chunk's
+ *                   offset in the arena and the value's offset in the chunk
+ * insn[0].off:      0
+ * insn[1].off:      0
+ * ldimm64 rewrite:  address of the value in cpu 0's chunk. The JIT adds
+ *                   smp_processor_id() * chunk_sz to step it into the
+ *                   running cpu's replica
+ * verifier type:    PTR_TO_ARENA
+ */
+#define BPF_PSEUDO_ARENA_PERCPU	7
+
 /* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative
  * offset to another bpf function
  */
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 0b0c9d72b353..8634778e8ba5 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -378,6 +378,7 @@ enum reloc_type {
 	RELO_SUBPROG_ADDR,
 	RELO_CORE,
 	RELO_INSN_ARRAY,
+	RELO_ARENA_PERCPU,
 };
 
 struct reloc_desc {
@@ -4934,7 +4935,8 @@ static int bpf_prog_compute_hash(struct bpf_program *prog)
 		purged[i] = prog->insns[i];
 		if (purged[i].code == (BPF_LD | BPF_IMM | BPF_DW) &&
 		    (purged[i].src_reg == BPF_PSEUDO_MAP_FD ||
-		     purged[i].src_reg == BPF_PSEUDO_MAP_VALUE)) {
+		     purged[i].src_reg == BPF_PSEUDO_MAP_VALUE ||
+		     purged[i].src_reg == BPF_PSEUDO_ARENA_PERCPU)) {
 			purged[i].imm = 0;
 			i++;
 			if (i >= prog->insns_cnt ||
@@ -5070,6 +5072,20 @@ static int bpf_program__record_reloc(struct bpf_program *prog,
 		return 0;
 	}
 
+	/* arena percpu data relocation */
+	if (shdr_idx == obj->efile.arena_percpu_shndx) {
+		if (obj->arena_map_idx < 0) {
+			pr_warn("prog '%s': bad arena percpu data relocation at insn %u, no arena maps defined\n",
+				prog->name, insn_idx);
+			return -LIBBPF_ERRNO__RELOC;
+		}
+		reloc_desc->type = RELO_ARENA_PERCPU;
+		reloc_desc->insn_idx = insn_idx;
+		reloc_desc->map_idx = obj->arena_map_idx;
+		reloc_desc->sym_off = sym->st_value;
+		return 0;
+	}
+
 	/* jump table data relocation */
 	if (shdr_idx == obj->efile.jumptables_data_shndx) {
 		reloc_desc->type = RELO_INSN_ARRAY;
@@ -7105,6 +7121,23 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
 						   relo->map_idx, map);
 			}
 			break;
+		case RELO_ARENA_PERCPU:
+			/*
+			 * resolve the symbol to its arena-relative offset in cpu 0's chunk. The JIT
+			 * will add smp_processor_id() * chunk_sz to it at run time, so the address
+			 * ends up in the running cpu's replica.
+			 */
+			insn[1].imm = insn[0].imm + relo->sym_off + obj->arena_percpu.offset;
+
+			map = &obj->maps[relo->map_idx];
+			if (map->autocreate) {
+				insn[0].src_reg = BPF_PSEUDO_ARENA_PERCPU;
+				insn[0].imm = map->fd;
+			} else {
+				poison_map_ldimm64(prog, i, relo->insn_idx, insn,
+						   relo->map_idx, map);
+			}
+			break;
 		case RELO_EXTERN_LD64:
 			ext = &obj->externs[relo->ext_idx];
 			if (ext->type == EXT_KCFG) {
-- 
2.47.3