Re: [PATCH] fix linux kernel BTF builds: increase max percpu variables by 10x

[Alan Maguire]

On 28/02/2024 09:20, Jiri Olsa wrote:
> On Tue, Feb 27, 2024 at 07:21:42PM -0800, John Hubbard wrote:
>> When building the Linux kernel with a distro .config, most or even all
>> possible kernel modules are built. This adds up to 4500+ modules, and
>> based on my testing, this causes the pahole utility to run out of space,
>> which shows up like this (CONFIG_DEBUG_INFO_BTF=y is required in order
>> to reproduce this):
>>
>>   LD      .tmp_vmlinux.btf
>>   BTF     .btf.vmlinux.bin.o
>> Reached the limit of per-CPU variables: 4096
>> ...repeated many times...
>> Reached the limit of per-CPU variables: 4096
>>   LD      .tmp_vmlinux.kallsyms1
>>   NM      .tmp_vmlinux.kallsyms1.syms
>>   KSYMS   .tmp_vmlinux.kallsyms1.S
>>   AS      .tmp_vmlinux.kallsyms1.S
>>   LD      .tmp_vmlinux.kallsyms2
>>   NM      .tmp_vmlinux.kallsyms2.syms
>>   KSYMS   .tmp_vmlinux.kallsyms2.S
>>   AS      .tmp_vmlinux.kallsyms2.S
>>   LD      vmlinux
>>   BTFIDS  vmlinux
>> libbpf: failed to find '.BTF' ELF section in vmlinux
>> FAILED: load BTF from vmlinux: No data available
>> make[2]: *** [scripts/Makefile.vmlinux:37: vmlinux] Error 255
>> make[2]: *** Deleting file 'vmlinux'
>> make[1]: *** [/kernel_work/linux-people/Makefile:1162: vmlinux] Error 2
>> make: *** [Makefile:240: __sub-make] Error 2
>>
>> Increasing MAX_PERCPU_VAR_CNT by 10x avoids running out of space, and
>> allows the build to succeed.
> 
> do you have an actual count of percpu variables for your config?
> 10x seems a lot to me
> 
> this might be a workaround, but we should make encoder->percpu.vars
> dynamically allocated like we do for functions
> 
> jirka
>

Good idea Jiri; John would you mind trying the attached patch? Thanks!

AlanFrom a254d14dee0313f01de1f1ea50784ed57c26511c Mon Sep 17 00:00:00 2001
From: Alan Maguire <alan.maguire@xxxxxxxxxx>
Date: Wed, 28 Feb 2024 11:56:38 +0000
Subject: [PATCH dwarves] btf_encoder: dynamically allocate the vars array for
 percpu variables

Use consistent method across allocating function and per-cpu variable
representations, based around (re)allocating the arrays based on demand.
This avoids issues where the number of per-CPU variables exceeds the
hardcoded limit.

Reported-by: John Hubbard <jhubbard@xxxxxxxxxx>
Suggested-by: Jiri Olsa <olsajiri@xxxxxxxxx>
Signed-off-by: Alan Maguire <alan.maguire@xxxxxxxxxx>
---
 btf_encoder.c | 38 +++++++++++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/btf_encoder.c b/btf_encoder.c
index fd04008..a43d702 100644
--- a/btf_encoder.c
+++ b/btf_encoder.c
@@ -50,8 +50,6 @@ struct elf_function {
 	struct btf_encoder_state state;
 };
 
-#define MAX_PERCPU_VAR_CNT 4096
-
 struct var_info {
 	uint64_t    addr;
 	const char *name;
@@ -80,8 +78,9 @@ struct btf_encoder {
 			  is_rel;
 	uint32_t	  array_index_id;
 	struct {
-		struct var_info vars[MAX_PERCPU_VAR_CNT];
+		struct var_info *vars;
 		int		var_cnt;
+		int		allocated;
 		uint32_t	shndx;
 		uint64_t	base_addr;
 		uint64_t	sec_sz;
@@ -983,6 +982,16 @@ static int functions_cmp(const void *_a, const void *_b)
 #define max(x, y) ((x) < (y) ? (y) : (x))
 #endif
 
+static void *reallocarray_grow(void *ptr, int *nmemb, size_t size)
+{
+	int new_nmemb = max(1000, *nmemb * 3 / 2);
+	void *new = realloc(ptr, new_nmemb * size);
+
+	if (new)
+		*nmemb = new_nmemb;
+	return new;
+}
+
 static int btf_encoder__collect_function(struct btf_encoder *encoder, GElf_Sym *sym)
 {
 	struct elf_function *new;
@@ -995,8 +1004,9 @@ static int btf_encoder__collect_function(struct btf_encoder *encoder, GElf_Sym *
 		return 0;
 
 	if (encoder->functions.cnt == encoder->functions.allocated) {
-		encoder->functions.allocated = max(1000, encoder->functions.allocated * 3 / 2);
-		new = realloc(encoder->functions.entries, encoder->functions.allocated * sizeof(*encoder->functions.entries));
+		new = reallocarray_grow(encoder->functions.entries,
+					&encoder->functions.allocated,
+					sizeof(*encoder->functions.entries));
 		if (!new) {
 			/*
 			 * The cleanup - delete_functions is called
@@ -1439,10 +1449,17 @@ static int btf_encoder__collect_percpu_var(struct btf_encoder *encoder, GElf_Sym
 	if (!encoder->is_rel)
 		addr -= encoder->percpu.base_addr;
 
-	if (encoder->percpu.var_cnt == MAX_PERCPU_VAR_CNT) {
-		fprintf(stderr, "Reached the limit of per-CPU variables: %d\n",
-			MAX_PERCPU_VAR_CNT);
-		return -1;
+	if (encoder->percpu.var_cnt == encoder->percpu.allocated) {
+		struct var_info *new;
+
+		new = reallocarray_grow(encoder->percpu.vars,
+					&encoder->percpu.allocated,
+					sizeof(*encoder->percpu.vars));
+		if (!new) {
+			fprintf(stderr, "Failed to allocate memory for variables\n");
+			return -1;
+		}	
+		encoder->percpu.vars = new;
 	}
 	encoder->percpu.vars[encoder->percpu.var_cnt].addr = addr;
 	encoder->percpu.vars[encoder->percpu.var_cnt].sz = size;
@@ -1720,6 +1737,9 @@ void btf_encoder__delete(struct btf_encoder *encoder)
 	encoder->functions.allocated = encoder->functions.cnt = 0;
 	free(encoder->functions.entries);
 	encoder->functions.entries = NULL;
+	encoder->percpu.allocated = encoder->percpu.var_cnt = 0;
+	free(encoder->percpu.vars);
+	encoder->percpu.vars = NULL;
 
 	free(encoder);
 }
-- 
2.39.3