[PATCH 11/17] tools/arch/x86/pmtctl: Add libpmtctl built-in metric definition support

[David E. Box]

Add support for compiling PMT metric definitions directly into
libpmtctl_core so tools can operate without requiring external JSON
definition files at runtime.

Built-in definitions avoid a runtime dependency on host filesystem data by
embedding generated metric definition tables directly into the library.
When no path argument is provided, pmt_metrics_load() serves definitions
from the built-in table.

Add a Python generator script to convert defs/*.json into a generated C
definition table used by the library build. An empty stub table remains
available for builds where generation is skipped, preserving the existing
runtime contract where pmt_metrics_load() returns PMTCTL_ERR_NOMETRICS when
no definitions are present.

Add a top-level 'make defs' target and update the README and CLI
documentation to describe the built-in definition workflow.

Assisted-by: GitHub-Copilot:claude-sonnet-4.6
Signed-off-by: David E. Box <david.e.box@xxxxxxxxxxxxxxx>
---
 tools/arch/x86/pmtctl/lib/Makefile            |  34 +-
 .../x86/pmtctl/scripts/gen_builtin_defs.py    | 405 ++++++++++++++++++
 2 files changed, 437 insertions(+), 2 deletions(-)
 create mode 100755 tools/arch/x86/pmtctl/scripts/gen_builtin_defs.py

diff --git a/tools/arch/x86/pmtctl/lib/Makefile b/tools/arch/x86/pmtctl/lib/Makefile
index e2aeff1935cf..2dbd4e5b95a0 100644
--- a/tools/arch/x86/pmtctl/lib/Makefile
+++ b/tools/arch/x86/pmtctl/lib/Makefile
@@ -1,5 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
+# Remove targets whose recipe exited non-zero so a failed codegen step
+# does not leave a truncated $@ behind that fools the next build.
+.DELETE_ON_ERROR:
+
 CC      := gcc
 AR      := ar
 RANLIB  := ranlib
@@ -33,8 +37,7 @@ CORE_SRC := \
 	metrics_db.c \
 	pmt_guid.c \
 	pmtctl.c \
-	metrics_provider.c \
-	builtin_defs_empty.c
+	metrics_provider.c
 
 HAVE_JANSSON ?= 0
 ifeq ($(HAVE_JANSSON),1)
@@ -45,6 +48,17 @@ endif
 
 CORE_OBJ := $(patsubst %.c,$(BUILDDIR)/%.o,$(CORE_SRC))
 
+# Built-in metric definitions: use the generated table when available,
+# otherwise fall back to the empty stub. Both sources export builtin_defs[]
+# and builtin_defs_count and are compiled to a fixed object name so the
+# choice is transparent to the rest of the library.
+# Use 'find' (recursive) to match the top-level Makefile; $(wildcard) would
+# silently miss any JSON files placed in defs/ subdirectories.
+DEFS_JSON ?= $(shell find ../defs -name '*.json' 2>/dev/null)
+BUILTIN_DEFS_SRC := $(if $(wildcard ../generated/builtin_defs.c),../generated/builtin_defs.c,builtin_defs_empty.c)
+BUILTIN_DEFS_OBJ := $(BUILDDIR)/builtin_defs.o
+CORE_OBJ += $(BUILTIN_DEFS_OBJ)
+
 PREFIX ?= /usr/local
 DESTDIR ?=
 INCLUDEDIR ?= $(PREFIX)/include/pmtctl
@@ -77,6 +91,22 @@ $(BUILDDIR)/%.o: %.c
 	@mkdir -p $(BUILDDIR)
 	$(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@
 
+$(BUILTIN_DEFS_OBJ): $(BUILTIN_DEFS_SRC)
+	@mkdir -p $(BUILDDIR)
+	$(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@
+
+../generated/builtin_defs.c: ../scripts/gen_builtin_defs.py $(DEFS_JSON)
+	@mkdir -p ../generated
+	@if [ -z "$(DEFS_JSON)" ]; then \
+		echo "No JSON files in defs/. Nothing to generate."; \
+	else \
+		if ! command -v python3 >/dev/null 2>&1; then \
+			echo "python3 is required to generate builtin defs but was not found." >&2; \
+			exit 1; \
+		fi; \
+		python3 ../scripts/gen_builtin_defs.py $(DEFS_JSON) > $@; \
+	fi
+
 install: all install-lib install-headers install-pkgconfig
 
 install-lib: all
diff --git a/tools/arch/x86/pmtctl/scripts/gen_builtin_defs.py b/tools/arch/x86/pmtctl/scripts/gen_builtin_defs.py
new file mode 100755
index 000000000000..5e34c8f9a1b3
--- /dev/null
+++ b/tools/arch/x86/pmtctl/scripts/gen_builtin_defs.py
@@ -0,0 +1,405 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Auto-generate builtin_defs.c from PMT perf-style JSON files.
+#
+"""Auto-generate builtin_defs.c from PMT perf-style JSON event files."""
+
+import sys
+import re
+import json
+from pathlib import Path
+
+# We only sanity-check that these fields exist in struct pmt_metric_def.
+EXPECTED_FIELDS = [
+    "event_name",
+    "description",
+    "group",
+    "platform_group",
+    "guid",
+    "sample_id",
+    "lsb",
+    "msb",
+]
+
+PMT_GUIDS_JSON_NAME = "pmt_guids.json"
+
+
+def find_project_root(start: Path) -> Path:
+    """
+    Walk upward until we find 'include/lib/metrics_db.h'.
+    This allows running the generator from anywhere:
+    - project root
+    - tools/
+    - build system
+    - editor integration
+    """
+    cur = start.resolve()
+    root = cur.anchor
+
+    while True:
+        candidate = cur / "include" / "lib" / "metrics_db.h"
+        if candidate.exists():
+            return cur
+        if str(cur) == root:
+            break
+        cur = cur.parent
+
+    sys.stderr.write(
+        "ERROR: Could not locate 'include/lib/metrics_db.h'. "
+        "Run from inside the project tree.\n"
+    )
+    sys.exit(1)
+
+
+PROJECT_ROOT = find_project_root(Path(__file__).parent)
+METRICS_DB_H = PROJECT_ROOT / "include" / "lib" / "metrics_db.h"
+
+
+def parse_struct_fields(header_path: Path):
+    """Parse struct pmt_metric_def from metrics_db.h and return [(type, name), ...]."""
+
+    sys.stderr.write(f"Using metrics_db.h at: {header_path}\n")
+
+    text = header_path.read_text()
+
+    m = re.search(r"struct\s+pmt_metric_def\s*\{([^}]*)\}", text, re.S)
+    if not m:
+        sys.stderr.write(
+            "ERROR: could not find struct pmt_metric_def in metrics_db.h\n"
+        )
+        sys.exit(1)
+
+    block = m.group(1)
+    fields: list[tuple[str, str]] = []
+
+    for raw in block.splitlines():
+        line = raw.strip()
+        if not line:
+            continue
+        if line.startswith("/*") or line.startswith("//"):
+            continue
+        if ";" not in line:
+            continue
+
+        # Take text before ';'
+        before = line.split(";", 1)[0]
+
+        # Strip trailing // and /* ... */ comments crudely
+        before = before.split("//", 1)[0]
+        before = before.split("/*", 1)[0]
+        before = before.rstrip()
+        if not before:
+            continue
+
+        # Find the trailing identifier (the field name)
+        m_name = re.search(r"([A-Za-z_][A-Za-z0-9_]*)$", before)
+        if not m_name:
+            continue
+
+        name = m_name.group(1)
+        type_part = before[: m_name.start()].rstrip()
+        if not type_part:
+            continue
+
+        fields.append((type_part, name))
+
+    sys.stderr.write("Parsed fields in struct pmt_metric_def:\n")
+    for t, n in fields:
+        sys.stderr.write(f"  {t} {n}\n")
+
+    return fields
+
+
+def check_expected_fields(fields):
+    """Verify that all EXPECTED_FIELDS are present in the parsed struct fields."""
+    names = [name for (_ty, name) in fields]
+    missing = [f for f in EXPECTED_FIELDS if f not in names]
+    if missing:
+        sys.stderr.write(
+            "ERROR: metrics_db.h struct pmt_metric_def is missing fields: "
+            + ", ".join(missing)
+            + "\n"
+        )
+        sys.exit(1)
+
+
+def parse_guid_from_pmu(pmu_str: str) -> int:
+    """
+    Example PMU strings:
+        "pmt_ep_22806802"
+        "pmt_ep_22806802_s0_r0" (future)
+    We take the last underscore-separated token and interpret it as hex.
+    """
+
+    if not pmu_str:
+        return 0
+
+    parts = pmu_str.split("_")
+    tail = parts[-1]
+    try:
+        return int(tail, 16)
+    except ValueError:
+        # Fallback: unknown format, just 0 it for now
+        return 0
+
+
+def decode_config_code(cfg: int):
+    """
+    ConfigCode mapping (from your spec):
+
+        bits[15:0]   = sample_id
+        bits[23:16]  = lsb
+        bits[31:24]  = msb
+        bits[63:32]  = reserved/0
+    """
+
+    sample_id = cfg & 0xFFFF
+    lsb = (cfg >> 16) & 0xFF
+    msb = (cfg >> 24) & 0xFF
+    return sample_id, lsb, msb
+
+
+def load_events_from_json(path: Path):
+    """Load and return the list of event dicts from a JSON file."""
+    with path.open("r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    if not isinstance(data, list):
+        raise ValueError(f"{path}: expected top-level JSON array")
+
+    return data
+
+
+def emit_header():
+    """Print the C file header and includes."""
+    print("/* Auto-generated by tools/gen_builtin_defs.py; do not edit. */")
+    print('#include "lib/metrics_db.h"')
+    print('#include "lib/pmt_guid.h"')
+    print()
+
+
+def emit_guid_table(guids: list[dict]):
+    """Emit the builtin_guids[] array and its count."""
+    print("const struct pmt_guid builtin_guids[] = {")
+    for g in guids:
+        print("    {")
+        print(f"        .guid        = 0x{g['guid']:08x},")
+        print(f"        .name        = {c_str(g.get('name') or None)},")
+        print(f"        .description = {c_str(g.get('description') or None)},")
+        print("    },")
+    print("};")
+    print()
+    print(f"const int builtin_guids_count = {len(guids)};")
+    print()
+
+
+def emit_defs_open():
+    """Print the opening brace for builtin_defs[]."""
+    print("const struct pmt_metric_def builtin_defs[] = {")
+    print()
+
+
+def emit_footer(total: int):
+    """Print the closing brace and builtin_defs_count definition."""
+    print("};")
+    print()
+    print(f"const int builtin_defs_count = {total};")
+
+
+def c_str(s):
+    """Return s as a quoted C string literal, or NULL for None."""
+    if s is None:
+        return "NULL"
+    s = str(s)
+    s = s.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{s}"'
+
+
+def emit_entry(ev: dict, guid_index: dict[int, int]):
+    """Print one C struct initializer for the given event dict."""
+    pmu = ev.get("PMU", "")
+    name = ev.get("EventName", "")
+    brief = ev.get("BriefDescription", "")
+    group = ev.get("MetricGroup", "")
+    platform_group = ev.get("PlatformGroup", "")
+
+    cfg_s = ev.get("ConfigCode", "0")
+
+    try:
+        cfg = int(cfg_s, 0)
+    except (ValueError, TypeError):
+        cfg = 0
+
+    sample_id, lsb, msb = decode_config_code(cfg)
+    guid = parse_guid_from_pmu(pmu)
+    idx = guid_index[guid]
+
+    print("    {")
+    print(f"        .event_name = {c_str(name)},")
+    print(f"        .description = {c_str(brief)},")
+    print(f"        .group       = {c_str(group)},")
+    print(f"        .platform_group = {c_str(platform_group)},")
+    print(f"        .guid        = &builtin_guids[{idx}], /* 0x{guid:08x} */")
+    print(f"        .sample_id   = {sample_id},")
+    print(f"        .lsb         = {lsb},")
+    print(f"        .msb         = {msb},")
+    print("    },")
+    print()
+
+
+def expand_paths(paths: list[str]) -> tuple[list[Path], Path | None]:
+    """
+    Expand arguments into JSON file paths.
+    If an argument is a directory, find all .json files in it.
+    If an argument is a file, include it directly.
+
+    Returns (event_json_paths, pmt_guids_json_path_or_None). The
+    pmt_guids.json sidecar (if encountered) is routed separately and
+    is never treated as a metric-event file.
+    """
+    json_paths: list[Path] = []
+    pmt_guids_path: Path | None = None
+
+    def _consider(p: Path) -> None:
+        nonlocal pmt_guids_path
+        if p.name == PMT_GUIDS_JSON_NAME:
+            if pmt_guids_path is not None and pmt_guids_path != p:
+                sys.stderr.write(
+                    f"WARNING: multiple {PMT_GUIDS_JSON_NAME} found; "
+                    f"using {pmt_guids_path}, ignoring {p}\n"
+                )
+                return
+            pmt_guids_path = p
+            return
+        json_paths.append(p)
+
+    for p_str in paths:
+        p = Path(p_str)
+        if p.is_dir():
+            json_files = sorted(p.glob("*.json"))
+            if not json_files:
+                sys.stderr.write(f"WARNING: no .json files found in directory {p}\n")
+            for jp in json_files:
+                _consider(jp)
+        elif p.is_file():
+            _consider(p)
+        else:
+            sys.stderr.write(f"WARNING: path does not exist: {p}\n")
+
+    return json_paths, pmt_guids_path
+
+
+def load_pmt_guids(path: Path) -> list[dict]:
+    """Load and validate the pmt_guids.json sidecar."""
+    with path.open("r", encoding="utf-8") as f:
+        data = json.load(f)
+    if not isinstance(data, list):
+        raise ValueError(f"{path}: expected top-level JSON array")
+
+    out: list[dict] = []
+    for entry in data:
+        guid_s = entry.get("guid")
+        if guid_s is None:
+            raise ValueError(f"{path}: entry missing 'guid'")
+        try:
+            guid = int(guid_s, 0) if isinstance(guid_s, str) else int(guid_s)
+        except (TypeError, ValueError) as ex:
+            raise ValueError(f"{path}: invalid guid {guid_s!r}: {ex}") from ex
+        out.append(
+            {
+                "guid": guid,
+                "name": entry.get("name") or "",
+                "description": entry.get("description") or "",
+            }
+        )
+    return out
+
+
+def _load_guid_meta(pmt_guids_path: Path | None) -> dict[int, dict]:
+    """Load GUID metadata (name + description per GUID) if available."""
+    guid_meta: dict[int, dict] = {}
+    if pmt_guids_path is not None:
+        sys.stderr.write(f"Loading GUID metadata from {pmt_guids_path}\n")
+        for g in load_pmt_guids(pmt_guids_path):
+            guid_meta[g["guid"]] = g
+    else:
+        sys.stderr.write(
+            f"WARNING: {PMT_GUIDS_JSON_NAME} not provided; "
+            "builtin_guids[] will have empty name/description.\n"
+        )
+    return guid_meta
+
+
+def _collect_events(
+    json_paths: list[Path],
+) -> tuple[list[tuple[Path, list[dict]]], list[int], dict[int, int]]:
+    """Load events from JSON paths and assign a stable GUID index."""
+    loaded: list[tuple[Path, list[dict]]] = []
+    seen_guids: list[int] = []
+    guid_index: dict[int, int] = {}
+    for jp in json_paths:
+        sys.stderr.write(f"Processing {jp}\n")
+        events = load_events_from_json(jp)
+        loaded.append((jp, events))
+        for ev in events:
+            guid = parse_guid_from_pmu(ev.get("PMU", ""))
+            if guid not in guid_index:
+                guid_index[guid] = len(seen_guids)
+                seen_guids.append(guid)
+    return loaded, seen_guids, guid_index
+
+
+def _build_guid_table(seen_guids: list[int], guid_meta: dict[int, dict]) -> list[dict]:
+    """Build the ordered guid table; merge in metadata."""
+    guid_table: list[dict] = []
+    for guid in seen_guids:
+        meta = guid_meta.get(guid)
+        if meta is None:
+            sys.stderr.write(
+                f"WARNING: no metadata for GUID 0x{guid:08x}; "
+                "emitting empty name/description.\n"
+            )
+            meta = {"guid": guid, "name": "", "description": ""}
+        guid_table.append(meta)
+    return guid_table
+
+
+def main(argv: list[str]) -> int:
+    """Entry point: parse arguments, emit C source to stdout."""
+    if len(argv) < 2:
+        sys.stderr.write(
+            "Usage: gen_builtin_defs.py <json|folder> [<json|folder> ...]\n"
+        )
+        return 1
+
+    # Sanity-check metrics_db.h
+    check_expected_fields(parse_struct_fields(METRICS_DB_H))
+
+    json_paths, pmt_guids_path = expand_paths(argv[1:])
+
+    if not json_paths:
+        sys.stderr.write("ERROR: no JSON files found to process\n")
+        return 1
+
+    guid_meta = _load_guid_meta(pmt_guids_path)
+    loaded, seen_guids, guid_index = _collect_events(json_paths)
+    guid_table = _build_guid_table(seen_guids, guid_meta)
+
+    # Emit C source.
+    emit_header()
+    emit_guid_table(guid_table)
+    emit_defs_open()
+
+    total = 0
+    for _jp, events in loaded:
+        for ev in events:
+            emit_entry(ev, guid_index)
+            total += 1
+
+    emit_footer(total)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv))
-- 
2.43.0