RE: [PATCH v2 3/8] selftests/resctrl: Read memory bandwidth from perf IMC counter and from resctrl file system

From: Moger, Babu
Date: Mon Oct 29 2018 - 18:01:17 EST

Next message: Pierre-Louis Bossart: "Re: Regression found (Stop-marking-clocks-as-CLK_IS_CRITICAL)"
Previous message: Linus Torvalds: "Re: Logitech high-resolution scrolling.."
In reply to: Fenghua Yu: "[PATCH v2 3/8] selftests/resctrl: Read memory bandwidth from perf IMC counter and from resctrl file system"
Next in thread: Moger, Babu: "RE: [PATCH v2 3/8] selftests/resctrl: Read memory bandwidth from perf IMC counter and from resctrl file system"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

Fenghua, Sai

> -----Original Message-----
> From: Fenghua Yu <fenghua.yu@xxxxxxxxx>
> Sent: Thursday, October 25, 2018 6:07 PM
> To: Thomas Gleixner <tglx@xxxxxxxxxxxxx>; Ingo Molnar
> <mingo@xxxxxxxxxx>; H Peter Anvin <hpa@xxxxxxxxx>; Tony Luck
> <tony.luck@xxxxxxxxx>; Peter Zijlstra <peterz@xxxxxxxxxxxxx>; Reinette
> Chatre <reinette.chatre@xxxxxxxxx>; Moger, Babu
> <Babu.Moger@xxxxxxx>; James Morse <james.morse@xxxxxxx>; Ravi V
> Shankar <ravi.v.shankar@xxxxxxxxx>; Sai Praneeth Prakhya
> <sai.praneeth.prakhya@xxxxxxxxx>; Arshiya Hayatkhan Pathan
> <arshiya.hayatkhan.pathan@xxxxxxxxx>
> Cc: linux-kernel <linux-kernel@xxxxxxxxxxxxxxx>; Fenghua Yu
> <fenghua.yu@xxxxxxxxx>
> Subject: [PATCH v2 3/8] selftests/resctrl: Read memory bandwidth from perf
> IMC counter and from resctrl file system
>
> From: Sai Praneeth Prakhya <sai.praneeth.prakhya@xxxxxxxxx>
>
> Total memory bandwidth can be monitored from perf IMC counter and from
> resctrl file system. Later the two will be compared to verify the total
> memory bandwidth read from resctrl is correct.
>
> Signed-off-by: Sai Praneeth Prakhya <sai.praneeth.prakhya@xxxxxxxxx>
> Signed-off-by: Arshiya Hayatkhan Pathan
> <arshiya.hayatkhan.pathan@xxxxxxxxx>
> Signed-off-by: Fenghua Yu <fenghua.yu@xxxxxxxxx>
> ---
> tools/testing/selftests/resctrl/membw.c | 403
> ++++++++++++++++++++++++++++++++
> 1 file changed, 403 insertions(+)
> create mode 100644 tools/testing/selftests/resctrl/membw.c
>
> diff --git a/tools/testing/selftests/resctrl/membw.c
> b/tools/testing/selftests/resctrl/membw.c
> new file mode 100644
> index 000000000000..1bced7e7f148
> --- /dev/null
> +++ b/tools/testing/selftests/resctrl/membw.c
> @@ -0,0 +1,403 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Memory bandwidth monitoring and allocation library
> + *
> + * Copyright (C) 2018 Intel Corporation
> + *
> + * Authors:
> + * Arshiya Hayatkhan Pathan <arshiya.hayatkhan.pathan@xxxxxxxxx>
> + * Sai Praneeth Prakhya <sai.praneeth.prakhya@xxxxxxxxx>,
> + * Fenghua Yu <fenghua.yu@xxxxxxxxx>
> + */
> +#include "resctrl.h"
> +
> +#define UNCORE_IMC "uncore_imc"
> +#define READ_FILE_NAME "events/cas_count_read"
> +#define WRITE_FILE_NAME "events/cas_count_write"
> +#define DYN_PMU_PATH "/sys/bus/event_source/devices"
> +#define SCALE 0.00006103515625
> +#define MAX_IMCS 20
> +#define MAX_TOKENS 5
> +#define READ 0
> +#define WRITE 1
> +#define CON_MON_MBM_LOCAL_BYTES_PATH
> \
> +
> "%s/%s/mon_groups/%s/mon_data/mon_L3_0%c/mbm_local_byte
> s"
> +
> +#define CON_MBM_LOCAL_BYTES_PATH \
> + "%s/%s/mon_data/mon_L3_0%c/mbm_local_bytes"
> +
> +#define MON_MBM_LOCAL_BYTES_PATH \
> + "%s/mon_groups/%s/mon_data/mon_L3_0%c/mbm_local_bytes"
> +
> +#define MBM_LOCAL_BYTES_PATH \
> + "%s/mon_data/mon_L3_0%c/mbm_local_bytes"
> +
> +struct membw_read_format {
> + __u64 value; /* The value of the event */
> + __u64 time_enabled; /* if PERF_FORMAT_TOTAL_TIME_ENABLED
> */
> + __u64 time_running; /* if PERF_FORMAT_TOTAL_TIME_RUNNING
> */
> + __u64 id; /* if PERF_FORMAT_ID */
> +};
> +
> +struct imc_counter_config {
> + __u32 type;
> + __u64 event;
> + __u64 umask;
> + struct perf_event_attr pe;
> + struct membw_read_format return_value;
> + int fd;
> +};
> +
> +static struct imc_counter_config imc_counters_config[MAX_IMCS][2];
> +static char mbm_total_path[1024];
> +static int imcs;
> +
> +void membw_initialize_perf_event_attr(int i, int j)
> +{
> + memset(&imc_counters_config[i][j].pe, 0,
> + sizeof(struct perf_event_attr));
> + imc_counters_config[i][j].pe.type = imc_counters_config[i][j].type;
> + imc_counters_config[i][j].pe.size = sizeof(struct perf_event_attr);
> + imc_counters_config[i][j].pe.disabled = 1;
> + imc_counters_config[i][j].pe.inherit = 1;
> + imc_counters_config[i][j].pe.exclude_guest = 1;
> + imc_counters_config[i][j].pe.config =
> + imc_counters_config[i][j].umask << 8 |
> + imc_counters_config[i][j].event;
> + imc_counters_config[i][j].pe.sample_type =
> PERF_SAMPLE_IDENTIFIER;
> + imc_counters_config[i][j].pe.read_format =
> + PERF_FORMAT_TOTAL_TIME_ENABLED |
> PERF_FORMAT_TOTAL_TIME_RUNNING;
> +}
> +
> +static int open_perf_event(int i, int cpu_no, int j)
> +{
> + imc_counters_config[i][j].fd =
> + perf_event_open(&imc_counters_config[i][j].pe, -1,
> cpu_no, -1,
> + PERF_FLAG_FD_CLOEXEC);
> +
> + if (imc_counters_config[i][j].fd == -1) {
> + fprintf(stderr, "Error opening leader %llx\n",
> + imc_counters_config[i][j].pe.config);
> +
> + return -1;
> + }
> +
> + return 0;
> +}
> +
> +void membw_ioctl_perf_event_ioc_reset_enable(int i, int j)
> +{
> + ioctl(imc_counters_config[i][j].fd, PERF_EVENT_IOC_RESET, 0);
> + ioctl(imc_counters_config[i][j].fd, PERF_EVENT_IOC_ENABLE, 0);
> +}
> +
> +void membw_ioctl_perf_event_ioc_disable(int i, int j)
> +{
> + ioctl(imc_counters_config[i][j].fd, PERF_EVENT_IOC_DISABLE, 0);
> +}
> +
> +/*
> + * get_event_and_umask: Parse config into event and umask
> + * @cas_count_cfg: Config
> + * @count: iMC number
> + * @op: Operation (read/write)
> + */
> +void get_event_and_umask(char *cas_count_cfg, int count, bool op)
> +{
> + char *token[MAX_TOKENS];
> + int i = 0;
> +
> + strcat(cas_count_cfg, ",");
> + token[0] = strtok(cas_count_cfg, "=,");
> +
> + for (i = 1; i < MAX_TOKENS; i++)
> + token[i] = strtok(NULL, "=,");
> +
> + for (i = 0; i < MAX_TOKENS; i++) {
> + if (!token[i])
> + break;
> + if (strcmp(token[i], "event") == 0) {
> + if (op == READ)
> + imc_counters_config[count][READ].event =
> + strtol(token[i + 1], NULL, 16);
> + else
> + imc_counters_config[count][WRITE].event =
> + strtol(token[i + 1], NULL, 16);
> + }
> + if (strcmp(token[i], "umask") == 0) {
> + if (op == READ)
> + imc_counters_config[count][READ].umask =
> + strtol(token[i + 1], NULL, 16);
> + else
> + imc_counters_config[count][WRITE].umask =
> + strtol(token[i + 1], NULL, 16);
> + }
> + }
> +}
> +
> +/* Get type and config (read and write) of an iMC counter */
> +static int read_from_imc_dir(char *imc_dir, int count)
> +{
> + char cas_count_cfg[1024], imc_counter_cfg[1024],
> imc_counter_type[1024];
> + FILE *fp;
> +
> + /* Get type of iMC counter */
> + sprintf(imc_counter_type, "%s%s", imc_dir, "type");
> + fp = fopen(imc_counter_type, "r");
> + if (!fp ||
> + fscanf(fp, "%u", &imc_counters_config[count][READ].type) <= 0 ||
> + fclose(fp) == EOF) {

Same comment as patch 2. If fscanf fails, will it fclose the file? I suggest to separate these checks.

> + perror("Could not get imc type");
> +
> + return errno;
> + }
> +
> + imc_counters_config[count][WRITE].type =
> + imc_counters_config[count][READ].type;
> +
> + /* Get read config */
> + sprintf(imc_counter_cfg, "%s%s", imc_dir, READ_FILE_NAME);
> + fp = fopen(imc_counter_cfg, "r");
> + if (!fp || fscanf(fp, "%s", cas_count_cfg) <= 0 || fclose(fp) == EOF) {

Same comment as above. If fscanf fails, will it fclose the file? I suggest to separate these checks.

> + perror("Could not get imc cas count read");
> +
> + return errno;
> + }
> +
> + get_event_and_umask(cas_count_cfg, count, READ);
> +
> + /* Get write config */
> + sprintf(imc_counter_cfg, "%s%s", imc_dir, WRITE_FILE_NAME);
> + fp = fopen(imc_counter_cfg, "r");
> + if (!fp || fscanf(fp, "%s", cas_count_cfg) <= 0 || fclose(fp) == EOF) {

Same comment as above.

> + perror("Could not get imc cas count write");
> +
> + return errno;
> + }
> +
> + get_event_and_umask(cas_count_cfg, count, WRITE);
> +
> + return 0;
> +}
> +
> +/*
> + * A system can have 'n' number of iMC (Integrated Memory Controller)
> + * counters, get that 'n'. For each iMC counter get it's type and config.
> + * Also, each counter has two configs, one for read and the other for write.
> + * A config again has two parts, event and umask.
> + * Enumerate all these details into an array of structures.
> + *
> + * Return: >= 0 on success. < 0 on failure.
> + */
> +static int num_of_imcs(void)
> +{
> + unsigned int count = 0;
> + char imc_dir[1024];
> + struct dirent *ep;
> + int ret;
> + DIR *dp;
> +
> + dp = opendir(DYN_PMU_PATH);
> + if (dp) {
> + while ((ep = readdir(dp))) {
> + if (strstr(ep->d_name, UNCORE_IMC)) {
> + sprintf(imc_dir, "%s/%s/", DYN_PMU_PATH,
> + ep->d_name);
> + ret = read_from_imc_dir(imc_dir, count);
> + if (ret)
> + return ret;
> +
> + count++;
> + }
> + }
> + closedir(dp);
> + if (count == 0) {
> + perror("Unable find iMC counters!\n");
> +
> + return -1;
> + }
> + } else {
> + perror("Unable to open PMU directory!\n");
> +
> + return -1;
> + }
> +
> + return count;
> +}
> +
> +static int initialize_mem_bw_imc(void)
> +{
> + int imc, j;
> +
> + imcs = num_of_imcs();
> + if (imcs < 0)
> + return imcs;
> +
> + /* Initialize perf_event_attr structures for all iMC's */
> + for (imc = 0; imc < imcs; imc++) {
> + for (j = 0; j < 2; j++)
> + membw_initialize_perf_event_attr(imc, j);
> + }
> +
> + return 0;
> +}
> +
> +/*
> + * get_mem_bw_imc: Memory band width as reported by iMC counters
> + * @cpu_no: CPU number that the benchmark PID is binded to
> + * @bw_report: Bandwidth report type (reads, writes)
> + *
> + * Memory B/W utilized by a process on a socket can be calculated using
> + * iMC counters. Perf events are used to read these counters.
> + *
> + * Return: >= 0 on success. < 0 on failure.
> + */
> +static float get_mem_bw_imc(int cpu_no, char *bw_report)
> +{
> + float reads, writes, of_mul_read, of_mul_write;
> + int imc, j, ret;
> +
> + /* Start all iMC counters to log values (both read and write) */
> + reads = 0, writes = 0, of_mul_read = 1, of_mul_write = 1;
> + for (imc = 0; imc < imcs; imc++) {
> + for (j = 0; j < 2; j++) {
> + ret = open_perf_event(imc, cpu_no, j);
> + if (ret)
> + return -1;
> + }
> + for (j = 0; j < 2; j++)
> + membw_ioctl_perf_event_ioc_reset_enable(imc, j);
> + }
> +
> + sleep(1);
> +
> + /* Stop counters after a second to get results (both read and write)
> */
> + for (imc = 0; imc < imcs; imc++) {
> + for (j = 0; j < 2; j++)
> + membw_ioctl_perf_event_ioc_disable(imc, j);
> + }
> +
> + /*
> + * Get results which are stored in struct type imc_counter_config
> + * Take over flow into consideration before calculating total b/w
> + */
> + for (imc = 0; imc < imcs; imc++) {
> + struct imc_counter_config *r =
> + &imc_counters_config[imc][READ];
> + struct imc_counter_config *w =
> + &imc_counters_config[imc][WRITE];
> +
> + if (read(r->fd, &r->return_value,
> + sizeof(struct membw_read_format)) == -1) {
> + perror("Couldn't get read b/w through iMC");
> +
> + return -1;
> + }
> +
> + if (read(w->fd, &w->return_value,
> + sizeof(struct membw_read_format)) == -1) {
> + perror("Couldn't get write bw through iMC");
> +
> + return -1;
> + }
> +
> + __u64 r_time_enabled = r->return_value.time_enabled;
> + __u64 r_time_running = r->return_value.time_running;
> +
> + if (r_time_enabled != r_time_running)
> + of_mul_read = (float)r_time_enabled /
> + (float)r_time_running;
> +
> + __u64 w_time_enabled = w->return_value.time_enabled;
> + __u64 w_time_running = w->return_value.time_running;
> +
> + if (w_time_enabled != w_time_running)
> + of_mul_write = (float)w_time_enabled /
> + (float)w_time_running;
> + reads += r->return_value.value * of_mul_read * SCALE;
> + writes += w->return_value.value * of_mul_write * SCALE;
> + }
> +
> + for (imc = 0; imc < imcs; imc++) {
> + close(imc_counters_config[imc][READ].fd);
> + close(imc_counters_config[imc][WRITE].fd);
> + }
> +
> + if (strcmp(bw_report, "reads") == 0)
> + return reads;
> +
> + if (strcmp(bw_report, "writes") == 0)
> + return writes;
> +
> + return (reads + writes);
> +}
> +
> +void set_mbm_path(const char *ctrlgrp, const char *mongrp, char
> sock_num)
> +{
> + if (ctrlgrp && mongrp)
> + sprintf(mbm_total_path,
> CON_MON_MBM_LOCAL_BYTES_PATH,
> + RESCTRL_PATH, ctrlgrp, mongrp, sock_num);
> + else if (!ctrlgrp && mongrp)
> + sprintf(mbm_total_path, MON_MBM_LOCAL_BYTES_PATH,
> RESCTRL_PATH,
> + mongrp, sock_num);
> + else if (ctrlgrp && !mongrp)
> + sprintf(mbm_total_path, CON_MBM_LOCAL_BYTES_PATH,
> RESCTRL_PATH,
> + ctrlgrp, sock_num);
> + else if (!ctrlgrp && !mongrp)
> + sprintf(mbm_total_path, MBM_LOCAL_BYTES_PATH,
> RESCTRL_PATH,
> + sock_num);
> +}
> +
> +/*
> + * initialize_mem_bw_resctrl: Appropriately populate "mbm_total_path"
> + * @ctrlgrp: Name of the control monitor group
> (con_mon grp)
> + * @mongrp: Name of the monitor group (mon grp)
> + * @cpu_no: CPU number that the benchmark PID is
> binded to
> + * @resctrl_val: Resctrl feature (Eg: mbm, mba.. etc)
> + */
> +static void initialize_mem_bw_resctrl(const char *ctrlgrp, const char
> *mongrp,
> + int cpu_no, char *resctrl_val)
> +{
> + char sock_num;
> +
> + sock_num = get_sock_num(cpu_no);
> + if (sock_num < 0)
> + return;
> +
> + if (strcmp(resctrl_val, "mbm") == 0)
> + set_mbm_path(ctrlgrp, mongrp, sock_num);
> +
> + if ((strcmp(resctrl_val, "mba") == 0)) {
> + if (ctrlgrp)
> + sprintf(mbm_total_path,
> CON_MBM_LOCAL_BYTES_PATH,
> + RESCTRL_PATH, ctrlgrp, sock_num);
> + else
> + sprintf(mbm_total_path, MBM_LOCAL_BYTES_PATH,
> + RESCTRL_PATH, sock_num);
> + }
> +}
> +
> +/*
> + * Get MBM Local bytes as reported by resctrl FS
> + * For MBM,
> + * 1. If con_mon grp and mon grp are given, then read from con_mon grp's
> mon grp
> + * 2. If only con_mon grp is given, then read from con_mon grp
> + * 3. If both are not given, then read from root con_mon grp
> + * For MBA,
> + * 1. If con_mon grp is given, then read from it
> + * 2. If con_mon grp is not given, then read from root con_mon grp
> + */
> +static unsigned long get_mem_bw_resctrl(void)
> +{
> + unsigned long mbm_total = 0;
> + FILE *fp;
> +
> + fp = fopen(mbm_total_path, "r");
> + if (!fp || fscanf(fp, "%lu", &mbm_total) <= 0 || fclose(fp) == EOF) {

Same comment as above. If fscanf fails, will it fclose the file? I suggest to separate these checks.

> + perror("Could not get mbm local bytes");
> +
> + return -1;
> + }
> +
> + return mbm_total;
> +}
> --
> 2.5.0

Next message: Pierre-Louis Bossart: "Re: Regression found (Stop-marking-clocks-as-CLK_IS_CRITICAL)"
Previous message: Linus Torvalds: "Re: Logitech high-resolution scrolling.."
In reply to: Fenghua Yu: "[PATCH v2 3/8] selftests/resctrl: Read memory bandwidth from perf IMC counter and from resctrl file system"
Next in thread: Moger, Babu: "RE: [PATCH v2 3/8] selftests/resctrl: Read memory bandwidth from perf IMC counter and from resctrl file system"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]