Re: [PATCH v11 24/24] perf session: Load data directory files for analysis

[Jiri Olsa]

On Tue, Aug 17, 2021 at 11:23:27AM +0300, Alexey Bayduraev wrote:
> Load data directory files and provide basic raw dump and aggregated
> analysis support of data directories in report mode, still with no
> memory consumption optimizations.
> 
> Design and implementation are based on the prototype [1], [2].
> 
> [1] git clone https://git.kernel.org/pub/scm/linux/kernel/git/jolsa/perf.git -b perf/record_threads
> [2] https://lore.kernel.org/lkml/20180913125450.21342-1-jolsa@xxxxxxxxxx/
> 
> Suggested-by: Jiri Olsa <jolsa@xxxxxxxxxx>
> Acked-by: Namhyung Kim <namhyung@xxxxxxxxx>
> Reviewed-by: Riccardo Mancini <rickyman7@xxxxxxxxx>
> Tested-by: Riccardo Mancini <rickyman7@xxxxxxxxx>
> Signed-off-by: Alexey Bayduraev <alexey.v.bayduraev@xxxxxxxxxxxxxxx>
> ---
>  tools/perf/util/session.c | 131 ++++++++++++++++++++++++++++++++++++++
>  1 file changed, 131 insertions(+)
> 
> diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c
> index b49b52768681..f5a82f66e36b 100644
> --- a/tools/perf/util/session.c
> +++ b/tools/perf/util/session.c
> @@ -48,6 +48,13 @@
>  #define NUM_MMAPS 128
>  #endif
>  
> +/*
> + * Processing 10MBs of data from each reader in sequence,
> + * because that's the way the ordered events sorting works
> + * most efficiently.
> + */
> +#define READER_MAX_SIZE (10 * 1024 * 1024)

why 10? ;-) I think we discussed this in your earlier iteration,
I put the 10M there to speedup the processing, because it worked
on my current data file for the test at that time

I think this needs some better justification and some setup based
on the env setup and data size perhaps?

jirka

> +
>  struct reader;
>  
>  typedef s64 (*reader_cb_t)(struct perf_session *session,
> @@ -65,6 +72,7 @@ struct reader_state {
>  	u64	 data_size;
>  	u64	 head;
>  	bool	 eof;
> +	u64	 size;
>  };
>  
>  enum {
> @@ -2328,6 +2336,7 @@ reader__read_event(struct reader *rd, struct perf_session *session,
>  	if (skip)
>  		size += skip;
>  
> +	st->size += size;
>  	st->head += size;
>  	st->file_pos += size;
>  
> @@ -2427,6 +2436,125 @@ static int __perf_session__process_events(struct perf_session *session)
>  	return err;
>  }
>  
> +/*
> + * This function reads, merge and process directory data.
> + * It assumens the version 1 of directory data, where each
> + * data file holds per-cpu data, already sorted by kernel.
> + */
> +static int __perf_session__process_dir_events(struct perf_session *session)
> +{
> +	struct perf_data *data = session->data;
> +	struct perf_tool *tool = session->tool;
> +	int i, ret = 0, readers = 1;
> +	struct ui_progress prog;
> +	u64 total_size = perf_data__size(session->data);
> +	struct reader *rd;
> +
> +	perf_tool__fill_defaults(tool);
> +
> +	ui_progress__init_size(&prog, total_size, "Sorting events...");
> +
> +	for (i = 0; i < data->dir.nr; i++) {
> +		if (data->dir.files[i].size)
> +			readers++;
> +	}
> +
> +	rd = session->readers = zalloc(readers * sizeof(struct reader));
> +	if (!rd)
> +		return -ENOMEM;
> +	session->nr_readers = readers;
> +	readers = 0;
> +
> +	rd[readers] = (struct reader) {
> +		.fd		 = perf_data__fd(session->data),
> +		.path		 = session->data->file.path,
> +		.data_size	 = session->header.data_size,
> +		.data_offset	 = session->header.data_offset,
> +		.in_place_update = session->data->in_place_update,
> +	};
> +	ret = reader__init(&rd[readers], NULL);
> +	if (ret)
> +		goto out_err;
> +	ret = reader__mmap(&rd[readers], session);
> +	if (ret != READER_OK) {
> +		if (ret == READER_EOF)
> +			ret = -EINVAL;
> +		goto out_err;
> +	}
> +	readers++;
> +
> +	for (i = 0; i < data->dir.nr; i++) {
> +		if (!data->dir.files[i].size)
> +			continue;
> +		rd[readers] = (struct reader) {
> +			.fd		 = data->dir.files[i].fd,
> +			.path		 = data->dir.files[i].path,
> +			.data_size	 = data->dir.files[i].size,
> +			.data_offset	 = 0,
> +			.in_place_update = session->data->in_place_update,
> +		};
> +		ret = reader__init(&rd[readers], NULL);
> +		if (ret)
> +			goto out_err;
> +		ret = reader__mmap(&rd[readers], session);
> +		if (ret != READER_OK) {
> +			if (ret == READER_EOF)
> +				ret = -EINVAL;
> +			goto out_err;
> +		}
> +		readers++;
> +	}
> +
> +	i = 0;
> +
> +	while ((ret >= 0) && readers) {
> +		if (session_done())
> +			return 0;
> +
> +		if (rd[i].state.eof) {
> +			i = (i + 1) % session->nr_readers;
> +			continue;
> +		}
> +
> +		ret = reader__read_event(&rd[i], session, &prog);
> +		if (ret < 0)
> +			break;
> +		if (ret == READER_NODATA) {
> +			ret = reader__mmap(&rd[i], session);
> +			if (ret < 0)
> +				goto out_err;
> +			if (ret == READER_EOF)
> +				readers--;
> +		}
> +
> +		if (rd[i].state.size >= READER_MAX_SIZE) {
> +			rd[i].state.size = 0;
> +			i = (i + 1) % session->nr_readers;
> +		}
> +	}
> +
> +	ret = ordered_events__flush(&session->ordered_events, OE_FLUSH__FINAL);
> +	if (ret)
> +		goto out_err;
> +
> +	ret = perf_session__flush_thread_stacks(session);
> +out_err:
> +	ui_progress__finish();
> +
> +	if (!tool->no_warn)
> +		perf_session__warn_about_errors(session);
> +
> +	/*
> +	 * We may switching perf.data output, make ordered_events
> +	 * reusable.
> +	 */
> +	ordered_events__reinit(&session->ordered_events);
> +
> +	session->one_mmap = false;
> +
> +	return ret;
> +}
> +
>  int perf_session__process_events(struct perf_session *session)
>  {
>  	if (perf_session__register_idle_thread(session) < 0)
> @@ -2435,6 +2563,9 @@ int perf_session__process_events(struct perf_session *session)
>  	if (perf_data__is_pipe(session->data))
>  		return __perf_session__process_pipe_events(session);
>  
> +	if (perf_data__is_dir(session->data))
> +		return __perf_session__process_dir_events(session);
> +
>  	return __perf_session__process_events(session);
>  }
>  
> -- 
> 2.19.0
>