From eebbd5c4115ac5326812e2d5258e63fa2b96fe99 Mon Sep 17 00:00:00 2001 From: Dmitrii Golovanov Date: Tue, 30 Apr 2024 10:33:32 +0200 Subject: [PATCH] twister: footprint: Add optional detailed JSON report on symbols New Twister option `--footprint-report` is introduced to collect and write detailed memory footprint results for symbols as an additional JSON file. By default, the new option is disabled. The new option implies and extends `--create-rom-ram-report`, so there are three choices: 'ROM', 'RAM', and 'all' to select what memory area symbols to report in `twister_footprint.json`. In case of the custom report name, or per-platform report, it is always composed with the rightmost '_footprint.json' suffix. The memory footprint report has similar structure as `twister.json` and compelements it having reduced set of test suite properties: - instead of `testcases` it contains `footprint` object with `rom.json` and `ram.json` artifacts embedded there; - other properites are limited to represent only the essential test suite context, thus to allow further data processing consistently and independently from the `twister.json`. - 'filtered' test instances are not included into the footprint report. Signed-off-by: Dmitrii Golovanov --- .../pylib/twister/twisterlib/environment.py | 15 ++++++ scripts/pylib/twister/twisterlib/reports.py | 52 +++++++++++++++++-- scripts/pylib/twister/twisterlib/runner.py | 2 + 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index 3d818e4e964..8a7b5b2f84f 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -385,6 +385,18 @@ structure in the main Zephyr tree: boards///""") help="Generate detailed json reports with ROM/RAM symbol sizes for each test image built " "using additional build option `--target footprint`.") + footprint_group.add_argument( + "--footprint-report", + nargs="?", + default=None, + choices=['all', 'ROM', 'RAM'], + const="all", + help="Select which memory area symbols' data to collect as 'footprint' property " + "of each test suite built, and report in 'twister_footprint.json' together " + "with the relevant execution metadata the same way as in `twister.json`. " + "Implies '--create-rom-ram-report' to generate the footprint data files. " + "No value means '%(const)s'. Default: %(default)s""") + footprint_group.add_argument( "--enable-size-report", action="store_true", @@ -790,6 +802,9 @@ def parse_arguments(parser, args, options = None, on_init=True): if options.last_metrics or options.compare_report: options.enable_size_report = True + if options.footprint_report: + options.create_rom_ram_report = True + if options.aggressive_no_clean: options.no_clean = True diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index 5521ec868c7..c6885420803 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -18,6 +18,16 @@ logger.setLevel(logging.DEBUG) class Reporting: + json_filters = { + 'twister.json': { + 'deny_suite': ['footprint'] + }, + 'footprint.json': { + 'deny_status': ['filtered'], + 'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable'] + } + } + def __init__(self, plan, env) -> None: self.plan = plan #FIXME self.instances = plan.instances @@ -28,6 +38,7 @@ class Reporting: self.timestamp = datetime.now().isoformat() self.outdir = os.path.abspath(env.options.outdir) self.instance_fail_count = plan.instance_fail_count + self.footprint = None @staticmethod def process_log(log_file): @@ -374,6 +385,13 @@ class Reporting: if instance.recording is not None: suite['recording'] = instance.recording + if (instance.status + and instance.status not in ["error", "filtered"] + and self.env.options.create_rom_ram_report + and self.env.options.footprint_report is not None): + # Init as empty data preparing for filtering properties. + suite['footprint'] = {} + # Pass suite properties through the context filters. if filters and 'allow_suite' in filters: suite = {k:v for k,v in suite.items() if k in filters['allow_suite']} @@ -381,6 +399,22 @@ class Reporting: if filters and 'deny_suite' in filters: suite = {k:v for k,v in suite.items() if k not in filters['deny_suite']} + # Compose external data only to these properties which pass filtering. + if 'footprint' in suite: + do_all = 'all' in self.env.options.footprint_report + footprint_files = { 'ROM': 'rom.json', 'RAM': 'ram.json' } + for k,v in footprint_files.items(): + if do_all or k in self.env.options.footprint_report: + footprint_fname = os.path.join(instance.build_dir, v) + try: + with open(footprint_fname, "rt") as footprint_json: + logger.debug(f"Collect footprint.{k} for '{instance.name}'") + suite['footprint'][k] = json.load(footprint_json) + except FileNotFoundError: + logger.error(f"Missing footprint.{k} for '{instance.name}'") + # + # + suites.append(suite) report["testsuites"] = suites @@ -585,7 +619,11 @@ class Reporting: if not no_update: json_file = filename + ".json" - self.json_report(json_file, version=self.env.version) + self.json_report(json_file, version=self.env.version, + filters=self.json_filters['twister.json']) + if self.env.options.footprint_report is not None: + self.json_report(filename + "_footprint.json", version=self.env.version, + filters=self.json_filters['footprint.json']) self.xunit_report(json_file, filename + ".xml", full_report=False) self.xunit_report(json_file, filename + "_report.xml", full_report=True) self.xunit_report_suites(json_file, filename + "_suite_report.xml") @@ -599,9 +637,15 @@ class Reporting: for platform in platforms.values(): if suffix: filename = os.path.join(outdir,"{}_{}.xml".format(platform.normalized_name, suffix)) - json_platform_file = os.path.join(outdir,"{}_{}.json".format(platform.normalized_name, suffix)) + json_platform_file = os.path.join(outdir,"{}_{}".format(platform.normalized_name, suffix)) else: filename = os.path.join(outdir,"{}.xml".format(platform.normalized_name)) - json_platform_file = os.path.join(outdir,"{}.json".format(platform.normalized_name)) + json_platform_file = os.path.join(outdir, platform.normalized_name) self.xunit_report(json_file, filename, platform.name, full_report=True) - self.json_report(json_platform_file, version=self.env.version, platform=platform.name) + self.json_report(json_platform_file + ".json", + version=self.env.version, platform=platform.name, + filters=self.json_filters['twister.json']) + if self.env.options.footprint_report is not None: + self.json_report(json_platform_file + "_footprint.json", + version=self.env.version, platform=platform.name, + filters=self.json_filters['footprint.json']) diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 6725134cc24..1ea52904607 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -764,6 +764,8 @@ class ProjectBuilder(FilterBuilder): 'build.log', 'device.log', 'recording.csv', + 'rom.json', + 'ram.json', # below ones are needed to make --test-only work as well 'Makefile', 'CMakeCache.txt',