twister: footprint: Add optional detailed JSON report on symbols

New Twister option `--footprint-report` is introduced to collect and
write detailed memory footprint results for symbols as an additional
JSON file. By default, the new option is disabled.

The new option implies and extends `--create-rom-ram-report`, so there
are three choices: 'ROM', 'RAM', and 'all' to select what memory area
symbols to report in `twister_footprint.json`.
In case of the custom report name, or per-platform report, it is always
composed with the rightmost '_footprint.json' suffix.

The memory footprint report has similar structure as `twister.json`
and compelements it having reduced set of test suite properties:
- instead of `testcases` it contains `footprint` object with
  `rom.json` and `ram.json` artifacts embedded there;
- other properites are limited to represent only the essential test
  suite context, thus to allow further data processing consistently
  and independently from the `twister.json`.
- 'filtered' test instances are not included into the footprint report.

Signed-off-by: Dmitrii Golovanov <dmitrii.golovanov@intel.com>

scripts/pylib/twister/twisterlib/environment.py

@@ -385,6 +385,18 @@ structure in the main Zephyr tree: boards/<arch>/<board_name>/""")
         help="Generate detailed json reports with ROM/RAM symbol sizes for each test image built "
              "using additional build option `--target footprint`.")
 
+    footprint_group.add_argument(
+        "--footprint-report",
+        nargs="?",
+        default=None,
+        choices=['all', 'ROM', 'RAM'],
+        const="all",
+        help="Select which memory area symbols' data to collect as 'footprint' property "
+             "of each test suite built, and report in 'twister_footprint.json' together "
+             "with the relevant execution metadata the same way as in `twister.json`. "
+             "Implies '--create-rom-ram-report' to generate the footprint data files. "
+             "No value means '%(const)s'. Default: %(default)s""")
+
     footprint_group.add_argument(
         "--enable-size-report",
         action="store_true",
@@ -790,6 +802,9 @@ def parse_arguments(parser, args, options = None, on_init=True):
     if options.last_metrics or options.compare_report:
         options.enable_size_report = True
 
+    if options.footprint_report:
+        options.create_rom_ram_report = True
+
     if options.aggressive_no_clean:
         options.no_clean = True
 

scripts/pylib/twister/twisterlib/reports.py

@@ -18,6 +18,16 @@ logger.setLevel(logging.DEBUG)
 
 class Reporting:
 
+    json_filters = {
+        'twister.json': {
+            'deny_suite': ['footprint']
+        },
+        'footprint.json': {
+            'deny_status': ['filtered'],
+            'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable']
+        }
+    }
+
     def __init__(self, plan, env) -> None:
         self.plan = plan #FIXME
         self.instances = plan.instances
@@ -28,6 +38,7 @@ class Reporting:
         self.timestamp = datetime.now().isoformat()
         self.outdir = os.path.abspath(env.options.outdir)
         self.instance_fail_count = plan.instance_fail_count
+        self.footprint = None
 
     @staticmethod
     def process_log(log_file):
@@ -374,6 +385,13 @@ class Reporting:
             if instance.recording is not None:
                 suite['recording'] = instance.recording
 
+            if (instance.status
+                    and instance.status not in ["error", "filtered"]
+                    and self.env.options.create_rom_ram_report
+                    and self.env.options.footprint_report is not None):
+                # Init as empty data preparing for filtering properties.
+                suite['footprint'] = {}
+
             # Pass suite properties through the context filters.
             if filters and 'allow_suite' in filters:
                 suite = {k:v for k,v in suite.items() if k in filters['allow_suite']}
@@ -381,6 +399,22 @@ class Reporting:
             if filters and 'deny_suite' in filters:
                 suite = {k:v for k,v in suite.items() if k not in filters['deny_suite']}
 
+            # Compose external data only to these properties which pass filtering.
+            if 'footprint' in suite:
+                do_all = 'all' in self.env.options.footprint_report
+                footprint_files = { 'ROM': 'rom.json', 'RAM': 'ram.json' }
+                for k,v in footprint_files.items():
+                    if do_all or k in self.env.options.footprint_report:
+                        footprint_fname = os.path.join(instance.build_dir, v)
+                        try:
+                            with open(footprint_fname, "rt") as footprint_json:
+                                logger.debug(f"Collect footprint.{k} for '{instance.name}'")
+                                suite['footprint'][k] = json.load(footprint_json)
+                        except FileNotFoundError:
+                            logger.error(f"Missing footprint.{k} for '{instance.name}'")
+                #
+            #
+
             suites.append(suite)
 
         report["testsuites"] = suites
@@ -585,7 +619,11 @@ class Reporting:
 
         if not no_update:
             json_file = filename + ".json"
-            self.json_report(json_file, version=self.env.version)
+            self.json_report(json_file, version=self.env.version,
+                             filters=self.json_filters['twister.json'])
+            if self.env.options.footprint_report is not None:
+                self.json_report(filename + "_footprint.json", version=self.env.version,
+                                 filters=self.json_filters['footprint.json'])
             self.xunit_report(json_file, filename + ".xml", full_report=False)
             self.xunit_report(json_file, filename + "_report.xml", full_report=True)
             self.xunit_report_suites(json_file, filename + "_suite_report.xml")
@@ -599,9 +637,15 @@ class Reporting:
         for platform in platforms.values():
             if suffix:
                 filename = os.path.join(outdir,"{}_{}.xml".format(platform.normalized_name, suffix))
-                json_platform_file = os.path.join(outdir,"{}_{}.json".format(platform.normalized_name, suffix))
+                json_platform_file = os.path.join(outdir,"{}_{}".format(platform.normalized_name, suffix))
             else:
                 filename = os.path.join(outdir,"{}.xml".format(platform.normalized_name))
-                json_platform_file = os.path.join(outdir,"{}.json".format(platform.normalized_name))
+                json_platform_file = os.path.join(outdir, platform.normalized_name)
             self.xunit_report(json_file, filename, platform.name, full_report=True)
-            self.json_report(json_platform_file, version=self.env.version, platform=platform.name)
+            self.json_report(json_platform_file + ".json",
+                             version=self.env.version, platform=platform.name,
+                             filters=self.json_filters['twister.json'])
+            if self.env.options.footprint_report is not None:
+                self.json_report(json_platform_file + "_footprint.json",
+                                 version=self.env.version, platform=platform.name,
+                                 filters=self.json_filters['footprint.json'])

scripts/pylib/twister/twisterlib/runner.py

@@ -764,6 +764,8 @@ class ProjectBuilder(FilterBuilder):
             'build.log',
             'device.log',
             'recording.csv',
+            'rom.json',
+            'ram.json',
             # below ones are needed to make --test-only work as well
             'Makefile',
             'CMakeCache.txt',