From f93f82f160b5f844f6fa9982e90318a11dae7212 Mon Sep 17 00:00:00 2001 From: Dmitrii Golovanov Date: Sat, 9 Dec 2023 21:00:07 +0100 Subject: [PATCH] twister: coverage: Data collection and reporting per-test instance With this change, the coverage data (GCOV dump) is extracted from the test log files after each of the test instance parallel execution, instead of post processing all the logs at the end of the Twister run. The new `--coverage-per-instance` mode extends Twister coverage operations to report coverage statistics on each test instance execution individually in addition to the default reporting mode which aggregates data to one report with all the test instances in the current scope of the Twister run. The split mode allows to identify precisely what amount of code coverage each test suite provides and to analyze its contribution to the overall test plan's coverage. Each test configuration's output directory will have its own coverage report and data files, so the overall disk space and the total execution time increase. Another new `--disable-coverage-aggregation` option allows to execute only the `--coverage-per-instance` mode when the aggregate coverage report for the whole Twister run scope is not needed. Signed-off-by: Dmitrii Golovanov --- scripts/pylib/twister/twisterlib/coverage.py | 138 +++++++++++++----- .../pylib/twister/twisterlib/environment.py | 32 +++- scripts/pylib/twister/twisterlib/reports.py | 1 + scripts/pylib/twister/twisterlib/runner.py | 27 +++- .../pylib/twister/twisterlib/testinstance.py | 4 +- .../pylib/twister/twisterlib/twister_main.py | 8 +- scripts/tests/twister/test_runner.py | 2 +- .../tests/twister_blackbox/test_coverage.py | 7 +- 8 files changed, 167 insertions(+), 52 deletions(-) diff --git a/scripts/pylib/twister/twisterlib/coverage.py b/scripts/pylib/twister/twisterlib/coverage.py index dd647c60ddc..a35201268ec 100644 --- a/scripts/pylib/twister/twisterlib/coverage.py +++ b/scripts/pylib/twister/twisterlib/coverage.py @@ -1,6 +1,6 @@ # vim: set syntax=python ts=4 : # -# Copyright (c) 2018-2022 Intel Corporation +# Copyright (c) 2018-2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 import contextlib @@ -31,6 +31,8 @@ class CoverageTool: self.gcov_tool = None self.base_dir = None self.output_formats = None + self.coverage_capture = True + self.coverage_report = True @staticmethod def factory(tool, jobs=None): @@ -109,7 +111,7 @@ class CoverageTool: def create_gcda_files(self, extracted_coverage_info): gcda_created = True - logger.debug("Generating gcda files") + logger.debug(f"Generating {len(extracted_coverage_info)} gcda files") for filename, hexdumps in extracted_coverage_info.items(): # if kobject_hash is given for coverage gcovr fails # hence skipping it problem only in gcovr v4.1 @@ -132,7 +134,7 @@ class CoverageTool: gcda_created = False return gcda_created - def generate(self, outdir): + def capture_data(self, outdir): coverage_completed = True for filename in glob.glob(f"{outdir}/**/handler.log", recursive=True): gcov_data = self.__class__.retrieve_gcov_data(filename) @@ -148,9 +150,15 @@ class CoverageTool: else: logger.error(f"Gcov data capture incomplete: {filename}") coverage_completed = False + return coverage_completed + def generate(self, outdir): + coverage_completed = self.capture_data(outdir) if self.coverage_capture else True + if not coverage_completed or not self.coverage_report: + return coverage_completed, {} + reports = {} with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog: - ret = self._generate(outdir, coveragelog) + ret, reports = self._generate(outdir, coveragelog) if ret == 0: report_log = { "html": "HTML report generated: {}".format( @@ -180,7 +188,7 @@ class CoverageTool: else: coverage_completed = False logger.debug(f"All coverage data processed: {coverage_completed}") - return coverage_completed + return coverage_completed, reports class Lcov(CoverageTool): @@ -274,6 +282,7 @@ class Lcov(CoverageTool): "--output-file", ztestfile] self.run_lcov(cmd, coveragelog) + files = [] if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0: cmd = ["--remove", ztestfile, os.path.join(self.base_dir, "tests/ztest/test/*"), @@ -289,12 +298,15 @@ class Lcov(CoverageTool): self.run_lcov(cmd, coveragelog) if 'html' not in self.output_formats.split(','): - return 0 + return 0, {} cmd = ["genhtml", "--legend", "--branch-coverage", "--prefix", self.base_dir, "-output-directory", os.path.join(outdir, "coverage")] + files - return self.run_command(cmd, coveragelog) + ret = self.run_command(cmd, coveragelog) + + # TODO: Add LCOV summary coverage report. + return ret, { 'report': coveragefile, 'ztest': ztestfile, 'summary': None } class Gcovr(CoverageTool): @@ -344,8 +356,9 @@ class Gcovr(CoverageTool): return [a for b in list for a in b] def _generate(self, outdir, coveragelog): - coveragefile = os.path.join(outdir, "coverage.json") - ztestfile = os.path.join(outdir, "ztest.json") + coverage_file = os.path.join(outdir, "coverage.json") + coverage_summary = os.path.join(outdir, "coverage_summary.json") + ztest_file = os.path.join(outdir, "ztest.json") excludes = Gcovr._interleave_list("-e", self.ignores) if len(self.ignore_branch_patterns) > 0: @@ -358,24 +371,37 @@ class Gcovr(CoverageTool): mode_options = ["--merge-mode-functions=separate"] # We want to remove tests/* and tests/ztest/test/* but save tests/ztest - cmd = ["gcovr", "-r", self.base_dir, + cmd = ["gcovr", "-v", "-r", self.base_dir, "--gcov-ignore-parse-errors=negative_hits.warn_once_per_file", "--gcov-executable", self.gcov_tool, "-e", "tests/*"] - cmd += excludes + mode_options + ["--json", "-o", coveragefile, outdir] + cmd += excludes + mode_options + ["--json", "-o", coverage_file, outdir] cmd_str = " ".join(cmd) - logger.debug(f"Running {cmd_str}...") - subprocess.call(cmd, stdout=coveragelog) + logger.debug(f"Running: {cmd_str}") + coveragelog.write(f"Running: {cmd_str}\n") + coveragelog.flush() + ret = subprocess.call(cmd, stdout=coveragelog, stderr=coveragelog) + if ret: + logger.error(f"GCOVR failed with {ret}") + return ret, {} - subprocess.call(["gcovr", "-r", self.base_dir, "--gcov-executable", - self.gcov_tool, "-f", "tests/ztest", "-e", - "tests/ztest/test/*", "--json", "-o", ztestfile, - outdir] + mode_options, stdout=coveragelog) + cmd = ["gcovr", "-v", "-r", self.base_dir] + mode_options + cmd += ["--gcov-executable", self.gcov_tool, + "-f", "tests/ztest", "-e", "tests/ztest/test/*", + "--json", "-o", ztest_file, outdir] + cmd_str = " ".join(cmd) + logger.debug(f"Running: {cmd_str}") + coveragelog.write(f"Running: {cmd_str}\n") + coveragelog.flush() + ret = subprocess.call(cmd, stdout=coveragelog, stderr=coveragelog) + if ret: + logger.error(f"GCOVR ztest stage failed with {ret}") + return ret, {} - if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0: - files = [coveragefile, ztestfile] + if os.path.exists(ztest_file) and os.path.getsize(ztest_file) > 0: + files = [coverage_file, ztest_file] else: - files = [coveragefile] + files = [coverage_file] subdir = os.path.join(outdir, "coverage") os.makedirs(subdir, exist_ok=True) @@ -396,21 +422,21 @@ class Gcovr(CoverageTool): [report_options[r] for r in self.output_formats.split(',')] ) - return subprocess.call( - ["gcovr", "-r", self.base_dir] \ - + mode_options + gcovr_options + tracefiles, stdout=coveragelog - ) + cmd = ["gcovr", "-v", "-r", self.base_dir] + mode_options + gcovr_options + tracefiles + cmd += ["--json-summary-pretty", "--json-summary", coverage_summary] + cmd_str = " ".join(cmd) + logger.debug(f"Running: {cmd_str}") + coveragelog.write(f"Running: {cmd_str}\n") + coveragelog.flush() + ret = subprocess.call(cmd, stdout=coveragelog, stderr=coveragelog) + if ret: + logger.error(f"GCOVR merge report stage failed with {ret}") + + return ret, { 'report': coverage_file, 'ztest': ztest_file, 'summary': coverage_summary } - -def run_coverage(testplan, options): - use_system_gcov = False +def choose_gcov_tool(options, is_system_gcov): gcov_tool = None - - for plat in options.coverage_platform: - _plat = testplan.get_platform(plat) - if _plat and (_plat.type in {"native", "unit"}): - use_system_gcov = True if not options.gcov_tool: zephyr_sdk_gcov_tool = os.path.join( os.environ.get("ZEPHYR_SDK_INSTALL_DIR", default=""), @@ -427,7 +453,7 @@ def run_coverage(testplan, options): except OSError: shutil.copy(llvm_cov, gcov_lnk) gcov_tool = gcov_lnk - elif use_system_gcov: + elif is_system_gcov: gcov_tool = "gcov" elif os.path.exists(zephyr_sdk_gcov_tool): gcov_tool = zephyr_sdk_gcov_tool @@ -439,10 +465,19 @@ def run_coverage(testplan, options): else: gcov_tool = str(options.gcov_tool) - logger.info("Generating coverage files...") - logger.info(f"Using gcov tool: {gcov_tool}") + return gcov_tool + + +def run_coverage_tool(options, outdir, is_system_gcov, coverage_capture, coverage_report): coverage_tool = CoverageTool.factory(options.coverage_tool, jobs=options.jobs) - coverage_tool.gcov_tool = gcov_tool + if not coverage_tool: + return False, {} + + coverage_tool.gcov_tool = str(choose_gcov_tool(options, is_system_gcov)) + logger.debug(f"Using gcov tool: {coverage_tool.gcov_tool}") + + coverage_tool.coverage_capture = coverage_capture + coverage_tool.coverage_report = coverage_report coverage_tool.base_dir = os.path.abspath(options.coverage_basedir) # Apply output format default if options.coverage_formats is not None: @@ -456,5 +491,32 @@ def run_coverage(testplan, options): # Ignore branch coverage on __ASSERT* macros # Covering the failing case is not desirable as it will immediately terminate the test. coverage_tool.add_ignore_branch_pattern(r"^\s*__ASSERT(?:_EVAL|_NO_MSG|_POST_ACTION)?\(.*") - coverage_completed = coverage_tool.generate(options.outdir) - return coverage_completed + return coverage_tool.generate(outdir) + + +def has_system_gcov(platform): + return platform and (platform.type in {"native", "unit"}) + + +def run_coverage(options, testplan): + """ Summary code coverage over the full test plan's scope. + """ + is_system_gcov = False + + for plat in options.coverage_platform: + if has_system_gcov(testplan.get_platform(plat)): + is_system_gcov = True + break + + return run_coverage_tool(options, options.outdir, is_system_gcov, + coverage_capture=False, + coverage_report=True) + + +def run_coverage_instance(options, instance): + """ Per-instance code coverage called by ProjectBuilder ('coverage' operation). + """ + is_system_gcov = has_system_gcov(instance.platform) + return run_coverage_tool(options, instance.build_dir, is_system_gcov, + coverage_capture=True, + coverage_report=options.coverage_per_instance) diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index 839bc9f49b3..7efb69473a8 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # vim: set syntax=python ts=4 : # -# Copyright (c) 2018-2024 Intel Corporation +# Copyright (c) 2018-2025 Intel Corporation # Copyright 2022 NXP # Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved. # @@ -380,7 +380,7 @@ structure in the main Zephyr tree: boards///""") "Default to what was selected with --platform.") coverage_group.add_argument("--coverage-tool", choices=['lcov', 'gcovr'], default='gcovr', - help="Tool to use to generate coverage report (%(default)s - default).") + help="Tool to use to generate coverage reports (%(default)s - default).") coverage_group.add_argument("--coverage-formats", action="store", default=None, help="Output formats to use for generated coverage reports " + @@ -390,6 +390,19 @@ structure in the main Zephyr tree: boards///""") " Valid options for 'lcov' tool are: " + ','.join(supported_coverage_formats['lcov']) + " (html,lcov - default).") + coverage_group.add_argument("--coverage-per-instance", action="store_true", default=False, + help="""Compose individual coverage reports for each test suite + when coverage reporting is enabled; it might run in addition to + the default aggregation mode which produces one coverage report for + all executed test suites. Default: %(default)s""") + + coverage_group.add_argument("--disable-coverage-aggregation", + action="store_true", default=False, + help="""Don't aggregate coverage report statistics for all the test suites + selected to run with enabled coverage. Requires another reporting mode to be + active (`--coverage-split`) to have at least one of these reporting modes. + Default: %(default)s""") + parser.add_argument( "--test-config", action="store", @@ -908,6 +921,21 @@ def parse_arguments( if options.enable_coverage and not options.coverage_platform: options.coverage_platform = options.platform + if ( + (not options.coverage) + and (options.disable_coverage_aggregation or options.coverage_per_instance) + ): + logger.error("Enable coverage reporting to set its aggregation mode.") + sys.exit(1) + + if ( + options.coverage + and options.disable_coverage_aggregation and (not options.coverage_per_instance) + ): + logger.error("At least one coverage reporting mode should be enabled: " + "either aggregation, or per-instance, or both.") + sys.exit(1) + if options.coverage_formats: for coverage_format in options.coverage_formats.split(','): if coverage_format not in supported_coverage_formats[options.coverage_tool]: diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index 249dad8cb91..2131c38a0dd 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -58,6 +58,7 @@ class Reporting: self.outdir = os.path.abspath(env.options.outdir) self.instance_fail_count = plan.instance_fail_count self.footprint = None + self.coverage_status = None @staticmethod diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 27f1d090852..b47899d82e8 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -1,6 +1,6 @@ # vim: set syntax=python ts=4 : # -# Copyright (c) 2018-2024 Intel Corporation +# Copyright (c) 2018-2025 Intel Corporation # Copyright 2022 NXP # SPDX-License-Identifier: Apache-2.0 @@ -42,6 +42,7 @@ from twisterlib.environment import ZEPHYR_BASE sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/build_helpers")) from domains import Domains +from twisterlib.coverage import run_coverage_instance from twisterlib.environment import TwisterEnv from twisterlib.harness import Ctest, HarnessImporter, Pytest from twisterlib.log_helper import log_command @@ -1130,7 +1131,7 @@ class ProjectBuilder(FilterBuilder): self.instance.handler.thread = None self.instance.handler.duts = None - next_op = 'report' + next_op = "coverage" if self.options.coverage else "report" additionals = { "status": self.instance.status, "reason": self.instance.reason @@ -1146,6 +1147,28 @@ class ProjectBuilder(FilterBuilder): finally: self._add_to_pipeline(pipeline, next_op, additionals) + # Run per-instance code coverage + elif op == "coverage": + try: + logger.debug(f"Run coverage for '{self.instance.name}'") + self.instance.coverage_status, self.instance.coverage = \ + run_coverage_instance(self.options, self.instance) + next_op = 'report' + additionals = { + "status": self.instance.status, + "reason": self.instance.reason + } + except StatusAttributeError as sae: + logger.error(str(sae)) + self.instance.status = TwisterStatus.ERROR + reason = f"Incorrect status assignment on {op}" + self.instance.reason = reason + self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason) + next_op = 'report' + additionals = {} + finally: + self._add_to_pipeline(pipeline, next_op, additionals) + # Report results and output progress to screen elif op == "report": try: diff --git a/scripts/pylib/twister/twisterlib/testinstance.py b/scripts/pylib/twister/twisterlib/testinstance.py index f3199a13271..f198bb58bb7 100644 --- a/scripts/pylib/twister/twisterlib/testinstance.py +++ b/scripts/pylib/twister/twisterlib/testinstance.py @@ -1,6 +1,6 @@ # vim: set syntax=python ts=4 : # -# Copyright (c) 2018-2024 Intel Corporation +# Copyright (c) 2018-2025 Intel Corporation # Copyright 2022 NXP # Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved. # @@ -59,6 +59,8 @@ class TestInstance: self.metrics = dict() self.handler = None self.recording = None + self.coverage = None + self.coverage_status = None self.outdir = outdir self.execution_time = 0 self.build_time = 0 diff --git a/scripts/pylib/twister/twisterlib/twister_main.py b/scripts/pylib/twister/twisterlib/twister_main.py index 9d4485c885e..2c847e49bb1 100644 --- a/scripts/pylib/twister/twisterlib/twister_main.py +++ b/scripts/pylib/twister/twisterlib/twister_main.py @@ -214,10 +214,10 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace): report.summary(runner.results, options.disable_unrecognized_section_test, duration) - coverage_completed = True - if options.coverage: + report.coverage_status = True + if options.coverage and not options.disable_coverage_aggregation: if not options.build_only: - coverage_completed = run_coverage(tplan, options) + report.coverage_status, report.coverage = run_coverage(options, tplan) else: logger.info("Skipping coverage report generation due to --build-only.") @@ -242,7 +242,7 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace): runner.results.failed or runner.results.error or (tplan.warnings and options.warnings_as_errors) - or (options.coverage and not coverage_completed) + or (options.coverage and not report.coverage_status) ): if env.options.quit_on_failure: logger.info("twister aborted because of a failure/error") diff --git a/scripts/tests/twister/test_runner.py b/scripts/tests/twister/test_runner.py index b5eec17b9da..63894e14aed 100644 --- a/scripts/tests/twister/test_runner.py +++ b/scripts/tests/twister/test_runner.py @@ -1245,7 +1245,7 @@ TESTDATA_6 = [ mock.ANY, ['run test: dummy instance name', 'run status: dummy instance name success'], - {'op': 'report', 'test': mock.ANY, 'status': 'success', 'reason': 'OK'}, + {'op': 'coverage', 'test': mock.ANY, 'status': 'success', 'reason': 'OK'}, 'success', 'OK', 0, diff --git a/scripts/tests/twister_blackbox/test_coverage.py b/scripts/tests/twister_blackbox/test_coverage.py index 5c1b6cec72d..aeb41e63daf 100644 --- a/scripts/tests/twister_blackbox/test_coverage.py +++ b/scripts/tests/twister_blackbox/test_coverage.py @@ -13,8 +13,7 @@ import pytest import sys import json -# pylint: disable=duplicate-code -# pylint: disable=no-name-in-module +# pylint: disable=duplicate-code, disable=no-name-in-module from conftest import TEST_DATA, ZEPHYR_BASE, testsuite_filename_mock, clear_log_in_test from twisterlib.testplan import TestPlan @@ -123,7 +122,7 @@ class TestCoverage: os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), ['qemu_x86'], 'gcovr', - 'Running gcovr -r' + 'Running: gcovr ' ), ( os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), @@ -136,7 +135,7 @@ class TestCoverage: ( os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), ['qemu_x86'], - ['The specified file does not exist.', r'\[Errno 13\] Permission denied:'], + ['GCOVR failed with '], ) ] TESTDATA_7 = [