From e11aecaed5aff6334a77b296de259b763a08e0ca Mon Sep 17 00:00:00 2001 From: Dmitrii Golovanov Date: Fri, 1 Nov 2024 12:07:02 +0100 Subject: [PATCH] twister: fix Ztest C++ test names extraction from ELF Fix Ztest test function name extraction from ELF symbols for C++ compiled binaries where symbol names need additional 'demangling' to match with corresponding test names. The `c++filt` utility (part of binutils) is called for demangling when it is needed. Twister test suite extension and adjustment. Signed-off-by: Dmitrii Golovanov --- scripts/pylib/twister/twisterlib/runner.py | 41 +++++++++++----- scripts/tests/twister/test_runner.py | 25 ++++++++-- .../test_data/test_config.yaml | 5 +- .../group1/subgroup1/CMakeLists.txt | 8 ++++ .../agnostic_cpp/group1/subgroup1/prj.conf | 1 + .../group1/subgroup1/src/main.cpp | 28 +++++++++++ .../group1/subgroup1/test_data.yaml | 10 ++++ .../group1/subgroup2/CMakeLists.txt | 8 ++++ .../agnostic_cpp/group1/subgroup2/prj.conf | 1 + .../group1/subgroup2/src/main.cpp | 27 +++++++++++ .../group1/subgroup2/test_data.yaml | 11 +++++ .../dummy/agnostic_cpp/group2/CMakeLists.txt | 8 ++++ .../tests/dummy/agnostic_cpp/group2/prj.conf | 1 + .../dummy/agnostic_cpp/group2/src/main.cpp | 48 +++++++++++++++++++ .../dummy/agnostic_cpp/group2/src/submain.cpp | 28 +++++++++++ .../dummy/agnostic_cpp/group2/test_data.yaml | 9 ++++ scripts/tests/twister_blackbox/test_filter.py | 15 +++--- .../tests/twister_blackbox/test_platform.py | 24 +++++++++- scripts/tests/twister_blackbox/test_report.py | 4 +- scripts/tests/twister_blackbox/test_runner.py | 2 +- .../tests/twister_blackbox/test_shuffle.py | 19 ++++---- 21 files changed, 288 insertions(+), 35 deletions(-) create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp create mode 100644 scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 83ca94f9fb5..172add5367e 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -814,6 +814,10 @@ class ProjectBuilder(FilterBuilder): self.env = env self.duts = None + @property + def trace(self) -> bool: + return self.options.verbose > 2 + def log_info(self, filename, inline_logs, log_testcases=False): filename = os.path.abspath(os.path.realpath(filename)) if inline_logs: @@ -1087,6 +1091,18 @@ class ProjectBuilder(FilterBuilder): self.instance.reason = reason self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason) + def demangle(self, symbol_name): + if symbol_name[:2] == '_Z': + try: + cpp_filt = subprocess.run('c++filt', input=symbol_name, text=True, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if self.trace: + logger.debug(f"Demangle: '{symbol_name}'==>'{cpp_filt.stdout}'") + return cpp_filt.stdout.strip() + except Exception as e: + logger.error(f"Failed to demangle '{symbol_name}': {e}") + return symbol_name + def determine_testcases(self, results): yaml_testsuite_name = self.instance.testsuite.id logger.debug(f"Determine test cases for test suite: {yaml_testsuite_name}") @@ -1102,19 +1118,22 @@ class ProjectBuilder(FilterBuilder): for sym in section.iter_symbols(): # It is only meant for new ztest fx because only new ztest fx exposes test functions # precisely. - + m_ = new_ztest_unit_test_regex.search(sym.name) + if not m_: + continue + # Demangle C++ symbols + m_ = new_ztest_unit_test_regex.search(self.demangle(sym.name)) + if not m_: + continue # The 1st capture group is new ztest suite name. # The 2nd capture group is new ztest unit test name. - matches = new_ztest_unit_test_regex.findall(sym.name) - if matches: - for m in matches: - new_ztest_suite = m[0] - if new_ztest_suite not in self.instance.testsuite.ztest_suite_names: - logger.warning(f"Unexpected Ztest suite '{new_ztest_suite}' " - f"not present in: {self.instance.testsuite.ztest_suite_names}") - test_func_name = m[1].replace("test_", "", 1) - testcase_id = f"{yaml_testsuite_name}.{new_ztest_suite}.{test_func_name}" - detected_cases.append(testcase_id) + new_ztest_suite = m_[1] + if new_ztest_suite not in self.instance.testsuite.ztest_suite_names: + logger.warning(f"Unexpected Ztest suite '{new_ztest_suite}' " + f"not present in: {self.instance.testsuite.ztest_suite_names}") + test_func_name = m_[2].replace("test_", "", 1) + testcase_id = f"{yaml_testsuite_name}.{new_ztest_suite}.{test_func_name}" + detected_cases.append(testcase_id) if detected_cases: logger.debug(f"Detected Ztest cases: [{', '.join(detected_cases)}] in {elf_file}") diff --git a/scripts/tests/twister/test_runner.py b/scripts/tests/twister/test_runner.py index 24e5e4fb779..29459d58549 100644 --- a/scripts/tests/twister/test_runner.py +++ b/scripts/tests/twister/test_runner.py @@ -56,6 +56,7 @@ def mocked_instance(tmp_path): def mocked_env(): env = mock.Mock() options = mock.Mock() + options.verbose = 2 env.options = options return env @@ -1571,6 +1572,24 @@ TESTDATA_7 = [ ('dummy_id.dummy_suite2_name.dummy_name2') ] ), + ( + [ + 'z_ztest_unit_test__dummy_suite2_name__test_dummy_name2', + 'z_ztest_unit_test__bad_suite3_name_no_test', + '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_dummy_name4E', + '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_bad_name1E', + '_ZN12_GLOBAL__N_1L51z_ztest_unit_test_dummy_suite3_name__test_bad_name2E', + '_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_dummy_name5E', + '_ZN15foobarnamespaceL54z_ztest_unit_test__dummy_suite3_name__test_dummy_name6E', + ], + [ + ('dummy_id.dummy_suite2_name.dummy_name2'), + ('dummy_id.dummy_suite3_name.dummy_name4'), + ('dummy_id.dummy_suite3_name.bad_name1E'), + ('dummy_id.dummy_suite3_name.dummy_name5'), + ('dummy_id.dummy_suite3_name.dummy_name6'), + ] + ), ( ['no match'], [] @@ -1580,10 +1599,11 @@ TESTDATA_7 = [ @pytest.mark.parametrize( 'symbols_names, added_tcs', TESTDATA_7, - ids=['two hits, one miss', 'nothing'] + ids=['two hits, one miss', 'demangle', 'nothing'] ) def test_projectbuilder_determine_testcases( mocked_jobserver, + mocked_env, symbols_names, added_tcs ): @@ -1603,9 +1623,8 @@ def test_projectbuilder_determine_testcases( instance_mock.testcases = [] instance_mock.testsuite.id = 'dummy_id' instance_mock.testsuite.ztest_suite_names = [] - env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, mocked_env, mocked_jobserver) with mock.patch('twisterlib.runner.ELFFile', elf_mock), \ mock.patch('builtins.open', mock.mock_open()): diff --git a/scripts/tests/twister_blackbox/test_data/test_config.yaml b/scripts/tests/twister_blackbox/test_data/test_config.yaml index d7e4828350c..6bfe24a4e81 100644 --- a/scripts/tests/twister_blackbox/test_data/test_config.yaml +++ b/scripts/tests/twister_blackbox/test_data/test_config.yaml @@ -6,9 +6,10 @@ levels: description: > A plan to be used verifying basic features adds: - - dummy.agnostic.* + - dummy.agnostic\..* - name: acceptance description: > More coverage adds: - - dummy.* + - dummy.agnostic\..* + - dummy.device\..* diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt new file mode 100644 index 00000000000..3ffe630f1ad --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(integration) + +FILE(GLOB app_sources src/*.cpp) +target_sources(app PRIVATE ${app_sources}) diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf new file mode 100644 index 00000000000..9467c292689 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp new file mode 100644 index 00000000000..b37d02646cf --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + + +// global namespace + +ZTEST_SUITE(a1_1_tests, NULL, NULL, NULL, NULL, NULL); + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a1_1_tests, test_assert) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml new file mode 100644 index 00000000000..7d8a6420541 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml @@ -0,0 +1,10 @@ +tests: + dummy.agnostic_cpp.group1.subgroup1: + platform_allow: + - native_sim + integration_platforms: + - native_sim + tags: + - agnostic + - cpp + - subgrouped diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt new file mode 100644 index 00000000000..3ffe630f1ad --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(integration) + +FILE(GLOB app_sources src/*.cpp) +target_sources(app PRIVATE ${app_sources}) diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf new file mode 100644 index 00000000000..9467c292689 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp new file mode 100644 index 00000000000..0ac835a959c --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +// global namespace + +ZTEST_SUITE(a1_2_tests, NULL, NULL, NULL, NULL, NULL); + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a1_2_tests, test_assert) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml new file mode 100644 index 00000000000..f0146d3e193 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml @@ -0,0 +1,11 @@ +tests: + dummy.agnostic_cpp.group1.subgroup2: + build_only: true + platform_allow: + - native_sim + integration_platforms: + - native_sim + tags: + - agnostic + - cpp + - subgrouped diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt new file mode 100644 index 00000000000..3ffe630f1ad --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(integration) + +FILE(GLOB app_sources src/*.cpp) +target_sources(app PRIVATE ${app_sources}) diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf new file mode 100644 index 00000000000..9467c292689 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp new file mode 100644 index 00000000000..1c86457e76a --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + + +namespace +{ + +ZTEST_SUITE(a2_tests, NULL, NULL, NULL, NULL, NULL); + +ZTEST_SUITE(a3_tests, NULL, NULL, NULL, NULL, NULL); + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a2_tests, test_assert1) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} + +ZTEST(a2_tests, test_assert2) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} + +ZTEST(a3_tests, test_assert1) +{ + zassert_true(1, "1 was false"); +} + +} // namsespace diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp new file mode 100644 index 00000000000..bf9c90536b2 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023-2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +namespace foo_namespace +{ + +/** + * @brief Test Asserts + * + * This test verifies various assert macros provided by ztest. + * + */ +ZTEST(a2_tests, test_assert3) +{ + zassert_true(1, "1 was false"); + zassert_false(0, "0 was true"); + zassert_is_null(NULL, "NULL was not NULL"); + zassert_not_null("foo", "\"foo\" was NULL"); + zassert_equal(1, 1, "1 was not equal to 1"); + zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL"); +} + +} // foo_namespace diff --git a/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml new file mode 100644 index 00000000000..1a93b591460 --- /dev/null +++ b/scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml @@ -0,0 +1,9 @@ +tests: + dummy.agnostic_cpp.group2: + platform_allow: + - native_sim + integration_platforms: + - native_sim + tags: + - agnostic + - cpp diff --git a/scripts/tests/twister_blackbox/test_filter.py b/scripts/tests/twister_blackbox/test_filter.py index d8dfd3575e4..3558296d40b 100644 --- a/scripts/tests/twister_blackbox/test_filter.py +++ b/scripts/tests/twister_blackbox/test_filter.py @@ -82,24 +82,27 @@ class TestFilter: pass @pytest.mark.parametrize( - 'tag, expected_test_count', + 'tags, expected_test_count', [ - ('device', 6), # dummy.agnostic.group1.subgroup1.a1_1_tests.assert + (['device', 'cpp'], 6), + # dummy.agnostic.group1.subgroup1.a1_1_tests.assert # dummy.agnostic.group1.subgroup2.a2_2_tests.assert # dummy.agnostic.group2.a2_tests.assert1 # dummy.agnostic.group2.a2_tests.assert2 # dummy.agnostic.group2.a2_tests.assert3 # dummy.agnostic.group2.a3_tests.assert1 - ('agnostic', 1) # dummy.device.group.assert + (['agnostic'], 1) # dummy.device.group.assert ], - ids=['no device', 'no agnostic'] + ids=['no device, no cpp', 'no agnostic'] ) @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) - def test_exclude_tag(self, out_path, tag, expected_test_count): + def test_exclude_tag(self, out_path, tags, expected_test_count): test_platforms = ['qemu_x86', 'intel_adl_crb'] path = os.path.join(TEST_DATA, 'tests', 'dummy') args = ['-i', '--outdir', out_path, '-T', path, '-y'] + \ - ['--exclude-tag', tag] + \ + [val for pair in zip( + ['--exclude-tag'] * len(tags), tags + ) for val in pair] + \ [val for pair in zip( ['-p'] * len(test_platforms), test_platforms ) for val in pair] diff --git a/scripts/tests/twister_blackbox/test_platform.py b/scripts/tests/twister_blackbox/test_platform.py index 2e97fa29348..0c644dcff5a 100644 --- a/scripts/tests/twister_blackbox/test_platform.py +++ b/scripts/tests/twister_blackbox/test_platform.py @@ -64,6 +64,27 @@ class TestPlatform: 'only_built': 0 } ), + ( + os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic_cpp'), + ['native_sim'], + { + 'selected_test_scenarios': 3, + 'selected_test_instances': 3, + 'executed_test_instances': 3, + 'skipped_configurations': 0, + 'skipped_by_static_filter': 0, + 'skipped_at_runtime': 0, + 'passed_configurations': 2, + 'built_configurations': 1, + 'failed_configurations': 0, + 'errored_configurations': 0, + 'executed_test_cases': 5, + 'skipped_test_cases': 0, + 'platform_count': 1, + 'executed_on_platform': 2, + 'only_built': 1 + } + ), ] @classmethod @@ -129,7 +150,7 @@ class TestPlatform: assert str(sys_exit.value) == '0' - assert len(filtered_j) == 14 + assert len(filtered_j) == 26 def test_platform(self, out_path): path = os.path.join(TEST_DATA, 'tests', 'dummy') @@ -250,6 +271,7 @@ class TestPlatform: ids=[ 'emulation_only tests/dummy/agnostic', 'emulation_only tests/dummy/device', + 'native_sim_only tests/dummy/agnostic_cpp', ] ) def test_emulation_only(self, capfd, out_path, test_path, test_platforms, expected): diff --git a/scripts/tests/twister_blackbox/test_report.py b/scripts/tests/twister_blackbox/test_report.py index 3a145fd59b3..bac6ddbbbfa 100644 --- a/scripts/tests/twister_blackbox/test_report.py +++ b/scripts/tests/twister_blackbox/test_report.py @@ -355,7 +355,7 @@ class TestReport: ( os.path.join(TEST_DATA, 'tests', 'dummy'), ['--detailed-skipped-report', '--report-filtered'], - {'qemu_x86/atom': 7, 'intel_adl_crb/alder_lake': 7} + {'qemu_x86/atom': 13, 'intel_adl_crb/alder_lake': 13} ), ], ids=['dummy tests', 'dummy tests with filtered'] @@ -392,7 +392,7 @@ class TestReport: 'test_path, report_filtered, expected_filtered_count', [ (os.path.join(TEST_DATA, 'tests', 'dummy'), False, 0), - (os.path.join(TEST_DATA, 'tests', 'dummy'), True, 4), + (os.path.join(TEST_DATA, 'tests', 'dummy'), True, 10), ], ids=['no filtered', 'with filtered'] ) diff --git a/scripts/tests/twister_blackbox/test_runner.py b/scripts/tests/twister_blackbox/test_runner.py index 41eea8987b6..e1b6a194703 100644 --- a/scripts/tests/twister_blackbox/test_runner.py +++ b/scripts/tests/twister_blackbox/test_runner.py @@ -591,7 +591,7 @@ class TestRunner: sys.stderr.write(err) for line in expected: - assert re.search(line, err) + assert re.search(line, err), f"no expected:'{line}' in '{err}'" assert str(sys_exit.value) == '0' diff --git a/scripts/tests/twister_blackbox/test_shuffle.py b/scripts/tests/twister_blackbox/test_shuffle.py index 412d97a619b..f45724b7b55 100644 --- a/scripts/tests/twister_blackbox/test_shuffle.py +++ b/scripts/tests/twister_blackbox/test_shuffle.py @@ -33,20 +33,21 @@ class TestShuffle: @pytest.mark.parametrize( 'seed, ratio, expected_order', [ - ('123', '1/2', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group1.subgroup2']), - ('123', '2/2', ['dummy.agnostic.group2', 'dummy.device.group']), - ('321', '1/2', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group2']), - ('321', '2/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']), - ('123', '1/3', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group1.subgroup2']), + ('123', '1/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']), + ('123', '2/2', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup1']), + ('321', '1/2', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup2']), + ('321', '2/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup1']), + ('123', '1/3', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']), ('123', '2/3', ['dummy.agnostic.group2']), - ('123', '3/3', ['dummy.device.group']), - ('321', '1/3', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group2']), + ('123', '3/3', ['dummy.agnostic.group1.subgroup1']), + ('321', '1/3', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup2']), ('321', '2/3', ['dummy.device.group']), - ('321', '3/3', ['dummy.agnostic.group1.subgroup2']) + ('321', '3/3', ['dummy.agnostic.group1.subgroup1']) ], ids=['first half, 123', 'second half, 123', 'first half, 321', 'second half, 321', 'first third, 123', 'middle third, 123', 'last third, 123', - 'first third, 321', 'middle third, 321', 'last third, 321'] + 'first third, 321', 'middle third, 321', 'last third, 321' +] ) @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) def test_shuffle_tests(self, out_path, seed, ratio, expected_order):