bartoszek/zephyr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

twister: fix Ztest C++ test names extraction from ELF

Browse Source 
Fix Ztest test function name extraction from ELF symbols
for C++ compiled binaries where symbol names need additional
'demangling' to match with corresponding test names.

The `c++filt` utility (part of binutils) is called for
demangling when it is needed.

Twister test suite extension and adjustment.

Signed-off-by: Dmitrii Golovanov <dmitrii.golovanov@intel.com>
This commit is contained in:
Dmitrii Golovanov 2024-11-01 12:07:02 +01:00 committed by Anas Nashif
parent 18451bca44
commit e11aecaed5
21 changed files with 288 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
scripts/pylib/twister/twisterlib/runner.py
View File

@ -814,6 +814,10 @@ class ProjectBuilder(FilterBuilder):
self.env = env
self.duts = None
@property
def trace(self) -> bool:
return self.options.verbose > 2
def log_info(self, filename, inline_logs, log_testcases=False):
filename = os.path.abspath(os.path.realpath(filename))
if inline_logs:
@ -1087,6 +1091,18 @@ class ProjectBuilder(FilterBuilder):
self.instance.reason = reason
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
def demangle(self, symbol_name):
if symbol_name[:2] == '_Z':
try:
cpp_filt = subprocess.run('c++filt', input=symbol_name, text=True, check=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if self.trace:
logger.debug(f"Demangle: '{symbol_name}'==>'{cpp_filt.stdout}'")
return cpp_filt.stdout.strip()
except Exception as e:
logger.error(f"Failed to demangle '{symbol_name}': {e}")
return symbol_name
def determine_testcases(self, results):
yaml_testsuite_name = self.instance.testsuite.id
logger.debug(f"Determine test cases for test suite: {yaml_testsuite_name}")
@ -1102,19 +1118,22 @@ class ProjectBuilder(FilterBuilder):
for sym in section.iter_symbols():
# It is only meant for new ztest fx because only new ztest fx exposes test functions
# precisely.
m_ = new_ztest_unit_test_regex.search(sym.name)
if not m_:
continue
# Demangle C++ symbols
m_ = new_ztest_unit_test_regex.search(self.demangle(sym.name))
if not m_:
continue
# The 1st capture group is new ztest suite name.
# The 2nd capture group is new ztest unit test name.
matches = new_ztest_unit_test_regex.findall(sym.name)
if matches:
for m in matches:
new_ztest_suite = m[0]
if new_ztest_suite not in self.instance.testsuite.ztest_suite_names:
logger.warning(f"Unexpected Ztest suite '{new_ztest_suite}' "
f"not present in: {self.instance.testsuite.ztest_suite_names}")
test_func_name = m[1].replace("test_", "", 1)
testcase_id = f"{yaml_testsuite_name}.{new_ztest_suite}.{test_func_name}"
detected_cases.append(testcase_id)
new_ztest_suite = m_[1]
if new_ztest_suite not in self.instance.testsuite.ztest_suite_names:
logger.warning(f"Unexpected Ztest suite '{new_ztest_suite}' "
f"not present in: {self.instance.testsuite.ztest_suite_names}")
test_func_name = m_[2].replace("test_", "", 1)
testcase_id = f"{yaml_testsuite_name}.{new_ztest_suite}.{test_func_name}"
detected_cases.append(testcase_id)
if detected_cases:
logger.debug(f"Detected Ztest cases: [{', '.join(detected_cases)}] in {elf_file}")

25
scripts/tests/twister/test_runner.py
View File

@ -56,6 +56,7 @@ def mocked_instance(tmp_path):
def mocked_env():
env = mock.Mock()
options = mock.Mock()
options.verbose = 2
env.options = options
return env
@ -1571,6 +1572,24 @@ TESTDATA_7 = [
('dummy_id.dummy_suite2_name.dummy_name2')
]
),
(
[
'z_ztest_unit_test__dummy_suite2_name__test_dummy_name2',
'z_ztest_unit_test__bad_suite3_name_no_test',
'_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_dummy_name4E',
'_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_bad_name1E',
'_ZN12_GLOBAL__N_1L51z_ztest_unit_test_dummy_suite3_name__test_bad_name2E',
'_ZN12_GLOBAL__N_1L54z_ztest_unit_test__dummy_suite3_name__test_dummy_name5E',
'_ZN15foobarnamespaceL54z_ztest_unit_test__dummy_suite3_name__test_dummy_name6E',
],
[
('dummy_id.dummy_suite2_name.dummy_name2'),
('dummy_id.dummy_suite3_name.dummy_name4'),
('dummy_id.dummy_suite3_name.bad_name1E'),
('dummy_id.dummy_suite3_name.dummy_name5'),
('dummy_id.dummy_suite3_name.dummy_name6'),
]
),
(
['no match'],
[]
@ -1580,10 +1599,11 @@ TESTDATA_7 = [
@pytest.mark.parametrize(
'symbols_names, added_tcs',
TESTDATA_7,
ids=['two hits, one miss', 'nothing']
ids=['two hits, one miss', 'demangle', 'nothing']
)
def test_projectbuilder_determine_testcases(
mocked_jobserver,
mocked_env,
symbols_names,
added_tcs
):
@ -1603,9 +1623,8 @@ def test_projectbuilder_determine_testcases(
instance_mock.testcases = []
instance_mock.testsuite.id = 'dummy_id'
instance_mock.testsuite.ztest_suite_names = []
env_mock = mock.Mock()
pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver)
pb = ProjectBuilder(instance_mock, mocked_env, mocked_jobserver)
with mock.patch('twisterlib.runner.ELFFile', elf_mock), \
mock.patch('builtins.open', mock.mock_open()):

5
scripts/tests/twister_blackbox/test_data/test_config.yaml
View File

@ -6,9 +6,10 @@ levels:
description: >
A plan to be used verifying basic features
adds:
- dummy.agnostic.*
- dummy.agnostic\..*
- name: acceptance
description: >
More coverage
adds:
- dummy.*
- dummy.agnostic\..*
- dummy.device\..*

8
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/CMakeLists.txt Normal file
View File

@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
cmake_minimum_required(VERSION 3.20.0)
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
project(integration)
FILE(GLOB app_sources src/*.cpp)
target_sources(app PRIVATE ${app_sources})

1
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/prj.conf Normal file
View File

@ -0,0 +1 @@
CONFIG_ZTEST=y

28
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/src/main.cpp Normal file
View File

@ -0,0 +1,28 @@
/*
* Copyright (c) 2023-2024 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/ztest.h>
// global namespace
ZTEST_SUITE(a1_1_tests, NULL, NULL, NULL, NULL, NULL);
/**
* @brief Test Asserts
*
* This test verifies various assert macros provided by ztest.
*
*/
ZTEST(a1_1_tests, test_assert)
{
zassert_true(1, "1 was false");
zassert_false(0, "0 was true");
zassert_is_null(NULL, "NULL was not NULL");
zassert_not_null("foo", "\"foo\" was NULL");
zassert_equal(1, 1, "1 was not equal to 1");
zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
}

10
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup1/test_data.yaml Normal file
View File

@ -0,0 +1,10 @@
tests:
dummy.agnostic_cpp.group1.subgroup1:
platform_allow:
- native_sim
integration_platforms:
- native_sim
tags:
- agnostic
- cpp
- subgrouped

8
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/CMakeLists.txt Normal file
View File

@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
cmake_minimum_required(VERSION 3.20.0)
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
project(integration)
FILE(GLOB app_sources src/*.cpp)
target_sources(app PRIVATE ${app_sources})

1
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/prj.conf Normal file
View File

@ -0,0 +1 @@
CONFIG_ZTEST=y

27
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/src/main.cpp Normal file
View File

@ -0,0 +1,27 @@
/*
* Copyright (c) 2023-2024 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/ztest.h>
// global namespace
ZTEST_SUITE(a1_2_tests, NULL, NULL, NULL, NULL, NULL);
/**
* @brief Test Asserts
*
* This test verifies various assert macros provided by ztest.
*
*/
ZTEST(a1_2_tests, test_assert)
{
zassert_true(1, "1 was false");
zassert_false(0, "0 was true");
zassert_is_null(NULL, "NULL was not NULL");
zassert_not_null("foo", "\"foo\" was NULL");
zassert_equal(1, 1, "1 was not equal to 1");
zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
}

11
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group1/subgroup2/test_data.yaml Normal file
View File

@ -0,0 +1,11 @@
tests:
dummy.agnostic_cpp.group1.subgroup2:
build_only: true
platform_allow:
- native_sim
integration_platforms:
- native_sim
tags:
- agnostic
- cpp
- subgrouped

8
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/CMakeLists.txt Normal file
View File

@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
cmake_minimum_required(VERSION 3.20.0)
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
project(integration)
FILE(GLOB app_sources src/*.cpp)
target_sources(app PRIVATE ${app_sources})

1
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/prj.conf Normal file
View File

@ -0,0 +1 @@
CONFIG_ZTEST=y

48
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/main.cpp Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2023-2024 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/ztest.h>
namespace
{
ZTEST_SUITE(a2_tests, NULL, NULL, NULL, NULL, NULL);
ZTEST_SUITE(a3_tests, NULL, NULL, NULL, NULL, NULL);
/**
* @brief Test Asserts
*
* This test verifies various assert macros provided by ztest.
*
*/
ZTEST(a2_tests, test_assert1)
{
zassert_true(1, "1 was false");
zassert_false(0, "0 was true");
zassert_is_null(NULL, "NULL was not NULL");
zassert_not_null("foo", "\"foo\" was NULL");
zassert_equal(1, 1, "1 was not equal to 1");
zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
}
ZTEST(a2_tests, test_assert2)
{
zassert_true(1, "1 was false");
zassert_false(0, "0 was true");
zassert_is_null(NULL, "NULL was not NULL");
zassert_not_null("foo", "\"foo\" was NULL");
zassert_equal(1, 1, "1 was not equal to 1");
zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
}
ZTEST(a3_tests, test_assert1)
{
zassert_true(1, "1 was false");
}
} // namsespace

28
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/src/submain.cpp Normal file
View File

@ -0,0 +1,28 @@
/*
* Copyright (c) 2023-2024 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/ztest.h>
namespace foo_namespace
{
/**
* @brief Test Asserts
*
* This test verifies various assert macros provided by ztest.
*
*/
ZTEST(a2_tests, test_assert3)
{
zassert_true(1, "1 was false");
zassert_false(0, "0 was true");
zassert_is_null(NULL, "NULL was not NULL");
zassert_not_null("foo", "\"foo\" was NULL");
zassert_equal(1, 1, "1 was not equal to 1");
zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
}
} // foo_namespace

9
scripts/tests/twister_blackbox/test_data/tests/dummy/agnostic_cpp/group2/test_data.yaml Normal file
View File

@ -0,0 +1,9 @@
tests:
dummy.agnostic_cpp.group2:
platform_allow:
- native_sim
integration_platforms:
- native_sim
tags:
- agnostic
- cpp

15
scripts/tests/twister_blackbox/test_filter.py
View File

@ -82,24 +82,27 @@ class TestFilter:
pass
@pytest.mark.parametrize(
'tag, expected_test_count',
'tags, expected_test_count',
[
('device', 6), # dummy.agnostic.group1.subgroup1.a1_1_tests.assert
(['device', 'cpp'], 6),
# dummy.agnostic.group1.subgroup1.a1_1_tests.assert
# dummy.agnostic.group1.subgroup2.a2_2_tests.assert
# dummy.agnostic.group2.a2_tests.assert1
# dummy.agnostic.group2.a2_tests.assert2
# dummy.agnostic.group2.a2_tests.assert3
# dummy.agnostic.group2.a3_tests.assert1
('agnostic', 1) # dummy.device.group.assert
(['agnostic'], 1) # dummy.device.group.assert
],
ids=['no device', 'no agnostic']
ids=['no device, no cpp', 'no agnostic']
)
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
def test_exclude_tag(self, out_path, tag, expected_test_count):
def test_exclude_tag(self, out_path, tags, expected_test_count):
test_platforms = ['qemu_x86', 'intel_adl_crb']
path = os.path.join(TEST_DATA, 'tests', 'dummy')
args = ['-i', '--outdir', out_path, '-T', path, '-y'] + \
['--exclude-tag', tag] + \
[val for pair in zip(
['--exclude-tag'] * len(tags), tags
) for val in pair] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair]

24
scripts/tests/twister_blackbox/test_platform.py
View File

@ -64,6 +64,27 @@ class TestPlatform:
'only_built': 0
}
),
(
os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic_cpp'),
['native_sim'],
{
'selected_test_scenarios': 3,
'selected_test_instances': 3,
'executed_test_instances': 3,
'skipped_configurations': 0,
'skipped_by_static_filter': 0,
'skipped_at_runtime': 0,
'passed_configurations': 2,
'built_configurations': 1,
'failed_configurations': 0,
'errored_configurations': 0,
'executed_test_cases': 5,
'skipped_test_cases': 0,
'platform_count': 1,
'executed_on_platform': 2,
'only_built': 1
}
),
]
@classmethod
@ -129,7 +150,7 @@ class TestPlatform:
assert str(sys_exit.value) == '0'
assert len(filtered_j) == 14
assert len(filtered_j) == 26
def test_platform(self, out_path):
path = os.path.join(TEST_DATA, 'tests', 'dummy')
@ -250,6 +271,7 @@ class TestPlatform:
ids=[
'emulation_only tests/dummy/agnostic',
'emulation_only tests/dummy/device',
'native_sim_only tests/dummy/agnostic_cpp',
]
)
def test_emulation_only(self, capfd, out_path, test_path, test_platforms, expected):

4
scripts/tests/twister_blackbox/test_report.py
View File

@ -355,7 +355,7 @@ class TestReport:
(
os.path.join(TEST_DATA, 'tests', 'dummy'),
['--detailed-skipped-report', '--report-filtered'],
{'qemu_x86/atom': 7, 'intel_adl_crb/alder_lake': 7}
{'qemu_x86/atom': 13, 'intel_adl_crb/alder_lake': 13}
),
],
ids=['dummy tests', 'dummy tests with filtered']
@ -392,7 +392,7 @@ class TestReport:
'test_path, report_filtered, expected_filtered_count',
[
(os.path.join(TEST_DATA, 'tests', 'dummy'), False, 0),
(os.path.join(TEST_DATA, 'tests', 'dummy'), True, 4),
(os.path.join(TEST_DATA, 'tests', 'dummy'), True, 10),
],
ids=['no filtered', 'with filtered']
)

2
scripts/tests/twister_blackbox/test_runner.py
View File

@ -591,7 +591,7 @@ class TestRunner:
sys.stderr.write(err)
for line in expected:
assert re.search(line, err)
assert re.search(line, err), f"no expected:'{line}' in '{err}'"
assert str(sys_exit.value) == '0'

19
scripts/tests/twister_blackbox/test_shuffle.py
View File

@ -33,20 +33,21 @@ class TestShuffle:
@pytest.mark.parametrize(
'seed, ratio, expected_order',
[
('123', '1/2', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group1.subgroup2']),
('123', '2/2', ['dummy.agnostic.group2', 'dummy.device.group']),
('321', '1/2', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group2']),
('321', '2/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']),
('123', '1/3', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group1.subgroup2']),
('123', '1/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']),
('123', '2/2', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup1']),
('321', '1/2', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup2']),
('321', '2/2', ['dummy.device.group', 'dummy.agnostic.group1.subgroup1']),
('123', '1/3', ['dummy.device.group', 'dummy.agnostic.group1.subgroup2']),
('123', '2/3', ['dummy.agnostic.group2']),
('123', '3/3', ['dummy.device.group']),
('321', '1/3', ['dummy.agnostic.group1.subgroup1', 'dummy.agnostic.group2']),
('123', '3/3', ['dummy.agnostic.group1.subgroup1']),
('321', '1/3', ['dummy.agnostic.group2', 'dummy.agnostic.group1.subgroup2']),
('321', '2/3', ['dummy.device.group']),
('321', '3/3', ['dummy.agnostic.group1.subgroup2'])
('321', '3/3', ['dummy.agnostic.group1.subgroup1'])
],
ids=['first half, 123', 'second half, 123', 'first half, 321', 'second half, 321',
'first third, 123', 'middle third, 123', 'last third, 123',
'first third, 321', 'middle third, 321', 'last third, 321']
'first third, 321', 'middle third, 321', 'last third, 321'
]
)
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
def test_shuffle_tests(self, out_path, seed, ratio, expected_order):