diff --git a/doc/develop/test/twister.rst b/doc/develop/test/twister.rst index 63fe6bcd395..64aa90f79b9 100644 --- a/doc/develop/test/twister.rst +++ b/doc/develop/test/twister.rst @@ -604,6 +604,18 @@ harness_config: Only one fixture can be defined per test scenario and the fixture name has to be unique across all tests in the test suite. + ztest_suite_repeat: (default 1) + This parameter specifies the number of times the entire test suite should be repeated. + + ztest_test_repeat: (default 1) + This parameter specifies the number of times each individual test within the test suite + should be repeated. + + ztest_test_shuffle: (default False) + This parameter indicates whether the order of the tests within the test suite should + be shuffled. When set to ``true``, the tests will be executed in a random order. + + The following is an example yaml file with robot harness_config options. diff --git a/scripts/pylib/twister/twisterlib/testinstance.py b/scripts/pylib/twister/twisterlib/testinstance.py index b205aa895fc..53e17b338ab 100644 --- a/scripts/pylib/twister/twisterlib/testinstance.py +++ b/scripts/pylib/twister/twisterlib/testinstance.py @@ -66,9 +66,11 @@ class TestInstance: self.build_time = 0 self.retries = 0 self.toolchain = toolchain - self.name = os.path.join(platform.name, toolchain, testsuite.name) self.dut = None + self.suite_repeat = None + self.test_repeat = None + self.test_shuffle = None if testsuite.detailed_test_id: self.build_dir = os.path.join( @@ -372,6 +374,23 @@ class TestInstance: content = "\n".join(new_config_list) + + if self.testsuite.harness_config: + self.suite_repeat = self.testsuite.harness_config.get('ztest_suite_repeat', None) + self.test_repeat = self.testsuite.harness_config.get('ztest_test_repeat', None) + self.test_shuffle = self.testsuite.harness_config.get('ztest_test_shuffle', False) + + + # Use suite_repeat and test_repeat values + if self.suite_repeat or self.test_repeat or self.test_shuffle: + content +="\nCONFIG_ZTEST_REPEAT=y" + if self.suite_repeat: + content += f"\nCONFIG_ZTEST_SUITE_REPEAT_COUNT={self.suite_repeat}" + if self.test_repeat: + content += f"\nCONFIG_ZTEST_TEST_REPEAT_COUNT={self.test_repeat}" + if self.test_shuffle: + content +="\nCONFIG_ZTEST_SHUFFLE=y" + if enable_coverage: for cp in coverage_platform: if cp in platform.aliases: diff --git a/scripts/schemas/twister/testsuite-schema.yaml b/scripts/schemas/twister/testsuite-schema.yaml index 796c342faa5..26bda49053d 100644 --- a/scripts/schemas/twister/testsuite-schema.yaml +++ b/scripts/schemas/twister/testsuite-schema.yaml @@ -186,6 +186,15 @@ schema;scenario-schema: "bsim_exe_name": type: str required: false + "ztest_suite_repeat": + type: int + required: false + "ztest_test_repeat": + type: int + required: false + "ztest_test_shuffle": + type: bool + required: false "min_ram": type: int required: false diff --git a/scripts/tests/twister/test_testinstance.py b/scripts/tests/twister/test_testinstance.py index 4c93e76bbe1..271d55e76a0 100644 --- a/scripts/tests/twister/test_testinstance.py +++ b/scripts/tests/twister/test_testinstance.py @@ -651,3 +651,38 @@ def test_testinstance_get_buildlog_file(tmp_path, testinstance, create_build_log if expected_error is None: assert res == str(build_log) + + +TESTDATA_9 = [ + ( + {'ztest_suite_repeat': 5, 'ztest_test_repeat': 10, 'ztest_test_shuffle': True}, + '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SUITE_REPEAT_COUNT=5\nCONFIG_ZTEST_TEST_REPEAT_COUNT=10\nCONFIG_ZTEST_SHUFFLE=y' + ), + ( + {'ztest_suite_repeat': 3}, + '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SUITE_REPEAT_COUNT=3' + ), + ( + {'ztest_test_repeat': 7}, + '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_TEST_REPEAT_COUNT=7' + ), + ( + {'ztest_test_shuffle': True}, + '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SHUFFLE=y' + ), + ( + {}, + '' + ), +] + +@pytest.mark.parametrize('harness_config, expected_content', TESTDATA_9) +def test_create_overlay_with_harness_config(class_testplan, all_testsuites_dict, platforms_list, harness_config, expected_content): + testsuite_path = 'scripts/tests/twister/test_data/testsuites/samples/test_app/sample_test.app' + class_testplan.testsuites = all_testsuites_dict + testsuite = class_testplan.testsuites.get(testsuite_path) + testsuite.harness_config = harness_config + class_testplan.platforms = platforms_list + platform = class_testplan.get_platform("demo_board_2") + testinstance = TestInstance(testsuite, platform,'zephyr', class_testplan.env.outdir) + assert testinstance.create_overlay(platform) == expected_content diff --git a/tests/ztest/repeat/CMakeLists.txt b/tests/ztest/repeat/CMakeLists.txt new file mode 100644 index 00000000000..54523fef0fa --- /dev/null +++ b/tests/ztest/repeat/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(repeat) + +target_sources(app PRIVATE src/main.c) diff --git a/tests/ztest/repeat/prj.conf b/tests/ztest/repeat/prj.conf new file mode 100644 index 00000000000..ce26aa164f2 --- /dev/null +++ b/tests/ztest/repeat/prj.conf @@ -0,0 +1,3 @@ +CONFIG_ZTEST=y +CONFIG_ZTEST_SUITE_REPEAT_COUNT=3 +CONFIG_ZTEST_TEST_REPEAT_COUNT=2 diff --git a/tests/ztest/repeat/src/main.c b/tests/ztest/repeat/src/main.c new file mode 100644 index 00000000000..62f14220324 --- /dev/null +++ b/tests/ztest/repeat/src/main.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ +#include + +extern struct ztest_suite_stats UTIL_CAT(z_ztest_suite_node_stats_, testsuite); +struct ztest_suite_stats *suite_stats = &UTIL_CAT(z_ztest_suite_node_stats_, testsuite); +extern struct ztest_unit_test_stats z_ztest_unit_test_stats_testsuite_test_repeating; +struct ztest_unit_test_stats *case_stats = + &z_ztest_unit_test_stats_testsuite_test_repeating; + +static struct test_data { + int suite_run; + int case_run; +} expected = {0, 0}; + +ZTEST(testsuite, test_repeating) +{ + expected.case_run++; + + if (case_stats->run_count > 0) { + zassert_true(case_stats->run_count == expected.case_run); + } + + if (suite_stats->run_count > 0) { + zassert_true(suite_stats->run_count + 1 == expected.suite_run); + } +} + +static void *repeat_setup(void) +{ + expected.suite_run++; + return NULL; +} + +ZTEST_SUITE(testsuite, NULL, repeat_setup, NULL, NULL, NULL); diff --git a/tests/ztest/repeat/testcase.yaml b/tests/ztest/repeat/testcase.yaml new file mode 100644 index 00000000000..6fd6110fe0b --- /dev/null +++ b/tests/ztest/repeat/testcase.yaml @@ -0,0 +1,30 @@ +tests: + testing.ztest.repeat_suite_regex: + platform_allow: qemu_x86 + harness: console + harness_config: + ztest_suite_repeat: 3 + ztest_test_repeat: 2 + type: multi_line + regex: + - "Running TESTSUITE testsuite" + - "Running TESTSUITE testsuite" + - "Running TESTSUITE testsuite" + + testing.ztest.repeat_testcase_regex: + platform_allow: qemu_x86 + harness: console + harness_config: + ztest_suite_repeat: 1 + ztest_test_repeat: 5 + type: multi_line + regex: + - "START - test_repeating" + - "START - test_repeating" + - "START - test_repeating" + - "START - test_repeating" + - "START - test_repeating" + + testing.ztest.repeat_regex: + platform_allow: qemu_x86 + harness: ztest