bb8b059e23
Problem ------- Board & SoC extensions are used to define out-of-tree board variants or SoC qualifiers. When a board is extended, it has multiple directories associated with it (each with its own `board.yml`), where twister should be able to find additional platform files to support these qualifiers. Currently, this doesn't work, because twister only traverses the primary BOARD_DIR and ignores the rest. The fix would've been trivial in the case of "legacy" platform files, i.e. those of the form `<normalized_board_target>.yaml`, but it's less straightforward for the newly introduced `twister.yaml` format. A `twister.yaml` file contains platform configuration that can be shared by multiple board targets and tweaked for specific targets by using the top-level `variants` key. Normally, there is at most one `twister.yaml` per board, but the file isn't necessarily unique to one board. Instead, it's unique to one directory, which may define multiple boards (as is the case with e.g. `boards/qemu/x86/`). With extensions in the picture, the goal is to initialize platforms when given multiple `twister.yaml` per board. The OOT files are expected to only provide information about OOT board targets, without being able to override in-tree targets (same principle as in the Zephyr build system). Solution -------- The `twister.yaml` handling is broken up into multiple passes - first loading all the files, then splitting the `variants` keys apart from the shared configuration, before constructing the Platform instances. The purpose of the split is to treat the variant information as global, instead of making unnecessary or faulty assumptions about locality. Remember that the build system can derive board target names not only from `board.yml`, but from `soc.yml` too. Considering that any board may end up using an OOT-extended SoC (and hence multiple `soc.yml` files), not every board target can be said to belong to some board dir. Unlike the variant data, the remaining top-level config is still rooted to the primary BOARD_DIR and inherited by the extension dirs from there. This is quite intuitive in most imagined cases, but there is a caveat: if a `twister.yaml` resides in an extension dir, then it is allowed to have a top-level config of its own, but it will be silently ignored. This is to support corner cases where, much like how a single board dir can define multiple boards, a single board dir can also extend multiple boards, or even do both. In those cases, the primary BOARD_DIR rule should make it unambiguous which config belongs to which board, even if it may seem counter-intuitive at first. For concrete examples of what this means, please see the newly added platform unit tests. As part of these functional changes, a good chunk of logic is moved out of `TestPlan.add_configurations()` into a new function in `platform.py`. This is because recombining the top-level and variant configs requires direct manipulation of the loaded YAML contents, which would be improper to do outside of the module responsible for encapsulating this data. Signed-off-by: Grzegorz Swiderski <grzegorz.swiderski@nordicsemi.no>
1794 lines
60 KiB
Python
1794 lines
60 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (c) 2020-2024 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
'''
|
|
This test file contains testsuites for testsuite.py module of twister
|
|
'''
|
|
import sys
|
|
import os
|
|
import mock
|
|
import pytest
|
|
|
|
from contextlib import nullcontext
|
|
|
|
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
|
|
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
|
|
|
from twisterlib.statuses import TwisterStatus
|
|
from twisterlib.testplan import TestPlan, change_skip_to_error_if_integration
|
|
from twisterlib.testinstance import TestInstance
|
|
from twisterlib.testsuite import TestSuite
|
|
from twisterlib.platform import Platform
|
|
from twisterlib.quarantine import Quarantine
|
|
from twisterlib.error import TwisterRuntimeError
|
|
|
|
|
|
def test_testplan_add_testsuites_short(class_testplan):
|
|
""" Testing add_testcase function of Testsuite class in twister """
|
|
# Test 1: Check the list of testsuites after calling add testsuites function is as expected
|
|
class_testplan.SAMPLE_FILENAME = 'test_sample_app.yaml'
|
|
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
|
|
class_testplan.add_testsuites()
|
|
|
|
tests_rel_dir = 'scripts/tests/twister/test_data/testsuites/tests/'
|
|
expected_testsuites = ['test_b.check_1',
|
|
'test_b.check_2',
|
|
'test_c.check_1',
|
|
'test_c.check_2',
|
|
'test_a.check_1',
|
|
'test_a.check_2',
|
|
'test_d.check_1',
|
|
'test_e.check_1',
|
|
'sample_test.app',
|
|
'test_config.main']
|
|
testsuite_list = []
|
|
for key in sorted(class_testplan.testsuites.keys()):
|
|
testsuite_list.append(os.path.basename(os.path.normpath(key)))
|
|
assert sorted(testsuite_list) == sorted(expected_testsuites)
|
|
|
|
# Test 2 : Assert Testcase name is expected & all testsuites values are testcase class objects
|
|
suite = class_testplan.testsuites.get(tests_rel_dir + 'test_a/test_a.check_1')
|
|
assert suite.name == tests_rel_dir + 'test_a/test_a.check_1'
|
|
assert all(isinstance(n, TestSuite) for n in class_testplan.testsuites.values())
|
|
|
|
@pytest.mark.parametrize("board_root_dir", [("board_config_file_not_exist"), ("board_config")])
|
|
def test_add_configurations_short(test_data, class_env, board_root_dir):
|
|
""" Testing add_configurations function of TestPlan class in Twister
|
|
Test : Asserting on default platforms list
|
|
"""
|
|
class_env.board_roots = [os.path.abspath(test_data + board_root_dir)]
|
|
plan = TestPlan(class_env)
|
|
plan.parse_configuration(config_file=class_env.test_config)
|
|
if board_root_dir == "board_config":
|
|
plan.add_configurations()
|
|
print(sorted(plan.default_platforms))
|
|
assert sorted(plan.default_platforms) == sorted(['demo_board_1/unit_testing', 'demo_board_3/unit_testing'])
|
|
elif board_root_dir == "board_config_file_not_exist":
|
|
plan.add_configurations()
|
|
assert sorted(plan.default_platforms) != sorted(['demo_board_1'])
|
|
|
|
|
|
def test_get_all_testsuites_short(class_testplan, all_testsuites_dict):
|
|
""" Testing get_all_testsuites function of TestPlan class in Twister """
|
|
plan = class_testplan
|
|
plan.testsuites = all_testsuites_dict
|
|
expected_tests = ['sample_test.app', 'test_a.check_1.1a',
|
|
'test_a.check_1.1c',
|
|
'test_a.check_1.2a', 'test_a.check_1.2b',
|
|
'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a',
|
|
'test_a.check_1.unit_1b', 'test_a.check_2.1a',
|
|
'test_a.check_2.1c', 'test_a.check_2.2a',
|
|
'test_a.check_2.2b', 'test_a.check_2.Unit_1c',
|
|
'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b',
|
|
'test_b.check_1', 'test_b.check_2', 'test_c.check_1',
|
|
'test_c.check_2', 'test_d.check_1.unit_1a',
|
|
'test_d.check_1.unit_1b',
|
|
'test_e.check_1.feature5.1a',
|
|
'test_e.check_1.feature5.1b',
|
|
'test_config.main']
|
|
|
|
assert sorted(plan.get_all_tests()) == sorted(expected_tests)
|
|
|
|
def test_get_platforms_short(class_testplan, platforms_list):
|
|
""" Testing get_platforms function of TestPlan class in Twister """
|
|
plan = class_testplan
|
|
plan.platforms = platforms_list
|
|
platform = plan.get_platform("demo_board_1")
|
|
assert isinstance(platform, Platform)
|
|
assert platform.name == "demo_board_1/unit_testing"
|
|
|
|
TESTDATA_PART1 = [
|
|
("toolchain_allow", ['gcc'], None, None, "Not in testsuite toolchain allow list"),
|
|
("platform_allow", ['demo_board_1/unit_testing'], None, None, "Not in testsuite platform allow list"),
|
|
("toolchain_exclude", ['zephyr'], None, None, "In test case toolchain exclude"),
|
|
("platform_exclude", ['demo_board_2'], None, None, "In test case platform exclude"),
|
|
("arch_exclude", ['x86'], None, None, "In test case arch exclude"),
|
|
("arch_allow", ['arm'], None, None, "Not in test case arch allow list"),
|
|
("skip", True, None, None, "Skip filter"),
|
|
("tags", set(['sensor', 'bluetooth']), "ignore_tags", ['bluetooth'], "Excluded tags per platform (exclude_tags)"),
|
|
("min_flash", "2024", "flash", "1024", "Not enough FLASH"),
|
|
("min_ram", "500", "ram", "256", "Not enough RAM"),
|
|
("None", "None", "env", ['BSIM_OUT_PATH', 'demo_env'], "Environment (BSIM_OUT_PATH, demo_env) not satisfied"),
|
|
("build_on_all", True, None, None, "Platform is excluded on command line."),
|
|
("build_on_all", True, "level", "foobar", "Unknown test level 'foobar'"),
|
|
(None, None, "supported_toolchains", ['gcc', 'xcc', 'xt-clang'], "Not supported by the toolchain"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("tc_attribute, tc_value, plat_attribute, plat_value, expected_discards",
|
|
TESTDATA_PART1)
|
|
def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list,
|
|
tc_attribute, tc_value, plat_attribute, plat_value, expected_discards):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 1: Response of apply_filters function have
|
|
appropriate values according to the filters
|
|
"""
|
|
plan = class_testplan
|
|
if tc_attribute is None and plat_attribute is None:
|
|
plan.apply_filters()
|
|
|
|
plan.platforms = platforms_list
|
|
plan.platform_names = [p.name for p in platforms_list]
|
|
plan.testsuites = all_testsuites_dict
|
|
for plat in plan.platforms:
|
|
if plat_attribute == "ignore_tags":
|
|
plat.ignore_tags = plat_value
|
|
if plat_attribute == "flash":
|
|
plat.flash = plat_value
|
|
if plat_attribute == "ram":
|
|
plat.ram = plat_value
|
|
if plat_attribute == "env":
|
|
plat.env = plat_value
|
|
plat.env_satisfied = False
|
|
if plat_attribute == "supported_toolchains":
|
|
plat.supported_toolchains = plat_value
|
|
for _, testcase in plan.testsuites.items():
|
|
if tc_attribute == "toolchain_allow":
|
|
testcase.toolchain_allow = tc_value
|
|
if tc_attribute == "platform_allow":
|
|
testcase.platform_allow = tc_value
|
|
if tc_attribute == "toolchain_exclude":
|
|
testcase.toolchain_exclude = tc_value
|
|
if tc_attribute == "platform_exclude":
|
|
testcase.platform_exclude = tc_value
|
|
if tc_attribute == "arch_exclude":
|
|
testcase.arch_exclude = tc_value
|
|
if tc_attribute == "arch_allow":
|
|
testcase.arch_allow = tc_value
|
|
if tc_attribute == "skip":
|
|
testcase.skip = tc_value
|
|
if tc_attribute == "tags":
|
|
testcase.tags = tc_value
|
|
if tc_attribute == "min_flash":
|
|
testcase.min_flash = tc_value
|
|
if tc_attribute == "min_ram":
|
|
testcase.min_ram = tc_value
|
|
|
|
if plat_attribute == "level":
|
|
plan.options.level = plat_value
|
|
|
|
if tc_attribute == "build_on_all":
|
|
for _, testcase in plan.testsuites.items():
|
|
testcase.build_on_all = tc_value
|
|
plan.apply_filters(exclude_platform=['demo_board_1'])
|
|
elif plat_attribute == "supported_toolchains":
|
|
plan.apply_filters(force_toolchain=False,
|
|
exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2/unit_testing'])
|
|
elif tc_attribute is None and plat_attribute is None:
|
|
plan.apply_filters()
|
|
else:
|
|
plan.apply_filters(exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2/unit_testing'])
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == TwisterStatus.FILTER, plan.instances.values()))
|
|
for d in filtered_instances:
|
|
assert d.reason == expected_discards
|
|
|
|
TESTDATA_PART2 = [
|
|
("runnable", "True", "Not runnable on device"),
|
|
("exclude_tag", ['test_a'], "Command line testsuite exclude filter"),
|
|
("run_individual_tests", ['scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1'], "TestSuite name filter"),
|
|
("arch", ['arm_test'], "Command line testsuite arch filter"),
|
|
("tag", ['test_d'], "Command line testsuite tag filter")
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("extra_filter, extra_filter_value, expected_discards", TESTDATA_PART2)
|
|
def test_apply_filters_part2(class_testplan, all_testsuites_dict,
|
|
platforms_list, extra_filter, extra_filter_value, expected_discards):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 2 : Response of apply_filters function (discard dictionary) have
|
|
appropriate values according to the filters
|
|
"""
|
|
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
kwargs = {
|
|
extra_filter : extra_filter_value,
|
|
"exclude_platform" : [
|
|
'demo_board_1'
|
|
],
|
|
"platform" : [
|
|
'demo_board_2'
|
|
]
|
|
}
|
|
class_testplan.apply_filters(**kwargs)
|
|
filtered_instances = list(filter(lambda item: item.status == TwisterStatus.FILTER, class_testplan.instances.values()))
|
|
for d in filtered_instances:
|
|
assert d.reason == expected_discards
|
|
|
|
|
|
TESTDATA_PART3 = [
|
|
(20, 20, -1, 0),
|
|
(-2, -1, 10, 20),
|
|
(0, 0, 0, 0)
|
|
]
|
|
|
|
@pytest.mark.parametrize("tc_min_flash, plat_flash, tc_min_ram, plat_ram",
|
|
TESTDATA_PART3)
|
|
def test_apply_filters_part3(class_testplan, all_testsuites_dict, platforms_list,
|
|
tc_min_flash, plat_flash, tc_min_ram, plat_ram):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 3 : Testing edge cases for ram and flash values of platforms & testsuites
|
|
"""
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
|
|
for plat in class_testplan.platforms:
|
|
plat.flash = plat_flash
|
|
plat.ram = plat_ram
|
|
for _, testcase in class_testplan.testsuites.items():
|
|
testcase.min_ram = tc_min_ram
|
|
testcase.min_flash = tc_min_flash
|
|
class_testplan.apply_filters(exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2'])
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == TwisterStatus.FILTER, class_testplan.instances.values()))
|
|
assert not filtered_instances
|
|
|
|
def test_add_instances_short(tmp_path, class_env, all_testsuites_dict, platforms_list):
|
|
""" Testing add_instances() function of TestPlan class in Twister
|
|
Test 1: instances dictionary keys have expected values (Platform Name + Testcase Name)
|
|
Test 2: Values of 'instances' dictionary in Testsuite class are an
|
|
instance of 'TestInstance' class
|
|
Test 3: Values of 'instances' dictionary have expected values.
|
|
"""
|
|
class_env.outdir = tmp_path
|
|
plan = TestPlan(class_env)
|
|
plan.platforms = platforms_list
|
|
platform = plan.get_platform("demo_board_2")
|
|
instance_list = []
|
|
for _, testcase in all_testsuites_dict.items():
|
|
instance = TestInstance(testcase, platform, 'zephyr', class_env.outdir)
|
|
instance_list.append(instance)
|
|
plan.add_instances(instance_list)
|
|
assert list(plan.instances.keys()) == \
|
|
[platform.name + '/zephyr/' + s for s in list(all_testsuites_dict.keys())]
|
|
assert all(isinstance(n, TestInstance) for n in list(plan.instances.values()))
|
|
assert list(plan.instances.values()) == instance_list
|
|
|
|
|
|
QUARANTINE_BASIC = {
|
|
'demo_board_1/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3',
|
|
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3'
|
|
}
|
|
|
|
QUARANTINE_WITH_REGEXP = {
|
|
'demo_board_2/unit_testing/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'a2 and c2 on x86',
|
|
'demo_board_1/unit_testing/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_3/unit_testing/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_2/unit_testing/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_2/unit_testing/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'a2 and c2 on x86'
|
|
}
|
|
|
|
QUARANTINE_PLATFORM = {
|
|
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'all on board_3',
|
|
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'all on board_3',
|
|
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all on board_3',
|
|
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3',
|
|
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3',
|
|
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3',
|
|
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3',
|
|
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_e/test_e.check_1' : 'all on board_3',
|
|
'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_config/test_config.main' : 'all on board_3'
|
|
}
|
|
|
|
QUARANTINE_MULTIFILES = {
|
|
**QUARANTINE_BASIC,
|
|
**QUARANTINE_WITH_REGEXP
|
|
}
|
|
|
|
@pytest.mark.parametrize(
|
|
("quarantine_files, quarantine_verify, expected_val"),
|
|
[
|
|
(['basic.yaml'], False, QUARANTINE_BASIC),
|
|
(['with_regexp.yaml'], False, QUARANTINE_WITH_REGEXP),
|
|
(['with_regexp.yaml'], True, QUARANTINE_WITH_REGEXP),
|
|
(['platform.yaml'], False, QUARANTINE_PLATFORM),
|
|
(['basic.yaml', 'with_regexp.yaml'], False, QUARANTINE_MULTIFILES),
|
|
(['empty.yaml'], False, {})
|
|
],
|
|
ids=[
|
|
'basic',
|
|
'with_regexp',
|
|
'quarantine_verify',
|
|
'platform',
|
|
'multifiles',
|
|
'empty'
|
|
])
|
|
def test_quarantine_short(class_testplan, platforms_list, test_data,
|
|
quarantine_files, quarantine_verify, expected_val):
|
|
""" Testing quarantine feature in Twister
|
|
"""
|
|
class_testplan.options.all = True
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
|
|
class_testplan.add_testsuites()
|
|
|
|
quarantine_list = [
|
|
os.path.join(test_data, 'quarantines', quarantine_file) for quarantine_file in quarantine_files
|
|
]
|
|
class_testplan.quarantine = Quarantine(quarantine_list)
|
|
class_testplan.options.quarantine_verify = quarantine_verify
|
|
class_testplan.apply_filters()
|
|
for testname, instance in class_testplan.instances.items():
|
|
if quarantine_verify:
|
|
if testname in expected_val:
|
|
assert instance.status == TwisterStatus.NONE
|
|
else:
|
|
assert instance.status == TwisterStatus.FILTER
|
|
assert instance.reason == "Not under quarantine"
|
|
else:
|
|
if testname in expected_val:
|
|
assert instance.status == TwisterStatus.FILTER
|
|
assert instance.reason == "Quarantine: " + expected_val[testname]
|
|
else:
|
|
assert instance.status == TwisterStatus.NONE
|
|
|
|
|
|
TESTDATA_PART4 = [
|
|
(os.path.join('test_d', 'test_d.check_1'), ['dummy'],
|
|
None, 'Snippet not supported'),
|
|
(os.path.join('test_c', 'test_c.check_1'), ['cdc-acm-console'],
|
|
0, None),
|
|
(os.path.join('test_d', 'test_d.check_1'), ['dummy', 'cdc-acm-console'],
|
|
2, 'Snippet not supported'),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'testpath, required_snippets, expected_filtered_len, expected_filtered_reason',
|
|
TESTDATA_PART4,
|
|
ids=['app', 'global', 'multiple']
|
|
)
|
|
def test_required_snippets_short(
|
|
class_testplan,
|
|
all_testsuites_dict,
|
|
platforms_list,
|
|
testpath,
|
|
required_snippets,
|
|
expected_filtered_len,
|
|
expected_filtered_reason
|
|
):
|
|
""" Testing required_snippets function of TestPlan class in Twister """
|
|
plan = class_testplan
|
|
testpath = os.path.join('scripts', 'tests', 'twister', 'test_data',
|
|
'testsuites', 'tests', testpath)
|
|
testsuite = class_testplan.testsuites.get(testpath)
|
|
plan.platforms = platforms_list
|
|
plan.platform_names = [p.name for p in platforms_list]
|
|
plan.testsuites = {testpath: testsuite}
|
|
|
|
for _, testcase in plan.testsuites.items():
|
|
testcase.exclude_platform = []
|
|
testcase.required_snippets = required_snippets
|
|
testcase.build_on_all = True
|
|
|
|
plan.apply_filters()
|
|
|
|
filtered_instances = list(
|
|
filter(lambda item: item.status == TwisterStatus.FILTER, plan.instances.values())
|
|
)
|
|
if expected_filtered_len is not None:
|
|
assert len(filtered_instances) == expected_filtered_len
|
|
if expected_filtered_reason is not None:
|
|
for d in filtered_instances:
|
|
assert d.reason == expected_filtered_reason
|
|
|
|
|
|
def test_testplan_get_level():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
lvl1 = mock.Mock()
|
|
lvl1.name = 'a lvl'
|
|
lvl2 = mock.Mock()
|
|
lvl2.name = 'a lvl'
|
|
lvl3 = mock.Mock()
|
|
lvl3.name = 'other lvl'
|
|
testplan.levels.append(lvl1)
|
|
testplan.levels.append(lvl2)
|
|
testplan.levels.append(lvl3)
|
|
|
|
name = 'a lvl'
|
|
|
|
res = testplan.get_level(name)
|
|
assert res == lvl1
|
|
|
|
res = testplan.get_level(name)
|
|
assert res == lvl1
|
|
|
|
lvl_missed = mock.Mock()
|
|
lvl_missed.name = 'missed lvl'
|
|
res = testplan.get_level('missed_lvl')
|
|
assert res is None
|
|
|
|
testplan.levels.remove(lvl1)
|
|
testplan.levels.remove(lvl2)
|
|
|
|
res = testplan.get_level(name)
|
|
assert res is None
|
|
|
|
|
|
TESTDATA_1 = [
|
|
('', {}),
|
|
(
|
|
"""\
|
|
levels:
|
|
- name: lvl1
|
|
adds:
|
|
- sc1
|
|
- sc2
|
|
inherits: []
|
|
- name: lvl2
|
|
adds:
|
|
- sc1-1
|
|
- sc1-2
|
|
inherits: [lvl1]
|
|
""",
|
|
{
|
|
'lvl1': ['sc1', 'sc2'],
|
|
'lvl2': ['sc1-1', 'sc1-2', 'sc1', 'sc2']
|
|
}
|
|
),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'config_yaml, expected_scenarios',
|
|
TESTDATA_1,
|
|
ids=['no config', 'valid config']
|
|
)
|
|
def test_testplan_parse_configuration(tmp_path, config_yaml, expected_scenarios):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.scenarios = ['sc1', 'sc1-1', 'sc1-2', 'sc2']
|
|
|
|
tmp_config_file = tmp_path / 'config_file.yaml'
|
|
if config_yaml:
|
|
tmp_config_file.write_text(config_yaml)
|
|
|
|
with pytest.raises(TwisterRuntimeError) if not config_yaml else nullcontext():
|
|
testplan.parse_configuration(tmp_config_file)
|
|
|
|
if not testplan.levels:
|
|
assert expected_scenarios == {}
|
|
for level in testplan.levels:
|
|
assert sorted(level.scenarios) == sorted(expected_scenarios[level.name])
|
|
|
|
|
|
TESTDATA_2 = [
|
|
([], [], False),
|
|
(['ts1.tc3'], [], True),
|
|
(['ts2.tc2'], ['- ts2'], False),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'sub_tests, expected_outs, expect_error',
|
|
TESTDATA_2,
|
|
ids=['no subtests', 'subtests not found', 'valid subtests']
|
|
)
|
|
def test_testplan_find_subtests(
|
|
capfd,
|
|
sub_tests,
|
|
expected_outs,
|
|
expect_error
|
|
):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.options = mock.Mock(sub_test=sub_tests)
|
|
testplan.run_individual_testsuite = []
|
|
testplan.testsuites = {
|
|
'ts1': mock.Mock(
|
|
testcases=[
|
|
mock.Mock(),
|
|
mock.Mock(),
|
|
]
|
|
),
|
|
'ts2': mock.Mock(
|
|
testcases=[
|
|
mock.Mock(),
|
|
mock.Mock(),
|
|
mock.Mock(),
|
|
]
|
|
)
|
|
}
|
|
testplan.testsuites['ts1'].name = 'ts1'
|
|
testplan.testsuites['ts1'].testcases[0].name = 'ts1.tc1'
|
|
testplan.testsuites['ts1'].testcases[1].name = 'ts1.tc2'
|
|
testplan.testsuites['ts2'].name = 'ts2'
|
|
testplan.testsuites['ts2'].testcases[0].name = 'ts2.tc1'
|
|
testplan.testsuites['ts2'].testcases[1].name = 'ts2.tc2'
|
|
testplan.testsuites['ts2'].testcases[2].name = 'ts2.tc3'
|
|
|
|
with pytest.raises(TwisterRuntimeError) if expect_error else nullcontext():
|
|
testplan.find_subtests()
|
|
|
|
out, err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stdout.write(err)
|
|
|
|
assert all([printout in out for printout in expected_outs])
|
|
|
|
|
|
TESTDATA_3 = [
|
|
(0, 0, [], False, [], TwisterRuntimeError, []),
|
|
(1, 1, [], False, [], TwisterRuntimeError, []),
|
|
(1, 0, [], True, [], TwisterRuntimeError, ['No quarantine list given to be verified']),
|
|
# (1, 0, ['qfile.yaml'], False, ['# empty'], None, ['Quarantine file qfile.yaml is empty']),
|
|
(1, 0, ['qfile.yaml'], False, ['- platforms:\n - demo_board_3\n comment: "board_3"'], None, []),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'added_testsuite_count, load_errors, ql, qv, ql_data, exception, expected_logs',
|
|
TESTDATA_3,
|
|
ids=['no tests', 'load errors', 'quarantine verify without quarantine list',
|
|
# 'empty quarantine file',
|
|
'valid quarantine file']
|
|
)
|
|
def test_testplan_discover(
|
|
tmp_path,
|
|
caplog,
|
|
added_testsuite_count,
|
|
load_errors,
|
|
ql,
|
|
qv,
|
|
ql_data,
|
|
exception,
|
|
expected_logs
|
|
):
|
|
for qf, data in zip(ql, ql_data):
|
|
tmp_qf = tmp_path / qf
|
|
tmp_qf.write_text(data)
|
|
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.options = mock.Mock(
|
|
test='ts1',
|
|
quarantine_list=[tmp_path / qf for qf in ql],
|
|
quarantine_verify=qv,
|
|
)
|
|
testplan.testsuites = {
|
|
'ts1': mock.Mock(id=1),
|
|
'ts2': mock.Mock(id=2),
|
|
}
|
|
testplan.run_individual_testsuite = 'ts0'
|
|
testplan.load_errors = load_errors
|
|
testplan.add_testsuites = mock.Mock(return_value=added_testsuite_count)
|
|
testplan.find_subtests = mock.Mock()
|
|
testplan.report_duplicates = mock.Mock()
|
|
testplan.parse_configuration = mock.Mock()
|
|
testplan.add_configurations = mock.Mock()
|
|
|
|
with pytest.raises(exception) if exception else nullcontext():
|
|
testplan.discover()
|
|
|
|
testplan.add_testsuites.assert_called_once_with(testsuite_filter='ts1')
|
|
assert all([log in caplog.text for log in expected_logs])
|
|
|
|
|
|
TESTDATA_4 = [
|
|
(None, None, None, None, '00',
|
|
TwisterRuntimeError, [], []),
|
|
(None, True, None, None, '6/4',
|
|
TwisterRuntimeError, set(['t-p3', 't-p4', 't-p1', 't-p2']), []),
|
|
(None, None, 'load_tests.json', None, '0/4',
|
|
TwisterRuntimeError, set(['lt-p1', 'lt-p3', 'lt-p4', 'lt-p2']), []),
|
|
('suffix', None, None, True, '2/4',
|
|
None, set(['ts-p4', 'ts-p2', 'ts-p1', 'ts-p3']), [2, 4]),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'report_suffix, only_failed, load_tests, test_only, subset,' \
|
|
' exception, expected_selected_platforms, expected_generate_subset_args',
|
|
TESTDATA_4,
|
|
ids=['apply_filters only', 'only failed', 'load tests', 'test only']
|
|
)
|
|
def test_testplan_load(
|
|
tmp_path,
|
|
report_suffix,
|
|
only_failed,
|
|
load_tests,
|
|
test_only,
|
|
subset,
|
|
exception,
|
|
expected_selected_platforms,
|
|
expected_generate_subset_args
|
|
):
|
|
twister_json = """\
|
|
{
|
|
"testsuites": [
|
|
{
|
|
"name": "ts1",
|
|
"platform": "t-p1",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts1",
|
|
"platform": "t-p2",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "t-p3",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "t-p4",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
}
|
|
]
|
|
}
|
|
"""
|
|
twister_file = tmp_path / 'twister.json'
|
|
twister_file.write_text(twister_json)
|
|
|
|
twister_suffix_json = """\
|
|
{
|
|
"testsuites": [
|
|
{
|
|
"name": "ts1",
|
|
"platform": "ts-p1",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts1",
|
|
"platform": "ts-p2",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "ts-p3",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "ts-p4",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
}
|
|
]
|
|
}
|
|
"""
|
|
twister_suffix_file = tmp_path / 'twister_suffix.json'
|
|
twister_suffix_file.write_text(twister_suffix_json)
|
|
|
|
load_tests_json = """\
|
|
{
|
|
"testsuites": [
|
|
{
|
|
"name": "ts1",
|
|
"platform": "lt-p1",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts1",
|
|
"platform": "lt-p2",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "lt-p3",
|
|
"toolchain": "zephyr",
|
|
\"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "lt-p4",
|
|
"toolchain": "zephyr",
|
|
"testcases": []
|
|
}
|
|
]
|
|
}
|
|
"""
|
|
load_tests_file = tmp_path / 'load_tests.json'
|
|
load_tests_file.write_text(load_tests_json)
|
|
|
|
testplan = TestPlan(env=mock.Mock(outdir=tmp_path))
|
|
testplan.testsuites = {
|
|
'ts1': mock.Mock(testcases=[], extra_configs=[]),
|
|
'ts2': mock.Mock(testcases=[], extra_configs=[]),
|
|
}
|
|
testplan.testsuites['ts1'].name = 'ts1'
|
|
testplan.testsuites['ts2'].name = 'ts2'
|
|
testplan.options = mock.Mock(
|
|
report_summary=None,
|
|
outdir=tmp_path,
|
|
report_suffix=report_suffix,
|
|
only_failed=only_failed,
|
|
load_tests=tmp_path / load_tests if load_tests else None,
|
|
test_only=test_only,
|
|
exclude_platform=['t-p0', 't-p1',
|
|
'ts-p0', 'ts-p1',
|
|
'lt-p0', 'lt-p1'],
|
|
platform=['t-p1', 't-p2', 't-p3', 't-p4',
|
|
'ts-p1', 'ts-p2', 'ts-p3', 'ts-p4',
|
|
'lt-p1', 'lt-p2', 'lt-p3', 'lt-p4'],
|
|
subset=subset
|
|
)
|
|
testplan.platforms=[mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(),
|
|
mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(),
|
|
mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock()]
|
|
testplan.platforms[0].name = 't-p1'
|
|
testplan.platforms[1].name = 't-p2'
|
|
testplan.platforms[2].name = 't-p3'
|
|
testplan.platforms[3].name = 't-p4'
|
|
testplan.platforms[4].name = 'ts-p1'
|
|
testplan.platforms[5].name = 'ts-p2'
|
|
testplan.platforms[6].name = 'ts-p3'
|
|
testplan.platforms[7].name = 'ts-p4'
|
|
testplan.platforms[8].name = 'lt-p1'
|
|
testplan.platforms[9].name = 'lt-p2'
|
|
testplan.platforms[10].name = 'lt-p3'
|
|
testplan.platforms[11].name = 'lt-p4'
|
|
testplan.platforms[0].aliases = ['t-p1']
|
|
testplan.platforms[1].aliases = ['t-p2']
|
|
testplan.platforms[2].aliases = ['t-p3']
|
|
testplan.platforms[3].aliases = ['t-p4']
|
|
testplan.platforms[4].aliases = ['ts-p1']
|
|
testplan.platforms[5].aliases = ['ts-p2']
|
|
testplan.platforms[6].aliases = ['ts-p3']
|
|
testplan.platforms[7].aliases = ['ts-p4']
|
|
testplan.platforms[8].aliases = ['lt-p1']
|
|
testplan.platforms[9].aliases = ['lt-p2']
|
|
testplan.platforms[10].aliases = ['lt-p3']
|
|
testplan.platforms[11].aliases = ['lt-p4']
|
|
testplan.platforms[0].normalized_name = 't-p1'
|
|
testplan.platforms[1].normalized_name = 't-p2'
|
|
testplan.platforms[2].normalized_name = 't-p3'
|
|
testplan.platforms[3].normalized_name = 't-p4'
|
|
testplan.platforms[4].normalized_name = 'ts-p1'
|
|
testplan.platforms[5].normalized_name = 'ts-p2'
|
|
testplan.platforms[6].normalized_name = 'ts-p3'
|
|
testplan.platforms[7].normalized_name = 'ts-p4'
|
|
testplan.platforms[8].normalized_name = 'lt-p1'
|
|
testplan.platforms[9].normalized_name = 'lt-p2'
|
|
testplan.platforms[10].normalized_name = 'lt-p3'
|
|
testplan.platforms[11].normalized_name = 'lt-p4'
|
|
testplan.generate_subset = mock.Mock()
|
|
testplan.apply_filters = mock.Mock()
|
|
|
|
with mock.patch('twisterlib.testinstance.TestInstance.create_overlay', mock.Mock()), \
|
|
mock.patch('twisterlib.testinstance.TestInstance.check_runnable', return_value=True), \
|
|
pytest.raises(exception) if exception else nullcontext():
|
|
testplan.load()
|
|
|
|
assert testplan.selected_platforms == expected_selected_platforms
|
|
if expected_generate_subset_args:
|
|
testplan.generate_subset.assert_called_once_with(*expected_generate_subset_args)
|
|
else:
|
|
testplan.generate_subset.assert_not_called()
|
|
|
|
|
|
TESTDATA_5 = [
|
|
(False, False, None, 1, 2,
|
|
['plat1/testA', 'plat1/testB', 'plat1/testC',
|
|
'plat3/testA', 'plat3/testB', 'plat3/testC']),
|
|
(False, False, None, 1, 5,
|
|
['plat1/testA',
|
|
'plat3/testA', 'plat3/testB', 'plat3/testC']),
|
|
(False, False, None, 2, 2,
|
|
['plat2/testA', 'plat2/testB']),
|
|
(True, False, None, 1, 2,
|
|
['plat1/testA', 'plat2/testA', 'plat1/testB',
|
|
'plat3/testA', 'plat3/testB', 'plat3/testC']),
|
|
(True, False, None, 2, 2,
|
|
['plat2/testB', 'plat1/testC']),
|
|
(True, True, 123, 1, 2,
|
|
['plat2/testA', 'plat2/testB', 'plat1/testC',
|
|
'plat3/testB', 'plat3/testA', 'plat3/testC']),
|
|
(True, True, 123, 2, 2,
|
|
['plat1/testB', 'plat1/testA']),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'device_testing, shuffle, seed, subset, sets, expected_subset',
|
|
TESTDATA_5,
|
|
ids=['subset 1', 'subset 1 out of 5', 'subset 2',
|
|
'device testing, subset 1', 'device testing, subset 2',
|
|
'device testing, shuffle with seed, subset 1',
|
|
'device testing, shuffle with seed, subset 2']
|
|
)
|
|
def test_testplan_generate_subset(
|
|
device_testing,
|
|
shuffle,
|
|
seed,
|
|
subset,
|
|
sets,
|
|
expected_subset
|
|
):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.options = mock.Mock(
|
|
device_testing=device_testing,
|
|
shuffle_tests=shuffle,
|
|
shuffle_tests_seed=seed
|
|
)
|
|
testplan.instances = {
|
|
'plat1/testA': mock.Mock(status=TwisterStatus.NONE),
|
|
'plat1/testB': mock.Mock(status=TwisterStatus.NONE),
|
|
'plat1/testC': mock.Mock(status=TwisterStatus.NONE),
|
|
'plat2/testA': mock.Mock(status=TwisterStatus.NONE),
|
|
'plat2/testB': mock.Mock(status=TwisterStatus.NONE),
|
|
'plat3/testA': mock.Mock(status=TwisterStatus.SKIP),
|
|
'plat3/testB': mock.Mock(status=TwisterStatus.SKIP),
|
|
'plat3/testC': mock.Mock(status=TwisterStatus.ERROR),
|
|
}
|
|
|
|
testplan.generate_subset(subset, sets)
|
|
|
|
assert [instance for instance in testplan.instances.keys()] == \
|
|
expected_subset
|
|
|
|
|
|
def test_testplan_handle_modules():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
|
|
modules = [mock.Mock(meta={'name': 'name1'}),
|
|
mock.Mock(meta={'name': 'name2'})]
|
|
|
|
with mock.patch('twisterlib.testplan.parse_modules', return_value=modules):
|
|
testplan.handle_modules()
|
|
|
|
assert testplan.modules == ['name1', 'name2']
|
|
|
|
|
|
TESTDATA_6 = [
|
|
(True, False, False, 0, 'report_test_tree'),
|
|
(True, True, False, 0, 'report_test_tree'),
|
|
(True, False, True, 0, 'report_test_tree'),
|
|
(True, True, True, 0, 'report_test_tree'),
|
|
(False, True, False, 0, 'report_test_list'),
|
|
(False, True, True, 0, 'report_test_list'),
|
|
(False, False, True, 0, 'report_tag_list'),
|
|
(False, False, False, 1, None),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'test_tree, list_tests, list_tags, expected_res, expected_method',
|
|
TESTDATA_6,
|
|
ids=['test tree', 'test tree + test list', 'test tree + tag list',
|
|
'test tree + test list + tag list', 'test list',
|
|
'test list + tag list', 'tag list', 'no report']
|
|
)
|
|
def test_testplan_report(
|
|
test_tree,
|
|
list_tests,
|
|
list_tags,
|
|
expected_res,
|
|
expected_method
|
|
):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.report_test_tree = mock.Mock()
|
|
testplan.report_test_list = mock.Mock()
|
|
testplan.report_tag_list = mock.Mock()
|
|
|
|
testplan.options = mock.Mock(
|
|
test_tree=test_tree,
|
|
list_tests=list_tests,
|
|
list_tags=list_tags,
|
|
)
|
|
|
|
res = testplan.report()
|
|
|
|
assert res == expected_res
|
|
|
|
methods = ['report_test_tree', 'report_test_list', 'report_tag_list']
|
|
if expected_method:
|
|
methods.remove(expected_method)
|
|
getattr(testplan, expected_method).assert_called_once()
|
|
for method in methods:
|
|
getattr(testplan, method).assert_not_called()
|
|
|
|
|
|
TESTDATA_7 = [
|
|
(
|
|
[
|
|
mock.Mock(
|
|
yamlfile='a.yaml',
|
|
scenarios=['scenario1', 'scenario2']
|
|
),
|
|
mock.Mock(
|
|
yamlfile='b.yaml',
|
|
scenarios=['scenario1']
|
|
)
|
|
],
|
|
TwisterRuntimeError,
|
|
'Duplicated test scenarios found:\n' \
|
|
'- scenario1 found in:\n' \
|
|
' - a.yaml\n' \
|
|
' - b.yaml\n',
|
|
[]
|
|
),
|
|
(
|
|
[
|
|
mock.Mock(
|
|
yamlfile='a.yaml',
|
|
scenarios=['scenario.a.1', 'scenario.a.2']
|
|
),
|
|
mock.Mock(
|
|
yamlfile='b.yaml',
|
|
scenarios=['scenario.b.1']
|
|
)
|
|
],
|
|
None,
|
|
None,
|
|
['No duplicates found.']
|
|
),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'testsuites, expected_error, error_msg, expected_logs',
|
|
TESTDATA_7,
|
|
ids=['a duplicate', 'no duplicates']
|
|
)
|
|
def test_testplan_report_duplicates(
|
|
capfd,
|
|
caplog,
|
|
testsuites,
|
|
expected_error,
|
|
error_msg,
|
|
expected_logs
|
|
):
|
|
def mock_get(name):
|
|
return list(filter(lambda x: name in x.scenarios, testsuites))
|
|
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.scenarios = [scenario for testsuite in testsuites \
|
|
for scenario in testsuite.scenarios]
|
|
testplan.get_testsuite = mock.Mock(side_effect=mock_get)
|
|
|
|
with pytest.raises(expected_error) if expected_error is not None else \
|
|
nullcontext() as err:
|
|
testplan.report_duplicates()
|
|
|
|
if expected_error:
|
|
assert str(err._excinfo[1]) == error_msg
|
|
|
|
assert all([log in caplog.text for log in expected_logs])
|
|
|
|
|
|
def test_testplan_report_tag_list(capfd):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.testsuites = {
|
|
'testsuite0': mock.Mock(tags=set(['tag1', 'tag2'])),
|
|
'testsuite1': mock.Mock(tags=set(['tag1', 'tag2', 'tag3'])),
|
|
'testsuite2': mock.Mock(tags=set(['tag1', 'tag3'])),
|
|
'testsuite3': mock.Mock(tags=set(['tag']))
|
|
}
|
|
|
|
testplan.report_tag_list()
|
|
|
|
out,err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stderr.write(err)
|
|
|
|
assert '- tag' in out
|
|
assert '- tag1' in out
|
|
assert '- tag2' in out
|
|
assert '- tag3' in out
|
|
|
|
|
|
def test_testplan_report_test_tree(capfd):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.get_tests_list = mock.Mock(
|
|
return_value=['1.dummy.case.1', '1.dummy.case.2',
|
|
'2.dummy.case.1', '2.dummy.case.2',
|
|
'3.dummy.case.1', '3.dummy.case.2',
|
|
'4.dummy.case.1', '4.dummy.case.2',
|
|
'5.dummy.case.1', '5.dummy.case.2',
|
|
'sample.group1.case1', 'sample.group1.case2',
|
|
'sample.group2.case', 'sample.group3.case1',
|
|
'sample.group3.case2', 'sample.group3.case3']
|
|
)
|
|
|
|
testplan.report_test_tree()
|
|
|
|
out,err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stderr.write(err)
|
|
|
|
expected = """
|
|
Testsuite
|
|
├── Samples
|
|
│ ├── group1
|
|
│ │ ├── sample.group1.case1
|
|
│ │ └── sample.group1.case2
|
|
│ ├── group2
|
|
│ │ └── sample.group2.case
|
|
│ └── group3
|
|
│ ├── sample.group3.case1
|
|
│ ├── sample.group3.case2
|
|
│ └── sample.group3.case3
|
|
└── Tests
|
|
├── 1
|
|
│ └── dummy
|
|
│ ├── 1.dummy.case.1
|
|
│ └── 1.dummy.case.2
|
|
├── 2
|
|
│ └── dummy
|
|
│ ├── 2.dummy.case.1
|
|
│ └── 2.dummy.case.2
|
|
├── 3
|
|
│ └── dummy
|
|
│ ├── 3.dummy.case.1
|
|
│ └── 3.dummy.case.2
|
|
├── 4
|
|
│ └── dummy
|
|
│ ├── 4.dummy.case.1
|
|
│ └── 4.dummy.case.2
|
|
└── 5
|
|
└── dummy
|
|
├── 5.dummy.case.1
|
|
└── 5.dummy.case.2
|
|
"""
|
|
expected = expected[1:]
|
|
|
|
assert expected in out
|
|
|
|
|
|
def test_testplan_report_test_list(capfd):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.get_tests_list = mock.Mock(
|
|
return_value=['4.dummy.case.1', '4.dummy.case.2',
|
|
'3.dummy.case.2', '2.dummy.case.2',
|
|
'1.dummy.case.1', '1.dummy.case.2',
|
|
'3.dummy.case.1', '2.dummy.case.1',
|
|
'5.dummy.case.1', '5.dummy.case.2']
|
|
)
|
|
|
|
testplan.report_test_list()
|
|
|
|
out,err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stderr.write(err)
|
|
|
|
assert ' - 1.dummy.case.1\n' \
|
|
' - 1.dummy.case.2\n' \
|
|
' - 2.dummy.case.1\n' \
|
|
' - 2.dummy.case.2\n' \
|
|
' - 3.dummy.case.1\n' \
|
|
' - 3.dummy.case.2\n' \
|
|
' - 4.dummy.case.1\n' \
|
|
' - 4.dummy.case.2\n' \
|
|
' - 5.dummy.case.1\n' \
|
|
' - 5.dummy.case.2\n' \
|
|
'10 total.' in out
|
|
|
|
|
|
def test_testplan_info(capfd):
|
|
TestPlan.info('dummy text')
|
|
|
|
out, err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stderr.write(err)
|
|
|
|
assert 'dummy text\n' in out
|
|
|
|
|
|
TESTDATA_8 = [
|
|
(False, ['p1e2/unit_testing', 'p2/unit_testing', 'p3/unit_testing'], ['p2/unit_testing', 'p3/unit_testing']),
|
|
(True, ['p1e2/unit_testing', 'p2/unit_testing', 'p3/unit_testing'], ['p3/unit_testing']),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'override_default_platforms, expected_platform_names, expected_defaults',
|
|
TESTDATA_8,
|
|
ids=['no override defaults', 'override defaults']
|
|
)
|
|
def test_testplan_add_configurations(
|
|
tmp_path,
|
|
override_default_platforms,
|
|
expected_platform_names,
|
|
expected_defaults
|
|
):
|
|
env = mock.Mock(board_roots=[tmp_path / 'boards'], soc_roots=[tmp_path], arch_roots=[tmp_path])
|
|
|
|
testplan = TestPlan(env=env)
|
|
|
|
testplan.test_config = {
|
|
'platforms': {
|
|
'override_default_platforms': override_default_platforms,
|
|
'default_platforms': ['p3', 'p1e1']
|
|
}
|
|
}
|
|
|
|
def mock_gen_plat(board_roots, soc_roots, arch_roots):
|
|
assert [tmp_path] == board_roots
|
|
assert [tmp_path] == soc_roots
|
|
assert [tmp_path] == arch_roots
|
|
|
|
platforms = [
|
|
mock.Mock(aliases=['p1e1/unit_testing', 'p1e1'], twister=False, default=False),
|
|
mock.Mock(aliases=['p1e2/unit_testing', 'p1e2'], twister=True, default=False),
|
|
mock.Mock(aliases=['p2/unit_testing', 'p2'], twister=True, default=True),
|
|
mock.Mock(aliases=['p3/unit_testing', 'p3'], twister=True, default=True),
|
|
]
|
|
for platform in platforms:
|
|
type(platform).name = mock.PropertyMock(return_value=platform.aliases[0])
|
|
yield platform
|
|
|
|
with mock.patch('twisterlib.testplan.generate_platforms', mock_gen_plat):
|
|
testplan.add_configurations()
|
|
|
|
if expected_defaults is not None:
|
|
print(expected_defaults)
|
|
print(testplan.default_platforms)
|
|
assert sorted(expected_defaults) == sorted(testplan.default_platforms)
|
|
if expected_platform_names is not None:
|
|
print(expected_platform_names)
|
|
print(testplan.platform_names)
|
|
platform_names = [p.name for p in testplan.platforms]
|
|
assert sorted(expected_platform_names) == sorted(platform_names)
|
|
|
|
|
|
def test_testplan_get_all_tests():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
tc1 = mock.Mock()
|
|
tc1.name = 'tc1'
|
|
tc2 = mock.Mock()
|
|
tc2.name = 'tc2'
|
|
tc3 = mock.Mock()
|
|
tc3.name = 'tc3'
|
|
tc4 = mock.Mock()
|
|
tc4.name = 'tc4'
|
|
tc5 = mock.Mock()
|
|
tc5.name = 'tc5'
|
|
ts1 = mock.Mock(testcases=[tc1, tc2])
|
|
ts2 = mock.Mock(testcases=[tc3, tc4, tc5])
|
|
testplan.testsuites = {
|
|
'ts1': ts1,
|
|
'ts2': ts2
|
|
}
|
|
|
|
res = testplan.get_all_tests()
|
|
|
|
assert sorted(res) == ['tc1', 'tc2', 'tc3', 'tc4', 'tc5']
|
|
|
|
|
|
TESTDATA_9 = [
|
|
([], False, True, 11, 1),
|
|
([], False, False, 7, 2),
|
|
([], True, False, 9, 1),
|
|
([], True, True, 9, 1),
|
|
([], True, False, 9, 1),
|
|
(['good_test/dummy.common.1', 'good_test/dummy.common.2', 'good_test/dummy.common.3'], False, True, 3, 1),
|
|
(['good_test/dummy.common.1', 'good_test/dummy.common.2',
|
|
'duplicate_test/dummy.common.1', 'duplicate_test/dummy.common.2'], False, True, 4, 1),
|
|
(['dummy.common.1', 'dummy.common.2'], False, False, 2, 1),
|
|
(['good_test/dummy.common.1', 'good_test/dummy.common.2', 'good_test/dummy.common.3'], True, True, 0, 1),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'testsuite_filter, use_alt_root, detailed_id, expected_suite_count, expected_errors',
|
|
TESTDATA_9,
|
|
ids=[
|
|
'no testsuite filter, detailed id',
|
|
'no testsuite filter, short id',
|
|
'no testsuite filter, alt root, detailed id',
|
|
'no filter, alt root, detailed id',
|
|
'no filter, alt root, short id',
|
|
'testsuite filter',
|
|
'testsuite filter and valid duplicate',
|
|
'testsuite filter, short id and duplicate',
|
|
'testsuite filter, alt root',
|
|
]
|
|
)
|
|
def test_testplan_add_testsuites(tmp_path, testsuite_filter, use_alt_root, detailed_id,
|
|
expected_errors, expected_suite_count):
|
|
# tmp_path
|
|
# ├ tests <- test root
|
|
# │ ├ good_test
|
|
# │ │ └ testcase.yaml
|
|
# │ ├ wrong_test
|
|
# │ │ └ testcase.yaml
|
|
# │ ├ good_sample
|
|
# │ │ └ sample.yaml
|
|
# │ ├ duplicate_test
|
|
# │ │ └ testcase.yaml
|
|
# │ └ others
|
|
# │ └ other.txt
|
|
# └ other_tests <- alternate test root
|
|
# └ good_test
|
|
# └ testcase.yaml
|
|
tmp_test_root_dir = tmp_path / 'tests'
|
|
tmp_test_root_dir.mkdir()
|
|
|
|
tmp_good_test_dir = tmp_test_root_dir / 'good_test'
|
|
tmp_good_test_dir.mkdir()
|
|
testcase_yaml_1 = """\
|
|
tests:
|
|
dummy.common.1:
|
|
build_on_all: true
|
|
dummy.common.2:
|
|
build_on_all: true
|
|
dummy.common.3:
|
|
build_on_all: true
|
|
dummy.special:
|
|
build_on_all: false
|
|
"""
|
|
testfile_1 = tmp_good_test_dir / 'testcase.yaml'
|
|
testfile_1.write_text(testcase_yaml_1)
|
|
|
|
tmp_bad_test_dir = tmp_test_root_dir / 'wrong_test'
|
|
tmp_bad_test_dir.mkdir()
|
|
testcase_yaml_2 = """\
|
|
tests:
|
|
wrong:
|
|
yaml: {]}
|
|
"""
|
|
testfile_2 = tmp_bad_test_dir / 'testcase.yaml'
|
|
testfile_2.write_text(testcase_yaml_2)
|
|
|
|
tmp_good_sample_dir = tmp_test_root_dir / 'good_sample'
|
|
tmp_good_sample_dir.mkdir()
|
|
samplecase_yaml_1 = """\
|
|
tests:
|
|
sample.dummy.common.1:
|
|
tags:
|
|
- samples
|
|
sample.dummy.common.2:
|
|
tags:
|
|
- samples
|
|
sample.dummy.special.1:
|
|
tags:
|
|
- samples
|
|
"""
|
|
samplefile_1 = tmp_good_sample_dir / 'sample.yaml'
|
|
samplefile_1.write_text(samplecase_yaml_1)
|
|
|
|
tmp_duplicate_test_dir = tmp_test_root_dir / 'duplicate_test'
|
|
tmp_duplicate_test_dir.mkdir()
|
|
# The duplicate needs to have the same number of tests as these configurations
|
|
# can be read either with duplicate_test first, or good_test first, so number
|
|
# of selected tests needs to be the same in both situations.
|
|
testcase_yaml_4 = """\
|
|
tests:
|
|
dummy.common.1:
|
|
build_on_all: true
|
|
dummy.common.2:
|
|
build_on_all: true
|
|
dummy.common.3:
|
|
build_on_all: true
|
|
dummy.special:
|
|
build_on_all: false
|
|
"""
|
|
testfile_4 = tmp_duplicate_test_dir / 'testcase.yaml'
|
|
testfile_4.write_text(testcase_yaml_4)
|
|
|
|
tmp_other_dir = tmp_test_root_dir / 'others'
|
|
tmp_other_dir.mkdir()
|
|
_ = tmp_other_dir / 'other.txt'
|
|
|
|
tmp_alt_test_root_dir = tmp_path / 'other_tests'
|
|
tmp_alt_test_root_dir.mkdir()
|
|
|
|
tmp_alt_good_test_dir = tmp_alt_test_root_dir / 'good_test'
|
|
tmp_alt_good_test_dir.mkdir()
|
|
testcase_yaml_3 = """\
|
|
tests:
|
|
dummy.alt.1:
|
|
build_on_all: true
|
|
dummy.alt.2:
|
|
build_on_all: true
|
|
"""
|
|
testfile_3 = tmp_alt_good_test_dir / 'testcase.yaml'
|
|
testfile_3.write_text(testcase_yaml_3)
|
|
|
|
env = mock.Mock(
|
|
test_roots=[tmp_test_root_dir],
|
|
options=mock.Mock(detailed_test_id=detailed_id),
|
|
alt_config_root=[tmp_alt_test_root_dir] if use_alt_root else []
|
|
)
|
|
|
|
testplan = TestPlan(env=env)
|
|
|
|
res = testplan.add_testsuites(testsuite_filter)
|
|
|
|
assert res == expected_suite_count
|
|
assert testplan.load_errors == expected_errors
|
|
|
|
|
|
def test_testplan_str():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.name = 'my name'
|
|
|
|
res = testplan.__str__()
|
|
|
|
assert res == 'my name'
|
|
|
|
|
|
TESTDATA_10 = [
|
|
('a platform', True),
|
|
('other platform', False),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'name, expect_found',
|
|
TESTDATA_10,
|
|
ids=['platform exists', 'no platform']
|
|
)
|
|
def test_testplan_get_platform(name, expect_found):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
p1 = mock.Mock()
|
|
p1.name = 'some platform'
|
|
p1.aliases = [p1.name]
|
|
p2 = mock.Mock()
|
|
p2.name = 'a platform'
|
|
p2.aliases = [p2.name]
|
|
testplan.platforms = [p1, p2]
|
|
|
|
res = testplan.get_platform(name)
|
|
|
|
if expect_found:
|
|
assert res.name == name
|
|
else:
|
|
assert res is None
|
|
|
|
|
|
TESTDATA_11 = [
|
|
(True, 'runnable'),
|
|
(False, 'buildable'),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'device_testing, expected_tfilter',
|
|
TESTDATA_11,
|
|
ids=['device testing', 'no device testing']
|
|
)
|
|
def test_testplan_load_from_file(caplog, device_testing, expected_tfilter):
|
|
def get_platform(name):
|
|
p = mock.Mock()
|
|
p.name = name
|
|
p.normalized_name = name
|
|
return p
|
|
|
|
ts1tc1 = mock.Mock()
|
|
ts1tc1.name = 'TS1.tc1'
|
|
ts1 = mock.Mock(testcases=[ts1tc1])
|
|
ts1.name = 'TestSuite 1'
|
|
ts1.toolchain = 'zephyr'
|
|
ts2 = mock.Mock(testcases=[])
|
|
ts2.name = 'TestSuite 2'
|
|
ts2.toolchain = 'zephyr'
|
|
ts3tc1 = mock.Mock()
|
|
ts3tc1.name = 'TS3.tc1'
|
|
ts3tc2 = mock.Mock()
|
|
ts3tc2.name = 'TS3.tc2'
|
|
ts3 = mock.Mock(testcases=[ts3tc1, ts3tc2])
|
|
ts3.name = 'TestSuite 3'
|
|
ts3.toolchain = 'zephyr'
|
|
ts4tc1 = mock.Mock()
|
|
ts4tc1.name = 'TS4.tc1'
|
|
ts4 = mock.Mock(testcases=[ts4tc1])
|
|
ts4.name = 'TestSuite 4'
|
|
ts4.toolchain = 'zephyr'
|
|
ts5 = mock.Mock(testcases=[])
|
|
ts5.name = 'TestSuite 5'
|
|
ts5.toolchain = 'zephyr'
|
|
|
|
testplan = TestPlan(env=mock.Mock(outdir=os.path.join('out', 'dir')))
|
|
testplan.options = mock.Mock(device_testing=device_testing, test_only=True, report_summary=None)
|
|
testplan.testsuites = {
|
|
'TestSuite 1': ts1,
|
|
'TestSuite 2': ts2,
|
|
'TestSuite 3': ts3,
|
|
'TestSuite 4': ts4,
|
|
'TestSuite 5': ts5
|
|
}
|
|
|
|
testplan.get_platform = mock.Mock(side_effect=get_platform)
|
|
|
|
testplan_data = """\
|
|
{
|
|
"testsuites": [
|
|
{
|
|
"name": "TestSuite 1",
|
|
"platform": "Platform 1",
|
|
"run_id": 1,
|
|
"execution_time": 60.00,
|
|
"used_ram": 4096,
|
|
"available_ram": 12278,
|
|
"used_rom": 1024,
|
|
"available_rom": 1047552,
|
|
"status": "passed",
|
|
"toolchain": "zephyr",
|
|
"reason": "OK",
|
|
"testcases": [
|
|
{
|
|
"identifier": "TS1.tc1",
|
|
"status": "passed",
|
|
"reason": "passed",
|
|
"execution_time": 60.00,
|
|
"log": ""
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "TestSuite 2",
|
|
"platform": "Platform 1",
|
|
"toolchain": "zephyr"
|
|
},
|
|
{
|
|
"name": "TestSuite 3",
|
|
"platform": "Platform 1",
|
|
"run_id": 1,
|
|
"execution_time": 360.00,
|
|
"used_ram": 4096,
|
|
"available_ram": 12278,
|
|
"used_rom": 1024,
|
|
"available_rom": 1047552,
|
|
"status": "error",
|
|
"toolchain": "zephyr",
|
|
"reason": "File Not Found Error",
|
|
"testcases": [
|
|
{
|
|
"identifier": "TS3.tc1",
|
|
"status": "error",
|
|
"reason": "File Not Found Error.",
|
|
"execution_time": 360.00,
|
|
"log": "[ERROR]: File 'dummy.yaml' not found!\\nClosing..."
|
|
},
|
|
{
|
|
"identifier": "TS3.tc2"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "TestSuite 4",
|
|
"platform": "Platform 1",
|
|
"execution_time": 360.00,
|
|
"used_ram": 4096,
|
|
"available_ram": 12278,
|
|
"used_rom": 1024,
|
|
"available_rom": 1047552,
|
|
"status": "skipped",
|
|
"toolchain": "zephyr",
|
|
"reason": "Not in requested test list.",
|
|
"testcases": [
|
|
{
|
|
"identifier": "TS4.tc1",
|
|
"status": "skipped",
|
|
"reason": "Not in requested test list.",
|
|
"execution_time": 360.00,
|
|
"log": "[INFO] Parsing..."
|
|
},
|
|
{
|
|
"identifier": "TS3.tc2"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "TestSuite 5",
|
|
"platform": "Platform 2",
|
|
"toolchain": "zephyr"
|
|
}
|
|
]
|
|
}
|
|
"""
|
|
|
|
filter_platform = ['Platform 1']
|
|
|
|
check_runnable_mock = mock.Mock(return_value=True)
|
|
|
|
with mock.patch('builtins.open', mock.mock_open(read_data=testplan_data)), \
|
|
mock.patch('twisterlib.testinstance.TestInstance.check_runnable', check_runnable_mock), \
|
|
mock.patch('twisterlib.testinstance.TestInstance.create_overlay', mock.Mock()):
|
|
testplan.load_from_file('dummy.yaml', filter_platform)
|
|
|
|
expected_instances = {
|
|
'Platform 1/zephyr/TestSuite 1': {
|
|
'metrics': {
|
|
'handler_time': 60.0,
|
|
'used_ram': 4096,
|
|
'used_rom': 1024,
|
|
'available_ram': 12278,
|
|
'available_rom': 1047552
|
|
},
|
|
'retries': 0,
|
|
'toolchain': 'zephyr',
|
|
'testcases': {
|
|
'TS1.tc1': {
|
|
'status': TwisterStatus.PASS,
|
|
'reason': 'passed',
|
|
'duration': 60.0,
|
|
'output': ''
|
|
}
|
|
}
|
|
},
|
|
'Platform 1/zephyr/TestSuite 2': {
|
|
'metrics': {
|
|
'handler_time': 0,
|
|
'used_ram': 0,
|
|
'used_rom': 0,
|
|
'available_ram': 0,
|
|
'available_rom': 0
|
|
},
|
|
'retries': 0,
|
|
'toolchain': 'zephyr',
|
|
'testcases': []
|
|
},
|
|
'Platform 1/zephyr/TestSuite 3': {
|
|
'metrics': {
|
|
'handler_time': 360.0,
|
|
'used_ram': 4096,
|
|
'used_rom': 1024,
|
|
'available_ram': 12278,
|
|
'available_rom': 1047552
|
|
},
|
|
'retries': 1,
|
|
'toolchain': 'zephyr',
|
|
'testcases': {
|
|
'TS3.tc1': {
|
|
'status': TwisterStatus.ERROR,
|
|
'reason': None,
|
|
'duration': 360.0,
|
|
'output': '[ERROR]: File \'dummy.yaml\' not found!\nClosing...'
|
|
},
|
|
'TS3.tc2': {
|
|
'status': TwisterStatus.NONE,
|
|
'reason': None,
|
|
'duration': 0,
|
|
'output': ''
|
|
}
|
|
}
|
|
},
|
|
'Platform 1/zephyr/TestSuite 4': {
|
|
'metrics': {
|
|
'handler_time': 360.0,
|
|
'used_ram': 4096,
|
|
'used_rom': 1024,
|
|
'available_ram': 12278,
|
|
'available_rom': 1047552
|
|
},
|
|
'retries': 0,
|
|
'toolchain': 'zephyr',
|
|
'testcases': {
|
|
'TS4.tc1': {
|
|
'status': TwisterStatus.SKIP,
|
|
'reason': 'Not in requested test list.',
|
|
'duration': 360.0,
|
|
'output': '[INFO] Parsing...'
|
|
}
|
|
}
|
|
},
|
|
}
|
|
|
|
for n, i in testplan.instances.items():
|
|
assert expected_instances[n]['metrics'] == i.metrics
|
|
assert expected_instances[n]['retries'] == i.retries
|
|
for t in i.testcases:
|
|
assert expected_instances[n]['testcases'][str(t)]['status'] == t.status
|
|
assert expected_instances[n]['testcases'][str(t)]['reason'] == t.reason
|
|
assert expected_instances[n]['testcases'][str(t)]['duration'] == t.duration
|
|
assert expected_instances[n]['testcases'][str(t)]['output'] == t.output
|
|
|
|
check_runnable_mock.assert_called_with(mock.ANY, mock.ANY)
|
|
|
|
expected_logs = [
|
|
'loading TestSuite 1...',
|
|
'loading TestSuite 2...',
|
|
'loading TestSuite 3...',
|
|
'loading TestSuite 4...',
|
|
]
|
|
assert all([log in caplog.text for log in expected_logs])
|
|
|
|
|
|
def test_testplan_add_instances():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
instance1 = mock.Mock()
|
|
instance1.name = 'instance 1'
|
|
instance2 = mock.Mock()
|
|
instance2.name = 'instance 2'
|
|
instance_list = [instance1, instance2]
|
|
|
|
testplan.add_instances(instance_list)
|
|
|
|
assert testplan.instances == {
|
|
'instance 1': instance1,
|
|
'instance 2': instance2,
|
|
}
|
|
|
|
|
|
def test_testplan_get_testcase():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.testsuites = {
|
|
'test1.suite0': mock.Mock(testcases=[mock.Mock(), mock.Mock()]),
|
|
'test1.suite1': mock.Mock(testcases=[mock.Mock(), mock.Mock()]),
|
|
'test1.suite2': mock.Mock(testcases=[mock.Mock(), mock.Mock()]),
|
|
'test1.suite3': mock.Mock(testcases=[])
|
|
}
|
|
|
|
testplan.testsuites['test1.suite0'].testcases[0].name = 'test1.suite0.case0'
|
|
testplan.testsuites['test1.suite0'].testcases[1].name = 'test1.suite0.case1'
|
|
#
|
|
testplan.testsuites['test1.suite1'].testcases[0].name = 'test1.suite1.case0'
|
|
testplan.testsuites['test1.suite1'].testcases[1].name = 'test1.suite1.case0' # in suite duplicate
|
|
#
|
|
testplan.testsuites['test1.suite2'].testcases[0].name = 'test1.suite2.case0'
|
|
testplan.testsuites['test1.suite2'].testcases[1].name = 'test1.suite1.case0' # out suite duplicate
|
|
|
|
id = 'test1.suite1.case0'
|
|
|
|
res = testplan.get_testcase(id)
|
|
|
|
assert len(res) == 3
|
|
assert testplan.testsuites['test1.suite1'] in res
|
|
assert testplan.testsuites['test1.suite2'] in res
|
|
|
|
|
|
def test_testplan_verify_platforms_existence(caplog):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.platform_names = ['a platform', 'other platform']
|
|
|
|
platform_names = ['other platform', 'some platform']
|
|
log_info = 'PLATFORM ERROR'
|
|
|
|
with pytest.raises(SystemExit) as se:
|
|
testplan.verify_platforms_existence(platform_names, log_info)
|
|
|
|
assert str(se.value) == '2'
|
|
assert 'PLATFORM ERROR - unrecognized platform - some platform'
|
|
|
|
|
|
TESTDATA_12 = [
|
|
(True),
|
|
(False)
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'exists',
|
|
TESTDATA_12,
|
|
ids=['links dir exists', 'links dir does not exist']
|
|
)
|
|
def test_testplan_create_build_dir_links(exists):
|
|
outdir = os.path.join('out', 'dir')
|
|
instances_linked = []
|
|
|
|
def mock_link(links_dir_path, instance):
|
|
assert links_dir_path == os.path.join(outdir, 'twister_links')
|
|
instances_linked.append(instance)
|
|
|
|
instances = {
|
|
'inst0': mock.Mock(status=TwisterStatus.PASS),
|
|
'inst1': mock.Mock(status=TwisterStatus.SKIP),
|
|
'inst2': mock.Mock(status=TwisterStatus.ERROR),
|
|
}
|
|
expected_instances = [instances['inst0'], instances['inst2']]
|
|
|
|
testplan = TestPlan(env=mock.Mock(outdir=outdir))
|
|
testplan._create_build_dir_link = mock.Mock(side_effect=mock_link)
|
|
testplan.instances = instances
|
|
|
|
with mock.patch('os.path.exists', return_value=exists), \
|
|
mock.patch('os.mkdir', mock.Mock()) as mkdir_mock:
|
|
testplan.create_build_dir_links()
|
|
|
|
if not exists:
|
|
mkdir_mock.assert_called_once()
|
|
|
|
assert expected_instances == instances_linked
|
|
|
|
|
|
TESTDATA_13 = [
|
|
('nt'),
|
|
('Linux')
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'os_name',
|
|
TESTDATA_13,
|
|
)
|
|
def test_testplan_create_build_dir_link(os_name):
|
|
def mock_makedirs(path, exist_ok=False):
|
|
assert exist_ok
|
|
assert path == instance_build_dir
|
|
|
|
def mock_symlink(source, target):
|
|
assert source == instance_build_dir
|
|
assert target == os.path.join('links', 'path', 'test_0')
|
|
|
|
def mock_call(cmd, shell=False):
|
|
assert shell
|
|
assert cmd == ['mklink', '/J', os.path.join('links', 'path', 'test_0'),
|
|
instance_build_dir]
|
|
|
|
def mock_join(*paths):
|
|
slash = "\\" if os.name == 'nt' else "/"
|
|
return slash.join(paths)
|
|
|
|
with mock.patch('os.name', os_name), \
|
|
mock.patch('os.symlink', side_effect=mock_symlink), \
|
|
mock.patch('os.makedirs', side_effect=mock_makedirs), \
|
|
mock.patch('subprocess.call', side_effect=mock_call), \
|
|
mock.patch('os.path.join', side_effect=mock_join):
|
|
|
|
testplan = TestPlan(env=mock.Mock())
|
|
links_dir_path = os.path.join('links', 'path')
|
|
instance_build_dir = os.path.join('some', 'far', 'off', 'build', 'dir')
|
|
instance = mock.Mock(build_dir=instance_build_dir)
|
|
testplan._create_build_dir_link(links_dir_path, instance)
|
|
|
|
assert instance.build_dir == os.path.join('links', 'path', 'test_0')
|
|
assert testplan.link_dir_counter == 1
|
|
|
|
|
|
TESTDATA_14 = [
|
|
('bad platform', 'dummy reason', [],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'quarantined', [],
|
|
TwisterStatus.ERROR, 'quarantined but is one of the integration platforms'),
|
|
('good platform', 'dummy reason', [{'type': 'command line filter'}],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'dummy reason', [{'type': 'Skip filter'}],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'dummy reason', [{'type': 'platform key filter'}],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'dummy reason', [{'type': 'Toolchain filter'}],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'dummy reason', [{'type': 'Module filter'}],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'dummy reason', [{'type': 'testsuite filter'}],
|
|
TwisterStatus.ERROR, 'dummy reason but is one of the integration platforms'),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'platform_name, reason, filters,' \
|
|
' expected_status, expected_reason',
|
|
TESTDATA_14,
|
|
ids=['wrong platform', 'quarantined', 'command line filtered',
|
|
'skip filtered', 'platform key filtered', 'toolchain filtered',
|
|
'module filtered', 'skip to error change']
|
|
)
|
|
def test_change_skip_to_error_if_integration(
|
|
platform_name,
|
|
reason,
|
|
filters,
|
|
expected_status,
|
|
expected_reason
|
|
):
|
|
options = mock.Mock()
|
|
platform = mock.Mock()
|
|
platform.name = platform_name
|
|
testsuite = mock.Mock(integration_platforms=['good platform', 'a platform'])
|
|
instance = mock.Mock(
|
|
testsuite=testsuite,
|
|
platform=platform,
|
|
filters=filters,
|
|
status='dummy status',
|
|
reason=reason
|
|
)
|
|
|
|
change_skip_to_error_if_integration(options, instance)
|
|
|
|
assert instance.status == expected_status
|
|
assert instance.reason == expected_reason
|