14d88f8425
Add support test levels and the ability to assign a specific test to one or more levels. Using command line options of twister it is then possible to select a level and just execute the tests included in this level. Additionally, a test configuration allows definiing level dependencies and additional inclusion of tests into a specific level if the test itself does not have this information already. In the configuration file you can include complete components using regular expressions and you can specify which test level to import from the same file, making management of levels easier. To help with testing outside of upstream CI infrastructure, additional options are available in the configuration file, which can be hosted locally. As of now, those options are available: - Ability to ignore default platforms as defined in board definitions (Those are mostly emulation platforms used to run tests in upstream CI) - Option to specify your own list of default platforms overriding what upstream defines. - Ability to override build_onl_all options used in some testscases. This will treat tests or sample as any other just build for default platforms you specify in the configuation file or on the command line. Signed-off-by: Anas Nashif <anas.nashif@intel.com>
339 lines
15 KiB
Python
339 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (c) 2020 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
'''
|
|
This test file contains testsuites for Testsuite class of twister
|
|
'''
|
|
import sys
|
|
import os
|
|
import pytest
|
|
|
|
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
|
|
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
|
|
|
from twisterlib.testplan import TestPlan
|
|
from twisterlib.testinstance import TestInstance
|
|
from twisterlib.testsuite import TestSuite
|
|
from twisterlib.platform import Platform
|
|
from twisterlib.quarantine import Quarantine
|
|
|
|
|
|
def test_testplan_add_testsuites(class_testplan):
|
|
""" Testing add_testcase function of Testsuite class in twister """
|
|
# Test 1: Check the list of testsuites after calling add testsuites function is as expected
|
|
class_testplan.SAMPLE_FILENAME = 'test_sample_app.yaml'
|
|
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
|
|
class_testplan.add_testsuites()
|
|
|
|
tests_rel_dir = 'scripts/tests/twister/test_data/testsuites/tests/'
|
|
expected_testsuites = ['test_b.check_1',
|
|
'test_b.check_2',
|
|
'test_c.check_1',
|
|
'test_c.check_2',
|
|
'test_a.check_1',
|
|
'test_a.check_2',
|
|
'test_d.check_1',
|
|
'sample_test.app',
|
|
'test_config.main']
|
|
testsuite_list = []
|
|
for key in sorted(class_testplan.testsuites.keys()):
|
|
testsuite_list.append(os.path.basename(os.path.normpath(key)))
|
|
assert sorted(testsuite_list) == sorted(expected_testsuites)
|
|
|
|
# Test 2 : Assert Testcase name is expected & all testsuites values are testcase class objects
|
|
suite = class_testplan.testsuites.get(tests_rel_dir + 'test_a/test_a.check_1')
|
|
assert suite.name == tests_rel_dir + 'test_a/test_a.check_1'
|
|
assert all(isinstance(n, TestSuite) for n in class_testplan.testsuites.values())
|
|
|
|
@pytest.mark.parametrize("board_root_dir", [("board_config_file_not_exist"), ("board_config")])
|
|
def test_add_configurations(test_data, class_env, board_root_dir):
|
|
""" Testing add_configurations function of TestPlan class in Twister
|
|
Test : Asserting on default platforms list
|
|
"""
|
|
class_env.board_roots = [os.path.abspath(test_data + board_root_dir)]
|
|
plan = TestPlan(class_env)
|
|
plan.parse_configuration(config_file=class_env.test_config)
|
|
if board_root_dir == "board_config":
|
|
plan.add_configurations()
|
|
assert sorted(plan.default_platforms) == sorted(['demo_board_1', 'demo_board_3'])
|
|
elif board_root_dir == "board_config_file_not_exist":
|
|
plan.add_configurations()
|
|
assert sorted(plan.default_platforms) != sorted(['demo_board_1'])
|
|
|
|
|
|
def test_get_all_testsuites(class_testplan, all_testsuites_dict):
|
|
""" Testing get_all_testsuites function of TestPlan class in Twister """
|
|
plan = class_testplan
|
|
plan.testsuites = all_testsuites_dict
|
|
expected_tests = ['sample_test.app', 'test_a.check_1.1a',
|
|
'test_a.check_1.1c',
|
|
'test_a.check_1.2a', 'test_a.check_1.2b',
|
|
'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a',
|
|
'test_a.check_1.unit_1b', 'test_a.check_2.1a',
|
|
'test_a.check_2.1c', 'test_a.check_2.2a',
|
|
'test_a.check_2.2b', 'test_a.check_2.Unit_1c',
|
|
'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b',
|
|
'test_b.check_1', 'test_b.check_2', 'test_c.check_1',
|
|
'test_c.check_2', 'test_d.check_1.unit_1a',
|
|
'test_d.check_1.unit_1b', 'test_config.main']
|
|
assert sorted(plan.get_all_tests()) == sorted(expected_tests)
|
|
|
|
def test_get_platforms(class_testplan, platforms_list):
|
|
""" Testing get_platforms function of TestPlan class in Twister """
|
|
plan = class_testplan
|
|
plan.platforms = platforms_list
|
|
platform = plan.get_platform("demo_board_1")
|
|
assert isinstance(platform, Platform)
|
|
assert platform.name == "demo_board_1"
|
|
|
|
TESTDATA_PART1 = [
|
|
("toolchain_allow", ['gcc'], None, None, "Not in testsuite toolchain allow list"),
|
|
("platform_allow", ['demo_board_1'], None, None, "Not in testsuite platform allow list"),
|
|
("toolchain_exclude", ['zephyr'], None, None, "In test case toolchain exclude"),
|
|
("platform_exclude", ['demo_board_2'], None, None, "In test case platform exclude"),
|
|
("arch_exclude", ['x86_demo'], None, None, "In test case arch exclude"),
|
|
("arch_allow", ['arm'], None, None, "Not in test case arch allow list"),
|
|
("skip", True, None, None, "Skip filter"),
|
|
("tags", set(['sensor', 'bluetooth']), "ignore_tags", ['bluetooth'], "Excluded tags per platform (exclude_tags)"),
|
|
("min_flash", "2024", "flash", "1024", "Not enough FLASH"),
|
|
("min_ram", "500", "ram", "256", "Not enough RAM"),
|
|
("None", "None", "env", ['BSIM_OUT_PATH', 'demo_env'], "Environment (BSIM_OUT_PATH, demo_env) not satisfied"),
|
|
("build_on_all", True, None, None, "Platform is excluded on command line."),
|
|
(None, None, "supported_toolchains", ['gcc'], "Not supported by the toolchain"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("tc_attribute, tc_value, plat_attribute, plat_value, expected_discards",
|
|
TESTDATA_PART1)
|
|
def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list,
|
|
tc_attribute, tc_value, plat_attribute, plat_value, expected_discards):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 1: Response of apply_filters function have
|
|
appropriate values according to the filters
|
|
"""
|
|
plan = class_testplan
|
|
if tc_attribute is None and plat_attribute is None:
|
|
plan.apply_filters()
|
|
|
|
plan.platforms = platforms_list
|
|
plan.platform_names = [p.name for p in platforms_list]
|
|
plan.testsuites = all_testsuites_dict
|
|
for plat in plan.platforms:
|
|
if plat_attribute == "ignore_tags":
|
|
plat.ignore_tags = plat_value
|
|
if plat_attribute == "flash":
|
|
plat.flash = plat_value
|
|
if plat_attribute == "ram":
|
|
plat.ram = plat_value
|
|
if plat_attribute == "env":
|
|
plat.env = plat_value
|
|
plat.env_satisfied = False
|
|
if plat_attribute == "supported_toolchains":
|
|
plat.supported_toolchains = plat_value
|
|
for _, testcase in plan.testsuites.items():
|
|
if tc_attribute == "toolchain_allow":
|
|
testcase.toolchain_allow = tc_value
|
|
if tc_attribute == "platform_allow":
|
|
testcase.platform_allow = tc_value
|
|
if tc_attribute == "toolchain_exclude":
|
|
testcase.toolchain_exclude = tc_value
|
|
if tc_attribute == "platform_exclude":
|
|
testcase.platform_exclude = tc_value
|
|
if tc_attribute == "arch_exclude":
|
|
testcase.arch_exclude = tc_value
|
|
if tc_attribute == "arch_allow":
|
|
testcase.arch_allow = tc_value
|
|
if tc_attribute == "skip":
|
|
testcase.skip = tc_value
|
|
if tc_attribute == "tags":
|
|
testcase.tags = tc_value
|
|
if tc_attribute == "min_flash":
|
|
testcase.min_flash = tc_value
|
|
if tc_attribute == "min_ram":
|
|
testcase.min_ram = tc_value
|
|
|
|
if tc_attribute == "build_on_all":
|
|
for _, testcase in plan.testsuites.items():
|
|
testcase.build_on_all = tc_value
|
|
plan.apply_filters(exclude_platform=['demo_board_1'])
|
|
elif plat_attribute == "supported_toolchains":
|
|
plan.apply_filters(force_toolchain=False,
|
|
exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2'])
|
|
elif tc_attribute is None and plat_attribute is None:
|
|
plan.apply_filters()
|
|
else:
|
|
plan.apply_filters(exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2'])
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", plan.instances.values()))
|
|
for d in filtered_instances:
|
|
assert d.reason == expected_discards
|
|
|
|
TESTDATA_PART2 = [
|
|
("runnable", "True", "Not runnable on device"),
|
|
("exclude_tag", ['test_a'], "Command line testsuite exclude filter"),
|
|
("run_individual_tests", ['scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1'], "TestSuite name filter"),
|
|
("arch", ['arm_test'], "Command line testsuite arch filter"),
|
|
("tag", ['test_d'], "Command line testsuite tag filter")
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("extra_filter, extra_filter_value, expected_discards", TESTDATA_PART2)
|
|
def test_apply_filters_part2(class_testplan, all_testsuites_dict,
|
|
platforms_list, extra_filter, extra_filter_value, expected_discards):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 2 : Response of apply_filters function (discard dictionary) have
|
|
appropriate values according to the filters
|
|
"""
|
|
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
kwargs = {
|
|
extra_filter : extra_filter_value,
|
|
"exclude_platform" : [
|
|
'demo_board_1'
|
|
],
|
|
"platform" : [
|
|
'demo_board_2'
|
|
]
|
|
}
|
|
class_testplan.apply_filters(**kwargs)
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
|
|
for d in filtered_instances:
|
|
assert d.reason == expected_discards
|
|
|
|
|
|
TESTDATA_PART3 = [
|
|
(20, 20, -1, 0),
|
|
(-2, -1, 10, 20),
|
|
(0, 0, 0, 0)
|
|
]
|
|
|
|
@pytest.mark.parametrize("tc_min_flash, plat_flash, tc_min_ram, plat_ram",
|
|
TESTDATA_PART3)
|
|
def test_apply_filters_part3(class_testplan, all_testsuites_dict, platforms_list,
|
|
tc_min_flash, plat_flash, tc_min_ram, plat_ram):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 3 : Testing edge cases for ram and flash values of platforms & testsuites
|
|
"""
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
|
|
for plat in class_testplan.platforms:
|
|
plat.flash = plat_flash
|
|
plat.ram = plat_ram
|
|
for _, testcase in class_testplan.testsuites.items():
|
|
testcase.min_ram = tc_min_ram
|
|
testcase.min_flash = tc_min_flash
|
|
class_testplan.apply_filters(exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2'])
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
|
|
assert not filtered_instances
|
|
|
|
def test_add_instances(test_data, class_env, all_testsuites_dict, platforms_list):
|
|
""" Testing add_instances() function of TestPlan class in Twister
|
|
Test 1: instances dictionary keys have expected values (Platform Name + Testcase Name)
|
|
Test 2: Values of 'instances' dictionary in Testsuite class are an
|
|
instance of 'TestInstance' class
|
|
Test 3: Values of 'instances' dictionary have expected values.
|
|
"""
|
|
class_env.outdir = test_data
|
|
plan = TestPlan(class_env)
|
|
plan.platforms = platforms_list
|
|
platform = plan.get_platform("demo_board_2")
|
|
instance_list = []
|
|
for _, testcase in all_testsuites_dict.items():
|
|
instance = TestInstance(testcase, platform, class_env.outdir)
|
|
instance_list.append(instance)
|
|
plan.add_instances(instance_list)
|
|
assert list(plan.instances.keys()) == \
|
|
[platform.name + '/' + s for s in list(all_testsuites_dict.keys())]
|
|
assert all(isinstance(n, TestInstance) for n in list(plan.instances.values()))
|
|
assert list(plan.instances.values()) == instance_list
|
|
|
|
|
|
QUARANTINE_BASIC = {
|
|
'demo_board_1/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3'
|
|
}
|
|
|
|
QUARANTINE_WITH_REGEXP = {
|
|
'demo_board_2/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'a2 and c2 on x86',
|
|
'demo_board_1/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_2/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_2/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'a2 and c2 on x86'
|
|
}
|
|
|
|
QUARANTINE_PLATFORM = {
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_config/test_config.main' : 'all on board_3'
|
|
}
|
|
|
|
QUARANTINE_MULTIFILES = {
|
|
**QUARANTINE_BASIC,
|
|
**QUARANTINE_WITH_REGEXP
|
|
}
|
|
|
|
@pytest.mark.parametrize(
|
|
("quarantine_files, quarantine_verify, expected_val"),
|
|
[
|
|
(['basic.yaml'], False, QUARANTINE_BASIC),
|
|
(['with_regexp.yaml'], False, QUARANTINE_WITH_REGEXP),
|
|
(['with_regexp.yaml'], True, QUARANTINE_WITH_REGEXP),
|
|
(['platform.yaml'], False, QUARANTINE_PLATFORM),
|
|
(['basic.yaml', 'with_regexp.yaml'], False, QUARANTINE_MULTIFILES),
|
|
(['empty.yaml'], False, {})
|
|
],
|
|
ids=[
|
|
'basic',
|
|
'with_regexp',
|
|
'quarantine_verify',
|
|
'platform',
|
|
'multifiles',
|
|
'empty'
|
|
])
|
|
def test_quarantine(class_testplan, platforms_list, test_data,
|
|
quarantine_files, quarantine_verify, expected_val):
|
|
""" Testing quarantine feature in Twister
|
|
"""
|
|
class_testplan.options.all = True
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
|
|
class_testplan.add_testsuites()
|
|
|
|
quarantine_list = [
|
|
os.path.join(test_data, 'quarantines', quarantine_file) for quarantine_file in quarantine_files
|
|
]
|
|
class_testplan.quarantine = Quarantine(quarantine_list)
|
|
class_testplan.options.quarantine_verify = quarantine_verify
|
|
class_testplan.apply_filters()
|
|
|
|
for testname, instance in class_testplan.instances.items():
|
|
if quarantine_verify:
|
|
if testname in expected_val:
|
|
assert not instance.status
|
|
else:
|
|
assert instance.status == 'filtered'
|
|
assert instance.reason == "Not under quarantine"
|
|
else:
|
|
print(testname)
|
|
if testname in expected_val:
|
|
assert instance.status == 'filtered'
|
|
assert instance.reason == "Quarantine: " + expected_val[testname]
|
|
else:
|
|
assert not instance.status
|