49595c7309
Skipped status fits quarantined items better than filtered. Filtered tests are by default removed from reports, which shouldn't be the case for quarantined tests. Adjust tests unit and blackbox tests accordingly. Signed-off-by: Maciej Perkowski <maciej.perkowski@nordicsemi.no>
129 lines
5.0 KiB
Python
129 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (c) 2024 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
"""
|
|
Blackbox tests for twister's command line functions related to the quarantine.
|
|
"""
|
|
|
|
import importlib
|
|
import mock
|
|
import os
|
|
import pytest
|
|
import re
|
|
import sys
|
|
import json
|
|
|
|
# pylint: disable=duplicate-code
|
|
# pylint: disable=no-name-in-module
|
|
from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
|
|
from twisterlib.testplan import TestPlan
|
|
|
|
|
|
class TestQuarantine:
|
|
@classmethod
|
|
def setup_class(cls):
|
|
apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
|
|
cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
|
|
cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
|
|
cls.twister_module = importlib.util.module_from_spec(cls.spec)
|
|
|
|
@classmethod
|
|
def teardown_class(cls):
|
|
pass
|
|
|
|
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
|
|
def test_quarantine_verify(self, out_path):
|
|
test_platforms = ['qemu_x86', 'intel_adl_crb']
|
|
path = os.path.join(TEST_DATA, 'tests', 'dummy')
|
|
quarantine_path = os.path.join(TEST_DATA, 'twister-quarantine-list.yml')
|
|
args = ['-i', '--outdir', out_path, '-T', path, '-y'] + \
|
|
['--quarantine-verify'] + \
|
|
['--quarantine-list', quarantine_path] + \
|
|
[val for pair in zip(
|
|
['-p'] * len(test_platforms), test_platforms
|
|
) for val in pair]
|
|
|
|
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
|
|
pytest.raises(SystemExit) as sys_exit:
|
|
self.loader.exec_module(self.twister_module)
|
|
|
|
with open(os.path.join(out_path, 'testplan.json')) as f:
|
|
j = json.load(f)
|
|
|
|
# Quarantine-verify "swaps" statuses. The ones that are in quarantine list
|
|
# should no longer be quarantined, and the ones that are not in the list
|
|
# should be quarantined. Remove "quarantined" tests from "verify" testplan
|
|
# to count what should be verified.
|
|
filtered_j = [
|
|
(ts['platform'], ts['name']) \
|
|
for ts in j['testsuites'] \
|
|
if ts['status'] != "skipped"
|
|
]
|
|
|
|
assert str(sys_exit.value) == '0'
|
|
|
|
assert len(filtered_j) == 2
|
|
|
|
@pytest.mark.parametrize(
|
|
'test_path, test_platforms, quarantine_directory',
|
|
[
|
|
(
|
|
os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
|
|
['qemu_x86', 'qemu_x86_64', 'intel_adl_crb'],
|
|
os.path.join(TEST_DATA, 'twister-quarantine-list.yml'),
|
|
),
|
|
],
|
|
ids=[
|
|
'quarantine',
|
|
],
|
|
)
|
|
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
|
|
def test_quarantine_list(self, capfd, out_path, test_path, test_platforms, quarantine_directory):
|
|
args = ['--outdir', out_path, '-T', test_path] +\
|
|
['--quarantine-list', quarantine_directory] + \
|
|
['-vv', '-ll', 'DEBUG'] + \
|
|
[val for pair in zip(
|
|
['-p'] * len(test_platforms), test_platforms
|
|
) for val in pair]
|
|
|
|
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
|
|
pytest.raises(SystemExit) as sys_exit:
|
|
self.loader.exec_module(self.twister_module)
|
|
|
|
out, err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stderr.write(err)
|
|
|
|
board1_match1 = re.search('agnostic/group2/dummy.agnostic.group2 SKIPPED: Quarantine: test '
|
|
'intel_adl_crb', err)
|
|
board1_match2 = re.search(
|
|
'agnostic/group1/subgroup2/dummy.agnostic.group1.subgroup2 SKIPPED: Quarantine: test '
|
|
'intel_adl_crb',
|
|
err)
|
|
qemu_64_match = re.search(
|
|
'agnostic/group1/subgroup2/dummy.agnostic.group1.subgroup2 SKIPPED: Quarantine: test '
|
|
'qemu_x86_64',
|
|
err)
|
|
all_platforms_match = re.search(
|
|
'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 SKIPPED: Quarantine: test '
|
|
'all platforms',
|
|
err)
|
|
all_platforms_match2 = re.search(
|
|
'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 SKIPPED: Quarantine: test '
|
|
'all platforms',
|
|
err)
|
|
all_platforms_match3 = re.search(
|
|
'agnostic/group1/subgroup1/dummy.agnostic.group1.subgroup1 SKIPPED: Quarantine: test '
|
|
'all platforms',
|
|
err)
|
|
|
|
assert board1_match1 and board1_match2, 'platform quarantine not working properly'
|
|
assert qemu_64_match, 'platform quarantine on scenario not working properly'
|
|
assert all_platforms_match and all_platforms_match2 and all_platforms_match3, 'scenario ' \
|
|
'quarantine' \
|
|
' not work ' \
|
|
'properly'
|
|
|
|
assert str(sys_exit.value) == '0'
|