da1066fa4b
More complex platforms require sysbuild to use always, even for such "simple" samples like hello_world. Such platforms can have `sysbuild: true` entry in their board_name.yaml used by twister. Using such entry will tell twister, that sysbuild must always be used on a given platform. Twister is aligned to have information about need of sysbuild at instance (platform + suite) level (was only at suite level before). Instance.sysbuild is true whenever a test suite or a platform requires sysbuild. Twister pytest unit tests are aligned with changes. Signed-off-by: Maciej Perkowski <Maciej.Perkowski@nordicsemi.no>
296 lines
9.3 KiB
Python
296 lines
9.3 KiB
Python
# Copyright (c) 2023 Nordic Semiconductor ASA
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
import textwrap
|
|
|
|
from unittest import mock
|
|
from pathlib import Path
|
|
|
|
from twisterlib.harness import Pytest
|
|
from twisterlib.testsuite import TestSuite
|
|
from twisterlib.testinstance import TestInstance
|
|
from twisterlib.platform import Platform
|
|
|
|
|
|
@pytest.fixture
|
|
def testinstance() -> TestInstance:
|
|
testsuite = TestSuite('.', 'samples/hello', 'unit.test')
|
|
testsuite.harness_config = {}
|
|
testsuite.ignore_faults = False
|
|
testsuite.sysbuild = False
|
|
platform = Platform()
|
|
|
|
testinstance = TestInstance(testsuite, platform, 'outdir')
|
|
testinstance.handler = mock.Mock()
|
|
testinstance.handler.options = mock.Mock()
|
|
testinstance.handler.options.verbose = 1
|
|
testinstance.handler.options.fixture = ['fixture1:option1', 'fixture2']
|
|
testinstance.handler.options.pytest_args = None
|
|
testinstance.handler.type_str = 'native'
|
|
return testinstance
|
|
|
|
|
|
@pytest.mark.parametrize('device_type', ['native', 'qemu'])
|
|
def test_pytest_command(testinstance: TestInstance, device_type):
|
|
pytest_harness = Pytest()
|
|
pytest_harness.configure(testinstance)
|
|
|
|
testinstance.handler.type_str = device_type
|
|
ref_command = [
|
|
'pytest',
|
|
'samples/hello/pytest',
|
|
f'--build-dir={testinstance.build_dir}',
|
|
f'--junit-xml={testinstance.build_dir}/report.xml',
|
|
f'--device-type={device_type}',
|
|
'--twister-fixture=fixture1:option1',
|
|
'--twister-fixture=fixture2'
|
|
]
|
|
|
|
command = pytest_harness.generate_command()
|
|
for c in ref_command:
|
|
assert c in command
|
|
|
|
|
|
def test_pytest_command_dut_scope(testinstance: TestInstance):
|
|
pytest_harness = Pytest()
|
|
dut_scope = 'session'
|
|
testinstance.testsuite.harness_config['pytest_dut_scope'] = dut_scope
|
|
pytest_harness.configure(testinstance)
|
|
command = pytest_harness.generate_command()
|
|
assert f'--dut-scope={dut_scope}' in command
|
|
|
|
|
|
def test_pytest_command_extra_args(testinstance: TestInstance):
|
|
pytest_harness = Pytest()
|
|
pytest_args = ['-k test1', '-m mark1']
|
|
testinstance.testsuite.harness_config['pytest_args'] = pytest_args
|
|
pytest_harness.configure(testinstance)
|
|
command = pytest_harness.generate_command()
|
|
for c in pytest_args:
|
|
assert c in command
|
|
|
|
|
|
def test_pytest_command_extra_args_in_options(testinstance: TestInstance):
|
|
pytest_harness = Pytest()
|
|
pytest_args_from_yaml = '-k test_from_yaml'
|
|
pytest_args_from_cmd = ['-k', 'test_from_cmd']
|
|
testinstance.testsuite.harness_config['pytest_args'] = [pytest_args_from_yaml]
|
|
testinstance.handler.options.pytest_args = pytest_args_from_cmd
|
|
pytest_harness.configure(testinstance)
|
|
command = pytest_harness.generate_command()
|
|
assert pytest_args_from_cmd[0] in command
|
|
assert pytest_args_from_cmd[1] in command
|
|
assert pytest_args_from_yaml not in command
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
('pytest_root', 'expected'),
|
|
[
|
|
(
|
|
['pytest/test_shell_help.py'],
|
|
['samples/hello/pytest/test_shell_help.py']
|
|
),
|
|
(
|
|
['pytest/test_shell_help.py', 'pytest/test_shell_version.py', 'test_dir'],
|
|
['samples/hello/pytest/test_shell_help.py',
|
|
'samples/hello/pytest/test_shell_version.py',
|
|
'samples/hello/test_dir']
|
|
),
|
|
(
|
|
['../shell/pytest/test_shell.py'],
|
|
['samples/shell/pytest/test_shell.py']
|
|
),
|
|
(
|
|
['/tmp/test_temp.py'],
|
|
['/tmp/test_temp.py']
|
|
),
|
|
(
|
|
['~/tmp/test_temp.py'],
|
|
['/home/joe/tmp/test_temp.py']
|
|
),
|
|
(
|
|
['$ZEPHYR_BASE/samples/subsys/testsuite/pytest/shell/pytest'],
|
|
['/zephyr_base/samples/subsys/testsuite/pytest/shell/pytest']
|
|
),
|
|
(
|
|
['pytest/test_shell_help.py::test_A', 'pytest/test_shell_help.py::test_B'],
|
|
['samples/hello/pytest/test_shell_help.py::test_A',
|
|
'samples/hello/pytest/test_shell_help.py::test_B']
|
|
),
|
|
(
|
|
['pytest/test_shell_help.py::test_A[param_a]'],
|
|
['samples/hello/pytest/test_shell_help.py::test_A[param_a]']
|
|
)
|
|
],
|
|
ids=[
|
|
'one_file',
|
|
'more_files',
|
|
'relative_path',
|
|
'absollute_path',
|
|
'user_dir',
|
|
'with_env_var',
|
|
'subtests',
|
|
'subtest_with_param'
|
|
]
|
|
)
|
|
def test_pytest_handle_source_list(testinstance: TestInstance, monkeypatch, pytest_root, expected):
|
|
monkeypatch.setenv('ZEPHYR_BASE', '/zephyr_base')
|
|
monkeypatch.setenv('HOME', '/home/joe')
|
|
testinstance.testsuite.harness_config['pytest_root'] = pytest_root
|
|
pytest_harness = Pytest()
|
|
pytest_harness.configure(testinstance)
|
|
command = pytest_harness.generate_command()
|
|
for pytest_src in expected:
|
|
assert pytest_src in command
|
|
|
|
|
|
def test_if_report_is_parsed(pytester, testinstance: TestInstance):
|
|
test_file_content = textwrap.dedent("""
|
|
def test_1():
|
|
pass
|
|
def test_2():
|
|
pass
|
|
""")
|
|
test_file = pytester.path / 'test_valid.py'
|
|
test_file.write_text(test_file_content)
|
|
report_file = Path('report.xml')
|
|
result = pytester.runpytest(
|
|
str(test_file),
|
|
f'--junit-xml={str(report_file)}'
|
|
)
|
|
result.assert_outcomes(passed=2)
|
|
assert report_file.is_file()
|
|
|
|
pytest_harness = Pytest()
|
|
pytest_harness.configure(testinstance)
|
|
pytest_harness.report_file = report_file
|
|
|
|
pytest_harness._update_test_status()
|
|
|
|
assert pytest_harness.state == "passed"
|
|
assert testinstance.status == "passed"
|
|
assert len(testinstance.testcases) == 2
|
|
for tc in testinstance.testcases:
|
|
assert tc.status == "passed"
|
|
|
|
|
|
def test_if_report_with_error(pytester, testinstance: TestInstance):
|
|
test_file_content = textwrap.dedent("""
|
|
def test_exp():
|
|
raise Exception('Test error')
|
|
def test_err():
|
|
assert False
|
|
""")
|
|
test_file = pytester.path / 'test_error.py'
|
|
test_file.write_text(test_file_content)
|
|
report_file = pytester.path / 'report.xml'
|
|
result = pytester.runpytest(
|
|
str(test_file),
|
|
f'--junit-xml={str(report_file)}'
|
|
)
|
|
result.assert_outcomes(failed=2)
|
|
assert report_file.is_file()
|
|
|
|
pytest_harness = Pytest()
|
|
pytest_harness.configure(testinstance)
|
|
pytest_harness.report_file = report_file
|
|
|
|
pytest_harness._update_test_status()
|
|
|
|
assert pytest_harness.state == "failed"
|
|
assert testinstance.status == "failed"
|
|
assert len(testinstance.testcases) == 2
|
|
for tc in testinstance.testcases:
|
|
assert tc.status == "failed"
|
|
assert tc.output
|
|
assert tc.reason
|
|
assert testinstance.reason
|
|
assert '2/2' in testinstance.reason
|
|
|
|
|
|
def test_if_report_with_skip(pytester, testinstance: TestInstance):
|
|
test_file_content = textwrap.dedent("""
|
|
import pytest
|
|
@pytest.mark.skip('Test skipped')
|
|
def test_skip_1():
|
|
pass
|
|
def test_skip_2():
|
|
pytest.skip('Skipped on runtime')
|
|
""")
|
|
test_file = pytester.path / 'test_skip.py'
|
|
test_file.write_text(test_file_content)
|
|
report_file = pytester.path / 'report.xml'
|
|
result = pytester.runpytest(
|
|
str(test_file),
|
|
f'--junit-xml={str(report_file)}'
|
|
)
|
|
result.assert_outcomes(skipped=2)
|
|
assert report_file.is_file()
|
|
|
|
pytest_harness = Pytest()
|
|
pytest_harness.configure(testinstance)
|
|
pytest_harness.report_file = report_file
|
|
|
|
pytest_harness._update_test_status()
|
|
|
|
assert pytest_harness.state == "skipped"
|
|
assert testinstance.status == "skipped"
|
|
assert len(testinstance.testcases) == 2
|
|
for tc in testinstance.testcases:
|
|
assert tc.status == "skipped"
|
|
|
|
|
|
def test_if_report_with_filter(pytester, testinstance: TestInstance):
|
|
test_file_content = textwrap.dedent("""
|
|
import pytest
|
|
def test_A():
|
|
pass
|
|
def test_B():
|
|
pass
|
|
""")
|
|
test_file = pytester.path / 'test_filter.py'
|
|
test_file.write_text(test_file_content)
|
|
report_file = pytester.path / 'report.xml'
|
|
result = pytester.runpytest(
|
|
str(test_file),
|
|
'-k', 'test_B',
|
|
f'--junit-xml={str(report_file)}'
|
|
)
|
|
result.assert_outcomes(passed=1)
|
|
assert report_file.is_file()
|
|
|
|
pytest_harness = Pytest()
|
|
pytest_harness.configure(testinstance)
|
|
pytest_harness.report_file = report_file
|
|
pytest_harness._update_test_status()
|
|
assert pytest_harness.state == "passed"
|
|
assert testinstance.status == "passed"
|
|
assert len(testinstance.testcases) == 1
|
|
|
|
|
|
def test_if_report_with_no_collected(pytester, testinstance: TestInstance):
|
|
test_file_content = textwrap.dedent("""
|
|
import pytest
|
|
def test_A():
|
|
pass
|
|
""")
|
|
test_file = pytester.path / 'test_filter.py'
|
|
test_file.write_text(test_file_content)
|
|
report_file = pytester.path / 'report.xml'
|
|
result = pytester.runpytest(
|
|
str(test_file),
|
|
'-k', 'test_B',
|
|
f'--junit-xml={str(report_file)}'
|
|
)
|
|
result.assert_outcomes(passed=0)
|
|
assert report_file.is_file()
|
|
|
|
pytest_harness = Pytest()
|
|
pytest_harness.configure(testinstance)
|
|
pytest_harness.report_file = report_file
|
|
pytest_harness._update_test_status()
|
|
assert pytest_harness.state == "skipped"
|
|
assert testinstance.status == "skipped"
|