Mbed-TLS/mbedtls-framework
1
0
Fork 0
mirror of https://github.com/Mbed-TLS/mbedtls-framework.git synced 2026-06-05 21:15:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Separate ignored from uncovered tests in coverage analysis

Browse Source 
For historical reasons, the "ignored" tests in outcome analysis are not
actually ignored: they must not be covered, otherwise the script complains
about an unnecessary exception. In coverage analysis, rename this behavior
to "uncovered", and have "ignored" tests be actually ignored. In driver test
parity analysis, which is now only done in the 3.6 LTS branch, keep the
historical behavior

Consuming branches are currently defining `IGNORED_TESTS` with the
expectation that the test cases must be uncovered. They will need to rename
their definition to `UNCOVERED_TESTS`.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
This commit is contained in:
Gilles Peskine
2026-04-05 20:29:58 +02:00
parent 1404bcbb8e
commit 30d14d340e
1 changed files with 25 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/mbedtls_framework/outcome_analysis.py
+25 -8
View File
@@ -220,20 +220,30 @@ class CoverageTask(Task):
"""Analyze test coverage."""
# Test cases whose suite and description are matched by an entry in
# IGNORED_TESTS are expected to be never executed.
# UNCOVERED_TESTS are expected to be never executed.
# Tests matched by IGNORED_TESTS are ignored entierly.
# All other test cases are expected to be executed at least once.
UNCOVERED_TESTS: TestCaseSetDescription = {}
IGNORED_TESTS: TestCaseSetDescription = {}
def __init__(self, options) -> None:
super().__init__(options)
self.full_coverage = options.full_coverage #type: bool
self.uncovered_tests = TestCaseSet(self.UNCOVERED_TESTS)
self.ignored_tests = TestCaseSet(self.IGNORED_TESTS)
@staticmethod
def section_name() -> str:
return "Analyze coverage"
def note_ignored_test(self, results: Results,
test_suite: str, test_description: str) -> None:
# pylint: disable=no-self-use # derived classes may need self
"""This method runs for each test case that's available and ignored."""
results.info('Test case was ignored: {};{}',
test_suite, test_description)
def run(self, results: Results, outcomes: Outcomes) -> None:
"""Check that all available test cases are executed at least once."""
# Make sure that the generated data files are present (and up-to-date).
@@ -254,21 +264,26 @@ class CoverageTask(Task):
for comp_outcomes in outcomes.values())
(test_suite, test_description) = suite_case.split(';')
ignored = self.ignored_tests.contains(test_suite, test_description)
if ignored:
self.note_ignored_test(results, test_suite, test_description)
if not hit and not ignored:
uncovered = self.uncovered_tests.contains(test_suite, test_description)
if not hit and not uncovered:
if self.full_coverage:
results.error('Test case not executed: {}', suite_case)
else:
results.warning('Test case not executed: {}', suite_case)
elif hit and ignored:
elif hit and uncovered:
# If a test case is no longer always skipped, we should remove
# it from the ignore list.
if self.full_coverage:
results.error('Test case was executed but marked as ignored for coverage: {}',
suite_case)
results.error(
'Test case was executed but marked as uncovered for coverage: {}',
suite_case)
else:
results.warning('Test case was executed but marked as ignored for coverage: {}',
suite_case)
results.warning(
'Test case was executed but marked as uncovered for coverage: {}',
suite_case)
class DriverVSReference(Task):
@@ -289,7 +304,9 @@ class DriverVSReference(Task):
DRIVER = ''
# Ignored test suites (without the test_suite_ prefix).
IGNORED_SUITES = [] #type: typing.List[str]
# Ignored test cases. Despite the name, these test case are not
# completely ignored: they must be skipped by drivers, indicating
# a spurious entry.
IGNORED_TESTS: TestCaseSetDescription = {}
def __init__(self, options) -> None: