From 30d14d340e5aef310dcdb2a5bd6d5c13f46adef0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 5 Apr 2026 20:29:58 +0200 Subject: [PATCH] Separate ignored from uncovered tests in coverage analysis For historical reasons, the "ignored" tests in outcome analysis are not actually ignored: they must not be covered, otherwise the script complains about an unnecessary exception. In coverage analysis, rename this behavior to "uncovered", and have "ignored" tests be actually ignored. In driver test parity analysis, which is now only done in the 3.6 LTS branch, keep the historical behavior Consuming branches are currently defining `IGNORED_TESTS` with the expectation that the test cases must be uncovered. They will need to rename their definition to `UNCOVERED_TESTS`. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/outcome_analysis.py | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/scripts/mbedtls_framework/outcome_analysis.py b/scripts/mbedtls_framework/outcome_analysis.py index 60bbf3adb..da6097637 100644 --- a/scripts/mbedtls_framework/outcome_analysis.py +++ b/scripts/mbedtls_framework/outcome_analysis.py @@ -220,20 +220,30 @@ class CoverageTask(Task): """Analyze test coverage.""" # Test cases whose suite and description are matched by an entry in - # IGNORED_TESTS are expected to be never executed. + # UNCOVERED_TESTS are expected to be never executed. + # Tests matched by IGNORED_TESTS are ignored entierly. # All other test cases are expected to be executed at least once. + UNCOVERED_TESTS: TestCaseSetDescription = {} IGNORED_TESTS: TestCaseSetDescription = {} def __init__(self, options) -> None: super().__init__(options) self.full_coverage = options.full_coverage #type: bool + self.uncovered_tests = TestCaseSet(self.UNCOVERED_TESTS) self.ignored_tests = TestCaseSet(self.IGNORED_TESTS) @staticmethod def section_name() -> str: return "Analyze coverage" + def note_ignored_test(self, results: Results, + test_suite: str, test_description: str) -> None: + # pylint: disable=no-self-use # derived classes may need self + """This method runs for each test case that's available and ignored.""" + results.info('Test case was ignored: {};{}', + test_suite, test_description) + def run(self, results: Results, outcomes: Outcomes) -> None: """Check that all available test cases are executed at least once.""" # Make sure that the generated data files are present (and up-to-date). @@ -254,21 +264,26 @@ class CoverageTask(Task): for comp_outcomes in outcomes.values()) (test_suite, test_description) = suite_case.split(';') ignored = self.ignored_tests.contains(test_suite, test_description) + if ignored: + self.note_ignored_test(results, test_suite, test_description) - if not hit and not ignored: + uncovered = self.uncovered_tests.contains(test_suite, test_description) + if not hit and not uncovered: if self.full_coverage: results.error('Test case not executed: {}', suite_case) else: results.warning('Test case not executed: {}', suite_case) - elif hit and ignored: + elif hit and uncovered: # If a test case is no longer always skipped, we should remove # it from the ignore list. if self.full_coverage: - results.error('Test case was executed but marked as ignored for coverage: {}', - suite_case) + results.error( + 'Test case was executed but marked as uncovered for coverage: {}', + suite_case) else: - results.warning('Test case was executed but marked as ignored for coverage: {}', - suite_case) + results.warning( + 'Test case was executed but marked as uncovered for coverage: {}', + suite_case) class DriverVSReference(Task): @@ -289,7 +304,9 @@ class DriverVSReference(Task): DRIVER = '' # Ignored test suites (without the test_suite_ prefix). IGNORED_SUITES = [] #type: typing.List[str] - + # Ignored test cases. Despite the name, these test case are not + # completely ignored: they must be skipped by drivers, indicating + # a spurious entry. IGNORED_TESTS: TestCaseSetDescription = {} def __init__(self, options) -> None: