From 2baaf60c5d54405cf6c17b653eba316b32c58387 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 7 Jan 2022 15:46:12 +0100 Subject: [PATCH 1/4] Don't error out if no opt-testcases/*.sh is found This can happen in an insufficiently populated out-of-tree build. Signed-off-by: Gilles Peskine --- tests/ssl-opt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 691c0e7d5b..762830486c 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -9013,7 +9013,7 @@ run_test "TLS 1.3: HelloRetryRequest check - gnutls" \ -c "Last error was: -0x6E00 - SSL - The handshake negotiation failed" \ -s "HELLO RETRY REQUEST was queued" -for i in $(ls opt-testcases/*.sh) +for i in opt-testcases/*.sh do . $i done From 5eb2b028627a674325dbbafb548290173dd7fdd0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 7 Jan 2022 15:47:02 +0100 Subject: [PATCH 2/4] Report correct test suite names for opt-testcases/* in outcome file In the outcome file, report each test case in the file it's in, rather than reporting them all from ssl-opt. This is more informative and matches what check_test_cases.py does. This fixes a bug whereby test cases from opt-testcases/* were not detected as having run on the CI, because analyze_outcomes.py (which uses check_test_cases.py) expects them in the containing file whereas they were reported in ssl-opt. Signed-off-by: Gilles Peskine --- tests/ssl-opt.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 762830486c..4dea6685d5 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -543,16 +543,18 @@ print_name() { # record_outcome [] # The test name must be in $NAME. +# Use $TEST_SUITE_NAME as the test suite name if set. record_outcome() { echo "$1" if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then printf '%s;%s;%s;%s;%s;%s\n' \ "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \ - "ssl-opt" "$NAME" \ + "${TEST_SUITE_NAME:-ssl-opt}" "$NAME" \ "$1" "${2-}" \ >>"$MBEDTLS_TEST_OUTCOME_FILE" fi } +unset TEST_SUITE_NAME # True if the presence of the given pattern in a log definitely indicates # that the test has failed. False if the presence is inconclusive. @@ -9015,8 +9017,11 @@ run_test "TLS 1.3: HelloRetryRequest check - gnutls" \ for i in opt-testcases/*.sh do - . $i + TEST_SUITE_NAME=${i##*/} + TEST_SUITE_NAME=${TEST_SUITE_NAME%.*} + . "$i" done +unset TEST_SUITE_NAME requires_openssl_tls1_3 requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3 From 686c292e8ac65ca7076d320185c3b6da53ee55db Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 7 Jan 2022 15:58:38 +0100 Subject: [PATCH 3/4] Move collect_available_test_cases to check_test_cases.py No behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 23 +---------------------- tests/scripts/check_test_cases.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 73f16bdb25..d06a0596f3 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -7,7 +7,6 @@ less likely to be useful. """ import argparse -import re import sys import traceback @@ -51,29 +50,9 @@ class TestCaseOutcomes: """ return len(self.successes) + len(self.failures) -class TestDescriptions(check_test_cases.TestDescriptionExplorer): - """Collect the available test cases.""" - - def __init__(self): - super().__init__() - self.descriptions = set() - - def process_test_case(self, _per_file_state, - file_name, _line_number, description): - """Record an available test case.""" - base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) - key = ';'.join([base_name, description.decode('utf-8')]) - self.descriptions.add(key) - -def collect_available_test_cases(): - """Collect the available test cases.""" - explorer = TestDescriptions() - explorer.walk_all() - return sorted(explorer.descriptions) - def analyze_coverage(results, outcomes): """Check that all available test cases are executed at least once.""" - available = collect_available_test_cases() + available = check_test_cases.collect_available_test_cases() for key in available: hits = outcomes[key].hits() if key in outcomes else 0 if hits == 0: diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index f9ae36c13d..01a9dd7662 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -137,6 +137,26 @@ state may override this method. '*.sh')): self.walk_ssl_opt_sh(ssl_opt_file_name) +class TestDescriptions(TestDescriptionExplorer): + """Collect the available test cases.""" + + def __init__(self): + super().__init__() + self.descriptions = set() + + def process_test_case(self, _per_file_state, + file_name, _line_number, description): + """Record an available test case.""" + base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) + key = ';'.join([base_name, description.decode('utf-8')]) + self.descriptions.add(key) + +def collect_available_test_cases(): + """Collect the available test cases.""" + explorer = TestDescriptions() + explorer.walk_all() + return sorted(explorer.descriptions) + class DescriptionChecker(TestDescriptionExplorer): """Check all test case descriptions. From 7e09105192246277acd5913a89be832b14e4ad2e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 7 Jan 2022 15:58:55 +0100 Subject: [PATCH 4/4] New option to list all test cases Occasionally useful for diagnosing issues with test reports. Signed-off-by: Gilles Peskine --- tests/scripts/check_test_cases.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 01a9dd7662..d84ed042c4 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -196,6 +196,9 @@ class DescriptionChecker(TestDescriptionExplorer): def main(): parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--list-all', + action='store_true', + help='List all test cases, without doing checks') parser.add_argument('--quiet', '-q', action='store_true', help='Hide warnings') @@ -203,6 +206,10 @@ def main(): action='store_false', dest='quiet', help='Show warnings (default: on; undoes --quiet)') options = parser.parse_args() + if options.list_all: + descriptions = collect_available_test_cases() + sys.stdout.write('\n'.join(descriptions + [''])) + return results = Results(options) checker = DescriptionChecker(results) checker.walk_all()