diff --git a/scripts/test_psa_compliance.py b/scripts/test_psa_compliance.py new file mode 100755 index 000000000..7c09afc19 --- /dev/null +++ b/scripts/test_psa_compliance.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Run the PSA Crypto API compliance test suite. +Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, +then compile and run the test suite. The clone is stored at /psa-arch-tests. +Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test +number - are ignored, while unexpected failures AND successes are reported as errors, to help +keep the list of known defects as up to date as possible. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +import argparse +import os +import re +import shutil +import subprocess +import sys +from typing import List + +#pylint: disable=unused-import +import scripts_path +from mbedtls_framework import build_tree + +# PSA Compliance tests we expect to fail due to known defects in Mbed TLS / +# TF-PSA-Crypto (or the test suite). +# The test numbers correspond to the numbers used by the console output of the test suite. +# Test number 2xx corresponds to the files in the folder +# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx +EXPECTED_FAILURES = {} # type: dict + +PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git' +PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC' + +#pylint: disable=too-many-branches,too-many-statements,too-many-locals +def main(library_build_dir: str): + root_dir = os.getcwd() + + in_tf_psa_crypto_repo = build_tree.looks_like_tf_psa_crypto_root(root_dir) + + crypto_name = build_tree.crypto_library_filename(root_dir) + + # Temporary, while the crypto library is still located in the library + # directory. This will not be the case anymore when it will be built by + # the TF-PSA-Crypto build system. + if in_tf_psa_crypto_repo: + library_subdir = build_tree.crypto_core_directory(root_dir, relative=True) + else: + library_subdir = 'library' + + crypto_lib_filename = (library_build_dir + '/' + + library_subdir + '/' + + 'lib' + crypto_name + '.a') + + if not os.path.exists(crypto_lib_filename): + #pylint: disable=bad-continuation + subprocess.check_call([ + 'cmake', '.', + '-GUnix Makefiles', + '-B' + library_build_dir + ]) + subprocess.check_call(['cmake', '--build', library_build_dir, + '--target', crypto_name]) + + psa_arch_tests_dir = 'psa-arch-tests' + os.makedirs(psa_arch_tests_dir, exist_ok=True) + try: + os.chdir(psa_arch_tests_dir) + + # Reuse existing local clone + subprocess.check_call(['git', 'init']) + subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) + subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) + + build_dir = 'api-tests/build' + try: + shutil.rmtree(build_dir) + except FileNotFoundError: + pass + os.mkdir(build_dir) + os.chdir(build_dir) + + # Temporary while the PSA compliance test suite is still run as part + # of Mbed TLS testing. When it is not the case anymore, the second case + # can be removed. + if in_tf_psa_crypto_repo: + extra_includes = ';{}/drivers/builtin/include'.format(root_dir) + elif os.path.isdir(os.path.join(root_dir, 'tf-psa-crypto')): + extra_includes = ';{}/tf-psa-crypto/include'.format(root_dir) + \ + (';{}/tf-psa-crypto/drivers/builtin/include'.format(root_dir)) + + #pylint: disable=bad-continuation + subprocess.check_call([ + 'cmake', '..', + '-GUnix Makefiles', + '-DTARGET=tgt_dev_apis_stdc', + '-DTOOLCHAIN=HOST_GCC', + '-DSUITE=CRYPTO', + '-DPSA_CRYPTO_LIB_FILENAME={}/{}'.format(root_dir, + crypto_lib_filename), + ('-DPSA_INCLUDE_PATHS={}/include' + extra_includes).format(root_dir) + ]) + subprocess.check_call(['cmake', '--build', '.']) + + proc = subprocess.Popen(['./psa-arch-tests-crypto'], + bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) + + test_re = re.compile( + '^TEST: (?P[0-9]*)|' + '^TEST RESULT: (?PFAILED|PASSED)' + ) + test = -1 + unexpected_successes = set(EXPECTED_FAILURES) + expected_failures = [] # type: List[int] + unexpected_failures = [] # type: List[int] + if proc.stdout is None: + return 1 + + for line in proc.stdout: + print(line, end='') + match = test_re.match(line) + if match is not None: + groupdict = match.groupdict() + test_num = groupdict['test_num'] + if test_num is not None: + test = int(test_num) + elif groupdict['test_result'] == 'FAILED': + try: + unexpected_successes.remove(test) + expected_failures.append(test) + print('Expected failure, ignoring') + except KeyError: + unexpected_failures.append(test) + print('ERROR: Unexpected failure') + elif test in unexpected_successes: + print('ERROR: Unexpected success') + proc.wait() + + print() + print('***** test_psa_compliance.py report ******') + print() + print('Expected failures:', ', '.join(str(i) for i in expected_failures)) + print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) + print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) + print() + if unexpected_successes or unexpected_failures: + if unexpected_successes: + print('Unexpected successes encountered.') + print('Please remove the corresponding tests from ' + 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') + print() + print('FAILED') + return 1 + else: + print('SUCCESS') + return 0 + finally: + os.chdir(root_dir) + +if __name__ == '__main__': + BUILD_DIR = 'out_of_source_build' + + # pylint: disable=invalid-name + parser = argparse.ArgumentParser() + parser.add_argument('--build-dir', nargs=1, + help='path to Mbed TLS / TF-PSA-Crypto build directory') + args = parser.parse_args() + + if args.build_dir is not None: + BUILD_DIR = args.build_dir[0] + + sys.exit(main(BUILD_DIR)) diff --git a/scripts/test_psa_constant_names.py b/scripts/test_psa_constant_names.py new file mode 100755 index 000000000..e697d1755 --- /dev/null +++ b/scripts/test_psa_constant_names.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Test the program psa_constant_names. +Gather constant names from header files and test cases. Compile a C program +to print out their numerical values, feed these numerical values to +psa_constant_names, and check that the output is the original name. +Return 0 if all test cases pass, 1 if the output was not always as expected, +or 1 (with a Python backtrace) if there was an operational error. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +import argparse +from collections import namedtuple +import os +import re +import subprocess +import sys +from typing import Iterable, List, Optional, Tuple + +import scripts_path # pylint: disable=unused-import +from mbedtls_framework import c_build_helper +from mbedtls_framework.macro_collector import InputsForTest, PSAMacroEnumerator +from mbedtls_framework import typing_util + +def gather_inputs(headers: Iterable[str], + test_suites: Iterable[str], + inputs_class=InputsForTest) -> PSAMacroEnumerator: + """Read the list of inputs to test psa_constant_names with.""" + inputs = inputs_class() + for header in headers: + inputs.parse_header(header) + for test_cases in test_suites: + inputs.parse_test_cases(test_cases) + inputs.add_numerical_values() + inputs.gather_arguments() + return inputs + +def run_c(type_word: str, + expressions: Iterable[str], + include_path: Optional[str] = None, + keep_c: bool = False) -> List[str]: + """Generate and run a program to print out numerical values of C expressions.""" + if type_word == 'status': + cast_to = 'long' + printf_format = '%ld' + else: + cast_to = 'unsigned long' + printf_format = '0x%08lx' + return c_build_helper.get_c_expression_values( + cast_to, printf_format, + expressions, + caller='test_psa_constant_names.py for {} values'.format(type_word), + file_label=type_word, + header='#include ', + include_path=include_path, + keep_c=keep_c + ) + +NORMALIZE_STRIP_RE = re.compile(r'\s+') +def normalize(expr: str) -> str: + """Normalize the C expression so as not to care about trivial differences. + + Currently "trivial differences" means whitespace. + """ + return re.sub(NORMALIZE_STRIP_RE, '', expr) + +ALG_TRUNCATED_TO_SELF_RE = \ + re.compile(r'PSA_ALG_AEAD_WITH_SHORTENED_TAG\(' + r'PSA_ALG_(?:CCM|CHACHA20_POLY1305|GCM)' + r', *16\)\Z') + +def is_simplifiable(expr: str) -> bool: + """Determine whether an expression is simplifiable. + + Simplifiable expressions can't be output in their input form, since + the output will be the simple form. Therefore they must be excluded + from testing. + """ + if ALG_TRUNCATED_TO_SELF_RE.match(expr): + return True + return False + +def collect_values(inputs: InputsForTest, + type_word: str, + include_path: Optional[str] = None, + keep_c: bool = False) -> Tuple[List[str], List[str]]: + """Generate expressions using known macro names and calculate their values. + + Return a list of pairs of (expr, value) where expr is an expression and + value is a string representation of its integer value. + """ + names = inputs.get_names(type_word) + expressions = sorted(expr + for expr in inputs.generate_expressions(names) + if not is_simplifiable(expr)) + values = run_c(type_word, expressions, + include_path=include_path, keep_c=keep_c) + return expressions, values + +class Tests: + """An object representing tests and their results.""" + + Error = namedtuple('Error', + ['type', 'expression', 'value', 'output']) + + def __init__(self, options) -> None: + self.options = options + self.count = 0 + self.errors = [] #type: List[Tests.Error] + + def run_one(self, inputs: InputsForTest, type_word: str) -> None: + """Test psa_constant_names for the specified type. + + Run the program on the names for this type. + Use the inputs to figure out what arguments to pass to macros that + take arguments. + """ + expressions, values = collect_values(inputs, type_word, + include_path=self.options.include, + keep_c=self.options.keep_c) + output_bytes = subprocess.check_output([self.options.program, + type_word] + values) + output = output_bytes.decode('ascii') + outputs = output.strip().split('\n') + self.count += len(expressions) + for expr, value, output in zip(expressions, values, outputs): + if self.options.show: + sys.stdout.write('{} {}\t{}\n'.format(type_word, value, output)) + if normalize(expr) != normalize(output): + self.errors.append(self.Error(type=type_word, + expression=expr, + value=value, + output=output)) + + def run_all(self, inputs: InputsForTest) -> None: + """Run psa_constant_names on all the gathered inputs.""" + for type_word in ['status', 'algorithm', 'ecc_curve', 'dh_group', + 'key_type', 'key_usage']: + self.run_one(inputs, type_word) + + def report(self, out: typing_util.Writable) -> None: + """Describe each case where the output is not as expected. + + Write the errors to ``out``. + Also write a total. + """ + for error in self.errors: + out.write('For {} "{}", got "{}" (value: {})\n' + .format(error.type, error.expression, + error.output, error.value)) + out.write('{} test cases'.format(self.count)) + if self.errors: + out.write(', {} FAIL\n'.format(len(self.errors))) + else: + out.write(' PASS\n') + +HEADERS = ['psa/crypto.h', 'psa/crypto_extra.h', 'psa/crypto_values.h'] +TEST_SUITES = ['tf-psa-crypto/tests/suites/test_suite_psa_crypto_metadata.data'] + +def main(): + parser = argparse.ArgumentParser(description=globals()['__doc__']) + parser.add_argument('--include', '-I', + action='append', default=['tf-psa-crypto/include', + 'tf-psa-crypto/drivers/builtin/include', + 'tf-psa-crypto/drivers/everest/include', + 'include'], + help='Directory for header files') + parser.add_argument('--keep-c', + action='store_true', dest='keep_c', default=False, + help='Keep the intermediate C file') + parser.add_argument('--no-keep-c', + action='store_false', dest='keep_c', + help='Don\'t keep the intermediate C file (default)') + parser.add_argument('--program', + default='tf-psa-crypto/programs/psa/psa_constant_names', + help='Program to test') + parser.add_argument('--show', + action='store_true', + help='Show tested values on stdout') + parser.add_argument('--no-show', + action='store_false', dest='show', + help='Don\'t show tested values (default)') + options = parser.parse_args() + headers = [os.path.join(options.include[0], h) for h in HEADERS] + inputs = gather_inputs(headers, TEST_SUITES) + tests = Tests(options) + tests.run_all(inputs) + tests.report(sys.stdout) + if tests.errors: + sys.exit(1) + +if __name__ == '__main__': + main()