Merge pull request #132 from valeriosetti/issue94-framework

[Framework] Move test_psa_*.py scripts to the framework

scripts/check-python-files.sh

@@ -38,7 +38,7 @@ can_pylint () {
 
 can_mypy () {
     # mypy 0.770 is too old:
-    #     tests/scripts/test_psa_constant_names.py:34: error: Cannot find implementation or library stub for module named 'mbedtls_framework'
+    #     framework/scripts/test_psa_constant_names.py:34: error: Cannot find implementation or library stub for module named 'mbedtls_framework'
     # mypy 0.780 from pip passed on the first commit containing this line.
     check_version mypy.version 0.780
 }

scripts/test_psa_compliance.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""Run the PSA Crypto API compliance test suite.
+Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF,
+then compile and run the test suite. The clone is stored at <repository root>/psa-arch-tests.
+Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test
+number - are ignored, while unexpected failures AND successes are reported as errors, to help
+keep the list of known defects as up to date as possible.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+import argparse
+import os
+import re
+import shutil
+import subprocess
+import sys
+from typing import List
+
+from mbedtls_framework import build_tree
+
+# PSA Compliance tests we expect to fail due to known defects in Mbed TLS /
+# TF-PSA-Crypto (or the test suite).
+# The test numbers correspond to the numbers used by the console output of the test suite.
+# Test number 2xx corresponds to the files in the folder
+# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx
+EXPECTED_FAILURES = {} # type: dict
+
+PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git'
+PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC'
+
+#pylint: disable=too-many-branches,too-many-statements,too-many-locals
+def main(library_build_dir: str):
+    root_dir = os.getcwd()
+
+    in_tf_psa_crypto_repo = build_tree.looks_like_tf_psa_crypto_root(root_dir)
+
+    crypto_name = build_tree.crypto_library_filename(root_dir)
+
+    # Temporary, while the crypto library is still located in the library
+    # directory. This will not be the case anymore when it will be built by
+    # the TF-PSA-Crypto build system.
+    if in_tf_psa_crypto_repo:
+        library_subdir = build_tree.crypto_core_directory(root_dir, relative=True)
+    else:
+        library_subdir = 'library'
+
+    crypto_lib_filename = (library_build_dir + '/' +
+                           library_subdir + '/' +
+                           'lib' + crypto_name + '.a')
+
+    # Temporary while the PSA compliance test suite is still run as part
+    # of Mbed TLS testing. When it is not the case anymore, the last case
+    # can be removed.
+    if in_tf_psa_crypto_repo:
+        extra_includes = ';{}/drivers/builtin/include'.format(root_dir)
+    elif build_tree.is_mbedtls_3_6():
+        extra_includes = ''
+    else:
+        extra_includes = ';{}/tf-psa-crypto/include'.format(root_dir) + \
+                            (';{}/tf-psa-crypto/drivers/builtin/include'.format(root_dir))
+
+    if not os.path.exists(crypto_lib_filename):
+        #pylint: disable=bad-continuation
+        subprocess.check_call([
+            'cmake', '.',
+                     '-GUnix Makefiles',
+                     '-B' + library_build_dir
+        ])
+        subprocess.check_call(['cmake', '--build', library_build_dir,
+                               '--target', crypto_name])
+
+    psa_arch_tests_dir = 'psa-arch-tests'
+    os.makedirs(psa_arch_tests_dir, exist_ok=True)
+    try:
+        os.chdir(psa_arch_tests_dir)
+
+        # Reuse existing local clone
+        subprocess.check_call(['git', 'init'])
+        subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF])
+        subprocess.check_call(['git', 'checkout', 'FETCH_HEAD'])
+
+        build_dir = 'api-tests/build'
+        try:
+            shutil.rmtree(build_dir)
+        except FileNotFoundError:
+            pass
+        os.mkdir(build_dir)
+        os.chdir(build_dir)
+
+        #pylint: disable=bad-continuation
+        subprocess.check_call([
+            'cmake', '..',
+                     '-GUnix Makefiles',
+                     '-DTARGET=tgt_dev_apis_stdc',
+                     '-DTOOLCHAIN=HOST_GCC',
+                     '-DSUITE=CRYPTO',
+                     '-DPSA_CRYPTO_LIB_FILENAME={}/{}'.format(root_dir,
+                                                              crypto_lib_filename),
+                     ('-DPSA_INCLUDE_PATHS={}/include' + extra_includes).format(root_dir)
+        ])
+        subprocess.check_call(['cmake', '--build', '.'])
+
+        proc = subprocess.Popen(['./psa-arch-tests-crypto'],
+                                bufsize=1, stdout=subprocess.PIPE, universal_newlines=True)
+
+        test_re = re.compile(
+            '^TEST: (?P<test_num>[0-9]*)|'
+            '^TEST RESULT: (?P<test_result>FAILED|PASSED)'
+        )
+        test = -1
+        unexpected_successes = set(EXPECTED_FAILURES)
+        expected_failures = [] # type: List[int]
+        unexpected_failures = [] # type: List[int]
+        if proc.stdout is None:
+            return 1
+
+        for line in proc.stdout:
+            print(line, end='')
+            match = test_re.match(line)
+            if match is not None:
+                groupdict = match.groupdict()
+                test_num = groupdict['test_num']
+                if test_num is not None:
+                    test = int(test_num)
+                elif groupdict['test_result'] == 'FAILED':
+                    try:
+                        unexpected_successes.remove(test)
+                        expected_failures.append(test)
+                        print('Expected failure, ignoring')
+                    except KeyError:
+                        unexpected_failures.append(test)
+                        print('ERROR: Unexpected failure')
+                elif test in unexpected_successes:
+                    print('ERROR: Unexpected success')
+        proc.wait()
+
+        print()
+        print('***** test_psa_compliance.py report ******')
+        print()
+        print('Expected failures:', ', '.join(str(i) for i in expected_failures))
+        print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures))
+        print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes)))
+        print()
+        if unexpected_successes or unexpected_failures:
+            if unexpected_successes:
+                print('Unexpected successes encountered.')
+                print('Please remove the corresponding tests from '
+                      'EXPECTED_FAILURES in tests/scripts/compliance_test.py')
+                print()
+            print('FAILED')
+            return 1
+        else:
+            print('SUCCESS')
+            return 0
+    finally:
+        os.chdir(root_dir)
+
+if __name__ == '__main__':
+    BUILD_DIR = 'out_of_source_build'
+
+    # pylint: disable=invalid-name
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--build-dir', nargs=1,
+                        help='path to Mbed TLS / TF-PSA-Crypto build directory')
+    args = parser.parse_args()
+
+    if args.build_dir is not None:
+        BUILD_DIR = args.build_dir[0]
+
+    sys.exit(main(BUILD_DIR))

scripts/test_psa_constant_names.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""Test the program psa_constant_names.
+Gather constant names from header files and test cases. Compile a C program
+to print out their numerical values, feed these numerical values to
+psa_constant_names, and check that the output is the original name.
+Return 0 if all test cases pass, 1 if the output was not always as expected,
+or 1 (with a Python backtrace) if there was an operational error.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+import argparse
+from collections import namedtuple
+import os
+import re
+import subprocess
+import sys
+from typing import Iterable, List, Optional, Tuple
+
+from mbedtls_framework import build_tree
+from mbedtls_framework import c_build_helper
+from mbedtls_framework.macro_collector import InputsForTest, PSAMacroEnumerator
+from mbedtls_framework import typing_util
+
+def gather_inputs(headers: Iterable[str],
+                  test_suites: Iterable[str],
+                  inputs_class=InputsForTest) -> PSAMacroEnumerator:
+    """Read the list of inputs to test psa_constant_names with."""
+    inputs = inputs_class()
+    for header in headers:
+        inputs.parse_header(header)
+    for test_cases in test_suites:
+        inputs.parse_test_cases(test_cases)
+    inputs.add_numerical_values()
+    inputs.gather_arguments()
+    return inputs
+
+def run_c(type_word: str,
+          expressions: Iterable[str],
+          include_path: Optional[str] = None,
+          keep_c: bool = False) -> List[str]:
+    """Generate and run a program to print out numerical values of C expressions."""
+    if type_word == 'status':
+        cast_to = 'long'
+        printf_format = '%ld'
+    else:
+        cast_to = 'unsigned long'
+        printf_format = '0x%08lx'
+    return c_build_helper.get_c_expression_values(
+        cast_to, printf_format,
+        expressions,
+        caller='test_psa_constant_names.py for {} values'.format(type_word),
+        file_label=type_word,
+        header='#include <psa/crypto.h>',
+        include_path=include_path,
+        keep_c=keep_c
+    )
+
+NORMALIZE_STRIP_RE = re.compile(r'\s+')
+def normalize(expr: str) -> str:
+    """Normalize the C expression so as not to care about trivial differences.
+
+    Currently "trivial differences" means whitespace.
+    """
+    return re.sub(NORMALIZE_STRIP_RE, '', expr)
+
+ALG_TRUNCATED_TO_SELF_RE = \
+    re.compile(r'PSA_ALG_AEAD_WITH_SHORTENED_TAG\('
+               r'PSA_ALG_(?:CCM|CHACHA20_POLY1305|GCM)'
+               r', *16\)\Z')
+
+def is_simplifiable(expr: str) -> bool:
+    """Determine whether an expression is simplifiable.
+
+    Simplifiable expressions can't be output in their input form, since
+    the output will be the simple form. Therefore they must be excluded
+    from testing.
+    """
+    if ALG_TRUNCATED_TO_SELF_RE.match(expr):
+        return True
+    return False
+
+def collect_values(inputs: InputsForTest,
+                   type_word: str,
+                   include_path: Optional[str] = None,
+                   keep_c: bool = False) -> Tuple[List[str], List[str]]:
+    """Generate expressions using known macro names and calculate their values.
+
+    Return a list of pairs of (expr, value) where expr is an expression and
+    value is a string representation of its integer value.
+    """
+    names = inputs.get_names(type_word)
+    expressions = sorted(expr
+                         for expr in inputs.generate_expressions(names)
+                         if not is_simplifiable(expr))
+    values = run_c(type_word, expressions,
+                   include_path=include_path, keep_c=keep_c)
+    return expressions, values
+
+class Tests:
+    """An object representing tests and their results."""
+
+    Error = namedtuple('Error',
+                       ['type', 'expression', 'value', 'output'])
+
+    def __init__(self, options) -> None:
+        self.options = options
+        self.count = 0
+        self.errors = [] #type: List[Tests.Error]
+
+    def run_one(self, inputs: InputsForTest, type_word: str) -> None:
+        """Test psa_constant_names for the specified type.
+
+        Run the program on the names for this type.
+        Use the inputs to figure out what arguments to pass to macros that
+        take arguments.
+        """
+        expressions, values = collect_values(inputs, type_word,
+                                             include_path=self.options.include,
+                                             keep_c=self.options.keep_c)
+        output_bytes = subprocess.check_output([self.options.program,
+                                                type_word] + values)
+        output = output_bytes.decode('ascii')
+        outputs = output.strip().split('\n')
+        self.count += len(expressions)
+        for expr, value, output in zip(expressions, values, outputs):
+            if self.options.show:
+                sys.stdout.write('{} {}\t{}\n'.format(type_word, value, output))
+            if normalize(expr) != normalize(output):
+                self.errors.append(self.Error(type=type_word,
+                                              expression=expr,
+                                              value=value,
+                                              output=output))
+
+    def run_all(self, inputs: InputsForTest) -> None:
+        """Run psa_constant_names on all the gathered inputs."""
+        for type_word in ['status', 'algorithm', 'ecc_curve', 'dh_group',
+                          'key_type', 'key_usage']:
+            self.run_one(inputs, type_word)
+
+    def report(self, out: typing_util.Writable) -> None:
+        """Describe each case where the output is not as expected.
+
+        Write the errors to ``out``.
+        Also write a total.
+        """
+        for error in self.errors:
+            out.write('For {} "{}", got "{}" (value: {})\n'
+                      .format(error.type, error.expression,
+                              error.output, error.value))
+        out.write('{} test cases'.format(self.count))
+        if self.errors:
+            out.write(', {} FAIL\n'.format(len(self.errors)))
+        else:
+            out.write(' PASS\n')
+
+HEADERS = ['psa/crypto.h', 'psa/crypto_extra.h', 'psa/crypto_values.h']
+
+if build_tree.is_mbedtls_3_6():
+    TEST_SUITES = ['tests/suites/test_suite_psa_crypto_metadata.data']
+else:
+    TEST_SUITES = ['tf-psa-crypto/tests/suites/test_suite_psa_crypto_metadata.data']
+
+def main():
+    parser = argparse.ArgumentParser(description=globals()['__doc__'])
+    if build_tree.is_mbedtls_3_6():
+        parser.add_argument('--include', '-I',
+                            action='append', default=['include'],
+                            help='Directory for header files')
+    else:
+        parser.add_argument('--include', '-I',
+                            action='append', default=['tf-psa-crypto/include',
+                                                      'tf-psa-crypto/drivers/builtin/include',
+                                                      'tf-psa-crypto/drivers/everest/include',
+                                                      'include'],
+                            help='Directory for header files')
+    parser.add_argument('--keep-c',
+                        action='store_true', dest='keep_c', default=False,
+                        help='Keep the intermediate C file')
+    parser.add_argument('--no-keep-c',
+                        action='store_false', dest='keep_c',
+                        help='Don\'t keep the intermediate C file (default)')
+    if build_tree.is_mbedtls_3_6():
+        parser.add_argument('--program',
+                            default='programs/psa/psa_constant_names',
+                            help='Program to test')
+    else:
+        parser.add_argument('--program',
+                            default='tf-psa-crypto/programs/psa/psa_constant_names',
+                            help='Program to test')
+    parser.add_argument('--show',
+                        action='store_true',
+                        help='Show tested values on stdout')
+    parser.add_argument('--no-show',
+                        action='store_false', dest='show',
+                        help='Don\'t show tested values (default)')
+    options = parser.parse_args()
+    headers = [os.path.join(options.include[0], h) for h in HEADERS]
+    inputs = gather_inputs(headers, TEST_SUITES)
+    tests = Tests(options)
+    tests.run_all(inputs)
+    tests.report(sys.stdout)
+    if tests.errors:
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()