Merge pull request #149 from ronald-cron-arm/make-generated-files

Enable checking of generated files and code style in TF-PSA-Crypto
Validated by the CI of https://github.com/Mbed-TLS/mbedtls/pull/10062 and https://github.com/Mbed-TLS/mbedtls/pull/10069.

scripts/code_style.py

@@ -41,16 +41,29 @@ def list_generated_files() -> FrozenSet[str]:
     would conform to the code style, but this would be difficult, especially
     with respect to the placement of line breaks in long logical lines.
     """
-    # Parse check-generated-files.sh to get an up-to-date list of
-    # generated files. Read the file rather than calling it so that
-    # this script only depends on Git, Python and uncrustify, and not other
-    # tools such as sh or grep which might not be available on Windows.
-    # This introduces a limitation: check-generated-files.sh must have
-    # the expected format and must list the files explicitly, not through
-    # wildcards or command substitution.
-    content = open(CHECK_GENERATED_FILES, encoding="utf-8").read()
-    checks = re.findall(CHECK_CALL_RE, content)
-    return frozenset(word for s in checks for word in s.split())
+    if build_tree.is_mbedtls_3_6():
+        # Parse check-generated-files.sh to get an up-to-date list of
+        # generated files. Read the file rather than calling it so that
+        # this script only depends on Git, Python and uncrustify, and not other
+        # tools such as sh or grep which might not be available on Windows.
+        # This introduces a limitation: check-generated-files.sh must have
+        # the expected format and must list the files explicitly, not through
+        # wildcards or command substitution.
+        content = open(CHECK_GENERATED_FILES, encoding="utf-8").read()
+        checks = re.findall(CHECK_CALL_RE, content)
+        return frozenset(word for s in checks for word in s.split())
+    else:
+        output = subprocess.check_output(["framework/scripts/make_generated_files.py",
+                                          "--list"], universal_newlines=True)
+        # psa_test_wrappers.[hc], generated by generate_psa_wrappers.py, are
+        # currently committed and unknown to make_generated_files.py. Add them
+        # here to the list of generated file as we do not want to check their
+        # coding style.
+        if build_tree.looks_like_tf_psa_crypto_root("."):
+            output += "tests/include/test/psa_test_wrappers.h\n"
+            output += "tests/src/psa_test_wrappers.c"
+
+        return frozenset(line for line in output.splitlines())
 
 # Check for comment string indicating an auto-generated file
 AUTOGEN_RE = re.compile(r"Warning[ :-]+This file is (now )?auto[ -]?generated",
@@ -72,7 +85,6 @@ def get_src_files(since: Optional[str]) -> List[str]:
     """
     file_patterns = ["*.[hc]",
                      "tests/suites/*.function",
-                     "tf-psa-crypto/tests/suites/*.function",
                      "scripts/data_files/*.fmt"]
     output = subprocess.check_output(["git", "ls-files"] + file_patterns,
                                      universal_newlines=True)
@@ -140,8 +152,8 @@ def get_src_files(since: Optional[str]) -> List[str]:
                              is_file_autogenerated(filename))]
     else:
         src_files = [filename for filename in src_files
-                     if not (filename.startswith("tf-psa-crypto/drivers/everest/") or
-                             filename.startswith("tf-psa-crypto/drivers/p256-m/") or
+                     if not (filename.startswith("drivers/everest/") or
+                             filename.startswith("drivers/p256-m/") or
                              filename in generated_files or
                              is_file_autogenerated(filename))]
     return src_files

scripts/generate_config_tests.py

@@ -12,6 +12,7 @@ from typing import Iterable, Iterator, List, Optional, Tuple
 
 import project_scripts # pylint: disable=unused-import
 import config
+from mbedtls_framework import build_tree
 from mbedtls_framework import config_common
 from mbedtls_framework import test_case
 from mbedtls_framework import test_data_generation
@@ -172,9 +173,10 @@ class ConfigTestGenerator(test_data_generation.TestGenerator):
             self.targets['test_suite_config.mbedtls_boolean'] = \
                 lambda: enumerate_boolean_setting_cases(self.mbedtls_config)
         if 'CryptoConfig' in config_members:
-            self.psa_config = config.CryptoConfig()
-            self.targets['test_suite_config.psa_boolean'] = \
-                lambda: enumerate_boolean_setting_cases(self.psa_config)
+            if build_tree.is_mbedtls_3_6():
+                self.psa_config = config.CryptoConfig()
+                self.targets['test_suite_config.psa_boolean'] = \
+                    lambda: enumerate_boolean_setting_cases(self.psa_config)
         elif 'TFPSACryptoConfig' in config_members:
             self.psa_config = config.TFPSACryptoConfig()
             self.targets['test_suite_config.psa_boolean'] = \

scripts/generate_test_cert_macros.py

@@ -54,7 +54,7 @@ INPUT_ARGS = [
 
 def main():
     parser = argparse.ArgumentParser()
-    default_output_path = os.path.join(TESTS_DIR, 'src', 'test_certs.h')
+    default_output_path = os.path.join(TESTS_DIR, 'include', 'test', 'test_certs.h')
     parser.add_argument('--output', type=str, default=default_output_path)
     parser.add_argument('--list-dependencies', action='store_true')
     args = parser.parse_args()

scripts/generate_test_keys.py

@@ -168,7 +168,7 @@ def collect_keys() -> Tuple[str, str]:
     return ''.join(arrays), '\n'.join(look_up_table)
 
 def main() -> None:
-    default_output_path = guess_project_root() + "/framework/tests/include/test/test_keys.h"
+    default_output_path = guess_project_root() + "/tests/include/test/test_keys.h"
 
     argparser = argparse.ArgumentParser()
     argparser.add_argument("--output", help="Output file", default=default_output_path)

scripts/make_generated_files.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+
+# make_generated_files.py
+#
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+"""
+Generate the TF-PSA-Crypto generated files
+"""
+import argparse
+import filecmp
+import shutil
+import subprocess
+import sys
+
+from pathlib import Path
+from typing import List, Optional
+
+from mbedtls_framework import build_tree
+
+class GenerationScript:
+    """
+    Representation of a script generating a configuration independent file.
+    """
+    # pylint: disable=too-few-public-methods
+    def __init__(self, script: Path, files: List[Path],
+                 output_dir_option: Optional[str] = None,
+                 output_file_option: Optional[str] = None):
+        """ Path from the root of Mbed TLS or TF-PSA-Crypto of the generation script """
+        self.script = script
+        """
+        List of the default paths from the Mbed TLS or TF-PSA-Crypto root of the
+        files the script generates.
+        """
+        self.files = files
+        """
+        Output directory script argument. Can be an empty string in case it is a
+        positional argument.
+        """
+        self.output_dir_option = output_dir_option
+        """
+        Output file script argument. Can be an empty string in case it is a
+        positional argument.
+        """
+        self.output_file_option = output_file_option
+
+def get_generation_script_files(generation_script: str):
+    """
+    Get the list of the default paths of the files that a given script
+    generates. It is assumed that the script supports the "--list" option.
+    """
+    files = []
+    output = subprocess.check_output([generation_script, "--list"],
+                                     universal_newlines=True)
+    for line in output.splitlines():
+        files.append(Path(line))
+
+    return files
+
+if build_tree.looks_like_tf_psa_crypto_root("."):
+    TF_PSA_CRYPTO_GENERATION_SCRIPTS = [
+        GenerationScript(
+            Path("scripts/generate_driver_wrappers.py"),
+            [Path("core/psa_crypto_driver_wrappers.h"),
+             Path("core/psa_crypto_driver_wrappers_no_static.c")],
+            "", None
+        ),
+        GenerationScript(
+            Path("framework/scripts/generate_test_keys.py"),
+            [Path("tests/include/test/test_keys.h")],
+            None, "--output"
+        ),
+        GenerationScript(
+            Path("scripts/generate_psa_constants.py"),
+            [Path("programs/psa/psa_constant_names_generated.c")],
+            "", None
+        ),
+        GenerationScript(
+            Path("framework/scripts/generate_bignum_tests.py"),
+            get_generation_script_files("framework/scripts/generate_bignum_tests.py"),
+            "--directory", None
+        ),
+        GenerationScript(
+            Path("framework/scripts/generate_config_tests.py"),
+            get_generation_script_files("framework/scripts/generate_config_tests.py"),
+            "--directory", None
+        ),
+        GenerationScript(
+            Path("framework/scripts/generate_ecp_tests.py"),
+            get_generation_script_files("framework/scripts/generate_ecp_tests.py"),
+            "--directory", None
+        ),
+        GenerationScript(
+            Path("framework/scripts/generate_psa_tests.py"),
+            get_generation_script_files("framework/scripts/generate_psa_tests.py"),
+            "--directory", None
+        ),
+    ]
+
+
+if build_tree.looks_like_mbedtls_root(".") and not build_tree.is_mbedtls_3_6():
+    MBEDTLS_GENERATION_SCRIPTS = [
+        GenerationScript(
+            Path("scripts/generate_errors.pl"),
+            [Path("library/error.c")],
+            None, "tf-psa-crypto/drivers/builtin/include/mbedtls \
+                   include/mbedtls/ \
+                   scripts/data_files"
+        ),
+        GenerationScript(
+            Path("scripts/generate_features.pl"),
+            [Path("library/version_features.c")],
+            None, "include/mbedtls/ scripts/data_files"
+        ),
+        GenerationScript(
+            Path("framework/scripts/generate_ssl_debug_helpers.py"),
+            [Path("library/ssl_debug_helpers_generated.c")],
+            "", None
+        ),
+        GenerationScript(
+            Path("framework/scripts/generate_test_keys.py"),
+            [Path("tests/include/test/test_keys.h")],
+            None, "--output"
+        ),
+        GenerationScript(
+            Path("framework/scripts/generate_test_cert_macros.py"),
+            [Path("tests/include/test/test_certs.h")],
+            None, "--output"
+        ),
+        GenerationScript(
+            Path("scripts/generate_query_config.pl"),
+            [Path("programs/test/query_config.c")],
+            None, "include/mbedtls/mbedtls_config.h \
+                   tf-psa-crypto/include/psa/crypto_config.h \
+                   scripts/data_files/query_config.fmt"
+        ),
+        GenerationScript(
+            Path("framework/scripts/generate_config_tests.py"),
+            get_generation_script_files("framework/scripts/generate_config_tests.py"),
+            "--directory", None
+        ),
+        GenerationScript(
+            Path("framework/scripts/generate_tls13_compat_tests.py"),
+            [Path("tests/opt-testcases/tls13-compat.sh")],
+            None, "--output"
+        ),
+        GenerationScript(
+            Path("scripts/generate_visualc_files.pl"),
+            get_generation_script_files("scripts/generate_visualc_files.pl"),
+            "--directory", None
+        ),
+    ]
+
+def get_generated_files(generation_scripts: List[GenerationScript]):
+    """
+    List the generated files in Mbed TLS or TF-PSA-Crypto. The path from root
+    is returned for each generated files.
+    """
+    files = []
+    for generation_script in generation_scripts:
+        files += generation_script.files
+
+    return files
+
+def make_generated_files(generation_scripts: List[GenerationScript]):
+    """
+    Generate the configuration independent files in their default location in
+    the Mbed TLS or TF-PSA-Crypto tree.
+    """
+    for generation_script in generation_scripts:
+        subprocess.run([str(generation_script.script)], check=True)
+
+def check_generated_files(generation_scripts: List[GenerationScript], root: Path):
+    """
+    Check that the given root directory contains the generated files as expected/
+    generated by this script.
+    """
+    for generation_script in generation_scripts:
+        for file in generation_script.files:
+            file = root / file
+            bak_file = file.with_name(file.name + ".bak")
+            if bak_file.exists():
+                bak_file.unlink()
+            file.rename(bak_file)
+
+        command = [str(generation_script.script)]
+        if generation_script.output_dir_option is not None:
+            command += [generation_script.output_dir_option,
+                        str(root / Path(generation_script.files[0].parent))]
+        elif generation_script.output_file_option is not None:
+            command += generation_script.output_file_option.split()
+            command += [str(root / Path(generation_script.files[0]))]
+        subprocess.run([item for item in command if item.strip()], check=True)
+
+        for file in generation_script.files:
+            file = root / file
+            bak_file = file.with_name(file.name + ".bak")
+            if not filecmp.cmp(file, bak_file):
+                ref_file = file.with_name(file.name + ".ref")
+                ref_file = root / ref_file
+                if ref_file.exists():
+                    ref_file.unlink()
+                shutil.copy(file, ref_file)
+                print(f"Generated file {file} not identical to the reference one {ref_file}.")
+            file.unlink()
+            bak_file.rename(file)
+
+def main():
+    """
+    Main function of this program
+    """
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--list', action='store_true',
+                        default=False, help='List generated files.')
+    parser.add_argument('--root', metavar='DIR',
+                        help='Root of the tree containing the generated files \
+                              to check (default: Mbed TLS or TF-PSA-Cryto root.)')
+    parser.add_argument('--check', action='store_true',
+                        default=False, help='Check the generated files in root')
+
+    args = parser.parse_args()
+
+    if not build_tree.looks_like_root("."):
+        raise RuntimeError("This script must be run from Mbed TLS or TF-PSA-Crypto root.")
+
+    if build_tree.looks_like_tf_psa_crypto_root("."):
+        generation_scripts = TF_PSA_CRYPTO_GENERATION_SCRIPTS
+    elif not build_tree.is_mbedtls_3_6():
+        generation_scripts = MBEDTLS_GENERATION_SCRIPTS
+    else:
+        raise Exception("No support for Mbed TLS 3.6")
+
+    if args.list:
+        files = get_generated_files(generation_scripts)
+        for file in files:
+            print(str(file))
+    elif args.check:
+        check_generated_files(generation_scripts, Path(args.root or "."))
+    else:
+        make_generated_files(generation_scripts)
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/mbedtls_framework/build_tree.py

@@ -73,10 +73,10 @@ def check_repo_path():
         raise Exception("This script must be run from Mbed TLS root")
 
 def chdir_to_root() -> None:
-    """Detect the root of the Mbed TLS source tree and change to it.
+    """Detect the root of the Mbed TLS or TF-PSA-Crypto source tree and change to it.
 
-    The current directory must be up to two levels deep inside an Mbed TLS
-    source tree.
+    The current directory must be up to two levels deep inside an Mbed TLS or
+    TF-PSA-Crypto source tree.
     """
     for d in [os.path.curdir,
               os.path.pardir,
@@ -84,7 +84,7 @@ def chdir_to_root() -> None:
         if looks_like_root(d):
             os.chdir(d)
             return
-    raise Exception('Mbed TLS source tree not found')
+    raise Exception('Mbed TLS or TF-PSA-Crypto source tree not found')
 
 def guess_project_root():
     """Guess project source code directory.

scripts/mbedtls_framework/code_wrapper/psa_wrapper.py

@@ -79,7 +79,7 @@ class PSAWrapper(c_wrapper_generator.Base):
         self.out_c_f = out_c_f
         self.out_h_f = out_h_f
 
-        self.mbedtls_root = build_tree.guess_mbedtls_root()
+        self.project_root = build_tree.guess_project_root()
         self.read_config(config)
         self.read_headers(in_headers)
 
@@ -99,18 +99,19 @@ class PSAWrapper(c_wrapper_generator.Base):
             c_parsing_helper.read_function_declarations(self.functions, header_path)
 
     def rel_path(self, filename: str, path_list: List[str] = ['include', 'psa']) -> str:
-        """Return the estimated path in relationship to the mbedtls_root.
+        """Return the estimated path in relationship to the project_root.
 
            The method allows overriding the targetted sub-directory.
-           Currently the default is set to mbedtls_root/include/psa."""
+           Currently the default is set to project_root/include/psa."""
         # Temporary, while Mbed TLS does not just rely on the TF-PSA-Crypto
         # build system to build its crypto library. When it does, the first
         # case can just be removed.
-        if not build_tree.is_mbedtls_3_6():
+        if build_tree.looks_like_mbedtls_root(self.project_root) and \
+           not build_tree.is_mbedtls_3_6():
             path_list = ['tf-psa-crypto' ] + path_list
-            return os.path.join(self.mbedtls_root, *path_list, filename)
+            return os.path.join(self.project_root, *path_list, filename)
 
-        return os.path.join(self.mbedtls_root, *path_list, filename)
+        return os.path.join(self.project_root, *path_list, filename)
 
     # Utility Methods
     @staticmethod