make_generated_files.py: Split data from code

Initial work to eventually have the list of
scripts that generate files in the branches
that consume the framework.

Signed-off-by: Ronald Cron <[email protected]>

scripts/check-python-files.sh

@@ -58,8 +58,13 @@ echo 'Running pylint ...'
 # Temporary workaround while moving the bulk of abi_check.py to the framework
 # Check abi_check.py separately from the rest of the files, so it's not flagged
 # for code duplication.
+# Temporary workaround for the transitional wrapper framework/scripts/make_generated_files.py
+# as well. Once make_generated_files.py exists in both MbedTLS:development and
+# TF-PSA-Crypto:development branches, we will be able to remove
+# framework/scripts/make_generated_files.py and, consequently, this exception.
 find framework/scripts/*.py framework/scripts/mbedtls_framework/*.py scripts/*.py tests/scripts/*.py \
-     -path scripts/abi_check.py \
+     ! -path scripts/abi_check.py \
+     ! -path framework/scripts/make_generated_files.py \
         -exec $PYTHON -m pylint {} + \
      -o -exec $PYTHON -m pylint {} + || {
     echo >&2 "pylint reported errors"

scripts/make_generated_files.py

@@ -1,77 +1,17 @@
 #!/usr/bin/env python3
+"""Generate, check and list the generated files
+Transitional wrapper to facilitate the migration of consuming branches.
+"""
 
-# make_generated_files.py
-#
 # Copyright The Mbed TLS Contributors
 # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
 
-"""
-Generate the TF-PSA-Crypto generated files
-"""
-import argparse
-import filecmp
-import shutil
-import subprocess
 import sys
-
 from pathlib import Path
-from typing import List, Optional
 
 from mbedtls_framework import build_tree
-
+from mbedtls_framework import generated_files
-class GenerationScript:
+from mbedtls_framework.generated_files import GenerationScript, get_generation_script_files
-    """
-    Representation of a script generating a configuration independent file.
-    """
-    # pylint: disable=too-few-public-methods,too-many-arguments
-    def __init__(self, script: Path, files: List[Path],
-                 output_dir_option: Optional[str] = None,
-                 output_file_option: Optional[str] = None,
-                 optional: bool = False) -> None:
-        # Path from the root of Mbed TLS or TF-PSA-Crypto of the generation script
-        self.script = script
-
-        # Executable to run the script, needed for Windows
-        if script.suffix == ".py":
-            self.exe = sys.executable
-        elif script.suffix == ".pl":
-            self.exe = "perl"
-
-        # List of the default paths from the Mbed TLS or TF-PSA-Crypto root of the
-        # files the script generates.
-        self.files = files
-
-        # Output directory script argument. Can be an empty string in case it is a
-        # positional argument.
-        self.output_dir_option = output_dir_option
-
-        # Output file script argument. Can be an empty string in case it is a
-        # positional argument.
-        self.output_file_option = output_file_option
-
-        # Optional files are skipped in --check mode if they don't exist.
-        # This normally shouldn't happen, but it can happen during transition
-        # periods where we're adding a new script or a new file, and a
-        # consuming repository hasn't been updated yet.
-        self.optional = optional
-
-def get_generation_script_files(generation_script: str) -> List[Path]:
-    """
-    Get the list of the default paths of the files that a given script
-    generates. It is assumed that the script supports the "--list" option.
-    """
-    files = []
-    if generation_script.endswith(".py"):
-        cmd = [sys.executable]
-    elif generation_script.endswith(".pl"):
-        cmd = ["perl"]
-    cmd += [generation_script, "--list"]
-
-    output = subprocess.check_output(cmd, universal_newlines=True)
-    for line in output.splitlines():
-        files.append(Path(line))
-
-    return files
 
 COMMON_GENERATION_SCRIPTS = [
     GenerationScript(
@@ -182,93 +122,7 @@ if build_tree.looks_like_mbedtls_root(".") and not build_tree.is_mbedtls_3_6():
                 get_generation_script_files("scripts/generate_visualc_files.pl"),
                 "--directory", None))
 
-def get_generated_files(generation_scripts: List[GenerationScript]) -> List[Path]:
-    """
-    List the generated files in Mbed TLS or TF-PSA-Crypto. The path from root
-    is returned for each generated files.
-    """
-    files = []
-    for generation_script in generation_scripts:
-        files += generation_script.files
-
-    return files
-
-def make_generated_files(generation_scripts: List[GenerationScript]) -> None:
-    """
-    Generate the configuration independent files in their default location in
-    the Mbed TLS or TF-PSA-Crypto tree.
-    """
-    for generation_script in generation_scripts:
-        subprocess.run([generation_script.exe, str(generation_script.script)], check=True)
-
-def check_generated_files(generation_scripts: List[GenerationScript],
-                          root: Path) -> bool:
-    """
-    Check that the given root directory contains the generated files as expected/
-    generated by this script.
-    """
-    ok = True
-    for generation_script in generation_scripts:
-        for file in generation_script.files:
-            file = root / file
-            if not file.exists():
-                # If the script is just being added, allow its files not
-                # to exist. This can happen, at least, when adding a new
-                # generation script in crypto: until mbedtls is updated,
-                # the files from that script won't be present when
-                # the updated crypto is built from mbedtls development.
-                if generation_script.optional:
-                    continue
-                raise Exception(f"Expected generated file does not exist: {file}")
-            bak_file = file.with_name(file.name + ".bak")
-            if bak_file.exists():
-                bak_file.unlink()
-            file.rename(bak_file)
-
-        command = [generation_script.exe, str(generation_script.script)]
-        if generation_script.output_dir_option is not None:
-            command += [generation_script.output_dir_option,
-                        str(root / Path(generation_script.files[0].parent))]
-        elif generation_script.output_file_option is not None:
-            command += generation_script.output_file_option.split()
-            command += [str(root / Path(generation_script.files[0]))]
-        subprocess.run([item for item in command if item.strip()], check=True)
-
-        for file in generation_script.files:
-            file = root / file
-            bak_file = file.with_name(file.name + ".bak")
-            if generation_script.optional and not bak_file.exists():
-                # This file is optional and didn't exist before, so
-                # there's nothing to compare to, or clean up.
-                continue
-            if not filecmp.cmp(file, bak_file):
-                ok = False
-                ref_file = file.with_name(file.name + ".ref")
-                ref_file = root / ref_file
-                if ref_file.exists():
-                    ref_file.unlink()
-                shutil.copy(file, ref_file)
-                print(f"Generated file {file} not identical to the reference one {ref_file}.")
-            file.unlink()
-            bak_file.rename(file)
-    return ok
-
 def main() -> int:
-    """
-    Main function of this program
-    """
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument('--list', action='store_true',
-                        default=False, help='List generated files.')
-    parser.add_argument('--root', metavar='DIR',
-                        help='Root of the tree containing the generated files \
-                              to check (default: Mbed TLS or TF-PSA-Cryto root.)')
-    parser.add_argument('--check', action='store_true',
-                        default=False, help='Check the generated files in root')
-
-    args = parser.parse_args()
-
     if not build_tree.looks_like_root("."):
         raise RuntimeError("This script must be run from Mbed TLS or TF-PSA-Crypto root.")
 
@@ -280,17 +134,7 @@ def main() -> int:
         raise Exception("No support for Mbed TLS 3.6")
     generation_scripts += COMMON_GENERATION_SCRIPTS
 
-    if args.list:
+    return generated_files.main(generation_scripts)
-        files = get_generated_files(generation_scripts)
-        for file in files:
-            print(str(file))
-        return 0
-    elif args.check:
-        ok = check_generated_files(generation_scripts, Path(args.root or "."))
-        return 0 if ok else 1
-    else:
-        make_generated_files(generation_scripts)
-        return 0 # Any error causes an exception
 
 if __name__ == "__main__":
     sys.exit(main())

scripts/mbedtls_framework/generated_files.py

@@ -0,0 +1,167 @@
+"""Generic code to generate, check and list the generated files
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+import argparse
+import filecmp
+import shutil
+import subprocess
+import sys
+
+from pathlib import Path
+from typing import List, Optional
+
+class GenerationScript:
+    """
+    Representation of a script generating a configuration independent file.
+    """
+    # pylint: disable=too-few-public-methods,too-many-arguments
+    def __init__(self, script: Path, files: List[Path],
+                 output_dir_option: Optional[str] = None,
+                 output_file_option: Optional[str] = None,
+                 optional: bool = False) -> None:
+        # Path from the root of Mbed TLS or TF-PSA-Crypto of the generation script
+        self.script = script
+
+        # Executable to run the script, needed for Windows
+        if script.suffix == ".py":
+            self.exe = sys.executable
+        elif script.suffix == ".pl":
+            self.exe = "perl"
+
+        # List of the default paths from the Mbed TLS or TF-PSA-Crypto root of the
+        # files the script generates.
+        self.files = files
+
+        # Output directory script argument. Can be an empty string in case it is a
+        # positional argument.
+        self.output_dir_option = output_dir_option
+
+        # Output file script argument. Can be an empty string in case it is a
+        # positional argument.
+        self.output_file_option = output_file_option
+
+        # Optional files are skipped in --check mode if they don't exist.
+        # This normally shouldn't happen, but it can happen during transition
+        # periods where we're adding a new script or a new file, and a
+        # consuming repository hasn't been updated yet.
+        self.optional = optional
+
+def get_generation_script_files(generation_script: str) -> List[Path]:
+    """
+    Get the list of the default paths of the files that a given script
+    generates. It is assumed that the script supports the "--list" option.
+    """
+    files = []
+    if generation_script.endswith(".py"):
+        cmd = [sys.executable]
+    elif generation_script.endswith(".pl"):
+        cmd = ["perl"]
+    cmd += [generation_script, "--list"]
+
+    output = subprocess.check_output(cmd, universal_newlines=True)
+    for line in output.splitlines():
+        files.append(Path(line))
+
+    return files
+
+def get_generated_files(generation_scripts: List[GenerationScript]) -> List[Path]:
+    """
+    List the generated files in Mbed TLS or TF-PSA-Crypto. The path from root
+    is returned for each generated files.
+    """
+    files = []
+    for generation_script in generation_scripts:
+        files += generation_script.files
+
+    return files
+
+def make_generated_files(generation_scripts: List[GenerationScript]) -> None:
+    """
+    Generate the configuration independent files in their default location in
+    the Mbed TLS or TF-PSA-Crypto tree.
+    """
+    for generation_script in generation_scripts:
+        subprocess.run([generation_script.exe, str(generation_script.script)], check=True)
+
+def check_generated_files(generation_scripts: List[GenerationScript],
+                          root: Path) -> bool:
+    """
+    Check that the given root directory contains the generated files as expected/
+    generated by this script.
+    """
+    ok = True
+    for generation_script in generation_scripts:
+        for file in generation_script.files:
+            file = root / file
+            if not file.exists():
+                # If the script is just being added, allow its files not
+                # to exist. This can happen, at least, when adding a new
+                # generation script in crypto: until mbedtls is updated,
+                # the files from that script won't be present when
+                # the updated crypto is built from mbedtls development.
+                if generation_script.optional:
+                    continue
+                raise Exception(f"Expected generated file does not exist: {file}")
+            bak_file = file.with_name(file.name + ".bak")
+            if bak_file.exists():
+                bak_file.unlink()
+            file.rename(bak_file)
+
+        command = [generation_script.exe, str(generation_script.script)]
+        if generation_script.output_dir_option is not None:
+            command += [generation_script.output_dir_option,
+                        str(root / Path(generation_script.files[0].parent))]
+        elif generation_script.output_file_option is not None:
+            command += generation_script.output_file_option.split()
+            command += [str(root / Path(generation_script.files[0]))]
+        subprocess.run([item for item in command if item.strip()], check=True)
+
+        for file in generation_script.files:
+            file = root / file
+            bak_file = file.with_name(file.name + ".bak")
+            if generation_script.optional and not bak_file.exists():
+                # This file is optional and didn't exist before, so
+                # there's nothing to compare to, or clean up.
+                continue
+            if not filecmp.cmp(file, bak_file):
+                ok = False
+                ref_file = file.with_name(file.name + ".ref")
+                ref_file = root / ref_file
+                if ref_file.exists():
+                    ref_file.unlink()
+                shutil.copy(file, ref_file)
+                print(f"Generated file {file} not identical to the reference one {ref_file}.")
+            file.unlink()
+            bak_file.rename(file)
+    return ok
+
+def main(generation_scripts: List[GenerationScript]) -> int:
+    """
+    Main function of this program
+    """
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--list', action='store_true',
+                        default=False, help='List generated files.')
+    parser.add_argument('--root', metavar='DIR',
+                        help='Root of the tree containing the generated files \
+                              to check (default: Mbed TLS or TF-PSA-Cryto root.)')
+    parser.add_argument('--check', action='store_true',
+                        default=False, help='Check the generated files in root')
+
+    args = parser.parse_args()
+
+    if args.list:
+        files = get_generated_files(generation_scripts)
+        for file in files:
+            print(str(file))
+        return 0
+    elif args.check:
+        ok = check_generated_files(generation_scripts, Path(args.root or "."))
+        return 0 if ok else 1
+    else:
+        make_generated_files(generation_scripts)
+        return 0 # Any error causes an exception