New utility Python module: generate_files_helper

A new utility to give a common interface to committed generated files.

For the time being, this module is only intended for committed generated
files, and `make_generated_files_common.py` is only intended for non-committed
generated files. The two may be unified at some point in the future.

Signed-off-by: Gilles Peskine <[email protected]>

scripts/mbedtls_framework/config_macros.py

@@ -6,9 +6,10 @@
 import glob
 import os
 import re
-from typing import FrozenSet, Iterable, Iterator
+from typing import FrozenSet, Iterable, Iterator, List
 
 from . import build_tree
+from . import generate_files_helper
 
 
 class ConfigMacros:
@@ -34,7 +35,7 @@ class ConfigMacros:
                              for line in input_)
 
 
-class Current(ConfigMacros):
+class Current(ConfigMacros, generate_files_helper.Generator):
     """Information about config-like macros parsed from the source code."""
 
     _SHADOW_FILE = 'scripts/data_files/config-options-current.txt'
@@ -136,6 +137,25 @@ class Current(ConfigMacros):
             for name in sorted(self.live_config_options()):
                 out.write(name + '\n')
 
+    # Implement the generate_files_helper.Generator interface
+    def generator_name(self) -> str:
+        """Name as a generate_files_helper.Generator."""
+        return 'options'
+
+    def target_files(self) -> List[str]:
+        """List the (single) generated file name."""
+        return [os.path.join(self._submodule, self._SHADOW_FILE)]
+
+    def outdated_files(self) -> List[str]:
+        """List the (single) generated file name if it is out of date."""
+        if self.is_shadow_file_up_to_date():
+            return []
+        else:
+            return self.target_files()
+
+    def update(self, always: bool) -> None:
+        """Update the shadow file from the live config file."""
+        self.update_shadow_file(always)
 
 
 class History(ConfigMacros):

scripts/mbedtls_framework/generate_files_helper.py

@@ -0,0 +1,147 @@
+"""Utilities for intermediate files that are generated, but platform-independent
+and configuration-independent.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+import argparse
+import sys
+from typing import Dict, Iterable, List, Sequence, Set
+
+
+class Generator:
+    """An abstract base class for generators of intermediate files."""
+
+    def generator_name(self) -> str:
+        """A name for this generator.
+
+        Generator names must be unique and should not be identical to
+        the name of any target.
+        """
+        raise NotImplementedError
+
+    def target_files(self) -> List[str]:
+        """The list of files targeted by this generator.
+
+        File names are relative to the project root.
+        """
+        raise NotImplementedError
+
+    def outdated_files(self) -> Iterable[str]:
+        """Return the list of targets that are out of date.
+
+        This is empty after running update().
+        Missing targets are considered out of date.
+        """
+        raise NotImplementedError
+
+    def update(self, always: bool) -> None:
+        """Update the target(s) of this generator.
+
+        If always is false, avoid changing the output file if it already has
+        the desired content. If always is true, make sure to update the
+        time stamp on the output file even if it already has the desired content.
+        """
+        raise NotImplementedError
+
+
+def assemble(available: Iterable[Generator]) -> Dict[str, Generator]:
+    """Assemble the generators into a dictionary with both names and targets as keys."""
+    by_ident = {} #type: Dict[str, Generator]
+    for generator in available:
+        ident = generator.generator_name()
+        if ident in by_ident:
+            raise Exception(f'Generator conflict: name "{ident}" of {generator} '
+                            f'already recorded for {by_ident[ident]}')
+        by_ident[ident] = generator
+        for ident in generator.target_files():
+            if ident in by_ident:
+                raise Exception(f'Generator conflict: target "{ident}" of {generator} '
+                                f'already recorded for {by_ident[ident]}')
+            by_ident[ident] = generator
+    return by_ident
+
+def list_names(available: Iterable[Generator]) -> List[str]:
+    """Return the list of generator names."""
+    return sorted(generator.generator_name() for generator in available)
+
+def list_targets(available: Iterable[Generator]) -> List[str]:
+    """Return the list of generator targets."""
+    return sorted(target
+                  for generator in available
+                  for target in generator.target_files())
+
+def select(available: Dict[str, Generator],
+           wanted: Iterable[str]) -> List[Generator]:
+    """Select generators by name or target."""
+    wanted_names = set() #type: Set[str]
+    for ident in wanted:
+        if ident not in available:
+            raise Exception(f'No generator found for {ident}')
+        wanted_names.add(ident)
+    return [available[name] for name in sorted(wanted_names)]
+
+def main(generators: Sequence[Generator],
+         description: str) -> None:
+    #pylint: disable=too-many-branches
+    """Command line entry point.
+    """
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('--always-update', '-U',
+                        action='store_true',
+                        help=('Update target files unconditionally '
+                              '(overrides --update)'))
+    parser.add_argument('--list',
+                        action='store_true',
+                        help='List generator names and targets and exit')
+    parser.add_argument('--list-names',
+                        action='store_true',
+                        help='List generator names and exit')
+    parser.add_argument('--list-targets',
+                        action='store_true',
+                        help='List generator targets and exit')
+    parser.add_argument('--update', '-u',
+                        action='store_true',
+                        help='Update target files if needed')
+    parser.add_argument('--verbose', '-v',
+                        action='store_true',
+                        help='Be more verbose')
+    parser.add_argument('idents', nargs='*', metavar='NAME|TARGET',
+                        help='List of generator names or targets (all targets if empty)')
+    args = parser.parse_args()
+
+    if args.list:
+        args.list_names = True
+        args.list_targets = True
+    if args.list_names:
+        for name in list_names(generators):
+            print(name)
+    if args.list_targets:
+        for target in list_targets(generators):
+            print(target)
+    if args.list_names or args.list_targets:
+        return
+
+    if args.idents:
+        available = assemble(generators)
+        wanted = select(available, args.idents) #type: Sequence[Generator]
+    else:
+        wanted = generators
+    if args.update or args.always_update:
+        for generator in wanted:
+            if args.verbose:
+                sys.stderr.write(f'Running generator {generator.generator_name()}...\n')
+            generator.update(args.always_update)
+    else:
+        outdated = [] #type: List[str]
+        for generator in wanted:
+            if args.verbose:
+                sys.stderr.write(f'Checking targets of generator {generator.generator_name()}...\n')
+            outdated += generator.outdated_files()
+        if outdated:
+            sys.stderr.write(f'Some targets are missing or out of date.\n')
+            for target in outdated:
+                print(target)
+            sys.stderr.write(f'Run {sys.argv[0]} -u and commit the result.')
+            sys.exit(1)