From 7f66782321a66b1b7e1e201b2cd745ce72b67040 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 23 Dec 2025 12:28:03 +0100 Subject: [PATCH 1/4] New module to load lists of config and adjusted macros Load the list of C preprocessor macros that are options in the configuration file (`mbedtls/mbedtls_config.h` or `psa/crypto_config.h`), and the list of C preprocessor macros that are defined in `*adjust*.h` but are not public options (derived internal macros). The new module `config_macros` supports both querying the current tree and querying historical data saved by `save_config_macros.sh`, with the same query interface. The part that queries historical data subsumes the `config_history` module. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/config_history.py | 2 + scripts/mbedtls_framework/config_macros.py | 99 +++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 scripts/mbedtls_framework/config_macros.py diff --git a/scripts/mbedtls_framework/config_history.py b/scripts/mbedtls_framework/config_history.py index b00dcb9db..baee92606 100644 --- a/scripts/mbedtls_framework/config_history.py +++ b/scripts/mbedtls_framework/config_history.py @@ -1,4 +1,6 @@ """Historical information about the library configuration. + +Note: this module is deprecated. Use config_macros.py instead. """ ## Copyright The Mbed TLS Contributors diff --git a/scripts/mbedtls_framework/config_macros.py b/scripts/mbedtls_framework/config_macros.py new file mode 100644 index 000000000..80c68fed6 --- /dev/null +++ b/scripts/mbedtls_framework/config_macros.py @@ -0,0 +1,99 @@ +"""Information about configuration macros and derived macros.""" + +## Copyright The Mbed TLS Contributors +## SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +import glob +import os +import re +from typing import FrozenSet, Iterable, Iterator + +from . import build_tree + + +class ConfigMacros: + """Information about configuration macros and derived macros.""" + + def __init__(self, public: FrozenSet[str], adjusted: FrozenSet[str]) -> None: + self._public = public + self._internal = adjusted - public + + def options(self) -> FrozenSet[str]: + """The set of configuration options in this product.""" + return self._public + + def internal(self) -> FrozenSet[str]: + """The set of internal option-like macros in this product.""" + return self._internal + + +class Current(ConfigMacros): + """Information about config-like macros parsed from the source code.""" + + _PUBLIC_CONFIG_HEADERS = [ + 'include/mbedtls/mbedtls_config.h', + 'include/psa/crypto_config.h', + ] + + _ADJUST_CONFIG_HEADERS = [ + 'include/**/*adjust*.h', + 'drivers/*/include/**/*adjust*.h', + ] + + _DEFINE_RE = re.compile(r'[/ ]*# *define *([A-Z_a-z][0-9A-Z_a-z]*)') + + def _list_files(self, patterns: Iterable[str]) -> Iterator[str]: + """Yield files matching the given glob patterns.""" + for pattern in patterns: + yield from glob.glob(os.path.join(self._root, self._submodule, + pattern), + recursive=True) + + def _search_file(self, filename: str) -> Iterator[str]: + """Yield macros defined in the given file.""" + with open(filename, encoding='utf-8') as input_: + for line in input_: + m = self._DEFINE_RE.match(line) + if m: + yield m.group(1) + + def _search_files(self, patterns: Iterable[str]) -> FrozenSet[str]: + """Yield macros defined in files matching the given glob patterns.""" + return frozenset(element + for filename in self._list_files(patterns) + for element in self._search_file(filename)) + + def __init__(self, submodule: str = '') -> None: + """Look for macros defined in the given submodule's source tree. + + If submodule is omitted or empty, look in the root module. + """ + self._root = build_tree.guess_project_root() + self._submodule = submodule + public = self._search_files(self._PUBLIC_CONFIG_HEADERS) + adjusted = self._search_files(self._ADJUST_CONFIG_HEADERS) + super().__init__(public, adjusted) + + +class History(ConfigMacros): + """Information about config-like macros in a previous version. + + Load files created by ``framework/scripts/save_config_history.sh``. + """ + + def _load_file(self, basename: str) -> FrozenSet[str]: + """Load macro names from the given file in the history directory.""" + filename = os.path.join(self._history_dir, basename) + with open(filename, encoding='ascii') as input_: + return frozenset(line.strip() + for line in input_) + + def __init__(self, project: str, version: str) -> None: + """Read information about the given project at the given version. + + The information must be present in history files in the framework. + """ + self._history_dir = os.path.join(build_tree.framework_root(), 'history') + public = self._load_file(f'config-options-{project}-{version}.txt') + adjusted = self._load_file(f'config-adjust-{project}-{version}.txt') + super().__init__(public, adjusted) From 530dbad92500807c06b2ed5b87abe9d6874ce9ad Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Dec 2025 15:42:48 +0100 Subject: [PATCH 2/4] Read current options from a shadow file rather than the config file The list of current options needs to be up-to-date, so we can't just use historical data. Users may edit the default config file (`include/tf-psa-crypto/crypto_config.h` or `include/mbedtls/mbedtls_config.h`). One of the reasons we use the list of config options is to prevent users from defining internal macros in their config file (in code generated by `generated_config_checks.py`). An internal macro is one that isn't listed in the official config file. So we need to know what macros are listed in the official config file, regardless of edits to the current config file. Hence we read a "shadow file" which contains the official list of options, rather than the "live" config file. This file will need to be kept up-to-date during development, but is not user-editable. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/config_macros.py | 26 ++++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/scripts/mbedtls_framework/config_macros.py b/scripts/mbedtls_framework/config_macros.py index 80c68fed6..a048670f4 100644 --- a/scripts/mbedtls_framework/config_macros.py +++ b/scripts/mbedtls_framework/config_macros.py @@ -26,10 +26,19 @@ class ConfigMacros: """The set of internal option-like macros in this product.""" return self._internal + @staticmethod + def _load_file(filename: str) -> FrozenSet[str]: + """Load macro names from the given file.""" + with open(filename, encoding='ascii') as input_: + return frozenset(line.strip() + for line in input_) + class Current(ConfigMacros): """Information about config-like macros parsed from the source code.""" + _SHADOW_FILE = 'scripts/data_files/config-options-current.txt' + _PUBLIC_CONFIG_HEADERS = [ 'include/mbedtls/mbedtls_config.h', 'include/psa/crypto_config.h', @@ -63,6 +72,10 @@ class Current(ConfigMacros): for filename in self._list_files(patterns) for element in self._search_file(filename)) + def shadow_file_path(self) -> str: + """The path to the option list shadow file.""" + return os.path.join(self._root, self._submodule, self._SHADOW_FILE) + def __init__(self, submodule: str = '') -> None: """Look for macros defined in the given submodule's source tree. @@ -70,7 +83,8 @@ class Current(ConfigMacros): """ self._root = build_tree.guess_project_root() self._submodule = submodule - public = self._search_files(self._PUBLIC_CONFIG_HEADERS) + shadow_file = self.shadow_file_path() + public = self._load_file(shadow_file) adjusted = self._search_files(self._ADJUST_CONFIG_HEADERS) super().__init__(public, adjusted) @@ -81,12 +95,10 @@ class History(ConfigMacros): Load files created by ``framework/scripts/save_config_history.sh``. """ - def _load_file(self, basename: str) -> FrozenSet[str]: + def _load_history_file(self, basename: str) -> FrozenSet[str]: """Load macro names from the given file in the history directory.""" filename = os.path.join(self._history_dir, basename) - with open(filename, encoding='ascii') as input_: - return frozenset(line.strip() - for line in input_) + return self._load_file(filename) def __init__(self, project: str, version: str) -> None: """Read information about the given project at the given version. @@ -94,6 +106,6 @@ class History(ConfigMacros): The information must be present in history files in the framework. """ self._history_dir = os.path.join(build_tree.framework_root(), 'history') - public = self._load_file(f'config-options-{project}-{version}.txt') - adjusted = self._load_file(f'config-adjust-{project}-{version}.txt') + public = self._load_history_file(f'config-options-{project}-{version}.txt') + adjusted = self._load_history_file(f'config-adjust-{project}-{version}.txt') super().__init__(public, adjusted) From 58957b982aff01edccbf745915321299eb11a111 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Dec 2025 15:47:39 +0100 Subject: [PATCH 3/4] Add code to update the shadow file Add code to compare the shadow file containing the list of config options with the options in the config file. Also add code to update (or just create) the shadow file. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/config_macros.py | 60 +++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/scripts/mbedtls_framework/config_macros.py b/scripts/mbedtls_framework/config_macros.py index a048670f4..769636891 100644 --- a/scripts/mbedtls_framework/config_macros.py +++ b/scripts/mbedtls_framework/config_macros.py @@ -76,18 +76,74 @@ class Current(ConfigMacros): """The path to the option list shadow file.""" return os.path.join(self._root, self._submodule, self._SHADOW_FILE) - def __init__(self, submodule: str = '') -> None: + def __init__(self, submodule: str = '', + shadow_missing_ok: bool = False) -> None: """Look for macros defined in the given submodule's source tree. If submodule is omitted or empty, look in the root module. + + If shadow_missing_ok is true, treat a missing shadow file as + if it was empty. This is intended for use only when regenerating + the shadow file. """ self._root = build_tree.guess_project_root() self._submodule = submodule shadow_file = self.shadow_file_path() - public = self._load_file(shadow_file) + try: + public = self._load_file(shadow_file) + except FileNotFoundError: + if not shadow_missing_ok: + raise + public = frozenset() adjusted = self._search_files(self._ADJUST_CONFIG_HEADERS) super().__init__(public, adjusted) + def live_config_options(self) -> FrozenSet[str]: + """Return config options from the config file (as opposed to the shadow file).""" + return self._search_files(self._PUBLIC_CONFIG_HEADERS) + + def compare_shadow_file(self) -> Iterator[str]: + """Compare the option list shadow file with the live config file. + + Yield the names that are only found in one of them, in a diff-like + format: prefixed by ``+`` if the name is missing from the shadow file, + or by ``-`` if the name is only in the shadow file. + """ + live = self.live_config_options() + for x in sorted(live | self._public): + if x not in live: + yield '+' + x + elif x not in self._public: + yield '-' + x + + def compare_shadow_file_verbosely(self) -> bool: + """Compare the shadow file with the live config file. Print differences. + + Return True if they have the same data, False otherwise. + """ + same = True + for line in self.compare_shadow_file(): + same = False + print(line) + return same + + def update_shadow_file(self, always_update: bool) -> None: + """Update the shadow file from the live config file. + + If always_update is false and the shadow file already has the desired + content, don't touch it. + """ + if not always_update: + try: + next(self.compare_shadow_file()) + except StopIteration: + # The file is already up-to-date. Don't touch it. + return + with open(self.shadow_file_path(), 'w') as out: + for name in sorted(self.live_config_options()): + out.write(name + '\n') + + class History(ConfigMacros): """Information about config-like macros in a previous version. From fa64d11a815bf855ac58ba0837473268e3f73297 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 2 Jan 2026 22:20:46 +0100 Subject: [PATCH 4/4] Simplify shadow file comparison interface Provide one method to just give a boolean result, and one to give a diff-like output as a simple string. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/config_macros.py | 39 +++++++++------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/scripts/mbedtls_framework/config_macros.py b/scripts/mbedtls_framework/config_macros.py index 769636891..a30fb8af0 100644 --- a/scripts/mbedtls_framework/config_macros.py +++ b/scripts/mbedtls_framework/config_macros.py @@ -102,30 +102,27 @@ class Current(ConfigMacros): """Return config options from the config file (as opposed to the shadow file).""" return self._search_files(self._PUBLIC_CONFIG_HEADERS) - def compare_shadow_file(self) -> Iterator[str]: + def is_shadow_file_up_to_date(self) -> bool: + """Whether the config options shadow file is up to date.""" + live = self.live_config_options() + return live == self._public + + def compare_shadow_file(self) -> str: """Compare the option list shadow file with the live config file. - Yield the names that are only found in one of them, in a diff-like - format: prefixed by ``+`` if the name is missing from the shadow file, - or by ``-`` if the name is only in the shadow file. + Return a string containing the names that are only found in one of + them, in a diff-like format: a line prefixed by ``+`` if the name + is missing from the shadow file, or by ``-`` if the name is only + in the shadow file. """ live = self.live_config_options() + diff = [] for x in sorted(live | self._public): if x not in live: - yield '+' + x + diff.append('+' + x + '\n') elif x not in self._public: - yield '-' + x - - def compare_shadow_file_verbosely(self) -> bool: - """Compare the shadow file with the live config file. Print differences. - - Return True if they have the same data, False otherwise. - """ - same = True - for line in self.compare_shadow_file(): - same = False - print(line) - return same + diff.append('-' + x + '\n') + return ''.join(diff) def update_shadow_file(self, always_update: bool) -> None: """Update the shadow file from the live config file. @@ -133,12 +130,8 @@ class Current(ConfigMacros): If always_update is false and the shadow file already has the desired content, don't touch it. """ - if not always_update: - try: - next(self.compare_shadow_file()) - except StopIteration: - # The file is already up-to-date. Don't touch it. - return + if not always_update and self.is_shadow_file_up_to_date(): + return with open(self.shadow_file_path(), 'w') as out: for name in sorted(self.live_config_options()): out.write(name + '\n')