From ad953606901d9fb75a5c79cbf6a010a691bb68f6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 27 Oct 2025 09:40:50 +0100 Subject: [PATCH 01/25] Create a release preparation script Set up the armature of a release script: command line interface, information about the build tree, abstract class for release preparation steps. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 294 +++++++++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100755 scripts/prepare_release.py diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py new file mode 100755 index 000000000..aed33294a --- /dev/null +++ b/scripts/prepare_release.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Prepare to release TF-PSA-Crypto or Mbed TLS. + +This script constructs a release candidate branch and release artifacts. +It must run from a clean git worktree with initialized, clean git submodules. +When making an Mbed TLS >=4 release, the tf-psa-crypto submodule should +already contain a TF-PSA-Crypto release candidate. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +################################################################ + +#### Tips for maintainers #### + +# This script insists on having a clean git checkout, including +# submodules. If you want to modify it, either make your changes in +# a separate worktree, or commit your changes first and update +# submodules in the parent repositories. + +#### Architectural notes #### + +# The process of preparing a release consists of some information +# gathering about the branch to release in the `Info` class, then +# a series of steps in `XxxStep` classes. + +# Information gathering is performed by the `Info` constructor. +# Any failure causes the script to abort. + +# The steps run sequentially in the order defined by `ALL_STEPS`. +# Each step starts by checking precondition in `step.assert_preconditions()`, +# then does its job in `step.run()`. `assert_preconditions` should +# not change any state. `run` is expected to change state and is +# not expected to clean up if an error occurs. + +# This script is expected to perform a release reliably, so it +# should try to detect dodgy conditions and abort. It should avoid +# silently outputting garbage. Nonetheless, human review of the +# result is still expected. + +################################################################ + + +import argparse +import os +import pathlib +import re +import subprocess +import sys +import typing +from typing import Dict, List, Optional, Sequence + + +PathOrString = typing.Union[os.PathLike, str] + + +class InformationGatheringError(Exception): + """Problem detected while gathering information.""" + pass + + +class Options(typing.NamedTuple): + """Options controlling the behavior of the release process. + + Each field here should have an associated command line option + declared in main(). + """ + # Directory where the release artifacts will be placed. + artifact_directory: pathlib.Path + # Version to release (None to read it from ChangeLog). + version: Optional[str] + +DEFAULT_OPTIONS = Options( + artifact_directory=pathlib.Path(os.pardir), + version=None) + + +class Info: + """Information about the intended release. + + This class contains information gathered from the product tree as well + as from the command line. + """ + + def _read_product_info(self) -> None: + """Read information from the source files.""" + with open(self.top_dir / 'ChangeLog', encoding='utf-8') as changelog: + # We deliberately read only a short portion at the top of the + # changelog file. This reduces the risk that we'll match an + # older release if the content near the top of the file doesn't + # have the expected format. + head = changelog.read(1000) + m = re.search(r'^= *(.*?) +([0-9][-.0-9A-Za-z]+) +branch released (\S+)\n', + head, re.M) + if not m: + raise InformationGatheringError( + 'Could not find version header line near the top of ChangeLog') + self._product_human_name = m.group(1) + self.old_version = m.group(2) + self.old_release_date = m.group(3) + + @staticmethod + def _product_machine_name_from_human_name(_human_name: str) -> str: + """Determine the name used in release tags and file names.""" + if _human_name == 'TF-PSA-Crypto': + return 'tf-psa-crypto' + elif _human_name == 'Mbed TLS': + return 'mbedtls' + else: + raise InformationGatheringError( + f'Could not determine product (found "{_human_name}" in ChangeLog)') + + def __init__(self, top_dir: str, options: Options) -> None: + """Parse the source files to obtain information about the product. + + Look for the product under `top_dir`. + + Optional parameters can override the supplied information: + * `version`: version to release. + """ + self.top_dir = pathlib.Path(top_dir) + self._read_product_info() + if options.version is None: + self._release_version = self.old_version + else: + self._release_version = options.version + self._product_machine_name = \ + self._product_machine_name_from_human_name(self._product_human_name) + + @property + def product_human_name(self) -> str: + """Human-readable product name, e.g. 'Mbed TLS'.""" + return self._product_human_name + + @property + def product_machine_name(self) -> str: + """Machine-frieldly product name, e.g. 'mbedtls'.""" + return self._product_machine_name + + @property + def version(self) -> str: + """Version string for the relase.""" + return self._release_version + + +class Step: + """A step on the release process. + + This is an abstract class containing some common tooling. + Subclasses must provide the following methods: + * `name()` returning a unique name for each step. + * `run()` doing the work. Raise an exception if something goes wrong. + + Subclasses may override the following methods: + * `assert_preconditions()` is called to perform sanity checks before + calling `run()`. + """ + + @classmethod + def name(cls) -> str: + """The name of this step.""" + raise NotImplementedError + + def __init__(self, options: Options, info: Info) -> None: + """Instantiate the release step for the given product directory. + + This constructor may analyze the contents of the product tree, + but it does not require other steps to have run. + """ + self.options = options + self.info = info + self._submodules = None #type: Optional[List[str]] + + @staticmethod + def _git_command(subcommand: List[str], + where: Optional[PathOrString] = None) -> List[str]: + cmd = ['git'] + if where is not None: + cmd += ['-C', str(where)] + return cmd + subcommand + + def call_git(self, cmd: List[str], + where: Optional[PathOrString] = None, + env: Optional[Dict[str, str]] = None) -> None: + """Run git in the source tree. + + Pass `where` to specify a submodule. + """ + subprocess.check_call(self._git_command(cmd, where), + cwd=self.info.top_dir, + env=env) + + def read_git(self, cmd: List[str], + where: Optional[PathOrString] = None, + env: Optional[Dict[str, str]] = None) -> bytes: + """Run git in the source tree and return the output. + + Pass `where` to specify a submodule. + """ + return subprocess.check_output(self._git_command(cmd, where), + cwd=self.info.top_dir, + env=env) + + @property + def submodules(self) -> Sequence[str]: + """List the git submodules (recursive, but not including the top level).""" + if self._submodules is None: + raw = self.read_git(['submodule', '--quiet', + 'foreach', '--recursive', + 'printf %s\\\\0 "$displaypath"']) + self._submodules = raw.decode('ascii').rstrip('\0').split('\0') + return self._submodules + + def assert_git_status(self) -> None: + """Abort if the working directory is not clean (no git changes). + + This includes git submodules. + """ + self.call_git(['diff', '--quiet']) + + + def assert_preconditions(self) -> None: + """Check whether the preconditions for this step have been achieved. + + If not, raise an exception. + """ + self.assert_git_status() + + def run(self) -> None: + """Perform the release preparation step. + + This may create commits in the source tree or its submodules. + """ + raise NotImplementedError + + +ALL_STEPS = [ +] #type: Sequence[typing.Type[Step]] + + +def run(options: Options, + top_dir: str, + from_: Optional[str] = None, + to: Optional[str] = None) -> None: + """Run the release process (or a segment thereof).""" + info = Info(top_dir, options) + from_reached = (from_ is None) + for step_class in ALL_STEPS: + step = step_class(options, info) + if not from_reached: + if step.name() != from_: + continue + from_reached = True + step.assert_preconditions() + step.run() + if step.name() == to: + break + +def main() -> None: + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + # Options that affect information gathering, or that affect the + # behavior of one or more steps, should have an associated field + # in the Options class. + parser.add_argument('--directory', + default=os.curdir, + help='Product toplevel directory') + parser.add_argument('--artifact-directory', '-a', + help='Directory where release artifacts will be placed') + parser.add_argument('--from', metavar='STEP', + dest='from_', + help='First step to run (default: run all steps)') + parser.add_argument('--list-steps', + action='store_true', + help='List release steps and exit') + parser.add_argument('--to', metavar='STEP', + help='Last step to run (default: run all steps)') + parser.add_argument('version', nargs='?', + help='The version to release (default: from ChangeLog)') + parser.set_defaults(**DEFAULT_OPTIONS._asdict()) + args = parser.parse_args() + if args.list_steps: + for step in ALL_STEPS: + sys.stdout.write(step.name() + '\n') + return + options = Options( + artifact_directory=pathlib.Path(args.artifact_directory).absolute(), + version=args.version) + run(options, args.directory, + from_=args.from_, to=args.to) + +if __name__ == '__main__': + main() From 3ef14048e073a68879a5cc0dcabb337063ef6454 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 27 Oct 2025 09:42:00 +0100 Subject: [PATCH 02/25] Fix absolute path in generated file That made the build non-reproducible. Addresses https://github.com/Mbed-TLS/mbedtls/issues/9521 Signed-off-by: Gilles Peskine --- data_files/test_certs.h.jinja2 | 4 +- scripts/generate_test_cert_macros.py | 72 ++++++++++++++-------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/data_files/test_certs.h.jinja2 b/data_files/test_certs.h.jinja2 index c420c7964..5be1c8e7b 100644 --- a/data_files/test_certs.h.jinja2 +++ b/data_files/test_certs.h.jinja2 @@ -10,7 +10,7 @@ {% for mode, name, value in macros %} {% if mode == 'string' %} -/* This is taken from {{value}}. */ +/* This is taken from {{data_dir}}/{{value}}. */ /* BEGIN FILE string macro {{name}} {{value}} */ #define {{name}}{{ '\\' | put_to_column(position=80-9-name|length)}} {% for line in value | read_lines %} @@ -21,7 +21,7 @@ /* END FILE */ {% endif %} {% if mode == 'binary' %} -/* This is generated from {{value}}. */ +/* This is generated from {{data_dir}}/{{value}}. */ /* BEGIN FILE binary macro {{name}} {{value}} */ #define {{name}} {% raw -%} { {%- endraw %} {{ '\\' | put_to_column(position=80-11-name|length)}} {% for line in value | read_as_c_array %} diff --git a/scripts/generate_test_cert_macros.py b/scripts/generate_test_cert_macros.py index f10ead6f7..e089e86ec 100755 --- a/scripts/generate_test_cert_macros.py +++ b/scripts/generate_test_cert_macros.py @@ -16,40 +16,40 @@ import jinja2 from mbedtls_framework.build_tree import guess_project_root TESTS_DIR = os.path.join(guess_project_root(), 'tests') -FRAMEWORK_DIR = os.path.join(guess_project_root(), 'framework') -DATA_FILES_PATH = os.path.join(FRAMEWORK_DIR, 'data_files') +DATA_DIR_RELATIVE = 'framework/data_files' +DATA_DIR_ABSOLUTE = os.path.join(guess_project_root(), DATA_DIR_RELATIVE) INPUT_ARGS = [ - ("string", "TEST_CA_CRT_EC_PEM", DATA_FILES_PATH + "/test-ca2.crt"), - ("binary", "TEST_CA_CRT_EC_DER", DATA_FILES_PATH + "/test-ca2.crt.der"), - ("string", "TEST_CA_KEY_EC_PEM", DATA_FILES_PATH + "/test-ca2.key.enc"), + ("string", "TEST_CA_CRT_EC_PEM", "test-ca2.crt"), + ("binary", "TEST_CA_CRT_EC_DER", "test-ca2.crt.der"), + ("string", "TEST_CA_KEY_EC_PEM", "test-ca2.key.enc"), ("password", "TEST_CA_PWD_EC_PEM", "PolarSSLTest"), - ("binary", "TEST_CA_KEY_EC_DER", DATA_FILES_PATH + "/test-ca2.key.der"), - ("string", "TEST_CA_CRT_RSA_SHA256_PEM", DATA_FILES_PATH + "/test-ca-sha256.crt"), - ("binary", "TEST_CA_CRT_RSA_SHA256_DER", DATA_FILES_PATH + "/test-ca-sha256.crt.der"), - ("string", "TEST_CA_CRT_RSA_SHA1_PEM", DATA_FILES_PATH + "/test-ca-sha1.crt"), - ("binary", "TEST_CA_CRT_RSA_SHA1_DER", DATA_FILES_PATH + "/test-ca-sha1.crt.der"), - ("string", "TEST_CA_KEY_RSA_PEM", DATA_FILES_PATH + "/test-ca.key"), + ("binary", "TEST_CA_KEY_EC_DER", "test-ca2.key.der"), + ("string", "TEST_CA_CRT_RSA_SHA256_PEM", "test-ca-sha256.crt"), + ("binary", "TEST_CA_CRT_RSA_SHA256_DER", "test-ca-sha256.crt.der"), + ("string", "TEST_CA_CRT_RSA_SHA1_PEM", "test-ca-sha1.crt"), + ("binary", "TEST_CA_CRT_RSA_SHA1_DER", "test-ca-sha1.crt.der"), + ("string", "TEST_CA_KEY_RSA_PEM", "test-ca.key"), ("password", "TEST_CA_PWD_RSA_PEM", "PolarSSLTest"), - ("binary", "TEST_CA_KEY_RSA_DER", DATA_FILES_PATH + "/test-ca.key.der"), - ("string", "TEST_SRV_CRT_EC_PEM", DATA_FILES_PATH + "/server5.crt"), - ("binary", "TEST_SRV_CRT_EC_DER", DATA_FILES_PATH + "/server5.crt.der"), - ("string", "TEST_SRV_KEY_EC_PEM", DATA_FILES_PATH + "/server5.key"), - ("binary", "TEST_SRV_KEY_EC_DER", DATA_FILES_PATH + "/server5.key.der"), - ("string", "TEST_SRV_CRT_RSA_SHA256_PEM", DATA_FILES_PATH + "/server2-sha256.crt"), - ("binary", "TEST_SRV_CRT_RSA_SHA256_DER", DATA_FILES_PATH + "/server2-sha256.crt.der"), - ("string", "TEST_SRV_CRT_RSA_SHA1_PEM", DATA_FILES_PATH + "/server2.crt"), - ("binary", "TEST_SRV_CRT_RSA_SHA1_DER", DATA_FILES_PATH + "/server2.crt.der"), - ("string", "TEST_SRV_KEY_RSA_PEM", DATA_FILES_PATH + "/server2.key"), - ("binary", "TEST_SRV_KEY_RSA_DER", DATA_FILES_PATH + "/server2.key.der"), - ("string", "TEST_CLI_CRT_EC_PEM", DATA_FILES_PATH + "/cli2.crt"), - ("binary", "TEST_CLI_CRT_EC_DER", DATA_FILES_PATH + "/cli2.crt.der"), - ("string", "TEST_CLI_KEY_EC_PEM", DATA_FILES_PATH + "/cli2.key"), - ("binary", "TEST_CLI_KEY_EC_DER", DATA_FILES_PATH + "/cli2.key.der"), - ("string", "TEST_CLI_CRT_RSA_PEM", DATA_FILES_PATH + "/cli-rsa-sha256.crt"), - ("binary", "TEST_CLI_CRT_RSA_DER", DATA_FILES_PATH + "/cli-rsa-sha256.crt.der"), - ("string", "TEST_CLI_KEY_RSA_PEM", DATA_FILES_PATH + "/cli-rsa.key"), - ("binary", "TEST_CLI_KEY_RSA_DER", DATA_FILES_PATH + "/cli-rsa.key.der"), + ("binary", "TEST_CA_KEY_RSA_DER", "test-ca.key.der"), + ("string", "TEST_SRV_CRT_EC_PEM", "server5.crt"), + ("binary", "TEST_SRV_CRT_EC_DER", "server5.crt.der"), + ("string", "TEST_SRV_KEY_EC_PEM", "server5.key"), + ("binary", "TEST_SRV_KEY_EC_DER", "server5.key.der"), + ("string", "TEST_SRV_CRT_RSA_SHA256_PEM", "server2-sha256.crt"), + ("binary", "TEST_SRV_CRT_RSA_SHA256_DER", "server2-sha256.crt.der"), + ("string", "TEST_SRV_CRT_RSA_SHA1_PEM", "server2.crt"), + ("binary", "TEST_SRV_CRT_RSA_SHA1_DER", "server2.crt.der"), + ("string", "TEST_SRV_KEY_RSA_PEM", "server2.key"), + ("binary", "TEST_SRV_KEY_RSA_DER", "server2.key.der"), + ("string", "TEST_CLI_CRT_EC_PEM", "cli2.crt"), + ("binary", "TEST_CLI_CRT_EC_DER", "cli2.crt.der"), + ("string", "TEST_CLI_KEY_EC_PEM", "cli2.key"), + ("binary", "TEST_CLI_KEY_EC_DER", "cli2.key.der"), + ("string", "TEST_CLI_CRT_RSA_PEM", "cli-rsa-sha256.crt"), + ("binary", "TEST_CLI_CRT_RSA_DER", "cli-rsa-sha256.crt.der"), + ("string", "TEST_CLI_KEY_RSA_PEM", "cli-rsa.key"), + ("binary", "TEST_CLI_KEY_RSA_DER", "cli-rsa.key.der"), ] def main(): @@ -62,7 +62,8 @@ def main(): if args.list_dependencies: files_list = [arg[2] for arg in INPUT_ARGS if arg[0] != "password"] - print(" ".join(files_list)) + print(" ".join(os.path.join(DATA_DIR_RELATIVE, filename) + for filename in files_list)) return generate(INPUT_ARGS, output=args.output) @@ -71,20 +72,20 @@ def main(): def generate(values=[], output=None): """Generate C header file. """ - template_loader = jinja2.FileSystemLoader(DATA_FILES_PATH) + template_loader = jinja2.FileSystemLoader(DATA_DIR_ABSOLUTE) template_env = jinja2.Environment( loader=template_loader, lstrip_blocks=True, trim_blocks=True, keep_trailing_newline=True) def read_as_c_array(filename): - with open(filename, 'rb') as f: + with open(os.path.join(DATA_DIR_ABSOLUTE, filename), 'rb') as f: data = f.read(12) while data: yield ', '.join(['{:#04x}'.format(b) for b in data]) data = f.read(12) def read_lines(filename): - with open(filename) as f: + with open(os.path.join(DATA_DIR_ABSOLUTE, filename)) as f: try: for line in f: yield line.strip() @@ -102,7 +103,8 @@ def generate(values=[], output=None): template = template_env.get_template('test_certs.h.jinja2') with open(output, 'w') as f: - f.write(template.render(macros=values)) + f.write(template.render(data_dir=DATA_DIR_RELATIVE, + macros=values)) if __name__ == '__main__': From f3e151299b6cde3ce53e17f86348f7c051c18f16 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 27 Oct 2025 09:44:57 +0100 Subject: [PATCH 03/25] Silence warning about timezone in datetime object Modern versions of Python emit a DeprecationWarning about `datetime.datetime.utcfromtimestamp()`. It didn't really matter here, but do stop using it. Signed-off-by: Gilles Peskine --- scripts/assemble_changelog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/assemble_changelog.py b/scripts/assemble_changelog.py index 69ee1b138..ef8c1e1fe 100755 --- a/scripts/assemble_changelog.py +++ b/scripts/assemble_changelog.py @@ -341,7 +341,8 @@ class EntryFileSortKey: text = subprocess.check_output(['git', 'show', '-s', '--format=%ct', commit_id]) - return datetime.datetime.utcfromtimestamp(int(text)) + return datetime.datetime.fromtimestamp(int(text), + datetime.timezone.utc) @staticmethod def file_timestamp(filename): From afb5d4f4c9ae7e817273a0e08ecb1b337b2a3bc7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 27 Oct 2025 10:46:39 +0100 Subject: [PATCH 04/25] Create release archive Create a release archive containing the git content and the generated files (including files from submodules), with `CMakeLists.txt` edited to turn off `GEN_FILES` by default. Arrange for the timestamps of generated files to be more recent than the timestamps of files checked into git, in a reproducible way. Uses GNU tar to assemble the pieces. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 257 ++++++++++++++++++++++++++++++++++++- 1 file changed, 256 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index aed33294a..59d6fe8cf 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -5,6 +5,10 @@ This script constructs a release candidate branch and release artifacts. It must run from a clean git worktree with initialized, clean git submodules. When making an Mbed TLS >=4 release, the tf-psa-crypto submodule should already contain a TF-PSA-Crypto release candidate. + +This script requires the following external tools: +- GNU tar (can be called ``gnutar`` or ``gtar``); +- ``sha256sum``. """ # Copyright The Mbed TLS Contributors @@ -39,17 +43,21 @@ already contain a TF-PSA-Crypto release candidate. # silently outputting garbage. Nonetheless, human review of the # result is still expected. +# If you add an external dependency to a step, please mention +# it in the docstring at the top. + ################################################################ import argparse +import datetime import os import pathlib import re import subprocess import sys import typing -from typing import Dict, List, Optional, Sequence +from typing import Callable, Dict, Iterator, List, Optional, Sequence PathOrString = typing.Union[os.PathLike, str] @@ -60,6 +68,24 @@ class InformationGatheringError(Exception): pass +def find_gnu_tar() -> str: + """Try to guess the command for GNU tar. + + This function never errors out. It defaults to "tar". + """ + for name in ['gnutar', 'gtar']: + try: + subprocess.check_call([name, '--version'], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + return name + except FileNotFoundError: + pass + except subprocess.CalledProcessError: + pass + return 'tar' + class Options(typing.NamedTuple): """Options controlling the behavior of the release process. @@ -68,11 +94,14 @@ class Options(typing.NamedTuple): """ # Directory where the release artifacts will be placed. artifact_directory: pathlib.Path + # GNU tar command. + tar_command: str # Version to release (None to read it from ChangeLog). version: Optional[str] DEFAULT_OPTIONS = Options( artifact_directory=pathlib.Path(os.pardir), + tar_command=find_gnu_tar(), version=None) @@ -212,6 +241,38 @@ class Step: self._submodules = raw.decode('ascii').rstrip('\0').split('\0') return self._submodules + def commit_timestamp(self, + where: Optional[PathOrString] = None, + what: str = 'HEAD') -> int: + """Return the timestamp (seconds since epoch) of the given commit. + + Pass `where` to specify a submodule. + Pass `what` to specify a commit (default: ``HEAD``). + """ + timestamp = self.read_git(['show', '--no-patch', '--pretty=%ct', what], + where=where) + return int(timestamp) + + def git_index_as_tree_ish(self, + where: Optional[PathOrString] = None) -> str: + """Return a git tree-ish corresponding to the index. + + Pass `where` to specify a submodule. + """ + raw = self.read_git(['write-tree'], where=where) + return raw.decode('ascii').strip() + + def commit_datetime(self, + where: Optional[PathOrString] = None, + what: str = 'HEAD') -> datetime.datetime: + """Return the time of the given commit. + + Pass `where` to specify a submodule. + Pass `what` to specify a commit (default: ``HEAD``). + """ + timestamp = self.commit_timestamp(where=where, what=what) + return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc) + def assert_git_status(self) -> None: """Abort if the working directory is not clean (no git changes). @@ -220,6 +281,36 @@ class Step: self.call_git(['diff', '--quiet']) + def artifact_base_name(self) -> str: + """The base name for a release artifact (file created for publishing). + + This contains the product name and version, with no directory part + or extension. For example "mbedtls-1.2.3". + """ + return '-'.join([self.info.product_machine_name, + self.info.version]) + + def artifact_path(self, extension: str) -> pathlib.Path: + """The path for a release artifact (file created for publishing). + + `extension` should start with a ".". + """ + file_name = self.artifact_base_name() + extension + return self.options.artifact_directory / file_name + + def edit_file(self, + path: PathOrString, + transform: Callable[[str], str]) -> None: + """Edit a text file. + + The path can be relative to the toplevel root or absolute. + """ + with open(self.info.top_dir / path, 'r+', encoding='utf-8') as file_: + new_content = transform(file_.read()) + file_.seek(0) + file_.truncate() + file_.write(new_content) + def assert_preconditions(self) -> None: """Check whether the preconditions for this step have been achieved. @@ -235,7 +326,168 @@ class Step: raise NotImplementedError +class ArchiveStep(Step): + """Prepare release archives and the associated checksum file.""" + + @classmethod + def name(cls) -> str: + return 'archive' + + def tar_git_files(self, plain_tar_path: str, prefix: str) -> None: + """Create an uncompressed tar files with the git contents.""" + # We archive the index, not the commit 'HEAD', because + # we may have modified some files to turn off GEN_FILES. + # We do use the commit mtime rather than the current time, + # however, for reproducibility. + # A downside of not archiving a commit is that the archive won't + # contain an extended header with the commit ID that + # `git get-tar-commit-id` could retrieve. If we change to releasing + # an exact commit, we should make sure that the commit gets published. + index = self.git_index_as_tree_ish() + mtime = self.commit_timestamp() + self.call_git(['archive', '--format=tar', + '--mtime', str(mtime), + '--prefix', prefix, + '--output', str(plain_tar_path), + index]) + for submodule in self.submodules: + index = self.git_index_as_tree_ish(where=submodule) + mtime = self.commit_timestamp(where=submodule) + data = self.read_git(['archive', '--format=tar', + '--mtime', str(mtime), + '--prefix', prefix + submodule + '/', + index], + where=submodule) + subprocess.run([self.options.tar_command, '--catenate', + '-f', plain_tar_path, + '/dev/stdin'], + input=data, + check=True) + + GEN_FILES_FILES = [ + 'CMakeLists.txt', + 'tf-psa-crypto/CMakeLists.txt', + ] + + @staticmethod + def set_gen_files_default_off(old_content: str) -> str: + """Make GEN_FILES default off in a build script.""" + # CMakeLists.txt + new_content = re.sub(r'(option\(GEN_FILES\b.*)\bON\)', + r'\g<1>OFF)', + old_content) + return new_content + + def turn_off_gen_files(self) -> None: + """Make GEN_FILES default off in build scripts.""" + for filename in self.GEN_FILES_FILES: + path = self.info.top_dir / filename + if path.exists(): + self.edit_file(path, self.set_gen_files_default_off) + self.call_git(['add', path.name], where=path.parent) + + def restore_gen_files(self) -> None: + """Restore build scripts affected by turn_off_gen_files().""" + for filename in self.GEN_FILES_FILES: + path = self.info.top_dir / filename + if path.exists(): + self.call_git(['reset', '--', path.name], + where=path.parent) + self.call_git(['checkout', '--', path.name], + where=path.parent) + + @staticmethod + def list_project_generated_files(project_dir: pathlib.Path) -> List[str]: + """Return the list of generated files in the given (sub)project. + + The returned file names are relative to the project root. + """ + raw_list = subprocess.check_output( + ['framework/scripts/make_generated_files.py', '--list'], + cwd=project_dir) + return raw_list.decode('ascii').rstrip('\n').split('\n') + + @staticmethod + def update_project_generated_files(project_dir: pathlib.Path) -> None: + """Update the list of generated files in the given (sub)project.""" + subprocess.check_call( + ['framework/scripts/make_generated_files.py'], + cwd=project_dir) + + def tar_add_project_generated_files(self, + plain_tar_path: str, + project_dir: pathlib.Path, + project_prefix: str, + file_list: List[str]) -> None: + transform = 's/^/' + project_prefix.replace('/', '\\/') + '/g' + commit_datetime = self.commit_datetime(project_dir) + file_datetime = commit_datetime + datetime.timedelta(seconds=1) + subprocess.check_call([self.options.tar_command, '-r', + '-f', plain_tar_path, + '--transform', transform, + '--owner=root:0', '--group=root:0', + '--mode=a+rX,u+w,go-w', + '--mtime', file_datetime.isoformat(), + '--'] + file_list, + cwd=project_dir) + + def tar_add_generated_files(self, plain_tar_path: str, prefix: str) -> None: + """Add generated files to an existing uncompressed tar file.""" + for project in [os.curdir] + list(self.submodules): + if project.endswith('/framework') or project == 'framework': + continue + project_dir = self.info.top_dir / project + project_prefix = (prefix if project == os.curdir else + prefix + project + '/') + file_list = self.list_project_generated_files(project_dir) + self.update_project_generated_files(project_dir) + self.tar_add_project_generated_files(plain_tar_path, + project_dir, + project_prefix, + file_list) + + def create_plain_tar(self, plain_tar_path: str, prefix: str) -> None: + """Create an uncompressed tar file for the release.""" + self.tar_git_files(plain_tar_path, prefix) + self.tar_add_generated_files(plain_tar_path, prefix) + + @staticmethod + def compress_tar(plain_tar_path: str) -> Iterator[str]: + """Compress the tar file. + + Remove the original, uncompressed file. + Yield the list of compressed files. + """ + compressed_path = plain_tar_path + '.bz2' + if os.path.exists(compressed_path): + os.remove(compressed_path) + subprocess.check_call(['bzip2', '-9', plain_tar_path]) + yield compressed_path + + def create_checksum_file(self, archive_paths: List[str]) -> None: + """Create a checksum file for the given files.""" + checksum_path = self.artifact_path('.txt') + relative_paths = [os.path.relpath(path, self.options.artifact_directory) + for path in archive_paths] + content = subprocess.check_output(['sha256sum', '--'] + relative_paths, + cwd=self.options.artifact_directory, + encoding='ascii') + with open(checksum_path, 'w') as out: + out.write(content) + + def run(self) -> None: + """Create the release artifacts.""" + self.turn_off_gen_files() + base_name = self.artifact_base_name() + plain_tar_path = str(self.artifact_path('.tar')) + self.create_plain_tar(plain_tar_path, base_name + '/') + compressed_paths = list(self.compress_tar(plain_tar_path)) + self.create_checksum_file(compressed_paths) + self.restore_gen_files() + + ALL_STEPS = [ + ArchiveStep, ] #type: Sequence[typing.Type[Step]] @@ -274,6 +526,8 @@ def main() -> None: parser.add_argument('--list-steps', action='store_true', help='List release steps and exit') + parser.add_argument('--tar-command', + help='GNU tar command') parser.add_argument('--to', metavar='STEP', help='Last step to run (default: run all steps)') parser.add_argument('version', nargs='?', @@ -286,6 +540,7 @@ def main() -> None: return options = Options( artifact_directory=pathlib.Path(args.artifact_directory).absolute(), + tar_command=args.tar_command, version=args.version) run(options, args.directory, from_=args.from_, to=args.to) From a802be7a02178765427657ebfb55898b13525f75 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 27 Oct 2025 10:51:13 +0100 Subject: [PATCH 05/25] Release preparation: assemble changelog Add a step to the release preparation script to assemble the changelog and commit that change into git. Make sure to create a changelog section for the desired version number even if there are no changes. Finalize that section by editing in a release date. Do nothing if the changelog already contains a finalized section for the desired version and release date. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 98 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 59d6fe8cf..32f8f9f58 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -5,6 +5,8 @@ This script constructs a release candidate branch and release artifacts. It must run from a clean git worktree with initialized, clean git submodules. When making an Mbed TLS >=4 release, the tf-psa-crypto submodule should already contain a TF-PSA-Crypto release candidate. +This script will update the checked out git branch, if any. +On normal exit, the worktree contains the release candidate commit. This script requires the following external tools: - GNU tar (can be called ``gnutar`` or ``gtar``); @@ -94,6 +96,8 @@ class Options(typing.NamedTuple): """ # Directory where the release artifacts will be placed. artifact_directory: pathlib.Path + # Release date (YYYY-mm-dd). + release_date: str # GNU tar command. tar_command: str # Version to release (None to read it from ChangeLog). @@ -101,6 +105,7 @@ class Options(typing.NamedTuple): DEFAULT_OPTIONS = Options( artifact_directory=pathlib.Path(os.pardir), + release_date=datetime.date.today().isoformat(), tar_command=find_gnu_tar(), version=None) @@ -280,6 +285,39 @@ class Step: """ self.call_git(['diff', '--quiet']) + def files_are_clean(self, *files: str, + where: Optional[PathOrString] = None) -> bool: + """Check whether the specified files are identical to their git version. + + With no files, check the whole work tree (including submodules). + + Pass `where` to specify a submodule (default: top level). + The file names are relative to the repository or submodule root, + and may not be in a submodule of `where`. + """ + try: + self.call_git(['diff', '--quiet', '--'] + list(files), + where=where) + return True + except subprocess.CalledProcessError as exn: + if exn.returncode != 1: + raise + return False + + def git_commit_maybe(self, + files: List[str], + message: str) -> None: + """Commit changes into Git. + + Do nothing if there are no changed files. + + The file names are relative to the toplevel directory, + and may not be in a submodule. + """ + if not self.files_are_clean(*files): + self.call_git(['add', '--'] + files) + self.call_git(['commit', '--signoff', + '-m', message]) def artifact_base_name(self) -> str: """The base name for a release artifact (file created for publishing). @@ -326,6 +364,62 @@ class Step: raise NotImplementedError +class AssembleChangelogStep(Step): + """Assemble the changelog and commit the result. + + Create a new changelog section if needed. + Do nothing if the changelog is already fine. + + Note: this step does not check or affect submodules. + """ + + @classmethod + def name(cls) -> str: + return 'changelog' + + def create_section(self, old_content: str) -> str: + """Create a new changelog section for the version that we're releasing.""" + product = self.info.product_human_name + version = self.info.version + new_section = f'= {product} {version} branch released xxxx-xx-xx\n\n' + return re.sub(r'(?=^=)', + new_section, + old_content, flags=re.MULTILINE, count=1) + + def release_date_needs_updating(self) -> bool: + """Whether the release date needs updating in the existing ChangeLog section.""" + m = re.match(r'([0-9]+)-([0-9]+)-([0-9]+)', self.info.old_release_date) + if not m: # presumably xxxx-xx-xx + return True + # The date format is a lexicographic, fixed-width format, + # so we can just compare the strings. + return self.info.old_release_date < self.options.release_date + + def finalize_release(self, old_content: str) -> str: + """Update the version and release date in the changelog content.""" + version = self.info.version + date = self.options.release_date + return re.sub(r'^(=.* )\S+( branch released )\S+$', + rf'\g<1>{version}\g<2>{date}', + old_content, flags=re.MULTILINE, count=1) + + def run(self) -> None: + """Assemble the changelog if needed.""" + subprocess.check_call(['framework/scripts/assemble_changelog.py'], + cwd=self.info.top_dir) + if self.files_are_clean('ChangeLog') and \ + self.info.old_version != self.info.version: + # Edge case: no change since the previous release. + # This could happen, for example, in an emergency release + # of Mbed TLS to ship a crypto bug fix, or when testing + # the release script. + self.edit_file('ChangeLog', self.create_section) + else: + self.edit_file('ChangeLog', self.finalize_release) + self.git_commit_maybe(['ChangeLog', 'ChangeLog.d'], + 'Assemble changelog and set release date') + + class ArchiveStep(Step): """Prepare release archives and the associated checksum file.""" @@ -487,6 +581,7 @@ class ArchiveStep(Step): ALL_STEPS = [ + AssembleChangelogStep, ArchiveStep, ] #type: Sequence[typing.Type[Step]] @@ -526,6 +621,8 @@ def main() -> None: parser.add_argument('--list-steps', action='store_true', help='List release steps and exit') + parser.add_argument('--release-date', + help='Release date (YYYY-mm-dd) (default: today)') parser.add_argument('--tar-command', help='GNU tar command') parser.add_argument('--to', metavar='STEP', @@ -540,6 +637,7 @@ def main() -> None: return options = Options( artifact_directory=pathlib.Path(args.artifact_directory).absolute(), + release_date=args.release_date, tar_command=args.tar_command, version=args.version) run(options, args.directory, From f3c8d9c1a9bd16d58e65820f59eb1420b7610306 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 27 Oct 2025 10:56:52 +0100 Subject: [PATCH 06/25] Recognize an empty version section Fix the case when there are no pending changes in the top version section. This is not generally expected to happen, but it can happen when following the release process and there have not yet been any changes since the last release. Signed-off-by: Gilles Peskine --- scripts/assemble_changelog.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/assemble_changelog.py b/scripts/assemble_changelog.py index ef8c1e1fe..51ed27bd2 100755 --- a/scripts/assemble_changelog.py +++ b/scripts/assemble_changelog.py @@ -116,7 +116,7 @@ class TextChangelogFormat(ChangelogFormat): # Look for an incomplete release date return not re.search(r'[0-9x]{4}-[0-9x]{2}-[0-9x]?x', title) - _top_version_re = re.compile(r'(?:\A|\n)(=[^\n]*\n+)(.*?\n)(?:=|$)', + _top_version_re = re.compile(r'(?:\A|\n)(=[^\n]*\n)(.*?)(?:\n=|\Z)', re.DOTALL) _name_re = re.compile(r'=\s(.*)\s[0-9x]+\.', re.DOTALL) @classmethod @@ -126,7 +126,7 @@ class TextChangelogFormat(ChangelogFormat): top_version_start = m.start(1) top_version_end = m.end(2) top_version_title = m.group(1) - top_version_body = m.group(2) + top_version_body = m.group(2).strip('\n') + '\n' name = re.match(cls._name_re, top_version_title).group(1) if cls.is_released_version(top_version_title): top_version_end = top_version_start @@ -136,11 +136,12 @@ class TextChangelogFormat(ChangelogFormat): top_version_title, top_version_body, changelog_file_content[top_version_end:]) + _newlines_only_re = re.compile(r'\n*\Z') _category_title_re = re.compile(r'(^\w.*)\n+', re.MULTILINE) @classmethod def split_categories(cls, version_body): """A category title is a line with the title in column 0.""" - if not version_body: + if cls._newlines_only_re.match(version_body): return [] title_matches = list(re.finditer(cls._category_title_re, version_body)) if not title_matches or title_matches[0].start() != 0: From 9254eb2fde1d48bc913c9680f205b607ca4bc485 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 27 Oct 2025 10:59:38 +0100 Subject: [PATCH 07/25] Release preparation: bump version Run `bump_version.sh` and commit that change into git. Do nothing if there is no version number changes. This commit only supports bumping the product version. Any ABI bump (SOVERSION changes) must have been done beforehand. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 32f8f9f58..4ce7a5319 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -420,6 +420,38 @@ class AssembleChangelogStep(Step): 'Assemble changelog and set release date') +class BumpVersionStep(Step): + """Bump the product version and commit the result. + + Do nothing if the product version is already as expected. + + Note: this step does not check or affect submodules. + + Note: this step does not currently handle ABI version bumps, + only product version bumps. + """ + + @classmethod + def name(cls) -> str: + return 'version' + + # Files and directories that may contain version information. + FILES_WITH_VERSION = [ + 'CMakeLists.txt', + 'doxygen', + 'include', + 'tests/suites', + ] + + def run(self) -> None: + """Bump the product version if needed.""" + subprocess.check_call(['scripts/bump_version.sh', + '--version', self.info.version], + cwd=self.info.top_dir) + self.git_commit_maybe(self.FILES_WITH_VERSION, + 'Bump version to ' + self.info.version) + + class ArchiveStep(Step): """Prepare release archives and the associated checksum file.""" @@ -582,6 +614,7 @@ class ArchiveStep(Step): ALL_STEPS = [ AssembleChangelogStep, + BumpVersionStep, ArchiveStep, ] #type: Sequence[typing.Type[Step]] From 29734aa6037d6ec62c18caf5be458a1a1ac7e491 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 27 Oct 2025 11:00:45 +0100 Subject: [PATCH 08/25] Create draft release notes Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 123 +++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 4ce7a5319..15fd6eeea 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -612,10 +612,133 @@ class ArchiveStep(Step): self.restore_gen_files() +class NotesStep(Step): + """Prepare draft release notes.""" + + def __init__(self, options: Options, info: Info) -> None: + super().__init__(options, info) + self.checksum_path = self.artifact_path('.txt') + + @classmethod + def name(cls) -> str: + return 'notes' + + def assert_preconditions(self) -> None: + assert self.checksum_path.exists() + + def read_changelog(self) -> Dict[Optional[str], str]: + """Return the section of the changelog for this release. + + The result is split into categories. + Use the index None to get the whole content. + """ + with open(self.info.top_dir / 'ChangeLog', encoding='utf-8') as inp: + whole_file = inp.read() + version_iter = re.finditer('^=.*', whole_file, re.MULTILINE) + start = next(version_iter).end() + end = next(version_iter).start() + whole = whole_file[start:end].strip() + sections = {None: whole} #type: Dict[Optional[str], str] + headings = [(m.group(), m.start(), m.end()) + for m in re.finditer(r'^\w.*', whole, re.MULTILINE)] + for (this, next_) in zip(headings, headings[1:] + [('', -1, -1)]): + title = this[0] + start = this[2] + end = next_[1] + sections[title] = whole[start:end].strip() + return sections + + @staticmethod + def advisory_items(changelog) -> Iterator[str]: + """Yield the items for the list of advisories.""" + bullets = [m.start() + for m in re.finditer(r'^ \* *', changelog, re.MULTILINE)] + for (this, next_) in zip(bullets, bullets[1:] + [-1]): + # We don't have the advisory title here, so make up something + # that the release handler will need to fill in. + content = changelog[this:next_] + teaser = re.sub(r'\n.*', '', + re.sub(r'^\W*', '', content), + re.DOTALL) + cve_match = re.search(r'CVE-[-0-9]+', content) + if cve_match: + cve_text = f' ({cve_match.group(0)})' + else: + cve_text = '' + # We don't have enough data to find the last component of the URL. + url = 'https://mbed-tls.readthedocs.io/en/latest/security-advisories/' + yield f'* [{teaser} …{cve_text}]({url})' + + def advisories(self, changelog) -> str: + """Format links to advisories for the release notes. + + The argument is just the "Security" section of the changelog, + without its heading. + """ + items = self.advisory_items(changelog) + return ('For full details, please see the following links:\n' + + '[TODO: this section needs manual editing!]\n' + + '\n'.join(items)) + + def notes(self) -> str: + """Construct draft release notes.""" + changelog = self.read_changelog() + advisories = (self.advisories(changelog['Security']) + if 'Security' in changelog + else 'None.') + with open(self.checksum_path, encoding='ascii') as inp: + checksums = inp.read() + # We currently emit a single archive file. If we switch to having + # multiple files, tweak this definition and the grammar where + # it's used. + m = re.search(r'\S+$', checksums) + if m is None: + raise Exception('Unable to determine archive file name') + archive_name = m.group(0) + # The very long lines here are deliberate. GitHub treats newlines + # in markdown as line breaks, not as spaces, so an ordinary paragraph + # needs to be in a single Python logical line. + return f'''\ +## Description + +This release of {self.info.product_human_name} provides new features, bug fixes and minor enhancements. \ +{'This release includes fixes for security issues.' if 'Security' in changelog else ''} + +## Security Advisories + +{advisories} + +## Release Notes + +{changelog[None]} + +## Who should update + +We recommend all users should update to take advantage of the bug fixes contained in this release at an appropriate point in their development lifecycle. + +## Note + +:grey_exclamation: `{archive_name}` is our official release file. `source.tar.gz` and `source.zip` are automatically generated snapshots that GitHub is generating. They do not include submodules or generated files, and [cannot be configured](https://github.com/orgs/community/discussions/6003). + +## Checksum + +The SHA256 hashes for the archives are: +``` +{checksums} +``` +''' + + def run(self) -> None: + content = self.notes() + with open(self.artifact_path('.md'), 'w') as out: + out.write(content) + + ALL_STEPS = [ AssembleChangelogStep, BumpVersionStep, ArchiveStep, + NotesStep, ] #type: Sequence[typing.Type[Step]] From 6d238071da3c27bf7d7525cd387b4851a736fae0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 28 Oct 2025 19:57:43 +0100 Subject: [PATCH 09/25] Instantiate all steps before starting to run one That way, if the initialization of a step fails, it'll happen before any external changes are done. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 15fd6eeea..005843d79 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -199,8 +199,7 @@ class Step: def __init__(self, options: Options, info: Info) -> None: """Instantiate the release step for the given product directory. - This constructor may analyze the contents of the product tree, - but it does not require other steps to have run. + All step constructors are executed before running the first step. """ self.options = options self.info = info @@ -742,23 +741,35 @@ ALL_STEPS = [ ] #type: Sequence[typing.Type[Step]] +def init_steps(options: Options, + info: Info, + #pylint: disable=dangerous-default-value + all_steps: Sequence[typing.Type[Step]] = ALL_STEPS, + from_: Optional[str] = None, + to: Optional[str] = None) -> Sequence[Step]: + """Initialize the selected steps without running them.""" + def iterator(): + from_reached = (from_ is None) + for step_class in all_steps: + step = step_class(options, info) + if not from_reached: + if step.name() != from_: + continue + from_reached = True + yield step + if step.name() == to: + break + return list(iterator()) + def run(options: Options, top_dir: str, from_: Optional[str] = None, to: Optional[str] = None) -> None: """Run the release process (or a segment thereof).""" info = Info(top_dir, options) - from_reached = (from_ is None) - for step_class in ALL_STEPS: - step = step_class(options, info) - if not from_reached: - if step.name() != from_: - continue - from_reached = True + for step in init_steps(options, info, from_=from_, to=to): step.assert_preconditions() step.run() - if step.name() == to: - break def main() -> None: """Command line entry point.""" From abca584eb2993d018108f1a5c52759159b590b5e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 28 Oct 2025 19:58:53 +0100 Subject: [PATCH 10/25] Step.edit_file now reports whether it made a change Don't change the disk file if the content hasn't changed. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 005843d79..bd1352d5b 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -337,16 +337,23 @@ class Step: def edit_file(self, path: PathOrString, - transform: Callable[[str], str]) -> None: + transform: Callable[[str], str]) -> bool: """Edit a text file. The path can be relative to the toplevel root or absolute. + + Return True if the file was modified, False otherwise. """ with open(self.info.top_dir / path, 'r+', encoding='utf-8') as file_: - new_content = transform(file_.read()) - file_.seek(0) - file_.truncate() - file_.write(new_content) + old_content = file_.read() + new_content = transform(old_content) + if old_content == new_content: + return False + else: + file_.seek(0) + file_.truncate() + file_.write(new_content) + return True def assert_preconditions(self) -> None: """Check whether the preconditions for this step have been achieved. From c8d7b91164f8973b4295de34ca72c9240d51ac8c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 28 Oct 2025 20:31:46 +0100 Subject: [PATCH 11/25] Error out if an invalid step name is specified Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index bd1352d5b..20398b1f9 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -751,22 +751,27 @@ ALL_STEPS = [ def init_steps(options: Options, info: Info, #pylint: disable=dangerous-default-value - all_steps: Sequence[typing.Type[Step]] = ALL_STEPS, + step_classes: Sequence[typing.Type[Step]] = ALL_STEPS, from_: Optional[str] = None, to: Optional[str] = None) -> Sequence[Step]: """Initialize the selected steps without running them.""" - def iterator(): - from_reached = (from_ is None) - for step_class in all_steps: - step = step_class(options, info) - if not from_reached: - if step.name() != from_: - continue - from_reached = True - yield step - if step.name() == to: + steps = [step_class(options, info) for step_class in step_classes] + if from_ is not None: + for n, step in enumerate(steps): + if step.name() == from_: + del steps[:n] break - return list(iterator()) + else: + raise Exception(f'Step name not found: {from_}') + if to is not None: + for n, step in enumerate(steps): + if step.name() == to: + del steps[n+1:] + break + else: + after_msg = f' after {from_}' if from_ is not None else '' + raise Exception(f'Step name not found{after_msg}: {to}') + return steps def run(options: Options, top_dir: str, From dceff748779b23b1af0a5d0b75caca1313705c7b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 29 Oct 2025 18:06:24 +0100 Subject: [PATCH 12/25] Better error message if there are uncommitted changes Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 20398b1f9..59187d0cf 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -360,7 +360,9 @@ class Step: If not, raise an exception. """ - self.assert_git_status() + if not self.files_are_clean(): + raise Exception('There are uncommitted changes (maybe in submodules) in ' + + str(self.info.top_dir)) def run(self) -> None: """Perform the release preparation step. From b1869372a07156fea4f9725a9450e10c6b29f200 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 29 Oct 2025 18:06:57 +0100 Subject: [PATCH 13/25] files_are_clean: also consider changes that might be in the index Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 59187d0cf..be7276ae8 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -295,7 +295,7 @@ class Step: and may not be in a submodule of `where`. """ try: - self.call_git(['diff', '--quiet', '--'] + list(files), + self.call_git(['diff', '--quiet', 'HEAD', '--'] + list(files), where=where) return True except subprocess.CalledProcessError as exn: From 7de727ed7065d5d91d422fd25c90c94c063cf68e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 29 Oct 2025 18:17:24 +0100 Subject: [PATCH 14/25] Change the version argument to --release-version/-r Don't use a positional argument, for consistency with the release date. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index be7276ae8..60d02e778 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -98,16 +98,16 @@ class Options(typing.NamedTuple): artifact_directory: pathlib.Path # Release date (YYYY-mm-dd). release_date: str + # Version to release (empty to read it from ChangeLog). + release_version: str # GNU tar command. tar_command: str - # Version to release (None to read it from ChangeLog). - version: Optional[str] DEFAULT_OPTIONS = Options( artifact_directory=pathlib.Path(os.pardir), release_date=datetime.date.today().isoformat(), - tar_command=find_gnu_tar(), - version=None) + release_version='', + tar_command=find_gnu_tar()) class Info: @@ -155,10 +155,10 @@ class Info: """ self.top_dir = pathlib.Path(top_dir) self._read_product_info() - if options.version is None: - self._release_version = self.old_version + if options.release_version: + self._release_version = options.release_version else: - self._release_version = options.version + self._release_version = self.old_version self._product_machine_name = \ self._product_machine_name_from_human_name(self._product_human_name) @@ -804,12 +804,12 @@ def main() -> None: help='List release steps and exit') parser.add_argument('--release-date', help='Release date (YYYY-mm-dd) (default: today)') + parser.add_argument('--release-version', '-r', + help='The version to release (default/empty: from ChangeLog)') parser.add_argument('--tar-command', help='GNU tar command') parser.add_argument('--to', metavar='STEP', help='Last step to run (default: run all steps)') - parser.add_argument('version', nargs='?', - help='The version to release (default: from ChangeLog)') parser.set_defaults(**DEFAULT_OPTIONS._asdict()) args = parser.parse_args() if args.list_steps: @@ -819,8 +819,8 @@ def main() -> None: options = Options( artifact_directory=pathlib.Path(args.artifact_directory).absolute(), release_date=args.release_date, - tar_command=args.tar_command, - version=args.version) + release_version=args.release_version, + tar_command=args.tar_command) run(options, args.directory, from_=args.from_, to=args.to) From bbc379d16213eb52ea77a418368b06153a54eb34 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 29 Oct 2025 18:18:48 +0100 Subject: [PATCH 15/25] Add short options for common options Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 60d02e778..9b2fa9f01 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -796,19 +796,19 @@ def main() -> None: help='Product toplevel directory') parser.add_argument('--artifact-directory', '-a', help='Directory where release artifacts will be placed') - parser.add_argument('--from', metavar='STEP', + parser.add_argument('--from', '-f', metavar='STEP', dest='from_', help='First step to run (default: run all steps)') parser.add_argument('--list-steps', action='store_true', help='List release steps and exit') - parser.add_argument('--release-date', + parser.add_argument('--release-date', '-d', help='Release date (YYYY-mm-dd) (default: today)') parser.add_argument('--release-version', '-r', help='The version to release (default/empty: from ChangeLog)') parser.add_argument('--tar-command', help='GNU tar command') - parser.add_argument('--to', metavar='STEP', + parser.add_argument('--to', '-t', metavar='STEP', help='Last step to run (default: run all steps)') parser.set_defaults(**DEFAULT_OPTIONS._asdict()) args = parser.parse_args() From f3ce1e8d2c8256ed8e57288ab41d1491089dd7eb Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 29 Oct 2025 18:22:22 +0100 Subject: [PATCH 16/25] Add --only-step option (short: -s) Rename `--from` and `--to` to `--from-step` and `--to-step` for consistency. They have short options so they don't need to be easy to type. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 41 +++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 9b2fa9f01..b07c16df3 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -754,34 +754,34 @@ def init_steps(options: Options, info: Info, #pylint: disable=dangerous-default-value step_classes: Sequence[typing.Type[Step]] = ALL_STEPS, - from_: Optional[str] = None, - to: Optional[str] = None) -> Sequence[Step]: + from_step: Optional[str] = None, + to_step: Optional[str] = None) -> Sequence[Step]: """Initialize the selected steps without running them.""" steps = [step_class(options, info) for step_class in step_classes] - if from_ is not None: + if from_step is not None: for n, step in enumerate(steps): - if step.name() == from_: + if step.name() == from_step: del steps[:n] break else: - raise Exception(f'Step name not found: {from_}') - if to is not None: + raise Exception(f'Step name not found: {from_step}') + if to_step is not None: for n, step in enumerate(steps): - if step.name() == to: + if step.name() == to_step: del steps[n+1:] break else: - after_msg = f' after {from_}' if from_ is not None else '' - raise Exception(f'Step name not found{after_msg}: {to}') + after_msg = f' after {from_step}' if from_step is not None else '' + raise Exception(f'Step name not found{after_msg}: {to_step}') return steps def run(options: Options, top_dir: str, - from_: Optional[str] = None, - to: Optional[str] = None) -> None: + from_step: Optional[str] = None, + to_step: Optional[str] = None) -> None: """Run the release process (or a segment thereof).""" info = Info(top_dir, options) - for step in init_steps(options, info, from_=from_, to=to): + for step in init_steps(options, info, from_step=from_step, to_step=to_step): step.assert_preconditions() step.run() @@ -796,33 +796,42 @@ def main() -> None: help='Product toplevel directory') parser.add_argument('--artifact-directory', '-a', help='Directory where release artifacts will be placed') - parser.add_argument('--from', '-f', metavar='STEP', - dest='from_', + parser.add_argument('--from-step', '--from', '-f', metavar='STEP', help='First step to run (default: run all steps)') parser.add_argument('--list-steps', action='store_true', help='List release steps and exit') + parser.add_argument('--only-step', '-s', metavar='STEP', + help=('Run only this step (default: run all steps) ' + '(equivalent to --from-step=STEP --to-step=STEP)')) parser.add_argument('--release-date', '-d', help='Release date (YYYY-mm-dd) (default: today)') parser.add_argument('--release-version', '-r', help='The version to release (default/empty: from ChangeLog)') parser.add_argument('--tar-command', help='GNU tar command') - parser.add_argument('--to', '-t', metavar='STEP', + parser.add_argument('--to-step', '-t', metavar='STEP', help='Last step to run (default: run all steps)') parser.set_defaults(**DEFAULT_OPTIONS._asdict()) args = parser.parse_args() + + # Process help-and-exit options if args.list_steps: for step in ALL_STEPS: sys.stdout.write(step.name() + '\n') return + + if args.only_step: + args.from_step = args.only_step + args.to_step = args.only_step options = Options( artifact_directory=pathlib.Path(args.artifact_directory).absolute(), release_date=args.release_date, release_version=args.release_version, tar_command=args.tar_command) + run(options, args.directory, - from_=args.from_, to=args.to) + from_step=args.from_step, to_step=args.to_step) if __name__ == '__main__': main() From a6c62f60642e53238d810b21d3b0f00ac4107b77 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 29 Oct 2025 18:47:08 +0100 Subject: [PATCH 17/25] Put default artifact directory under the source tree Using the parent of the source directory as an artifact directory makes sense if the parent of the source directory contains multiple projects or branches of Mbed TLS or TF-PSA-Crypto. It doesn't work for people who have clutter there. Also, the default artifact directory was the parent of the _current_ directory, whatever it is, and not the parent of the source directory. Passing `--directory` without `--artifact-directory` resulted in a nonsensical artifact directory location. Default to placing artifacts in `release-artifacts` in the source tree. This way, you can have multiple release candidates for the same branch in sibling directories, and their artifacts won't overwrite each other. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index b07c16df3..b79bbedcf 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -95,7 +95,8 @@ class Options(typing.NamedTuple): declared in main(). """ # Directory where the release artifacts will be placed. - artifact_directory: pathlib.Path + # If None: "{source_directory}/release-artifacts" + artifact_directory: Optional[pathlib.Path] # Release date (YYYY-mm-dd). release_date: str # Version to release (empty to read it from ChangeLog). @@ -104,7 +105,7 @@ class Options(typing.NamedTuple): tar_command: str DEFAULT_OPTIONS = Options( - artifact_directory=pathlib.Path(os.pardir), + artifact_directory=None, release_date=datetime.date.today().isoformat(), release_version='', tar_command=find_gnu_tar()) @@ -153,7 +154,7 @@ class Info: Optional parameters can override the supplied information: * `version`: version to release. """ - self.top_dir = pathlib.Path(top_dir) + self.top_dir = pathlib.Path(top_dir).absolute() self._read_product_info() if options.release_version: self._release_version = options.release_version @@ -203,6 +204,10 @@ class Step: """ self.options = options self.info = info + if options.artifact_directory is None: + self.artifact_directory = pathlib.Path(self.info.top_dir) / 'release-artifacts' + else: + self.artifact_directory = pathlib.Path(options.artifact_directory).absolute() self._submodules = None #type: Optional[List[str]] @staticmethod @@ -333,7 +338,7 @@ class Step: `extension` should start with a ".". """ file_name = self.artifact_base_name() + extension - return self.options.artifact_directory / file_name + return self.artifact_directory / file_name def edit_file(self, path: PathOrString, @@ -601,16 +606,17 @@ class ArchiveStep(Step): def create_checksum_file(self, archive_paths: List[str]) -> None: """Create a checksum file for the given files.""" checksum_path = self.artifact_path('.txt') - relative_paths = [os.path.relpath(path, self.options.artifact_directory) + relative_paths = [os.path.relpath(path, self.artifact_directory) for path in archive_paths] content = subprocess.check_output(['sha256sum', '--'] + relative_paths, - cwd=self.options.artifact_directory, + cwd=self.artifact_directory, encoding='ascii') with open(checksum_path, 'w') as out: out.write(content) def run(self) -> None: """Create the release artifacts.""" + self.artifact_directory.mkdir(exist_ok=True) self.turn_off_gen_files() base_name = self.artifact_base_name() plain_tar_path = str(self.artifact_path('.tar')) @@ -791,11 +797,12 @@ def main() -> None: # Options that affect information gathering, or that affect the # behavior of one or more steps, should have an associated field # in the Options class. - parser.add_argument('--directory', + parser.add_argument('--directory', metavar='DIR', default=os.curdir, - help='Product toplevel directory') - parser.add_argument('--artifact-directory', '-a', - help='Directory where release artifacts will be placed') + help='Product toplevel directory (default: .)') + parser.add_argument('--artifact-directory', '-a', metavar='DIR', + help=('Directory where release artifacts will be placed ' + '(default/empty: <--directory>/release-artifacts)')) parser.add_argument('--from-step', '--from', '-f', metavar='STEP', help='First step to run (default: run all steps)') parser.add_argument('--list-steps', @@ -824,8 +831,12 @@ def main() -> None: if args.only_step: args.from_step = args.only_step args.to_step = args.only_step + if args.artifact_directory is None: + artifact_directory = None + else: + artifact_directory = pathlib.Path(args.artifact_directory) options = Options( - artifact_directory=pathlib.Path(args.artifact_directory).absolute(), + artifact_directory=artifact_directory, release_date=args.release_date, release_version=args.release_version, tar_command=args.tar_command) From 9ad6c3962d5e44eb593d8194175697a3dd5af072 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 29 Oct 2025 19:11:05 +0100 Subject: [PATCH 18/25] Move git submodule gathering to the Info class The list of submodules won't change during the process. So get it once and for all. In support of this, move some basic git-related methods to the Info class. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 81 +++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index b79bbedcf..04438b6e8 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -118,8 +118,46 @@ class Info: as from the command line. """ + + @staticmethod + def _git_command(subcommand: List[str], + where: Optional[PathOrString] = None) -> List[str]: + cmd = ['git'] + if where is not None: + cmd += ['-C', str(where)] + return cmd + subcommand + + def call_git(self, cmd: List[str], + where: Optional[PathOrString] = None, + env: Optional[Dict[str, str]] = None) -> None: + """Run git in the source tree. + + Pass `where` to specify a submodule. + """ + subprocess.check_call(self._git_command(cmd, where), + cwd=self.top_dir, + env=env) + + def read_git(self, cmd: List[str], + where: Optional[PathOrString] = None, + env: Optional[Dict[str, str]] = None) -> bytes: + """Run git in the source tree and return the output. + + Pass `where` to specify a submodule. + """ + return subprocess.check_output(self._git_command(cmd, where), + cwd=self.top_dir, + env=env) + + def _read_file_tree(self) -> None: + """Find information about files in this branch.""" + raw = self.read_git(['submodule', '--quiet', + 'foreach', '--recursive', + 'printf %s\\\\0 "$displaypath"']) + self._submodules = raw.decode('ascii').rstrip('\0').split('\0') + def _read_product_info(self) -> None: - """Read information from the source files.""" + """Read product information (name, version) from the source files.""" with open(self.top_dir / 'ChangeLog', encoding='utf-8') as changelog: # We deliberately read only a short portion at the top of the # changelog file. This reduces the risk that we'll match an @@ -155,6 +193,7 @@ class Info: * `version`: version to release. """ self.top_dir = pathlib.Path(top_dir).absolute() + self._read_file_tree() self._read_product_info() if options.release_version: self._release_version = options.release_version @@ -173,6 +212,15 @@ class Info: """Machine-frieldly product name, e.g. 'mbedtls'.""" return self._product_machine_name + @property + def submodules(self) -> Sequence[str]: + """Submodules present in the source tree (including nested ones). + + This is a list of paths relative to the root of the main tree. + Note that the main tree itself is not included. + """ + return self._submodules + @property def version(self) -> str: """Version string for the relase.""" @@ -208,15 +256,6 @@ class Step: self.artifact_directory = pathlib.Path(self.info.top_dir) / 'release-artifacts' else: self.artifact_directory = pathlib.Path(options.artifact_directory).absolute() - self._submodules = None #type: Optional[List[str]] - - @staticmethod - def _git_command(subcommand: List[str], - where: Optional[PathOrString] = None) -> List[str]: - cmd = ['git'] - if where is not None: - cmd += ['-C', str(where)] - return cmd + subcommand def call_git(self, cmd: List[str], where: Optional[PathOrString] = None, @@ -225,9 +264,7 @@ class Step: Pass `where` to specify a submodule. """ - subprocess.check_call(self._git_command(cmd, where), - cwd=self.info.top_dir, - env=env) + self.info.call_git(cmd, where=where, env=env) def read_git(self, cmd: List[str], where: Optional[PathOrString] = None, @@ -236,19 +273,7 @@ class Step: Pass `where` to specify a submodule. """ - return subprocess.check_output(self._git_command(cmd, where), - cwd=self.info.top_dir, - env=env) - - @property - def submodules(self) -> Sequence[str]: - """List the git submodules (recursive, but not including the top level).""" - if self._submodules is None: - raw = self.read_git(['submodule', '--quiet', - 'foreach', '--recursive', - 'printf %s\\\\0 "$displaypath"']) - self._submodules = raw.decode('ascii').rstrip('\0').split('\0') - return self._submodules + return self.info.read_git(cmd, where=where, env=env) def commit_timestamp(self, where: Optional[PathOrString] = None, @@ -489,7 +514,7 @@ class ArchiveStep(Step): '--prefix', prefix, '--output', str(plain_tar_path), index]) - for submodule in self.submodules: + for submodule in self.info.submodules: index = self.git_index_as_tree_ish(where=submodule) mtime = self.commit_timestamp(where=submodule) data = self.read_git(['archive', '--format=tar', @@ -572,7 +597,7 @@ class ArchiveStep(Step): def tar_add_generated_files(self, plain_tar_path: str, prefix: str) -> None: """Add generated files to an existing uncompressed tar file.""" - for project in [os.curdir] + list(self.submodules): + for project in [os.curdir] + list(self.info.submodules): if project.endswith('/framework') or project == 'framework': continue project_dir = self.info.top_dir / project From 6f7b78e2747e8171e320197fb52550f2dd9d3bf5 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Thu, 5 Mar 2026 14:38:29 +0000 Subject: [PATCH 19/25] prepare_release: Support for MLDSA submodule Signed-off-by: Minos Galanakis --- scripts/prepare_release.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 04438b6e8..cba345d35 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -603,6 +603,10 @@ class ArchiveStep(Step): project_dir = self.info.top_dir / project project_prefix = (prefix if project == os.curdir else prefix + project + '/') + gen_script = project_dir / 'framework/scripts/make_generated_files.py' + # Skip submodules that do not use the framework build machinery. + if not gen_script.exists(): + continue file_list = self.list_project_generated_files(project_dir) self.update_project_generated_files(project_dir) self.tar_add_project_generated_files(plain_tar_path, From 25e1c591734a145b7917a166210e5b064d40d928 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 3 Jul 2026 14:42:09 +0200 Subject: [PATCH 20/25] Note the need for git >=2.40.0 Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index cba345d35..f6c160df1 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -9,6 +9,7 @@ This script will update the checked out git branch, if any. On normal exit, the worktree contains the release candidate commit. This script requires the following external tools: +- Git >= 2.40.0 (for ``git archive --mtime=...``). - GNU tar (can be called ``gnutar`` or ``gtar``); - ``sha256sum``. """ @@ -509,6 +510,7 @@ class ArchiveStep(Step): # an exact commit, we should make sure that the commit gets published. index = self.git_index_as_tree_ish() mtime = self.commit_timestamp() + # Note that `git archive --mtime=...` requires Git >= 2.40.0. self.call_git(['archive', '--format=tar', '--mtime', str(mtime), '--prefix', prefix, From c58fbebd96fed1106eed0661b526b07e142cb0be Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 5 Jul 2026 20:57:44 +0200 Subject: [PATCH 21/25] Add experimental --ignore-submodules option Allow the user to ignore all changes in submodules. This is probably not exactly what we want: in practice, we'd like to be able to use this script with submodules that are out-to-date, when testing a new version of TF-PSA-Crypto against the current Mbed TLS. But that is more difficult to implement. Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index f6c160df1..64e64fbac 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -98,6 +98,9 @@ class Options(typing.NamedTuple): # Directory where the release artifacts will be placed. # If None: "{source_directory}/release-artifacts" artifact_directory: Optional[pathlib.Path] + # Whether to go ahead if submodules are out of date or if there + # are modified files inside submodules. + ignore_submodules: bool # Release date (YYYY-mm-dd). release_date: str # Version to release (empty to read it from ChangeLog). @@ -107,6 +110,7 @@ class Options(typing.NamedTuple): DEFAULT_OPTIONS = Options( artifact_directory=None, + ignore_submodules=False, release_date=datetime.date.today().isoformat(), release_version='', tar_command=find_gnu_tar()) @@ -316,6 +320,7 @@ class Step: self.call_git(['diff', '--quiet']) def files_are_clean(self, *files: str, + ignore_submodules: Optional[str] = None, where: Optional[PathOrString] = None) -> bool: """Check whether the specified files are identical to their git version. @@ -326,8 +331,12 @@ class Step: and may not be in a submodule of `where`. """ try: - self.call_git(['diff', '--quiet', 'HEAD', '--'] + list(files), - where=where) + cmd = ['diff', '--quiet'] + if ignore_submodules: + cmd += ['--ignore-submodules=' + ignore_submodules] + cmd += ['HEAD', '--'] + cmd += list(files) + self.call_git(cmd, where=where) return True except subprocess.CalledProcessError as exn: if exn.returncode != 1: @@ -391,7 +400,11 @@ class Step: If not, raise an exception. """ - if not self.files_are_clean(): + if self.options.ignore_submodules: + ignore_submodules = 'all' + else: + ignore_submodules = 'none' + if not self.files_are_clean(ignore_submodules=ignore_submodules): raise Exception('There are uncommitted changes (maybe in submodules) in ' + str(self.info.top_dir)) @@ -836,6 +849,9 @@ def main() -> None: '(default/empty: <--directory>/release-artifacts)')) parser.add_argument('--from-step', '--from', '-f', metavar='STEP', help='First step to run (default: run all steps)') + parser.add_argument('--ignore-submodules', + action='store_true', + help='Ignore changes to submodules and changed files within submodules') parser.add_argument('--list-steps', action='store_true', help='List release steps and exit') @@ -868,6 +884,7 @@ def main() -> None: artifact_directory = pathlib.Path(args.artifact_directory) options = Options( artifact_directory=artifact_directory, + ignore_submodules=args.ignore_submodules, release_date=args.release_date, release_version=args.release_version, tar_command=args.tar_command) From 2e1c8c9510d88328a679715d98240222b258228f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 20 Jul 2026 14:00:38 +0200 Subject: [PATCH 22/25] Use absolute paths in dependency outputs This fixes an error when `tests/Makefile` in Mbed TLS invokes `../framwework/scripts/generate_test_cert_macros.py`: this needs paths that are either absolute or relative to the current directory (which is `tests`), rather than relative to the repository root. Signed-off-by: Gilles Peskine --- scripts/generate_test_cert_macros.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_test_cert_macros.py b/scripts/generate_test_cert_macros.py index e089e86ec..ac6cf248a 100755 --- a/scripts/generate_test_cert_macros.py +++ b/scripts/generate_test_cert_macros.py @@ -62,7 +62,7 @@ def main(): if args.list_dependencies: files_list = [arg[2] for arg in INPUT_ARGS if arg[0] != "password"] - print(" ".join(os.path.join(DATA_DIR_RELATIVE, filename) + print(" ".join(os.path.join(DATA_DIR_ABSOLUTE, filename) for filename in files_list)) return From 81cc340bb8a6d835a90f58f805e5cb24da9f5818 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 5 Aug 2026 19:00:15 +0200 Subject: [PATCH 23/25] Minor documentation fixes Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 64e64fbac..980507b1a 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -214,7 +214,7 @@ class Info: @property def product_machine_name(self) -> str: - """Machine-frieldly product name, e.g. 'mbedtls'.""" + """Machine-friendly product name, e.g. 'mbedtls'.""" return self._product_machine_name @property @@ -228,7 +228,7 @@ class Info: @property def version(self) -> str: - """Version string for the relase.""" + """Version string for the release.""" return self._release_version @@ -443,7 +443,7 @@ class AssembleChangelogStep(Step): m = re.match(r'([0-9]+)-([0-9]+)-([0-9]+)', self.info.old_release_date) if not m: # presumably xxxx-xx-xx return True - # The date format is a lexicographic, fixed-width format, + # The date format is a lexicographic-order, fixed-width format, # so we can just compare the strings. return self.info.old_release_date < self.options.release_date @@ -527,7 +527,7 @@ class ArchiveStep(Step): self.call_git(['archive', '--format=tar', '--mtime', str(mtime), '--prefix', prefix, - '--output', str(plain_tar_path), + '--output', plain_tar_path, index]) for submodule in self.info.submodules: index = self.git_index_as_tree_ish(where=submodule) @@ -588,7 +588,7 @@ class ArchiveStep(Step): @staticmethod def update_project_generated_files(project_dir: pathlib.Path) -> None: - """Update the list of generated files in the given (sub)project.""" + """Regenerate the generated files in the given (sub)project.""" subprocess.check_call( ['framework/scripts/make_generated_files.py'], cwd=project_dir) From c92117bf3276d73d892483c61287a8115d88e70b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 5 Aug 2026 19:12:54 +0200 Subject: [PATCH 24/25] Remove unused ability to override env in git calls Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 980507b1a..679a32fbb 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -133,26 +133,22 @@ class Info: return cmd + subcommand def call_git(self, cmd: List[str], - where: Optional[PathOrString] = None, - env: Optional[Dict[str, str]] = None) -> None: + where: Optional[PathOrString] = None) -> None: """Run git in the source tree. Pass `where` to specify a submodule. """ subprocess.check_call(self._git_command(cmd, where), - cwd=self.top_dir, - env=env) + cwd=self.top_dir) def read_git(self, cmd: List[str], - where: Optional[PathOrString] = None, - env: Optional[Dict[str, str]] = None) -> bytes: + where: Optional[PathOrString] = None) -> bytes: """Run git in the source tree and return the output. Pass `where` to specify a submodule. """ return subprocess.check_output(self._git_command(cmd, where), - cwd=self.top_dir, - env=env) + cwd=self.top_dir) def _read_file_tree(self) -> None: """Find information about files in this branch.""" @@ -263,22 +259,20 @@ class Step: self.artifact_directory = pathlib.Path(options.artifact_directory).absolute() def call_git(self, cmd: List[str], - where: Optional[PathOrString] = None, - env: Optional[Dict[str, str]] = None) -> None: + where: Optional[PathOrString] = None) -> None: """Run git in the source tree. Pass `where` to specify a submodule. """ - self.info.call_git(cmd, where=where, env=env) + self.info.call_git(cmd, where=where) def read_git(self, cmd: List[str], - where: Optional[PathOrString] = None, - env: Optional[Dict[str, str]] = None) -> bytes: + where: Optional[PathOrString] = None) -> bytes: """Run git in the source tree and return the output. Pass `where` to specify a submodule. """ - return self.info.read_git(cmd, where=where, env=env) + return self.info.read_git(cmd, where=where) def commit_timestamp(self, where: Optional[PathOrString] = None, From a61e26eebe9166ba1f692f10e22b916db30e4788 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 5 Aug 2026 19:13:10 +0200 Subject: [PATCH 25/25] Remove unused function release_date_needs_updating Signed-off-by: Gilles Peskine --- scripts/prepare_release.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/prepare_release.py b/scripts/prepare_release.py index 679a32fbb..43887eff7 100755 --- a/scripts/prepare_release.py +++ b/scripts/prepare_release.py @@ -432,15 +432,6 @@ class AssembleChangelogStep(Step): new_section, old_content, flags=re.MULTILINE, count=1) - def release_date_needs_updating(self) -> bool: - """Whether the release date needs updating in the existing ChangeLog section.""" - m = re.match(r'([0-9]+)-([0-9]+)-([0-9]+)', self.info.old_release_date) - if not m: # presumably xxxx-xx-xx - return True - # The date format is a lexicographic-order, fixed-width format, - # so we can just compare the strings. - return self.info.old_release_date < self.options.release_date - def finalize_release(self, old_content: str) -> str: """Update the version and release date in the changelog content.""" version = self.info.version