mirror of
https://github.com/Mbed-TLS/mbedtls-framework.git
synced 2026-09-21 09:27:22 +00:00
882 lines
34 KiB
Python
Executable File
882 lines
34 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Prepare to release TF-PSA-Crypto or Mbed TLS.
|
|
|
|
This script constructs a release candidate branch and release artifacts.
|
|
It must run from a clean git worktree with initialized, clean git submodules.
|
|
When making an Mbed TLS >=4 release, the tf-psa-crypto submodule should
|
|
already contain a TF-PSA-Crypto release candidate.
|
|
This script will update the checked out git branch, if any.
|
|
On normal exit, the worktree contains the release candidate commit.
|
|
|
|
This script requires the following external tools:
|
|
- Git >= 2.40.0 (for ``git archive --mtime=...``).
|
|
- GNU tar (can be called ``gnutar`` or ``gtar``);
|
|
- ``sha256sum``.
|
|
"""
|
|
|
|
# Copyright The Mbed TLS Contributors
|
|
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
|
|
|
|
################################################################
|
|
|
|
#### Tips for maintainers ####
|
|
|
|
# This script insists on having a clean git checkout, including
|
|
# submodules. If you want to modify it, either make your changes in
|
|
# a separate worktree, or commit your changes first and update
|
|
# submodules in the parent repositories.
|
|
|
|
#### Architectural notes ####
|
|
|
|
# The process of preparing a release consists of some information
|
|
# gathering about the branch to release in the `Info` class, then
|
|
# a series of steps in `XxxStep` classes.
|
|
|
|
# Information gathering is performed by the `Info` constructor.
|
|
# Any failure causes the script to abort.
|
|
|
|
# The steps run sequentially in the order defined by `ALL_STEPS`.
|
|
# Each step starts by checking precondition in `step.assert_preconditions()`,
|
|
# then does its job in `step.run()`. `assert_preconditions` should
|
|
# not change any state. `run` is expected to change state and is
|
|
# not expected to clean up if an error occurs.
|
|
|
|
# This script is expected to perform a release reliably, so it
|
|
# should try to detect dodgy conditions and abort. It should avoid
|
|
# silently outputting garbage. Nonetheless, human review of the
|
|
# result is still expected.
|
|
|
|
# If you add an external dependency to a step, please mention
|
|
# it in the docstring at the top.
|
|
|
|
################################################################
|
|
|
|
|
|
import argparse
|
|
import datetime
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import typing
|
|
from typing import Callable, Dict, Iterator, List, Optional, Sequence
|
|
|
|
|
|
PathOrString = typing.Union[os.PathLike, str]
|
|
|
|
|
|
class InformationGatheringError(Exception):
|
|
"""Problem detected while gathering information."""
|
|
pass
|
|
|
|
|
|
def find_gnu_tar() -> str:
|
|
"""Try to guess the command for GNU tar.
|
|
|
|
This function never errors out. It defaults to "tar".
|
|
"""
|
|
for name in ['gnutar', 'gtar']:
|
|
try:
|
|
subprocess.check_call([name, '--version'],
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL)
|
|
return name
|
|
except FileNotFoundError:
|
|
pass
|
|
except subprocess.CalledProcessError:
|
|
pass
|
|
return 'tar'
|
|
|
|
class Options(typing.NamedTuple):
|
|
"""Options controlling the behavior of the release process.
|
|
|
|
Each field here should have an associated command line option
|
|
declared in main().
|
|
"""
|
|
# Directory where the release artifacts will be placed.
|
|
# If None: "{source_directory}/release-artifacts"
|
|
artifact_directory: Optional[pathlib.Path]
|
|
# Whether to go ahead if submodules are out of date or if there
|
|
# are modified files inside submodules.
|
|
ignore_submodules: bool
|
|
# Release date (YYYY-mm-dd).
|
|
release_date: str
|
|
# Version to release (empty to read it from ChangeLog).
|
|
release_version: str
|
|
# GNU tar command.
|
|
tar_command: str
|
|
|
|
DEFAULT_OPTIONS = Options(
|
|
artifact_directory=None,
|
|
ignore_submodules=False,
|
|
release_date=datetime.date.today().isoformat(),
|
|
release_version='',
|
|
tar_command=find_gnu_tar())
|
|
|
|
|
|
class Info:
|
|
"""Information about the intended release.
|
|
|
|
This class contains information gathered from the product tree as well
|
|
as from the command line.
|
|
"""
|
|
|
|
|
|
@staticmethod
|
|
def _git_command(subcommand: List[str],
|
|
where: Optional[PathOrString] = None) -> List[str]:
|
|
cmd = ['git']
|
|
if where is not None:
|
|
cmd += ['-C', str(where)]
|
|
return cmd + subcommand
|
|
|
|
def call_git(self, cmd: List[str],
|
|
where: Optional[PathOrString] = None) -> None:
|
|
"""Run git in the source tree.
|
|
|
|
Pass `where` to specify a submodule.
|
|
"""
|
|
subprocess.check_call(self._git_command(cmd, where),
|
|
cwd=self.top_dir)
|
|
|
|
def read_git(self, cmd: List[str],
|
|
where: Optional[PathOrString] = None) -> bytes:
|
|
"""Run git in the source tree and return the output.
|
|
|
|
Pass `where` to specify a submodule.
|
|
"""
|
|
return subprocess.check_output(self._git_command(cmd, where),
|
|
cwd=self.top_dir)
|
|
|
|
def _read_file_tree(self) -> None:
|
|
"""Find information about files in this branch."""
|
|
raw = self.read_git(['submodule', '--quiet',
|
|
'foreach', '--recursive',
|
|
'printf %s\\\\0 "$displaypath"'])
|
|
self._submodules = raw.decode('ascii').rstrip('\0').split('\0')
|
|
|
|
def _read_product_info(self) -> None:
|
|
"""Read product information (name, version) from the source files."""
|
|
with open(self.top_dir / 'ChangeLog', encoding='utf-8') as changelog:
|
|
# We deliberately read only a short portion at the top of the
|
|
# changelog file. This reduces the risk that we'll match an
|
|
# older release if the content near the top of the file doesn't
|
|
# have the expected format.
|
|
head = changelog.read(1000)
|
|
m = re.search(r'^= *(.*?) +([0-9][-.0-9A-Za-z]+) +branch released (\S+)\n',
|
|
head, re.M)
|
|
if not m:
|
|
raise InformationGatheringError(
|
|
'Could not find version header line near the top of ChangeLog')
|
|
self._product_human_name = m.group(1)
|
|
self.old_version = m.group(2)
|
|
self.old_release_date = m.group(3)
|
|
|
|
@staticmethod
|
|
def _product_machine_name_from_human_name(_human_name: str) -> str:
|
|
"""Determine the name used in release tags and file names."""
|
|
if _human_name == 'TF-PSA-Crypto':
|
|
return 'tf-psa-crypto'
|
|
elif _human_name == 'Mbed TLS':
|
|
return 'mbedtls'
|
|
else:
|
|
raise InformationGatheringError(
|
|
f'Could not determine product (found "{_human_name}" in ChangeLog)')
|
|
|
|
def __init__(self, top_dir: str, options: Options) -> None:
|
|
"""Parse the source files to obtain information about the product.
|
|
|
|
Look for the product under `top_dir`.
|
|
|
|
Optional parameters can override the supplied information:
|
|
* `version`: version to release.
|
|
"""
|
|
self.top_dir = pathlib.Path(top_dir).absolute()
|
|
self._read_file_tree()
|
|
self._read_product_info()
|
|
if options.release_version:
|
|
self._release_version = options.release_version
|
|
else:
|
|
self._release_version = self.old_version
|
|
self._product_machine_name = \
|
|
self._product_machine_name_from_human_name(self._product_human_name)
|
|
|
|
@property
|
|
def product_human_name(self) -> str:
|
|
"""Human-readable product name, e.g. 'Mbed TLS'."""
|
|
return self._product_human_name
|
|
|
|
@property
|
|
def product_machine_name(self) -> str:
|
|
"""Machine-friendly product name, e.g. 'mbedtls'."""
|
|
return self._product_machine_name
|
|
|
|
@property
|
|
def submodules(self) -> Sequence[str]:
|
|
"""Submodules present in the source tree (including nested ones).
|
|
|
|
This is a list of paths relative to the root of the main tree.
|
|
Note that the main tree itself is not included.
|
|
"""
|
|
return self._submodules
|
|
|
|
@property
|
|
def version(self) -> str:
|
|
"""Version string for the release."""
|
|
return self._release_version
|
|
|
|
|
|
class Step:
|
|
"""A step on the release process.
|
|
|
|
This is an abstract class containing some common tooling.
|
|
Subclasses must provide the following methods:
|
|
* `name()` returning a unique name for each step.
|
|
* `run()` doing the work. Raise an exception if something goes wrong.
|
|
|
|
Subclasses may override the following methods:
|
|
* `assert_preconditions()` is called to perform sanity checks before
|
|
calling `run()`.
|
|
"""
|
|
|
|
@classmethod
|
|
def name(cls) -> str:
|
|
"""The name of this step."""
|
|
raise NotImplementedError
|
|
|
|
def __init__(self, options: Options, info: Info) -> None:
|
|
"""Instantiate the release step for the given product directory.
|
|
|
|
All step constructors are executed before running the first step.
|
|
"""
|
|
self.options = options
|
|
self.info = info
|
|
if options.artifact_directory is None:
|
|
self.artifact_directory = pathlib.Path(self.info.top_dir) / 'release-artifacts'
|
|
else:
|
|
self.artifact_directory = pathlib.Path(options.artifact_directory).absolute()
|
|
|
|
def call_git(self, cmd: List[str],
|
|
where: Optional[PathOrString] = None) -> None:
|
|
"""Run git in the source tree.
|
|
|
|
Pass `where` to specify a submodule.
|
|
"""
|
|
self.info.call_git(cmd, where=where)
|
|
|
|
def read_git(self, cmd: List[str],
|
|
where: Optional[PathOrString] = None) -> bytes:
|
|
"""Run git in the source tree and return the output.
|
|
|
|
Pass `where` to specify a submodule.
|
|
"""
|
|
return self.info.read_git(cmd, where=where)
|
|
|
|
def commit_timestamp(self,
|
|
where: Optional[PathOrString] = None,
|
|
what: str = 'HEAD') -> int:
|
|
"""Return the timestamp (seconds since epoch) of the given commit.
|
|
|
|
Pass `where` to specify a submodule.
|
|
Pass `what` to specify a commit (default: ``HEAD``).
|
|
"""
|
|
timestamp = self.read_git(['show', '--no-patch', '--pretty=%ct', what],
|
|
where=where)
|
|
return int(timestamp)
|
|
|
|
def git_index_as_tree_ish(self,
|
|
where: Optional[PathOrString] = None) -> str:
|
|
"""Return a git tree-ish corresponding to the index.
|
|
|
|
Pass `where` to specify a submodule.
|
|
"""
|
|
raw = self.read_git(['write-tree'], where=where)
|
|
return raw.decode('ascii').strip()
|
|
|
|
def commit_datetime(self,
|
|
where: Optional[PathOrString] = None,
|
|
what: str = 'HEAD') -> datetime.datetime:
|
|
"""Return the time of the given commit.
|
|
|
|
Pass `where` to specify a submodule.
|
|
Pass `what` to specify a commit (default: ``HEAD``).
|
|
"""
|
|
timestamp = self.commit_timestamp(where=where, what=what)
|
|
return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc)
|
|
|
|
def assert_git_status(self) -> None:
|
|
"""Abort if the working directory is not clean (no git changes).
|
|
|
|
This includes git submodules.
|
|
"""
|
|
self.call_git(['diff', '--quiet'])
|
|
|
|
def files_are_clean(self, *files: str,
|
|
ignore_submodules: Optional[str] = None,
|
|
where: Optional[PathOrString] = None) -> bool:
|
|
"""Check whether the specified files are identical to their git version.
|
|
|
|
With no files, check the whole work tree (including submodules).
|
|
|
|
Pass `where` to specify a submodule (default: top level).
|
|
The file names are relative to the repository or submodule root,
|
|
and may not be in a submodule of `where`.
|
|
"""
|
|
try:
|
|
cmd = ['diff', '--quiet']
|
|
if ignore_submodules:
|
|
cmd += ['--ignore-submodules=' + ignore_submodules]
|
|
cmd += ['HEAD', '--']
|
|
cmd += list(files)
|
|
self.call_git(cmd, where=where)
|
|
return True
|
|
except subprocess.CalledProcessError as exn:
|
|
if exn.returncode != 1:
|
|
raise
|
|
return False
|
|
|
|
def git_commit_maybe(self,
|
|
files: List[str],
|
|
message: str) -> None:
|
|
"""Commit changes into Git.
|
|
|
|
Do nothing if there are no changed files.
|
|
|
|
The file names are relative to the toplevel directory,
|
|
and may not be in a submodule.
|
|
"""
|
|
if not self.files_are_clean(*files):
|
|
self.call_git(['add', '--'] + files)
|
|
self.call_git(['commit', '--signoff',
|
|
'-m', message])
|
|
|
|
def artifact_base_name(self) -> str:
|
|
"""The base name for a release artifact (file created for publishing).
|
|
|
|
This contains the product name and version, with no directory part
|
|
or extension. For example "mbedtls-1.2.3".
|
|
"""
|
|
return '-'.join([self.info.product_machine_name,
|
|
self.info.version])
|
|
|
|
def artifact_path(self, extension: str) -> pathlib.Path:
|
|
"""The path for a release artifact (file created for publishing).
|
|
|
|
`extension` should start with a ".".
|
|
"""
|
|
file_name = self.artifact_base_name() + extension
|
|
return self.artifact_directory / file_name
|
|
|
|
def edit_file(self,
|
|
path: PathOrString,
|
|
transform: Callable[[str], str]) -> bool:
|
|
"""Edit a text file.
|
|
|
|
The path can be relative to the toplevel root or absolute.
|
|
|
|
Return True if the file was modified, False otherwise.
|
|
"""
|
|
with open(self.info.top_dir / path, 'r+', encoding='utf-8') as file_:
|
|
old_content = file_.read()
|
|
new_content = transform(old_content)
|
|
if old_content == new_content:
|
|
return False
|
|
else:
|
|
file_.seek(0)
|
|
file_.truncate()
|
|
file_.write(new_content)
|
|
return True
|
|
|
|
def assert_preconditions(self) -> None:
|
|
"""Check whether the preconditions for this step have been achieved.
|
|
|
|
If not, raise an exception.
|
|
"""
|
|
if self.options.ignore_submodules:
|
|
ignore_submodules = 'all'
|
|
else:
|
|
ignore_submodules = 'none'
|
|
if not self.files_are_clean(ignore_submodules=ignore_submodules):
|
|
raise Exception('There are uncommitted changes (maybe in submodules) in ' +
|
|
str(self.info.top_dir))
|
|
|
|
def run(self) -> None:
|
|
"""Perform the release preparation step.
|
|
|
|
This may create commits in the source tree or its submodules.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
class AssembleChangelogStep(Step):
|
|
"""Assemble the changelog and commit the result.
|
|
|
|
Create a new changelog section if needed.
|
|
Do nothing if the changelog is already fine.
|
|
|
|
Note: this step does not check or affect submodules.
|
|
"""
|
|
|
|
@classmethod
|
|
def name(cls) -> str:
|
|
return 'changelog'
|
|
|
|
def create_section(self, old_content: str) -> str:
|
|
"""Create a new changelog section for the version that we're releasing."""
|
|
product = self.info.product_human_name
|
|
version = self.info.version
|
|
new_section = f'= {product} {version} branch released xxxx-xx-xx\n\n'
|
|
return re.sub(r'(?=^=)',
|
|
new_section,
|
|
old_content, flags=re.MULTILINE, count=1)
|
|
|
|
def finalize_release(self, old_content: str) -> str:
|
|
"""Update the version and release date in the changelog content."""
|
|
version = self.info.version
|
|
date = self.options.release_date
|
|
return re.sub(r'^(=.* )\S+( branch released )\S+$',
|
|
rf'\g<1>{version}\g<2>{date}',
|
|
old_content, flags=re.MULTILINE, count=1)
|
|
|
|
def run(self) -> None:
|
|
"""Assemble the changelog if needed."""
|
|
subprocess.check_call(['framework/scripts/assemble_changelog.py'],
|
|
cwd=self.info.top_dir)
|
|
if self.files_are_clean('ChangeLog') and \
|
|
self.info.old_version != self.info.version:
|
|
# Edge case: no change since the previous release.
|
|
# This could happen, for example, in an emergency release
|
|
# of Mbed TLS to ship a crypto bug fix, or when testing
|
|
# the release script.
|
|
self.edit_file('ChangeLog', self.create_section)
|
|
else:
|
|
self.edit_file('ChangeLog', self.finalize_release)
|
|
self.git_commit_maybe(['ChangeLog', 'ChangeLog.d'],
|
|
'Assemble changelog and set release date')
|
|
|
|
|
|
class BumpVersionStep(Step):
|
|
"""Bump the product version and commit the result.
|
|
|
|
Do nothing if the product version is already as expected.
|
|
|
|
Note: this step does not check or affect submodules.
|
|
|
|
Note: this step does not currently handle ABI version bumps,
|
|
only product version bumps.
|
|
"""
|
|
|
|
@classmethod
|
|
def name(cls) -> str:
|
|
return 'version'
|
|
|
|
# Files and directories that may contain version information.
|
|
FILES_WITH_VERSION = [
|
|
'CMakeLists.txt',
|
|
'doxygen',
|
|
'include',
|
|
'tests/suites',
|
|
]
|
|
|
|
def run(self) -> None:
|
|
"""Bump the product version if needed."""
|
|
subprocess.check_call(['scripts/bump_version.sh',
|
|
'--version', self.info.version],
|
|
cwd=self.info.top_dir)
|
|
self.git_commit_maybe(self.FILES_WITH_VERSION,
|
|
'Bump version to ' + self.info.version)
|
|
|
|
|
|
class ArchiveStep(Step):
|
|
"""Prepare release archives and the associated checksum file."""
|
|
|
|
@classmethod
|
|
def name(cls) -> str:
|
|
return 'archive'
|
|
|
|
def tar_git_files(self, plain_tar_path: str, prefix: str) -> None:
|
|
"""Create an uncompressed tar files with the git contents."""
|
|
# We archive the index, not the commit 'HEAD', because
|
|
# we may have modified some files to turn off GEN_FILES.
|
|
# We do use the commit mtime rather than the current time,
|
|
# however, for reproducibility.
|
|
# A downside of not archiving a commit is that the archive won't
|
|
# contain an extended header with the commit ID that
|
|
# `git get-tar-commit-id` could retrieve. If we change to releasing
|
|
# an exact commit, we should make sure that the commit gets published.
|
|
index = self.git_index_as_tree_ish()
|
|
mtime = self.commit_timestamp()
|
|
# Note that `git archive --mtime=...` requires Git >= 2.40.0.
|
|
self.call_git(['archive', '--format=tar',
|
|
'--mtime', str(mtime),
|
|
'--prefix', prefix,
|
|
'--output', plain_tar_path,
|
|
index])
|
|
for submodule in self.info.submodules:
|
|
index = self.git_index_as_tree_ish(where=submodule)
|
|
mtime = self.commit_timestamp(where=submodule)
|
|
data = self.read_git(['archive', '--format=tar',
|
|
'--mtime', str(mtime),
|
|
'--prefix', prefix + submodule + '/',
|
|
index],
|
|
where=submodule)
|
|
subprocess.run([self.options.tar_command, '--catenate',
|
|
'-f', plain_tar_path,
|
|
'/dev/stdin'],
|
|
input=data,
|
|
check=True)
|
|
|
|
GEN_FILES_FILES = [
|
|
'CMakeLists.txt',
|
|
'tf-psa-crypto/CMakeLists.txt',
|
|
]
|
|
|
|
@staticmethod
|
|
def set_gen_files_default_off(old_content: str) -> str:
|
|
"""Make GEN_FILES default off in a build script."""
|
|
# CMakeLists.txt
|
|
new_content = re.sub(r'(option\(GEN_FILES\b.*)\bON\)',
|
|
r'\g<1>OFF)',
|
|
old_content)
|
|
return new_content
|
|
|
|
def turn_off_gen_files(self) -> None:
|
|
"""Make GEN_FILES default off in build scripts."""
|
|
for filename in self.GEN_FILES_FILES:
|
|
path = self.info.top_dir / filename
|
|
if path.exists():
|
|
self.edit_file(path, self.set_gen_files_default_off)
|
|
self.call_git(['add', path.name], where=path.parent)
|
|
|
|
def restore_gen_files(self) -> None:
|
|
"""Restore build scripts affected by turn_off_gen_files()."""
|
|
for filename in self.GEN_FILES_FILES:
|
|
path = self.info.top_dir / filename
|
|
if path.exists():
|
|
self.call_git(['reset', '--', path.name],
|
|
where=path.parent)
|
|
self.call_git(['checkout', '--', path.name],
|
|
where=path.parent)
|
|
|
|
@staticmethod
|
|
def list_project_generated_files(project_dir: pathlib.Path) -> List[str]:
|
|
"""Return the list of generated files in the given (sub)project.
|
|
|
|
The returned file names are relative to the project root.
|
|
"""
|
|
raw_list = subprocess.check_output(
|
|
['framework/scripts/make_generated_files.py', '--list'],
|
|
cwd=project_dir)
|
|
return raw_list.decode('ascii').rstrip('\n').split('\n')
|
|
|
|
@staticmethod
|
|
def update_project_generated_files(project_dir: pathlib.Path) -> None:
|
|
"""Regenerate the generated files in the given (sub)project."""
|
|
subprocess.check_call(
|
|
['framework/scripts/make_generated_files.py'],
|
|
cwd=project_dir)
|
|
|
|
def tar_add_project_generated_files(self,
|
|
plain_tar_path: str,
|
|
project_dir: pathlib.Path,
|
|
project_prefix: str,
|
|
file_list: List[str]) -> None:
|
|
transform = 's/^/' + project_prefix.replace('/', '\\/') + '/g'
|
|
commit_datetime = self.commit_datetime(project_dir)
|
|
file_datetime = commit_datetime + datetime.timedelta(seconds=1)
|
|
subprocess.check_call([self.options.tar_command, '-r',
|
|
'-f', plain_tar_path,
|
|
'--transform', transform,
|
|
'--owner=root:0', '--group=root:0',
|
|
'--mode=a+rX,u+w,go-w',
|
|
'--mtime', file_datetime.isoformat(),
|
|
'--'] + file_list,
|
|
cwd=project_dir)
|
|
|
|
def tar_add_generated_files(self, plain_tar_path: str, prefix: str) -> None:
|
|
"""Add generated files to an existing uncompressed tar file."""
|
|
for project in [os.curdir] + list(self.info.submodules):
|
|
if project.endswith('/framework') or project == 'framework':
|
|
continue
|
|
project_dir = self.info.top_dir / project
|
|
project_prefix = (prefix if project == os.curdir else
|
|
prefix + project + '/')
|
|
gen_script = project_dir / 'framework/scripts/make_generated_files.py'
|
|
# Skip submodules that do not use the framework build machinery.
|
|
if not gen_script.exists():
|
|
continue
|
|
file_list = self.list_project_generated_files(project_dir)
|
|
self.update_project_generated_files(project_dir)
|
|
self.tar_add_project_generated_files(plain_tar_path,
|
|
project_dir,
|
|
project_prefix,
|
|
file_list)
|
|
|
|
def create_plain_tar(self, plain_tar_path: str, prefix: str) -> None:
|
|
"""Create an uncompressed tar file for the release."""
|
|
self.tar_git_files(plain_tar_path, prefix)
|
|
self.tar_add_generated_files(plain_tar_path, prefix)
|
|
|
|
@staticmethod
|
|
def compress_tar(plain_tar_path: str) -> Iterator[str]:
|
|
"""Compress the tar file.
|
|
|
|
Remove the original, uncompressed file.
|
|
Yield the list of compressed files.
|
|
"""
|
|
compressed_path = plain_tar_path + '.bz2'
|
|
if os.path.exists(compressed_path):
|
|
os.remove(compressed_path)
|
|
subprocess.check_call(['bzip2', '-9', plain_tar_path])
|
|
yield compressed_path
|
|
|
|
def create_checksum_file(self, archive_paths: List[str]) -> None:
|
|
"""Create a checksum file for the given files."""
|
|
checksum_path = self.artifact_path('.txt')
|
|
relative_paths = [os.path.relpath(path, self.artifact_directory)
|
|
for path in archive_paths]
|
|
content = subprocess.check_output(['sha256sum', '--'] + relative_paths,
|
|
cwd=self.artifact_directory,
|
|
encoding='ascii')
|
|
with open(checksum_path, 'w') as out:
|
|
out.write(content)
|
|
|
|
def run(self) -> None:
|
|
"""Create the release artifacts."""
|
|
self.artifact_directory.mkdir(exist_ok=True)
|
|
self.turn_off_gen_files()
|
|
base_name = self.artifact_base_name()
|
|
plain_tar_path = str(self.artifact_path('.tar'))
|
|
self.create_plain_tar(plain_tar_path, base_name + '/')
|
|
compressed_paths = list(self.compress_tar(plain_tar_path))
|
|
self.create_checksum_file(compressed_paths)
|
|
self.restore_gen_files()
|
|
|
|
|
|
class NotesStep(Step):
|
|
"""Prepare draft release notes."""
|
|
|
|
def __init__(self, options: Options, info: Info) -> None:
|
|
super().__init__(options, info)
|
|
self.checksum_path = self.artifact_path('.txt')
|
|
|
|
@classmethod
|
|
def name(cls) -> str:
|
|
return 'notes'
|
|
|
|
def assert_preconditions(self) -> None:
|
|
assert self.checksum_path.exists()
|
|
|
|
def read_changelog(self) -> Dict[Optional[str], str]:
|
|
"""Return the section of the changelog for this release.
|
|
|
|
The result is split into categories.
|
|
Use the index None to get the whole content.
|
|
"""
|
|
with open(self.info.top_dir / 'ChangeLog', encoding='utf-8') as inp:
|
|
whole_file = inp.read()
|
|
version_iter = re.finditer('^=.*', whole_file, re.MULTILINE)
|
|
start = next(version_iter).end()
|
|
end = next(version_iter).start()
|
|
whole = whole_file[start:end].strip()
|
|
sections = {None: whole} #type: Dict[Optional[str], str]
|
|
headings = [(m.group(), m.start(), m.end())
|
|
for m in re.finditer(r'^\w.*', whole, re.MULTILINE)]
|
|
for (this, next_) in zip(headings, headings[1:] + [('', -1, -1)]):
|
|
title = this[0]
|
|
start = this[2]
|
|
end = next_[1]
|
|
sections[title] = whole[start:end].strip()
|
|
return sections
|
|
|
|
@staticmethod
|
|
def advisory_items(changelog) -> Iterator[str]:
|
|
"""Yield the items for the list of advisories."""
|
|
bullets = [m.start()
|
|
for m in re.finditer(r'^ \* *', changelog, re.MULTILINE)]
|
|
for (this, next_) in zip(bullets, bullets[1:] + [-1]):
|
|
# We don't have the advisory title here, so make up something
|
|
# that the release handler will need to fill in.
|
|
content = changelog[this:next_]
|
|
teaser = re.sub(r'\n.*', '',
|
|
re.sub(r'^\W*', '', content),
|
|
re.DOTALL)
|
|
cve_match = re.search(r'CVE-[-0-9]+', content)
|
|
if cve_match:
|
|
cve_text = f' ({cve_match.group(0)})'
|
|
else:
|
|
cve_text = ''
|
|
# We don't have enough data to find the last component of the URL.
|
|
url = 'https://mbed-tls.readthedocs.io/en/latest/security-advisories/'
|
|
yield f'* [{teaser} …{cve_text}]({url})'
|
|
|
|
def advisories(self, changelog) -> str:
|
|
"""Format links to advisories for the release notes.
|
|
|
|
The argument is just the "Security" section of the changelog,
|
|
without its heading.
|
|
"""
|
|
items = self.advisory_items(changelog)
|
|
return ('For full details, please see the following links:\n' +
|
|
'[TODO: this section needs manual editing!]\n' +
|
|
'\n'.join(items))
|
|
|
|
def notes(self) -> str:
|
|
"""Construct draft release notes."""
|
|
changelog = self.read_changelog()
|
|
advisories = (self.advisories(changelog['Security'])
|
|
if 'Security' in changelog
|
|
else 'None.')
|
|
with open(self.checksum_path, encoding='ascii') as inp:
|
|
checksums = inp.read()
|
|
# We currently emit a single archive file. If we switch to having
|
|
# multiple files, tweak this definition and the grammar where
|
|
# it's used.
|
|
m = re.search(r'\S+$', checksums)
|
|
if m is None:
|
|
raise Exception('Unable to determine archive file name')
|
|
archive_name = m.group(0)
|
|
# The very long lines here are deliberate. GitHub treats newlines
|
|
# in markdown as line breaks, not as spaces, so an ordinary paragraph
|
|
# needs to be in a single Python logical line.
|
|
return f'''\
|
|
## Description
|
|
|
|
This release of {self.info.product_human_name} provides new features, bug fixes and minor enhancements. \
|
|
{'This release includes fixes for security issues.' if 'Security' in changelog else ''}
|
|
|
|
## Security Advisories
|
|
|
|
{advisories}
|
|
|
|
## Release Notes
|
|
|
|
{changelog[None]}
|
|
|
|
## Who should update
|
|
|
|
We recommend all users should update to take advantage of the bug fixes contained in this release at an appropriate point in their development lifecycle.
|
|
|
|
## Note
|
|
|
|
:grey_exclamation: `{archive_name}` is our official release file. `source.tar.gz` and `source.zip` are automatically generated snapshots that GitHub is generating. They do not include submodules or generated files, and [cannot be configured](https://github.com/orgs/community/discussions/6003).
|
|
|
|
## Checksum
|
|
|
|
The SHA256 hashes for the archives are:
|
|
```
|
|
{checksums}
|
|
```
|
|
'''
|
|
|
|
def run(self) -> None:
|
|
content = self.notes()
|
|
with open(self.artifact_path('.md'), 'w') as out:
|
|
out.write(content)
|
|
|
|
|
|
ALL_STEPS = [
|
|
AssembleChangelogStep,
|
|
BumpVersionStep,
|
|
ArchiveStep,
|
|
NotesStep,
|
|
] #type: Sequence[typing.Type[Step]]
|
|
|
|
|
|
def init_steps(options: Options,
|
|
info: Info,
|
|
#pylint: disable=dangerous-default-value
|
|
step_classes: Sequence[typing.Type[Step]] = ALL_STEPS,
|
|
from_step: Optional[str] = None,
|
|
to_step: Optional[str] = None) -> Sequence[Step]:
|
|
"""Initialize the selected steps without running them."""
|
|
steps = [step_class(options, info) for step_class in step_classes]
|
|
if from_step is not None:
|
|
for n, step in enumerate(steps):
|
|
if step.name() == from_step:
|
|
del steps[:n]
|
|
break
|
|
else:
|
|
raise Exception(f'Step name not found: {from_step}')
|
|
if to_step is not None:
|
|
for n, step in enumerate(steps):
|
|
if step.name() == to_step:
|
|
del steps[n+1:]
|
|
break
|
|
else:
|
|
after_msg = f' after {from_step}' if from_step is not None else ''
|
|
raise Exception(f'Step name not found{after_msg}: {to_step}')
|
|
return steps
|
|
|
|
def run(options: Options,
|
|
top_dir: str,
|
|
from_step: Optional[str] = None,
|
|
to_step: Optional[str] = None) -> None:
|
|
"""Run the release process (or a segment thereof)."""
|
|
info = Info(top_dir, options)
|
|
for step in init_steps(options, info, from_step=from_step, to_step=to_step):
|
|
step.assert_preconditions()
|
|
step.run()
|
|
|
|
def main() -> None:
|
|
"""Command line entry point."""
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
# Options that affect information gathering, or that affect the
|
|
# behavior of one or more steps, should have an associated field
|
|
# in the Options class.
|
|
parser.add_argument('--directory', metavar='DIR',
|
|
default=os.curdir,
|
|
help='Product toplevel directory (default: .)')
|
|
parser.add_argument('--artifact-directory', '-a', metavar='DIR',
|
|
help=('Directory where release artifacts will be placed '
|
|
'(default/empty: <--directory>/release-artifacts)'))
|
|
parser.add_argument('--from-step', '--from', '-f', metavar='STEP',
|
|
help='First step to run (default: run all steps)')
|
|
parser.add_argument('--ignore-submodules',
|
|
action='store_true',
|
|
help='Ignore changes to submodules and changed files within submodules')
|
|
parser.add_argument('--list-steps',
|
|
action='store_true',
|
|
help='List release steps and exit')
|
|
parser.add_argument('--only-step', '-s', metavar='STEP',
|
|
help=('Run only this step (default: run all steps) '
|
|
'(equivalent to --from-step=STEP --to-step=STEP)'))
|
|
parser.add_argument('--release-date', '-d',
|
|
help='Release date (YYYY-mm-dd) (default: today)')
|
|
parser.add_argument('--release-version', '-r',
|
|
help='The version to release (default/empty: from ChangeLog)')
|
|
parser.add_argument('--tar-command',
|
|
help='GNU tar command')
|
|
parser.add_argument('--to-step', '-t', metavar='STEP',
|
|
help='Last step to run (default: run all steps)')
|
|
parser.set_defaults(**DEFAULT_OPTIONS._asdict())
|
|
args = parser.parse_args()
|
|
|
|
# Process help-and-exit options
|
|
if args.list_steps:
|
|
for step in ALL_STEPS:
|
|
sys.stdout.write(step.name() + '\n')
|
|
return
|
|
|
|
if args.only_step:
|
|
args.from_step = args.only_step
|
|
args.to_step = args.only_step
|
|
if args.artifact_directory is None:
|
|
artifact_directory = None
|
|
else:
|
|
artifact_directory = pathlib.Path(args.artifact_directory)
|
|
options = Options(
|
|
artifact_directory=artifact_directory,
|
|
ignore_submodules=args.ignore_submodules,
|
|
release_date=args.release_date,
|
|
release_version=args.release_version,
|
|
tar_command=args.tar_command)
|
|
|
|
run(options, args.directory,
|
|
from_step=args.from_step, to_step=args.to_step)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|