Merge pull request #117 from Harry-Ramsey/tf-psa-crypto-check-names

Adapt check_names.py for independent TF-PSA-Crypto testing
This commit is contained in:
Ronald Cron
2025-01-31 11:58:43 +01:00
committed by GitHub
+322 -156
View File
@@ -8,12 +8,14 @@ This script confirms that the naming of all symbols and identifiers in Mbed TLS
are consistent with the house style and are also self-consistent. It only runs
on Linux and macOS since it depends on nm.
It contains two major Python classes, CodeParser and NameChecker. They both have
a comprehensive "run-all" function (comprehensive_parse() and perform_checks())
but the individual functions can also be used for specific needs.
It contains three major Python classes, TFPSACryptoCodeParser,
MBEDTLSCodeParser and NameChecker. They all have a comprehensive "run-all"
function (comprehensive_parse() and perform_checks()) but the individual
functions can also be used for specific needs.
CodeParser makes heavy use of regular expressions to parse the code, and is
dependent on the current code formatting. Many Python C parser libraries require
CodeParser(a inherent base class for TFPSACryptoCodeParser and MBEDTLSCodeParser)
makes heavy use of regular expressions to parse the code, and is dependent on
the current code formatting. Many Python C parser libraries require
preprocessed C code, which means no macro parsing. Compiler tools are also not
very helpful when we want the exact location in the original source (which
becomes impossible when e.g. comments are stripped).
@@ -43,6 +45,7 @@ import enum
import shutil
import subprocess
import logging
import tempfile
import project_scripts # pylint: disable=unused-import
from mbedtls_framework import build_tree
@@ -212,7 +215,8 @@ class CodeParser():
"""
def __init__(self, log):
self.log = log
build_tree.check_repo_path()
if not build_tree.looks_like_root(os.getcwd()):
raise Exception("This script must be run from Mbed TLS or TF-PSA-Crypto root")
# Memo for storing "glob expression": set(filepaths)
self.files = {}
@@ -221,126 +225,21 @@ class CodeParser():
# Note that "*" can match directory separators in exclude lists.
self.excluded_files = ["*/bn_mul", "*/compat-2.x.h"]
def comprehensive_parse(self):
def _parse(self, all_macros, enum_consts, identifiers,
excluded_identifiers, mbed_psa_words, symbols):
# pylint: disable=too-many-arguments
"""
Comprehensive ("default") function to call each parsing function and
retrieve various elements of the code, together with the source location.
Parse macros, enums, identifiers, excluded identifiers, Mbed PSA word and Symbols.
Returns a dict of parsed item key to the corresponding List of Matches.
"""
self.log.info("Parsing source code...")
self.log.debug(
"The following files are excluded from the search: {}"
.format(str(self.excluded_files))
)
all_macros = {"public": [], "internal": [], "private":[]}
if build_tree.is_mbedtls_3_6():
all_macros["public"] = self.parse_macros([
"include/mbedtls/*.h",
"include/psa/*.h",
"3rdparty/everest/include/everest/everest.h",
"3rdparty/everest/include/everest/x25519.h"
])
all_macros["internal"] = self.parse_macros([
"library/*.h",
"framework/tests/include/test/drivers/*.h",
])
all_macros["private"] = self.parse_macros([
"library/*.c",
])
enum_consts = self.parse_enum_consts([
"include/mbedtls/*.h",
"include/psa/*.h",
"library/*.h",
"library/*.c",
"3rdparty/everest/include/everest/everest.h",
"3rdparty/everest/include/everest/x25519.h"
])
identifiers, excluded_identifiers = self.parse_identifiers([
"include/mbedtls/*.h",
"include/psa/*.h",
"library/*.h",
"3rdparty/everest/include/everest/everest.h",
"3rdparty/everest/include/everest/x25519.h"
], ["3rdparty/p256-m/p256-m/p256-m.h"])
mbed_psa_words = self.parse_mbed_psa_words([
"include/mbedtls/*.h",
"include/psa/*.h",
"library/*.h",
"3rdparty/everest/include/everest/everest.h",
"3rdparty/everest/include/everest/x25519.h",
"library/*.c",
"3rdparty/everest/library/everest.c",
"3rdparty/everest/library/x25519.c"
], ["library/psa_crypto_driver_wrappers.h"])
else:
all_macros["public"] = self.parse_macros([
"include/mbedtls/*.h",
"include/psa/*.h",
"tf-psa-crypto/include/psa/*.h",
"tf-psa-crypto/include/tf-psa-crypto/*.h",
"tf-psa-crypto/drivers/builtin/include/mbedtls/*.h",
"tf-psa-crypto/drivers/everest/include/everest/everest.h",
"tf-psa-crypto/drivers/everest/include/everest/x25519.h"
])
all_macros["internal"] = self.parse_macros([
"library/*.h",
"tf-psa-crypto/core/*.h",
"tf-psa-crypto/drivers/builtin/src/*.h",
"framework/tests/include/test/drivers/*.h",
])
all_macros["private"] = self.parse_macros([
"library/*.c",
"tf-psa-crypto/core/*.c",
"tf-psa-crypto/drivers/builtin/src/*.c",
])
enum_consts = self.parse_enum_consts([
"include/mbedtls/*.h",
"include/psa/*.h",
"tf-psa-crypto/include/psa/*.h",
"tf-psa-crypto/include/tf-psa-crypto/*.h",
"tf-psa-crypto/drivers/builtin/include/mbedtls/*.h",
"library/*.h",
"tf-psa-crypto/core/*.h",
"tf-psa-crypto/drivers/builtin/src/*.h",
"library/*.c",
"tf-psa-crypto/core/*.c",
"tf-psa-crypto/drivers/builtin/src/*.c",
"tf-psa-crypto/drivers/everest/include/everest/everest.h",
"tf-psa-crypto/drivers/everest/include/everest/x25519.h"
])
identifiers, excluded_identifiers = self.parse_identifiers([
"include/mbedtls/*.h",
"include/psa/*.h",
"tf-psa-crypto/include/psa/*.h",
"tf-psa-crypto/include/tf-psa-crypto/*.h",
"tf-psa-crypto/drivers/builtin/include/mbedtls/*.h",
"library/*.h",
"tf-psa-crypto/core/*.h",
"tf-psa-crypto/drivers/builtin/src/*.h",
"tf-psa-crypto/drivers/everest/include/everest/everest.h",
"tf-psa-crypto/drivers/everest/include/everest/x25519.h"
], ["tf-psa-crypto/drivers/p256-m/p256-m/p256-m.h"])
mbed_psa_words = self.parse_mbed_psa_words([
"include/mbedtls/*.h",
"include/psa/*.h",
"tf-psa-crypto/include/psa/*.h",
"tf-psa-crypto/include/tf-psa-crypto/*.h",
"tf-psa-crypto/drivers/builtin/include/mbedtls/*.h",
"library/*.h",
"tf-psa-crypto/core/*.h",
"tf-psa-crypto/drivers/builtin/src/*.h",
"tf-psa-crypto/drivers/everest/include/everest/everest.h",
"tf-psa-crypto/drivers/everest/include/everest/x25519.h",
"library/*.c",
"tf-psa-crypto/core/*.c",
"tf-psa-crypto/drivers/builtin/src/*.c",
"tf-psa-crypto/drivers/everest/library/everest.c",
"tf-psa-crypto/drivers/everest/library/x25519.c"
], ["tf-psa-crypto/core/psa_crypto_driver_wrappers.h"])
symbols = self.parse_symbols()
# Remove identifier macros like mbedtls_printf or mbedtls_calloc
identifiers_justname = [x.name for x in identifiers]
actual_macros = {"public": [], "internal": []}
@@ -725,6 +624,304 @@ class CodeParser():
return (included_identifiers, excluded_identifiers)
def parse_symbols(self):
"""
Compile a library, and parse the object files using nm to retrieve the
list of referenced symbols. Exceptions thrown here are rethrown because
they would be critical errors that void several tests, and thus needs
to halt the program. This is explicitly done for clarity.
Returns a List of unique symbols defined and used in the libraries.
"""
raise NotImplementedError("parse_symbols must be implemented by a code parser")
def comprehensive_parse(self):
"""
(Must be defined as a class method)
Comprehensive ("default") function to call each parsing function and
retrieve various elements of the code, together with the source location.
Returns a dict of parsed item key to the corresponding List of Matches.
"""
raise NotImplementedError("comprehension_parse must be implemented by a code parser")
def parse_symbols_from_nm(self, object_files):
"""
Run nm to retrieve the list of referenced symbols in each object file.
Does not return the position data since it is of no use.
Args:
* object_files: a List of compiled object filepaths to search through.
Returns a List of unique symbols defined and used in any of the object
files.
"""
nm_undefined_regex = re.compile(r"^\S+: +U |^$|^\S+:$")
nm_valid_regex = re.compile(r"^\S+( [0-9A-Fa-f]+)* . _*(?P<symbol>\w+)")
exclusions = ("FStar", "Hacl")
symbols = []
# Gather all outputs of nm
nm_output = ""
for lib in object_files:
nm_output += subprocess.run(
["nm", "-og", lib],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=True
).stdout
for line in nm_output.splitlines():
if not nm_undefined_regex.search(line):
symbol = nm_valid_regex.search(line)
if (symbol and not symbol.group("symbol").startswith(exclusions)):
symbols.append(symbol.group("symbol"))
else:
self.log.error(line)
return symbols
class TFPSACryptoCodeParser(CodeParser):
"""
Class for retrieving files and parsing TF-PSA-Crypto code. This can be used
independently of the checks that NameChecker performs.
"""
def __init__(self, log):
super().__init__(log)
if not build_tree.looks_like_tf_psa_crypto_root(os.getcwd()):
raise Exception("This script must be run from TF-PSA-Crypto root.")
def comprehensive_parse(self):
"""
Comprehensive ("default") function to call each parsing function and
retrieve various elements of the code, together with the source location.
Returns a dict of parsed item key to the corresponding List of Matches.
"""
all_macros = {"public": [], "internal": [], "private":[]}
all_macros["public"] = self.parse_macros([
"include/psa/*.h",
"include/tf-psa-crypto/*.h",
"drivers/builtin/include/mbedtls/*.h",
"drivers/everest/include/everest/everest.h",
"drivers/everest/include/everest/x25519.h"
])
all_macros["internal"] = self.parse_macros([
"core/*.h",
"drivers/builtin/src/*.h",
"framework/tests/include/test/drivers/*.h",
])
all_macros["private"] = self.parse_macros([
"core/*.c",
"drivers/builtin/src/*.c",
])
enum_consts = self.parse_enum_consts([
"include/psa/*.h",
"include/tf-psa-crypto/*.h",
"drivers/builtin/include/mbedtls/*.h",
"core/*.h",
"drivers/builtin/src/*.h",
"core/*.c",
"drivers/builtin/src/*.c",
"drivers/everest/include/everest/everest.h",
"drivers/everest/include/everest/x25519.h"
])
identifiers, excluded_identifiers = self.parse_identifiers([
"include/psa/*.h",
"include/tf-psa-crypto/*.h",
"drivers/builtin/include/mbedtls/*.h",
"core/*.h",
"drivers/builtin/src/*.h",
"drivers/everest/include/everest/everest.h",
"drivers/everest/include/everest/x25519.h"
], ["drivers/p256-m/p256-m/p256-m.h"])
mbed_psa_words = self.parse_mbed_psa_words([
"include/psa/*.h",
"include/tf-psa-crypto/*.h",
"drivers/builtin/include/mbedtls/*.h",
"core/*.h",
"drivers/builtin/src/*.h",
"drivers/everest/include/everest/everest.h",
"drivers/everest/include/everest/x25519.h",
"core/*.c",
"drivers/builtin/src/*.c",
"drivers/everest/library/everest.c",
"drivers/everest/library/x25519.c"
], ["core/psa_crypto_driver_wrappers.h"])
symbols = self.parse_symbols()
return self._parse(all_macros, enum_consts, identifiers,
excluded_identifiers, mbed_psa_words, symbols)
def parse_symbols(self):
"""
Compile the TF-PSA-Crypto libraries, and parse the
object files using nm to retrieve the list of referenced symbols.
Exceptions thrown here are rethrown because they would be critical
errors that void several tests, and thus needs to halt the program. This
is explicitly done for clarity.
Returns a List of unique symbols defined and used in the libraries.
"""
self.log.info("Compiling...")
symbols = []
# Back up the config and atomically compile with the full configuration.
shutil.copy(
"include/psa/crypto_config.h",
"include/psa/crypto_config.h.bak"
)
try:
# Use check=True in all subprocess calls so that failures are raised
# as exceptions and logged.
subprocess.run(
["python3", "scripts/config.py", "full"],
universal_newlines=True,
check=True
)
my_environment = os.environ.copy()
my_environment["CFLAGS"] = "-fno-asynchronous-unwind-tables"
source_dir = os.getcwd()
build_dir = tempfile.mkdtemp()
os.chdir(build_dir)
subprocess.run(
["cmake", "-DGEN_FILES=ON", source_dir],
universal_newlines=True,
check=True
)
subprocess.run(
["make"],
env=my_environment,
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=True
)
# Perform object file analysis using nm
symbols = self.parse_symbols_from_nm([
build_dir + "/drivers/builtin/libbuiltin.a",
build_dir + "/drivers/p256-m/libp256m.a",
build_dir + "/drivers/everest/libeverest.a",
build_dir + "/core/libtfpsacrypto.a"
])
os.chdir(source_dir)
shutil.rmtree(build_dir)
except subprocess.CalledProcessError as error:
self.log.debug(error.output)
raise error
finally:
# Put back the original config regardless of there being errors.
# Works also for keyboard interrupts.
shutil.move(
"include/psa/crypto_config.h.bak",
"include/psa/crypto_config.h"
)
return symbols
class MBEDTLSCodeParser(CodeParser):
"""
Class for retrieving files and parsing Mbed TLS code. This can be used
independently of the checks that NameChecker performs.
"""
def __init__(self, log):
super().__init__(log)
if not build_tree.looks_like_mbedtls_root(os.getcwd()):
raise Exception("This script must be run from Mbed TLS root.")
def comprehensive_parse(self):
"""
Comprehensive ("default") function to call each parsing function and
retrieve various elements of the code, together with the source location.
Returns a dict of parsed item key to the corresponding List of Matches.
"""
all_macros = {"public": [], "internal": [], "private":[]}
# TF-PSA-Crypto is in the same repo in 3.6 so initalise variable here.
tf_psa_crypto_parse_result = {}
if build_tree.is_mbedtls_3_6():
all_macros["public"] = self.parse_macros([
"include/mbedtls/*.h",
"include/psa/*.h",
"3rdparty/everest/include/everest/everest.h",
"3rdparty/everest/include/everest/x25519.h"
])
all_macros["internal"] = self.parse_macros([
"library/*.h",
"framework/tests/include/test/drivers/*.h",
])
all_macros["private"] = self.parse_macros([
"library/*.c",
])
enum_consts = self.parse_enum_consts([
"include/mbedtls/*.h",
"include/psa/*.h",
"library/*.h",
"library/*.c",
"3rdparty/everest/include/everest/everest.h",
"3rdparty/everest/include/everest/x25519.h"
])
identifiers, excluded_identifiers = self.parse_identifiers([
"include/mbedtls/*.h",
"include/psa/*.h",
"library/*.h",
"3rdparty/everest/include/everest/everest.h",
"3rdparty/everest/include/everest/x25519.h"
], ["3rdparty/p256-m/p256-m/p256-m.h"])
mbed_psa_words = self.parse_mbed_psa_words([
"include/mbedtls/*.h",
"include/psa/*.h",
"library/*.h",
"3rdparty/everest/include/everest/everest.h",
"3rdparty/everest/include/everest/x25519.h",
"library/*.c",
"3rdparty/everest/library/everest.c",
"3rdparty/everest/library/x25519.c"
], ["library/psa_crypto_driver_wrappers.h"])
else:
all_macros = {"public": [], "internal": [], "private":[]}
all_macros["public"] = self.parse_macros([
"include/mbedtls/*.h",
])
all_macros["internal"] = self.parse_macros([
"library/*.h",
"framework/tests/include/test/drivers/*.h",
])
all_macros["private"] = self.parse_macros([
"library/*.c",
])
enum_consts = self.parse_enum_consts([
"include/mbedtls/*.h",
"library/*.h",
"library/*.c",
])
identifiers, excluded_identifiers = self.parse_identifiers([
"include/mbedtls/*.h",
"library/*.h",
])
mbed_psa_words = self.parse_mbed_psa_words([
"include/mbedtls/*.h",
"library/*.h",
"library/*.c",
])
os.chdir("./tf-psa-crypto")
tf_psa_crypto_code_parser = TFPSACryptoCodeParser(self.log)
tf_psa_crypto_parse_result = tf_psa_crypto_code_parser.comprehensive_parse()
os.chdir("../")
symbols = self.parse_symbols()
mbedtls_parse_result = self._parse(all_macros, enum_consts,
identifiers, excluded_identifiers,
mbed_psa_words, symbols)
# Combile results for Mbed TLS and TF-PSA-Crypto
for key in tf_psa_crypto_parse_result:
mbedtls_parse_result[key] += tf_psa_crypto_parse_result[key]
return mbedtls_parse_result
def parse_symbols(self):
"""
Compile the Mbed TLS libraries, and parse the TLS, Crypto, and x509
@@ -794,44 +991,6 @@ class CodeParser():
return symbols
def parse_symbols_from_nm(self, object_files):
"""
Run nm to retrieve the list of referenced symbols in each object file.
Does not return the position data since it is of no use.
Args:
* object_files: a List of compiled object filepaths to search through.
Returns a List of unique symbols defined and used in any of the object
files.
"""
nm_undefined_regex = re.compile(r"^\S+: +U |^$|^\S+:$")
nm_valid_regex = re.compile(r"^\S+( [0-9A-Fa-f]+)* . _*(?P<symbol>\w+)")
exclusions = ("FStar", "Hacl")
symbols = []
# Gather all outputs of nm
nm_output = ""
for lib in object_files:
nm_output += subprocess.run(
["nm", "-og", lib],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=True
).stdout
for line in nm_output.splitlines():
if not nm_undefined_regex.search(line):
symbol = nm_valid_regex.search(line)
if (symbol and not symbol.group("symbol").startswith(exclusions)):
symbols.append(symbol.group("symbol"))
else:
self.log.error(line)
return symbols
class NameChecker():
"""
Representation of the core name checking operation performed by this script.
@@ -1016,8 +1175,15 @@ def main():
log.addHandler(logging.StreamHandler())
try:
code_parser = CodeParser(log)
parse_result = code_parser.comprehensive_parse()
if build_tree.looks_like_tf_psa_crypto_root(os.getcwd()):
tf_psa_crypto_code_parser = TFPSACryptoCodeParser(log)
parse_result = tf_psa_crypto_code_parser.comprehensive_parse()
elif build_tree.looks_like_mbedtls_root(os.getcwd()):
# Mbed TLS uses TF-PSA-Crypto, so we need to parse TF-PSA-Crypto too
mbedtls_code_parser = MBEDTLSCodeParser(log)
parse_result = mbedtls_code_parser.comprehensive_parse()
else:
raise Exception("This script must be run from Mbed TLS or TF-PSA-Crypto root")
except Exception: # pylint: disable=broad-except
traceback.print_exc()
sys.exit(2)