From 9c5b27894252ac752dd9896293edfdfb7c660c3d Mon Sep 17 00:00:00 2001 From: Harry Ramsey Date: Thu, 9 Jan 2025 17:08:40 +0000 Subject: [PATCH 1/6] Enable independent TF-PSA-Crypto name checks This commit separates CodeParser into three classes. CodeParser a base class containing methods for parsing .c and .h files as well as retrieving symbols from compiled libraries. As well as two subclasses of CodeParser: TF-PSA-Crypto_CodeParser and MBEDTLS_CodeParser, which are responsible for parsing the TF-PSA-Crypto and Mbed TLS libraries. Signed-off-by: Harry Ramsey --- scripts/check_names.py | 459 +++++++++++++++++++++++++++-------------- 1 file changed, 308 insertions(+), 151 deletions(-) diff --git a/scripts/check_names.py b/scripts/check_names.py index 12daae090..f53f54667 100755 --- a/scripts/check_names.py +++ b/scripts/check_names.py @@ -212,7 +212,8 @@ class CodeParser(): """ def __init__(self, log): self.log = log - build_tree.check_repo_path() + if not build_tree.looks_like_root(os.getcwd()): + raise Exception("This script must be run from Mbed TLS or TF-PSA-Crypto root") # Memo for storing "glob expression": set(filepaths) self.files = {} @@ -221,126 +222,19 @@ class CodeParser(): # Note that "*" can match directory separators in exclude lists. self.excluded_files = ["*/bn_mul", "*/compat-2.x.h"] - def comprehensive_parse(self): + def _parse(self, all_macros, enum_consts, identifiers, excluded_identifiers, mbed_psa_words, symbols): """ - Comprehensive ("default") function to call each parsing function and - retrieve various elements of the code, together with the source location. + Parse macros, enums, identifiers, excluded identifiers, Mbed PSA word and Symbols. Returns a dict of parsed item key to the corresponding List of Matches. """ + self.log.info("Parsing source code...") self.log.debug( "The following files are excluded from the search: {}" .format(str(self.excluded_files)) ) - all_macros = {"public": [], "internal": [], "private":[]} - if build_tree.is_mbedtls_3_6(): - all_macros["public"] = self.parse_macros([ - "include/mbedtls/*.h", - "include/psa/*.h", - "3rdparty/everest/include/everest/everest.h", - "3rdparty/everest/include/everest/x25519.h" - ]) - all_macros["internal"] = self.parse_macros([ - "library/*.h", - "framework/tests/include/test/drivers/*.h", - ]) - all_macros["private"] = self.parse_macros([ - "library/*.c", - ]) - enum_consts = self.parse_enum_consts([ - "include/mbedtls/*.h", - "include/psa/*.h", - "library/*.h", - "library/*.c", - "3rdparty/everest/include/everest/everest.h", - "3rdparty/everest/include/everest/x25519.h" - ]) - identifiers, excluded_identifiers = self.parse_identifiers([ - "include/mbedtls/*.h", - "include/psa/*.h", - "library/*.h", - "3rdparty/everest/include/everest/everest.h", - "3rdparty/everest/include/everest/x25519.h" - ], ["3rdparty/p256-m/p256-m/p256-m.h"]) - mbed_psa_words = self.parse_mbed_psa_words([ - "include/mbedtls/*.h", - "include/psa/*.h", - "library/*.h", - "3rdparty/everest/include/everest/everest.h", - "3rdparty/everest/include/everest/x25519.h", - "library/*.c", - "3rdparty/everest/library/everest.c", - "3rdparty/everest/library/x25519.c" - ], ["library/psa_crypto_driver_wrappers.h"]) - else: - all_macros["public"] = self.parse_macros([ - "include/mbedtls/*.h", - "include/psa/*.h", - "tf-psa-crypto/include/psa/*.h", - "tf-psa-crypto/include/tf-psa-crypto/*.h", - "tf-psa-crypto/drivers/builtin/include/mbedtls/*.h", - "tf-psa-crypto/drivers/everest/include/everest/everest.h", - "tf-psa-crypto/drivers/everest/include/everest/x25519.h" - ]) - all_macros["internal"] = self.parse_macros([ - "library/*.h", - "tf-psa-crypto/core/*.h", - "tf-psa-crypto/drivers/builtin/src/*.h", - "framework/tests/include/test/drivers/*.h", - ]) - all_macros["private"] = self.parse_macros([ - "library/*.c", - "tf-psa-crypto/core/*.c", - "tf-psa-crypto/drivers/builtin/src/*.c", - ]) - enum_consts = self.parse_enum_consts([ - "include/mbedtls/*.h", - "include/psa/*.h", - "tf-psa-crypto/include/psa/*.h", - "tf-psa-crypto/include/tf-psa-crypto/*.h", - "tf-psa-crypto/drivers/builtin/include/mbedtls/*.h", - "library/*.h", - "tf-psa-crypto/core/*.h", - "tf-psa-crypto/drivers/builtin/src/*.h", - "library/*.c", - "tf-psa-crypto/core/*.c", - "tf-psa-crypto/drivers/builtin/src/*.c", - "tf-psa-crypto/drivers/everest/include/everest/everest.h", - "tf-psa-crypto/drivers/everest/include/everest/x25519.h" - ]) - identifiers, excluded_identifiers = self.parse_identifiers([ - "include/mbedtls/*.h", - "include/psa/*.h", - "tf-psa-crypto/include/psa/*.h", - "tf-psa-crypto/include/tf-psa-crypto/*.h", - "tf-psa-crypto/drivers/builtin/include/mbedtls/*.h", - "library/*.h", - "tf-psa-crypto/core/*.h", - "tf-psa-crypto/drivers/builtin/src/*.h", - "tf-psa-crypto/drivers/everest/include/everest/everest.h", - "tf-psa-crypto/drivers/everest/include/everest/x25519.h" - ], ["tf-psa-crypto/drivers/p256-m/p256-m/p256-m.h"]) - mbed_psa_words = self.parse_mbed_psa_words([ - "include/mbedtls/*.h", - "include/psa/*.h", - "tf-psa-crypto/include/psa/*.h", - "tf-psa-crypto/include/tf-psa-crypto/*.h", - "tf-psa-crypto/drivers/builtin/include/mbedtls/*.h", - "library/*.h", - "tf-psa-crypto/core/*.h", - "tf-psa-crypto/drivers/builtin/src/*.h", - "tf-psa-crypto/drivers/everest/include/everest/everest.h", - "tf-psa-crypto/drivers/everest/include/everest/x25519.h", - "library/*.c", - "tf-psa-crypto/core/*.c", - "tf-psa-crypto/drivers/builtin/src/*.c", - "tf-psa-crypto/drivers/everest/library/everest.c", - "tf-psa-crypto/drivers/everest/library/x25519.c" - ], ["tf-psa-crypto/core/psa_crypto_driver_wrappers.h"]) - symbols = self.parse_symbols() - # Remove identifier macros like mbedtls_printf or mbedtls_calloc identifiers_justname = [x.name for x in identifiers] actual_macros = {"public": [], "internal": []} @@ -725,6 +619,294 @@ class CodeParser(): return (included_identifiers, excluded_identifiers) + def parse_symbols(self): + """ + Compile a library, and parse the object files using nm to retrieve the + list of referenced symbols. Exceptions thrown here are rethrown because + they would be critical errors that void several tests, and thus needs + to halt the program. This is explicitly done for clarity. + + Returns a List of unique symbols defined and used in the libraries. + """ + raise NotImplementedError("parse_symbols must be implemented by a code parser") + + def comprehensive_parse(self): + """ + (Must be defined as a class method) + Comprehensive ("default") function to call each parsing function and + retrieve various elements of the code, together with the source location. + + Returns a dict of parsed item key to the corresponding List of Matches. + """ + raise NotImplementedError("comprehension_parse must be implemented by a code parser") + + def parse_symbols_from_nm(self, object_files): + """ + Run nm to retrieve the list of referenced symbols in each object file. + Does not return the position data since it is of no use. + + Args: + * object_files: a List of compiled object filepaths to search through. + + Returns a List of unique symbols defined and used in any of the object + files. + """ + nm_undefined_regex = re.compile(r"^\S+: +U |^$|^\S+:$") + nm_valid_regex = re.compile(r"^\S+( [0-9A-Fa-f]+)* . _*(?P\w+)") + exclusions = ("FStar", "Hacl") + symbols = [] + # Gather all outputs of nm + nm_output = "" + for lib in object_files: + nm_output += subprocess.run( + ["nm", "-og", lib], + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True + ).stdout + for line in nm_output.splitlines(): + if not nm_undefined_regex.search(line): + symbol = nm_valid_regex.search(line) + if (symbol and not symbol.group("symbol").startswith(exclusions)): + symbols.append(symbol.group("symbol")) + else: + self.log.error(line) + return symbols + +class TF_PSA_Crypto_CodeParser(CodeParser): + """ + Class for retrieving files and parsing TF-PSA-Crypto code. This can be used + independently of the checks that NameChecker performs, for example for + list_internal_identifiers.py. + """ + + def __init__(self, log): + super().__init__(log) + if build_tree.looks_like_tf_psa_crypto_root(os.getcwd()): + self.source_dir = os.getcwd() + else: + self.source_dir = os.path.join(os.getcwd(), "tf-psa-crypto") + self.out_of_source_dir = os.path.join(os.getcwd(), "out_of_source_dir") + if not os.path.exists(self.out_of_source_dir): + os.mkdir(self.out_of_source_dir) + + def comprehensive_parse(self): + """ + Comprehensive ("default") function to call each parsing function and + retrieve various elements of the code, together with the source location. + + Returns a dict of parsed item key to the corresponding List of Matches. + """ + all_macros = {"public": [], "internal": [], "private":[]} + all_macros["public"] = self.parse_macros([ + self.source_dir + "/include/psa/*.h", + self.source_dir + "/include/tf-psa-crypto/*.h", + self.source_dir + "/drivers/builtin/include/mbedtls/*.h", + self.source_dir + "/drivers/everest/include/everest/everest.h", + self.source_dir + "/drivers/everest/include/everest/x25519.h" + ]) + all_macros["internal"] = self.parse_macros([ + self.source_dir + "/core/*.h", + self.source_dir + "/drivers/builtin/src/*.h", + self.source_dir + "/framework/tests/include/test/drivers/*.h", + ]) + all_macros["private"] = self.parse_macros([ + self.source_dir + "/core/*.c", + self.source_dir + "/drivers/builtin/src/*.c", + ]) + enum_consts = self.parse_enum_consts([ + self.source_dir + "/include/psa/*.h", + self.source_dir + "/include/tf-psa-crypto/*.h", + self.source_dir + "/drivers/builtin/include/mbedtls/*.h", + self.source_dir + "/core/*.h", + self.source_dir + "/drivers/builtin/src/*.h", + self.source_dir + "/core/*.c", + self.source_dir + "/drivers/builtin/src/*.c", + self.source_dir + "/drivers/everest/include/everest/everest.h", + self.source_dir + "/drivers/everest/include/everest/x25519.h" + ]) + identifiers, excluded_identifiers = self.parse_identifiers([ + self.source_dir + "/include/psa/*.h", + self.source_dir + "/include/tf-psa-crypto/*.h", + self.source_dir + "/drivers/builtin/include/mbedtls/*.h", + self.source_dir + "/core/*.h", + self.source_dir + "/drivers/builtin/src/*.h", + self.source_dir + "/drivers/everest/include/everest/everest.h", + self.source_dir + "/drivers/everest/include/everest/x25519.h" + ], [self.source_dir + "/drivers/p256-m/p256-m/p256-m.h"]) + mbed_psa_words = self.parse_mbed_psa_words([ + self.source_dir + "/include/psa/*.h", + self.source_dir + "/include/tf-psa-crypto/*.h", + self.source_dir + "/drivers/builtin/include/mbedtls/*.h", + self.source_dir + "/core/*.h", + self.source_dir + "/drivers/builtin/src/*.h", + self.source_dir + "/drivers/everest/include/everest/everest.h", + self.source_dir + "/drivers/everest/include/everest/x25519.h", + self.source_dir + "/core/*.c", + self.source_dir + "/drivers/builtin/src/*.c", + self.source_dir + "/drivers/everest/library/everest.c", + self.source_dir + "/drivers/everest/library/x25519.c" + ], [self.source_dir + "/core/psa_crypto_driver_wrappers.h"]) + symbols = self.parse_symbols() + + return self._parse(all_macros, enum_consts, identifiers, excluded_identifiers, mbed_psa_words, symbols) + + def parse_symbols(self): + """ + Compile the TF-PSA-Crypto libraries, and parse the + object files using nm to retrieve the list of referenced symbols. + Exceptions thrown here are rethrown because they would be critical + errors that void several tests, and thus needs to halt the program. This + is explicitly done for clarity. + + Returns a List of unique symbols defined and used in the libraries. + """ + self.log.info("Compiling...") + symbols = [] + + # Back up the config and atomically compile with the full configuration. + shutil.copy( + self.source_dir + "/include/psa/crypto_config.h", + self.source_dir + "/include/psa/crypto_config.h.bak" + ) + try: + # Use check=True in all subprocess calls so that failures are raised + # as exceptions and logged. + subprocess.run( + ["python3", "scripts/config.py", "full"], + universal_newlines=True, + check=True + ) + my_environment = os.environ.copy() + my_environment["CFLAGS"] = "-fno-asynchronous-unwind-tables" + + # Run make clean separately to lib to prevent unwanted behavior when + # make is invoked with parallelism. + previous_dir = os.getcwd() + os.chdir(self.out_of_source_dir) + subprocess.run( + ["cmake", "-DGEN_FILES=ON", self.source_dir], + universal_newlines=True, + check=True + ) + subprocess.run( + ["make"], + env=my_environment, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True + ) + + # Perform object file analysis using nm + symbols = self.parse_symbols_from_nm([ + self.out_of_source_dir + "/drivers/builtin/libbuiltin.a", + self.out_of_source_dir + "/drivers/p256-m/libp256m.a", + self.out_of_source_dir + "/drivers/everest/libeverest.a", + self.out_of_source_dir + "/core/libtfpsacrypto.a" + ]) + + os.chdir(previous_dir) + except subprocess.CalledProcessError as error: + self.log.debug(error.output) + raise error + finally: + # Put back the original config regardless of there being errors. + # Works also for keyboard interrupts. + shutil.move( + self.source_dir + "/include/psa/crypto_config.h.bak", + self.source_dir + "/include/psa/crypto_config.h" + ) + + return symbols + +class MBEDTLS_CodeParser(CodeParser): + """ + Class for retrieving files and parsing Mbed TLS code. This can be used + independently of the checks that NameChecker performs, for example for + list_internal_identifiers.py. + """ + + def comprehensive_parse(self): + """ + Comprehensive ("default") function to call each parsing function and + retrieve various elements of the code, together with the source location. + + Returns a dict of parsed item key to the corresponding List of Matches. + """ + if build_tree.is_mbedtls_3_6(): + all_macros["public"] = self.parse_macros([ + "include/mbedtls/*.h", + "include/psa/*.h", + "3rdparty/everest/include/everest/everest.h", + "3rdparty/everest/include/everest/x25519.h" + ]) + all_macros["internal"] = self.parse_macros([ + "library/*.h", + "framework/tests/include/test/drivers/*.h", + ]) + all_macros["private"] = self.parse_macros([ + "library/*.c", + ]) + enum_consts = self.parse_enum_consts([ + "include/mbedtls/*.h", + "include/psa/*.h", + "library/*.h", + "library/*.c", + "3rdparty/everest/include/everest/everest.h", + "3rdparty/everest/include/everest/x25519.h" + ]) + identifiers, excluded_identifiers = self.parse_identifiers([ + "include/mbedtls/*.h", + "include/psa/*.h", + "library/*.h", + "3rdparty/everest/include/everest/everest.h", + "3rdparty/everest/include/everest/x25519.h" + ], ["3rdparty/p256-m/p256-m/p256-m.h"]) + mbed_psa_words = self.parse_mbed_psa_words([ + "include/mbedtls/*.h", + "include/psa/*.h", + "library/*.h", + "3rdparty/everest/include/everest/everest.h", + "3rdparty/everest/include/everest/x25519.h", + "library/*.c", + "3rdparty/everest/library/everest.c", + "3rdparty/everest/library/x25519.c" + ], ["library/psa_crypto_driver_wrappers.h"]) + else: + all_macros = {"public": [], "internal": [], "private":[]} + all_macros["public"] = self.parse_macros([ + "include/mbedtls/*.h", + "include/psa/*.h", + ]) + all_macros["internal"] = self.parse_macros([ + "library/*.h", + "framework/tests/include/test/drivers/*.h", + ]) + all_macros["private"] = self.parse_macros([ + "library/*.c", + ]) + enum_consts = self.parse_enum_consts([ + "include/mbedtls/*.h", + "include/psa/*.h", + "library/*.h", + "library/*.c", + ]) + identifiers, excluded_identifiers = self.parse_identifiers([ + "include/mbedtls/*.h", + "include/psa/*.h", + "library/*.h", + ]) + mbed_psa_words = self.parse_mbed_psa_words([ + "include/mbedtls/*.h", + "include/psa/*.h", + "library/*.h", + "library/*.c", + ]) + symbols = self.parse_symbols() + return self._parse(all_macros, enum_consts, identifiers, excluded_identifiers, mbed_psa_words, symbols) + def parse_symbols(self): """ Compile the Mbed TLS libraries, and parse the TLS, Crypto, and x509 @@ -794,44 +976,6 @@ class CodeParser(): return symbols - def parse_symbols_from_nm(self, object_files): - """ - Run nm to retrieve the list of referenced symbols in each object file. - Does not return the position data since it is of no use. - - Args: - * object_files: a List of compiled object filepaths to search through. - - Returns a List of unique symbols defined and used in any of the object - files. - """ - nm_undefined_regex = re.compile(r"^\S+: +U |^$|^\S+:$") - nm_valid_regex = re.compile(r"^\S+( [0-9A-Fa-f]+)* . _*(?P\w+)") - exclusions = ("FStar", "Hacl") - - symbols = [] - - # Gather all outputs of nm - nm_output = "" - for lib in object_files: - nm_output += subprocess.run( - ["nm", "-og", lib], - universal_newlines=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=True - ).stdout - - for line in nm_output.splitlines(): - if not nm_undefined_regex.search(line): - symbol = nm_valid_regex.search(line) - if (symbol and not symbol.group("symbol").startswith(exclusions)): - symbols.append(symbol.group("symbol")) - else: - self.log.error(line) - - return symbols - class NameChecker(): """ Representation of the core name checking operation performed by this script. @@ -1016,8 +1160,21 @@ def main(): log.addHandler(logging.StreamHandler()) try: - code_parser = CodeParser(log) - parse_result = code_parser.comprehensive_parse() + if build_tree.looks_like_tf_psa_crypto_root(os.getcwd()): + tf_psa_crypto_code_parser = TF_PSA_Crypto_CodeParser(log) + parse_result = tf_psa_crypto_code_parser.comprehensive_parse() + elif build_tree.looks_like_mbedtls_root(os.getcwd()): + # Mbed TLS uses TF-PSA-Crypto, so we need to parse TF-PSA-Crypto too + tf_psa_crypto_code_parser = TF_PSA_Crypto_CodeParser(log) + tf_psa_crypto_parse_result = tf_psa_crypto_code_parser.comprehensive_parse() + mbedtls_code_parser = MBEDTLS_CodeParser(log) + mbedtls_parse_result = mbedtls_code_parser.comprehensive_parse() + # Combine parse results together for NameChecker + parse_result = {} + for key in tf_psa_crypto_parse_result: + parse_result[key] = tf_psa_crypto_parse_result[key] + mbedtls_parse_result[key] + else: + raise Exception("This script must be run from Mbed TLS or TF-PSA-Crypto root") except Exception: # pylint: disable=broad-except traceback.print_exc() sys.exit(2) From f1367be9058aee3ae68b7e5262ca570ffa27c0f7 Mon Sep 17 00:00:00 2001 From: Harry Ramsey Date: Thu, 9 Jan 2025 20:40:16 +0000 Subject: [PATCH 2/6] Fix pylint errors This commit fixes numerous pylint errors where possible. For simplicty, it is easier to keep the many arguments instead of refactoring into an object. Signed-off-by: Harry Ramsey --- scripts/check_names.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/check_names.py b/scripts/check_names.py index f53f54667..ac9571bfe 100755 --- a/scripts/check_names.py +++ b/scripts/check_names.py @@ -222,7 +222,9 @@ class CodeParser(): # Note that "*" can match directory separators in exclude lists. self.excluded_files = ["*/bn_mul", "*/compat-2.x.h"] - def _parse(self, all_macros, enum_consts, identifiers, excluded_identifiers, mbed_psa_words, symbols): + def _parse(self, all_macros, enum_consts, identifiers, + excluded_identifiers, mbed_psa_words, symbols): + # pylint: disable=too-many-arguments """ Parse macros, enums, identifiers, excluded identifiers, Mbed PSA word and Symbols. @@ -674,7 +676,7 @@ class CodeParser(): self.log.error(line) return symbols -class TF_PSA_Crypto_CodeParser(CodeParser): +class TFPSACryptoCodeParser(CodeParser): """ Class for retrieving files and parsing TF-PSA-Crypto code. This can be used independently of the checks that NameChecker performs, for example for @@ -750,7 +752,8 @@ class TF_PSA_Crypto_CodeParser(CodeParser): ], [self.source_dir + "/core/psa_crypto_driver_wrappers.h"]) symbols = self.parse_symbols() - return self._parse(all_macros, enum_consts, identifiers, excluded_identifiers, mbed_psa_words, symbols) + return self._parse(all_macros, enum_consts, identifiers, + excluded_identifiers, mbed_psa_words, symbols) def parse_symbols(self): """ @@ -821,7 +824,7 @@ class TF_PSA_Crypto_CodeParser(CodeParser): return symbols -class MBEDTLS_CodeParser(CodeParser): +class MBEDTLSCodeParser(CodeParser): """ Class for retrieving files and parsing Mbed TLS code. This can be used independently of the checks that NameChecker performs, for example for @@ -835,6 +838,7 @@ class MBEDTLS_CodeParser(CodeParser): Returns a dict of parsed item key to the corresponding List of Matches. """ + all_macros = {"public": [], "internal": [], "private":[]} if build_tree.is_mbedtls_3_6(): all_macros["public"] = self.parse_macros([ "include/mbedtls/*.h", @@ -905,7 +909,8 @@ class MBEDTLS_CodeParser(CodeParser): "library/*.c", ]) symbols = self.parse_symbols() - return self._parse(all_macros, enum_consts, identifiers, excluded_identifiers, mbed_psa_words, symbols) + return self._parse(all_macros, enum_consts, identifiers, + excluded_identifiers, mbed_psa_words, symbols) def parse_symbols(self): """ @@ -1161,13 +1166,13 @@ def main(): try: if build_tree.looks_like_tf_psa_crypto_root(os.getcwd()): - tf_psa_crypto_code_parser = TF_PSA_Crypto_CodeParser(log) + tf_psa_crypto_code_parser = TFPSACryptoCodeParser(log) parse_result = tf_psa_crypto_code_parser.comprehensive_parse() elif build_tree.looks_like_mbedtls_root(os.getcwd()): # Mbed TLS uses TF-PSA-Crypto, so we need to parse TF-PSA-Crypto too - tf_psa_crypto_code_parser = TF_PSA_Crypto_CodeParser(log) + tf_psa_crypto_code_parser = TFPSACryptoCodeParser(log) tf_psa_crypto_parse_result = tf_psa_crypto_code_parser.comprehensive_parse() - mbedtls_code_parser = MBEDTLS_CodeParser(log) + mbedtls_code_parser = MBEDTLSCodeParser(log) mbedtls_parse_result = mbedtls_code_parser.comprehensive_parse() # Combine parse results together for NameChecker parse_result = {} From a58d44adbe684883f218c751feed4de6bc16b00e Mon Sep 17 00:00:00 2001 From: Harry Ramsey Date: Thu, 23 Jan 2025 13:22:31 +0000 Subject: [PATCH 3/6] Remove psa include for Mbed TLS Code Parser This commit removes unncessary TF-PSA-crypto includes for the Mbed TLS code parser class. Signed-off-by: Harry Ramsey --- scripts/check_names.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/check_names.py b/scripts/check_names.py index ac9571bfe..ae8b1fd2e 100755 --- a/scripts/check_names.py +++ b/scripts/check_names.py @@ -882,7 +882,6 @@ class MBEDTLSCodeParser(CodeParser): all_macros = {"public": [], "internal": [], "private":[]} all_macros["public"] = self.parse_macros([ "include/mbedtls/*.h", - "include/psa/*.h", ]) all_macros["internal"] = self.parse_macros([ "library/*.h", @@ -893,18 +892,15 @@ class MBEDTLSCodeParser(CodeParser): ]) enum_consts = self.parse_enum_consts([ "include/mbedtls/*.h", - "include/psa/*.h", "library/*.h", "library/*.c", ]) identifiers, excluded_identifiers = self.parse_identifiers([ "include/mbedtls/*.h", - "include/psa/*.h", "library/*.h", ]) mbed_psa_words = self.parse_mbed_psa_words([ "include/mbedtls/*.h", - "include/psa/*.h", "library/*.h", "library/*.c", ]) From b54f3279b700ed91c1f9a95dfeddd7c39ecd4fc7 Mon Sep 17 00:00:00 2001 From: Harry Ramsey Date: Thu, 23 Jan 2025 13:51:02 +0000 Subject: [PATCH 4/6] Improve documentation of check_names.py This commit improves the documentation of check_names.py specifically for the new class MBEDTLSCodeParser and TFPSACryptoCodeParser. Signed-off-by: Harry Ramsey --- scripts/check_names.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/check_names.py b/scripts/check_names.py index ae8b1fd2e..dc93f02d6 100755 --- a/scripts/check_names.py +++ b/scripts/check_names.py @@ -8,12 +8,14 @@ This script confirms that the naming of all symbols and identifiers in Mbed TLS are consistent with the house style and are also self-consistent. It only runs on Linux and macOS since it depends on nm. -It contains two major Python classes, CodeParser and NameChecker. They both have -a comprehensive "run-all" function (comprehensive_parse() and perform_checks()) -but the individual functions can also be used for specific needs. +It contains three major Python classes, TFPSACryptoCodeParser, +MBEDTLSCodeParser and NameChecker. They all have a comprehensive "run-all" +function (comprehensive_parse() and perform_checks()) but the individual +functions can also be used for specific needs. -CodeParser makes heavy use of regular expressions to parse the code, and is -dependent on the current code formatting. Many Python C parser libraries require +CodeParser(a inherent base class for TFPSACryptoCodeParser and MBEDTLSCodeParser) +makes heavy use of regular expressions to parse the code, and is dependent on +the current code formatting. Many Python C parser libraries require preprocessed C code, which means no macro parsing. Compiler tools are also not very helpful when we want the exact location in the original source (which becomes impossible when e.g. comments are stripped). @@ -679,8 +681,7 @@ class CodeParser(): class TFPSACryptoCodeParser(CodeParser): """ Class for retrieving files and parsing TF-PSA-Crypto code. This can be used - independently of the checks that NameChecker performs, for example for - list_internal_identifiers.py. + independently of the checks that NameChecker performs. """ def __init__(self, log): @@ -827,8 +828,7 @@ class TFPSACryptoCodeParser(CodeParser): class MBEDTLSCodeParser(CodeParser): """ Class for retrieving files and parsing Mbed TLS code. This can be used - independently of the checks that NameChecker performs, for example for - list_internal_identifiers.py. + independently of the checks that NameChecker performs. """ def comprehensive_parse(self): From eaaf91ecd9e2000944ea42305a1d692ba07c5118 Mon Sep 17 00:00:00 2001 From: Harry Ramsey Date: Fri, 24 Jan 2025 10:02:50 +0000 Subject: [PATCH 5/6] Refactor MBEDTLSCodeParer in check_names.py This commit refactors MBEDTLSCodeParser to contain code necessary for obtaining the required parse results from TF-PSA-Crypto. Signed-off-by: Harry Ramsey --- scripts/check_names.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/scripts/check_names.py b/scripts/check_names.py index dc93f02d6..e1b5aa152 100755 --- a/scripts/check_names.py +++ b/scripts/check_names.py @@ -831,6 +831,13 @@ class MBEDTLSCodeParser(CodeParser): independently of the checks that NameChecker performs. """ + def __init__(self, log): + super().__init__(log) + if not build_tree.looks_like_mbedtls_root(os.getcwd()): + raise Exception("This script must be run from Mbed TLS root.") + + self.tf_psa_crypto_code_parser = TFPSACryptoCodeParser(log) + def comprehensive_parse(self): """ Comprehensive ("default") function to call each parsing function and @@ -839,6 +846,9 @@ class MBEDTLSCodeParser(CodeParser): Returns a dict of parsed item key to the corresponding List of Matches. """ all_macros = {"public": [], "internal": [], "private":[]} + # TF-PSA-Crypto is in the same repo in 3.6 so initalise variable here. + tf_psa_crypto_parse_result = {} + if build_tree.is_mbedtls_3_6(): all_macros["public"] = self.parse_macros([ "include/mbedtls/*.h", @@ -904,9 +914,16 @@ class MBEDTLSCodeParser(CodeParser): "library/*.h", "library/*.c", ]) - symbols = self.parse_symbols() - return self._parse(all_macros, enum_consts, identifiers, - excluded_identifiers, mbed_psa_words, symbols) + tf_psa_crypto_parse_result = self.tf_psa_crypto_code_parser.comprehensive_parse() + + symbols = self.parse_symbols() + mbedtls_parse_result = self._parse(all_macros, enum_consts, + identifiers, excluded_identifiers, + mbed_psa_words, symbols) + # Combile results for Mbed TLS and TF-PSA-Crypto + for key in tf_psa_crypto_parse_result: + mbedtls_parse_result[key] += tf_psa_crypto_parse_result[key] + return mbedtls_parse_result def parse_symbols(self): """ @@ -1166,14 +1183,8 @@ def main(): parse_result = tf_psa_crypto_code_parser.comprehensive_parse() elif build_tree.looks_like_mbedtls_root(os.getcwd()): # Mbed TLS uses TF-PSA-Crypto, so we need to parse TF-PSA-Crypto too - tf_psa_crypto_code_parser = TFPSACryptoCodeParser(log) - tf_psa_crypto_parse_result = tf_psa_crypto_code_parser.comprehensive_parse() mbedtls_code_parser = MBEDTLSCodeParser(log) - mbedtls_parse_result = mbedtls_code_parser.comprehensive_parse() - # Combine parse results together for NameChecker - parse_result = {} - for key in tf_psa_crypto_parse_result: - parse_result[key] = tf_psa_crypto_parse_result[key] + mbedtls_parse_result[key] + parse_result = mbedtls_code_parser.comprehensive_parse() else: raise Exception("This script must be run from Mbed TLS or TF-PSA-Crypto root") except Exception: # pylint: disable=broad-except From e65f6848791369dd98e4aed765708f5c89936a10 Mon Sep 17 00:00:00 2001 From: Harry Ramsey Date: Fri, 24 Jan 2025 10:29:43 +0000 Subject: [PATCH 6/6] Refactor TFPSACryptoCodeParser in check_names.py This commit refactors TFPSACryptoCodeParser to throw exceptions for usage in MbedTLS 3.6 as it is not compatible. This commit also removes comments not related to building TF-PSA-Crypto. Signed-off-by: Harry Ramsey --- scripts/check_names.py | 123 ++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 63 deletions(-) diff --git a/scripts/check_names.py b/scripts/check_names.py index e1b5aa152..0f5427574 100755 --- a/scripts/check_names.py +++ b/scripts/check_names.py @@ -45,6 +45,7 @@ import enum import shutil import subprocess import logging +import tempfile import project_scripts # pylint: disable=unused-import from mbedtls_framework import build_tree @@ -686,13 +687,8 @@ class TFPSACryptoCodeParser(CodeParser): def __init__(self, log): super().__init__(log) - if build_tree.looks_like_tf_psa_crypto_root(os.getcwd()): - self.source_dir = os.getcwd() - else: - self.source_dir = os.path.join(os.getcwd(), "tf-psa-crypto") - self.out_of_source_dir = os.path.join(os.getcwd(), "out_of_source_dir") - if not os.path.exists(self.out_of_source_dir): - os.mkdir(self.out_of_source_dir) + if not build_tree.looks_like_tf_psa_crypto_root(os.getcwd()): + raise Exception("This script must be run from TF-PSA-Crypto root.") def comprehensive_parse(self): """ @@ -703,54 +699,54 @@ class TFPSACryptoCodeParser(CodeParser): """ all_macros = {"public": [], "internal": [], "private":[]} all_macros["public"] = self.parse_macros([ - self.source_dir + "/include/psa/*.h", - self.source_dir + "/include/tf-psa-crypto/*.h", - self.source_dir + "/drivers/builtin/include/mbedtls/*.h", - self.source_dir + "/drivers/everest/include/everest/everest.h", - self.source_dir + "/drivers/everest/include/everest/x25519.h" + "include/psa/*.h", + "include/tf-psa-crypto/*.h", + "drivers/builtin/include/mbedtls/*.h", + "drivers/everest/include/everest/everest.h", + "drivers/everest/include/everest/x25519.h" ]) all_macros["internal"] = self.parse_macros([ - self.source_dir + "/core/*.h", - self.source_dir + "/drivers/builtin/src/*.h", - self.source_dir + "/framework/tests/include/test/drivers/*.h", + "core/*.h", + "drivers/builtin/src/*.h", + "framework/tests/include/test/drivers/*.h", ]) all_macros["private"] = self.parse_macros([ - self.source_dir + "/core/*.c", - self.source_dir + "/drivers/builtin/src/*.c", + "core/*.c", + "drivers/builtin/src/*.c", ]) enum_consts = self.parse_enum_consts([ - self.source_dir + "/include/psa/*.h", - self.source_dir + "/include/tf-psa-crypto/*.h", - self.source_dir + "/drivers/builtin/include/mbedtls/*.h", - self.source_dir + "/core/*.h", - self.source_dir + "/drivers/builtin/src/*.h", - self.source_dir + "/core/*.c", - self.source_dir + "/drivers/builtin/src/*.c", - self.source_dir + "/drivers/everest/include/everest/everest.h", - self.source_dir + "/drivers/everest/include/everest/x25519.h" + "include/psa/*.h", + "include/tf-psa-crypto/*.h", + "drivers/builtin/include/mbedtls/*.h", + "core/*.h", + "drivers/builtin/src/*.h", + "core/*.c", + "drivers/builtin/src/*.c", + "drivers/everest/include/everest/everest.h", + "drivers/everest/include/everest/x25519.h" ]) identifiers, excluded_identifiers = self.parse_identifiers([ - self.source_dir + "/include/psa/*.h", - self.source_dir + "/include/tf-psa-crypto/*.h", - self.source_dir + "/drivers/builtin/include/mbedtls/*.h", - self.source_dir + "/core/*.h", - self.source_dir + "/drivers/builtin/src/*.h", - self.source_dir + "/drivers/everest/include/everest/everest.h", - self.source_dir + "/drivers/everest/include/everest/x25519.h" - ], [self.source_dir + "/drivers/p256-m/p256-m/p256-m.h"]) + "include/psa/*.h", + "include/tf-psa-crypto/*.h", + "drivers/builtin/include/mbedtls/*.h", + "core/*.h", + "drivers/builtin/src/*.h", + "drivers/everest/include/everest/everest.h", + "drivers/everest/include/everest/x25519.h" + ], ["drivers/p256-m/p256-m/p256-m.h"]) mbed_psa_words = self.parse_mbed_psa_words([ - self.source_dir + "/include/psa/*.h", - self.source_dir + "/include/tf-psa-crypto/*.h", - self.source_dir + "/drivers/builtin/include/mbedtls/*.h", - self.source_dir + "/core/*.h", - self.source_dir + "/drivers/builtin/src/*.h", - self.source_dir + "/drivers/everest/include/everest/everest.h", - self.source_dir + "/drivers/everest/include/everest/x25519.h", - self.source_dir + "/core/*.c", - self.source_dir + "/drivers/builtin/src/*.c", - self.source_dir + "/drivers/everest/library/everest.c", - self.source_dir + "/drivers/everest/library/x25519.c" - ], [self.source_dir + "/core/psa_crypto_driver_wrappers.h"]) + "include/psa/*.h", + "include/tf-psa-crypto/*.h", + "drivers/builtin/include/mbedtls/*.h", + "core/*.h", + "drivers/builtin/src/*.h", + "drivers/everest/include/everest/everest.h", + "drivers/everest/include/everest/x25519.h", + "core/*.c", + "drivers/builtin/src/*.c", + "drivers/everest/library/everest.c", + "drivers/everest/library/x25519.c" + ], ["core/psa_crypto_driver_wrappers.h"]) symbols = self.parse_symbols() return self._parse(all_macros, enum_consts, identifiers, @@ -771,8 +767,8 @@ class TFPSACryptoCodeParser(CodeParser): # Back up the config and atomically compile with the full configuration. shutil.copy( - self.source_dir + "/include/psa/crypto_config.h", - self.source_dir + "/include/psa/crypto_config.h.bak" + "include/psa/crypto_config.h", + "include/psa/crypto_config.h.bak" ) try: # Use check=True in all subprocess calls so that failures are raised @@ -785,12 +781,11 @@ class TFPSACryptoCodeParser(CodeParser): my_environment = os.environ.copy() my_environment["CFLAGS"] = "-fno-asynchronous-unwind-tables" - # Run make clean separately to lib to prevent unwanted behavior when - # make is invoked with parallelism. - previous_dir = os.getcwd() - os.chdir(self.out_of_source_dir) + source_dir = os.getcwd() + build_dir = tempfile.mkdtemp() + os.chdir(build_dir) subprocess.run( - ["cmake", "-DGEN_FILES=ON", self.source_dir], + ["cmake", "-DGEN_FILES=ON", source_dir], universal_newlines=True, check=True ) @@ -805,13 +800,14 @@ class TFPSACryptoCodeParser(CodeParser): # Perform object file analysis using nm symbols = self.parse_symbols_from_nm([ - self.out_of_source_dir + "/drivers/builtin/libbuiltin.a", - self.out_of_source_dir + "/drivers/p256-m/libp256m.a", - self.out_of_source_dir + "/drivers/everest/libeverest.a", - self.out_of_source_dir + "/core/libtfpsacrypto.a" + build_dir + "/drivers/builtin/libbuiltin.a", + build_dir + "/drivers/p256-m/libp256m.a", + build_dir + "/drivers/everest/libeverest.a", + build_dir + "/core/libtfpsacrypto.a" ]) - os.chdir(previous_dir) + os.chdir(source_dir) + shutil.rmtree(build_dir) except subprocess.CalledProcessError as error: self.log.debug(error.output) raise error @@ -819,8 +815,8 @@ class TFPSACryptoCodeParser(CodeParser): # Put back the original config regardless of there being errors. # Works also for keyboard interrupts. shutil.move( - self.source_dir + "/include/psa/crypto_config.h.bak", - self.source_dir + "/include/psa/crypto_config.h" + "include/psa/crypto_config.h.bak", + "include/psa/crypto_config.h" ) return symbols @@ -836,8 +832,6 @@ class MBEDTLSCodeParser(CodeParser): if not build_tree.looks_like_mbedtls_root(os.getcwd()): raise Exception("This script must be run from Mbed TLS root.") - self.tf_psa_crypto_code_parser = TFPSACryptoCodeParser(log) - def comprehensive_parse(self): """ Comprehensive ("default") function to call each parsing function and @@ -914,7 +908,10 @@ class MBEDTLSCodeParser(CodeParser): "library/*.h", "library/*.c", ]) - tf_psa_crypto_parse_result = self.tf_psa_crypto_code_parser.comprehensive_parse() + os.chdir("./tf-psa-crypto") + tf_psa_crypto_code_parser = TFPSACryptoCodeParser(self.log) + tf_psa_crypto_parse_result = tf_psa_crypto_code_parser.comprehensive_parse() + os.chdir("../") symbols = self.parse_symbols() mbedtls_parse_result = self._parse(all_macros, enum_consts,