From 9b987bd2045d05e2cad87ebac025437832242bf4 Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Mon, 23 Sep 2024 11:53:32 +0200 Subject: [PATCH 1/8] config.py: add get-all and get-all-enabled commands Both commands use regex expressions to get a list of symbols. The difference between the 2 is that: - get-all returns both enabled and commented out symbols while - get-all-enabled returns only enabled ones. Signed-off-by: Valerio Setti --- scripts/mbedtls_framework/config_common.py | 31 ++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scripts/mbedtls_framework/config_common.py b/scripts/mbedtls_framework/config_common.py index 6f7c58b20..95d53ef27 100644 --- a/scripts/mbedtls_framework/config_common.py +++ b/scripts/mbedtls_framework/config_common.py @@ -95,6 +95,18 @@ class Config: else: return default + def get_matching(self, regexs, only_enabled): + """Get all symbols matching one of the regexs.""" + if not regexs: + return None + match_list = [] + regex = re.compile('|'.join(regexs)) + for setting in self.settings.values(): + if regex.search(setting.name): + if (not only_enabled) or (only_enabled and setting.active): + match_list.append(setting.name) + return match_list + def __setitem__(self, name, value): """If name is known, set its value. @@ -353,6 +365,7 @@ class ConfigTool(metaclass=ABCMeta): subparser.set_defaults(adapter=function) def _common_parser_options(self, default_file_path): + # pylint: disable=too-many-branches """Common parser options for config manipulation tool.""" self.parser.add_argument( @@ -392,12 +405,22 @@ class ConfigTool(metaclass=ABCMeta): 'unset-all', help="""Comment out all #define whose name contains a match for REGEX.""") parser_unset_all.add_argument('regexs', metavar='REGEX', nargs='*') + parser_get_all = self.subparsers.add_parser( + 'get-all', + help="""Get all #define whose name contains a match for REGEX.""") + parser_get_all.add_argument('regexs', metavar='REGEX', nargs='*') + parser_get_all_enabled = self.subparsers.add_parser( + 'get-all-enabled', + help="""Get all enabled #define whose name contains a match for REGEX.""") + parser_get_all_enabled.add_argument('regexs', metavar='REGEX', nargs='*') + def custom_parser_options(self): """Adds custom options for the parser. Designed for overridden by descendant.""" pass def main(self): + # pylint: disable=too-many-branches """Common main fuction for config manipulation tool.""" args = self.args @@ -412,6 +435,14 @@ class ConfigTool(metaclass=ABCMeta): if value: sys.stdout.write(value + '\n') return 0 if args.symbol in config else 1 + elif args.command == 'get-all': + match_list = config.get_matching(args.regexs, False) + if match_list is not None: + sys.stdout.write("\n".join(match_list)) + elif args.command == 'get-all-enabled': + match_list = config.get_matching(args.regexs, True) + if match_list is not None: + sys.stdout.write("\n".join(match_list)) elif args.command == 'set': if not args.force and args.symbol not in config.settings: sys.stderr.write( From 40a28bd2afed71a774a4900ea0b855fbf9ebf7f3 Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Thu, 26 Sep 2024 12:36:50 +0200 Subject: [PATCH 2/8] config_common.py: make get_matching() a generator Signed-off-by: Valerio Setti --- scripts/mbedtls_framework/config_common.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/mbedtls_framework/config_common.py b/scripts/mbedtls_framework/config_common.py index 95d53ef27..d594dc189 100644 --- a/scripts/mbedtls_framework/config_common.py +++ b/scripts/mbedtls_framework/config_common.py @@ -98,14 +98,12 @@ class Config: def get_matching(self, regexs, only_enabled): """Get all symbols matching one of the regexs.""" if not regexs: - return None - match_list = [] + return regex = re.compile('|'.join(regexs)) for setting in self.settings.values(): if regex.search(setting.name): - if (not only_enabled) or (only_enabled and setting.active): - match_list.append(setting.name) - return match_list + if setting.active or not only_enabled: + yield setting.name def __setitem__(self, name, value): """If name is known, set its value. From 6b04996bdaaf83c77ae0ef23ea3ecfa6b6e9b104 Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Thu, 26 Sep 2024 13:28:02 +0200 Subject: [PATCH 3/8] config_common.py: remove unnecessary if statements Signed-off-by: Valerio Setti --- scripts/mbedtls_framework/config_common.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/mbedtls_framework/config_common.py b/scripts/mbedtls_framework/config_common.py index d594dc189..75ab52901 100644 --- a/scripts/mbedtls_framework/config_common.py +++ b/scripts/mbedtls_framework/config_common.py @@ -435,12 +435,10 @@ class ConfigTool(metaclass=ABCMeta): return 0 if args.symbol in config else 1 elif args.command == 'get-all': match_list = config.get_matching(args.regexs, False) - if match_list is not None: - sys.stdout.write("\n".join(match_list)) + sys.stdout.write("\n".join(match_list)) elif args.command == 'get-all-enabled': match_list = config.get_matching(args.regexs, True) - if match_list is not None: - sys.stdout.write("\n".join(match_list)) + sys.stdout.write("\n".join(match_list)) elif args.command == 'set': if not args.force and args.symbol not in config.settings: sys.stderr.write( From b6dc14c3c8fb6eb77cc4e661f2613161cb8bfc1c Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Thu, 26 Sep 2024 17:23:17 +0200 Subject: [PATCH 4/8] generate_test_code: ease checks on the test data depends_on In case of depends_on elements that include a conditional check on some symbol's value, we allow the comparison element to be anything and not just a fixed value as it was before. This allows for more complex depends_on conditions where build symbols and macros are used on both sides of the comparison. Signed-off-by: Valerio Setti --- scripts/generate_test_code.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 6a69f9d3d..996350c56 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -193,10 +193,19 @@ BEGIN_CASE_REGEX = r'/\*\s*BEGIN_CASE\s*(?P.*?)\s*\*/' END_CASE_REGEX = r'/\*\s*END_CASE\s*\*/' DEPENDENCY_REGEX = r'depends_on:(?P.*)' +# This can be something like [!]MBEDTLS_xxx C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*' +# This is a generic relation operator: ==, !=, >[=], <[=] CONDITION_OPERATOR_REGEX = r'[!=]=|[<>]=?' -# forbid 0ddd which might be accidentally octal or accidentally decimal -CONDITION_VALUE_REGEX = r'[-+]?(0x[0-9a-f]+|0|[1-9][0-9]*)' +# This can be (almost) anything as long as: +# - it starts with a number or a letter or a "(" +# - it contains only +# - numbers +# - letters +# - spaces +# - math operators, i.e "+"", "-", "*", "/" +# - parentheses, i.e. "()" +CONDITION_VALUE_REGEX = r'[\d|\w|\(][\s_\(\)0-9a-zA-Z\+\-\*\/]*' CONDITION_REGEX = r'({})(?:\s*({})\s*({}))?$'.format(C_IDENTIFIER_REGEX, CONDITION_OPERATOR_REGEX, CONDITION_VALUE_REGEX) From a6e9c083a1c577facf7933a43553133a1dad665e Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Fri, 4 Oct 2024 06:36:30 +0200 Subject: [PATCH 5/8] generate_test_code.py: add check for numerical format Do not allow values starting with a 0 because they can be either accidentally octal or accidentally decimal. Hex values are not affected by this change. Signed-off-by: Valerio Setti --- scripts/generate_test_code.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 996350c56..1cd8c3838 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -209,6 +209,10 @@ CONDITION_VALUE_REGEX = r'[\d|\w|\(][\s_\(\)0-9a-zA-Z\+\-\*\/]*' CONDITION_REGEX = r'({})(?:\s*({})\s*({}))?$'.format(C_IDENTIFIER_REGEX, CONDITION_OPERATOR_REGEX, CONDITION_VALUE_REGEX) +# Match numerical values that start with a 0 because they can be accidentally +# octal or accidentally decimal. Hexadecimal values starting with '0x' are +# valid of course. +INVALID_NUMBER_FORMAT_REGEX = r'(0[0-9]+)' TEST_FUNCTION_VALIDATION_REGEX = r'\s*void\s+(?P\w+)\s*\(' FUNCTION_ARG_LIST_END_REGEX = r'.*\)' EXIT_LABEL_REGEX = r'^exit:' @@ -407,6 +411,8 @@ def validate_dependency(dependency): :return: input dependency stripped of leading & trailing white spaces. """ dependency = dependency.strip() + if re.match(INVALID_NUMBER_FORMAT_REGEX, dependency, re.I): + raise GeneratorInputError('Invalid numerical format %s' % dependency) if not re.match(CONDITION_REGEX, dependency, re.I): raise GeneratorInputError('Invalid dependency %s' % dependency) return dependency From f8025e4362db37cbe37ba12aa891a093e37a54fe Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Fri, 4 Oct 2024 06:52:40 +0200 Subject: [PATCH 6/8] generate_test_code.py: allow bitwise operators in depends_on expressions Signed-off-by: Valerio Setti --- scripts/generate_test_code.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 1cd8c3838..5d071a9c9 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -203,9 +203,10 @@ CONDITION_OPERATOR_REGEX = r'[!=]=|[<>]=?' # - numbers # - letters # - spaces -# - math operators, i.e "+"", "-", "*", "/" +# - math operators, i.e "+", "-", "*", "/" +# - bitwise operators, i.e. "^", "|", "&", "~", "<<", ">>" # - parentheses, i.e. "()" -CONDITION_VALUE_REGEX = r'[\d|\w|\(][\s_\(\)0-9a-zA-Z\+\-\*\/]*' +CONDITION_VALUE_REGEX = r'[\d|\w|\(][\s_\(\)0-9a-zA-Z\+\-\*\/\^\|\&\~\<\>]*' CONDITION_REGEX = r'({})(?:\s*({})\s*({}))?$'.format(C_IDENTIFIER_REGEX, CONDITION_OPERATOR_REGEX, CONDITION_VALUE_REGEX) From bf87497fdcce01c32bb15b07cb7a5edf822d21ac Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Fri, 4 Oct 2024 13:26:41 +0200 Subject: [PATCH 7/8] generate_test_code.py: improve detection of ambiguous numerical values Signed-off-by: Valerio Setti --- scripts/generate_test_code.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 5d071a9c9..1cc18ad37 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -213,7 +213,7 @@ CONDITION_REGEX = r'({})(?:\s*({})\s*({}))?$'.format(C_IDENTIFIER_REGEX, # Match numerical values that start with a 0 because they can be accidentally # octal or accidentally decimal. Hexadecimal values starting with '0x' are # valid of course. -INVALID_NUMBER_FORMAT_REGEX = r'(0[0-9]+)' +AMBIGUOUS_INTEGER_REGEX = r'\b0[0-9]+' TEST_FUNCTION_VALIDATION_REGEX = r'\s*void\s+(?P\w+)\s*\(' FUNCTION_ARG_LIST_END_REGEX = r'.*\)' EXIT_LABEL_REGEX = r'^exit:' @@ -412,8 +412,9 @@ def validate_dependency(dependency): :return: input dependency stripped of leading & trailing white spaces. """ dependency = dependency.strip() - if re.match(INVALID_NUMBER_FORMAT_REGEX, dependency, re.I): - raise GeneratorInputError('Invalid numerical format %s' % dependency) + m = re.search(AMBIGUOUS_INTEGER_REGEX, dependency) + if m: + raise GeneratorInputError('Ambiguous integer literal: '+ m.group(0)) if not re.match(CONDITION_REGEX, dependency, re.I): raise GeneratorInputError('Invalid dependency %s' % dependency) return dependency From c9d6bf45342b072c74dd0a622304d6f750a9bc8c Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Fri, 11 Oct 2024 15:12:02 +0200 Subject: [PATCH 8/8] generate_test_code.py: simplify regex expression "\w" already matches "[a-zA-Z0-9_]" so CONDITION_VALUE_REGEX can be simplified as proposed in this commit. Signed-off-by: Valerio Setti --- scripts/generate_test_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 1cc18ad37..2ac00add2 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -206,7 +206,7 @@ CONDITION_OPERATOR_REGEX = r'[!=]=|[<>]=?' # - math operators, i.e "+", "-", "*", "/" # - bitwise operators, i.e. "^", "|", "&", "~", "<<", ">>" # - parentheses, i.e. "()" -CONDITION_VALUE_REGEX = r'[\d|\w|\(][\s_\(\)0-9a-zA-Z\+\-\*\/\^\|\&\~\<\>]*' +CONDITION_VALUE_REGEX = r'[\w|\(][\s\w\(\)\+\-\*\/\^\|\&\~\<\>]*' CONDITION_REGEX = r'({})(?:\s*({})\s*({}))?$'.format(C_IDENTIFIER_REGEX, CONDITION_OPERATOR_REGEX, CONDITION_VALUE_REGEX)