From db200b4eae55c100761ce1144451d99819b549f3 Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Tue, 6 Mar 2018 11:49:41 +0000 Subject: [PATCH 001/490] Rename generate_code.py -> generate_test_code.py --- scripts/generate_test_code.py | 726 +++++++++++++ scripts/test_generate_test_code.py | 1524 ++++++++++++++++++++++++++++ 2 files changed, 2250 insertions(+) create mode 100644 scripts/generate_test_code.py create mode 100644 scripts/test_generate_test_code.py diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py new file mode 100644 index 000000000..38b0d7547 --- /dev/null +++ b/scripts/generate_test_code.py @@ -0,0 +1,726 @@ +# Test suites code generator. +# +# Copyright (C) 2018, ARM Limited, All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file is part of mbed TLS (https://tls.mbed.org) + +""" +Test Suite code generator. + +Generates a test source file using following input files: + +test_suite_xyz.function - Read test functions from test suite functions file. +test_suite_xyz.data - Read test functions and their dependencies to generate + dispatch and dependency check code. +main template - Substitute generated test function dispatch code, dependency + checking code. +platform .function - Read host or target platform implementation for + dispatching test cases from .data file. +helper .function - Read common reusable functions. +""" + + +import os +import re +import argparse +import shutil + + +BEGIN_HEADER_REGEX = '/\*\s*BEGIN_HEADER\s*\*/' +END_HEADER_REGEX = '/\*\s*END_HEADER\s*\*/' + +BEGIN_SUITE_HELPERS_REGEX = '/\*\s*BEGIN_SUITE_HELPERS\s*\*/' +END_SUITE_HELPERS_REGEX = '/\*\s*END_SUITE_HELPERS\s*\*/' + +BEGIN_DEP_REGEX = 'BEGIN_DEPENDENCIES' +END_DEP_REGEX = 'END_DEPENDENCIES' + +BEGIN_CASE_REGEX = '/\*\s*BEGIN_CASE\s*(.*?)\s*\*/' +END_CASE_REGEX = '/\*\s*END_CASE\s*\*/' + + +class InvalidFileFormat(Exception): + """ + Exception to indicate invalid file format. + """ + pass + + +class FileWrapper(file): + """ + File wrapper class. Provides reading with line no. tracking. + """ + + def __init__(self, file_name): + """ + Init file handle. + + :param file_name: File path to open. + """ + super(FileWrapper, self).__init__(file_name, 'r') + self.line_no = 0 + + def next(self): + """ + Iterator return impl. + :return: Line read from file. + """ + line = super(FileWrapper, self).next() + if line: + self.line_no += 1 + return line + + def readline(self, limit=0): + """ + Wrap the base class readline. + + :param limit: limit to match file.readline([limit]) + :return: Line read from file. + """ + return self.next() + + +def split_dep(dep): + """ + Split NOT character '!' from dependency. Used by gen_deps() + + :param dep: Dependency list + :return: list of tuples where index 0 has '!' if there was a '!' before the dependency string + """ + return ('!', dep[1:]) if dep[0] == '!' else ('', dep) + + +def gen_deps(deps): + """ + Generates dependency i.e. if def and endif code + + :param deps: List of dependencies. + :return: if defined and endif code with macro annotations for readability. + """ + dep_start = ''.join(['#if %sdefined(%s)\n' % split_dep(x) for x in deps]) + dep_end = ''.join(['#endif /* %s */\n' % x for x in reversed(deps)]) + + return dep_start, dep_end + + +def gen_deps_one_line(deps): + """ + Generates dependency checks in one line. Useful for writing code in #else case. + + :param deps: List of dependencies. + :return: ifdef code + """ + defines = ('#if ' if len(deps) else '') + ' && '.join(['%sdefined(%s)' % split_dep(x) for x in deps]) + return defines + + +def gen_function_wrapper(name, locals, args_dispatch): + """ + Creates test function wrapper code. A wrapper has the code to unpack parameters from parameters[] array. + + :param name: Test function name + :param locals: Local variables declaration code + :param args_dispatch: List of dispatch arguments. Ex: ['(char *)params[0]', '*((int *)params[1])'] + :return: Test function wrapper. + """ + # Then create the wrapper + wrapper = ''' +void {name}_wrapper( void ** params ) +{{ + {unused_params} +{locals} + {name}( {args} ); +}} +'''.format(name=name, unused_params='(void)params;' if len(args_dispatch) == 0 else '', + args=', '.join(args_dispatch), + locals=locals) + return wrapper + + +def gen_dispatch(name, deps): + """ + Generates dispatch code for the test function table. + + :param name: Test function name + :param deps: List of dependencies + :return: Dispatch code. + """ + if len(deps): + ifdef = gen_deps_one_line(deps) + dispatch_code = ''' +{ifdef} + {name}_wrapper, +#else + NULL, +#endif +'''.format(ifdef=ifdef, name=name) + else: + dispatch_code = ''' + {name}_wrapper, +'''.format(name=name) + + return dispatch_code + + +def parse_until_pattern(funcs_f, end_regex): + """ + Parses function headers or helper code until end pattern. + + :param funcs_f: file object for .functions file + :param end_regex: Pattern to stop parsing + :return: Test suite headers code + """ + headers = '#line %d "%s"\n' % (funcs_f.line_no + 1, funcs_f.name) + for line in funcs_f: + if re.search(end_regex, line): + break + headers += line + else: + raise InvalidFileFormat("file: %s - end pattern [%s] not found!" % (funcs_f.name, end_regex)) + + return headers + + +def parse_suite_deps(funcs_f): + """ + Parses test suite dependencies. + + :param funcs_f: file object for .functions file + :return: List of test suite dependencies. + """ + deps = [] + for line in funcs_f: + m = re.search('depends_on\:(.*)', line.strip()) + if m: + deps += [x.strip() for x in m.group(1).split(':')] + if re.search(END_DEP_REGEX, line): + break + else: + raise InvalidFileFormat("file: %s - end dependency pattern [%s] not found!" % (funcs_f.name, END_DEP_REGEX)) + + return deps + + +def parse_function_deps(line): + """ + Parses function dependencies. + + :param line: Line from .functions file that has dependencies. + :return: List of dependencies. + """ + deps = [] + m = re.search(BEGIN_CASE_REGEX, line) + dep_str = m.group(1) + if len(dep_str): + m = re.search('depends_on:(.*)', dep_str) + if m: + deps = [x.strip() for x in m.group(1).strip().split(':')] + return deps + + +def parse_function_signature(line): + """ + Parsing function signature + + :param line: Line from .functions file that has a function signature. + :return: function name, argument list, local variables for wrapper function and argument dispatch code. + """ + args = [] + locals = '' + args_dispatch = [] + m = re.search('\s*void\s+(\w+)\s*\(', line, re.I) + if not m: + raise ValueError("Test function should return 'void'\n%s" % line) + name = m.group(1) + line = line[len(m.group(0)):] + arg_idx = 0 + for arg in line[:line.find(')')].split(','): + arg = arg.strip() + if arg == '': + continue + if re.search('int\s+.*', arg.strip()): + args.append('int') + args_dispatch.append('*( (int *) params[%d] )' % arg_idx) + elif re.search('char\s*\*\s*.*', arg.strip()): + args.append('char*') + args_dispatch.append('(char *) params[%d]' % arg_idx) + elif re.search('HexParam_t\s*\*\s*.*', arg.strip()): + args.append('hex') + # create a structure + locals += """ HexParam_t hex%d = {%s, %s}; +""" % (arg_idx, '(uint8_t *) params[%d]' % arg_idx, '*( (uint32_t *) params[%d] )' % (arg_idx + 1)) + + args_dispatch.append('&hex%d' % arg_idx) + arg_idx += 1 + else: + raise ValueError("Test function arguments can only be 'int', 'char *' or 'HexParam_t'\n%s" % line) + arg_idx += 1 + + return name, args, locals, args_dispatch + + +def parse_function_code(funcs_f, deps, suite_deps): + """ + Parses out a function from function file object and generates function and dispatch code. + + :param funcs_f: file object of the functions file. + :param deps: List of dependencies + :param suite_deps: List of test suite dependencies + :return: Function name, arguments, function code and dispatch code. + """ + code = '#line %d "%s"\n' % (funcs_f.line_no + 1, funcs_f.name) + for line in funcs_f: + # Check function signature + m = re.match('.*?\s+(\w+)\s*\(', line, re.I) + if m: + # check if we have full signature i.e. split in more lines + if not re.match('.*\)', line): + for lin in funcs_f: + line += lin + if re.search('.*?\)', line): + break + name, args, locals, args_dispatch = parse_function_signature(line) + code += line.replace(name, 'test_' + name) + name = 'test_' + name + break + else: + raise InvalidFileFormat("file: %s - Test functions not found!" % funcs_f.name) + + for line in funcs_f: + if re.search(END_CASE_REGEX, line): + break + code += line + else: + raise InvalidFileFormat("file: %s - end case pattern [%s] not found!" % (funcs_f.name, END_CASE_REGEX)) + + # Add exit label if not present + if code.find('exit:') == -1: + s = code.rsplit('}', 1) + if len(s) == 2: + code = """exit: + ;; +}""".join(s) + + code += gen_function_wrapper(name, locals, args_dispatch) + ifdef, endif = gen_deps(deps) + dispatch_code = gen_dispatch(name, suite_deps + deps) + return name, args, ifdef + code + endif, dispatch_code + + +def parse_functions(funcs_f): + """ + Returns functions code pieces + + :param funcs_f: file object of the functions file. + :return: List of test suite dependencies, test function dispatch code, function code and + a dict with function identifiers and arguments info. + """ + suite_headers = '' + suite_helpers = '' + suite_deps = [] + suite_functions = '' + func_info = {} + function_idx = 0 + dispatch_code = '' + for line in funcs_f: + if re.search(BEGIN_HEADER_REGEX, line): + headers = parse_until_pattern(funcs_f, END_HEADER_REGEX) + suite_headers += headers + elif re.search(BEGIN_SUITE_HELPERS_REGEX, line): + helpers = parse_until_pattern(funcs_f, END_SUITE_HELPERS_REGEX) + suite_helpers += helpers + elif re.search(BEGIN_DEP_REGEX, line): + deps = parse_suite_deps(funcs_f) + suite_deps += deps + elif re.search(BEGIN_CASE_REGEX, line): + deps = parse_function_deps(line) + func_name, args, func_code, func_dispatch = parse_function_code(funcs_f, deps, suite_deps) + suite_functions += func_code + # Generate dispatch code and enumeration info + assert func_name not in func_info, "file: %s - function %s re-declared at line %d" % \ + (funcs_f.name, func_name, funcs_f.line_no) + func_info[func_name] = (function_idx, args) + dispatch_code += '/* Function Id: %d */\n' % function_idx + dispatch_code += func_dispatch + function_idx += 1 + + ifdef, endif = gen_deps(suite_deps) + func_code = ifdef + suite_headers + suite_helpers + suite_functions + endif + return suite_deps, dispatch_code, func_code, func_info + + +def escaped_split(str, ch): + """ + Split str on character ch but ignore escaped \{ch} + Since return value is used to write back to the intermediate data file. + Any escape characters in the input are retained in the output. + + :param str: String to split + :param ch: split character + :return: List of splits + """ + if len(ch) > 1: + raise ValueError('Expected split character. Found string!') + out = [] + part = '' + escape = False + for i in range(len(str)): + if not escape and str[i] == ch: + out.append(part) + part = '' + else: + part += str[i] + escape = not escape and str[i] == '\\' + if len(part): + out.append(part) + return out + + +def parse_test_data(data_f, debug=False): + """ + Parses .data file + + :param data_f: file object of the data file. + :return: Generator that yields test name, function name, dependency list and function argument list. + """ + STATE_READ_NAME = 0 + STATE_READ_ARGS = 1 + state = STATE_READ_NAME + deps = [] + name = '' + for line in data_f: + line = line.strip() + if len(line) and line[0] == '#': # Skip comments + continue + + # Blank line indicates end of test + if len(line) == 0: + assert state != STATE_READ_ARGS, "Newline before arguments. " \ + "Test function and arguments missing for %s" % name + continue + + if state == STATE_READ_NAME: + # Read test name + name = line + state = STATE_READ_ARGS + elif state == STATE_READ_ARGS: + # Check dependencies + m = re.search('depends_on\:(.*)', line) + if m: + deps = [x.strip() for x in m.group(1).split(':') if len(x.strip())] + else: + # Read test vectors + parts = escaped_split(line, ':') + function = parts[0] + args = parts[1:] + yield name, function, deps, args + deps = [] + state = STATE_READ_NAME + assert state != STATE_READ_ARGS, "Newline before arguments. " \ + "Test function and arguments missing for %s" % name + + +def gen_dep_check(dep_id, dep): + """ + Generate code for the dependency. + + :param dep_id: Dependency identifier + :param dep: Dependency macro + :return: Dependency check code + """ + assert dep_id > -1, "Dependency Id should be a positive integer." + noT, dep = ('!', dep[1:]) if dep[0] == '!' else ('', dep) + assert len(dep) > 0, "Dependency should not be an empty string." + dep_check = ''' + case {id}: + {{ +#if {noT}defined({macro}) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + }} + break;'''.format(noT=noT, macro=dep, id=dep_id) + return dep_check + + +def gen_expression_check(exp_id, exp): + """ + Generates code for expression check + + :param exp_id: Expression Identifier + :param exp: Expression/Macro + :return: Expression check code + """ + assert exp_id > -1, "Expression Id should be a positive integer." + assert len(exp) > 0, "Expression should not be an empty string." + exp_code = ''' + case {exp_id}: + {{ + *out_value = {expression}; + }} + break;'''.format(exp_id=exp_id, expression=exp) + return exp_code + + +def write_deps(out_data_f, test_deps, unique_deps): + """ + Write dependencies to intermediate test data file. + It also returns dependency check code. + + :param out_data_f: Output intermediate data file + :param test_deps: Dependencies + :param unique_deps: Mutable list to track unique dependencies that are global to this re-entrant function. + :return: returns dependency check code. + """ + dep_check_code = '' + if len(test_deps): + out_data_f.write('depends_on') + for dep in test_deps: + if dep not in unique_deps: + unique_deps.append(dep) + dep_id = unique_deps.index(dep) + dep_check_code += gen_dep_check(dep_id, dep) + else: + dep_id = unique_deps.index(dep) + out_data_f.write(':' + str(dep_id)) + out_data_f.write('\n') + return dep_check_code + + +def write_parameters(out_data_f, test_args, func_args, unique_expressions): + """ + Writes test parameters to the intermediate data file. + Also generates expression code. + + :param out_data_f: Output intermediate data file + :param test_args: Test parameters + :param func_args: Function arguments + :param unique_expressions: Mutable list to track unique expressions that are global to this re-entrant function. + :return: Returns expression check code. + """ + expression_code = '' + for i in xrange(len(test_args)): + typ = func_args[i] + val = test_args[i] + + # check if val is a non literal int val + if typ == 'int' and not re.match('(\d+$)|((0x)?[0-9a-fA-F]+$)', val): # its an expression + typ = 'exp' + if val not in unique_expressions: + unique_expressions.append(val) + # exp_id can be derived from len(). But for readability and consistency with case of existing let's + # use index(). + exp_id = unique_expressions.index(val) + expression_code += gen_expression_check(exp_id, val) + val = exp_id + else: + val = unique_expressions.index(val) + out_data_f.write(':' + typ + ':' + str(val)) + out_data_f.write('\n') + return expression_code + + +def gen_suite_deps_checks(suite_deps, dep_check_code, expression_code): + """ + Adds preprocessor checks for test suite dependencies. + + :param suite_deps: Test suite dependencies read from the .functions file. + :param dep_check_code: Dependency check code + :param expression_code: Expression check code + :return: Dependency and expression code guarded by test suite dependencies. + """ + if len(suite_deps): + ifdef = gen_deps_one_line(suite_deps) + dep_check_code = ''' +{ifdef} +{code} +#endif +'''.format(ifdef=ifdef, code=dep_check_code) + expression_code = ''' +{ifdef} +{code} +#endif +'''.format(ifdef=ifdef, code=expression_code) + return dep_check_code, expression_code + + +def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): + """ + Generates dependency checks, expression code and intermediate data file from test data file. + + :param data_f: Data file object + :param out_data_f:Output intermediate data file + :param func_info: Dict keyed by function and with function id and arguments info + :param suite_deps: Test suite deps + :return: Returns dependency and expression check code + """ + unique_deps = [] + unique_expressions = [] + dep_check_code = '' + expression_code = '' + for test_name, function_name, test_deps, test_args in parse_test_data(data_f): + out_data_f.write(test_name + '\n') + + # Write deps + dep_check_code += write_deps(out_data_f, test_deps, unique_deps) + + # Write test function name + test_function_name = 'test_' + function_name + assert test_function_name in func_info, "Function %s not found!" % test_function_name + func_id, func_args = func_info[test_function_name] + out_data_f.write(str(func_id)) + + # Write parameters + assert len(test_args) == len(func_args), \ + "Invalid number of arguments in test %s. See function %s signature." % (test_name, function_name) + expression_code += write_parameters(out_data_f, test_args, func_args, unique_expressions) + + # Write a newline as test case separator + out_data_f.write('\n') + + dep_check_code, expression_code = gen_suite_deps_checks(suite_deps, dep_check_code, expression_code) + return dep_check_code, expression_code + + +def generate_code(funcs_file, data_file, template_file, platform_file, help_file, suites_dir, c_file, out_data_file): + """ + Generate mbed-os test code. + + :param funcs_file: Functions file object + :param data_file: Data file object + :param template_file: Template file object + :param platform_file: Platform file object + :param help_file: Helper functions file object + :param suites_dir: Test suites dir + :param c_file: Output C file object + :param out_data_file: Output intermediate data file object + :return: + """ + for name, path in [('Functions file', funcs_file), + ('Data file', data_file), + ('Template file', template_file), + ('Platform file', platform_file), + ('Help code file', help_file), + ('Suites dir', suites_dir)]: + if not os.path.exists(path): + raise IOError("ERROR: %s [%s] not found!" % (name, path)) + + snippets = {'generator_script' : os.path.basename(__file__)} + + # Read helpers + with open(help_file, 'r') as help_f, open(platform_file, 'r') as platform_f: + snippets['test_common_helper_file'] = help_file + snippets['test_common_helpers'] = help_f.read() + snippets['test_platform_file'] = platform_file + snippets['platform_code'] = platform_f.read().replace('DATA_FILE', + out_data_file.replace('\\', '\\\\')) # escape '\' + + # Function code + with FileWrapper(funcs_file) as funcs_f, open(data_file, 'r') as data_f, open(out_data_file, 'w') as out_data_f: + suite_deps, dispatch_code, func_code, func_info = parse_functions(funcs_f) + snippets['functions_code'] = func_code + snippets['dispatch_code'] = dispatch_code + dep_check_code, expression_code = gen_from_test_data(data_f, out_data_f, func_info, suite_deps) + snippets['dep_check_code'] = dep_check_code + snippets['expression_code'] = expression_code + + snippets['test_file'] = c_file + snippets['test_main_file'] = template_file + snippets['test_case_file'] = funcs_file + snippets['test_case_data_file'] = data_file + # Read Template + # Add functions + # + with open(template_file, 'r') as template_f, open(c_file, 'w') as c_f: + line_no = 1 + for line in template_f.readlines(): + snippets['line_no'] = line_no + 1 # Increment as it sets next line number + code = line.format(**snippets) + c_f.write(code) + line_no += 1 + + +def check_cmd(): + """ + Command line parser. + + :return: + """ + parser = argparse.ArgumentParser(description='Generate code for mbed-os tests.') + + parser.add_argument("-f", "--functions-file", + dest="funcs_file", + help="Functions file", + metavar="FUNCTIONS", + required=True) + + parser.add_argument("-d", "--data-file", + dest="data_file", + help="Data file", + metavar="DATA", + required=True) + + parser.add_argument("-t", "--template-file", + dest="template_file", + help="Template file", + metavar="TEMPLATE", + required=True) + + parser.add_argument("-s", "--suites-dir", + dest="suites_dir", + help="Suites dir", + metavar="SUITES", + required=True) + + parser.add_argument("--help-file", + dest="help_file", + help="Help file", + metavar="HELPER", + required=True) + + parser.add_argument("-p", "--platform-file", + dest="platform_file", + help="Platform code file", + metavar="PLATFORM_FILE", + required=True) + + parser.add_argument("-o", "--out-dir", + dest="out_dir", + help="Dir where generated code and scripts are copied", + metavar="OUT_DIR", + required=True) + + args = parser.parse_args() + + data_file_name = os.path.basename(args.data_file) + data_name = os.path.splitext(data_file_name)[0] + + out_c_file = os.path.join(args.out_dir, data_name + '.c') + out_data_file = os.path.join(args.out_dir, data_file_name) + + out_c_file_dir = os.path.dirname(out_c_file) + out_data_file_dir = os.path.dirname(out_data_file) + for d in [out_c_file_dir, out_data_file_dir]: + if not os.path.exists(d): + os.makedirs(d) + + generate_code(args.funcs_file, args.data_file, args.template_file, args.platform_file, + args.help_file, args.suites_dir, out_c_file, out_data_file) + + +if __name__ == "__main__": + check_cmd() diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py new file mode 100644 index 000000000..08b6fb3a6 --- /dev/null +++ b/scripts/test_generate_test_code.py @@ -0,0 +1,1524 @@ +# Unit test for generate_test_code.py +# +# Copyright (C) 2018, ARM Limited, All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file is part of mbed TLS (https://tls.mbed.org) + +from StringIO import StringIO +from unittest import TestCase, main as unittest_main +from mock import patch +from generate_test_code import * + + +""" +Unit tests for generate_test_code.py +""" + + +class GenDep(TestCase): + """ + Test suite for function gen_dep() + """ + + def test_deps_list(self): + """ + Test that gen_dep() correctly creates deps for given dependency list. + :return: + """ + deps = ['DEP1', 'DEP2'] + dep_start, dep_end = gen_deps(deps) + ifdef1, ifdef2 = dep_start.splitlines() + endif1, endif2 = dep_end.splitlines() + self.assertEqual(ifdef1, '#if defined(DEP1)', 'ifdef generated incorrectly') + self.assertEqual(ifdef2, '#if defined(DEP2)', 'ifdef generated incorrectly') + self.assertEqual(endif1, '#endif /* DEP2 */', 'endif generated incorrectly') + self.assertEqual(endif2, '#endif /* DEP1 */', 'endif generated incorrectly') + + def test_disabled_deps_list(self): + """ + Test that gen_dep() correctly creates deps for given dependency list. + :return: + """ + deps = ['!DEP1', '!DEP2'] + dep_start, dep_end = gen_deps(deps) + ifdef1, ifdef2 = dep_start.splitlines() + endif1, endif2 = dep_end.splitlines() + self.assertEqual(ifdef1, '#if !defined(DEP1)', 'ifdef generated incorrectly') + self.assertEqual(ifdef2, '#if !defined(DEP2)', 'ifdef generated incorrectly') + self.assertEqual(endif1, '#endif /* !DEP2 */', 'endif generated incorrectly') + self.assertEqual(endif2, '#endif /* !DEP1 */', 'endif generated incorrectly') + + def test_mixed_deps_list(self): + """ + Test that gen_dep() correctly creates deps for given dependency list. + :return: + """ + deps = ['!DEP1', 'DEP2'] + dep_start, dep_end = gen_deps(deps) + ifdef1, ifdef2 = dep_start.splitlines() + endif1, endif2 = dep_end.splitlines() + self.assertEqual(ifdef1, '#if !defined(DEP1)', 'ifdef generated incorrectly') + self.assertEqual(ifdef2, '#if defined(DEP2)', 'ifdef generated incorrectly') + self.assertEqual(endif1, '#endif /* DEP2 */', 'endif generated incorrectly') + self.assertEqual(endif2, '#endif /* !DEP1 */', 'endif generated incorrectly') + + def test_empty_deps_list(self): + """ + Test that gen_dep() correctly creates deps for given dependency list. + :return: + """ + deps = [] + dep_start, dep_end = gen_deps(deps) + self.assertEqual(dep_start, '', 'ifdef generated incorrectly') + self.assertEqual(dep_end, '', 'ifdef generated incorrectly') + + def test_large_deps_list(self): + """ + Test that gen_dep() correctly creates deps for given dependency list. + :return: + """ + deps = [] + count = 10 + for i in range(count): + deps.append('DEP%d' % i) + dep_start, dep_end = gen_deps(deps) + self.assertEqual(len(dep_start.splitlines()), count, 'ifdef generated incorrectly') + self.assertEqual(len(dep_end.splitlines()), count, 'ifdef generated incorrectly') + + +class GenDepOneLine(TestCase): + """ + Test Suite for testing gen_deps_one_line() + """ + + def test_deps_list(self): + """ + Test that gen_dep() correctly creates deps for given dependency list. + :return: + """ + deps = ['DEP1', 'DEP2'] + dep_str = gen_deps_one_line(deps) + self.assertEqual(dep_str, '#if defined(DEP1) && defined(DEP2)', 'ifdef generated incorrectly') + + def test_disabled_deps_list(self): + """ + Test that gen_dep() correctly creates deps for given dependency list. + :return: + """ + deps = ['!DEP1', '!DEP2'] + dep_str = gen_deps_one_line(deps) + self.assertEqual(dep_str, '#if !defined(DEP1) && !defined(DEP2)', 'ifdef generated incorrectly') + + def test_mixed_deps_list(self): + """ + Test that gen_dep() correctly creates deps for given dependency list. + :return: + """ + deps = ['!DEP1', 'DEP2'] + dep_str = gen_deps_one_line(deps) + self.assertEqual(dep_str, '#if !defined(DEP1) && defined(DEP2)', 'ifdef generated incorrectly') + + def test_empty_deps_list(self): + """ + Test that gen_dep() correctly creates deps for given dependency list. + :return: + """ + deps = [] + dep_str = gen_deps_one_line(deps) + self.assertEqual(dep_str, '', 'ifdef generated incorrectly') + + def test_large_deps_list(self): + """ + Test that gen_dep() correctly creates deps for given dependency list. + :return: + """ + deps = [] + count = 10 + for i in range(count): + deps.append('DEP%d' % i) + dep_str = gen_deps_one_line(deps) + expected = '#if ' + ' && '.join(['defined(%s)' % x for x in deps]) + self.assertEqual(dep_str, expected, 'ifdef generated incorrectly') + + +class GenFunctionWrapper(TestCase): + """ + Test Suite for testing gen_function_wrapper() + """ + + def test_params_unpack(self): + """ + Test that params are properly unpacked in the function call. + + :return: + """ + code = gen_function_wrapper('test_a', '', ('a', 'b', 'c', 'd')) + expected = ''' +void test_a_wrapper( void ** params ) +{ + + + test_a( a, b, c, d ); +} +''' + self.assertEqual(code, expected) + + def test_local(self): + """ + Test that params are properly unpacked in the function call. + + :return: + """ + code = gen_function_wrapper('test_a', 'int x = 1;', ('x', 'b', 'c', 'd')) + expected = ''' +void test_a_wrapper( void ** params ) +{ + +int x = 1; + test_a( x, b, c, d ); +} +''' + self.assertEqual(code, expected) + + def test_empty_params(self): + """ + Test that params are properly unpacked in the function call. + + :return: + """ + code = gen_function_wrapper('test_a', '', ()) + expected = ''' +void test_a_wrapper( void ** params ) +{ + (void)params; + + test_a( ); +} +''' + self.assertEqual(code, expected) + + +class GenDispatch(TestCase): + """ + Test suite for testing gen_dispatch() + """ + + def test_dispatch(self): + """ + Test that dispatch table entry is generated correctly. + :return: + """ + code = gen_dispatch('test_a', ['DEP1', 'DEP2']) + expected = ''' +#if defined(DEP1) && defined(DEP2) + test_a_wrapper, +#else + NULL, +#endif +''' + self.assertEqual(code, expected) + + def test_empty_deps(self): + """ + Test empty dependency list. + :return: + """ + code = gen_dispatch('test_a', []) + expected = ''' + test_a_wrapper, +''' + self.assertEqual(code, expected) + + +class StringIOWrapper(StringIO, object): + """ + file like class to mock file object in tests. + """ + def __init__(self, file_name, data, line_no = 1): + """ + Init file handle. + + :param file_name: + :param data: + :param line_no: + """ + super(StringIOWrapper, self).__init__(data) + self.line_no = line_no + self.name = file_name + + def next(self): + """ + Iterator return impl. + :return: + """ + line = super(StringIOWrapper, self).next() + return line + + def readline(self, limit=0): + """ + Wrap the base class readline. + + :param limit: + :return: + """ + line = super(StringIOWrapper, self).readline() + if line: + self.line_no += 1 + return line + + +class ParseUntilPattern(TestCase): + """ + Test Suite for testing parse_until_pattern(). + """ + + def test_suite_headers(self): + """ + Test that suite headers are parsed correctly. + + :return: + """ + data = '''#include "mbedtls/ecp.h" + +#define ECP_PF_UNKNOWN -1 +/* END_HEADER */ +''' + expected = '''#line 1 "test_suite_ut.function" +#include "mbedtls/ecp.h" + +#define ECP_PF_UNKNOWN -1 +''' + s = StringIOWrapper('test_suite_ut.function', data, line_no=0) + headers = parse_until_pattern(s, END_HEADER_REGEX) + self.assertEqual(headers, expected) + + def test_line_no(self): + """ + Test that #line is set to correct line no. in source .function file. + + :return: + """ + data = '''#include "mbedtls/ecp.h" + +#define ECP_PF_UNKNOWN -1 +/* END_HEADER */ +''' + offset_line_no = 5 + expected = '''#line %d "test_suite_ut.function" +#include "mbedtls/ecp.h" + +#define ECP_PF_UNKNOWN -1 +''' % (offset_line_no + 1) + s = StringIOWrapper('test_suite_ut.function', data, offset_line_no) + headers = parse_until_pattern(s, END_HEADER_REGEX) + self.assertEqual(headers, expected) + + def test_no_end_header_comment(self): + """ + Test that InvalidFileFormat is raised when end header comment is missing. + :return: + """ + data = '''#include "mbedtls/ecp.h" + +#define ECP_PF_UNKNOWN -1 + +''' + s = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(InvalidFileFormat, parse_until_pattern, s, END_HEADER_REGEX) + + +class ParseSuiteDeps(TestCase): + """ + Test Suite for testing parse_suite_deps(). + """ + + def test_suite_deps(self): + """ + + :return: + """ + data = ''' + * depends_on:MBEDTLS_ECP_C + * END_DEPENDENCIES + */ +''' + expected = ['MBEDTLS_ECP_C'] + s = StringIOWrapper('test_suite_ut.function', data) + deps = parse_suite_deps(s) + self.assertEqual(deps, expected) + + def test_no_end_dep_comment(self): + """ + Test that InvalidFileFormat is raised when end dep comment is missing. + :return: + """ + data = ''' +* depends_on:MBEDTLS_ECP_C +''' + s = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(InvalidFileFormat, parse_suite_deps, s) + + def test_deps_split(self): + """ + Test that InvalidFileFormat is raised when end dep comment is missing. + :return: + """ + data = ''' + * depends_on:MBEDTLS_ECP_C:A:B: C : D :F : G: !H + * END_DEPENDENCIES + */ +''' + expected = ['MBEDTLS_ECP_C', 'A', 'B', 'C', 'D', 'F', 'G', '!H'] + s = StringIOWrapper('test_suite_ut.function', data) + deps = parse_suite_deps(s) + self.assertEqual(deps, expected) + + +class ParseFuncDeps(TestCase): + """ + Test Suite for testing parse_function_deps() + """ + + def test_function_deps(self): + """ + Test that parse_function_deps() correctly parses function dependencies. + :return: + """ + line = '/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */' + expected = ['MBEDTLS_ENTROPY_NV_SEED', 'MBEDTLS_FS_IO'] + deps = parse_function_deps(line) + self.assertEqual(deps, expected) + + def test_no_deps(self): + """ + Test that parse_function_deps() correctly parses function dependencies. + :return: + """ + line = '/* BEGIN_CASE */' + deps = parse_function_deps(line) + self.assertEqual(deps, []) + + def test_poorly_defined_deps(self): + """ + Test that parse_function_deps() correctly parses function dependencies. + :return: + """ + line = '/* BEGIN_CASE depends_on:MBEDTLS_FS_IO: A : !B:C : F*/' + deps = parse_function_deps(line) + self.assertEqual(deps, ['MBEDTLS_FS_IO', 'A', '!B', 'C', 'F']) + + +class ParseFuncSignature(TestCase): + """ + Test Suite for parse_function_signature(). + """ + + def test_int_and_char_params(self): + """ + Test int and char parameters parsing + :return: + """ + line = 'void entropy_threshold( char * a, int b, int result )' + name, args, local, arg_dispatch = parse_function_signature(line) + self.assertEqual(name, 'entropy_threshold') + self.assertEqual(args, ['char*', 'int', 'int']) + self.assertEqual(local, '') + self.assertEqual(arg_dispatch, ['(char *) params[0]', '*( (int *) params[1] )', '*( (int *) params[2] )']) + + def test_hex_params(self): + """ + Test hex parameters parsing + :return: + """ + line = 'void entropy_threshold( char * a, HexParam_t * h, int result )' + name, args, local, arg_dispatch = parse_function_signature(line) + self.assertEqual(name, 'entropy_threshold') + self.assertEqual(args, ['char*', 'hex', 'int']) + self.assertEqual(local, ' HexParam_t hex1 = {(uint8_t *) params[1], *( (uint32_t *) params[2] )};\n') + self.assertEqual(arg_dispatch, ['(char *) params[0]', '&hex1', '*( (int *) params[3] )']) + + def test_non_void_function(self): + """ + Test invalid signature (non void). + :return: + """ + line = 'int entropy_threshold( char * a, HexParam_t * h, int result )' + self.assertRaises(ValueError, parse_function_signature, line) + + def test_unsupported_arg(self): + """ + Test unsupported arguments (not among int, char * and HexParam_t) + :return: + """ + line = 'int entropy_threshold( char * a, HexParam_t * h, int * result )' + self.assertRaises(ValueError, parse_function_signature, line) + + def test_no_params(self): + """ + Test no parameters. + :return: + """ + line = 'void entropy_threshold()' + name, args, local, arg_dispatch = parse_function_signature(line) + self.assertEqual(name, 'entropy_threshold') + self.assertEqual(args, []) + self.assertEqual(local, '') + self.assertEqual(arg_dispatch, []) + + +class ParseFunctionCode(TestCase): + """ + Test suite for testing parse_function_code() + """ + + def test_no_function(self): + """ + Test no test function found. + :return: + """ + data = ''' +No +test +function +''' + s = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(InvalidFileFormat, parse_function_code, s, [], []) + + def test_no_end_case_comment(self): + """ + Test missing end case. + :return: + """ + data = ''' +void test_func() +{ +} +''' + s = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(InvalidFileFormat, parse_function_code, s, [], []) + + @patch("generate_test_code.parse_function_signature") + def test_parse_function_signature_called(self, parse_function_signature_mock): + """ + Test parse_function_code() + :return: + """ + parse_function_signature_mock.return_value = ('test_func', [], '', []) + data = ''' +void test_func() +{ +} +''' + s = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(InvalidFileFormat, parse_function_code, s, [], []) + self.assertTrue(parse_function_signature_mock.called) + parse_function_signature_mock.assert_called_with('void test_func()\n') + + @patch("generate_test_code.gen_dispatch") + @patch("generate_test_code.gen_deps") + @patch("generate_test_code.gen_function_wrapper") + @patch("generate_test_code.parse_function_signature") + def test_return(self, parse_function_signature_mock, + gen_function_wrapper_mock, + gen_deps_mock, + gen_dispatch_mock): + """ + Test generated code. + :return: + """ + parse_function_signature_mock.return_value = ('func', [], '', []) + gen_function_wrapper_mock.return_value = '' + gen_deps_mock.side_effect = gen_deps + gen_dispatch_mock.side_effect = gen_dispatch + data = ''' +void func() +{ + ba ba black sheep + have you any wool +} +/* END_CASE */ +''' + s = StringIOWrapper('test_suite_ut.function', data) + name, arg, code, dispatch_code = parse_function_code(s, [], []) + + #self.assertRaises(InvalidFileFormat, parse_function_code, s, [], []) + self.assertTrue(parse_function_signature_mock.called) + parse_function_signature_mock.assert_called_with('void func()\n') + gen_function_wrapper_mock.assert_called_with('test_func', '', []) + self.assertEqual(name, 'test_func') + self.assertEqual(arg, []) + expected = '''#line 2 "test_suite_ut.function" +void test_func() +{ + ba ba black sheep + have you any wool +exit: + ;; +} +''' + self.assertEqual(code, expected) + self.assertEqual(dispatch_code, "\n test_func_wrapper,\n") + + @patch("generate_test_code.gen_dispatch") + @patch("generate_test_code.gen_deps") + @patch("generate_test_code.gen_function_wrapper") + @patch("generate_test_code.parse_function_signature") + def test_with_exit_label(self, parse_function_signature_mock, + gen_function_wrapper_mock, + gen_deps_mock, + gen_dispatch_mock): + """ + Test when exit label is present. + :return: + """ + parse_function_signature_mock.return_value = ('func', [], '', []) + gen_function_wrapper_mock.return_value = '' + gen_deps_mock.side_effect = gen_deps + gen_dispatch_mock.side_effect = gen_dispatch + data = ''' +void func() +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} +/* END_CASE */ +''' + s = StringIOWrapper('test_suite_ut.function', data) + name, arg, code, dispatch_code = parse_function_code(s, [], []) + + expected = '''#line 2 "test_suite_ut.function" +void test_func() +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} +''' + self.assertEqual(code, expected) + + +class ParseFunction(TestCase): + """ + Test Suite for testing parse_functions() + """ + + @patch("generate_test_code.parse_until_pattern") + def test_begin_header(self, parse_until_pattern_mock): + """ + Test that begin header is checked and parse_until_pattern() is called. + :return: + """ + def stop(this): + raise Exception + parse_until_pattern_mock.side_effect = stop + data = '''/* BEGIN_HEADER */ +#include "mbedtls/ecp.h" + +#define ECP_PF_UNKNOWN -1 +/* END_HEADER */ +''' + s = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(Exception, parse_functions, s) + parse_until_pattern_mock.assert_called_with(s, END_HEADER_REGEX) + self.assertEqual(s.line_no, 2) + + @patch("generate_test_code.parse_until_pattern") + def test_begin_helper(self, parse_until_pattern_mock): + """ + Test that begin helper is checked and parse_until_pattern() is called. + :return: + """ + def stop(this): + raise Exception + parse_until_pattern_mock.side_effect = stop + data = '''/* BEGIN_SUITE_HELPERS */ +void print_helloworld() +{ + printf ("Hello World!\n"); +} +/* END_SUITE_HELPERS */ +''' + s = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(Exception, parse_functions, s) + parse_until_pattern_mock.assert_called_with(s, END_SUITE_HELPERS_REGEX) + self.assertEqual(s.line_no, 2) + + @patch("generate_test_code.parse_suite_deps") + def test_begin_dep(self, parse_suite_deps_mock): + """ + Test that begin dep is checked and parse_suite_deps() is called. + :return: + """ + def stop(this): + raise Exception + parse_suite_deps_mock.side_effect = stop + data = '''/* BEGIN_DEPENDENCIES + * depends_on:MBEDTLS_ECP_C + * END_DEPENDENCIES + */ +''' + s = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(Exception, parse_functions, s) + parse_suite_deps_mock.assert_called_with(s) + self.assertEqual(s.line_no, 2) + + @patch("generate_test_code.parse_function_deps") + def test_begin_function_dep(self, parse_function_deps_mock): + """ + Test that begin dep is checked and parse_function_deps() is called. + :return: + """ + def stop(this): + raise Exception + parse_function_deps_mock.side_effect = stop + + deps_str = '/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */\n' + data = '''%svoid test_func() +{ +} +''' % deps_str + s = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(Exception, parse_functions, s) + parse_function_deps_mock.assert_called_with(deps_str) + self.assertEqual(s.line_no, 2) + + @patch("generate_test_code.parse_function_code") + @patch("generate_test_code.parse_function_deps") + def test_return(self, parse_function_deps_mock, parse_function_code_mock): + """ + Test that begin case is checked and parse_function_code() is called. + :return: + """ + def stop(this): + raise Exception + parse_function_deps_mock.return_value = [] + in_func_code= '''void test_func() +{ +} +''' + func_dispatch = ''' + test_func_wrapper, +''' + parse_function_code_mock.return_value = 'test_func', [], in_func_code, func_dispatch + deps_str = '/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */\n' + data = '''%svoid test_func() +{ +} +''' % deps_str + s = StringIOWrapper('test_suite_ut.function', data) + suite_deps, dispatch_code, func_code, func_info = parse_functions(s) + parse_function_deps_mock.assert_called_with(deps_str) + parse_function_code_mock.assert_called_with(s, [], []) + self.assertEqual(s.line_no, 5) + self.assertEqual(suite_deps, []) + expected_dispatch_code = '''/* Function Id: 0 */ + + test_func_wrapper, +''' + self.assertEqual(dispatch_code, expected_dispatch_code) + self.assertEqual(func_code, in_func_code) + self.assertEqual(func_info, {'test_func': (0, [])}) + + def test_parsing(self): + """ + Test case parsing. + :return: + """ + data = '''/* BEGIN_HEADER */ +#include "mbedtls/ecp.h" + +#define ECP_PF_UNKNOWN -1 +/* END_HEADER */ + +/* BEGIN_DEPENDENCIES + * depends_on:MBEDTLS_ECP_C + * END_DEPENDENCIES + */ + +/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */ +void func1() +{ +} +/* END_CASE */ + +/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */ +void func2() +{ +} +/* END_CASE */ +''' + s = StringIOWrapper('test_suite_ut.function', data) + suite_deps, dispatch_code, func_code, func_info = parse_functions(s) + self.assertEqual(s.line_no, 23) + self.assertEqual(suite_deps, ['MBEDTLS_ECP_C']) + + expected_dispatch_code = '''/* Function Id: 0 */ + +#if defined(MBEDTLS_ECP_C) && defined(MBEDTLS_ENTROPY_NV_SEED) && defined(MBEDTLS_FS_IO) + test_func1_wrapper, +#else + NULL, +#endif +/* Function Id: 1 */ + +#if defined(MBEDTLS_ECP_C) && defined(MBEDTLS_ENTROPY_NV_SEED) && defined(MBEDTLS_FS_IO) + test_func2_wrapper, +#else + NULL, +#endif +''' + self.assertEqual(dispatch_code, expected_dispatch_code) + expected_func_code = '''#if defined(MBEDTLS_ECP_C) +#line 3 "test_suite_ut.function" +#include "mbedtls/ecp.h" + +#define ECP_PF_UNKNOWN -1 +#if defined(MBEDTLS_ENTROPY_NV_SEED) +#if defined(MBEDTLS_FS_IO) +#line 14 "test_suite_ut.function" +void test_func1() +{ +exit: + ;; +} + +void test_func1_wrapper( void ** params ) +{ + (void)params; + + test_func1( ); +} +#endif /* MBEDTLS_FS_IO */ +#endif /* MBEDTLS_ENTROPY_NV_SEED */ +#if defined(MBEDTLS_ENTROPY_NV_SEED) +#if defined(MBEDTLS_FS_IO) +#line 20 "test_suite_ut.function" +void test_func2() +{ +exit: + ;; +} + +void test_func2_wrapper( void ** params ) +{ + (void)params; + + test_func2( ); +} +#endif /* MBEDTLS_FS_IO */ +#endif /* MBEDTLS_ENTROPY_NV_SEED */ +#endif /* MBEDTLS_ECP_C */ +''' + self.assertEqual(func_code, expected_func_code) + self.assertEqual(func_info, {'test_func1': (0, []), 'test_func2': (1, [])}) + + def test_same_function_name(self): + """ + Test name conflict. + :return: + """ + data = '''/* BEGIN_HEADER */ +#include "mbedtls/ecp.h" + +#define ECP_PF_UNKNOWN -1 +/* END_HEADER */ + +/* BEGIN_DEPENDENCIES + * depends_on:MBEDTLS_ECP_C + * END_DEPENDENCIES + */ + +/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */ +void func() +{ +} +/* END_CASE */ + +/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */ +void func() +{ +} +/* END_CASE */ +''' + s = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(AssertionError, parse_functions, s) + + +class ExcapedSplit(TestCase): + """ + Test suite for testing escaped_split(). + Note: Since escaped_split() output is used to write back to the intermediate data file. Any escape characters + in the input are retained in the output. + """ + + def test_invalid_input(self): + """ + Test when input split character is not a character. + :return: + """ + self.assertRaises(ValueError, escaped_split, '', 'string') + + def test_empty_string(self): + """ + Test empty strig input. + :return: + """ + splits = escaped_split('', ':') + self.assertEqual(splits, []) + + def test_no_escape(self): + """ + Test with no escape character. The behaviour should be same as str.split() + :return: + """ + s = 'yahoo:google' + splits = escaped_split(s, ':') + self.assertEqual(splits, s.split(':')) + + def test_escaped_input(self): + """ + Test imput that has escaped delimiter. + :return: + """ + s = 'yahoo\:google:facebook' + splits = escaped_split(s, ':') + self.assertEqual(splits, ['yahoo\:google', 'facebook']) + + def test_escaped_escape(self): + """ + Test imput that has escaped delimiter. + :return: + """ + s = 'yahoo\\\:google:facebook' + splits = escaped_split(s, ':') + self.assertEqual(splits, ['yahoo\\\\', 'google', 'facebook']) + + def test_all_at_once(self): + """ + Test imput that has escaped delimiter. + :return: + """ + s = 'yahoo\\\:google:facebook\:instagram\\\:bbc\\\\:wikipedia' + splits = escaped_split(s, ':') + self.assertEqual(splits, ['yahoo\\\\', 'google', 'facebook\:instagram\\\\', 'bbc\\\\', 'wikipedia']) + + +class ParseTestData(TestCase): + """ + Test suite for parse test data. + """ + + def test_parser(self): + """ + Test that tests are parsed correctly from data file. + :return: + """ + data = """ +Diffie-Hellman full exchange #1 +dhm_do_dhm:10:"23":10:"5" + +Diffie-Hellman full exchange #2 +dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" + +Diffie-Hellman full exchange #3 +dhm_do_dhm:10:"9345098382739712938719287391879381271":10:"9345098792137312973297123912791271" + +Diffie-Hellman selftest +dhm_selftest: +""" + s = StringIOWrapper('test_suite_ut.function', data) + tests = [(name, function, deps, args) for name, function, deps, args in parse_test_data(s)] + t1, t2, t3, t4 = tests + self.assertEqual(t1[0], 'Diffie-Hellman full exchange #1') + self.assertEqual(t1[1], 'dhm_do_dhm') + self.assertEqual(t1[2], []) + self.assertEqual(t1[3], ['10', '"23"', '10', '"5"']) + + self.assertEqual(t2[0], 'Diffie-Hellman full exchange #2') + self.assertEqual(t2[1], 'dhm_do_dhm') + self.assertEqual(t2[2], []) + self.assertEqual(t2[3], ['10', '"93450983094850938450983409623"', '10', '"9345098304850938450983409622"']) + + self.assertEqual(t3[0], 'Diffie-Hellman full exchange #3') + self.assertEqual(t3[1], 'dhm_do_dhm') + self.assertEqual(t3[2], []) + self.assertEqual(t3[3], ['10', '"9345098382739712938719287391879381271"', '10', '"9345098792137312973297123912791271"']) + + self.assertEqual(t4[0], 'Diffie-Hellman selftest') + self.assertEqual(t4[1], 'dhm_selftest') + self.assertEqual(t4[2], []) + self.assertEqual(t4[3], []) + + def test_with_dependencies(self): + """ + Test that tests with dependencies are parsed. + :return: + """ + data = """ +Diffie-Hellman full exchange #1 +depends_on:YAHOO +dhm_do_dhm:10:"23":10:"5" + +Diffie-Hellman full exchange #2 +dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" + +""" + s = StringIOWrapper('test_suite_ut.function', data) + tests = [(name, function, deps, args) for name, function, deps, args in parse_test_data(s)] + t1, t2 = tests + self.assertEqual(t1[0], 'Diffie-Hellman full exchange #1') + self.assertEqual(t1[1], 'dhm_do_dhm') + self.assertEqual(t1[2], ['YAHOO']) + self.assertEqual(t1[3], ['10', '"23"', '10', '"5"']) + + self.assertEqual(t2[0], 'Diffie-Hellman full exchange #2') + self.assertEqual(t2[1], 'dhm_do_dhm') + self.assertEqual(t2[2], []) + self.assertEqual(t2[3], ['10', '"93450983094850938450983409623"', '10', '"9345098304850938450983409622"']) + + def test_no_args(self): + """ + Test AssertionError is raised when test function name and args line is missing. + :return: + """ + data = """ +Diffie-Hellman full exchange #1 +depends_on:YAHOO + + +Diffie-Hellman full exchange #2 +dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" + +""" + s = StringIOWrapper('test_suite_ut.function', data) + e = None + try: + for x, y, z, a in parse_test_data(s): + pass + except AssertionError, e: + pass + self.assertEqual(type(e), AssertionError) + + def test_incomplete_data(self): + """ + Test AssertionError is raised when test function name and args line is missing. + :return: + """ + data = """ +Diffie-Hellman full exchange #1 +depends_on:YAHOO +""" + s = StringIOWrapper('test_suite_ut.function', data) + e = None + try: + for x, y, z, a in parse_test_data(s): + pass + except AssertionError, e: + pass + self.assertEqual(type(e), AssertionError) + + +class GenDepCheck(TestCase): + """ + Test suite for gen_dep_check(). It is assumed this function is called with valid inputs. + """ + + def test_gen_dep_check(self): + """ + Test that dependency check code generated correctly. + :return: + """ + expected = """ + case 5: + { +#if defined(YAHOO) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break;""" + out = gen_dep_check(5, 'YAHOO') + self.assertEqual(out, expected) + + def test_noT(self): + """ + Test dependency with !. + :return: + """ + expected = """ + case 5: + { +#if !defined(YAHOO) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break;""" + out = gen_dep_check(5, '!YAHOO') + self.assertEqual(out, expected) + + def test_empty_dependency(self): + """ + Test invalid dependency input. + :return: + """ + self.assertRaises(AssertionError, gen_dep_check, 5, '!') + + def test_negative_dep_id(self): + """ + Test invalid dependency input. + :return: + """ + self.assertRaises(AssertionError, gen_dep_check, -1, 'YAHOO') + + +class GenExpCheck(TestCase): + """ + Test suite for gen_expression_check(). It is assumed this function is called with valid inputs. + """ + + def test_gen_exp_check(self): + """ + Test that expression check code generated correctly. + :return: + """ + expected = """ + case 5: + { + *out_value = YAHOO; + } + break;""" + out = gen_expression_check(5, 'YAHOO') + self.assertEqual(out, expected) + + def test_invalid_expression(self): + """ + Test invalid expression input. + :return: + """ + self.assertRaises(AssertionError, gen_expression_check, 5, '') + + def test_negative_exp_id(self): + """ + Test invalid expression id. + :return: + """ + self.assertRaises(AssertionError, gen_expression_check, -1, 'YAHOO') + + +class WriteDeps(TestCase): + """ + Test suite for testing write_deps. + """ + + def test_no_test_deps(self): + """ + Test when test_deps is empty. + :return: + """ + s = StringIOWrapper('test_suite_ut.data', '') + unique_deps = [] + dep_check_code = write_deps(s, [], unique_deps) + self.assertEqual(dep_check_code, '') + self.assertEqual(len(unique_deps), 0) + self.assertEqual(s.getvalue(), '') + + def test_unique_dep_ids(self): + """ + + :return: + """ + s = StringIOWrapper('test_suite_ut.data', '') + unique_deps = [] + dep_check_code = write_deps(s, ['DEP3', 'DEP2', 'DEP1'], unique_deps) + expect_dep_check_code = ''' + case 0: + { +#if defined(DEP3) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break; + case 1: + { +#if defined(DEP2) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break; + case 2: + { +#if defined(DEP1) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break;''' + self.assertEqual(dep_check_code, expect_dep_check_code) + self.assertEqual(len(unique_deps), 3) + self.assertEqual(s.getvalue(), 'depends_on:0:1:2\n') + + def test_dep_id_repeat(self): + """ + + :return: + """ + s = StringIOWrapper('test_suite_ut.data', '') + unique_deps = [] + dep_check_code = '' + dep_check_code += write_deps(s, ['DEP3', 'DEP2'], unique_deps) + dep_check_code += write_deps(s, ['DEP2', 'DEP1'], unique_deps) + dep_check_code += write_deps(s, ['DEP1', 'DEP3'], unique_deps) + expect_dep_check_code = ''' + case 0: + { +#if defined(DEP3) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break; + case 1: + { +#if defined(DEP2) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break; + case 2: + { +#if defined(DEP1) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break;''' + self.assertEqual(dep_check_code, expect_dep_check_code) + self.assertEqual(len(unique_deps), 3) + self.assertEqual(s.getvalue(), 'depends_on:0:1\ndepends_on:1:2\ndepends_on:2:0\n') + + +class WriteParams(TestCase): + """ + Test Suite for testing write_parameters(). + """ + + def test_no_params(self): + """ + Test with empty test_args + :return: + """ + s = StringIOWrapper('test_suite_ut.data', '') + unique_expressions = [] + expression_code = write_parameters(s, [], [], unique_expressions) + self.assertEqual(len(unique_expressions), 0) + self.assertEqual(expression_code, '') + self.assertEqual(s.getvalue(), '\n') + + def test_no_exp_param(self): + """ + Test when there is no macro or expression in the params. + :return: + """ + s = StringIOWrapper('test_suite_ut.data', '') + unique_expressions = [] + expression_code = write_parameters(s, ['"Yahoo"', '"abcdef00"', '0'], ['char*', 'hex', 'int'], + unique_expressions) + self.assertEqual(len(unique_expressions), 0) + self.assertEqual(expression_code, '') + self.assertEqual(s.getvalue(), ':char*:"Yahoo":hex:"abcdef00":int:0\n') + + def test_hex_format_int_param(self): + """ + Test int parameter in hex format. + :return: + """ + s = StringIOWrapper('test_suite_ut.data', '') + unique_expressions = [] + expression_code = write_parameters(s, ['"Yahoo"', '"abcdef00"', '0xAA'], ['char*', 'hex', 'int'], + unique_expressions) + self.assertEqual(len(unique_expressions), 0) + self.assertEqual(expression_code, '') + self.assertEqual(s.getvalue(), ':char*:"Yahoo":hex:"abcdef00":int:0xAA\n') + + def test_with_exp_param(self): + """ + Test when there is macro or expression in the params. + :return: + """ + s = StringIOWrapper('test_suite_ut.data', '') + unique_expressions = [] + expression_code = write_parameters(s, ['"Yahoo"', '"abcdef00"', '0', 'MACRO1', 'MACRO2', 'MACRO3'], + ['char*', 'hex', 'int', 'int', 'int', 'int'], + unique_expressions) + self.assertEqual(len(unique_expressions), 3) + self.assertEqual(unique_expressions, ['MACRO1', 'MACRO2', 'MACRO3']) + expected_expression_code = ''' + case 0: + { + *out_value = MACRO1; + } + break; + case 1: + { + *out_value = MACRO2; + } + break; + case 2: + { + *out_value = MACRO3; + } + break;''' + self.assertEqual(expression_code, expected_expression_code) + self.assertEqual(s.getvalue(), ':char*:"Yahoo":hex:"abcdef00":int:0:exp:0:exp:1:exp:2\n') + + def test_with_repeate_calls(self): + """ + Test when write_parameter() is called with same macro or expression. + :return: + """ + s = StringIOWrapper('test_suite_ut.data', '') + unique_expressions = [] + expression_code = '' + expression_code += write_parameters(s, ['"Yahoo"', 'MACRO1', 'MACRO2'], ['char*', 'int', 'int'], + unique_expressions) + expression_code += write_parameters(s, ['"abcdef00"', 'MACRO2', 'MACRO3'], ['hex', 'int', 'int'], + unique_expressions) + expression_code += write_parameters(s, ['0', 'MACRO3', 'MACRO1'], ['int', 'int', 'int'], + unique_expressions) + self.assertEqual(len(unique_expressions), 3) + self.assertEqual(unique_expressions, ['MACRO1', 'MACRO2', 'MACRO3']) + expected_expression_code = ''' + case 0: + { + *out_value = MACRO1; + } + break; + case 1: + { + *out_value = MACRO2; + } + break; + case 2: + { + *out_value = MACRO3; + } + break;''' + self.assertEqual(expression_code, expected_expression_code) + expected_data_file = ''':char*:"Yahoo":exp:0:exp:1 +:hex:"abcdef00":exp:1:exp:2 +:int:0:exp:2:exp:0 +''' + self.assertEqual(s.getvalue(), expected_data_file) + + +class GenTestSuiteDepsChecks(TestCase): + """ + + """ + def test_empty_suite_deps(self): + """ + Test with empty suite_deps list. + + :return: + """ + dep_check_code, expression_code = gen_suite_deps_checks([], 'DEP_CHECK_CODE', 'EXPRESSION_CODE') + self.assertEqual(dep_check_code, 'DEP_CHECK_CODE') + self.assertEqual(expression_code, 'EXPRESSION_CODE') + + def test_suite_deps(self): + """ + Test with suite_deps list. + + :return: + """ + dep_check_code, expression_code = gen_suite_deps_checks(['SUITE_DEP'], 'DEP_CHECK_CODE', 'EXPRESSION_CODE') + exprectd_dep_check_code = ''' +#if defined(SUITE_DEP) +DEP_CHECK_CODE +#endif +''' + expected_expression_code = ''' +#if defined(SUITE_DEP) +EXPRESSION_CODE +#endif +''' + self.assertEqual(dep_check_code, exprectd_dep_check_code) + self.assertEqual(expression_code, expected_expression_code) + + def test_no_dep_no_exp(self): + """ + Test when there are no dependency and expression code. + :return: + """ + dep_check_code, expression_code = gen_suite_deps_checks([], '', '') + self.assertEqual(dep_check_code, '') + self.assertEqual(expression_code, '') + + +class GenFromTestData(TestCase): + """ + Test suite for gen_from_test_data() + """ + + @patch("generate_test_code.write_deps") + @patch("generate_test_code.write_parameters") + @patch("generate_test_code.gen_suite_deps_checks") + def test_intermediate_data_file(self, gen_suite_deps_checks_mock, write_parameters_mock, write_deps_mock): + """ + Test that intermediate data file is written with expected data. + :return: + """ + data = ''' +My test +depends_on:DEP1 +func1:0 +''' + data_f = StringIOWrapper('test_suite_ut.data', data) + out_data_f = StringIOWrapper('test_suite_ut.datax', '') + func_info = {'test_func1': (1, ('int',))} + suite_deps = [] + write_parameters_mock.side_effect = write_parameters + write_deps_mock.side_effect = write_deps + gen_suite_deps_checks_mock.side_effect = gen_suite_deps_checks + gen_from_test_data(data_f, out_data_f, func_info, suite_deps) + write_deps_mock.assert_called_with(out_data_f, ['DEP1'], ['DEP1']) + write_parameters_mock.assert_called_with(out_data_f, ['0'], ('int',), []) + expected_dep_check_code = ''' + case 0: + { +#if defined(DEP1) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break;''' + gen_suite_deps_checks_mock.assert_called_with(suite_deps, expected_dep_check_code, '') + + def test_function_not_found(self): + """ + Test that AssertError is raised when function info in not found. + :return: + """ + data = ''' +My test +depends_on:DEP1 +func1:0 +''' + data_f = StringIOWrapper('test_suite_ut.data', data) + out_data_f = StringIOWrapper('test_suite_ut.datax', '') + func_info = {'test_func2': (1, ('int',))} + suite_deps = [] + self.assertRaises(AssertionError, gen_from_test_data, data_f, out_data_f, func_info, suite_deps) + + def test_different_func_args(self): + """ + Test that AssertError is raised when no. of parameters and function args differ. + :return: + """ + data = ''' +My test +depends_on:DEP1 +func1:0 +''' + data_f = StringIOWrapper('test_suite_ut.data', data) + out_data_f = StringIOWrapper('test_suite_ut.datax', '') + func_info = {'test_func2': (1, ('int','hex'))} + suite_deps = [] + self.assertRaises(AssertionError, gen_from_test_data, data_f, out_data_f, func_info, suite_deps) + + def test_output(self): + """ + Test that intermediate data file is written with expected data. + :return: + """ + data = ''' +My test 1 +depends_on:DEP1 +func1:0:0xfa:MACRO1:MACRO2 + +My test 2 +depends_on:DEP1:DEP2 +func2:"yahoo":88:MACRO1 +''' + data_f = StringIOWrapper('test_suite_ut.data', data) + out_data_f = StringIOWrapper('test_suite_ut.datax', '') + func_info = {'test_func1': (0, ('int', 'int', 'int', 'int')), 'test_func2': (1, ('char*', 'int', 'int'))} + suite_deps = [] + dep_check_code, expression_code = gen_from_test_data(data_f, out_data_f, func_info, suite_deps) + expected_dep_check_code = ''' + case 0: + { +#if defined(DEP1) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break; + case 1: + { +#if defined(DEP2) + ret = DEPENDENCY_SUPPORTED; +#else + ret = DEPENDENCY_NOT_SUPPORTED; +#endif + } + break;''' + expecrted_data = '''My test 1 +depends_on:0 +0:int:0:int:0xfa:exp:0:exp:1 + +My test 2 +depends_on:0:1 +1:char*:"yahoo":int:88:exp:0 + +''' + expected_expression_code = ''' + case 0: + { + *out_value = MACRO1; + } + break; + case 1: + { + *out_value = MACRO2; + } + break;''' + self.assertEqual(dep_check_code, expected_dep_check_code) + self.assertEqual(out_data_f.getvalue(), expecrted_data) + self.assertEqual(expression_code, expected_expression_code) + + +if __name__=='__main__': + unittest_main() From 435a8f476b26c0eda03403637bf1c70f704d0145 Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Wed, 11 Apr 2018 23:46:37 +0100 Subject: [PATCH 002/490] Python3 compatible generate_test_code.py --- scripts/generate_test_code.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) mode change 100644 => 100755 scripts/generate_test_code.py diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py old mode 100644 new mode 100755 index 38b0d7547..bf4ddb82c --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Test suites code generator. # # Copyright (C) 2018, ARM Limited, All Rights Reserved @@ -33,8 +34,10 @@ helper .function - Read common reusable functions. """ +import io import os import re +import sys import argparse import shutil @@ -59,7 +62,7 @@ class InvalidFileFormat(Exception): pass -class FileWrapper(file): +class FileWrapper(io.FileIO): """ File wrapper class. Provides reading with line no. tracking. """ @@ -73,24 +76,17 @@ class FileWrapper(file): super(FileWrapper, self).__init__(file_name, 'r') self.line_no = 0 - def next(self): + def __next__(self): """ Iterator return impl. :return: Line read from file. """ - line = super(FileWrapper, self).next() + line = super(FileWrapper, self).__next__() if line: self.line_no += 1 - return line - - def readline(self, limit=0): - """ - Wrap the base class readline. - - :param limit: limit to match file.readline([limit]) - :return: Line read from file. - """ - return self.next() + # Convert byte array to string with correct encoding + return line.decode(sys.getdefaultencoding()) + return None def split_dep(dep): @@ -513,7 +509,7 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): :return: Returns expression check code. """ expression_code = '' - for i in xrange(len(test_args)): + for i in range(len(test_args)): typ = func_args[i] val = test_args[i] From 1d9e82576760e56cbf1e009e67b9f9d47e3aac9b Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Wed, 13 Jun 2018 16:31:26 +0100 Subject: [PATCH 003/490] Remove white spaces caught by check-files.py --- scripts/generate_test_code.py | 34 +++--- scripts/test_generate_test_code.py | 190 ++++++++++++++--------------- 2 files changed, 112 insertions(+), 112 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index bf4ddb82c..3ff7a41d9 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -57,20 +57,20 @@ END_CASE_REGEX = '/\*\s*END_CASE\s*\*/' class InvalidFileFormat(Exception): """ - Exception to indicate invalid file format. + Exception to indicate invalid file format. """ pass class FileWrapper(io.FileIO): """ - File wrapper class. Provides reading with line no. tracking. + File wrapper class. Provides reading with line no. tracking. """ def __init__(self, file_name): """ Init file handle. - + :param file_name: File path to open. """ super(FileWrapper, self).__init__(file_name, 'r') @@ -174,7 +174,7 @@ def gen_dispatch(name, deps): def parse_until_pattern(funcs_f, end_regex): """ Parses function headers or helper code until end pattern. - + :param funcs_f: file object for .functions file :param end_regex: Pattern to stop parsing :return: Test suite headers code @@ -193,7 +193,7 @@ def parse_until_pattern(funcs_f, end_regex): def parse_suite_deps(funcs_f): """ Parses test suite dependencies. - + :param funcs_f: file object for .functions file :return: List of test suite dependencies. """ @@ -213,7 +213,7 @@ def parse_suite_deps(funcs_f): def parse_function_deps(line): """ Parses function dependencies. - + :param line: Line from .functions file that has dependencies. :return: List of dependencies. """ @@ -230,7 +230,7 @@ def parse_function_deps(line): def parse_function_signature(line): """ Parsing function signature - + :param line: Line from .functions file that has a function signature. :return: function name, argument list, local variables for wrapper function and argument dispatch code. """ @@ -271,7 +271,7 @@ def parse_function_signature(line): def parse_function_code(funcs_f, deps, suite_deps): """ Parses out a function from function file object and generates function and dispatch code. - + :param funcs_f: file object of the functions file. :param deps: List of dependencies :param suite_deps: List of test suite dependencies @@ -319,7 +319,7 @@ def parse_function_code(funcs_f, deps, suite_deps): def parse_functions(funcs_f): """ Returns functions code pieces - + :param funcs_f: file object of the functions file. :return: List of test suite dependencies, test function dispatch code, function code and a dict with function identifiers and arguments info. @@ -361,7 +361,7 @@ def parse_functions(funcs_f): def escaped_split(str, ch): """ Split str on character ch but ignore escaped \{ch} - Since return value is used to write back to the intermediate data file. + Since return value is used to write back to the intermediate data file. Any escape characters in the input are retained in the output. :param str: String to split @@ -388,7 +388,7 @@ def escaped_split(str, ch): def parse_test_data(data_f, debug=False): """ Parses .data file - + :param data_f: file object of the data file. :return: Generator that yields test name, function name, dependency list and function argument list. """ @@ -432,7 +432,7 @@ def parse_test_data(data_f, debug=False): def gen_dep_check(dep_id, dep): """ Generate code for the dependency. - + :param dep_id: Dependency identifier :param dep: Dependency macro :return: Dependency check code @@ -456,7 +456,7 @@ def gen_dep_check(dep_id, dep): def gen_expression_check(exp_id, exp): """ Generates code for expression check - + :param exp_id: Expression Identifier :param exp: Expression/Macro :return: Expression check code @@ -476,7 +476,7 @@ def write_deps(out_data_f, test_deps, unique_deps): """ Write dependencies to intermediate test data file. It also returns dependency check code. - + :param out_data_f: Output intermediate data file :param test_deps: Dependencies :param unique_deps: Mutable list to track unique dependencies that are global to this re-entrant function. @@ -501,7 +501,7 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): """ Writes test parameters to the intermediate data file. Also generates expression code. - + :param out_data_f: Output intermediate data file :param test_args: Test parameters :param func_args: Function arguments @@ -533,7 +533,7 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): def gen_suite_deps_checks(suite_deps, dep_check_code, expression_code): """ Adds preprocessor checks for test suite dependencies. - + :param suite_deps: Test suite dependencies read from the .functions file. :param dep_check_code: Dependency check code :param expression_code: Expression check code @@ -557,7 +557,7 @@ def gen_suite_deps_checks(suite_deps, dep_check_code, expression_code): def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): """ Generates dependency checks, expression code and intermediate data file from test data file. - + :param data_f: Data file object :param out_data_f:Output intermediate data file :param func_info: Dict keyed by function and with function id and arguments info diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 08b6fb3a6..4e225dc56 100644 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -36,7 +36,7 @@ class GenDep(TestCase): def test_deps_list(self): """ Test that gen_dep() correctly creates deps for given dependency list. - :return: + :return: """ deps = ['DEP1', 'DEP2'] dep_start, dep_end = gen_deps(deps) @@ -50,7 +50,7 @@ class GenDep(TestCase): def test_disabled_deps_list(self): """ Test that gen_dep() correctly creates deps for given dependency list. - :return: + :return: """ deps = ['!DEP1', '!DEP2'] dep_start, dep_end = gen_deps(deps) @@ -64,7 +64,7 @@ class GenDep(TestCase): def test_mixed_deps_list(self): """ Test that gen_dep() correctly creates deps for given dependency list. - :return: + :return: """ deps = ['!DEP1', 'DEP2'] dep_start, dep_end = gen_deps(deps) @@ -78,7 +78,7 @@ class GenDep(TestCase): def test_empty_deps_list(self): """ Test that gen_dep() correctly creates deps for given dependency list. - :return: + :return: """ deps = [] dep_start, dep_end = gen_deps(deps) @@ -88,7 +88,7 @@ class GenDep(TestCase): def test_large_deps_list(self): """ Test that gen_dep() correctly creates deps for given dependency list. - :return: + :return: """ deps = [] count = 10 @@ -107,7 +107,7 @@ class GenDepOneLine(TestCase): def test_deps_list(self): """ Test that gen_dep() correctly creates deps for given dependency list. - :return: + :return: """ deps = ['DEP1', 'DEP2'] dep_str = gen_deps_one_line(deps) @@ -162,14 +162,14 @@ class GenFunctionWrapper(TestCase): def test_params_unpack(self): """ Test that params are properly unpacked in the function call. - - :return: + + :return: """ code = gen_function_wrapper('test_a', '', ('a', 'b', 'c', 'd')) expected = ''' void test_a_wrapper( void ** params ) { - + test_a( a, b, c, d ); } @@ -179,14 +179,14 @@ void test_a_wrapper( void ** params ) def test_local(self): """ Test that params are properly unpacked in the function call. - - :return: + + :return: """ code = gen_function_wrapper('test_a', 'int x = 1;', ('x', 'b', 'c', 'd')) expected = ''' void test_a_wrapper( void ** params ) { - + int x = 1; test_a( x, b, c, d ); } @@ -196,8 +196,8 @@ int x = 1; def test_empty_params(self): """ Test that params are properly unpacked in the function call. - - :return: + + :return: """ code = gen_function_wrapper('test_a', '', ()) expected = ''' @@ -219,7 +219,7 @@ class GenDispatch(TestCase): def test_dispatch(self): """ Test that dispatch table entry is generated correctly. - :return: + :return: """ code = gen_dispatch('test_a', ['DEP1', 'DEP2']) expected = ''' @@ -234,7 +234,7 @@ class GenDispatch(TestCase): def test_empty_deps(self): """ Test empty dependency list. - :return: + :return: """ code = gen_dispatch('test_a', []) expected = ''' @@ -250,8 +250,8 @@ class StringIOWrapper(StringIO, object): def __init__(self, file_name, data, line_no = 1): """ Init file handle. - - :param file_name: + + :param file_name: :param data: :param line_no: """ @@ -262,7 +262,7 @@ class StringIOWrapper(StringIO, object): def next(self): """ Iterator return impl. - :return: + :return: """ line = super(StringIOWrapper, self).next() return line @@ -270,9 +270,9 @@ class StringIOWrapper(StringIO, object): def readline(self, limit=0): """ Wrap the base class readline. - - :param limit: - :return: + + :param limit: + :return: """ line = super(StringIOWrapper, self).readline() if line: @@ -288,8 +288,8 @@ class ParseUntilPattern(TestCase): def test_suite_headers(self): """ Test that suite headers are parsed correctly. - - :return: + + :return: """ data = '''#include "mbedtls/ecp.h" @@ -307,9 +307,9 @@ class ParseUntilPattern(TestCase): def test_line_no(self): """ - Test that #line is set to correct line no. in source .function file. - - :return: + Test that #line is set to correct line no. in source .function file. + + :return: """ data = '''#include "mbedtls/ecp.h" @@ -329,7 +329,7 @@ class ParseUntilPattern(TestCase): def test_no_end_header_comment(self): """ Test that InvalidFileFormat is raised when end header comment is missing. - :return: + :return: """ data = '''#include "mbedtls/ecp.h" @@ -347,8 +347,8 @@ class ParseSuiteDeps(TestCase): def test_suite_deps(self): """ - - :return: + + :return: """ data = ''' * depends_on:MBEDTLS_ECP_C @@ -363,7 +363,7 @@ class ParseSuiteDeps(TestCase): def test_no_end_dep_comment(self): """ Test that InvalidFileFormat is raised when end dep comment is missing. - :return: + :return: """ data = ''' * depends_on:MBEDTLS_ECP_C @@ -374,10 +374,10 @@ class ParseSuiteDeps(TestCase): def test_deps_split(self): """ Test that InvalidFileFormat is raised when end dep comment is missing. - :return: + :return: """ data = ''' - * depends_on:MBEDTLS_ECP_C:A:B: C : D :F : G: !H + * depends_on:MBEDTLS_ECP_C:A:B: C : D :F : G: !H * END_DEPENDENCIES */ ''' @@ -389,13 +389,13 @@ class ParseSuiteDeps(TestCase): class ParseFuncDeps(TestCase): """ - Test Suite for testing parse_function_deps() + Test Suite for testing parse_function_deps() """ def test_function_deps(self): """ Test that parse_function_deps() correctly parses function dependencies. - :return: + :return: """ line = '/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */' expected = ['MBEDTLS_ENTROPY_NV_SEED', 'MBEDTLS_FS_IO'] @@ -405,7 +405,7 @@ class ParseFuncDeps(TestCase): def test_no_deps(self): """ Test that parse_function_deps() correctly parses function dependencies. - :return: + :return: """ line = '/* BEGIN_CASE */' deps = parse_function_deps(line) @@ -414,7 +414,7 @@ class ParseFuncDeps(TestCase): def test_poorly_defined_deps(self): """ Test that parse_function_deps() correctly parses function dependencies. - :return: + :return: """ line = '/* BEGIN_CASE depends_on:MBEDTLS_FS_IO: A : !B:C : F*/' deps = parse_function_deps(line) @@ -423,13 +423,13 @@ class ParseFuncDeps(TestCase): class ParseFuncSignature(TestCase): """ - Test Suite for parse_function_signature(). + Test Suite for parse_function_signature(). """ def test_int_and_char_params(self): """ Test int and char parameters parsing - :return: + :return: """ line = 'void entropy_threshold( char * a, int b, int result )' name, args, local, arg_dispatch = parse_function_signature(line) @@ -441,7 +441,7 @@ class ParseFuncSignature(TestCase): def test_hex_params(self): """ Test hex parameters parsing - :return: + :return: """ line = 'void entropy_threshold( char * a, HexParam_t * h, int result )' name, args, local, arg_dispatch = parse_function_signature(line) @@ -453,7 +453,7 @@ class ParseFuncSignature(TestCase): def test_non_void_function(self): """ Test invalid signature (non void). - :return: + :return: """ line = 'int entropy_threshold( char * a, HexParam_t * h, int result )' self.assertRaises(ValueError, parse_function_signature, line) @@ -461,7 +461,7 @@ class ParseFuncSignature(TestCase): def test_unsupported_arg(self): """ Test unsupported arguments (not among int, char * and HexParam_t) - :return: + :return: """ line = 'int entropy_threshold( char * a, HexParam_t * h, int * result )' self.assertRaises(ValueError, parse_function_signature, line) @@ -469,7 +469,7 @@ class ParseFuncSignature(TestCase): def test_no_params(self): """ Test no parameters. - :return: + :return: """ line = 'void entropy_threshold()' name, args, local, arg_dispatch = parse_function_signature(line) @@ -487,7 +487,7 @@ class ParseFunctionCode(TestCase): def test_no_function(self): """ Test no test function found. - :return: + :return: """ data = ''' No @@ -500,7 +500,7 @@ function def test_no_end_case_comment(self): """ Test missing end case. - :return: + :return: """ data = ''' void test_func() @@ -514,7 +514,7 @@ void test_func() def test_parse_function_signature_called(self, parse_function_signature_mock): """ Test parse_function_code() - :return: + :return: """ parse_function_signature_mock.return_value = ('test_func', [], '', []) data = ''' @@ -537,7 +537,7 @@ void test_func() gen_dispatch_mock): """ Test generated code. - :return: + :return: """ parse_function_signature_mock.return_value = ('func', [], '', []) gen_function_wrapper_mock.return_value = '' @@ -582,7 +582,7 @@ exit: gen_dispatch_mock): """ Test when exit label is present. - :return: + :return: """ parse_function_signature_mock.return_value = ('func', [], '', []) gen_function_wrapper_mock.return_value = '' @@ -624,7 +624,7 @@ class ParseFunction(TestCase): def test_begin_header(self, parse_until_pattern_mock): """ Test that begin header is checked and parse_until_pattern() is called. - :return: + :return: """ def stop(this): raise Exception @@ -644,7 +644,7 @@ class ParseFunction(TestCase): def test_begin_helper(self, parse_until_pattern_mock): """ Test that begin helper is checked and parse_until_pattern() is called. - :return: + :return: """ def stop(this): raise Exception @@ -665,7 +665,7 @@ void print_helloworld() def test_begin_dep(self, parse_suite_deps_mock): """ Test that begin dep is checked and parse_suite_deps() is called. - :return: + :return: """ def stop(this): raise Exception @@ -684,7 +684,7 @@ void print_helloworld() def test_begin_function_dep(self, parse_function_deps_mock): """ Test that begin dep is checked and parse_function_deps() is called. - :return: + :return: """ def stop(this): raise Exception @@ -705,7 +705,7 @@ void print_helloworld() def test_return(self, parse_function_deps_mock, parse_function_code_mock): """ Test that begin case is checked and parse_function_code() is called. - :return: + :return: """ def stop(this): raise Exception @@ -740,7 +740,7 @@ void print_helloworld() def test_parsing(self): """ Test case parsing. - :return: + :return: """ data = '''/* BEGIN_HEADER */ #include "mbedtls/ecp.h" @@ -833,7 +833,7 @@ void test_func2_wrapper( void ** params ) def test_same_function_name(self): """ Test name conflict. - :return: + :return: """ data = '''/* BEGIN_HEADER */ #include "mbedtls/ecp.h" @@ -872,14 +872,14 @@ class ExcapedSplit(TestCase): def test_invalid_input(self): """ Test when input split character is not a character. - :return: + :return: """ self.assertRaises(ValueError, escaped_split, '', 'string') def test_empty_string(self): """ Test empty strig input. - :return: + :return: """ splits = escaped_split('', ':') self.assertEqual(splits, []) @@ -887,7 +887,7 @@ class ExcapedSplit(TestCase): def test_no_escape(self): """ Test with no escape character. The behaviour should be same as str.split() - :return: + :return: """ s = 'yahoo:google' splits = escaped_split(s, ':') @@ -896,7 +896,7 @@ class ExcapedSplit(TestCase): def test_escaped_input(self): """ Test imput that has escaped delimiter. - :return: + :return: """ s = 'yahoo\:google:facebook' splits = escaped_split(s, ':') @@ -905,7 +905,7 @@ class ExcapedSplit(TestCase): def test_escaped_escape(self): """ Test imput that has escaped delimiter. - :return: + :return: """ s = 'yahoo\\\:google:facebook' splits = escaped_split(s, ':') @@ -914,7 +914,7 @@ class ExcapedSplit(TestCase): def test_all_at_once(self): """ Test imput that has escaped delimiter. - :return: + :return: """ s = 'yahoo\\\:google:facebook\:instagram\\\:bbc\\\\:wikipedia' splits = escaped_split(s, ':') @@ -929,7 +929,7 @@ class ParseTestData(TestCase): def test_parser(self): """ Test that tests are parsed correctly from data file. - :return: + :return: """ data = """ Diffie-Hellman full exchange #1 @@ -970,7 +970,7 @@ dhm_selftest: def test_with_dependencies(self): """ Test that tests with dependencies are parsed. - :return: + :return: """ data = """ Diffie-Hellman full exchange #1 @@ -997,7 +997,7 @@ dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" def test_no_args(self): """ Test AssertionError is raised when test function name and args line is missing. - :return: + :return: """ data = """ Diffie-Hellman full exchange #1 @@ -1020,7 +1020,7 @@ dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" def test_incomplete_data(self): """ Test AssertionError is raised when test function name and args line is missing. - :return: + :return: """ data = """ Diffie-Hellman full exchange #1 @@ -1038,13 +1038,13 @@ depends_on:YAHOO class GenDepCheck(TestCase): """ - Test suite for gen_dep_check(). It is assumed this function is called with valid inputs. + Test suite for gen_dep_check(). It is assumed this function is called with valid inputs. """ def test_gen_dep_check(self): """ Test that dependency check code generated correctly. - :return: + :return: """ expected = """ case 5: @@ -1062,7 +1062,7 @@ class GenDepCheck(TestCase): def test_noT(self): """ Test dependency with !. - :return: + :return: """ expected = """ case 5: @@ -1080,27 +1080,27 @@ class GenDepCheck(TestCase): def test_empty_dependency(self): """ Test invalid dependency input. - :return: + :return: """ self.assertRaises(AssertionError, gen_dep_check, 5, '!') def test_negative_dep_id(self): """ Test invalid dependency input. - :return: + :return: """ self.assertRaises(AssertionError, gen_dep_check, -1, 'YAHOO') class GenExpCheck(TestCase): """ - Test suite for gen_expression_check(). It is assumed this function is called with valid inputs. + Test suite for gen_expression_check(). It is assumed this function is called with valid inputs. """ def test_gen_exp_check(self): """ Test that expression check code generated correctly. - :return: + :return: """ expected = """ case 5: @@ -1114,14 +1114,14 @@ class GenExpCheck(TestCase): def test_invalid_expression(self): """ Test invalid expression input. - :return: + :return: """ self.assertRaises(AssertionError, gen_expression_check, 5, '') def test_negative_exp_id(self): """ Test invalid expression id. - :return: + :return: """ self.assertRaises(AssertionError, gen_expression_check, -1, 'YAHOO') @@ -1134,7 +1134,7 @@ class WriteDeps(TestCase): def test_no_test_deps(self): """ Test when test_deps is empty. - :return: + :return: """ s = StringIOWrapper('test_suite_ut.data', '') unique_deps = [] @@ -1145,8 +1145,8 @@ class WriteDeps(TestCase): def test_unique_dep_ids(self): """ - - :return: + + :return: """ s = StringIOWrapper('test_suite_ut.data', '') unique_deps = [] @@ -1185,8 +1185,8 @@ class WriteDeps(TestCase): def test_dep_id_repeat(self): """ - - :return: + + :return: """ s = StringIOWrapper('test_suite_ut.data', '') unique_deps = [] @@ -1235,7 +1235,7 @@ class WriteParams(TestCase): def test_no_params(self): """ Test with empty test_args - :return: + :return: """ s = StringIOWrapper('test_suite_ut.data', '') unique_expressions = [] @@ -1247,7 +1247,7 @@ class WriteParams(TestCase): def test_no_exp_param(self): """ Test when there is no macro or expression in the params. - :return: + :return: """ s = StringIOWrapper('test_suite_ut.data', '') unique_expressions = [] @@ -1260,7 +1260,7 @@ class WriteParams(TestCase): def test_hex_format_int_param(self): """ Test int parameter in hex format. - :return: + :return: """ s = StringIOWrapper('test_suite_ut.data', '') unique_expressions = [] @@ -1273,7 +1273,7 @@ class WriteParams(TestCase): def test_with_exp_param(self): """ Test when there is macro or expression in the params. - :return: + :return: """ s = StringIOWrapper('test_suite_ut.data', '') unique_expressions = [] @@ -1304,7 +1304,7 @@ class WriteParams(TestCase): def test_with_repeate_calls(self): """ Test when write_parameter() is called with same macro or expression. - :return: + :return: """ s = StringIOWrapper('test_suite_ut.data', '') unique_expressions = [] @@ -1343,13 +1343,13 @@ class WriteParams(TestCase): class GenTestSuiteDepsChecks(TestCase): """ - + """ def test_empty_suite_deps(self): """ Test with empty suite_deps list. - - :return: + + :return: """ dep_check_code, expression_code = gen_suite_deps_checks([], 'DEP_CHECK_CODE', 'EXPRESSION_CODE') self.assertEqual(dep_check_code, 'DEP_CHECK_CODE') @@ -1358,8 +1358,8 @@ class GenTestSuiteDepsChecks(TestCase): def test_suite_deps(self): """ Test with suite_deps list. - - :return: + + :return: """ dep_check_code, expression_code = gen_suite_deps_checks(['SUITE_DEP'], 'DEP_CHECK_CODE', 'EXPRESSION_CODE') exprectd_dep_check_code = ''' @@ -1377,8 +1377,8 @@ EXPRESSION_CODE def test_no_dep_no_exp(self): """ - Test when there are no dependency and expression code. - :return: + Test when there are no dependency and expression code. + :return: """ dep_check_code, expression_code = gen_suite_deps_checks([], '', '') self.assertEqual(dep_check_code, '') @@ -1396,7 +1396,7 @@ class GenFromTestData(TestCase): def test_intermediate_data_file(self, gen_suite_deps_checks_mock, write_parameters_mock, write_deps_mock): """ Test that intermediate data file is written with expected data. - :return: + :return: """ data = ''' My test @@ -1428,7 +1428,7 @@ func1:0 def test_function_not_found(self): """ Test that AssertError is raised when function info in not found. - :return: + :return: """ data = ''' My test @@ -1444,7 +1444,7 @@ func1:0 def test_different_func_args(self): """ Test that AssertError is raised when no. of parameters and function args differ. - :return: + :return: """ data = ''' My test @@ -1460,7 +1460,7 @@ func1:0 def test_output(self): """ Test that intermediate data file is written with expected data. - :return: + :return: """ data = ''' My test 1 From aae525e7da73ed255246883e4c564c699f7cf9b4 Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Thu, 14 Jun 2018 10:21:42 +0100 Subject: [PATCH 004/490] Give execute permissions to Python scripts --- scripts/test_generate_test_code.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/test_generate_test_code.py diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py old mode 100644 new mode 100755 From 0c0970a4b03af9d4f421ccfb109d9e4be719247f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 18 Jun 2018 17:51:40 +0200 Subject: [PATCH 005/490] Don't generate lines with only whitespace --- scripts/generate_test_code.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 3ff7a41d9..45fb1f574 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -136,11 +136,11 @@ def gen_function_wrapper(name, locals, args_dispatch): wrapper = ''' void {name}_wrapper( void ** params ) {{ - {unused_params} -{locals} +{unused_params}{locals} {name}( {args} ); }} -'''.format(name=name, unused_params='(void)params;' if len(args_dispatch) == 0 else '', +'''.format(name=name, + unused_params='' if args_dispatch else ' (void) params;\n', args=', '.join(args_dispatch), locals=locals) return wrapper From e1dec038494c4d19a6e167059201a435ef94064f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 18 Jun 2018 17:51:56 +0200 Subject: [PATCH 006/490] Fix generation of #line directives in Python 2 When using Python 2 (which is done in the Makefile), all #line directives from the test code were generated with the line number 1. This traces back to the change in the method name for generators in Python 2 (next) vs Python 3 (__next__). Override both methods so that the script remains compatible with both Python 2 and Python 3. --- scripts/generate_test_code.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 45fb1f574..78bbaa399 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -76,17 +76,24 @@ class FileWrapper(io.FileIO): super(FileWrapper, self).__init__(file_name, 'r') self.line_no = 0 + # Override the generator function in a way that works in both Python 2 + # and Python 3. def __next__(self): """ Iterator return impl. :return: Line read from file. """ - line = super(FileWrapper, self).__next__() + parent = super(FileWrapper, self) + if hasattr(parent, '__next__'): + line = parent.__next__() # Python 3 + else: + line = parent.next() # Python 2 if line: self.line_no += 1 # Convert byte array to string with correct encoding return line.decode(sys.getdefaultencoding()) return None + next = __next__ def split_dep(dep): From 5e6e5c8e37fad4438afb335141a7e507b613eeeb Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Tue, 26 Jun 2018 14:06:52 +0100 Subject: [PATCH 007/490] Fix generate_test_code.py unit tests --- scripts/generate_test_code.py | 2 +- scripts/test_generate_test_code.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 78bbaa399..f668128e6 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -147,7 +147,7 @@ void {name}_wrapper( void ** params ) {name}( {args} ); }} '''.format(name=name, - unused_params='' if args_dispatch else ' (void) params;\n', + unused_params='' if args_dispatch else ' (void)params;\n', args=', '.join(args_dispatch), locals=locals) return wrapper diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 4e225dc56..a4debbae4 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -170,7 +170,6 @@ class GenFunctionWrapper(TestCase): void test_a_wrapper( void ** params ) { - test_a( a, b, c, d ); } ''' @@ -186,7 +185,6 @@ void test_a_wrapper( void ** params ) expected = ''' void test_a_wrapper( void ** params ) { - int x = 1; test_a( x, b, c, d ); } From 3862f620ce415273d669b00906256a05358c51cf Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Tue, 26 Jun 2018 14:35:25 +0100 Subject: [PATCH 008/490] Replace asserts with exceptions in generate_test_code.py --- scripts/generate_test_code.py | 49 +++++++++++++++++++++--------- scripts/test_generate_test_code.py | 27 ++++++++-------- 2 files changed, 49 insertions(+), 27 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index f668128e6..22066f7e6 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -62,6 +62,13 @@ class InvalidFileFormat(Exception): pass +class GeneratorInputError(Exception): + """ + Exception to indicate error in the input to the generator. + """ + pass + + class FileWrapper(io.FileIO): """ File wrapper class. Provides reading with line no. tracking. @@ -353,8 +360,10 @@ def parse_functions(funcs_f): func_name, args, func_code, func_dispatch = parse_function_code(funcs_f, deps, suite_deps) suite_functions += func_code # Generate dispatch code and enumeration info - assert func_name not in func_info, "file: %s - function %s re-declared at line %d" % \ - (funcs_f.name, func_name, funcs_f.line_no) + if func_name in func_info: + raise GeneratorInputError( + "file: %s - function %s re-declared at line %d" % \ + (funcs_f.name, func_name, funcs_f.line_no)) func_info[func_name] = (function_idx, args) dispatch_code += '/* Function Id: %d */\n' % function_idx dispatch_code += func_dispatch @@ -411,8 +420,9 @@ def parse_test_data(data_f, debug=False): # Blank line indicates end of test if len(line) == 0: - assert state != STATE_READ_ARGS, "Newline before arguments. " \ - "Test function and arguments missing for %s" % name + if state == STATE_READ_ARGS: + raise GeneratorInputError("Newline before arguments. " \ + "Test function and arguments missing for %s" % name) continue if state == STATE_READ_NAME: @@ -432,8 +442,9 @@ def parse_test_data(data_f, debug=False): yield name, function, deps, args deps = [] state = STATE_READ_NAME - assert state != STATE_READ_ARGS, "Newline before arguments. " \ - "Test function and arguments missing for %s" % name + if state == STATE_READ_ARGS: + raise GeneratorInputError("Newline before arguments. " \ + "Test function and arguments missing for %s" % name) def gen_dep_check(dep_id, dep): @@ -444,9 +455,11 @@ def gen_dep_check(dep_id, dep): :param dep: Dependency macro :return: Dependency check code """ - assert dep_id > -1, "Dependency Id should be a positive integer." + if dep_id < 0: + raise GeneratorInputError("Dependency Id should be a positive integer.") noT, dep = ('!', dep[1:]) if dep[0] == '!' else ('', dep) - assert len(dep) > 0, "Dependency should not be an empty string." + if len(dep) == 0: + raise GeneratorInputError("Dependency should not be an empty string.") dep_check = ''' case {id}: {{ @@ -468,8 +481,10 @@ def gen_expression_check(exp_id, exp): :param exp: Expression/Macro :return: Expression check code """ - assert exp_id > -1, "Expression Id should be a positive integer." - assert len(exp) > 0, "Expression should not be an empty string." + if exp_id < 0: + raise GeneratorInputError("Expression Id should be a positive integer.") + if len(exp) == 0: + raise GeneratorInputError("Expression should not be an empty string.") exp_code = ''' case {exp_id}: {{ @@ -583,13 +598,15 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): # Write test function name test_function_name = 'test_' + function_name - assert test_function_name in func_info, "Function %s not found!" % test_function_name + if test_function_name not in func_info: + raise GeneratorInputError("Function %s not found!" % test_function_name) func_id, func_args = func_info[test_function_name] out_data_f.write(str(func_id)) # Write parameters - assert len(test_args) == len(func_args), \ - "Invalid number of arguments in test %s. See function %s signature." % (test_name, function_name) + if len(test_args) != len(func_args): + raise GeneratorInputError("Invalid number of arguments in test %s. See function %s signature." % (test_name, + function_name)) expression_code += write_parameters(out_data_f, test_args, func_args, unique_expressions) # Write a newline as test case separator @@ -726,4 +743,8 @@ def check_cmd(): if __name__ == "__main__": - check_cmd() + try: + check_cmd() + except GeneratorInputError as e: + script_name = os.path.basename(sys.argv[0]) + print("%s: input error: %s" % (script_name, str(e))) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index a4debbae4..9964ab9f6 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # Unit test for generate_test_code.py # # Copyright (C) 2018, ARM Limited, All Rights Reserved @@ -857,7 +858,7 @@ void func() /* END_CASE */ ''' s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(AssertionError, parse_functions, s) + self.assertRaises(GeneratorInputError, parse_functions, s) class ExcapedSplit(TestCase): @@ -994,7 +995,7 @@ dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" def test_no_args(self): """ - Test AssertionError is raised when test function name and args line is missing. + Test GeneratorInputError is raised when test function name and args line is missing. :return: """ data = """ @@ -1011,13 +1012,13 @@ dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" try: for x, y, z, a in parse_test_data(s): pass - except AssertionError, e: + except GeneratorInputError as e: pass - self.assertEqual(type(e), AssertionError) + self.assertEqual(type(e), GeneratorInputError) def test_incomplete_data(self): """ - Test AssertionError is raised when test function name and args line is missing. + Test GeneratorInputError is raised when test function name and args line is missing. :return: """ data = """ @@ -1029,9 +1030,9 @@ depends_on:YAHOO try: for x, y, z, a in parse_test_data(s): pass - except AssertionError, e: + except GeneratorInputError as e: pass - self.assertEqual(type(e), AssertionError) + self.assertEqual(type(e), GeneratorInputError) class GenDepCheck(TestCase): @@ -1080,14 +1081,14 @@ class GenDepCheck(TestCase): Test invalid dependency input. :return: """ - self.assertRaises(AssertionError, gen_dep_check, 5, '!') + self.assertRaises(GeneratorInputError, gen_dep_check, 5, '!') def test_negative_dep_id(self): """ Test invalid dependency input. :return: """ - self.assertRaises(AssertionError, gen_dep_check, -1, 'YAHOO') + self.assertRaises(GeneratorInputError, gen_dep_check, -1, 'YAHOO') class GenExpCheck(TestCase): @@ -1114,14 +1115,14 @@ class GenExpCheck(TestCase): Test invalid expression input. :return: """ - self.assertRaises(AssertionError, gen_expression_check, 5, '') + self.assertRaises(GeneratorInputError, gen_expression_check, 5, '') def test_negative_exp_id(self): """ Test invalid expression id. :return: """ - self.assertRaises(AssertionError, gen_expression_check, -1, 'YAHOO') + self.assertRaises(GeneratorInputError, gen_expression_check, -1, 'YAHOO') class WriteDeps(TestCase): @@ -1437,7 +1438,7 @@ func1:0 out_data_f = StringIOWrapper('test_suite_ut.datax', '') func_info = {'test_func2': (1, ('int',))} suite_deps = [] - self.assertRaises(AssertionError, gen_from_test_data, data_f, out_data_f, func_info, suite_deps) + self.assertRaises(GeneratorInputError, gen_from_test_data, data_f, out_data_f, func_info, suite_deps) def test_different_func_args(self): """ @@ -1453,7 +1454,7 @@ func1:0 out_data_f = StringIOWrapper('test_suite_ut.datax', '') func_info = {'test_func2': (1, ('int','hex'))} suite_deps = [] - self.assertRaises(AssertionError, gen_from_test_data, data_f, out_data_f, func_info, suite_deps) + self.assertRaises(GeneratorInputError, gen_from_test_data, data_f, out_data_f, func_info, suite_deps) def test_output(self): """ From 394558abfcee8b99a2712e154c1e29104f1ee4b6 Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Tue, 26 Jun 2018 16:57:37 +0100 Subject: [PATCH 009/490] Print line number with data file error --- scripts/generate_test_code.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 22066f7e6..c62b5b9a8 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -421,8 +421,9 @@ def parse_test_data(data_f, debug=False): # Blank line indicates end of test if len(line) == 0: if state == STATE_READ_ARGS: - raise GeneratorInputError("Newline before arguments. " \ - "Test function and arguments missing for %s" % name) + raise GeneratorInputError("[%s:%d] Newline before arguments. " \ + "Test function and arguments missing for %s" % \ + (data_f.name, data_f.line_no, name)) continue if state == STATE_READ_NAME: @@ -443,8 +444,9 @@ def parse_test_data(data_f, debug=False): deps = [] state = STATE_READ_NAME if state == STATE_READ_ARGS: - raise GeneratorInputError("Newline before arguments. " \ - "Test function and arguments missing for %s" % name) + raise GeneratorInputError("[%s:%d] Newline before arguments. " \ + "Test function and arguments missing for %s" % \ + (data_f.name, data_f.line_no, name)) def gen_dep_check(dep_id, dep): @@ -650,7 +652,7 @@ def generate_code(funcs_file, data_file, template_file, platform_file, help_file out_data_file.replace('\\', '\\\\')) # escape '\' # Function code - with FileWrapper(funcs_file) as funcs_f, open(data_file, 'r') as data_f, open(out_data_file, 'w') as out_data_f: + with FileWrapper(funcs_file) as funcs_f, FileWrapper(data_file) as data_f, open(out_data_file, 'w') as out_data_f: suite_deps, dispatch_code, func_code, func_info = parse_functions(funcs_f) snippets['functions_code'] = func_code snippets['dispatch_code'] = dispatch_code From 1e246c1ab325c2ac2e3c619b663b7ecaf0407345 Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Thu, 28 Jun 2018 13:10:19 +0100 Subject: [PATCH 010/490] Change intermediate data file extension to .datax --- scripts/generate_test_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index c62b5b9a8..ccb2d5fe1 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -732,7 +732,7 @@ def check_cmd(): data_name = os.path.splitext(data_file_name)[0] out_c_file = os.path.join(args.out_dir, data_name + '.c') - out_data_file = os.path.join(args.out_dir, data_file_name) + out_data_file = os.path.join(args.out_dir, data_name + '.datax') out_c_file_dir = os.path.dirname(out_c_file) out_data_file_dir = os.path.dirname(out_data_file) From e3fd69c1598a796649987ff44335746f5c305fca Mon Sep 17 00:00:00 2001 From: Azim Khan Date: Thu, 28 Jun 2018 16:47:12 +0100 Subject: [PATCH 011/490] Strip whitespaces added by decode() function --- scripts/generate_test_code.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index ccb2d5fe1..33da990df 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -97,8 +97,9 @@ class FileWrapper(io.FileIO): line = parent.next() # Python 2 if line: self.line_no += 1 - # Convert byte array to string with correct encoding - return line.decode(sys.getdefaultencoding()) + # Convert byte array to string with correct encoding and + # strip any whitespaces added in the decoding process. + return line.decode(sys.getdefaultencoding()).strip() + "\n" return None next = __next__ From 0163be2d38cf4ddb2063fea2a780a7b742d87877 Mon Sep 17 00:00:00 2001 From: Azim Khan Date: Thu, 28 Jun 2018 16:49:13 +0100 Subject: [PATCH 012/490] Wrap code to 79 character limit --- scripts/generate_test_code.py | 182 +++++++++++++++++++++------------- 1 file changed, 115 insertions(+), 67 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 33da990df..b2d49129e 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -23,14 +23,18 @@ Test Suite code generator. Generates a test source file using following input files: -test_suite_xyz.function - Read test functions from test suite functions file. -test_suite_xyz.data - Read test functions and their dependencies to generate - dispatch and dependency check code. -main template - Substitute generated test function dispatch code, dependency - checking code. -platform .function - Read host or target platform implementation for - dispatching test cases from .data file. -helper .function - Read common reusable functions. +test_suite_xyz.function - Read test functions from test suite + functions file. +test_suite_xyz.data - Read test functions and their + dependencies to generate dispatch and + dependency check code. +main_test.function - Template to substitute generated test + function dispatch code, dependency + checking code. +platform .function - Read host or target platform + implementation for dispatching test + cases from .data file. +helpers.function - Read common reusable functions. """ @@ -83,8 +87,8 @@ class FileWrapper(io.FileIO): super(FileWrapper, self).__init__(file_name, 'r') self.line_no = 0 - # Override the generator function in a way that works in both Python 2 - # and Python 3. + # Override the generator function in a way that works in both + # Python 2 and Python 3. def __next__(self): """ Iterator return impl. @@ -109,7 +113,8 @@ def split_dep(dep): Split NOT character '!' from dependency. Used by gen_deps() :param dep: Dependency list - :return: list of tuples where index 0 has '!' if there was a '!' before the dependency string + :return: list of tuples where index 0 has '!' if there was a '!' + before the dependency string """ return ('!', dep[1:]) if dep[0] == '!' else ('', dep) @@ -119,7 +124,8 @@ def gen_deps(deps): Generates dependency i.e. if def and endif code :param deps: List of dependencies. - :return: if defined and endif code with macro annotations for readability. + :return: if defined and endif code with macro annotations for + readability. """ dep_start = ''.join(['#if %sdefined(%s)\n' % split_dep(x) for x in deps]) dep_end = ''.join(['#endif /* %s */\n' % x for x in reversed(deps)]) @@ -129,22 +135,26 @@ def gen_deps(deps): def gen_deps_one_line(deps): """ - Generates dependency checks in one line. Useful for writing code in #else case. + Generates dependency checks in one line. Useful for writing code + in #else case. :param deps: List of dependencies. :return: ifdef code """ - defines = ('#if ' if len(deps) else '') + ' && '.join(['%sdefined(%s)' % split_dep(x) for x in deps]) + defines = '#if ' if len(deps) else '' + defines += ' && '.join(['%sdefined(%s)' % split_dep(x) for x in deps]) return defines def gen_function_wrapper(name, locals, args_dispatch): """ - Creates test function wrapper code. A wrapper has the code to unpack parameters from parameters[] array. + Creates test function wrapper code. A wrapper has the code to + unpack parameters from parameters[] array. :param name: Test function name :param locals: Local variables declaration code - :param args_dispatch: List of dispatch arguments. Ex: ['(char *)params[0]', '*((int *)params[1])'] + :param args_dispatch: List of dispatch arguments. + Ex: ['(char *)params[0]', '*((int *)params[1])'] :return: Test function wrapper. """ # Then create the wrapper @@ -200,7 +210,8 @@ def parse_until_pattern(funcs_f, end_regex): break headers += line else: - raise InvalidFileFormat("file: %s - end pattern [%s] not found!" % (funcs_f.name, end_regex)) + raise InvalidFileFormat("file: %s - end pattern [%s] not found!" % + (funcs_f.name, end_regex)) return headers @@ -220,7 +231,8 @@ def parse_suite_deps(funcs_f): if re.search(END_DEP_REGEX, line): break else: - raise InvalidFileFormat("file: %s - end dependency pattern [%s] not found!" % (funcs_f.name, END_DEP_REGEX)) + raise InvalidFileFormat("file: %s - end dependency pattern [%s]" + " not found!" % (funcs_f.name, END_DEP_REGEX)) return deps @@ -246,8 +258,10 @@ def parse_function_signature(line): """ Parsing function signature - :param line: Line from .functions file that has a function signature. - :return: function name, argument list, local variables for wrapper function and argument dispatch code. + :param line: Line from .functions file that has a function + signature. + :return: function name, argument list, local variables for + wrapper function and argument dispatch code. """ args = [] locals = '' @@ -271,13 +285,16 @@ def parse_function_signature(line): elif re.search('HexParam_t\s*\*\s*.*', arg.strip()): args.append('hex') # create a structure + pointer_initializer = '(uint8_t *) params[%d]' % arg_idx + len_initializer = '*( (uint32_t *) params[%d] )' % (arg_idx+1) locals += """ HexParam_t hex%d = {%s, %s}; -""" % (arg_idx, '(uint8_t *) params[%d]' % arg_idx, '*( (uint32_t *) params[%d] )' % (arg_idx + 1)) +""" % (arg_idx, pointer_initializer, len_initializer) args_dispatch.append('&hex%d' % arg_idx) arg_idx += 1 else: - raise ValueError("Test function arguments can only be 'int', 'char *' or 'HexParam_t'\n%s" % line) + raise ValueError("Test function arguments can only be 'int', " + "'char *' or 'HexParam_t'\n%s" % line) arg_idx += 1 return name, args, locals, args_dispatch @@ -285,7 +302,8 @@ def parse_function_signature(line): def parse_function_code(funcs_f, deps, suite_deps): """ - Parses out a function from function file object and generates function and dispatch code. + Parses out a function from function file object and generates + function and dispatch code. :param funcs_f: file object of the functions file. :param deps: List of dependencies @@ -308,14 +326,16 @@ def parse_function_code(funcs_f, deps, suite_deps): name = 'test_' + name break else: - raise InvalidFileFormat("file: %s - Test functions not found!" % funcs_f.name) + raise InvalidFileFormat("file: %s - Test functions not found!" % + funcs_f.name) for line in funcs_f: if re.search(END_CASE_REGEX, line): break code += line else: - raise InvalidFileFormat("file: %s - end case pattern [%s] not found!" % (funcs_f.name, END_CASE_REGEX)) + raise InvalidFileFormat("file: %s - end case pattern [%s] not " + "found!" % (funcs_f.name, END_CASE_REGEX)) # Add exit label if not present if code.find('exit:') == -1: @@ -336,8 +356,9 @@ def parse_functions(funcs_f): Returns functions code pieces :param funcs_f: file object of the functions file. - :return: List of test suite dependencies, test function dispatch code, function code and - a dict with function identifiers and arguments info. + :return: List of test suite dependencies, test function dispatch + code, function code and a dict with function identifiers + and arguments info. """ suite_headers = '' suite_helpers = '' @@ -358,7 +379,8 @@ def parse_functions(funcs_f): suite_deps += deps elif re.search(BEGIN_CASE_REGEX, line): deps = parse_function_deps(line) - func_name, args, func_code, func_dispatch = parse_function_code(funcs_f, deps, suite_deps) + func_name, args, func_code, func_dispatch =\ + parse_function_code(funcs_f, deps, suite_deps) suite_functions += func_code # Generate dispatch code and enumeration info if func_name in func_info: @@ -378,8 +400,9 @@ def parse_functions(funcs_f): def escaped_split(str, ch): """ Split str on character ch but ignore escaped \{ch} - Since return value is used to write back to the intermediate data file. - Any escape characters in the input are retained in the output. + Since, return value is used to write back to the intermediate + data file, any escape characters in the input are retained in the + output. :param str: String to split :param ch: split character @@ -407,7 +430,8 @@ def parse_test_data(data_f, debug=False): Parses .data file :param data_f: file object of the data file. - :return: Generator that yields test name, function name, dependency list and function argument list. + :return: Generator that yields test name, function name, + dependency list and function argument list. """ STATE_READ_NAME = 0 STATE_READ_ARGS = 1 @@ -422,9 +446,10 @@ def parse_test_data(data_f, debug=False): # Blank line indicates end of test if len(line) == 0: if state == STATE_READ_ARGS: - raise GeneratorInputError("[%s:%d] Newline before arguments. " \ - "Test function and arguments missing for %s" % \ - (data_f.name, data_f.line_no, name)) + raise GeneratorInputError("[%s:%d] Newline before arguments. " + "Test function and arguments " + "missing for %s" % + (data_f.name, data_f.line_no, name)) continue if state == STATE_READ_NAME: @@ -435,7 +460,8 @@ def parse_test_data(data_f, debug=False): # Check dependencies m = re.search('depends_on\:(.*)', line) if m: - deps = [x.strip() for x in m.group(1).split(':') if len(x.strip())] + deps = [x.strip() for x in m.group(1).split(':') if len( + x.strip())] else: # Read test vectors parts = escaped_split(line, ':') @@ -445,9 +471,9 @@ def parse_test_data(data_f, debug=False): deps = [] state = STATE_READ_NAME if state == STATE_READ_ARGS: - raise GeneratorInputError("[%s:%d] Newline before arguments. " \ - "Test function and arguments missing for %s" % \ - (data_f.name, data_f.line_no, name)) + raise GeneratorInputError("[%s:%d] Newline before arguments. " + "Test function and arguments missing for " + "%s" % (data_f.name, data_f.line_no, name)) def gen_dep_check(dep_id, dep): @@ -459,7 +485,8 @@ def gen_dep_check(dep_id, dep): :return: Dependency check code """ if dep_id < 0: - raise GeneratorInputError("Dependency Id should be a positive integer.") + raise GeneratorInputError("Dependency Id should be a positive " + "integer.") noT, dep = ('!', dep[1:]) if dep[0] == '!' else ('', dep) if len(dep) == 0: raise GeneratorInputError("Dependency should not be an empty string.") @@ -485,7 +512,8 @@ def gen_expression_check(exp_id, exp): :return: Expression check code """ if exp_id < 0: - raise GeneratorInputError("Expression Id should be a positive integer.") + raise GeneratorInputError("Expression Id should be a positive " + "integer.") if len(exp) == 0: raise GeneratorInputError("Expression should not be an empty string.") exp_code = ''' @@ -504,7 +532,8 @@ def write_deps(out_data_f, test_deps, unique_deps): :param out_data_f: Output intermediate data file :param test_deps: Dependencies - :param unique_deps: Mutable list to track unique dependencies that are global to this re-entrant function. + :param unique_deps: Mutable list to track unique dependencies + that are global to this re-entrant function. :return: returns dependency check code. """ dep_check_code = '' @@ -530,7 +559,8 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): :param out_data_f: Output intermediate data file :param test_args: Test parameters :param func_args: Function arguments - :param unique_expressions: Mutable list to track unique expressions that are global to this re-entrant function. + :param unique_expressions: Mutable list to track unique + expressions that are global to this re-entrant function. :return: Returns expression check code. """ expression_code = '' @@ -538,13 +568,14 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): typ = func_args[i] val = test_args[i] - # check if val is a non literal int val - if typ == 'int' and not re.match('(\d+$)|((0x)?[0-9a-fA-F]+$)', val): # its an expression + # check if val is a non literal int val (i.e. an expression) + if typ == 'int' and not re.match('(\d+$)|((0x)?[0-9a-fA-F]+$)', val): typ = 'exp' if val not in unique_expressions: unique_expressions.append(val) - # exp_id can be derived from len(). But for readability and consistency with case of existing let's - # use index(). + # exp_id can be derived from len(). But for + # readability and consistency with case of existing + # let's use index(). exp_id = unique_expressions.index(val) expression_code += gen_expression_check(exp_id, val) val = exp_id @@ -559,10 +590,12 @@ def gen_suite_deps_checks(suite_deps, dep_check_code, expression_code): """ Adds preprocessor checks for test suite dependencies. - :param suite_deps: Test suite dependencies read from the .functions file. + :param suite_deps: Test suite dependencies read from the + .functions file. :param dep_check_code: Dependency check code :param expression_code: Expression check code - :return: Dependency and expression code guarded by test suite dependencies. + :return: Dependency and expression code guarded by test suite + dependencies. """ if len(suite_deps): ifdef = gen_deps_one_line(suite_deps) @@ -581,11 +614,13 @@ def gen_suite_deps_checks(suite_deps, dep_check_code, expression_code): def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): """ - Generates dependency checks, expression code and intermediate data file from test data file. + Generates dependency checks, expression code and intermediate + data file from test data file. :param data_f: Data file object :param out_data_f:Output intermediate data file - :param func_info: Dict keyed by function and with function id and arguments info + :param func_info: Dict keyed by function and with function id + and arguments info :param suite_deps: Test suite deps :return: Returns dependency and expression check code """ @@ -593,7 +628,8 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): unique_expressions = [] dep_check_code = '' expression_code = '' - for test_name, function_name, test_deps, test_args in parse_test_data(data_f): + for test_name, function_name, test_deps, test_args in parse_test_data( + data_f): out_data_f.write(test_name + '\n') # Write deps @@ -602,24 +638,29 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): # Write test function name test_function_name = 'test_' + function_name if test_function_name not in func_info: - raise GeneratorInputError("Function %s not found!" % test_function_name) + raise GeneratorInputError("Function %s not found!" % + test_function_name) func_id, func_args = func_info[test_function_name] out_data_f.write(str(func_id)) # Write parameters if len(test_args) != len(func_args): - raise GeneratorInputError("Invalid number of arguments in test %s. See function %s signature." % (test_name, - function_name)) - expression_code += write_parameters(out_data_f, test_args, func_args, unique_expressions) + raise GeneratorInputError("Invalid number of arguments in test " + "%s. See function %s signature." % ( + test_name, function_name)) + expression_code += write_parameters(out_data_f, test_args, func_args, + unique_expressions) # Write a newline as test case separator out_data_f.write('\n') - dep_check_code, expression_code = gen_suite_deps_checks(suite_deps, dep_check_code, expression_code) + dep_check_code, expression_code = gen_suite_deps_checks( + suite_deps, dep_check_code, expression_code) return dep_check_code, expression_code -def generate_code(funcs_file, data_file, template_file, platform_file, help_file, suites_dir, c_file, out_data_file): +def generate_code(funcs_file, data_file, template_file, platform_file, + help_file, suites_dir, c_file, out_data_file): """ Generate mbed-os test code. @@ -645,19 +686,23 @@ def generate_code(funcs_file, data_file, template_file, platform_file, help_file snippets = {'generator_script' : os.path.basename(__file__)} # Read helpers - with open(help_file, 'r') as help_f, open(platform_file, 'r') as platform_f: + with open(help_file, 'r') as help_f, open(platform_file, 'r') as \ + platform_f: snippets['test_common_helper_file'] = help_file snippets['test_common_helpers'] = help_f.read() snippets['test_platform_file'] = platform_file - snippets['platform_code'] = platform_f.read().replace('DATA_FILE', - out_data_file.replace('\\', '\\\\')) # escape '\' + snippets['platform_code'] = platform_f.read().replace( + 'DATA_FILE', out_data_file.replace('\\', '\\\\')) # escape '\' # Function code - with FileWrapper(funcs_file) as funcs_f, FileWrapper(data_file) as data_f, open(out_data_file, 'w') as out_data_f: - suite_deps, dispatch_code, func_code, func_info = parse_functions(funcs_f) + with FileWrapper(funcs_file) as funcs_f, FileWrapper(data_file) as \ + data_f, open(out_data_file, 'w') as out_data_f: + suite_deps, dispatch_code, func_code, func_info = parse_functions( + funcs_f) snippets['functions_code'] = func_code snippets['dispatch_code'] = dispatch_code - dep_check_code, expression_code = gen_from_test_data(data_f, out_data_f, func_info, suite_deps) + dep_check_code, expression_code = gen_from_test_data( + data_f, out_data_f, func_info, suite_deps) snippets['dep_check_code'] = dep_check_code snippets['expression_code'] = expression_code @@ -671,7 +716,8 @@ def generate_code(funcs_file, data_file, template_file, platform_file, help_file with open(template_file, 'r') as template_f, open(c_file, 'w') as c_f: line_no = 1 for line in template_f.readlines(): - snippets['line_no'] = line_no + 1 # Increment as it sets next line number + # Update line number. +1 as #line directive sets next line number + snippets['line_no'] = line_no + 1 code = line.format(**snippets) c_f.write(code) line_no += 1 @@ -683,7 +729,8 @@ def check_cmd(): :return: """ - parser = argparse.ArgumentParser(description='Generate code for mbed-os tests.') + parser = argparse.ArgumentParser( + description='Generate code for mbed-os tests.') parser.add_argument("-f", "--functions-file", dest="funcs_file", @@ -741,8 +788,9 @@ def check_cmd(): if not os.path.exists(d): os.makedirs(d) - generate_code(args.funcs_file, args.data_file, args.template_file, args.platform_file, - args.help_file, args.suites_dir, out_c_file, out_data_file) + generate_code(args.funcs_file, args.data_file, args.template_file, + args.platform_file, args.help_file, args.suites_dir, + out_c_file, out_data_file) if __name__ == "__main__": From 8042f5e2ef293d19d661f5d0661064260a94facf Mon Sep 17 00:00:00 2001 From: Azim Khan Date: Fri, 29 Jun 2018 02:36:57 +0100 Subject: [PATCH 013/490] Improve documentation in generate_test_code.py --- scripts/generate_test_code.py | 176 ++++++++++++++++++++-------------- 1 file changed, 106 insertions(+), 70 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index b2d49129e..047b13001 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -19,22 +19,18 @@ # This file is part of mbed TLS (https://tls.mbed.org) """ -Test Suite code generator. +This script dynamically generates test suite code for Mbed TLS, by +taking following input files. -Generates a test source file using following input files: - -test_suite_xyz.function - Read test functions from test suite - functions file. -test_suite_xyz.data - Read test functions and their - dependencies to generate dispatch and - dependency check code. +test_suite_xyz.function - Test suite functions file contains test + functions. +test_suite_xyz.data - Contains test case vectors. main_test.function - Template to substitute generated test - function dispatch code, dependency - checking code. -platform .function - Read host or target platform - implementation for dispatching test - cases from .data file. -helpers.function - Read common reusable functions. + functions, dispatch code, dependency + checking code etc. +platform .function - Platform specific initialization and + platform code. +helpers.function - Common/reusable data and functions. """ @@ -43,7 +39,6 @@ import os import re import sys import argparse -import shutil BEGIN_HEADER_REGEX = '/\*\s*BEGIN_HEADER\s*\*/' @@ -59,39 +54,39 @@ BEGIN_CASE_REGEX = '/\*\s*BEGIN_CASE\s*(.*?)\s*\*/' END_CASE_REGEX = '/\*\s*END_CASE\s*\*/' -class InvalidFileFormat(Exception): - """ - Exception to indicate invalid file format. - """ - pass - - class GeneratorInputError(Exception): """ - Exception to indicate error in the input to the generator. + Exception to indicate error in the input files to this script. + This includes missing patterns, test function names and other + parsing errors. """ pass class FileWrapper(io.FileIO): """ - File wrapper class. Provides reading with line no. tracking. + This class extends built-in io.FileIO class with attribute line_no, + that indicates line number for the line that is read. """ def __init__(self, file_name): """ - Init file handle. + Instantiate the base class and initialize the line number to 0. :param file_name: File path to open. """ super(FileWrapper, self).__init__(file_name, 'r') self.line_no = 0 - # Override the generator function in a way that works in both - # Python 2 and Python 3. def __next__(self): """ - Iterator return impl. + Python 2 iterator method. This method overrides base class's + next method and extends the next method to count the line + numbers as each line is read. + + It works for both Python 2 and Python 3 by checking iterator + method name in the base iterator object. + :return: Line read from file. """ parent = super(FileWrapper, self) @@ -105,6 +100,8 @@ class FileWrapper(io.FileIO): # strip any whitespaces added in the decoding process. return line.decode(sys.getdefaultencoding()).strip() + "\n" return None + + # Python 3 iterator method next = __next__ @@ -113,15 +110,22 @@ def split_dep(dep): Split NOT character '!' from dependency. Used by gen_deps() :param dep: Dependency list - :return: list of tuples where index 0 has '!' if there was a '!' - before the dependency string + :return: string tuple. Ex: ('!', MACRO) for !MACRO and ('', MACRO) for + MACRO. """ return ('!', dep[1:]) if dep[0] == '!' else ('', dep) def gen_deps(deps): """ - Generates dependency i.e. if def and endif code + Test suite data and functions specifies compile time dependencies. + This function generates C preprocessor code from the input + dependency list. Caller uses the generated preprocessor code to + wrap dependent code. + A dependency in the input list can have a leading '!' character + to negate a condition. '!' is separated from the dependency using + function split_dep() and proper preprocessor check is generated + accordingly. :param deps: List of dependencies. :return: if defined and endif code with macro annotations for @@ -135,8 +139,8 @@ def gen_deps(deps): def gen_deps_one_line(deps): """ - Generates dependency checks in one line. Useful for writing code - in #else case. + Similar to gen_deps() but generates dependency checks in one line. + Useful for generating code with #else block. :param deps: List of dependencies. :return: ifdef code @@ -173,7 +177,12 @@ void {name}_wrapper( void ** params ) def gen_dispatch(name, deps): """ - Generates dispatch code for the test function table. + Test suite code template main_test.function defines a C function + array to contain test case functions. This function generates an + initializer entry for a function in that array. The entry is + composed of a compile time check for the test function + dependencies. At compile time the test function is assigned when + dependencies are met, else NULL is assigned. :param name: Test function name :param deps: List of dependencies @@ -198,11 +207,12 @@ def gen_dispatch(name, deps): def parse_until_pattern(funcs_f, end_regex): """ - Parses function headers or helper code until end pattern. + Matches pattern end_regex to the lines read from the file object. + Returns the lines read until end pattern is matched. :param funcs_f: file object for .functions file :param end_regex: Pattern to stop parsing - :return: Test suite headers code + :return: Lines read before the end pattern """ headers = '#line %d "%s"\n' % (funcs_f.line_no + 1, funcs_f.name) for line in funcs_f: @@ -210,7 +220,7 @@ def parse_until_pattern(funcs_f, end_regex): break headers += line else: - raise InvalidFileFormat("file: %s - end pattern [%s] not found!" % + raise GeneratorInputError("file: %s - end pattern [%s] not found!" % (funcs_f.name, end_regex)) return headers @@ -218,7 +228,10 @@ def parse_until_pattern(funcs_f, end_regex): def parse_suite_deps(funcs_f): """ - Parses test suite dependencies. + Parses test suite dependencies specified at the top of a + .function file, that starts with pattern BEGIN_DEPENDENCIES + and end with END_DEPENDENCIES. Dependencies are specified + after pattern 'depends_on:' and are delimited by ':'. :param funcs_f: file object for .functions file :return: List of test suite dependencies. @@ -231,7 +244,7 @@ def parse_suite_deps(funcs_f): if re.search(END_DEP_REGEX, line): break else: - raise InvalidFileFormat("file: %s - end dependency pattern [%s]" + raise GeneratorInputError("file: %s - end dependency pattern [%s]" " not found!" % (funcs_f.name, END_DEP_REGEX)) return deps @@ -239,7 +252,9 @@ def parse_suite_deps(funcs_f): def parse_function_deps(line): """ - Parses function dependencies. + Parses function dependencies, that are in the same line as + comment BEGIN_CASE. Dependencies are specified after pattern + 'depends_on:' and are delimited by ':'. :param line: Line from .functions file that has dependencies. :return: List of dependencies. @@ -256,7 +271,9 @@ def parse_function_deps(line): def parse_function_signature(line): """ - Parsing function signature + Parses test function signature for validation and generates + a dispatch wrapper function that translates input test vectors + read from the data file into test function arguments. :param line: Line from .functions file that has a function signature. @@ -266,6 +283,7 @@ def parse_function_signature(line): args = [] locals = '' args_dispatch = [] + # Check if the test function returns void. m = re.search('\s*void\s+(\w+)\s*\(', line, re.I) if not m: raise ValueError("Test function should return 'void'\n%s" % line) @@ -326,7 +344,7 @@ def parse_function_code(funcs_f, deps, suite_deps): name = 'test_' + name break else: - raise InvalidFileFormat("file: %s - Test functions not found!" % + raise GeneratorInputError("file: %s - Test functions not found!" % funcs_f.name) for line in funcs_f: @@ -334,7 +352,7 @@ def parse_function_code(funcs_f, deps, suite_deps): break code += line else: - raise InvalidFileFormat("file: %s - end case pattern [%s] not " + raise GeneratorInputError("file: %s - end case pattern [%s] not " "found!" % (funcs_f.name, END_CASE_REGEX)) # Add exit label if not present @@ -353,7 +371,8 @@ def parse_function_code(funcs_f, deps, suite_deps): def parse_functions(funcs_f): """ - Returns functions code pieces + Parses a test_suite_xxx.function file and returns information + for generating a C source file for the test suite. :param funcs_f: file object of the functions file. :return: List of test suite dependencies, test function dispatch @@ -427,7 +446,13 @@ def escaped_split(str, ch): def parse_test_data(data_f, debug=False): """ - Parses .data file + Parses .data file for each test case name, test function name, + test dependencies and test arguments. This information is + correlated with the test functions file for generating an + intermediate data file replacing the strings for test function + names, dependencies and integer constant expressions with + identifiers. Mainly for optimising space for on-target + execution. :param data_f: file object of the data file. :return: Generator that yields test name, function name, @@ -478,7 +503,8 @@ def parse_test_data(data_f, debug=False): def gen_dep_check(dep_id, dep): """ - Generate code for the dependency. + Generate code for checking dependency with the associated + identifier. :param dep_id: Dependency identifier :param dep: Dependency macro @@ -505,7 +531,8 @@ def gen_dep_check(dep_id, dep): def gen_expression_check(exp_id, exp): """ - Generates code for expression check + Generates code for evaluating an integer expression using + associated expression Id. :param exp_id: Expression Identifier :param exp: Expression/Macro @@ -527,8 +554,9 @@ def gen_expression_check(exp_id, exp): def write_deps(out_data_f, test_deps, unique_deps): """ - Write dependencies to intermediate test data file. - It also returns dependency check code. + Write dependencies to intermediate test data file, replacing + the string form with identifiers. Also, generates dependency + check code. :param out_data_f: Output intermediate data file :param test_deps: Dependencies @@ -553,8 +581,9 @@ def write_deps(out_data_f, test_deps, unique_deps): def write_parameters(out_data_f, test_args, func_args, unique_expressions): """ - Writes test parameters to the intermediate data file. - Also generates expression code. + Writes test parameters to the intermediate data file, replacing + the string form with identifiers. Also, generates expression + check code. :param out_data_f: Output intermediate data file :param test_args: Test parameters @@ -588,7 +617,7 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): def gen_suite_deps_checks(suite_deps, dep_check_code, expression_code): """ - Adds preprocessor checks for test suite dependencies. + Generates preprocessor checks for test suite dependencies. :param suite_deps: Test suite dependencies read from the .functions file. @@ -614,8 +643,14 @@ def gen_suite_deps_checks(suite_deps, dep_check_code, expression_code): def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): """ - Generates dependency checks, expression code and intermediate - data file from test data file. + This function reads test case name, dependencies and test vectors + from the .data file. This information is correlated with the test + functions file for generating an intermediate data file replacing + the strings for test function names, dependencies and integer + constant expressions with identifiers. Mainly for optimising + space for on-target execution. + It also generates test case dependency check code and expression + evaluation code. :param data_f: Data file object :param out_data_f:Output intermediate data file @@ -660,15 +695,16 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): def generate_code(funcs_file, data_file, template_file, platform_file, - help_file, suites_dir, c_file, out_data_file): + helpers_file, suites_dir, c_file, out_data_file): """ - Generate mbed-os test code. + Generates C source code from test suite file, data file, common + helpers file and platform file. :param funcs_file: Functions file object :param data_file: Data file object :param template_file: Template file object :param platform_file: Platform file object - :param help_file: Helper functions file object + :param helpers_file: Helper functions file object :param suites_dir: Test suites dir :param c_file: Output C file object :param out_data_file: Output intermediate data file object @@ -678,7 +714,7 @@ def generate_code(funcs_file, data_file, template_file, platform_file, ('Data file', data_file), ('Template file', template_file), ('Platform file', platform_file), - ('Help code file', help_file), + ('Helpers code file', helpers_file), ('Suites dir', suites_dir)]: if not os.path.exists(path): raise IOError("ERROR: %s [%s] not found!" % (name, path)) @@ -686,9 +722,9 @@ def generate_code(funcs_file, data_file, template_file, platform_file, snippets = {'generator_script' : os.path.basename(__file__)} # Read helpers - with open(help_file, 'r') as help_f, open(platform_file, 'r') as \ + with open(helpers_file, 'r') as help_f, open(platform_file, 'r') as \ platform_f: - snippets['test_common_helper_file'] = help_file + snippets['test_common_helper_file'] = helpers_file snippets['test_common_helpers'] = help_f.read() snippets['test_platform_file'] = platform_file snippets['platform_code'] = platform_f.read().replace( @@ -730,36 +766,36 @@ def check_cmd(): :return: """ parser = argparse.ArgumentParser( - description='Generate code for mbed-os tests.') + description='Dynamically generate test suite code.') parser.add_argument("-f", "--functions-file", dest="funcs_file", help="Functions file", - metavar="FUNCTIONS", + metavar="FUNCTIONS_FILE", required=True) parser.add_argument("-d", "--data-file", dest="data_file", help="Data file", - metavar="DATA", + metavar="DATA_FILE", required=True) parser.add_argument("-t", "--template-file", dest="template_file", help="Template file", - metavar="TEMPLATE", + metavar="TEMPLATE_FILE", required=True) parser.add_argument("-s", "--suites-dir", dest="suites_dir", help="Suites dir", - metavar="SUITES", + metavar="SUITES_DIR", required=True) - parser.add_argument("--help-file", - dest="help_file", - help="Help file", - metavar="HELPER", + parser.add_argument("--helpers-file", + dest="helpers_file", + help="Helpers file", + metavar="HELPERS_FILE", required=True) parser.add_argument("-p", "--platform-file", @@ -789,7 +825,7 @@ def check_cmd(): os.makedirs(d) generate_code(args.funcs_file, args.data_file, args.template_file, - args.platform_file, args.help_file, args.suites_dir, + args.platform_file, args.helpers_file, args.suites_dir, out_c_file, out_data_file) From 74a6faafb7348a52e200cfffddbe03c9ea972726 Mon Sep 17 00:00:00 2001 From: Azim Khan Date: Fri, 29 Jun 2018 11:05:32 +0100 Subject: [PATCH 014/490] Rename HexParam_t -> data_t for consistent coding style --- scripts/generate_test_code.py | 8 ++++---- scripts/test_generate_test_code.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 047b13001..c4c11fc39 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -300,19 +300,19 @@ def parse_function_signature(line): elif re.search('char\s*\*\s*.*', arg.strip()): args.append('char*') args_dispatch.append('(char *) params[%d]' % arg_idx) - elif re.search('HexParam_t\s*\*\s*.*', arg.strip()): + elif re.search('data_t\s*\*\s*.*', arg.strip()): args.append('hex') # create a structure pointer_initializer = '(uint8_t *) params[%d]' % arg_idx len_initializer = '*( (uint32_t *) params[%d] )' % (arg_idx+1) - locals += """ HexParam_t hex%d = {%s, %s}; + locals += """ data_t data%d = {%s, %s}; """ % (arg_idx, pointer_initializer, len_initializer) - args_dispatch.append('&hex%d' % arg_idx) + args_dispatch.append('&data%d' % arg_idx) arg_idx += 1 else: raise ValueError("Test function arguments can only be 'int', " - "'char *' or 'HexParam_t'\n%s" % line) + "'char *' or 'data_t'\n%s" % line) arg_idx += 1 return name, args, locals, args_dispatch diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 9964ab9f6..f1088a32a 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -442,11 +442,11 @@ class ParseFuncSignature(TestCase): Test hex parameters parsing :return: """ - line = 'void entropy_threshold( char * a, HexParam_t * h, int result )' + line = 'void entropy_threshold( char * a, data_t * h, int result )' name, args, local, arg_dispatch = parse_function_signature(line) self.assertEqual(name, 'entropy_threshold') self.assertEqual(args, ['char*', 'hex', 'int']) - self.assertEqual(local, ' HexParam_t hex1 = {(uint8_t *) params[1], *( (uint32_t *) params[2] )};\n') + self.assertEqual(local, ' data_t hex1 = {(uint8_t *) params[1], *( (uint32_t *) params[2] )};\n') self.assertEqual(arg_dispatch, ['(char *) params[0]', '&hex1', '*( (int *) params[3] )']) def test_non_void_function(self): @@ -454,15 +454,15 @@ class ParseFuncSignature(TestCase): Test invalid signature (non void). :return: """ - line = 'int entropy_threshold( char * a, HexParam_t * h, int result )' + line = 'int entropy_threshold( char * a, data_t * h, int result )' self.assertRaises(ValueError, parse_function_signature, line) def test_unsupported_arg(self): """ - Test unsupported arguments (not among int, char * and HexParam_t) + Test unsupported arguments (not among int, char * and data_t) :return: """ - line = 'int entropy_threshold( char * a, HexParam_t * h, int * result )' + line = 'int entropy_threshold( char * a, data_t * h, int * result )' self.assertRaises(ValueError, parse_function_signature, line) def test_no_params(self): From 9573f388d91bd5993b5e0d0d6328f9374010cb8c Mon Sep 17 00:00:00 2001 From: Azim Khan Date: Mon, 2 Jul 2018 16:01:04 +0100 Subject: [PATCH 015/490] Add test suite framework summary --- scripts/generate_test_code.py | 151 +++++++++++++++++++++++++++++++--- 1 file changed, 140 insertions(+), 11 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index c4c11fc39..a9ec566e6 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -19,18 +19,147 @@ # This file is part of mbed TLS (https://tls.mbed.org) """ -This script dynamically generates test suite code for Mbed TLS, by -taking following input files. +This script is a key part of Mbed TLS test suites framework. For +understanding the script it is important to understand the +framework. This doc string contains a summary of the framework +and explains the function of this script. + +Mbed TLS test suites: +===================== +Scope: +------ +The test suites focus on unit testing the crypto primitives and also +include x509 parser tests. Tests can be added to test any MBED TLS +module. However, the framework is not capable of testing SSL +protocol, since that requires full stack execution and that is best +tested as part of the system test. + +Test case definition: +--------------------- +Tests are defined in a test_suite_[.].data +file. A test definition contains: + test name + optional build macro dependencies + test function + test parameters + +Test dependencies are build macros that can be specified to indicate +the build config in which the test is valid. For example if a test +depends on a feature that is only enabled by defining a macro. Then +that macro should be specified as a dependency of the test. + +Test function is the function that implements the test steps. This +function is specified for different tests that perform same steps +with different parameters. + +Test parameters are specified in string form separated by ':'. +Parameters can be of type string, binary data specified as hex +string and integer constants specified as integer, macro or +as an expression. Following is an example test definition: + +X509 CRL Unsupported critical extension (issuingDistributionPoint) +depends_on:MBEDTLS_PEM_PARSE_C:MBEDTLS_RSA_C:MBEDTLS_SHA256_C +mbedtls_x509_crl_parse:"data_files/crl-idp.pem":MBEDTLS_ERR_X509_INVALID_EXTENSIONS + MBEDTLS_ERR_ASN1_UNEXPECTED_TAG + +Test functions: +--------------- +Test functions are coded in C in test_suite_.function files. +Functions file is itself not compilable and contains special +format patterns to specify test suite dependencies, start and end +of functions and function dependencies. Check any existing functions +file for example. + +Execution: +---------- +Tests are executed in 3 steps: +- Generating test_suite_[.].c file + for each corresponding .data file. +- Building each source file into executables. +- Running each executable and printing report. + +Generating C test source requires more than just the test functions. +Following extras are required: +- Process main() +- Reading .data file and dispatching test cases. +- Platform specific test case execution +- Dependency checking +- Integer expression evaluation +- Test function dispatch + +Build dependencies and integer expressions (in the test parameters) +are specified as strings in the .data file. Their run time value is +not known at the generation stage. Hence, they need to be translated +into run time evaluations. This script generates the run time checks +for dependencies and integer expressions. + +Similarly, function names have to be translated into function calls. +This script also generates code for function dispatch. + +The extra code mentioned here is either generated by this script +or it comes from the input files: helpers file, platform file and +the template file. + +Helper file: +------------ +Helpers file contains common helper/utility functions and data. + +Platform file: +-------------- +Platform file contains platform specific setup code and test case +dispatch code. For example, host_test.function reads test data +file from host's file system and dispatches tests. +In case of on-target target_test.function tests are not dispatched +on target. Target code is kept minimum and only test functions are +dispatched. Test case dispatch is done on the host using tools like +Greentea. + +Template file: +--------- +Template file for example main_test.function is a template C file in +which generated code and code from input files is substituted to +generate a compilable C file. It also contains skeleton functions for +dependency checks, expression evaluation and function dispatch. These +functions are populated with checks and return codes by this script. + +Template file contains "replacement" fields that are formatted +strings processed by Python str.format() method. + +This script: +============ +Core function of this script is to fill the template file with +code that is generated or read from helpers and platform files. + +This script replaces following fields in the template and generates +the test source file: + +{test_common_helpers} <-- All common code from helpers.function + is substituted here. +{functions_code} <-- Test functions are substituted here + from the input test_suit_xyz.function + file. C preprocessor checks are generated + for the build dependencies specified + in the input file. This script also + generates wrappers for the test + functions with code to expand the + string parameters read from the data + file. +{expression_code} <-- This script enumerates the + expressions in the .data file and + generates code to handle enumerated + expression Ids and return the values. +{dep_check_code} <-- This script enumerates all + build dependencies and generate + code to handle enumerated build + dependency Id and return status: if + the dependency is defined or not. +{dispatch_code} <-- This script enumerates the functions + specified in the input test data file + and generates the initializer for the + function table in the template + file. +{platform_code} <-- Platform specific setup and test + dispatch code. -test_suite_xyz.function - Test suite functions file contains test - functions. -test_suite_xyz.data - Contains test case vectors. -main_test.function - Template to substitute generated test - functions, dispatch code, dependency - checking code etc. -platform .function - Platform specific initialization and - platform code. -helpers.function - Common/reusable data and functions. """ From dbb46928eb5a487ff8a4a03d967629f820408c62 Mon Sep 17 00:00:00 2001 From: Azim Khan Date: Tue, 3 Jul 2018 11:57:54 +0100 Subject: [PATCH 016/490] Fix style errors reported by pylint --- scripts/generate_test_code.py | 525 +++++++++++-------- scripts/test_generate_test_code.py | 812 +++++++++++++++++------------ 2 files changed, 785 insertions(+), 552 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index a9ec566e6..a28a73669 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -16,7 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# This file is part of mbed TLS (https://tls.mbed.org) +# This file is part of Mbed TLS (https://tls.mbed.org) """ This script is a key part of Mbed TLS test suites framework. For @@ -29,7 +29,7 @@ Mbed TLS test suites: Scope: ------ The test suites focus on unit testing the crypto primitives and also -include x509 parser tests. Tests can be added to test any MBED TLS +include x509 parser tests. Tests can be added to test any Mbed TLS module. However, the framework is not capable of testing SSL protocol, since that requires full stack execution and that is best tested as part of the system test. @@ -59,7 +59,8 @@ as an expression. Following is an example test definition: X509 CRL Unsupported critical extension (issuingDistributionPoint) depends_on:MBEDTLS_PEM_PARSE_C:MBEDTLS_RSA_C:MBEDTLS_SHA256_C -mbedtls_x509_crl_parse:"data_files/crl-idp.pem":MBEDTLS_ERR_X509_INVALID_EXTENSIONS + MBEDTLS_ERR_ASN1_UNEXPECTED_TAG +mbedtls_x509_crl_parse:"data_files/crl-idp.pem":\ + MBEDTLS_ERR_X509_INVALID_EXTENSIONS + MBEDTLS_ERR_ASN1_UNEXPECTED_TAG Test functions: --------------- @@ -170,17 +171,17 @@ import sys import argparse -BEGIN_HEADER_REGEX = '/\*\s*BEGIN_HEADER\s*\*/' -END_HEADER_REGEX = '/\*\s*END_HEADER\s*\*/' +BEGIN_HEADER_REGEX = r'/\*\s*BEGIN_HEADER\s*\*/' +END_HEADER_REGEX = r'/\*\s*END_HEADER\s*\*/' -BEGIN_SUITE_HELPERS_REGEX = '/\*\s*BEGIN_SUITE_HELPERS\s*\*/' -END_SUITE_HELPERS_REGEX = '/\*\s*END_SUITE_HELPERS\s*\*/' +BEGIN_SUITE_HELPERS_REGEX = r'/\*\s*BEGIN_SUITE_HELPERS\s*\*/' +END_SUITE_HELPERS_REGEX = r'/\*\s*END_SUITE_HELPERS\s*\*/' -BEGIN_DEP_REGEX = 'BEGIN_DEPENDENCIES' -END_DEP_REGEX = 'END_DEPENDENCIES' +BEGIN_DEP_REGEX = r'BEGIN_DEPENDENCIES' +END_DEP_REGEX = r'END_DEPENDENCIES' -BEGIN_CASE_REGEX = '/\*\s*BEGIN_CASE\s*(.*?)\s*\*/' -END_CASE_REGEX = '/\*\s*END_CASE\s*\*/' +BEGIN_CASE_REGEX = r'/\*\s*BEGIN_CASE\s*(.*?)\s*\*/' +END_CASE_REGEX = r'/\*\s*END_CASE\s*\*/' class GeneratorInputError(Exception): @@ -192,7 +193,7 @@ class GeneratorInputError(Exception): pass -class FileWrapper(io.FileIO): +class FileWrapper(io.FileIO, object): """ This class extends built-in io.FileIO class with attribute line_no, that indicates line number for the line that is read. @@ -205,9 +206,9 @@ class FileWrapper(io.FileIO): :param file_name: File path to open. """ super(FileWrapper, self).__init__(file_name, 'r') - self.line_no = 0 + self._line_no = 0 - def __next__(self): + def next(self): """ Python 2 iterator method. This method overrides base class's next method and extends the next method to count the line @@ -220,23 +221,31 @@ class FileWrapper(io.FileIO): """ parent = super(FileWrapper, self) if hasattr(parent, '__next__'): - line = parent.__next__() # Python 3 + line = parent.__next__() # Python 3 else: - line = parent.next() # Python 2 - if line: - self.line_no += 1 + line = parent.next() # Python 2 + if line is not None: + self._line_no += 1 # Convert byte array to string with correct encoding and # strip any whitespaces added in the decoding process. return line.decode(sys.getdefaultencoding()).strip() + "\n" return None # Python 3 iterator method - next = __next__ + __next__ = next + + def get_line_no(self): + """ + Gives current line number. + """ + return self._line_no + + line_no = property(get_line_no) def split_dep(dep): """ - Split NOT character '!' from dependency. Used by gen_deps() + Split NOT character '!' from dependency. Used by gen_dependencies() :param dep: Dependency list :return: string tuple. Ex: ('!', MACRO) for !MACRO and ('', MACRO) for @@ -245,7 +254,7 @@ def split_dep(dep): return ('!', dep[1:]) if dep[0] == '!' else ('', dep) -def gen_deps(deps): +def gen_dependencies(dependencies): """ Test suite data and functions specifies compile time dependencies. This function generates C preprocessor code from the input @@ -256,36 +265,39 @@ def gen_deps(deps): function split_dep() and proper preprocessor check is generated accordingly. - :param deps: List of dependencies. + :param dependencies: List of dependencies. :return: if defined and endif code with macro annotations for readability. """ - dep_start = ''.join(['#if %sdefined(%s)\n' % split_dep(x) for x in deps]) - dep_end = ''.join(['#endif /* %s */\n' % x for x in reversed(deps)]) + dep_start = ''.join(['#if %sdefined(%s)\n' % (x, y) for x, y in + map(split_dep, dependencies)]) + dep_end = ''.join(['#endif /* %s */\n' % + x for x in reversed(dependencies)]) return dep_start, dep_end -def gen_deps_one_line(deps): +def gen_dependencies_one_line(dependencies): """ - Similar to gen_deps() but generates dependency checks in one line. + Similar to gen_dependencies() but generates dependency checks in one line. Useful for generating code with #else block. - :param deps: List of dependencies. - :return: ifdef code + :param dependencies: List of dependencies. + :return: Preprocessor check code """ - defines = '#if ' if len(deps) else '' - defines += ' && '.join(['%sdefined(%s)' % split_dep(x) for x in deps]) + defines = '#if ' if dependencies else '' + defines += ' && '.join(['%sdefined(%s)' % (x, y) for x, y in map( + split_dep, dependencies)]) return defines -def gen_function_wrapper(name, locals, args_dispatch): +def gen_function_wrapper(name, local_vars, args_dispatch): """ Creates test function wrapper code. A wrapper has the code to unpack parameters from parameters[] array. :param name: Test function name - :param locals: Local variables declaration code + :param local_vars: Local variables declaration code :param args_dispatch: List of dispatch arguments. Ex: ['(char *)params[0]', '*((int *)params[1])'] :return: Test function wrapper. @@ -300,11 +312,11 @@ void {name}_wrapper( void ** params ) '''.format(name=name, unused_params='' if args_dispatch else ' (void)params;\n', args=', '.join(args_dispatch), - locals=locals) + locals=local_vars) return wrapper -def gen_dispatch(name, deps): +def gen_dispatch(name, dependencies): """ Test suite code template main_test.function defines a C function array to contain test case functions. This function generates an @@ -314,18 +326,18 @@ def gen_dispatch(name, deps): dependencies are met, else NULL is assigned. :param name: Test function name - :param deps: List of dependencies + :param dependencies: List of dependencies :return: Dispatch code. """ - if len(deps): - ifdef = gen_deps_one_line(deps) + if dependencies: + preprocessor_check = gen_dependencies_one_line(dependencies) dispatch_code = ''' -{ifdef} +{preprocessor_check} {name}_wrapper, #else NULL, #endif -'''.format(ifdef=ifdef, name=name) +'''.format(preprocessor_check=preprocessor_check, name=name) else: dispatch_code = ''' {name}_wrapper, @@ -350,12 +362,12 @@ def parse_until_pattern(funcs_f, end_regex): headers += line else: raise GeneratorInputError("file: %s - end pattern [%s] not found!" % - (funcs_f.name, end_regex)) + (funcs_f.name, end_regex)) return headers -def parse_suite_deps(funcs_f): +def parse_suite_dependencies(funcs_f): """ Parses test suite dependencies specified at the top of a .function file, that starts with pattern BEGIN_DEPENDENCIES @@ -365,21 +377,22 @@ def parse_suite_deps(funcs_f): :param funcs_f: file object for .functions file :return: List of test suite dependencies. """ - deps = [] + dependencies = [] for line in funcs_f: - m = re.search('depends_on\:(.*)', line.strip()) - if m: - deps += [x.strip() for x in m.group(1).split(':')] + match = re.search('depends_on:(.*)', line.strip()) + if match: + dependencies += [x.strip() for x in match.group(1).split(':')] if re.search(END_DEP_REGEX, line): break else: raise GeneratorInputError("file: %s - end dependency pattern [%s]" - " not found!" % (funcs_f.name, END_DEP_REGEX)) + " not found!" % (funcs_f.name, + END_DEP_REGEX)) - return deps + return dependencies -def parse_function_deps(line): +def parse_function_dependencies(line): """ Parses function dependencies, that are in the same line as comment BEGIN_CASE. Dependencies are specified after pattern @@ -388,14 +401,15 @@ def parse_function_deps(line): :param line: Line from .functions file that has dependencies. :return: List of dependencies. """ - deps = [] - m = re.search(BEGIN_CASE_REGEX, line) - dep_str = m.group(1) - if len(dep_str): - m = re.search('depends_on:(.*)', dep_str) - if m: - deps = [x.strip() for x in m.group(1).strip().split(':')] - return deps + dependencies = [] + match = re.search(BEGIN_CASE_REGEX, line) + dep_str = match.group(1) + if dep_str: + match = re.search('depends_on:(.*)', dep_str) + if match: + dependencies = [x.strip() + for x in match.group(1).strip().split(':')] + return dependencies def parse_function_signature(line): @@ -410,31 +424,31 @@ def parse_function_signature(line): wrapper function and argument dispatch code. """ args = [] - locals = '' + local_vars = '' args_dispatch = [] # Check if the test function returns void. - m = re.search('\s*void\s+(\w+)\s*\(', line, re.I) - if not m: + match = re.search(r'\s*void\s+(\w+)\s*\(', line, re.I) + if not match: raise ValueError("Test function should return 'void'\n%s" % line) - name = m.group(1) - line = line[len(m.group(0)):] + name = match.group(1) + line = line[len(match.group(0)):] arg_idx = 0 for arg in line[:line.find(')')].split(','): arg = arg.strip() if arg == '': continue - if re.search('int\s+.*', arg.strip()): + if re.search(r'int\s+.*', arg.strip()): args.append('int') args_dispatch.append('*( (int *) params[%d] )' % arg_idx) - elif re.search('char\s*\*\s*.*', arg.strip()): + elif re.search(r'char\s*\*\s*.*', arg.strip()): args.append('char*') args_dispatch.append('(char *) params[%d]' % arg_idx) - elif re.search('data_t\s*\*\s*.*', arg.strip()): + elif re.search(r'data_t\s*\*\s*.*', arg.strip()): args.append('hex') # create a structure pointer_initializer = '(uint8_t *) params[%d]' % arg_idx len_initializer = '*( (uint32_t *) params[%d] )' % (arg_idx+1) - locals += """ data_t data%d = {%s, %s}; + local_vars += """ data_t data%d = {%s, %s}; """ % (arg_idx, pointer_initializer, len_initializer) args_dispatch.append('&data%d' % arg_idx) @@ -444,37 +458,38 @@ def parse_function_signature(line): "'char *' or 'data_t'\n%s" % line) arg_idx += 1 - return name, args, locals, args_dispatch + return name, args, local_vars, args_dispatch -def parse_function_code(funcs_f, deps, suite_deps): +def parse_function_code(funcs_f, dependencies, suite_dependencies): """ Parses out a function from function file object and generates function and dispatch code. :param funcs_f: file object of the functions file. - :param deps: List of dependencies - :param suite_deps: List of test suite dependencies + :param dependencies: List of dependencies + :param suite_dependencies: List of test suite dependencies :return: Function name, arguments, function code and dispatch code. """ code = '#line %d "%s"\n' % (funcs_f.line_no + 1, funcs_f.name) for line in funcs_f: # Check function signature - m = re.match('.*?\s+(\w+)\s*\(', line, re.I) - if m: + match = re.match(r'.*?\s+(\w+)\s*\(', line, re.I) + if match: # check if we have full signature i.e. split in more lines - if not re.match('.*\)', line): + if not re.match(r'.*\)', line): for lin in funcs_f: line += lin - if re.search('.*?\)', line): + if re.search(r'.*?\)', line): break - name, args, locals, args_dispatch = parse_function_signature(line) + name, args, local_vars, args_dispatch = parse_function_signature( + line) code += line.replace(name, 'test_' + name) name = 'test_' + name break else: raise GeneratorInputError("file: %s - Test functions not found!" % - funcs_f.name) + funcs_f.name) for line in funcs_f: if re.search(END_CASE_REGEX, line): @@ -482,20 +497,22 @@ def parse_function_code(funcs_f, deps, suite_deps): code += line else: raise GeneratorInputError("file: %s - end case pattern [%s] not " - "found!" % (funcs_f.name, END_CASE_REGEX)) + "found!" % (funcs_f.name, END_CASE_REGEX)) # Add exit label if not present if code.find('exit:') == -1: - s = code.rsplit('}', 1) - if len(s) == 2: + split_code = code.rsplit('}', 1) + if len(split_code) == 2: code = """exit: ;; -}""".join(s) +}""".join(split_code) - code += gen_function_wrapper(name, locals, args_dispatch) - ifdef, endif = gen_deps(deps) - dispatch_code = gen_dispatch(name, suite_deps + deps) - return name, args, ifdef + code + endif, dispatch_code + code += gen_function_wrapper(name, local_vars, args_dispatch) + preprocessor_check_start, preprocessor_check_end = \ + gen_dependencies(dependencies) + dispatch_code = gen_dispatch(name, suite_dependencies + dependencies) + return (name, args, preprocessor_check_start + code + + preprocessor_check_end, dispatch_code) def parse_functions(funcs_f): @@ -508,9 +525,8 @@ def parse_functions(funcs_f): code, function code and a dict with function identifiers and arguments info. """ - suite_headers = '' suite_helpers = '' - suite_deps = [] + suite_dependencies = [] suite_functions = '' func_info = {} function_idx = 0 @@ -518,62 +534,61 @@ def parse_functions(funcs_f): for line in funcs_f: if re.search(BEGIN_HEADER_REGEX, line): headers = parse_until_pattern(funcs_f, END_HEADER_REGEX) - suite_headers += headers + suite_helpers += headers elif re.search(BEGIN_SUITE_HELPERS_REGEX, line): helpers = parse_until_pattern(funcs_f, END_SUITE_HELPERS_REGEX) suite_helpers += helpers elif re.search(BEGIN_DEP_REGEX, line): - deps = parse_suite_deps(funcs_f) - suite_deps += deps + suite_dependencies += parse_suite_dependencies(funcs_f) elif re.search(BEGIN_CASE_REGEX, line): - deps = parse_function_deps(line) + dependencies = parse_function_dependencies(line) func_name, args, func_code, func_dispatch =\ - parse_function_code(funcs_f, deps, suite_deps) + parse_function_code(funcs_f, dependencies, suite_dependencies) suite_functions += func_code # Generate dispatch code and enumeration info if func_name in func_info: raise GeneratorInputError( - "file: %s - function %s re-declared at line %d" % \ + "file: %s - function %s re-declared at line %d" % (funcs_f.name, func_name, funcs_f.line_no)) func_info[func_name] = (function_idx, args) dispatch_code += '/* Function Id: %d */\n' % function_idx dispatch_code += func_dispatch function_idx += 1 - ifdef, endif = gen_deps(suite_deps) - func_code = ifdef + suite_headers + suite_helpers + suite_functions + endif - return suite_deps, dispatch_code, func_code, func_info + func_code = (suite_helpers + + suite_functions).join(gen_dependencies(suite_dependencies)) + return suite_dependencies, dispatch_code, func_code, func_info -def escaped_split(str, ch): +def escaped_split(inp_str, split_char): """ - Split str on character ch but ignore escaped \{ch} + Split inp_str on character split_char but ignore if escaped. Since, return value is used to write back to the intermediate data file, any escape characters in the input are retained in the output. - :param str: String to split - :param ch: split character + :param inp_str: String to split + :param split_char: split character :return: List of splits """ - if len(ch) > 1: + if len(split_char) > 1: raise ValueError('Expected split character. Found string!') out = [] part = '' escape = False - for i in range(len(str)): - if not escape and str[i] == ch: + for character in inp_str: + if not escape and character == split_char: out.append(part) part = '' else: - part += str[i] - escape = not escape and str[i] == '\\' - if len(part): + part += character + escape = not escape and character == '\\' + if part: out.append(part) return out -def parse_test_data(data_f, debug=False): +def parse_test_data(data_f): """ Parses .data file for each test case name, test function name, test dependencies and test arguments. This information is @@ -587,44 +602,44 @@ def parse_test_data(data_f, debug=False): :return: Generator that yields test name, function name, dependency list and function argument list. """ - STATE_READ_NAME = 0 - STATE_READ_ARGS = 1 - state = STATE_READ_NAME - deps = [] + __state_read_name = 0 + __state_read_args = 1 + state = __state_read_name + dependencies = [] name = '' for line in data_f: line = line.strip() - if len(line) and line[0] == '#': # Skip comments + if line and line[0] == '#': # Skip comments continue # Blank line indicates end of test - if len(line) == 0: - if state == STATE_READ_ARGS: + if not line: + if state == __state_read_args: raise GeneratorInputError("[%s:%d] Newline before arguments. " "Test function and arguments " "missing for %s" % (data_f.name, data_f.line_no, name)) continue - if state == STATE_READ_NAME: + if state == __state_read_name: # Read test name name = line - state = STATE_READ_ARGS - elif state == STATE_READ_ARGS: + state = __state_read_args + elif state == __state_read_args: # Check dependencies - m = re.search('depends_on\:(.*)', line) - if m: - deps = [x.strip() for x in m.group(1).split(':') if len( - x.strip())] + match = re.search('depends_on:(.*)', line) + if match: + dependencies = [x.strip() for x in match.group(1).split(':') + if len(x.strip())] else: # Read test vectors parts = escaped_split(line, ':') - function = parts[0] + test_function = parts[0] args = parts[1:] - yield name, function, deps, args - deps = [] - state = STATE_READ_NAME - if state == STATE_READ_ARGS: + yield name, test_function, dependencies, args + dependencies = [] + state = __state_read_name + if state == __state_read_args: raise GeneratorInputError("[%s:%d] Newline before arguments. " "Test function and arguments missing for " "%s" % (data_f.name, data_f.line_no, name)) @@ -642,19 +657,19 @@ def gen_dep_check(dep_id, dep): if dep_id < 0: raise GeneratorInputError("Dependency Id should be a positive " "integer.") - noT, dep = ('!', dep[1:]) if dep[0] == '!' else ('', dep) - if len(dep) == 0: + _not, dep = ('!', dep[1:]) if dep[0] == '!' else ('', dep) + if not dep: raise GeneratorInputError("Dependency should not be an empty string.") dep_check = ''' case {id}: {{ -#if {noT}defined({macro}) +#if {_not}defined({macro}) ret = DEPENDENCY_SUPPORTED; #else ret = DEPENDENCY_NOT_SUPPORTED; #endif }} - break;'''.format(noT=noT, macro=dep, id=dep_id) + break;'''.format(_not=_not, macro=dep, id=dep_id) return dep_check @@ -670,7 +685,7 @@ def gen_expression_check(exp_id, exp): if exp_id < 0: raise GeneratorInputError("Expression Id should be a positive " "integer.") - if len(exp) == 0: + if not exp: raise GeneratorInputError("Expression should not be an empty string.") exp_code = ''' case {exp_id}: @@ -681,28 +696,28 @@ def gen_expression_check(exp_id, exp): return exp_code -def write_deps(out_data_f, test_deps, unique_deps): +def write_dependencies(out_data_f, test_dependencies, unique_dependencies): """ Write dependencies to intermediate test data file, replacing the string form with identifiers. Also, generates dependency check code. :param out_data_f: Output intermediate data file - :param test_deps: Dependencies - :param unique_deps: Mutable list to track unique dependencies + :param test_dependencies: Dependencies + :param unique_dependencies: Mutable list to track unique dependencies that are global to this re-entrant function. :return: returns dependency check code. """ dep_check_code = '' - if len(test_deps): + if test_dependencies: out_data_f.write('depends_on') - for dep in test_deps: - if dep not in unique_deps: - unique_deps.append(dep) - dep_id = unique_deps.index(dep) + for dep in test_dependencies: + if dep not in unique_dependencies: + unique_dependencies.append(dep) + dep_id = unique_dependencies.index(dep) dep_check_code += gen_dep_check(dep_id, dep) else: - dep_id = unique_deps.index(dep) + dep_id = unique_dependencies.index(dep) out_data_f.write(':' + str(dep_id)) out_data_f.write('\n') return dep_check_code @@ -722,12 +737,12 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): :return: Returns expression check code. """ expression_code = '' - for i in range(len(test_args)): + for i, _ in enumerate(test_args): typ = func_args[i] val = test_args[i] # check if val is a non literal int val (i.e. an expression) - if typ == 'int' and not re.match('(\d+$)|((0x)?[0-9a-fA-F]+$)', val): + if typ == 'int' and not re.match(r'(\d+$)|((0x)?[0-9a-fA-F]+$)', val): typ = 'exp' if val not in unique_expressions: unique_expressions.append(val) @@ -744,33 +759,33 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): return expression_code -def gen_suite_deps_checks(suite_deps, dep_check_code, expression_code): +def gen_suite_dep_checks(suite_dependencies, dep_check_code, expression_code): """ Generates preprocessor checks for test suite dependencies. - :param suite_deps: Test suite dependencies read from the + :param suite_dependencies: Test suite dependencies read from the .functions file. :param dep_check_code: Dependency check code :param expression_code: Expression check code :return: Dependency and expression code guarded by test suite dependencies. """ - if len(suite_deps): - ifdef = gen_deps_one_line(suite_deps) + if suite_dependencies: + preprocessor_check = gen_dependencies_one_line(suite_dependencies) dep_check_code = ''' -{ifdef} +{preprocessor_check} {code} #endif -'''.format(ifdef=ifdef, code=dep_check_code) +'''.format(preprocessor_check=preprocessor_check, code=dep_check_code) expression_code = ''' -{ifdef} +{preprocessor_check} {code} #endif -'''.format(ifdef=ifdef, code=expression_code) +'''.format(preprocessor_check=preprocessor_check, code=expression_code) return dep_check_code, expression_code -def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): +def gen_from_test_data(data_f, out_data_f, func_info, suite_dependencies): """ This function reads test case name, dependencies and test vectors from the .data file. This information is correlated with the test @@ -785,19 +800,20 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): :param out_data_f:Output intermediate data file :param func_info: Dict keyed by function and with function id and arguments info - :param suite_deps: Test suite deps + :param suite_dependencies: Test suite dependencies :return: Returns dependency and expression check code """ - unique_deps = [] + unique_dependencies = [] unique_expressions = [] dep_check_code = '' expression_code = '' - for test_name, function_name, test_deps, test_args in parse_test_data( - data_f): + for test_name, function_name, test_dependencies, test_args in \ + parse_test_data(data_f): out_data_f.write(test_name + '\n') - # Write deps - dep_check_code += write_deps(out_data_f, test_deps, unique_deps) + # Write dependencies + dep_check_code += write_dependencies(out_data_f, test_dependencies, + unique_dependencies) # Write test function name test_function_name = 'test_' + function_name @@ -810,35 +826,143 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): # Write parameters if len(test_args) != len(func_args): raise GeneratorInputError("Invalid number of arguments in test " - "%s. See function %s signature." % ( - test_name, function_name)) + "%s. See function %s signature." % + (test_name, function_name)) expression_code += write_parameters(out_data_f, test_args, func_args, unique_expressions) # Write a newline as test case separator out_data_f.write('\n') - dep_check_code, expression_code = gen_suite_deps_checks( - suite_deps, dep_check_code, expression_code) + dep_check_code, expression_code = gen_suite_dep_checks( + suite_dependencies, dep_check_code, expression_code) return dep_check_code, expression_code -def generate_code(funcs_file, data_file, template_file, platform_file, - helpers_file, suites_dir, c_file, out_data_file): +def add_input_info(funcs_file, data_file, template_file, + c_file, snippets): """ - Generates C source code from test suite file, data file, common - helpers file and platform file. + Add generator input info in snippets. :param funcs_file: Functions file object :param data_file: Data file object :param template_file: Template file object - :param platform_file: Platform file object - :param helpers_file: Helper functions file object - :param suites_dir: Test suites dir :param c_file: Output C file object - :param out_data_file: Output intermediate data file object + :param snippets: Dictionary to contain code pieces to be + substituted in the template. :return: """ + snippets['test_file'] = c_file + snippets['test_main_file'] = template_file + snippets['test_case_file'] = funcs_file + snippets['test_case_data_file'] = data_file + + +def read_code_from_input_files(platform_file, helpers_file, + out_data_file, snippets): + """ + Read code from input files and create substitutions for replacement + strings in the template file. + + :param platform_file: Platform file object + :param helpers_file: Helper functions file object + :param out_data_file: Output intermediate data file object + :param snippets: Dictionary to contain code pieces to be + substituted in the template. + :return: + """ + # Read helpers + with open(helpers_file, 'r') as help_f, open(platform_file, 'r') as \ + platform_f: + snippets['test_common_helper_file'] = helpers_file + snippets['test_common_helpers'] = help_f.read() + snippets['test_platform_file'] = platform_file + snippets['platform_code'] = platform_f.read().replace( + 'DATA_FILE', out_data_file.replace('\\', '\\\\')) # escape '\' + + +def write_test_source_file(template_file, c_file, snippets): + """ + Write output source file with generated source code. + + :param template_file: Template file name + :param c_file: Output source file + :param snippets: Generated and code snippets + :return: + """ + with open(template_file, 'r') as template_f, open(c_file, 'w') as c_f: + line_no = 1 + for line in template_f.readlines(): + # Update line number. +1 as #line directive sets next line number + snippets['line_no'] = line_no + 1 + code = line.format(**snippets) + c_f.write(code) + line_no += 1 + + +def parse_function_file(funcs_file, snippets): + """ + Parse function file and generate function dispatch code. + + :param funcs_file: Functions file name + :param snippets: Dictionary to contain code pieces to be + substituted in the template. + :return: + """ + with FileWrapper(funcs_file) as funcs_f: + suite_dependencies, dispatch_code, func_code, func_info = \ + parse_functions(funcs_f) + snippets['functions_code'] = func_code + snippets['dispatch_code'] = dispatch_code + return suite_dependencies, func_info + + +def generate_intermediate_data_file(data_file, out_data_file, + suite_dependencies, func_info, snippets): + """ + Generates intermediate data file from input data file and + information read from functions file. + + :param data_file: Data file name + :param out_data_file: Output/Intermediate data file + :param suite_dependencies: List of suite dependencies. + :param func_info: Function info parsed from functions file. + :param snippets: Dictionary to contain code pieces to be + substituted in the template. + :return: + """ + with FileWrapper(data_file) as data_f, \ + open(out_data_file, 'w') as out_data_f: + dep_check_code, expression_code = gen_from_test_data( + data_f, out_data_f, func_info, suite_dependencies) + snippets['dep_check_code'] = dep_check_code + snippets['expression_code'] = expression_code + + +def generate_code(**input_info): + """ + Generates C source code from test suite file, data file, common + helpers file and platform file. + + input_info expands to following parameters: + funcs_file: Functions file object + data_file: Data file object + template_file: Template file object + platform_file: Platform file object + helpers_file: Helper functions file object + suites_dir: Test suites dir + c_file: Output C file object + out_data_file: Output intermediate data file object + :return: + """ + funcs_file = input_info['funcs_file'] + data_file = input_info['data_file'] + template_file = input_info['template_file'] + platform_file = input_info['platform_file'] + helpers_file = input_info['helpers_file'] + suites_dir = input_info['suites_dir'] + c_file = input_info['c_file'] + out_data_file = input_info['out_data_file'] for name, path in [('Functions file', funcs_file), ('Data file', data_file), ('Template file', template_file), @@ -848,44 +972,15 @@ def generate_code(funcs_file, data_file, template_file, platform_file, if not os.path.exists(path): raise IOError("ERROR: %s [%s] not found!" % (name, path)) - snippets = {'generator_script' : os.path.basename(__file__)} - - # Read helpers - with open(helpers_file, 'r') as help_f, open(platform_file, 'r') as \ - platform_f: - snippets['test_common_helper_file'] = helpers_file - snippets['test_common_helpers'] = help_f.read() - snippets['test_platform_file'] = platform_file - snippets['platform_code'] = platform_f.read().replace( - 'DATA_FILE', out_data_file.replace('\\', '\\\\')) # escape '\' - - # Function code - with FileWrapper(funcs_file) as funcs_f, FileWrapper(data_file) as \ - data_f, open(out_data_file, 'w') as out_data_f: - suite_deps, dispatch_code, func_code, func_info = parse_functions( - funcs_f) - snippets['functions_code'] = func_code - snippets['dispatch_code'] = dispatch_code - dep_check_code, expression_code = gen_from_test_data( - data_f, out_data_f, func_info, suite_deps) - snippets['dep_check_code'] = dep_check_code - snippets['expression_code'] = expression_code - - snippets['test_file'] = c_file - snippets['test_main_file'] = template_file - snippets['test_case_file'] = funcs_file - snippets['test_case_data_file'] = data_file - # Read Template - # Add functions - # - with open(template_file, 'r') as template_f, open(c_file, 'w') as c_f: - line_no = 1 - for line in template_f.readlines(): - # Update line number. +1 as #line directive sets next line number - snippets['line_no'] = line_no + 1 - code = line.format(**snippets) - c_f.write(code) - line_no += 1 + snippets = {'generator_script': os.path.basename(__file__)} + read_code_from_input_files(platform_file, helpers_file, + out_data_file, snippets) + add_input_info(funcs_file, data_file, template_file, + c_file, snippets) + suite_dependencies, func_info = parse_function_file(funcs_file, snippets) + generate_intermediate_data_file(data_file, out_data_file, + suite_dependencies, func_info, snippets) + write_test_source_file(template_file, c_file, snippets) def check_cmd(): @@ -949,18 +1044,20 @@ def check_cmd(): out_c_file_dir = os.path.dirname(out_c_file) out_data_file_dir = os.path.dirname(out_data_file) - for d in [out_c_file_dir, out_data_file_dir]: - if not os.path.exists(d): - os.makedirs(d) + for directory in [out_c_file_dir, out_data_file_dir]: + if not os.path.exists(directory): + os.makedirs(directory) - generate_code(args.funcs_file, args.data_file, args.template_file, - args.platform_file, args.helpers_file, args.suites_dir, - out_c_file, out_data_file) + generate_code(funcs_file=args.funcs_file, data_file=args.data_file, + template_file=args.template_file, + platform_file=args.platform_file, + helpers_file=args.helpers_file, suites_dir=args.suites_dir, + c_file=out_c_file, out_data_file=out_data_file) if __name__ == "__main__": try: check_cmd() - except GeneratorInputError as e: - script_name = os.path.basename(sys.argv[0]) - print("%s: input error: %s" % (script_name, str(e))) + except GeneratorInputError as err: + print("%s: input error: %s" % + (os.path.basename(sys.argv[0]), str(err))) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index f1088a32a..f0a935d20 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Unit test for generate_test_code.py # # Copyright (C) 2018, ARM Limited, All Rights Reserved @@ -16,143 +16,184 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# This file is part of mbed TLS (https://tls.mbed.org) - -from StringIO import StringIO -from unittest import TestCase, main as unittest_main -from mock import patch -from generate_test_code import * - +# This file is part of Mbed TLS (https://tls.mbed.org) """ Unit tests for generate_test_code.py """ +import sys +from StringIO import StringIO +from unittest import TestCase, main as unittest_main +from mock import patch +from generate_test_code import gen_dependencies, gen_dependencies_one_line +from generate_test_code import gen_function_wrapper, gen_dispatch +from generate_test_code import parse_until_pattern, GeneratorInputError +from generate_test_code import parse_suite_dependencies +from generate_test_code import parse_function_dependencies +from generate_test_code import parse_function_signature, parse_function_code +from generate_test_code import parse_functions, END_HEADER_REGEX +from generate_test_code import END_SUITE_HELPERS_REGEX, escaped_split +from generate_test_code import parse_test_data, gen_dep_check +from generate_test_code import gen_expression_check, write_dependencies +from generate_test_code import write_parameters, gen_suite_dep_checks +from generate_test_code import gen_from_test_data + + class GenDep(TestCase): """ Test suite for function gen_dep() """ - def test_deps_list(self): + def test_dependencies_list(self): """ - Test that gen_dep() correctly creates deps for given dependency list. + Test that gen_dep() correctly creates dependencies for given + dependency list. :return: """ - deps = ['DEP1', 'DEP2'] - dep_start, dep_end = gen_deps(deps) - ifdef1, ifdef2 = dep_start.splitlines() + dependencies = ['DEP1', 'DEP2'] + dep_start, dep_end = gen_dependencies(dependencies) + preprocessor1, preprocessor2 = dep_start.splitlines() endif1, endif2 = dep_end.splitlines() - self.assertEqual(ifdef1, '#if defined(DEP1)', 'ifdef generated incorrectly') - self.assertEqual(ifdef2, '#if defined(DEP2)', 'ifdef generated incorrectly') - self.assertEqual(endif1, '#endif /* DEP2 */', 'endif generated incorrectly') - self.assertEqual(endif2, '#endif /* DEP1 */', 'endif generated incorrectly') + self.assertEqual(preprocessor1, '#if defined(DEP1)', + 'Preprocessor generated incorrectly') + self.assertEqual(preprocessor2, '#if defined(DEP2)', + 'Preprocessor generated incorrectly') + self.assertEqual(endif1, '#endif /* DEP2 */', + 'Preprocessor generated incorrectly') + self.assertEqual(endif2, '#endif /* DEP1 */', + 'Preprocessor generated incorrectly') - def test_disabled_deps_list(self): + def test_disabled_dependencies_list(self): """ - Test that gen_dep() correctly creates deps for given dependency list. + Test that gen_dep() correctly creates dependencies for given + dependency list. :return: """ - deps = ['!DEP1', '!DEP2'] - dep_start, dep_end = gen_deps(deps) - ifdef1, ifdef2 = dep_start.splitlines() + dependencies = ['!DEP1', '!DEP2'] + dep_start, dep_end = gen_dependencies(dependencies) + preprocessor1, preprocessor2 = dep_start.splitlines() endif1, endif2 = dep_end.splitlines() - self.assertEqual(ifdef1, '#if !defined(DEP1)', 'ifdef generated incorrectly') - self.assertEqual(ifdef2, '#if !defined(DEP2)', 'ifdef generated incorrectly') - self.assertEqual(endif1, '#endif /* !DEP2 */', 'endif generated incorrectly') - self.assertEqual(endif2, '#endif /* !DEP1 */', 'endif generated incorrectly') + self.assertEqual(preprocessor1, '#if !defined(DEP1)', + 'Preprocessor generated incorrectly') + self.assertEqual(preprocessor2, '#if !defined(DEP2)', + 'Preprocessor generated incorrectly') + self.assertEqual(endif1, '#endif /* !DEP2 */', + 'Preprocessor generated incorrectly') + self.assertEqual(endif2, '#endif /* !DEP1 */', + 'Preprocessor generated incorrectly') - def test_mixed_deps_list(self): + def test_mixed_dependencies_list(self): """ - Test that gen_dep() correctly creates deps for given dependency list. + Test that gen_dep() correctly creates dependencies for given + dependency list. :return: """ - deps = ['!DEP1', 'DEP2'] - dep_start, dep_end = gen_deps(deps) - ifdef1, ifdef2 = dep_start.splitlines() + dependencies = ['!DEP1', 'DEP2'] + dep_start, dep_end = gen_dependencies(dependencies) + preprocessor1, preprocessor2 = dep_start.splitlines() endif1, endif2 = dep_end.splitlines() - self.assertEqual(ifdef1, '#if !defined(DEP1)', 'ifdef generated incorrectly') - self.assertEqual(ifdef2, '#if defined(DEP2)', 'ifdef generated incorrectly') - self.assertEqual(endif1, '#endif /* DEP2 */', 'endif generated incorrectly') - self.assertEqual(endif2, '#endif /* !DEP1 */', 'endif generated incorrectly') + self.assertEqual(preprocessor1, '#if !defined(DEP1)', + 'Preprocessor generated incorrectly') + self.assertEqual(preprocessor2, '#if defined(DEP2)', + 'Preprocessor generated incorrectly') + self.assertEqual(endif1, '#endif /* DEP2 */', + 'Preprocessor generated incorrectly') + self.assertEqual(endif2, '#endif /* !DEP1 */', + 'Preprocessor generated incorrectly') - def test_empty_deps_list(self): + def test_empty_dependencies_list(self): """ - Test that gen_dep() correctly creates deps for given dependency list. + Test that gen_dep() correctly creates dependencies for given + dependency list. :return: """ - deps = [] - dep_start, dep_end = gen_deps(deps) - self.assertEqual(dep_start, '', 'ifdef generated incorrectly') - self.assertEqual(dep_end, '', 'ifdef generated incorrectly') + dependencies = [] + dep_start, dep_end = gen_dependencies(dependencies) + self.assertEqual(dep_start, '', 'Preprocessor generated incorrectly') + self.assertEqual(dep_end, '', 'Preprocessor generated incorrectly') - def test_large_deps_list(self): + def test_large_dependencies_list(self): """ - Test that gen_dep() correctly creates deps for given dependency list. + Test that gen_dep() correctly creates dependencies for given + dependency list. :return: """ - deps = [] + dependencies = [] count = 10 for i in range(count): - deps.append('DEP%d' % i) - dep_start, dep_end = gen_deps(deps) - self.assertEqual(len(dep_start.splitlines()), count, 'ifdef generated incorrectly') - self.assertEqual(len(dep_end.splitlines()), count, 'ifdef generated incorrectly') + dependencies.append('DEP%d' % i) + dep_start, dep_end = gen_dependencies(dependencies) + self.assertEqual(len(dep_start.splitlines()), count, + 'Preprocessor generated incorrectly') + self.assertEqual(len(dep_end.splitlines()), count, + 'Preprocessor generated incorrectly') class GenDepOneLine(TestCase): """ - Test Suite for testing gen_deps_one_line() + Test Suite for testing gen_dependencies_one_line() """ - def test_deps_list(self): + def test_dependencies_list(self): """ - Test that gen_dep() correctly creates deps for given dependency list. + Test that gen_dep() correctly creates dependencies for given + dependency list. :return: """ - deps = ['DEP1', 'DEP2'] - dep_str = gen_deps_one_line(deps) - self.assertEqual(dep_str, '#if defined(DEP1) && defined(DEP2)', 'ifdef generated incorrectly') + dependencies = ['DEP1', 'DEP2'] + dep_str = gen_dependencies_one_line(dependencies) + self.assertEqual(dep_str, '#if defined(DEP1) && defined(DEP2)', + 'Preprocessor generated incorrectly') - def test_disabled_deps_list(self): + def test_disabled_dependencies_list(self): """ - Test that gen_dep() correctly creates deps for given dependency list. + Test that gen_dep() correctly creates dependencies for given + dependency list. :return: """ - deps = ['!DEP1', '!DEP2'] - dep_str = gen_deps_one_line(deps) - self.assertEqual(dep_str, '#if !defined(DEP1) && !defined(DEP2)', 'ifdef generated incorrectly') + dependencies = ['!DEP1', '!DEP2'] + dep_str = gen_dependencies_one_line(dependencies) + self.assertEqual(dep_str, '#if !defined(DEP1) && !defined(DEP2)', + 'Preprocessor generated incorrectly') - def test_mixed_deps_list(self): + def test_mixed_dependencies_list(self): """ - Test that gen_dep() correctly creates deps for given dependency list. + Test that gen_dep() correctly creates dependencies for given + dependency list. :return: """ - deps = ['!DEP1', 'DEP2'] - dep_str = gen_deps_one_line(deps) - self.assertEqual(dep_str, '#if !defined(DEP1) && defined(DEP2)', 'ifdef generated incorrectly') + dependencies = ['!DEP1', 'DEP2'] + dep_str = gen_dependencies_one_line(dependencies) + self.assertEqual(dep_str, '#if !defined(DEP1) && defined(DEP2)', + 'Preprocessor generated incorrectly') - def test_empty_deps_list(self): + def test_empty_dependencies_list(self): """ - Test that gen_dep() correctly creates deps for given dependency list. + Test that gen_dep() correctly creates dependencies for given + dependency list. :return: """ - deps = [] - dep_str = gen_deps_one_line(deps) - self.assertEqual(dep_str, '', 'ifdef generated incorrectly') + dependencies = [] + dep_str = gen_dependencies_one_line(dependencies) + self.assertEqual(dep_str, '', 'Preprocessor generated incorrectly') - def test_large_deps_list(self): + def test_large_dependencies_list(self): """ - Test that gen_dep() correctly creates deps for given dependency list. + Test that gen_dep() correctly creates dependencies for given + dependency list. :return: """ - deps = [] + dependencies = [] count = 10 for i in range(count): - deps.append('DEP%d' % i) - dep_str = gen_deps_one_line(deps) - expected = '#if ' + ' && '.join(['defined(%s)' % x for x in deps]) - self.assertEqual(dep_str, expected, 'ifdef generated incorrectly') + dependencies.append('DEP%d' % i) + dep_str = gen_dependencies_one_line(dependencies) + expected = '#if ' + ' && '.join(['defined(%s)' % + x for x in dependencies]) + self.assertEqual(dep_str, expected, + 'Preprocessor generated incorrectly') class GenFunctionWrapper(TestCase): @@ -182,7 +223,8 @@ void test_a_wrapper( void ** params ) :return: """ - code = gen_function_wrapper('test_a', 'int x = 1;', ('x', 'b', 'c', 'd')) + code = gen_function_wrapper('test_a', + 'int x = 1;', ('x', 'b', 'c', 'd')) expected = ''' void test_a_wrapper( void ** params ) { @@ -230,7 +272,7 @@ class GenDispatch(TestCase): ''' self.assertEqual(code, expected) - def test_empty_deps(self): + def test_empty_dependencies(self): """ Test empty dependency list. :return: @@ -246,7 +288,7 @@ class StringIOWrapper(StringIO, object): """ file like class to mock file object in tests. """ - def __init__(self, file_name, data, line_no = 1): + def __init__(self, file_name, data, line_no=1): """ Init file handle. @@ -260,17 +302,28 @@ class StringIOWrapper(StringIO, object): def next(self): """ - Iterator return impl. - :return: - """ - line = super(StringIOWrapper, self).next() - return line + Iterator method. This method overrides base class's + next method and extends the next method to count the line + numbers as each line is read. - def readline(self, limit=0): + :return: Line read from file. + """ + parent = super(StringIOWrapper, self) + line = parent.next() # Python 2 + if line: + self.line_no += 1 + # Convert byte array to string with correct encoding and + # strip any whitespaces added in the decoding process. + return line.decode(sys.getdefaultencoding()).strip() + "\n" + return None + + __next__ = next + + def readline(self, length=0): """ Wrap the base class readline. - :param limit: + :param length: :return: """ line = super(StringIOWrapper, self).readline() @@ -300,8 +353,8 @@ class ParseUntilPattern(TestCase): #define ECP_PF_UNKNOWN -1 ''' - s = StringIOWrapper('test_suite_ut.function', data, line_no=0) - headers = parse_until_pattern(s, END_HEADER_REGEX) + stream = StringIOWrapper('test_suite_ut.function', data, line_no=0) + headers = parse_until_pattern(stream, END_HEADER_REGEX) self.assertEqual(headers, expected) def test_line_no(self): @@ -321,13 +374,15 @@ class ParseUntilPattern(TestCase): #define ECP_PF_UNKNOWN -1 ''' % (offset_line_no + 1) - s = StringIOWrapper('test_suite_ut.function', data, offset_line_no) - headers = parse_until_pattern(s, END_HEADER_REGEX) + stream = StringIOWrapper('test_suite_ut.function', data, + offset_line_no) + headers = parse_until_pattern(stream, END_HEADER_REGEX) self.assertEqual(headers, expected) def test_no_end_header_comment(self): """ - Test that InvalidFileFormat is raised when end header comment is missing. + Test that InvalidFileFormat is raised when end header comment is + missing. :return: """ data = '''#include "mbedtls/ecp.h" @@ -335,16 +390,17 @@ class ParseUntilPattern(TestCase): #define ECP_PF_UNKNOWN -1 ''' - s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(InvalidFileFormat, parse_until_pattern, s, END_HEADER_REGEX) + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(GeneratorInputError, parse_until_pattern, stream, + END_HEADER_REGEX) -class ParseSuiteDeps(TestCase): +class ParseSuiteDependencies(TestCase): """ - Test Suite for testing parse_suite_deps(). + Test Suite for testing parse_suite_dependencies(). """ - def test_suite_deps(self): + def test_suite_dependencies(self): """ :return: @@ -355,9 +411,9 @@ class ParseSuiteDeps(TestCase): */ ''' expected = ['MBEDTLS_ECP_C'] - s = StringIOWrapper('test_suite_ut.function', data) - deps = parse_suite_deps(s) - self.assertEqual(deps, expected) + stream = StringIOWrapper('test_suite_ut.function', data) + dependencies = parse_suite_dependencies(stream) + self.assertEqual(dependencies, expected) def test_no_end_dep_comment(self): """ @@ -367,10 +423,11 @@ class ParseSuiteDeps(TestCase): data = ''' * depends_on:MBEDTLS_ECP_C ''' - s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(InvalidFileFormat, parse_suite_deps, s) + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(GeneratorInputError, parse_suite_dependencies, + stream) - def test_deps_split(self): + def test_dependencies_split(self): """ Test that InvalidFileFormat is raised when end dep comment is missing. :return: @@ -381,43 +438,47 @@ class ParseSuiteDeps(TestCase): */ ''' expected = ['MBEDTLS_ECP_C', 'A', 'B', 'C', 'D', 'F', 'G', '!H'] - s = StringIOWrapper('test_suite_ut.function', data) - deps = parse_suite_deps(s) - self.assertEqual(deps, expected) + stream = StringIOWrapper('test_suite_ut.function', data) + dependencies = parse_suite_dependencies(stream) + self.assertEqual(dependencies, expected) -class ParseFuncDeps(TestCase): +class ParseFuncDependencies(TestCase): """ - Test Suite for testing parse_function_deps() + Test Suite for testing parse_function_dependencies() """ - def test_function_deps(self): + def test_function_dependencies(self): """ - Test that parse_function_deps() correctly parses function dependencies. + Test that parse_function_dependencies() correctly parses function + dependencies. :return: """ - line = '/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */' + line = '/* BEGIN_CASE ' \ + 'depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */' expected = ['MBEDTLS_ENTROPY_NV_SEED', 'MBEDTLS_FS_IO'] - deps = parse_function_deps(line) - self.assertEqual(deps, expected) + dependencies = parse_function_dependencies(line) + self.assertEqual(dependencies, expected) - def test_no_deps(self): + def test_no_dependencies(self): """ - Test that parse_function_deps() correctly parses function dependencies. + Test that parse_function_dependencies() correctly parses function + dependencies. :return: """ line = '/* BEGIN_CASE */' - deps = parse_function_deps(line) - self.assertEqual(deps, []) + dependencies = parse_function_dependencies(line) + self.assertEqual(dependencies, []) - def test_poorly_defined_deps(self): + def test_tolerance(self): """ - Test that parse_function_deps() correctly parses function dependencies. + Test that parse_function_dependencies() correctly parses function + dependencies. :return: """ line = '/* BEGIN_CASE depends_on:MBEDTLS_FS_IO: A : !B:C : F*/' - deps = parse_function_deps(line) - self.assertEqual(deps, ['MBEDTLS_FS_IO', 'A', '!B', 'C', 'F']) + dependencies = parse_function_dependencies(line) + self.assertEqual(dependencies, ['MBEDTLS_FS_IO', 'A', '!B', 'C', 'F']) class ParseFuncSignature(TestCase): @@ -435,7 +496,9 @@ class ParseFuncSignature(TestCase): self.assertEqual(name, 'entropy_threshold') self.assertEqual(args, ['char*', 'int', 'int']) self.assertEqual(local, '') - self.assertEqual(arg_dispatch, ['(char *) params[0]', '*( (int *) params[1] )', '*( (int *) params[2] )']) + self.assertEqual(arg_dispatch, ['(char *) params[0]', + '*( (int *) params[1] )', + '*( (int *) params[2] )']) def test_hex_params(self): """ @@ -446,8 +509,12 @@ class ParseFuncSignature(TestCase): name, args, local, arg_dispatch = parse_function_signature(line) self.assertEqual(name, 'entropy_threshold') self.assertEqual(args, ['char*', 'hex', 'int']) - self.assertEqual(local, ' data_t hex1 = {(uint8_t *) params[1], *( (uint32_t *) params[2] )};\n') - self.assertEqual(arg_dispatch, ['(char *) params[0]', '&hex1', '*( (int *) params[3] )']) + self.assertEqual(local, + ' data_t hex1 = {(uint8_t *) params[1], ' + '*( (uint32_t *) params[2] )};\n') + self.assertEqual(arg_dispatch, ['(char *) params[0]', + '&hex1', + '*( (int *) params[3] )']) def test_non_void_function(self): """ @@ -493,8 +560,9 @@ No test function ''' - s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(InvalidFileFormat, parse_function_code, s, [], []) + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(GeneratorInputError, parse_function_code, stream, [], + []) def test_no_end_case_comment(self): """ @@ -506,11 +574,13 @@ void test_func() { } ''' - s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(InvalidFileFormat, parse_function_code, s, [], []) + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(GeneratorInputError, parse_function_code, stream, [], + []) @patch("generate_test_code.parse_function_signature") - def test_parse_function_signature_called(self, parse_function_signature_mock): + def test_function_called(self, + parse_function_signature_mock): """ Test parse_function_code() :return: @@ -521,26 +591,27 @@ void test_func() { } ''' - s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(InvalidFileFormat, parse_function_code, s, [], []) + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(GeneratorInputError, parse_function_code, + stream, [], []) self.assertTrue(parse_function_signature_mock.called) parse_function_signature_mock.assert_called_with('void test_func()\n') @patch("generate_test_code.gen_dispatch") - @patch("generate_test_code.gen_deps") + @patch("generate_test_code.gen_dependencies") @patch("generate_test_code.gen_function_wrapper") @patch("generate_test_code.parse_function_signature") def test_return(self, parse_function_signature_mock, - gen_function_wrapper_mock, - gen_deps_mock, - gen_dispatch_mock): + gen_function_wrapper_mock, + gen_dependencies_mock, + gen_dispatch_mock): """ Test generated code. :return: """ parse_function_signature_mock.return_value = ('func', [], '', []) gen_function_wrapper_mock.return_value = '' - gen_deps_mock.side_effect = gen_deps + gen_dependencies_mock.side_effect = gen_dependencies gen_dispatch_mock.side_effect = gen_dispatch data = ''' void func() @@ -550,10 +621,9 @@ void func() } /* END_CASE */ ''' - s = StringIOWrapper('test_suite_ut.function', data) - name, arg, code, dispatch_code = parse_function_code(s, [], []) + stream = StringIOWrapper('test_suite_ut.function', data) + name, arg, code, dispatch_code = parse_function_code(stream, [], []) - #self.assertRaises(InvalidFileFormat, parse_function_code, s, [], []) self.assertTrue(parse_function_signature_mock.called) parse_function_signature_mock.assert_called_with('void func()\n') gen_function_wrapper_mock.assert_called_with('test_func', '', []) @@ -572,20 +642,20 @@ exit: self.assertEqual(dispatch_code, "\n test_func_wrapper,\n") @patch("generate_test_code.gen_dispatch") - @patch("generate_test_code.gen_deps") + @patch("generate_test_code.gen_dependencies") @patch("generate_test_code.gen_function_wrapper") @patch("generate_test_code.parse_function_signature") def test_with_exit_label(self, parse_function_signature_mock, - gen_function_wrapper_mock, - gen_deps_mock, - gen_dispatch_mock): + gen_function_wrapper_mock, + gen_dependencies_mock, + gen_dispatch_mock): """ Test when exit label is present. :return: """ parse_function_signature_mock.return_value = ('func', [], '', []) gen_function_wrapper_mock.return_value = '' - gen_deps_mock.side_effect = gen_deps + gen_dependencies_mock.side_effect = gen_dependencies gen_dispatch_mock.side_effect = gen_dispatch data = ''' void func() @@ -598,8 +668,8 @@ exit: } /* END_CASE */ ''' - s = StringIOWrapper('test_suite_ut.function', data) - name, arg, code, dispatch_code = parse_function_code(s, [], []) + stream = StringIOWrapper('test_suite_ut.function', data) + _, _, code, _ = parse_function_code(stream, [], []) expected = '''#line 2 "test_suite_ut.function" void test_func() @@ -625,7 +695,8 @@ class ParseFunction(TestCase): Test that begin header is checked and parse_until_pattern() is called. :return: """ - def stop(this): + def stop(*_unused): + """Stop when parse_until_pattern is called.""" raise Exception parse_until_pattern_mock.side_effect = stop data = '''/* BEGIN_HEADER */ @@ -634,10 +705,10 @@ class ParseFunction(TestCase): #define ECP_PF_UNKNOWN -1 /* END_HEADER */ ''' - s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(Exception, parse_functions, s) - parse_until_pattern_mock.assert_called_with(s, END_HEADER_REGEX) - self.assertEqual(s.line_no, 2) + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(Exception, parse_functions, stream) + parse_until_pattern_mock.assert_called_with(stream, END_HEADER_REGEX) + self.assertEqual(stream.line_no, 2) @patch("generate_test_code.parse_until_pattern") def test_begin_helper(self, parse_until_pattern_mock): @@ -645,89 +716,97 @@ class ParseFunction(TestCase): Test that begin helper is checked and parse_until_pattern() is called. :return: """ - def stop(this): + def stop(*_unused): + """Stop when parse_until_pattern is called.""" raise Exception parse_until_pattern_mock.side_effect = stop data = '''/* BEGIN_SUITE_HELPERS */ -void print_helloworld() +void print_hello_world() { - printf ("Hello World!\n"); + printf("Hello World!\n"); } /* END_SUITE_HELPERS */ ''' - s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(Exception, parse_functions, s) - parse_until_pattern_mock.assert_called_with(s, END_SUITE_HELPERS_REGEX) - self.assertEqual(s.line_no, 2) + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(Exception, parse_functions, stream) + parse_until_pattern_mock.assert_called_with(stream, + END_SUITE_HELPERS_REGEX) + self.assertEqual(stream.line_no, 2) - @patch("generate_test_code.parse_suite_deps") - def test_begin_dep(self, parse_suite_deps_mock): + @patch("generate_test_code.parse_suite_dependencies") + def test_begin_dep(self, parse_suite_dependencies_mock): """ - Test that begin dep is checked and parse_suite_deps() is called. + Test that begin dep is checked and parse_suite_dependencies() is + called. :return: """ - def stop(this): + def stop(*_unused): + """Stop when parse_until_pattern is called.""" raise Exception - parse_suite_deps_mock.side_effect = stop + parse_suite_dependencies_mock.side_effect = stop data = '''/* BEGIN_DEPENDENCIES * depends_on:MBEDTLS_ECP_C * END_DEPENDENCIES */ ''' - s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(Exception, parse_functions, s) - parse_suite_deps_mock.assert_called_with(s) - self.assertEqual(s.line_no, 2) + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(Exception, parse_functions, stream) + parse_suite_dependencies_mock.assert_called_with(stream) + self.assertEqual(stream.line_no, 2) - @patch("generate_test_code.parse_function_deps") - def test_begin_function_dep(self, parse_function_deps_mock): + @patch("generate_test_code.parse_function_dependencies") + def test_begin_function_dep(self, func_mock): """ - Test that begin dep is checked and parse_function_deps() is called. + Test that begin dep is checked and parse_function_dependencies() is + called. :return: """ - def stop(this): + def stop(*_unused): + """Stop when parse_until_pattern is called.""" raise Exception - parse_function_deps_mock.side_effect = stop + func_mock.side_effect = stop - deps_str = '/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */\n' + dependencies_str = '/* BEGIN_CASE ' \ + 'depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */\n' data = '''%svoid test_func() { } -''' % deps_str - s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(Exception, parse_functions, s) - parse_function_deps_mock.assert_called_with(deps_str) - self.assertEqual(s.line_no, 2) +''' % dependencies_str + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(Exception, parse_functions, stream) + func_mock.assert_called_with(dependencies_str) + self.assertEqual(stream.line_no, 2) @patch("generate_test_code.parse_function_code") - @patch("generate_test_code.parse_function_deps") - def test_return(self, parse_function_deps_mock, parse_function_code_mock): + @patch("generate_test_code.parse_function_dependencies") + def test_return(self, func_mock1, func_mock2): """ Test that begin case is checked and parse_function_code() is called. :return: """ - def stop(this): - raise Exception - parse_function_deps_mock.return_value = [] - in_func_code= '''void test_func() + func_mock1.return_value = [] + in_func_code = '''void test_func() { } ''' func_dispatch = ''' test_func_wrapper, ''' - parse_function_code_mock.return_value = 'test_func', [], in_func_code, func_dispatch - deps_str = '/* BEGIN_CASE depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */\n' + func_mock2.return_value = 'test_func', [],\ + in_func_code, func_dispatch + dependencies_str = '/* BEGIN_CASE ' \ + 'depends_on:MBEDTLS_ENTROPY_NV_SEED:MBEDTLS_FS_IO */\n' data = '''%svoid test_func() { } -''' % deps_str - s = StringIOWrapper('test_suite_ut.function', data) - suite_deps, dispatch_code, func_code, func_info = parse_functions(s) - parse_function_deps_mock.assert_called_with(deps_str) - parse_function_code_mock.assert_called_with(s, [], []) - self.assertEqual(s.line_no, 5) - self.assertEqual(suite_deps, []) +''' % dependencies_str + stream = StringIOWrapper('test_suite_ut.function', data) + suite_dependencies, dispatch_code, func_code, func_info = \ + parse_functions(stream) + func_mock1.assert_called_with(dependencies_str) + func_mock2.assert_called_with(stream, [], []) + self.assertEqual(stream.line_no, 5) + self.assertEqual(suite_dependencies, []) expected_dispatch_code = '''/* Function Id: 0 */ test_func_wrapper, @@ -764,10 +843,11 @@ void func2() } /* END_CASE */ ''' - s = StringIOWrapper('test_suite_ut.function', data) - suite_deps, dispatch_code, func_code, func_info = parse_functions(s) - self.assertEqual(s.line_no, 23) - self.assertEqual(suite_deps, ['MBEDTLS_ECP_C']) + stream = StringIOWrapper('test_suite_ut.function', data) + suite_dependencies, dispatch_code, func_code, func_info = \ + parse_functions(stream) + self.assertEqual(stream.line_no, 23) + self.assertEqual(suite_dependencies, ['MBEDTLS_ECP_C']) expected_dispatch_code = '''/* Function Id: 0 */ @@ -827,7 +907,8 @@ void test_func2_wrapper( void ** params ) #endif /* MBEDTLS_ECP_C */ ''' self.assertEqual(func_code, expected_func_code) - self.assertEqual(func_info, {'test_func1': (0, []), 'test_func2': (1, [])}) + self.assertEqual(func_info, {'test_func1': (0, []), + 'test_func2': (1, [])}) def test_same_function_name(self): """ @@ -857,15 +938,16 @@ void func() } /* END_CASE */ ''' - s = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(GeneratorInputError, parse_functions, s) + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaises(GeneratorInputError, parse_functions, stream) -class ExcapedSplit(TestCase): +class EscapedSplit(TestCase): """ Test suite for testing escaped_split(). - Note: Since escaped_split() output is used to write back to the intermediate data file. Any escape characters - in the input are retained in the output. + Note: Since escaped_split() output is used to write back to the + intermediate data file. Any escape characters in the input are + retained in the output. """ def test_invalid_input(self): @@ -877,7 +959,7 @@ class ExcapedSplit(TestCase): def test_empty_string(self): """ - Test empty strig input. + Test empty string input. :return: """ splits = escaped_split('', ':') @@ -885,39 +967,42 @@ class ExcapedSplit(TestCase): def test_no_escape(self): """ - Test with no escape character. The behaviour should be same as str.split() + Test with no escape character. The behaviour should be same as + str.split() :return: """ - s = 'yahoo:google' - splits = escaped_split(s, ':') - self.assertEqual(splits, s.split(':')) + test_str = 'yahoo:google' + splits = escaped_split(test_str, ':') + self.assertEqual(splits, test_str.split(':')) def test_escaped_input(self): """ - Test imput that has escaped delimiter. + Test input that has escaped delimiter. :return: """ - s = 'yahoo\:google:facebook' - splits = escaped_split(s, ':') - self.assertEqual(splits, ['yahoo\:google', 'facebook']) + test_str = r'yahoo\:google:facebook' + splits = escaped_split(test_str, ':') + self.assertEqual(splits, [r'yahoo\:google', 'facebook']) def test_escaped_escape(self): """ - Test imput that has escaped delimiter. + Test input that has escaped delimiter. :return: """ - s = 'yahoo\\\:google:facebook' - splits = escaped_split(s, ':') - self.assertEqual(splits, ['yahoo\\\\', 'google', 'facebook']) + test_str = r'yahoo\\\:google:facebook' + splits = escaped_split(test_str, ':') + self.assertEqual(splits, [r'yahoo\\\\', 'google', 'facebook']) def test_all_at_once(self): """ - Test imput that has escaped delimiter. + Test input that has escaped delimiter. :return: """ - s = 'yahoo\\\:google:facebook\:instagram\\\:bbc\\\\:wikipedia' - splits = escaped_split(s, ':') - self.assertEqual(splits, ['yahoo\\\\', 'google', 'facebook\:instagram\\\\', 'bbc\\\\', 'wikipedia']) + test_str = r'yahoo\\\:google:facebook\:instagram\\\:bbc\\\\:wikipedia' + splits = escaped_split(test_str, ':') + self.assertEqual(splits, [r'yahoo\\\\', r'google', + r'facebook\:instagram\\\\', + r'bbc\\\\', r'wikipedia']) class ParseTestData(TestCase): @@ -943,28 +1028,34 @@ dhm_do_dhm:10:"9345098382739712938719287391879381271":10:"9345098792137312973297 Diffie-Hellman selftest dhm_selftest: """ - s = StringIOWrapper('test_suite_ut.function', data) - tests = [(name, function, deps, args) for name, function, deps, args in parse_test_data(s)] - t1, t2, t3, t4 = tests - self.assertEqual(t1[0], 'Diffie-Hellman full exchange #1') - self.assertEqual(t1[1], 'dhm_do_dhm') - self.assertEqual(t1[2], []) - self.assertEqual(t1[3], ['10', '"23"', '10', '"5"']) + stream = StringIOWrapper('test_suite_ut.function', data) + tests = [(name, test_function, dependencies, args) + for name, test_function, dependencies, args in + parse_test_data(stream)] + test1, test2, test3, test4 = tests + self.assertEqual(test1[0], 'Diffie-Hellman full exchange #1') + self.assertEqual(test1[1], 'dhm_do_dhm') + self.assertEqual(test1[2], []) + self.assertEqual(test1[3], ['10', '"23"', '10', '"5"']) - self.assertEqual(t2[0], 'Diffie-Hellman full exchange #2') - self.assertEqual(t2[1], 'dhm_do_dhm') - self.assertEqual(t2[2], []) - self.assertEqual(t2[3], ['10', '"93450983094850938450983409623"', '10', '"9345098304850938450983409622"']) + self.assertEqual(test2[0], 'Diffie-Hellman full exchange #2') + self.assertEqual(test2[1], 'dhm_do_dhm') + self.assertEqual(test2[2], []) + self.assertEqual(test2[3], ['10', '"93450983094850938450983409623"', + '10', '"9345098304850938450983409622"']) - self.assertEqual(t3[0], 'Diffie-Hellman full exchange #3') - self.assertEqual(t3[1], 'dhm_do_dhm') - self.assertEqual(t3[2], []) - self.assertEqual(t3[3], ['10', '"9345098382739712938719287391879381271"', '10', '"9345098792137312973297123912791271"']) + self.assertEqual(test3[0], 'Diffie-Hellman full exchange #3') + self.assertEqual(test3[1], 'dhm_do_dhm') + self.assertEqual(test3[2], []) + self.assertEqual(test3[3], ['10', + '"9345098382739712938719287391879381271"', + '10', + '"9345098792137312973297123912791271"']) - self.assertEqual(t4[0], 'Diffie-Hellman selftest') - self.assertEqual(t4[1], 'dhm_selftest') - self.assertEqual(t4[2], []) - self.assertEqual(t4[3], []) + self.assertEqual(test4[0], 'Diffie-Hellman selftest') + self.assertEqual(test4[1], 'dhm_selftest') + self.assertEqual(test4[2], []) + self.assertEqual(test4[3], []) def test_with_dependencies(self): """ @@ -980,22 +1071,26 @@ Diffie-Hellman full exchange #2 dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" """ - s = StringIOWrapper('test_suite_ut.function', data) - tests = [(name, function, deps, args) for name, function, deps, args in parse_test_data(s)] - t1, t2 = tests - self.assertEqual(t1[0], 'Diffie-Hellman full exchange #1') - self.assertEqual(t1[1], 'dhm_do_dhm') - self.assertEqual(t1[2], ['YAHOO']) - self.assertEqual(t1[3], ['10', '"23"', '10', '"5"']) + stream = StringIOWrapper('test_suite_ut.function', data) + tests = [(name, function_name, dependencies, args) + for name, function_name, dependencies, args in + parse_test_data(stream)] + test1, test2 = tests + self.assertEqual(test1[0], 'Diffie-Hellman full exchange #1') + self.assertEqual(test1[1], 'dhm_do_dhm') + self.assertEqual(test1[2], ['YAHOO']) + self.assertEqual(test1[3], ['10', '"23"', '10', '"5"']) - self.assertEqual(t2[0], 'Diffie-Hellman full exchange #2') - self.assertEqual(t2[1], 'dhm_do_dhm') - self.assertEqual(t2[2], []) - self.assertEqual(t2[3], ['10', '"93450983094850938450983409623"', '10', '"9345098304850938450983409622"']) + self.assertEqual(test2[0], 'Diffie-Hellman full exchange #2') + self.assertEqual(test2[1], 'dhm_do_dhm') + self.assertEqual(test2[2], []) + self.assertEqual(test2[3], ['10', '"93450983094850938450983409623"', + '10', '"9345098304850938450983409622"']) def test_no_args(self): """ - Test GeneratorInputError is raised when test function name and args line is missing. + Test GeneratorInputError is raised when test function name and + args line is missing. :return: """ data = """ @@ -1007,37 +1102,39 @@ Diffie-Hellman full exchange #2 dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" """ - s = StringIOWrapper('test_suite_ut.function', data) - e = None + stream = StringIOWrapper('test_suite_ut.function', data) + err = None try: - for x, y, z, a in parse_test_data(s): + for _, _, _, _ in parse_test_data(stream): pass - except GeneratorInputError as e: + except GeneratorInputError as err: pass - self.assertEqual(type(e), GeneratorInputError) + self.assertEqual(type(err), GeneratorInputError) def test_incomplete_data(self): """ - Test GeneratorInputError is raised when test function name and args line is missing. + Test GeneratorInputError is raised when test function name + and args line is missing. :return: """ data = """ Diffie-Hellman full exchange #1 depends_on:YAHOO """ - s = StringIOWrapper('test_suite_ut.function', data) - e = None + stream = StringIOWrapper('test_suite_ut.function', data) + err = None try: - for x, y, z, a in parse_test_data(s): + for _, _, _, _ in parse_test_data(stream): pass - except GeneratorInputError as e: + except GeneratorInputError as err: pass - self.assertEqual(type(e), GeneratorInputError) + self.assertEqual(type(err), GeneratorInputError) class GenDepCheck(TestCase): """ - Test suite for gen_dep_check(). It is assumed this function is called with valid inputs. + Test suite for gen_dep_check(). It is assumed this function is + called with valid inputs. """ def test_gen_dep_check(self): @@ -1058,7 +1155,7 @@ class GenDepCheck(TestCase): out = gen_dep_check(5, 'YAHOO') self.assertEqual(out, expected) - def test_noT(self): + def test_not_defined_dependency(self): """ Test dependency with !. :return: @@ -1093,7 +1190,8 @@ class GenDepCheck(TestCase): class GenExpCheck(TestCase): """ - Test suite for gen_expression_check(). It is assumed this function is called with valid inputs. + Test suite for gen_expression_check(). It is assumed this function + is called with valid inputs. """ def test_gen_exp_check(self): @@ -1122,34 +1220,36 @@ class GenExpCheck(TestCase): Test invalid expression id. :return: """ - self.assertRaises(GeneratorInputError, gen_expression_check, -1, 'YAHOO') + self.assertRaises(GeneratorInputError, gen_expression_check, + -1, 'YAHOO') -class WriteDeps(TestCase): +class WriteDependencies(TestCase): """ - Test suite for testing write_deps. + Test suite for testing write_dependencies. """ - def test_no_test_deps(self): + def test_no_test_dependencies(self): """ - Test when test_deps is empty. + Test when test dependencies input is empty. :return: """ - s = StringIOWrapper('test_suite_ut.data', '') - unique_deps = [] - dep_check_code = write_deps(s, [], unique_deps) + stream = StringIOWrapper('test_suite_ut.data', '') + unique_dependencies = [] + dep_check_code = write_dependencies(stream, [], unique_dependencies) self.assertEqual(dep_check_code, '') - self.assertEqual(len(unique_deps), 0) - self.assertEqual(s.getvalue(), '') + self.assertEqual(len(unique_dependencies), 0) + self.assertEqual(stream.getvalue(), '') def test_unique_dep_ids(self): """ :return: """ - s = StringIOWrapper('test_suite_ut.data', '') - unique_deps = [] - dep_check_code = write_deps(s, ['DEP3', 'DEP2', 'DEP1'], unique_deps) + stream = StringIOWrapper('test_suite_ut.data', '') + unique_dependencies = [] + dep_check_code = write_dependencies(stream, ['DEP3', 'DEP2', 'DEP1'], + unique_dependencies) expect_dep_check_code = ''' case 0: { @@ -1179,20 +1279,23 @@ class WriteDeps(TestCase): } break;''' self.assertEqual(dep_check_code, expect_dep_check_code) - self.assertEqual(len(unique_deps), 3) - self.assertEqual(s.getvalue(), 'depends_on:0:1:2\n') + self.assertEqual(len(unique_dependencies), 3) + self.assertEqual(stream.getvalue(), 'depends_on:0:1:2\n') def test_dep_id_repeat(self): """ :return: """ - s = StringIOWrapper('test_suite_ut.data', '') - unique_deps = [] + stream = StringIOWrapper('test_suite_ut.data', '') + unique_dependencies = [] dep_check_code = '' - dep_check_code += write_deps(s, ['DEP3', 'DEP2'], unique_deps) - dep_check_code += write_deps(s, ['DEP2', 'DEP1'], unique_deps) - dep_check_code += write_deps(s, ['DEP1', 'DEP3'], unique_deps) + dep_check_code += write_dependencies(stream, ['DEP3', 'DEP2'], + unique_dependencies) + dep_check_code += write_dependencies(stream, ['DEP2', 'DEP1'], + unique_dependencies) + dep_check_code += write_dependencies(stream, ['DEP1', 'DEP3'], + unique_dependencies) expect_dep_check_code = ''' case 0: { @@ -1222,8 +1325,9 @@ class WriteDeps(TestCase): } break;''' self.assertEqual(dep_check_code, expect_dep_check_code) - self.assertEqual(len(unique_deps), 3) - self.assertEqual(s.getvalue(), 'depends_on:0:1\ndepends_on:1:2\ndepends_on:2:0\n') + self.assertEqual(len(unique_dependencies), 3) + self.assertEqual(stream.getvalue(), + 'depends_on:0:1\ndepends_on:1:2\ndepends_on:2:0\n') class WriteParams(TestCase): @@ -1236,48 +1340,57 @@ class WriteParams(TestCase): Test with empty test_args :return: """ - s = StringIOWrapper('test_suite_ut.data', '') + stream = StringIOWrapper('test_suite_ut.data', '') unique_expressions = [] - expression_code = write_parameters(s, [], [], unique_expressions) + expression_code = write_parameters(stream, [], [], unique_expressions) self.assertEqual(len(unique_expressions), 0) self.assertEqual(expression_code, '') - self.assertEqual(s.getvalue(), '\n') + self.assertEqual(stream.getvalue(), '\n') def test_no_exp_param(self): """ Test when there is no macro or expression in the params. :return: """ - s = StringIOWrapper('test_suite_ut.data', '') + stream = StringIOWrapper('test_suite_ut.data', '') unique_expressions = [] - expression_code = write_parameters(s, ['"Yahoo"', '"abcdef00"', '0'], ['char*', 'hex', 'int'], + expression_code = write_parameters(stream, ['"Yahoo"', '"abcdef00"', + '0'], + ['char*', 'hex', 'int'], unique_expressions) self.assertEqual(len(unique_expressions), 0) self.assertEqual(expression_code, '') - self.assertEqual(s.getvalue(), ':char*:"Yahoo":hex:"abcdef00":int:0\n') + self.assertEqual(stream.getvalue(), + ':char*:"Yahoo":hex:"abcdef00":int:0\n') def test_hex_format_int_param(self): """ Test int parameter in hex format. :return: """ - s = StringIOWrapper('test_suite_ut.data', '') + stream = StringIOWrapper('test_suite_ut.data', '') unique_expressions = [] - expression_code = write_parameters(s, ['"Yahoo"', '"abcdef00"', '0xAA'], ['char*', 'hex', 'int'], + expression_code = write_parameters(stream, + ['"Yahoo"', '"abcdef00"', '0xAA'], + ['char*', 'hex', 'int'], unique_expressions) self.assertEqual(len(unique_expressions), 0) self.assertEqual(expression_code, '') - self.assertEqual(s.getvalue(), ':char*:"Yahoo":hex:"abcdef00":int:0xAA\n') + self.assertEqual(stream.getvalue(), + ':char*:"Yahoo":hex:"abcdef00":int:0xAA\n') def test_with_exp_param(self): """ Test when there is macro or expression in the params. :return: """ - s = StringIOWrapper('test_suite_ut.data', '') + stream = StringIOWrapper('test_suite_ut.data', '') unique_expressions = [] - expression_code = write_parameters(s, ['"Yahoo"', '"abcdef00"', '0', 'MACRO1', 'MACRO2', 'MACRO3'], - ['char*', 'hex', 'int', 'int', 'int', 'int'], + expression_code = write_parameters(stream, + ['"Yahoo"', '"abcdef00"', '0', + 'MACRO1', 'MACRO2', 'MACRO3'], + ['char*', 'hex', 'int', + 'int', 'int', 'int'], unique_expressions) self.assertEqual(len(unique_expressions), 3) self.assertEqual(unique_expressions, ['MACRO1', 'MACRO2', 'MACRO3']) @@ -1298,21 +1411,29 @@ class WriteParams(TestCase): } break;''' self.assertEqual(expression_code, expected_expression_code) - self.assertEqual(s.getvalue(), ':char*:"Yahoo":hex:"abcdef00":int:0:exp:0:exp:1:exp:2\n') + self.assertEqual(stream.getvalue(), + ':char*:"Yahoo":hex:"abcdef00":int:0:exp:0:exp:1' + ':exp:2\n') - def test_with_repeate_calls(self): + def test_with_repeat_calls(self): """ Test when write_parameter() is called with same macro or expression. :return: """ - s = StringIOWrapper('test_suite_ut.data', '') + stream = StringIOWrapper('test_suite_ut.data', '') unique_expressions = [] expression_code = '' - expression_code += write_parameters(s, ['"Yahoo"', 'MACRO1', 'MACRO2'], ['char*', 'int', 'int'], + expression_code += write_parameters(stream, + ['"Yahoo"', 'MACRO1', 'MACRO2'], + ['char*', 'int', 'int'], unique_expressions) - expression_code += write_parameters(s, ['"abcdef00"', 'MACRO2', 'MACRO3'], ['hex', 'int', 'int'], + expression_code += write_parameters(stream, + ['"abcdef00"', 'MACRO2', 'MACRO3'], + ['hex', 'int', 'int'], unique_expressions) - expression_code += write_parameters(s, ['0', 'MACRO3', 'MACRO1'], ['int', 'int', 'int'], + expression_code += write_parameters(stream, + ['0', 'MACRO3', 'MACRO1'], + ['int', 'int', 'int'], unique_expressions) self.assertEqual(len(unique_expressions), 3) self.assertEqual(unique_expressions, ['MACRO1', 'MACRO2', 'MACRO3']) @@ -1337,31 +1458,34 @@ class WriteParams(TestCase): :hex:"abcdef00":exp:1:exp:2 :int:0:exp:2:exp:0 ''' - self.assertEqual(s.getvalue(), expected_data_file) + self.assertEqual(stream.getvalue(), expected_data_file) -class GenTestSuiteDepsChecks(TestCase): +class GenTestSuiteDependenciesChecks(TestCase): """ - + Test suite for testing gen_suite_dep_checks() """ - def test_empty_suite_deps(self): + def test_empty_suite_dependencies(self): """ - Test with empty suite_deps list. + Test with empty suite_dependencies list. :return: """ - dep_check_code, expression_code = gen_suite_deps_checks([], 'DEP_CHECK_CODE', 'EXPRESSION_CODE') + dep_check_code, expression_code = \ + gen_suite_dep_checks([], 'DEP_CHECK_CODE', 'EXPRESSION_CODE') self.assertEqual(dep_check_code, 'DEP_CHECK_CODE') self.assertEqual(expression_code, 'EXPRESSION_CODE') - def test_suite_deps(self): + def test_suite_dependencies(self): """ - Test with suite_deps list. + Test with suite_dependencies list. :return: """ - dep_check_code, expression_code = gen_suite_deps_checks(['SUITE_DEP'], 'DEP_CHECK_CODE', 'EXPRESSION_CODE') - exprectd_dep_check_code = ''' + dep_check_code, expression_code = \ + gen_suite_dep_checks(['SUITE_DEP'], 'DEP_CHECK_CODE', + 'EXPRESSION_CODE') + expected_dep_check_code = ''' #if defined(SUITE_DEP) DEP_CHECK_CODE #endif @@ -1371,7 +1495,7 @@ DEP_CHECK_CODE EXPRESSION_CODE #endif ''' - self.assertEqual(dep_check_code, exprectd_dep_check_code) + self.assertEqual(dep_check_code, expected_dep_check_code) self.assertEqual(expression_code, expected_expression_code) def test_no_dep_no_exp(self): @@ -1379,7 +1503,7 @@ EXPRESSION_CODE Test when there are no dependency and expression code. :return: """ - dep_check_code, expression_code = gen_suite_deps_checks([], '', '') + dep_check_code, expression_code = gen_suite_dep_checks([], '', '') self.assertEqual(dep_check_code, '') self.assertEqual(expression_code, '') @@ -1389,10 +1513,13 @@ class GenFromTestData(TestCase): Test suite for gen_from_test_data() """ - @patch("generate_test_code.write_deps") + @staticmethod + @patch("generate_test_code.write_dependencies") @patch("generate_test_code.write_parameters") - @patch("generate_test_code.gen_suite_deps_checks") - def test_intermediate_data_file(self, gen_suite_deps_checks_mock, write_parameters_mock, write_deps_mock): + @patch("generate_test_code.gen_suite_dependencies_checks") + def test_intermediate_data_file(func_mock1, + write_parameters_mock, + write_dependencies_mock): """ Test that intermediate data file is written with expected data. :return: @@ -1405,13 +1532,15 @@ func1:0 data_f = StringIOWrapper('test_suite_ut.data', data) out_data_f = StringIOWrapper('test_suite_ut.datax', '') func_info = {'test_func1': (1, ('int',))} - suite_deps = [] + suite_dependencies = [] write_parameters_mock.side_effect = write_parameters - write_deps_mock.side_effect = write_deps - gen_suite_deps_checks_mock.side_effect = gen_suite_deps_checks - gen_from_test_data(data_f, out_data_f, func_info, suite_deps) - write_deps_mock.assert_called_with(out_data_f, ['DEP1'], ['DEP1']) - write_parameters_mock.assert_called_with(out_data_f, ['0'], ('int',), []) + write_dependencies_mock.side_effect = write_dependencies + func_mock1.side_effect = gen_suite_dep_checks + gen_from_test_data(data_f, out_data_f, func_info, suite_dependencies) + write_dependencies_mock.assert_called_with(out_data_f, + ['DEP1'], ['DEP1']) + write_parameters_mock.assert_called_with(out_data_f, ['0'], + ('int',), []) expected_dep_check_code = ''' case 0: { @@ -1422,7 +1551,8 @@ func1:0 #endif } break;''' - gen_suite_deps_checks_mock.assert_called_with(suite_deps, expected_dep_check_code, '') + func_mock1.assert_called_with( + suite_dependencies, expected_dep_check_code, '') def test_function_not_found(self): """ @@ -1437,12 +1567,14 @@ func1:0 data_f = StringIOWrapper('test_suite_ut.data', data) out_data_f = StringIOWrapper('test_suite_ut.datax', '') func_info = {'test_func2': (1, ('int',))} - suite_deps = [] - self.assertRaises(GeneratorInputError, gen_from_test_data, data_f, out_data_f, func_info, suite_deps) + suite_dependencies = [] + self.assertRaises(GeneratorInputError, gen_from_test_data, + data_f, out_data_f, func_info, suite_dependencies) def test_different_func_args(self): """ - Test that AssertError is raised when no. of parameters and function args differ. + Test that AssertError is raised when no. of parameters and + function args differ. :return: """ data = ''' @@ -1452,9 +1584,10 @@ func1:0 ''' data_f = StringIOWrapper('test_suite_ut.data', data) out_data_f = StringIOWrapper('test_suite_ut.datax', '') - func_info = {'test_func2': (1, ('int','hex'))} - suite_deps = [] - self.assertRaises(GeneratorInputError, gen_from_test_data, data_f, out_data_f, func_info, suite_deps) + func_info = {'test_func2': (1, ('int', 'hex'))} + suite_dependencies = [] + self.assertRaises(GeneratorInputError, gen_from_test_data, data_f, + out_data_f, func_info, suite_dependencies) def test_output(self): """ @@ -1472,9 +1605,12 @@ func2:"yahoo":88:MACRO1 ''' data_f = StringIOWrapper('test_suite_ut.data', data) out_data_f = StringIOWrapper('test_suite_ut.datax', '') - func_info = {'test_func1': (0, ('int', 'int', 'int', 'int')), 'test_func2': (1, ('char*', 'int', 'int'))} - suite_deps = [] - dep_check_code, expression_code = gen_from_test_data(data_f, out_data_f, func_info, suite_deps) + func_info = {'test_func1': (0, ('int', 'int', 'int', 'int')), + 'test_func2': (1, ('char*', 'int', 'int'))} + suite_dependencies = [] + dep_check_code, expression_code = \ + gen_from_test_data(data_f, out_data_f, func_info, + suite_dependencies) expected_dep_check_code = ''' case 0: { @@ -1494,7 +1630,7 @@ func2:"yahoo":88:MACRO1 #endif } break;''' - expecrted_data = '''My test 1 + expected_data = '''My test 1 depends_on:0 0:int:0:int:0xfa:exp:0:exp:1 @@ -1515,9 +1651,9 @@ depends_on:0:1 } break;''' self.assertEqual(dep_check_code, expected_dep_check_code) - self.assertEqual(out_data_f.getvalue(), expecrted_data) + self.assertEqual(out_data_f.getvalue(), expected_data) self.assertEqual(expression_code, expected_expression_code) -if __name__=='__main__': +if __name__ == '__main__': unittest_main() From b9e4cdd5f78a73c9a7250d18b9ab5a82f7904fe0 Mon Sep 17 00:00:00 2001 From: Azim Khan Date: Wed, 4 Jul 2018 23:29:46 +0100 Subject: [PATCH 017/490] Incorporated code revoew comments. --- scripts/generate_test_code.py | 131 +++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 35 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index a28a73669..036ed1c02 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Test suites code generator. # -# Copyright (C) 2018, ARM Limited, All Rights Reserved +# Copyright (C) 2018, Arm Limited, All Rights Reserved # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -180,9 +180,19 @@ END_SUITE_HELPERS_REGEX = r'/\*\s*END_SUITE_HELPERS\s*\*/' BEGIN_DEP_REGEX = r'BEGIN_DEPENDENCIES' END_DEP_REGEX = r'END_DEPENDENCIES' -BEGIN_CASE_REGEX = r'/\*\s*BEGIN_CASE\s*(.*?)\s*\*/' +BEGIN_CASE_REGEX = r'/\*\s*BEGIN_CASE\s*(?P.*?)\s*\*/' END_CASE_REGEX = r'/\*\s*END_CASE\s*\*/' +DEPENDENCY_REGEX = r'depends_on:(?P.*)' +C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*' +TEST_FUNCTION_VALIDATION_REGEX = r'\s*void\s+(\w+)\s*\(' +INT_CHECK_REGEX = r'int\s+.*' +CHAR_CHECK_REGEX = r'char\s*\*\s*.*' +DATA_T_CHECK_REGEX = r'data_t\s*\*\s*.*' +FUNCTION_ARG_LIST_START_REGEX = r'.*?\s+(\w+)\s*\(' +FUNCTION_ARG_LIST_END_REGEX = r'.*\)' +EXIT_LABEL_REGEX = r'^exit:' + class GeneratorInputError(Exception): """ @@ -228,7 +238,7 @@ class FileWrapper(io.FileIO, object): self._line_no += 1 # Convert byte array to string with correct encoding and # strip any whitespaces added in the decoding process. - return line.decode(sys.getdefaultencoding()).strip() + "\n" + return line.decode(sys.getdefaultencoding()).rstrip() + '\n' return None # Python 3 iterator method @@ -351,7 +361,7 @@ def parse_until_pattern(funcs_f, end_regex): Matches pattern end_regex to the lines read from the file object. Returns the lines read until end pattern is matched. - :param funcs_f: file object for .functions file + :param funcs_f: file object for .function file :param end_regex: Pattern to stop parsing :return: Lines read before the end pattern """ @@ -367,6 +377,31 @@ def parse_until_pattern(funcs_f, end_regex): return headers +def validate_dependency(dependency): + """ + Validates a C macro and raises GeneratorInputError on invalid input. + :param dependency: Input macro dependency + :return: input dependency stripped of leading & trailing white spaces. + """ + dependency = dependency.strip() + if not re.match(C_IDENTIFIER_REGEX, dependency, re.I): + raise GeneratorInputError('Invalid dependency %s' % dependency) + return dependency + + +def parse_dependencies(inp_str): + """ + Parses dependencies out of inp_str, validates them and returns a + list of macros. + + :param inp_str: Input string with macros delimited by ':'. + :return: list of dependencies + """ + dependencies = [dep for dep in map(validate_dependency, + inp_str.split(':'))] + return dependencies + + def parse_suite_dependencies(funcs_f): """ Parses test suite dependencies specified at the top of a @@ -374,14 +409,18 @@ def parse_suite_dependencies(funcs_f): and end with END_DEPENDENCIES. Dependencies are specified after pattern 'depends_on:' and are delimited by ':'. - :param funcs_f: file object for .functions file + :param funcs_f: file object for .function file :return: List of test suite dependencies. """ dependencies = [] for line in funcs_f: - match = re.search('depends_on:(.*)', line.strip()) + match = re.search(DEPENDENCY_REGEX, line.strip()) if match: - dependencies += [x.strip() for x in match.group(1).split(':')] + try: + dependencies = parse_dependencies(match.group('dependencies')) + except GeneratorInputError as error: + raise GeneratorInputError( + str(error) + " - %s:%d" % (funcs_f.name, funcs_f.line_no)) if re.search(END_DEP_REGEX, line): break else: @@ -398,19 +437,18 @@ def parse_function_dependencies(line): comment BEGIN_CASE. Dependencies are specified after pattern 'depends_on:' and are delimited by ':'. - :param line: Line from .functions file that has dependencies. + :param line: Line from .function file that has dependencies. :return: List of dependencies. """ dependencies = [] match = re.search(BEGIN_CASE_REGEX, line) - dep_str = match.group(1) + dep_str = match.group('depends_on') if dep_str: - match = re.search('depends_on:(.*)', dep_str) + match = re.search(DEPENDENCY_REGEX, dep_str) if match: - dependencies = [x.strip() - for x in match.group(1).strip().split(':')] - return dependencies + dependencies += parse_dependencies(match.group('dependencies')) + return dependencies def parse_function_signature(line): """ @@ -418,7 +456,7 @@ def parse_function_signature(line): a dispatch wrapper function that translates input test vectors read from the data file into test function arguments. - :param line: Line from .functions file that has a function + :param line: Line from .function file that has a function signature. :return: function name, argument list, local variables for wrapper function and argument dispatch code. @@ -427,23 +465,27 @@ def parse_function_signature(line): local_vars = '' args_dispatch = [] # Check if the test function returns void. - match = re.search(r'\s*void\s+(\w+)\s*\(', line, re.I) + match = re.search(TEST_FUNCTION_VALIDATION_REGEX, line, re.I) if not match: raise ValueError("Test function should return 'void'\n%s" % line) name = match.group(1) line = line[len(match.group(0)):] arg_idx = 0 + # Process arguments, ex: arg1, arg2 ) + # This script assumes that the argument list is terminated by ')' + # i.e. the test functions will not have a function pointer + # argument. for arg in line[:line.find(')')].split(','): arg = arg.strip() if arg == '': continue - if re.search(r'int\s+.*', arg.strip()): + if re.search(INT_CHECK_REGEX, arg.strip()): args.append('int') args_dispatch.append('*( (int *) params[%d] )' % arg_idx) - elif re.search(r'char\s*\*\s*.*', arg.strip()): + elif re.search(CHAR_CHECK_REGEX, arg.strip()): args.append('char*') args_dispatch.append('(char *) params[%d]' % arg_idx) - elif re.search(r'data_t\s*\*\s*.*', arg.strip()): + elif re.search(DATA_T_CHECK_REGEX, arg.strip()): args.append('hex') # create a structure pointer_initializer = '(uint8_t *) params[%d]' % arg_idx @@ -472,21 +514,25 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): :return: Function name, arguments, function code and dispatch code. """ code = '#line %d "%s"\n' % (funcs_f.line_no + 1, funcs_f.name) + has_exit_label = False for line in funcs_f: - # Check function signature - match = re.match(r'.*?\s+(\w+)\s*\(', line, re.I) + # Check function signature. This script expects function name + # and return type to be specified at the same line. + match = re.match(FUNCTION_ARG_LIST_START_REGEX, line, re.I) if match: # check if we have full signature i.e. split in more lines - if not re.match(r'.*\)', line): + if not re.match(FUNCTION_ARG_LIST_END_REGEX, line): for lin in funcs_f: line += lin - if re.search(r'.*?\)', line): + if re.search(FUNCTION_ARG_LIST_END_REGEX, line): break name, args, local_vars, args_dispatch = parse_function_signature( line) - code += line.replace(name, 'test_' + name) + code += line.replace(name, 'test_' + name, 1) name = 'test_' + name break + else: + code += line else: raise GeneratorInputError("file: %s - Test functions not found!" % funcs_f.name) @@ -494,6 +540,9 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): for line in funcs_f: if re.search(END_CASE_REGEX, line): break + if not has_exit_label: + has_exit_label = \ + re.search(EXIT_LABEL_REGEX, line.strip()) is not None code += line else: raise GeneratorInputError("file: %s - end case pattern [%s] not " @@ -504,7 +553,7 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): split_code = code.rsplit('}', 1) if len(split_code) == 2: code = """exit: - ;; + ; }""".join(split_code) code += gen_function_wrapper(name, local_vars, args_dispatch) @@ -541,7 +590,12 @@ def parse_functions(funcs_f): elif re.search(BEGIN_DEP_REGEX, line): suite_dependencies += parse_suite_dependencies(funcs_f) elif re.search(BEGIN_CASE_REGEX, line): - dependencies = parse_function_dependencies(line) + try: + dependencies = parse_function_dependencies(line) + except GeneratorInputError as error: + raise GeneratorInputError( + "%s:%d: %s" % (funcs_f.name, funcs_f.line_no, + str(error))) func_name, args, func_code, func_dispatch =\ parse_function_code(funcs_f, dependencies, suite_dependencies) suite_functions += func_code @@ -568,7 +622,7 @@ def escaped_split(inp_str, split_char): output. :param inp_str: String to split - :param split_char: split character + :param split_char: Split character :return: List of splits """ if len(split_char) > 1: @@ -609,7 +663,8 @@ def parse_test_data(data_f): name = '' for line in data_f: line = line.strip() - if line and line[0] == '#': # Skip comments + # Skip comments + if line.startswith('#'): continue # Blank line indicates end of test @@ -627,10 +682,15 @@ def parse_test_data(data_f): state = __state_read_args elif state == __state_read_args: # Check dependencies - match = re.search('depends_on:(.*)', line) + match = re.search(DEPENDENCY_REGEX, line) if match: - dependencies = [x.strip() for x in match.group(1).split(':') - if len(x.strip())] + try: + dependencies = parse_dependencies( + match.group('dependencies')) + except GeneratorInputError as error: + raise GeneratorInputError( + str(error) + " - %s:%d" % + (data_f.name, data_f.line_no)) else: # Read test vectors parts = escaped_split(line, ':') @@ -742,7 +802,8 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): val = test_args[i] # check if val is a non literal int val (i.e. an expression) - if typ == 'int' and not re.match(r'(\d+$)|((0x)?[0-9a-fA-F]+$)', val): + if typ == 'int' and not re.match(r'(\d+|0x[0-9a-f]+)$', + val, re.I): typ = 'exp' if val not in unique_expressions: unique_expressions.append(val) @@ -764,7 +825,7 @@ def gen_suite_dep_checks(suite_dependencies, dep_check_code, expression_code): Generates preprocessor checks for test suite dependencies. :param suite_dependencies: Test suite dependencies read from the - .functions file. + .function file. :param dep_check_code: Dependency check code :param expression_code: Expression check code :return: Dependency and expression code guarded by test suite @@ -797,7 +858,7 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_dependencies): evaluation code. :param data_f: Data file object - :param out_data_f:Output intermediate data file + :param out_data_f: Output intermediate data file :param func_info: Dict keyed by function and with function id and arguments info :param suite_dependencies: Test suite dependencies @@ -983,7 +1044,7 @@ def generate_code(**input_info): write_test_source_file(template_file, c_file, snippets) -def check_cmd(): +def main(): """ Command line parser. @@ -1057,7 +1118,7 @@ def check_cmd(): if __name__ == "__main__": try: - check_cmd() + main() except GeneratorInputError as err: print("%s: input error: %s" % (os.path.basename(sys.argv[0]), str(err))) From f1416334fcdd41cbf525f16d40828d446e8089e3 Mon Sep 17 00:00:00 2001 From: Azim Khan Date: Thu, 5 Jul 2018 14:20:08 +0100 Subject: [PATCH 018/490] Fixed unit tests in test_generate_test_code.py --- scripts/generate_test_code.py | 1 + scripts/test_generate_test_code.py | 61 ++++++++++++++---------------- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 036ed1c02..b744d7c07 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -450,6 +450,7 @@ def parse_function_dependencies(line): return dependencies + def parse_function_signature(line): """ Parses test function signature for validation and generates diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index f0a935d20..29d9e4f44 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Unit test for generate_test_code.py # -# Copyright (C) 2018, ARM Limited, All Rights Reserved +# Copyright (C) 2018, Arm Limited, All Rights Reserved # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -23,7 +23,6 @@ Unit tests for generate_test_code.py """ -import sys from StringIO import StringIO from unittest import TestCase, main as unittest_main from mock import patch @@ -288,7 +287,7 @@ class StringIOWrapper(StringIO, object): """ file like class to mock file object in tests. """ - def __init__(self, file_name, data, line_no=1): + def __init__(self, file_name, data, line_no=0): """ Init file handle. @@ -308,14 +307,8 @@ class StringIOWrapper(StringIO, object): :return: Line read from file. """ - parent = super(StringIOWrapper, self) - line = parent.next() # Python 2 - if line: - self.line_no += 1 - # Convert byte array to string with correct encoding and - # strip any whitespaces added in the decoding process. - return line.decode(sys.getdefaultencoding()).strip() + "\n" - return None + line = super(StringIOWrapper, self).next() + return line __next__ = next @@ -327,7 +320,7 @@ class StringIOWrapper(StringIO, object): :return: """ line = super(StringIOWrapper, self).readline() - if line: + if line is not None: self.line_no += 1 return line @@ -510,10 +503,10 @@ class ParseFuncSignature(TestCase): self.assertEqual(name, 'entropy_threshold') self.assertEqual(args, ['char*', 'hex', 'int']) self.assertEqual(local, - ' data_t hex1 = {(uint8_t *) params[1], ' + ' data_t data1 = {(uint8_t *) params[1], ' '*( (uint32_t *) params[2] )};\n') self.assertEqual(arg_dispatch, ['(char *) params[0]', - '&hex1', + '&data1', '*( (int *) params[3] )']) def test_non_void_function(self): @@ -629,13 +622,14 @@ void func() gen_function_wrapper_mock.assert_called_with('test_func', '', []) self.assertEqual(name, 'test_func') self.assertEqual(arg, []) - expected = '''#line 2 "test_suite_ut.function" + expected = '''#line 1 "test_suite_ut.function" + void test_func() { ba ba black sheep have you any wool exit: - ;; + ; } ''' self.assertEqual(code, expected) @@ -671,7 +665,8 @@ exit: stream = StringIOWrapper('test_suite_ut.function', data) _, _, code, _ = parse_function_code(stream, [], []) - expected = '''#line 2 "test_suite_ut.function" + expected = '''#line 1 "test_suite_ut.function" + void test_func() { ba ba black sheep @@ -708,7 +703,7 @@ class ParseFunction(TestCase): stream = StringIOWrapper('test_suite_ut.function', data) self.assertRaises(Exception, parse_functions, stream) parse_until_pattern_mock.assert_called_with(stream, END_HEADER_REGEX) - self.assertEqual(stream.line_no, 2) + self.assertEqual(stream.line_no, 1) @patch("generate_test_code.parse_until_pattern") def test_begin_helper(self, parse_until_pattern_mock): @@ -731,7 +726,7 @@ void print_hello_world() self.assertRaises(Exception, parse_functions, stream) parse_until_pattern_mock.assert_called_with(stream, END_SUITE_HELPERS_REGEX) - self.assertEqual(stream.line_no, 2) + self.assertEqual(stream.line_no, 1) @patch("generate_test_code.parse_suite_dependencies") def test_begin_dep(self, parse_suite_dependencies_mock): @@ -752,7 +747,7 @@ void print_hello_world() stream = StringIOWrapper('test_suite_ut.function', data) self.assertRaises(Exception, parse_functions, stream) parse_suite_dependencies_mock.assert_called_with(stream) - self.assertEqual(stream.line_no, 2) + self.assertEqual(stream.line_no, 1) @patch("generate_test_code.parse_function_dependencies") def test_begin_function_dep(self, func_mock): @@ -775,7 +770,7 @@ void print_hello_world() stream = StringIOWrapper('test_suite_ut.function', data) self.assertRaises(Exception, parse_functions, stream) func_mock.assert_called_with(dependencies_str) - self.assertEqual(stream.line_no, 2) + self.assertEqual(stream.line_no, 1) @patch("generate_test_code.parse_function_code") @patch("generate_test_code.parse_function_dependencies") @@ -866,17 +861,17 @@ void func2() ''' self.assertEqual(dispatch_code, expected_dispatch_code) expected_func_code = '''#if defined(MBEDTLS_ECP_C) -#line 3 "test_suite_ut.function" +#line 2 "test_suite_ut.function" #include "mbedtls/ecp.h" #define ECP_PF_UNKNOWN -1 #if defined(MBEDTLS_ENTROPY_NV_SEED) #if defined(MBEDTLS_FS_IO) -#line 14 "test_suite_ut.function" +#line 13 "test_suite_ut.function" void test_func1() { exit: - ;; + ; } void test_func1_wrapper( void ** params ) @@ -889,11 +884,11 @@ void test_func1_wrapper( void ** params ) #endif /* MBEDTLS_ENTROPY_NV_SEED */ #if defined(MBEDTLS_ENTROPY_NV_SEED) #if defined(MBEDTLS_FS_IO) -#line 20 "test_suite_ut.function" +#line 19 "test_suite_ut.function" void test_func2() { exit: - ;; + ; } void test_func2_wrapper( void ** params ) @@ -989,20 +984,20 @@ class EscapedSplit(TestCase): Test input that has escaped delimiter. :return: """ - test_str = r'yahoo\\\:google:facebook' + test_str = r'yahoo\\:google:facebook' splits = escaped_split(test_str, ':') - self.assertEqual(splits, [r'yahoo\\\\', 'google', 'facebook']) + self.assertEqual(splits, [r'yahoo\\', 'google', 'facebook']) def test_all_at_once(self): """ Test input that has escaped delimiter. :return: """ - test_str = r'yahoo\\\:google:facebook\:instagram\\\:bbc\\\\:wikipedia' + test_str = r'yahoo\\:google:facebook\:instagram\\:bbc\\:wikipedia' splits = escaped_split(test_str, ':') - self.assertEqual(splits, [r'yahoo\\\\', r'google', - r'facebook\:instagram\\\\', - r'bbc\\\\', r'wikipedia']) + self.assertEqual(splits, [r'yahoo\\', r'google', + r'facebook\:instagram\\', + r'bbc\\', r'wikipedia']) class ParseTestData(TestCase): @@ -1516,7 +1511,7 @@ class GenFromTestData(TestCase): @staticmethod @patch("generate_test_code.write_dependencies") @patch("generate_test_code.write_parameters") - @patch("generate_test_code.gen_suite_dependencies_checks") + @patch("generate_test_code.gen_suite_dep_checks") def test_intermediate_data_file(func_mock1, write_parameters_mock, write_dependencies_mock): From 3769c2dd0f4575cd83b74ff6160d8f855be19148 Mon Sep 17 00:00:00 2001 From: Azim Khan Date: Thu, 5 Jul 2018 17:31:46 +0100 Subject: [PATCH 019/490] Make test function parsing robust This commit enhances parsing of the test function in generate_test_code.py for cases where return type and function name are on separate lines. --- scripts/generate_test_code.py | 46 ++++++----- scripts/test_generate_test_code.py | 122 ++++++++++++++++++++--------- 2 files changed, 112 insertions(+), 56 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index b744d7c07..b01bd3511 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -185,11 +185,10 @@ END_CASE_REGEX = r'/\*\s*END_CASE\s*\*/' DEPENDENCY_REGEX = r'depends_on:(?P.*)' C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*' -TEST_FUNCTION_VALIDATION_REGEX = r'\s*void\s+(\w+)\s*\(' +TEST_FUNCTION_VALIDATION_REGEX = r'\s*void\s+(?P\w+)\s*\(' INT_CHECK_REGEX = r'int\s+.*' CHAR_CHECK_REGEX = r'char\s*\*\s*.*' DATA_T_CHECK_REGEX = r'data_t\s*\*\s*.*' -FUNCTION_ARG_LIST_START_REGEX = r'.*?\s+(\w+)\s*\(' FUNCTION_ARG_LIST_END_REGEX = r'.*\)' EXIT_LABEL_REGEX = r'^exit:' @@ -451,7 +450,7 @@ def parse_function_dependencies(line): return dependencies -def parse_function_signature(line): +def parse_function_arguments(line): """ Parses test function signature for validation and generates a dispatch wrapper function that translates input test vectors @@ -459,19 +458,15 @@ def parse_function_signature(line): :param line: Line from .function file that has a function signature. - :return: function name, argument list, local variables for + :return: argument list, local variables for wrapper function and argument dispatch code. """ args = [] local_vars = '' args_dispatch = [] - # Check if the test function returns void. - match = re.search(TEST_FUNCTION_VALIDATION_REGEX, line, re.I) - if not match: - raise ValueError("Test function should return 'void'\n%s" % line) - name = match.group(1) - line = line[len(match.group(0)):] arg_idx = 0 + # Remove characters before arguments + line = line[line.find('(') + 1:] # Process arguments, ex: arg1, arg2 ) # This script assumes that the argument list is terminated by ')' # i.e. the test functions will not have a function pointer @@ -501,7 +496,7 @@ def parse_function_signature(line): "'char *' or 'data_t'\n%s" % line) arg_idx += 1 - return name, args, local_vars, args_dispatch + return args, local_vars, args_dispatch def parse_function_code(funcs_f, dependencies, suite_dependencies): @@ -514,30 +509,38 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): :param suite_dependencies: List of test suite dependencies :return: Function name, arguments, function code and dispatch code. """ - code = '#line %d "%s"\n' % (funcs_f.line_no + 1, funcs_f.name) + line_directive = '#line %d "%s"\n' % (funcs_f.line_no + 1, funcs_f.name) + code = '' has_exit_label = False for line in funcs_f: - # Check function signature. This script expects function name - # and return type to be specified at the same line. - match = re.match(FUNCTION_ARG_LIST_START_REGEX, line, re.I) + # Check function signature. Function signature may be split + # across multiple lines. Here we try to find the start of + # arguments list, then remove '\n's and apply the regex to + # detect function start. + up_to_arg_list_start = code + line[:line.find('(') + 1] + match = re.match(TEST_FUNCTION_VALIDATION_REGEX, + up_to_arg_list_start.replace('\n', ' '), re.I) if match: # check if we have full signature i.e. split in more lines + name = match.group('func_name') if not re.match(FUNCTION_ARG_LIST_END_REGEX, line): for lin in funcs_f: line += lin if re.search(FUNCTION_ARG_LIST_END_REGEX, line): break - name, args, local_vars, args_dispatch = parse_function_signature( + args, local_vars, args_dispatch = parse_function_arguments( line) - code += line.replace(name, 'test_' + name, 1) - name = 'test_' + name - break - else: code += line + break + code += line else: raise GeneratorInputError("file: %s - Test functions not found!" % funcs_f.name) + # Prefix test function name with 'test_' + code = code.replace(name, 'test_' + name, 1) + name = 'test_' + name + for line in funcs_f: if re.search(END_CASE_REGEX, line): break @@ -557,7 +560,8 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): ; }""".join(split_code) - code += gen_function_wrapper(name, local_vars, args_dispatch) + code = line_directive + code + gen_function_wrapper(name, local_vars, + args_dispatch) preprocessor_check_start, preprocessor_check_end = \ gen_dependencies(dependencies) dispatch_code = gen_dispatch(name, suite_dependencies + dependencies) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 29d9e4f44..149159c8c 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -31,7 +31,7 @@ from generate_test_code import gen_function_wrapper, gen_dispatch from generate_test_code import parse_until_pattern, GeneratorInputError from generate_test_code import parse_suite_dependencies from generate_test_code import parse_function_dependencies -from generate_test_code import parse_function_signature, parse_function_code +from generate_test_code import parse_function_arguments, parse_function_code from generate_test_code import parse_functions, END_HEADER_REGEX from generate_test_code import END_SUITE_HELPERS_REGEX, escaped_split from generate_test_code import parse_test_data, gen_dep_check @@ -476,7 +476,7 @@ class ParseFuncDependencies(TestCase): class ParseFuncSignature(TestCase): """ - Test Suite for parse_function_signature(). + Test Suite for parse_function_arguments(). """ def test_int_and_char_params(self): @@ -485,8 +485,7 @@ class ParseFuncSignature(TestCase): :return: """ line = 'void entropy_threshold( char * a, int b, int result )' - name, args, local, arg_dispatch = parse_function_signature(line) - self.assertEqual(name, 'entropy_threshold') + args, local, arg_dispatch = parse_function_arguments(line) self.assertEqual(args, ['char*', 'int', 'int']) self.assertEqual(local, '') self.assertEqual(arg_dispatch, ['(char *) params[0]', @@ -499,8 +498,7 @@ class ParseFuncSignature(TestCase): :return: """ line = 'void entropy_threshold( char * a, data_t * h, int result )' - name, args, local, arg_dispatch = parse_function_signature(line) - self.assertEqual(name, 'entropy_threshold') + args, local, arg_dispatch = parse_function_arguments(line) self.assertEqual(args, ['char*', 'hex', 'int']) self.assertEqual(local, ' data_t data1 = {(uint8_t *) params[1], ' @@ -509,21 +507,13 @@ class ParseFuncSignature(TestCase): '&data1', '*( (int *) params[3] )']) - def test_non_void_function(self): - """ - Test invalid signature (non void). - :return: - """ - line = 'int entropy_threshold( char * a, data_t * h, int result )' - self.assertRaises(ValueError, parse_function_signature, line) - def test_unsupported_arg(self): """ Test unsupported arguments (not among int, char * and data_t) :return: """ - line = 'int entropy_threshold( char * a, data_t * h, int * result )' - self.assertRaises(ValueError, parse_function_signature, line) + line = 'void entropy_threshold( char * a, data_t * h, char result )' + self.assertRaises(ValueError, parse_function_arguments, line) def test_no_params(self): """ @@ -531,8 +521,7 @@ class ParseFuncSignature(TestCase): :return: """ line = 'void entropy_threshold()' - name, args, local, arg_dispatch = parse_function_signature(line) - self.assertEqual(name, 'entropy_threshold') + args, local, arg_dispatch = parse_function_arguments(line) self.assertEqual(args, []) self.assertEqual(local, '') self.assertEqual(arg_dispatch, []) @@ -554,8 +543,9 @@ test function ''' stream = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(GeneratorInputError, parse_function_code, stream, [], - []) + err_msg = 'file: test_suite_ut.function - Test functions not found!' + self.assertRaisesRegexp(GeneratorInputError, err_msg, + parse_function_code, stream, [], []) def test_no_end_case_comment(self): """ @@ -568,17 +558,19 @@ void test_func() } ''' stream = StringIOWrapper('test_suite_ut.function', data) - self.assertRaises(GeneratorInputError, parse_function_code, stream, [], - []) + err_msg = r'file: test_suite_ut.function - '\ + 'end case pattern .*? not found!' + self.assertRaisesRegexp(GeneratorInputError, err_msg, + parse_function_code, stream, [], []) - @patch("generate_test_code.parse_function_signature") + @patch("generate_test_code.parse_function_arguments") def test_function_called(self, - parse_function_signature_mock): + parse_function_arguments_mock): """ Test parse_function_code() :return: """ - parse_function_signature_mock.return_value = ('test_func', [], '', []) + parse_function_arguments_mock.return_value = ([], '', []) data = ''' void test_func() { @@ -587,14 +579,14 @@ void test_func() stream = StringIOWrapper('test_suite_ut.function', data) self.assertRaises(GeneratorInputError, parse_function_code, stream, [], []) - self.assertTrue(parse_function_signature_mock.called) - parse_function_signature_mock.assert_called_with('void test_func()\n') + self.assertTrue(parse_function_arguments_mock.called) + parse_function_arguments_mock.assert_called_with('void test_func()\n') @patch("generate_test_code.gen_dispatch") @patch("generate_test_code.gen_dependencies") @patch("generate_test_code.gen_function_wrapper") - @patch("generate_test_code.parse_function_signature") - def test_return(self, parse_function_signature_mock, + @patch("generate_test_code.parse_function_arguments") + def test_return(self, parse_function_arguments_mock, gen_function_wrapper_mock, gen_dependencies_mock, gen_dispatch_mock): @@ -602,7 +594,7 @@ void test_func() Test generated code. :return: """ - parse_function_signature_mock.return_value = ('func', [], '', []) + parse_function_arguments_mock.return_value = ([], '', []) gen_function_wrapper_mock.return_value = '' gen_dependencies_mock.side_effect = gen_dependencies gen_dispatch_mock.side_effect = gen_dispatch @@ -617,8 +609,8 @@ void func() stream = StringIOWrapper('test_suite_ut.function', data) name, arg, code, dispatch_code = parse_function_code(stream, [], []) - self.assertTrue(parse_function_signature_mock.called) - parse_function_signature_mock.assert_called_with('void func()\n') + self.assertTrue(parse_function_arguments_mock.called) + parse_function_arguments_mock.assert_called_with('void func()\n') gen_function_wrapper_mock.assert_called_with('test_func', '', []) self.assertEqual(name, 'test_func') self.assertEqual(arg, []) @@ -638,8 +630,8 @@ exit: @patch("generate_test_code.gen_dispatch") @patch("generate_test_code.gen_dependencies") @patch("generate_test_code.gen_function_wrapper") - @patch("generate_test_code.parse_function_signature") - def test_with_exit_label(self, parse_function_signature_mock, + @patch("generate_test_code.parse_function_arguments") + def test_with_exit_label(self, parse_function_arguments_mock, gen_function_wrapper_mock, gen_dependencies_mock, gen_dispatch_mock): @@ -647,7 +639,7 @@ exit: Test when exit label is present. :return: """ - parse_function_signature_mock.return_value = ('func', [], '', []) + parse_function_arguments_mock.return_value = ([], '', []) gen_function_wrapper_mock.return_value = '' gen_dependencies_mock.side_effect = gen_dependencies gen_dispatch_mock.side_effect = gen_dispatch @@ -675,6 +667,66 @@ exit: yes sir yes sir 3 bags full } +''' + self.assertEqual(code, expected) + + def test_non_void_function(self): + """ + Test invalid signature (non void). + :return: + """ + data = 'int entropy_threshold( char * a, data_t * h, int result )' + err_msg = 'file: test_suite_ut.function - Test functions not found!' + stream = StringIOWrapper('test_suite_ut.function', data) + self.assertRaisesRegexp(GeneratorInputError, err_msg, + parse_function_code, stream, [], []) + + @patch("generate_test_code.gen_dispatch") + @patch("generate_test_code.gen_dependencies") + @patch("generate_test_code.gen_function_wrapper") + @patch("generate_test_code.parse_function_arguments") + def test_functio_name_on_newline(self, parse_function_arguments_mock, + gen_function_wrapper_mock, + gen_dependencies_mock, + gen_dispatch_mock): + """ + Test when exit label is present. + :return: + """ + parse_function_arguments_mock.return_value = ([], '', []) + gen_function_wrapper_mock.return_value = '' + gen_dependencies_mock.side_effect = gen_dependencies + gen_dispatch_mock.side_effect = gen_dispatch + data = ''' +void + + +func() +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} +/* END_CASE */ +''' + stream = StringIOWrapper('test_suite_ut.function', data) + _, _, code, _ = parse_function_code(stream, [], []) + + expected = '''#line 1 "test_suite_ut.function" + +void + + +test_func() +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} ''' self.assertEqual(code, expected) From 12347463e9cd33fb4f7334dbb25a3c17eb6edab4 Mon Sep 17 00:00:00 2001 From: Azim Khan Date: Thu, 5 Jul 2018 17:53:11 +0100 Subject: [PATCH 020/490] Replaced escaped_split() logic with regex --- scripts/generate_test_code.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index b01bd3511..ece35dfb4 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -632,18 +632,10 @@ def escaped_split(inp_str, split_char): """ if len(split_char) > 1: raise ValueError('Expected split character. Found string!') - out = [] - part = '' - escape = False - for character in inp_str: - if not escape and character == split_char: - out.append(part) - part = '' - else: - part += character - escape = not escape and character == '\\' - if part: - out.append(part) + out = re.sub(r'(\\.)|' + split_char, + lambda m: m.group(1) or '\n', inp_str, + len(inp_str)).split('\n') + out = filter(lambda x: x or False, out) return out From 804a0e22379a630b0461a6a4baec55b659837143 Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Fri, 6 Jul 2018 00:29:09 +0100 Subject: [PATCH 021/490] Fix Pylint errors in Python scripts --- scripts/generate_test_code.py | 54 +++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index ece35dfb4..2468063d1 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -499,6 +499,33 @@ def parse_function_arguments(line): return args, local_vars, args_dispatch +def generate_function_code(name, code, local_vars, args_dispatch, + dependencies): + """ + Generate function code with preprocessor checks and parameter dispatch + wrapper. + + :param name: Function name + :param code: Function code + :param local_vars: Local variables for function wrapper + :param args_dispatch: Argument dispatch code + :param dependencies: Preprocessor dependencies list + :return: Final function code + """ + # Add exit label if not present + if code.find('exit:') == -1: + split_code = code.rsplit('}', 1) + if len(split_code) == 2: + code = """exit: + ; +}""".join(split_code) + + code += gen_function_wrapper(name, local_vars, args_dispatch) + preprocessor_check_start, preprocessor_check_end = \ + gen_dependencies(dependencies) + return preprocessor_check_start + code + preprocessor_check_end + + def parse_function_code(funcs_f, dependencies, suite_dependencies): """ Parses out a function from function file object and generates @@ -552,21 +579,11 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): raise GeneratorInputError("file: %s - end case pattern [%s] not " "found!" % (funcs_f.name, END_CASE_REGEX)) - # Add exit label if not present - if code.find('exit:') == -1: - split_code = code.rsplit('}', 1) - if len(split_code) == 2: - code = """exit: - ; -}""".join(split_code) - - code = line_directive + code + gen_function_wrapper(name, local_vars, - args_dispatch) - preprocessor_check_start, preprocessor_check_end = \ - gen_dependencies(dependencies) + code = line_directive + code + code = generate_function_code(name, code, local_vars, args_dispatch, + dependencies) dispatch_code = gen_dispatch(name, suite_dependencies + dependencies) - return (name, args, preprocessor_check_start + code + - preprocessor_check_end, dispatch_code) + return (name, args, code, dispatch_code) def parse_functions(funcs_f): @@ -587,11 +604,10 @@ def parse_functions(funcs_f): dispatch_code = '' for line in funcs_f: if re.search(BEGIN_HEADER_REGEX, line): - headers = parse_until_pattern(funcs_f, END_HEADER_REGEX) - suite_helpers += headers + suite_helpers += parse_until_pattern(funcs_f, END_HEADER_REGEX) elif re.search(BEGIN_SUITE_HELPERS_REGEX, line): - helpers = parse_until_pattern(funcs_f, END_SUITE_HELPERS_REGEX) - suite_helpers += helpers + suite_helpers += parse_until_pattern(funcs_f, + END_SUITE_HELPERS_REGEX) elif re.search(BEGIN_DEP_REGEX, line): suite_dependencies += parse_suite_dependencies(funcs_f) elif re.search(BEGIN_CASE_REGEX, line): @@ -635,7 +651,7 @@ def escaped_split(inp_str, split_char): out = re.sub(r'(\\.)|' + split_char, lambda m: m.group(1) or '\n', inp_str, len(inp_str)).split('\n') - out = filter(lambda x: x or False, out) + out = [x for x in out if x] return out From 777514380fbdaf919ddc040496dcd4c41a450142 Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Fri, 6 Jul 2018 00:29:50 +0100 Subject: [PATCH 022/490] Fix Python 2 & 3 compatibility in test_generate_test_code.py --- scripts/test_generate_test_code.py | 73 ++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 13 deletions(-) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 149159c8c..2ef12e18d 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -23,9 +23,19 @@ Unit tests for generate_test_code.py """ -from StringIO import StringIO +try: + # Python 2 + from StringIO import StringIO +except ImportError: + # Python 3 + from io import StringIO from unittest import TestCase, main as unittest_main -from mock import patch +try: + # Python 2 + from mock import patch +except ImportError: + # Python 3 + from unittest.mock import patch from generate_test_code import gen_dependencies, gen_dependencies_one_line from generate_test_code import gen_function_wrapper, gen_dispatch from generate_test_code import parse_until_pattern, GeneratorInputError @@ -307,9 +317,16 @@ class StringIOWrapper(StringIO, object): :return: Line read from file. """ - line = super(StringIOWrapper, self).next() + parent = super(StringIOWrapper, self) + if getattr(parent, 'next', None): + # Python 2 + line = parent.next() + else: + # Python 3 + line = parent.__next__() return line + # Python 3 __next__ = next def readline(self, length=0): @@ -532,6 +549,38 @@ class ParseFunctionCode(TestCase): Test suite for testing parse_function_code() """ + def assert_raises_regex(self, exp, regex, func, *args): + """ + Python 2 & 3 portable wrapper of assertRaisesRegex(p)? function. + + :param exp: Exception type expected to be raised by cb. + :param regex: Expected exception message + :param func: callable object under test + :param args: variable positional arguments + """ + parent = super(ParseFunctionCode, self) + + # Pylint does not appreciate that the super method called + # conditionally can be available in other Python version + # then that of Pylint. + # Workaround is to call the method via getattr. + # Pylint ignores that the method got via getattr is + # conditionally executed. Method has to be a callable. + # Hence, using a dummy callable for getattr default. + dummy = lambda *x: None + # First Python 3 assertRaisesRegex is checked, since Python 2 + # assertRaisesRegexp is also available in Python 3 but is + # marked deprecated. + for name in ('assertRaisesRegex', 'assertRaisesRegexp'): + method = getattr(parent, name, dummy) + if method is not dummy: + method(exp, regex, func, *args) + break + else: + raise AttributeError(" 'ParseFunctionCode' object has no attribute" + " 'assertRaisesRegex' or 'assertRaisesRegexp'" + ) + def test_no_function(self): """ Test no test function found. @@ -544,8 +593,8 @@ function ''' stream = StringIOWrapper('test_suite_ut.function', data) err_msg = 'file: test_suite_ut.function - Test functions not found!' - self.assertRaisesRegexp(GeneratorInputError, err_msg, - parse_function_code, stream, [], []) + self.assert_raises_regex(GeneratorInputError, err_msg, + parse_function_code, stream, [], []) def test_no_end_case_comment(self): """ @@ -560,8 +609,8 @@ void test_func() stream = StringIOWrapper('test_suite_ut.function', data) err_msg = r'file: test_suite_ut.function - '\ 'end case pattern .*? not found!' - self.assertRaisesRegexp(GeneratorInputError, err_msg, - parse_function_code, stream, [], []) + self.assert_raises_regex(GeneratorInputError, err_msg, + parse_function_code, stream, [], []) @patch("generate_test_code.parse_function_arguments") def test_function_called(self, @@ -678,8 +727,8 @@ exit: data = 'int entropy_threshold( char * a, data_t * h, int result )' err_msg = 'file: test_suite_ut.function - Test functions not found!' stream = StringIOWrapper('test_suite_ut.function', data) - self.assertRaisesRegexp(GeneratorInputError, err_msg, - parse_function_code, stream, [], []) + self.assert_raises_regex(GeneratorInputError, err_msg, + parse_function_code, stream, [], []) @patch("generate_test_code.gen_dispatch") @patch("generate_test_code.gen_dependencies") @@ -1155,8 +1204,7 @@ dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" for _, _, _, _ in parse_test_data(stream): pass except GeneratorInputError as err: - pass - self.assertEqual(type(err), GeneratorInputError) + self.assertEqual(type(err), GeneratorInputError) def test_incomplete_data(self): """ @@ -1174,8 +1222,7 @@ depends_on:YAHOO for _, _, _, _ in parse_test_data(stream): pass except GeneratorInputError as err: - pass - self.assertEqual(type(err), GeneratorInputError) + self.assertEqual(type(err), GeneratorInputError) class GenDepCheck(TestCase): From fd142ec3d2fba3816e848adede6990336f83763c Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Wed, 18 Jul 2018 12:50:49 +0100 Subject: [PATCH 023/490] Fix macro validation regex --- scripts/generate_test_code.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 2468063d1..77e235dec 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -184,7 +184,7 @@ BEGIN_CASE_REGEX = r'/\*\s*BEGIN_CASE\s*(?P.*?)\s*\*/' END_CASE_REGEX = r'/\*\s*END_CASE\s*\*/' DEPENDENCY_REGEX = r'depends_on:(?P.*)' -C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*' +C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*$' TEST_FUNCTION_VALIDATION_REGEX = r'\s*void\s+(?P\w+)\s*\(' INT_CHECK_REGEX = r'int\s+.*' CHAR_CHECK_REGEX = r'char\s*\*\s*.*' @@ -1133,5 +1133,5 @@ if __name__ == "__main__": try: main() except GeneratorInputError as err: - print("%s: input error: %s" % - (os.path.basename(sys.argv[0]), str(err))) + sys.exit("%s: input error: %s" % + (os.path.basename(sys.argv[0]), str(err))) From 26a139b8bcb4e8c57004cc741a6b0cad695e7dc0 Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Wed, 18 Jul 2018 17:48:37 +0100 Subject: [PATCH 024/490] Style fixes --- scripts/generate_test_code.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 77e235dec..26d1c29cb 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -57,10 +57,9 @@ Parameters can be of type string, binary data specified as hex string and integer constants specified as integer, macro or as an expression. Following is an example test definition: -X509 CRL Unsupported critical extension (issuingDistributionPoint) -depends_on:MBEDTLS_PEM_PARSE_C:MBEDTLS_RSA_C:MBEDTLS_SHA256_C -mbedtls_x509_crl_parse:"data_files/crl-idp.pem":\ - MBEDTLS_ERR_X509_INVALID_EXTENSIONS + MBEDTLS_ERR_ASN1_UNEXPECTED_TAG + AES 128 GCM Encrypt and decrypt 8 bytes + depends_on:MBEDTLS_AES_C:MBEDTLS_GCM_C + enc_dec_buf:MBEDTLS_CIPHER_AES_128_GCM:"AES-128-GCM":128:8:-1 Test functions: --------------- @@ -965,13 +964,11 @@ def write_test_source_file(template_file, c_file, snippets): :return: """ with open(template_file, 'r') as template_f, open(c_file, 'w') as c_f: - line_no = 1 - for line in template_f.readlines(): + for line_no, line in enumerate(template_f.readlines(), 1): # Update line number. +1 as #line directive sets next line number snippets['line_no'] = line_no + 1 code = line.format(**snippets) c_f.write(code) - line_no += 1 def parse_function_file(funcs_file, snippets): From ccf34edd5af5981218c2dce49680afbdd46b295e Mon Sep 17 00:00:00 2001 From: Mohammad Azim Khan Date: Thu, 19 Jul 2018 11:32:30 +0100 Subject: [PATCH 025/490] Less obscure test suites template --- scripts/generate_test_code.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 26d1c29cb..ce6f88c3c 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -122,7 +122,7 @@ dependency checks, expression evaluation and function dispatch. These functions are populated with checks and return codes by this script. Template file contains "replacement" fields that are formatted -strings processed by Python str.format() method. +strings processed by Python string.Template.substitute() method. This script: ============ @@ -132,9 +132,9 @@ code that is generated or read from helpers and platform files. This script replaces following fields in the template and generates the test source file: -{test_common_helpers} <-- All common code from helpers.function +$test_common_helpers <-- All common code from helpers.function is substituted here. -{functions_code} <-- Test functions are substituted here +$functions_code <-- Test functions are substituted here from the input test_suit_xyz.function file. C preprocessor checks are generated for the build dependencies specified @@ -143,21 +143,21 @@ the test source file: functions with code to expand the string parameters read from the data file. -{expression_code} <-- This script enumerates the +$expression_code <-- This script enumerates the expressions in the .data file and generates code to handle enumerated expression Ids and return the values. -{dep_check_code} <-- This script enumerates all +$dep_check_code <-- This script enumerates all build dependencies and generate code to handle enumerated build dependency Id and return status: if the dependency is defined or not. -{dispatch_code} <-- This script enumerates the functions +$dispatch_code <-- This script enumerates the functions specified in the input test data file and generates the initializer for the function table in the template file. -{platform_code} <-- Platform specific setup and test +$platform_code <-- Platform specific setup and test dispatch code. """ @@ -167,6 +167,7 @@ import io import os import re import sys +import string import argparse @@ -967,7 +968,7 @@ def write_test_source_file(template_file, c_file, snippets): for line_no, line in enumerate(template_f.readlines(), 1): # Update line number. +1 as #line directive sets next line number snippets['line_no'] = line_no + 1 - code = line.format(**snippets) + code = string.Template(line).substitute(**snippets) c_f.write(code) From d66683a0fa63583a744ebbb1c0651b5c29e8587a Mon Sep 17 00:00:00 2001 From: Ron Eldor Date: Mon, 26 Nov 2018 14:23:14 +0200 Subject: [PATCH 026/490] Add conditional dependency to tests Add a way to check compile time defionitions values, for determining whether to skip tests. --- scripts/generate_test_code.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index ce6f88c3c..cb80c764b 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -184,7 +184,7 @@ BEGIN_CASE_REGEX = r'/\*\s*BEGIN_CASE\s*(?P.*?)\s*\*/' END_CASE_REGEX = r'/\*\s*END_CASE\s*\*/' DEPENDENCY_REGEX = r'depends_on:(?P.*)' -C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*$' +C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*(([<>]=?|==)[0-9]*)?$' TEST_FUNCTION_VALIDATION_REGEX = r'\s*void\s+(?P\w+)\s*\(' INT_CHECK_REGEX = r'int\s+.*' CHAR_CHECK_REGEX = r'char\s*\*\s*.*' @@ -255,6 +255,7 @@ class FileWrapper(io.FileIO, object): def split_dep(dep): """ Split NOT character '!' from dependency. Used by gen_dependencies() + Determine condition MACRO and definition MACRO. :param dep: Dependency list :return: string tuple. Ex: ('!', MACRO) for !MACRO and ('', MACRO) for @@ -731,18 +732,19 @@ def gen_dep_check(dep_id, dep): raise GeneratorInputError("Dependency Id should be a positive " "integer.") _not, dep = ('!', dep[1:]) if dep[0] == '!' else ('', dep) + _defined = '' if re.search(r'(<=?|>=?|==)', dep) else 'defined' if not dep: raise GeneratorInputError("Dependency should not be an empty string.") dep_check = ''' case {id}: {{ -#if {_not}defined({macro}) +#if {_not}{_defined}({macro}) ret = DEPENDENCY_SUPPORTED; #else ret = DEPENDENCY_NOT_SUPPORTED; #endif }} - break;'''.format(_not=_not, macro=dep, id=dep_id) + break;'''.format(_not=_not, _defined=_defined, macro=dep, id=dep_id) return dep_check From ee47e29dcc8a4281c6a29e2968af954724f8ca9a Mon Sep 17 00:00:00 2001 From: Ron Eldor Date: Tue, 27 Nov 2018 16:35:20 +0200 Subject: [PATCH 027/490] Separate REGEX of MACRO to groups Seperate the REGEX into identifier, condition and value, into groups, to behandled differently. --- scripts/generate_test_code.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index cb80c764b..125802442 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -184,7 +184,13 @@ BEGIN_CASE_REGEX = r'/\*\s*BEGIN_CASE\s*(?P.*?)\s*\*/' END_CASE_REGEX = r'/\*\s*END_CASE\s*\*/' DEPENDENCY_REGEX = r'depends_on:(?P.*)' -C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*(([<>]=?|==)[0-9]*)?$' +C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*' +CONDITION_OPERATOR_REGEX = r'[!=]=|[<>]=?' +# forbid 0ddd which might be accidentally octal or accidentally decimal +CONDITION_VALUE_REGEX = r'[-+]?(0x[0-9a-f]+|0|[1-9][0-9]*)' +CONDITION_REGEX = r'({})(?:\s*({})\s*({}))?$'.format(C_IDENTIFIER_REGEX, + CONDITION_OPERATOR_REGEX, + CONDITION_VALUE_REGEX) TEST_FUNCTION_VALIDATION_REGEX = r'\s*void\s+(?P\w+)\s*\(' INT_CHECK_REGEX = r'int\s+.*' CHAR_CHECK_REGEX = r'char\s*\*\s*.*' @@ -255,7 +261,6 @@ class FileWrapper(io.FileIO, object): def split_dep(dep): """ Split NOT character '!' from dependency. Used by gen_dependencies() - Determine condition MACRO and definition MACRO. :param dep: Dependency list :return: string tuple. Ex: ('!', MACRO) for !MACRO and ('', MACRO) for @@ -384,7 +389,7 @@ def validate_dependency(dependency): :return: input dependency stripped of leading & trailing white spaces. """ dependency = dependency.strip() - if not re.match(C_IDENTIFIER_REGEX, dependency, re.I): + if not re.match(CONDITION_REGEX, dependency, re.I): raise GeneratorInputError('Invalid dependency %s' % dependency) return dependency @@ -732,19 +737,29 @@ def gen_dep_check(dep_id, dep): raise GeneratorInputError("Dependency Id should be a positive " "integer.") _not, dep = ('!', dep[1:]) if dep[0] == '!' else ('', dep) - _defined = '' if re.search(r'(<=?|>=?|==)', dep) else 'defined' if not dep: raise GeneratorInputError("Dependency should not be an empty string.") + + dependency = re.match(CONDITION_REGEX, dep, re.I) + if not dependency: + raise GeneratorInputError('Invalid dependency %s' % dep) + + _defined = '' if dependency.group(2) else 'defined' + _cond = dependency.group(2) if dependency.group(2) else '' + _value = dependency.group(3) if dependency.group(3) else '' + dep_check = ''' case {id}: {{ -#if {_not}{_defined}({macro}) +#if {_not}{_defined}({macro}{_cond}{_value}) ret = DEPENDENCY_SUPPORTED; #else ret = DEPENDENCY_NOT_SUPPORTED; #endif }} - break;'''.format(_not=_not, _defined=_defined, macro=dep, id=dep_id) + break;'''.format(_not=_not, _defined=_defined, + macro=dependency.group(1), id=dep_id, + _cond=_cond, _value=_value) return dep_check From 1d5278187982f560f63efed3211f2caef8631a27 Mon Sep 17 00:00:00 2001 From: Andrzej Kurek Date: Thu, 31 Jan 2019 08:20:20 -0500 Subject: [PATCH 028/490] Merge development commit 8e76332 into development-psa Additional changes to temporarily enable running tests: ssl_srv.c and test_suite_ecdh use mbedtls_ecp_group_load instead of mbedtls_ecdh_setup test_suite_ctr_drbg uses mbedtls_ctr_drbg_update instead of mbedtls_ctr_drbg_update_ret --- scripts/generate_test_code.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index ce6f88c3c..125802442 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -184,7 +184,13 @@ BEGIN_CASE_REGEX = r'/\*\s*BEGIN_CASE\s*(?P.*?)\s*\*/' END_CASE_REGEX = r'/\*\s*END_CASE\s*\*/' DEPENDENCY_REGEX = r'depends_on:(?P.*)' -C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*$' +C_IDENTIFIER_REGEX = r'!?[a-z_][a-z0-9_]*' +CONDITION_OPERATOR_REGEX = r'[!=]=|[<>]=?' +# forbid 0ddd which might be accidentally octal or accidentally decimal +CONDITION_VALUE_REGEX = r'[-+]?(0x[0-9a-f]+|0|[1-9][0-9]*)' +CONDITION_REGEX = r'({})(?:\s*({})\s*({}))?$'.format(C_IDENTIFIER_REGEX, + CONDITION_OPERATOR_REGEX, + CONDITION_VALUE_REGEX) TEST_FUNCTION_VALIDATION_REGEX = r'\s*void\s+(?P\w+)\s*\(' INT_CHECK_REGEX = r'int\s+.*' CHAR_CHECK_REGEX = r'char\s*\*\s*.*' @@ -383,7 +389,7 @@ def validate_dependency(dependency): :return: input dependency stripped of leading & trailing white spaces. """ dependency = dependency.strip() - if not re.match(C_IDENTIFIER_REGEX, dependency, re.I): + if not re.match(CONDITION_REGEX, dependency, re.I): raise GeneratorInputError('Invalid dependency %s' % dependency) return dependency @@ -733,16 +739,27 @@ def gen_dep_check(dep_id, dep): _not, dep = ('!', dep[1:]) if dep[0] == '!' else ('', dep) if not dep: raise GeneratorInputError("Dependency should not be an empty string.") + + dependency = re.match(CONDITION_REGEX, dep, re.I) + if not dependency: + raise GeneratorInputError('Invalid dependency %s' % dep) + + _defined = '' if dependency.group(2) else 'defined' + _cond = dependency.group(2) if dependency.group(2) else '' + _value = dependency.group(3) if dependency.group(3) else '' + dep_check = ''' case {id}: {{ -#if {_not}defined({macro}) +#if {_not}{_defined}({macro}{_cond}{_value}) ret = DEPENDENCY_SUPPORTED; #else ret = DEPENDENCY_NOT_SUPPORTED; #endif }} - break;'''.format(_not=_not, macro=dep, id=dep_id) + break;'''.format(_not=_not, _defined=_defined, + macro=dependency.group(1), id=dep_id, + _cond=_cond, _value=_value) return dep_check From 644398e115b06435b3bda0131aa4d8fd84fd80a7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Feb 2019 21:39:42 +0100 Subject: [PATCH 029/490] Silence pylint Silence pylint in specific places where we're doing slightly unusual or dodgy, but correct. --- scripts/generate_test_code.py | 2 +- scripts/test_generate_test_code.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 125802442..1fff09992 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -238,7 +238,7 @@ class FileWrapper(io.FileIO, object): if hasattr(parent, '__next__'): line = parent.__next__() # Python 3 else: - line = parent.next() # Python 2 + line = parent.next() # Python 2 # pylint: disable=no-member if line is not None: self._line_no += 1 # Convert byte array to string with correct encoding and diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 2ef12e18d..6d7113e18 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -22,7 +22,7 @@ Unit tests for generate_test_code.py """ - +# pylint: disable=wrong-import-order try: # Python 2 from StringIO import StringIO @@ -36,6 +36,7 @@ try: except ImportError: # Python 3 from unittest.mock import patch +# pylint: enable=wrong-import-order from generate_test_code import gen_dependencies, gen_dependencies_one_line from generate_test_code import gen_function_wrapper, gen_dispatch from generate_test_code import parse_until_pattern, GeneratorInputError @@ -336,6 +337,7 @@ class StringIOWrapper(StringIO, object): :param length: :return: """ + # pylint: disable=unused-argument line = super(StringIOWrapper, self).readline() if line is not None: self.line_no += 1 From 50b0acdfa43d405fef682d66f1197f00035de9dc Mon Sep 17 00:00:00 2001 From: Jaeden Amero Date: Wed, 27 Feb 2019 17:09:45 +0000 Subject: [PATCH 030/490] tests: Update generator with Mbed Crypto comments Update comments in the top of the test code generator script with the name of the parent project. --- scripts/generate_test_code.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 1fff09992..3a25a8433 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -24,15 +24,12 @@ understanding the script it is important to understand the framework. This doc string contains a summary of the framework and explains the function of this script. -Mbed TLS test suites: -===================== +Mbed Crypto test suites: +======================== Scope: ------ -The test suites focus on unit testing the crypto primitives and also -include x509 parser tests. Tests can be added to test any Mbed TLS -module. However, the framework is not capable of testing SSL -protocol, since that requires full stack execution and that is best -tested as part of the system test. +The test suites focus on unit testing the crypto primitives. Tests can be added +to test any Mbed Crypto module. Test case definition: --------------------- From 2fc0d575a578cd1674bb87c255c9c16e406b44f3 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 26 Feb 2020 18:25:12 +0100 Subject: [PATCH 031/490] Revert "tests: Update generator with Mbed Crypto comments" This reverts commit 50b0acdfa43d405fef682d66f1197f00035de9dc. --- scripts/generate_test_code.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 3a25a8433..1fff09992 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -24,12 +24,15 @@ understanding the script it is important to understand the framework. This doc string contains a summary of the framework and explains the function of this script. -Mbed Crypto test suites: -======================== +Mbed TLS test suites: +===================== Scope: ------ -The test suites focus on unit testing the crypto primitives. Tests can be added -to test any Mbed Crypto module. +The test suites focus on unit testing the crypto primitives and also +include x509 parser tests. Tests can be added to test any Mbed TLS +module. However, the framework is not capable of testing SSL +protocol, since that requires full stack execution and that is best +tested as part of the system test. Test case definition: --------------------- From 17ecabc40c87aabffb1d1774d6108f4f3a80fad5 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 24 Mar 2020 18:25:17 +0100 Subject: [PATCH 032/490] Pylint: abide by useless-object-inheritance warnings Inheriting from object is a remainder of Python 2 habits and is just clutter in Python 3. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 2 +- scripts/test_generate_test_code.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 1fff09992..c0f99f74f 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -208,7 +208,7 @@ class GeneratorInputError(Exception): pass -class FileWrapper(io.FileIO, object): +class FileWrapper(io.FileIO): """ This class extends built-in io.FileIO class with attribute line_no, that indicates line number for the line that is read. diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 6d7113e18..e39b29b83 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -294,7 +294,7 @@ class GenDispatch(TestCase): self.assertEqual(code, expected) -class StringIOWrapper(StringIO, object): +class StringIOWrapper(StringIO): """ file like class to mock file object in tests. """ From 4c650bced19be3006769d9a22dc3f6fcd3317291 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 24 Mar 2020 18:36:56 +0100 Subject: [PATCH 033/490] Pylint: minor code simplifications Simplify the code in minor ways. Each of this changes fixes a warning from Pylint 2.4 that doesn't appear with Pylint 1.7. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 3 +-- scripts/test_generate_test_code.py | 10 ++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index c0f99f74f..21f816ea9 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -402,8 +402,7 @@ def parse_dependencies(inp_str): :param inp_str: Input string with macros delimited by ':'. :return: list of dependencies """ - dependencies = [dep for dep in map(validate_dependency, - inp_str.split(':'))] + dependencies = list(map(validate_dependency, inp_str.split(':'))) return dependencies diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index e39b29b83..c8e8c5ce1 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -1127,9 +1127,8 @@ Diffie-Hellman selftest dhm_selftest: """ stream = StringIOWrapper('test_suite_ut.function', data) - tests = [(name, test_function, dependencies, args) - for name, test_function, dependencies, args in - parse_test_data(stream)] + # List of (name, function_name, dependencies, args) + tests = list(parse_test_data(stream)) test1, test2, test3, test4 = tests self.assertEqual(test1[0], 'Diffie-Hellman full exchange #1') self.assertEqual(test1[1], 'dhm_do_dhm') @@ -1170,9 +1169,8 @@ dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" """ stream = StringIOWrapper('test_suite_ut.function', data) - tests = [(name, function_name, dependencies, args) - for name, function_name, dependencies, args in - parse_test_data(stream)] + # List of (name, function_name, dependencies, args) + tests = list(parse_test_data(stream)) test1, test2 = tests self.assertEqual(test1[0], 'Diffie-Hellman full exchange #1') self.assertEqual(test1[1], 'dhm_do_dhm') From 8c59c00d166620b59b0dab413704f813f6185691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Fri, 7 Aug 2020 13:07:28 +0200 Subject: [PATCH 034/490] Update copyright notices to use Linux Foundation guidance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As a result, the copyright of contributors other than Arm is now acknowledged, and the years of publishing are no longer tracked in the source files. Also remove the now-redundant lines declaring that the files are part of MbedTLS. This commit was generated using the following script: # ======================== #!/bin/sh # Find files find '(' -path './.git' -o -path './3rdparty' ')' -prune -o -type f -print | xargs sed -bi ' # Replace copyright attribution line s/Copyright.*Arm.*/Copyright The Mbed TLS Contributors/I # Remove redundant declaration and the preceding line $!N /This file is part of Mbed TLS/Id P D ' # ======================== Signed-off-by: Bence Szépkúti --- scripts/generate_test_code.py | 4 +--- scripts/test_generate_test_code.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 21f816ea9..7382fb6ec 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Test suites code generator. # -# Copyright (C) 2018, Arm Limited, All Rights Reserved +# Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,8 +15,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# This file is part of Mbed TLS (https://tls.mbed.org) """ This script is a key part of Mbed TLS test suites framework. For diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index c8e8c5ce1..000c2a701 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Unit test for generate_test_code.py # -# Copyright (C) 2018, Arm Limited, All Rights Reserved +# Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,8 +15,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# This file is part of Mbed TLS (https://tls.mbed.org) """ Unit tests for generate_test_code.py From dc334f11687585a95dd22f6eb097c7d18d4971e3 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 12 Aug 2020 02:11:38 +0200 Subject: [PATCH 035/490] test_generate_test_code: remove Python 2 compatibility code This makes the code cleaner. As a bonus, mypy no longer gets confused. Signed-off-by: Gilles Peskine --- scripts/test_generate_test_code.py | 74 +++++------------------------- 1 file changed, 11 insertions(+), 63 deletions(-) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 000c2a701..9bf66f1cc 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -20,21 +20,10 @@ Unit tests for generate_test_code.py """ -# pylint: disable=wrong-import-order -try: - # Python 2 - from StringIO import StringIO -except ImportError: - # Python 3 - from io import StringIO +from io import StringIO from unittest import TestCase, main as unittest_main -try: - # Python 2 - from mock import patch -except ImportError: - # Python 3 - from unittest.mock import patch -# pylint: enable=wrong-import-order +from unittest.mock import patch + from generate_test_code import gen_dependencies, gen_dependencies_one_line from generate_test_code import gen_function_wrapper, gen_dispatch from generate_test_code import parse_until_pattern, GeneratorInputError @@ -317,25 +306,16 @@ class StringIOWrapper(StringIO): :return: Line read from file. """ parent = super(StringIOWrapper, self) - if getattr(parent, 'next', None): - # Python 2 - line = parent.next() - else: - # Python 3 - line = parent.__next__() + line = parent.__next__() return line - # Python 3 - __next__ = next - - def readline(self, length=0): + def readline(self, _length=0): """ Wrap the base class readline. :param length: :return: """ - # pylint: disable=unused-argument line = super(StringIOWrapper, self).readline() if line is not None: self.line_no += 1 @@ -549,38 +529,6 @@ class ParseFunctionCode(TestCase): Test suite for testing parse_function_code() """ - def assert_raises_regex(self, exp, regex, func, *args): - """ - Python 2 & 3 portable wrapper of assertRaisesRegex(p)? function. - - :param exp: Exception type expected to be raised by cb. - :param regex: Expected exception message - :param func: callable object under test - :param args: variable positional arguments - """ - parent = super(ParseFunctionCode, self) - - # Pylint does not appreciate that the super method called - # conditionally can be available in other Python version - # then that of Pylint. - # Workaround is to call the method via getattr. - # Pylint ignores that the method got via getattr is - # conditionally executed. Method has to be a callable. - # Hence, using a dummy callable for getattr default. - dummy = lambda *x: None - # First Python 3 assertRaisesRegex is checked, since Python 2 - # assertRaisesRegexp is also available in Python 3 but is - # marked deprecated. - for name in ('assertRaisesRegex', 'assertRaisesRegexp'): - method = getattr(parent, name, dummy) - if method is not dummy: - method(exp, regex, func, *args) - break - else: - raise AttributeError(" 'ParseFunctionCode' object has no attribute" - " 'assertRaisesRegex' or 'assertRaisesRegexp'" - ) - def test_no_function(self): """ Test no test function found. @@ -593,8 +541,8 @@ function ''' stream = StringIOWrapper('test_suite_ut.function', data) err_msg = 'file: test_suite_ut.function - Test functions not found!' - self.assert_raises_regex(GeneratorInputError, err_msg, - parse_function_code, stream, [], []) + self.assertRaisesRegex(GeneratorInputError, err_msg, + parse_function_code, stream, [], []) def test_no_end_case_comment(self): """ @@ -609,8 +557,8 @@ void test_func() stream = StringIOWrapper('test_suite_ut.function', data) err_msg = r'file: test_suite_ut.function - '\ 'end case pattern .*? not found!' - self.assert_raises_regex(GeneratorInputError, err_msg, - parse_function_code, stream, [], []) + self.assertRaisesRegex(GeneratorInputError, err_msg, + parse_function_code, stream, [], []) @patch("generate_test_code.parse_function_arguments") def test_function_called(self, @@ -727,8 +675,8 @@ exit: data = 'int entropy_threshold( char * a, data_t * h, int result )' err_msg = 'file: test_suite_ut.function - Test functions not found!' stream = StringIOWrapper('test_suite_ut.function', data) - self.assert_raises_regex(GeneratorInputError, err_msg, - parse_function_code, stream, [], []) + self.assertRaisesRegex(GeneratorInputError, err_msg, + parse_function_code, stream, [], []) @patch("generate_test_code.gen_dispatch") @patch("generate_test_code.gen_dependencies") From 20159462f1eb506f1c8f715316706d9757c3527c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 11 Dec 2020 00:30:53 +0100 Subject: [PATCH 036/490] Move get_c_expression_values into a separate module Create a directory mbedtls_dev intended to contain various Python module for use by Python scripts located anywhere in the Mbed TLS source tree. Move get_c_expression_values and its auxiliary functions into a new Python module mbedtls_dev.c_build_helper. Signed-off-by: Gilles Peskine --- scripts/framework_dev/c_build_helper.py | 138 ++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 scripts/framework_dev/c_build_helper.py diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py new file mode 100644 index 000000000..680760247 --- /dev/null +++ b/scripts/framework_dev/c_build_helper.py @@ -0,0 +1,138 @@ +"""Generate and run C code. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import platform +import subprocess +import sys +import tempfile + +def remove_file_if_exists(filename): + """Remove the specified file, ignoring errors.""" + if not filename: + return + try: + os.remove(filename) + except OSError: + pass + +def create_c_file(file_label): + """Create a temporary C file. + + * ``file_label``: a string that will be included in the file name. + + Return ```(c_file, c_name, exe_name)``` where ``c_file`` is a Python + stream open for writing to the file, ``c_name`` is the name of the file + and ``exe_name`` is the name of the executable that will be produced + by compiling the file. + """ + c_fd, c_name = tempfile.mkstemp(prefix='tmp-{}-'.format(file_label), + suffix='.c') + exe_suffix = '.exe' if platform.system() == 'Windows' else '' + exe_name = c_name[:-2] + exe_suffix + remove_file_if_exists(exe_name) + c_file = os.fdopen(c_fd, 'w', encoding='ascii') + return c_file, c_name, exe_name + +def generate_c_printf_expressions(c_file, cast_to, printf_format, expressions): + """Generate C instructions to print the value of ``expressions``. + + Write the code with ``c_file``'s ``write`` method. + + Each expression is cast to the type ``cast_to`` and printed with the + printf format ``printf_format``. + """ + for expr in expressions: + c_file.write(' printf("{}\\n", ({}) {});\n' + .format(printf_format, cast_to, expr)) + +def generate_c_file(c_file, + caller, header, + main_generator): + """Generate a temporary C source file. + + * ``c_file`` is an open stream on the C source file. + * ``caller``: an informational string written in a comment at the top + of the file. + * ``header``: extra code to insert before any function in the generated + C file. + * ``main_generator``: a function called with ``c_file`` as its sole argument + to generate the body of the ``main()`` function. + """ + c_file.write('/* Generated by {} */' + .format(caller)) + c_file.write(''' +#include +''') + c_file.write(header) + c_file.write(''' +int main(void) +{ +''') + main_generator(c_file) + c_file.write(''' return 0; +} +''') + +def get_c_expression_values( + cast_to, printf_format, + expressions, + caller=__name__, file_label='', + header='', include_path=None, + keep_c=False, +): # pylint: disable=too-many-arguments + """Generate and run a program to print out numerical values for expressions. + + * ``cast_to``: a C type. + * ``printf_format``: a printf format suitable for the type ``cast_to``. + * ``header``: extra code to insert before any function in the generated + C file. + * ``expressions``: a list of C language expressions that have the type + ``type``. + * ``include_path``: a list of directories containing header files. + * ``keep_c``: if true, keep the temporary C file (presumably for debugging + purposes). + + Return the list of values of the ``expressions``. + """ + if include_path is None: + include_path = [] + c_name = None + exe_name = None + try: + c_file, c_name, exe_name = create_c_file(file_label) + generate_c_file( + c_file, caller, header, + lambda c_file: generate_c_printf_expressions(c_file, + cast_to, printf_format, + expressions) + ) + c_file.close() + cc = os.getenv('CC', 'cc') + subprocess.check_call([cc] + + ['-I' + dir for dir in include_path] + + ['-o', exe_name, c_name]) + if keep_c: + sys.stderr.write('List of {} tests kept at {}\n' + .format(caller, c_name)) + else: + os.remove(c_name) + output = subprocess.check_output([exe_name]) + return output.decode('ascii').strip().split('\n') + finally: + remove_file_if_exists(exe_name) From 84ee5c1afd3bb201eaaa4c9eb394aa501ae0178d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 19 Jan 2021 21:19:02 +0100 Subject: [PATCH 037/490] Minor documentation fixes Signed-off-by: Gilles Peskine --- scripts/framework_dev/c_build_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 680760247..5c587a16b 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -103,7 +103,7 @@ def get_c_expression_values( * ``header``: extra code to insert before any function in the generated C file. * ``expressions``: a list of C language expressions that have the type - ``type``. + ``cast_to``. * ``include_path``: a list of directories containing header files. * ``keep_c``: if true, keep the temporary C file (presumably for debugging purposes). From 90d79507c741f21527338cf36791a423f823fd7a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Jan 2021 21:40:45 +0100 Subject: [PATCH 038/490] Move PSAMacroCollector to a module of its own This will make it possible to use the class from other scripts. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 114 +++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 scripts/framework_dev/macro_collector.py diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py new file mode 100644 index 000000000..f6f26f8c8 --- /dev/null +++ b/scripts/framework_dev/macro_collector.py @@ -0,0 +1,114 @@ +"""Collect macro definitions from header files. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +class PSAMacroCollector: + """Collect PSA crypto macro definitions from C header files. + """ + + def __init__(self): + self.statuses = set() + self.key_types = set() + self.key_types_from_curve = {} + self.key_types_from_group = {} + self.ecc_curves = set() + self.dh_groups = set() + self.algorithms = set() + self.hash_algorithms = set() + self.ka_algorithms = set() + self.algorithms_from_hash = {} + self.key_usages = set() + + # "#define" followed by a macro name with either no parameters + # or a single parameter and a non-empty expansion. + # Grab the macro name in group 1, the parameter name if any in group 2 + # and the expansion in group 3. + _define_directive_re = re.compile(r'\s*#\s*define\s+(\w+)' + + r'(?:\s+|\((\w+)\)\s*)' + + r'(.+)') + _deprecated_definition_re = re.compile(r'\s*MBEDTLS_DEPRECATED') + + def read_line(self, line): + """Parse a C header line and record the PSA identifier it defines if any. + This function analyzes lines that start with "#define PSA_" + (up to non-significant whitespace) and skips all non-matching lines. + """ + # pylint: disable=too-many-branches + m = re.match(self._define_directive_re, line) + if not m: + return + name, parameter, expansion = m.groups() + expansion = re.sub(r'/\*.*?\*/|//.*', r' ', expansion) + if re.match(self._deprecated_definition_re, expansion): + # Skip deprecated values, which are assumed to be + # backward compatibility aliases that share + # numerical values with non-deprecated values. + return + if name.endswith('_FLAG') or name.endswith('MASK'): + # Macro only to build actual values + return + elif (name.startswith('PSA_ERROR_') or name == 'PSA_SUCCESS') \ + and not parameter: + self.statuses.add(name) + elif name.startswith('PSA_KEY_TYPE_') and not parameter: + self.key_types.add(name) + elif name.startswith('PSA_KEY_TYPE_') and parameter == 'curve': + self.key_types_from_curve[name] = name[:13] + 'IS_' + name[13:] + elif name.startswith('PSA_KEY_TYPE_') and parameter == 'group': + self.key_types_from_group[name] = name[:13] + 'IS_' + name[13:] + elif name.startswith('PSA_ECC_FAMILY_') and not parameter: + self.ecc_curves.add(name) + elif name.startswith('PSA_DH_FAMILY_') and not parameter: + self.dh_groups.add(name) + elif name.startswith('PSA_ALG_') and not parameter: + if name in ['PSA_ALG_ECDSA_BASE', + 'PSA_ALG_RSA_PKCS1V15_SIGN_BASE']: + # Ad hoc skipping of duplicate names for some numerical values + return + self.algorithms.add(name) + # Ad hoc detection of hash algorithms + if re.search(r'0x020000[0-9A-Fa-f]{2}', expansion): + self.hash_algorithms.add(name) + # Ad hoc detection of key agreement algorithms + if re.search(r'0x09[0-9A-Fa-f]{2}0000', expansion): + self.ka_algorithms.add(name) + elif name.startswith('PSA_ALG_') and parameter == 'hash_alg': + if name in ['PSA_ALG_DSA', 'PSA_ALG_ECDSA']: + # A naming irregularity + tester = name[:8] + 'IS_RANDOMIZED_' + name[8:] + else: + tester = name[:8] + 'IS_' + name[8:] + self.algorithms_from_hash[name] = tester + elif name.startswith('PSA_KEY_USAGE_') and not parameter: + self.key_usages.add(name) + else: + # Other macro without parameter + return + + _nonascii_re = re.compile(rb'[^\x00-\x7f]+') + _continued_line_re = re.compile(rb'\\\r?\n\Z') + def read_file(self, header_file): + for line in header_file: + m = re.search(self._continued_line_re, line) + while m: + cont = next(header_file) + line = line[:m.start(0)] + cont + m = re.search(self._continued_line_re, line) + line = re.sub(self._nonascii_re, rb'', line).decode('ascii') + self.read_line(line) From d1d053a5330fafef923874e34db603eeb1bd67f0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Jan 2021 22:41:45 +0100 Subject: [PATCH 039/490] Factor out is_internal_name as a separate method Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index f6f26f8c8..0f7be604d 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -35,6 +35,10 @@ class PSAMacroCollector: self.algorithms_from_hash = {} self.key_usages = set() + def is_internal_name(self, name): + """Whether this is an internal macro. Internal macros will be skipped.""" + return name.endswith('_FLAG') or name.endswith('MASK') + # "#define" followed by a macro name with either no parameters # or a single parameter and a non-empty expansion. # Grab the macro name in group 1, the parameter name if any in group 2 @@ -60,7 +64,7 @@ class PSAMacroCollector: # backward compatibility aliases that share # numerical values with non-deprecated values. return - if name.endswith('_FLAG') or name.endswith('MASK'): + if self.is_internal_name(name): # Macro only to build actual values return elif (name.startswith('PSA_ERROR_') or name == 'PSA_SUCCESS') \ From ea4a4c73fedc33fbbd05c506a000c30670c95f2b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Jan 2021 22:42:14 +0100 Subject: [PATCH 040/490] MacroCollector: default to not including intermediate macros By default, exclude macros whose numerical value is not a valid member of the semantic type (e.g. PSA_ALG_xxx_BASE is not itself an algorithm, only an intermediate value used to construct others). But do include them with include_intermediate=True, which generate_psa_constants.py does. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 0f7be604d..ca277795c 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -22,7 +22,15 @@ class PSAMacroCollector: """Collect PSA crypto macro definitions from C header files. """ - def __init__(self): + def __init__(self, include_intermediate=False): + """Set up an object to collect PSA macro definitions. + + Call the read_file method of the constructed object on each header file. + + * include_intermediate: if true, include intermediate macros such as + PSA_XXX_BASE that do not designate semantic values. + """ + self.include_intermediate = include_intermediate self.statuses = set() self.key_types = set() self.key_types_from_curve = {} @@ -37,6 +45,11 @@ class PSAMacroCollector: def is_internal_name(self, name): """Whether this is an internal macro. Internal macros will be skipped.""" + if not self.include_intermediate: + if name.endswith('_BASE') or name.endswith('_NONE'): + return True + if '_CATEGORY_' in name: + return True return name.endswith('_FLAG') or name.endswith('MASK') # "#define" followed by a macro name with either no parameters From aaca60574576cf9a49e88cfab48967da2e0b4188 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 25 Jan 2021 22:44:36 +0100 Subject: [PATCH 041/490] Check if the last word is 'MASK', not if it ends with 'MASK' At the moment it makes no difference, but it could if e.g. a new algorithm was called 'foomask'. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index ca277795c..b98e40e5f 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -50,7 +50,7 @@ class PSAMacroCollector: return True if '_CATEGORY_' in name: return True - return name.endswith('_FLAG') or name.endswith('MASK') + return name.endswith('_FLAG') or name.endswith('_MASK') # "#define" followed by a macro name with either no parameters # or a single parameter and a non-empty expansion. From 94addc4ae6dcbe5cbe45ac6f415ad868fc6ce9e5 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 26 Jan 2021 21:23:56 +0100 Subject: [PATCH 042/490] Framework for knowledge about key types New Python module intended to gather knowledge about key types and cryptographic mechanisms, such as the ability to create test data for a given key type and the determination of whether an algorithm is compatible with a key type. This commit just creates a class for knowledge about key types. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 51 +++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 scripts/framework_dev/crypto_knowledge.py diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py new file mode 100644 index 000000000..e79557953 --- /dev/null +++ b/scripts/framework_dev/crypto_knowledge.py @@ -0,0 +1,51 @@ +"""Knowledge about cryptographic mechanisms implemented in Mbed TLS. + +This module is entirely based on the PSA API. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import List, Optional + +class KeyType: + """Knowledge about a PSA key type.""" + + def __init__(self, name: str, params: Optional[List[str]] = None): + """Analyze a key type. + + The key type must be specified in PSA syntax. In its simplest form, + this is a string 'PSA_KEY_TYPE_xxx' which is the name of a PSA key + type macro. For key types that take arguments, the arguments can + be passed either through the optional argument `params` or by + passing an expression of the form 'PSA_KEY_TYPE_xxx(param1, param2)' + as the a string. + """ + self.name = name.strip() + if params is None: + if '(' in self.name: + m = re.match(r'(\w+)\s*\((.*)\)\Z', self.name) + assert m is not None + self.name = m.group(1) + params = ','.split(m.group(2)) + if params is None: + self.params = params + else: + self.params = [param.strip() for param in params] + self.expression = self.name + if self.params is not None: + self.expression += '(' + ', '.join(self.params) + ')' + self.private_type = re.sub(r'_PUBLIC_KEY\Z', r'_KEY_PAIR', self.name) From c6131c4b0f89701cf4333d46a2d97eba7c8ca7b4 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 26 Jan 2021 21:25:34 +0100 Subject: [PATCH 043/490] Enumerate sizes to test for each key type Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 38 ++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index e79557953..eea796268 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -19,7 +19,7 @@ This module is entirely based on the PSA API. # limitations under the License. import re -from typing import List, Optional +from typing import List, Optional, Tuple class KeyType: """Knowledge about a PSA key type.""" @@ -49,3 +49,39 @@ class KeyType: if self.params is not None: self.expression += '(' + ', '.join(self.params) + ')' self.private_type = re.sub(r'_PUBLIC_KEY\Z', r'_KEY_PAIR', self.name) + + ECC_KEY_SIZES = { + 'PSA_ECC_FAMILY_SECP_K1': (192, 224, 256), + 'PSA_ECC_FAMILY_SECP_R1': (192, 225, 256, 384, 521), + 'PSA_ECC_FAMILY_SECP_R2': (160,), + 'PSA_ECC_FAMILY_SECT_K1': (163, 233, 239, 283, 409, 571), + 'PSA_ECC_FAMILY_SECT_R1': (163, 233, 283, 409, 571), + 'PSA_ECC_FAMILY_SECT_R2': (163,), + 'PSA_ECC_FAMILY_BRAINPOOL_P_R1': (160, 192, 224, 256, 320, 384, 512), + 'PSA_ECC_FAMILY_MONTGOMERY': (255, 448), + } + KEY_TYPE_SIZES = { + 'PSA_KEY_TYPE_AES': (128, 192, 256), # exhaustive + 'PSA_KEY_TYPE_ARC4': (8, 128, 2048), # extremes + sensible + 'PSA_KEY_TYPE_ARIA': (128, 192, 256), # exhaustive + 'PSA_KEY_TYPE_CAMELLIA': (128, 192, 256), # exhaustive + 'PSA_KEY_TYPE_CHACHA20': (256,), # exhaustive + 'PSA_KEY_TYPE_DERIVE': (120, 128), # sample + 'PSA_KEY_TYPE_DES': (64, 128, 192), # exhaustive + 'PSA_KEY_TYPE_HMAC': (128, 160, 224, 256, 384, 512), # standard size for each supported hash + 'PSA_KEY_TYPE_RAW_DATA': (8, 40, 128), # sample + 'PSA_KEY_TYPE_RSA_KEY_PAIR': (1024, 1536), # small sample + } + def sizes_to_test(self) -> Tuple[int, ...]: + """Return a tuple of key sizes to test. + + For key types that only allow a single size, or only a small set of + sizes, these are all the possible sizes. For key types that allow a + wide range of sizes, these are a representative sample of sizes, + excluding large sizes for which a typical resource-constrained platform + may run out of memory. + """ + if self.private_type == 'PSA_KEY_TYPE_ECC_KEY_PAIR': + assert self.params is not None + return self.ECC_KEY_SIZES[self.params[0]] + return self.KEY_TYPE_SIZES[self.private_type] From d521f6a113d832c966e01f783ad4f6eba053f1ab Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 26 Jan 2021 21:26:26 +0100 Subject: [PATCH 044/490] Create sample key material for symmetric keys Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index eea796268..2e0fa2fd1 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -85,3 +85,29 @@ class KeyType: assert self.params is not None return self.ECC_KEY_SIZES[self.params[0]] return self.KEY_TYPE_SIZES[self.private_type] + + # "48657265006973206b6579a064617461" + DATA_BLOCK = b'Here\000is key\240data' + def key_material(self, bits: int) -> bytes: + """Return a byte string containing suitable key material with the given bit length. + + Use the PSA export representation. The resulting byte string is one that + can be obtained with the following code: + ``` + psa_set_key_type(&attributes, `self.expression`); + psa_set_key_bits(&attributes, `bits`); + psa_set_key_usage_flags(&attributes, PSA_KEY_USAGE_EXPORT); + psa_generate_key(&attributes, &id); + psa_export_key(id, `material`, ...); + ``` + """ + if bits % 8 != 0: + raise ValueError('Non-integer number of bytes: {} bits'.format(bits)) + length = bits // 8 + if self.name == 'PSA_KEY_TYPE_DES': + # "644573206b457901644573206b457902644573206b457904" + des3 = b'dEs kEy\001dEs kEy\002dEs kEy\004' + return des3[:length] + # TODO: ECC, RSA + return b''.join([self.DATA_BLOCK] * (length // len(self.DATA_BLOCK)) + + [self.DATA_BLOCK[:length % len(self.DATA_BLOCK)]]) From 03685fa59ffc416768050aee8f2a28e4de3a9d74 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 26 Jan 2021 21:27:22 +0100 Subject: [PATCH 045/490] New Python module for generating Mbed TLS test cases Signed-off-by: Gilles Peskine --- scripts/framework_dev/test_case.py | 88 ++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 scripts/framework_dev/test_case.py diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py new file mode 100644 index 000000000..8ef18510c --- /dev/null +++ b/scripts/framework_dev/test_case.py @@ -0,0 +1,88 @@ +"""Library for generating Mbed TLS test data. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import binascii +from typing import Any, List, Optional +import typing_extensions #pylint: disable=import-error + +class Writable(typing_extensions.Protocol): + """Abstract class for typing hints.""" + # pylint: disable=no-self-use,too-few-public-methods,unused-argument + def write(self, text: str) -> Any: + ... + + +def hex_string(data: bytes) -> str: + return '"' + binascii.hexlify(data).decode('ascii') + '"' + + +class MissingDescription(Exception): + pass + +class MissingFunction(Exception): + pass + +class TestCase: + """An Mbed TLS test case.""" + + def __init__(self, description: Optional[str] = None): + self.comments = [] #type: List[str] + self.description = description #type: Optional[str] + self.dependencies = [] #type: List[str] + self.function = None #type: Optional[str] + self.arguments = [] #type: List[str] + + def add_comment(self, *lines: str) -> None: + self.comments += lines + + def set_description(self, description: str) -> None: + self.description = description + + def set_dependencies(self, dependencies: List[str]) -> None: + self.dependencies = dependencies + + def set_function(self, function: str) -> None: + self.function = function + + def set_arguments(self, arguments: List[str]) -> None: + self.arguments = arguments + + def check_completeness(self) -> None: + if self.description is None: + raise MissingDescription + if self.function is None: + raise MissingFunction + + def write(self, out: Writable) -> None: + """Write the .data file paragraph for this test case. + + The output starts and ends with a single newline character. If the + surrounding code writes lines (consisting of non-newline characters + and a final newline), you will end up with a blank line before, but + not after the test case. + """ + self.check_completeness() + assert self.description is not None # guide mypy + assert self.function is not None # guide mypy + out.write('\n') + for line in self.comments: + out.write('# ' + line + '\n') + out.write(self.description + '\n') + if self.dependencies: + out.write('depends_on:' + ':'.join(self.dependencies) + '\n') + out.write(self.function + ':' + ':'.join(self.arguments) + '\n') From 57da51be87dac6fffb09d049c9ffec6f13d82405 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 26 Jan 2021 21:35:01 +0100 Subject: [PATCH 046/490] New function to write a whole .data file Signed-off-by: Gilles Peskine --- scripts/framework_dev/test_case.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index 8ef18510c..a6ed8fc90 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -17,7 +17,8 @@ # limitations under the License. import binascii -from typing import Any, List, Optional +import sys +from typing import Any, Iterable, List, Optional import typing_extensions #pylint: disable=import-error class Writable(typing_extensions.Protocol): @@ -86,3 +87,21 @@ class TestCase: if self.dependencies: out.write('depends_on:' + ':'.join(self.dependencies) + '\n') out.write(self.function + ':' + ':'.join(self.arguments) + '\n') + + + +def write_data_file(filename: str, + test_cases: Iterable[TestCase], + caller: Optional[str] = None) -> None: + """Write the test cases to the specified file. + + If the file already exists, it is overwritten. + """ + if caller is None: + caller = sys.argv[0] + with open(filename, 'w') as out: + out.write('# Automatically generated by {}. Do not edit!\n' + .format(caller)) + for tc in test_cases: + tc.write(out) + out.write('\n# End of automatically generated file.\n') From 3718577dae97a46e8bf2ea613dae6a73569f3594 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 27 Jan 2021 12:43:24 +0100 Subject: [PATCH 047/490] New module for key material for asymmetric key types Asymmetric keys can't just be arbitrary byte strings: the public key has to match the private key and the private key usually has nontrivial constraints. In order to have deterministic test data and not to rely on cryptographic dependencies in the Python script, hard-code some test keys. In this commit, copy some test keys from test_suite_psa_crypto.data. Signed-off-by: Gilles Peskine --- scripts/framework_dev/asymmetric_key_data.py | 78 ++++++++++++++++++++ scripts/framework_dev/crypto_knowledge.py | 11 ++- 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 scripts/framework_dev/asymmetric_key_data.py diff --git a/scripts/framework_dev/asymmetric_key_data.py b/scripts/framework_dev/asymmetric_key_data.py new file mode 100644 index 000000000..0ba7b7cb4 --- /dev/null +++ b/scripts/framework_dev/asymmetric_key_data.py @@ -0,0 +1,78 @@ +"""Sample key material for asymmetric key types. + +Meant for use in crypto_knowledge.py. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import binascii +import re +from typing import Dict + +STR_TRANS_REMOVE_BLANKS = str.maketrans('', '', ' \t\n\r') + +def unhexlify(text: str) -> bytes: + return binascii.unhexlify(text.translate(STR_TRANS_REMOVE_BLANKS)) + +def construct_asymmetric_key_data(src) -> Dict[str, Dict[int, bytes]]: + """Split key pairs into separate table entries and convert hex to bytes. + + Input format: src[abbreviated_type][size] = (private_key_hex, public_key_hex) + Output format: dst['PSA_KEY_TYPE_xxx'][size] = key_bytes + """ + dst = {} #type: Dict[str, Dict[int, bytes]] + for typ in src: + private = 'PSA_KEY_TYPE_' + re.sub(r'(\(|\Z)', r'_KEY_PAIR\1', typ, 1) + public = 'PSA_KEY_TYPE_' + re.sub(r'(\(|\Z)', r'_PUBLIC_KEY\1', typ, 1) + dst[private] = {} + dst[public] = {} + for size in src[typ]: + dst[private][size] = unhexlify(src[typ][size][0]) + dst[public][size] = unhexlify(src[typ][size][1]) + return dst + +## These are valid keys that don't try to exercise any edge cases. They're +## either test vectors from some specification, or randomly generated. All +## pairs consist of a private key and its public key. +#pylint: disable=line-too-long +ASYMMETRIC_KEY_DATA = construct_asymmetric_key_data({ + 'ECC(PSA_ECC_FAMILY_SECP_R1)': { + 256: ("49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee", + "047772656f814b399279d5e1f1781fac6f099a3c5ca1b0e35351834b08b65e0b572590cdaf8f769361bcf34acfc11e5e074e8426bdde04be6e653945449617de45"), + 384: ("3f5d8d9be280b5696cc5cc9f94cf8af7e6b61dd6592b2ab2b3a4c607450417ec327dcdcaed7c10053d719a0574f0a76a", + "04d9c662b50ba29ca47990450e043aeaf4f0c69b15676d112f622a71c93059af999691c5680d2b44d111579db12f4a413a2ed5c45fcfb67b5b63e00b91ebe59d09a6b1ac2c0c4282aa12317ed5914f999bc488bb132e8342cc36f2ca5e3379c747"), + 521: ("01b1b6ad07bb79e7320da59860ea28e055284f6058f279de666e06d435d2af7bda28d99fa47b7dd0963e16b0073078ee8b8a38d966a582f46d19ff95df3ad9685aae", + "04001de142d54f69eb038ee4b7af9d3ca07736fd9cf719eb354d69879ee7f3c136fb0fbf9f08f86be5fa128ec1a051d3e6c643e85ada8ffacf3663c260bd2c844b6f5600cee8e48a9e65d09cadd89f235dee05f3b8a646be715f1f67d5b434e0ff23a1fc07ef7740193e40eeff6f3bcdfd765aa9155033524fe4f205f5444e292c4c2f6ac1"), + }, + 'RSA': { + 1024: (""" +3082025e + 020100 + 02818100af057d396ee84fb75fdbb5c2b13c7fe5a654aa8aa2470b541ee1feb0b12d25c79711531249e1129628042dbbb6c120d1443524ef4c0e6e1d8956eeb2077af12349ddeee54483bc06c2c61948cd02b202e796aebd94d3a7cbf859c2c1819c324cb82b9cd34ede263a2abffe4733f077869e8660f7d6834da53d690ef7985f6bc3 + 0203010001 + 02818100874bf0ffc2f2a71d14671ddd0171c954d7fdbf50281e4f6d99ea0e1ebcf82faa58e7b595ffb293d1abe17f110b37c48cc0f36c37e84d876621d327f64bbe08457d3ec4098ba2fa0a319fba411c2841ed7be83196a8cdf9daa5d00694bc335fc4c32217fe0488bce9cb7202e59468b1ead119000477db2ca797fac19eda3f58c1 + 024100e2ab760841bb9d30a81d222de1eb7381d82214407f1b975cbbfe4e1a9467fd98adbd78f607836ca5be1928b9d160d97fd45c12d6b52e2c9871a174c66b488113 + 024100c5ab27602159ae7d6f20c3c2ee851e46dc112e689e28d5fcbbf990a99ef8a90b8bb44fd36467e7fc1789ceb663abda338652c3c73f111774902e840565927091 + 024100b6cdbd354f7df579a63b48b3643e353b84898777b48b15f94e0bfc0567a6ae5911d57ad6409cf7647bf96264e9bd87eb95e263b7110b9a1f9f94acced0fafa4d + 024071195eec37e8d257decfc672b07ae639f10cbb9b0c739d0c809968d644a94e3fd6ed9287077a14583f379058f76a8aecd43c62dc8c0f41766650d725275ac4a1 + 024100bb32d133edc2e048d463388b7be9cb4be29f4b6250be603e70e3647501c97ddde20a4e71be95fd5e71784e25aca4baf25be5738aae59bbfe1c997781447a2b24 +""", """ + 308189 + 02818100af057d396ee84fb75fdbb5c2b13c7fe5a654aa8aa2470b541ee1feb0b12d25c79711531249e1129628042dbbb6c120d1443524ef4c0e6e1d8956eeb2077af12349ddeee54483bc06c2c61948cd02b202e796aebd94d3a7cbf859c2c1819c324cb82b9cd34ede263a2abffe4733f077869e8660f7d6834da53d690ef7985f6bc3 + 0203010001 +"""), + }, +}) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 2e0fa2fd1..65dbc347d 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -21,6 +21,8 @@ This module is entirely based on the PSA API. import re from typing import List, Optional, Tuple +from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA + class KeyType: """Knowledge about a PSA key type.""" @@ -101,13 +103,18 @@ class KeyType: psa_export_key(id, `material`, ...); ``` """ + if self.expression in ASYMMETRIC_KEY_DATA: + if bits not in ASYMMETRIC_KEY_DATA[self.expression]: + raise ValueError('No key data for {}-bit {}' + .format(bits, self.expression)) + return ASYMMETRIC_KEY_DATA[self.expression][bits] if bits % 8 != 0: - raise ValueError('Non-integer number of bytes: {} bits'.format(bits)) + raise ValueError('Non-integer number of bytes: {} bits for {}' + .format(bits, self.expression)) length = bits // 8 if self.name == 'PSA_KEY_TYPE_DES': # "644573206b457901644573206b457902644573206b457904" des3 = b'dEs kEy\001dEs kEy\002dEs kEy\004' return des3[:length] - # TODO: ECC, RSA return b''.join([self.DATA_BLOCK] * (length // len(self.DATA_BLOCK)) + [self.DATA_BLOCK[:length % len(self.DATA_BLOCK)]]) From b68a3588ac108799ece69faea399598c082e227c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 27 Jan 2021 12:48:22 +0100 Subject: [PATCH 048/490] Add some randomly generated keys RSA-1536: ``` openssl genrsa 1536 2>/dev/null | openssl rsa -outform DER -pubout |hexlify ``` then formatted manually. ECC except Montgomery: ``` function e { openssl genpkey -algorithm ec -pkeyopt ec_paramgen_curve:$1 -text |perl -0777 -pe 's/.*\npriv:([\n 0-9a-f:]*)pub:([\n 0-9a-f:]*).*/"$1","$2"/s or die; y/\n ://d; s/,/,\n /;' |xsel -b; } ``` Curve448: ``` openssl genpkey -algorithm x448 ``` then formatted manually. Signed-off-by: Gilles Peskine --- scripts/framework_dev/asymmetric_key_data.py | 82 ++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/scripts/framework_dev/asymmetric_key_data.py b/scripts/framework_dev/asymmetric_key_data.py index 0ba7b7cb4..1efe44959 100644 --- a/scripts/framework_dev/asymmetric_key_data.py +++ b/scripts/framework_dev/asymmetric_key_data.py @@ -49,7 +49,17 @@ def construct_asymmetric_key_data(src) -> Dict[str, Dict[int, bytes]]: ## pairs consist of a private key and its public key. #pylint: disable=line-too-long ASYMMETRIC_KEY_DATA = construct_asymmetric_key_data({ + 'ECC(PSA_ECC_FAMILY_SECP_K1)': { + 192: ("297ac1722ccac7589ecb240dc719842538ca974beb79f228", + "0426b7bb38da649ac2138fc050c6548b32553dab68afebc36105d325b75538c12323cb0764789ecb992671beb2b6bef2f5"), + 224: ("0024122bf020fa113f6c0ac978dfbd41f749257a9468febdbe0dc9f7e8", + "042cc7335f4b76042bed44ef45959a62aa215f7a5ff0c8111b8c44ed654ee71c1918326ad485b2d599fe2a6eab096ee26d977334d2bac6d61d"), + 256: ("7fa06fa02d0e911b9a47fdc17d2d962ca01e2f31d60c6212d0ed7e3bba23a7b9", + "045c39154579efd667adc73a81015a797d2c8682cdfbd3c3553c4a185d481cdc50e42a0e1cbc3ca29a32a645e927f54beaed14c9dbbf8279d725f5495ca924b24d"), + }, 'ECC(PSA_ECC_FAMILY_SECP_R1)': { + 225: ("872f203b3ad35b7f2ecc803c3a0e1e0b1ed61cc1afe71b189cd4c995", + "046f00eadaa949fee3e9e1c7fa1247eecec86a0dce46418b9bd3117b981d4bd0ae7a990de912f9d060d6cb531a42d22e394ac29e81804bf160"), 256: ("49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee", "047772656f814b399279d5e1f1781fac6f099a3c5ca1b0e35351834b08b65e0b572590cdaf8f769361bcf34acfc11e5e074e8426bdde04be6e653945449617de45"), 384: ("3f5d8d9be280b5696cc5cc9f94cf8af7e6b61dd6592b2ab2b3a4c607450417ec327dcdcaed7c10053d719a0574f0a76a", @@ -57,6 +67,62 @@ ASYMMETRIC_KEY_DATA = construct_asymmetric_key_data({ 521: ("01b1b6ad07bb79e7320da59860ea28e055284f6058f279de666e06d435d2af7bda28d99fa47b7dd0963e16b0073078ee8b8a38d966a582f46d19ff95df3ad9685aae", "04001de142d54f69eb038ee4b7af9d3ca07736fd9cf719eb354d69879ee7f3c136fb0fbf9f08f86be5fa128ec1a051d3e6c643e85ada8ffacf3663c260bd2c844b6f5600cee8e48a9e65d09cadd89f235dee05f3b8a646be715f1f67d5b434e0ff23a1fc07ef7740193e40eeff6f3bcdfd765aa9155033524fe4f205f5444e292c4c2f6ac1"), }, + 'ECC(PSA_ECC_FAMILY_SECP_R2)': { + 160: ("00bf539a1cdda0d7f71a50a3f98aec0a2e8e4ced1e", + "049570d541398665adb5cfa16f5af73b3196926bbd4b876bdb80f8eab20d0f540c22f4de9c140f6d7b"), + }, + 'ECC(PSA_ECC_FAMILY_SECT_K1)': { + 163: ("03ebc8fcded2d6ab72ec0f75bdb4fd080481273e71", + "0406f88f90b4b65950f06ce433afdb097e320f433dc2062b8a65db8fafd3c110f46bc45663fbf021ee7eb9"), + 233: ("41f08485ce587b06061c087e76e247c359de2ba9927ee013b2f1ed9ca8", + "0401e9d7189189f773bd8f71be2c10774ba18842434dfa9312595ea545104400f45a9d5675647513ba75b079fe66a29daac2ec86a6a5d4e75c5f290c1f"), + 239: ("1a8069ce2c2c8bdd7087f2a6ab49588797e6294e979495602ab9650b9c61", + "04068d76b9f4508762c2379db9ee8b87ad8d86d9535132ffba3b5680440cfa28eb133d4232faf1c9aba96af11aefe634a551440800d5f8185105d3072d"), + 283: ("006d627885dd48b9ec6facb5b3865377d755b75a5d51440e45211c1f600e15eff8a881a0", + "0405f48374debceaadb46ba385fd92048fcc5b9af1a1c90408bf94a68b9378df1cbfdfb6fb026a96bea06d8f181bf10c020adbcc88b6ecff96bdc564a9649c247cede601c4be63afc3"), + 409: ("3ff5e74d932fa77db139b7c948c81e4069c72c24845574064beea8976b70267f1c6f9a503e3892ea1dcbb71fcea423faa370a8", + "04012c587f69f68b308ba6dcb238797f4e22290ca939ae806604e2b5ab4d9caef5a74a98fd87c4f88d292dd39d92e556e16c6ecc3c019a105826eef507cd9a04119f54d5d850b3720b3792d5d03410e9105610f7e4b420166ed45604a7a1f229d80975ba6be2060e8b"), + 571: ("005008c97b4a161c0db1bac6452c72846d57337aa92d8ecb4a66eb01d2f29555ffb61a5317225dcc8ca6917d91789e227efc0bfe9eeda7ee21998cd11c3c9885056b0e55b4f75d51", + "04050172a7fd7adf98e4e2ed2742faa5cd12731a15fb0dbbdf75b1c3cc771a4369af6f2fa00e802735650881735759ea9c79961ded18e0daa0ac59afb1d513b5bbda9962e435f454fc020b4afe1445c2302ada07d295ec2580f8849b2dfa7f956b09b4cbe4c88d3b1c217049f75d3900d36df0fa12689256b58dd2ef784ebbeb0564600cf47a841485f8cf897a68accd5a"), + }, + 'ECC(PSA_ECC_FAMILY_SECT_R1)': { + 163: ("009b05dc82d46d64a04a22e6e5ca70ca1231e68c50", + "0400465eeb9e7258b11e33c02266bfe834b20bcb118700772796ee4704ec67651bd447e3011959a79a04cb"), + 233: ("00e5e42834e3c78758088b905deea975f28dc20ef6173e481f96e88afe7f", + "0400cd68c8af4430c92ec7a7048becfdf00a6bae8d1b4c37286f2d336f2a0e017eca3748f4ad6d435c85867aa014eea1bd6d9d005bbd8319cab629001d"), + 283: ("004cecad915f6f3c9bbbd92d1eb101eda23f16c7dad60a57c87c7e1fd2b29b22f6d666ad", + "04052f9ff887254c2d1440ba9e30f13e2185ba53c373b2c410dae21cf8c167f796c08134f601cbc4c570bffbc2433082cf4d9eb5ba173ecb8caec15d66a02673f60807b2daa729b765"), + 409: ("00c22422d265721a3ae2b3b2baeb77bee50416e19877af97b5fc1c700a0a88916ecb9050135883accb5e64edc77a3703f4f67a64", + "0401aa25466b1d291846db365957b25431591e50d9c109fe2106e93bb369775896925b15a7bfec397406ab4fe6f6b1a13bf8fdcb9300fa5500a813228676b0a6c572ed96b0f4aec7e87832e7e20f17ca98ecdfd36f59c82bddb8665f1f357a73900e827885ec9e1f22"), + 571: ("026ac1cdf92a13a1b8d282da9725847908745138f5c6706b52d164e3675fcfbf86fc3e6ab2de732193267db029dd35a0599a94a118f480231cfc6ccca2ebfc1d8f54176e0f5656a1", + "040708f3403ee9948114855c17572152a08f8054d486defef5f29cbffcfb7cfd9280746a1ac5f751a6ad902ec1e0525120e9be56f03437af196fbe60ee7856e3542ab2cf87880632d80290e39b1a2bd03c6bbf6225511c567bd2ff41d2325dc58346f2b60b1feee4dc8b2af2296c2dc52b153e0556b5d24152b07f690c3fa24e4d1d19efbdeb1037833a733654d2366c74"), + }, + 'ECC(PSA_ECC_FAMILY_SECT_R2)': { + 163: ("0210b482a458b4822d0cb21daa96819a67c8062d34", + "0403692601144c32a6cfa369ae20ae5d43c1c764678c037bafe80c6fd2e42b7ced96171d9c5367fd3dca6f"), + }, + 'ECC(PSA_ECC_FAMILY_BRAINPOOL_P_R1)': { + 160: ("69502c4fdaf48d4fa617bdd24498b0406d0eeaac", + "04d4b9186816358e2f9c59cf70748cb70641b22fbab65473db4b4e22a361ed7e3de7e8a8ddc4130c5c"), + 192: ("1688a2c5fbf4a3c851d76a98c3ec88f445a97996283db59f", + "043fdd168c179ff5363dd71dcd58de9617caad791ae0c37328be9ca0bfc79cebabf6a95d1c52df5b5f3c8b1a2441cf6c88"), + 224: ("a69835dafeb5da5ab89c59860dddebcfd80b529a99f59b880882923c", + "045fbea378fc8583b3837e3f21a457c31eaf20a54e18eb11d104b3adc47f9d1c97eb9ea4ac21740d70d88514b98bf0bc31addac1d19c4ab3cc"), + 256: ("2161d6f2db76526fa62c16f356a80f01f32f776784b36aa99799a8b7662080ff", + "04768c8cae4abca6306db0ed81b0c4a6215c378066ec6d616c146e13f1c7df809b96ab6911c27d8a02339f0926840e55236d3d1efbe2669d090e4c4c660fada91d"), + 320: ("61b8daa7a6e5aa9fccf1ef504220b2e5a5b8c6dc7475d16d3172d7db0b2778414e4f6e8fa2032ead", + "049caed8fb4742956cc2ad12a9a1c995e21759ef26a07bc2054136d3d2f28bb331a70e26c4c687275ab1f434be7871e115d2350c0c5f61d4d06d2bcdb67f5cb63fdb794e5947c87dc6849a58694e37e6cd"), + 384: ("3dd92e750d90d7d39fc1885cd8ad12ea9441f22b9334b4d965202adb1448ce24c5808a85dd9afc229af0a3124f755bcb", + "04719f9d093a627e0d350385c661cebf00c61923566fe9006a3107af1d871bc6bb68985fd722ea32be316f8e783b7cd1957785f66cfc0cb195dd5c99a8e7abaa848553a584dfd2b48e76d445fe00dd8be59096d877d4696d23b4bc8db14724e66a"), + 512: ("372c9778f69f726cbca3f4a268f16b4d617d10280d79a6a029cd51879fe1012934dfe5395455337df6906dc7d6d2eea4dbb2065c0228f73b3ed716480e7d71d2", + "0438b7ec92b61c5c6c7fbc28a4ec759d48fcd4e2e374defd5c4968a54dbef7510e517886fbfc38ea39aa529359d70a7156c35d3cbac7ce776bdb251dd64bce71234424ee7049eed072f0dbc4d79996e175d557e263763ae97095c081e73e7db2e38adc3d4c9a0487b1ede876dc1fca61c902e9a1d8722b8612928f18a24845591a"), + }, + 'ECC(PSA_ECC_FAMILY_MONTGOMERY)': { + 255: ("70076d0a7318a57d3c16c17251b26645df4c2f87ebc0992ab177fba51db92c6a", + "8520f0098930a754748b7ddcb43ef75a0dbf3a0d26381af4eba4a98eaa9b4e6a"), + 448: ("e4e49f52686f9ee3b638528f721f1596196ffd0a1cddb64c3f216f06541805cfeb1a286dc78018095cdfec050e8007b5f4908962ba20d6c1", + "c0d3a5a2b416a573dc9909f92f134ac01323ab8f8e36804e578588ba2d09fe7c3e737f771ca112825b548a0ffded6d6a2fd09a3e77dec30e"), + }, 'RSA': { 1024: (""" 3082025e @@ -73,6 +139,22 @@ ASYMMETRIC_KEY_DATA = construct_asymmetric_key_data({ 308189 02818100af057d396ee84fb75fdbb5c2b13c7fe5a654aa8aa2470b541ee1feb0b12d25c79711531249e1129628042dbbb6c120d1443524ef4c0e6e1d8956eeb2077af12349ddeee54483bc06c2c61948cd02b202e796aebd94d3a7cbf859c2c1819c324cb82b9cd34ede263a2abffe4733f077869e8660f7d6834da53d690ef7985f6bc3 0203010001 +"""), + 1536: (""" +3082037b + 020100 + 0281c100c870feb6ca6b1d2bd9f2dd99e20f1fe2d7e5192de662229dbe162bd1ba66336a7182903ca0b72796cd441c83d24bcdc3e9a2f5e4399c8a043f1c3ddf04754a66d4cfe7b3671a37dd31a9b4c13bfe06ee90f9d94ddaa06de67a52ac863e68f756736ceb014405a6160579640f831dddccc34ad0b05070e3f9954a58d1815813e1b83bcadba814789c87f1ef2ba5d738b793ec456a67360eea1b5faf1c7cc7bf24f3b2a9d0f8958b1096e0f0c335f8888d0c63a51c3c0337214fa3f5efdf6dcc35 + 0203010001 + 0281c06d2d670047973a87752a9d5bc14f3dae00acb01f593aa0e24cf4a49f932931de4bbfb332e2d38083da80bc0b6d538edba479f7f77d0deffb4a28e6e67ff6273585bb4cd862535c946605ab0809d65f0e38f76e4ec2c3d9b8cd6e14bcf667943892cd4b34cc6420a439abbf3d7d35ef73976dd6f9cbde35a51fa5213f0107f83e3425835d16d3c9146fc9e36ce75a09bb66cdff21dd5a776899f1cb07e282cca27be46510e9c799f0d8db275a6be085d9f3f803218ee3384265bfb1a3640e8ca1 + 026100e6848c31d466fffefc547e3a3b0d3785de6f78b0dd12610843512e495611a0675509b1650b27415009838dd8e68eec6e7530553b637d602424643b33e8bc5b762e1799bc79d56b13251d36d4f201da2182416ce13574e88278ff04467ad602d9 + 026100de994fdf181f02be2bf9e5f5e4e517a94993b827d1eaf609033e3a6a6f2396ae7c44e9eb594cf1044cb3ad32ea258f0c82963b27bb650ed200cde82cb993374be34be5b1c7ead5446a2b82a4486e8c1810a0b01551609fb0841d474bada802bd + 026076ddae751b73a959d0bfb8ff49e7fcd378e9be30652ecefe35c82cb8003bc29cc60ae3809909baf20c95db9516fe680865417111d8b193dbcf30281f1249de57c858bf1ba32f5bb1599800e8398a9ef25c7a642c95261da6f9c17670e97265b1 + 0260732482b837d5f2a9443e23c1aa0106d83e82f6c3424673b5fdc3769c0f992d1c5c93991c7038e882fcda04414df4d7a5f4f698ead87851ce37344b60b72d7b70f9c60cae8566e7a257f8e1bef0e89df6e4c2f9d24d21d9f8889e4c7eccf91751 + 026009050d94493da8f00a4ddbe9c800afe3d44b43f78a48941a79b2814a1f0b81a18a8b2347642a03b27998f5a18de9abc9ae0e54ab8294feac66dc87e854cce6f7278ac2710cb5878b592ffeb1f4f0a1853e4e8d1d0561b6efcc831a296cf7eeaf +""", """ +3081c9 + 0281c100c870feb6ca6b1d2bd9f2dd99e20f1fe2d7e5192de662229dbe162bd1ba66336a7182903ca0b72796cd441c83d24bcdc3e9a2f5e4399c8a043f1c3ddf04754a66d4cfe7b3671a37dd31a9b4c13bfe06ee90f9d94ddaa06de67a52ac863e68f756736ceb014405a6160579640f831dddccc34ad0b05070e3f9954a58d1815813e1b83bcadba814789c87f1ef2ba5d738b793ec456a67360eea1b5faf1c7cc7bf24f3b2a9d0f8958b1096e0f0c335f8888d0c63a51c3c0337214fa3f5efdf6dcc35 + 0203010001 """), }, }) From 9334d71c18a9cda71c4aacbec7129056500f6279 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 27 Jan 2021 13:11:59 +0100 Subject: [PATCH 049/490] Don't consider secp192r1 SECP192R1 is declared in the PSA API specification, but it's an old one that Mbed TLS doesn't support and even OpenSSL doesn't support. We don't have test vectors for it. Just skip it. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 65dbc347d..e9dad1d2d 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -54,7 +54,7 @@ class KeyType: ECC_KEY_SIZES = { 'PSA_ECC_FAMILY_SECP_K1': (192, 224, 256), - 'PSA_ECC_FAMILY_SECP_R1': (192, 225, 256, 384, 521), + 'PSA_ECC_FAMILY_SECP_R1': (225, 256, 384, 521), 'PSA_ECC_FAMILY_SECP_R2': (160,), 'PSA_ECC_FAMILY_SECT_K1': (163, 233, 239, 283, 409, 571), 'PSA_ECC_FAMILY_SECT_R1': (163, 233, 283, 409, 571), From 68261d11d001689ff1a270bd9ffef1cdec85db1d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 27 Jan 2021 18:30:40 +0100 Subject: [PATCH 050/490] Use the base name of the generating script, not the full path Otherwise the generation is sensitive to trivial differences such as running `tests/scripts/generate_psa_tests.py` vs `./tests/scripts/generate_psa_tests.py` vs an absolute path. Signed-off-by: Gilles Peskine --- scripts/framework_dev/test_case.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index a6ed8fc90..c476d1ee9 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -17,6 +17,7 @@ # limitations under the License. import binascii +import os import sys from typing import Any, Iterable, List, Optional import typing_extensions #pylint: disable=import-error @@ -98,7 +99,7 @@ def write_data_file(filename: str, If the file already exists, it is overwritten. """ if caller is None: - caller = sys.argv[0] + caller = os.path.basename(sys.argv[0]) with open(filename, 'w') as out: out.write('# Automatically generated by {}. Do not edit!\n' .format(caller)) From d3a5a233b1b8de196bd8c5929ce315ed0184f274 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 16 Feb 2021 14:29:22 +0100 Subject: [PATCH 051/490] Improve documentation of crypto_knowledge.KeyType Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 25 +++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index e9dad1d2d..fb96ec07e 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -30,27 +30,40 @@ class KeyType: """Analyze a key type. The key type must be specified in PSA syntax. In its simplest form, - this is a string 'PSA_KEY_TYPE_xxx' which is the name of a PSA key + `name` is a string 'PSA_KEY_TYPE_xxx' which is the name of a PSA key type macro. For key types that take arguments, the arguments can be passed either through the optional argument `params` or by passing an expression of the form 'PSA_KEY_TYPE_xxx(param1, param2)' - as the a string. + in `name` as a string. """ self.name = name.strip() + """The key type macro name (``PSA_KEY_TYPE_xxx``). + + For key types constructed from a macro with arguments, this is the + name of the macro, and the arguments are in `self.params`. + """ if params is None: if '(' in self.name: m = re.match(r'(\w+)\s*\((.*)\)\Z', self.name) assert m is not None self.name = m.group(1) params = ','.split(m.group(2)) - if params is None: - self.params = params - else: - self.params = [param.strip() for param in params] + self.params = (None if params is None else + [param.strip() for param in params]) + """The parameters of the key type, if there are any. + + None if the key type is a macro without arguments. + """ self.expression = self.name + """A C expression whose value is the key type encoding.""" if self.params is not None: self.expression += '(' + ', '.join(self.params) + ')' self.private_type = re.sub(r'_PUBLIC_KEY\Z', r'_KEY_PAIR', self.name) + """The key type macro name for the corresponding key pair type. + + For everything other than a public key type, this is the same as + `self.name`. + """ ECC_KEY_SIZES = { 'PSA_ECC_FAMILY_SECP_K1': (192, 224, 256), From 60aa49614a2a09b24c1416489cfbf311daac3aaa Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 16 Feb 2021 18:06:59 +0100 Subject: [PATCH 052/490] Do not require typing_extensions at runtime There are type annotations that indirectly depend on the typing_extensions module (on Python 3.5-3.7: Protocol was added to the core typing module in 3.8). The typing_extensions module is not installed by default, so the code didn't run on a pristine Python installation. To avoid depending on a non-default module, make the dependency on typing_extensions optional. (It's still required to run mypy, but installing mypy takes care of providing typing_extensions.) If it isn't available, provide a substitute definition that's just good enough to get the scripts to run. Move this ugly code to its own module to avoid the temptation of spreading such ugliness all over the place. It's likely to be used in other modules anyway. Signed-off-by: Gilles Peskine --- scripts/framework_dev/test_case.py | 12 +++------ scripts/framework_dev/typing_util.py | 39 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 scripts/framework_dev/typing_util.py diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index c476d1ee9..d01e1432b 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -19,15 +19,9 @@ import binascii import os import sys -from typing import Any, Iterable, List, Optional -import typing_extensions #pylint: disable=import-error - -class Writable(typing_extensions.Protocol): - """Abstract class for typing hints.""" - # pylint: disable=no-self-use,too-few-public-methods,unused-argument - def write(self, text: str) -> Any: - ... +from typing import Iterable, List, Optional +from mbedtls_dev import typing_util def hex_string(data: bytes) -> str: return '"' + binascii.hexlify(data).decode('ascii') + '"' @@ -70,7 +64,7 @@ class TestCase: if self.function is None: raise MissingFunction - def write(self, out: Writable) -> None: + def write(self, out: typing_util.Writable) -> None: """Write the .data file paragraph for this test case. The output starts and ends with a single newline character. If the diff --git a/scripts/framework_dev/typing_util.py b/scripts/framework_dev/typing_util.py new file mode 100644 index 000000000..4c344492c --- /dev/null +++ b/scripts/framework_dev/typing_util.py @@ -0,0 +1,39 @@ +"""Auxiliary definitions used in type annotations. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +# The typing_extensions module is necessary for type annotations that are +# checked with mypy. It is only used for type annotations or to define +# things that are themselves only used for type annotations. It is not +# available on a default Python installation. Therefore, try loading +# what we need from it for the sake of mypy (which depends on, or comes +# with, typing_extensions), and if not define substitutes that lack the +# static type information but are good enough at runtime. +try: + from typing_extensions import Protocol #pylint: disable=import-error +except ImportError: + class Protocol: #type: ignore + #pylint: disable=too-few-public-methods + pass + +class Writable(Protocol): + """Abstract class for typing hints.""" + # pylint: disable=no-self-use,too-few-public-methods,unused-argument + def write(self, text: str) -> Any: + ... From 3404b89c57ed47eca05fc2d32ce3eff21f24854f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Feb 2021 18:04:28 +0100 Subject: [PATCH 053/490] KeyType: do a sanity check on the key type expression Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index fb96ec07e..4ff4f16ef 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -36,6 +36,7 @@ class KeyType: passing an expression of the form 'PSA_KEY_TYPE_xxx(param1, param2)' in `name` as a string. """ + self.name = name.strip() """The key type macro name (``PSA_KEY_TYPE_xxx``). @@ -54,10 +55,13 @@ class KeyType: None if the key type is a macro without arguments. """ + assert re.match(r'PSA_KEY_TYPE_\w+\Z', self.name) + self.expression = self.name """A C expression whose value is the key type encoding.""" if self.params is not None: self.expression += '(' + ', '.join(self.params) + ')' + self.private_type = re.sub(r'_PUBLIC_KEY\Z', r'_KEY_PAIR', self.name) """The key type macro name for the corresponding key pair type. From a6bfa7ef2ef6a3b71db41503d8f80a64e7caba0d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 10 Mar 2021 00:59:53 +0100 Subject: [PATCH 054/490] Add some type annotations Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 27 ++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index b98e40e5f..7ebd8f7c3 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -17,12 +17,13 @@ # limitations under the License. import re +from typing import Dict, Set class PSAMacroCollector: """Collect PSA crypto macro definitions from C header files. """ - def __init__(self, include_intermediate=False): + def __init__(self, include_intermediate: bool = False) -> None: """Set up an object to collect PSA macro definitions. Call the read_file method of the constructed object on each header file. @@ -31,19 +32,19 @@ class PSAMacroCollector: PSA_XXX_BASE that do not designate semantic values. """ self.include_intermediate = include_intermediate - self.statuses = set() - self.key_types = set() - self.key_types_from_curve = {} - self.key_types_from_group = {} - self.ecc_curves = set() - self.dh_groups = set() - self.algorithms = set() - self.hash_algorithms = set() - self.ka_algorithms = set() - self.algorithms_from_hash = {} - self.key_usages = set() + self.statuses = set() #type: Set[str] + self.key_types = set() #type: Set[str] + self.key_types_from_curve = {} #type: Dict[str, str] + self.key_types_from_group = {} #type: Dict[str, str] + self.ecc_curves = set() #type: Set[str] + self.dh_groups = set() #type: Set[str] + self.algorithms = set() #type: Set[str] + self.hash_algorithms = set() #type: Set[str] + self.ka_algorithms = set() #type: Set[str] + self.algorithms_from_hash = {} #type: Dict[str, str] + self.key_usages = set() #type: Set[str] - def is_internal_name(self, name): + def is_internal_name(self, name: str) -> bool: """Whether this is an internal macro. Internal macros will be skipped.""" if not self.include_intermediate: if name.endswith('_BASE') or name.endswith('_NONE'): From 79082016c27f8ff61e29917d2cd7404411bc9db2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 10 Mar 2021 01:02:39 +0100 Subject: [PATCH 055/490] Move PSAMacroEnumerator to macro_collector It's useful for more than test_psa_constant_names. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 109 ++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 7ebd8f7c3..c9e6ec337 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -16,8 +16,115 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools import re -from typing import Dict, Set +from typing import Dict, Iterable, Iterator, List, Set + + +class PSAMacroEnumerator: + """Information about constructors of various PSA Crypto types. + + This includes macro names as well as information about their arguments + when applicable. + + This class only provides ways to enumerate expressions that evaluate to + values of the covered types. Derived classes are expected to populate + the set of known constructors of each kind, as well as populate + `self.arguments_for` for arguments that are not of a kind that is + enumerated here. + """ + + def __init__(self) -> None: + """Set up an empty set of known constructor macros. + """ + self.statuses = set() #type: Set[str] + self.algorithms = set() #type: Set[str] + self.ecc_curves = set() #type: Set[str] + self.dh_groups = set() #type: Set[str] + self.key_types = set() #type: Set[str] + self.key_usage_flags = set() #type: Set[str] + self.hash_algorithms = set() #type: Set[str] + self.mac_algorithms = set() #type: Set[str] + self.ka_algorithms = set() #type: Set[str] + self.kdf_algorithms = set() #type: Set[str] + self.aead_algorithms = set() #type: Set[str] + # macro name -> list of argument names + self.argspecs = {} #type: Dict[str, List[str]] + # argument name -> list of values + self.arguments_for = { + 'mac_length': [], + 'min_mac_length': [], + 'tag_length': [], + 'min_tag_length': [], + } #type: Dict[str, List[str]] + + def gather_arguments(self) -> None: + """Populate the list of values for macro arguments. + + Call this after parsing all the inputs. + """ + self.arguments_for['hash_alg'] = sorted(self.hash_algorithms) + self.arguments_for['mac_alg'] = sorted(self.mac_algorithms) + self.arguments_for['ka_alg'] = sorted(self.ka_algorithms) + self.arguments_for['kdf_alg'] = sorted(self.kdf_algorithms) + self.arguments_for['aead_alg'] = sorted(self.aead_algorithms) + self.arguments_for['curve'] = sorted(self.ecc_curves) + self.arguments_for['group'] = sorted(self.dh_groups) + + @staticmethod + def _format_arguments(name: str, arguments: Iterable[str]) -> str: + """Format a macro call with arguments..""" + return name + '(' + ', '.join(arguments) + ')' + + _argument_split_re = re.compile(r' *, *') + @classmethod + def _argument_split(cls, arguments: str) -> List[str]: + return re.split(cls._argument_split_re, arguments) + + def distribute_arguments(self, name: str) -> Iterator[str]: + """Generate macro calls with each tested argument set. + + If name is a macro without arguments, just yield "name". + If name is a macro with arguments, yield a series of + "name(arg1,...,argN)" where each argument takes each possible + value at least once. + """ + try: + if name not in self.argspecs: + yield name + return + argspec = self.argspecs[name] + if argspec == []: + yield name + '()' + return + argument_lists = [self.arguments_for[arg] for arg in argspec] + arguments = [values[0] for values in argument_lists] + yield self._format_arguments(name, arguments) + # Dear Pylint, enumerate won't work here since we're modifying + # the array. + # pylint: disable=consider-using-enumerate + for i in range(len(arguments)): + for value in argument_lists[i][1:]: + arguments[i] = value + yield self._format_arguments(name, arguments) + arguments[i] = argument_lists[0][0] + except BaseException as e: + raise Exception('distribute_arguments({})'.format(name)) from e + + def generate_expressions(self, names: Iterable[str]) -> Iterator[str]: + """Generate expressions covering values constructed from the given names. + + `names` can be any iterable collection of macro names. + + For example: + * ``generate_expressions(['PSA_ALG_CMAC', 'PSA_ALG_HMAC'])`` + generates ``'PSA_ALG_CMAC'`` as well as ``'PSA_ALG_HMAC(h)'`` for + every known hash algorithm ``h``. + * ``macros.generate_expressions(macros.key_types)`` generates all + key types. + """ + return itertools.chain(*map(self.distribute_arguments, names)) + class PSAMacroCollector: """Collect PSA crypto macro definitions from C header files. From 3915ed231ce5e4df3ef68b49750a3dae8f786722 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 10 Mar 2021 01:25:50 +0100 Subject: [PATCH 056/490] Hook up PSAMacroCollector to PSAMacroEnumerator Make it possible to enumerate the key types, algorithms, etc. collected by PSAMacroCollector. This commit ensures that all fields of PSAMacroEnumerator are filled by code inspection. Testing of the result may reveal more work to be done in later commits. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 46 +++++++++++++++--------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index c9e6ec337..a2192baf4 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -126,7 +126,7 @@ class PSAMacroEnumerator: return itertools.chain(*map(self.distribute_arguments, names)) -class PSAMacroCollector: +class PSAMacroCollector(PSAMacroEnumerator): """Collect PSA crypto macro definitions from C header files. """ @@ -138,18 +138,11 @@ class PSAMacroCollector: * include_intermediate: if true, include intermediate macros such as PSA_XXX_BASE that do not designate semantic values. """ + super().__init__() self.include_intermediate = include_intermediate - self.statuses = set() #type: Set[str] - self.key_types = set() #type: Set[str] self.key_types_from_curve = {} #type: Dict[str, str] self.key_types_from_group = {} #type: Dict[str, str] - self.ecc_curves = set() #type: Set[str] - self.dh_groups = set() #type: Set[str] - self.algorithms = set() #type: Set[str] - self.hash_algorithms = set() #type: Set[str] - self.ka_algorithms = set() #type: Set[str] self.algorithms_from_hash = {} #type: Dict[str, str] - self.key_usages = set() #type: Set[str] def is_internal_name(self, name: str) -> bool: """Whether this is an internal macro. Internal macros will be skipped.""" @@ -160,6 +153,30 @@ class PSAMacroCollector: return True return name.endswith('_FLAG') or name.endswith('_MASK') + def record_algorithm_subtype(self, name: str, expansion: str) -> None: + """Record the subtype of an algorithm constructor. + + Given a ``PSA_ALG_xxx`` macro name and its expansion, if the algorithm + is of a subtype that is tracked in its own set, add it to the relevant + set. + """ + # This code is very ad hoc and fragile. It should be replaced by + # something more robust. + if re.match(r'MAC(?:_|\Z)', name): + self.mac_algorithms.add(name) + elif re.match(r'KDF(?:_|\Z)', name): + self.kdf_algorithms.add(name) + elif re.search(r'0x020000[0-9A-Fa-f]{2}', expansion): + self.hash_algorithms.add(name) + elif re.search(r'0x03[0-9A-Fa-f]{6}', expansion): + self.mac_algorithms.add(name) + elif re.search(r'0x05[0-9A-Fa-f]{6}', expansion): + self.aead_algorithms.add(name) + elif re.search(r'0x09[0-9A-Fa-f]{2}0000', expansion): + self.ka_algorithms.add(name) + elif re.search(r'0x08[0-9A-Fa-f]{6}', expansion): + self.kdf_algorithms.add(name) + # "#define" followed by a macro name with either no parameters # or a single parameter and a non-empty expansion. # Grab the macro name in group 1, the parameter name if any in group 2 @@ -180,6 +197,8 @@ class PSAMacroCollector: return name, parameter, expansion = m.groups() expansion = re.sub(r'/\*.*?\*/|//.*', r' ', expansion) + if parameter: + self.argspecs[name] = [parameter] if re.match(self._deprecated_definition_re, expansion): # Skip deprecated values, which are assumed to be # backward compatibility aliases that share @@ -207,12 +226,7 @@ class PSAMacroCollector: # Ad hoc skipping of duplicate names for some numerical values return self.algorithms.add(name) - # Ad hoc detection of hash algorithms - if re.search(r'0x020000[0-9A-Fa-f]{2}', expansion): - self.hash_algorithms.add(name) - # Ad hoc detection of key agreement algorithms - if re.search(r'0x09[0-9A-Fa-f]{2}0000', expansion): - self.ka_algorithms.add(name) + self.record_algorithm_subtype(name, expansion) elif name.startswith('PSA_ALG_') and parameter == 'hash_alg': if name in ['PSA_ALG_DSA', 'PSA_ALG_ECDSA']: # A naming irregularity @@ -221,7 +235,7 @@ class PSAMacroCollector: tester = name[:8] + 'IS_' + name[8:] self.algorithms_from_hash[name] = tester elif name.startswith('PSA_KEY_USAGE_') and not parameter: - self.key_usages.add(name) + self.key_usage_flags.add(name) else: # Other macro without parameter return From 334233f3061f51aa7e48e1cea901b1f477256725 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Feb 2021 14:34:37 +0100 Subject: [PATCH 057/490] New python module to encode a PSA key for storage Construct an object given the attributes and material for a PSA crypto key and get the Mbed TLS storage representation. The code to generate the storage representation was written based on the specification in docs/architecture/mbed-crypto-storage-specification.md, without looking at the code. The data in the unit tests is from the AES-128 format_storage_data_check test case in test_suite_psa_crypto_persistent_key.data, tweaked manually. This commit creates a basic framework for using symbolic values for attributes, but does not yet implement obtaining the corresponding numerical values from an external source. Signed-off-by: Gilles Peskine --- scripts/framework_dev/psa_storage.py | 184 +++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 scripts/framework_dev/psa_storage.py diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py new file mode 100644 index 000000000..3e15c6726 --- /dev/null +++ b/scripts/framework_dev/psa_storage.py @@ -0,0 +1,184 @@ +"""Knowledge about the PSA key store as implemented in Mbed TLS. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import struct +from typing import Optional, Union +import unittest + + +class Expr: + """Representation of a C expression with a known or knowable numerical value.""" + def __init__(self, content: Union[int, str]): + if isinstance(content, int): + digits = 8 if content > 0xffff else 4 + self.string = '{0:#0{1}x}'.format(content, digits + 2) + self.value_if_known = content #type: Optional[int] + else: + self.string = content + self.value_if_known = None + + value_cache = { + # Hard-coded for initial testing + 'PSA_KEY_LIFETIME_PERSISTENT': 0x00000001, + 'PSA_KEY_TYPE_RAW_DATA': 0x1001, + } + + def update_cache(self) -> None: + pass #not implemented yet + + @staticmethod + def normalize(string: str) -> str: + """Put the given C expression in a canonical form. + + This function is only intended to give correct results for the + relatively simple kind of C expression typically used with this + module. + """ + return re.sub(r'\s+', r'', string) + + def value(self) -> int: + """Return the numerical value of the expression.""" + if self.value_if_known is None: + if re.match(r'([0-9]+|0x[0-9a-f]+)\Z', self.string, re.I): + return int(self.string, 0) + normalized = self.normalize(self.string) + if normalized not in self.value_cache: + self.update_cache() + self.value_if_known = self.value_cache[normalized] + return self.value_if_known + +Exprable = Union[str, int, Expr] +"""Something that can be converted to a C expression with a known numerical value.""" + +def as_expr(thing: Exprable) -> Expr: + """Return an `Expr` object for `thing`. + + If `thing` is already an `Expr` object, return it. Otherwise build a new + `Expr` object from `thing`. `thing` can be an integer or a string that + contains a C expression. + """ + if isinstance(thing, Expr): + return thing + else: + return Expr(thing) + + +class Key: + """Representation of a PSA crypto key object and its storage encoding. + """ + + LATEST_VERSION = 0 + """The latest version of the storage format.""" + + def __init__(self, *, + version: Optional[int] = None, + id: Optional[int] = None, #pylint: disable=redefined-builtin + lifetime: Exprable = 'PSA_KEY_LIFETIME_PERSISTENT', + type: Exprable, #pylint: disable=redefined-builtin + bits: int, + usage: Exprable, alg: Exprable, alg2: Exprable, + material: bytes #pylint: disable=used-before-assignment + ) -> None: + self.version = self.LATEST_VERSION if version is None else version + self.id = id #pylint: disable=invalid-name #type: Optional[int] + self.lifetime = as_expr(lifetime) #type: Expr + self.type = as_expr(type) #type: Expr + self.bits = bits #type: int + self.usage = as_expr(usage) #type: Expr + self.alg = as_expr(alg) #type: Expr + self.alg2 = as_expr(alg2) #type: Expr + self.material = material #type: bytes + + MAGIC = b'PSA\000KEY\000' + + @staticmethod + def pack( + fmt: str, + *args: Union[int, Expr] + ) -> bytes: #pylint: disable=used-before-assignment + """Pack the given arguments into a byte string according to the given format. + + This function is similar to `struct.pack`, but with the following differences: + * All integer values are encoded with standard sizes and in + little-endian representation. `fmt` must not include an endianness + prefix. + * Arguments can be `Expr` objects instead of integers. + * Only integer-valued elements are supported. + """ + return struct.pack('<' + fmt, # little-endian, standard sizes + *[arg.value() if isinstance(arg, Expr) else arg + for arg in args]) + + def bytes(self) -> bytes: + """Return the representation of the key in storage as a byte array. + + This is the content of the PSA storage file. When PSA storage is + implemented over stdio files, this does not include any wrapping made + by the PSA-storage-over-stdio-file implementation. + """ + header = self.MAGIC + self.pack('L', self.version) + if self.version == 0: + attributes = self.pack('LHHLLL', + self.lifetime, self.type, self.bits, + self.usage, self.alg, self.alg2) + material = self.pack('L', len(self.material)) + self.material + else: + raise NotImplementedError + return header + attributes + material + + def hex(self) -> str: + """Return the representation of the key as a hexadecimal string. + + This is the hexadecimal representation of `self.bytes`. + """ + return self.bytes().hex() + + +class TestKey(unittest.TestCase): + # pylint: disable=line-too-long + """A few smoke tests for the functionality of the `Key` class.""" + + def test_numerical(self): + key = Key(version=0, + id=1, lifetime=0x00000001, + type=0x2400, bits=128, + usage=0x00000300, alg=0x05500200, alg2=0x04c01000, + material=b'@ABCDEFGHIJKLMNO') + expected_hex = '505341004b45590000000000010000000024800000030000000250050010c00410000000404142434445464748494a4b4c4d4e4f' + self.assertEqual(key.bytes(), bytes.fromhex(expected_hex)) + self.assertEqual(key.hex(), expected_hex) + + def test_names(self): + length = 0xfff8 // 8 # PSA_MAX_KEY_BITS in bytes + key = Key(version=0, + id=1, lifetime='PSA_KEY_LIFETIME_PERSISTENT', + type='PSA_KEY_TYPE_RAW_DATA', bits=length*8, + usage=0, alg=0, alg2=0, + material=b'\x00' * length) + expected_hex = '505341004b45590000000000010000000110f8ff000000000000000000000000ff1f0000' + '00' * length + self.assertEqual(key.bytes(), bytes.fromhex(expected_hex)) + self.assertEqual(key.hex(), expected_hex) + + def test_defaults(self): + key = Key(type=0x1001, bits=8, + usage=0, alg=0, alg2=0, + material=b'\x2a') + expected_hex = '505341004b455900000000000100000001100800000000000000000000000000010000002a' + self.assertEqual(key.bytes(), bytes.fromhex(expected_hex)) + self.assertEqual(key.hex(), expected_hex) From 03f6ac05015692337d6ea16b488efcfa6bace2de Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 10 Mar 2021 01:32:38 +0100 Subject: [PATCH 058/490] Obtain the values of expressions by running C code Signed-off-by: Gilles Peskine --- scripts/framework_dev/psa_storage.py | 29 +++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index 3e15c6726..3a740072e 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -18,12 +18,15 @@ import re import struct -from typing import Optional, Union +from typing import Dict, List, Optional, Set, Union import unittest +from mbedtls_dev import c_build_helper + class Expr: """Representation of a C expression with a known or knowable numerical value.""" + def __init__(self, content: Union[int, str]): if isinstance(content, int): digits = 8 if content > 0xffff else 4 @@ -31,16 +34,28 @@ class Expr: self.value_if_known = content #type: Optional[int] else: self.string = content + self.unknown_values.add(self.normalize(content)) self.value_if_known = None - value_cache = { - # Hard-coded for initial testing - 'PSA_KEY_LIFETIME_PERSISTENT': 0x00000001, - 'PSA_KEY_TYPE_RAW_DATA': 0x1001, - } + value_cache = {} #type: Dict[str, int] + """Cache of known values of expressions.""" + + unknown_values = set() #type: Set[str] + """Expressions whose values are not present in `value_cache` yet.""" def update_cache(self) -> None: - pass #not implemented yet + """Update `value_cache` for expressions registered in `unknown_values`.""" + expressions = sorted(self.unknown_values) + values = c_build_helper.get_c_expression_values( + 'unsigned long', '%lu', + expressions, + header=""" + #include + """, + include_path=['include']) #type: List[str] + for e, v in zip(expressions, values): + self.value_cache[e] = int(v, 0) + self.unknown_values.clear() @staticmethod def normalize(string: str) -> str: From 1855f8ffeaa6a5df7e9fde854a71dddcdf33bf74 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 10 Mar 2021 15:07:16 +0100 Subject: [PATCH 059/490] Cover all key types Generate test cases for all key types. These test cases cover the key representation (checked with export) and the encoding of the key type and the bit-size. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 4ff4f16ef..02c09608d 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -19,14 +19,14 @@ This module is entirely based on the PSA API. # limitations under the License. import re -from typing import List, Optional, Tuple +from typing import Iterable, Optional, Tuple from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA class KeyType: """Knowledge about a PSA key type.""" - def __init__(self, name: str, params: Optional[List[str]] = None): + def __init__(self, name: str, params: Optional[Iterable[str]] = None): """Analyze a key type. The key type must be specified in PSA syntax. In its simplest form, From 46d1eca3c981ee92039d953581e6f0caf973f36b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 24 Feb 2021 21:49:40 +0100 Subject: [PATCH 060/490] New elliptic curve family: twisted Edwards Add an elliptic curve family for the twisted Edwards curves Edwards25519 and Edwards448 ("Goldilocks"). As with Montgomery curves, since these are the only two curves in common use, the family has a generic name. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 02c09608d..642e7254f 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -78,6 +78,7 @@ class KeyType: 'PSA_ECC_FAMILY_SECT_R2': (163,), 'PSA_ECC_FAMILY_BRAINPOOL_P_R1': (160, 192, 224, 256, 320, 384, 512), 'PSA_ECC_FAMILY_MONTGOMERY': (255, 448), + 'PSA_ECC_FAMILY_TWISTED_EDWARDS': (256, 448), } KEY_TYPE_SIZES = { 'PSA_KEY_TYPE_AES': (128, 192, 256), # exhaustive From cf3374a7873a36e9db9ede3e3cc29eb1923d551f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 16 Mar 2021 18:25:14 +0100 Subject: [PATCH 061/490] Consistently describe Ed25519 as a 255-bit curve The coordinates are over $F_{2^{255}-19}$, so by the general definition of the bit size associated with the curve in the specification, the value for size attribute of keys is 255. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 642e7254f..500aceafd 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -78,7 +78,7 @@ class KeyType: 'PSA_ECC_FAMILY_SECT_R2': (163,), 'PSA_ECC_FAMILY_BRAINPOOL_P_R1': (160, 192, 224, 256, 320, 384, 512), 'PSA_ECC_FAMILY_MONTGOMERY': (255, 448), - 'PSA_ECC_FAMILY_TWISTED_EDWARDS': (256, 448), + 'PSA_ECC_FAMILY_TWISTED_EDWARDS': (255, 448), } KEY_TYPE_SIZES = { 'PSA_KEY_TYPE_AES': (128, 192, 256), # exhaustive From ae9c3211156e830645c652613dd5979674bbc2a6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 16 Mar 2021 18:32:24 +0100 Subject: [PATCH 062/490] Add key material for twisted Edwards curves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the test keys from RFC 8032 (§7.1 Ed25519 "TEST 1", §7.4 Ed448 "Blank"). This replaces the generic byte-sized data used for unknown key types which no longer works now that Ed25519 is considered to have 255 bits. Re-generate the automatically generated test data accordingly. Signed-off-by: Gilles Peskine --- scripts/framework_dev/asymmetric_key_data.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/framework_dev/asymmetric_key_data.py b/scripts/framework_dev/asymmetric_key_data.py index 1efe44959..6fd6223f3 100644 --- a/scripts/framework_dev/asymmetric_key_data.py +++ b/scripts/framework_dev/asymmetric_key_data.py @@ -123,6 +123,12 @@ ASYMMETRIC_KEY_DATA = construct_asymmetric_key_data({ 448: ("e4e49f52686f9ee3b638528f721f1596196ffd0a1cddb64c3f216f06541805cfeb1a286dc78018095cdfec050e8007b5f4908962ba20d6c1", "c0d3a5a2b416a573dc9909f92f134ac01323ab8f8e36804e578588ba2d09fe7c3e737f771ca112825b548a0ffded6d6a2fd09a3e77dec30e"), }, + 'ECC(PSA_ECC_FAMILY_TWISTED_EDWARDS)': { + 255: ("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60", + "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a"), + 448: ("6c82a562cb808d10d632be89c8513ebf6c929f34ddfa8c9f63c9960ef6e348a3528c8a3fcc2f044e39a3fc5b94492f8f032e7549a20098f95b", + "5fd7449b59b461fd2ce787ec616ad46a1da1342485a70e1f8a0ea75d80e96778edf124769b46c7061bd6783df1e50f6cd1fa1abeafe8256180"), + }, 'RSA': { 1024: (""" 3082025e From 382fdd0fdd11d2b909f3e605c6db3888ceb05f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Mon, 3 May 2021 11:02:56 +0200 Subject: [PATCH 063/490] Add new key types to crypto_knowledge.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- scripts/framework_dev/crypto_knowledge.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 500aceafd..aa5279027 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -89,6 +89,9 @@ class KeyType: 'PSA_KEY_TYPE_DERIVE': (120, 128), # sample 'PSA_KEY_TYPE_DES': (64, 128, 192), # exhaustive 'PSA_KEY_TYPE_HMAC': (128, 160, 224, 256, 384, 512), # standard size for each supported hash + 'PSA_KEY_TYPE_PASSWORD': (48, 168, 336), # sample + 'PSA_KEY_TYPE_PASSWORD_HASH': (128, 256), # sample + 'PSA_KEY_TYPE_PEPPER': (128, 256), # sample 'PSA_KEY_TYPE_RAW_DATA': (8, 40, 128), # sample 'PSA_KEY_TYPE_RSA_KEY_PAIR': (1024, 1536), # small sample } From 86bfa78a5e5142322fec6b881464128054bcdc99 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 22 Apr 2021 00:20:47 +0200 Subject: [PATCH 064/490] Allow running source file generators from a subdirectory Signed-off-by: Gilles Peskine --- scripts/framework_dev/build_tree.py | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 scripts/framework_dev/build_tree.py diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py new file mode 100644 index 000000000..772410473 --- /dev/null +++ b/scripts/framework_dev/build_tree.py @@ -0,0 +1,38 @@ +"""Mbed TLS build tree information and manipulation. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +def looks_like_mbedtls_root(path: str) -> bool: + """Whether the given directory looks like the root of the Mbed TLS source tree.""" + return all(os.path.isdir(os.path.join(path, subdir)) + for subdir in ['include', 'library', 'programs', 'tests']) + +def chdir_to_root() -> None: + """Detect the root of the Mbed TLS source tree and change to it. + + The current directory must be up to two levels deep inside an Mbed TLS + source tree. + """ + for d in [os.path.curdir, + os.path.pardir, + os.path.join(os.path.pardir, os.path.pardir)]: + if looks_like_mbedtls_root(d): + os.chdir(d) + return + raise Exception('Mbed TLS source tree not found') From 37a81c23cda947f524165e2cefe9e36921a6d815 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 23 Apr 2021 22:07:25 +0200 Subject: [PATCH 065/490] If $CC looks like MSVC, use a compatible command line syntax Signed-off-by: Gilles Peskine --- scripts/framework_dev/c_build_helper.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 5c587a16b..9215d648c 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -108,6 +108,10 @@ def get_c_expression_values( * ``keep_c``: if true, keep the temporary C file (presumably for debugging purposes). + Use the C compiler specified by the ``CC`` environment variable, defaulting + to ``cc``. If ``CC`` looks like MSVC, use its command line syntax, + otherwise assume the compiler supports Unix traditional ``-I`` and ``-o``. + Return the list of values of the ``expressions``. """ if include_path is None: @@ -124,9 +128,14 @@ def get_c_expression_values( ) c_file.close() cc = os.getenv('CC', 'cc') - subprocess.check_call([cc] + - ['-I' + dir for dir in include_path] + - ['-o', exe_name, c_name]) + cc_is_msvc = ('\\' + cc).lower().endswith('\\cl.exe') + cmd = [cc] + cmd += ['-I' + dir for dir in include_path] + # MSVC doesn't support -o to specify the output file, but we use its + # default output file name. + if not cc_is_msvc: + cmd += ['-o', exe_name] + subprocess.check_call(cmd + [c_name]) if keep_c: sys.stderr.write('List of {} tests kept at {}\n' .format(caller, c_name)) From c0f85315b62326c288403f03e0a59d9b5fe1d2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Thu, 6 May 2021 15:14:16 +0200 Subject: [PATCH 066/490] Specify output name for MSVC as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The output file was being created in the working directory, instead of the temp directory. Signed-off-by: Bence Szépkúti --- scripts/framework_dev/c_build_helper.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 9215d648c..ddef10fbe 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -95,7 +95,7 @@ def get_c_expression_values( caller=__name__, file_label='', header='', include_path=None, keep_c=False, -): # pylint: disable=too-many-arguments +): # pylint: disable=too-many-arguments, too-many-locals """Generate and run a program to print out numerical values for expressions. * ``cast_to``: a C type. @@ -131,10 +131,9 @@ def get_c_expression_values( cc_is_msvc = ('\\' + cc).lower().endswith('\\cl.exe') cmd = [cc] cmd += ['-I' + dir for dir in include_path] - # MSVC doesn't support -o to specify the output file, but we use its - # default output file name. - if not cc_is_msvc: - cmd += ['-o', exe_name] + # MSVC has deprecated using -o to specify the output file. + output_opt = '-Fe' if cc_is_msvc else '-o' + cmd += [output_opt + exe_name] subprocess.check_call(cmd + [c_name]) if keep_c: sys.stderr.write('List of {} tests kept at {}\n' From 1ff00fe5102c99fe71771ce892e6af32da6d4261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Fri, 7 May 2021 15:10:39 +0200 Subject: [PATCH 067/490] Improve MSVC detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- scripts/framework_dev/c_build_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index ddef10fbe..e9e65c30a 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -128,7 +128,7 @@ def get_c_expression_values( ) c_file.close() cc = os.getenv('CC', 'cc') - cc_is_msvc = ('\\' + cc).lower().endswith('\\cl.exe') + cc_is_msvc = os.path.split(cc)[1].lower() in ('cl', 'cl.exe') cmd = [cc] cmd += ['-I' + dir for dir in include_path] # MSVC has deprecated using -o to specify the output file. From 687a4602f4bc4b2e65d0e5f18ad0cfdcb2ff00d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 10 May 2021 20:50:07 +0200 Subject: [PATCH 068/490] Detect MSVC without relying on compiler filename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- scripts/framework_dev/c_build_helper.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index e9e65c30a..569ebf064 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -89,6 +89,7 @@ int main(void) } ''') +_cc_is_msvc = None #pylint: disable=invalid-name def get_c_expression_values( cast_to, printf_format, expressions, @@ -128,11 +129,20 @@ def get_c_expression_values( ) c_file.close() cc = os.getenv('CC', 'cc') - cc_is_msvc = os.path.split(cc)[1].lower() in ('cl', 'cl.exe') cmd = [cc] + + global _cc_is_msvc #pylint: disable=global-statement,invalid-name + if _cc_is_msvc is None: + proc = subprocess.Popen(cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + universal_newlines=True) + _cc_is_msvc = 'Microsoft (R) C/C++ Optimizing Compiler' in \ + proc.communicate()[1] + cmd += ['-I' + dir for dir in include_path] # MSVC has deprecated using -o to specify the output file. - output_opt = '-Fe' if cc_is_msvc else '-o' + output_opt = '-Fe' if _cc_is_msvc else '-o' cmd += [output_opt + exe_name] subprocess.check_call(cmd + [c_name]) if keep_c: From cdc22fde061f62eaedc2f437fe72c683bb1bf809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 10 May 2021 23:56:29 +0200 Subject: [PATCH 069/490] Clean up object files produced by MSVC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- scripts/framework_dev/c_build_helper.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 569ebf064..4ce96ebfa 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -45,9 +45,11 @@ def create_c_file(file_label): suffix='.c') exe_suffix = '.exe' if platform.system() == 'Windows' else '' exe_name = c_name[:-2] + exe_suffix + obj_name = c_name[:-2] + '.obj' remove_file_if_exists(exe_name) + remove_file_if_exists(obj_name) c_file = os.fdopen(c_fd, 'w', encoding='ascii') - return c_file, c_name, exe_name + return c_file, c_name, exe_name, obj_name def generate_c_printf_expressions(c_file, cast_to, printf_format, expressions): """Generate C instructions to print the value of ``expressions``. @@ -120,7 +122,7 @@ def get_c_expression_values( c_name = None exe_name = None try: - c_file, c_name, exe_name = create_c_file(file_label) + c_file, c_name, exe_name, obj_name = create_c_file(file_label) generate_c_file( c_file, caller, header, lambda c_file: generate_c_printf_expressions(c_file, @@ -141,15 +143,20 @@ def get_c_expression_values( proc.communicate()[1] cmd += ['-I' + dir for dir in include_path] - # MSVC has deprecated using -o to specify the output file. - output_opt = '-Fe' if _cc_is_msvc else '-o' - cmd += [output_opt + exe_name] + if _cc_is_msvc: + # MSVC has deprecated using -o to specify the output file, + # and produces an object file in the working directory by default. + cmd += ['-Fe' + exe_name, '-Fo' + obj_name] + else: + cmd += ['-o' + exe_name] subprocess.check_call(cmd + [c_name]) if keep_c: sys.stderr.write('List of {} tests kept at {}\n' .format(caller, c_name)) else: os.remove(c_name) + if _cc_is_msvc: + os.remove(obj_name) output = subprocess.check_output([exe_name]) return output.decode('ascii').strip().split('\n') finally: From 6696996ab4f4bd6c8ab2baee34ef2ab8c9b38456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Wed, 12 May 2021 09:49:45 +0200 Subject: [PATCH 070/490] Add notice of caching whether the compiler is MSVC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- scripts/framework_dev/c_build_helper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 4ce96ebfa..d02deb35a 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -115,6 +115,9 @@ def get_c_expression_values( to ``cc``. If ``CC`` looks like MSVC, use its command line syntax, otherwise assume the compiler supports Unix traditional ``-I`` and ``-o``. + NOTE: This function only checks the identity of the compiler referred to by + ``CC`` on its first invocation, and caches the result. + Return the list of values of the ``expressions``. """ if include_path is None: From cba30c422687ec78287b7c53803a70d641c35034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Wed, 12 May 2021 10:11:53 +0200 Subject: [PATCH 071/490] Move object file handling out of create_c_file() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- scripts/framework_dev/c_build_helper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index d02deb35a..54c4e93d1 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -45,11 +45,9 @@ def create_c_file(file_label): suffix='.c') exe_suffix = '.exe' if platform.system() == 'Windows' else '' exe_name = c_name[:-2] + exe_suffix - obj_name = c_name[:-2] + '.obj' remove_file_if_exists(exe_name) - remove_file_if_exists(obj_name) c_file = os.fdopen(c_fd, 'w', encoding='ascii') - return c_file, c_name, exe_name, obj_name + return c_file, c_name, exe_name def generate_c_printf_expressions(c_file, cast_to, printf_format, expressions): """Generate C instructions to print the value of ``expressions``. @@ -124,8 +122,9 @@ def get_c_expression_values( include_path = [] c_name = None exe_name = None + obj_name = None try: - c_file, c_name, exe_name, obj_name = create_c_file(file_label) + c_file, c_name, exe_name = create_c_file(file_label) generate_c_file( c_file, caller, header, lambda c_file: generate_c_printf_expressions(c_file, @@ -149,6 +148,7 @@ def get_c_expression_values( if _cc_is_msvc: # MSVC has deprecated using -o to specify the output file, # and produces an object file in the working directory by default. + obj_name = exe_name[:-4] + '.obj' cmd += ['-Fe' + exe_name, '-Fo' + obj_name] else: cmd += ['-o' + exe_name] From 8403239d5b262056a968274664deab7139dbedff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Wed, 12 May 2021 10:21:39 +0200 Subject: [PATCH 072/490] Remove object file in finally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- scripts/framework_dev/c_build_helper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 54c4e93d1..833db465f 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -158,9 +158,8 @@ def get_c_expression_values( .format(caller, c_name)) else: os.remove(c_name) - if _cc_is_msvc: - os.remove(obj_name) output = subprocess.check_output([exe_name]) return output.decode('ascii').strip().split('\n') finally: remove_file_if_exists(exe_name) + remove_file_if_exists(obj_name) From 3e5b72a06ff75703aa18a68515ff4f336ff871dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Wed, 12 May 2021 10:28:30 +0200 Subject: [PATCH 073/490] Remove caching of cc_is_msvc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- scripts/framework_dev/c_build_helper.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 833db465f..18c1e7606 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -89,7 +89,6 @@ int main(void) } ''') -_cc_is_msvc = None #pylint: disable=invalid-name def get_c_expression_values( cast_to, printf_format, expressions, @@ -113,9 +112,6 @@ def get_c_expression_values( to ``cc``. If ``CC`` looks like MSVC, use its command line syntax, otherwise assume the compiler supports Unix traditional ``-I`` and ``-o``. - NOTE: This function only checks the identity of the compiler referred to by - ``CC`` on its first invocation, and caches the result. - Return the list of values of the ``expressions``. """ if include_path is None: @@ -135,17 +131,15 @@ def get_c_expression_values( cc = os.getenv('CC', 'cc') cmd = [cc] - global _cc_is_msvc #pylint: disable=global-statement,invalid-name - if _cc_is_msvc is None: - proc = subprocess.Popen(cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - universal_newlines=True) - _cc_is_msvc = 'Microsoft (R) C/C++ Optimizing Compiler' in \ - proc.communicate()[1] + proc = subprocess.Popen(cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + universal_newlines=True) + cc_is_msvc = 'Microsoft (R) C/C++ Optimizing Compiler' in \ + proc.communicate()[1] cmd += ['-I' + dir for dir in include_path] - if _cc_is_msvc: + if cc_is_msvc: # MSVC has deprecated using -o to specify the output file, # and produces an object file in the working directory by default. obj_name = exe_name[:-4] + '.obj' From 802570683d50aea19e4b1ae2c335e177758a8d4a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Mar 2021 19:09:05 +0200 Subject: [PATCH 074/490] Move InputsForTest to macro_collector.py This is useful to generate PSA tests for more than constant names. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 215 ++++++++++++++++++++++- 1 file changed, 214 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index a2192baf4..0c9a9a5b2 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -18,7 +18,55 @@ import itertools import re -from typing import Dict, Iterable, Iterator, List, Set +from typing import Dict, Iterable, Iterator, List, Optional, Pattern, Set, Tuple, Union + + +class ReadFileLineException(Exception): + def __init__(self, filename: str, line_number: Union[int, str]) -> None: + message = 'in {} at {}'.format(filename, line_number) + super(ReadFileLineException, self).__init__(message) + self.filename = filename + self.line_number = line_number + + +class read_file_lines: + # Dear Pylint, conventionally, a context manager class name is lowercase. + # pylint: disable=invalid-name,too-few-public-methods + """Context manager to read a text file line by line. + + ``` + with read_file_lines(filename) as lines: + for line in lines: + process(line) + ``` + is equivalent to + ``` + with open(filename, 'r') as input_file: + for line in input_file: + process(line) + ``` + except that if process(line) raises an exception, then the read_file_lines + snippet annotates the exception with the file name and line number. + """ + def __init__(self, filename: str, binary: bool = False) -> None: + self.filename = filename + self.line_number = 'entry' #type: Union[int, str] + self.generator = None #type: Optional[Iterable[Tuple[int, str]]] + self.binary = binary + def __enter__(self) -> 'read_file_lines': + self.generator = enumerate(open(self.filename, + 'rb' if self.binary else 'r')) + return self + def __iter__(self) -> Iterator[str]: + assert self.generator is not None + for line_number, content in self.generator: + self.line_number = line_number + yield content + self.line_number = 'exit' + def __exit__(self, exc_type, exc_value, exc_traceback) -> None: + if exc_type is not None: + raise ReadFileLineException(self.filename, self.line_number) \ + from exc_value class PSAMacroEnumerator: @@ -251,3 +299,168 @@ class PSAMacroCollector(PSAMacroEnumerator): m = re.search(self._continued_line_re, line) line = re.sub(self._nonascii_re, rb'', line).decode('ascii') self.read_line(line) + + +class InputsForTest(PSAMacroEnumerator): + # pylint: disable=too-many-instance-attributes + """Accumulate information about macros to test. +enumerate + This includes macro names as well as information about their arguments + when applicable. + """ + + def __init__(self) -> None: + super().__init__() + self.all_declared = set() #type: Set[str] + # Sets of names per type + self.statuses.add('PSA_SUCCESS') + self.algorithms.add('0xffffffff') + self.ecc_curves.add('0xff') + self.dh_groups.add('0xff') + self.key_types.add('0xffff') + self.key_usage_flags.add('0x80000000') + + # Hard-coded values for unknown algorithms + # + # These have to have values that are correct for their respective + # PSA_ALG_IS_xxx macros, but are also not currently assigned and are + # not likely to be assigned in the near future. + self.hash_algorithms.add('0x020000fe') # 0x020000ff is PSA_ALG_ANY_HASH + self.mac_algorithms.add('0x03007fff') + self.ka_algorithms.add('0x09fc0000') + self.kdf_algorithms.add('0x080000ff') + # For AEAD algorithms, the only variability is over the tag length, + # and this only applies to known algorithms, so don't test an + # unknown algorithm. + + # Identifier prefixes + self.table_by_prefix = { + 'ERROR': self.statuses, + 'ALG': self.algorithms, + 'ECC_CURVE': self.ecc_curves, + 'DH_GROUP': self.dh_groups, + 'KEY_TYPE': self.key_types, + 'KEY_USAGE': self.key_usage_flags, + } #type: Dict[str, Set[str]] + # Test functions + self.table_by_test_function = { + # Any function ending in _algorithm also gets added to + # self.algorithms. + 'key_type': [self.key_types], + 'block_cipher_key_type': [self.key_types], + 'stream_cipher_key_type': [self.key_types], + 'ecc_key_family': [self.ecc_curves], + 'ecc_key_types': [self.ecc_curves], + 'dh_key_family': [self.dh_groups], + 'dh_key_types': [self.dh_groups], + 'hash_algorithm': [self.hash_algorithms], + 'mac_algorithm': [self.mac_algorithms], + 'cipher_algorithm': [], + 'hmac_algorithm': [self.mac_algorithms], + 'aead_algorithm': [self.aead_algorithms], + 'key_derivation_algorithm': [self.kdf_algorithms], + 'key_agreement_algorithm': [self.ka_algorithms], + 'asymmetric_signature_algorithm': [], + 'asymmetric_signature_wildcard': [self.algorithms], + 'asymmetric_encryption_algorithm': [], + 'other_algorithm': [], + } #type: Dict[str, List[Set[str]]] + self.arguments_for['mac_length'] += ['1', '63'] + self.arguments_for['min_mac_length'] += ['1', '63'] + self.arguments_for['tag_length'] += ['1', '63'] + self.arguments_for['min_tag_length'] += ['1', '63'] + + def get_names(self, type_word: str) -> Set[str]: + """Return the set of known names of values of the given type.""" + return { + 'status': self.statuses, + 'algorithm': self.algorithms, + 'ecc_curve': self.ecc_curves, + 'dh_group': self.dh_groups, + 'key_type': self.key_types, + 'key_usage': self.key_usage_flags, + }[type_word] + + # Regex for interesting header lines. + # Groups: 1=macro name, 2=type, 3=argument list (optional). + _header_line_re = \ + re.compile(r'#define +' + + r'(PSA_((?:(?:DH|ECC|KEY)_)?[A-Z]+)_\w+)' + + r'(?:\(([^\n()]*)\))?') + # Regex of macro names to exclude. + _excluded_name_re = re.compile(r'_(?:GET|IS|OF)_|_(?:BASE|FLAG|MASK)\Z') + # Additional excluded macros. + _excluded_names = set([ + # Macros that provide an alternative way to build the same + # algorithm as another macro. + 'PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG', + 'PSA_ALG_FULL_LENGTH_MAC', + # Auxiliary macro whose name doesn't fit the usual patterns for + # auxiliary macros. + 'PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG_CASE', + ]) + def parse_header_line(self, line: str) -> None: + """Parse a C header line, looking for "#define PSA_xxx".""" + m = re.match(self._header_line_re, line) + if not m: + return + name = m.group(1) + self.all_declared.add(name) + if re.search(self._excluded_name_re, name) or \ + name in self._excluded_names: + return + dest = self.table_by_prefix.get(m.group(2)) + if dest is None: + return + dest.add(name) + if m.group(3): + self.argspecs[name] = self._argument_split(m.group(3)) + + _nonascii_re = re.compile(rb'[^\x00-\x7f]+') #type: Pattern + def parse_header(self, filename: str) -> None: + """Parse a C header file, looking for "#define PSA_xxx".""" + with read_file_lines(filename, binary=True) as lines: + for line in lines: + line = re.sub(self._nonascii_re, rb'', line).decode('ascii') + self.parse_header_line(line) + + _macro_identifier_re = re.compile(r'[A-Z]\w+') + def generate_undeclared_names(self, expr: str) -> Iterable[str]: + for name in re.findall(self._macro_identifier_re, expr): + if name not in self.all_declared: + yield name + + def accept_test_case_line(self, function: str, argument: str) -> bool: + #pylint: disable=unused-argument + undeclared = list(self.generate_undeclared_names(argument)) + if undeclared: + raise Exception('Undeclared names in test case', undeclared) + return True + + def add_test_case_line(self, function: str, argument: str) -> None: + """Parse a test case data line, looking for algorithm metadata tests.""" + sets = [] + if function.endswith('_algorithm'): + sets.append(self.algorithms) + if function == 'key_agreement_algorithm' and \ + argument.startswith('PSA_ALG_KEY_AGREEMENT('): + # We only want *raw* key agreement algorithms as such, so + # exclude ones that are already chained with a KDF. + # Keep the expression as one to test as an algorithm. + function = 'other_algorithm' + sets += self.table_by_test_function[function] + if self.accept_test_case_line(function, argument): + for s in sets: + s.add(argument) + + # Regex matching a *.data line containing a test function call and + # its arguments. The actual definition is partly positional, but this + # regex is good enough in practice. + _test_case_line_re = re.compile(r'(?!depends_on:)(\w+):([^\n :][^:\n]*)') + def parse_test_cases(self, filename: str) -> None: + """Parse a test case file (*.data), looking for algorithm metadata tests.""" + with read_file_lines(filename) as lines: + for line in lines: + m = re.match(self._test_case_line_re, line) + if m: + self.add_test_case_line(m.group(1), m.group(2)) From d78f34902da5ea0c5b5ad2ea9405b16d893568ad Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 30 Mar 2021 21:46:35 +0200 Subject: [PATCH 075/490] Use InputsForTest in generate_psa_tests In generate_psa_tests, use InputsForTest rather than PSAMacroCollector to gather values. This way, the enumeration of values to test includes values used in metadata tests in addition to constructors parsed from header files. This allows greater coverage of values built from constructors with arguments. This doesn't make a difference yet, but it will once algorithm constructors with arguments are supported in generate_psa_tests. Make the injection of numerical values optional. They are useful for test_psa_constant_names, so keep them there. Don't use them for not-supported tests: they might make sense, but the current code wouldn't work since it doesn't know how to make up fake key material or what dependencies to generate. Don't use them for storage tests: they only make sense for supported values. Don't inject 'PSA_SUCCESS': that's superfluous. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 45 ++++++++++++------------ 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 0c9a9a5b2..f5f81c8a4 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -301,7 +301,7 @@ class PSAMacroCollector(PSAMacroEnumerator): self.read_line(line) -class InputsForTest(PSAMacroEnumerator): +class InputsForTest(PSAMacroCollector): # pylint: disable=too-many-instance-attributes """Accumulate information about macros to test. enumerate @@ -312,27 +312,6 @@ enumerate def __init__(self) -> None: super().__init__() self.all_declared = set() #type: Set[str] - # Sets of names per type - self.statuses.add('PSA_SUCCESS') - self.algorithms.add('0xffffffff') - self.ecc_curves.add('0xff') - self.dh_groups.add('0xff') - self.key_types.add('0xffff') - self.key_usage_flags.add('0x80000000') - - # Hard-coded values for unknown algorithms - # - # These have to have values that are correct for their respective - # PSA_ALG_IS_xxx macros, but are also not currently assigned and are - # not likely to be assigned in the near future. - self.hash_algorithms.add('0x020000fe') # 0x020000ff is PSA_ALG_ANY_HASH - self.mac_algorithms.add('0x03007fff') - self.ka_algorithms.add('0x09fc0000') - self.kdf_algorithms.add('0x080000ff') - # For AEAD algorithms, the only variability is over the tag length, - # and this only applies to known algorithms, so don't test an - # unknown algorithm. - # Identifier prefixes self.table_by_prefix = { 'ERROR': self.statuses, @@ -370,6 +349,28 @@ enumerate self.arguments_for['tag_length'] += ['1', '63'] self.arguments_for['min_tag_length'] += ['1', '63'] + def add_numerical_values(self) -> None: + """Add numerical values that are not supported to the known identifiers.""" + # Sets of names per type + self.algorithms.add('0xffffffff') + self.ecc_curves.add('0xff') + self.dh_groups.add('0xff') + self.key_types.add('0xffff') + self.key_usage_flags.add('0x80000000') + + # Hard-coded values for unknown algorithms + # + # These have to have values that are correct for their respective + # PSA_ALG_IS_xxx macros, but are also not currently assigned and are + # not likely to be assigned in the near future. + self.hash_algorithms.add('0x020000fe') # 0x020000ff is PSA_ALG_ANY_HASH + self.mac_algorithms.add('0x03007fff') + self.ka_algorithms.add('0x09fc0000') + self.kdf_algorithms.add('0x080000ff') + # For AEAD algorithms, the only variability is over the tag length, + # and this only applies to known algorithms, so don't test an + # unknown algorithm. + def get_names(self, type_word: str) -> Set[str]: """Return the set of known names of values of the given type.""" return { From 3f1a60144dbaf523f0b9888c2708ba4813a6073c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 12 Apr 2021 13:41:52 +0200 Subject: [PATCH 076/490] Fix KeyType with parameters passed in the name argument Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index aa5279027..94a97e7e9 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -33,7 +33,7 @@ class KeyType: `name` is a string 'PSA_KEY_TYPE_xxx' which is the name of a PSA key type macro. For key types that take arguments, the arguments can be passed either through the optional argument `params` or by - passing an expression of the form 'PSA_KEY_TYPE_xxx(param1, param2)' + passing an expression of the form 'PSA_KEY_TYPE_xxx(param1, ...)' in `name` as a string. """ @@ -48,7 +48,7 @@ class KeyType: m = re.match(r'(\w+)\s*\((.*)\)\Z', self.name) assert m is not None self.name = m.group(1) - params = ','.split(m.group(2)) + params = m.group(2).split(',') self.params = (None if params is None else [param.strip() for param in params]) """The parameters of the key type, if there are any. From f333c244834993f072f3ca5cfc23b18eed1b89f3 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 19 Apr 2021 13:50:25 +0200 Subject: [PATCH 077/490] Expand psa_generate_tests to support constructor arguments In macro_collector.py, base InputsForTest on PSAMacroEnumerator rather than PSAMacroCollector. It didn't make much sense to use PSAMacroCollector anymore since InputsForTest didn't use anything other than the constructor. psa_generate_tests now generates arguments for more macros. In particular, it now collects macro arguments from test_suite_psa_crypto_metadata. Algorithms with parameters are now supported. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index f5f81c8a4..3e28f6248 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -105,6 +105,16 @@ class PSAMacroEnumerator: 'tag_length': [], 'min_tag_length': [], } #type: Dict[str, List[str]] + self.include_intermediate = False + + def is_internal_name(self, name: str) -> bool: + """Whether this is an internal macro. Internal macros will be skipped.""" + if not self.include_intermediate: + if name.endswith('_BASE') or name.endswith('_NONE'): + return True + if '_CATEGORY_' in name: + return True + return name.endswith('_FLAG') or name.endswith('_MASK') def gather_arguments(self) -> None: """Populate the list of values for macro arguments. @@ -192,15 +202,6 @@ class PSAMacroCollector(PSAMacroEnumerator): self.key_types_from_group = {} #type: Dict[str, str] self.algorithms_from_hash = {} #type: Dict[str, str] - def is_internal_name(self, name: str) -> bool: - """Whether this is an internal macro. Internal macros will be skipped.""" - if not self.include_intermediate: - if name.endswith('_BASE') or name.endswith('_NONE'): - return True - if '_CATEGORY_' in name: - return True - return name.endswith('_FLAG') or name.endswith('_MASK') - def record_algorithm_subtype(self, name: str, expansion: str) -> None: """Record the subtype of an algorithm constructor. @@ -301,7 +302,7 @@ class PSAMacroCollector(PSAMacroEnumerator): self.read_line(line) -class InputsForTest(PSAMacroCollector): +class InputsForTest(PSAMacroEnumerator): # pylint: disable=too-many-instance-attributes """Accumulate information about macros to test. enumerate @@ -408,7 +409,8 @@ enumerate name = m.group(1) self.all_declared.add(name) if re.search(self._excluded_name_re, name) or \ - name in self._excluded_names: + name in self._excluded_names or \ + self.is_internal_name(name): return dest = self.table_by_prefix.get(m.group(2)) if dest is None: From 3e2cf340c10f0dc9f2cd238066ee54f65ec52606 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Apr 2021 15:36:58 +0200 Subject: [PATCH 078/490] Normalize whitespace in test arguments Avoid ending up with test cases that only differ in whitespace in an argument. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 3e28f6248..9c3c72319 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -131,7 +131,11 @@ class PSAMacroEnumerator: @staticmethod def _format_arguments(name: str, arguments: Iterable[str]) -> str: - """Format a macro call with arguments..""" + """Format a macro call with arguments. + + The resulting format is consistent with + `InputsForTest.normalize_argument`. + """ return name + '(' + ', '.join(arguments) + ')' _argument_split_re = re.compile(r' *, *') @@ -440,6 +444,15 @@ enumerate raise Exception('Undeclared names in test case', undeclared) return True + @staticmethod + def normalize_argument(argument: str) -> str: + """Normalize whitespace in the given C expression. + + The result uses the same whitespace as + ` PSAMacroEnumerator.distribute_arguments`. + """ + return re.sub(r',', r', ', re.sub(r' +', r'', argument)) + def add_test_case_line(self, function: str, argument: str) -> None: """Parse a test case data line, looking for algorithm metadata tests.""" sets = [] @@ -454,7 +467,7 @@ enumerate sets += self.table_by_test_function[function] if self.accept_test_case_line(function, argument): for s in sets: - s.add(argument) + s.add(self.normalize_argument(argument)) # Regex matching a *.data line containing a test function call and # its arguments. The actual definition is partly positional, but this From fe3c01c45df07ed907c3f45c609c8c0d23b8b1c6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Apr 2021 15:37:34 +0200 Subject: [PATCH 079/490] Remove duplicates from enumerated test inputs When generating expressions to construct test case data, there can be duplicate values, for example if a value of the form C(A) is present as such in test_suite_psa_crypto_metadata.data and also constructed by enumerating the argument A for the constructor C. Eliminate such duplicates in generate_expressions. This commit removes many test cases that were exact duplicates (and were near-duplicates differing only in whitespace before the whitespace normalization). Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 9c3c72319..ec8e18435 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -173,6 +173,15 @@ class PSAMacroEnumerator: except BaseException as e: raise Exception('distribute_arguments({})'.format(name)) from e + def distribute_arguments_without_duplicates( + self, seen: Set[str], name: str + ) -> Iterator[str]: + """Same as `distribute_arguments`, but don't repeat seen results.""" + for result in self.distribute_arguments(name): + if result not in seen: + seen.add(result) + yield result + def generate_expressions(self, names: Iterable[str]) -> Iterator[str]: """Generate expressions covering values constructed from the given names. @@ -185,7 +194,11 @@ class PSAMacroEnumerator: * ``macros.generate_expressions(macros.key_types)`` generates all key types. """ - return itertools.chain(*map(self.distribute_arguments, names)) + seen = set() #type: Set[str] + return itertools.chain(*( + self.distribute_arguments_without_duplicates(seen, name) + for name in names + )) class PSAMacroCollector(PSAMacroEnumerator): From 295d7f640ab3bad0d42746a592e93e3cdd1e71b8 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 20 May 2021 21:37:06 +0200 Subject: [PATCH 080/490] Document include_intermediate in PSAMacroEnumerator Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index ec8e18435..0e76435f3 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -105,6 +105,10 @@ class PSAMacroEnumerator: 'tag_length': [], 'min_tag_length': [], } #type: Dict[str, List[str]] + # Whether to include intermediate macros in enumerations. Intermediate + # macros serve as category headers and are not valid values of their + # type. See `is_internal_name`. + # Always false in this class, may be set to true in derived classes. self.include_intermediate = False def is_internal_name(self, name: str) -> bool: From fbc0c6cc525a1b69ecdda379ae71cf1d9b58115f Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Mon, 19 Apr 2021 15:12:46 +0100 Subject: [PATCH 081/490] PSA PAKE: add to PSA constant name test Signed-off-by: Janos Follath --- scripts/framework_dev/macro_collector.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 0e76435f3..395e038ca 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -95,6 +95,7 @@ class PSAMacroEnumerator: self.mac_algorithms = set() #type: Set[str] self.ka_algorithms = set() #type: Set[str] self.kdf_algorithms = set() #type: Set[str] + self.pake_algorithms = set() #type: Set[str] self.aead_algorithms = set() #type: Set[str] # macro name -> list of argument names self.argspecs = {} #type: Dict[str, List[str]] @@ -364,6 +365,7 @@ enumerate 'asymmetric_signature_algorithm': [], 'asymmetric_signature_wildcard': [self.algorithms], 'asymmetric_encryption_algorithm': [], + 'pake_algorithm': [self.pake_algorithms], 'other_algorithm': [], } #type: Dict[str, List[Set[str]]] self.arguments_for['mac_length'] += ['1', '63'] @@ -389,6 +391,7 @@ enumerate self.mac_algorithms.add('0x03007fff') self.ka_algorithms.add('0x09fc0000') self.kdf_algorithms.add('0x080000ff') + self.pake_algorithms.add('0x0a0000ff') # For AEAD algorithms, the only variability is over the tag length, # and this only applies to known algorithms, so don't test an # unknown algorithm. From 6d5f13a4ccf6180f956f1c70d1b718f658441fae Mon Sep 17 00:00:00 2001 From: TRodziewicz Date: Mon, 31 May 2021 17:58:57 +0200 Subject: [PATCH 082/490] Remove MD2, MD4, RC4, Blowfish and XTEA Signed-off-by: TRodziewicz --- scripts/framework_dev/crypto_knowledge.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 94a97e7e9..4b4e2df51 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -82,7 +82,6 @@ class KeyType: } KEY_TYPE_SIZES = { 'PSA_KEY_TYPE_AES': (128, 192, 256), # exhaustive - 'PSA_KEY_TYPE_ARC4': (8, 128, 2048), # extremes + sensible 'PSA_KEY_TYPE_ARIA': (128, 192, 256), # exhaustive 'PSA_KEY_TYPE_CAMELLIA': (128, 192, 256), # exhaustive 'PSA_KEY_TYPE_CHACHA20': (256,), # exhaustive From 0126f0e5127091b7f757d59f83a9a5b4b94ed16b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 22 Jun 2021 11:26:19 +0200 Subject: [PATCH 083/490] Tweak MSVC detection to work with non-English Visual Studio Fix #4699 Signed-off-by: Gilles Peskine --- scripts/framework_dev/c_build_helper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 18c1e7606..459afba42 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -135,8 +135,7 @@ def get_c_expression_values( stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, universal_newlines=True) - cc_is_msvc = 'Microsoft (R) C/C++ Optimizing Compiler' in \ - proc.communicate()[1] + cc_is_msvc = 'Microsoft (R) C/C++' in proc.communicate()[1] cmd += ['-I' + dir for dir in include_path] if cc_is_msvc: From 013782492a9da83ee95a8bb716fef19687d72948 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Apr 2021 20:03:53 +0200 Subject: [PATCH 084/490] Add lifetime metadata tests Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 395e038ca..bc432be9f 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -367,6 +367,7 @@ enumerate 'asymmetric_encryption_algorithm': [], 'pake_algorithm': [self.pake_algorithms], 'other_algorithm': [], + 'lifetime': [], } #type: Dict[str, List[Set[str]]] self.arguments_for['mac_length'] += ['1', '63'] self.arguments_for['min_mac_length'] += ['1', '63'] From 2e4ae56614882350df882added3d83b6bc43c0ea Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Apr 2021 21:39:27 +0200 Subject: [PATCH 085/490] Collect lifetime constructors Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index bc432be9f..6eb0d00d1 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -81,11 +81,15 @@ class PSAMacroEnumerator: `self.arguments_for` for arguments that are not of a kind that is enumerated here. """ + #pylint: disable=too-many-instance-attributes def __init__(self) -> None: """Set up an empty set of known constructor macros. """ self.statuses = set() #type: Set[str] + self.lifetimes = set() #type: Set[str] + self.locations = set() #type: Set[str] + self.persistence_levels = set() #type: Set[str] self.algorithms = set() #type: Set[str] self.ecc_curves = set() #type: Set[str] self.dh_groups = set() #type: Set[str] @@ -133,6 +137,9 @@ class PSAMacroEnumerator: self.arguments_for['aead_alg'] = sorted(self.aead_algorithms) self.arguments_for['curve'] = sorted(self.ecc_curves) self.arguments_for['group'] = sorted(self.dh_groups) + self.arguments_for['persistence'] = sorted(self.persistence_levels) + self.arguments_for['location'] = sorted(self.locations) + self.arguments_for['lifetime'] = sorted(self.lifetimes) @staticmethod def _format_arguments(name: str, arguments: Iterable[str]) -> str: @@ -341,6 +348,9 @@ enumerate 'ALG': self.algorithms, 'ECC_CURVE': self.ecc_curves, 'DH_GROUP': self.dh_groups, + 'KEY_LIFETIME': self.lifetimes, + 'KEY_LOCATION': self.locations, + 'KEY_PERSISTENCE': self.persistence_levels, 'KEY_TYPE': self.key_types, 'KEY_USAGE': self.key_usage_flags, } #type: Dict[str, Set[str]] @@ -367,7 +377,7 @@ enumerate 'asymmetric_encryption_algorithm': [], 'pake_algorithm': [self.pake_algorithms], 'other_algorithm': [], - 'lifetime': [], + 'lifetime': [self.lifetimes], } #type: Dict[str, List[Set[str]]] self.arguments_for['mac_length'] += ['1', '63'] self.arguments_for['min_mac_length'] += ['1', '63'] From ed2a9facfedeaa28bbff733909ad40d978a1c486 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Apr 2021 22:05:34 +0200 Subject: [PATCH 086/490] Add storage tests for lifetimes Test keys with various persistence levels, enumerated from the metadata tests. For read-only keys, do not attempt to create or destroy the key through the API, only to read a key that has been injected into storage directly through filesystem access. Do not test keys with a non-default location, since they require a driver and we do not yet have a dependency mechanism to require the presence of a driver for a specific location value. Signed-off-by: Gilles Peskine --- scripts/framework_dev/psa_storage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index 3a740072e..45f0380e7 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -164,6 +164,10 @@ class Key: """ return self.bytes().hex() + def location_value(self) -> int: + """The numerical value of the location encoded in the key's lifetime.""" + return self.lifetime.value() >> 8 + class TestKey(unittest.TestCase): # pylint: disable=line-too-long From 0d3d15cdcc397c8e30f077f8437585a7cdd26384 Mon Sep 17 00:00:00 2001 From: gabor-mezei-arm Date: Thu, 24 Jun 2021 10:04:38 +0200 Subject: [PATCH 087/490] Add key usage policy extension support for key generation Signed-off-by: gabor-mezei-arm --- scripts/framework_dev/psa_storage.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index 45f0380e7..4cd3dfe91 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -101,6 +101,12 @@ class Key: LATEST_VERSION = 0 """The latest version of the storage format.""" + EXTENDABLE_USAGE_FLAGS = { + Expr('PSA_KEY_USAGE_SIGN_HASH'): Expr('PSA_KEY_USAGE_SIGN_MESSAGE'), + Expr('PSA_KEY_USAGE_VERIFY_HASH'): Expr('PSA_KEY_USAGE_VERIFY_MESSAGE') + } #type: Dict[Expr, Expr] + """The extendable usage flags with the corresponding extension flags.""" + def __init__(self, *, version: Optional[int] = None, id: Optional[int] = None, #pylint: disable=redefined-builtin @@ -108,18 +114,27 @@ class Key: type: Exprable, #pylint: disable=redefined-builtin bits: int, usage: Exprable, alg: Exprable, alg2: Exprable, - material: bytes #pylint: disable=used-before-assignment + material: bytes, #pylint: disable=used-before-assignment + usage_extension: bool = True ) -> None: self.version = self.LATEST_VERSION if version is None else version self.id = id #pylint: disable=invalid-name #type: Optional[int] self.lifetime = as_expr(lifetime) #type: Expr self.type = as_expr(type) #type: Expr self.bits = bits #type: int - self.usage = as_expr(usage) #type: Expr + self.original_usage = as_expr(usage) #type: Expr + self.updated_usage = self.original_usage #type: Expr self.alg = as_expr(alg) #type: Expr self.alg2 = as_expr(alg2) #type: Expr self.material = material #type: bytes + if usage_extension: + for flag, extension in self.EXTENDABLE_USAGE_FLAGS.items(): + if self.original_usage.value() & flag.value() and \ + self.original_usage.value() & extension.value() == 0: + self.updated_usage = Expr(self.updated_usage.string + + ' | ' + extension.string) + MAGIC = b'PSA\000KEY\000' @staticmethod @@ -151,7 +166,7 @@ class Key: if self.version == 0: attributes = self.pack('LHHLLL', self.lifetime, self.type, self.bits, - self.usage, self.alg, self.alg2) + self.updated_usage, self.alg, self.alg2) material = self.pack('L', len(self.material)) + self.material else: raise NotImplementedError From b27fddfdb714b97fd7f1a781bb952a3970481350 Mon Sep 17 00:00:00 2001 From: gabor-mezei-arm Date: Thu, 24 Jun 2021 10:16:44 +0200 Subject: [PATCH 088/490] Add test case generation for usage extensions when loading keys Add test cases validating that if a stored key only had the hash policy, then after loading it psa_get_key_attributes reports that it also has the message policy, and the key can be used with message functions. Signed-off-by: gabor-mezei-arm --- scripts/framework_dev/macro_collector.py | 6 ++++-- scripts/framework_dev/psa_storage.py | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 6eb0d00d1..f9ef5f915 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -101,6 +101,7 @@ class PSAMacroEnumerator: self.kdf_algorithms = set() #type: Set[str] self.pake_algorithms = set() #type: Set[str] self.aead_algorithms = set() #type: Set[str] + self.sign_algorithms = set() #type: Set[str] # macro name -> list of argument names self.argspecs = {} #type: Dict[str, List[str]] # argument name -> list of values @@ -135,6 +136,7 @@ class PSAMacroEnumerator: self.arguments_for['ka_alg'] = sorted(self.ka_algorithms) self.arguments_for['kdf_alg'] = sorted(self.kdf_algorithms) self.arguments_for['aead_alg'] = sorted(self.aead_algorithms) + self.arguments_for['sign_alg'] = sorted(self.sign_algorithms) self.arguments_for['curve'] = sorted(self.ecc_curves) self.arguments_for['group'] = sorted(self.dh_groups) self.arguments_for['persistence'] = sorted(self.persistence_levels) @@ -368,11 +370,11 @@ enumerate 'hash_algorithm': [self.hash_algorithms], 'mac_algorithm': [self.mac_algorithms], 'cipher_algorithm': [], - 'hmac_algorithm': [self.mac_algorithms], + 'hmac_algorithm': [self.mac_algorithms, self.sign_algorithms], 'aead_algorithm': [self.aead_algorithms], 'key_derivation_algorithm': [self.kdf_algorithms], 'key_agreement_algorithm': [self.ka_algorithms], - 'asymmetric_signature_algorithm': [], + 'asymmetric_signature_algorithm': [self.sign_algorithms], 'asymmetric_signature_wildcard': [self.algorithms], 'asymmetric_encryption_algorithm': [], 'pake_algorithm': [self.pake_algorithms], diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index 4cd3dfe91..ff2fdd434 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -107,6 +107,14 @@ class Key: } #type: Dict[Expr, Expr] """The extendable usage flags with the corresponding extension flags.""" + EXTENDABLE_USAGE_FLAGS_KEY_RESTRICTION = { + 'PSA_KEY_USAGE_SIGN_HASH': '.*KEY_PAIR', + 'PSA_KEY_USAGE_VERIFY_HASH': '.*KEY.*' + } #type: Dict[str, str] + """The key type filter for the extendable usage flags. + The filter is a regexp. + """ + def __init__(self, *, version: Optional[int] = None, id: Optional[int] = None, #pylint: disable=redefined-builtin From 55aaa41dcd56d8dd229dad37bd83d82827a8324b Mon Sep 17 00:00:00 2001 From: gabor-mezei-arm Date: Mon, 28 Jun 2021 16:27:29 +0200 Subject: [PATCH 089/490] Add better name for variables Signed-off-by: gabor-mezei-arm --- scripts/framework_dev/psa_storage.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index ff2fdd434..cde6c80db 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -101,18 +101,18 @@ class Key: LATEST_VERSION = 0 """The latest version of the storage format.""" - EXTENDABLE_USAGE_FLAGS = { + IMPLICIT_USAGE_FLAGS = { Expr('PSA_KEY_USAGE_SIGN_HASH'): Expr('PSA_KEY_USAGE_SIGN_MESSAGE'), Expr('PSA_KEY_USAGE_VERIFY_HASH'): Expr('PSA_KEY_USAGE_VERIFY_MESSAGE') } #type: Dict[Expr, Expr] - """The extendable usage flags with the corresponding extension flags.""" + """Mapping of usage flags to the flags that they imply.""" - EXTENDABLE_USAGE_FLAGS_KEY_RESTRICTION = { + IMPLICIT_USAGE_FLAGS_KEY_RESTRICTION = { 'PSA_KEY_USAGE_SIGN_HASH': '.*KEY_PAIR', 'PSA_KEY_USAGE_VERIFY_HASH': '.*KEY.*' } #type: Dict[str, str] - """The key type filter for the extendable usage flags. - The filter is a regexp. + """Use a regexp to determine key types for which signature is possible + when using the actual usage flag. """ def __init__(self, *, @@ -137,7 +137,7 @@ class Key: self.material = material #type: bytes if usage_extension: - for flag, extension in self.EXTENDABLE_USAGE_FLAGS.items(): + for flag, extension in self.IMPLICIT_USAGE_FLAGS.items(): if self.original_usage.value() & flag.value() and \ self.original_usage.value() & extension.value() == 0: self.updated_usage = Expr(self.updated_usage.string + From fff8750002aecfb5441c227e5077da1c21b707b6 Mon Sep 17 00:00:00 2001 From: gabor-mezei-arm Date: Mon, 28 Jun 2021 16:54:11 +0200 Subject: [PATCH 090/490] Use string in dict instead of Expr object Signed-off-by: gabor-mezei-arm --- scripts/framework_dev/psa_storage.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index cde6c80db..0a8a0a3da 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -102,9 +102,9 @@ class Key: """The latest version of the storage format.""" IMPLICIT_USAGE_FLAGS = { - Expr('PSA_KEY_USAGE_SIGN_HASH'): Expr('PSA_KEY_USAGE_SIGN_MESSAGE'), - Expr('PSA_KEY_USAGE_VERIFY_HASH'): Expr('PSA_KEY_USAGE_VERIFY_MESSAGE') - } #type: Dict[Expr, Expr] + 'PSA_KEY_USAGE_SIGN_HASH': 'PSA_KEY_USAGE_SIGN_MESSAGE', + 'PSA_KEY_USAGE_VERIFY_HASH': 'PSA_KEY_USAGE_VERIFY_MESSAGE' + } #type: Dict[str, str] """Mapping of usage flags to the flags that they imply.""" IMPLICIT_USAGE_FLAGS_KEY_RESTRICTION = { @@ -138,10 +138,10 @@ class Key: if usage_extension: for flag, extension in self.IMPLICIT_USAGE_FLAGS.items(): - if self.original_usage.value() & flag.value() and \ - self.original_usage.value() & extension.value() == 0: + if self.original_usage.value() & Expr(flag).value() and \ + self.original_usage.value() & Expr(extension).value() == 0: self.updated_usage = Expr(self.updated_usage.string + - ' | ' + extension.string) + ' | ' + extension) MAGIC = b'PSA\000KEY\000' From 813ad98a05229e78dcb77bb5ca358c950e86d39a Mon Sep 17 00:00:00 2001 From: gabor-mezei-arm Date: Mon, 28 Jun 2021 17:40:32 +0200 Subject: [PATCH 091/490] Rename variables and funcions Signed-off-by: gabor-mezei-arm --- scripts/framework_dev/psa_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index 0a8a0a3da..5ff173831 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -123,7 +123,7 @@ class Key: bits: int, usage: Exprable, alg: Exprable, alg2: Exprable, material: bytes, #pylint: disable=used-before-assignment - usage_extension: bool = True + implicit_usage: bool = True ) -> None: self.version = self.LATEST_VERSION if version is None else version self.id = id #pylint: disable=invalid-name #type: Optional[int] @@ -136,7 +136,7 @@ class Key: self.alg2 = as_expr(alg2) #type: Expr self.material = material #type: bytes - if usage_extension: + if implicit_usage: for flag, extension in self.IMPLICIT_USAGE_FLAGS.items(): if self.original_usage.value() & Expr(flag).value() and \ self.original_usage.value() & Expr(extension).value() == 0: From c716dc504113b27c303c5509a0b11c44c2b22418 Mon Sep 17 00:00:00 2001 From: gabor-mezei-arm Date: Mon, 28 Jun 2021 20:02:11 +0200 Subject: [PATCH 092/490] Move key type validation to crypto_knowledge Signed-off-by: gabor-mezei-arm --- scripts/framework_dev/crypto_knowledge.py | 17 ++++++++++++++++- scripts/framework_dev/psa_storage.py | 8 -------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 4b4e2df51..01a8c5dc3 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -19,7 +19,7 @@ This module is entirely based on the PSA API. # limitations under the License. import re -from typing import Iterable, Optional, Tuple +from typing import Iterable, Optional, Tuple, Dict from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA @@ -138,3 +138,18 @@ class KeyType: return des3[:length] return b''.join([self.DATA_BLOCK] * (length // len(self.DATA_BLOCK)) + [self.DATA_BLOCK[:length % len(self.DATA_BLOCK)]]) + + KEY_TYPE_FOR_SIGNATURE = { + 'PSA_KEY_USAGE_SIGN_HASH': '.*KEY_PAIR', + 'PSA_KEY_USAGE_VERIFY_HASH': '.*KEY.*' + } #type: Dict[str, str] + """Use a regexp to determine key types for which signature is possible + when using the actual usage flag. + """ + def is_valid_for_signature(self, usage: str) -> bool: + """Determine if the key type is compatible with the specified + signitute type. + + """ + # This is just temporaly solution for the implicit usage flags. + return re.match(self.KEY_TYPE_FOR_SIGNATURE[usage], self.name) is not None diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index 5ff173831..88992a6fc 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -107,14 +107,6 @@ class Key: } #type: Dict[str, str] """Mapping of usage flags to the flags that they imply.""" - IMPLICIT_USAGE_FLAGS_KEY_RESTRICTION = { - 'PSA_KEY_USAGE_SIGN_HASH': '.*KEY_PAIR', - 'PSA_KEY_USAGE_VERIFY_HASH': '.*KEY.*' - } #type: Dict[str, str] - """Use a regexp to determine key types for which signature is possible - when using the actual usage flag. - """ - def __init__(self, *, version: Optional[int] = None, id: Optional[int] = None, #pylint: disable=redefined-builtin From ddb9ea7fd719caee7b707386b647616f0d4b100a Mon Sep 17 00:00:00 2001 From: gabor-mezei-arm Date: Tue, 29 Jun 2021 11:17:14 +0200 Subject: [PATCH 093/490] Keep the imported classes sorted Signed-off-by: gabor-mezei-arm --- scripts/framework_dev/crypto_knowledge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 01a8c5dc3..089d4fe1f 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -19,7 +19,7 @@ This module is entirely based on the PSA API. # limitations under the License. import re -from typing import Iterable, Optional, Tuple, Dict +from typing import Dict, Iterable, Optional, Tuple from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA From bc82a54103eed5767c9f76b7ef3c614ea22a5413 Mon Sep 17 00:00:00 2001 From: gabor-mezei-arm Date: Tue, 29 Jun 2021 11:17:54 +0200 Subject: [PATCH 094/490] Use regexp pattern instaed of string Signed-off-by: gabor-mezei-arm --- scripts/framework_dev/crypto_knowledge.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 089d4fe1f..c38955b0e 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -19,7 +19,7 @@ This module is entirely based on the PSA API. # limitations under the License. import re -from typing import Dict, Iterable, Optional, Tuple +from typing import Dict, Iterable, Optional, Pattern, Tuple from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA @@ -140,9 +140,9 @@ class KeyType: [self.DATA_BLOCK[:length % len(self.DATA_BLOCK)]]) KEY_TYPE_FOR_SIGNATURE = { - 'PSA_KEY_USAGE_SIGN_HASH': '.*KEY_PAIR', - 'PSA_KEY_USAGE_VERIFY_HASH': '.*KEY.*' - } #type: Dict[str, str] + 'PSA_KEY_USAGE_SIGN_HASH': re.compile('.*KEY_PAIR'), + 'PSA_KEY_USAGE_VERIFY_HASH': re.compile('.*KEY.*') + } #type: Dict[str, Pattern] """Use a regexp to determine key types for which signature is possible when using the actual usage flag. """ From c59614ceb68e1cef6daeb2b04c781e8a202cf38c Mon Sep 17 00:00:00 2001 From: gabor-mezei-arm Date: Tue, 29 Jun 2021 15:29:24 +0200 Subject: [PATCH 095/490] Refactor handlibg of the key usage flags Move implicit usage flags handling to the StorageKey class. Create a subclass for test case data. Signed-off-by: gabor-mezei-arm --- scripts/framework_dev/psa_storage.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index 88992a6fc..45f0380e7 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -101,12 +101,6 @@ class Key: LATEST_VERSION = 0 """The latest version of the storage format.""" - IMPLICIT_USAGE_FLAGS = { - 'PSA_KEY_USAGE_SIGN_HASH': 'PSA_KEY_USAGE_SIGN_MESSAGE', - 'PSA_KEY_USAGE_VERIFY_HASH': 'PSA_KEY_USAGE_VERIFY_MESSAGE' - } #type: Dict[str, str] - """Mapping of usage flags to the flags that they imply.""" - def __init__(self, *, version: Optional[int] = None, id: Optional[int] = None, #pylint: disable=redefined-builtin @@ -114,27 +108,18 @@ class Key: type: Exprable, #pylint: disable=redefined-builtin bits: int, usage: Exprable, alg: Exprable, alg2: Exprable, - material: bytes, #pylint: disable=used-before-assignment - implicit_usage: bool = True + material: bytes #pylint: disable=used-before-assignment ) -> None: self.version = self.LATEST_VERSION if version is None else version self.id = id #pylint: disable=invalid-name #type: Optional[int] self.lifetime = as_expr(lifetime) #type: Expr self.type = as_expr(type) #type: Expr self.bits = bits #type: int - self.original_usage = as_expr(usage) #type: Expr - self.updated_usage = self.original_usage #type: Expr + self.usage = as_expr(usage) #type: Expr self.alg = as_expr(alg) #type: Expr self.alg2 = as_expr(alg2) #type: Expr self.material = material #type: bytes - if implicit_usage: - for flag, extension in self.IMPLICIT_USAGE_FLAGS.items(): - if self.original_usage.value() & Expr(flag).value() and \ - self.original_usage.value() & Expr(extension).value() == 0: - self.updated_usage = Expr(self.updated_usage.string + - ' | ' + extension) - MAGIC = b'PSA\000KEY\000' @staticmethod @@ -166,7 +151,7 @@ class Key: if self.version == 0: attributes = self.pack('LHHLLL', self.lifetime, self.type, self.bits, - self.updated_usage, self.alg, self.alg2) + self.usage, self.alg, self.alg2) material = self.pack('L', len(self.material)) + self.material else: raise NotImplementedError From 1f0cb3bb6f1119c34c2c2bb8bad818e44c5f4077 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 4 Oct 2021 18:10:16 +0200 Subject: [PATCH 096/490] Break out algorithm_tester() as a separate method No intended behavior change. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 26 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index f9ef5f915..bd2d296c0 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -233,6 +233,25 @@ class PSAMacroCollector(PSAMacroEnumerator): self.key_types_from_group = {} #type: Dict[str, str] self.algorithms_from_hash = {} #type: Dict[str, str] + @staticmethod + def algorithm_tester(name: str) -> str: + """The predicate for whether an algorithm is built from the given constructor. + + The given name must be the name of an algorithm constructor of the + form ``PSA_ALG_xxx`` which is used as ``PSA_ALG_xxx(yyy)`` to build + an algorithm value. Return the corresponding predicate macro which + is used as ``predicate(alg)`` to test whether ``alg`` can be built + as ``PSA_ALG_xxx(yyy)``. The predicate is usually called + ``PSA_ALG_IS_xxx``. + """ + prefix = 'PSA_ALG_' + assert name.startswith(prefix) + midfix = 'IS_' + suffix = name[len(prefix):] + if suffix in ['DSA', 'ECDSA']: + midfix += 'RANDOMIZED_' + return prefix + midfix + suffix + def record_algorithm_subtype(self, name: str, expansion: str) -> None: """Record the subtype of an algorithm constructor. @@ -308,12 +327,7 @@ class PSAMacroCollector(PSAMacroEnumerator): self.algorithms.add(name) self.record_algorithm_subtype(name, expansion) elif name.startswith('PSA_ALG_') and parameter == 'hash_alg': - if name in ['PSA_ALG_DSA', 'PSA_ALG_ECDSA']: - # A naming irregularity - tester = name[:8] + 'IS_RANDOMIZED_' + name[8:] - else: - tester = name[:8] + 'IS_' + name[8:] - self.algorithms_from_hash[name] = tester + self.algorithms_from_hash[name] = self.algorithm_tester(name) elif name.startswith('PSA_KEY_USAGE_') and not parameter: self.key_usage_flags.add(name) else: From 129f8332d12deaf9ca5e0c6b6c999deb2bb170b7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 4 Oct 2021 18:10:38 +0200 Subject: [PATCH 097/490] New algorithm PSA_ALG_RSA_PSS_ANY_SALT This is a variant of PSA_ALG_RSA_PSS which currently has exactly the same behavior, but is intended to have a different behavior when verifying signatures. In a subsequent commit, PSA_ALG_RSA_PSS will change to requiring the salt length to be what it would produce when signing, as is currently documented, whereas PSA_ALG_RSA_PSS_ANY_SALT will retain the current behavior of allowing any salt length (including 0). Changes in this commit: * New algorithm constructor PSA_ALG_RSA_PSS_ANY_SALT. * New predicates PSA_ALG_IS_RSA_PSS_STANDARD_SALT (corresponding to PSA_ALG_RSA_PSS) and PSA_ALG_IS_RSA_PSS_ANY_SALT (corresponding to PSA_ALG_RSA_PSS_ANY_SALT). * Support for the new predicates in macro_collector.py (needed for generate_psa_constant_names). Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index bd2d296c0..bf82f13dc 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -250,6 +250,8 @@ class PSAMacroCollector(PSAMacroEnumerator): suffix = name[len(prefix):] if suffix in ['DSA', 'ECDSA']: midfix += 'RANDOMIZED_' + elif suffix == 'RSA_PSS': + suffix += '_STANDARD_SALT' return prefix + midfix + suffix def record_algorithm_subtype(self, name: str, expansion: str) -> None: From fbcfb5bd731c912148a142e569a273d29226730a Mon Sep 17 00:00:00 2001 From: Przemyslaw Stekiel Date: Fri, 15 Oct 2021 15:21:51 +0200 Subject: [PATCH 098/490] Add test class for key generation Genertae test_suite_psa_crypto_generate_key.generated.data. Use test_suite_psa_crypto_generate_key.function as a test function. Signed-off-by: Przemyslaw Stekiel --- scripts/framework_dev/test_case.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index d01e1432b..11117fcdd 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -42,6 +42,7 @@ class TestCase: self.dependencies = [] #type: List[str] self.function = None #type: Optional[str] self.arguments = [] #type: List[str] + self.result = '' #type: str def add_comment(self, *lines: str) -> None: self.comments += lines @@ -58,6 +59,9 @@ class TestCase: def set_arguments(self, arguments: List[str]) -> None: self.arguments = arguments + def set_result(self, result: str) -> None: + self.result = result + def check_completeness(self) -> None: if self.description is None: raise MissingDescription @@ -81,9 +85,11 @@ class TestCase: out.write(self.description + '\n') if self.dependencies: out.write('depends_on:' + ':'.join(self.dependencies) + '\n') - out.write(self.function + ':' + ':'.join(self.arguments) + '\n') - - + out.write(self.function + ':' + ':'.join(self.arguments)) + if self.result: + out.write(':' + self.result + '\n') + else: + out.write('\n') def write_data_file(filename: str, test_cases: Iterable[TestCase], From 62905811a99112708fe808efa5137a7a549946b7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 14 Sep 2021 11:28:22 +0200 Subject: [PATCH 099/490] Remove on-target testing It was unmaintained and untested, and the fear of breaking it was holding us back. Resolves #4934. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 7382fb6ec..f5750aacf 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -106,10 +106,6 @@ Platform file: Platform file contains platform specific setup code and test case dispatch code. For example, host_test.function reads test data file from host's file system and dispatches tests. -In case of on-target target_test.function tests are not dispatched -on target. Target code is kept minimum and only test functions are -dispatched. Test case dispatch is done on the host using tools like -Greentea. Template file: --------- From 5794184006e80be78c2792aebe258c5d60323f58 Mon Sep 17 00:00:00 2001 From: Przemyslaw Stekiel Date: Tue, 2 Nov 2021 10:50:44 +0100 Subject: [PATCH 100/490] generate_psa_tests.py: add key generation result to test case argument list, add comments Signed-off-by: Przemyslaw Stekiel --- scripts/framework_dev/test_case.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index 11117fcdd..8ec211546 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -42,7 +42,6 @@ class TestCase: self.dependencies = [] #type: List[str] self.function = None #type: Optional[str] self.arguments = [] #type: List[str] - self.result = '' #type: str def add_comment(self, *lines: str) -> None: self.comments += lines @@ -59,9 +58,6 @@ class TestCase: def set_arguments(self, arguments: List[str]) -> None: self.arguments = arguments - def set_result(self, result: str) -> None: - self.result = result - def check_completeness(self) -> None: if self.description is None: raise MissingDescription @@ -86,10 +82,6 @@ class TestCase: if self.dependencies: out.write('depends_on:' + ':'.join(self.dependencies) + '\n') out.write(self.function + ':' + ':'.join(self.arguments)) - if self.result: - out.write(':' + self.result + '\n') - else: - out.write('\n') def write_data_file(filename: str, test_cases: Iterable[TestCase], From f9dc19f46555910c57604bfa7122e8c9e106544d Mon Sep 17 00:00:00 2001 From: Przemyslaw Stekiel Date: Tue, 9 Nov 2021 14:40:12 +0100 Subject: [PATCH 101/490] test_case.py: add new line between test cases Signed-off-by: Przemyslaw Stekiel --- scripts/framework_dev/test_case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index 8ec211546..6a46e4209 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -81,7 +81,7 @@ class TestCase: out.write(self.description + '\n') if self.dependencies: out.write('depends_on:' + ':'.join(self.dependencies) + '\n') - out.write(self.function + ':' + ':'.join(self.arguments)) + out.write(self.function + ':' + ':'.join(self.arguments) + '\n') def write_data_file(filename: str, test_cases: Iterable[TestCase], From c9fd1d07cad6c06744720e2a2fdec08eb436908e Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Thu, 2 Dec 2021 13:51:26 +0800 Subject: [PATCH 102/490] fix test_cmake_as_package fail Signed-off-by: Jerry Yu --- scripts/framework_dev/build_tree.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index 772410473..aee68f140 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -17,12 +17,15 @@ # limitations under the License. import os +import inspect + def looks_like_mbedtls_root(path: str) -> bool: """Whether the given directory looks like the root of the Mbed TLS source tree.""" return all(os.path.isdir(os.path.join(path, subdir)) for subdir in ['include', 'library', 'programs', 'tests']) + def chdir_to_root() -> None: """Detect the root of the Mbed TLS source tree and change to it. @@ -36,3 +39,21 @@ def chdir_to_root() -> None: os.chdir(d) return raise Exception('Mbed TLS source tree not found') + + +def guess_mbedtls_root(): + """Guess mbedTLS source code directory. + + Return the first possible mbedTLS root directory + """ + dirs = set({}) + for i in inspect.stack(): + path = os.path.dirname(i.filename) + for d in ['.', os.path.pardir, os.path.join(*([os.path.pardir]*2))]: + d = os.path.abspath(os.path.join(path, d)) + if d in dirs: + continue + dirs.add(d) + if looks_like_mbedtls_root(d): + return d + raise Exception('Mbed TLS source tree not found') From 24afabae0b118fc2d1bb12c8e4e121d6c0dbf415 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Fri, 10 Dec 2021 14:21:27 +0800 Subject: [PATCH 103/490] fix build fail Signed-off-by: Jerry Yu --- scripts/framework_dev/build_tree.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index aee68f140..3920d0ed6 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -47,9 +47,10 @@ def guess_mbedtls_root(): Return the first possible mbedTLS root directory """ dirs = set({}) - for i in inspect.stack(): - path = os.path.dirname(i.filename) - for d in ['.', os.path.pardir, os.path.join(*([os.path.pardir]*2))]: + for frame in inspect.stack(): + path = os.path.dirname(frame.filename) + for d in ['.', os.path.pardir] \ + + [os.path.join(*([os.path.pardir]*i)) for i in range(2, 10)]: d = os.path.abspath(os.path.join(path, d)) if d in dirs: continue From 7520dddd40ffe49475c997b76fd4522ecb2f2a1b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 4 Mar 2022 20:02:00 +0100 Subject: [PATCH 104/490] Ensure files get closed when they go out of scope This is automatic in CPython but not guaranteed by the language. Be friendly to other Python implementations. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index bf82f13dc..987779d0e 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -18,7 +18,7 @@ import itertools import re -from typing import Dict, Iterable, Iterator, List, Optional, Pattern, Set, Tuple, Union +from typing import Dict, IO, Iterable, Iterator, List, Optional, Pattern, Set, Tuple, Union class ReadFileLineException(Exception): @@ -50,12 +50,13 @@ class read_file_lines: """ def __init__(self, filename: str, binary: bool = False) -> None: self.filename = filename + self.file = None #type: Optional[IO[str]] self.line_number = 'entry' #type: Union[int, str] self.generator = None #type: Optional[Iterable[Tuple[int, str]]] self.binary = binary def __enter__(self) -> 'read_file_lines': - self.generator = enumerate(open(self.filename, - 'rb' if self.binary else 'r')) + self.file = open(self.filename, 'rb' if self.binary else 'r') + self.generator = enumerate(self.file) return self def __iter__(self) -> Iterator[str]: assert self.generator is not None @@ -64,6 +65,8 @@ class read_file_lines: yield content self.line_number = 'exit' def __exit__(self, exc_type, exc_value, exc_traceback) -> None: + if self.file is not None: + self.file.close() if exc_type is not None: raise ReadFileLineException(self.filename, self.line_number) \ from exc_value From e082e79d5247f8d0e280965f3c44cdaedbfaf46b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 29 Apr 2021 20:19:57 +0200 Subject: [PATCH 105/490] Add missing type annotation Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index c38955b0e..6b38aac16 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -26,7 +26,7 @@ from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA class KeyType: """Knowledge about a PSA key type.""" - def __init__(self, name: str, params: Optional[Iterable[str]] = None): + def __init__(self, name: str, params: Optional[Iterable[str]] = None) -> None: """Analyze a key type. The key type must be specified in PSA syntax. In its simplest form, From d11899c9881068dd3caea9b6c3a60a57e1b3b531 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 29 Apr 2021 20:38:01 +0200 Subject: [PATCH 106/490] Add knowledge of algorithms Determine the category of operations supported by an algorithm based on its name. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 151 ++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 6b38aac16..072d53d93 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -18,11 +18,21 @@ This module is entirely based on the PSA API. # See the License for the specific language governing permissions and # limitations under the License. +import enum import re from typing import Dict, Iterable, Optional, Pattern, Tuple from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA + +BLOCK_MAC_MODES = frozenset(['CBC_MAC', 'CMAC']) +BLOCK_CIPHER_MODES = frozenset([ + 'CTR', 'CFB', 'OFB', 'XTS', 'CCM_STAR_NO_TAG', + 'ECB_NO_PADDING', 'CBC_NO_PADDING', 'CBC_PKCS7', +]) +BLOCK_AEAD_MODES = frozenset(['CCM', 'GCM']) + + class KeyType: """Knowledge about a PSA key type.""" @@ -153,3 +163,144 @@ class KeyType: """ # This is just temporaly solution for the implicit usage flags. return re.match(self.KEY_TYPE_FOR_SIGNATURE[usage], self.name) is not None + + +class AlgorithmCategory(enum.Enum): + """PSA algorithm categories.""" + # The numbers are aligned with the category bits in numerical values of + # algorithms. + HASH = 2 + MAC = 3 + CIPHER = 4 + AEAD = 5 + SIGN = 6 + ASYMMETRIC_ENCRYPTION = 7 + KEY_DERIVATION = 8 + KEY_AGREEMENT = 9 + PAKE = 10 + + def requires_key(self) -> bool: + return self not in {self.HASH, self.KEY_DERIVATION} + + +class AlgorithmNotRecognized(Exception): + def __init__(self, expr: str) -> None: + super().__init__('Algorithm not recognized: ' + expr) + self.expr = expr + + +class Algorithm: + """Knowledge about a PSA algorithm.""" + + @staticmethod + def determine_base(expr: str) -> str: + """Return an expression for the "base" of the algorithm. + + This strips off variants of algorithms such as MAC truncation. + + This function does not attempt to detect invalid inputs. + """ + m = re.match(r'PSA_ALG_(?:' + r'(?:TRUNCATED|AT_LEAST_THIS_LENGTH)_MAC|' + r'AEAD_WITH_(?:SHORTENED|AT_LEAST_THIS_LENGTH)_TAG' + r')\((.*),[^,]+\)\Z', expr) + if m: + expr = m.group(1) + return expr + + @staticmethod + def determine_head(expr: str) -> str: + """Return the head of an algorithm expression. + + The head is the first (outermost) constructor, without its PSA_ALG_ + prefix, and with some normalization of similar algorithms. + """ + m = re.match(r'PSA_ALG_(?:DETERMINISTIC_)?(\w+)', expr) + if not m: + raise AlgorithmNotRecognized(expr) + head = m.group(1) + if head == 'KEY_AGREEMENT': + m = re.match(r'PSA_ALG_KEY_AGREEMENT\s*\(\s*PSA_ALG_(\w+)', expr) + if not m: + raise AlgorithmNotRecognized(expr) + head = m.group(1) + head = re.sub(r'_ANY\Z', r'', head) + if re.match(r'ED[0-9]+PH\Z', head): + head = 'EDDSA_PREHASH' + return head + + CATEGORY_FROM_HEAD = { + 'SHA': AlgorithmCategory.HASH, + 'SHAKE256_512': AlgorithmCategory.HASH, + 'MD': AlgorithmCategory.HASH, + 'RIPEMD': AlgorithmCategory.HASH, + 'ANY_HASH': AlgorithmCategory.HASH, + 'HMAC': AlgorithmCategory.MAC, + 'STREAM_CIPHER': AlgorithmCategory.CIPHER, + 'CHACHA20_POLY1305': AlgorithmCategory.AEAD, + 'DSA': AlgorithmCategory.SIGN, + 'ECDSA': AlgorithmCategory.SIGN, + 'EDDSA': AlgorithmCategory.SIGN, + 'PURE_EDDSA': AlgorithmCategory.SIGN, + 'RSA_PSS': AlgorithmCategory.SIGN, + 'RSA_PKCS1V15_SIGN': AlgorithmCategory.SIGN, + 'RSA_PKCS1V15_CRYPT': AlgorithmCategory.ASYMMETRIC_ENCRYPTION, + 'RSA_OAEP': AlgorithmCategory.ASYMMETRIC_ENCRYPTION, + 'HKDF': AlgorithmCategory.KEY_DERIVATION, + 'TLS12_PRF': AlgorithmCategory.KEY_DERIVATION, + 'TLS12_PSK_TO_MS': AlgorithmCategory.KEY_DERIVATION, + 'PBKDF': AlgorithmCategory.KEY_DERIVATION, + 'ECDH': AlgorithmCategory.KEY_AGREEMENT, + 'FFDH': AlgorithmCategory.KEY_AGREEMENT, + # KEY_AGREEMENT(...) is a key derivation with a key agreement component + 'KEY_AGREEMENT': AlgorithmCategory.KEY_DERIVATION, + 'JPAKE': AlgorithmCategory.PAKE, + } + for x in BLOCK_MAC_MODES: + CATEGORY_FROM_HEAD[x] = AlgorithmCategory.MAC + for x in BLOCK_CIPHER_MODES: + CATEGORY_FROM_HEAD[x] = AlgorithmCategory.CIPHER + for x in BLOCK_AEAD_MODES: + CATEGORY_FROM_HEAD[x] = AlgorithmCategory.AEAD + + def determine_category(self, expr: str, head: str) -> AlgorithmCategory: + """Return the category of the given algorithm expression. + + This function does not attempt to detect invalid inputs. + """ + prefix = head + while prefix: + if prefix in self.CATEGORY_FROM_HEAD: + return self.CATEGORY_FROM_HEAD[prefix] + if re.match(r'.*[0-9]\Z', prefix): + prefix = re.sub(r'_*[0-9]+\Z', r'', prefix) + else: + prefix = re.sub(r'_*[^_]*\Z', r'', prefix) + raise AlgorithmNotRecognized(expr) + + @staticmethod + def determine_wildcard(expr) -> bool: + """Whether the given algorithm expression is a wildcard. + + This function does not attempt to detect invalid inputs. + """ + if re.search(r'\bPSA_ALG_ANY_HASH\b', expr): + return True + if re.search(r'_AT_LEAST_', expr): + return True + return False + + def __init__(self, expr: str) -> None: + """Analyze an algorithm value. + + The algorithm must be expressed as a C expression containing only + calls to PSA algorithm constructor macros and numeric literals. + + This class is only programmed to handle valid expressions. Invalid + expressions may result in exceptions or in nonsensical results. + """ + self.expression = re.sub(r'\s+', r'', expr) + self.base_expression = self.determine_base(self.expression) + self.head = self.determine_head(self.base_expression) + self.category = self.determine_category(self.base_expression, self.head) + self.is_wildcard = self.determine_wildcard(self.expression) From ced7064605d5f116ba9325ed943841e2d5e4f8de Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 29 Apr 2021 20:38:47 +0200 Subject: [PATCH 107/490] Add knowledge of the compatibility of key types and algorithms Determine key types that are compatible with an algorithm based on their names. Key derivation and PAKE are not yet supported. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 072d53d93..adda02f52 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -25,6 +25,7 @@ from typing import Dict, Iterable, Optional, Pattern, Tuple from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA +BLOCK_CIPHERS = frozenset(['AES', 'ARIA', 'CAMELLIA', 'DES']) BLOCK_MAC_MODES = frozenset(['CBC_MAC', 'CMAC']) BLOCK_CIPHER_MODES = frozenset([ 'CTR', 'CFB', 'OFB', 'XTS', 'CCM_STAR_NO_TAG', @@ -32,6 +33,25 @@ BLOCK_CIPHER_MODES = frozenset([ ]) BLOCK_AEAD_MODES = frozenset(['CCM', 'GCM']) +class EllipticCurveCategory(enum.Enum): + """Categorization of elliptic curve families. + + The category of a curve determines what algorithms are defined over it. + """ + + SHORT_WEIERSTRASS = 0 + MONTGOMERY = 1 + TWISTED_EDWARDS = 2 + + @staticmethod + def from_family(family: str) -> 'EllipticCurveCategory': + if family == 'PSA_ECC_FAMILY_MONTGOMERY': + return EllipticCurveCategory.MONTGOMERY + if family == 'PSA_ECC_FAMILY_TWISTED_EDWARDS': + return EllipticCurveCategory.TWISTED_EDWARDS + # Default to SW, which most curves belong to. + return EllipticCurveCategory.SHORT_WEIERSTRASS + class KeyType: """Knowledge about a PSA key type.""" @@ -72,6 +92,11 @@ class KeyType: if self.params is not None: self.expression += '(' + ', '.join(self.params) + ')' + m = re.match(r'PSA_KEY_TYPE_(\w+)', self.name) + assert m + self.head = re.sub(r'_(?:PUBLIC_KEY|KEY_PAIR)\Z', r'', m.group(1)) + """The key type macro name, with common prefixes and suffixes stripped.""" + self.private_type = re.sub(r'_PUBLIC_KEY\Z', r'_KEY_PAIR', self.name) """The key type macro name for the corresponding key pair type. @@ -164,6 +189,43 @@ class KeyType: # This is just temporaly solution for the implicit usage flags. return re.match(self.KEY_TYPE_FOR_SIGNATURE[usage], self.name) is not None + def can_do(self, alg: 'Algorithm') -> bool: + """Whether this key type can be used for operations with the given algorithm. + + This function does not currently handle key derivation or PAKE. + """ + #pylint: disable=too-many-return-statements + if alg.is_wildcard: + return False + if self.head == 'HMAC' and alg.head == 'HMAC': + return True + if self.head in BLOCK_CIPHERS and \ + alg.head in frozenset.union(BLOCK_MAC_MODES, + BLOCK_CIPHER_MODES, + BLOCK_AEAD_MODES): + return True + if self.head == 'CHACHA20' and alg.head == 'CHACHA20_POLY1305': + return True + if self.head in {'ARC4', 'CHACHA20'} and \ + alg.head == 'STREAM_CIPHER': + return True + if self.head == 'RSA' and alg.head.startswith('RSA_'): + return True + if self.head == 'ECC': + assert self.params is not None + eccc = EllipticCurveCategory.from_family(self.params[0]) + if alg.head == 'ECDH' and \ + eccc in {EllipticCurveCategory.SHORT_WEIERSTRASS, + EllipticCurveCategory.MONTGOMERY}: + return True + if alg.head == 'ECDSA' and \ + eccc == EllipticCurveCategory.SHORT_WEIERSTRASS: + return True + if alg.head in {'PURE_EDDSA', 'EDDSA_PREHASH'} and \ + eccc == EllipticCurveCategory.TWISTED_EDWARDS: + return True + return False + class AlgorithmCategory(enum.Enum): """PSA algorithm categories.""" From 3c7a06342a906a6b8e7c095e7450c3d64b51f433 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 29 Apr 2021 20:54:40 +0200 Subject: [PATCH 108/490] A key agreement algorithm can contain a key derivation PSA_ALG_KEY_AGREEMENT(..., kdf) is a valid key derivation algorithm when kdf is one. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index adda02f52..73890f81a 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -366,3 +366,23 @@ class Algorithm: self.head = self.determine_head(self.base_expression) self.category = self.determine_category(self.base_expression, self.head) self.is_wildcard = self.determine_wildcard(self.expression) + + def is_key_agreement_with_derivation(self) -> bool: + """Whether this is a combined key agreement and key derivation algorithm.""" + if self.category != AlgorithmCategory.KEY_AGREEMENT: + return False + m = re.match(r'PSA_ALG_KEY_AGREEMENT\(\w+,\s*(.*)\)\Z', self.expression) + if not m: + return False + kdf_alg = m.group(1) + # Assume kdf_alg is either a valid KDF or 0. + return not re.match(r'(?:0[Xx])?0+\s*\Z', kdf_alg) + + def can_do(self, category: AlgorithmCategory) -> bool: + """Whether this algorithm fits the specified operation category.""" + if category == self.category: + return True + if category == AlgorithmCategory.KEY_DERIVATION and \ + self.is_key_agreement_with_derivation(): + return True + return False From 2f3d9a972111d314a3811ac7822abfacaec45931 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 29 Apr 2021 21:56:59 +0200 Subject: [PATCH 109/490] Test attempts to use a public key for a private-key operation Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 73890f81a..1bd011fe2 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -104,6 +104,10 @@ class KeyType: `self.name`. """ + def is_public(self) -> bool: + """Whether the key type is for public keys.""" + return self.name.endswith('_PUBLIC_KEY') + ECC_KEY_SIZES = { 'PSA_ECC_FAMILY_SECP_K1': (192, 224, 256), 'PSA_ECC_FAMILY_SECP_R1': (225, 256, 384, 521), @@ -242,8 +246,17 @@ class AlgorithmCategory(enum.Enum): PAKE = 10 def requires_key(self) -> bool: + """Whether operations in this category are set up with a key.""" return self not in {self.HASH, self.KEY_DERIVATION} + def is_asymmetric(self) -> bool: + """Whether operations in this category involve asymmetric keys.""" + return self in { + self.SIGN, + self.ASYMMETRIC_ENCRYPTION, + self.KEY_AGREEMENT + } + class AlgorithmNotRecognized(Exception): def __init__(self, expr: str) -> None: From 10cdcf511a46281781b3137b3fb821bfa0afcb8c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 17 Mar 2022 12:52:24 +0100 Subject: [PATCH 110/490] Remove ad hoc is_valid_for_signature method Use the new generic is_public method. Impact on generated cases: there are new HMAC test cases for SIGN_HASH. It was a bug that these test cases were previously not generated. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 1bd011fe2..82487d97f 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -20,7 +20,7 @@ This module is entirely based on the PSA API. import enum import re -from typing import Dict, Iterable, Optional, Pattern, Tuple +from typing import Iterable, Optional, Tuple from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA @@ -178,21 +178,6 @@ class KeyType: return b''.join([self.DATA_BLOCK] * (length // len(self.DATA_BLOCK)) + [self.DATA_BLOCK[:length % len(self.DATA_BLOCK)]]) - KEY_TYPE_FOR_SIGNATURE = { - 'PSA_KEY_USAGE_SIGN_HASH': re.compile('.*KEY_PAIR'), - 'PSA_KEY_USAGE_VERIFY_HASH': re.compile('.*KEY.*') - } #type: Dict[str, Pattern] - """Use a regexp to determine key types for which signature is possible - when using the actual usage flag. - """ - def is_valid_for_signature(self, usage: str) -> bool: - """Determine if the key type is compatible with the specified - signitute type. - - """ - # This is just temporaly solution for the implicit usage flags. - return re.match(self.KEY_TYPE_FOR_SIGNATURE[usage], self.name) is not None - def can_do(self, alg: 'Algorithm') -> bool: """Whether this key type can be used for operations with the given algorithm. From 33d68a09db0ef5c522d97bdb9ce1ff32c543f2bc Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 17 Mar 2022 23:42:25 +0100 Subject: [PATCH 111/490] Unify the code to shorten expressions The output of generate_psa_tests.py is almost unchanged: the differences are only spaces after commas (now consistently omitted). Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 82487d97f..f2775265c 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -25,6 +25,20 @@ from typing import Iterable, Optional, Tuple from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA +def short_expression(original: str) -> str: + """Abbreviate the expression, keeping it human-readable. + + If `level` is 0, just remove parts that are implicit from context, + such as a leading ``PSA_KEY_TYPE_``. + For larger values of `level`, also abbreviate some names in an + unambiguous, but ad hoc way. + """ + short = original + short = re.sub(r'\bPSA_(?:ALG|ECC_FAMILY|KEY_[A-Z]+)_', r'', short) + short = re.sub(r' +', r'', short) + return short + + BLOCK_CIPHERS = frozenset(['AES', 'ARIA', 'CAMELLIA', 'DES']) BLOCK_MAC_MODES = frozenset(['CBC_MAC', 'CMAC']) BLOCK_CIPHER_MODES = frozenset([ @@ -104,6 +118,13 @@ class KeyType: `self.name`. """ + def short_expression(self) -> str: + """Abbreviate the expression, keeping it human-readable. + + See `crypto_knowledge.short_expression`. + """ + return short_expression(self.expression) + def is_public(self) -> bool: """Whether the key type is for public keys.""" return self.name.endswith('_PUBLIC_KEY') @@ -376,6 +397,14 @@ class Algorithm: # Assume kdf_alg is either a valid KDF or 0. return not re.match(r'(?:0[Xx])?0+\s*\Z', kdf_alg) + + def short_expression(self) -> str: + """Abbreviate the expression, keeping it human-readable. + + See `crypto_knowledge.short_expression`. + """ + return short_expression(self.expression) + def can_do(self, category: AlgorithmCategory) -> bool: """Whether this algorithm fits the specified operation category.""" if category == self.category: From a1cc59d6097fb19979a81de802ea5af60d92a0ef Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 18 Mar 2022 00:02:15 +0100 Subject: [PATCH 112/490] Abbreviate descriptions of generated PSA storage tests This currently makes all the descriptions unambiguous even when truncated at 66 characters, as the unit test framework does. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index f2775265c..434ecf33e 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -25,7 +25,7 @@ from typing import Iterable, Optional, Tuple from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA -def short_expression(original: str) -> str: +def short_expression(original: str, level: int = 0) -> str: """Abbreviate the expression, keeping it human-readable. If `level` is 0, just remove parts that are implicit from context, @@ -36,6 +36,15 @@ def short_expression(original: str) -> str: short = original short = re.sub(r'\bPSA_(?:ALG|ECC_FAMILY|KEY_[A-Z]+)_', r'', short) short = re.sub(r' +', r'', short) + if level >= 1: + short = re.sub(r'PUBLIC_KEY\b', r'PUB', short) + short = re.sub(r'KEY_PAIR\b', r'PAIR', short) + short = re.sub(r'\bBRAINPOOL_P', r'BP', short) + short = re.sub(r'\bMONTGOMERY\b', r'MGM', short) + short = re.sub(r'AEAD_WITH_SHORTENED_TAG\b', r'AEAD_SHORT', short) + short = re.sub(r'\bDETERMINISTIC_', r'DET_', short) + short = re.sub(r'\bKEY_AGREEMENT\b', r'KA', short) + short = re.sub(r'_PSK_TO_MS\b', r'_PSK2MS', short) return short @@ -118,12 +127,12 @@ class KeyType: `self.name`. """ - def short_expression(self) -> str: + def short_expression(self, level: int = 0) -> str: """Abbreviate the expression, keeping it human-readable. See `crypto_knowledge.short_expression`. """ - return short_expression(self.expression) + return short_expression(self.expression, level=level) def is_public(self) -> bool: """Whether the key type is for public keys.""" @@ -398,12 +407,12 @@ class Algorithm: return not re.match(r'(?:0[Xx])?0+\s*\Z', kdf_alg) - def short_expression(self) -> str: + def short_expression(self, level: int = 0) -> str: """Abbreviate the expression, keeping it human-readable. See `crypto_knowledge.short_expression`. """ - return short_expression(self.expression) + return short_expression(self.expression, level=level) def can_do(self, category: AlgorithmCategory) -> bool: """Whether this algorithm fits the specified operation category.""" From 5afd5e3dbd09843eb6f9b61d3fa4298475684d11 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 18 Mar 2022 09:58:09 +0100 Subject: [PATCH 113/490] Storage format tests: exercise operations with keys In key read tests, add usage flags that are suitable for the key type and algorithm. This way, the call to exercise_key() in the test not only checks that exporting the key is possible, but also that operations on the key are possible. This triggers a number of failures in edge cases where the generator generates combinations that are not valid, which will be fixed in subsequent commits. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 30 ++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 434ecf33e..6f499403a 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -20,7 +20,7 @@ This module is entirely based on the PSA API. import enum import re -from typing import Iterable, Optional, Tuple +from typing import Iterable, List, Optional, Tuple from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA @@ -422,3 +422,31 @@ class Algorithm: self.is_key_agreement_with_derivation(): return True return False + + def usage_flags(self, public: bool = False) -> List[str]: + """The list of usage flags describing operations that can perform this algorithm. + + If public is true, only return public-key operations, not private-key operations. + """ + if self.category == AlgorithmCategory.HASH: + flags = [] + elif self.category == AlgorithmCategory.MAC: + flags = ['SIGN_HASH', 'SIGN_MESSAGE', + 'VERIFY_HASH', 'VERIFY_MESSAGE'] + elif self.category == AlgorithmCategory.CIPHER or \ + self.category == AlgorithmCategory.AEAD: + flags = ['DECRYPT', 'ENCRYPT'] + elif self.category == AlgorithmCategory.SIGN: + flags = ['VERIFY_HASH', 'VERIFY_MESSAGE'] + if not public: + flags += ['SIGN_HASH', 'SIGN_MESSAGE'] + elif self.category == AlgorithmCategory.ASYMMETRIC_ENCRYPTION: + flags = ['ENCRYPT'] + if not public: + flags += ['DECRYPT'] + elif self.category == AlgorithmCategory.KEY_DERIVATION or \ + self.category == AlgorithmCategory.KEY_AGREEMENT: + flags = ['DERIVE'] + else: + raise AlgorithmNotRecognized(self.expression) + return ['PSA_KEY_USAGE_' + flag for flag in flags] From 2200591cd37e15af1c016b7cce5a8c1788354d68 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 18 Mar 2022 10:18:58 +0100 Subject: [PATCH 114/490] 64-bit block ciphers are incompatible with some modes Only allow selected modes with 64-bit block ciphers (i.e. DES). This removes some storage tests and creates corresponding op_fail tests. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 6f499403a..bd4d94ee8 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -218,6 +218,12 @@ class KeyType: return False if self.head == 'HMAC' and alg.head == 'HMAC': return True + if self.head == 'DES': + # 64-bit block ciphers only allow a reduced set of modes. + return alg.head in [ + 'CBC_NO_PADDING', 'CBC_PKCS7', + 'ECB_NO_PADDING', + ] if self.head in BLOCK_CIPHERS and \ alg.head in frozenset.union(BLOCK_MAC_MODES, BLOCK_CIPHER_MODES, From 73f0cd46661e1fa23f632ab8b7c7513fc59d6a93 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 18 Mar 2022 18:46:00 +0100 Subject: [PATCH 115/490] Test more truncated MAC and short AEAD tag lengths The current macro collector only tried the minimum and maximum expressible lengths for PSA_ALG_TRUNCATED_MAC and PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG. This was good enough for psa_constant_names, but it's weak for exercising keys, in particular because it doesn't include any valid AEAD tag length. So cover more lengths. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 987779d0e..218a718f3 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -400,10 +400,26 @@ enumerate 'other_algorithm': [], 'lifetime': [self.lifetimes], } #type: Dict[str, List[Set[str]]] - self.arguments_for['mac_length'] += ['1', '63'] - self.arguments_for['min_mac_length'] += ['1', '63'] - self.arguments_for['tag_length'] += ['1', '63'] - self.arguments_for['min_tag_length'] += ['1', '63'] + mac_lengths = [str(n) for n in [ + 1, # minimum expressible + 4, # minimum allowed by policy + 13, # an odd size in a plausible range + 14, # an even non-power-of-two size in a plausible range + 16, # same as full size for at least one algorithm + 63, # maximum expressible + ]] + self.arguments_for['mac_length'] += mac_lengths + self.arguments_for['min_mac_length'] += mac_lengths + aead_lengths = [str(n) for n in [ + 1, # minimum expressible + 4, # minimum allowed by policy + 13, # an odd size in a plausible range + 14, # an even non-power-of-two size in a plausible range + 16, # same as full size for at least one algorithm + 63, # maximum expressible + ]] + self.arguments_for['tag_length'] += aead_lengths + self.arguments_for['min_tag_length'] += aead_lengths def add_numerical_values(self) -> None: """Add numerical values that are not supported to the known identifiers.""" From 0f8a73c2685c10b3061f7e5ee6af8df80af83360 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 19 Mar 2022 10:36:07 +0100 Subject: [PATCH 116/490] Fix invalid argument enumeration when there are >=3 arguments This bug had no impact since currently no macro has more than 2 arguments. Signed-off-by: Gilles Peskine --- scripts/framework_dev/macro_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 218a718f3..3cad2a3f6 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -186,7 +186,7 @@ class PSAMacroEnumerator: for value in argument_lists[i][1:]: arguments[i] = value yield self._format_arguments(name, arguments) - arguments[i] = argument_lists[0][0] + arguments[i] = argument_lists[i][0] except BaseException as e: raise Exception('distribute_arguments({})'.format(name)) from e From dc707bcf4c82652d96a96f4984c82c13a7989ba7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 19 Mar 2022 10:37:33 +0100 Subject: [PATCH 117/490] Reject invalid MAC and AEAD truncations Reject algorithms of the form PSA_ALG_TRUNCATED_MAC(...) or PSA_ALG_AEAD_WITH_SHORTENED_TAG(...) when the truncation length is invalid or not accepted by policy in Mbed TLS. This is done in KeyType.can_do, so in generate_psa_tests.py, keys will be tested for operation failure with this algorithm if the algorithm is rejected, and for storage if the algorithm is accepted. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 53 ++++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index bd4d94ee8..7d2e755f6 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -20,7 +20,7 @@ This module is entirely based on the PSA API. import enum import re -from typing import Iterable, List, Optional, Tuple +from typing import FrozenSet, Iterable, List, Optional, Tuple from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA @@ -216,6 +216,8 @@ class KeyType: #pylint: disable=too-many-return-statements if alg.is_wildcard: return False + if alg.is_invalid_truncation(): + return False if self.head == 'HMAC' and alg.head == 'HMAC': return True if self.head == 'DES': @@ -420,8 +422,55 @@ class Algorithm: """ return short_expression(self.expression, level=level) + PERMITTED_TAG_LENGTHS = { + 'PSA_ALG_CCM': frozenset([4, 6, 8, 10, 12, 14, 16]), + 'PSA_ALG_CHACHA20_POLY1305': frozenset([16]), + 'PSA_ALG_GCM': frozenset([4, 8, 12, 13, 14, 15, 16]), + } + MAC_LENGTH = { + 'PSA_ALG_CBC_MAC': 16, + 'PSA_ALG_CMAC': 16, + 'PSA_ALG_HMAC(PSA_ALG_MD5)': 16, + 'PSA_ALG_HMAC(PSA_ALG_SHA_1)': 20, + } + HMAC_WITH_NOMINAL_LENGTH_RE = re.compile(r'PSA_ALG_HMAC\(\w+([0-9])+\)\Z') + @classmethod + def mac_or_tag_length(cls, base: str) -> FrozenSet[int]: + """Return the set of permitted lengths for the given MAC or AEAD tag.""" + if base in cls.PERMITTED_TAG_LENGTHS: + return cls.PERMITTED_TAG_LENGTHS[base] + max_length = cls.MAC_LENGTH.get(base, None) + if max_length is None: + m = cls.HMAC_WITH_NOMINAL_LENGTH_RE.match(base) + if m: + max_length = int(m.group(1)) // 8 + if max_length is None: + raise ValueError('Unknown permitted lengths for ' + base) + return frozenset(range(4, max_length + 1)) + + TRUNCATED_ALG_RE = re.compile( + r'(?PPSA_ALG_(?:AEAD_WITH_SHORTENED_TAG|TRUNCATED_MAC))' + r'\((?P.*),' + r'(?P0[Xx][0-9A-Fa-f]+|[1-9][0-9]*|0[0-9]*)[LUlu]*\)\Z') + def is_invalid_truncation(self) -> bool: + """False for a MAC or AEAD algorithm truncated to an invalid length. + + True for a MAC or AEAD algorithm truncated to a valid length or to + a length that cannot be determined. True for anything other than + a truncated MAC or AEAD. + """ + m = self.TRUNCATED_ALG_RE.match(self.expression) + if m: + base = m.group('base') + to_length = int(m.group('length'), 0) + permitted_lengths = self.mac_or_tag_length(base) + if to_length not in permitted_lengths: + return True + return False + def can_do(self, category: AlgorithmCategory) -> bool: - """Whether this algorithm fits the specified operation category.""" + """Whether this algorithm can perform operations in the given category. + """ if category == self.category: return True if category == AlgorithmCategory.KEY_DERIVATION and \ From 556211285fc0ef14a9b4c80b7d3f7327fc3d8bee Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 19 Mar 2022 10:49:43 +0100 Subject: [PATCH 118/490] Reject block cipher modes that are not implemented in Mbed TLS Mbed TLS doesn't support certain block cipher mode combinations. This limitation should probably be lifted, but for now, test them as unsupported. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 7d2e755f6..55eb54d59 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -230,6 +230,9 @@ class KeyType: alg.head in frozenset.union(BLOCK_MAC_MODES, BLOCK_CIPHER_MODES, BLOCK_AEAD_MODES): + if alg.head in ['CMAC', 'OFB'] and \ + self.head in ['ARIA', 'CAMELLIA']: + return False # not implemented in Mbed TLS return True if self.head == 'CHACHA20' and alg.head == 'CHACHA20_POLY1305': return True From dfe08acdc9abd649517975a64f6f9e675648903c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 19 Mar 2022 12:09:13 +0100 Subject: [PATCH 119/490] Don't exercise OAEP with small key and large hash RSA-OAEP requires the key to be larger than a function of the hash size. Ideally such combinations would be detected as a key/algorithm incompatibility. However key/algorithm compatibility is currently tested between the key type and the algorithm without considering the key size, and this is inconvenient to change. So as a workaround, dispense OAEP-with-too-small-hash from exercising, without including it in the automatic operation-failure test generation. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 33 ++++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 55eb54d59..1491a844a 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -213,7 +213,7 @@ class KeyType: This function does not currently handle key derivation or PAKE. """ - #pylint: disable=too-many-return-statements + #pylint: disable=too-many-branches,too-many-return-statements if alg.is_wildcard: return False if alg.is_invalid_truncation(): @@ -425,28 +425,41 @@ class Algorithm: """ return short_expression(self.expression, level=level) + HASH_LENGTH = { + 'PSA_ALG_MD5': 16, + 'PSA_ALG_SHA_1': 20, + } + HASH_LENGTH_BITS_RE = re.compile(r'([0-9]+)\Z') + @classmethod + def hash_length(cls, alg: str) -> int: + """The length of the given hash algorithm, in bytes.""" + if alg in cls.HASH_LENGTH: + return cls.HASH_LENGTH[alg] + m = cls.HASH_LENGTH_BITS_RE.search(alg) + if m: + return int(m.group(1)) // 8 + raise ValueError('Unknown hash length for ' + alg) + PERMITTED_TAG_LENGTHS = { 'PSA_ALG_CCM': frozenset([4, 6, 8, 10, 12, 14, 16]), 'PSA_ALG_CHACHA20_POLY1305': frozenset([16]), 'PSA_ALG_GCM': frozenset([4, 8, 12, 13, 14, 15, 16]), } MAC_LENGTH = { - 'PSA_ALG_CBC_MAC': 16, - 'PSA_ALG_CMAC': 16, - 'PSA_ALG_HMAC(PSA_ALG_MD5)': 16, - 'PSA_ALG_HMAC(PSA_ALG_SHA_1)': 20, + 'PSA_ALG_CBC_MAC': 16, # actually the block cipher length + 'PSA_ALG_CMAC': 16, # actually the block cipher length } - HMAC_WITH_NOMINAL_LENGTH_RE = re.compile(r'PSA_ALG_HMAC\(\w+([0-9])+\)\Z') + HMAC_RE = re.compile(r'PSA_ALG_HMAC\((.*)\)\Z') @classmethod - def mac_or_tag_length(cls, base: str) -> FrozenSet[int]: + def mac_or_tag_lengths(cls, base: str) -> FrozenSet[int]: """Return the set of permitted lengths for the given MAC or AEAD tag.""" if base in cls.PERMITTED_TAG_LENGTHS: return cls.PERMITTED_TAG_LENGTHS[base] max_length = cls.MAC_LENGTH.get(base, None) if max_length is None: - m = cls.HMAC_WITH_NOMINAL_LENGTH_RE.match(base) + m = cls.HMAC_RE.match(base) if m: - max_length = int(m.group(1)) // 8 + max_length = cls.hash_length(m.group(1)) if max_length is None: raise ValueError('Unknown permitted lengths for ' + base) return frozenset(range(4, max_length + 1)) @@ -466,7 +479,7 @@ class Algorithm: if m: base = m.group('base') to_length = int(m.group('length'), 0) - permitted_lengths = self.mac_or_tag_length(base) + permitted_lengths = self.mac_or_tag_lengths(base) if to_length not in permitted_lengths: return True return False From 2b0f4937f7d710c6f5e6eec0c3e78f1b0ab0b261 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 19 Mar 2022 12:16:45 +0100 Subject: [PATCH 120/490] Public keys can't be used as private-key inputs to key agreement The PSA API does not use public key objects in key agreement operations: it imports the public key as a formatted byte string. So a public key object with a key agreement algorithm is not a valid combination. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 1491a844a..54de0def6 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -241,6 +241,13 @@ class KeyType: return True if self.head == 'RSA' and alg.head.startswith('RSA_'): return True + if alg.category == AlgorithmCategory.KEY_AGREEMENT and \ + self.is_public(): + # The PSA API does not use public key objects in key agreement + # operations: it imports the public key as a formatted byte string. + # So a public key object with a key agreement algorithm is not + # a valid combination. + return False if self.head == 'ECC': assert self.params is not None eccc = EllipticCurveCategory.from_family(self.params[0]) From cd29386c3a10c072ccb08294ab641787181ce589 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 5 Apr 2022 16:31:16 +0200 Subject: [PATCH 121/490] Fix digits in octal constant Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 54de0def6..238a34bb8 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -474,7 +474,7 @@ class Algorithm: TRUNCATED_ALG_RE = re.compile( r'(?PPSA_ALG_(?:AEAD_WITH_SHORTENED_TAG|TRUNCATED_MAC))' r'\((?P.*),' - r'(?P0[Xx][0-9A-Fa-f]+|[1-9][0-9]*|0[0-9]*)[LUlu]*\)\Z') + r'(?P0[Xx][0-9A-Fa-f]+|[1-9][0-9]*|0[0-7]*)[LUlu]*\)\Z') def is_invalid_truncation(self) -> bool: """False for a MAC or AEAD algorithm truncated to an invalid length. From fe53bc5ca509510f3dbf17dac37662517efb6bbe Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 12 Apr 2022 18:51:01 +0200 Subject: [PATCH 122/490] Rename and document mac_or_tag_lengths -> permitted_truncations No behavior change. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 238a34bb8..592fc0afe 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -458,8 +458,14 @@ class Algorithm: } HMAC_RE = re.compile(r'PSA_ALG_HMAC\((.*)\)\Z') @classmethod - def mac_or_tag_lengths(cls, base: str) -> FrozenSet[int]: - """Return the set of permitted lengths for the given MAC or AEAD tag.""" + def permitted_truncations(cls, base: str) -> FrozenSet[int]: + """Permitted output lengths for the given MAC or AEAD base algorithm. + + For a MAC algorithm, this is the set of truncation lengths that + Mbed TLS supports. + For an AEAD algorithm, this is the set of truncation lengths that + are permitted by the algorithm specification. + """ if base in cls.PERMITTED_TAG_LENGTHS: return cls.PERMITTED_TAG_LENGTHS[base] max_length = cls.MAC_LENGTH.get(base, None) @@ -486,7 +492,7 @@ class Algorithm: if m: base = m.group('base') to_length = int(m.group('length'), 0) - permitted_lengths = self.mac_or_tag_lengths(base) + permitted_lengths = self.permitted_truncations(base) if to_length not in permitted_lengths: return True return False From 7e67daa1b0514f88aafbe65614e946aeea82a534 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 20 Jun 2022 19:10:35 +0200 Subject: [PATCH 123/490] Add warnings to test code and data about storage format stability Signed-off-by: Gilles Peskine --- scripts/framework_dev/psa_storage.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index 45f0380e7..a06dce13b 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -1,4 +1,9 @@ """Knowledge about the PSA key store as implemented in Mbed TLS. + +Note that if you need to make a change that affects how keys are +stored, this may indicate that the key store is changing in a +backward-incompatible way! Think carefully about backward compatibility +before changing how test data is constructed or validated. """ # Copyright The Mbed TLS Contributors @@ -146,6 +151,11 @@ class Key: This is the content of the PSA storage file. When PSA storage is implemented over stdio files, this does not include any wrapping made by the PSA-storage-over-stdio-file implementation. + + Note that if you need to make a change in this function, + this may indicate that the key store is changing in a + backward-incompatible way! Think carefully about backward + compatibility before making any change here. """ header = self.MAGIC + self.pack('L', self.version) if self.version == 0: From 1c3f1181727162bd6f0a50f0d6ae27d6a2f4d66d Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 11:30:03 +0100 Subject: [PATCH 124/490] Separate common test generation classes/functions Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 149 +++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 scripts/framework_dev/test_generation.py diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py new file mode 100644 index 000000000..2414f3a4b --- /dev/null +++ b/scripts/framework_dev/test_generation.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Common test generation classes and main function. + +These are used both by generate_psa_tests.py and generate_bignum_tests.py. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import posixpath +import re +from typing import Callable, Dict, Iterable, List, Type, TypeVar + +from mbedtls_dev import build_tree +from mbedtls_dev import test_case + +T = TypeVar('T') #pylint: disable=invalid-name + + +class BaseTarget: + """Base target for test case generation. + + Attributes: + count: Counter for test class. + desc: Short description of test case. + func: Function which the class generates tests for. + gen_file: File to write generated tests to. + title: Description of the test function/purpose. + """ + count = 0 + desc = "" + func = "" + gen_file = "" + title = "" + + def __init__(self) -> None: + type(self).count += 1 + + @property + def args(self) -> List[str]: + """Create list of arguments for test case.""" + return [] + + @property + def description(self) -> str: + """Create a numbered test description.""" + return "{} #{} {}".format(self.title, self.count, self.desc) + + def create_test_case(self) -> test_case.TestCase: + """Generate test case from the current object.""" + tc = test_case.TestCase() + tc.set_description(self.description) + tc.set_function(self.func) + tc.set_arguments(self.args) + + return tc + + @classmethod + def generate_tests(cls): + """Generate test cases for the target subclasses.""" + for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): + yield from subclass.generate_tests() + + +class TestGenerator: + """Generate test data.""" + def __init__(self, options) -> None: + self.test_suite_directory = self.get_option(options, 'directory', + 'tests/suites') + + @staticmethod + def get_option(options, name: str, default: T) -> T: + value = getattr(options, name, None) + return default if value is None else value + + def filename_for(self, basename: str) -> str: + """The location of the data file with the specified base name.""" + return posixpath.join(self.test_suite_directory, basename + '.data') + + def write_test_data_file(self, basename: str, + test_cases: Iterable[test_case.TestCase]) -> None: + """Write the test cases to a .data file. + + The output file is ``basename + '.data'`` in the test suite directory. + """ + filename = self.filename_for(basename) + test_case.write_data_file(filename, test_cases) + + # Note that targets whose names contain 'test_format' have their content + # validated by `abi_check.py`. + TARGETS = {} # type: Dict[str, Callable[..., test_case.TestCase]] + + def generate_target(self, name: str, *target_args) -> None: + """Generate cases and write to data file for a target. + + For target callables which require arguments, override this function + and pass these arguments using super() (see PSATestGenerator). + """ + test_cases = self.TARGETS[name](*target_args) + self.write_test_data_file(name, test_cases) + +def main(args, generator_class: Type[TestGenerator] = TestGenerator): + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--list', action='store_true', + help='List available targets and exit') + parser.add_argument('--list-for-cmake', action='store_true', + help='Print \';\'-separated list of available targets and exit') + parser.add_argument('--directory', metavar='DIR', + help='Output directory (default: tests/suites)') + parser.add_argument('targets', nargs='*', metavar='TARGET', + help='Target file to generate (default: all; "-": none)') + options = parser.parse_args(args) + build_tree.chdir_to_root() + generator = generator_class(options) + if options.list: + for name in sorted(generator.TARGETS): + print(generator.filename_for(name)) + return + # List in a cmake list format (i.e. ';'-separated) + if options.list_for_cmake: + print(';'.join(generator.filename_for(name) + for name in sorted(generator.TARGETS)), end='') + return + if options.targets: + # Allow "-" as a special case so you can run + # ``generate_xxx_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] + else: + options.targets = sorted(generator.TARGETS) + for target in options.targets: + generator.generate_target(target) From 09159cd3a20b51c48fc48a50c9650cbcb4600e16 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 23 Aug 2022 14:21:53 +0100 Subject: [PATCH 125/490] Remove abbreviations and clarify attributes Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 34 ++++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 2414f3a4b..bb70b9c72 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -35,37 +35,37 @@ class BaseTarget: """Base target for test case generation. Attributes: - count: Counter for test class. - desc: Short description of test case. - func: Function which the class generates tests for. - gen_file: File to write generated tests to. - title: Description of the test function/purpose. + count: Counter for test cases from this class. + case_description: Short description of the test case. This may be + automatically generated using the class, or manually set. + target_basename: Basename of file to write generated tests to. This + should be specified in a child class of BaseTarget. + test_function: Test function which the class generates cases for. + test_name: A common name or description of the test function. This can + be the function's name, or a short summary of its purpose. """ count = 0 - desc = "" - func = "" - gen_file = "" - title = "" + case_description = "" + target_basename = "" + test_function = "" + test_name = "" def __init__(self) -> None: type(self).count += 1 - @property - def args(self) -> List[str]: - """Create list of arguments for test case.""" + def arguments(self) -> List[str]: return [] - @property def description(self) -> str: """Create a numbered test description.""" - return "{} #{} {}".format(self.title, self.count, self.desc) + return "{} #{} {}".format(self.test_name, self.count, self.case_description) def create_test_case(self) -> test_case.TestCase: """Generate test case from the current object.""" tc = test_case.TestCase() - tc.set_description(self.description) - tc.set_function(self.func) - tc.set_arguments(self.args) + tc.set_description(self.description()) + tc.set_function(self.test_function) + tc.set_arguments(self.arguments()) return tc From 7a64e5474109e1b004e92712e5c81ac4a9c6ca27 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 23 Aug 2022 16:07:37 +0100 Subject: [PATCH 126/490] Add details to docstrings Clarification is added to docstrings, mostly in abstract classes. Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 38 +++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index bb70b9c72..712c7996b 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -23,6 +23,8 @@ import argparse import os import posixpath import re + +from abc import abstractmethod from typing import Callable, Dict, Iterable, List, Type, TypeVar from mbedtls_dev import build_tree @@ -53,15 +55,34 @@ class BaseTarget: def __init__(self) -> None: type(self).count += 1 + @abstractmethod def arguments(self) -> List[str]: - return [] + """Get the list of arguments for the test case. + + Override this method to provide the list of arguments required for + generating the test_function. + + Returns: + List of arguments required for the test function. + """ + pass def description(self) -> str: - """Create a numbered test description.""" + """Create a test description. + + Creates a description of the test case, including a name for the test + function, and describing the specific test case. This should inform a + reader of the purpose of the case. The case description may be + generated in the class, or provided manually as needed. + + Returns: + Description for the test case. + """ return "{} #{} {}".format(self.test_name, self.count, self.case_description) + def create_test_case(self) -> test_case.TestCase: - """Generate test case from the current object.""" + """Generate TestCase from the current object.""" tc = test_case.TestCase() tc.set_description(self.description()) tc.set_function(self.test_function) @@ -71,7 +92,16 @@ class BaseTarget: @classmethod def generate_tests(cls): - """Generate test cases for the target subclasses.""" + """Generate test cases for the target subclasses. + + Classes will iterate over its subclasses, calling this method in each. + In abstract classes, no further changes are needed, as there is no + function to generate tests for. + In classes which do implement a test function, this should be overrided + and a means to use `create_test_case()` should be added. In most cases + the subclasses can still be iterated over, as either the class will + have none, or it may continue. + """ for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() From cce48b1d956574f39f9449e25693dfc898daf8d1 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 12:18:25 +0100 Subject: [PATCH 127/490] Use ABCMeta for abstract classes Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 712c7996b..b825df07b 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -24,7 +24,7 @@ import os import posixpath import re -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from typing import Callable, Dict, Iterable, List, Type, TypeVar from mbedtls_dev import build_tree @@ -33,7 +33,7 @@ from mbedtls_dev import test_case T = TypeVar('T') #pylint: disable=invalid-name -class BaseTarget: +class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. Attributes: @@ -94,13 +94,12 @@ class BaseTarget: def generate_tests(cls): """Generate test cases for the target subclasses. - Classes will iterate over its subclasses, calling this method in each. - In abstract classes, no further changes are needed, as there is no + During generation, each class will iterate over any subclasses, calling + this method in each. + In abstract classes, no tests will be generated, as there is no function to generate tests for. - In classes which do implement a test function, this should be overrided - and a means to use `create_test_case()` should be added. In most cases - the subclasses can still be iterated over, as either the class will - have none, or it may continue. + In classes which do implement a test function, this should be overridden + and a means to use `create_test_case()` should be added. """ for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() From e49a5f5d321482908aad68181e5e5ec5bb5e1513 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 12:42:00 +0100 Subject: [PATCH 128/490] Split generate_tests to reduce code complexity Previous implementation mixed the test case generation and the recursive generation calls together. A separate method is added to generate test cases for the current class' test function. This reduces the need to override generate_tests(). Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 33 +++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index b825df07b..aeb551d05 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -25,7 +25,7 @@ import posixpath import re from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Iterable, List, Type, TypeVar +from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar from mbedtls_dev import build_tree from mbedtls_dev import test_case @@ -91,16 +91,31 @@ class BaseTarget(metaclass=ABCMeta): return tc @classmethod - def generate_tests(cls): - """Generate test cases for the target subclasses. + @abstractmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + """Generate test cases for the test function. - During generation, each class will iterate over any subclasses, calling - this method in each. - In abstract classes, no tests will be generated, as there is no - function to generate tests for. - In classes which do implement a test function, this should be overridden - and a means to use `create_test_case()` should be added. + This will be called in classes where `test_function` is set. + Implementations should yield TestCase objects, by creating instances + of the class with appropriate input data, and then calling + `create_test_case()` on each. """ + pass + + @classmethod + def generate_tests(cls) -> Iterator[test_case.TestCase]: + """Generate test cases for the class and its subclasses. + + In classes with `test_function` set, `generate_function_tests()` is + used to generate test cases first. + In all classes, this method will iterate over its subclasses, and + yield from `generate_tests()` in each. + + Calling this method on a class X will yield test cases from all classes + derived from X. + """ + if cls.test_function: + yield from cls.generate_function_tests() for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() From 1a87c81b34452363ad6dc298e806f6fc432ccc5d Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 17:04:07 +0100 Subject: [PATCH 129/490] Use __new__() for case counting Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index aeb551d05..f1e085d4e 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -52,8 +52,9 @@ class BaseTarget(metaclass=ABCMeta): test_function = "" test_name = "" - def __init__(self) -> None: - type(self).count += 1 + def __new__(cls, *args, **kwargs): + cls.count += 1 + return super().__new__(cls) @abstractmethod def arguments(self) -> List[str]: From 29a49ffc8d85ff3aafe8a6c4e8312c554a631faa Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 17:20:29 +0100 Subject: [PATCH 130/490] Remove trailing whitespace in description Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index f1e085d4e..23d9c7e55 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -79,7 +79,9 @@ class BaseTarget(metaclass=ABCMeta): Returns: Description for the test case. """ - return "{} #{} {}".format(self.test_name, self.count, self.case_description) + return "{} #{} {}".format( + self.test_name, self.count, self.case_description + ).strip() def create_test_case(self) -> test_case.TestCase: From dd5cac5a46121a5edf4223bd71d3e017e99232c9 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 24 Aug 2022 18:09:10 +0100 Subject: [PATCH 131/490] Disable pylint unused arg in __new__ Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 23d9c7e55..652b9a1f8 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -53,6 +53,7 @@ class BaseTarget(metaclass=ABCMeta): test_name = "" def __new__(cls, *args, **kwargs): + # pylint: disable=unused-argument cls.count += 1 return super().__new__(cls) From 19fd8e7fac42121894757f9cf4b698867e2e0e00 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 09:56:51 +0100 Subject: [PATCH 132/490] Raise NotImplementedError in abstract methods Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 652b9a1f8..a90547349 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -67,7 +67,7 @@ class BaseTarget(metaclass=ABCMeta): Returns: List of arguments required for the test function. """ - pass + raise NotImplementedError def description(self) -> str: """Create a test description. @@ -104,7 +104,7 @@ class BaseTarget(metaclass=ABCMeta): of the class with appropriate input data, and then calling `create_test_case()` on each. """ - pass + raise NotImplementedError @classmethod def generate_tests(cls) -> Iterator[test_case.TestCase]: From 4424ae2081691dbad33f364235ac7e2bc33f6c27 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 10:02:06 +0100 Subject: [PATCH 133/490] Fix TARGET types and code style Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index a90547349..11c085f6b 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Common test generation classes and main function. These are used both by generate_psa_tests.py and generate_bignum_tests.py. @@ -150,7 +149,7 @@ class TestGenerator: # Note that targets whose names contain 'test_format' have their content # validated by `abi_check.py`. - TARGETS = {} # type: Dict[str, Callable[..., test_case.TestCase]] + TARGETS = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] def generate_target(self, name: str, *target_args) -> None: """Generate cases and write to data file for a target. From 9cfe5b7dd7e4d64c380c0fcedfabecdb8cce5d33 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 11:30:17 +0100 Subject: [PATCH 134/490] Use argparser default for directory Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 11c085f6b..4803c24b6 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -126,13 +126,7 @@ class BaseTarget(metaclass=ABCMeta): class TestGenerator: """Generate test data.""" def __init__(self, options) -> None: - self.test_suite_directory = self.get_option(options, 'directory', - 'tests/suites') - - @staticmethod - def get_option(options, name: str, default: T) -> T: - value = getattr(options, name, None) - return default if value is None else value + self.test_suite_directory = getattr(options, 'directory') def filename_for(self, basename: str) -> str: """The location of the data file with the specified base name.""" @@ -167,7 +161,7 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): help='List available targets and exit') parser.add_argument('--list-for-cmake', action='store_true', help='Print \';\'-separated list of available targets and exit') - parser.add_argument('--directory', metavar='DIR', + parser.add_argument('--directory', default="tests/suites", metavar='DIR', help='Output directory (default: tests/suites)') parser.add_argument('targets', nargs='*', metavar='TARGET', help='Target file to generate (default: all; "-": none)') From 68e44c4f6ccc8e5b8f881b09b32e4b0de3354962 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 12:29:46 +0100 Subject: [PATCH 135/490] Clarify documentation Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 26 +++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 4803c24b6..9e004a69f 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -35,6 +35,8 @@ T = TypeVar('T') #pylint: disable=invalid-name class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. + This should be derived from for file Targets. + Attributes: count: Counter for test cases from this class. case_description: Short description of the test case. This may be @@ -43,7 +45,8 @@ class BaseTarget(metaclass=ABCMeta): should be specified in a child class of BaseTarget. test_function: Test function which the class generates cases for. test_name: A common name or description of the test function. This can - be the function's name, or a short summary of its purpose. + be `test_function`, a clearer equivalent, or a short summary of the + test function's purpose. """ count = 0 case_description = "" @@ -61,7 +64,7 @@ class BaseTarget(metaclass=ABCMeta): """Get the list of arguments for the test case. Override this method to provide the list of arguments required for - generating the test_function. + the `test_function`. Returns: List of arguments required for the test function. @@ -69,12 +72,12 @@ class BaseTarget(metaclass=ABCMeta): raise NotImplementedError def description(self) -> str: - """Create a test description. + """Create a test case description. Creates a description of the test case, including a name for the test - function, and describing the specific test case. This should inform a - reader of the purpose of the case. The case description may be - generated in the class, or provided manually as needed. + function, a case number, and a description the specific test case. + This should inform a reader what is being tested, and provide context + for the test case. Returns: Description for the test case. @@ -85,7 +88,7 @@ class BaseTarget(metaclass=ABCMeta): def create_test_case(self) -> test_case.TestCase: - """Generate TestCase from the current object.""" + """Generate TestCase from the instance.""" tc = test_case.TestCase() tc.set_description(self.description()) tc.set_function(self.test_function) @@ -96,7 +99,7 @@ class BaseTarget(metaclass=ABCMeta): @classmethod @abstractmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - """Generate test cases for the test function. + """Generate test cases for the class test function. This will be called in classes where `test_function` is set. Implementations should yield TestCase objects, by creating instances @@ -111,11 +114,10 @@ class BaseTarget(metaclass=ABCMeta): In classes with `test_function` set, `generate_function_tests()` is used to generate test cases first. - In all classes, this method will iterate over its subclasses, and - yield from `generate_tests()` in each. - Calling this method on a class X will yield test cases from all classes - derived from X. + In all classes, this method will iterate over its subclasses, and + yield from `generate_tests()` in each. Calling this method on a class X + will yield test cases from all classes derived from X. """ if cls.test_function: yield from cls.generate_function_tests() From b15b56191ddd224b36b9683b6f49018dd8f2cb75 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 12:49:41 +0100 Subject: [PATCH 136/490] Use argparser default for targets Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 9e004a69f..b22d58f99 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -166,6 +166,7 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): parser.add_argument('--directory', default="tests/suites", metavar='DIR', help='Output directory (default: tests/suites)') parser.add_argument('targets', nargs='*', metavar='TARGET', + default=sorted(generator_class.TARGETS), help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) build_tree.chdir_to_root() @@ -179,14 +180,11 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): print(';'.join(generator.filename_for(name) for name in sorted(generator.TARGETS)), end='') return - if options.targets: - # Allow "-" as a special case so you can run - # ``generate_xxx_tests.py - $targets`` and it works uniformly whether - # ``$targets`` is empty or not. - options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] - else: - options.targets = sorted(generator.TARGETS) + # Allow "-" as a special case so you can run + # ``generate_xxx_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] for target in options.targets: generator.generate_target(target) From 396e8cd4c3bd5b628b3c684c24fd555e27089315 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 13:21:45 +0100 Subject: [PATCH 137/490] Fix trailing whitespace Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index b22d58f99..2981a7470 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -166,7 +166,7 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): parser.add_argument('--directory', default="tests/suites", metavar='DIR', help='Output directory (default: tests/suites)') parser.add_argument('targets', nargs='*', metavar='TARGET', - default=sorted(generator_class.TARGETS), + default=sorted(generator_class.TARGETS), help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) build_tree.chdir_to_root() From b52a2a34c0fff9c4d09f38703b8d5d6a7141da8c Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 25 Aug 2022 16:27:05 +0100 Subject: [PATCH 138/490] Modify wording in docstrings Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 2981a7470..e833008b5 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -35,7 +35,8 @@ T = TypeVar('T') #pylint: disable=invalid-name class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. - This should be derived from for file Targets. + Derive directly from this class when adding new file Targets, setting + `target_basename`. Attributes: count: Counter for test cases from this class. @@ -113,7 +114,7 @@ class BaseTarget(metaclass=ABCMeta): """Generate test cases for the class and its subclasses. In classes with `test_function` set, `generate_function_tests()` is - used to generate test cases first. + called to generate test cases first. In all classes, this method will iterate over its subclasses, and yield from `generate_tests()` in each. Calling this method on a class X From 237b40f49f72175af15f4014f4ac4a1de1c7270c Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 31 Aug 2022 17:01:38 +0100 Subject: [PATCH 139/490] Add dependencies attribute to BaseTarget Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index e833008b5..9eaa7e28f 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -42,6 +42,7 @@ class BaseTarget(metaclass=ABCMeta): count: Counter for test cases from this class. case_description: Short description of the test case. This may be automatically generated using the class, or manually set. + dependencies: A list of dependencies required for the test case. target_basename: Basename of file to write generated tests to. This should be specified in a child class of BaseTarget. test_function: Test function which the class generates cases for. @@ -51,6 +52,7 @@ class BaseTarget(metaclass=ABCMeta): """ count = 0 case_description = "" + dependencies: List[str] = [] target_basename = "" test_function = "" test_name = "" @@ -94,6 +96,7 @@ class BaseTarget(metaclass=ABCMeta): tc.set_description(self.description()) tc.set_function(self.test_function) tc.set_arguments(self.arguments()) + tc.set_dependencies(self.dependencies) return tc From aa79a969990419c292d8e096cc0db8e075175e8a Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 31 Aug 2022 17:16:44 +0100 Subject: [PATCH 140/490] Use Python 3.5 style typing for dependencies Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 9eaa7e28f..1adf8e24f 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -52,7 +52,7 @@ class BaseTarget(metaclass=ABCMeta): """ count = 0 case_description = "" - dependencies: List[str] = [] + dependencies = [] # type: List[str] target_basename = "" test_function = "" test_name = "" From 682eea4a142a0aa6ca4e7add96e64b4d301e5e55 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 2 Sep 2022 11:56:34 +0100 Subject: [PATCH 141/490] Rework TestGenerator to add file targets BaseTarget-derived targets are now added to TestGenerator.targets in initialization. This reduces repeated code in generate_xxx_tests.py scripts which use this framework. Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 29 +++++++++++++++--------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 1adf8e24f..57eb2be1f 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -133,6 +133,11 @@ class TestGenerator: """Generate test data.""" def __init__(self, options) -> None: self.test_suite_directory = getattr(options, 'directory') + # Add file Targets which have been declared in other modules + self.targets.update({ + subclass.target_basename: subclass.generate_tests + for subclass in BaseTarget.__subclasses__() + }) def filename_for(self, basename: str) -> str: """The location of the data file with the specified base name.""" @@ -149,7 +154,7 @@ class TestGenerator: # Note that targets whose names contain 'test_format' have their content # validated by `abi_check.py`. - TARGETS = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] + targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] def generate_target(self, name: str, *target_args) -> None: """Generate cases and write to data file for a target. @@ -157,7 +162,7 @@ class TestGenerator: For target callables which require arguments, override this function and pass these arguments using super() (see PSATestGenerator). """ - test_cases = self.TARGETS[name](*target_args) + test_cases = self.targets[name](*target_args) self.write_test_data_file(name, test_cases) def main(args, generator_class: Type[TestGenerator] = TestGenerator): @@ -170,25 +175,27 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): parser.add_argument('--directory', default="tests/suites", metavar='DIR', help='Output directory (default: tests/suites)') parser.add_argument('targets', nargs='*', metavar='TARGET', - default=sorted(generator_class.TARGETS), help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) build_tree.chdir_to_root() generator = generator_class(options) if options.list: - for name in sorted(generator.TARGETS): + for name in sorted(generator.targets): print(generator.filename_for(name)) return # List in a cmake list format (i.e. ';'-separated) if options.list_for_cmake: print(';'.join(generator.filename_for(name) - for name in sorted(generator.TARGETS)), end='') + for name in sorted(generator.targets)), end='') return - # Allow "-" as a special case so you can run - # ``generate_xxx_tests.py - $targets`` and it works uniformly whether - # ``$targets`` is empty or not. - options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] + if options.targets: + # Allow "-" as a special case so you can run + # ``generate_xxx_tests.py - $targets`` and it works uniformly whether + # ``$targets`` is empty or not. + options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) + for target in options.targets + if target != '-'] + else: + options.targets = sorted(generator.targets) for target in options.targets: generator.generate_target(target) From 8d458e36b9aa75a24afb6e22695740f03e52c973 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 2 Sep 2022 12:57:37 +0100 Subject: [PATCH 142/490] Remove unused imports Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 57eb2be1f..682f7b036 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -193,8 +193,8 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): # ``generate_xxx_tests.py - $targets`` and it works uniformly whether # ``$targets`` is empty or not. options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) - for target in options.targets - if target != '-'] + for target in options.targets + if target != '-'] else: options.targets = sorted(generator.targets) for target in options.targets: From 6bcfb9052e0aad7e17bf263b525ef129f0f56871 Mon Sep 17 00:00:00 2001 From: Andrzej Kurek Date: Fri, 29 Jul 2022 10:00:16 -0400 Subject: [PATCH 143/490] Add an EC J-PAKE KDF to transform K -> SHA256(K.X) for TLS 1.2 TLS uses it to derive the session secret. The algorithm takes a serialized point in an uncompressed form, extracts the X coordinate and computes SHA256 of it. It is only expected to work with P-256. Fixes #5978. Signed-off-by: Andrzej Kurek --- scripts/framework_dev/crypto_knowledge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 592fc0afe..f52ca9ac8 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -357,6 +357,7 @@ class Algorithm: 'HKDF': AlgorithmCategory.KEY_DERIVATION, 'TLS12_PRF': AlgorithmCategory.KEY_DERIVATION, 'TLS12_PSK_TO_MS': AlgorithmCategory.KEY_DERIVATION, + 'TLS12_ECJPAKE_TO_PMS': AlgorithmCategory.KEY_DERIVATION, 'PBKDF': AlgorithmCategory.KEY_DERIVATION, 'ECDH': AlgorithmCategory.KEY_AGREEMENT, 'FFDH': AlgorithmCategory.KEY_AGREEMENT, From 544b2c77f848cd1c145f2700bd004d7259c86e78 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 12:59:32 +0100 Subject: [PATCH 144/490] Update comments/docstrings in TestGenerator Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 682f7b036..f1b268229 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -130,10 +130,12 @@ class BaseTarget(metaclass=ABCMeta): class TestGenerator: - """Generate test data.""" + """Generate test cases and write to data files.""" def __init__(self, options) -> None: self.test_suite_directory = getattr(options, 'directory') - # Add file Targets which have been declared in other modules + # Update `targets` with an entry for each child class of BaseTarget. + # Each entry represents a file generated by the BaseTarget framework, + # and enables generating the .data files using the CLI. self.targets.update({ subclass.target_basename: subclass.generate_tests for subclass in BaseTarget.__subclasses__() From d862e5bf829a4d2590e348407a9df853692bb065 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 13:02:40 +0100 Subject: [PATCH 145/490] Add toggle for test case count in descriptions Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index f1b268229..81af7ba27 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -43,6 +43,7 @@ class BaseTarget(metaclass=ABCMeta): case_description: Short description of the test case. This may be automatically generated using the class, or manually set. dependencies: A list of dependencies required for the test case. + show_test_count: Toggle for inclusion of `count` in the test description. target_basename: Basename of file to write generated tests to. This should be specified in a child class of BaseTarget. test_function: Test function which the class generates cases for. @@ -53,6 +54,7 @@ class BaseTarget(metaclass=ABCMeta): count = 0 case_description = "" dependencies = [] # type: List[str] + show_test_count = True target_basename = "" test_function = "" test_name = "" @@ -78,16 +80,19 @@ class BaseTarget(metaclass=ABCMeta): """Create a test case description. Creates a description of the test case, including a name for the test - function, a case number, and a description the specific test case. - This should inform a reader what is being tested, and provide context - for the test case. + function, an optional case count, and a description of the specific + test case. This should inform a reader what is being tested, and + provide context for the test case. Returns: Description for the test case. """ - return "{} #{} {}".format( - self.test_name, self.count, self.case_description - ).strip() + if self.show_test_count: + return "{} #{} {}".format( + self.test_name, self.count, self.case_description + ).strip() + else: + return "{} {}".format(self.test_name, self.case_description).strip() def create_test_case(self) -> test_case.TestCase: From 1ccf89c573c1c4eb882c0604c3ef1d818b231f0e Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 13:39:20 +0100 Subject: [PATCH 146/490] Remove argparser default for directory This reverts commit 9cfe5b7dd7e4d64c380c0fcedfabecdb8cce5d33. Adds a comment to explain reasoning for current implementation. Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 81af7ba27..87cb43ecd 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -137,7 +137,7 @@ class BaseTarget(metaclass=ABCMeta): class TestGenerator: """Generate test cases and write to data files.""" def __init__(self, options) -> None: - self.test_suite_directory = getattr(options, 'directory') + self.test_suite_directory = getattr(options, 'directory', 'tests/suites') # Update `targets` with an entry for each child class of BaseTarget. # Each entry represents a file generated by the BaseTarget framework, # and enables generating the .data files using the CLI. @@ -179,8 +179,12 @@ def main(args, generator_class: Type[TestGenerator] = TestGenerator): help='List available targets and exit') parser.add_argument('--list-for-cmake', action='store_true', help='Print \';\'-separated list of available targets and exit') - parser.add_argument('--directory', default="tests/suites", metavar='DIR', + parser.add_argument('--directory', metavar='DIR', help='Output directory (default: tests/suites)') + # The `--directory` option is interpreted relative to the directory from + # which the script is invoked, but the default is relative to the root of + # the mbedtls tree. The default should not be set above, but instead after + # `build_tree.chdir_to_root()` is called. parser.add_argument('targets', nargs='*', metavar='TARGET', help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) From 9e4b9e65fabf4a75afb8a76fe6b42a92a1bf6eb2 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 14 Sep 2022 16:26:54 +0100 Subject: [PATCH 147/490] Update references to file targets in docstrings Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index 87cb43ecd..c9a73c4ad 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -35,8 +35,9 @@ T = TypeVar('T') #pylint: disable=invalid-name class BaseTarget(metaclass=ABCMeta): """Base target for test case generation. - Derive directly from this class when adding new file Targets, setting - `target_basename`. + Child classes of this class represent an output file, and can be referred + to as file targets. These indicate where test cases will be written to for + all subclasses of the file target, which is set by `target_basename`. Attributes: count: Counter for test cases from this class. From b648877a7ab2027748eb2267c94e352ff614a357 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Thu, 15 Sep 2022 09:02:07 +0100 Subject: [PATCH 148/490] Fix setting for default test suite directory Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index c9a73c4ad..a82f79e67 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -138,7 +138,8 @@ class BaseTarget(metaclass=ABCMeta): class TestGenerator: """Generate test cases and write to data files.""" def __init__(self, options) -> None: - self.test_suite_directory = getattr(options, 'directory', 'tests/suites') + self.test_suite_directory = self.get_option(options, 'directory', + 'tests/suites') # Update `targets` with an entry for each child class of BaseTarget. # Each entry represents a file generated by the BaseTarget framework, # and enables generating the .data files using the CLI. @@ -147,6 +148,11 @@ class TestGenerator: for subclass in BaseTarget.__subclasses__() }) + @staticmethod + def get_option(options, name: str, default: T) -> T: + value = getattr(options, name, None) + return default if value is None else value + def filename_for(self, basename: str) -> str: """The location of the data file with the specified base name.""" return posixpath.join(self.test_suite_directory, basename + '.data') From 644c1bac32d1ed47f67770a180ce5c2d47040c53 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 16 Sep 2022 17:03:54 +0100 Subject: [PATCH 149/490] Use a script specific description in CLI help Previous changes used the docstring of the test_generation module, which does not inform a user about the script. Signed-off-by: Werner Lewis --- scripts/framework_dev/test_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_generation.py index a82f79e67..a88425f46 100644 --- a/scripts/framework_dev/test_generation.py +++ b/scripts/framework_dev/test_generation.py @@ -179,9 +179,9 @@ class TestGenerator: test_cases = self.targets[name](*target_args) self.write_test_data_file(name, test_cases) -def main(args, generator_class: Type[TestGenerator] = TestGenerator): +def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator): """Command line entry point.""" - parser = argparse.ArgumentParser(description=__doc__) + parser = argparse.ArgumentParser(description=description) parser.add_argument('--list', action='store_true', help='List available targets and exit') parser.add_argument('--list-for-cmake', action='store_true', From 2e1174db8532c4c875e5afc5238ff267de3ae9a9 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 16 Sep 2022 21:41:47 +0200 Subject: [PATCH 150/490] More precise name for test data generation We have Python code both for test code generation (tests/scripts/generate_test_code.py) and now for test data generation. Avoid the ambiguous expression "test generation". This commit renames the Python module and adjusts all references to it. A subsequent commit will adjust the documentation. Signed-off-by: Gilles Peskine --- .../framework_dev/{test_generation.py => test_data_generation.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/framework_dev/{test_generation.py => test_data_generation.py} (100%) diff --git a/scripts/framework_dev/test_generation.py b/scripts/framework_dev/test_data_generation.py similarity index 100% rename from scripts/framework_dev/test_generation.py rename to scripts/framework_dev/test_data_generation.py From ab9786adc1cebf1e089d695100936b7fc3dbb992 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 16 Sep 2022 22:02:37 +0200 Subject: [PATCH 151/490] Clarify the descriptions of test-case-data-related modules Signed-off-by: Gilles Peskine --- scripts/framework_dev/test_case.py | 2 +- scripts/framework_dev/test_data_generation.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index 6a46e4209..d8f7b6004 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -1,4 +1,4 @@ -"""Library for generating Mbed TLS test data. +"""Library for constructing an Mbed TLS test case. """ # Copyright The Mbed TLS Contributors diff --git a/scripts/framework_dev/test_data_generation.py b/scripts/framework_dev/test_data_generation.py index a88425f46..f8e86ed8c 100644 --- a/scripts/framework_dev/test_data_generation.py +++ b/scripts/framework_dev/test_data_generation.py @@ -1,4 +1,7 @@ -"""Common test generation classes and main function. +"""Common code for test data generation. + +This module defines classes that are of general use to automatically +generate .data files for unit tests, as well as a main function. These are used both by generate_psa_tests.py and generate_bignum_tests.py. """ From 871ee08fcdb00c2d45a48bcaefafd0087811a004 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 16 Sep 2022 22:22:53 +0200 Subject: [PATCH 152/490] generate_*_tests.py --directory: fix handling of relative path The option to --directory was intended to be relative to the current directory when the script is invoked, which is the intuitive behavior. But this was not implemented correctly, and it was actually interpreted relative to the mbedtls root (which the script chdir's into). Fix this. Signed-off-by: Gilles Peskine --- scripts/framework_dev/test_data_generation.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scripts/framework_dev/test_data_generation.py b/scripts/framework_dev/test_data_generation.py index f8e86ed8c..36c947505 100644 --- a/scripts/framework_dev/test_data_generation.py +++ b/scripts/framework_dev/test_data_generation.py @@ -141,8 +141,7 @@ class BaseTarget(metaclass=ABCMeta): class TestGenerator: """Generate test cases and write to data files.""" def __init__(self, options) -> None: - self.test_suite_directory = self.get_option(options, 'directory', - 'tests/suites') + self.test_suite_directory = options.directory # Update `targets` with an entry for each child class of BaseTarget. # Each entry represents a file generated by the BaseTarget framework, # and enables generating the .data files using the CLI. @@ -151,11 +150,6 @@ class TestGenerator: for subclass in BaseTarget.__subclasses__() }) - @staticmethod - def get_option(options, name: str, default: T) -> T: - value = getattr(options, name, None) - return default if value is None else value - def filename_for(self, basename: str) -> str: """The location of the data file with the specified base name.""" return posixpath.join(self.test_suite_directory, basename + '.data') @@ -189,16 +183,24 @@ def main(args, description: str, generator_class: Type[TestGenerator] = TestGene help='List available targets and exit') parser.add_argument('--list-for-cmake', action='store_true', help='Print \';\'-separated list of available targets and exit') + # If specified explicitly, this option may be a path relative to the + # current directory when the script is invoked. The default value + # is relative to the mbedtls root, which we don't know yet. So we + # can't set a string as the default value here. parser.add_argument('--directory', metavar='DIR', help='Output directory (default: tests/suites)') - # The `--directory` option is interpreted relative to the directory from - # which the script is invoked, but the default is relative to the root of - # the mbedtls tree. The default should not be set above, but instead after - # `build_tree.chdir_to_root()` is called. parser.add_argument('targets', nargs='*', metavar='TARGET', help='Target file to generate (default: all; "-": none)') options = parser.parse_args(args) + + # Change to the mbedtls root, to keep things simple. But first, adjust + # command line options that might be relative paths. + if options.directory is None: + options.directory = 'tests/suites' + else: + options.directory = os.path.abspath(options.directory) build_tree.chdir_to_root() + generator = generator_class(options) if options.list: for name in sorted(generator.targets): From 6ba3691145e5fc449870ffa79aac3a4e32f9c1a2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 16 Sep 2022 22:35:18 +0200 Subject: [PATCH 153/490] Use relative imports when importing other modules in the same directory We were using absolute imports under the assumption that the /scripts directory is in the path. This worked in normal use because every one of our Python scripts either were in the /scripts directory, or added the /scripts directory to the module search path in order to reference mbedtls_dev. However, this broke things like ``` python3 -m unittest scripts/mbedtls_dev/psa_storage.py ``` Fix this by using relative imports. Relative imports are only supposed to be used inside a package (Python doesn't complain, but Pylint does). So make /scripts/mbedtls_dev a proper package by creating __init__.py. Signed-off-by: Gilles Peskine --- scripts/framework_dev/__init__.py | 3 +++ scripts/framework_dev/crypto_knowledge.py | 2 +- scripts/framework_dev/psa_storage.py | 2 +- scripts/framework_dev/test_case.py | 2 +- scripts/framework_dev/test_data_generation.py | 4 ++-- 5 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 scripts/framework_dev/__init__.py diff --git a/scripts/framework_dev/__init__.py b/scripts/framework_dev/__init__.py new file mode 100644 index 000000000..c5bddaf74 --- /dev/null +++ b/scripts/framework_dev/__init__.py @@ -0,0 +1,3 @@ +# This file needs to exist to make mbedtls_dev a package. +# Among other things, this allows modules in this directory to make +# relative impotrs. diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 592fc0afe..1ce549903 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -22,7 +22,7 @@ import enum import re from typing import FrozenSet, Iterable, List, Optional, Tuple -from mbedtls_dev.asymmetric_key_data import ASYMMETRIC_KEY_DATA +from .asymmetric_key_data import ASYMMETRIC_KEY_DATA def short_expression(original: str, level: int = 0) -> str: diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index a06dce13b..bae99383d 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -26,7 +26,7 @@ import struct from typing import Dict, List, Optional, Set, Union import unittest -from mbedtls_dev import c_build_helper +from . import c_build_helper class Expr: diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index d8f7b6004..43ddf203b 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -21,7 +21,7 @@ import os import sys from typing import Iterable, List, Optional -from mbedtls_dev import typing_util +from . import typing_util def hex_string(data: bytes) -> str: return '"' + binascii.hexlify(data).decode('ascii') + '"' diff --git a/scripts/framework_dev/test_data_generation.py b/scripts/framework_dev/test_data_generation.py index 36c947505..cdb1c03b8 100644 --- a/scripts/framework_dev/test_data_generation.py +++ b/scripts/framework_dev/test_data_generation.py @@ -29,8 +29,8 @@ import re from abc import ABCMeta, abstractmethod from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar -from mbedtls_dev import build_tree -from mbedtls_dev import test_case +from . import build_tree +from . import test_case T = TypeVar('T') #pylint: disable=invalid-name From 495a0d791c78f3e289215cd53cc5c252e8e6b462 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 18 Sep 2022 21:17:09 +0200 Subject: [PATCH 154/490] Unify check_repo_path We had 4 identical copies of the check_repo_path function. Replace them by a single copy in the build_tree module where it naturally belongs. Signed-off-by: Gilles Peskine --- scripts/framework_dev/build_tree.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index 3920d0ed6..f52b785d9 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -25,6 +25,13 @@ def looks_like_mbedtls_root(path: str) -> bool: return all(os.path.isdir(os.path.join(path, subdir)) for subdir in ['include', 'library', 'programs', 'tests']) +def check_repo_path(): + """ + Check that the current working directory is the project root, and throw + an exception if not. + """ + if not all(os.path.isdir(d) for d in ["include", "library", "tests"]): + raise Exception("This script must be run from Mbed TLS root") def chdir_to_root() -> None: """Detect the root of the Mbed TLS source tree and change to it. From 65bfc98c5c8eb4a170f004583d3fcf1d5072ad87 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Sep 2022 22:00:06 +0200 Subject: [PATCH 155/490] Replace the output file atomically When writing the new .data file, first write the new content, then replace the target. This way, there isn't a temporary state in which the file is partially written. This temporary state can be misleading if the build is interrupted. It's annoying if you're watching changes to the output and the changes appear as emptying the file following by the new version appearing. Now interrupted builds don't leave a file that appears to be up to date but isn't, and when watching the output, there's a single transition to the new version. Signed-off-by: Gilles Peskine --- scripts/framework_dev/test_case.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index 43ddf203b..8f0870367 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -92,9 +92,11 @@ def write_data_file(filename: str, """ if caller is None: caller = os.path.basename(sys.argv[0]) - with open(filename, 'w') as out: + tempfile = filename + '.new' + with open(tempfile, 'w') as out: out.write('# Automatically generated by {}. Do not edit!\n' .format(caller)) for tc in test_cases: tc.write(out) out.write('\n# End of automatically generated file.\n') + os.replace(tempfile, filename) From e3fb2c7a3ccefa2bf04ee33914da26116b3690e0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 29 Sep 2022 18:48:41 +0200 Subject: [PATCH 156/490] Documentation typo Signed-off-by: Gilles Peskine --- scripts/framework_dev/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/__init__.py b/scripts/framework_dev/__init__.py index c5bddaf74..15b0d60dd 100644 --- a/scripts/framework_dev/__init__.py +++ b/scripts/framework_dev/__init__.py @@ -1,3 +1,3 @@ # This file needs to exist to make mbedtls_dev a package. # Among other things, this allows modules in this directory to make -# relative impotrs. +# relative imports. From 042868aecd1ef32f2e64691682859a00041baaf8 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Sep 2022 22:00:06 +0200 Subject: [PATCH 157/490] Replace the output file atomically When writing the new .data file, first write the new content, then replace the target. This way, there isn't a temporary state in which the file is partially written. This temporary state can be misleading if the build is interrupted. It's annoying if you're watching changes to the output and the changes appear as emptying the file following by the new version appearing. Now interrupted builds don't leave a file that appears to be up to date but isn't, and when watching the output, there's a single transition to the new version. Signed-off-by: Gilles Peskine --- scripts/framework_dev/test_case.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index 6a46e4209..d0afa592f 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -92,9 +92,11 @@ def write_data_file(filename: str, """ if caller is None: caller = os.path.basename(sys.argv[0]) - with open(filename, 'w') as out: + tempfile = filename + '.new' + with open(tempfile, 'w') as out: out.write('# Automatically generated by {}. Do not edit!\n' .format(caller)) for tc in test_cases: tc.write(out) out.write('\n# End of automatically generated file.\n') + os.replace(tempfile, filename) From 3fcb428a9f817e7282f06faeee9e8500e2a79575 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 30 Sep 2022 16:28:43 +0100 Subject: [PATCH 158/490] Add module for bignum_core test generation Separate file is added for classes used to generate cases for tests in bignum_core.function. Common elements of the BignumOperation class are added to classes in a new common file, for use across files. Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_common.py | 111 ++++++++++++++++++ scripts/framework_dev/bignum_core.py | 72 ++++++++++++ scripts/framework_dev/test_data_generation.py | 1 + 3 files changed, 184 insertions(+) create mode 100644 scripts/framework_dev/bignum_common.py create mode 100644 scripts/framework_dev/bignum_core.py diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py new file mode 100644 index 000000000..b20fe815d --- /dev/null +++ b/scripts/framework_dev/bignum_common.py @@ -0,0 +1,111 @@ +"""Common features for bignum in test generation framework.""" +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import typing + +from abc import abstractmethod +from typing import Iterator, List, Tuple, TypeVar + +T = TypeVar('T') #pylint: disable=invalid-name + +def hex_to_int(val: str) -> int: + return int(val, 16) if val else 0 + +def quote_str(val) -> str: + return "\"{}\"".format(val) + +def bound_mpi8(val: int) -> int: + """First number exceeding 8-byte limbs needed for given input value.""" + return bound_mpi8_limbs(limbs_mpi8(val)) + +def bound_mpi4(val: int) -> int: + """First number exceeding 4-byte limbs needed for given input value.""" + return bound_mpi4_limbs(limbs_mpi4(val)) + +def bound_mpi8_limbs(limbs: int) -> int: + """First number exceeding maximum of given 8-byte limbs.""" + bits = 64 * limbs + return 1 << bits + +def bound_mpi4_limbs(limbs: int) -> int: + """First number exceeding maximum of given 4-byte limbs.""" + bits = 32 * limbs + return 1 << bits + +def limbs_mpi8(val: int) -> int: + """Return the number of 8-byte limbs required to store value.""" + return (val.bit_length() + 63) // 64 + +def limbs_mpi4(val: int) -> int: + """Return the number of 4-byte limbs required to store value.""" + return (val.bit_length() + 31) // 32 + +def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: + """Return all pair combinations from input values. + + The return value is cast, as older versions of mypy are unable to derive + the specific type returned by itertools.combinations_with_replacement. + """ + return typing.cast( + List[Tuple[T, T]], + list(itertools.combinations_with_replacement(values, 2)) + ) + + +class OperationCommon: + """Common features for bignum binary operations. + + This adds functionality common in binary operation tests. + + Attributes: + symbol: Symbol to use for the operation in case description. + input_values: List of values to use as test case inputs. These are + combined to produce pairs of values. + input_cases: List of tuples containing pairs of test case inputs. This + can be used to implement specific pairs of inputs. + """ + symbol = "" + input_values = [] # type: List[str] + input_cases = [] # type: List[Tuple[str, str]] + + def __init__(self, val_a: str, val_b: str) -> None: + self.arg_a = val_a + self.arg_b = val_b + self.int_a = hex_to_int(val_a) + self.int_b = hex_to_int(val_b) + + def arguments(self) -> List[str]: + return [quote_str(self.arg_a), quote_str(self.arg_b), self.result()] + + @abstractmethod + def result(self) -> str: + """Get the result of the operation. + + This could be calculated during initialization and stored as `_result` + and then returned, or calculated when the method is called. + """ + raise NotImplementedError + + @classmethod + def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: + """Generator to yield pairs of inputs. + + Combinations are first generated from all input values, and then + specific cases provided. + """ + yield from combination_pairs(cls.input_values) + yield from cls.input_cases diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py new file mode 100644 index 000000000..b30f8581a --- /dev/null +++ b/scripts/framework_dev/bignum_core.py @@ -0,0 +1,72 @@ +"""Framework classes for generation of bignum core test cases.""" +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABCMeta +from typing import Iterator + +from . import test_case +from . import test_data_generation +from . import bignum_common + + +class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): + #pylint: disable=abstract-method + """Target for bignum core test case generation.""" + target_basename = 'test_suite_bignum_core.generated' + + +class BignumCoreOperation(bignum_common.OperationCommon, BignumCoreTarget, metaclass=ABCMeta): + #pylint: disable=abstract-method + """Common features for bignum core operations.""" + input_values = [ + "0", "1", "3", "f", "fe", "ff", "100", "ff00", "fffe", "ffff", "10000", + "fffffffe", "ffffffff", "100000000", "1f7f7f7f7f7f7f", + "8000000000000000", "fefefefefefefefe", "fffffffffffffffe", + "ffffffffffffffff", "10000000000000000", "1234567890abcdef0", + "fffffffffffffffffefefefefefefefe", "fffffffffffffffffffffffffffffffe", + "ffffffffffffffffffffffffffffffff", "100000000000000000000000000000000", + "1234567890abcdef01234567890abcdef0", + "fffffffffffffffffffffffffffffffffffffffffffffffffefefefefefefefe", + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "10000000000000000000000000000000000000000000000000000000000000000", + "1234567890abcdef01234567890abcdef01234567890abcdef01234567890abcdef0", + ( + "4df72d07b4b71c8dacb6cffa954f8d88254b6277099308baf003fab73227f34029" + "643b5a263f66e0d3c3fa297ef71755efd53b8fb6cb812c6bbf7bcf179298bd9947" + "c4c8b14324140a2c0f5fad7958a69050a987a6096e9f055fb38edf0c5889eca4a0" + "cfa99b45fbdeee4c696b328ddceae4723945901ec025076b12b" + ) + ] + + def description(self) -> str: + """Generate a description for the test case. + + If not set, case_description uses the form A `symbol` B, where symbol + is used to represent the operation. Descriptions of each value are + generated to provide some context to the test case. + """ + if not self.case_description: + self.case_description = "{} {} {}".format( + self.arg_a, self.symbol, self.arg_b + ) + return super().description() + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for a_value, b_value in cls.get_value_pairs(): + yield cls(a_value, b_value).create_test_case() + diff --git a/scripts/framework_dev/test_data_generation.py b/scripts/framework_dev/test_data_generation.py index cdb1c03b8..eec0f9d97 100644 --- a/scripts/framework_dev/test_data_generation.py +++ b/scripts/framework_dev/test_data_generation.py @@ -148,6 +148,7 @@ class TestGenerator: self.targets.update({ subclass.target_basename: subclass.generate_tests for subclass in BaseTarget.__subclasses__() + if subclass.target_basename }) def filename_for(self, basename: str) -> str: From 95d8233d62461862bf99938fe98cb1d035c70437 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 30 Sep 2022 16:32:19 +0100 Subject: [PATCH 159/490] Add test generation for mpi_core_add_if Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_core.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index b30f8581a..711ec69ff 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -70,3 +70,22 @@ class BignumCoreOperation(bignum_common.OperationCommon, BignumCoreTarget, metac for a_value, b_value in cls.get_value_pairs(): yield cls(a_value, b_value).create_test_case() + + +class BignumCoreAddIf(BignumCoreOperation): + """Test cases for bignum core add if.""" + count = 0 + symbol = "+" + test_function = "mpi_core_add_if" + test_name = "mbedtls_mpi_core_add_if" + + def result(self) -> str: + tmp = self.int_a + self.int_b + bound_val = max(self.int_a, self.int_b) + bound_4 = bignum_common.bound_mpi4(bound_val) + bound_8 = bignum_common.bound_mpi8(bound_val) + carry_4, remainder_4 = divmod(tmp, bound_4) + carry_8, remainder_8 = divmod(tmp, bound_8) + return "\"{:x}\":{}:\"{:x}\":{}".format( + remainder_4, carry_4, remainder_8, carry_8 + ) From 17ba308e0bdd95beb2a409494532205a0172cfe0 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Fri, 30 Sep 2022 16:33:11 +0100 Subject: [PATCH 160/490] Add test generation for mpi_core_sub Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_core.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 711ec69ff..a8d6ec216 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -89,3 +89,24 @@ class BignumCoreAddIf(BignumCoreOperation): return "\"{:x}\":{}:\"{:x}\":{}".format( remainder_4, carry_4, remainder_8, carry_8 ) + + +class BignumCoreSub(BignumCoreOperation): + """Test cases for bignum core sub.""" + count = 0 + symbol = "-" + test_function = "mpi_core_sub" + test_name = "mbedtls_mpi_core_sub" + + def result(self) -> str: + if self.int_a >= self.int_b: + result_4 = result_8 = self.int_a - self.int_b + carry = 0 + else: + bound_val = max(self.int_a, self.int_b) + bound_4 = bignum_common.bound_mpi4(bound_val) + result_4 = bound_4 + self.int_a - self.int_b + bound_8 = bignum_common.bound_mpi8(bound_val) + result_8 = bound_8 + self.int_a - self.int_b + carry = 1 + return "\"{:x}\":\"{:x}\":{}".format(result_4, result_8, carry) From 5348c5c73027bbc4de30924ca5b5813c6f70be52 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 4 Oct 2022 10:07:13 +0100 Subject: [PATCH 161/490] Add flag for unique combinations in operations Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_common.py | 13 ++++++++++++- scripts/framework_dev/bignum_core.py | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index b20fe815d..7857dd65c 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -77,10 +77,14 @@ class OperationCommon: combined to produce pairs of values. input_cases: List of tuples containing pairs of test case inputs. This can be used to implement specific pairs of inputs. + unique_combinations_only: Boolean to select if test case combinations + must be unique. If True, only A,B or B,A would be included as a test + case. If False, both A,B and B,A would be included. """ symbol = "" input_values = [] # type: List[str] input_cases = [] # type: List[Tuple[str, str]] + unique_combinations_only = True def __init__(self, val_a: str, val_b: str) -> None: self.arg_a = val_a @@ -107,5 +111,12 @@ class OperationCommon: Combinations are first generated from all input values, and then specific cases provided. """ - yield from combination_pairs(cls.input_values) + if cls.unique_combinations_only: + yield from combination_pairs(cls.input_values) + else: + yield from ( + (a, b) + for a in cls.input_values + for b in cls.input_values + ) yield from cls.input_cases diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index a8d6ec216..f2e3db74b 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -97,6 +97,7 @@ class BignumCoreSub(BignumCoreOperation): symbol = "-" test_function = "mpi_core_sub" test_name = "mbedtls_mpi_core_sub" + unique_combinations_only = False def result(self) -> str: if self.int_a >= self.int_b: From f4235bd7da25563b57dbe36293f9711490008bac Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 4 Oct 2022 10:08:26 +0100 Subject: [PATCH 162/490] Add test generation for mpi_core_mla Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_core.py | 74 +++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index f2e3db74b..47b09fd40 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -15,13 +15,12 @@ # limitations under the License. from abc import ABCMeta -from typing import Iterator +from typing import Iterator, List from . import test_case from . import test_data_generation from . import bignum_common - class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): #pylint: disable=abstract-method """Target for bignum core test case generation.""" @@ -111,3 +110,74 @@ class BignumCoreSub(BignumCoreOperation): result_8 = bound_8 + self.int_a - self.int_b carry = 1 return "\"{:x}\":\"{:x}\":{}".format(result_4, result_8, carry) + + +class BignumCoreMLA(BignumCoreOperation): + """Test cases for fixed-size multiply accumulate.""" + count = 0 + test_function = "mpi_core_mla" + test_name = "mbedtls_mpi_core_mla" + unique_combinations_only = False + + input_values = [ + "0", "1", "fffe", "ffffffff", "100000000", "20000000000000", + "ffffffffffffffff", "10000000000000000", "1234567890abcdef0", + "fffffffffffffffffefefefefefefefe", + "100000000000000000000000000000000", + "1234567890abcdef01234567890abcdef0", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "1234567890abcdef01234567890abcdef01234567890abcdef01234567890abcdef0", + ( + "4df72d07b4b71c8dacb6cffa954f8d88254b6277099308baf003fab73227f" + "34029643b5a263f66e0d3c3fa297ef71755efd53b8fb6cb812c6bbf7bcf17" + "9298bd9947c4c8b14324140a2c0f5fad7958a69050a987a6096e9f055fb38" + "edf0c5889eca4a0cfa99b45fbdeee4c696b328ddceae4723945901ec02507" + "6b12b" + ) + ] # type: List[str] + input_scalars = [ + "0", "3", "fe", "ff", "ffff", "10000", "ffffffff", "100000000", + "7f7f7f7f7f7f7f7f", "8000000000000000", "fffffffffffffffe" + ] # type: List[str] + + def __init__(self, val_a: str, val_b: str, val_s: str) -> None: + super().__init__(val_a, val_b) + self.arg_scalar = val_s + self.int_scalar = bignum_common.hex_to_int(val_s) + if bignum_common.limbs_mpi4(self.int_scalar) > 1: + self.dependencies = ["MBEDTLS_HAVE_INT64"] + + def arguments(self) -> List[str]: + return [ + bignum_common.quote_str(self.arg_a), + bignum_common.quote_str(self.arg_b), + bignum_common.quote_str(self.arg_scalar), + self.result() + ] + + def description(self) -> str: + """Override and add the additional scalar.""" + if not self.case_description: + self.case_description = "0x{} + 0x{} * 0x{}".format( + self.arg_a, self.arg_b, self.arg_scalar + ) + return super().description() + + def result(self) -> str: + result = self.int_a + (self.int_b * self.int_scalar) + bound_val = max(self.int_a, self.int_b) + bound_4 = bignum_common.bound_mpi4(bound_val) + bound_8 = bignum_common.bound_mpi8(bound_val) + carry_4, remainder_4 = divmod(result, bound_4) + carry_8, remainder_8 = divmod(result, bound_8) + return "\"{:x}\":\"{:x}\":\"{:x}\":\"{:x}\"".format( + remainder_4, carry_4, remainder_8, carry_8 + ) + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + """Override for additional scalar input.""" + for a_value, b_value in cls.get_value_pairs(): + for s_value in cls.input_scalars: + cur_op = cls(a_value, b_value, s_value) + yield cur_op.create_test_case() From 159231d77b229d8f4b3bac7460f67e5cbd048e14 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 4 Oct 2022 10:10:40 +0100 Subject: [PATCH 163/490] Add test generation for mpi_core_montmul Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_common.py | 15 + scripts/framework_dev/bignum_core.py | 430 ++++++++++++++++++++++++- 2 files changed, 444 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 7857dd65c..c340cd53a 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -22,6 +22,21 @@ from typing import Iterator, List, Tuple, TypeVar T = TypeVar('T') #pylint: disable=invalid-name +def invmod(a: int, n: int) -> int: + """Return inverse of a to modulo n. + + Equivalent to pow(a, -1, n) in Python 3.8+. Implementation is equivalent + to long_invmod() in CPython. + """ + b, c = 1, 0 + while n: + q, r = divmod(a, n) + a, b, c, n = n, c, b - q*c, r + # at this point a is the gcd of the original inputs + if a == 1: + return b + raise ValueError("Not invertible") + def hex_to_int(val: str) -> int: return int(val, 16) if val else 0 diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 47b09fd40..f04db2648 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -15,7 +15,7 @@ # limitations under the License. from abc import ABCMeta -from typing import Iterator, List +from typing import Iterator, List, Tuple from . import test_case from . import test_data_generation @@ -181,3 +181,431 @@ class BignumCoreMLA(BignumCoreOperation): for s_value in cls.input_scalars: cur_op = cls(a_value, b_value, s_value) yield cur_op.create_test_case() + + +class BignumCoreMontmul(BignumCoreTarget): + """Test cases for Montgomery multiplication.""" + count = 0 + test_function = "mpi_core_montmul" + test_name = "mbedtls_mpi_core_montmul" + + start_2_mpi4 = False + start_2_mpi8 = False + + replay_test_cases = [ + (2, 1, 1, 1, "19", "1", "1D"), (2, 1, 1, 1, "7", "1", "9"), + (2, 1, 1, 1, "4", "1", "9"), + ( + 12, 1, 6, 1, ( + "3C246D0E059A93A266288A7718419EC741661B474C58C032C5EDAF92709402" + "B07CC8C7CE0B781C641A1EA8DB2F4343" + ), "1", ( + "66A198186C18C10B2F5ED9B522752A9830B69916E535C8F047518A889A43A5" + "94B6BED27A168D31D4A52F88925AA8F5" + ) + ), ( + 8, 1, 4, 1, + "1E442976B0E63D64FCCE74B999E470CA9888165CB75BFA1F340E918CE03C6211", + "1", "B3A119602EE213CDE28581ECD892E0F592A338655DCE4CA88054B3D124D0E561" + ), ( + 22, 1, 11, 1, ( + "7CF5AC97304E0B63C65413F57249F59994B0FED1D2A8D3D83ED5FA38560FFB" + "82392870D6D08F87D711917FD7537E13B7E125BE407E74157776839B0AC9DB" + "23CBDFC696104353E4D2780B2B4968F8D8542306BCA7A2366E" + ), "1", ( + "284139EA19C139EBE09A8111926AAA39A2C2BE12ED487A809D3CB5BC558547" + "25B4CDCB5734C58F90B2F60D99CC1950CDBC8D651793E93C9C6F0EAD752500" + "A32C56C62082912B66132B2A6AA42ADA923E1AD22CEB7BA0123" + ) + ) + ] # type: List[Tuple[int, int, int, int, str, str, str]] + + random_test_cases = [ + ("2", "2", "3", ""), ("1", "2", "3", ""), ("2", "1", "3", ""), + ("6", "5", "7", ""), ("3", "4", "7", ""), ("1", "6", "7", ""), ("5", "6", "7", ""), + ("3", "4", "B", ""), ("7", "4", "B", ""), ("9", "7", "B", ""), ("2", "a", "B", ""), + ("25", "16", "29", "(0x29 is prime)"), ("8", "28", "29", ""), + ("18", "21", "29", ""), ("15", "f", "29", ""), + ("e2", "ea", "FF", ""), ("43", "72", "FF", ""), + ("d8", "70", "FF", ""), ("3c", "7c", "FF", ""), + ("99", "b9", "101", "(0x101 is prime)"), ("65", "b2", "101", ""), + ("81", "32", "101", ""), ("51", "dd", "101", ""), + ("d5", "143", "38B", "(0x38B is prime)"), ("3d", "387", "38B", ""), + ("160", "2e5", "38B", ""), ("10f", "137", "38B", ""), + ("7dac", "25a", "8003", "(0x8003 is prime)"), ("6f1c", "3286", "8003", ""), + ("59ed", "2f3f", "8003", ""), ("6893", "736d", "8003", ""), + ("d199", "2832", "10001", "(0x10001 is prime)"), ("c3b2", "3e5b", "10001", ""), + ("abe4", "214e", "10001", ""), ("4360", "a05d", "10001", ""), + ("3f5a1", "165b2", "7F7F7", ""), ("3bd29", "37863", "7F7F7", ""), + ("60c47", "64819", "7F7F7", ""), ("16584", "12c49", "7F7F7", ""), + ("1ff03f", "610347", "800009", "(0x800009 is prime)"), ("340fd5", "19812e", "800009", ""), + ("3fe2e8", "4d0dc7", "800009", ""), ("40356", "e6392", "800009", ""), + ("dd8a1d", "266c0e", "100002B", "(0x100002B is prime)"), + ("3fa1cb", "847fd6", "100002B", ""), ("5f439d", "5c3196", "100002B", ""), + ("18d645", "f72dc6", "100002B", ""), + ("20051ad", "37def6e", "37EEE9D", "(0x37EEE9D is prime)"), + ("2ec140b", "3580dbf", "37EEE9D", ""), ("1d91b46", "190d4fc", "37EEE9D", ""), + ("34e488d", "1224d24", "37EEE9D", ""), + ("2a4fe2cb", "263466a9", "8000000B", "(0x8000000B is prime)"), + ("5643fe94", "29a1aefa", "8000000B", ""), ("29633513", "7b007ac4", "8000000B", ""), + ("2439cef5", "5c9d5a47", "8000000B", ""), + ("4de3cfaa", "50dea178", "8CD626B9", "(0x8CD626B9 is prime)"), + ("b8b8563", "10dbbbac", "8CD626B9", ""), ("4e8a6151", "5574ec19", "8CD626B9", ""), + ("69224878", "309cfc23", "8CD626B9", ""), + ("fb6f7fb6", "afb05423", "10000000F", "(0x10000000F is prime)"), + ("8391a243", "26034dcd", "10000000F", ""), ("d26b98c", "14b2d6aa", "10000000F", ""), + ("6b9f1371", "a21daf1d", "10000000F", ""), + ( + "9f49435ad", "c8264ade8", "174876E7E9", + "0x174876E7E9 is prime (dec) 99999999977" + ), + ("c402da434", "1fb427acf", "174876E7E9", ""), + ("f6ebc2bb1", "1096d39f2a", "174876E7E9", ""), + ("153b7f7b6b", "878fda8ff", "174876E7E9", ""), + ("2c1adbb8d6", "4384d2d3c6", "8000000017", "(0x8000000017 is prime)"), + ("2e4f9cf5fb", "794f3443d9", "8000000017", ""), + ("149e495582", "3802b8f7b7", "8000000017", ""), + ("7b9d49df82", "69c68a442a", "8000000017", ""), + ("683a134600", "6dd80ea9f6", "864CB9076D", "(0x864CB9076D is prime)"), + ("13a870ff0d", "59b099694a", "864CB9076D", ""), + ("37d06b0e63", "4d2147e46f", "864CB9076D", ""), + ("661714f8f4", "22e55df507", "864CB9076D", ""), + ("2f0a96363", "52693307b4", "F7F7F7F7F7", ""), + ("3c85078e64", "f2275ecb6d", "F7F7F7F7F7", ""), + ("352dae68d1", "707775b4c6", "F7F7F7F7F7", ""), + ("37ae0f3e0b", "912113040f", "F7F7F7F7F7", ""), + ("6dada15e31", "f58ed9eff7", "1000000000F", "(0x1000000000F is prime)"), + ("69627a7c89", "cfb5ebd13d", "1000000000F", ""), + ("a5e1ad239b", "afc030c731", "1000000000F", ""), + ("f1cc45f4c5", "c64ad607c8", "1000000000F", ""), + ("2ebad87d2e31", "4c72d90bca78", "800000000005", "(0x800000000005 is prime)"), + ("a30b3cc50d", "29ac4fe59490", "800000000005", ""), + ("33674e9647b4", "5ec7ee7e72d3", "800000000005", ""), + ("3d956f474f61", "74070040257d", "800000000005", ""), + ("48348e3717d6", "43fcb4399571", "800795D9BA47", "(0x800795D9BA47 is prime)"), + ("5234c03cc99b", "2f3cccb87803", "800795D9BA47", ""), + ("3ed13db194ab", "44b8f4ba7030", "800795D9BA47", ""), + ("1c11e843bfdb", "95bd1b47b08", "800795D9BA47", ""), + ("a81d11cb81fd", "1e5753a3f33d", "1000000000015", "(0x1000000000015 is prime)"), + ("688c4db99232", "36fc0cf7ed", "1000000000015", ""), + ("f0720cc07e07", "fc76140ed903", "1000000000015", ""), + ("2ec61f8d17d1", "d270c85e36d2", "1000000000015", ""), + ( + "6a24cd3ab63820", "ed4aad55e5e348", "100000000000051", + "(0x100000000000051 is prime)" + ), + ("e680c160d3b248", "31e0d8840ed510", "100000000000051", ""), + ("a80637e9aebc38", "bb81decc4e1738", "100000000000051", ""), + ("9afa5a59e9d630", "be9e65a6d42938", "100000000000051", ""), + ("ab5e104eeb71c000", "2cffbd639e9fea00", "ABCDEF0123456789", ""), + ("197b867547f68a00", "44b796cf94654800", "ABCDEF0123456789", ""), + ("329f9483a04f2c00", "9892f76961d0f000", "ABCDEF0123456789", ""), + ("4a2e12dfb4545000", "1aa3e89a69794500", "ABCDEF0123456789", ""), + ( + "8b9acdf013d140f000", "12e4ceaefabdf2b2f00", "25A55A46E5DA99C71C7", + "0x25A55A46E5DA99C71C7 is the 3rd repunit prime(dec) 11111111111111111111111" + ), + ("1b8d960ea277e3f5500", "14418aa980e37dd000", "25A55A46E5DA99C71C7", ""), + ("7314524977e8075980", "8172fa45618ccd0d80", "25A55A46E5DA99C71C7", ""), + ("ca14f031769be63580", "147a2f3cf2964ca9400", "25A55A46E5DA99C71C7", ""), + ( + "18532ba119d5cd0cf39735c0000", "25f9838e31634844924733000000", + "314DC643FB763F2B8C0E2DE00879", + "0x314DC643FB763F2B8C0E2DE00879 is (dec)99999999977^3" + ), + ( + "a56e2d2517519e3970e70c40000", "ec27428d4bb380458588fa80000", + "314DC643FB763F2B8C0E2DE00879", "" + ), + ( + "1cb5e8257710e8653fff33a00000", "15fdd42fe440fd3a1d121380000", + "314DC643FB763F2B8C0E2DE00879", "" + ), + ( + "e50d07a65fc6f93e538ce040000", "1f4b059ca609f3ce597f61240000", + "314DC643FB763F2B8C0E2DE00879", "" + ), + ( + "1ea3ade786a095d978d387f30df9f20000000", + "127c448575f04af5a367a7be06c7da0000000", + "47BF19662275FA2F6845C74942ED1D852E521", + "0x47BF19662275FA2F6845C74942ED1D852E521 is (dec) 99999999977^4" + ), + ( + "16e15b0ca82764e72e38357b1f10a20000000", + "43e2355d8514bbe22b0838fdc3983a0000000", + "47BF19662275FA2F6845C74942ED1D852E521", "" + ), + ( + "be39332529d93f25c3d116c004c620000000", + "5cccec42370a0a2c89c6772da801a0000000", + "47BF19662275FA2F6845C74942ED1D852E521", "" + ), + ( + "ecaa468d90de0eeda474d39b3e1fc0000000", + "1e714554018de6dc0fe576bfd3b5660000000", + "47BF19662275FA2F6845C74942ED1D852E521", "" + ), + ( + "32298816711c5dce46f9ba06e775c4bedfc770e6700000000000000", + "8ee751fd5fb24f0b4a653cb3a0c8b7d9e724574d168000000000000", + "97EDD86E4B5C4592C6D32064AC55C888A7245F07CA3CC455E07C931", + ( + "0x97EDD86E4B5C4592C6D32064AC55C888A7245F07CA3CC455E07C931" + " is (dec) 99999999977^6" + ) + ), + ( + "29213b9df3cfd15f4b428645b67b677c29d1378d810000000000000", + "6cbb732c65e10a28872394dfdd1936d5171c3c3aac0000000000000", + "97EDD86E4B5C4592C6D32064AC55C888A7245F07CA3CC455E07C931", "" + ), + ( + "6f18db06ad4abc52c0c50643dd13098abccd4a232f0000000000000", + "7e6bf41f2a86098ad51f98dfc10490ba3e8081bc830000000000000", + "97EDD86E4B5C4592C6D32064AC55C888A7245F07CA3CC455E07C931", "" + ), + ( + "62d3286cd706ad9d73caff63f1722775d7e8c731208000000000000", + "530f7ba02ae2b04c2fe3e3d27ec095925631a6c2528000000000000", + "97EDD86E4B5C4592C6D32064AC55C888A7245F07CA3CC455E07C931", "" + ), + ( + "a6c6503e3c031fdbf6009a89ed60582b7233c5a85de28b16000000000000000", + "75c8ed18270b583f16d442a467d32bf95c5e491e9b8523798000000000000000", + "DD15FE80B731872AC104DB37832F7E75A244AA2631BC87885B861E8F20375499", + ( + "0xDD15FE80B731872AC104DB37832F7E75A244AA2631BC87885B861E8F20375499" + "is (dec) 99999999977^7" + ) + ), + ( + "bf84d1f85cf6b51e04d2c8f4ffd03532d852053cf99b387d4000000000000000", + "397ba5a743c349f4f28bc583ecd5f06e0a25f9c6d98f09134000000000000000", + "DD15FE80B731872AC104DB37832F7E75A244AA2631BC87885B861E8F20375499", "" + ), + ( + "6db11c3a4152ed1a2aa6fa34b0903ec82ea1b88908dcb482000000000000000", + "ac8ac576a74ad6ca48f201bf89f77350ce86e821358d85920000000000000000", + "DD15FE80B731872AC104DB37832F7E75A244AA2631BC87885B861E8F20375499", "" + ), + ( + "3001d96d7fe8b733f33687646fc3017e3ac417eb32e0ec708000000000000000", + "925ddbdac4174e8321a48a32f79640e8cf7ec6f46ea235a80000000000000000", + "DD15FE80B731872AC104DB37832F7E75A244AA2631BC87885B861E8F20375499", "" + ), + ( + "1029048755f2e60dd98c8de6d9989226b6bb4f0db8e46bd1939de560000000000000000000", + "51bb7270b2e25cec0301a03e8275213bb6c2f6e6ec93d4d46d36ca0000000000000000000", + "141B8EBD9009F84C241879A1F680FACCED355DA36C498F73E96E880CF78EA5F96146380E41", + ( + "0x141B8EBD9009F84C241879A1F680FACCED355DA36C498F73E96E880CF78EA5F96146" + "380E41 is 99999999977^8" + ) + ), + ( + "1c5337ff982b3ad6611257dbff5bbd7a9920ba2d4f5838a0cc681ce000000000000000000", + "520c5d049ca4702031ba728591b665c4d4ccd3b2b86864d4c160fd2000000000000000000", + "141B8EBD9009F84C241879A1F680FACCED355DA36C498F73E96E880CF78EA5F96146380E41", + "" + ), + ( + "57074dfa00e42f6555bae624b7f0209f218adf57f73ed34ab0ff90c000000000000000000", + "41eb14b6c07bfd3d1fe4f4a610c17cc44fcfcda695db040e011065000000000000000000", + "141B8EBD9009F84C241879A1F680FACCED355DA36C498F73E96E880CF78EA5F96146380E41", + "" + ), + ( + "d8ed7feed2fe855e6997ad6397f776158573d425031bf085a615784000000000000000000", + "6f121dcd18c578ab5e229881006007bb6d319b179f11015fe958b9c000000000000000000", + "141B8EBD9009F84C241879A1F680FACCED355DA36C498F73E96E880CF78EA5F96146380E41", + "" + ), + ( + ( + "2a462b156180ea5fe550d3758c764e06fae54e626b5f503265a09df76edbdfbf" + "a1e6000000000000000000000000" + ), ( + "1136f41d1879fd4fb9e49e0943a46b6704d77c068ee237c3121f9071cfd3e6a0" + "0315800000000000000000000000" + ), ( + "2A94608DE88B6D5E9F8920F5ABB06B24CC35AE1FBACC87D075C621C3E2833EC90" + "2713E40F51E3B3C214EDFABC451" + ), ( + "0x2A94608DE88B6D5E9F8920F5ABB06B24CC35AE1FBACC87D075C621C3E2833EC" + "902713E40F51E3B3C214EDFABC451 is (dec) 99999999977^10" + ) + ), + ( + ( + "c1ac3800dfb3c6954dea391d206200cf3c47f795bf4a5603b4cb88ae7e574de47" + "40800000000000000000000000" + ), ( + "c0d16eda0549ede42fa0deb4635f7b7ce061fadea02ee4d85cba4c4f709603419" + "3c800000000000000000000000" + ), ( + "2A94608DE88B6D5E9F8920F5ABB06B24CC35AE1FBACC87D075C621C3E2833EC90" + "2713E40F51E3B3C214EDFABC451" + ), "" + ), + ( + ( + "19e45bb7633094d272588ad2e43bcb3ee341991c6731b6fa9d47c4018d7ce7bba" + "5ee800000000000000000000000" + ), ( + "1e4f83166ae59f6b9cc8fd3e7677ed8bfc01bb99c98bd3eb084246b64c1e18c33" + "65b800000000000000000000000" + ), ( + "2A94608DE88B6D5E9F8920F5ABB06B24CC35AE1FBACC87D075C621C3E2833EC90" + "2713E40F51E3B3C214EDFABC451" + ), "" + ), + ( + ( + "1aa93395fad5f9b7f20b8f9028a054c0bb7c11bb8520e6a95e5a34f06cb70bcdd" + "01a800000000000000000000000" + ), ( + "54b45afa5d4310192f8d224634242dd7dcfb342318df3d9bd37b4c614788ba13b" + "8b000000000000000000000000" + ), ( + "2A94608DE88B6D5E9F8920F5ABB06B24CC35AE1FBACC87D075C621C3E2833EC90" + "2713E40F51E3B3C214EDFABC451" + ), "" + ), + ( + ( + "544f2628a28cfb5ce0a1b7180ee66b49716f1d9476c466c57f0c4b23089917843" + "06d48f78686115ee19e25400000000000000000000000000000000" + ), ( + "677eb31ef8d66c120fa872a60cd47f6e10cbfdf94f90501bd7883cba03d185be0" + "a0148d1625745e9c4c827300000000000000000000000000000000" + ), ( + "8335616AED761F1F7F44E6BD49E807B82E3BF2BF11BFA6AF813C808DBF33DBFA1" + "1DABD6E6144BEF37C6800000000000000000000000000000000051" + ), ( + "0x8335616AED761F1F7F44E6BD49E807B82E3BF2BF11BFA6AF813C808DBF33DBF" + "A11DABD6E6144BEF37C6800000000000000000000000000000000051 is prime," + " (dec) 10^143 + 3^4" + ) + ), + ( + ( + "76bb3470985174915e9993522aec989666908f9e8cf5cb9f037bf4aee33d8865c" + "b6464174795d07e30015b80000000000000000000000000000000" + ), ( + "6aaaf60d5784dcef612d133613b179a317532ecca0eed40b8ad0c01e6d4a6d8c7" + "9a52af190abd51739009a900000000000000000000000000000000" + ), ( + "8335616AED761F1F7F44E6BD49E807B82E3BF2BF11BFA6AF813C808DBF33DBFA1" + "1DABD6E6144BEF37C6800000000000000000000000000000000051" + ), "" + ), + ( + ( + "6cfdd6e60912e441d2d1fc88f421b533f0103a5322ccd3f4db84861643ad63fd6" + "3d1d8cfbc1d498162786ba00000000000000000000000000000000" + ), ( + "1177246ec5e93814816465e7f8f248b350d954439d35b2b5d75d917218e7fd5fb" + "4c2f6d0667f9467fdcf33400000000000000000000000000000000" + ), ( + "8335616AED761F1F7F44E6BD49E807B82E3BF2BF11BFA6AF813C808DBF33DBFA1" + "1DABD6E6144BEF37C6800000000000000000000000000000000051" + ), "" + ), + ( + ( + "7a09a0b0f8bbf8057116fb0277a9bdf3a91b5eaa8830d448081510d8973888be5" + "a9f0ad04facb69aa3715f00000000000000000000000000000000" + ), ( + "764dec6c05a1c0d87b649efa5fd94c91ea28bffb4725d4ab4b33f1a3e8e3b314d" + "799020e244a835a145ec9800000000000000000000000000000000" + ), ( + "8335616AED761F1F7F44E6BD49E807B82E3BF2BF11BFA6AF813C808DBF33DBFA1" + "1DABD6E6144BEF37C6800000000000000000000000000000000051" + ), "" + ) + ] # type: List[Tuple[str, str, str, str]] + + def __init__( + self, val_a: str, val_b: str, val_n: str, case_description: str = "" + ): + self.case_description = case_description + self.arg_a = val_a + self.int_a = bignum_common.hex_to_int(val_a) + self.arg_b = val_b + self.int_b = bignum_common.hex_to_int(val_b) + self.arg_n = val_n + self.int_n = bignum_common.hex_to_int(val_n) + + limbs_a4 = bignum_common.limbs_mpi4(self.int_a) + limbs_a8 = bignum_common.limbs_mpi8(self.int_a) + self.limbs_b4 = bignum_common.limbs_mpi4(self.int_b) + self.limbs_b8 = bignum_common.limbs_mpi8(self.int_b) + self.limbs_an4 = bignum_common.limbs_mpi4(self.int_n) + self.limbs_an8 = bignum_common.limbs_mpi8(self.int_n) + + if limbs_a4 > self.limbs_an4 or limbs_a8 > self.limbs_an8: + raise Exception("Limbs of input A ({}) exceeds N ({})".format( + self.arg_a, self.arg_n + )) + + def arguments(self) -> List[str]: + return [ + str(self.limbs_an4), str(self.limbs_b4), + str(self.limbs_an8), str(self.limbs_b8), + bignum_common.quote_str(self.arg_a), + bignum_common.quote_str(self.arg_b), + bignum_common.quote_str(self.arg_n), + self.result() + ] + + def description(self) -> str: + if self.case_description != "replay": + if not self.start_2_mpi4 and self.limbs_an4 > 1: + tmp = "(start of 2-MPI 4-byte bignums) " + self.__class__.start_2_mpi4 = True + elif not self.start_2_mpi8 and self.limbs_an8 > 1: + tmp = "(start of 2-MPI 8-byte bignums) " + self.__class__.start_2_mpi8 = True + else: + tmp = "(gen) " + self.case_description = tmp + self.case_description + return super().description() + + def result(self) -> str: + """Get the result of the operation.""" + r4 = bignum_common.bound_mpi4_limbs(self.limbs_an4) + i4 = bignum_common.invmod(r4, self.int_n) + x4 = self.int_a * self.int_b * i4 + x4 = x4 % self.int_n + + r8 = bignum_common.bound_mpi8_limbs(self.limbs_an8) + i8 = bignum_common.invmod(r8, self.int_n) + x8 = self.int_a * self.int_b * i8 + x8 = x8 % self.int_n + return "\"{:x}\":\"{:x}\"".format(x4, x8) + + def set_limbs( + self, limbs_an4: int, limbs_b4: int, limbs_an8: int, limbs_b8: int + ) -> None: + """Set number of limbs for each input. + + Replaces default values set during initialization. + """ + self.limbs_an4 = limbs_an4 + self.limbs_b4 = limbs_b4 + self.limbs_an8 = limbs_an8 + self.limbs_b8 = limbs_b8 + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + """Generate replay and randomly generated test cases.""" + # Test cases which replay captured invocations during unit test runs. + for limbs_an4, limbs_b4, limbs_an8, limbs_b8, a, b, n in cls.replay_test_cases: + cur_op = cls(a, b, n, case_description="replay") + cur_op.set_limbs(limbs_an4, limbs_b4, limbs_an8, limbs_b8) + yield cur_op.create_test_case() + # Test cases generated randomly + for a, b, n, description in cls.random_test_cases: + cur_op = cls(a, b, n, case_description=description) + yield cur_op.create_test_case() From bb19f169854b77fc5d7cdff0af1816ba9f405b6b Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 4 Oct 2022 14:57:39 +0100 Subject: [PATCH 164/490] Add function to generate random montmul cases Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_core.py | 98 +++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index f04db2648..bbd0ac4af 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -14,8 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import random + from abc import ABCMeta -from typing import Iterator, List, Tuple +from typing import Dict, Iterator, List, Tuple from . import test_case from . import test_data_generation @@ -605,7 +607,99 @@ class BignumCoreMontmul(BignumCoreTarget): cur_op = cls(a, b, n, case_description="replay") cur_op.set_limbs(limbs_an4, limbs_b4, limbs_an8, limbs_b8) yield cur_op.create_test_case() - # Test cases generated randomly + # Random test cases can be generated using mpi_modmul_case_generate() + # Uses a mixture of primes and odd numbers as N, with four randomly + # generated cases for each N. for a, b, n, description in cls.random_test_cases: cur_op = cls(a, b, n, case_description=description) yield cur_op.create_test_case() + + +def mpi_modmul_case_generate() -> None: + """Generate valid inputs for montmul tests using moduli. + + For each modulus, generates random values for A and B and simple descriptions + for the test case. + """ + moduli = [ + ("3", ""), ("7", ""), ("B", ""), ("29", ""), ("FF", ""), + ("101", ""), ("38B", ""), ("8003", ""), ("10001", ""), + ("7F7F7", ""), ("800009", ""), ("100002B", ""), ("37EEE9D", ""), + ("8000000B", ""), ("8CD626B9", ""), ("10000000F", ""), + ("174876E7E9", "is prime (dec) 99999999977"), + ("8000000017", ""), ("864CB9076D", ""), ("F7F7F7F7F7", ""), + ("1000000000F", ""), ("800000000005", ""), ("800795D9BA47", ""), + ("1000000000015", ""), ("100000000000051", ""), ("ABCDEF0123456789", ""), + ( + "25A55A46E5DA99C71C7", + "is the 3rd repunit prime (dec) 11111111111111111111111" + ), + ("314DC643FB763F2B8C0E2DE00879", "is (dec)99999999977^3"), + ("47BF19662275FA2F6845C74942ED1D852E521", "is (dec) 99999999977^4"), + ( + "97EDD86E4B5C4592C6D32064AC55C888A7245F07CA3CC455E07C931", + "is (dec) 99999999977^6" + ), + ( + "DD15FE80B731872AC104DB37832F7E75A244AA2631BC87885B861E8F20375499", + "is (dec) 99999999977^7" + ), + ( + "141B8EBD9009F84C241879A1F680FACCED355DA36C498F73E96E880CF78EA5F96146380E41", + "is (dec) 99999999977^8" + ), + ( + ( + "2A94608DE88B6D5E9F8920F5ABB06B24CC35AE1FBACC87D075C621C3E283" + "3EC902713E40F51E3B3C214EDFABC451" + ), + "is (dec) 99999999977^10" + ), + ( + "8335616AED761F1F7F44E6BD49E807B82E3BF2BF11BFA6AF813C808DBF33DBFA11" + "DABD6E6144BEF37C6800000000000000000000000000000000051", + "is prime, (dec) 10^143 + 3^4" + ) + ] # type: List[Tuple[str, str]] + primes = [ + "3", "7", "B", "29", "101", "38B", "8003", "10001", "800009", + "100002B", "37EEE9D", "8000000B", "8CD626B9", + # From here they require > 1 4-byte MPI + "10000000F", "174876E7E9", "8000000017", "864CB9076D", "1000000000F", + "800000000005", "800795D9BA47", "1000000000015", "100000000000051", + # From here they require > 1 8-byte MPI + "25A55A46E5DA99C71C7", # this is 11111111111111111111111 decimal + # 10^143 + 3^4: (which is prime) + # 100000000000000000000000000000000000000000000000000000000000000000000000000000 + # 000000000000000000000000000000000000000000000000000000000000000081 + ( + "8335616AED761F1F7F44E6BD49E807B82E3BF2BF11BFA6AF813C808DBF33DBFA11" + "DABD6E6144BEF37C6800000000000000000000000000000000051" + ) + ] # type: List[str] + generated_inputs = [] + for mod, description in moduli: + n = bignum_common.hex_to_int(mod) + mod_read = "{:x}".format(n) + if mod_read != mod.lower(): + raise ValueError("Read modulus not equal to input.") + case_count = 3 if n < 5 else 4 + cases = {} # type: Dict[int, int] + i = 0 + while i < case_count: + a = random.randint(1, n) + b = random.randint(1, n) + if cases.get(a) == b: + continue + cases[a] = b + if description: + out_description = "0x{} {}".format(mod_read, description) + elif i == 0 and len(mod) > 1 and mod in primes: + out_description = "(0x{} is prime)" + else: + out_description = "" + generated_inputs.append( + ("{:x}".format(a), "{:x}".format(b), mod, out_description) + ) + i += 1 + print(generated_inputs) From e074e30e5fdf5ce0ad621cf7ea082427373faf26 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 12 Oct 2022 14:53:17 +0100 Subject: [PATCH 165/490] Redefine result() method to return List Many bignum tests have multiple calculated result values, so return these as a list, rather than formatting as a string. Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_common.py | 6 ++-- scripts/framework_dev/bignum_core.py | 47 ++++++++++++++++---------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index c340cd53a..88fa4dfd0 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -108,10 +108,12 @@ class OperationCommon: self.int_b = hex_to_int(val_b) def arguments(self) -> List[str]: - return [quote_str(self.arg_a), quote_str(self.arg_b), self.result()] + return [ + quote_str(self.arg_a), quote_str(self.arg_b) + ] + self.result() @abstractmethod - def result(self) -> str: + def result(self) -> List[str]: """Get the result of the operation. This could be calculated during initialization and stored as `_result` diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index bbd0ac4af..b4dff8938 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -80,16 +80,19 @@ class BignumCoreAddIf(BignumCoreOperation): test_function = "mpi_core_add_if" test_name = "mbedtls_mpi_core_add_if" - def result(self) -> str: + def result(self) -> List[str]: tmp = self.int_a + self.int_b bound_val = max(self.int_a, self.int_b) bound_4 = bignum_common.bound_mpi4(bound_val) bound_8 = bignum_common.bound_mpi8(bound_val) carry_4, remainder_4 = divmod(tmp, bound_4) carry_8, remainder_8 = divmod(tmp, bound_8) - return "\"{:x}\":{}:\"{:x}\":{}".format( - remainder_4, carry_4, remainder_8, carry_8 - ) + return [ + "\"{:x}\"".format(remainder_4), + str(carry_4), + "\"{:x}\"".format(remainder_8), + str(carry_8) + ] class BignumCoreSub(BignumCoreOperation): @@ -100,7 +103,7 @@ class BignumCoreSub(BignumCoreOperation): test_name = "mbedtls_mpi_core_sub" unique_combinations_only = False - def result(self) -> str: + def result(self) -> List[str]: if self.int_a >= self.int_b: result_4 = result_8 = self.int_a - self.int_b carry = 0 @@ -111,7 +114,11 @@ class BignumCoreSub(BignumCoreOperation): bound_8 = bignum_common.bound_mpi8(bound_val) result_8 = bound_8 + self.int_a - self.int_b carry = 1 - return "\"{:x}\":\"{:x}\":{}".format(result_4, result_8, carry) + return [ + "\"{:x}\"".format(result_4), + "\"{:x}\"".format(result_8), + str(carry) + ] class BignumCoreMLA(BignumCoreOperation): @@ -153,9 +160,8 @@ class BignumCoreMLA(BignumCoreOperation): return [ bignum_common.quote_str(self.arg_a), bignum_common.quote_str(self.arg_b), - bignum_common.quote_str(self.arg_scalar), - self.result() - ] + bignum_common.quote_str(self.arg_scalar) + ] + self.result() def description(self) -> str: """Override and add the additional scalar.""" @@ -165,16 +171,19 @@ class BignumCoreMLA(BignumCoreOperation): ) return super().description() - def result(self) -> str: + def result(self) -> List[str]: result = self.int_a + (self.int_b * self.int_scalar) bound_val = max(self.int_a, self.int_b) bound_4 = bignum_common.bound_mpi4(bound_val) bound_8 = bignum_common.bound_mpi8(bound_val) carry_4, remainder_4 = divmod(result, bound_4) carry_8, remainder_8 = divmod(result, bound_8) - return "\"{:x}\":\"{:x}\":\"{:x}\":\"{:x}\"".format( - remainder_4, carry_4, remainder_8, carry_8 - ) + return [ + "\"{:x}\"".format(remainder_4), + "\"{:x}\"".format(carry_4), + "\"{:x}\"".format(remainder_8), + "\"{:x}\"".format(carry_8) + ] @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: @@ -557,9 +566,8 @@ class BignumCoreMontmul(BignumCoreTarget): str(self.limbs_an8), str(self.limbs_b8), bignum_common.quote_str(self.arg_a), bignum_common.quote_str(self.arg_b), - bignum_common.quote_str(self.arg_n), - self.result() - ] + bignum_common.quote_str(self.arg_n) + ] + self.result() def description(self) -> str: if self.case_description != "replay": @@ -574,7 +582,7 @@ class BignumCoreMontmul(BignumCoreTarget): self.case_description = tmp + self.case_description return super().description() - def result(self) -> str: + def result(self) -> List[str]: """Get the result of the operation.""" r4 = bignum_common.bound_mpi4_limbs(self.limbs_an4) i4 = bignum_common.invmod(r4, self.int_n) @@ -585,7 +593,10 @@ class BignumCoreMontmul(BignumCoreTarget): i8 = bignum_common.invmod(r8, self.int_n) x8 = self.int_a * self.int_b * i8 x8 = x8 % self.int_n - return "\"{:x}\":\"{:x}\"".format(x4, x8) + return [ + "\"{:x}\"".format(x4), + "\"{:x}\"".format(x8) + ] def set_limbs( self, limbs_an4: int, limbs_b4: int, limbs_an8: int, limbs_b8: int From a1cb8682106a429451aeffedd17122256fc1632f Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 18 Oct 2022 15:42:51 +0100 Subject: [PATCH 166/490] Add missing space in string Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index b4dff8938..a300225ab 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -387,7 +387,7 @@ class BignumCoreMontmul(BignumCoreTarget): "DD15FE80B731872AC104DB37832F7E75A244AA2631BC87885B861E8F20375499", ( "0xDD15FE80B731872AC104DB37832F7E75A244AA2631BC87885B861E8F20375499" - "is (dec) 99999999977^7" + " is (dec) 99999999977^7" ) ), ( From 4f655f93d273080149cd32085ffcebf6094ffbcb Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 19 Oct 2022 13:37:12 +0100 Subject: [PATCH 167/490] Remove unnecessary check Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index a300225ab..1bd248266 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -692,8 +692,6 @@ def mpi_modmul_case_generate() -> None: for mod, description in moduli: n = bignum_common.hex_to_int(mod) mod_read = "{:x}".format(n) - if mod_read != mod.lower(): - raise ValueError("Read modulus not equal to input.") case_count = 3 if n < 5 else 4 cases = {} # type: Dict[int, int] i = 0 From 22addd3276bdb9d35dfe8fde8578170e19d92d40 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 19 Oct 2022 13:50:10 +0100 Subject: [PATCH 168/490] Pass bits_in_limb parameter to duplicated methods Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_common.py | 31 ++++++++------------------ scripts/framework_dev/bignum_core.py | 30 ++++++++++++------------- 2 files changed, 24 insertions(+), 37 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 88fa4dfd0..c81770ed7 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -43,31 +43,18 @@ def hex_to_int(val: str) -> int: def quote_str(val) -> str: return "\"{}\"".format(val) -def bound_mpi8(val: int) -> int: - """First number exceeding 8-byte limbs needed for given input value.""" - return bound_mpi8_limbs(limbs_mpi8(val)) +def bound_mpi(val: int, bits_in_limb: int) -> int: + """First number exceeding number of limbs needed for given input value.""" + return bound_mpi_limbs(limbs_mpi(val, bits_in_limb), bits_in_limb) -def bound_mpi4(val: int) -> int: - """First number exceeding 4-byte limbs needed for given input value.""" - return bound_mpi4_limbs(limbs_mpi4(val)) - -def bound_mpi8_limbs(limbs: int) -> int: - """First number exceeding maximum of given 8-byte limbs.""" - bits = 64 * limbs +def bound_mpi_limbs(limbs: int, bits_in_limb: int) -> int: + """First number exceeding maximum of given number of limbs.""" + bits = bits_in_limb * limbs return 1 << bits -def bound_mpi4_limbs(limbs: int) -> int: - """First number exceeding maximum of given 4-byte limbs.""" - bits = 32 * limbs - return 1 << bits - -def limbs_mpi8(val: int) -> int: - """Return the number of 8-byte limbs required to store value.""" - return (val.bit_length() + 63) // 64 - -def limbs_mpi4(val: int) -> int: - """Return the number of 4-byte limbs required to store value.""" - return (val.bit_length() + 31) // 32 +def limbs_mpi(val: int, bits_in_limb: int) -> int: + """Return the number of limbs required to store value.""" + return (val.bit_length() + bits_in_limb - 1) // bits_in_limb def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: """Return all pair combinations from input values. diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 1bd248266..3652ac20a 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -83,8 +83,8 @@ class BignumCoreAddIf(BignumCoreOperation): def result(self) -> List[str]: tmp = self.int_a + self.int_b bound_val = max(self.int_a, self.int_b) - bound_4 = bignum_common.bound_mpi4(bound_val) - bound_8 = bignum_common.bound_mpi8(bound_val) + bound_4 = bignum_common.bound_mpi(bound_val, 32) + bound_8 = bignum_common.bound_mpi(bound_val, 64) carry_4, remainder_4 = divmod(tmp, bound_4) carry_8, remainder_8 = divmod(tmp, bound_8) return [ @@ -109,9 +109,9 @@ class BignumCoreSub(BignumCoreOperation): carry = 0 else: bound_val = max(self.int_a, self.int_b) - bound_4 = bignum_common.bound_mpi4(bound_val) + bound_4 = bignum_common.bound_mpi(bound_val, 32) result_4 = bound_4 + self.int_a - self.int_b - bound_8 = bignum_common.bound_mpi8(bound_val) + bound_8 = bignum_common.bound_mpi(bound_val, 64) result_8 = bound_8 + self.int_a - self.int_b carry = 1 return [ @@ -153,7 +153,7 @@ class BignumCoreMLA(BignumCoreOperation): super().__init__(val_a, val_b) self.arg_scalar = val_s self.int_scalar = bignum_common.hex_to_int(val_s) - if bignum_common.limbs_mpi4(self.int_scalar) > 1: + if bignum_common.limbs_mpi(self.int_scalar, 32) > 1: self.dependencies = ["MBEDTLS_HAVE_INT64"] def arguments(self) -> List[str]: @@ -174,8 +174,8 @@ class BignumCoreMLA(BignumCoreOperation): def result(self) -> List[str]: result = self.int_a + (self.int_b * self.int_scalar) bound_val = max(self.int_a, self.int_b) - bound_4 = bignum_common.bound_mpi4(bound_val) - bound_8 = bignum_common.bound_mpi8(bound_val) + bound_4 = bignum_common.bound_mpi(bound_val, 32) + bound_8 = bignum_common.bound_mpi(bound_val, 64) carry_4, remainder_4 = divmod(result, bound_4) carry_8, remainder_8 = divmod(result, bound_8) return [ @@ -548,12 +548,12 @@ class BignumCoreMontmul(BignumCoreTarget): self.arg_n = val_n self.int_n = bignum_common.hex_to_int(val_n) - limbs_a4 = bignum_common.limbs_mpi4(self.int_a) - limbs_a8 = bignum_common.limbs_mpi8(self.int_a) - self.limbs_b4 = bignum_common.limbs_mpi4(self.int_b) - self.limbs_b8 = bignum_common.limbs_mpi8(self.int_b) - self.limbs_an4 = bignum_common.limbs_mpi4(self.int_n) - self.limbs_an8 = bignum_common.limbs_mpi8(self.int_n) + limbs_a4 = bignum_common.limbs_mpi(self.int_a, 32) + limbs_a8 = bignum_common.limbs_mpi(self.int_a, 64) + self.limbs_b4 = bignum_common.limbs_mpi(self.int_b, 32) + self.limbs_b8 = bignum_common.limbs_mpi(self.int_b, 64) + self.limbs_an4 = bignum_common.limbs_mpi(self.int_n, 32) + self.limbs_an8 = bignum_common.limbs_mpi(self.int_n, 64) if limbs_a4 > self.limbs_an4 or limbs_a8 > self.limbs_an8: raise Exception("Limbs of input A ({}) exceeds N ({})".format( @@ -584,12 +584,12 @@ class BignumCoreMontmul(BignumCoreTarget): def result(self) -> List[str]: """Get the result of the operation.""" - r4 = bignum_common.bound_mpi4_limbs(self.limbs_an4) + r4 = bignum_common.bound_mpi_limbs(self.limbs_an4, 32) i4 = bignum_common.invmod(r4, self.int_n) x4 = self.int_a * self.int_b * i4 x4 = x4 % self.int_n - r8 = bignum_common.bound_mpi8_limbs(self.limbs_an8) + r8 = bignum_common.bound_mpi_limbs(self.limbs_an8, 64) i8 = bignum_common.invmod(r8, self.int_n) x8 = self.int_a * self.int_b * i8 x8 = x8 % self.int_n From 8600dcd4ddd557dc147a9a643f4cf31fda9aa766 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Mon, 17 Oct 2022 10:16:56 +0100 Subject: [PATCH 169/490] Bignum Core: add limb size specific test generation In Bignum Core the result also involves a carry and both the result and the carry depend on the size of the limbs. Before this change both 32 and 64 bit specific result have been passed to the test functions. Moving this decision out of the tests makes the test functions easier to write and read and the test cases easier to read and debug. The change doesn't make writing the generator script any harder and might even make reading it easier. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 3652ac20a..23db980e8 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -72,6 +72,24 @@ class BignumCoreOperation(bignum_common.OperationCommon, BignumCoreTarget, metac yield cls(a_value, b_value).create_test_case() +class BignumCoreOperationArchSplit(BignumCoreOperation): + #pylint: disable=abstract-method + """Common features for bignum core operations where the result depends on + the limb size.""" + + def __init__(self, val_a: str, val_b: str, bits_in_limb: int) -> None: + super().__init__(val_a, val_b) + self.bits_in_limb = bits_in_limb + if self.bits_in_limb == 32: + self.dependencies = ["MBEDTLS_HAVE_INT32"] + elif self.bits_in_limb == 64: + self.dependencies = ["MBEDTLS_HAVE_INT64"] + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for a_value, b_value in cls.get_value_pairs(): + yield cls(a_value, b_value, 32).create_test_case() + yield cls(a_value, b_value, 64).create_test_case() class BignumCoreAddIf(BignumCoreOperation): """Test cases for bignum core add if.""" From faceb57d9e23b5b7789a4a76da06c43e94b5c755 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Mon, 17 Oct 2022 10:25:29 +0100 Subject: [PATCH 170/490] mpi_core_add_if: simplify tests Use the new, limb size aware base class to generate tests for mpi_core_add_if(). Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 23db980e8..0b6ee9717 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -91,7 +91,7 @@ class BignumCoreOperationArchSplit(BignumCoreOperation): yield cls(a_value, b_value, 32).create_test_case() yield cls(a_value, b_value, 64).create_test_case() -class BignumCoreAddIf(BignumCoreOperation): +class BignumCoreAddIf(BignumCoreOperationArchSplit): """Test cases for bignum core add if.""" count = 0 symbol = "+" @@ -99,17 +99,15 @@ class BignumCoreAddIf(BignumCoreOperation): test_name = "mbedtls_mpi_core_add_if" def result(self) -> List[str]: - tmp = self.int_a + self.int_b + result = self.int_a + self.int_b bound_val = max(self.int_a, self.int_b) - bound_4 = bignum_common.bound_mpi(bound_val, 32) - bound_8 = bignum_common.bound_mpi(bound_val, 64) - carry_4, remainder_4 = divmod(tmp, bound_4) - carry_8, remainder_8 = divmod(tmp, bound_8) + + bound = bignum_common.bound_mpi(bound_val, self.bits_in_limb) + carry, result = divmod(result, bound) + return [ - "\"{:x}\"".format(remainder_4), - str(carry_4), - "\"{:x}\"".format(remainder_8), - str(carry_8) + "\"{:x}\"".format(result), + str(carry) ] From 47c1b10296fc5139fa7098fe5a2aec865a4df3b0 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Mon, 17 Oct 2022 11:21:22 +0100 Subject: [PATCH 171/490] Bignum Core test: move bound to constructor We will need it to pad parameters in the base class, but it is useful because every child class would need to calculate it anyway. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 0b6ee9717..8327df1b8 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -79,11 +79,13 @@ class BignumCoreOperationArchSplit(BignumCoreOperation): def __init__(self, val_a: str, val_b: str, bits_in_limb: int) -> None: super().__init__(val_a, val_b) + bound_val = max(self.int_a, self.int_b) self.bits_in_limb = bits_in_limb + self.bound = bignum_common.bound_mpi(bound_val, self.bits_in_limb) if self.bits_in_limb == 32: self.dependencies = ["MBEDTLS_HAVE_INT32"] elif self.bits_in_limb == 64: - self.dependencies = ["MBEDTLS_HAVE_INT64"] + self.dependencies = ["MBDTLS_HAVE_INT64"] @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: @@ -100,10 +102,8 @@ class BignumCoreAddIf(BignumCoreOperationArchSplit): def result(self) -> List[str]: result = self.int_a + self.int_b - bound_val = max(self.int_a, self.int_b) - bound = bignum_common.bound_mpi(bound_val, self.bits_in_limb) - carry, result = divmod(result, bound) + carry, result = divmod(result, self.bound) return [ "\"{:x}\"".format(result), From 17940f836a7c8c08713b0fb8f69f0d70a1091b99 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Mon, 17 Oct 2022 13:47:13 +0100 Subject: [PATCH 172/490] mpi_core_add_if test: Remove dependency on old API Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 8327df1b8..ab582d3a7 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -61,8 +61,8 @@ class BignumCoreOperation(bignum_common.OperationCommon, BignumCoreTarget, metac generated to provide some context to the test case. """ if not self.case_description: - self.case_description = "{} {} {}".format( - self.arg_a, self.symbol, self.arg_b + self.case_description = "{:x} {} {:x}".format( + self.int_a, self.symbol, self.int_b ) return super().description() @@ -82,10 +82,20 @@ class BignumCoreOperationArchSplit(BignumCoreOperation): bound_val = max(self.int_a, self.int_b) self.bits_in_limb = bits_in_limb self.bound = bignum_common.bound_mpi(bound_val, self.bits_in_limb) + limbs = bignum_common.limbs_mpi(bound_val, self.bits_in_limb) + byte_len = limbs*self.bits_in_limb//8 + self.hex_digits = 2*byte_len if self.bits_in_limb == 32: self.dependencies = ["MBEDTLS_HAVE_INT32"] elif self.bits_in_limb == 64: - self.dependencies = ["MBDTLS_HAVE_INT64"] + self.dependencies = ["MBEDTLS_HAVE_INT64"] + else: + raise ValueError("Invalid number of bits in limb!") + self.arg_a = self.arg_a.zfill(self.hex_digits) + self.arg_b = self.arg_b.zfill(self.hex_digits) + + def pad_to_limbs(self, val) -> str: + return "{:x}".format(val).zfill(self.hex_digits) @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: @@ -106,11 +116,10 @@ class BignumCoreAddIf(BignumCoreOperationArchSplit): carry, result = divmod(result, self.bound) return [ - "\"{:x}\"".format(result), + bignum_common.quote_str(self.pad_to_limbs(result)), str(carry) ] - class BignumCoreSub(BignumCoreOperation): """Test cases for bignum core sub.""" count = 0 From 7fac23c22fc87e8f4e678c705ef82b476437c726 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Sep 2022 23:13:33 +0200 Subject: [PATCH 173/490] Bignum core: test shift_r Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_core.py | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 3652ac20a..e6310f32b 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -29,6 +29,47 @@ class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): target_basename = 'test_suite_bignum_core.generated' +class BignumCoreShiftR(BignumCoreTarget, metaclass=ABCMeta): + """Test cases for mbedtls_bignum_core_shift_r().""" + count = 0 + test_function = "mpi_core_shift_r" + test_name = "Core shift right" + + DATA = [ + ('00', '0', [0, 1, 8]), + ('01', '1', [0, 1, 2, 8, 64]), + ('dee5ca1a7ef10a75', '64-bit', + list(range(11)) + [31, 32, 33, 63, 64, 65, 71, 72]), + ('002e7ab0070ad57001', '[leading 0 limb]', + [0, 1, 8, 63, 64]), + ('a1055eb0bb1efa1150ff', '80-bit', + [0, 1, 8, 63, 64, 65, 72, 79, 80, 81, 88, 128, 129, 136]), + ('020100000000000000001011121314151617', '138-bit', + [0, 1, 8, 9, 16, 72, 73, 136, 137, 138, 144]), + ] + + def __init__(self, input_hex: str, descr: str, count: int) -> None: + self.input_hex = input_hex + self.number_description = descr + self.shift_count = count + self.result = bignum_common.hex_to_int(input_hex) >> count + + def arguments(self) -> List[str]: + return ['"{}"'.format(self.input_hex), + str(self.shift_count), + '"{:0{}x}"'.format(self.result, len(self.input_hex))] + + def description(self) -> str: + return 'Core shift {} >> {}'.format(self.number_description, + self.shift_count) + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for input_hex, descr, counts in cls.DATA: + for count in counts: + yield cls(input_hex, descr, count).create_test_case() + + class BignumCoreOperation(bignum_common.OperationCommon, BignumCoreTarget, metaclass=ABCMeta): #pylint: disable=abstract-method """Common features for bignum core operations.""" From 69eb1f2002a369227ef7b8caee415d4e3dc5952f Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Thu, 20 Oct 2022 12:09:30 +0100 Subject: [PATCH 174/490] Fix style in bignum_core.py Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index ab582d3a7..2e641953d 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -82,9 +82,9 @@ class BignumCoreOperationArchSplit(BignumCoreOperation): bound_val = max(self.int_a, self.int_b) self.bits_in_limb = bits_in_limb self.bound = bignum_common.bound_mpi(bound_val, self.bits_in_limb) - limbs = bignum_common.limbs_mpi(bound_val, self.bits_in_limb) - byte_len = limbs*self.bits_in_limb//8 - self.hex_digits = 2*byte_len + limbs = bignum_common.limbs_mpi(bound_val, self.bits_in_limb) + byte_len = limbs * self.bits_in_limb // 8 + self.hex_digits = 2 * byte_len if self.bits_in_limb == 32: self.dependencies = ["MBEDTLS_HAVE_INT32"] elif self.bits_in_limb == 64: From 4e8eb1c156f0b2f9bd3342501d204dcaacf763e2 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Tue, 25 Oct 2022 12:45:50 +0100 Subject: [PATCH 175/490] Extend the unit tests for mbedtls_mpi_core_add_if() to also test mbedtls_mpi_core_add() Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index e46364b1a..0d238e714 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -144,12 +144,12 @@ class BignumCoreOperationArchSplit(BignumCoreOperation): yield cls(a_value, b_value, 32).create_test_case() yield cls(a_value, b_value, 64).create_test_case() -class BignumCoreAddIf(BignumCoreOperationArchSplit): - """Test cases for bignum core add if.""" +class BignumCoreAddAndAddIf(BignumCoreOperationArchSplit): + """Test cases for bignum core add and add-if.""" count = 0 symbol = "+" - test_function = "mpi_core_add_if" - test_name = "mbedtls_mpi_core_add_if" + test_function = "mpi_core_add_and_add_if" + test_name = "mpi_core_add_and_add_if" def result(self) -> List[str]: result = self.int_a + self.int_b From 819db0041cabd89c4904c8143497b94d10bca9ae Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 26 Oct 2022 19:10:29 +0100 Subject: [PATCH 176/490] Add mbedtls_mpi_core_ct_uint_table_lookup tests Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 0d238e714..e88f469a1 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -69,6 +69,44 @@ class BignumCoreShiftR(BignumCoreTarget, metaclass=ABCMeta): for count in counts: yield cls(input_hex, descr, count).create_test_case() +class BignumCoreCTLookup(BignumCoreTarget, metaclass=ABCMeta): + """Test cases for mbedtls_mpi_core_ct_uint_table_lookup().""" + count = 0 + test_function = "mpi_core_ct_uint_table_lookup" + test_name = "Constant time MPI table lookup" + + bitsizes = [ + (32, "One limb"), + (192, "Smallest curve sized"), + (512, "Largest curve sized"), + (2048, "Small FF/RSA sized"), + (4096, "Large FF/RSA sized"), + ] + + window_sizes = [ 0, 1, 2, 3, 4, 5, 6 ] + + def __init__(self, + bitsize: int, descr: str, window_size: int) -> None: + self.bitsize = bitsize + self.bitsize_description = descr + self.window_size = window_size + + def arguments(self) -> List[str]: + return [str(self.bitsize), str(self.window_size)] + + def description(self) -> str: + return '{} - {} MPI with {} bit window'.format( + BignumCoreCTLookup.test_name, + self.bitsize_description, + self.window_size + ) + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for bitsize, bitsize_description in cls.bitsizes: + for window_size in cls.window_sizes: + yield (cls(bitsize, bitsize_description, window_size) + .create_test_case()) class BignumCoreOperation(bignum_common.OperationCommon, BignumCoreTarget, metaclass=ABCMeta): #pylint: disable=abstract-method From 05120d5c5a9a2fec961a949760c227d2869c5330 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Mon, 31 Oct 2022 14:32:46 +0000 Subject: [PATCH 177/490] Make pylint happy Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index e88f469a1..f8ba12b02 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -76,14 +76,14 @@ class BignumCoreCTLookup(BignumCoreTarget, metaclass=ABCMeta): test_name = "Constant time MPI table lookup" bitsizes = [ - (32, "One limb"), - (192, "Smallest curve sized"), - (512, "Largest curve sized"), - (2048, "Small FF/RSA sized"), - (4096, "Large FF/RSA sized"), - ] + (32, "One limb"), + (192, "Smallest curve sized"), + (512, "Largest curve sized"), + (2048, "Small FF/RSA sized"), + (4096, "Large FF/RSA sized"), + ] - window_sizes = [ 0, 1, 2, 3, 4, 5, 6 ] + window_sizes = [0, 1, 2, 3, 4, 5, 6] def __init__(self, bitsize: int, descr: str, window_size: int) -> None: @@ -96,10 +96,10 @@ class BignumCoreCTLookup(BignumCoreTarget, metaclass=ABCMeta): def description(self) -> str: return '{} - {} MPI with {} bit window'.format( - BignumCoreCTLookup.test_name, - self.bitsize_description, - self.window_size - ) + BignumCoreCTLookup.test_name, + self.bitsize_description, + self.window_size + ) @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: From b85b6115b812da6bf3603e13977d5e7d63e1161b Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Mon, 31 Oct 2022 15:32:28 +0000 Subject: [PATCH 178/490] mpi_core_ct_uint_table_lookup: style and docs Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index f8ba12b02..9929e13fa 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -71,7 +71,6 @@ class BignumCoreShiftR(BignumCoreTarget, metaclass=ABCMeta): class BignumCoreCTLookup(BignumCoreTarget, metaclass=ABCMeta): """Test cases for mbedtls_mpi_core_ct_uint_table_lookup().""" - count = 0 test_function = "mpi_core_ct_uint_table_lookup" test_name = "Constant time MPI table lookup" From 632cb3ba9f11e3260e816aa0236427b84e22b47c Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 2 Nov 2022 14:35:17 +0000 Subject: [PATCH 179/490] Add merge slots to Bignum files Legacy Bignum is excluded as it doesn't get regular extensions like new ones. Each slot uses comments of their respective filetype. Since .data files don't have a syntax for comments, dummy test cases are used. (These test cases will never be executed and no noise will be added to tests.) Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 9929e13fa..87098075f 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -815,3 +815,44 @@ def mpi_modmul_case_generate() -> None: ) i += 1 print(generated_inputs) + +# BEGIN MERGE SLOT 1 + +# END MERGE SLOT 1 + +# BEGIN MERGE SLOT 2 + +# END MERGE SLOT 2 + +# BEGIN MERGE SLOT 3 + +# END MERGE SLOT 3 + +# BEGIN MERGE SLOT 4 + +# END MERGE SLOT 4 + +# BEGIN MERGE SLOT 5 + +# END MERGE SLOT 5 + +# BEGIN MERGE SLOT 6 + +# END MERGE SLOT 6 + +# BEGIN MERGE SLOT 7 + +# END MERGE SLOT 7 + +# BEGIN MERGE SLOT 8 + +# END MERGE SLOT 8 + +# BEGIN MERGE SLOT 9 + +# END MERGE SLOT 9 + +# BEGIN MERGE SLOT 10 + +# END MERGE SLOT 10 + From d304f892630676d6f5c16af9fd865f4a4e0ccfdb Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 2 Nov 2022 14:40:58 +0000 Subject: [PATCH 180/490] Add script for generating mod_raw test cases This commit only adds the boilerplate, no actual tests are added. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_mod_raw.py | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 scripts/framework_dev/bignum_mod_raw.py diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py new file mode 100644 index 000000000..767073171 --- /dev/null +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -0,0 +1,29 @@ +"""Framework classes for generation of bignum mod_raw test cases.""" +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random + +from abc import ABCMeta +from typing import Dict, Iterator, List, Tuple + +from . import test_data_generation +from . import bignum_common + +class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): + #pylint: disable=abstract-method + """Target for bignum mod_raw test case generation.""" + target_basename = 'test_suite_bignum_mod_raw.generated' + From 95dd5f36e913853d0f853a4120c1ca377db9072d Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 2 Nov 2022 14:44:08 +0000 Subject: [PATCH 181/490] Add script for generating mod test cases This commit only adds the boilerplate, no actual tests are added. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_mod.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 scripts/framework_dev/bignum_mod.py diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py new file mode 100644 index 000000000..3bed853c6 --- /dev/null +++ b/scripts/framework_dev/bignum_mod.py @@ -0,0 +1,29 @@ +"""Framework classes for generation of bignum mod test cases.""" +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random + +from abc import ABCMeta +from typing import Dict, Iterator, List, Tuple + +from . import test_data_generation +from . import bignum_common + +class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): + #pylint: disable=abstract-method + """Target for bignum mod test case generation.""" + target_basename = 'test_suite_bignum_mod.generated' + From c1c0d9fd4ac653bf32724ced7350e5f0a3b71f88 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 2 Nov 2022 14:46:23 +0000 Subject: [PATCH 182/490] Add merge slots to raw and mod_raw test generation Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_mod.py | 40 +++++++++++++++++++++++++ scripts/framework_dev/bignum_mod_raw.py | 40 +++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 3bed853c6..f4f05912f 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -27,3 +27,43 @@ class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): """Target for bignum mod test case generation.""" target_basename = 'test_suite_bignum_mod.generated' +# BEGIN MERGE SLOT 1 + +# END MERGE SLOT 1 + +# BEGIN MERGE SLOT 2 + +# END MERGE SLOT 2 + +# BEGIN MERGE SLOT 3 + +# END MERGE SLOT 3 + +# BEGIN MERGE SLOT 4 + +# END MERGE SLOT 4 + +# BEGIN MERGE SLOT 5 + +# END MERGE SLOT 5 + +# BEGIN MERGE SLOT 6 + +# END MERGE SLOT 6 + +# BEGIN MERGE SLOT 7 + +# END MERGE SLOT 7 + +# BEGIN MERGE SLOT 8 + +# END MERGE SLOT 8 + +# BEGIN MERGE SLOT 9 + +# END MERGE SLOT 9 + +# BEGIN MERGE SLOT 10 + +# END MERGE SLOT 10 + diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 767073171..d5d48fe32 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -27,3 +27,43 @@ class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): """Target for bignum mod_raw test case generation.""" target_basename = 'test_suite_bignum_mod_raw.generated' +# BEGIN MERGE SLOT 1 + +# END MERGE SLOT 1 + +# BEGIN MERGE SLOT 2 + +# END MERGE SLOT 2 + +# BEGIN MERGE SLOT 3 + +# END MERGE SLOT 3 + +# BEGIN MERGE SLOT 4 + +# END MERGE SLOT 4 + +# BEGIN MERGE SLOT 5 + +# END MERGE SLOT 5 + +# BEGIN MERGE SLOT 6 + +# END MERGE SLOT 6 + +# BEGIN MERGE SLOT 7 + +# END MERGE SLOT 7 + +# BEGIN MERGE SLOT 8 + +# END MERGE SLOT 8 + +# BEGIN MERGE SLOT 9 + +# END MERGE SLOT 9 + +# BEGIN MERGE SLOT 10 + +# END MERGE SLOT 10 + From f67a3ba2e957ec502961aafd122f9ecd5b557181 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 2 Nov 2022 16:15:25 +0000 Subject: [PATCH 183/490] Make pylint happy Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 1 - scripts/framework_dev/bignum_mod.py | 5 ----- scripts/framework_dev/bignum_mod_raw.py | 5 ----- 3 files changed, 11 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 87098075f..0cc86b809 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -855,4 +855,3 @@ def mpi_modmul_case_generate() -> None: # BEGIN MERGE SLOT 10 # END MERGE SLOT 10 - diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index f4f05912f..117e1ee7b 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -14,13 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import random - from abc import ABCMeta -from typing import Dict, Iterator, List, Tuple from . import test_data_generation -from . import bignum_common class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): #pylint: disable=abstract-method @@ -66,4 +62,3 @@ class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): # BEGIN MERGE SLOT 10 # END MERGE SLOT 10 - diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index d5d48fe32..2a4886ca9 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -14,13 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import random - from abc import ABCMeta -from typing import Dict, Iterator, List, Tuple from . import test_data_generation -from . import bignum_common class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): #pylint: disable=abstract-method @@ -66,4 +62,3 @@ class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): # BEGIN MERGE SLOT 10 # END MERGE SLOT 10 - From fb50f95629b53706a8f24ad603dd3a1a89d8d0c6 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Thu, 3 Nov 2022 08:42:54 +0000 Subject: [PATCH 184/490] Fix bignum test generator class names Co-authored-by: minosgalanakis <30719586+minosgalanakis@users.noreply.github.com> Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_mod.py | 2 +- scripts/framework_dev/bignum_mod_raw.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 117e1ee7b..2bd7fbbda 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -18,7 +18,7 @@ from abc import ABCMeta from . import test_data_generation -class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): +class BignumModTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): #pylint: disable=abstract-method """Target for bignum mod test case generation.""" target_basename = 'test_suite_bignum_mod.generated' diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 2a4886ca9..2e059b26e 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -18,7 +18,7 @@ from abc import ABCMeta from . import test_data_generation -class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): +class BignumModRawTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): #pylint: disable=abstract-method """Target for bignum mod_raw test case generation.""" target_basename = 'test_suite_bignum_mod_raw.generated' From 4cb520ea20840d91e0e04a77c73895150d27ce22 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Thu, 3 Nov 2022 14:46:18 +0000 Subject: [PATCH 185/490] Add merge slots to bignum_common.py Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index c81770ed7..279668fd5 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -124,3 +124,43 @@ class OperationCommon: for b in cls.input_values ) yield from cls.input_cases + +# BEGIN MERGE SLOT 1 + +# END MERGE SLOT 1 + +# BEGIN MERGE SLOT 2 + +# END MERGE SLOT 2 + +# BEGIN MERGE SLOT 3 + +# END MERGE SLOT 3 + +# BEGIN MERGE SLOT 4 + +# END MERGE SLOT 4 + +# BEGIN MERGE SLOT 5 + +# END MERGE SLOT 5 + +# BEGIN MERGE SLOT 6 + +# END MERGE SLOT 6 + +# BEGIN MERGE SLOT 7 + +# END MERGE SLOT 7 + +# BEGIN MERGE SLOT 8 + +# END MERGE SLOT 8 + +# BEGIN MERGE SLOT 9 + +# END MERGE SLOT 9 + +# BEGIN MERGE SLOT 10 + +# END MERGE SLOT 10 From 300186b309c2a8dd354569942e3a888bf16c6fe3 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 3 Nov 2022 17:49:29 +0000 Subject: [PATCH 186/490] Change test templating syntax to be valid C For the benefit of auto-formatting tools, move from the '$placeholder' templating syntax to a new syntax of the form: __MBEDTLS_TEST_TEMPLATE__PLACEHOLDER This change allows the test code template to be almost entirely valid C. Signed-off-by: David Horstmann --- scripts/generate_test_code.py | 75 ++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index f5750aacf..6d65986c8 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -126,33 +126,33 @@ code that is generated or read from helpers and platform files. This script replaces following fields in the template and generates the test source file: -$test_common_helpers <-- All common code from helpers.function - is substituted here. -$functions_code <-- Test functions are substituted here - from the input test_suit_xyz.function - file. C preprocessor checks are generated - for the build dependencies specified - in the input file. This script also - generates wrappers for the test - functions with code to expand the - string parameters read from the data - file. -$expression_code <-- This script enumerates the - expressions in the .data file and - generates code to handle enumerated - expression Ids and return the values. -$dep_check_code <-- This script enumerates all - build dependencies and generate - code to handle enumerated build - dependency Id and return status: if - the dependency is defined or not. -$dispatch_code <-- This script enumerates the functions - specified in the input test data file - and generates the initializer for the - function table in the template - file. -$platform_code <-- Platform specific setup and test - dispatch code. +__MBEDTLS_TEST_TEMPLATE__TEST_COMMON_HELPERS <-- All common code from helpers.function + is substituted here. +__MBEDTLS_TEST_TEMPLATE__FUNCTIONS_CODE <-- Test functions are substituted here + from the input test_suit_xyz.function + file. C preprocessor checks are generated + for the build dependencies specified + in the input file. This script also + generates wrappers for the test + functions with code to expand the + string parameters read from the data + file. +__MBEDTLS_TEST_TEMPLATE__EXPRESSION_CODE <-- This script enumerates the + expressions in the .data file and + generates code to handle enumerated + expression Ids and return the values. +__MBEDTLS_TEST_TEMPLATE__DEP_CHECK_CODE <-- This script enumerates all + build dependencies and generate + code to handle enumerated build + dependency Id and return status: if + the dependency is defined or not. +__MBEDTLS_TEST_TEMPLATE__DISPATCH_CODE <-- This script enumerates the functions + specified in the input test data file + and generates the initializer for the + function table in the template + file. +__MBEDTLS_TEST_TEMPLATE__PLATFORM_CODE <-- Platform specific setup and test + dispatch code. """ @@ -974,11 +974,30 @@ def write_test_source_file(template_file, c_file, snippets): :param snippets: Generated and code snippets :return: """ + + # Create a placeholder pattern with the correct named capture groups + # to override the default provided with Template. + # Match nothing (no way of escaping placeholders). + escaped = "(?P(?!))" + # Match the "__MBEDTLS_TEST_TEMPLATE__PLACEHOLDER_NAME" pattern. + named = "__MBEDTLS_TEST_TEMPLATE__(?P[A-Z][_A-Z0-9]*)" + # Match nothing (no braced placeholder syntax). + braced = "(?P(?!))" + # If not already matched, a "__MBEDTLS_TEST_TEMPLATE__" prefix is invalid. + invalid = "(?P__MBEDTLS_TEST_TEMPLATE__)" + placeholder_pattern = re.compile(escaped \ + + "|" + named \ + + "|" + braced \ + + "|" + invalid) + with open(template_file, 'r') as template_f, open(c_file, 'w') as c_f: for line_no, line in enumerate(template_f.readlines(), 1): # Update line number. +1 as #line directive sets next line number snippets['line_no'] = line_no + 1 - code = string.Template(line).substitute(**snippets) + template = string.Template(line) + template.pattern = placeholder_pattern + snippets = {k.upper():v for (k, v) in snippets.items()} + code = template.substitute(**snippets) c_f.write(code) From 7a2d73679529d4da5942d8454ca1e828c066adf9 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Wed, 9 Nov 2022 17:27:33 +0000 Subject: [PATCH 187/490] Minor improvements to test code script Signed-off-by: David Horstmann --- scripts/generate_test_code.py | 65 ++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 6d65986c8..938f24cf4 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -126,33 +126,39 @@ code that is generated or read from helpers and platform files. This script replaces following fields in the template and generates the test source file: -__MBEDTLS_TEST_TEMPLATE__TEST_COMMON_HELPERS <-- All common code from helpers.function - is substituted here. -__MBEDTLS_TEST_TEMPLATE__FUNCTIONS_CODE <-- Test functions are substituted here - from the input test_suit_xyz.function - file. C preprocessor checks are generated - for the build dependencies specified - in the input file. This script also - generates wrappers for the test - functions with code to expand the - string parameters read from the data - file. -__MBEDTLS_TEST_TEMPLATE__EXPRESSION_CODE <-- This script enumerates the - expressions in the .data file and - generates code to handle enumerated - expression Ids and return the values. -__MBEDTLS_TEST_TEMPLATE__DEP_CHECK_CODE <-- This script enumerates all - build dependencies and generate - code to handle enumerated build - dependency Id and return status: if - the dependency is defined or not. -__MBEDTLS_TEST_TEMPLATE__DISPATCH_CODE <-- This script enumerates the functions - specified in the input test data file - and generates the initializer for the - function table in the template - file. -__MBEDTLS_TEST_TEMPLATE__PLATFORM_CODE <-- Platform specific setup and test - dispatch code. +__MBEDTLS_TEST_TEMPLATE__TEST_COMMON_HELPERS + All common code from helpers.function + is substituted here. +__MBEDTLS_TEST_TEMPLATE__FUNCTIONS_CODE + Test functions are substituted here + from the input test_suit_xyz.function + file. C preprocessor checks are generated + for the build dependencies specified + in the input file. This script also + generates wrappers for the test + functions with code to expand the + string parameters read from the data + file. +__MBEDTLS_TEST_TEMPLATE__EXPRESSION_CODE + This script enumerates the + expressions in the .data file and + generates code to handle enumerated + expression Ids and return the values. +__MBEDTLS_TEST_TEMPLATE__DEP_CHECK_CODE + This script enumerates all + build dependencies and generate + code to handle enumerated build + dependency Id and return status: if + the dependency is defined or not. +__MBEDTLS_TEST_TEMPLATE__DISPATCH_CODE + This script enumerates the functions + specified in the input test data file + and generates the initializer for the + function table in the template + file. +__MBEDTLS_TEST_TEMPLATE__PLATFORM_CODE + Platform specific setup and test + dispatch code. """ @@ -985,10 +991,7 @@ def write_test_source_file(template_file, c_file, snippets): braced = "(?P(?!))" # If not already matched, a "__MBEDTLS_TEST_TEMPLATE__" prefix is invalid. invalid = "(?P__MBEDTLS_TEST_TEMPLATE__)" - placeholder_pattern = re.compile(escaped \ - + "|" + named \ - + "|" + braced \ - + "|" + invalid) + placeholder_pattern = re.compile("|".join([escaped, named, braced, invalid])) with open(template_file, 'r') as template_f, open(c_file, 'w') as c_f: for line_no, line in enumerate(template_f.readlines(), 1): From 0693262c7215761ba13296372df9422297679f67 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 9 Nov 2022 11:46:47 +0000 Subject: [PATCH 188/490] bignum_mod_raw.py: Added BignumModRawOperation This patch is adding a basic instantance of `BignumModRawOperation` and creates an `BignumModRawOperationArchSplit` class, copying over the implementation of `BignumCoreRawOperationArchSplit`. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 38 +++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 2e059b26e..1127ced8d 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -15,8 +15,11 @@ # limitations under the License. from abc import ABCMeta +from typing import Dict, Iterator, List +from . import test_case from . import test_data_generation +from . import bignum_common class BignumModRawTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): #pylint: disable=abstract-method @@ -48,7 +51,42 @@ class BignumModRawTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): # END MERGE SLOT 6 # BEGIN MERGE SLOT 7 +class BignumModRawOperation(bignum_common.OperationCommon, BignumModRawTarget, metaclass=ABCMeta): + #pylint: disable=abstract-method + pass +class BignumModRawOperationArchSplit(BignumModRawOperation): + #pylint: disable=abstract-method + """Common features for bignum core operations where the result depends on + the limb size.""" + + def __init__(self, val_a: str, val_b: str, bits_in_limb: int) -> None: + super().__init__(val_a, val_b) + bound_val = max(self.int_a, self.int_b) + self.bits_in_limb = bits_in_limb + self.bound = bignum_common.bound_mpi(bound_val, self.bits_in_limb) + limbs = bignum_common.limbs_mpi(bound_val, self.bits_in_limb) + byte_len = limbs * self.bits_in_limb // 8 + self.hex_digits = 2 * byte_len + if self.bits_in_limb == 32: + self.dependencies = ["MBEDTLS_HAVE_INT32"] + elif self.bits_in_limb == 64: + self.dependencies = ["MBEDTLS_HAVE_INT64"] + else: + raise ValueError("Invalid number of bits in limb!") + self.arg_a = self.arg_a.zfill(self.hex_digits) + self.arg_b = self.arg_b.zfill(self.hex_digits) + self.arg_a_int = bignum_common.hex_to_int(self.arg_a) + self.arg_b_int = bignum_common.hex_to_int(self.arg_b) + + def pad_to_limbs(self, val) -> str: + return "{:x}".format(val).zfill(self.hex_digits) + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for a_value, b_value in cls.get_value_pairs(): + yield cls(a_value, b_value, 32).create_test_case() + yield cls(a_value, b_value, 64).create_test_case() # END MERGE SLOT 7 # BEGIN MERGE SLOT 8 From b26e55ec7bf6305448a31fea048689fbc09930fc Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 9 Nov 2022 12:36:02 +0000 Subject: [PATCH 189/490] bignum_mod_raw.py: Refactoring `BignumModRawOperation` This patch modifies the BignumModRawOperation class to provide special access to key members commonly used in tests. It binds the module's getters to conversion functions which enable automatic conversions such as: * hex to int. * zero padding hex strings. * common Montgomery constants such as R, R^2 and R^01 are now be calculated upon access. class `BignumModRawOperationArchSplit` is also updated to utilise the new design. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 83 ++++++++++++++++++------- 1 file changed, 59 insertions(+), 24 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 1127ced8d..19942ed16 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -53,40 +53,75 @@ class BignumModRawTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): # BEGIN MERGE SLOT 7 class BignumModRawOperation(bignum_common.OperationCommon, BignumModRawTarget, metaclass=ABCMeta): #pylint: disable=abstract-method - pass + """Target for bignum mod_raw test case generation.""" + + def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: + super().__init__(val_a=val_a, val_b=val_b) + self.val_n = val_n + self.bits_in_limb = bits_in_limb + + @property + def int_n(self) -> int: + return bignum_common.hex_to_int(self.val_n) + + @property + def boundary(self) -> int: + data_in = [self.int_a, self.int_b, self.int_n] + return max([n for n in data_in if n is not None]) + + @property + def limbs(self) -> int: + return bignum_common.limbs_mpi(self.boundary, self.bits_in_limb) + + @property + def hex_digits(self) -> int: + return 2 * (self.limbs * self.bits_in_limb // 8) + + @property + def hex_n(self) -> str: + return "{:x}".format(self.int_n).zfill(self.hex_digits) + + @property + def hex_a(self) -> str: + return "{:x}".format(self.int_a).zfill(self.hex_digits) + + @property + def hex_b(self) -> str: + return "{:x}".format(self.int_b).zfill(self.hex_digits) + + @property + def r(self) -> int: # pylint: disable=invalid-name + l = bignum_common.limbs_mpi(self.int_n, self.bits_in_limb) + return bignum_common.bound_mpi_limbs(l, self.bits_in_limb) + + @property + def r_inv(self) -> int: + return bignum_common.invmod(self.r, self.int_n) + + @property + def r_sqrt(self) -> int: # pylint: disable=invalid-name + return pow(self.r, 2) class BignumModRawOperationArchSplit(BignumModRawOperation): #pylint: disable=abstract-method - """Common features for bignum core operations where the result depends on + """Common features for bignum mod raw operations where the result depends on the limb size.""" - def __init__(self, val_a: str, val_b: str, bits_in_limb: int) -> None: - super().__init__(val_a, val_b) - bound_val = max(self.int_a, self.int_b) - self.bits_in_limb = bits_in_limb - self.bound = bignum_common.bound_mpi(bound_val, self.bits_in_limb) - limbs = bignum_common.limbs_mpi(bound_val, self.bits_in_limb) - byte_len = limbs * self.bits_in_limb // 8 - self.hex_digits = 2 * byte_len - if self.bits_in_limb == 32: - self.dependencies = ["MBEDTLS_HAVE_INT32"] - elif self.bits_in_limb == 64: - self.dependencies = ["MBEDTLS_HAVE_INT64"] - else: - raise ValueError("Invalid number of bits in limb!") - self.arg_a = self.arg_a.zfill(self.hex_digits) - self.arg_b = self.arg_b.zfill(self.hex_digits) - self.arg_a_int = bignum_common.hex_to_int(self.arg_a) - self.arg_b_int = bignum_common.hex_to_int(self.arg_b) + limb_sizes = [32, 64] # type: List[int] - def pad_to_limbs(self, val) -> str: - return "{:x}".format(val).zfill(self.hex_digits) + def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: + super().__init__(val_n=val_n, val_a=val_a, val_b=val_b, bits_in_limb=bits_in_limb) + + if bits_in_limb not in self.limb_sizes: + raise ValueError("Invalid number of bits in limb!") + + self.dependencies = ["MBEDTLS_HAVE_INT{:d}".format(bits_in_limb)] @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: for a_value, b_value in cls.get_value_pairs(): - yield cls(a_value, b_value, 32).create_test_case() - yield cls(a_value, b_value, 64).create_test_case() + for bil in cls.limb_sizes: + yield cls(a_value, b_value, bits_in_limb=bil).create_test_case() # END MERGE SLOT 7 # BEGIN MERGE SLOT 8 From 7f881cfddb07e348d4f6f64e3bea214ba71162ad Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Thu, 10 Nov 2022 11:33:25 +0000 Subject: [PATCH 190/490] bignum_mod_raw.py: Moved Classes outside of slots This patch moves `BignumModRawOperation` and `BignumModRawOperationArchSplit` outside of the scaffolding merge slot. It also renames `r_sqrt` property to `r2`. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 53 +++++++++++++------------ 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 19942ed16..1465e3ed7 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -26,31 +26,6 @@ class BignumModRawTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): """Target for bignum mod_raw test case generation.""" target_basename = 'test_suite_bignum_mod_raw.generated' -# BEGIN MERGE SLOT 1 - -# END MERGE SLOT 1 - -# BEGIN MERGE SLOT 2 - -# END MERGE SLOT 2 - -# BEGIN MERGE SLOT 3 - -# END MERGE SLOT 3 - -# BEGIN MERGE SLOT 4 - -# END MERGE SLOT 4 - -# BEGIN MERGE SLOT 5 - -# END MERGE SLOT 5 - -# BEGIN MERGE SLOT 6 - -# END MERGE SLOT 6 - -# BEGIN MERGE SLOT 7 class BignumModRawOperation(bignum_common.OperationCommon, BignumModRawTarget, metaclass=ABCMeta): #pylint: disable=abstract-method """Target for bignum mod_raw test case generation.""" @@ -99,7 +74,7 @@ class BignumModRawOperation(bignum_common.OperationCommon, BignumModRawTarget, m return bignum_common.invmod(self.r, self.int_n) @property - def r_sqrt(self) -> int: # pylint: disable=invalid-name + def r2(self) -> int: # pylint: disable=invalid-name return pow(self.r, 2) class BignumModRawOperationArchSplit(BignumModRawOperation): @@ -122,6 +97,32 @@ class BignumModRawOperationArchSplit(BignumModRawOperation): for a_value, b_value in cls.get_value_pairs(): for bil in cls.limb_sizes: yield cls(a_value, b_value, bits_in_limb=bil).create_test_case() +# BEGIN MERGE SLOT 1 + +# END MERGE SLOT 1 + +# BEGIN MERGE SLOT 2 + +# END MERGE SLOT 2 + +# BEGIN MERGE SLOT 3 + +# END MERGE SLOT 3 + +# BEGIN MERGE SLOT 4 + +# END MERGE SLOT 4 + +# BEGIN MERGE SLOT 5 + +# END MERGE SLOT 5 + +# BEGIN MERGE SLOT 6 + +# END MERGE SLOT 6 + +# BEGIN MERGE SLOT 7 + # END MERGE SLOT 7 # BEGIN MERGE SLOT 8 From 646b3a393c7905ebcfea161fadfe6f3d1f634a97 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 10 Nov 2022 19:33:25 +0100 Subject: [PATCH 191/490] Remove some Python 2 compatibility code Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 938f24cf4..d994f6bf9 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -220,25 +220,17 @@ class FileWrapper(io.FileIO): :param file_name: File path to open. """ - super(FileWrapper, self).__init__(file_name, 'r') + super().__init__(file_name, 'r') self._line_no = 0 - def next(self): + def __next__(self): """ - Python 2 iterator method. This method overrides base class's - next method and extends the next method to count the line - numbers as each line is read. - - It works for both Python 2 and Python 3 by checking iterator - method name in the base iterator object. + This method overrides base class's __next__ method and extends it + method to count the line numbers as each line is read. :return: Line read from file. """ - parent = super(FileWrapper, self) - if hasattr(parent, '__next__'): - line = parent.__next__() # Python 3 - else: - line = parent.next() # Python 2 # pylint: disable=no-member + line = super().__next__() if line is not None: self._line_no += 1 # Convert byte array to string with correct encoding and @@ -246,9 +238,6 @@ class FileWrapper(io.FileIO): return line.decode(sys.getdefaultencoding()).rstrip() + '\n' return None - # Python 3 iterator method - __next__ = next - def get_line_no(self): """ Gives current line number. From 9e28ccb7df351fb9f09772b00332b2e2dc11c7ef Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 11 Nov 2022 16:36:51 +0100 Subject: [PATCH 192/490] Fix typo and copypasta Signed-off-by: Gilles Peskine --- scripts/test_generate_test_code.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 9bf66f1cc..9796463e4 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -682,12 +682,12 @@ exit: @patch("generate_test_code.gen_dependencies") @patch("generate_test_code.gen_function_wrapper") @patch("generate_test_code.parse_function_arguments") - def test_functio_name_on_newline(self, parse_function_arguments_mock, - gen_function_wrapper_mock, - gen_dependencies_mock, - gen_dispatch_mock): + def test_function_name_on_newline(self, parse_function_arguments_mock, + gen_function_wrapper_mock, + gen_dependencies_mock, + gen_dispatch_mock): """ - Test when exit label is present. + Test with line break before the function name. :return: """ parse_function_arguments_mock.return_value = ([], '', []) From 828cf6cc3f36eea0ca6c3313eee7cc965c5c6442 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 11 Nov 2022 16:37:16 +0100 Subject: [PATCH 193/490] Allow comments in prototypes of unit test functions Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 38 +++++++++++- scripts/test_generate_test_code.py | 96 ++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index d994f6bf9..a1eaa957f 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -519,6 +519,41 @@ def generate_function_code(name, code, local_vars, args_dispatch, gen_dependencies(dependencies) return preprocessor_check_start + code + preprocessor_check_end +COMMENT_START_REGEX = re.compile(r'/[*/]') + +def skip_comments(line, stream): + """Remove comments in line. + + If the line contains an unfinished comment, read more lines from stream + until the line that contains the comment. + + :return: The original line with inner comments replaced by spaces. + Trailing comments and whitespace may be removed completely. + """ + pos = 0 + while True: + opening = COMMENT_START_REGEX.search(line, pos) + if not opening: + break + if line[opening.start(0) + 1] == '/': # //... + continuation = line + while continuation.endswith('\\\n'): + # This errors out if the file ends with an unfinished line + # comment. That's acceptable to not complicat the code further. + continuation = next(stream) + return line[:opening.start(0)].rstrip() + '\n' + # Parsing /*...*/, looking for the end + closing = line.find('*/', opening.end(0)) + while closing == -1: + # This errors out if the file ends with an unfinished block + # comment. That's acceptable to not complicat the code further. + line += next(stream) + closing = line.find('*/', opening.end(0)) + pos = closing + 2 + line = (line[:opening.start(0)] + + ' ' * (pos - opening.start(0)) + + line[pos:]) + return re.sub(r' +(\n|\Z)', r'\1', line) def parse_function_code(funcs_f, dependencies, suite_dependencies): """ @@ -538,6 +573,7 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): # across multiple lines. Here we try to find the start of # arguments list, then remove '\n's and apply the regex to # detect function start. + line = skip_comments(line, funcs_f) up_to_arg_list_start = code + line[:line.find('(') + 1] match = re.match(TEST_FUNCTION_VALIDATION_REGEX, up_to_arg_list_start.replace('\n', ' '), re.I) @@ -546,7 +582,7 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): name = match.group('func_name') if not re.match(FUNCTION_ARG_LIST_END_REGEX, line): for lin in funcs_f: - line += lin + line += skip_comments(lin, funcs_f) if re.search(FUNCTION_ARG_LIST_END_REGEX, line): break args, local_vars, args_dispatch = parse_function_arguments( diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 9796463e4..f36675397 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -724,6 +724,102 @@ exit: yes sir yes sir 3 bags full } +''' + self.assertEqual(code, expected) + + @patch("generate_test_code.gen_dispatch") + @patch("generate_test_code.gen_dependencies") + @patch("generate_test_code.gen_function_wrapper") + @patch("generate_test_code.parse_function_arguments") + def test_case_starting_with_comment(self, parse_function_arguments_mock, + gen_function_wrapper_mock, + gen_dependencies_mock, + gen_dispatch_mock): + """ + Test with comments before the function signature + :return: + """ + parse_function_arguments_mock.return_value = ([], '', []) + gen_function_wrapper_mock.return_value = '' + gen_dependencies_mock.side_effect = gen_dependencies + gen_dispatch_mock.side_effect = gen_dispatch + data = '''/* comment */ +/* more + * comment */ +// this is\ +still \ +a comment +void func() +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} +/* END_CASE */ +''' + stream = StringIOWrapper('test_suite_ut.function', data) + _, _, code, _ = parse_function_code(stream, [], []) + + expected = '''#line 1 "test_suite_ut.function" + + + +void test_func() +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} +''' + self.assertEqual(code, expected) + + @patch("generate_test_code.gen_dispatch") + @patch("generate_test_code.gen_dependencies") + @patch("generate_test_code.gen_function_wrapper") + @patch("generate_test_code.parse_function_arguments") + def test_comment_in_prototype(self, parse_function_arguments_mock, + gen_function_wrapper_mock, + gen_dependencies_mock, + gen_dispatch_mock): + """ + Test with comments in the function prototype + :return: + """ + parse_function_arguments_mock.return_value = ([], '', []) + gen_function_wrapper_mock.return_value = '' + gen_dependencies_mock.side_effect = gen_dependencies + gen_dispatch_mock.side_effect = gen_dispatch + data = ''' +void func( int x, // (line \\ + comment) + int y /* lone closing parenthesis) */ ) +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} +/* END_CASE */ +''' + stream = StringIOWrapper('test_suite_ut.function', data) + _, _, code, _ = parse_function_code(stream, [], []) + + expected = '''#line 1 "test_suite_ut.function" + +void test_func( int x, + int y ) +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} ''' self.assertEqual(code, expected) From d2c8454d0e981044e140ef0bbec9b1a186433544 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 9 Nov 2022 19:23:53 +0000 Subject: [PATCH 194/490] bignum_mod_raw.py: Added BignumModRawConvertToMont This patch adds test class for 'mpi_mod_raw_to_mont_rep()`. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 88 +++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 1465e3ed7..cea7ec78d 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -122,7 +122,95 @@ class BignumModRawOperationArchSplit(BignumModRawOperation): # END MERGE SLOT 6 # BEGIN MERGE SLOT 7 +class BignumModRawConvertToMont(BignumModRawOperationArchSplit): + """ Test cases for mpi_mod_raw_to_mont_rep(). """ + test_function = "mpi_mod_raw_to_mont_rep" + test_name = "Convert into Mont: " + + test_data_moduli = ["b", + "fd", + "eeff99aa37", + "eeff99aa11", + "800000000005", + "7fffffffffffffff", + "80fe000a10000001", + "25a55a46e5da99c71c7", + "1058ad82120c3a10196bb36229c1", + "7e35b84cb19ea5bc57ec37f5e431462fa962d98c1e63738d4657f" + "18ad6532e6adc3eafe67f1e5fa262af94cee8d3e7268593942a2a" + "98df75154f8c914a282f8b", + "8335616aed761f1f7f44e6bd49e807b82e3bf2bf11bfa63", + "ffcece570f2f991013f26dd5b03c4c5b65f97be5905f36cb4664f" + "2c78ff80aa8135a4aaf57ccb8a0aca2f394909a74cef1ef6758a6" + "4d11e2c149c393659d124bfc94196f0ce88f7d7d567efa5a649e2" + "deefaa6e10fdc3deac60d606bf63fc540ac95294347031aefd73d" + "6a9ee10188aaeb7a90d920894553cb196881691cadc51808715a0" + "7e8b24fcb1a63df047c7cdf084dd177ba368c806f3d51ddb5d389" + "8c863e687ecaf7d649a57a46264a582f94d3c8f2edaf59f77a7f6" + "bdaf83c991e8f06abe220ec8507386fce8c3da84c6c3903ab8f3a" + "d4630a204196a7dbcbd9bcca4e40ec5cc5c09938d49f5e1e6181d" + "b8896f33bb12e6ef73f12ec5c5ea7a8a337" + ] + + test_input_numbers = ["0", + "1", + "97", + "f5", + "6f5c3", + "745bfe50f7", + "ffa1f9924123", + "334a8b983c79bd", + "5b84f632b58f3461", + "19acd15bc38008e1", + "ffffffffffffffff", + "54ce6a6bb8247fa0427cfc75a6b0599", + "fecafe8eca052f154ce6a6bb8247fa019558bfeecce9bb9", + "a87d7a56fa4bfdc7da42ef798b9cf6843d4c54794698cb14d72" + "851dec9586a319f4bb6d5695acbd7c92e7a42a5ede6972adcbc" + "f68425265887f2d721f462b7f1b91531bac29fa648facb8e3c6" + "1bd5ae42d5a59ba1c89a95897bfe541a8ce1d633b98f379c481" + "6f25e21f6ac49286b261adb4b78274fe5f61c187581f213e84b" + "2a821e341ef956ecd5de89e6c1a35418cd74a549379d2d4594a" + "577543147f8e35b3514e62cf3e89d1156cdc91ab5f4c928fbd6" + "9148c35df5962fed381f4d8a62852a36823d5425f7487c13a12" + "523473fb823aa9d6ea5f42e794e15f2c1a8785cf6b7d51a4617" + "947fb3baf674f74a673cf1d38126983a19ed52c7439fab42c2185" + ] + + descr_tpl = '{} #{} N: \"{}\" A: \"{}\".' + + def result(self) -> List[str]: + return [self.hex_x] + + def arguments(self) -> List[str]: + return [bignum_common.quote_str(n) for n in [self.hex_n, + self.hex_a, + self.hex_x]] + + def description(self) -> str: + return self.descr_tpl.format(self.test_name, + self.count, + self.int_n, + self.int_a) + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for bil in [32, 64]: + for n in cls.test_data_moduli: + for i in cls.test_input_numbers: + # Skip invalid combinations where A.limbs > N.limbs + if bignum_common.hex_to_int(i) > bignum_common.hex_to_int(n): + continue + yield cls(n, i, bits_in_limb=bil).create_test_case() + + @property + def x(self) -> int: # pylint: disable=invalid-name + return (self.int_a * self.r) % self.int_n + + @property + def hex_x(self) -> str: + return "{:x}".format(self.x).zfill(self.hex_digits) # END MERGE SLOT 7 # BEGIN MERGE SLOT 8 From defdabde74f7b7a2d1097666231912464f4a9c4a Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 9 Nov 2022 19:36:16 +0000 Subject: [PATCH 195/490] bignum_mod_raw.py: Added BignumModRawConvertfromMont This patch adds test class for 'mpi_mod_raw_from_mont_rep()`. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index cea7ec78d..bd694a608 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -211,6 +211,37 @@ class BignumModRawConvertToMont(BignumModRawOperationArchSplit): @property def hex_x(self) -> str: return "{:x}".format(self.x).zfill(self.hex_digits) + +class BignumModRawConvertFromMont(BignumModRawConvertToMont): + """ Test cases for mpi_mod_raw_from_mont_rep(). """ + + test_function = "mpi_mod_raw_from_mont_rep" + test_name = "Convert from Mont: " + + test_input_numbers = ["0", + "1", + "3ca", + "539ed428", + "7dfe5c6beb35a2d6", + "dca8de1c2adfc6d7aafb9b48e", + "a7d17b6c4be72f3d5c16bf9c1af6fc933", + "2fec97beec546f9553142ed52f147845463f579", + "378dc83b8bc5a7b62cba495af4919578dce6d4f175cadc4f", + "b6415f2a1a8e48a518345db11f56db3829c8f2c6415ab4a395a" + "b3ac2ea4cbef4af86eb18a84eb6ded4c6ecbfc4b59c2879a675" + "487f687adea9d197a84a5242a5cf6125ce19a6ad2e7341f1c57" + "d43ea4f4c852a51cb63dabcd1c9de2b827a3146a3d175b35bea" + "41ae75d2a286a3e9d43623152ac513dcdea1d72a7da846a8ab3" + "58d9be4926c79cfb287cf1cf25b689de3b912176be5dcaf4d4c" + "6e7cb839a4a3243a6c47c1e2c99d65c59d6fa3672575c2f1ca8" + "de6a32e854ec9d8ec635c96af7679fce26d7d159e4a9da3bd74" + "e1272c376cd926d74fe3fb164a5935cff3d5cdb92b35fe2cea32" + "138a7e6bfbc319ebd1725dacb9a359cbf693f2ecb785efb9d627" + ] + + @property + def x(self): # pylint: disable=invalid-name + return (self.int_a * self.r_inv) % self.int_n # END MERGE SLOT 7 # BEGIN MERGE SLOT 8 From f4e2f08c9d4ba0ce6d42d936fa9380a8eab6fb50 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 9 Nov 2022 21:57:52 +0100 Subject: [PATCH 196/490] For binary operations, test both x op y and y op x This exposes a bug in mbedtls_mpi_add_mpi() and mbedtls_mpi_sub_mpi() which will be fixed in a subsequent commit. Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_common.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 279668fd5..c2891fc61 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -57,15 +57,8 @@ def limbs_mpi(val: int, bits_in_limb: int) -> int: return (val.bit_length() + bits_in_limb - 1) // bits_in_limb def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: - """Return all pair combinations from input values. - - The return value is cast, as older versions of mypy are unable to derive - the specific type returned by itertools.combinations_with_replacement. - """ - return typing.cast( - List[Tuple[T, T]], - list(itertools.combinations_with_replacement(values, 2)) - ) + """Return all pair combinations from input values.""" + return [(x, y) for x in values for y in values] class OperationCommon: From 78d4b3f8f915d4b29b31d05bb40d7719b7ddbfe7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 15 Nov 2022 20:43:33 +0100 Subject: [PATCH 197/490] Add negative zero as an input to automatically generated tests Although negative zero is officially unsupported, we've had bugs related to it in the past. So do test functions with a negative zero input. There will likely be cases where we don't want to accept negative zero as if it was valid, because it's too hard to handle. We'll add exceptions on a case by case basis. For the functions that are currently tested by the generated tests, the new test cases pass. Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_common.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index c2891fc61..1eb4ca7df 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -38,7 +38,13 @@ def invmod(a: int, n: int) -> int: raise ValueError("Not invertible") def hex_to_int(val: str) -> int: - return int(val, 16) if val else 0 + """Implement the syntax accepted by mbedtls_test_read_mpi(). + + This is a superset of what is accepted by mbedtls_test_read_mpi_core(). + """ + if val == '' or val == '-': + return 0 + return int(val, 16) def quote_str(val) -> str: return "\"{}\"".format(val) From 9cdf97748c479a52554423f8aa68240ef1494616 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 10 Nov 2022 09:15:21 +0100 Subject: [PATCH 198/490] Pacify pylint Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_common.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 1eb4ca7df..8b11bc283 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -14,9 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import itertools -import typing - from abc import abstractmethod from typing import Iterator, List, Tuple, TypeVar @@ -42,7 +39,7 @@ def hex_to_int(val: str) -> int: This is a superset of what is accepted by mbedtls_test_read_mpi_core(). """ - if val == '' or val == '-': + if val in ['', '-']: return 0 return int(val, 16) From ff4538c50182f29d554f9a3f49f502fcd662f040 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 18 Nov 2022 22:24:56 +0100 Subject: [PATCH 199/490] Fix intended backslash in test data Signed-off-by: Gilles Peskine --- scripts/test_generate_test_code.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index f36675397..ae1aa2af4 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -746,8 +746,8 @@ exit: data = '''/* comment */ /* more * comment */ -// this is\ -still \ +// this is\\ +still \\ a comment void func() { From 65641aa3945991eb186f37a1705d1e3d16572f9a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 18 Nov 2022 22:25:18 +0100 Subject: [PATCH 200/490] Add test cases for comment nesting Add a test case that would fail if all line comments were parsed before block comments, and a test case that would fail if all block comments were parsed before line comments. Signed-off-by: Gilles Peskine --- scripts/test_generate_test_code.py | 88 ++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index ae1aa2af4..d8b8cf979 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -820,6 +820,94 @@ exit: yes sir yes sir 3 bags full } +''' + self.assertEqual(code, expected) + + @patch("generate_test_code.gen_dispatch") + @patch("generate_test_code.gen_dependencies") + @patch("generate_test_code.gen_function_wrapper") + @patch("generate_test_code.parse_function_arguments") + def test_line_comment_in_block_comment(self, parse_function_arguments_mock, + gen_function_wrapper_mock, + gen_dependencies_mock, + gen_dispatch_mock): + """ + Test with line comment in block comment. + :return: + """ + parse_function_arguments_mock.return_value = ([], '', []) + gen_function_wrapper_mock.return_value = '' + gen_dependencies_mock.side_effect = gen_dependencies + gen_dispatch_mock.side_effect = gen_dispatch + data = ''' +void func( int x /* // */ ) +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} +/* END_CASE */ +''' + stream = StringIOWrapper('test_suite_ut.function', data) + _, _, code, _ = parse_function_code(stream, [], []) + + expected = '''#line 1 "test_suite_ut.function" + +void test_func( int x ) +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} +''' + self.assertEqual(code, expected) + + @patch("generate_test_code.gen_dispatch") + @patch("generate_test_code.gen_dependencies") + @patch("generate_test_code.gen_function_wrapper") + @patch("generate_test_code.parse_function_arguments") + def test_block_comment_in_line_comment(self, parse_function_arguments_mock, + gen_function_wrapper_mock, + gen_dependencies_mock, + gen_dispatch_mock): + """ + Test with block comment in line comment. + :return: + """ + parse_function_arguments_mock.return_value = ([], '', []) + gen_function_wrapper_mock.return_value = '' + gen_dependencies_mock.side_effect = gen_dependencies + gen_dispatch_mock.side_effect = gen_dispatch + data = ''' +// /* +void func( int x ) +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} +/* END_CASE */ +''' + stream = StringIOWrapper('test_suite_ut.function', data) + _, _, code, _ = parse_function_code(stream, [], []) + + expected = '''#line 1 "test_suite_ut.function" + + +void test_func( int x ) +{ + ba ba black sheep + have you any wool +exit: + yes sir yes sir + 3 bags full +} ''' self.assertEqual(code, expected) From 659ee85e78247bdd8b914c9237f9a1af32ccc113 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 18 Nov 2022 22:26:03 +0100 Subject: [PATCH 201/490] Typos in comments Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index a1eaa957f..e70fffb93 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -539,14 +539,14 @@ def skip_comments(line, stream): continuation = line while continuation.endswith('\\\n'): # This errors out if the file ends with an unfinished line - # comment. That's acceptable to not complicat the code further. + # comment. That's acceptable to not complicate the code further. continuation = next(stream) return line[:opening.start(0)].rstrip() + '\n' # Parsing /*...*/, looking for the end closing = line.find('*/', opening.end(0)) while closing == -1: # This errors out if the file ends with an unfinished block - # comment. That's acceptable to not complicat the code further. + # comment. That's acceptable to not complicate the code further. line += next(stream) closing = line.find('*/', opening.end(0)) pos = closing + 2 From 1a012eaa583a97f3dede60c44734b5254fd4749f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 18 Nov 2022 22:27:37 +0100 Subject: [PATCH 202/490] Explain space preservation Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index e70fffb93..c1d1c1cbf 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -550,6 +550,13 @@ def skip_comments(line, stream): line += next(stream) closing = line.find('*/', opening.end(0)) pos = closing + 2 + # Replace inner comment by spaces. There needs to be at least one space + # for things like 'int/*ihatespaces*/foo'. Go further and preserve the + # width of the comment, this way column positions in error messages + # remain correct. + # TODO: It would be better to preserve line breaks, to get accurate + # line numbers if there's something interesting after a comment on + # the same line. line = (line[:opening.start(0)] + ' ' * (pos - opening.start(0)) + line[pos:]) From 48b93d77f4f5590eb4abac5088d46ea65c9e9443 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 9 Nov 2022 12:14:14 +0000 Subject: [PATCH 203/490] Split test generator base class The class BaseTarget served two purposes: - track test cases and target files for generation - provide an abstract base class for individual test groups Splitting these allows decoupling these two and to have further common superclasses across targets. No intended change in generated test cases. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 5 ++-- scripts/framework_dev/bignum_core.py | 14 +++++----- scripts/framework_dev/bignum_mod.py | 6 ++-- scripts/framework_dev/bignum_mod_raw.py | 4 +-- scripts/framework_dev/test_data_generation.py | 28 ++++++++++++------- 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 8b11bc283..ba30be40e 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -17,6 +17,8 @@ from abc import abstractmethod from typing import Iterator, List, Tuple, TypeVar +from . import test_data_generation + T = TypeVar('T') #pylint: disable=invalid-name def invmod(a: int, n: int) -> int: @@ -63,8 +65,7 @@ def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: """Return all pair combinations from input values.""" return [(x, y) for x in values for y in values] - -class OperationCommon: +class OperationCommon(test_data_generation.BaseTest): """Common features for bignum binary operations. This adds functionality common in binary operation tests. diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 0cc86b809..db9d1b7ca 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -16,20 +16,19 @@ import random -from abc import ABCMeta from typing import Dict, Iterator, List, Tuple from . import test_case from . import test_data_generation from . import bignum_common -class BignumCoreTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): - #pylint: disable=abstract-method +class BignumCoreTarget(test_data_generation.BaseTarget): + #pylint: disable=abstract-method, too-few-public-methods """Target for bignum core test case generation.""" target_basename = 'test_suite_bignum_core.generated' -class BignumCoreShiftR(BignumCoreTarget, metaclass=ABCMeta): +class BignumCoreShiftR(BignumCoreTarget, test_data_generation.BaseTest): """Test cases for mbedtls_bignum_core_shift_r().""" count = 0 test_function = "mpi_core_shift_r" @@ -69,7 +68,7 @@ class BignumCoreShiftR(BignumCoreTarget, metaclass=ABCMeta): for count in counts: yield cls(input_hex, descr, count).create_test_case() -class BignumCoreCTLookup(BignumCoreTarget, metaclass=ABCMeta): +class BignumCoreCTLookup(BignumCoreTarget, test_data_generation.BaseTest): """Test cases for mbedtls_mpi_core_ct_uint_table_lookup().""" test_function = "mpi_core_ct_uint_table_lookup" test_name = "Constant time MPI table lookup" @@ -107,7 +106,8 @@ class BignumCoreCTLookup(BignumCoreTarget, metaclass=ABCMeta): yield (cls(bitsize, bitsize_description, window_size) .create_test_case()) -class BignumCoreOperation(bignum_common.OperationCommon, BignumCoreTarget, metaclass=ABCMeta): +class BignumCoreOperation(BignumCoreTarget, bignum_common.OperationCommon, + metaclass=ABCMeta): #pylint: disable=abstract-method """Common features for bignum core operations.""" input_values = [ @@ -297,7 +297,7 @@ class BignumCoreMLA(BignumCoreOperation): yield cur_op.create_test_case() -class BignumCoreMontmul(BignumCoreTarget): +class BignumCoreMontmul(BignumCoreTarget, test_data_generation.BaseTest): """Test cases for Montgomery multiplication.""" count = 0 test_function = "mpi_core_montmul" diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 2bd7fbbda..a604cc0c5 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -14,12 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from abc import ABCMeta - from . import test_data_generation -class BignumModTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): - #pylint: disable=abstract-method +class BignumModTarget(test_data_generation.BaseTarget): + #pylint: disable=abstract-method, too-few-public-methods """Target for bignum mod test case generation.""" target_basename = 'test_suite_bignum_mod.generated' diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index bd694a608..4f12d9a86 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -21,8 +21,8 @@ from . import test_case from . import test_data_generation from . import bignum_common -class BignumModRawTarget(test_data_generation.BaseTarget, metaclass=ABCMeta): - #pylint: disable=abstract-method +class BignumModRawTarget(test_data_generation.BaseTarget): + #pylint: disable=abstract-method, too-few-public-methods """Target for bignum mod_raw test case generation.""" target_basename = 'test_suite_bignum_mod_raw.generated' diff --git a/scripts/framework_dev/test_data_generation.py b/scripts/framework_dev/test_data_generation.py index eec0f9d97..3d703eec7 100644 --- a/scripts/framework_dev/test_data_generation.py +++ b/scripts/framework_dev/test_data_generation.py @@ -25,6 +25,7 @@ import argparse import os import posixpath import re +import inspect from abc import ABCMeta, abstractmethod from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar @@ -35,12 +36,8 @@ from . import test_case T = TypeVar('T') #pylint: disable=invalid-name -class BaseTarget(metaclass=ABCMeta): - """Base target for test case generation. - - Child classes of this class represent an output file, and can be referred - to as file targets. These indicate where test cases will be written to for - all subclasses of the file target, which is set by `target_basename`. +class BaseTest(metaclass=ABCMeta): + """Base class for test case generation. Attributes: count: Counter for test cases from this class. @@ -48,8 +45,6 @@ class BaseTarget(metaclass=ABCMeta): automatically generated using the class, or manually set. dependencies: A list of dependencies required for the test case. show_test_count: Toggle for inclusion of `count` in the test description. - target_basename: Basename of file to write generated tests to. This - should be specified in a child class of BaseTarget. test_function: Test function which the class generates cases for. test_name: A common name or description of the test function. This can be `test_function`, a clearer equivalent, or a short summary of the @@ -59,7 +54,6 @@ class BaseTarget(metaclass=ABCMeta): case_description = "" dependencies = [] # type: List[str] show_test_count = True - target_basename = "" test_function = "" test_name = "" @@ -121,6 +115,20 @@ class BaseTarget(metaclass=ABCMeta): """ raise NotImplementedError + +class BaseTarget: + """Base target for test case generation. + + Child classes of this class represent an output file, and can be referred + to as file targets. These indicate where test cases will be written to for + all subclasses of the file target, which is set by `target_basename`. + + Attributes: + target_basename: Basename of file to write generated tests to. This + should be specified in a child class of BaseTarget. + """ + target_basename = "" + @classmethod def generate_tests(cls) -> Iterator[test_case.TestCase]: """Generate test cases for the class and its subclasses. @@ -132,7 +140,7 @@ class BaseTarget(metaclass=ABCMeta): yield from `generate_tests()` in each. Calling this method on a class X will yield test cases from all classes derived from X. """ - if cls.test_function: + if issubclass(cls, BaseTest) and not inspect.isabstract(cls): yield from cls.generate_function_tests() for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() From 94ccc5f903002d04254cf412c6081d97abf9e1e9 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 9 Nov 2022 12:31:23 +0000 Subject: [PATCH 204/490] Bignum test: Move identical function to superclass No intended change in generated test cases. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 6 ++++++ scripts/framework_dev/bignum_core.py | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index ba30be40e..02241141f 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -17,6 +17,7 @@ from abc import abstractmethod from typing import Iterator, List, Tuple, TypeVar +from . import test_case from . import test_data_generation T = TypeVar('T') #pylint: disable=invalid-name @@ -122,6 +123,11 @@ class OperationCommon(test_data_generation.BaseTest): ) yield from cls.input_cases + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for a_value, b_value in cls.get_value_pairs(): + yield cls(a_value, b_value).create_test_case() + # BEGIN MERGE SLOT 1 # END MERGE SLOT 1 diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index db9d1b7ca..a1c2e1bc6 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -144,11 +144,6 @@ class BignumCoreOperation(BignumCoreTarget, bignum_common.OperationCommon, ) return super().description() - @classmethod - def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - for a_value, b_value in cls.get_value_pairs(): - yield cls(a_value, b_value).create_test_case() - class BignumCoreOperationArchSplit(BignumCoreOperation): #pylint: disable=abstract-method From 53887a7f805cac8e1e96c3c8a106c79d0adf8b0b Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 9 Nov 2022 13:24:46 +0000 Subject: [PATCH 205/490] Bignum test: move archsplit to superclass We need arch split tests in different modules, moving it to the common module makes it reusable. No intended changes in the generated tests. (The position of the core_add_if tests changed, but they are still all there.) Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 46 ++++++++++++++ scripts/framework_dev/bignum_core.py | 88 ++++++++------------------ 2 files changed, 73 insertions(+), 61 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 02241141f..7ab788be0 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -97,6 +97,19 @@ class OperationCommon(test_data_generation.BaseTest): quote_str(self.arg_a), quote_str(self.arg_b) ] + self.result() + def description(self) -> str: + """Generate a description for the test case. + + If not set, case_description uses the form A `symbol` B, where symbol + is used to represent the operation. Descriptions of each value are + generated to provide some context to the test case. + """ + if not self.case_description: + self.case_description = "{:x} {} {:x}".format( + self.int_a, self.symbol, self.int_b + ) + return super().description() + @abstractmethod def result(self) -> List[str]: """Get the result of the operation. @@ -128,6 +141,39 @@ class OperationCommon(test_data_generation.BaseTest): for a_value, b_value in cls.get_value_pairs(): yield cls(a_value, b_value).create_test_case() + +class OperationCommonArchSplit(OperationCommon): + #pylint: disable=abstract-method + """Common features for operations where the result depends on + the limb size.""" + + def __init__(self, val_a: str, val_b: str, bits_in_limb: int) -> None: + super().__init__(val_a, val_b) + bound_val = max(self.int_a, self.int_b) + self.bits_in_limb = bits_in_limb + self.bound = bound_mpi(bound_val, self.bits_in_limb) + limbs = limbs_mpi(bound_val, self.bits_in_limb) + byte_len = limbs * self.bits_in_limb // 8 + self.hex_digits = 2 * byte_len + if self.bits_in_limb == 32: + self.dependencies = ["MBEDTLS_HAVE_INT32"] + elif self.bits_in_limb == 64: + self.dependencies = ["MBEDTLS_HAVE_INT64"] + else: + raise ValueError("Invalid number of bits in limb!") + self.arg_a = self.arg_a.zfill(self.hex_digits) + self.arg_b = self.arg_b.zfill(self.hex_digits) + + def pad_to_limbs(self, val) -> str: + return "{:x}".format(val).zfill(self.hex_digits) + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for a_value, b_value in cls.get_value_pairs(): + yield cls(a_value, b_value, 32).create_test_case() + yield cls(a_value, b_value, 64).create_test_case() + + # BEGIN MERGE SLOT 1 # END MERGE SLOT 1 diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index a1c2e1bc6..591e53c20 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -106,75 +106,41 @@ class BignumCoreCTLookup(BignumCoreTarget, test_data_generation.BaseTest): yield (cls(bitsize, bitsize_description, window_size) .create_test_case()) -class BignumCoreOperation(BignumCoreTarget, bignum_common.OperationCommon, - metaclass=ABCMeta): +INPUT_VALUES = [ + "0", "1", "3", "f", "fe", "ff", "100", "ff00", "fffe", "ffff", "10000", + "fffffffe", "ffffffff", "100000000", "1f7f7f7f7f7f7f", + "8000000000000000", "fefefefefefefefe", "fffffffffffffffe", + "ffffffffffffffff", "10000000000000000", "1234567890abcdef0", + "fffffffffffffffffefefefefefefefe", "fffffffffffffffffffffffffffffffe", + "ffffffffffffffffffffffffffffffff", "100000000000000000000000000000000", + "1234567890abcdef01234567890abcdef0", + "fffffffffffffffffffffffffffffffffffffffffffffffffefefefefefefefe", + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "10000000000000000000000000000000000000000000000000000000000000000", + "1234567890abcdef01234567890abcdef01234567890abcdef01234567890abcdef0", + ( + "4df72d07b4b71c8dacb6cffa954f8d88254b6277099308baf003fab73227f34029" + "643b5a263f66e0d3c3fa297ef71755efd53b8fb6cb812c6bbf7bcf179298bd9947" + "c4c8b14324140a2c0f5fad7958a69050a987a6096e9f055fb38edf0c5889eca4a0" + "cfa99b45fbdeee4c696b328ddceae4723945901ec025076b12b" + ) +] + + +class BignumCoreOperation(BignumCoreTarget, bignum_common.OperationCommon): #pylint: disable=abstract-method """Common features for bignum core operations.""" - input_values = [ - "0", "1", "3", "f", "fe", "ff", "100", "ff00", "fffe", "ffff", "10000", - "fffffffe", "ffffffff", "100000000", "1f7f7f7f7f7f7f", - "8000000000000000", "fefefefefefefefe", "fffffffffffffffe", - "ffffffffffffffff", "10000000000000000", "1234567890abcdef0", - "fffffffffffffffffefefefefefefefe", "fffffffffffffffffffffffffffffffe", - "ffffffffffffffffffffffffffffffff", "100000000000000000000000000000000", - "1234567890abcdef01234567890abcdef0", - "fffffffffffffffffffffffffffffffffffffffffffffffffefefefefefefefe", - "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe", - "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "10000000000000000000000000000000000000000000000000000000000000000", - "1234567890abcdef01234567890abcdef01234567890abcdef01234567890abcdef0", - ( - "4df72d07b4b71c8dacb6cffa954f8d88254b6277099308baf003fab73227f34029" - "643b5a263f66e0d3c3fa297ef71755efd53b8fb6cb812c6bbf7bcf179298bd9947" - "c4c8b14324140a2c0f5fad7958a69050a987a6096e9f055fb38edf0c5889eca4a0" - "cfa99b45fbdeee4c696b328ddceae4723945901ec025076b12b" - ) - ] - - def description(self) -> str: - """Generate a description for the test case. - - If not set, case_description uses the form A `symbol` B, where symbol - is used to represent the operation. Descriptions of each value are - generated to provide some context to the test case. - """ - if not self.case_description: - self.case_description = "{:x} {} {:x}".format( - self.int_a, self.symbol, self.int_b - ) - return super().description() + input_values = INPUT_VALUES -class BignumCoreOperationArchSplit(BignumCoreOperation): +class BignumCoreOperationArchSplit(BignumCoreTarget, + bignum_common.OperationCommonArchSplit): #pylint: disable=abstract-method """Common features for bignum core operations where the result depends on the limb size.""" + input_values = INPUT_VALUES - def __init__(self, val_a: str, val_b: str, bits_in_limb: int) -> None: - super().__init__(val_a, val_b) - bound_val = max(self.int_a, self.int_b) - self.bits_in_limb = bits_in_limb - self.bound = bignum_common.bound_mpi(bound_val, self.bits_in_limb) - limbs = bignum_common.limbs_mpi(bound_val, self.bits_in_limb) - byte_len = limbs * self.bits_in_limb // 8 - self.hex_digits = 2 * byte_len - if self.bits_in_limb == 32: - self.dependencies = ["MBEDTLS_HAVE_INT32"] - elif self.bits_in_limb == 64: - self.dependencies = ["MBEDTLS_HAVE_INT64"] - else: - raise ValueError("Invalid number of bits in limb!") - self.arg_a = self.arg_a.zfill(self.hex_digits) - self.arg_b = self.arg_b.zfill(self.hex_digits) - - def pad_to_limbs(self, val) -> str: - return "{:x}".format(val).zfill(self.hex_digits) - - @classmethod - def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - for a_value, b_value in cls.get_value_pairs(): - yield cls(a_value, b_value, 32).create_test_case() - yield cls(a_value, b_value, 64).create_test_case() class BignumCoreAddAndAddIf(BignumCoreOperationArchSplit): """Test cases for bignum core add and add-if.""" From f27c617795b592f93fb618c61518a59a3c48e6ac Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Wed, 9 Nov 2022 16:04:41 +0000 Subject: [PATCH 206/490] Make pylint happy Signed-off-by: Janos Follath --- scripts/framework_dev/test_data_generation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/framework_dev/test_data_generation.py b/scripts/framework_dev/test_data_generation.py index 3d703eec7..02aa51051 100644 --- a/scripts/framework_dev/test_data_generation.py +++ b/scripts/framework_dev/test_data_generation.py @@ -117,6 +117,7 @@ class BaseTest(metaclass=ABCMeta): class BaseTarget: + #pylint: disable=too-few-public-methods """Base target for test case generation. Child classes of this class represent an output file, and can be referred @@ -141,6 +142,7 @@ class BaseTarget: will yield test cases from all classes derived from X. """ if issubclass(cls, BaseTest) and not inspect.isabstract(cls): + #pylint: disable=no-member yield from cls.generate_function_tests() for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() From ba8301fef7a0a19ee1ed55e367f169d92219b9ac Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Thu, 17 Nov 2022 13:32:43 +0000 Subject: [PATCH 207/490] Bignum Tests: Move ModOperation to common The class BignumModRawOperation implements functionality that are needed in other modules, therefore we move it to common. No intended changes to test cases. The order of add_and_add_if and sub tests have been switched. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 52 +++++++++++++++++++++++ scripts/framework_dev/bignum_mod_raw.py | 55 +------------------------ 2 files changed, 54 insertions(+), 53 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 7ab788be0..28e27b039 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -142,6 +142,58 @@ class OperationCommon(test_data_generation.BaseTest): yield cls(a_value, b_value).create_test_case() +class ModOperationCommon(OperationCommon): + #pylint: disable=abstract-method + """Target for bignum mod_raw test case generation.""" + + def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: + super().__init__(val_a=val_a, val_b=val_b) + self.val_n = val_n + self.bits_in_limb = bits_in_limb + + @property + def int_n(self) -> int: + return hex_to_int(self.val_n) + + @property + def boundary(self) -> int: + data_in = [self.int_a, self.int_b, self.int_n] + return max([n for n in data_in if n is not None]) + + @property + def limbs(self) -> int: + return limbs_mpi(self.boundary, self.bits_in_limb) + + @property + def hex_digits(self) -> int: + return 2 * (self.limbs * self.bits_in_limb // 8) + + @property + def hex_n(self) -> str: + return "{:x}".format(self.int_n).zfill(self.hex_digits) + + @property + def hex_a(self) -> str: + return "{:x}".format(self.int_a).zfill(self.hex_digits) + + @property + def hex_b(self) -> str: + return "{:x}".format(self.int_b).zfill(self.hex_digits) + + @property + def r(self) -> int: # pylint: disable=invalid-name + l = limbs_mpi(self.int_n, self.bits_in_limb) + return bound_mpi_limbs(l, self.bits_in_limb) + + @property + def r_inv(self) -> int: + return invmod(self.r, self.int_n) + + @property + def r2(self) -> int: # pylint: disable=invalid-name + return pow(self.r, 2) + + class OperationCommonArchSplit(OperationCommon): #pylint: disable=abstract-method """Common features for operations where the result depends on diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 4f12d9a86..884e2ef4a 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from abc import ABCMeta from typing import Dict, Iterator, List from . import test_case @@ -26,58 +25,8 @@ class BignumModRawTarget(test_data_generation.BaseTarget): """Target for bignum mod_raw test case generation.""" target_basename = 'test_suite_bignum_mod_raw.generated' -class BignumModRawOperation(bignum_common.OperationCommon, BignumModRawTarget, metaclass=ABCMeta): - #pylint: disable=abstract-method - """Target for bignum mod_raw test case generation.""" - - def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: - super().__init__(val_a=val_a, val_b=val_b) - self.val_n = val_n - self.bits_in_limb = bits_in_limb - - @property - def int_n(self) -> int: - return bignum_common.hex_to_int(self.val_n) - - @property - def boundary(self) -> int: - data_in = [self.int_a, self.int_b, self.int_n] - return max([n for n in data_in if n is not None]) - - @property - def limbs(self) -> int: - return bignum_common.limbs_mpi(self.boundary, self.bits_in_limb) - - @property - def hex_digits(self) -> int: - return 2 * (self.limbs * self.bits_in_limb // 8) - - @property - def hex_n(self) -> str: - return "{:x}".format(self.int_n).zfill(self.hex_digits) - - @property - def hex_a(self) -> str: - return "{:x}".format(self.int_a).zfill(self.hex_digits) - - @property - def hex_b(self) -> str: - return "{:x}".format(self.int_b).zfill(self.hex_digits) - - @property - def r(self) -> int: # pylint: disable=invalid-name - l = bignum_common.limbs_mpi(self.int_n, self.bits_in_limb) - return bignum_common.bound_mpi_limbs(l, self.bits_in_limb) - - @property - def r_inv(self) -> int: - return bignum_common.invmod(self.r, self.int_n) - - @property - def r2(self) -> int: # pylint: disable=invalid-name - return pow(self.r, 2) - -class BignumModRawOperationArchSplit(BignumModRawOperation): +class BignumModRawOperationArchSplit(bignum_common.ModOperationCommon, + BignumModRawTarget): #pylint: disable=abstract-method """Common features for bignum mod raw operations where the result depends on the limb size.""" From 4a3dfb8b0038348e39e77c5083a1871a59ebe07f Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Thu, 17 Nov 2022 13:38:56 +0000 Subject: [PATCH 208/490] Bignum Tests: move ModOperationArchSplit to common The class BignumModRawOperationArchSplit has functionality that are needed in other modules, therefore moving it to bignum_common. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 22 ++++++++++++++++++++++ scripts/framework_dev/bignum_mod_raw.py | 24 ++---------------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 28e27b039..b853d1136 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -226,6 +226,28 @@ class OperationCommonArchSplit(OperationCommon): yield cls(a_value, b_value, 64).create_test_case() +class ModOperationCommonArchSplit(ModOperationCommon): + #pylint: disable=abstract-method + """Common features for bignum mod raw operations where the result depends on + the limb size.""" + + limb_sizes = [32, 64] # type: List[int] + + def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: + super().__init__(val_n=val_n, val_a=val_a, val_b=val_b, bits_in_limb=bits_in_limb) + + if bits_in_limb not in self.limb_sizes: + raise ValueError("Invalid number of bits in limb!") + + self.dependencies = ["MBEDTLS_HAVE_INT{:d}".format(bits_in_limb)] + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for a_value, b_value in cls.get_value_pairs(): + for bil in cls.limb_sizes: + yield cls(a_value, b_value, bits_in_limb=bil).create_test_case() + + # BEGIN MERGE SLOT 1 # END MERGE SLOT 1 diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 884e2ef4a..58a93fc5d 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -25,27 +25,6 @@ class BignumModRawTarget(test_data_generation.BaseTarget): """Target for bignum mod_raw test case generation.""" target_basename = 'test_suite_bignum_mod_raw.generated' -class BignumModRawOperationArchSplit(bignum_common.ModOperationCommon, - BignumModRawTarget): - #pylint: disable=abstract-method - """Common features for bignum mod raw operations where the result depends on - the limb size.""" - - limb_sizes = [32, 64] # type: List[int] - - def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: - super().__init__(val_n=val_n, val_a=val_a, val_b=val_b, bits_in_limb=bits_in_limb) - - if bits_in_limb not in self.limb_sizes: - raise ValueError("Invalid number of bits in limb!") - - self.dependencies = ["MBEDTLS_HAVE_INT{:d}".format(bits_in_limb)] - - @classmethod - def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - for a_value, b_value in cls.get_value_pairs(): - for bil in cls.limb_sizes: - yield cls(a_value, b_value, bits_in_limb=bil).create_test_case() # BEGIN MERGE SLOT 1 # END MERGE SLOT 1 @@ -71,7 +50,8 @@ class BignumModRawOperationArchSplit(bignum_common.ModOperationCommon, # END MERGE SLOT 6 # BEGIN MERGE SLOT 7 -class BignumModRawConvertToMont(BignumModRawOperationArchSplit): +class BignumModRawConvertToMont(bignum_common.ModOperationCommonArchSplit, + BignumModRawTarget): """ Test cases for mpi_mod_raw_to_mont_rep(). """ test_function = "mpi_mod_raw_to_mont_rep" From 63795ab8ac48a18b76558cc6de4960e0c43e2fb1 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Thu, 17 Nov 2022 14:42:40 +0000 Subject: [PATCH 209/490] Bignum Tests: remove ModOperationCommonArchSplit The functionality of ModOperationCommonArchSplit is needed in several subclasses, therefore moving it to a superclass. There is another, redundant ArchSplit class, which will be removed in a later commit. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 49 +++++++++++-------------- scripts/framework_dev/bignum_mod_raw.py | 3 +- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index b853d1136..cbbbf9f67 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -80,17 +80,29 @@ class OperationCommon(test_data_generation.BaseTest): unique_combinations_only: Boolean to select if test case combinations must be unique. If True, only A,B or B,A would be included as a test case. If False, both A,B and B,A would be included. + arch_split: Boolean to select if different test cases are needed + depending on the architecture/limb size. This will cause test + objects being generated with different architectures. Individual + test objects can tell their architecture by accessing the + bits_in_limb instance variable. """ symbol = "" input_values = [] # type: List[str] input_cases = [] # type: List[Tuple[str, str]] unique_combinations_only = True + arch_split = False + limb_sizes = [32, 64] # type: List[int] - def __init__(self, val_a: str, val_b: str) -> None: + def __init__(self, val_a: str, val_b: str, bits_in_limb: int = 64) -> None: self.arg_a = val_a self.arg_b = val_b self.int_a = hex_to_int(val_a) self.int_b = hex_to_int(val_b) + if bits_in_limb not in self.limb_sizes: + raise ValueError("Invalid number of bits in limb!") + if self.arch_split: + self.dependencies = ["MBEDTLS_HAVE_INT{:d}".format(bits_in_limb)] + self.bits_in_limb = bits_in_limb def arguments(self) -> List[str]: return [ @@ -139,17 +151,22 @@ class OperationCommon(test_data_generation.BaseTest): @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: for a_value, b_value in cls.get_value_pairs(): - yield cls(a_value, b_value).create_test_case() + if cls.arch_split: + for bil in cls.limb_sizes: + yield cls(a_value, b_value, + bits_in_limb=bil).create_test_case() + else: + yield cls(a_value, b_value).create_test_case() class ModOperationCommon(OperationCommon): #pylint: disable=abstract-method """Target for bignum mod_raw test case generation.""" - def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: - super().__init__(val_a=val_a, val_b=val_b) + def __init__(self, val_n: str, val_a: str, val_b: str = "0", + bits_in_limb: int = 64) -> None: + super().__init__(val_a=val_a, val_b=val_b, bits_in_limb=bits_in_limb) self.val_n = val_n - self.bits_in_limb = bits_in_limb @property def int_n(self) -> int: @@ -226,28 +243,6 @@ class OperationCommonArchSplit(OperationCommon): yield cls(a_value, b_value, 64).create_test_case() -class ModOperationCommonArchSplit(ModOperationCommon): - #pylint: disable=abstract-method - """Common features for bignum mod raw operations where the result depends on - the limb size.""" - - limb_sizes = [32, 64] # type: List[int] - - def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: - super().__init__(val_n=val_n, val_a=val_a, val_b=val_b, bits_in_limb=bits_in_limb) - - if bits_in_limb not in self.limb_sizes: - raise ValueError("Invalid number of bits in limb!") - - self.dependencies = ["MBEDTLS_HAVE_INT{:d}".format(bits_in_limb)] - - @classmethod - def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - for a_value, b_value in cls.get_value_pairs(): - for bil in cls.limb_sizes: - yield cls(a_value, b_value, bits_in_limb=bil).create_test_case() - - # BEGIN MERGE SLOT 1 # END MERGE SLOT 1 diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 58a93fc5d..f44acef73 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -50,12 +50,13 @@ class BignumModRawTarget(test_data_generation.BaseTarget): # END MERGE SLOT 6 # BEGIN MERGE SLOT 7 -class BignumModRawConvertToMont(bignum_common.ModOperationCommonArchSplit, +class BignumModRawConvertToMont(bignum_common.ModOperationCommon, BignumModRawTarget): """ Test cases for mpi_mod_raw_to_mont_rep(). """ test_function = "mpi_mod_raw_to_mont_rep" test_name = "Convert into Mont: " + arch_split = True test_data_moduli = ["b", "fd", From c0e5d323718f2729da055babd026090d9bbb81ed Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Thu, 17 Nov 2022 15:13:02 +0000 Subject: [PATCH 210/490] Bignum Tests: move properties to superclass Move properties that are needed in several children to the superclass. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 40 ++++++++++++++------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index cbbbf9f67..7d52749f8 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -104,6 +104,27 @@ class OperationCommon(test_data_generation.BaseTest): self.dependencies = ["MBEDTLS_HAVE_INT{:d}".format(bits_in_limb)] self.bits_in_limb = bits_in_limb + @property + def boundary(self) -> int: + data_in = [self.int_a, self.int_b] + return max([n for n in data_in if n is not None]) + + @property + def limbs(self) -> int: + return limbs_mpi(self.boundary, self.bits_in_limb) + + @property + def hex_digits(self) -> int: + return 2 * (self.limbs * self.bits_in_limb // 8) + + @property + def hex_a(self) -> str: + return "{:x}".format(self.int_a).zfill(self.hex_digits) + + @property + def hex_b(self) -> str: + return "{:x}".format(self.int_b).zfill(self.hex_digits) + def arguments(self) -> List[str]: return [ quote_str(self.arg_a), quote_str(self.arg_b) @@ -177,26 +198,10 @@ class ModOperationCommon(OperationCommon): data_in = [self.int_a, self.int_b, self.int_n] return max([n for n in data_in if n is not None]) - @property - def limbs(self) -> int: - return limbs_mpi(self.boundary, self.bits_in_limb) - - @property - def hex_digits(self) -> int: - return 2 * (self.limbs * self.bits_in_limb // 8) - @property def hex_n(self) -> str: return "{:x}".format(self.int_n).zfill(self.hex_digits) - @property - def hex_a(self) -> str: - return "{:x}".format(self.int_a).zfill(self.hex_digits) - - @property - def hex_b(self) -> str: - return "{:x}".format(self.int_b).zfill(self.hex_digits) - @property def r(self) -> int: # pylint: disable=invalid-name l = limbs_mpi(self.int_n, self.bits_in_limb) @@ -221,9 +226,6 @@ class OperationCommonArchSplit(OperationCommon): bound_val = max(self.int_a, self.int_b) self.bits_in_limb = bits_in_limb self.bound = bound_mpi(bound_val, self.bits_in_limb) - limbs = limbs_mpi(bound_val, self.bits_in_limb) - byte_len = limbs * self.bits_in_limb // 8 - self.hex_digits = 2 * byte_len if self.bits_in_limb == 32: self.dependencies = ["MBEDTLS_HAVE_INT32"] elif self.bits_in_limb == 64: From 0aa8cdba90c95a7fea268d2214c50ad4fec379e5 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Thu, 17 Nov 2022 20:33:51 +0000 Subject: [PATCH 211/490] Bignum Tests: remove OperationCommonArchSplit The ArchSplit functionality was duplicated and moved to OperationCommon from the other copy. The remnants of the functionality is moved to the only subclass using this. There is no semantic change to the generated tests. The order has changed however: core_add tests have been moved before core_mla tests and the order of the 64 and 32 bit versions have been swapped. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 52 +++++++------------------ scripts/framework_dev/bignum_core.py | 24 ++++++------ scripts/framework_dev/bignum_mod_raw.py | 2 +- 3 files changed, 29 insertions(+), 49 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 7d52749f8..0784f845f 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -80,17 +80,18 @@ class OperationCommon(test_data_generation.BaseTest): unique_combinations_only: Boolean to select if test case combinations must be unique. If True, only A,B or B,A would be included as a test case. If False, both A,B and B,A would be included. - arch_split: Boolean to select if different test cases are needed - depending on the architecture/limb size. This will cause test - objects being generated with different architectures. Individual - test objects can tell their architecture by accessing the - bits_in_limb instance variable. + input_style: Controls the way how test data is passed to the functions + in the generated test cases. "variable" passes them as they are + defined in the python source. "arch_split" pads the values with + zeroes depending on the architecture/limb size. If this is set, + test cases are generated for all architectures. """ symbol = "" input_values = [] # type: List[str] input_cases = [] # type: List[Tuple[str, str]] unique_combinations_only = True - arch_split = False + input_styles = ["variable", "arch_split"] # type: List[str] + input_style = "variable" # type: str limb_sizes = [32, 64] # type: List[int] def __init__(self, val_a: str, val_b: str, bits_in_limb: int = 64) -> None: @@ -100,7 +101,7 @@ class OperationCommon(test_data_generation.BaseTest): self.int_b = hex_to_int(val_b) if bits_in_limb not in self.limb_sizes: raise ValueError("Invalid number of bits in limb!") - if self.arch_split: + if self.input_style == "arch_split": self.dependencies = ["MBEDTLS_HAVE_INT{:d}".format(bits_in_limb)] self.bits_in_limb = bits_in_limb @@ -109,6 +110,10 @@ class OperationCommon(test_data_generation.BaseTest): data_in = [self.int_a, self.int_b] return max([n for n in data_in if n is not None]) + @property + def limb_boundary(self) -> int: + return bound_mpi(self.boundary, self.bits_in_limb) + @property def limbs(self) -> int: return limbs_mpi(self.boundary, self.bits_in_limb) @@ -171,8 +176,10 @@ class OperationCommon(test_data_generation.BaseTest): @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + if cls.input_style not in cls.input_styles: + raise ValueError("Unknown input style!") for a_value, b_value in cls.get_value_pairs(): - if cls.arch_split: + if cls.input_style == "arch_split": for bil in cls.limb_sizes: yield cls(a_value, b_value, bits_in_limb=bil).create_test_case() @@ -216,35 +223,6 @@ class ModOperationCommon(OperationCommon): return pow(self.r, 2) -class OperationCommonArchSplit(OperationCommon): - #pylint: disable=abstract-method - """Common features for operations where the result depends on - the limb size.""" - - def __init__(self, val_a: str, val_b: str, bits_in_limb: int) -> None: - super().__init__(val_a, val_b) - bound_val = max(self.int_a, self.int_b) - self.bits_in_limb = bits_in_limb - self.bound = bound_mpi(bound_val, self.bits_in_limb) - if self.bits_in_limb == 32: - self.dependencies = ["MBEDTLS_HAVE_INT32"] - elif self.bits_in_limb == 64: - self.dependencies = ["MBEDTLS_HAVE_INT64"] - else: - raise ValueError("Invalid number of bits in limb!") - self.arg_a = self.arg_a.zfill(self.hex_digits) - self.arg_b = self.arg_b.zfill(self.hex_digits) - - def pad_to_limbs(self, val) -> str: - return "{:x}".format(val).zfill(self.hex_digits) - - @classmethod - def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - for a_value, b_value in cls.get_value_pairs(): - yield cls(a_value, b_value, 32).create_test_case() - yield cls(a_value, b_value, 64).create_test_case() - - # BEGIN MERGE SLOT 1 # END MERGE SLOT 1 diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 591e53c20..749403705 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -106,6 +106,7 @@ class BignumCoreCTLookup(BignumCoreTarget, test_data_generation.BaseTest): yield (cls(bitsize, bitsize_description, window_size) .create_test_case()) + INPUT_VALUES = [ "0", "1", "3", "f", "fe", "ff", "100", "ff00", "fffe", "ffff", "10000", "fffffffe", "ffffffff", "100000000", "1f7f7f7f7f7f7f", @@ -127,38 +128,39 @@ INPUT_VALUES = [ ) ] - class BignumCoreOperation(BignumCoreTarget, bignum_common.OperationCommon): #pylint: disable=abstract-method """Common features for bignum core operations.""" input_values = INPUT_VALUES -class BignumCoreOperationArchSplit(BignumCoreTarget, - bignum_common.OperationCommonArchSplit): - #pylint: disable=abstract-method - """Common features for bignum core operations where the result depends on - the limb size.""" - input_values = INPUT_VALUES - - -class BignumCoreAddAndAddIf(BignumCoreOperationArchSplit): +class BignumCoreAddAndAddIf(BignumCoreOperation): """Test cases for bignum core add and add-if.""" count = 0 symbol = "+" test_function = "mpi_core_add_and_add_if" test_name = "mpi_core_add_and_add_if" + input_style = "arch_split" + + def __init__(self, val_a: str, val_b: str, bits_in_limb: int) -> None: + super().__init__(val_a, val_b) + self.arg_a = self.arg_a.zfill(self.hex_digits) + self.arg_b = self.arg_b.zfill(self.hex_digits) + + def pad_to_limbs(self, val) -> str: + return "{:x}".format(val).zfill(self.hex_digits) def result(self) -> List[str]: result = self.int_a + self.int_b - carry, result = divmod(result, self.bound) + carry, result = divmod(result, self.limb_boundary) return [ bignum_common.quote_str(self.pad_to_limbs(result)), str(carry) ] + class BignumCoreSub(BignumCoreOperation): """Test cases for bignum core sub.""" count = 0 diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index f44acef73..b330c493d 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -56,7 +56,7 @@ class BignumModRawConvertToMont(bignum_common.ModOperationCommon, test_function = "mpi_mod_raw_to_mont_rep" test_name = "Convert into Mont: " - arch_split = True + input_style = "arch_split" test_data_moduli = ["b", "fd", From a39cb83d757704054146207a9e4d5cacd3d86ecf Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Fri, 18 Nov 2022 16:05:46 +0000 Subject: [PATCH 212/490] Bignum tests: make args use input_style Before arg_ attributes were the arguments as they were defined in the python script. Turning these into properties and having them take the form respect the style set in input_style makes the class easier to use and more consistent. This change makes the hex_ properties redundant and therefore they are removed. There are no semantic changes to the generated test cases. (The order of appearance of 64 and 32 bit mpi_core_add_and_add_if test cases has changed.) Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 30 +++++++++++++++++-------- scripts/framework_dev/bignum_core.py | 10 +-------- scripts/framework_dev/bignum_mod_raw.py | 4 ++-- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 0784f845f..907c0b6d5 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -95,8 +95,8 @@ class OperationCommon(test_data_generation.BaseTest): limb_sizes = [32, 64] # type: List[int] def __init__(self, val_a: str, val_b: str, bits_in_limb: int = 64) -> None: - self.arg_a = val_a - self.arg_b = val_b + self.val_a = val_a + self.val_b = val_b self.int_a = hex_to_int(val_a) self.int_b = hex_to_int(val_b) if bits_in_limb not in self.limb_sizes: @@ -122,13 +122,25 @@ class OperationCommon(test_data_generation.BaseTest): def hex_digits(self) -> int: return 2 * (self.limbs * self.bits_in_limb // 8) - @property - def hex_a(self) -> str: - return "{:x}".format(self.int_a).zfill(self.hex_digits) + def format_arg(self, val) -> str: + if self.input_style not in self.input_styles: + raise ValueError("Unknown input style!") + if self.input_style == "variable": + return val + else: + return val.zfill(self.hex_digits) + + def format_result(self, res) -> str: + res_str = '{:x}'.format(res) + return quote_str(self.format_arg(res_str)) @property - def hex_b(self) -> str: - return "{:x}".format(self.int_b).zfill(self.hex_digits) + def arg_a(self) -> str: + return self.format_arg(self.val_a) + + @property + def arg_b(self) -> str: + return self.format_arg(self.val_b) def arguments(self) -> List[str]: return [ @@ -206,8 +218,8 @@ class ModOperationCommon(OperationCommon): return max([n for n in data_in if n is not None]) @property - def hex_n(self) -> str: - return "{:x}".format(self.int_n).zfill(self.hex_digits) + def arg_n(self) -> str: + return self.format_arg(self.val_n) @property def r(self) -> int: # pylint: disable=invalid-name diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 749403705..48390b98c 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -142,21 +142,13 @@ class BignumCoreAddAndAddIf(BignumCoreOperation): test_name = "mpi_core_add_and_add_if" input_style = "arch_split" - def __init__(self, val_a: str, val_b: str, bits_in_limb: int) -> None: - super().__init__(val_a, val_b) - self.arg_a = self.arg_a.zfill(self.hex_digits) - self.arg_b = self.arg_b.zfill(self.hex_digits) - - def pad_to_limbs(self, val) -> str: - return "{:x}".format(val).zfill(self.hex_digits) - def result(self) -> List[str]: result = self.int_a + self.int_b carry, result = divmod(result, self.limb_boundary) return [ - bignum_common.quote_str(self.pad_to_limbs(result)), + self.format_result(result), str(carry) ] diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index b330c493d..e2d8cd698 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -114,8 +114,8 @@ class BignumModRawConvertToMont(bignum_common.ModOperationCommon, return [self.hex_x] def arguments(self) -> List[str]: - return [bignum_common.quote_str(n) for n in [self.hex_n, - self.hex_a, + return [bignum_common.quote_str(n) for n in [self.arg_n, + self.arg_a, self.hex_x]] def description(self) -> str: From aa0450934bea5dfd6076033977b4920affe6c815 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Fri, 18 Nov 2022 16:48:45 +0000 Subject: [PATCH 213/490] Bignum tests: make n an attribute Having int_ variants as an attribute has the advantage of the input being validated when the object is instantiated. In theory otherwise if a particular int_ attribute is not accessed, then the invalid argument is passed to the tests as it is. (This would in all likelihood detected by the actual test cases, still, it is more robust like this.) There are no semantic changes to the generated test cases. (The order of appearance of 64 and 32 bit mpi_core_add_and_add_if test cases has changed.) Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 907c0b6d5..58eb11ebd 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -97,6 +97,8 @@ class OperationCommon(test_data_generation.BaseTest): def __init__(self, val_a: str, val_b: str, bits_in_limb: int = 64) -> None: self.val_a = val_a self.val_b = val_b + # Setting the int versions here as opposed to making them @properties + # provides earlier/more robust input validation. self.int_a = hex_to_int(val_a) self.int_b = hex_to_int(val_b) if bits_in_limb not in self.limb_sizes: @@ -207,10 +209,9 @@ class ModOperationCommon(OperationCommon): bits_in_limb: int = 64) -> None: super().__init__(val_a=val_a, val_b=val_b, bits_in_limb=bits_in_limb) self.val_n = val_n - - @property - def int_n(self) -> int: - return hex_to_int(self.val_n) + # Setting the int versions here as opposed to making them @properties + # provides earlier/more robust input validation. + self.int_n = hex_to_int(val_n) @property def boundary(self) -> int: From 5b3fba52188013eae4c3fbbdebdd926170269f84 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Fri, 18 Nov 2022 17:49:13 +0000 Subject: [PATCH 214/490] Bignum tests: add arity Add the ability to control the number of operands, by setting the arity class attribute. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 30 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 58eb11ebd..ecff206a3 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -85,6 +85,8 @@ class OperationCommon(test_data_generation.BaseTest): defined in the python source. "arch_split" pads the values with zeroes depending on the architecture/limb size. If this is set, test cases are generated for all architectures. + arity: the number of operands for the operation. Currently supported + values are 1 and 2. """ symbol = "" input_values = [] # type: List[str] @@ -93,8 +95,10 @@ class OperationCommon(test_data_generation.BaseTest): input_styles = ["variable", "arch_split"] # type: List[str] input_style = "variable" # type: str limb_sizes = [32, 64] # type: List[int] + arities = [1, 2] + arity = 2 - def __init__(self, val_a: str, val_b: str, bits_in_limb: int = 64) -> None: + def __init__(self, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: self.val_a = val_a self.val_b = val_b # Setting the int versions here as opposed to making them @properties @@ -109,8 +113,11 @@ class OperationCommon(test_data_generation.BaseTest): @property def boundary(self) -> int: - data_in = [self.int_a, self.int_b] - return max([n for n in data_in if n is not None]) + if self.arity == 1: + return self.int_a + elif self.arity == 2: + return max(self.int_a, self.int_b) + raise ValueError("Unsupported number of operands!") @property def limb_boundary(self) -> int: @@ -142,12 +149,15 @@ class OperationCommon(test_data_generation.BaseTest): @property def arg_b(self) -> str: + if self.arity == 1: + raise AttributeError("Operation is unary and doesn't have arg_b!") return self.format_arg(self.val_b) def arguments(self) -> List[str]: - return [ - quote_str(self.arg_a), quote_str(self.arg_b) - ] + self.result() + args = [quote_str(self.arg_a)] + if self.arity == 2: + args.append(quote_str(self.arg_b)) + return args + self.result() def description(self) -> str: """Generate a description for the test case. @@ -192,6 +202,8 @@ class OperationCommon(test_data_generation.BaseTest): def generate_function_tests(cls) -> Iterator[test_case.TestCase]: if cls.input_style not in cls.input_styles: raise ValueError("Unknown input style!") + if cls.arity not in cls.arities: + raise ValueError("Unsupported number of operands!") for a_value, b_value in cls.get_value_pairs(): if cls.input_style == "arch_split": for bil in cls.limb_sizes: @@ -215,13 +227,15 @@ class ModOperationCommon(OperationCommon): @property def boundary(self) -> int: - data_in = [self.int_a, self.int_b, self.int_n] - return max([n for n in data_in if n is not None]) + return self.int_n @property def arg_n(self) -> str: return self.format_arg(self.val_n) + def arguments(self) -> List[str]: + return [quote_str(self.arg_n)] + super().arguments() + @property def r(self) -> int: # pylint: disable=invalid-name l = limbs_mpi(self.int_n, self.bits_in_limb) From 485ab27102f158c5716870149b663bc0200eab88 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Fri, 18 Nov 2022 17:51:02 +0000 Subject: [PATCH 215/490] Bignum tests: use arity in bignum_mod_raw This makes a couple of properties redundant which are cleaned up. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_mod_raw.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index e2d8cd698..6c217c235 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -57,6 +57,7 @@ class BignumModRawConvertToMont(bignum_common.ModOperationCommon, test_function = "mpi_mod_raw_to_mont_rep" test_name = "Convert into Mont: " input_style = "arch_split" + arity = 1 test_data_moduli = ["b", "fd", @@ -111,12 +112,8 @@ class BignumModRawConvertToMont(bignum_common.ModOperationCommon, descr_tpl = '{} #{} N: \"{}\" A: \"{}\".' def result(self) -> List[str]: - return [self.hex_x] - - def arguments(self) -> List[str]: - return [bignum_common.quote_str(n) for n in [self.arg_n, - self.arg_a, - self.hex_x]] + result = (self.int_a * self.r) % self.int_n + return [self.format_result(result)] def description(self) -> str: return self.descr_tpl.format(self.test_name, @@ -134,13 +131,6 @@ class BignumModRawConvertToMont(bignum_common.ModOperationCommon, continue yield cls(n, i, bits_in_limb=bil).create_test_case() - @property - def x(self) -> int: # pylint: disable=invalid-name - return (self.int_a * self.r) % self.int_n - - @property - def hex_x(self) -> str: - return "{:x}".format(self.x).zfill(self.hex_digits) class BignumModRawConvertFromMont(BignumModRawConvertToMont): """ Test cases for mpi_mod_raw_from_mont_rep(). """ @@ -169,9 +159,11 @@ class BignumModRawConvertFromMont(BignumModRawConvertToMont): "138a7e6bfbc319ebd1725dacb9a359cbf693f2ecb785efb9d627" ] - @property - def x(self): # pylint: disable=invalid-name - return (self.int_a * self.r_inv) % self.int_n + def result(self) -> List[str]: + result = (self.int_a * self.r_inv) % self.int_n + return [self.format_result(result)] + + # END MERGE SLOT 7 # BEGIN MERGE SLOT 8 From 046fe7d4defdd45ca87a3552b67c779b90f59489 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Fri, 18 Nov 2022 18:15:24 +0000 Subject: [PATCH 216/490] Bignum tests: add support for filtering Sometimes we don't want all possible combinations of the input data and sometimes not all combinations make sense. We are adding a convenient way to decide on a case by case basis. Now child classes only need to implement the is_valid method and the invalid cases will be filtered out automatically. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index ecff206a3..b22846b71 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -172,6 +172,10 @@ class OperationCommon(test_data_generation.BaseTest): ) return super().description() + @property + def is_valid(self) -> bool: + return True + @abstractmethod def result(self) -> List[str]: """Get the result of the operation. @@ -204,13 +208,18 @@ class OperationCommon(test_data_generation.BaseTest): raise ValueError("Unknown input style!") if cls.arity not in cls.arities: raise ValueError("Unsupported number of operands!") - for a_value, b_value in cls.get_value_pairs(): - if cls.input_style == "arch_split": - for bil in cls.limb_sizes: - yield cls(a_value, b_value, - bits_in_limb=bil).create_test_case() - else: - yield cls(a_value, b_value).create_test_case() + if cls.input_style == "arch_split": + test_objects = (cls(a_value, b_value, bits_in_limb=bil) + for a_value, b_value in cls.get_value_pairs() + for bil in cls.limb_sizes) + else: + test_objects = (cls(a_value, b_value) for + a_value, b_value in cls.get_value_pairs()) + yield from (valid_test_object.create_test_case() + for valid_test_object in filter( + lambda test_object: test_object.is_valid, + test_objects + )) class ModOperationCommon(OperationCommon): From 22e36adcb2955b7c2eea934c1cab27f4eed6db18 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sat, 19 Nov 2022 10:42:20 +0000 Subject: [PATCH 217/490] Bignum tests: automate modulo test object generation Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 37 +++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index b22846b71..7d7170d17 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -209,12 +209,12 @@ class OperationCommon(test_data_generation.BaseTest): if cls.arity not in cls.arities: raise ValueError("Unsupported number of operands!") if cls.input_style == "arch_split": - test_objects = (cls(a_value, b_value, bits_in_limb=bil) - for a_value, b_value in cls.get_value_pairs() + test_objects = (cls(a, b, bits_in_limb=bil) + for a, b in cls.get_value_pairs() for bil in cls.limb_sizes) else: - test_objects = (cls(a_value, b_value) for - a_value, b_value in cls.get_value_pairs()) + test_objects = (cls(a, b) + for a, b in cls.get_value_pairs()) yield from (valid_test_object.create_test_case() for valid_test_object in filter( lambda test_object: test_object.is_valid, @@ -225,6 +225,7 @@ class OperationCommon(test_data_generation.BaseTest): class ModOperationCommon(OperationCommon): #pylint: disable=abstract-method """Target for bignum mod_raw test case generation.""" + moduli = [] # type: List[str] def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: @@ -258,6 +259,34 @@ class ModOperationCommon(OperationCommon): def r2(self) -> int: # pylint: disable=invalid-name return pow(self.r, 2) + @property + def is_valid(self) -> bool: + if self.int_a >= self.int_n: + return False + if self.arity == 2 and self.int_b >= self.int_n: + return False + return True + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + if cls.input_style not in cls.input_styles: + raise ValueError("Unknown input style!") + if cls.arity not in cls.arities: + raise ValueError("Unsupported number of operands!") + if cls.input_style == "arch_split": + test_objects = (cls(n, a, b, bits_in_limb=bil) + for n in cls.moduli + for a, b in cls.get_value_pairs() + for bil in cls.limb_sizes) + else: + test_objects = (cls(n, a, b) + for n in cls.moduli + for a, b in cls.get_value_pairs()) + yield from (valid_test_object.create_test_case() + for valid_test_object in filter( + lambda test_object: test_object.is_valid, + test_objects + )) # BEGIN MERGE SLOT 1 From b7b2db8adf956926b8a29b73fb798e30caa4ca96 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sat, 19 Nov 2022 12:48:17 +0000 Subject: [PATCH 218/490] Bignum test: remove type restrictrion The special case list type depends on the arity and the subclass. Remove type restriction to make defining special case lists more flexible and natural. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 16 +++++++++++----- scripts/framework_dev/bignum_core.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 7d7170d17..ed321d7c3 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -15,7 +15,8 @@ # limitations under the License. from abc import abstractmethod -from typing import Iterator, List, Tuple, TypeVar +from typing import Iterator, List, Tuple, TypeVar, Any +from itertools import chain from . import test_case from . import test_data_generation @@ -90,7 +91,7 @@ class OperationCommon(test_data_generation.BaseTest): """ symbol = "" input_values = [] # type: List[str] - input_cases = [] # type: List[Tuple[str, str]] + input_cases = [] # type: List[Any] unique_combinations_only = True input_styles = ["variable", "arch_split"] # type: List[str] input_style = "variable" # type: str @@ -200,7 +201,6 @@ class OperationCommon(test_data_generation.BaseTest): for a in cls.input_values for b in cls.input_values ) - yield from cls.input_cases @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: @@ -212,14 +212,20 @@ class OperationCommon(test_data_generation.BaseTest): test_objects = (cls(a, b, bits_in_limb=bil) for a, b in cls.get_value_pairs() for bil in cls.limb_sizes) + special_cases = (cls(*args, bits_in_limb=bil) # type: ignore + for args in cls.input_cases + for bil in cls.limb_sizes) else: test_objects = (cls(a, b) for a, b in cls.get_value_pairs()) + special_cases = (cls(*args) for args in cls.input_cases) yield from (valid_test_object.create_test_case() for valid_test_object in filter( lambda test_object: test_object.is_valid, - test_objects - )) + chain(test_objects, special_cases) + ) + ) + class ModOperationCommon(OperationCommon): diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 48390b98c..1bfc652ef 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -243,6 +243,16 @@ class BignumCoreMLA(BignumCoreOperation): "\"{:x}\"".format(carry_8) ] + @classmethod + def get_value_pairs(cls) -> Iterator[Tuple[str, str]]: + """Generator to yield pairs of inputs. + + Combinations are first generated from all input values, and then + specific cases provided. + """ + yield from super().get_value_pairs() + yield from cls.input_cases + @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: """Override for additional scalar input.""" From f4918759b4534ff4c4d1ba5a2fc7643367fb2dd1 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sat, 19 Nov 2022 14:18:02 +0000 Subject: [PATCH 219/490] Bignum tests: add special cases to mod Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index ed321d7c3..6fd42d1e7 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -273,6 +273,15 @@ class ModOperationCommon(OperationCommon): return False return True + @classmethod + def input_cases_args(cls) -> Iterator[Tuple[Any, Any, Any]]: + if cls.arity == 1: + yield from ((n, a, "0") for a, n in cls.input_cases) + elif cls.arity == 2: + yield from ((n, a, b) for a, b, n in cls.input_cases) + else: + raise ValueError("Unsupported number of operands!") + @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: if cls.input_style not in cls.input_styles: @@ -284,14 +293,18 @@ class ModOperationCommon(OperationCommon): for n in cls.moduli for a, b in cls.get_value_pairs() for bil in cls.limb_sizes) + special_cases = (cls(*args, bits_in_limb=bil) + for args in cls.input_cases_args() + for bil in cls.limb_sizes) else: test_objects = (cls(n, a, b) for n in cls.moduli for a, b in cls.get_value_pairs()) + special_cases = (cls(*args) for args in cls.input_cases_args()) yield from (valid_test_object.create_test_case() for valid_test_object in filter( lambda test_object: test_object.is_valid, - test_objects + chain(test_objects, special_cases) )) # BEGIN MERGE SLOT 1 From b25cf2242112a74162c7af629b7d7d743273e774 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sat, 19 Nov 2022 14:55:43 +0000 Subject: [PATCH 220/490] Bignum tests: complete support for unary operators There are no intended changes to generated tests. (The ordering of tests in the mod_raw module has changed.) Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 19 +-- scripts/framework_dev/bignum_mod_raw.py | 149 +++++++++++------------- 2 files changed, 81 insertions(+), 87 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 6fd42d1e7..318e25ca1 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -193,14 +193,19 @@ class OperationCommon(test_data_generation.BaseTest): Combinations are first generated from all input values, and then specific cases provided. """ - if cls.unique_combinations_only: - yield from combination_pairs(cls.input_values) + if cls.arity == 1: + yield from ((a, "0") for a in cls.input_values) + elif cls.arity == 2: + if cls.unique_combinations_only: + yield from combination_pairs(cls.input_values) + else: + yield from ( + (a, b) + for a in cls.input_values + for b in cls.input_values + ) else: - yield from ( - (a, b) - for a in cls.input_values - for b in cls.input_values - ) + raise ValueError("Unsupported number of operands!") @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 6c217c235..087c8dc87 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -14,9 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Iterator, List +from typing import Dict, List -from . import test_case from . import test_data_generation from . import bignum_common @@ -59,55 +58,55 @@ class BignumModRawConvertToMont(bignum_common.ModOperationCommon, input_style = "arch_split" arity = 1 - test_data_moduli = ["b", - "fd", - "eeff99aa37", - "eeff99aa11", - "800000000005", - "7fffffffffffffff", - "80fe000a10000001", - "25a55a46e5da99c71c7", - "1058ad82120c3a10196bb36229c1", - "7e35b84cb19ea5bc57ec37f5e431462fa962d98c1e63738d4657f" - "18ad6532e6adc3eafe67f1e5fa262af94cee8d3e7268593942a2a" - "98df75154f8c914a282f8b", - "8335616aed761f1f7f44e6bd49e807b82e3bf2bf11bfa63", - "ffcece570f2f991013f26dd5b03c4c5b65f97be5905f36cb4664f" - "2c78ff80aa8135a4aaf57ccb8a0aca2f394909a74cef1ef6758a6" - "4d11e2c149c393659d124bfc94196f0ce88f7d7d567efa5a649e2" - "deefaa6e10fdc3deac60d606bf63fc540ac95294347031aefd73d" - "6a9ee10188aaeb7a90d920894553cb196881691cadc51808715a0" - "7e8b24fcb1a63df047c7cdf084dd177ba368c806f3d51ddb5d389" - "8c863e687ecaf7d649a57a46264a582f94d3c8f2edaf59f77a7f6" - "bdaf83c991e8f06abe220ec8507386fce8c3da84c6c3903ab8f3a" - "d4630a204196a7dbcbd9bcca4e40ec5cc5c09938d49f5e1e6181d" - "b8896f33bb12e6ef73f12ec5c5ea7a8a337" - ] + moduli = ["b", + "fd", + "eeff99aa37", + "eeff99aa11", + "800000000005", + "7fffffffffffffff", + "80fe000a10000001", + "25a55a46e5da99c71c7", + "1058ad82120c3a10196bb36229c1", + "7e35b84cb19ea5bc57ec37f5e431462fa962d98c1e63738d4657f" + "18ad6532e6adc3eafe67f1e5fa262af94cee8d3e7268593942a2a" + "98df75154f8c914a282f8b", + "8335616aed761f1f7f44e6bd49e807b82e3bf2bf11bfa63", + "ffcece570f2f991013f26dd5b03c4c5b65f97be5905f36cb4664f" + "2c78ff80aa8135a4aaf57ccb8a0aca2f394909a74cef1ef6758a6" + "4d11e2c149c393659d124bfc94196f0ce88f7d7d567efa5a649e2" + "deefaa6e10fdc3deac60d606bf63fc540ac95294347031aefd73d" + "6a9ee10188aaeb7a90d920894553cb196881691cadc51808715a0" + "7e8b24fcb1a63df047c7cdf084dd177ba368c806f3d51ddb5d389" + "8c863e687ecaf7d649a57a46264a582f94d3c8f2edaf59f77a7f6" + "bdaf83c991e8f06abe220ec8507386fce8c3da84c6c3903ab8f3a" + "d4630a204196a7dbcbd9bcca4e40ec5cc5c09938d49f5e1e6181d" + "b8896f33bb12e6ef73f12ec5c5ea7a8a337" + ] - test_input_numbers = ["0", - "1", - "97", - "f5", - "6f5c3", - "745bfe50f7", - "ffa1f9924123", - "334a8b983c79bd", - "5b84f632b58f3461", - "19acd15bc38008e1", - "ffffffffffffffff", - "54ce6a6bb8247fa0427cfc75a6b0599", - "fecafe8eca052f154ce6a6bb8247fa019558bfeecce9bb9", - "a87d7a56fa4bfdc7da42ef798b9cf6843d4c54794698cb14d72" - "851dec9586a319f4bb6d5695acbd7c92e7a42a5ede6972adcbc" - "f68425265887f2d721f462b7f1b91531bac29fa648facb8e3c6" - "1bd5ae42d5a59ba1c89a95897bfe541a8ce1d633b98f379c481" - "6f25e21f6ac49286b261adb4b78274fe5f61c187581f213e84b" - "2a821e341ef956ecd5de89e6c1a35418cd74a549379d2d4594a" - "577543147f8e35b3514e62cf3e89d1156cdc91ab5f4c928fbd6" - "9148c35df5962fed381f4d8a62852a36823d5425f7487c13a12" - "523473fb823aa9d6ea5f42e794e15f2c1a8785cf6b7d51a4617" - "947fb3baf674f74a673cf1d38126983a19ed52c7439fab42c2185" - ] + input_values = ["0", + "1", + "97", + "f5", + "6f5c3", + "745bfe50f7", + "ffa1f9924123", + "334a8b983c79bd", + "5b84f632b58f3461", + "19acd15bc38008e1", + "ffffffffffffffff", + "54ce6a6bb8247fa0427cfc75a6b0599", + "fecafe8eca052f154ce6a6bb8247fa019558bfeecce9bb9", + "a87d7a56fa4bfdc7da42ef798b9cf6843d4c54794698cb14d72" + "851dec9586a319f4bb6d5695acbd7c92e7a42a5ede6972adcbc" + "f68425265887f2d721f462b7f1b91531bac29fa648facb8e3c6" + "1bd5ae42d5a59ba1c89a95897bfe541a8ce1d633b98f379c481" + "6f25e21f6ac49286b261adb4b78274fe5f61c187581f213e84b" + "2a821e341ef956ecd5de89e6c1a35418cd74a549379d2d4594a" + "577543147f8e35b3514e62cf3e89d1156cdc91ab5f4c928fbd6" + "9148c35df5962fed381f4d8a62852a36823d5425f7487c13a12" + "523473fb823aa9d6ea5f42e794e15f2c1a8785cf6b7d51a4617" + "947fb3baf674f74a673cf1d38126983a19ed52c7439fab42c2185" + ] descr_tpl = '{} #{} N: \"{}\" A: \"{}\".' @@ -121,16 +120,6 @@ class BignumModRawConvertToMont(bignum_common.ModOperationCommon, self.int_n, self.int_a) - @classmethod - def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - for bil in [32, 64]: - for n in cls.test_data_moduli: - for i in cls.test_input_numbers: - # Skip invalid combinations where A.limbs > N.limbs - if bignum_common.hex_to_int(i) > bignum_common.hex_to_int(n): - continue - yield cls(n, i, bits_in_limb=bil).create_test_case() - class BignumModRawConvertFromMont(BignumModRawConvertToMont): """ Test cases for mpi_mod_raw_from_mont_rep(). """ @@ -138,26 +127,26 @@ class BignumModRawConvertFromMont(BignumModRawConvertToMont): test_function = "mpi_mod_raw_from_mont_rep" test_name = "Convert from Mont: " - test_input_numbers = ["0", - "1", - "3ca", - "539ed428", - "7dfe5c6beb35a2d6", - "dca8de1c2adfc6d7aafb9b48e", - "a7d17b6c4be72f3d5c16bf9c1af6fc933", - "2fec97beec546f9553142ed52f147845463f579", - "378dc83b8bc5a7b62cba495af4919578dce6d4f175cadc4f", - "b6415f2a1a8e48a518345db11f56db3829c8f2c6415ab4a395a" - "b3ac2ea4cbef4af86eb18a84eb6ded4c6ecbfc4b59c2879a675" - "487f687adea9d197a84a5242a5cf6125ce19a6ad2e7341f1c57" - "d43ea4f4c852a51cb63dabcd1c9de2b827a3146a3d175b35bea" - "41ae75d2a286a3e9d43623152ac513dcdea1d72a7da846a8ab3" - "58d9be4926c79cfb287cf1cf25b689de3b912176be5dcaf4d4c" - "6e7cb839a4a3243a6c47c1e2c99d65c59d6fa3672575c2f1ca8" - "de6a32e854ec9d8ec635c96af7679fce26d7d159e4a9da3bd74" - "e1272c376cd926d74fe3fb164a5935cff3d5cdb92b35fe2cea32" - "138a7e6bfbc319ebd1725dacb9a359cbf693f2ecb785efb9d627" - ] + input_values = ["0", + "1", + "3ca", + "539ed428", + "7dfe5c6beb35a2d6", + "dca8de1c2adfc6d7aafb9b48e", + "a7d17b6c4be72f3d5c16bf9c1af6fc933", + "2fec97beec546f9553142ed52f147845463f579", + "378dc83b8bc5a7b62cba495af4919578dce6d4f175cadc4f", + "b6415f2a1a8e48a518345db11f56db3829c8f2c6415ab4a395a" + "b3ac2ea4cbef4af86eb18a84eb6ded4c6ecbfc4b59c2879a675" + "487f687adea9d197a84a5242a5cf6125ce19a6ad2e7341f1c57" + "d43ea4f4c852a51cb63dabcd1c9de2b827a3146a3d175b35bea" + "41ae75d2a286a3e9d43623152ac513dcdea1d72a7da846a8ab3" + "58d9be4926c79cfb287cf1cf25b689de3b912176be5dcaf4d4c" + "6e7cb839a4a3243a6c47c1e2c99d65c59d6fa3672575c2f1ca8" + "de6a32e854ec9d8ec635c96af7679fce26d7d159e4a9da3bd74" + "e1272c376cd926d74fe3fb164a5935cff3d5cdb92b35fe2cea32" + "138a7e6bfbc319ebd1725dacb9a359cbf693f2ecb785efb9d627" + ] def result(self) -> List[str]: result = (self.int_a * self.r_inv) % self.int_n From fd0ea1660998d163740c517419d068ff42b77275 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sat, 19 Nov 2022 15:05:19 +0000 Subject: [PATCH 221/490] Bignum tests: improve mod descriptions There are no semantic changes to the generated tests. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 23 +++++++++++++++++++---- scripts/framework_dev/bignum_mod_raw.py | 12 +++--------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 318e25ca1..9e92b8e61 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -168,9 +168,14 @@ class OperationCommon(test_data_generation.BaseTest): generated to provide some context to the test case. """ if not self.case_description: - self.case_description = "{:x} {} {:x}".format( - self.int_a, self.symbol, self.int_b - ) + if self.arity == 1: + self.case_description = "{} {:x}".format( + self.symbol, self.int_a + ) + elif self.arity == 2: + self.case_description = "{:x} {} {:x}".format( + self.int_a, self.symbol, self.int_b + ) return super().description() @property @@ -232,7 +237,6 @@ class OperationCommon(test_data_generation.BaseTest): ) - class ModOperationCommon(OperationCommon): #pylint: disable=abstract-method """Target for bignum mod_raw test case generation.""" @@ -278,6 +282,17 @@ class ModOperationCommon(OperationCommon): return False return True + def description(self) -> str: + """Generate a description for the test case. + + It uses the form A `symbol` B mod N, where symbol is used to represent + the operation. + """ + + if not self.case_description: + return super().description() + " mod {:x}".format(self.int_n) + return super().description() + @classmethod def input_cases_args(cls) -> Iterator[Tuple[Any, Any, Any]]: if cls.arity == 1: diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 087c8dc87..b23fbb2dc 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -55,6 +55,7 @@ class BignumModRawConvertToMont(bignum_common.ModOperationCommon, test_function = "mpi_mod_raw_to_mont_rep" test_name = "Convert into Mont: " + symbol = "R *" input_style = "arch_split" arity = 1 @@ -108,24 +109,17 @@ class BignumModRawConvertToMont(bignum_common.ModOperationCommon, "947fb3baf674f74a673cf1d38126983a19ed52c7439fab42c2185" ] - descr_tpl = '{} #{} N: \"{}\" A: \"{}\".' - def result(self) -> List[str]: result = (self.int_a * self.r) % self.int_n return [self.format_result(result)] - def description(self) -> str: - return self.descr_tpl.format(self.test_name, - self.count, - self.int_n, - self.int_a) - class BignumModRawConvertFromMont(BignumModRawConvertToMont): """ Test cases for mpi_mod_raw_from_mont_rep(). """ - + count = 0 test_function = "mpi_mod_raw_from_mont_rep" test_name = "Convert from Mont: " + symbol = "1/R *" input_values = ["0", "1", From 34f1dbb64ebe658672ee2b241b9e4fa2bd848da5 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sat, 19 Nov 2022 15:55:53 +0000 Subject: [PATCH 222/490] Bignum tests: add support for fixed width input Only fixed width input_style uses the default value of the bits_in_limb parameter, so set it to 32 in order to have less leading zeroes. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 9e92b8e61..b68653a03 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -93,13 +93,13 @@ class OperationCommon(test_data_generation.BaseTest): input_values = [] # type: List[str] input_cases = [] # type: List[Any] unique_combinations_only = True - input_styles = ["variable", "arch_split"] # type: List[str] + input_styles = ["variable", "fixed", "arch_split"] # type: List[str] input_style = "variable" # type: str limb_sizes = [32, 64] # type: List[int] arities = [1, 2] arity = 2 - def __init__(self, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: + def __init__(self, val_a: str, val_b: str = "0", bits_in_limb: int = 32) -> None: self.val_a = val_a self.val_b = val_b # Setting the int versions here as opposed to making them @properties From c9ba9d1b2c383ffaa8974b7ea6fbf1164b06e23b Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sun, 20 Nov 2022 10:56:05 +0000 Subject: [PATCH 223/490] Bignum Tests: add test data The goal of this commit is to add some constants that can be used to define datasets and add test data in a more readable and reusable manner. All platforms using ECC need to support calculations with at least 192 bits, therefore constants for this length are added. We are not using a curve prime as those will be tested elsewhere and it is better not to play favourites. All platforms using RSA or FFDH need to support calculations with at least 1024 bits, therefore numbers of this size are added too. A safe prime is added for both sizes as it makes all elements generators (except 0 and 1 of course), which in turn makes some tests more effective. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_data.py | 109 +++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 scripts/framework_dev/bignum_data.py diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py new file mode 100644 index 000000000..78fbb8c04 --- /dev/null +++ b/scripts/framework_dev/bignum_data.py @@ -0,0 +1,109 @@ +"""Base values and datasets for bignum generated tests and helper functions that +produced them.""" +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random + +# Functions calling these were used to produce test data and are here only for +# reproducability, they are not used by the test generation framework/classes +try: + from Cryptodome.Util.number import isPrime, getPrime #type: ignore #pylint: disable=import-error +except ImportError: + pass + +# Generated by bignum_common.gen_safe_prime(192,1) +SAFE_PRIME_192_BIT_SEED_1 = "d1c127a667786703830500038ebaef20e5a3e2dc378fb75b" + +# First number generated by random.getrandbits(192) - seed(2,2), not a prime +RANDOM_192_BIT_SEED_2_NO1 = "177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973" + +# Second number generated by random.getrandbits(192) - seed(2,2), not a prime +RANDOM_192_BIT_SEED_2_NO2 = "cf1822ffbc6887782b491044d5e341245c6e433715ba2bdd" + +# Third number generated by random.getrandbits(192) - seed(2,2), not a prime +RANDOM_192_BIT_SEED_2_NO3 = "3653f8dd9b1f282e4067c3584ee207f8da94e3e8ab73738f" + +# Fourth number generated by random.getrandbits(192) - seed(2,2), not a prime +RANDOM_192_BIT_SEED_2_NO4 = "ffed9235288bc781ae66267594c9c9500925e4749b575bd1" + +# Ninth number generated by random.getrandbits(192) - seed(2,2), not a prime +RANDOM_192_BIT_SEED_2_NO9 = "2a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f" + +# Generated by bignum_common.gen_safe_prime(1024,3) +SAFE_PRIME_1024_BIT_SEED_3 = ("c93ba7ec74d96f411ba008bdb78e63ff11bb5df46a51e16b" + "2c9d156f8e4e18abf5e052cb01f47d0d1925a77f60991577" + "e128fb6f52f34a27950a594baadd3d8057abeb222cf3cca9" + "62db16abf79f2ada5bd29ab2f51244bf295eff9f6aaba130" + "2efc449b128be75eeaca04bc3c1a155d11d14e8be32a2c82" + "87b3996cf6ad5223") + +# First number generated by random.getrandbits(1024) - seed(4,2), not a prime +RANDOM_1024_BIT_SEED_4_NO1 = ("6905269ed6f0b09f165c8ce36e2f24b43000de01b2ed40ed" + "3addccb2c33be0ac79d679346d4ac7a5c3902b38963dc6e8" + "534f45738d048ec0f1099c6c3e1b258fd724452ccea71ff4" + "a14876aeaff1a098ca5996666ceab360512bd13110722311" + "710cf5327ac435a7a97c643656412a9b8a1abcd1a6916c74" + "da4f9fc3c6da5d7") + +# Second number generated by random.getrandbits(1024) - seed(4,2), not a prime +RANDOM_1024_BIT_SEED_4_NO2 = ("f1cfd99216df648647adec26793d0e453f5082492d83a823" + "3fb62d2c81862fc9634f806fabf4a07c566002249b191bf4" + "d8441b5616332aca5f552773e14b0190d93936e1daca3c06" + "f5ff0c03bb5d7385de08caa1a08179104a25e4664f5253a0" + "2a3187853184ff27459142deccea264542a00403ce80c4b0" + "a4042bb3d4341aad") + +# Third number generated by random.getrandbits(1024) - seed(4,2), not a prime +RANDOM_1024_BIT_SEED_4_NO3 = ("14c15c910b11ad28cc21ce88d0060cc54278c2614e1bcb38" + "3bb4a570294c4ea3738d243a6e58d5ca49c7b59b995253fd" + "6c79a3de69f85e3131f3b9238224b122c3e4a892d9196ada" + "4fcfa583e1df8af9b474c7e89286a1754abcb06ae8abb93f" + "01d89a024cdce7a6d7288ff68c320f89f1347e0cdd905ecf" + "d160c5d0ef412ed6") + +# Fourth number generated by random.getrandbits(1024) - seed(4,2), not a prime +RANDOM_1024_BIT_SEED_4_NO4 = ("32decd6b8efbc170a26a25c852175b7a96b98b5fbf37a2be" + "6f98bca35b17b9662f0733c846bbe9e870ef55b1a1f65507" + "a2909cb633e238b4e9dd38b869ace91311021c9e32111ac1" + "ac7cc4a4ff4dab102522d53857c49391b36cc9aa78a330a1" + "a5e333cb88dcf94384d4cd1f47ca7883ff5a52f1a05885ac" + "7671863c0bdbc23a") + +# Fifth number generated by random.getrandbits(1024) - seed(4,2), not a prime +RANDOM_1024_BIT_SEED_4_NO5 = ("53be4721f5b9e1f5acdac615bc20f6264922b9ccf469aef8" + "f6e7d078e55b85dd1525f363b281b8885b69dc230af5ac87" + "0692b534758240df4a7a03052d733dcdef40af2e54c0ce68" + "1f44ebd13cc75f3edcb285f89d8cf4d4950b16ffc3e1ac3b" + "4708d9893a973000b54a23020fc5b043d6e4a51519d9c9cc" + "52d32377e78131c1") + +def __gen_safe_prime(bits, seed): + ''' + Generate a safe prime. + + This function is intended for generating constants offline and shouldn't be + used in test generation classes. + + Requires pycryptodomex for getPrime and isPrime and python 3.9 or later for + randbytes. + ''' + rng = random.Random() + # We want reproducability across python versions + rng.seed(seed, version=2) + while True: + prime = 2*getPrime(bits-1, rng.randbytes)+1 #pylint: disable=no-member + if isPrime(prime, 1e-30): + return prime From f19dff3903c2456f1ed841cd4cc517c9d827fbaa Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sun, 20 Nov 2022 11:58:12 +0000 Subject: [PATCH 224/490] Bignum tests: add default datasets Add data for small values, 192 bit and 1024 bit values, primes, non-primes odd, even, and some typical corner cases. All subclasses override this for the time being so there are no changes to the test cases. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 5 +++-- scripts/framework_dev/bignum_data.py | 27 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index b68653a03..e03c1c3f8 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -20,6 +20,7 @@ from itertools import chain from . import test_case from . import test_data_generation +from .bignum_data import INPUTS_DEFAULT, MODULI_DEFAULT T = TypeVar('T') #pylint: disable=invalid-name @@ -90,7 +91,7 @@ class OperationCommon(test_data_generation.BaseTest): values are 1 and 2. """ symbol = "" - input_values = [] # type: List[str] + input_values = INPUTS_DEFAULT # type: List[str] input_cases = [] # type: List[Any] unique_combinations_only = True input_styles = ["variable", "fixed", "arch_split"] # type: List[str] @@ -240,7 +241,7 @@ class OperationCommon(test_data_generation.BaseTest): class ModOperationCommon(OperationCommon): #pylint: disable=abstract-method """Target for bignum mod_raw test case generation.""" - moduli = [] # type: List[str] + moduli = MODULI_DEFAULT # type: List[str] def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py index 78fbb8c04..74d21d0ca 100644 --- a/scripts/framework_dev/bignum_data.py +++ b/scripts/framework_dev/bignum_data.py @@ -90,6 +90,33 @@ RANDOM_1024_BIT_SEED_4_NO5 = ("53be4721f5b9e1f5acdac615bc20f6264922b9ccf469aef8" "4708d9893a973000b54a23020fc5b043d6e4a51519d9c9cc" "52d32377e78131c1") +# Adding 192 bit and 1024 bit numbers because these are the shortest required +# for ECC and RSA respectively. +INPUTS_DEFAULT = [ + "0", "1", # corner cases + "2", "3", # small primes + "4", # non-prime even + "38", # small random + SAFE_PRIME_192_BIT_SEED_1, # prime + RANDOM_192_BIT_SEED_2_NO1, # not a prime + RANDOM_192_BIT_SEED_2_NO2, # not a prime + SAFE_PRIME_1024_BIT_SEED_3, # prime + RANDOM_1024_BIT_SEED_4_NO1, # not a prime + RANDOM_1024_BIT_SEED_4_NO3, # not a prime + RANDOM_1024_BIT_SEED_4_NO2, # largest (not a prime) + ] + +# Only odd moduli are present as in the new bignum code only odd moduli are +# supported for now. +MODULI_DEFAULT = [ + "53", # safe prime + "45", # non-prime + SAFE_PRIME_192_BIT_SEED_1, # safe prime + RANDOM_192_BIT_SEED_2_NO4, # not a prime + SAFE_PRIME_1024_BIT_SEED_3, # safe prime + RANDOM_1024_BIT_SEED_4_NO5, # not a prime + ] + def __gen_safe_prime(bits, seed): ''' Generate a safe prime. From 83786c24a855fbd2ac072de24f9891bb0683a9a8 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sun, 20 Nov 2022 12:45:58 +0000 Subject: [PATCH 225/490] Bignum tests: remove deprecated dataset Remove old dataset that was overriding the defaults in bignum_core. This will change the datasets for core_sub and core_add to the default inherited from bignum_common. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 1bfc652ef..deff6a8a6 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -107,31 +107,9 @@ class BignumCoreCTLookup(BignumCoreTarget, test_data_generation.BaseTest): .create_test_case()) -INPUT_VALUES = [ - "0", "1", "3", "f", "fe", "ff", "100", "ff00", "fffe", "ffff", "10000", - "fffffffe", "ffffffff", "100000000", "1f7f7f7f7f7f7f", - "8000000000000000", "fefefefefefefefe", "fffffffffffffffe", - "ffffffffffffffff", "10000000000000000", "1234567890abcdef0", - "fffffffffffffffffefefefefefefefe", "fffffffffffffffffffffffffffffffe", - "ffffffffffffffffffffffffffffffff", "100000000000000000000000000000000", - "1234567890abcdef01234567890abcdef0", - "fffffffffffffffffffffffffffffffffffffffffffffffffefefefefefefefe", - "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe", - "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "10000000000000000000000000000000000000000000000000000000000000000", - "1234567890abcdef01234567890abcdef01234567890abcdef01234567890abcdef0", - ( - "4df72d07b4b71c8dacb6cffa954f8d88254b6277099308baf003fab73227f34029" - "643b5a263f66e0d3c3fa297ef71755efd53b8fb6cb812c6bbf7bcf179298bd9947" - "c4c8b14324140a2c0f5fad7958a69050a987a6096e9f055fb38edf0c5889eca4a0" - "cfa99b45fbdeee4c696b328ddceae4723945901ec025076b12b" - ) -] - class BignumCoreOperation(BignumCoreTarget, bignum_common.OperationCommon): #pylint: disable=abstract-method """Common features for bignum core operations.""" - input_values = INPUT_VALUES class BignumCoreAddAndAddIf(BignumCoreOperation): From 52bb6af4218dc2e01dabd507201bb833d14a266c Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sun, 20 Nov 2022 12:52:53 +0000 Subject: [PATCH 226/490] Bignum tests: flatten class hierarchy in _core There is no semantic changes to the generated tests, the order of the test blocks has changed. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index deff6a8a6..806e13193 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -107,12 +107,7 @@ class BignumCoreCTLookup(BignumCoreTarget, test_data_generation.BaseTest): .create_test_case()) -class BignumCoreOperation(BignumCoreTarget, bignum_common.OperationCommon): - #pylint: disable=abstract-method - """Common features for bignum core operations.""" - - -class BignumCoreAddAndAddIf(BignumCoreOperation): +class BignumCoreAddAndAddIf(BignumCoreTarget, bignum_common.OperationCommon): """Test cases for bignum core add and add-if.""" count = 0 symbol = "+" @@ -131,7 +126,7 @@ class BignumCoreAddAndAddIf(BignumCoreOperation): ] -class BignumCoreSub(BignumCoreOperation): +class BignumCoreSub(BignumCoreTarget, bignum_common.OperationCommon): """Test cases for bignum core sub.""" count = 0 symbol = "-" @@ -157,7 +152,7 @@ class BignumCoreSub(BignumCoreOperation): ] -class BignumCoreMLA(BignumCoreOperation): +class BignumCoreMLA(BignumCoreTarget, bignum_common.OperationCommon): """Test cases for fixed-size multiply accumulate.""" count = 0 test_function = "mpi_core_mla" From f8435619dfd875a42fcc44f28912e0b502cd5622 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sun, 20 Nov 2022 13:32:54 +0000 Subject: [PATCH 227/490] Bignum tests: set unique combinations off by default Normally we need all the combinations, unique combinations make sense only if the operation is commutative. No changes to generated tests. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_common.py | 2 +- scripts/framework_dev/bignum_core.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index e03c1c3f8..67ea78db4 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -93,7 +93,7 @@ class OperationCommon(test_data_generation.BaseTest): symbol = "" input_values = INPUTS_DEFAULT # type: List[str] input_cases = [] # type: List[Any] - unique_combinations_only = True + unique_combinations_only = False input_styles = ["variable", "fixed", "arch_split"] # type: List[str] input_style = "variable" # type: str limb_sizes = [32, 64] # type: List[int] diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 806e13193..4910daea8 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -114,6 +114,7 @@ class BignumCoreAddAndAddIf(BignumCoreTarget, bignum_common.OperationCommon): test_function = "mpi_core_add_and_add_if" test_name = "mpi_core_add_and_add_if" input_style = "arch_split" + unique_combinations_only = True def result(self) -> List[str]: result = self.int_a + self.int_b @@ -132,7 +133,6 @@ class BignumCoreSub(BignumCoreTarget, bignum_common.OperationCommon): symbol = "-" test_function = "mpi_core_sub" test_name = "mbedtls_mpi_core_sub" - unique_combinations_only = False def result(self) -> List[str]: if self.int_a >= self.int_b: @@ -157,7 +157,6 @@ class BignumCoreMLA(BignumCoreTarget, bignum_common.OperationCommon): count = 0 test_function = "mpi_core_mla" test_name = "mbedtls_mpi_core_mla" - unique_combinations_only = False input_values = [ "0", "1", "fffe", "ffffffff", "100000000", "20000000000000", From 3e50eaa1a38ec5bfcf38cb63ceaa8b0a304efa48 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Sun, 20 Nov 2022 13:40:25 +0000 Subject: [PATCH 228/490] Bignum tests: use default dataset in mod_raw While at it, flatten class hierarchy as well. Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_mod_raw.py | 79 ++----------------------- 1 file changed, 5 insertions(+), 74 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index b23fbb2dc..60f2feded 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -49,98 +49,29 @@ class BignumModRawTarget(test_data_generation.BaseTarget): # END MERGE SLOT 6 # BEGIN MERGE SLOT 7 + class BignumModRawConvertToMont(bignum_common.ModOperationCommon, BignumModRawTarget): """ Test cases for mpi_mod_raw_to_mont_rep(). """ - test_function = "mpi_mod_raw_to_mont_rep" test_name = "Convert into Mont: " symbol = "R *" input_style = "arch_split" arity = 1 - moduli = ["b", - "fd", - "eeff99aa37", - "eeff99aa11", - "800000000005", - "7fffffffffffffff", - "80fe000a10000001", - "25a55a46e5da99c71c7", - "1058ad82120c3a10196bb36229c1", - "7e35b84cb19ea5bc57ec37f5e431462fa962d98c1e63738d4657f" - "18ad6532e6adc3eafe67f1e5fa262af94cee8d3e7268593942a2a" - "98df75154f8c914a282f8b", - "8335616aed761f1f7f44e6bd49e807b82e3bf2bf11bfa63", - "ffcece570f2f991013f26dd5b03c4c5b65f97be5905f36cb4664f" - "2c78ff80aa8135a4aaf57ccb8a0aca2f394909a74cef1ef6758a6" - "4d11e2c149c393659d124bfc94196f0ce88f7d7d567efa5a649e2" - "deefaa6e10fdc3deac60d606bf63fc540ac95294347031aefd73d" - "6a9ee10188aaeb7a90d920894553cb196881691cadc51808715a0" - "7e8b24fcb1a63df047c7cdf084dd177ba368c806f3d51ddb5d389" - "8c863e687ecaf7d649a57a46264a582f94d3c8f2edaf59f77a7f6" - "bdaf83c991e8f06abe220ec8507386fce8c3da84c6c3903ab8f3a" - "d4630a204196a7dbcbd9bcca4e40ec5cc5c09938d49f5e1e6181d" - "b8896f33bb12e6ef73f12ec5c5ea7a8a337" - ] - - input_values = ["0", - "1", - "97", - "f5", - "6f5c3", - "745bfe50f7", - "ffa1f9924123", - "334a8b983c79bd", - "5b84f632b58f3461", - "19acd15bc38008e1", - "ffffffffffffffff", - "54ce6a6bb8247fa0427cfc75a6b0599", - "fecafe8eca052f154ce6a6bb8247fa019558bfeecce9bb9", - "a87d7a56fa4bfdc7da42ef798b9cf6843d4c54794698cb14d72" - "851dec9586a319f4bb6d5695acbd7c92e7a42a5ede6972adcbc" - "f68425265887f2d721f462b7f1b91531bac29fa648facb8e3c6" - "1bd5ae42d5a59ba1c89a95897bfe541a8ce1d633b98f379c481" - "6f25e21f6ac49286b261adb4b78274fe5f61c187581f213e84b" - "2a821e341ef956ecd5de89e6c1a35418cd74a549379d2d4594a" - "577543147f8e35b3514e62cf3e89d1156cdc91ab5f4c928fbd6" - "9148c35df5962fed381f4d8a62852a36823d5425f7487c13a12" - "523473fb823aa9d6ea5f42e794e15f2c1a8785cf6b7d51a4617" - "947fb3baf674f74a673cf1d38126983a19ed52c7439fab42c2185" - ] - def result(self) -> List[str]: result = (self.int_a * self.r) % self.int_n return [self.format_result(result)] -class BignumModRawConvertFromMont(BignumModRawConvertToMont): +class BignumModRawConvertFromMont(bignum_common.ModOperationCommon, + BignumModRawTarget): """ Test cases for mpi_mod_raw_from_mont_rep(). """ - count = 0 test_function = "mpi_mod_raw_from_mont_rep" test_name = "Convert from Mont: " symbol = "1/R *" - - input_values = ["0", - "1", - "3ca", - "539ed428", - "7dfe5c6beb35a2d6", - "dca8de1c2adfc6d7aafb9b48e", - "a7d17b6c4be72f3d5c16bf9c1af6fc933", - "2fec97beec546f9553142ed52f147845463f579", - "378dc83b8bc5a7b62cba495af4919578dce6d4f175cadc4f", - "b6415f2a1a8e48a518345db11f56db3829c8f2c6415ab4a395a" - "b3ac2ea4cbef4af86eb18a84eb6ded4c6ecbfc4b59c2879a675" - "487f687adea9d197a84a5242a5cf6125ce19a6ad2e7341f1c57" - "d43ea4f4c852a51cb63dabcd1c9de2b827a3146a3d175b35bea" - "41ae75d2a286a3e9d43623152ac513dcdea1d72a7da846a8ab3" - "58d9be4926c79cfb287cf1cf25b689de3b912176be5dcaf4d4c" - "6e7cb839a4a3243a6c47c1e2c99d65c59d6fa3672575c2f1ca8" - "de6a32e854ec9d8ec635c96af7679fce26d7d159e4a9da3bd74" - "e1272c376cd926d74fe3fb164a5935cff3d5cdb92b35fe2cea32" - "138a7e6bfbc319ebd1725dacb9a359cbf693f2ecb785efb9d627" - ] + input_style = "arch_split" + arity = 1 def result(self) -> List[str]: result = (self.int_a * self.r_inv) % self.int_n From df6a3a6b584cb7cfa75648063a3fb04c7274afe6 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Tue, 22 Nov 2022 15:07:31 +0000 Subject: [PATCH 229/490] Add unit tests for mbedtls_mpi_core_sub_int(), MPI A - scalar b Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_core.py | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 4910daea8..b8e2a3123 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -763,6 +763,37 @@ def mpi_modmul_case_generate() -> None: # BEGIN MERGE SLOT 3 +class BignumCoreSubInt(BignumCoreTarget, bignum_common.OperationCommon): + """Test cases for bignum core sub int.""" + count = 0 + symbol = "-" + test_function = "mpi_core_sub_int" + test_name = "mpi_core_sub_int" + input_style = "arch_split" + + @property + def is_valid(self) -> bool: + # This is "sub int", so b is only one limb + if bignum_common.limbs_mpi(self.int_b, self.bits_in_limb) > 1: + return False + return True + + # Overriding because we don't want leading zeros on b + @property + def arg_b(self) -> str: + return self.val_b + + def result(self) -> List[str]: + result = self.int_a - self.int_b + + borrow, result = divmod(result, self.limb_boundary) + + # Borrow will be -1 if non-zero, but we want it to be 1 in the test data + return [ + self.format_result(result), + str(-borrow) + ] + # END MERGE SLOT 3 # BEGIN MERGE SLOT 4 From ada7ca6d621947141413ac16706580e1fb868c28 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Tue, 22 Nov 2022 21:37:10 +0000 Subject: [PATCH 230/490] mpi_core_exp_mod: add generated tests Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 4910daea8..f85fb2e36 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -755,6 +755,27 @@ def mpi_modmul_case_generate() -> None: # BEGIN MERGE SLOT 1 +class BignumCoreExpMod(BignumCoreTarget, bignum_common.ModOperationCommon): + """Test cases for bignum core exponentiation.""" + symbol = "^" + test_function = "mpi_core_exp_mod" + test_name = "Core modular exponentiation" + input_style = "fixed" + + def result(self) -> List[str]: + result = pow(self.int_a, self.int_b, self.int_n) + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + # The base needs to be canonical, but the exponent can be larger than + # the modulus (see for example exponent blinding) + if self.int_a < self.int_n: + return True + else: + return False + + # END MERGE SLOT 1 # BEGIN MERGE SLOT 2 From 7050cabb74d864d2ab379f231c1723c0a4c148d5 Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Tue, 22 Nov 2022 21:50:22 +0000 Subject: [PATCH 231/490] Make pylint happy Signed-off-by: Janos Follath --- scripts/framework_dev/bignum_core.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index f85fb2e36..ed6ecbd00 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -770,11 +770,7 @@ class BignumCoreExpMod(BignumCoreTarget, bignum_common.ModOperationCommon): def is_valid(self) -> bool: # The base needs to be canonical, but the exponent can be larger than # the modulus (see for example exponent blinding) - if self.int_a < self.int_n: - return True - else: - return False - + return bool(self.int_a < self.int_n) # END MERGE SLOT 1 From 93397c75b3f462302af1f3789e0d6e7872cfa267 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 15 Nov 2022 18:51:20 +0100 Subject: [PATCH 232/490] Add generated test for low level subtraction with modulus Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod_raw.py | 90 +++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 60f2feded..5d4bda2a7 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -30,6 +30,96 @@ class BignumModRawTarget(test_data_generation.BaseTarget): # BEGIN MERGE SLOT 2 +class BignumModRawSub(BignumModRawOperation): + """Test cases for bignum mod raw sub.""" + count = 0 + symbol = "-" + test_function = "mpi_mod_raw_sub" + test_name = "mbedtls_mpi_mod_raw_sub" + unique_combinations_only = False + + input_values = [ + "0", "1", "fe", "ff", "fffe", "ffff", + "fffffffffffffffe", "ffffffffffffffff", + "fffffffffffffffffffffffffffffffe", + "ffffffffffffffffffffffffffffffff", + "1234567890abcdef01234567890abcdef0", + "3653f8dd9b1f282e4067c3584ee207f8da94e3e8ab73738f", + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "1234567890abcdef01234567890abcdef01234567890abcdef01234567890abcdef0", + ( + "14c15c910b11ad28cc21ce88d0060cc54278c2614e1bcb383bb4a570294c4ea3" + "738d243a6e58d5ca49c7b59b995253fd6c79a3de69f85e3131f3b9238224b122" + "c3e4a892d9196ada4fcfa583e1df8af9b474c7e89286a1754abcb06ae8abb93f" + "01d89a024cdce7a6d7288ff68c320f89f1347e0cdd905ecfd160c5d0ef412ed6" + ) + ] + + modulus_values = [ + "7", "ff", + "d1c127a667786703830500038ebaef20e5a3e2dc378fb75b" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff43", + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff67", + ( + "c93ba7ec74d96f411ba008bdb78e63ff11bb5df46a51e16b2c9d156f8e4e18ab" + "f5e052cb01f47d0d1925a77f60991577e128fb6f52f34a27950a594baadd3d80" + "57abeb222cf3cca962db16abf79f2ada5bd29ab2f51244bf295eff9f6aaba130" + "2efc449b128be75eeaca04bc3c1a155d11d14e8be32a2c8287b3996cf6ad5223" + ), + ( + "5c083126e978d4fdf3b645a1cac083126e978d4fdf3b645a1cac083126e978d4" + "fdf3b645a1cac083126e978d4fdf3b645a1cac083126e978d4fdf3b645a1cac0" + "83126e978d4fdf3b645a1cac083126e978d4fdf3b645a1cac083126e978d4fdf" + "3b645a1cac083126e978d4fdf3b645a1cac083126e978d4fdf3b645a1cac05d2" + ) + ] + + descr_tpl = '{} #{} \"{}\" - \"{}\" % \"{}\".' + + BITS_IN_LIMB = 32 + + @property + def boundary(self) -> int: + return self.int_n + + @property + def x(self): # pylint: disable=invalid-name + return (self.int_a - self.int_b) % self.int_n if self.int_n > 0 else 0 + + @property + def hex_x(self) -> str: + return format(self.x, 'x').zfill(self.hex_digits) + + def description(self) -> str: + return self.descr_tpl.format(self.test_name, + self.count, + self.int_a, + self.int_b, + self.int_n) + + def arguments(self) -> List[str]: + return [bignum_common.quote_str(n) for n in [self.hex_a, + self.hex_b, + self.hex_n, + self.hex_x]] + + def result(self) -> List[str]: + return [self.hex_x] + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + for a_value, b_value in cls.get_value_pairs(): + int_a = bignum_common.hex_to_int(a_value) + int_b = bignum_common.hex_to_int(b_value) + highest = max(int_a, int_b) + + # Choose a modulus bigger then the arguments + for n_value in cls.modulus_values: + int_n = bignum_common.hex_to_int(n_value) + if highest < int_n: + yield cls(n_value, a_value, b_value, cls.BITS_IN_LIMB).create_test_case() + # END MERGE SLOT 2 # BEGIN MERGE SLOT 3 From 83f1f09145c776d4baea3c96a5994d46a70240a4 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Wed, 23 Nov 2022 16:45:05 +0100 Subject: [PATCH 233/490] Update the test case generator Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod_raw.py | 93 +++---------------------- 1 file changed, 11 insertions(+), 82 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 5d4bda2a7..c27104854 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -30,95 +30,24 @@ class BignumModRawTarget(test_data_generation.BaseTarget): # BEGIN MERGE SLOT 2 -class BignumModRawSub(BignumModRawOperation): - """Test cases for bignum mod raw sub.""" - count = 0 +class BignumModRawSub(bignum_common.ModOperationCommon, + BignumModRawTarget): + """Test cases for bignum mpi_mod_raw_sub().""" symbol = "-" test_function = "mpi_mod_raw_sub" test_name = "mbedtls_mpi_mod_raw_sub" - unique_combinations_only = False - - input_values = [ - "0", "1", "fe", "ff", "fffe", "ffff", - "fffffffffffffffe", "ffffffffffffffff", - "fffffffffffffffffffffffffffffffe", - "ffffffffffffffffffffffffffffffff", - "1234567890abcdef01234567890abcdef0", - "3653f8dd9b1f282e4067c3584ee207f8da94e3e8ab73738f", - "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe", - "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "1234567890abcdef01234567890abcdef01234567890abcdef01234567890abcdef0", - ( - "14c15c910b11ad28cc21ce88d0060cc54278c2614e1bcb383bb4a570294c4ea3" - "738d243a6e58d5ca49c7b59b995253fd6c79a3de69f85e3131f3b9238224b122" - "c3e4a892d9196ada4fcfa583e1df8af9b474c7e89286a1754abcb06ae8abb93f" - "01d89a024cdce7a6d7288ff68c320f89f1347e0cdd905ecfd160c5d0ef412ed6" - ) - ] - - modulus_values = [ - "7", "ff", - "d1c127a667786703830500038ebaef20e5a3e2dc378fb75b" - "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff43", - "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff67", - ( - "c93ba7ec74d96f411ba008bdb78e63ff11bb5df46a51e16b2c9d156f8e4e18ab" - "f5e052cb01f47d0d1925a77f60991577e128fb6f52f34a27950a594baadd3d80" - "57abeb222cf3cca962db16abf79f2ada5bd29ab2f51244bf295eff9f6aaba130" - "2efc449b128be75eeaca04bc3c1a155d11d14e8be32a2c8287b3996cf6ad5223" - ), - ( - "5c083126e978d4fdf3b645a1cac083126e978d4fdf3b645a1cac083126e978d4" - "fdf3b645a1cac083126e978d4fdf3b645a1cac083126e978d4fdf3b645a1cac0" - "83126e978d4fdf3b645a1cac083126e978d4fdf3b645a1cac083126e978d4fdf" - "3b645a1cac083126e978d4fdf3b645a1cac083126e978d4fdf3b645a1cac05d2" - ) - ] - - descr_tpl = '{} #{} \"{}\" - \"{}\" % \"{}\".' - - BITS_IN_LIMB = 32 - - @property - def boundary(self) -> int: - return self.int_n - - @property - def x(self): # pylint: disable=invalid-name - return (self.int_a - self.int_b) % self.int_n if self.int_n > 0 else 0 - - @property - def hex_x(self) -> str: - return format(self.x, 'x').zfill(self.hex_digits) - - def description(self) -> str: - return self.descr_tpl.format(self.test_name, - self.count, - self.int_a, - self.int_b, - self.int_n) + input_style = "fixed" + arity = 2 def arguments(self) -> List[str]: - return [bignum_common.quote_str(n) for n in [self.hex_a, - self.hex_b, - self.hex_n, - self.hex_x]] + return [bignum_common.quote_str(n) for n in [self.arg_a, + self.arg_b, + self.arg_n] + ] + self.result() def result(self) -> List[str]: - return [self.hex_x] - - @classmethod - def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - for a_value, b_value in cls.get_value_pairs(): - int_a = bignum_common.hex_to_int(a_value) - int_b = bignum_common.hex_to_int(b_value) - highest = max(int_a, int_b) - - # Choose a modulus bigger then the arguments - for n_value in cls.modulus_values: - int_n = bignum_common.hex_to_int(n_value) - if highest < int_n: - yield cls(n_value, a_value, b_value, cls.BITS_IN_LIMB).create_test_case() + result = (self.int_a - self.int_b) % self.int_n + return [self.format_result(result)] # END MERGE SLOT 2 From 3ad9c25f0ce64930593eb61a0f3c1be2a09c1662 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Thu, 24 Nov 2022 15:56:53 +0000 Subject: [PATCH 234/490] Add test generation for mbedtls_mpi_mod_raw_add() Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_mod_raw.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 60f2feded..ee144aa1e 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -42,6 +42,25 @@ class BignumModRawTarget(test_data_generation.BaseTarget): # BEGIN MERGE SLOT 5 +class BignumModRawAdd(bignum_common.ModOperationCommon, + BignumModRawTarget): + """Test cases for bignum mpi_mod_raw_add().""" + symbol = "+" + test_function = "mpi_mod_raw_add" + test_name = "mbedtls_mpi_mod_raw_add" + input_style = "fixed" + arity = 2 + + def arguments(self) -> List[str]: + return [bignum_common.quote_str(n) for n in [self.arg_a, + self.arg_b, + self.arg_n] + ] + self.result() + + def result(self) -> List[str]: + result = (self.int_a + self.int_b) % self.int_n + return [self.format_result(result)] + # END MERGE SLOT 5 # BEGIN MERGE SLOT 6 From 6aa5f49e203bcc6ada27d653abb443427e225d79 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Thu, 24 Nov 2022 21:29:23 +0000 Subject: [PATCH 235/490] Change order of test arguments for bignum_mod_raw to simplify Python script Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_mod_raw.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index ee144aa1e..4c53a7f4c 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -51,12 +51,6 @@ class BignumModRawAdd(bignum_common.ModOperationCommon, input_style = "fixed" arity = 2 - def arguments(self) -> List[str]: - return [bignum_common.quote_str(n) for n in [self.arg_a, - self.arg_b, - self.arg_n] - ] + self.result() - def result(self) -> List[str]: result = (self.int_a + self.int_b) % self.int_n return [self.format_result(result)] From 8500c017bb6b3f32006336b40d00298fc9ec8f59 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 29 Nov 2022 22:03:32 +0100 Subject: [PATCH 236/490] Preserve line breaks in comments before test functions This way line numbers match better in error messages. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 10 ++++------ scripts/test_generate_test_code.py | 1 + 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index c1d1c1cbf..b7c8e2854 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -552,14 +552,12 @@ def skip_comments(line, stream): pos = closing + 2 # Replace inner comment by spaces. There needs to be at least one space # for things like 'int/*ihatespaces*/foo'. Go further and preserve the - # width of the comment, this way column positions in error messages - # remain correct. - # TODO: It would be better to preserve line breaks, to get accurate - # line numbers if there's something interesting after a comment on - # the same line. + # width of the comment and line breaks, this way positions in error + # messages remain correct. line = (line[:opening.start(0)] + - ' ' * (pos - opening.start(0)) + + re.sub(r'.', r' ', line[opening.start(0):pos]) + line[pos:]) + # Strip whitespace at the end of lines (it's irrelevant to error messages). return re.sub(r' +(\n|\Z)', r'\1', line) def parse_function_code(funcs_f, dependencies, suite_dependencies): diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index d8b8cf979..8b2bf78d8 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -766,6 +766,7 @@ exit: + void test_func() { ba ba black sheep From 1005b20f73231ce3bb2e447eea3459aa1486e229 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 16:38:49 +0100 Subject: [PATCH 237/490] Preserve line breaks from continued line comments The commit "Preserve line breaks in comments before test functions" only handled block comments. This commit handles line comments. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 6 +++++- scripts/test_generate_test_code.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index b7c8e2854..f19d30b61 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -537,11 +537,15 @@ def skip_comments(line, stream): break if line[opening.start(0) + 1] == '/': # //... continuation = line + # Count the number of line breaks, to keep line numbers aligned + # in the output. + line_count = 1 while continuation.endswith('\\\n'): # This errors out if the file ends with an unfinished line # comment. That's acceptable to not complicate the code further. continuation = next(stream) - return line[:opening.start(0)].rstrip() + '\n' + line_count += 1 + return line[:opening.start(0)].rstrip() + '\n' * line_count # Parsing /*...*/, looking for the end closing = line.find('*/', opening.end(0)) while closing == -1: diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 8b2bf78d8..d23d74219 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -767,6 +767,8 @@ exit: + + void test_func() { ba ba black sheep @@ -813,6 +815,7 @@ exit: expected = '''#line 1 "test_suite_ut.function" void test_func( int x, + int y ) { ba ba black sheep From b2ae5d83dc6ad2873034def28609a735e2637484 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Wed, 30 Nov 2022 16:34:07 +0000 Subject: [PATCH 238/490] Add imports to bignum_mod Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_mod.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index a604cc0c5..81ece0727 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -14,7 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict, List # pylint: disable=unused-import + from . import test_data_generation +from . import bignum_common # pylint: disable=unused-import class BignumModTarget(test_data_generation.BaseTarget): #pylint: disable=abstract-method, too-few-public-methods From 737f7550ceffc67ddd32713a79e38fadd568d6f8 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Thu, 1 Dec 2022 14:27:37 +0000 Subject: [PATCH 239/490] Implement mbedtls_mpi_mod_sub() Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_mod.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 81ece0727..5f4e362b0 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -34,6 +34,25 @@ class BignumModTarget(test_data_generation.BaseTarget): # BEGIN MERGE SLOT 3 +class BignumModSub(bignum_common.ModOperationCommon, BignumModTarget): + """Test cases for bignum mpi_mod_sub().""" + symbol = "-" + test_function = "mpi_mod_sub" + test_name = "mbedtls_mpi_mod_sub" + input_style = "fixed" + arity = 2 + + # To make negative tests easier, append 0 for success to the generated cases + def arguments(self) -> List[str]: + return [bignum_common.quote_str(n) for n in [self.arg_n, + self.arg_a, + self.arg_b] + ] + self.result() + ["0"] + + def result(self) -> List[str]: + result = (self.int_a - self.int_b) % self.int_n + return [self.format_result(result)] + # END MERGE SLOT 3 # BEGIN MERGE SLOT 4 From 7e6222714cc3044fbad999ea04694f1bf5dd3215 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Sun, 4 Dec 2022 17:19:59 +0000 Subject: [PATCH 240/490] Fix typos prior to release Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py index 74d21d0ca..e6ed30005 100644 --- a/scripts/framework_dev/bignum_data.py +++ b/scripts/framework_dev/bignum_data.py @@ -18,7 +18,7 @@ produced them.""" import random # Functions calling these were used to produce test data and are here only for -# reproducability, they are not used by the test generation framework/classes +# reproducibility, they are not used by the test generation framework/classes try: from Cryptodome.Util.number import isPrime, getPrime #type: ignore #pylint: disable=import-error except ImportError: @@ -128,7 +128,7 @@ def __gen_safe_prime(bits, seed): randbytes. ''' rng = random.Random() - # We want reproducability across python versions + # We want reproducibility across python versions rng.seed(seed, version=2) while True: prime = 2*getPrime(bits-1, rng.randbytes)+1 #pylint: disable=no-member From a46b92f8d52da35cfd037d42d8e4e43a2321220b Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Mon, 5 Dec 2022 15:47:40 +0000 Subject: [PATCH 241/490] Apply review comments Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_mod.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 5f4e362b0..aa06fe863 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -42,16 +42,11 @@ class BignumModSub(bignum_common.ModOperationCommon, BignumModTarget): input_style = "fixed" arity = 2 - # To make negative tests easier, append 0 for success to the generated cases - def arguments(self) -> List[str]: - return [bignum_common.quote_str(n) for n in [self.arg_n, - self.arg_a, - self.arg_b] - ] + self.result() + ["0"] - def result(self) -> List[str]: result = (self.int_a - self.int_b) % self.int_n - return [self.format_result(result)] + # To make negative tests easier, append 0 for success to the + # generated cases + return [self.format_result(result), "0"] # END MERGE SLOT 3 From 7623ac7f42f66114a42a31b506f7255ceb3fcff3 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Tue, 6 Dec 2022 10:46:30 +0000 Subject: [PATCH 242/490] Require input to mbedtls_mpi_core_exp_mod() to already be in Montgomery form Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_core.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 2960d242d..a000bde07 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -759,12 +759,23 @@ class BignumCoreExpMod(BignumCoreTarget, bignum_common.ModOperationCommon): """Test cases for bignum core exponentiation.""" symbol = "^" test_function = "mpi_core_exp_mod" - test_name = "Core modular exponentiation" + test_name = "Core modular exponentiation (Mongtomery form only)" input_style = "fixed" + def arguments(self) -> List[str]: + # Input 'a' has to be given in Montgomery form + mont_a = (self.int_a * self.r) % self.int_n + arg_mont_a = self.format_arg('{:x}'.format(mont_a)) + return [bignum_common.quote_str(n) for n in [self.arg_n, + arg_mont_a, + self.arg_b] + ] + self.result() + def result(self) -> List[str]: + # Result has to be given in Montgomery form result = pow(self.int_a, self.int_b, self.int_n) - return [self.format_result(result)] + mont_result = (result * self.r) % self.int_n + return [self.format_result(mont_result)] @property def is_valid(self) -> bool: From 4f26d33b07699b5a3242d1142ad8c3120bd13081 Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 29 Nov 2022 12:25:05 +0000 Subject: [PATCH 243/490] Implement mbedtls_mpi_mod_add() Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_mod.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index aa06fe863..9086c2eb5 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -55,6 +55,20 @@ class BignumModSub(bignum_common.ModOperationCommon, BignumModTarget): # END MERGE SLOT 4 # BEGIN MERGE SLOT 5 +class BignumModAdd(bignum_common.ModOperationCommon, BignumModTarget): + """Test cases for bignum mpi_mod_add().""" + count = 0 + symbol = "+" + test_function = "mpi_mod_add" + test_name = "mbedtls_mpi_mod_add" + input_style = "fixed" + + def result(self) -> List[str]: + result = (self.int_a + self.int_b) % self.int_n + # To make negative tests easier, append "0" for success to the + # generated cases + return [self.format_result(result), "0"] + # END MERGE SLOT 5 From b9bc950ef52b93ee39fdbc66c19a2b887ae648dd Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Tue, 6 Dec 2022 10:14:57 +0000 Subject: [PATCH 244/490] Re-enable pylint unused warnings Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_mod.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 9086c2eb5..a16699a64 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -14,10 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, List # pylint: disable=unused-import +from typing import Dict, List from . import test_data_generation -from . import bignum_common # pylint: disable=unused-import +from . import bignum_common class BignumModTarget(test_data_generation.BaseTarget): #pylint: disable=abstract-method, too-few-public-methods From dbdca131030ecf35c7357122fd079c85d6415f99 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Tue, 6 Dec 2022 12:20:43 +0000 Subject: [PATCH 245/490] Separate out to_montgomery and from_montgomery for bignum tests Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_common.py | 6 ++++++ scripts/framework_dev/bignum_core.py | 6 +++--- scripts/framework_dev/bignum_mod_raw.py | 5 ++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 67ea78db4..81bc28e19 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -251,6 +251,12 @@ class ModOperationCommon(OperationCommon): # provides earlier/more robust input validation. self.int_n = hex_to_int(val_n) + def to_montgomery(self, val) -> int: + return (val * self.r) % self.int_n + + def from_montgomery(self, val) -> int: + return (val * self.r_inv) % self.int_n + @property def boundary(self) -> int: return self.int_n diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index a000bde07..118a659cf 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -764,7 +764,7 @@ class BignumCoreExpMod(BignumCoreTarget, bignum_common.ModOperationCommon): def arguments(self) -> List[str]: # Input 'a' has to be given in Montgomery form - mont_a = (self.int_a * self.r) % self.int_n + mont_a = self.to_montgomery(self.int_a) arg_mont_a = self.format_arg('{:x}'.format(mont_a)) return [bignum_common.quote_str(n) for n in [self.arg_n, arg_mont_a, @@ -772,9 +772,9 @@ class BignumCoreExpMod(BignumCoreTarget, bignum_common.ModOperationCommon): ] + self.result() def result(self) -> List[str]: - # Result has to be given in Montgomery form + # Result has to be given in Montgomery form too result = pow(self.int_a, self.int_b, self.int_n) - mont_result = (result * self.r) % self.int_n + mont_result = self.to_montgomery(result) return [self.format_result(mont_result)] @property diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 0bbad5dd9..d05479a00 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -92,10 +92,9 @@ class BignumModRawConvertToMont(bignum_common.ModOperationCommon, arity = 1 def result(self) -> List[str]: - result = (self.int_a * self.r) % self.int_n + result = self.to_montgomery(self.int_a) return [self.format_result(result)] - class BignumModRawConvertFromMont(bignum_common.ModOperationCommon, BignumModRawTarget): """ Test cases for mpi_mod_raw_from_mont_rep(). """ @@ -106,7 +105,7 @@ class BignumModRawConvertFromMont(bignum_common.ModOperationCommon, arity = 1 def result(self) -> List[str]: - result = (self.int_a * self.r_inv) % self.int_n + result = self.from_montgomery(self.int_a) return [self.format_result(result)] From 895d5df5e82dcf8efdc4184e3b99643ca27422c7 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Tue, 6 Dec 2022 12:36:00 +0000 Subject: [PATCH 246/490] Add type annotations Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 81bc28e19..3ff8b2f35 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -251,10 +251,10 @@ class ModOperationCommon(OperationCommon): # provides earlier/more robust input validation. self.int_n = hex_to_int(val_n) - def to_montgomery(self, val) -> int: + def to_montgomery(self, val: int) -> int: return (val * self.r) % self.int_n - def from_montgomery(self, val) -> int: + def from_montgomery(self, val: int) -> int: return (val * self.r_inv) % self.int_n @property From 94f917bb805387b7d4ea2785a8d96e9c3d0157e7 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Tue, 1 Nov 2022 15:46:16 +0000 Subject: [PATCH 247/490] Add script to run Uncrustify Signed-off-by: David Horstmann --- scripts/code_style.py | 153 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100755 scripts/code_style.py diff --git a/scripts/code_style.py b/scripts/code_style.py new file mode 100755 index 000000000..7923db451 --- /dev/null +++ b/scripts/code_style.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Check or fix the code style by running Uncrustify. +""" +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import io +import os +import subprocess +import sys +from typing import List + +CONFIG_FILE = "codestyle.cfg" +UNCRUSTIFY_EXE = "uncrustify" +UNCRUSTIFY_ARGS = ["-c", CONFIG_FILE] +STDOUT_UTF8 = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') +STDERR_UTF8 = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8') + +def get_src_files() -> List[str]: + """ + Use git ls-files to get a list of the source files + """ + git_ls_files_cmd = ["git", "ls-files", \ + "*.[hc]", \ + "tests/suites/*.function", \ + "scripts/data_files/*.fmt"] + + result = subprocess.run(git_ls_files_cmd, stdout=subprocess.PIPE, \ + stderr=STDERR_UTF8, check=False) + + if result.returncode != 0: + print("Error: git ls-files returned: "+str(result.returncode), \ + file=STDERR_UTF8) + return [] + else: + src_files = str(result.stdout, "utf-8").split() + # Don't correct style for files in 3rdparty/ + src_files = list(filter( \ + lambda filename: not filename.startswith("3rdparty/"), \ + src_files)) + return src_files + +def get_uncrustify_version() -> str: + """ + Get the version string from Uncrustify + """ + version_args = ["--version"] + result = subprocess.run([UNCRUSTIFY_EXE] + version_args, \ + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if result.returncode != 0: + print("Error getting version: "+str(result.stderr, "utf-8"), \ + file=STDERR_UTF8) + return "" + else: + return str(result.stdout, "utf-8") + +def check_style_is_correct(src_file_list: List[str]) -> bool: + """ + Check the code style and output a diff foir each file whose style is + incorrect. + """ + style_correct = True + for src_file in src_file_list: + uncrustify_cmd = [UNCRUSTIFY_EXE] + UNCRUSTIFY_ARGS + [src_file] + subprocess.run(uncrustify_cmd, stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, check=False) + + # Uncrustify makes changes to the code and places the result in a new + # file with the extension ".uncrustify". To get the changes (if any) + # simply diff the 2 files. + diff_cmd = ["diff", "-u", src_file, src_file+".uncrustify"] + result = subprocess.run(diff_cmd, stdout=subprocess.PIPE, \ + stderr=STDERR_UTF8, check=False) + if len(result.stdout) > 0: + print(src_file+" - Incorrect code style.", file=STDOUT_UTF8) + print("File changed - diff:", file=STDOUT_UTF8) + print(str(result.stdout, "utf-8"), file=STDOUT_UTF8) + style_correct = False + else: + print(src_file+" - OK.", file=STDOUT_UTF8) + + # Tidy up artifact + os.remove(src_file+".uncrustify") + + return style_correct + +def fix_style_single_pass(src_file_list: List[str]) -> None: + """ + Run Uncrustify once over the source files. + """ + code_change_args = UNCRUSTIFY_ARGS + ["--no-backup"] + for src_file in src_file_list: + uncrustify_cmd = [UNCRUSTIFY_EXE] + code_change_args + [src_file] + subprocess.run(uncrustify_cmd, check=False, stdout=STDOUT_UTF8, \ + stderr=STDERR_UTF8) + +def fix_style(src_file_list: List[str]) -> int: + """ + Fix the code style. This takes 2 passes of Uncrustify. + """ + fix_style_single_pass(src_file_list) + fix_style_single_pass(src_file_list) + + # Guard against future changes that cause the codebase to require + # more passes. + if not check_style_is_correct(src_file_list): + print("Error: Code style still incorrect after second run of Uncrustify.", \ + file=STDERR_UTF8) + return 1 + else: + return 0 + +def main() -> int: + """ + Main with command line arguments. + """ + uncrustify_version = get_uncrustify_version() + if "0.75.1" not in uncrustify_version: + print("Warning: Using unsupported Uncrustify version '" \ + + uncrustify_version + "'", file=STDOUT_UTF8) + + src_files = get_src_files() + + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--fix', action='store_true', \ + help='modify source files to fix the code style') + + args = parser.parse_args() + + if args.fix: + # Fix mode + return fix_style(src_files) + else: + # Check mode + if check_style_is_correct(src_files): + return 0 + else: + return 1 + +if __name__ == '__main__': + sys.exit(main()) From ccd0d4453e9e20d6af3a35a913023da9d8b58c7c Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Thu, 8 Dec 2022 09:44:10 +0000 Subject: [PATCH 248/490] Bignum: Implement mbedtls_mpi_mod_raw_inv_prime() and tests Fixes #6023. Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_common.py | 4 +++- scripts/framework_dev/bignum_data.py | 14 ++++++++--- scripts/framework_dev/bignum_mod_raw.py | 31 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 3ff8b2f35..0339b1ad1 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -99,6 +99,7 @@ class OperationCommon(test_data_generation.BaseTest): limb_sizes = [32, 64] # type: List[int] arities = [1, 2] arity = 2 + suffix = False # for arity = 1, symbol can be prefix (default) or suffix def __init__(self, val_a: str, val_b: str = "0", bits_in_limb: int = 32) -> None: self.val_a = val_a @@ -170,7 +171,8 @@ class OperationCommon(test_data_generation.BaseTest): """ if not self.case_description: if self.arity == 1: - self.case_description = "{} {:x}".format( + format_string = "{1:x} {0}" if self.suffix else "{0} {1:x}" + self.case_description = format_string.format( self.symbol, self.int_a ) elif self.arity == 2: diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py index e6ed30005..965893320 100644 --- a/scripts/framework_dev/bignum_data.py +++ b/scripts/framework_dev/bignum_data.py @@ -90,8 +90,8 @@ RANDOM_1024_BIT_SEED_4_NO5 = ("53be4721f5b9e1f5acdac615bc20f6264922b9ccf469aef8" "4708d9893a973000b54a23020fc5b043d6e4a51519d9c9cc" "52d32377e78131c1") -# Adding 192 bit and 1024 bit numbers because these are the shortest required -# for ECC and RSA respectively. +# Adding 192 bit and 1024 bit numbers because these are the shortest required +# for ECC and RSA respectively. INPUTS_DEFAULT = [ "0", "1", # corner cases "2", "3", # small primes @@ -110,13 +110,21 @@ INPUTS_DEFAULT = [ # supported for now. MODULI_DEFAULT = [ "53", # safe prime - "45", # non-prime + "45", # non-prime SAFE_PRIME_192_BIT_SEED_1, # safe prime RANDOM_192_BIT_SEED_2_NO4, # not a prime SAFE_PRIME_1024_BIT_SEED_3, # safe prime RANDOM_1024_BIT_SEED_4_NO5, # not a prime ] +# Some functions, e.g. mbedtls_mpi_mod_raw_inv_prime(), only support prime moduli. +ONLY_PRIME_MODULI = [ + "53", # safe prime + "8ac72304057392b5", # 9999999997777777333 (longer, not safe, prime) + SAFE_PRIME_192_BIT_SEED_1, # safe prime + SAFE_PRIME_1024_BIT_SEED_3, # safe prime + ] + def __gen_safe_prime(bits, seed): ''' Generate a safe prime. diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index d05479a00..1a23a60ea 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -18,6 +18,7 @@ from typing import Dict, List from . import test_data_generation from . import bignum_common +from .bignum_data import ONLY_PRIME_MODULI class BignumModRawTarget(test_data_generation.BaseTarget): #pylint: disable=abstract-method, too-few-public-methods @@ -53,6 +54,36 @@ class BignumModRawSub(bignum_common.ModOperationCommon, # BEGIN MERGE SLOT 3 +class BignumModRawInvPrime(bignum_common.ModOperationCommon, + BignumModRawTarget): + """Test cases for bignum mpi_mod_raw_inv_prime().""" + moduli = ONLY_PRIME_MODULI + symbol = "^ -1" + test_function = "mpi_mod_raw_inv_prime" + test_name = "mbedtls_mpi_mod_raw_inv_prime (Montgomery form only)" + input_style = "fixed" + arity = 1 + suffix = True + + @property + def is_valid(self) -> bool: + return self.int_a > 0 and self.int_a < self.int_n + + def arguments(self) -> List[str]: + # Input has to be given in Montgomery form + mont_a = self.to_montgomery(self.int_a) + arg_mont_a = self.format_arg('{:x}'.format(mont_a)) + return [bignum_common.quote_str(n) for n in [self.arg_n, + arg_mont_a] + ] + self.result() + + def result(self) -> List[str]: + result = bignum_common.invmod(self.int_a, self.int_n) + if result < 0: + result += self.int_n + mont_result = self.to_montgomery(result) + return [self.format_result(mont_result)] + # END MERGE SLOT 3 # BEGIN MERGE SLOT 4 From 1244dc78436fa9e761e8a7bf9ef95f37cfc8ab6e Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 8 Dec 2022 13:12:21 +0000 Subject: [PATCH 249/490] Miscellaneous improvements to code style script Signed-off-by: David Horstmann --- scripts/code_style.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 7923db451..7de223a16 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -32,9 +32,9 @@ def get_src_files() -> List[str]: """ Use git ls-files to get a list of the source files """ - git_ls_files_cmd = ["git", "ls-files", \ - "*.[hc]", \ - "tests/suites/*.function", \ + git_ls_files_cmd = ["git", "ls-files", + "*.[hc]", + "tests/suites/*.function", "scripts/data_files/*.fmt"] result = subprocess.run(git_ls_files_cmd, stdout=subprocess.PIPE, \ @@ -56,8 +56,7 @@ def get_uncrustify_version() -> str: """ Get the version string from Uncrustify """ - version_args = ["--version"] - result = subprocess.run([UNCRUSTIFY_EXE] + version_args, \ + result = subprocess.run([UNCRUSTIFY_EXE, "--version"], \ stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if result.returncode != 0: print("Error getting version: "+str(result.stderr, "utf-8"), \ From 33be369438848edd398a622dfc807560e4033911 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 8 Dec 2022 14:33:52 +0000 Subject: [PATCH 250/490] Use helper function for error printing Signed-off-by: David Horstmann --- scripts/code_style.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 7de223a16..068298aec 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -28,6 +28,9 @@ UNCRUSTIFY_ARGS = ["-c", CONFIG_FILE] STDOUT_UTF8 = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') STDERR_UTF8 = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8') +def print_err(*args): + print("Error: ", *args, file=STDERR_UTF8) + def get_src_files() -> List[str]: """ Use git ls-files to get a list of the source files @@ -41,8 +44,7 @@ def get_src_files() -> List[str]: stderr=STDERR_UTF8, check=False) if result.returncode != 0: - print("Error: git ls-files returned: "+str(result.returncode), \ - file=STDERR_UTF8) + print_err("git ls-files returned: "+str(result.returncode)) return [] else: src_files = str(result.stdout, "utf-8").split() @@ -59,8 +61,7 @@ def get_uncrustify_version() -> str: result = subprocess.run([UNCRUSTIFY_EXE, "--version"], \ stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if result.returncode != 0: - print("Error getting version: "+str(result.stderr, "utf-8"), \ - file=STDERR_UTF8) + print_err("Could not get Uncrustify version:", str(result.stderr, "utf-8")) return "" else: return str(result.stdout, "utf-8") @@ -115,8 +116,7 @@ def fix_style(src_file_list: List[str]) -> int: # Guard against future changes that cause the codebase to require # more passes. if not check_style_is_correct(src_file_list): - print("Error: Code style still incorrect after second run of Uncrustify.", \ - file=STDERR_UTF8) + print("Code style still incorrect after second run of Uncrustify.") return 1 else: return 0 From bd602bfff7eb5e9ad5d0f6d04ea20e09fbca1cfd Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 8 Dec 2022 14:36:10 +0000 Subject: [PATCH 251/490] Fix typo in code style script Signed-off-by: David Horstmann --- scripts/code_style.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 068298aec..be333f093 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -68,7 +68,7 @@ def get_uncrustify_version() -> str: def check_style_is_correct(src_file_list: List[str]) -> bool: """ - Check the code style and output a diff foir each file whose style is + Check the code style and output a diff for each file whose style is incorrect. """ style_correct = True From 384baec8628e0a33874aff0858546969fe751155 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 8 Dec 2022 14:44:36 +0000 Subject: [PATCH 252/490] Use constant for supported Uncrustify version Define and report the supported Uncrustify version (and remove extra newlines from version output). Signed-off-by: David Horstmann --- scripts/code_style.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index be333f093..4aa9a40a0 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -22,6 +22,7 @@ import subprocess import sys from typing import List +UNCRUSTIFY_SUPPORTED_VERSION = "0.75.1" CONFIG_FILE = "codestyle.cfg" UNCRUSTIFY_EXE = "uncrustify" UNCRUSTIFY_ARGS = ["-c", CONFIG_FILE] @@ -125,10 +126,11 @@ def main() -> int: """ Main with command line arguments. """ - uncrustify_version = get_uncrustify_version() - if "0.75.1" not in uncrustify_version: + uncrustify_version = get_uncrustify_version().strip() + if UNCRUSTIFY_SUPPORTED_VERSION not in uncrustify_version: print("Warning: Using unsupported Uncrustify version '" \ - + uncrustify_version + "'", file=STDOUT_UTF8) + + uncrustify_version + "' (Note: The only supported version" \ + "is " + UNCRUSTIFY_SUPPORTED_VERSION + ")", file=STDOUT_UTF8) src_files = get_src_files() From 95cc15e4b703e10e2ff4b6d5af0390130e768bc0 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 8 Dec 2022 14:56:18 +0000 Subject: [PATCH 253/490] Explain that the script is only for the future Signed-off-by: David Horstmann --- scripts/code_style.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/code_style.py b/scripts/code_style.py index 4aa9a40a0..f93df7b6f 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 """Check or fix the code style by running Uncrustify. + +Note: The code style enforced by this script is not yet introduced to +Mbed TLS. At present this script will only be used to prepare for a future +change of code style. """ # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 From 00f1df84eb1873e29a75b5206d9723c1ed356db4 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 8 Dec 2022 15:04:20 +0000 Subject: [PATCH 254/490] Add spaces around '+' Signed-off-by: David Horstmann --- scripts/code_style.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index f93df7b6f..588484856 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -49,7 +49,7 @@ def get_src_files() -> List[str]: stderr=STDERR_UTF8, check=False) if result.returncode != 0: - print_err("git ls-files returned: "+str(result.returncode)) + print_err("git ls-files returned: " + str(result.returncode)) return [] else: src_files = str(result.stdout, "utf-8").split() @@ -85,19 +85,19 @@ def check_style_is_correct(src_file_list: List[str]) -> bool: # Uncrustify makes changes to the code and places the result in a new # file with the extension ".uncrustify". To get the changes (if any) # simply diff the 2 files. - diff_cmd = ["diff", "-u", src_file, src_file+".uncrustify"] + diff_cmd = ["diff", "-u", src_file, src_file + ".uncrustify"] result = subprocess.run(diff_cmd, stdout=subprocess.PIPE, \ stderr=STDERR_UTF8, check=False) if len(result.stdout) > 0: - print(src_file+" - Incorrect code style.", file=STDOUT_UTF8) + print(src_file + " - Incorrect code style.", file=STDOUT_UTF8) print("File changed - diff:", file=STDOUT_UTF8) print(str(result.stdout, "utf-8"), file=STDOUT_UTF8) style_correct = False else: - print(src_file+" - OK.", file=STDOUT_UTF8) + print(src_file + " - OK.", file=STDOUT_UTF8) # Tidy up artifact - os.remove(src_file+".uncrustify") + os.remove(src_file + ".uncrustify") return style_correct From 4085b067c280deb6ac2c7fa47d4eda8b45abdaff Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 8 Dec 2022 17:03:01 +0000 Subject: [PATCH 255/490] Fixup: Config file name in code style script Signed-off-by: David Horstmann --- scripts/code_style.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 588484856..ebf0cd48e 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -27,7 +27,7 @@ import sys from typing import List UNCRUSTIFY_SUPPORTED_VERSION = "0.75.1" -CONFIG_FILE = "codestyle.cfg" +CONFIG_FILE = ".uncrustify.cfg" UNCRUSTIFY_EXE = "uncrustify" UNCRUSTIFY_ARGS = ["-c", CONFIG_FILE] STDOUT_UTF8 = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') From 82858900d8a1c0ad564b028307c885f4e29aa90b Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 8 Dec 2022 17:38:27 +0000 Subject: [PATCH 256/490] Reindent line continuations for pylint Signed-off-by: David Horstmann --- scripts/code_style.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index ebf0cd48e..68cd55620 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -41,9 +41,9 @@ def get_src_files() -> List[str]: Use git ls-files to get a list of the source files """ git_ls_files_cmd = ["git", "ls-files", - "*.[hc]", - "tests/suites/*.function", - "scripts/data_files/*.fmt"] + "*.[hc]", + "tests/suites/*.function", + "scripts/data_files/*.fmt"] result = subprocess.run(git_ls_files_cmd, stdout=subprocess.PIPE, \ stderr=STDERR_UTF8, check=False) From 7c4a9211df1b5e86a18e863d8eeba93d8c16be87 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Fri, 9 Dec 2022 10:58:46 +0000 Subject: [PATCH 257/490] Have BignumModRawInvPrime() do Montgomery conversion in arg_a() Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_mod_raw.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 1a23a60ea..048642667 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -69,13 +69,11 @@ class BignumModRawInvPrime(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return self.int_a > 0 and self.int_a < self.int_n - def arguments(self) -> List[str]: + @property + def arg_a(self) -> str: # Input has to be given in Montgomery form mont_a = self.to_montgomery(self.int_a) - arg_mont_a = self.format_arg('{:x}'.format(mont_a)) - return [bignum_common.quote_str(n) for n in [self.arg_n, - arg_mont_a] - ] + self.result() + return self.format_arg('{:x}'.format(mont_a)) def result(self) -> List[str]: result = bignum_common.invmod(self.int_a, self.int_n) From f3e57e83da89656d80ade22b7929a7f953def91f Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 7 Dec 2022 18:10:46 +0000 Subject: [PATCH 258/490] bignum_mod_raw.py: Added BignumModRawModNegate. This patch adds autogenerated inputs for the `mpi_mod_raw_neg()` test in the bignum_mod_raw suite. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 0bbad5dd9..34d26f9bb 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -109,7 +109,18 @@ class BignumModRawConvertFromMont(bignum_common.ModOperationCommon, result = (self.int_a * self.r_inv) % self.int_n return [self.format_result(result)] +class BignumModRawModNegate(bignum_common.ModOperationCommon, + BignumModRawTarget): + """ Test cases for mpi_mod_raw_neg(). """ + test_function = "mpi_mod_raw_neg" + test_name = "Modular negation: " + symbol = "(-A)" + input_style = "arch_split" + arity = 1 + def result(self) -> List[str]: + result = (self.int_n - self.int_a) % self.int_n + return [self.format_result(result)] # END MERGE SLOT 7 # BEGIN MERGE SLOT 8 From 2e38c1e04769a927d93a81fac6bc4ad2dae1efb7 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Thu, 8 Dec 2022 11:48:26 +0000 Subject: [PATCH 259/490] bignum_mod_raw.py: Changed the symbol for modular negation to "-". Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 34d26f9bb..c3cb70bed 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -114,7 +114,7 @@ class BignumModRawModNegate(bignum_common.ModOperationCommon, """ Test cases for mpi_mod_raw_neg(). """ test_function = "mpi_mod_raw_neg" test_name = "Modular negation: " - symbol = "(-A)" + symbol = "-" input_style = "arch_split" arity = 1 From 05d5f45ba7b0ddf65c0ed8780a9ada77613c692b Mon Sep 17 00:00:00 2001 From: Werner Lewis Date: Mon, 12 Dec 2022 17:04:16 +0000 Subject: [PATCH 260/490] Refactor mpi_core_sub tests to use arch_split Tests are refactored to generate separate cases for 32-bit and 64-bit limbs using arch_split. Duplicate arguments and branching in the test function is removed. Signed-off-by: Werner Lewis --- scripts/framework_dev/bignum_core.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 118a659cf..158ada99d 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -130,24 +130,20 @@ class BignumCoreAddAndAddIf(BignumCoreTarget, bignum_common.OperationCommon): class BignumCoreSub(BignumCoreTarget, bignum_common.OperationCommon): """Test cases for bignum core sub.""" count = 0 + input_style = "arch_split" symbol = "-" test_function = "mpi_core_sub" test_name = "mbedtls_mpi_core_sub" def result(self) -> List[str]: if self.int_a >= self.int_b: - result_4 = result_8 = self.int_a - self.int_b + result = self.int_a - self.int_b carry = 0 else: - bound_val = max(self.int_a, self.int_b) - bound_4 = bignum_common.bound_mpi(bound_val, 32) - result_4 = bound_4 + self.int_a - self.int_b - bound_8 = bignum_common.bound_mpi(bound_val, 64) - result_8 = bound_8 + self.int_a - self.int_b + result = self.limb_boundary + self.int_a - self.int_b carry = 1 return [ - "\"{:x}\"".format(result_4), - "\"{:x}\"".format(result_8), + self.format_result(result), str(carry) ] From fa9cea300eb0602c2c53fbaaf9cccd4808dde400 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Wed, 7 Dec 2022 16:04:15 +0100 Subject: [PATCH 261/490] Add generated tests for mod_raw_mul Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod_raw.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 6fc4c919b..29e3339f5 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -50,6 +50,25 @@ class BignumModRawSub(bignum_common.ModOperationCommon, result = (self.int_a - self.int_b) % self.int_n return [self.format_result(result)] +class BignumModRawMul(bignum_common.ModOperationCommon, + BignumModRawTarget): + """Test cases for bignum mpi_mod_raw_mul().""" + symbol = "*" + test_function = "mpi_mod_raw_mul" + test_name = "mbedtls_mpi_mod_raw_mul" + input_style = "arch_split" + arity = 2 + + def arguments(self) -> List[str]: + return [bignum_common.quote_str(n) for n in [self.arg_a, + self.arg_b, + self.arg_n] + ] + self.result() + + def result(self) -> List[str]: + result = (self.int_a * self.int_b) % self.int_n + return [self.format_result(result)] + # END MERGE SLOT 2 # BEGIN MERGE SLOT 3 From 7fe60095c1d6305156a229af81a17700baa65bca Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Mon, 12 Dec 2022 16:54:57 +0000 Subject: [PATCH 262/490] Add mbedtls_mpi_core_check_zero_ct() and tests Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_core.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 158ada99d..1a8c22bfa 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -818,6 +818,20 @@ class BignumCoreSubInt(BignumCoreTarget, bignum_common.OperationCommon): str(-borrow) ] +class BignumCoreZeroCheckCT(BignumCoreTarget, bignum_common.OperationCommon): + """Test cases for bignum core zero check (constant flow).""" + count = 0 + symbol = "== 0" + test_function = "mpi_core_check_zero_ct" + test_name = "mpi_core_check_zero_ct" + input_style = "variable" + arity = 1 + suffix = True + + def result(self) -> List[str]: + result = 1 if self.int_a == 0 else 0 + return [str(result)] + # END MERGE SLOT 3 # BEGIN MERGE SLOT 4 From d818d40ca3552f835988a97ced6333bd8c89652a Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Wed, 14 Dec 2022 08:27:18 +0000 Subject: [PATCH 263/490] mbedtls_mpi_mod_raw_inv_prime() tests should be arch_split Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_data.py | 3 +++ scripts/framework_dev/bignum_mod_raw.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py index 965893320..0a48e538d 100644 --- a/scripts/framework_dev/bignum_data.py +++ b/scripts/framework_dev/bignum_data.py @@ -121,6 +121,9 @@ MODULI_DEFAULT = [ ONLY_PRIME_MODULI = [ "53", # safe prime "8ac72304057392b5", # 9999999997777777333 (longer, not safe, prime) + # The next prime has a different R in Montgomery form depending on + # whether 32- or 64-bit MPIs are used. + "152d02c7e14af67fe0bf", # 99999999999999999991999 SAFE_PRIME_192_BIT_SEED_1, # safe prime SAFE_PRIME_1024_BIT_SEED_3, # safe prime ] diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 6fc4c919b..0084898be 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -61,7 +61,7 @@ class BignumModRawInvPrime(bignum_common.ModOperationCommon, symbol = "^ -1" test_function = "mpi_mod_raw_inv_prime" test_name = "mbedtls_mpi_mod_raw_inv_prime (Montgomery form only)" - input_style = "fixed" + input_style = "arch_split" arity = 1 suffix = True From 460f5c3f383eb87688222de3ee7d2833bdec6e62 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Thu, 15 Dec 2022 16:59:40 +0000 Subject: [PATCH 264/490] Add tests for mbedtls_mpi_mod_inv() Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_mod.py | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index a16699a64..9f98131bd 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -18,6 +18,7 @@ from typing import Dict, List from . import test_data_generation from . import bignum_common +from .bignum_data import ONLY_PRIME_MODULI class BignumModTarget(test_data_generation.BaseTarget): #pylint: disable=abstract-method, too-few-public-methods @@ -48,6 +49,56 @@ class BignumModSub(bignum_common.ModOperationCommon, BignumModTarget): # generated cases return [self.format_result(result), "0"] +class BignumModInvNonMont(bignum_common.ModOperationCommon, BignumModTarget): + """Test cases for bignum mpi_mod_inv() - not in Montgomery form.""" + moduli = ONLY_PRIME_MODULI # for now only prime moduli supported + symbol = "^ -1" + test_function = "mpi_mod_inv_non_mont" + test_name = "mbedtls_mpi_mod_inv non-Mont. form" + input_style = "fixed" + arity = 1 + suffix = True + + @property + def is_valid(self) -> bool: + return self.int_a > 0 and self.int_a < self.int_n + + def result(self) -> List[str]: + result = bignum_common.invmod(self.int_a, self.int_n) + if result < 0: + result += self.int_n + # To make negative tests easier, append 0 for success to the + # generated cases + return [self.format_result(result), "0"] + +class BignumModInvMont(bignum_common.ModOperationCommon, BignumModTarget): + """Test cases for bignum mpi_mod_inv() - Montgomery form.""" + moduli = ONLY_PRIME_MODULI # for now only prime moduli supported + symbol = "^ -1" + test_function = "mpi_mod_inv_mont" + test_name = "mbedtls_mpi_mod_inv Mont. form" + input_style = "arch_split" # Mont. form requires arch_split + arity = 1 + suffix = True + + @property + def is_valid(self) -> bool: + return self.int_a > 0 and self.int_a < self.int_n + + @property + def arg_a(self) -> str: + mont_a = self.to_montgomery(self.int_a) + return self.format_arg('{:x}'.format(mont_a)) + + def result(self) -> List[str]: + result = bignum_common.invmod(self.int_a, self.int_n) + if result < 0: + result += self.int_n + mont_result = self.to_montgomery(result) + # To make negative tests easier, append 0 for success to the + # generated cases + return [self.format_result(mont_result), "0"] + # END MERGE SLOT 3 # BEGIN MERGE SLOT 4 From e2228085d15ff6fa153df567b7d86b57000ef61c Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Fri, 16 Dec 2022 03:53:17 +0000 Subject: [PATCH 265/490] Attempt to pacify pylint in bignum tests Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_common.py | 17 +++++++++++++++++ scripts/framework_dev/bignum_core.py | 10 +--------- scripts/framework_dev/bignum_mod.py | 24 +++++------------------- scripts/framework_dev/bignum_mod_raw.py | 16 +++------------- 4 files changed, 26 insertions(+), 41 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 0339b1ad1..dd4fc3684 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -39,6 +39,11 @@ def invmod(a: int, n: int) -> int: return b raise ValueError("Not invertible") +def invmod_positive(a: int, n: int) -> int: + """Return a non-negative inverse of a to modulo n.""" + inv = invmod(a, n) + return inv if inv >= 0 else inv + n + def hex_to_int(val: str) -> int: """Implement the syntax accepted by mbedtls_test_read_mpi(). @@ -244,6 +249,8 @@ class ModOperationCommon(OperationCommon): #pylint: disable=abstract-method """Target for bignum mod_raw test case generation.""" moduli = MODULI_DEFAULT # type: List[str] + mongtomgery_form_a = False + disallow_zero_a = False def __init__(self, val_n: str, val_a: str, val_b: str = "0", bits_in_limb: int = 64) -> None: @@ -263,6 +270,14 @@ class ModOperationCommon(OperationCommon): def boundary(self) -> int: return self.int_n + @property + def arg_a(self) -> str: + if self.mongtomgery_form_a: + value_a = self.to_montgomery(self.int_a) + else: + value_a = self.int_a + return self.format_arg('{:x}'.format(value_a)) + @property def arg_n(self) -> str: return self.format_arg(self.val_n) @@ -287,6 +302,8 @@ class ModOperationCommon(OperationCommon): def is_valid(self) -> bool: if self.int_a >= self.int_n: return False + if self.disallow_zero_a and self.int_a == 0: + return False if self.arity == 2 and self.int_b >= self.int_n: return False return True diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 1a8c22bfa..3bd7f111c 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -757,15 +757,7 @@ class BignumCoreExpMod(BignumCoreTarget, bignum_common.ModOperationCommon): test_function = "mpi_core_exp_mod" test_name = "Core modular exponentiation (Mongtomery form only)" input_style = "fixed" - - def arguments(self) -> List[str]: - # Input 'a' has to be given in Montgomery form - mont_a = self.to_montgomery(self.int_a) - arg_mont_a = self.format_arg('{:x}'.format(mont_a)) - return [bignum_common.quote_str(n) for n in [self.arg_n, - arg_mont_a, - self.arg_b] - ] + self.result() + mongtomgery_form_a = True def result(self) -> List[str]: # Result has to be given in Montgomery form too diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 9f98131bd..642887354 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -58,15 +58,10 @@ class BignumModInvNonMont(bignum_common.ModOperationCommon, BignumModTarget): input_style = "fixed" arity = 1 suffix = True - - @property - def is_valid(self) -> bool: - return self.int_a > 0 and self.int_a < self.int_n + disallow_zero_a = True def result(self) -> List[str]: - result = bignum_common.invmod(self.int_a, self.int_n) - if result < 0: - result += self.int_n + result = bignum_common.invmod_positive(self.int_a, self.int_n) # To make negative tests easier, append 0 for success to the # generated cases return [self.format_result(result), "0"] @@ -80,20 +75,11 @@ class BignumModInvMont(bignum_common.ModOperationCommon, BignumModTarget): input_style = "arch_split" # Mont. form requires arch_split arity = 1 suffix = True - - @property - def is_valid(self) -> bool: - return self.int_a > 0 and self.int_a < self.int_n - - @property - def arg_a(self) -> str: - mont_a = self.to_montgomery(self.int_a) - return self.format_arg('{:x}'.format(mont_a)) + disallow_zero_a = True + mongtomgery_form_a = True def result(self) -> List[str]: - result = bignum_common.invmod(self.int_a, self.int_n) - if result < 0: - result += self.int_n + result = bignum_common.invmod_positive(self.int_a, self.int_n) mont_result = self.to_montgomery(result) # To make negative tests easier, append 0 for success to the # generated cases diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 0084898be..461b1f2b9 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -64,21 +64,11 @@ class BignumModRawInvPrime(bignum_common.ModOperationCommon, input_style = "arch_split" arity = 1 suffix = True - - @property - def is_valid(self) -> bool: - return self.int_a > 0 and self.int_a < self.int_n - - @property - def arg_a(self) -> str: - # Input has to be given in Montgomery form - mont_a = self.to_montgomery(self.int_a) - return self.format_arg('{:x}'.format(mont_a)) + mongtomgery_form_a = True + disallow_zero_a = True def result(self) -> List[str]: - result = bignum_common.invmod(self.int_a, self.int_n) - if result < 0: - result += self.int_n + result = bignum_common.invmod_positive(self.int_a, self.int_n) mont_result = self.to_montgomery(result) return [self.format_result(mont_result)] From 4c60f0e3b070dac5546302ddf9ac691c4d3be7a3 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Thu, 15 Dec 2022 15:00:44 +0100 Subject: [PATCH 266/490] Generate operands in Mongomery representation for the test function Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod_raw.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 29e3339f5..296e2d2d1 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -60,14 +60,14 @@ class BignumModRawMul(bignum_common.ModOperationCommon, arity = 2 def arguments(self) -> List[str]: - return [bignum_common.quote_str(n) for n in [self.arg_a, - self.arg_b, - self.arg_n] + return [self.format_result(self.to_montgomery(self.int_a)), + self.format_result(self.to_montgomery(self.int_b)), + bignum_common.quote_str(self.arg_n) ] + self.result() def result(self) -> List[str]: result = (self.int_a * self.int_b) % self.int_n - return [self.format_result(result)] + return [self.format_result(self.to_montgomery(result))] # END MERGE SLOT 2 From f571cc9895f423f60b8fff7102135c58bc3ef002 Mon Sep 17 00:00:00 2001 From: Tom Cosgrove Date: Fri, 16 Dec 2022 16:10:36 +0000 Subject: [PATCH 267/490] Fix typos Signed-off-by: Tom Cosgrove --- scripts/framework_dev/bignum_common.py | 4 ++-- scripts/framework_dev/bignum_core.py | 2 +- scripts/framework_dev/bignum_mod.py | 2 +- scripts/framework_dev/bignum_mod_raw.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index dd4fc3684..c4efabfcc 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -249,7 +249,7 @@ class ModOperationCommon(OperationCommon): #pylint: disable=abstract-method """Target for bignum mod_raw test case generation.""" moduli = MODULI_DEFAULT # type: List[str] - mongtomgery_form_a = False + montgomery_form_a = False disallow_zero_a = False def __init__(self, val_n: str, val_a: str, val_b: str = "0", @@ -272,7 +272,7 @@ class ModOperationCommon(OperationCommon): @property def arg_a(self) -> str: - if self.mongtomgery_form_a: + if self.montgomery_form_a: value_a = self.to_montgomery(self.int_a) else: value_a = self.int_a diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 3bd7f111c..24d37cbc7 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -757,7 +757,7 @@ class BignumCoreExpMod(BignumCoreTarget, bignum_common.ModOperationCommon): test_function = "mpi_core_exp_mod" test_name = "Core modular exponentiation (Mongtomery form only)" input_style = "fixed" - mongtomgery_form_a = True + montgomery_form_a = True def result(self) -> List[str]: # Result has to be given in Montgomery form too diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 642887354..25afe3053 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -76,7 +76,7 @@ class BignumModInvMont(bignum_common.ModOperationCommon, BignumModTarget): arity = 1 suffix = True disallow_zero_a = True - mongtomgery_form_a = True + montgomery_form_a = True def result(self) -> List[str]: result = bignum_common.invmod_positive(self.int_a, self.int_n) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 461b1f2b9..ae855e829 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -64,7 +64,7 @@ class BignumModRawInvPrime(bignum_common.ModOperationCommon, input_style = "arch_split" arity = 1 suffix = True - mongtomgery_form_a = True + montgomery_form_a = True disallow_zero_a = True def result(self) -> List[str]: From 9e7ce40189ed54780b90e3d3422923d0f4b96386 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 19 Dec 2022 00:48:58 +0100 Subject: [PATCH 268/490] Don't touch the style of generated files Ideally the result of the generator would conform to the code style, but this would be difficult, especially with respect to the placement of line breaks in long logical lines. So, to avoid surprises when checking the style of generated files (which happens in releases and in long-time support branches), systematically skip generated files. Signed-off-by: Gilles Peskine --- scripts/code_style.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 68cd55620..8e82b93fb 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -22,9 +22,10 @@ change of code style. import argparse import io import os +import re import subprocess import sys -from typing import List +from typing import FrozenSet, List UNCRUSTIFY_SUPPORTED_VERSION = "0.75.1" CONFIG_FILE = ".uncrustify.cfg" @@ -32,10 +33,33 @@ UNCRUSTIFY_EXE = "uncrustify" UNCRUSTIFY_ARGS = ["-c", CONFIG_FILE] STDOUT_UTF8 = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') STDERR_UTF8 = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8') +CHECK_GENERATED_FILES = "tests/scripts/check-generated-files.sh" def print_err(*args): print("Error: ", *args, file=STDERR_UTF8) +# Match FILENAME(s) in "check SCRIPT (FILENAME...)" +CHECK_CALL_RE = re.compile(r"\n\s*check\s+[^\s#$&*?;|]+([^\n#$&*?;|]+)", + re.ASCII) +def list_generated_files() -> FrozenSet[str]: + """Return the names of generated files. + + We don't reformat generated files, since the result might be different + from the output of the generator. Ideally the result of the generator + would conform to the code style, but this would be difficult, especially + with respect to the placement of line breaks in long logical lines. + """ + # Parse check-generated-files.sh to get an up-to-date list of + # generated files. Read the file rather than calling it so that + # this script only depends on Git, Python and uncrustify, and not other + # tools such as sh or grep which might not be available on Windows. + # This introduces a limitation: check-generated-files.sh must have + # the expected format and must list the files explicitly, not through + # wildcards or command substitution. + content = open(CHECK_GENERATED_FILES, encoding="utf-8").read() + checks = re.findall(CHECK_CALL_RE, content) + return frozenset(word for s in checks for word in s.split()) + def get_src_files() -> List[str]: """ Use git ls-files to get a list of the source files @@ -52,11 +76,14 @@ def get_src_files() -> List[str]: print_err("git ls-files returned: " + str(result.returncode)) return [] else: + generated_files = list_generated_files() src_files = str(result.stdout, "utf-8").split() - # Don't correct style for files in 3rdparty/ - src_files = list(filter( \ - lambda filename: not filename.startswith("3rdparty/"), \ - src_files)) + # Don't correct style for third-party files (and, for simplicity, + # companion files in the same subtree), or for automatically + # generated files (we're correcting the templates instead). + src_files = [filename for filename in src_files + if not (filename.startswith("3rdparty/") or + filename in generated_files)] return src_files def get_uncrustify_version() -> str: From 5100c7acae4f092d0c7135d11ac2ecce7aea9938 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 13 Dec 2022 10:53:50 +0100 Subject: [PATCH 269/490] Add tests for mod_mul Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 25afe3053..e0fc15939 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -31,6 +31,25 @@ class BignumModTarget(test_data_generation.BaseTarget): # BEGIN MERGE SLOT 2 +class BignumModMul(bignum_common.ModOperationCommon, + BignumModTarget): + """Test cases for bignum mpi_mod_mul().""" + symbol = "*" + test_function = "mpi_mod_mul" + test_name = "mbedtls_mpi_mod_mul" + input_style = "arch_split" + arity = 2 + + def arguments(self) -> List[str]: + return [bignum_common.quote_str(n) for n in [self.arg_a, + self.arg_b, + self.arg_n] + ] + self.result() + + def result(self) -> List[str]: + result = (self.int_a * self.int_b) % self.int_n + return [self.format_result(result)] + # END MERGE SLOT 2 # BEGIN MERGE SLOT 3 From 3f90400d9be573b26570a26296362dd57f9d79f1 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Fri, 16 Dec 2022 15:25:02 +0100 Subject: [PATCH 270/490] Generate operands in Mongomery representation for the test function Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index e0fc15939..1960723f1 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -41,14 +41,14 @@ class BignumModMul(bignum_common.ModOperationCommon, arity = 2 def arguments(self) -> List[str]: - return [bignum_common.quote_str(n) for n in [self.arg_a, - self.arg_b, - self.arg_n] + return [self.format_result(self.to_montgomery(self.int_a)), + self.format_result(self.to_montgomery(self.int_b)), + bignum_common.quote_str(self.arg_n) ] + self.result() def result(self) -> List[str]: result = (self.int_a * self.int_b) % self.int_n - return [self.format_result(result)] + return [self.format_result(self.to_montgomery(result))] # END MERGE SLOT 2 From ffa2f407c14f82cfea1bf1d0a365ec276db4aa99 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Fri, 16 Dec 2022 17:18:28 +0100 Subject: [PATCH 271/490] Supress pylint's duplicated code warning Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 1960723f1..a83e1360a 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -33,6 +33,7 @@ class BignumModTarget(test_data_generation.BaseTarget): class BignumModMul(bignum_common.ModOperationCommon, BignumModTarget): + # pylint:disable=duplicate-code """Test cases for bignum mpi_mod_mul().""" symbol = "*" test_function = "mpi_mod_mul" From 64cddf76b0cc37742d84182b27a9d6bdc5120549 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 20 Dec 2022 19:12:22 +0100 Subject: [PATCH 272/490] Add some missing type annotations Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 0339b1ad1..9db567b7c 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -48,7 +48,7 @@ def hex_to_int(val: str) -> int: return 0 return int(val, 16) -def quote_str(val) -> str: +def quote_str(val: str) -> str: return "\"{}\"".format(val) def bound_mpi(val: int, bits_in_limb: int) -> int: @@ -134,7 +134,7 @@ class OperationCommon(test_data_generation.BaseTest): def hex_digits(self) -> int: return 2 * (self.limbs * self.bits_in_limb // 8) - def format_arg(self, val) -> str: + def format_arg(self, val: str) -> str: if self.input_style not in self.input_styles: raise ValueError("Unknown input style!") if self.input_style == "variable": @@ -142,7 +142,7 @@ class OperationCommon(test_data_generation.BaseTest): else: return val.zfill(self.hex_digits) - def format_result(self, res) -> str: + def format_result(self, res: int) -> str: res_str = '{:x}'.format(res) return quote_str(self.format_arg(res_str)) From cf5587b2e7a80788a7f6cf90762fc389ecfe0428 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 20 Dec 2022 19:16:54 +0100 Subject: [PATCH 273/490] Mod operations: fill arguments to the width of the modulus With the default input style (which is "variable"), fill all bignum test case arguments to the same width as the modulus. Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 9db567b7c..03055f02b 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -267,6 +267,12 @@ class ModOperationCommon(OperationCommon): def arg_n(self) -> str: return self.format_arg(self.val_n) + def format_arg(self, val: str) -> str: + if self.input_style == "variable": + return val.zfill(len(hex(self.int_n)) - 2) + else: + return super().format_arg(val) + def arguments(self) -> List[str]: return [quote_str(self.arg_n)] + super().arguments() From 01466686ef01949ad85ccfd79dcb56377922a1a5 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 20 Dec 2022 19:19:18 +0100 Subject: [PATCH 274/490] Helpers for generating representation-aware test cases Add a class for modulus representations (mbedtls_mpi_mod_rep_selector). Add a method to convert a number to any representation. Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_common.py | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 03055f02b..7b2669af7 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -15,6 +15,7 @@ # limitations under the License. from abc import abstractmethod +import enum from typing import Iterator, List, Tuple, TypeVar, Any from itertools import chain @@ -240,6 +241,23 @@ class OperationCommon(test_data_generation.BaseTest): ) +class ModulusRepresentation(enum.Enum): + """Representation selector of a modulus.""" + # Numerical values aligned with the type mbedtls_mpi_mod_rep_selector + INVALID = 0 + MONTGOMERY = 2 + OPT_RED = 3 + + def symbol(self) -> str: + """The C symbol for this representation selector.""" + return 'MBEDTLS_MPI_MOD_REP_' + self.name + + @classmethod + def supported_representations(cls) -> List['ModulusRepresentation']: + """Return all representations that are supported in positive test cases.""" + return [cls.MONTGOMERY, cls.OPT_RED] + + class ModOperationCommon(OperationCommon): #pylint: disable=abstract-method """Target for bignum mod_raw test case generation.""" @@ -259,6 +277,17 @@ class ModOperationCommon(OperationCommon): def from_montgomery(self, val: int) -> int: return (val * self.r_inv) % self.int_n + def convert_from_canonical(self, canonical: int, + rep: ModulusRepresentation) -> int: + """Convert values from canonical representation to the given representation.""" + if rep is ModulusRepresentation.MONTGOMERY: + return self.to_montgomery(canonical) + elif rep is ModulusRepresentation.OPT_RED: + return canonical + else: + raise ValueError('Modulus representation not supported: {}' + .format(rep.name)) + @property def boundary(self) -> int: return self.int_n From 05c96e4e55ed5615428e822c04ca1e53fa81ff07 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 20 Dec 2022 19:30:47 +0100 Subject: [PATCH 275/490] Generate test cases for mpi_mod_raw_canonical_to_modulus_rep Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_mod_raw.py | 36 ++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 6fc4c919b..c50458149 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -14,8 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, List +from typing import Dict, Iterator, List +from . import test_case from . import test_data_generation from . import bignum_common from .bignum_data import ONLY_PRIME_MODULI @@ -107,6 +108,39 @@ class BignumModRawAdd(bignum_common.ModOperationCommon, # BEGIN MERGE SLOT 6 +class BignumModRawCanonicalToModulusRep(bignum_common.ModOperationCommon, + BignumModRawTarget): + """Test cases for mpi_mod_raw_canonical_to_modulus_rep.""" + test_function = "mpi_mod_raw_canonical_to_modulus_rep" + test_name = "Rep canon->mod" + arity = 1 + + def __init__(self, + val_n: str, val_a: str, + rep: bignum_common.ModulusRepresentation) -> None: + super().__init__(val_n=val_n, val_a=val_a) + self.rep = rep + + def result(self) -> List[str]: + result = self.convert_from_canonical(self.int_a, self.rep) + return [self.format_result(result)] + + def arguments(self) -> List[str]: + return ([bignum_common.quote_str(self.arg_n), self.rep.symbol(), + bignum_common.quote_str(self.arg_a)] + + self.result()) + + @classmethod + def generate_function_tests(cls) -> Iterator[test_case.TestCase]: + representations = \ + bignum_common.ModulusRepresentation.supported_representations() + for rep in representations: + for n in cls.moduli: + for a in cls.input_values: + test_object = cls(n, a, rep) + if test_object.is_valid: + yield test_object.create_test_case() + # END MERGE SLOT 6 # BEGIN MERGE SLOT 7 From 1241e5b2cc72d4051aab845cea0bcd91c2e2220a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 20 Dec 2022 19:51:22 +0100 Subject: [PATCH 276/490] Generate test cases for mpi_mod_raw_modulus_to_canonical_rep as well Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_mod_raw.py | 39 ++++++++++++++++++------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index c50458149..800bcb13e 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -108,23 +108,18 @@ class BignumModRawAdd(bignum_common.ModOperationCommon, # BEGIN MERGE SLOT 6 -class BignumModRawCanonicalToModulusRep(bignum_common.ModOperationCommon, - BignumModRawTarget): - """Test cases for mpi_mod_raw_canonical_to_modulus_rep.""" - test_function = "mpi_mod_raw_canonical_to_modulus_rep" - test_name = "Rep canon->mod" +class BignumModRawConvertRep(bignum_common.ModOperationCommon, + BignumModRawTarget): + # This is an abstract class, it's ok to have unimplemented methods. + #pylint: disable=abstract-method + """Test cases for representation conversion.""" arity = 1 - def __init__(self, - val_n: str, val_a: str, + def __init__(self, val_n: str, val_a: str, rep: bignum_common.ModulusRepresentation) -> None: super().__init__(val_n=val_n, val_a=val_a) self.rep = rep - def result(self) -> List[str]: - result = self.convert_from_canonical(self.int_a, self.rep) - return [self.format_result(result)] - def arguments(self) -> List[str]: return ([bignum_common.quote_str(self.arg_n), self.rep.symbol(), bignum_common.quote_str(self.arg_a)] + @@ -141,6 +136,28 @@ class BignumModRawCanonicalToModulusRep(bignum_common.ModOperationCommon, if test_object.is_valid: yield test_object.create_test_case() +class BignumModRawCanonicalToModulusRep(BignumModRawConvertRep): + """Test cases for mpi_mod_raw_canonical_to_modulus_rep.""" + test_function = "mpi_mod_raw_canonical_to_modulus_rep" + test_name = "Rep canon->mod" + + def result(self) -> List[str]: + result = self.convert_from_canonical(self.int_a, self.rep) + return [self.format_result(result)] + +class BignumModRawModulusToCanonicalRep(BignumModRawConvertRep): + """Test cases for mpi_mod_raw_modulus_to_canonical_rep.""" + test_function = "mpi_mod_raw_modulus_to_canonical_rep" + test_name = "Rep mod->canon" + + @property + def arg_a(self) -> str: + converted_a = self.convert_from_canonical(self.int_a, self.rep) + return self.format_arg(hex(converted_a)[2:]) + + def result(self) -> List[str]: + return [self.format_result(self.int_a)] + # END MERGE SLOT 6 # BEGIN MERGE SLOT 7 From 73bea827e802763839661980c0bad6f91b35fb68 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 20 Dec 2022 22:39:15 +0100 Subject: [PATCH 277/490] Fix representation conversion with 32-bit limbs The Montgomery representation depends on the limb size. So the representation conversion test cases need separate 64-bit and 32-bit cases when the representation is Montgomery. Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_mod_raw.py | 26 +++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 800bcb13e..e108fe3a0 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Iterator, List +from typing import Iterator, List, Optional, Union from . import test_case from . import test_data_generation @@ -115,9 +115,13 @@ class BignumModRawConvertRep(bignum_common.ModOperationCommon, """Test cases for representation conversion.""" arity = 1 - def __init__(self, val_n: str, val_a: str, + def __init__(self, val_n: str, val_a: str, bits_in_limb: Optional[int], rep: bignum_common.ModulusRepresentation) -> None: - super().__init__(val_n=val_n, val_a=val_a) + if bits_in_limb is None: + super().__init__(val_n=val_n, val_a=val_a) + else: + self.input_style = "arch_split" + super().__init__(val_n=val_n, val_a=val_a, bits_in_limb=bits_in_limb) self.rep = rep def arguments(self) -> List[str]: @@ -125,16 +129,26 @@ class BignumModRawConvertRep(bignum_common.ModOperationCommon, bignum_common.quote_str(self.arg_a)] + self.result()) + def description(self) -> str: + base = super().description() + mod_with_rep = 'mod({})'.format(self.rep.name) + return base.replace('mod', mod_with_rep, 1) + @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: representations = \ bignum_common.ModulusRepresentation.supported_representations() for rep in representations: + if rep is bignum_common.ModulusRepresentation.MONTGOMERY: + limb_sizes = cls.limb_sizes #type: Union[List[int], List[None]] + else: + limb_sizes = [None] # no dependency on limb size for n in cls.moduli: for a in cls.input_values: - test_object = cls(n, a, rep) - if test_object.is_valid: - yield test_object.create_test_case() + for bil in limb_sizes: + test_object = cls(n, a, bil, rep) + if test_object.is_valid: + yield test_object.create_test_case() class BignumModRawCanonicalToModulusRep(BignumModRawConvertRep): """Test cases for mpi_mod_raw_canonical_to_modulus_rep.""" From 456cd39e5af88cbc23928f9701e0e69b01df9681 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 21 Dec 2022 17:30:10 +0000 Subject: [PATCH 278/490] bignum_common: Adjusted `format_arg` to always size input according to modulo. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_common.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 7b2669af7..bb64ce079 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -297,10 +297,7 @@ class ModOperationCommon(OperationCommon): return self.format_arg(self.val_n) def format_arg(self, val: str) -> str: - if self.input_style == "variable": - return val.zfill(len(hex(self.int_n)) - 2) - else: - return super().format_arg(val) + return super().format_arg(val).zfill(self.hex_digits) def arguments(self) -> List[str]: return [quote_str(self.arg_n)] + super().arguments() From 6441c384f2aa254393b20f9ae6c019fa8b4a7c00 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 21 Dec 2022 17:31:56 +0000 Subject: [PATCH 279/490] bignum_mod_raw: Simplified `BignumModRawCanonicalToFromModulusRep` output expressions. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index e108fe3a0..a2336f95a 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -156,8 +156,7 @@ class BignumModRawCanonicalToModulusRep(BignumModRawConvertRep): test_name = "Rep canon->mod" def result(self) -> List[str]: - result = self.convert_from_canonical(self.int_a, self.rep) - return [self.format_result(result)] + return [self.format_result(self.convert_from_canonical(self.int_a, self.rep))] class BignumModRawModulusToCanonicalRep(BignumModRawConvertRep): """Test cases for mpi_mod_raw_modulus_to_canonical_rep.""" @@ -166,8 +165,7 @@ class BignumModRawModulusToCanonicalRep(BignumModRawConvertRep): @property def arg_a(self) -> str: - converted_a = self.convert_from_canonical(self.int_a, self.rep) - return self.format_arg(hex(converted_a)[2:]) + return self.format_arg("{:x}".format(self.convert_from_canonical(self.int_a, self.rep))) def result(self) -> List[str]: return [self.format_result(self.int_a)] From 23d2c6512e696f6bf10d3d20aa8471bb3e2f5edd Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 21 Dec 2022 17:34:15 +0000 Subject: [PATCH 280/490] bignum_common.py: Introduce the set_representation setter. This patch adds the default representation attribute through a setter() method in `BignumModRawConvertRep()` It also adds standard common template properties: symbol = "" input_style = "arch_split" arity = 1 Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index a2336f95a..b244e0973 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -113,16 +113,13 @@ class BignumModRawConvertRep(bignum_common.ModOperationCommon, # This is an abstract class, it's ok to have unimplemented methods. #pylint: disable=abstract-method """Test cases for representation conversion.""" + symbol = "" + input_style = "arch_split" arity = 1 + rep = bignum_common.ModulusRepresentation.INVALID - def __init__(self, val_n: str, val_a: str, bits_in_limb: Optional[int], - rep: bignum_common.ModulusRepresentation) -> None: - if bits_in_limb is None: - super().__init__(val_n=val_n, val_a=val_a) - else: - self.input_style = "arch_split" - super().__init__(val_n=val_n, val_a=val_a, bits_in_limb=bits_in_limb) - self.rep = rep + def set_representation(self, r: bignum_common.ModulusRepresentation) -> bool: + self.rep = r def arguments(self) -> List[str]: return ([bignum_common.quote_str(self.arg_n), self.rep.symbol(), From e65933d5c1d8e5e1875dd38b0828631e3c04a34e Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 21 Dec 2022 17:38:16 +0000 Subject: [PATCH 281/490] bignum_common.py: Refactored `BignumModRawConvertRep.generate_function_tests()` This patch adjusts the test generating method to calculate all possible combinations for (modulo, input, limb_sizes, representation). Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index b244e0973..ebbc970e5 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -133,17 +133,13 @@ class BignumModRawConvertRep(bignum_common.ModOperationCommon, @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: - representations = \ - bignum_common.ModulusRepresentation.supported_representations() - for rep in representations: - if rep is bignum_common.ModulusRepresentation.MONTGOMERY: - limb_sizes = cls.limb_sizes #type: Union[List[int], List[None]] - else: - limb_sizes = [None] # no dependency on limb size + + for rep in bignum_common.ModulusRepresentation.supported_representations(): for n in cls.moduli: for a in cls.input_values: - for bil in limb_sizes: - test_object = cls(n, a, bil, rep) + for bil in cls.limb_sizes: + test_object = cls(n, a, bits_in_limb=bil) + test_object.set_representation(rep) if test_object.is_valid: yield test_object.create_test_case() From e0ca288cd2202f8ebaee95d1a50cf9be0c3bb62c Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 21 Dec 2022 17:41:30 +0000 Subject: [PATCH 282/490] bignum_mod_raw.py: Added a filtering logic to `BignumModRawConvertRep.generate_function_tests()` This patch introduces a hybrid approach to input_styles, and will remove the dependency requirements from test cases with `ModulusRepresentation.OPT_RED` As a result it is reducing testing input duplication. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_mod_raw.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index ebbc970e5..98605d655 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -140,6 +140,11 @@ class BignumModRawConvertRep(bignum_common.ModOperationCommon, for bil in cls.limb_sizes: test_object = cls(n, a, bits_in_limb=bil) test_object.set_representation(rep) + #Filters out the duplicate + if rep == bignum_common.ModulusRepresentation.OPT_RED: + test_object.dependencies= [] + if bil == 64: + continue if test_object.is_valid: yield test_object.create_test_case() From ce700dd00adaff07114471ffa8caf48ab539771e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Dec 2022 20:12:31 +0100 Subject: [PATCH 283/490] Fix type declaration Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_mod_raw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 98605d655..89ae825fc 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -118,7 +118,7 @@ class BignumModRawConvertRep(bignum_common.ModOperationCommon, arity = 1 rep = bignum_common.ModulusRepresentation.INVALID - def set_representation(self, r: bignum_common.ModulusRepresentation) -> bool: + def set_representation(self, r: bignum_common.ModulusRepresentation) -> None: self.rep = r def arguments(self) -> List[str]: From 69cce1bfe0f3e51ec7b310276274e3fa36449aa9 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Dec 2022 20:18:23 +0100 Subject: [PATCH 284/490] Split the high nesting of BignumModRawConvertRep.generate_function_tests Pylint complains about the nesting. It's not wrong. No behavior change. Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_mod_raw.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 89ae825fc..352e438ab 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -131,22 +131,27 @@ class BignumModRawConvertRep(bignum_common.ModOperationCommon, mod_with_rep = 'mod({})'.format(self.rep.name) return base.replace('mod', mod_with_rep, 1) + @classmethod + def test_cases_for_values(cls, rep: bignum_common.ModulusRepresentation, + n: str, a: str) -> Iterator[test_case.TestCase]: + for bil in cls.limb_sizes: + test_object = cls(n, a, bits_in_limb=bil) + test_object.set_representation(rep) + #Filters out the duplicate + if rep == bignum_common.ModulusRepresentation.OPT_RED: + test_object.dependencies= [] + if bil == 64: + continue + if test_object.is_valid: + yield test_object.create_test_case() + @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: for rep in bignum_common.ModulusRepresentation.supported_representations(): for n in cls.moduli: for a in cls.input_values: - for bil in cls.limb_sizes: - test_object = cls(n, a, bits_in_limb=bil) - test_object.set_representation(rep) - #Filters out the duplicate - if rep == bignum_common.ModulusRepresentation.OPT_RED: - test_object.dependencies= [] - if bil == 64: - continue - if test_object.is_valid: - yield test_object.create_test_case() + yield from cls.test_cases_for_values(rep, n, a) class BignumModRawCanonicalToModulusRep(BignumModRawConvertRep): """Test cases for mpi_mod_raw_canonical_to_modulus_rep.""" From 82d44392dc122c26fd5f5eb0f34183beef94f225 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Dec 2022 20:20:44 +0100 Subject: [PATCH 285/490] Pacify pylint Except for missing documentation, which will come in a subsequent commit. Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_mod_raw.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 352e438ab..4628bffe4 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Iterator, List, Optional, Union +from typing import Iterator, List from . import test_case from . import test_data_generation @@ -139,7 +139,7 @@ class BignumModRawConvertRep(bignum_common.ModOperationCommon, test_object.set_representation(rep) #Filters out the duplicate if rep == bignum_common.ModulusRepresentation.OPT_RED: - test_object.dependencies= [] + test_object.dependencies = [] if bil == 64: continue if test_object.is_valid: From c6b69e08165a1cb0ccda48ca4ae53a4ca94f1387 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Dec 2022 20:28:29 +0100 Subject: [PATCH 286/490] Simplify logic and document test_cases_for_values Explain what's going on in BignumModRawConvertRep.test_case_for_values. Simplify the logic and the interdependencies related to limb sizes: * Montgomery is the special case, so base the decisions on it. * As soon as we've encountered one limb size, no matter what it is, give up. No behavior change, other than changing the numbering of test cases (which previously included more skipped test cases). Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_mod_raw.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 4628bffe4..21add3baa 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -134,17 +134,32 @@ class BignumModRawConvertRep(bignum_common.ModOperationCommon, @classmethod def test_cases_for_values(cls, rep: bignum_common.ModulusRepresentation, n: str, a: str) -> Iterator[test_case.TestCase]: + """Emit test cases for the given values (if any). + + This may emit no test cases if a isn't valid for the modulus n, + or multiple test cases if rep requires different data depending + on the limb size. + """ for bil in cls.limb_sizes: test_object = cls(n, a, bits_in_limb=bil) test_object.set_representation(rep) - #Filters out the duplicate - if rep == bignum_common.ModulusRepresentation.OPT_RED: + # The class is set to having separate test cases for each limb + # size, because the Montgomery representation requires it. + # But other representations don't require it. So for other + # representations, emit a single test case with no dependency + # on the limb size. + if rep is not bignum_common.ModulusRepresentation.MONTGOMERY: test_object.dependencies = [] - if bil == 64: - continue if test_object.is_valid: yield test_object.create_test_case() + if rep is not bignum_common.ModulusRepresentation.MONTGOMERY: + # A single test case (emitted, or skipped due to invalidity) + # is enough, since this test case doesn't depend on the + # limb size. + break + # The parent class doesn't support non-bignum parameters. So we override + # test generation, in order to have the representation as a parameter. @classmethod def generate_function_tests(cls) -> Iterator[test_case.TestCase]: From 1642bcae213d9a097caa46d1983152fd6c65dcc2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 21 Dec 2022 20:33:30 +0100 Subject: [PATCH 287/490] More robust dependency filtering Only remove the MBEDTLS_HAVE_INTnn dependency, not any other dependency that might be present. No behavior change, this is just robustness. Signed-off-by: Gilles Peskine --- scripts/framework_dev/bignum_mod_raw.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 21add3baa..fa95e24fd 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -149,7 +149,9 @@ class BignumModRawConvertRep(bignum_common.ModOperationCommon, # representations, emit a single test case with no dependency # on the limb size. if rep is not bignum_common.ModulusRepresentation.MONTGOMERY: - test_object.dependencies = [] + test_object.dependencies = \ + [dep for dep in test_object.dependencies + if not dep.startswith('MBEDTLS_HAVE_INT')] if test_object.is_valid: yield test_object.create_test_case() if rep is not bignum_common.ModulusRepresentation.MONTGOMERY: From 1a1deb85a1bd88dc4747f344d138e095c09ba2c2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 22 Dec 2022 16:34:01 +0100 Subject: [PATCH 288/490] Support restyling only the specified files Signed-off-by: Gilles Peskine --- scripts/code_style.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 8e82b93fb..77f6a1b6a 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -163,14 +163,26 @@ def main() -> int: + uncrustify_version + "' (Note: The only supported version" \ "is " + UNCRUSTIFY_SUPPORTED_VERSION + ")", file=STDOUT_UTF8) - src_files = get_src_files() - parser = argparse.ArgumentParser() - parser.add_argument('-f', '--fix', action='store_true', \ - help='modify source files to fix the code style') + parser.add_argument('-f', '--fix', action='store_true', + help='modify source files to fix the code style') + # --files is almost useless: it only matters if there are no files + # ('code_style.py' without arguments checks all files known to Git, + # 'code_style.py --files' does nothing). 'code_style.py --files ...' is + # intended as a stable ("porcelain") way to restyle a possibly empty + # set of files. + parser.add_argument('--files', action='store_true', + help='only check the specified files (default with non-option arguments)') + parser.add_argument('operands', nargs='*', metavar='FILE', + help='files to check (if none: check files that are known to git)') args = parser.parse_args() + if args.files or args.operands: + src_files = args.operands + else: + src_files = get_src_files() + if args.fix: # Fix mode return fix_style(src_files) From c46bb0a929a5c6f4cf4f44fbc9dcd81fa7d797fb Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 23 Dec 2022 18:15:19 +0100 Subject: [PATCH 289/490] Documentation improvements Signed-off-by: Gilles Peskine --- scripts/code_style.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 77f6a1b6a..23bb21b3a 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -159,18 +159,20 @@ def main() -> int: """ uncrustify_version = get_uncrustify_version().strip() if UNCRUSTIFY_SUPPORTED_VERSION not in uncrustify_version: - print("Warning: Using unsupported Uncrustify version '" \ - + uncrustify_version + "' (Note: The only supported version" \ - "is " + UNCRUSTIFY_SUPPORTED_VERSION + ")", file=STDOUT_UTF8) + print("Warning: Using unsupported Uncrustify version '" + + uncrustify_version + "'", file=STDOUT_UTF8) + print("Note: The only supported version is " + + UNCRUSTIFY_SUPPORTED_VERSION, file=STDOUT_UTF8) parser = argparse.ArgumentParser() parser.add_argument('-f', '--fix', action='store_true', - help='modify source files to fix the code style') + help=('modify source files to fix the code style ' + '(default: print diff, do not modify files)')) # --files is almost useless: it only matters if there are no files # ('code_style.py' without arguments checks all files known to Git, - # 'code_style.py --files' does nothing). 'code_style.py --files ...' is - # intended as a stable ("porcelain") way to restyle a possibly empty - # set of files. + # 'code_style.py --files' does nothing). In particular, + # 'code_style.py --files ...' is intended as a stable ("porcelain") way + # to restyle a possibly empty set of files. parser.add_argument('--files', action='store_true', help='only check the specified files (default with non-option arguments)') parser.add_argument('operands', nargs='*', metavar='FILE', From 50597976fc731207b4acd1d2a379c0b898333c0f Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Wed, 4 Jan 2023 18:33:25 +0000 Subject: [PATCH 290/490] Check Uncrustify returncode in code_style.py Signed-off-by: David Horstmann --- scripts/code_style.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 8e82b93fb..a23d3a5ce 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -106,8 +106,12 @@ def check_style_is_correct(src_file_list: List[str]) -> bool: style_correct = True for src_file in src_file_list: uncrustify_cmd = [UNCRUSTIFY_EXE] + UNCRUSTIFY_ARGS + [src_file] - subprocess.run(uncrustify_cmd, stdout=subprocess.PIPE, \ + result = subprocess.run(uncrustify_cmd, stdout=subprocess.PIPE, \ stderr=subprocess.PIPE, check=False) + if result.returncode != 0: + print_err("Uncrustify returned " + str(result.returncode) + \ + " correcting file " + src_file) + return False # Uncrustify makes changes to the code and places the result in a new # file with the extension ".uncrustify". To get the changes (if any) @@ -135,15 +139,22 @@ def fix_style_single_pass(src_file_list: List[str]) -> None: code_change_args = UNCRUSTIFY_ARGS + ["--no-backup"] for src_file in src_file_list: uncrustify_cmd = [UNCRUSTIFY_EXE] + code_change_args + [src_file] - subprocess.run(uncrustify_cmd, check=False, stdout=STDOUT_UTF8, \ - stderr=STDERR_UTF8) + result = subprocess.run(uncrustify_cmd, check=False, \ + stdout=STDOUT_UTF8, stderr=STDERR_UTF8) + if result.returncode != 0: + print_err("Uncrustify with file returned: " + \ + str(result.returncode) + " correcting file " + \ + src_file) + return False def fix_style(src_file_list: List[str]) -> int: """ Fix the code style. This takes 2 passes of Uncrustify. """ - fix_style_single_pass(src_file_list) - fix_style_single_pass(src_file_list) + if fix_style_single_pass(src_file_list) != True: + return 1 + if fix_style_single_pass(src_file_list) != True: + return 1 # Guard against future changes that cause the codebase to require # more passes. From 3b1c55d7ca4e9261be5d78a298324a5617009b85 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 5 Jan 2023 09:59:35 +0000 Subject: [PATCH 291/490] Fix incorrect typing of function in code_style.py Signed-off-by: David Horstmann --- scripts/code_style.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index a23d3a5ce..2c7a77966 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -132,7 +132,7 @@ def check_style_is_correct(src_file_list: List[str]) -> bool: return style_correct -def fix_style_single_pass(src_file_list: List[str]) -> None: +def fix_style_single_pass(src_file_list: List[str]) -> bool: """ Run Uncrustify once over the source files. """ @@ -146,6 +146,7 @@ def fix_style_single_pass(src_file_list: List[str]) -> None: str(result.returncode) + " correcting file " + \ src_file) return False + return True def fix_style(src_file_list: List[str]) -> int: """ From 50dfacaa7e2139ac08e4ff3d8793f4864d3e71cf Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 5 Jan 2023 10:02:09 +0000 Subject: [PATCH 292/490] Fix pylint warnings about comparison to True Signed-off-by: David Horstmann --- scripts/code_style.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 2c7a77966..aae3e2492 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -152,9 +152,9 @@ def fix_style(src_file_list: List[str]) -> int: """ Fix the code style. This takes 2 passes of Uncrustify. """ - if fix_style_single_pass(src_file_list) != True: + if not fix_style_single_pass(src_file_list): return 1 - if fix_style_single_pass(src_file_list) != True: + if not fix_style_single_pass(src_file_list): return 1 # Guard against future changes that cause the codebase to require From 8dc85d91075c94d2636e618dedeb278300b8f761 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 12 Jan 2023 15:45:32 +0100 Subject: [PATCH 293/490] Fix example command Signed-off-by: Gilles Peskine --- scripts/code_style.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 23bb21b3a..a0f35c7c2 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -171,8 +171,8 @@ def main() -> int: # --files is almost useless: it only matters if there are no files # ('code_style.py' without arguments checks all files known to Git, # 'code_style.py --files' does nothing). In particular, - # 'code_style.py --files ...' is intended as a stable ("porcelain") way - # to restyle a possibly empty set of files. + # 'code_style.py --fix --files ...' is intended as a stable ("porcelain") + # way to restyle a possibly empty set of files. parser.add_argument('--files', action='store_true', help='only check the specified files (default with non-option arguments)') parser.add_argument('operands', nargs='*', metavar='FILE', From 990bb782cccf8f66ea02609e67d071b9e6da9764 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 16 Jan 2023 16:53:29 +0100 Subject: [PATCH 294/490] Add test generation support for the ecp module Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 26 ++++++++++++++++++++++++++ scripts/framework_dev/ecp_common.py | 23 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 scripts/framework_dev/ecp.py create mode 100644 scripts/framework_dev/ecp_common.py diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py new file mode 100644 index 000000000..59a0f31d2 --- /dev/null +++ b/scripts/framework_dev/ecp.py @@ -0,0 +1,26 @@ +"""Framework classes for generation of ecp test cases.""" +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +from . import test_case +from . import test_data_generation +from . import ecp_common + +class EcpTarget(test_data_generation.BaseTarget): + #pylint: disable=abstract-method, too-few-public-methods + """Target for ecp test case generation.""" + target_basename = 'test_suite_ecp.generated' diff --git a/scripts/framework_dev/ecp_common.py b/scripts/framework_dev/ecp_common.py new file mode 100644 index 000000000..1f2ba0aae --- /dev/null +++ b/scripts/framework_dev/ecp_common.py @@ -0,0 +1,23 @@ +"""Common features for ecp in test generation framework.""" +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import test_case +from . import test_data_generation +from . import bignum_common + +class EcpOperationCommon(bignum_common.ModOperationCommon): + """Target for ecp test case generation.""" + pass \ No newline at end of file From 12d70a92eeef87b8e7b177d777bb84555a199f53 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 16 Jan 2023 16:54:48 +0100 Subject: [PATCH 295/490] Add generated test for ecp quasi-reduction Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 59a0f31d2..713dd96df 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -24,3 +24,30 @@ class EcpTarget(test_data_generation.BaseTarget): #pylint: disable=abstract-method, too-few-public-methods """Target for ecp test case generation.""" target_basename = 'test_suite_ecp.generated' + +class EcpQuasiReduction(ecp_common.EcpOperationCommon, + EcpTarget): + """Test cases for ecp quasi_reduction().""" + symbol = "-" + test_function = "ecp_quasi_reduction" + test_name = "mbedtls_ecp_quasi_reduction" + input_style = "fixed" + arity = 1 + + # Extend the default values with n < x < 2n + input_values = ecp_common.EcpOperationCommon.input_values + [ + "73", + "ebeddd7b4fefae8755bbfb9c181a73347096b3ec70d1a021", + ("1f4e1d074d0b50e8d8818f9a9e5df9959f902bb955fd24fd3d791175226ad8c1" + "fcb6d59fa41a3dcb25412009e5e356eb65b50ca67782285290420b45b32f0d63" + "7c9ee549a52ad8d631ba4945435c9aec77227ec59faff878b71b920a3d631929" + "d636c9a409d6ffdcd95e2568e128596811fb9ade15e69f6efd509381ebbf3599") + ] # type: List[str] + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return bool(self.int_a < 2 * self.int_n) From 9b226cb9baba5bd4291204431b9ab06eb64855a9 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 17 Jan 2023 16:34:24 +0100 Subject: [PATCH 296/490] Fix pylint issues Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 15 +++++++-------- scripts/framework_dev/ecp_common.py | 4 +--- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 713dd96df..f9d6af51d 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -16,7 +16,6 @@ from typing import List -from . import test_case from . import test_data_generation from . import ecp_common @@ -36,13 +35,13 @@ class EcpQuasiReduction(ecp_common.EcpOperationCommon, # Extend the default values with n < x < 2n input_values = ecp_common.EcpOperationCommon.input_values + [ - "73", - "ebeddd7b4fefae8755bbfb9c181a73347096b3ec70d1a021", - ("1f4e1d074d0b50e8d8818f9a9e5df9959f902bb955fd24fd3d791175226ad8c1" - "fcb6d59fa41a3dcb25412009e5e356eb65b50ca67782285290420b45b32f0d63" - "7c9ee549a52ad8d631ba4945435c9aec77227ec59faff878b71b920a3d631929" - "d636c9a409d6ffdcd95e2568e128596811fb9ade15e69f6efd509381ebbf3599") - ] # type: List[str] + "73", + "ebeddd7b4fefae8755bbfb9c181a73347096b3ec70d1a021", + ("1f4e1d074d0b50e8d8818f9a9e5df9959f902bb955fd24fd3d791175226ad8c1" + "fcb6d59fa41a3dcb25412009e5e356eb65b50ca67782285290420b45b32f0d63" + "7c9ee549a52ad8d631ba4945435c9aec77227ec59faff878b71b920a3d631929" + "d636c9a409d6ffdcd95e2568e128596811fb9ade15e69f6efd509381ebbf3599") + ] # type: List[str] def result(self) -> List[str]: result = self.int_a % self.int_n diff --git a/scripts/framework_dev/ecp_common.py b/scripts/framework_dev/ecp_common.py index 1f2ba0aae..5b10a5dfc 100644 --- a/scripts/framework_dev/ecp_common.py +++ b/scripts/framework_dev/ecp_common.py @@ -14,10 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import test_case -from . import test_data_generation from . import bignum_common class EcpOperationCommon(bignum_common.ModOperationCommon): """Target for ecp test case generation.""" - pass \ No newline at end of file + pass From df4464a49e735bb84d35fb77dd22cd3f7d23fea9 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Wed, 18 Jan 2023 10:56:13 +0100 Subject: [PATCH 297/490] Fix lint issues Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp_common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/framework_dev/ecp_common.py b/scripts/framework_dev/ecp_common.py index 5b10a5dfc..311b9121c 100644 --- a/scripts/framework_dev/ecp_common.py +++ b/scripts/framework_dev/ecp_common.py @@ -17,5 +17,6 @@ from . import bignum_common class EcpOperationCommon(bignum_common.ModOperationCommon): + #pylint: disable=abstract-method """Target for ecp test case generation.""" pass From f048545f9e530bc9b0bcb10f53340faebe7b29b6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 15 Dec 2022 22:41:34 +0100 Subject: [PATCH 298/490] Refactoring: new method Algorithm.is_valid_for_operation No intended behavior change. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 1a033210b..a56c8638e 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -214,9 +214,7 @@ class KeyType: This function does not currently handle key derivation or PAKE. """ #pylint: disable=too-many-branches,too-many-return-statements - if alg.is_wildcard: - return False - if alg.is_invalid_truncation(): + if not alg.is_valid_for_operation(): return False if self.head == 'HMAC' and alg.head == 'HMAC': return True @@ -498,6 +496,19 @@ class Algorithm: return True return False + def is_valid_for_operation(self) -> bool: + """Whether this algorithm construction is valid for an operation. + + This function assumes that the algorithm is constructed in a + "grammatically" correct way, and only rejects semantically invalid + combinations. + """ + if self.is_wildcard: + return False + if self.is_invalid_truncation(): + return False + return True + def can_do(self, category: AlgorithmCategory) -> bool: """Whether this algorithm can perform operations in the given category. """ From 3325957ae46d72264a8e93c965ec956c17f4903a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 16 Dec 2022 00:20:50 +0100 Subject: [PATCH 299/490] A key agreement cannot be chained with PSA_ALG_TLS12_ECJPAKE_TO_PMS Test accordingly. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_knowledge.py | 35 +++++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index a56c8638e..3029c801d 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -246,6 +246,8 @@ class KeyType: # So a public key object with a key agreement algorithm is not # a valid combination. return False + if alg.is_invalid_key_agreement_with_derivation(): + return False if self.head == 'ECC': assert self.params is not None eccc = EllipticCurveCategory.from_family(self.params[0]) @@ -412,17 +414,38 @@ class Algorithm: self.category = self.determine_category(self.base_expression, self.head) self.is_wildcard = self.determine_wildcard(self.expression) - def is_key_agreement_with_derivation(self) -> bool: - """Whether this is a combined key agreement and key derivation algorithm.""" + def get_key_agreement_derivation(self) -> Optional[str]: + """For a combined key agreement and key derivation algorithm, get the derivation part. + + For anything else, return None. + """ if self.category != AlgorithmCategory.KEY_AGREEMENT: - return False + return None m = re.match(r'PSA_ALG_KEY_AGREEMENT\(\w+,\s*(.*)\)\Z', self.expression) if not m: - return False + return None kdf_alg = m.group(1) # Assume kdf_alg is either a valid KDF or 0. - return not re.match(r'(?:0[Xx])?0+\s*\Z', kdf_alg) + if re.match(r'(?:0[Xx])?0+\s*\Z', kdf_alg): + return None + return kdf_alg + KEY_DERIVATIONS_INCOMPATIBLE_WITH_AGREEMENT = frozenset([ + 'PSA_ALG_TLS12_ECJPAKE_TO_PMS', # secret input in specific format + ]) + def is_valid_key_agreement_with_derivation(self) -> bool: + """Whether this is a valid combined key agreement and key derivation algorithm.""" + kdf_alg = self.get_key_agreement_derivation() + if kdf_alg is None: + return False + return kdf_alg not in self.KEY_DERIVATIONS_INCOMPATIBLE_WITH_AGREEMENT + + def is_invalid_key_agreement_with_derivation(self) -> bool: + """Whether this is an invalid combined key agreement and key derivation algorithm.""" + kdf_alg = self.get_key_agreement_derivation() + if kdf_alg is None: + return False + return kdf_alg in self.KEY_DERIVATIONS_INCOMPATIBLE_WITH_AGREEMENT def short_expression(self, level: int = 0) -> str: """Abbreviate the expression, keeping it human-readable. @@ -515,7 +538,7 @@ class Algorithm: if category == self.category: return True if category == AlgorithmCategory.KEY_DERIVATION and \ - self.is_key_agreement_with_derivation(): + self.is_valid_key_agreement_with_derivation(): return True return False From bf903a0d2886e3cce31246c31b6f69a9c2176954 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 23 Jan 2023 16:13:43 +0100 Subject: [PATCH 300/490] Move the quasi reduction fixing function to bignum_mod_raw Rename the function to 'fix_quasi_reduction' to better suite its functionality. Also changed the name prefix to suite for the new module. Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod_raw.py | 27 +++++++++++++ scripts/framework_dev/ecp.py | 52 ------------------------- scripts/framework_dev/ecp_common.py | 22 ----------- 3 files changed, 27 insertions(+), 74 deletions(-) delete mode 100644 scripts/framework_dev/ecp.py delete mode 100644 scripts/framework_dev/ecp_common.py diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index f9d9899f6..5645685a5 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -51,6 +51,33 @@ class BignumModRawSub(bignum_common.ModOperationCommon, result = (self.int_a - self.int_b) % self.int_n return [self.format_result(result)] +class BignumModRawFixQuasiReduction(bignum_common.ModOperationCommon, + BignumModRawTarget): + """Test cases for ecp quasi_reduction().""" + symbol = "-" + test_function = "mpi_mod_raw_fix_quasi_reduction" + test_name = "mbedtls_mpi_mod_raw_fix_quasi_reduction" + input_style = "fixed" + arity = 1 + + # Extend the default values with n < x < 2n + input_values = bignum_common.ModOperationCommon.input_values + [ + "73", + "ebeddd7b4fefae8755bbfb9c181a73347096b3ec70d1a021", + ("1f4e1d074d0b50e8d8818f9a9e5df9959f902bb955fd24fd3d791175226ad8c1" + "fcb6d59fa41a3dcb25412009e5e356eb65b50ca67782285290420b45b32f0d63" + "7c9ee549a52ad8d631ba4945435c9aec77227ec59faff878b71b920a3d631929" + "d636c9a409d6ffdcd95e2568e128596811fb9ade15e69f6efd509381ebbf3599") + ] # type: List[str] + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return bool(self.int_a < 2 * self.int_n) + class BignumModRawMul(bignum_common.ModOperationCommon, BignumModRawTarget): """Test cases for bignum mpi_mod_raw_mul().""" diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py deleted file mode 100644 index f9d6af51d..000000000 --- a/scripts/framework_dev/ecp.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Framework classes for generation of ecp test cases.""" -# Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List - -from . import test_data_generation -from . import ecp_common - -class EcpTarget(test_data_generation.BaseTarget): - #pylint: disable=abstract-method, too-few-public-methods - """Target for ecp test case generation.""" - target_basename = 'test_suite_ecp.generated' - -class EcpQuasiReduction(ecp_common.EcpOperationCommon, - EcpTarget): - """Test cases for ecp quasi_reduction().""" - symbol = "-" - test_function = "ecp_quasi_reduction" - test_name = "mbedtls_ecp_quasi_reduction" - input_style = "fixed" - arity = 1 - - # Extend the default values with n < x < 2n - input_values = ecp_common.EcpOperationCommon.input_values + [ - "73", - "ebeddd7b4fefae8755bbfb9c181a73347096b3ec70d1a021", - ("1f4e1d074d0b50e8d8818f9a9e5df9959f902bb955fd24fd3d791175226ad8c1" - "fcb6d59fa41a3dcb25412009e5e356eb65b50ca67782285290420b45b32f0d63" - "7c9ee549a52ad8d631ba4945435c9aec77227ec59faff878b71b920a3d631929" - "d636c9a409d6ffdcd95e2568e128596811fb9ade15e69f6efd509381ebbf3599") - ] # type: List[str] - - def result(self) -> List[str]: - result = self.int_a % self.int_n - return [self.format_result(result)] - - @property - def is_valid(self) -> bool: - return bool(self.int_a < 2 * self.int_n) diff --git a/scripts/framework_dev/ecp_common.py b/scripts/framework_dev/ecp_common.py deleted file mode 100644 index 311b9121c..000000000 --- a/scripts/framework_dev/ecp_common.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Common features for ecp in test generation framework.""" -# Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from . import bignum_common - -class EcpOperationCommon(bignum_common.ModOperationCommon): - #pylint: disable=abstract-method - """Target for ecp test case generation.""" - pass From c4fe40de36e357acea153c5d1d683a89202883e7 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Mon, 16 Jan 2023 18:20:08 +0000 Subject: [PATCH 301/490] Remove provisional notice on code style script Since code style is now enforced, the notice is wrong. Remove it to avoid confusion. Signed-off-by: David Horstmann --- scripts/code_style.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 3958e870f..463a349ff 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -1,9 +1,5 @@ #!/usr/bin/env python3 """Check or fix the code style by running Uncrustify. - -Note: The code style enforced by this script is not yet introduced to -Mbed TLS. At present this script will only be used to prepare for a future -change of code style. """ # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 From edb88c4f7306e34bc1e4b1ebea62dfee9410836b Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Mon, 16 Jan 2023 18:28:21 +0000 Subject: [PATCH 302/490] Document that the script must be run from the root Signed-off-by: David Horstmann --- scripts/code_style.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/code_style.py b/scripts/code_style.py index 463a349ff..e26d42e9c 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 """Check or fix the code style by running Uncrustify. + +This script must be run from the root of a Git work tree containing Mbed TLS. """ # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 From 20694ab33358a3b20c2832cb45686b2aed85f31e Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Mon, 16 Jan 2023 18:32:56 +0000 Subject: [PATCH 303/490] Change print to print_err for an error message Signed-off-by: David Horstmann --- scripts/code_style.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index e26d42e9c..6222dbad4 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -158,7 +158,7 @@ def fix_style(src_file_list: List[str]) -> int: # Guard against future changes that cause the codebase to require # more passes. if not check_style_is_correct(src_file_list): - print("Code style still incorrect after second run of Uncrustify.") + print_err("Code style still incorrect after second run of Uncrustify.") return 1 else: return 0 From b9c3833a09bf8dc6cc9af97731ed1138aecef552 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Mon, 16 Jan 2023 18:53:01 +0000 Subject: [PATCH 304/490] Remove overly verbose output on success Signed-off-by: David Horstmann --- scripts/code_style.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 6222dbad4..5d05fd8a9 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -122,8 +122,6 @@ def check_style_is_correct(src_file_list: List[str]) -> bool: print("File changed - diff:", file=STDOUT_UTF8) print(str(result.stdout, "utf-8"), file=STDOUT_UTF8) style_correct = False - else: - print(src_file + " - OK.", file=STDOUT_UTF8) # Tidy up artifact os.remove(src_file + ".uncrustify") From 21157c0cfca224c014c0dd0014cc01fa49d1a223 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 24 Jan 2023 17:38:26 +0100 Subject: [PATCH 305/490] Fix pylint issues Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod_raw.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 5645685a5..4a9459460 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -62,13 +62,13 @@ class BignumModRawFixQuasiReduction(bignum_common.ModOperationCommon, # Extend the default values with n < x < 2n input_values = bignum_common.ModOperationCommon.input_values + [ - "73", - "ebeddd7b4fefae8755bbfb9c181a73347096b3ec70d1a021", - ("1f4e1d074d0b50e8d8818f9a9e5df9959f902bb955fd24fd3d791175226ad8c1" - "fcb6d59fa41a3dcb25412009e5e356eb65b50ca67782285290420b45b32f0d63" - "7c9ee549a52ad8d631ba4945435c9aec77227ec59faff878b71b920a3d631929" - "d636c9a409d6ffdcd95e2568e128596811fb9ade15e69f6efd509381ebbf3599") - ] # type: List[str] + "73", + "ebeddd7b4fefae8755bbfb9c181a73347096b3ec70d1a021", + ("1f4e1d074d0b50e8d8818f9a9e5df9959f902bb955fd24fd3d791175226ad8c1" + "fcb6d59fa41a3dcb25412009e5e356eb65b50ca67782285290420b45b32f0d63" + "7c9ee549a52ad8d631ba4945435c9aec77227ec59faff878b71b920a3d631929" + "d636c9a409d6ffdcd95e2568e128596811fb9ade15e69f6efd509381ebbf3599") + ] # type: List[str] def result(self) -> List[str]: result = self.int_a % self.int_n From 7dce673f1481cc2bfe518a5e73830c6dc300f745 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Tue, 24 Jan 2023 16:56:18 +0000 Subject: [PATCH 306/490] Add basic output on success Whilst it is true that "silence is golden", no output at all could be disconcerting and it makes searching in a CI log more difficult. Add a simple status message that says "Checked N files, style ok". Signed-off-by: David Horstmann --- scripts/code_style.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/code_style.py b/scripts/code_style.py index 5d05fd8a9..f333c643c 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -199,6 +199,8 @@ def main() -> int: else: # Check mode if check_style_is_correct(src_files): + print("Checked {} files, style ok.".format(len(src_files)), + file=STDOUT_UTF8) return 0 else: return 1 From 42c4850227296fba4ce86e324d299bbbc3ac7c82 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 24 Jan 2023 18:02:52 +0100 Subject: [PATCH 307/490] Use reproductable random numbers Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod_raw.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 4a9459460..69fe7b8e6 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -63,11 +63,15 @@ class BignumModRawFixQuasiReduction(bignum_common.ModOperationCommon, # Extend the default values with n < x < 2n input_values = bignum_common.ModOperationCommon.input_values + [ "73", - "ebeddd7b4fefae8755bbfb9c181a73347096b3ec70d1a021", - ("1f4e1d074d0b50e8d8818f9a9e5df9959f902bb955fd24fd3d791175226ad8c1" - "fcb6d59fa41a3dcb25412009e5e356eb65b50ca67782285290420b45b32f0d63" - "7c9ee549a52ad8d631ba4945435c9aec77227ec59faff878b71b920a3d631929" - "d636c9a409d6ffdcd95e2568e128596811fb9ade15e69f6efd509381ebbf3599") + + # First number generated by random.getrandbits(1024) - seed(3,2) + "ea7b5bf55eb561a4216363698b529b4a97b750923ceb3ffd", + + # First number generated by random.getrandbits(1024) - seed(1,2) + ("cd447e35b8b6d8fe442e3d437204e52db2221a58008a05a6c4647159c324c985" + "9b810e766ec9d28663ca828dd5f4b3b2e4b06ce60741c7a87ce42c8218072e8c" + "35bf992dc9e9c616612e7696a6cecc1b78e510617311d8a3c2ce6f447ed4d57b" + "1e2feb89414c343c1027c4d1c386bbc4cd613e30d8f16adf91b7584a2265b1f5") ] # type: List[str] def result(self) -> List[str]: From 2abb5388a080ff99aaef38b95555a8ce72d22b3d Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Tue, 24 Jan 2023 18:08:49 +0000 Subject: [PATCH 308/490] Output diff without capturing it Instead of capturing the output of diff and printing it, let diff do its own outputting and se the return code to decide what to do. This also means that the conversion of stdout to UTF-8 is not necessary, as the reason it was needed was for printing diffs of files with UTF-8 characters in them. Signed-off-by: David Horstmann --- scripts/code_style.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index f333c643c..d9c61a5af 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -115,13 +115,14 @@ def check_style_is_correct(src_file_list: List[str]) -> bool: # file with the extension ".uncrustify". To get the changes (if any) # simply diff the 2 files. diff_cmd = ["diff", "-u", src_file, src_file + ".uncrustify"] - result = subprocess.run(diff_cmd, stdout=subprocess.PIPE, \ - stderr=STDERR_UTF8, check=False) - if len(result.stdout) > 0: - print(src_file + " - Incorrect code style.", file=STDOUT_UTF8) - print("File changed - diff:", file=STDOUT_UTF8) - print(str(result.stdout, "utf-8"), file=STDOUT_UTF8) + cp = subprocess.run(diff_cmd, check=False) + + if cp.returncode == 1: + print(src_file + " changed - code style is incorrect.", file=STDOUT_UTF8) style_correct = False + elif cp.returncode != 0: + raise subprocess.CalledProcessError(cp.returncode, cp.args, + cp.stdout, cp.stderr) # Tidy up artifact os.remove(src_file + ".uncrustify") From 3a0d806e0ee13e7fc02508bf9d6c28b72f0fbd5b Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Tue, 24 Jan 2023 18:36:41 +0000 Subject: [PATCH 309/490] Don't wrap stdout and stderr in UTF-8 wrapper This is no longer needed as we only print ASCII text directly Signed-off-by: David Horstmann --- scripts/code_style.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index d9c61a5af..d3ae624ec 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -18,7 +18,6 @@ This script must be run from the root of a Git work tree containing Mbed TLS. # See the License for the specific language governing permissions and # limitations under the License. import argparse -import io import os import re import subprocess @@ -29,12 +28,10 @@ UNCRUSTIFY_SUPPORTED_VERSION = "0.75.1" CONFIG_FILE = ".uncrustify.cfg" UNCRUSTIFY_EXE = "uncrustify" UNCRUSTIFY_ARGS = ["-c", CONFIG_FILE] -STDOUT_UTF8 = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') -STDERR_UTF8 = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8') CHECK_GENERATED_FILES = "tests/scripts/check-generated-files.sh" def print_err(*args): - print("Error: ", *args, file=STDERR_UTF8) + print("Error: ", *args, file=sys.stderr) # Match FILENAME(s) in "check SCRIPT (FILENAME...)" CHECK_CALL_RE = re.compile(r"\n\s*check\s+[^\s#$&*?;|]+([^\n#$&*?;|]+)", @@ -67,8 +64,8 @@ def get_src_files() -> List[str]: "tests/suites/*.function", "scripts/data_files/*.fmt"] - result = subprocess.run(git_ls_files_cmd, stdout=subprocess.PIPE, \ - stderr=STDERR_UTF8, check=False) + result = subprocess.run(git_ls_files_cmd, stdout=subprocess.PIPE, + check=False) if result.returncode != 0: print_err("git ls-files returned: " + str(result.returncode)) @@ -118,7 +115,7 @@ def check_style_is_correct(src_file_list: List[str]) -> bool: cp = subprocess.run(diff_cmd, check=False) if cp.returncode == 1: - print(src_file + " changed - code style is incorrect.", file=STDOUT_UTF8) + print(src_file + " changed - code style is incorrect.") style_correct = False elif cp.returncode != 0: raise subprocess.CalledProcessError(cp.returncode, cp.args, @@ -136,8 +133,7 @@ def fix_style_single_pass(src_file_list: List[str]) -> bool: code_change_args = UNCRUSTIFY_ARGS + ["--no-backup"] for src_file in src_file_list: uncrustify_cmd = [UNCRUSTIFY_EXE] + code_change_args + [src_file] - result = subprocess.run(uncrustify_cmd, check=False, \ - stdout=STDOUT_UTF8, stderr=STDERR_UTF8) + result = subprocess.run(uncrustify_cmd, check=False) if result.returncode != 0: print_err("Uncrustify with file returned: " + \ str(result.returncode) + " correcting file " + \ @@ -169,9 +165,9 @@ def main() -> int: uncrustify_version = get_uncrustify_version().strip() if UNCRUSTIFY_SUPPORTED_VERSION not in uncrustify_version: print("Warning: Using unsupported Uncrustify version '" + - uncrustify_version + "'", file=STDOUT_UTF8) + uncrustify_version + "'") print("Note: The only supported version is " + - UNCRUSTIFY_SUPPORTED_VERSION, file=STDOUT_UTF8) + UNCRUSTIFY_SUPPORTED_VERSION) parser = argparse.ArgumentParser() parser.add_argument('-f', '--fix', action='store_true', @@ -200,8 +196,7 @@ def main() -> int: else: # Check mode if check_style_is_correct(src_files): - print("Checked {} files, style ok.".format(len(src_files)), - file=STDOUT_UTF8) + print("Checked {} files, style ok.".format(len(src_files))) return 0 else: return 1 From 2813823a811e5b903ab51735e1b1e2150da19509 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Tue, 24 Jan 2023 18:59:07 +0000 Subject: [PATCH 310/490] Give proper Dict type hints in crypto_knowledge.py This prevents a return type error in a later function that uses the dictionaries here properly typed. Signed-off-by: David Horstmann --- scripts/framework_dev/crypto_knowledge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 1a033210b..4dac33fb9 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -20,7 +20,7 @@ This module is entirely based on the PSA API. import enum import re -from typing import FrozenSet, Iterable, List, Optional, Tuple +from typing import FrozenSet, Iterable, List, Optional, Tuple, Dict from .asymmetric_key_data import ASYMMETRIC_KEY_DATA @@ -148,7 +148,7 @@ class KeyType: 'PSA_ECC_FAMILY_BRAINPOOL_P_R1': (160, 192, 224, 256, 320, 384, 512), 'PSA_ECC_FAMILY_MONTGOMERY': (255, 448), 'PSA_ECC_FAMILY_TWISTED_EDWARDS': (255, 448), - } + } # type: Dict[str, Tuple[int, ...]] KEY_TYPE_SIZES = { 'PSA_KEY_TYPE_AES': (128, 192, 256), # exhaustive 'PSA_KEY_TYPE_ARIA': (128, 192, 256), # exhaustive @@ -162,7 +162,7 @@ class KeyType: 'PSA_KEY_TYPE_PEPPER': (128, 256), # sample 'PSA_KEY_TYPE_RAW_DATA': (8, 40, 128), # sample 'PSA_KEY_TYPE_RSA_KEY_PAIR': (1024, 1536), # small sample - } + } # type: Dict[str, Tuple[int, ...]] def sizes_to_test(self) -> Tuple[int, ...]: """Return a tuple of key sizes to test. From f6866ceb7553dd542a3544695c1de2392af93e0e Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Wed, 25 Jan 2023 11:39:04 +0000 Subject: [PATCH 311/490] Remove unnecessary '\' linebreak characters Signed-off-by: David Horstmann --- scripts/code_style.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index d3ae624ec..dd8305faf 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -85,8 +85,9 @@ def get_uncrustify_version() -> str: """ Get the version string from Uncrustify """ - result = subprocess.run([UNCRUSTIFY_EXE, "--version"], \ - stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + result = subprocess.run([UNCRUSTIFY_EXE, "--version"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + check=False) if result.returncode != 0: print_err("Could not get Uncrustify version:", str(result.stderr, "utf-8")) return "" @@ -101,11 +102,11 @@ def check_style_is_correct(src_file_list: List[str]) -> bool: style_correct = True for src_file in src_file_list: uncrustify_cmd = [UNCRUSTIFY_EXE] + UNCRUSTIFY_ARGS + [src_file] - result = subprocess.run(uncrustify_cmd, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, check=False) + result = subprocess.run(uncrustify_cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=False) if result.returncode != 0: - print_err("Uncrustify returned " + str(result.returncode) + \ - " correcting file " + src_file) + print_err("Uncrustify returned " + str(result.returncode) + + " correcting file " + src_file) return False # Uncrustify makes changes to the code and places the result in a new @@ -135,9 +136,9 @@ def fix_style_single_pass(src_file_list: List[str]) -> bool: uncrustify_cmd = [UNCRUSTIFY_EXE] + code_change_args + [src_file] result = subprocess.run(uncrustify_cmd, check=False) if result.returncode != 0: - print_err("Uncrustify with file returned: " + \ - str(result.returncode) + " correcting file " + \ - src_file) + print_err("Uncrustify with file returned: " + + str(result.returncode) + " correcting file " + + src_file) return False return True From 613468f287d52be46cfaa563ed1cb2220564bd55 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Thu, 26 Jan 2023 12:27:29 +0100 Subject: [PATCH 312/490] Add dependency for generated test cases Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod_raw.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 69fe7b8e6..e1ff0cd98 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -59,6 +59,7 @@ class BignumModRawFixQuasiReduction(bignum_common.ModOperationCommon, test_name = "mbedtls_mpi_mod_raw_fix_quasi_reduction" input_style = "fixed" arity = 1 + dependencies = ["MBEDTLS_TEST_HOOKS"] # Extend the default values with n < x < 2n input_values = bignum_common.ModOperationCommon.input_values + [ From 1d1957ab1956385cc2cdc3a811d55ad1b32aceda Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Fri, 27 Jan 2023 14:33:50 +0100 Subject: [PATCH 313/490] Revert "Add dependency for generated test cases" The 'MBEDTLS_TEST_HOOKS' belongs to a test function and not to a test case. This reverts commit 613468f287d52be46cfaa563ed1cb2220564bd55. Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod_raw.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index e1ff0cd98..69fe7b8e6 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -59,7 +59,6 @@ class BignumModRawFixQuasiReduction(bignum_common.ModOperationCommon, test_name = "mbedtls_mpi_mod_raw_fix_quasi_reduction" input_style = "fixed" arity = 1 - dependencies = ["MBEDTLS_TEST_HOOKS"] # Extend the default values with n < x < 2n input_values = bignum_common.ModOperationCommon.input_values + [ From da4b0c43adb941c619dbc39291f14ae79c032ddc Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Fri, 27 Jan 2023 14:37:42 +0100 Subject: [PATCH 314/490] Shorten the prefix of the test case belongs to the fix quasi-reduction function Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_mod_raw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 69fe7b8e6..d197b5491 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -56,7 +56,7 @@ class BignumModRawFixQuasiReduction(bignum_common.ModOperationCommon, """Test cases for ecp quasi_reduction().""" symbol = "-" test_function = "mpi_mod_raw_fix_quasi_reduction" - test_name = "mbedtls_mpi_mod_raw_fix_quasi_reduction" + test_name = "fix_quasi_reduction" input_style = "fixed" arity = 1 From 65386fd9f7c7bf6fbd7997d33caaa05ec0a22c21 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Wed, 18 Jan 2023 11:52:56 +0000 Subject: [PATCH 315/490] c_build_helper.py: Move compile to helper Move compilation to a separate helper function in c_build_helper.py to allow more generic use. Signed-off-by: David Horstmann --- scripts/framework_dev/c_build_helper.py | 39 ++++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 459afba42..7e0cac730 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -89,6 +89,27 @@ int main(void) } ''') +def compile_c_file(c_filename, exe_filename, include_dirs): + cc = os.getenv('CC', 'cc') + cmd = [cc] + + proc = subprocess.Popen(cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + universal_newlines=True) + cc_is_msvc = 'Microsoft (R) C/C++' in proc.communicate()[1] + + cmd += ['-I' + dir for dir in include_dirs] + if cc_is_msvc: + # MSVC has deprecated using -o to specify the output file, + # and produces an object file in the working directory by default. + obj_filename = exe_filename[:-4] + '.obj' + cmd += ['-Fe' + exe_filename, '-Fo' + obj_filename] + else: + cmd += ['-o' + exe_filename] + + subprocess.check_call(cmd + [c_filename]) + def get_c_expression_values( cast_to, printf_format, expressions, @@ -128,24 +149,8 @@ def get_c_expression_values( expressions) ) c_file.close() - cc = os.getenv('CC', 'cc') - cmd = [cc] - proc = subprocess.Popen(cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - universal_newlines=True) - cc_is_msvc = 'Microsoft (R) C/C++' in proc.communicate()[1] - - cmd += ['-I' + dir for dir in include_path] - if cc_is_msvc: - # MSVC has deprecated using -o to specify the output file, - # and produces an object file in the working directory by default. - obj_name = exe_name[:-4] + '.obj' - cmd += ['-Fe' + exe_name, '-Fo' + obj_name] - else: - cmd += ['-o' + exe_name] - subprocess.check_call(cmd + [c_name]) + compile_c_file(c_name, exe_name, include_path) if keep_c: sys.stderr.write('List of {} tests kept at {}\n' .format(caller, c_name)) From c3642e68adabc97ff710ea35eea2306a9c561067 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Thu, 26 Jan 2023 19:11:57 +0000 Subject: [PATCH 316/490] Make c_build_helper module respect HOSTCC If HOSTCC is set, use that to generate files, otherwise use CC. This should make cross-compilation with generated files slightly easier. Signed-off-by: David Horstmann --- scripts/framework_dev/c_build_helper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 7e0cac730..6d2805759 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -90,7 +90,10 @@ int main(void) ''') def compile_c_file(c_filename, exe_filename, include_dirs): - cc = os.getenv('CC', 'cc') + # Respect $HOSTCC if it is set + cc = os.getenv('HOSTCC', None) + if cc is None: + cc = os.getenv('CC', 'cc') cmd = [cc] proc = subprocess.Popen(cmd, From 2995eb94b9be83badce4e181758c39a07fc97e11 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Mon, 30 Jan 2023 09:50:59 +0000 Subject: [PATCH 317/490] Add docstring for new compile function. Signed-off-by: David Horstmann --- scripts/framework_dev/c_build_helper.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 6d2805759..9bd17d608 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -90,6 +90,13 @@ int main(void) ''') def compile_c_file(c_filename, exe_filename, include_dirs): + """Compile a C source file with the host compiler. + + * ``c_filename``: the name of the source file to compile. + * ``exe_filename``: the name for the executable to be created. + * ``include_dirs``: a list of paths to include directories to be passed + with the -I switch. + """ # Respect $HOSTCC if it is set cc = os.getenv('HOSTCC', None) if cc is None: From 6d6209fe23f0640daa3d13a1df1adbbf63832929 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 11 Jan 2023 14:50:10 +0100 Subject: [PATCH 318/490] Switch to the new code style Signed-off-by: Gilles Peskine From fc04802939526dc41f7c4704221a4eb697f2b1df Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 16 Jan 2023 16:53:29 +0100 Subject: [PATCH 319/490] Add test generation support for the ecp module Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 scripts/framework_dev/ecp.py diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py new file mode 100644 index 000000000..77381cfd3 --- /dev/null +++ b/scripts/framework_dev/ecp.py @@ -0,0 +1,25 @@ +"""Framework classes for generation of ecp test cases.""" +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +from . import test_case +from . import test_data_generation + +class EcpTarget(test_data_generation.BaseTarget): + #pylint: disable=abstract-method, too-few-public-methods + """Target for ecp test case generation.""" + target_basename = 'test_suite_ecp.generated' From e5c94b879ce3dd9a5c1a5eed75335c50ab8dfdba Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Wed, 25 Jan 2023 18:09:49 +0100 Subject: [PATCH 320/490] Add test generation for ecp_mod_p192_raw Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 50 ++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 77381cfd3..7e1dafce4 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -18,8 +18,58 @@ from typing import List from . import test_case from . import test_data_generation +from . import bignum_common class EcpTarget(test_data_generation.BaseTarget): #pylint: disable=abstract-method, too-few-public-methods """Target for ecp test case generation.""" target_basename = 'test_suite_ecp.generated' + +class EcpP192R1Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ecp quasi_reduction().""" + symbol = "-" + test_function = "ecp_mod_p192_raw" + test_name = "ecp_mod_p192_raw" + input_style = "fixed" + arity = 1 + + moduli = [ "fffffffffffffffffffffffffffffffeffffffffffffffff" ] # type: List[str] + + input_values = [ + "0", "1", + + # First 8 number generated by random.getrandbits(384) - seed(2,2) + ("cf1822ffbc6887782b491044d5e341245c6e433715ba2bdd" + "177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("ffed9235288bc781ae66267594c9c9500925e4749b575bd1" + "3653f8dd9b1f282e4067c3584ee207f8da94e3e8ab73738f"), + ("ef8acd128b4f2fc15f3f57ebf30b94fa82523e86feac7eb7" + "dc38f519b91751dacdbd47d364be8049a372db8f6e405d93"), + ("e8624fab5186ee32ee8d7ee9770348a05d300cb90706a045" + "defc044a09325626e6b58de744ab6cce80877b6f71e1f6d2"), + ("2d3d854e061b90303b08c6e33c7295782d6c797f8f7d9b78" + "2a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f"), + ("fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f1" + "5c14bc4a829e07b0829a48d422fe99a22c70501e533c9135"), + ("97eeab64ca2ce6bc5d3fd983c34c769fe89204e2e8168561" + "867e5e15bc01bfce6a27e0dfcbf8754472154e76e4c11ab2"), + ("bd143fa9b714210c665d7435c1066932f4767f26294365b2" + "721dea3bf63f23d0dbe53fcafb2147df5ca495fa5a91c89b"), + + # Next 2 number generated by random.getrandbits(192) + "47733e847d718d733ff98ff387c56473a7a83ee0761ebfd2", + "cbd4d3e2d4dec9ef83f0be4e80371eb97f81375eecc1cb63" + ] + + @property + def arg_a(self) -> str: + return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True From 4377cb5a54ab86d1f74a7745c5741d952c53a091 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 31 Jan 2023 14:35:17 +0100 Subject: [PATCH 321/490] Fix pylint issues Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 7e1dafce4..bcb6bfb95 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -16,7 +16,6 @@ from typing import List -from . import test_case from . import test_data_generation from . import bignum_common @@ -34,7 +33,7 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, input_style = "fixed" arity = 1 - moduli = [ "fffffffffffffffffffffffffffffffeffffffffffffffff" ] # type: List[str] + moduli = ["fffffffffffffffffffffffffffffffeffffffffffffffff"] # type: List[str] input_values = [ "0", "1", From 8da7cd30fb4eab40bf5c5220cc0ead045abc29e6 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Mon, 6 Feb 2023 14:29:02 +0800 Subject: [PATCH 322/490] code_style.py: Apply exclusions to the file list This commit rename `--files` options to `--subset` and it means to check a subset of the files known to git. Signed-off-by: Pengyu Lv --- scripts/code_style.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index dd8305faf..4a5fb68c1 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -174,22 +174,19 @@ def main() -> int: parser.add_argument('-f', '--fix', action='store_true', help=('modify source files to fix the code style ' '(default: print diff, do not modify files)')) - # --files is almost useless: it only matters if there are no files - # ('code_style.py' without arguments checks all files known to Git, - # 'code_style.py --files' does nothing). In particular, - # 'code_style.py --fix --files ...' is intended as a stable ("porcelain") - # way to restyle a possibly empty set of files. - parser.add_argument('--files', action='store_true', - help='only check the specified files (default with non-option arguments)') + parser.add_argument('--subset', action='store_true', + help=('check a subset of the files known to git ' + '(default: empty FILE means full set)')) parser.add_argument('operands', nargs='*', metavar='FILE', - help='files to check (if none: check files that are known to git)') + help='files to check') args = parser.parse_args() - if args.files or args.operands: - src_files = args.operands - else: - src_files = get_src_files() + all_src_files = get_src_files() + src_files = args.operands if args.operands else all_src_files + if args.subset: + # We are to check a subset of the default list + src_files = [f for f in args.operands if f in all_src_files] if args.fix: # Fix mode From dc41dda7aad44de94a0bf8a1f859a926439fd0ee Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Mon, 6 Feb 2023 14:27:30 +0800 Subject: [PATCH 323/490] code_style.py: Add helpers to print warning and skipped files Signed-off-by: Pengyu Lv --- scripts/code_style.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/code_style.py b/scripts/code_style.py index 4a5fb68c1..61b1ab0e6 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -33,6 +33,17 @@ CHECK_GENERATED_FILES = "tests/scripts/check-generated-files.sh" def print_err(*args): print("Error: ", *args, file=sys.stderr) +def print_warn(*args): + print("Warn:", *args, file=sys.stderr) + +# Print the file names that will be skipped and the help message +def print_skip(files_to_skip): + print() + print(*files_to_skip, sep=", SKIP\n", end=", SKIP\n") + print_warn("The listed files will be skipped because\n" + "they are not included in the default list.") + print() + # Match FILENAME(s) in "check SCRIPT (FILENAME...)" CHECK_CALL_RE = re.compile(r"\n\s*check\s+[^\s#$&*?;|]+([^\n#$&*?;|]+)", re.ASCII) @@ -187,6 +198,9 @@ def main() -> int: if args.subset: # We are to check a subset of the default list src_files = [f for f in args.operands if f in all_src_files] + skip_src_files = [f for f in args.operands if f not in src_files] + if skip_src_files: + print_skip(skip_src_files) if args.fix: # Fix mode From 2fc0c2595dd20ebb4667522d96142b49fd4573cc Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 10 Feb 2023 10:55:29 +0800 Subject: [PATCH 324/490] print skipped file names to stdout Signed-off-by: Pengyu Lv --- scripts/code_style.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 61b1ab0e6..85008bec1 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -33,15 +33,12 @@ CHECK_GENERATED_FILES = "tests/scripts/check-generated-files.sh" def print_err(*args): print("Error: ", *args, file=sys.stderr) -def print_warn(*args): - print("Warn:", *args, file=sys.stderr) - # Print the file names that will be skipped and the help message def print_skip(files_to_skip): print() print(*files_to_skip, sep=", SKIP\n", end=", SKIP\n") - print_warn("The listed files will be skipped because\n" - "they are not included in the default list.") + print("Warn: The listed files will be skipped because\n" + "they are not included in the default list.") print() # Match FILENAME(s) in "check SCRIPT (FILENAME...)" From 39d3cb56d61ce4f729cbbf19a74f006f8985b142 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 10 Feb 2023 11:06:36 +0800 Subject: [PATCH 325/490] adjust help message Signed-off-by: Pengyu Lv --- scripts/code_style.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 85008bec1..65c9cccfb 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -184,9 +184,10 @@ def main() -> int: '(default: print diff, do not modify files)')) parser.add_argument('--subset', action='store_true', help=('check a subset of the files known to git ' - '(default: empty FILE means full set)')) + '(default: check all files passed as arguments, ' + 'known to git or not)')) parser.add_argument('operands', nargs='*', metavar='FILE', - help='files to check') + help='files to check (if none: check files that are known to git)') args = parser.parse_args() From 45f9a2f25869042e55eaa1e78765b2d743b4c458 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 13 Feb 2023 14:15:08 +0100 Subject: [PATCH 326/490] Add more test cases Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index bcb6bfb95..93cd2123f 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -38,6 +38,9 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, input_values = [ "0", "1", + # Modulus - 1 + "fffffffffffffffffffffffffffffffefffffffffffffffe", + # First 8 number generated by random.getrandbits(384) - seed(2,2) ("cf1822ffbc6887782b491044d5e341245c6e433715ba2bdd" "177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), From 2923ed0cdf3f26ce0ee1e716243e37a45dfa1c44 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Tue, 14 Feb 2023 10:29:53 +0800 Subject: [PATCH 327/490] Improve readability Signed-off-by: Pengyu Lv --- scripts/code_style.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 65c9cccfb..eaf1f6b88 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -191,12 +191,12 @@ def main() -> int: args = parser.parse_args() - all_src_files = get_src_files() - src_files = args.operands if args.operands else all_src_files + covered = frozenset(get_src_files()) + src_files = args.operands if args.operands else covered if args.subset: # We are to check a subset of the default list - src_files = [f for f in args.operands if f in all_src_files] - skip_src_files = [f for f in args.operands if f not in src_files] + src_files = [f for f in args.operands if f in covered] + skip_src_files = [f for f in args.operands if f not in covered] if skip_src_files: print_skip(skip_src_files) From 8508172d120934a184dc349111f2ce8ae083aa47 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Wed, 15 Feb 2023 10:20:40 +0800 Subject: [PATCH 328/490] Only check files known to git Signed-off-by: Pengyu Lv --- scripts/code_style.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index eaf1f6b88..e40a20cfc 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -37,8 +37,8 @@ def print_err(*args): def print_skip(files_to_skip): print() print(*files_to_skip, sep=", SKIP\n", end=", SKIP\n") - print("Warn: The listed files will be skipped because\n" - "they are not included in the default list.") + print("Warning: The listed files will be skipped because\n" + "they are not known to git.") print() # Match FILENAME(s) in "check SCRIPT (FILENAME...)" @@ -182,23 +182,27 @@ def main() -> int: parser.add_argument('-f', '--fix', action='store_true', help=('modify source files to fix the code style ' '(default: print diff, do not modify files)')) + # --subset is almost useless: it only matters if there are no files + # ('code_style.py' without arguments checks all files known to Git, + # 'code_style.py --subset' does nothing). In particular, + # 'code_style.py --fix --subset ...' is intended as a stable ("porcelain") + # way to restyle a possibly empty set of files. parser.add_argument('--subset', action='store_true', - help=('check a subset of the files known to git ' - '(default: check all files passed as arguments, ' - 'known to git or not)')) + help='only check the specified files (default with non-option arguments)') parser.add_argument('operands', nargs='*', metavar='FILE', - help='files to check (if none: check files that are known to git)') + help='files to check (files MUST be known to git, if none: check all)') args = parser.parse_args() covered = frozenset(get_src_files()) - src_files = args.operands if args.operands else covered - if args.subset: - # We are to check a subset of the default list + # We only check files that are known to git + if args.subset or args.operands: src_files = [f for f in args.operands if f in covered] skip_src_files = [f for f in args.operands if f not in covered] if skip_src_files: print_skip(skip_src_files) + else: + src_files = covered if args.fix: # Fix mode From 83a19cdbd8fe5f738765fed088c51f8170caf522 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Wed, 15 Feb 2023 16:58:09 +0800 Subject: [PATCH 329/490] Fix CI failure Signed-off-by: Pengyu Lv --- scripts/code_style.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index e40a20cfc..c31fb2949 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -202,7 +202,7 @@ def main() -> int: if skip_src_files: print_skip(skip_src_files) else: - src_files = covered + src_files = list(covered) if args.fix: # Fix mode From 1fc352934639a7a1a1e9875e6d7abd032d94d0bc Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 6 Feb 2023 15:49:42 +0100 Subject: [PATCH 330/490] Add test generation for ecp_mod_p521_raw Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 83 ++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 93cd2123f..96ddd057f 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -75,3 +75,86 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, @property def is_valid(self) -> bool: return True + +class EcpP521R1Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ecp quasi_reduction().""" + test_function = "ecp_mod_p521_raw" + test_name = "ecp_mod_p521_raw" + input_style = "fixed" + arity = 1 + + moduli = [("01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + ] # type: List[str] + + input_values = [ + "0", "1", + + # Test case for overflow during addition + ("0001efffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "000001ef" + "0000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000000000000000000000000000000000f000000"), + + # First 8 number generated by random.getrandbits(1042) - seed(2,2) + ("0003cc2e82523e86feac7eb7dc38f519b91751dacdbd47d364be8049a372db8f" + "6e405d93ffed9235288bc781ae66267594c9c9500925e4749b575bd13653f8dd" + "9b1f282e" + "4067c3584ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" + "5c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("00017052829e07b0829a48d422fe99a22c70501e533c91352d3d854e061b9030" + "3b08c6e33c7295782d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c5055" + "6c71c4a6" + "6148a86fe8624fab5186ee32ee8d7ee9770348a05d300cb90706a045defc044a" + "09325626e6b58de744ab6cce80877b6f71e1f6d2ef8acd128b4f2fc15f3f57eb"), + ("00021f15a7a83ee0761ebfd2bd143fa9b714210c665d7435c1066932f4767f26" + "294365b2721dea3bf63f23d0dbe53fcafb2147df5ca495fa5a91c89b97eeab64" + "ca2ce6bc" + "5d3fd983c34c769fe89204e2e8168561867e5e15bc01bfce6a27e0dfcbf87544" + "72154e76e4c11ab2fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f1"), + ("000381bc2a838af8d5c44a4eb3172062d08f1bb2531d6460f0caeef038c89b38" + "a8acb5137c9260dc74e088a9b9492f258ebdbfe3eb9ac688b9d39cca91551e82" + "59cc60b1" + "7604e4b4e73695c3e652c71a74667bffe202849da9643a295a9ac6decbd4d3e2" + "d4dec9ef83f0be4e80371eb97f81375eecc1cb6347733e847d718d733ff98ff3"), + ("00034816c8c69069134bccd3e1cf4f589f8e4ce0af29d115ef24bd625dd961e6" + "830b54fa7d28f93435339774bb1e386c4fd5079e681b8f5896838b769da59b74" + "a6c3181c" + "81e220df848b1df78feb994a81167346d4c0dca8b4c9e755cc9c3adcf515a823" + "4da4daeb4f3f87777ad1f45ae9500ec9c5e2486c44a4a8f69dc8db48e86ec9c6"), + ("000397846c4454b90f756132e16dce72f18e859835e1f291d322a7353ead4efe" + "440e2b4fda9c025a22f1a83185b98f5fc11e60de1b343f52ea748db9e020307a" + "aeb6db2c" + "3a038a709779ac1f45e9dd320c855fdfa7251af0930cdbd30f0ad2a81b2d19a2" + "beaa14a7ff3fe32a30ffc4eed0a7bd04e85bfcdd0227eeb7b9d7d01f5769da05"), + ("00002c3296e6bc4d62b47204007ee4fab105d83e85e951862f0981aebc1b00d9" + "2838e766ef9b6bf2d037fe2e20b6a8464174e75a5f834da70569c018eb2b5693" + "babb7fbb" + "0a76c196067cfdcb11457d9cf45e2fa01d7f4275153924800600571fac3a5b26" + "3fdf57cd2c0064975c3747465cc36c270e8a35b10828d569c268a20eb78ac332"), + ("00009d23b4917fc09f20dbb0dcc93f0e66dfe717c17313394391b6e2e6eacb0f" + "0bb7be72bd6d25009aeb7fa0c4169b148d2f527e72daf0a54ef25c0707e33868" + "7d1f7157" + "5653a45c49390aa51cf5192bbf67da14be11d56ba0b4a2969d8055a9f03f2d71" + "581d8e830112ff0f0948eccaf8877acf26c377c13f719726fd70bddacb4deeec"), + + # Next 2 number generated by random.getrandbits(521) + ("12b84ae65e920a63ac1f2b64df6dff07870c9d531ae72a47403063238da1a1fe" + "3f9d6a179fa50f96cd4aff9261aa92c0e6f17ec940639bc2ccdf572df00790813e3"), + ("166049dd332a73fa0b26b75196cf87eb8a09b27ec714307c68c425424a1574f1" + "eedf5b0f16cdfdb839424d201e653f53d6883ca1c107ca6e706649889c0c7f38608") + ] + + @property + def arg_a(self) -> str: + return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits - 2 * self.bits_in_limb // 8) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True From 6b348dec5e1fd496f20e60d3256291518200fe53 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 6 Feb 2023 18:03:39 +0100 Subject: [PATCH 331/490] Fix pylint issues Create a new function for calculating the number of hex digits needed for a certain amount of limbs. Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_common.py | 6 +++++- scripts/framework_dev/ecp.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 242217554..5319ec68b 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -74,6 +74,10 @@ def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: """Return all pair combinations from input values.""" return [(x, y) for x in values for y in values] +def hex_digits_for_limb(limbs: int, bits_in_limb: int) -> int: + """ Retrun the hex digits need for a number of limbs. """ + return 2 * (limbs * bits_in_limb // 8) + class OperationCommon(test_data_generation.BaseTest): """Common features for bignum binary operations. @@ -138,7 +142,7 @@ class OperationCommon(test_data_generation.BaseTest): @property def hex_digits(self) -> int: - return 2 * (self.limbs * self.bits_in_limb // 8) + return hex_digits_for_limb(self.limbs, self.bits_in_limb) def format_arg(self, val: str) -> str: if self.input_style not in self.input_styles: diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 96ddd057f..c167f6b6f 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -149,7 +149,9 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, @property def arg_a(self) -> str: - return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits - 2 * self.bits_in_limb // 8) + # Number of limbs: 2 * N - 1 + hex_digits = bignum_common.hex_digits_for_limb(2 * self.limbs - 1, self.bits_in_limb) + return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) def result(self) -> List[str]: result = self.int_a % self.int_n From 0f228522a333485cbf3b770536a21a0b286eec7a Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 7 Feb 2023 12:46:54 +0100 Subject: [PATCH 332/490] Fix 32-bit issues The 521 bit needs different limb alignment for different word sizes. Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index c167f6b6f..e0fb00035 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -81,7 +81,7 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, """Test cases for ecp quasi_reduction().""" test_function = "ecp_mod_p521_raw" test_name = "ecp_mod_p521_raw" - input_style = "fixed" + input_style = "arch_split" arity = 1 moduli = [("01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" From d38808048ebede6860f676497400c603fdab5e8f Mon Sep 17 00:00:00 2001 From: Janos Follath Date: Tue, 7 Feb 2023 15:27:44 +0000 Subject: [PATCH 333/490] Add corner case to mod_p521 tests Signed-off-by: Janos Follath Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index e0fb00035..fa70dedb5 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -91,6 +91,13 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, input_values = [ "0", "1", + # Corner case: maximum canonical P521 multiplication result + ("0003ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "fffff800" + "0000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000004"), + # Test case for overflow during addition ("0001efffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" From 44fbcafc8efd61777b25bb0a987ea8a645f2cb72 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Wed, 15 Feb 2023 16:52:33 +0100 Subject: [PATCH 334/490] Restrict input parameter size for ecp_mod_p521_raw The imput mpi parameter must have twice as many limbs as the modulus. Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index fa70dedb5..d436d0a35 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -81,7 +81,7 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, """Test cases for ecp quasi_reduction().""" test_function = "ecp_mod_p521_raw" test_name = "ecp_mod_p521_raw" - input_style = "arch_split" + input_style = "fixed" arity = 1 moduli = [("01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" @@ -156,9 +156,8 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, @property def arg_a(self) -> str: - # Number of limbs: 2 * N - 1 - hex_digits = bignum_common.hex_digits_for_limb(2 * self.limbs - 1, self.bits_in_limb) - return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) + # Number of limbs: 2 * N + return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits) def result(self) -> List[str]: result = self.int_a % self.int_n From 8959be7f3d749b4dd6f32c4abda2a3b5d30e8457 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Wed, 15 Feb 2023 17:04:40 +0100 Subject: [PATCH 335/490] Revert the addition of hex digit calculator function This reverts commit 0f83e15e670565147daa32fd1fac510759520e26. Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_common.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 5319ec68b..242217554 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -74,10 +74,6 @@ def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: """Return all pair combinations from input values.""" return [(x, y) for x in values for y in values] -def hex_digits_for_limb(limbs: int, bits_in_limb: int) -> int: - """ Retrun the hex digits need for a number of limbs. """ - return 2 * (limbs * bits_in_limb // 8) - class OperationCommon(test_data_generation.BaseTest): """Common features for bignum binary operations. @@ -142,7 +138,7 @@ class OperationCommon(test_data_generation.BaseTest): @property def hex_digits(self) -> int: - return hex_digits_for_limb(self.limbs, self.bits_in_limb) + return 2 * (self.limbs * self.bits_in_limb // 8) def format_arg(self, val: str) -> str: if self.input_style not in self.input_styles: From f43a2c9028b3c0e3eb447774f84f51611fc3fdaf Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Thu, 16 Feb 2023 10:25:08 +0100 Subject: [PATCH 336/490] Fix tests for 32bit Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index d436d0a35..6370d258a 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -81,7 +81,7 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, """Test cases for ecp quasi_reduction().""" test_function = "ecp_mod_p521_raw" test_name = "ecp_mod_p521_raw" - input_style = "fixed" + input_style = "arch_split" arity = 1 moduli = [("01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" From 217efa06ace86439ff0e030e1dfb0e1379398057 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 14 Feb 2023 18:25:23 +0100 Subject: [PATCH 337/490] Use a common function to calculate the number of hex digits Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_common.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 242217554..5319ec68b 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -74,6 +74,10 @@ def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: """Return all pair combinations from input values.""" return [(x, y) for x in values for y in values] +def hex_digits_for_limb(limbs: int, bits_in_limb: int) -> int: + """ Retrun the hex digits need for a number of limbs. """ + return 2 * (limbs * bits_in_limb // 8) + class OperationCommon(test_data_generation.BaseTest): """Common features for bignum binary operations. @@ -138,7 +142,7 @@ class OperationCommon(test_data_generation.BaseTest): @property def hex_digits(self) -> int: - return 2 * (self.limbs * self.bits_in_limb // 8) + return hex_digits_for_limb(self.limbs, self.bits_in_limb) def format_arg(self, val: str) -> str: if self.input_style not in self.input_styles: From 7011783395af93335a36d8507bf917029aabb945 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 14 Feb 2023 18:26:36 +0100 Subject: [PATCH 338/490] Add test generation for ecp_mod_p224_raw Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 6370d258a..da0ae3741 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -76,6 +76,55 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True +class EcpP224R1Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ecp quasi_reduction().""" + symbol = "-" + test_function = "ecp_mod_p224_raw" + test_name = "ecp_mod_p224_raw" + input_style = "arch_split" + arity = 1 + + moduli = ["ffffffffffffffffffffffffffffffff000000000000000000000001"] # type: List[str] + + input_values = [ + "0", "1", + + # First 8 number generated by random.getrandbits(448) - seed(2,2) + ("da94e3e8ab73738fcf1822ffbc6887782b491044d5e341245c6e4337" + "15ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("cdbd47d364be8049a372db8f6e405d93ffed9235288bc781ae662675" + "94c9c9500925e4749b575bd13653f8dd9b1f282e4067c3584ee207f8"), + ("defc044a09325626e6b58de744ab6cce80877b6f71e1f6d2ef8acd12" + "8b4f2fc15f3f57ebf30b94fa82523e86feac7eb7dc38f519b91751da"), + ("2d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71c4a6" + "6148a86fe8624fab5186ee32ee8d7ee9770348a05d300cb90706a045"), + ("8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0829a48d4" + "22fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578"), + ("97eeab64ca2ce6bc5d3fd983c34c769fe89204e2e8168561867e5e15" + "bc01bfce6a27e0dfcbf8754472154e76e4c11ab2fec3f6b32e8d4b8a"), + ("a7a83ee0761ebfd2bd143fa9b714210c665d7435c1066932f4767f26" + "294365b2721dea3bf63f23d0dbe53fcafb2147df5ca495fa5a91c89b"), + ("74667bffe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e" + "80371eb97f81375eecc1cb6347733e847d718d733ff98ff387c56473"), + + # Next 2 number generated by random.getrandbits(224) + "eb9ac688b9d39cca91551e8259cc60b17604e4b4e73695c3e652c71a", + "f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f258ebdbfe3" + ] + + @property + def arg_a(self) -> str: + hex_digits = bignum_common.hex_digits_for_limb(448 // self.bits_in_limb, self.bits_in_limb) + return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True class EcpP521R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ecp quasi_reduction().""" From 35626cc7aba8df15d7885c4b713ddd1ab8698258 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 27 Feb 2023 15:59:34 +0100 Subject: [PATCH 339/490] Add more test cases for P224 testing Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index da0ae3741..4f529d15c 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -90,6 +90,21 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, input_values = [ "0", "1", + # Modulus - 1 + "ffffffffffffffffffffffffffffffff000000000000000000000000", + + # Maximum canonical P224 multiplication result + ("ffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + + # Generate an overflow during reduction + ("00000000000000000000000000010000000070000000002000001000" + "FFFFFFFFFFFF9FFFFFFFFFE00000EFFF000070000000002000001003"), + + # Generate an underflow during reduction + ("00000001000000000000000000000000000000000000000000000000" + "00000000000DC0000000000000000001000000010000000100000003"), + # First 8 number generated by random.getrandbits(448) - seed(2,2) ("da94e3e8ab73738fcf1822ffbc6887782b491044d5e341245c6e4337" "15ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), From c49413534e6685112648d580b17f4ae3af0d9600 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Wed, 1 Mar 2023 16:50:00 +0100 Subject: [PATCH 340/490] Use lower case hex number Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 4f529d15c..e01b80fbc 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -99,11 +99,11 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, # Generate an overflow during reduction ("00000000000000000000000000010000000070000000002000001000" - "FFFFFFFFFFFF9FFFFFFFFFE00000EFFF000070000000002000001003"), + "ffffffffffff9fffffffffe00000efff000070000000002000001003"), # Generate an underflow during reduction ("00000001000000000000000000000000000000000000000000000000" - "00000000000DC0000000000000000001000000010000000100000003"), + "00000000000dc0000000000000000001000000010000000100000003"), # First 8 number generated by random.getrandbits(448) - seed(2,2) ("da94e3e8ab73738fcf1822ffbc6887782b491044d5e341245c6e4337" @@ -140,6 +140,7 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, @property def is_valid(self) -> bool: return True + class EcpP521R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ecp quasi_reduction().""" From f1cbbec6bc70172fe8aace8b8ae9517893252888 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 6 Mar 2023 16:13:42 +0100 Subject: [PATCH 341/490] Remove unnecessary function override Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index e01b80fbc..6440ba05d 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -72,10 +72,6 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, result = self.int_a % self.int_n return [self.format_result(result)] - @property - def is_valid(self) -> bool: - return True - class EcpP224R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ecp quasi_reduction().""" @@ -137,10 +133,6 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, result = self.int_a % self.int_n return [self.format_result(result)] - @property - def is_valid(self) -> bool: - return True - class EcpP521R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ecp quasi_reduction().""" @@ -227,7 +219,3 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, def result(self) -> List[str]: result = self.int_a % self.int_n return [self.format_result(result)] - - @property - def is_valid(self) -> bool: - return True From 1c1eb3c554e0d62d61cb36a090b2fdc0f72aa925 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 6 Mar 2023 16:15:43 +0100 Subject: [PATCH 342/490] Code style: have two empty lines before and after class definitions Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 6440ba05d..ff31baedc 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -19,11 +19,13 @@ from typing import List from . import test_data_generation from . import bignum_common + class EcpTarget(test_data_generation.BaseTarget): #pylint: disable=abstract-method, too-few-public-methods """Target for ecp test case generation.""" target_basename = 'test_suite_ecp.generated' + class EcpP192R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ecp quasi_reduction().""" @@ -72,6 +74,7 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, result = self.int_a % self.int_n return [self.format_result(result)] + class EcpP224R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ecp quasi_reduction().""" @@ -133,6 +136,7 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, result = self.int_a % self.int_n return [self.format_result(result)] + class EcpP521R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ecp quasi_reduction().""" From e0a84d74c6d937a66705c3537619826c67d65523 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 6 Mar 2023 16:26:18 +0100 Subject: [PATCH 343/490] Correct the maximum canonical value in tests Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index ff31baedc..556209f46 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -93,8 +93,8 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, "ffffffffffffffffffffffffffffffff000000000000000000000000", # Maximum canonical P224 multiplication result - ("ffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + ("fffffffffffffffffffffffffffffffe000000000000000000000000" + "00000001000000000000000000000000000000000000000000000000"), # Generate an overflow during reduction ("00000000000000000000000000010000000070000000002000001000" From c9709b3829ee5ae15514d7c8cd4ca9fc9cb49169 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 6 Mar 2023 16:57:25 +0100 Subject: [PATCH 344/490] The is_valid() function is needed to not filter out test cases Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 556209f46..354b23416 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -74,6 +74,10 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, result = self.int_a % self.int_n return [self.format_result(result)] + @property + def is_valid(self) -> bool: + return True + class EcpP224R1Raw(bignum_common.ModOperationCommon, EcpTarget): @@ -136,6 +140,10 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, result = self.int_a % self.int_n return [self.format_result(result)] + @property + def is_valid(self) -> bool: + return True + class EcpP521R1Raw(bignum_common.ModOperationCommon, EcpTarget): @@ -223,3 +231,7 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, def result(self) -> List[str]: result = self.int_a % self.int_n return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True From 340f92f9d0ebff252462bea0a4e75711a0009923 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Wed, 8 Mar 2023 14:06:04 +0100 Subject: [PATCH 345/490] Add test generation for ecp_mod_p256_raw Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 73 ++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 354b23416..f448925ab 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -145,6 +145,79 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, return True +class EcpP256R1Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ecp quasi_reduction().""" + symbol = "-" + test_function = "ecp_mod_p256_raw" + test_name = "ecp_mod_p256_raw" + input_style = "fixed" + arity = 1 + + moduli = ["ffffffff00000001000000000000000000000000ffffffffffffffffffffffff"] # type: List[str] + + input_values = [ + "0", "1", + + # Modulus - 1 + "ffffffff00000001000000000000000000000000fffffffffffffffffffffffe", + + # Maximum canonical P256 multiplication result + ("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + + # Generate an overflow during reduction + ("0000000000000000000000010000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000ffffffff"), + + # Generate an underflow during reduction + ("0000000000000000000000000000000000000000000000000000000000000010" + "ffffffff00000000000000000000000000000000000000000000000000000000"), + + # Generate an overflow during carry reduction + ("aaaaaaaa00000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000aaaaaaacaaaaaaaaaaaaaaaa00000000"), + + # Generate an underflow during carry reduction + ("000000000000000000000001ffffffff00000000000000000000000000000000" + "0000000000000000000000000000000000000002000000020000000100000002"), + + # First 8 number generated by random.getrandbits(512) - seed(2,2) + ("4067c3584ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" + "5c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("82523e86feac7eb7dc38f519b91751dacdbd47d364be8049a372db8f6e405d93" + "ffed9235288bc781ae66267594c9c9500925e4749b575bd13653f8dd9b1f282e"), + ("e8624fab5186ee32ee8d7ee9770348a05d300cb90706a045defc044a09325626" + "e6b58de744ab6cce80877b6f71e1f6d2ef8acd128b4f2fc15f3f57ebf30b94fa"), + ("829a48d422fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578" + "2d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f"), + ("e89204e2e8168561867e5e15bc01bfce6a27e0dfcbf8754472154e76e4c11ab2" + "fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0"), + ("bd143fa9b714210c665d7435c1066932f4767f26294365b2721dea3bf63f23d0" + "dbe53fcafb2147df5ca495fa5a91c89b97eeab64ca2ce6bc5d3fd983c34c769f"), + ("74667bffe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e80371eb9" + "7f81375eecc1cb6347733e847d718d733ff98ff387c56473a7a83ee0761ebfd2"), + ("d08f1bb2531d6460f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f25" + "8ebdbfe3eb9ac688b9d39cca91551e8259cc60b17604e4b4e73695c3e652c71a"), + + # Next 2 number generated by random.getrandbits(256) + "c5e2486c44a4a8f69dc8db48e86ec9c6e06f291b2a838af8d5c44a4eb3172062", + "d4c0dca8b4c9e755cc9c3adcf515a8234da4daeb4f3f87777ad1f45ae9500ec9" + ] + + @property + def arg_a(self) -> str: + return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True + + class EcpP521R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ecp quasi_reduction().""" From fa85e66429522fc3ec9383844674603b11f1eb93 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Thu, 9 Mar 2023 13:41:10 +0100 Subject: [PATCH 346/490] Add and fix comments Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index f448925ab..e24517100 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -147,7 +147,7 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, class EcpP256R1Raw(bignum_common.ModOperationCommon, EcpTarget): - """Test cases for ecp quasi_reduction().""" + """Test cases for ECP P256 fast reduction.""" symbol = "-" test_function = "ecp_mod_p256_raw" test_name = "ecp_mod_p256_raw" From a970c388807864ec9cbd013e3cd47af4ac0d7763 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Thu, 9 Mar 2023 13:43:15 +0100 Subject: [PATCH 347/490] Fix maximum cannonical value Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index e24517100..ffe48fc5d 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -163,8 +163,8 @@ class EcpP256R1Raw(bignum_common.ModOperationCommon, "ffffffff00000001000000000000000000000000fffffffffffffffffffffffe", # Maximum canonical P256 multiplication result - ("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + ("fffffffe00000002fffffffe0000000100000001fffffffe00000001fffffffc" + "00000003fffffffcfffffffffffffffffffffffc000000000000000000000004"), # Generate an overflow during reduction ("0000000000000000000000010000000000000000000000000000000000000000" From 6dc69474dffb9484dd349710ab1b2d173a435108 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Mon, 6 Mar 2023 11:39:54 +0000 Subject: [PATCH 348/490] ecp test generator: Added EcpPp384R1Raw(). Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 80 ++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index ffe48fc5d..6d3bb0536 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -144,6 +144,86 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True +class EcpPp384R1Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ecp quasi_reduction modulo p384.""" + test_function = "ecp_mod_p384_raw" + test_name = "ecp_mod_p384_raw" + input_style = "arch_split" + arity = 1 + + moduli = [("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "fffffeffffffff0000000000000000ffffffff") + ] # type: List[str] + + input_values = [ + "0", "1", + + # Modulus - 1 + ("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffef" + "fffffff0000000000000000fffffffe"), + + # Maximum canonical P384 multiplication result + ("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "fdfffffffe0000000000000001fffffffc0000000000000000000000000000000" + "10000000200000000fffffffe000000020000000400000000fffffffc00000004"), + + # Testing with overflow in A(12) + A(21) + A(20); + ("497811378624857a2c2af60d70583376545484cfae5c812fe2999fc1abb51d18b" + "559e8ca3b50aaf263fdf8f24bdfb98fffffffff20e65bf9099e4e73a5e8b517cf" + "4fbeb8fd1750fdae6d43f2e53f82d5ffffffffffffffffcc6f1e06111c62e0"), + + # Testing with underflow in A(13) + A(22) + A(23) - A(12) - A(20); + ("dfdd25e96777406b3c04b8c7b406f5fcf287e1e576003a092852a6fbe517f2712" + "b68abef41dbd35183a0614fb7222606ffffffff84396eee542f18a9189d94396c" + "784059c17a9f18f807214ef32f2f10ffffffff8a77fac20000000000000000"), + + # First 8 number generated by random.getrandbits(768) - seed(2,2) + ("ffed9235288bc781ae66267594c9c9500925e4749b575bd13653f8dd9b1f282e" + "4067c3584ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" + "5c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("e8624fab5186ee32ee8d7ee9770348a05d300cb90706a045defc044a09325626" + "e6b58de744ab6cce80877b6f71e1f6d2ef8acd128b4f2fc15f3f57ebf30b94fa" + "82523e86feac7eb7dc38f519b91751dacdbd47d364be8049a372db8f6e405d93"), + ("fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0" + "829a48d422fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578" + "2d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f"), + ("bd143fa9b714210c665d7435c1066932f4767f26294365b2721dea3bf63f23d0" + "dbe53fcafb2147df5ca495fa5a91c89b97eeab64ca2ce6bc5d3fd983c34c769f" + "e89204e2e8168561867e5e15bc01bfce6a27e0dfcbf8754472154e76e4c11ab2"), + ("8ebdbfe3eb9ac688b9d39cca91551e8259cc60b17604e4b4e73695c3e652c71a" + "74667bffe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e80371eb9" + "7f81375eecc1cb6347733e847d718d733ff98ff387c56473a7a83ee0761ebfd2"), + ("d4c0dca8b4c9e755cc9c3adcf515a8234da4daeb4f3f87777ad1f45ae9500ec9" + "c5e2486c44a4a8f69dc8db48e86ec9c6e06f291b2a838af8d5c44a4eb3172062" + "d08f1bb2531d6460f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f25"), + ("227eeb7b9d7d01f5769da05d205bbfcc8c69069134bccd3e1cf4f589f8e4ce0a" + "f29d115ef24bd625dd961e6830b54fa7d28f93435339774bb1e386c4fd5079e6" + "81b8f5896838b769da59b74a6c3181c81e220df848b1df78feb994a81167346"), + ("d322a7353ead4efe440e2b4fda9c025a22f1a83185b98f5fc11e60de1b343f52" + "ea748db9e020307aaeb6db2c3a038a709779ac1f45e9dd320c855fdfa7251af0" + "930cdbd30f0ad2a81b2d19a2beaa14a7ff3fe32a30ffc4eed0a7bd04e85bfcdd"), + + # Next 2 number generated by random.getrandbits(384) + ("5c3747465cc36c270e8a35b10828d569c268a20eb78ac332e5e138e26c4454b9" + "0f756132e16dce72f18e859835e1f291"), + ("eb2b5693babb7fbb0a76c196067cfdcb11457d9cf45e2fa01d7f427515392480" + "0600571fac3a5b263fdf57cd2c006497") + ] + + @property + def arg_a(self) -> str: + hex_digits = bignum_common.hex_digits_for_limb((766 // self.bits_in_limb) + 1, + self.bits_in_limb) + return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True class EcpP256R1Raw(bignum_common.ModOperationCommon, EcpTarget): From 636abecb7a60d8729d1bbb13c6fa2b2dae206623 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Thu, 9 Mar 2023 11:15:15 +0000 Subject: [PATCH 349/490] ecp_curves: Minor rework for p384 This patch adjusts formatting, documentation and testing. Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 6d3bb0536..10fcc5e47 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -144,12 +144,13 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True -class EcpPp384R1Raw(bignum_common.ModOperationCommon, - EcpTarget): + +class EcpP384R1Raw(bignum_common.ModOperationCommon, + EcpTarget): """Test cases for ecp quasi_reduction modulo p384.""" test_function = "ecp_mod_p384_raw" test_name = "ecp_mod_p384_raw" - input_style = "arch_split" + input_style = "fixed" arity = 1 moduli = [("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" @@ -164,7 +165,7 @@ class EcpPp384R1Raw(bignum_common.ModOperationCommon, "fffffff0000000000000000fffffffe"), # Maximum canonical P384 multiplication result - ("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + ("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" "fdfffffffe0000000000000001fffffffc0000000000000000000000000000000" "10000000200000000fffffffe000000020000000400000000fffffffc00000004"), @@ -178,6 +179,16 @@ class EcpPp384R1Raw(bignum_common.ModOperationCommon, "b68abef41dbd35183a0614fb7222606ffffffff84396eee542f18a9189d94396c" "784059c17a9f18f807214ef32f2f10ffffffff8a77fac20000000000000000"), + # Testing with overflow in A(23) + A(20) + A(19) - A(22); + ("783753f8a5afba6c1862eead1deb2fcdd907272be3ffd18542b24a71ee8b26ca" + "b0aa33513610ff973042bbe1637cc9fc99ad36c7f703514572cf4f5c3044469a" + "8f5be6312c19e5d3f8fc1ac6ffffffffffffffff8c86252400000000ffffffff"), + + # Testing with underflow in A(23) + A(20) + A(19) - A(22); + ("65e1d2362fce922663b7fd517586e88842a9b4bd092e93e6251c9c69f278cbf8" + "285d99ae3b53da5ba36e56701e2b17c225f1239556c5f00117fa140218b46ebd8" + "e34f50d0018701fa8a0a5cc00000000000000004410bcb4ffffffff00000000"), + # First 8 number generated by random.getrandbits(768) - seed(2,2) ("ffed9235288bc781ae66267594c9c9500925e4749b575bd13653f8dd9b1f282e" "4067c3584ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" @@ -213,9 +224,7 @@ class EcpPp384R1Raw(bignum_common.ModOperationCommon, @property def arg_a(self) -> str: - hex_digits = bignum_common.hex_digits_for_limb((766 // self.bits_in_limb) + 1, - self.bits_in_limb) - return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) + return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits) def result(self) -> List[str]: result = self.int_a % self.int_n From 75bdc544d8bde79cebc4c492451e25aeaf608517 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Mon, 20 Mar 2023 12:20:46 +0000 Subject: [PATCH 350/490] EcpP384R1Raw: Added test case for 2nd round of carry reduction. Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 10fcc5e47..aee871831 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -189,6 +189,11 @@ class EcpP384R1Raw(bignum_common.ModOperationCommon, "285d99ae3b53da5ba36e56701e2b17c225f1239556c5f00117fa140218b46ebd8" "e34f50d0018701fa8a0a5cc00000000000000004410bcb4ffffffff00000000"), + # Testing the second round of carry reduction + ("000000000000000000000000ffffffffffffffffffffffffffffffffffffffff" + "ffffffffffffffff00000000000000000000000000000000ffffffff00000000" + "000000000000000100000000000000000000000000000000ffffffff00000001"), + # First 8 number generated by random.getrandbits(768) - seed(2,2) ("ffed9235288bc781ae66267594c9c9500925e4749b575bd13653f8dd9b1f282e" "4067c3584ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" From ce6c440243577cc83e503adf86546ae57850747f Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 28 Mar 2023 15:04:48 +0200 Subject: [PATCH 351/490] Add test cases for P192 fast reduction testing Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index aee871831..c5ad3c8f1 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -43,6 +43,24 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, # Modulus - 1 "fffffffffffffffffffffffffffffffefffffffffffffffe", + # Modulus + 1 + "ffffffffffffffffffffffffffffffff0000000000000000", + + # 2^192 - 1 + "ffffffffffffffffffffffffffffffffffffffffffffffff", + + # Maximum canonical P192 multiplication result + ("fffffffffffffffffffffffffffffffdfffffffffffffffc" + "000000000000000100000000000000040000000000000004"), + + # Generate an overflow during reduction + ("00000000000000000000000000000001ffffffffffffffff" + "ffffffffffffffffffffffffffffffff0000000000000000"), + + # Generate an overflow during carry reduction + ("ffffffffffffffff00000000000000010000000000000000" + "fffffffffffffffeffffffffffffffff0000000000000000"), + # First 8 number generated by random.getrandbits(384) - seed(2,2) ("cf1822ffbc6887782b491044d5e341245c6e433715ba2bdd" "177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), From 59e748a8827476c2013accf16d014a66c7d94bde Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 28 Mar 2023 15:05:57 +0200 Subject: [PATCH 352/490] Add test cases for P224 fast reduction testing Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index c5ad3c8f1..daa669db0 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -114,6 +114,12 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, # Modulus - 1 "ffffffffffffffffffffffffffffffff000000000000000000000000", + # Modulus + 1 + "ffffffffffffffffffffffffffffffff000000000000000000000002", + + # 2^224 - 1 + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + # Maximum canonical P224 multiplication result ("fffffffffffffffffffffffffffffffe000000000000000000000000" "00000001000000000000000000000000000000000000000000000000"), From bf594792fd2e50d47cd1b2739b3a82604626e2d6 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 28 Mar 2023 15:06:51 +0200 Subject: [PATCH 353/490] Add test cases for P521 fast reduction testing Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index daa669db0..8f714c908 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -351,7 +351,15 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, input_values = [ "0", "1", - # Corner case: maximum canonical P521 multiplication result + # Modulus - 1 + ("01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"), + + # Modulus + 1 + ("020000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000000000000000000000000000000000000000000"), + + # Maximum canonical P521 multiplication result ("0003ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" "fffff800" From 83abbc1815549acb60f7aa760c06abb901bde7b9 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 28 Mar 2023 15:29:40 +0200 Subject: [PATCH 354/490] Typo: reformat numbers Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 114 ++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 50 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 8f714c908..8b602c6e7 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -177,78 +177,92 @@ class EcpP384R1Raw(bignum_common.ModOperationCommon, input_style = "fixed" arity = 1 - moduli = [("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "fffffeffffffff0000000000000000ffffffff") + moduli = [("ffffffffffffffffffffffffffffffffffffffffffffffff" + "fffffffffffffffeffffffff0000000000000000ffffffff") ] # type: List[str] input_values = [ "0", "1", # Modulus - 1 - ("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffef" - "fffffff0000000000000000fffffffe"), + ("ffffffffffffffffffffffffffffffffffffffffffffffff" + "fffffffffffffffeffffffff0000000000000000fffffffe"), # Maximum canonical P384 multiplication result - ("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "fdfffffffe0000000000000001fffffffc0000000000000000000000000000000" - "10000000200000000fffffffe000000020000000400000000fffffffc00000004"), + ("ffffffffffffffffffffffffffffffffffffffffffffffff" + "fffffffffffffffdfffffffe0000000000000001fffffffc" + "000000000000000000000000000000010000000200000000" + "fffffffe000000020000000400000000fffffffc00000004"), # Testing with overflow in A(12) + A(21) + A(20); - ("497811378624857a2c2af60d70583376545484cfae5c812fe2999fc1abb51d18b" - "559e8ca3b50aaf263fdf8f24bdfb98fffffffff20e65bf9099e4e73a5e8b517cf" - "4fbeb8fd1750fdae6d43f2e53f82d5ffffffffffffffffcc6f1e06111c62e0"), + ("497811378624857a2c2af60d70583376545484cfae5c812f" + "e2999fc1abb51d18b559e8ca3b50aaf263fdf8f24bdfb98f" + "ffffffff20e65bf9099e4e73a5e8b517cf4fbeb8fd1750fd" + "ae6d43f2e53f82d5ffffffffffffffffcc6f1e06111c62e0"), # Testing with underflow in A(13) + A(22) + A(23) - A(12) - A(20); - ("dfdd25e96777406b3c04b8c7b406f5fcf287e1e576003a092852a6fbe517f2712" - "b68abef41dbd35183a0614fb7222606ffffffff84396eee542f18a9189d94396c" - "784059c17a9f18f807214ef32f2f10ffffffff8a77fac20000000000000000"), + ("dfdd25e96777406b3c04b8c7b406f5fcf287e1e576003a09" + "2852a6fbe517f2712b68abef41dbd35183a0614fb7222606" + "ffffffff84396eee542f18a9189d94396c784059c17a9f18" + "f807214ef32f2f10ffffffff8a77fac20000000000000000"), # Testing with overflow in A(23) + A(20) + A(19) - A(22); - ("783753f8a5afba6c1862eead1deb2fcdd907272be3ffd18542b24a71ee8b26ca" - "b0aa33513610ff973042bbe1637cc9fc99ad36c7f703514572cf4f5c3044469a" - "8f5be6312c19e5d3f8fc1ac6ffffffffffffffff8c86252400000000ffffffff"), + ("783753f8a5afba6c1862eead1deb2fcdd907272be3ffd185" + "42b24a71ee8b26cab0aa33513610ff973042bbe1637cc9fc" + "99ad36c7f703514572cf4f5c3044469a8f5be6312c19e5d3" + "f8fc1ac6ffffffffffffffff8c86252400000000ffffffff"), # Testing with underflow in A(23) + A(20) + A(19) - A(22); - ("65e1d2362fce922663b7fd517586e88842a9b4bd092e93e6251c9c69f278cbf8" - "285d99ae3b53da5ba36e56701e2b17c225f1239556c5f00117fa140218b46ebd8" - "e34f50d0018701fa8a0a5cc00000000000000004410bcb4ffffffff00000000"), + ("65e1d2362fce922663b7fd517586e88842a9b4bd092e93e6" + "251c9c69f278cbf8285d99ae3b53da5ba36e56701e2b17c2" + "25f1239556c5f00117fa140218b46ebd8e34f50d0018701f" + "a8a0a5cc00000000000000004410bcb4ffffffff00000000"), # Testing the second round of carry reduction - ("000000000000000000000000ffffffffffffffffffffffffffffffffffffffff" - "ffffffffffffffff00000000000000000000000000000000ffffffff00000000" - "000000000000000100000000000000000000000000000000ffffffff00000001"), + ("000000000000000000000000ffffffffffffffffffffffff" + "ffffffffffffffffffffffffffffffff0000000000000000" + "0000000000000000ffffffff000000000000000000000001" + "00000000000000000000000000000000ffffffff00000001"), # First 8 number generated by random.getrandbits(768) - seed(2,2) - ("ffed9235288bc781ae66267594c9c9500925e4749b575bd13653f8dd9b1f282e" - "4067c3584ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" - "5c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), - ("e8624fab5186ee32ee8d7ee9770348a05d300cb90706a045defc044a09325626" - "e6b58de744ab6cce80877b6f71e1f6d2ef8acd128b4f2fc15f3f57ebf30b94fa" - "82523e86feac7eb7dc38f519b91751dacdbd47d364be8049a372db8f6e405d93"), - ("fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0" - "829a48d422fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578" - "2d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f"), - ("bd143fa9b714210c665d7435c1066932f4767f26294365b2721dea3bf63f23d0" - "dbe53fcafb2147df5ca495fa5a91c89b97eeab64ca2ce6bc5d3fd983c34c769f" - "e89204e2e8168561867e5e15bc01bfce6a27e0dfcbf8754472154e76e4c11ab2"), - ("8ebdbfe3eb9ac688b9d39cca91551e8259cc60b17604e4b4e73695c3e652c71a" - "74667bffe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e80371eb9" - "7f81375eecc1cb6347733e847d718d733ff98ff387c56473a7a83ee0761ebfd2"), - ("d4c0dca8b4c9e755cc9c3adcf515a8234da4daeb4f3f87777ad1f45ae9500ec9" - "c5e2486c44a4a8f69dc8db48e86ec9c6e06f291b2a838af8d5c44a4eb3172062" - "d08f1bb2531d6460f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f25"), - ("227eeb7b9d7d01f5769da05d205bbfcc8c69069134bccd3e1cf4f589f8e4ce0a" - "f29d115ef24bd625dd961e6830b54fa7d28f93435339774bb1e386c4fd5079e6" - "81b8f5896838b769da59b74a6c3181c81e220df848b1df78feb994a81167346"), - ("d322a7353ead4efe440e2b4fda9c025a22f1a83185b98f5fc11e60de1b343f52" - "ea748db9e020307aaeb6db2c3a038a709779ac1f45e9dd320c855fdfa7251af0" - "930cdbd30f0ad2a81b2d19a2beaa14a7ff3fe32a30ffc4eed0a7bd04e85bfcdd"), + ("ffed9235288bc781ae66267594c9c9500925e4749b575bd1" + "3653f8dd9b1f282e4067c3584ee207f8da94e3e8ab73738f" + "cf1822ffbc6887782b491044d5e341245c6e433715ba2bdd" + "177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("e8624fab5186ee32ee8d7ee9770348a05d300cb90706a045" + "defc044a09325626e6b58de744ab6cce80877b6f71e1f6d2" + "ef8acd128b4f2fc15f3f57ebf30b94fa82523e86feac7eb7" + "dc38f519b91751dacdbd47d364be8049a372db8f6e405d93"), + ("fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f1" + "5c14bc4a829e07b0829a48d422fe99a22c70501e533c9135" + "2d3d854e061b90303b08c6e33c7295782d6c797f8f7d9b78" + "2a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f"), + ("bd143fa9b714210c665d7435c1066932f4767f26294365b2" + "721dea3bf63f23d0dbe53fcafb2147df5ca495fa5a91c89b" + "97eeab64ca2ce6bc5d3fd983c34c769fe89204e2e8168561" + "867e5e15bc01bfce6a27e0dfcbf8754472154e76e4c11ab2"), + ("8ebdbfe3eb9ac688b9d39cca91551e8259cc60b17604e4b4" + "e73695c3e652c71a74667bffe202849da9643a295a9ac6de" + "cbd4d3e2d4dec9ef83f0be4e80371eb97f81375eecc1cb63" + "47733e847d718d733ff98ff387c56473a7a83ee0761ebfd2"), + ("d4c0dca8b4c9e755cc9c3adcf515a8234da4daeb4f3f8777" + "7ad1f45ae9500ec9c5e2486c44a4a8f69dc8db48e86ec9c6" + "e06f291b2a838af8d5c44a4eb3172062d08f1bb2531d6460" + "f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f25"), + ("0227eeb7b9d7d01f5769da05d205bbfcc8c69069134bccd3" + "e1cf4f589f8e4ce0af29d115ef24bd625dd961e6830b54fa" + "7d28f93435339774bb1e386c4fd5079e681b8f5896838b76" + "9da59b74a6c3181c81e220df848b1df78feb994a81167346"), + ("d322a7353ead4efe440e2b4fda9c025a22f1a83185b98f5f" + "c11e60de1b343f52ea748db9e020307aaeb6db2c3a038a70" + "9779ac1f45e9dd320c855fdfa7251af0930cdbd30f0ad2a8" + "1b2d19a2beaa14a7ff3fe32a30ffc4eed0a7bd04e85bfcdd"), # Next 2 number generated by random.getrandbits(384) - ("5c3747465cc36c270e8a35b10828d569c268a20eb78ac332e5e138e26c4454b9" - "0f756132e16dce72f18e859835e1f291"), - ("eb2b5693babb7fbb0a76c196067cfdcb11457d9cf45e2fa01d7f427515392480" - "0600571fac3a5b263fdf57cd2c006497") + ("5c3747465cc36c270e8a35b10828d569c268a20eb78ac332" + "e5e138e26c4454b90f756132e16dce72f18e859835e1f291"), + ("eb2b5693babb7fbb0a76c196067cfdcb11457d9cf45e2fa0" + "1d7f4275153924800600571fac3a5b263fdf57cd2c006497") ] @property From ca9b17ce2919ba3bf08d368063fd8af5b523fe9c Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 28 Mar 2023 15:30:32 +0200 Subject: [PATCH 355/490] Add test cases for P384 fast reduction testing Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 8b602c6e7..88112c561 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -188,6 +188,14 @@ class EcpP384R1Raw(bignum_common.ModOperationCommon, ("ffffffffffffffffffffffffffffffffffffffffffffffff" "fffffffffffffffeffffffff0000000000000000fffffffe"), + # Modulus + 1 + ("ffffffffffffffffffffffffffffffffffffffffffffffff" + "fffffffffffffffeffffffff000000000000000100000000"), + + # 2^384 - 1 + ("ffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffffffffffffffffffffffffffffffffffffffffffff"), + # Maximum canonical P384 multiplication result ("ffffffffffffffffffffffffffffffffffffffffffffffff" "fffffffffffffffdfffffffe0000000000000001fffffffc" From 5f5421c6cbffc29ed10067b6b20b90a80d46ec26 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 28 Mar 2023 15:31:05 +0200 Subject: [PATCH 356/490] Add test cases for P256 fast reduction testing Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 88112c561..d661002c8 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -302,6 +302,12 @@ class EcpP256R1Raw(bignum_common.ModOperationCommon, # Modulus - 1 "ffffffff00000001000000000000000000000000fffffffffffffffffffffffe", + # Modulus + 1 + "ffffffff00000001000000000000000000000001000000000000000000000000", + + # 2^256 - 1 + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + # Maximum canonical P256 multiplication result ("fffffffe00000002fffffffe0000000100000001fffffffe00000001fffffffc" "00000003fffffffcfffffffffffffffffffffffc000000000000000000000004"), From 32116976d753da4e5e547a97677fe73a657cd29b Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 28 Mar 2023 15:32:47 +0200 Subject: [PATCH 357/490] Typo: reorder testing classes Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 158 +++++++++++++++++------------------ 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index d661002c8..0afeca6a2 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -169,6 +169,85 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, return True +class EcpP256R1Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ECP P256 fast reduction.""" + symbol = "-" + test_function = "ecp_mod_p256_raw" + test_name = "ecp_mod_p256_raw" + input_style = "fixed" + arity = 1 + + moduli = ["ffffffff00000001000000000000000000000000ffffffffffffffffffffffff"] # type: List[str] + + input_values = [ + "0", "1", + + # Modulus - 1 + "ffffffff00000001000000000000000000000000fffffffffffffffffffffffe", + + # Modulus + 1 + "ffffffff00000001000000000000000000000001000000000000000000000000", + + # 2^256 - 1 + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + + # Maximum canonical P256 multiplication result + ("fffffffe00000002fffffffe0000000100000001fffffffe00000001fffffffc" + "00000003fffffffcfffffffffffffffffffffffc000000000000000000000004"), + + # Generate an overflow during reduction + ("0000000000000000000000010000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000ffffffff"), + + # Generate an underflow during reduction + ("0000000000000000000000000000000000000000000000000000000000000010" + "ffffffff00000000000000000000000000000000000000000000000000000000"), + + # Generate an overflow during carry reduction + ("aaaaaaaa00000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000aaaaaaacaaaaaaaaaaaaaaaa00000000"), + + # Generate an underflow during carry reduction + ("000000000000000000000001ffffffff00000000000000000000000000000000" + "0000000000000000000000000000000000000002000000020000000100000002"), + + # First 8 number generated by random.getrandbits(512) - seed(2,2) + ("4067c3584ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" + "5c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("82523e86feac7eb7dc38f519b91751dacdbd47d364be8049a372db8f6e405d93" + "ffed9235288bc781ae66267594c9c9500925e4749b575bd13653f8dd9b1f282e"), + ("e8624fab5186ee32ee8d7ee9770348a05d300cb90706a045defc044a09325626" + "e6b58de744ab6cce80877b6f71e1f6d2ef8acd128b4f2fc15f3f57ebf30b94fa"), + ("829a48d422fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578" + "2d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f"), + ("e89204e2e8168561867e5e15bc01bfce6a27e0dfcbf8754472154e76e4c11ab2" + "fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0"), + ("bd143fa9b714210c665d7435c1066932f4767f26294365b2721dea3bf63f23d0" + "dbe53fcafb2147df5ca495fa5a91c89b97eeab64ca2ce6bc5d3fd983c34c769f"), + ("74667bffe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e80371eb9" + "7f81375eecc1cb6347733e847d718d733ff98ff387c56473a7a83ee0761ebfd2"), + ("d08f1bb2531d6460f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f25" + "8ebdbfe3eb9ac688b9d39cca91551e8259cc60b17604e4b4e73695c3e652c71a"), + + # Next 2 number generated by random.getrandbits(256) + "c5e2486c44a4a8f69dc8db48e86ec9c6e06f291b2a838af8d5c44a4eb3172062", + "d4c0dca8b4c9e755cc9c3adcf515a8234da4daeb4f3f87777ad1f45ae9500ec9" + ] + + @property + def arg_a(self) -> str: + return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True + + class EcpP384R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ecp quasi_reduction modulo p384.""" @@ -285,85 +364,6 @@ class EcpP384R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True -class EcpP256R1Raw(bignum_common.ModOperationCommon, - EcpTarget): - """Test cases for ECP P256 fast reduction.""" - symbol = "-" - test_function = "ecp_mod_p256_raw" - test_name = "ecp_mod_p256_raw" - input_style = "fixed" - arity = 1 - - moduli = ["ffffffff00000001000000000000000000000000ffffffffffffffffffffffff"] # type: List[str] - - input_values = [ - "0", "1", - - # Modulus - 1 - "ffffffff00000001000000000000000000000000fffffffffffffffffffffffe", - - # Modulus + 1 - "ffffffff00000001000000000000000000000001000000000000000000000000", - - # 2^256 - 1 - "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - - # Maximum canonical P256 multiplication result - ("fffffffe00000002fffffffe0000000100000001fffffffe00000001fffffffc" - "00000003fffffffcfffffffffffffffffffffffc000000000000000000000004"), - - # Generate an overflow during reduction - ("0000000000000000000000010000000000000000000000000000000000000000" - "00000000000000000000000000000000000000000000000000000000ffffffff"), - - # Generate an underflow during reduction - ("0000000000000000000000000000000000000000000000000000000000000010" - "ffffffff00000000000000000000000000000000000000000000000000000000"), - - # Generate an overflow during carry reduction - ("aaaaaaaa00000000000000000000000000000000000000000000000000000000" - "00000000000000000000000000000000aaaaaaacaaaaaaaaaaaaaaaa00000000"), - - # Generate an underflow during carry reduction - ("000000000000000000000001ffffffff00000000000000000000000000000000" - "0000000000000000000000000000000000000002000000020000000100000002"), - - # First 8 number generated by random.getrandbits(512) - seed(2,2) - ("4067c3584ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" - "5c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), - ("82523e86feac7eb7dc38f519b91751dacdbd47d364be8049a372db8f6e405d93" - "ffed9235288bc781ae66267594c9c9500925e4749b575bd13653f8dd9b1f282e"), - ("e8624fab5186ee32ee8d7ee9770348a05d300cb90706a045defc044a09325626" - "e6b58de744ab6cce80877b6f71e1f6d2ef8acd128b4f2fc15f3f57ebf30b94fa"), - ("829a48d422fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578" - "2d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f"), - ("e89204e2e8168561867e5e15bc01bfce6a27e0dfcbf8754472154e76e4c11ab2" - "fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0"), - ("bd143fa9b714210c665d7435c1066932f4767f26294365b2721dea3bf63f23d0" - "dbe53fcafb2147df5ca495fa5a91c89b97eeab64ca2ce6bc5d3fd983c34c769f"), - ("74667bffe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e80371eb9" - "7f81375eecc1cb6347733e847d718d733ff98ff387c56473a7a83ee0761ebfd2"), - ("d08f1bb2531d6460f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f25" - "8ebdbfe3eb9ac688b9d39cca91551e8259cc60b17604e4b4e73695c3e652c71a"), - - # Next 2 number generated by random.getrandbits(256) - "c5e2486c44a4a8f69dc8db48e86ec9c6e06f291b2a838af8d5c44a4eb3172062", - "d4c0dca8b4c9e755cc9c3adcf515a8234da4daeb4f3f87777ad1f45ae9500ec9" - ] - - @property - def arg_a(self) -> str: - return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits) - - def result(self) -> List[str]: - result = self.int_a % self.int_n - return [self.format_result(result)] - - @property - def is_valid(self) -> bool: - return True - - class EcpP521R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ecp quasi_reduction().""" From 9be2525c5d393833fc49df419634f744f75f8dcd Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 28 Mar 2023 15:34:49 +0200 Subject: [PATCH 358/490] Fix comments Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 0afeca6a2..1c03205c1 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -28,7 +28,7 @@ class EcpTarget(test_data_generation.BaseTarget): class EcpP192R1Raw(bignum_common.ModOperationCommon, EcpTarget): - """Test cases for ecp quasi_reduction().""" + """Test cases for ECP P192 fast reduction.""" symbol = "-" test_function = "ecp_mod_p192_raw" test_name = "ecp_mod_p192_raw" @@ -99,7 +99,7 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, class EcpP224R1Raw(bignum_common.ModOperationCommon, EcpTarget): - """Test cases for ecp quasi_reduction().""" + """Test cases for ECP P224 fast reduction.""" symbol = "-" test_function = "ecp_mod_p224_raw" test_name = "ecp_mod_p224_raw" @@ -250,7 +250,7 @@ class EcpP256R1Raw(bignum_common.ModOperationCommon, class EcpP384R1Raw(bignum_common.ModOperationCommon, EcpTarget): - """Test cases for ecp quasi_reduction modulo p384.""" + """Test cases for ECP P384 fast reduction.""" test_function = "ecp_mod_p384_raw" test_name = "ecp_mod_p384_raw" input_style = "fixed" @@ -366,7 +366,7 @@ class EcpP384R1Raw(bignum_common.ModOperationCommon, class EcpP521R1Raw(bignum_common.ModOperationCommon, EcpTarget): - """Test cases for ecp quasi_reduction().""" + """Test cases for ECP P521 fast reduction.""" test_function = "ecp_mod_p521_raw" test_name = "ecp_mod_p521_raw" input_style = "arch_split" From dc86aa7bd8f2cd925562851d73d523fc30caa255 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Fri, 31 Mar 2023 16:03:14 +0200 Subject: [PATCH 359/490] Fix 0 limb size for value 0 Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 5319ec68b..aa2cd2588 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -68,7 +68,8 @@ def bound_mpi_limbs(limbs: int, bits_in_limb: int) -> int: def limbs_mpi(val: int, bits_in_limb: int) -> int: """Return the number of limbs required to store value.""" - return (val.bit_length() + bits_in_limb - 1) // bits_in_limb + bit_length = max(val.bit_length(), 1) + return (bit_length + bits_in_limb - 1) // bits_in_limb def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: """Return all pair combinations from input values.""" From 3f37a253cd754e98c8191268a2a0e0792f35ffac Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Fri, 31 Mar 2023 16:04:42 +0200 Subject: [PATCH 360/490] Add generated test for core_mul Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_core.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 24d37cbc7..f7b175f08 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -230,6 +230,30 @@ class BignumCoreMLA(BignumCoreTarget, bignum_common.OperationCommon): yield cur_op.create_test_case() +class BignumCoreMul(BignumCoreTarget, bignum_common.OperationCommon): + """Test cases for bignum core multiplication.""" + count = 0 + input_style = "arch_split" + symbol = "*" + test_function = "mpi_core_mul" + test_name = "mbedtls_mpi_core_mul" + arity = 2 + + def format_arg(self, val: str) -> str: + return val + + def format_result(self, res: int) -> str: + res_str = '{:x}'.format(res) + a_limbs = bignum_common.limbs_mpi(self.int_a, self.bits_in_limb) + b_limbs = bignum_common.limbs_mpi(self.int_b, self.bits_in_limb) + hex_digits = bignum_common.hex_digits_for_limb(a_limbs + b_limbs, self.bits_in_limb) + return bignum_common.quote_str(self.format_arg(res_str).zfill(hex_digits)) + + def result(self) -> List[str]: + result = self.int_a * self.int_b + return [self.format_result(result)] + + class BignumCoreMontmul(BignumCoreTarget, test_data_generation.BaseTest): """Test cases for Montgomery multiplication.""" count = 0 From 8645c275c018d9ae677ee19ca317e9168adf8799 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 3 Apr 2023 17:26:44 +0200 Subject: [PATCH 361/490] Multplication is simmetric so only generate unique combinations Signed-off-by: Gabor Mezei --- scripts/framework_dev/bignum_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index f7b175f08..e914ae72d 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -238,6 +238,7 @@ class BignumCoreMul(BignumCoreTarget, bignum_common.OperationCommon): test_function = "mpi_core_mul" test_name = "mbedtls_mpi_core_mul" arity = 2 + unique_combinations_only = True def format_arg(self, val: str) -> str: return val From f3c99a923bf2b9a3a45e820eb168577d9b264ca6 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Tue, 21 Mar 2023 10:04:12 +0000 Subject: [PATCH 362/490] bignum: Removed merge scaffolding. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_common.py | 40 ------------------------- scripts/framework_dev/bignum_core.py | 38 ----------------------- scripts/framework_dev/bignum_mod.py | 37 ----------------------- scripts/framework_dev/bignum_mod_raw.py | 34 --------------------- 4 files changed, 149 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index aa2cd2588..b942070e8 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -389,43 +389,3 @@ class ModOperationCommon(OperationCommon): lambda test_object: test_object.is_valid, chain(test_objects, special_cases) )) - -# BEGIN MERGE SLOT 1 - -# END MERGE SLOT 1 - -# BEGIN MERGE SLOT 2 - -# END MERGE SLOT 2 - -# BEGIN MERGE SLOT 3 - -# END MERGE SLOT 3 - -# BEGIN MERGE SLOT 4 - -# END MERGE SLOT 4 - -# BEGIN MERGE SLOT 5 - -# END MERGE SLOT 5 - -# BEGIN MERGE SLOT 6 - -# END MERGE SLOT 6 - -# BEGIN MERGE SLOT 7 - -# END MERGE SLOT 7 - -# BEGIN MERGE SLOT 8 - -# END MERGE SLOT 8 - -# BEGIN MERGE SLOT 9 - -# END MERGE SLOT 9 - -# BEGIN MERGE SLOT 10 - -# END MERGE SLOT 10 diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index e914ae72d..5801caef5 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -774,7 +774,6 @@ def mpi_modmul_case_generate() -> None: i += 1 print(generated_inputs) -# BEGIN MERGE SLOT 1 class BignumCoreExpMod(BignumCoreTarget, bignum_common.ModOperationCommon): """Test cases for bignum core exponentiation.""" @@ -796,13 +795,6 @@ class BignumCoreExpMod(BignumCoreTarget, bignum_common.ModOperationCommon): # the modulus (see for example exponent blinding) return bool(self.int_a < self.int_n) -# END MERGE SLOT 1 - -# BEGIN MERGE SLOT 2 - -# END MERGE SLOT 2 - -# BEGIN MERGE SLOT 3 class BignumCoreSubInt(BignumCoreTarget, bignum_common.OperationCommon): """Test cases for bignum core sub int.""" @@ -848,33 +840,3 @@ class BignumCoreZeroCheckCT(BignumCoreTarget, bignum_common.OperationCommon): def result(self) -> List[str]: result = 1 if self.int_a == 0 else 0 return [str(result)] - -# END MERGE SLOT 3 - -# BEGIN MERGE SLOT 4 - -# END MERGE SLOT 4 - -# BEGIN MERGE SLOT 5 - -# END MERGE SLOT 5 - -# BEGIN MERGE SLOT 6 - -# END MERGE SLOT 6 - -# BEGIN MERGE SLOT 7 - -# END MERGE SLOT 7 - -# BEGIN MERGE SLOT 8 - -# END MERGE SLOT 8 - -# BEGIN MERGE SLOT 9 - -# END MERGE SLOT 9 - -# BEGIN MERGE SLOT 10 - -# END MERGE SLOT 10 diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index a83e1360a..77c7b1bbd 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -25,11 +25,6 @@ class BignumModTarget(test_data_generation.BaseTarget): """Target for bignum mod test case generation.""" target_basename = 'test_suite_bignum_mod.generated' -# BEGIN MERGE SLOT 1 - -# END MERGE SLOT 1 - -# BEGIN MERGE SLOT 2 class BignumModMul(bignum_common.ModOperationCommon, BignumModTarget): @@ -51,9 +46,6 @@ class BignumModMul(bignum_common.ModOperationCommon, result = (self.int_a * self.int_b) % self.int_n return [self.format_result(self.to_montgomery(result))] -# END MERGE SLOT 2 - -# BEGIN MERGE SLOT 3 class BignumModSub(bignum_common.ModOperationCommon, BignumModTarget): """Test cases for bignum mpi_mod_sub().""" @@ -105,13 +97,7 @@ class BignumModInvMont(bignum_common.ModOperationCommon, BignumModTarget): # generated cases return [self.format_result(mont_result), "0"] -# END MERGE SLOT 3 -# BEGIN MERGE SLOT 4 - -# END MERGE SLOT 4 - -# BEGIN MERGE SLOT 5 class BignumModAdd(bignum_common.ModOperationCommon, BignumModTarget): """Test cases for bignum mpi_mod_add().""" count = 0 @@ -125,26 +111,3 @@ class BignumModAdd(bignum_common.ModOperationCommon, BignumModTarget): # To make negative tests easier, append "0" for success to the # generated cases return [self.format_result(result), "0"] - - -# END MERGE SLOT 5 - -# BEGIN MERGE SLOT 6 - -# END MERGE SLOT 6 - -# BEGIN MERGE SLOT 7 - -# END MERGE SLOT 7 - -# BEGIN MERGE SLOT 8 - -# END MERGE SLOT 8 - -# BEGIN MERGE SLOT 9 - -# END MERGE SLOT 9 - -# BEGIN MERGE SLOT 10 - -# END MERGE SLOT 10 diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index d197b5491..7121f2f49 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -26,11 +26,6 @@ class BignumModRawTarget(test_data_generation.BaseTarget): """Target for bignum mod_raw test case generation.""" target_basename = 'test_suite_bignum_mod_raw.generated' -# BEGIN MERGE SLOT 1 - -# END MERGE SLOT 1 - -# BEGIN MERGE SLOT 2 class BignumModRawSub(bignum_common.ModOperationCommon, BignumModRawTarget): @@ -101,9 +96,6 @@ class BignumModRawMul(bignum_common.ModOperationCommon, result = (self.int_a * self.int_b) % self.int_n return [self.format_result(self.to_montgomery(result))] -# END MERGE SLOT 2 - -# BEGIN MERGE SLOT 3 class BignumModRawInvPrime(bignum_common.ModOperationCommon, BignumModRawTarget): @@ -123,13 +115,6 @@ class BignumModRawInvPrime(bignum_common.ModOperationCommon, mont_result = self.to_montgomery(result) return [self.format_result(mont_result)] -# END MERGE SLOT 3 - -# BEGIN MERGE SLOT 4 - -# END MERGE SLOT 4 - -# BEGIN MERGE SLOT 5 class BignumModRawAdd(bignum_common.ModOperationCommon, BignumModRawTarget): @@ -144,9 +129,6 @@ class BignumModRawAdd(bignum_common.ModOperationCommon, result = (self.int_a + self.int_b) % self.int_n return [self.format_result(result)] -# END MERGE SLOT 5 - -# BEGIN MERGE SLOT 6 class BignumModRawConvertRep(bignum_common.ModOperationCommon, BignumModRawTarget): @@ -230,9 +212,6 @@ class BignumModRawModulusToCanonicalRep(BignumModRawConvertRep): def result(self) -> List[str]: return [self.format_result(self.int_a)] -# END MERGE SLOT 6 - -# BEGIN MERGE SLOT 7 class BignumModRawConvertToMont(bignum_common.ModOperationCommon, BignumModRawTarget): @@ -272,16 +251,3 @@ class BignumModRawModNegate(bignum_common.ModOperationCommon, def result(self) -> List[str]: result = (self.int_n - self.int_a) % self.int_n return [self.format_result(result)] -# END MERGE SLOT 7 - -# BEGIN MERGE SLOT 8 - -# END MERGE SLOT 8 - -# BEGIN MERGE SLOT 9 - -# END MERGE SLOT 9 - -# BEGIN MERGE SLOT 10 - -# END MERGE SLOT 10 From 32f3bcc007aec890c623df45a0c1dcb9a373cb88 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Tue, 4 Apr 2023 16:05:54 +0800 Subject: [PATCH 363/490] cert_audit: Initial script for auditing expiry date We introduce the script to audit the expiry date of X509 files (i.e. crt/crl/csr files) in tests/data_files/ folder. This commit add basic classes and the framework for auditing and "-a" option to list all valid crt/crl/csr files it found. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 282 ++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100755 scripts/audit-validity-dates.py diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py new file mode 100755 index 000000000..67d409665 --- /dev/null +++ b/scripts/audit-validity-dates.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# +# copyright the mbed tls contributors +# spdx-license-identifier: apache-2.0 +# +# licensed under the apache license, version 2.0 (the "license"); you may +# not use this file except in compliance with the license. +# you may obtain a copy of the license at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Audit validity date of X509 crt/crl/csr + +This script is used to audit the validity date of crt/crl/csr used for testing. +The files are in tests/data_files/ while some data are in test suites data in +tests/suites/*.data files. +""" + +import os +import sys +import re +import typing +import types +import argparse +import datetime +from enum import Enum + +from cryptography import x509 + +class DataType(Enum): + CRT = 1 # Certificate + CRL = 2 # Certificate Revocation List + CSR = 3 # Certificate Signing Request + +class DataFormat(Enum): + PEM = 1 # Privacy-Enhanced Mail + DER = 2 # Distinguished Encoding Rules + +class AuditData: + """Store file, type and expiration date for audit.""" + #pylint: disable=too-few-public-methods + def __init__(self, data_type: DataType): + self.data_type = data_type + self.filename = "" + self.not_valid_after: datetime.datetime + self.not_valid_before: datetime.datetime + + def fill_validity_duration(self, x509_obj): + """Fill expiration_date field from a x509 object""" + # Certificate expires after "not_valid_after" + # Certificate is invalid before "not_valid_before" + if self.data_type == DataType.CRT: + self.not_valid_after = x509_obj.not_valid_after + self.not_valid_before = x509_obj.not_valid_before + # CertificateRevocationList expires after "next_update" + # CertificateRevocationList is invalid before "last_update" + elif self.data_type == DataType.CRL: + self.not_valid_after = x509_obj.next_update + self.not_valid_before = x509_obj.last_update + # CertificateSigningRequest is always valid. + elif self.data_type == DataType.CSR: + self.not_valid_after = datetime.datetime.max + self.not_valid_before = datetime.datetime.min + else: + raise ValueError("Unsupported file_type: {}".format(self.data_type)) + +class X509Parser(): + """A parser class to parse crt/crl/csr file or data in PEM/DER format.""" + PEM_REGEX = br'-{5}BEGIN (?P.*?)-{5}\n(?P.*?)-{5}END (?P=type)-{5}\n' + PEM_TAG_REGEX = br'-{5}BEGIN (?P.*?)-{5}\n' + PEM_TAGS = { + DataType.CRT: 'CERTIFICATE', + DataType.CRL: 'X509 CRL', + DataType.CSR: 'CERTIFICATE REQUEST' + } + + def __init__(self, backends: dict): + self.backends = backends + self.__generate_parsers() + + def __generate_parser(self, data_type: DataType): + """Parser generator for a specific DataType""" + tag = self.PEM_TAGS[data_type] + pem_loader = self.backends[data_type][DataFormat.PEM] + der_loader = self.backends[data_type][DataFormat.DER] + def wrapper(data: bytes): + pem_type = X509Parser.pem_data_type(data) + # It is in PEM format with target tag + if pem_type == tag: + return pem_loader(data) + # It is in PEM format without target tag + if pem_type: + return None + # It might be in DER format + try: + result = der_loader(data) + except ValueError: + result = None + return result + wrapper.__name__ = "{}.parser[{}]".format(type(self).__name__, tag) + return wrapper + + def __generate_parsers(self): + """Generate parsers for all support DataType""" + self.parsers = {} + for data_type, _ in self.PEM_TAGS.items(): + self.parsers[data_type] = self.__generate_parser(data_type) + + def __getitem__(self, item): + return self.parsers[item] + + @staticmethod + def pem_data_type(data: bytes) -> str: + """Get the tag from the data in PEM format + + :param data: data to be checked in binary mode. + :return: PEM tag or "" when no tag detected. + """ + m = re.search(X509Parser.PEM_TAG_REGEX, data) + if m is not None: + return m.group('type').decode('UTF-8') + else: + return "" + +class Auditor: + """A base class for audit.""" + def __init__(self, verbose): + self.verbose = verbose + self.default_files = [] + self.audit_data = [] + self.parser = X509Parser({ + DataType.CRT: { + DataFormat.PEM: x509.load_pem_x509_certificate, + DataFormat.DER: x509.load_der_x509_certificate + }, + DataType.CRL: { + DataFormat.PEM: x509.load_pem_x509_crl, + DataFormat.DER: x509.load_der_x509_crl + }, + DataType.CSR: { + DataFormat.PEM: x509.load_pem_x509_csr, + DataFormat.DER: x509.load_der_x509_csr + }, + }) + + def error(self, *args): + #pylint: disable=no-self-use + print("Error: ", *args, file=sys.stderr) + + def warn(self, *args): + if self.verbose: + print("Warn: ", *args, file=sys.stderr) + + def parse_file(self, filename: str) -> typing.List[AuditData]: + """ + Parse a list of AuditData from file. + + :param filename: name of the file to parse. + :return list of AuditData parsed from the file. + """ + with open(filename, 'rb') as f: + data = f.read() + result_list = [] + result = self.parse_bytes(data) + if result is not None: + result.filename = filename + result_list.append(result) + return result_list + + def parse_bytes(self, data: bytes): + """Parse AuditData from bytes.""" + for data_type in list(DataType): + try: + result = self.parser[data_type](data) + except ValueError as val_error: + result = None + self.warn(val_error) + if result is not None: + audit_data = AuditData(data_type) + audit_data.fill_validity_duration(result) + return audit_data + return None + + def walk_all(self, file_list): + """ + Iterate over all the files in the list and get audit data. + """ + if not file_list: + file_list = self.default_files + for filename in file_list: + data_list = self.parse_file(filename) + self.audit_data.extend(data_list) + + def for_each(self, do, *args, **kwargs): + """ + Sort the audit data and iterate over them. + """ + if not isinstance(do, types.FunctionType): + return + for d in self.audit_data: + do(d, *args, **kwargs) + + @staticmethod + def find_test_dir(): + """Get the relative path for the MbedTLS test directory.""" + if os.path.isdir('tests'): + tests_dir = 'tests' + elif os.path.isdir('suites'): + tests_dir = '.' + elif os.path.isdir('../suites'): + tests_dir = '..' + else: + raise Exception("Mbed TLS source tree not found") + return tests_dir + +class TestDataAuditor(Auditor): + """Class for auditing files in tests/data_files/""" + def __init__(self, verbose): + super().__init__(verbose) + self.default_files = self.collect_default_files() + + def collect_default_files(self): + """collect all files in tests/data_files/""" + test_dir = self.find_test_dir() + test_data_folder = os.path.join(test_dir, 'data_files') + data_files = [] + for (dir_path, _, file_names) in os.walk(test_data_folder): + data_files.extend(os.path.join(dir_path, file_name) + for file_name in file_names) + return data_files + + +def list_all(audit_data: AuditData): + print("{}\t{}\t{}\t{}".format( + audit_data.not_valid_before.isoformat(timespec='seconds'), + audit_data.not_valid_after.isoformat(timespec='seconds'), + audit_data.data_type.name, + audit_data.filename)) + +def main(): + """ + Perform argument parsing. + """ + parser = argparse.ArgumentParser( + description='Audit script for X509 crt/crl/csr files.' + ) + + parser.add_argument('-a', '--all', + action='store_true', + help='list the information of all files') + parser.add_argument('-v', '--verbose', + action='store_true', dest='verbose', + help='Show warnings') + parser.add_argument('-f', '--file', dest='file', + help='file to audit (Debug only)', + metavar='FILE') + + args = parser.parse_args() + + # start main routine + td_auditor = TestDataAuditor(args.verbose) + + if args.file: + data_files = [args.file] + else: + data_files = td_auditor.default_files + + td_auditor.walk_all(data_files) + + if args.all: + td_auditor.for_each(list_all) + + print("\nDone!\n") + +if __name__ == "__main__": + main() From 109399cd85974a7a9a84c9dfeda7201dd4c7e9a9 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Thu, 6 Apr 2023 14:33:41 +0800 Subject: [PATCH 364/490] cert_audit: Support audit on test suite data files Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 43 ++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 67d409665..0d1425b28 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -29,6 +29,7 @@ import typing import types import argparse import datetime +import glob from enum import Enum from cryptography import x509 @@ -226,7 +227,7 @@ class TestDataAuditor(Auditor): self.default_files = self.collect_default_files() def collect_default_files(self): - """collect all files in tests/data_files/""" + """Collect all files in tests/data_files/""" test_dir = self.find_test_dir() test_data_folder = os.path.join(test_dir, 'data_files') data_files = [] @@ -235,6 +236,38 @@ class TestDataAuditor(Auditor): for file_name in file_names) return data_files +class SuiteDataAuditor(Auditor): + """Class for auditing files in tests/suites/*.data""" + def __init__(self, options): + super().__init__(options) + self.default_files = self.collect_default_files() + + def collect_default_files(self): + """Collect all files in tests/suites/*.data""" + test_dir = self.find_test_dir() + suites_data_folder = os.path.join(test_dir, 'suites') + # collect all data files in tests/suites (114 in total) + data_files = glob.glob(os.path.join(suites_data_folder, '*.data')) + return data_files + + def parse_file(self, filename: str): + """Parse AuditData from file.""" + with open(filename, 'r') as f: + data = f.read() + audit_data_list = [] + # extract hex strings from the data file. + hex_strings = re.findall(r'"(?P[0-9a-fA-F]+)"', data) + for hex_str in hex_strings: + # We regard hex string with odd number length as invaild data. + if len(hex_str) & 1: + continue + bytes_data = bytes.fromhex(hex_str) + audit_data = self.parse_bytes(bytes_data) + if audit_data is None: + continue + audit_data.filename = filename + audit_data_list.append(audit_data) + return audit_data_list def list_all(audit_data: AuditData): print("{}\t{}\t{}\t{}".format( @@ -265,16 +298,24 @@ def main(): # start main routine td_auditor = TestDataAuditor(args.verbose) + sd_auditor = SuiteDataAuditor(args.verbose) if args.file: data_files = [args.file] + suite_data_files = [args.file] else: data_files = td_auditor.default_files + suite_data_files = sd_auditor.default_files td_auditor.walk_all(data_files) + # TODO: Improve the method for auditing test suite data files + # It takes 6 times longer than td_auditor.walk_all(), + # typically 0.827 s VS 0.147 s. + sd_auditor.walk_all(suite_data_files) if args.all: td_auditor.for_each(list_all) + sd_auditor.for_each(list_all) print("\nDone!\n") From e5624fc2be620ab5400eb6a4727f428a923787d8 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Tue, 21 Mar 2023 12:08:37 +0000 Subject: [PATCH 365/490] test_suite_ecp: Introduced `ecp_mod_p_generic_raw` This patch replaces similiarly structured test functions for: * MBEDTLS_ECP_DP_SECP192R1 * MBEDTLS_ECP_DP_SECP224R1 * MBEDTLS_ECP_DP_SECP256R1 * MBEDTLS_ECP_DP_SECP384R1 * MBEDTLS_ECP_DP_BP512R1R1 with a more generic version, which adjusts the parameters, based on the `curve_id` field, provided by the testing data. The python test framework has been updated to provide that extra field. Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 1c03205c1..0f4651151 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -30,7 +30,7 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ECP P192 fast reduction.""" symbol = "-" - test_function = "ecp_mod_p192_raw" + test_function = "ecp_mod_p_generic_raw" test_name = "ecp_mod_p192_raw" input_style = "fixed" arity = 1 @@ -96,12 +96,16 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True + def arguments(self): + args = super().arguments() + return ["MBEDTLS_ECP_DP_SECP192R1"] + args + class EcpP224R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ECP P224 fast reduction.""" symbol = "-" - test_function = "ecp_mod_p224_raw" + test_function = "ecp_mod_p_generic_raw" test_name = "ecp_mod_p224_raw" input_style = "arch_split" arity = 1 @@ -168,12 +172,16 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True + def arguments(self): + args = super().arguments() + return ["MBEDTLS_ECP_DP_SECP224R1"] + args + class EcpP256R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ECP P256 fast reduction.""" symbol = "-" - test_function = "ecp_mod_p256_raw" + test_function = "ecp_mod_p_generic_raw" test_name = "ecp_mod_p256_raw" input_style = "fixed" arity = 1 @@ -247,11 +255,15 @@ class EcpP256R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True + def arguments(self): + args = super().arguments() + return ["MBEDTLS_ECP_DP_SECP256R1"] + args + class EcpP384R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ECP P384 fast reduction.""" - test_function = "ecp_mod_p384_raw" + test_function = "ecp_mod_p_generic_raw" test_name = "ecp_mod_p384_raw" input_style = "fixed" arity = 1 @@ -364,10 +376,15 @@ class EcpP384R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True + def arguments(self): + args = super().arguments() + return ["MBEDTLS_ECP_DP_SECP384R1"] + args + + class EcpP521R1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ECP P521 fast reduction.""" - test_function = "ecp_mod_p521_raw" + test_function = "ecp_mod_p_generic_raw" test_name = "ecp_mod_p521_raw" input_style = "arch_split" arity = 1 @@ -462,3 +479,7 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, @property def is_valid(self) -> bool: return True + + def arguments(self): + args = super().arguments() + return ["MBEDTLS_ECP_DP_SECP521R1"] + args From 47f70171eb7071ca4b54240c48b41f70711c350e Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Thu, 6 Apr 2023 16:33:10 +0100 Subject: [PATCH 366/490] ecp.py: Set test-dependencies as attributes. This patch enables declaring dependencie as test-class members. ECP curve functions have been updated to use the new capability. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_common.py | 5 ++++- scripts/framework_dev/ecp.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index b942070e8..d8ef4a84f 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -17,6 +17,7 @@ from abc import abstractmethod import enum from typing import Iterator, List, Tuple, TypeVar, Any +from copy import deepcopy from itertools import chain from . import test_case @@ -104,6 +105,7 @@ class OperationCommon(test_data_generation.BaseTest): symbol = "" input_values = INPUTS_DEFAULT # type: List[str] input_cases = [] # type: List[Any] + dependencies = [] # type: List[Any] unique_combinations_only = False input_styles = ["variable", "fixed", "arch_split"] # type: List[str] input_style = "variable" # type: str @@ -119,10 +121,11 @@ class OperationCommon(test_data_generation.BaseTest): # provides earlier/more robust input validation. self.int_a = hex_to_int(val_a) self.int_b = hex_to_int(val_b) + self.dependencies = deepcopy(self.dependencies) if bits_in_limb not in self.limb_sizes: raise ValueError("Invalid number of bits in limb!") if self.input_style == "arch_split": - self.dependencies = ["MBEDTLS_HAVE_INT{:d}".format(bits_in_limb)] + self.dependencies.append("MBEDTLS_HAVE_INT{:d}".format(bits_in_limb)) self.bits_in_limb = bits_in_limb @property diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 0f4651151..d1d23c130 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -34,6 +34,7 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p192_raw" input_style = "fixed" arity = 1 + dependencies = ["MBEDTLS_ECP_DP_SECP192R1_ENABLED"] moduli = ["fffffffffffffffffffffffffffffffeffffffffffffffff"] # type: List[str] @@ -109,6 +110,7 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p224_raw" input_style = "arch_split" arity = 1 + dependencies = ["MBEDTLS_ECP_DP_SECP224R1_ENABLED"] moduli = ["ffffffffffffffffffffffffffffffff000000000000000000000001"] # type: List[str] @@ -185,6 +187,7 @@ class EcpP256R1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p256_raw" input_style = "fixed" arity = 1 + dependencies = ["MBEDTLS_ECP_DP_SECP256R1_ENABLED"] moduli = ["ffffffff00000001000000000000000000000000ffffffffffffffffffffffff"] # type: List[str] @@ -267,6 +270,7 @@ class EcpP384R1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p384_raw" input_style = "fixed" arity = 1 + dependencies = ["MBEDTLS_ECP_DP_SECP384R1_ENABLED"] moduli = [("ffffffffffffffffffffffffffffffffffffffffffffffff" "fffffffffffffffeffffffff0000000000000000ffffffff") @@ -388,6 +392,7 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p521_raw" input_style = "arch_split" arity = 1 + dependencies = ["MBEDTLS_ECP_DP_SECP521R1_ENABLED"] moduli = [("01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") From 8e8cc2bac1cf3164884bd5e972cff496c3af92fd Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 7 Apr 2023 18:04:07 +0800 Subject: [PATCH 367/490] cert_audit: Parse more information from test suite data file Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 126 +++++++++++++++++++++++++++----- 1 file changed, 107 insertions(+), 19 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 0d1425b28..5e22bfca9 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -34,6 +34,9 @@ from enum import Enum from cryptography import x509 +# reuse the function to parse *.data file in tests/suites/ +from generate_test_code import parse_test_data as parse_suite_data + class DataType(Enum): CRT = 1 # Certificate CRL = 2 # Certificate Revocation List @@ -129,6 +132,32 @@ class X509Parser(): else: return "" + @staticmethod + def check_hex_string(hex_str: str) -> bool: + """Check if the hex string is possibly DER data.""" + hex_len = len(hex_str) + # At least 6 hex char for 3 bytes: Type + Length + Content + if hex_len < 6: + return False + # Check if Type (1 byte) is SEQUENCE. + if hex_str[0:2] != '30': + return False + # Check LENGTH (1 byte) value + content_len = int(hex_str[2:4], base=16) + consumed = 4 + if content_len in (128, 255): + # Indefinite or Reserved + return False + elif content_len > 127: + # Definite, Long + length_len = (content_len - 128) * 2 + content_len = int(hex_str[consumed:consumed+length_len], base=16) + consumed += length_len + # Check LENGTH + if hex_len != content_len * 2 + consumed: + return False + return True + class Auditor: """A base class for audit.""" def __init__(self, verbose): @@ -236,6 +265,64 @@ class TestDataAuditor(Auditor): for file_name in file_names) return data_files +class FileWrapper(): + """ + This a stub class of generate_test_code.FileWrapper. + + This class reads the whole file to memory before iterating + over the lines. + """ + + def __init__(self, file_name): + """ + Read the file and initialize the line number to 0. + + :param file_name: File path to open. + """ + with open(file_name, 'rb') as f: + self.buf = f.read() + self.buf_len = len(self.buf) + self._line_no = 0 + self._line_start = 0 + + def __iter__(self): + """Make the class iterable.""" + return self + + def __next__(self): + """ + This method for returning a line of the file per iteration. + + :return: Line read from file. + """ + # If we reach the end of the file. + if not self._line_start < self.buf_len: + raise StopIteration + + line_end = self.buf.find(b'\n', self._line_start) + 1 + if line_end > 0: + # Find the first LF as the end of the new line. + line = self.buf[self._line_start:line_end] + self._line_start = line_end + self._line_no += 1 + else: + # No LF found. We are at the last line without LF. + line = self.buf[self._line_start:] + self._line_start = self.buf_len + self._line_no += 1 + + # Convert byte array to string with correct encoding and + # strip any whitespaces added in the decoding process. + return line.decode(sys.getdefaultencoding()).rstrip() + '\n' + + def get_line_no(self): + """ + Gives current line number. + """ + return self._line_no + + line_no = property(get_line_no) + class SuiteDataAuditor(Auditor): """Class for auditing files in tests/suites/*.data""" def __init__(self, options): @@ -246,27 +333,31 @@ class SuiteDataAuditor(Auditor): """Collect all files in tests/suites/*.data""" test_dir = self.find_test_dir() suites_data_folder = os.path.join(test_dir, 'suites') - # collect all data files in tests/suites (114 in total) data_files = glob.glob(os.path.join(suites_data_folder, '*.data')) return data_files def parse_file(self, filename: str): - """Parse AuditData from file.""" - with open(filename, 'r') as f: - data = f.read() + """ + Parse a list of AuditData from file. + + :param filename: name of the file to parse. + :return list of AuditData parsed from the file. + """ audit_data_list = [] - # extract hex strings from the data file. - hex_strings = re.findall(r'"(?P[0-9a-fA-F]+)"', data) - for hex_str in hex_strings: - # We regard hex string with odd number length as invaild data. - if len(hex_str) & 1: - continue - bytes_data = bytes.fromhex(hex_str) - audit_data = self.parse_bytes(bytes_data) - if audit_data is None: - continue - audit_data.filename = filename - audit_data_list.append(audit_data) + data_f = FileWrapper(filename) + for _, _, _, test_args in parse_suite_data(data_f): + for test_arg in test_args: + match = re.match(r'"(?P[0-9a-fA-F]+)"', test_arg) + if not match: + continue + if not X509Parser.check_hex_string(match.group('data')): + continue + audit_data = self.parse_bytes(bytes.fromhex(match.group('data'))) + if audit_data is None: + continue + audit_data.filename = filename + audit_data_list.append(audit_data) + return audit_data_list def list_all(audit_data: AuditData): @@ -308,9 +399,6 @@ def main(): suite_data_files = sd_auditor.default_files td_auditor.walk_all(data_files) - # TODO: Improve the method for auditing test suite data files - # It takes 6 times longer than td_auditor.walk_all(), - # typically 0.827 s VS 0.147 s. sd_auditor.walk_all(suite_data_files) if args.all: From 3b7a9416e3599816304e6fccab59dca789135fe6 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Tue, 11 Apr 2023 13:39:31 +0800 Subject: [PATCH 368/490] cert_audit: Introduce not-[before|after] option Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 37 ++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 5e22bfca9..85c0bd9dd 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -26,7 +26,6 @@ import os import sys import re import typing -import types import argparse import datetime import glob @@ -227,15 +226,6 @@ class Auditor: data_list = self.parse_file(filename) self.audit_data.extend(data_list) - def for_each(self, do, *args, **kwargs): - """ - Sort the audit data and iterate over them. - """ - if not isinstance(do, types.FunctionType): - return - for d in self.audit_data: - do(d, *args, **kwargs) - @staticmethod def find_test_dir(): """Get the relative path for the MbedTLS test directory.""" @@ -381,6 +371,12 @@ def main(): parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', help='Show warnings') + parser.add_argument('--not-before', dest='not_before', + help='not valid before this date(UTC), YYYY-MM-DD', + metavar='DATE') + parser.add_argument('--not-after', dest='not_after', + help='not valid after this date(UTC), YYYY-MM-DD', + metavar='DATE') parser.add_argument('-f', '--file', dest='file', help='file to audit (Debug only)', metavar='FILE') @@ -398,12 +394,29 @@ def main(): data_files = td_auditor.default_files suite_data_files = sd_auditor.default_files + if args.not_before: + not_before_date = datetime.datetime.fromisoformat(args.not_before) + else: + not_before_date = datetime.datetime.today() + if args.not_after: + not_after_date = datetime.datetime.fromisoformat(args.not_after) + else: + not_after_date = not_before_date + td_auditor.walk_all(data_files) sd_auditor.walk_all(suite_data_files) + audit_results = td_auditor.audit_data + sd_auditor.audit_data + + # we filter out the files whose validity duration covers the provide + # duration. + filter_func = lambda d: (not_before_date < d.not_valid_before) or \ + (d.not_valid_after < not_after_date) if args.all: - td_auditor.for_each(list_all) - sd_auditor.for_each(list_all) + filter_func = None + + for d in filter(filter_func, audit_results): + list_all(d) print("\nDone!\n") From 2a4abc479c320fb029e137242a9e351924fbc545 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Tue, 11 Apr 2023 15:05:29 +0800 Subject: [PATCH 369/490] cert_audit: Fill validity dates in AuditData constructor Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 85c0bd9dd..472041e16 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -48,11 +48,10 @@ class DataFormat(Enum): class AuditData: """Store file, type and expiration date for audit.""" #pylint: disable=too-few-public-methods - def __init__(self, data_type: DataType): + def __init__(self, data_type: DataType, x509_obj): self.data_type = data_type self.filename = "" - self.not_valid_after: datetime.datetime - self.not_valid_before: datetime.datetime + self.fill_validity_duration(x509_obj) def fill_validity_duration(self, x509_obj): """Fill expiration_date field from a x509 object""" @@ -211,8 +210,7 @@ class Auditor: result = None self.warn(val_error) if result is not None: - audit_data = AuditData(data_type) - audit_data.fill_validity_duration(result) + audit_data = AuditData(data_type, result) return audit_data return None From a678c0a4bcd56abc5769d87bcb8ccb63e5a1f20d Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Tue, 11 Apr 2023 16:30:54 +0800 Subject: [PATCH 370/490] cert_audit: Disable pylint error for importing cryptography This is to make CI happy. The script requires cryptography >= 35.0.0, which is only available for Python >= 3.6. But both ubuntu-16.04 and Travis CI are using Python 3.5.x. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 472041e16..3f1987030 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -31,7 +31,10 @@ import datetime import glob from enum import Enum -from cryptography import x509 +# The script requires cryptography >= 35.0.0 which is only available +# for Python >= 3.6. Disable the pylint error here until we were +# using modern system on our CI. +from cryptography import x509 #pylint: disable=import-error # reuse the function to parse *.data file in tests/suites/ from generate_test_code import parse_test_data as parse_suite_data From adda9daaad2f2a2c2681bfa5b2655a46450eef95 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Thu, 13 Apr 2023 14:42:37 +0800 Subject: [PATCH 371/490] cert_audit: Make FILE as positional argument Make FILE as positional argument so that we can pass multiple files to the script. This commit also contains some help message improvements. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 3f1987030..577179d0b 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -15,11 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Audit validity date of X509 crt/crl/csr +"""Audit validity date of X509 crt/crl/csr. This script is used to audit the validity date of crt/crl/csr used for testing. -The files are in tests/data_files/ while some data are in test suites data in -tests/suites/*.data files. +It prints the information of X509 data whose validity duration does not cover +the provided validity duration. The data are collected from tests/data_files/ +and tests/suites/*.data files by default. """ import os @@ -362,24 +363,23 @@ def main(): """ Perform argument parsing. """ - parser = argparse.ArgumentParser( - description='Audit script for X509 crt/crl/csr files.' - ) + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-a', '--all', action='store_true', - help='list the information of all files') + help='list the information of all the files') parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', - help='Show warnings') + help='show warnings') parser.add_argument('--not-before', dest='not_before', - help='not valid before this date(UTC), YYYY-MM-DD', + help=('not valid before this date (UTC, YYYY-MM-DD). ' + 'Default: today'), metavar='DATE') parser.add_argument('--not-after', dest='not_after', - help='not valid after this date(UTC), YYYY-MM-DD', + help=('not valid after this date (UTC, YYYY-MM-DD). ' + 'Default: not-before'), metavar='DATE') - parser.add_argument('-f', '--file', dest='file', - help='file to audit (Debug only)', + parser.add_argument('files', nargs='*', help='files to audit', metavar='FILE') args = parser.parse_args() @@ -388,9 +388,9 @@ def main(): td_auditor = TestDataAuditor(args.verbose) sd_auditor = SuiteDataAuditor(args.verbose) - if args.file: - data_files = [args.file] - suite_data_files = [args.file] + if args.files: + data_files = args.files + suite_data_files = args.files else: data_files = td_auditor.default_files suite_data_files = sd_auditor.default_files @@ -408,7 +408,7 @@ def main(): sd_auditor.walk_all(suite_data_files) audit_results = td_auditor.audit_data + sd_auditor.audit_data - # we filter out the files whose validity duration covers the provide + # we filter out the files whose validity duration covers the provided # duration. filter_func = lambda d: (not_before_date < d.not_valid_before) or \ (d.not_valid_after < not_after_date) From ac593c760279093dce063c4d9068ffb1787796f3 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Thu, 13 Apr 2023 15:55:30 +0800 Subject: [PATCH 372/490] cert_audit: Output line/argument number for *.data files Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 577179d0b..537cf40f0 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -338,7 +338,7 @@ class SuiteDataAuditor(Auditor): audit_data_list = [] data_f = FileWrapper(filename) for _, _, _, test_args in parse_suite_data(data_f): - for test_arg in test_args: + for idx, test_arg in enumerate(test_args): match = re.match(r'"(?P[0-9a-fA-F]+)"', test_arg) if not match: continue @@ -347,7 +347,9 @@ class SuiteDataAuditor(Auditor): audit_data = self.parse_bytes(bytes.fromhex(match.group('data'))) if audit_data is None: continue - audit_data.filename = filename + audit_data.filename = "{}:{}:{}".format(filename, + data_f.line_no, + idx + 1) audit_data_list.append(audit_data) return audit_data_list From 2184cf25fbb6ec57df90a453dea491548592e79b Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 11 Apr 2023 15:43:12 +0200 Subject: [PATCH 373/490] Add generated tests for ecp_mod_p192k1 Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index d1d23c130..b211c98eb 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -488,3 +488,66 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, def arguments(self): args = super().arguments() return ["MBEDTLS_ECP_DP_SECP521R1"] + args + + +class EcpP192K1Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ECP P192K1 fast reduction.""" + symbol = "-" + test_function = "ecp_mod_p192k1" + test_name = "ecp_mod_p192k1" + input_style = "fixed" + arity = 1 + + moduli = ["fffffffffffffffffffffffffffffffffffffffeffffee37"] # type: List[str] + + input_values = [ + "0", "1", + + # Modulus - 1 + "fffffffffffffffffffffffffffffffffffffffeffffee36", + + # Modulus + 1 + "fffffffffffffffffffffffffffffffffffffffeffffee38", + + # 2^192 - 1 + "ffffffffffffffffffffffffffffffffffffffffffffffff", + + # Maximum canonical P192K1 multiplication result + ("fffffffffffffffffffffffffffffffffffffffdffffdc6c" + "0000000000000000000000000000000100002394013c7364"), + + # First 8 number generated by random.getrandbits(384) - seed(2,2) + ("cf1822ffbc6887782b491044d5e341245c6e433715ba2bdd" + "177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("ffed9235288bc781ae66267594c9c9500925e4749b575bd1" + "3653f8dd9b1f282e4067c3584ee207f8da94e3e8ab73738f"), + ("ef8acd128b4f2fc15f3f57ebf30b94fa82523e86feac7eb7" + "dc38f519b91751dacdbd47d364be8049a372db8f6e405d93"), + ("e8624fab5186ee32ee8d7ee9770348a05d300cb90706a045" + "defc044a09325626e6b58de744ab6cce80877b6f71e1f6d2"), + ("2d3d854e061b90303b08c6e33c7295782d6c797f8f7d9b78" + "2a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f"), + ("fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f1" + "5c14bc4a829e07b0829a48d422fe99a22c70501e533c9135"), + ("97eeab64ca2ce6bc5d3fd983c34c769fe89204e2e8168561" + "867e5e15bc01bfce6a27e0dfcbf8754472154e76e4c11ab2"), + ("bd143fa9b714210c665d7435c1066932f4767f26294365b2" + "721dea3bf63f23d0dbe53fcafb2147df5ca495fa5a91c89b"), + + # Next 2 number generated by random.getrandbits(192) + "47733e847d718d733ff98ff387c56473a7a83ee0761ebfd2", + "cbd4d3e2d4dec9ef83f0be4e80371eb97f81375eecc1cb63" + ] + + @property + def arg_a(self) -> str: + return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True From 133136269afb19521d9e7475134b239406cac3bb Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Thu, 13 Apr 2023 13:11:05 +0200 Subject: [PATCH 374/490] Add dependency for P192K1 tests Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index b211c98eb..636ff20a9 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -498,6 +498,7 @@ class EcpP192K1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p192k1" input_style = "fixed" arity = 1 + dependencies = ["MBEDTLS_ECP_DP_SECP119K1_ENABLED"] moduli = ["fffffffffffffffffffffffffffffffffffffffeffffee37"] # type: List[str] From 7b02aa3dc1ce090faae4168828d7ffc6035f3055 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 17 Apr 2023 14:56:03 +0200 Subject: [PATCH 375/490] Fix test case dependency Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 636ff20a9..8a711ce1f 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -498,7 +498,7 @@ class EcpP192K1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p192k1" input_style = "fixed" arity = 1 - dependencies = ["MBEDTLS_ECP_DP_SECP119K1_ENABLED"] + dependencies = ["MBEDTLS_ECP_DP_SECP192K1_ENABLED"] moduli = ["fffffffffffffffffffffffffffffffffffffffeffffee37"] # type: List[str] From 17d14fdf1cc4ebb69c9b7458503e3d5ff7bcd33a Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Tue, 18 Apr 2023 15:43:25 +0800 Subject: [PATCH 376/490] cert_audit: Improve documentation This commit is a collection of improving the documentation in the script: * Restore uppercase in the license header. * Reword the script description. * Reword the docstring of AuditData.fill_validity_duration * Rename AuditData.filename to *.location Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 537cf40f0..9ab8806d6 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 # -# copyright the mbed tls contributors -# spdx-license-identifier: apache-2.0 +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 # -# licensed under the apache license, version 2.0 (the "license"); you may -# not use this file except in compliance with the license. -# you may obtain a copy of the license at +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -18,9 +18,9 @@ """Audit validity date of X509 crt/crl/csr. This script is used to audit the validity date of crt/crl/csr used for testing. -It prints the information of X509 data whose validity duration does not cover -the provided validity duration. The data are collected from tests/data_files/ -and tests/suites/*.data files by default. +It would print the information about X.509 data if the validity period of the +X.509 data didn't cover the provided validity period. The data are collected +from tests/data_files/ and tests/suites/*.data files by default. """ import os @@ -50,15 +50,15 @@ class DataFormat(Enum): DER = 2 # Distinguished Encoding Rules class AuditData: - """Store file, type and expiration date for audit.""" + """Store data location, type and validity period of X.509 objects.""" #pylint: disable=too-few-public-methods def __init__(self, data_type: DataType, x509_obj): self.data_type = data_type - self.filename = "" + self.location = "" self.fill_validity_duration(x509_obj) def fill_validity_duration(self, x509_obj): - """Fill expiration_date field from a x509 object""" + """Read validity period from an X.509 object.""" # Certificate expires after "not_valid_after" # Certificate is invalid before "not_valid_before" if self.data_type == DataType.CRT: @@ -76,7 +76,7 @@ class AuditData: else: raise ValueError("Unsupported file_type: {}".format(self.data_type)) -class X509Parser(): +class X509Parser: """A parser class to parse crt/crl/csr file or data in PEM/DER format.""" PEM_REGEX = br'-{5}BEGIN (?P.*?)-{5}\n(?P.*?)-{5}END (?P=type)-{5}\n' PEM_TAG_REGEX = br'-{5}BEGIN (?P.*?)-{5}\n' @@ -201,7 +201,7 @@ class Auditor: result_list = [] result = self.parse_bytes(data) if result is not None: - result.filename = filename + result.location = filename result_list.append(result) return result_list @@ -347,9 +347,9 @@ class SuiteDataAuditor(Auditor): audit_data = self.parse_bytes(bytes.fromhex(match.group('data'))) if audit_data is None: continue - audit_data.filename = "{}:{}:{}".format(filename, - data_f.line_no, - idx + 1) + audit_data.location = "{}:{}:#{}".format(filename, + data_f.line_no, + idx + 1) audit_data_list.append(audit_data) return audit_data_list @@ -359,7 +359,7 @@ def list_all(audit_data: AuditData): audit_data.not_valid_before.isoformat(timespec='seconds'), audit_data.not_valid_after.isoformat(timespec='seconds'), audit_data.data_type.name, - audit_data.filename)) + audit_data.location)) def main(): """ From 6f97d92b661ae562350afd71a672fdf5a1baf733 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Tue, 18 Apr 2023 17:00:47 +0800 Subject: [PATCH 377/490] cert_audit: Code refinement This commit is a collection of code refinements from review comments. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 9ab8806d6..575da12d0 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -86,7 +86,12 @@ class X509Parser: DataType.CSR: 'CERTIFICATE REQUEST' } - def __init__(self, backends: dict): + def __init__(self, + backends: + typing.Dict[DataType, + typing.Dict[DataFormat, + typing.Callable[[bytes], object]]]) \ + -> None: self.backends = backends self.__generate_parsers() @@ -122,7 +127,7 @@ class X509Parser: return self.parsers[item] @staticmethod - def pem_data_type(data: bytes) -> str: + def pem_data_type(data: bytes) -> typing.Optional[str]: """Get the tag from the data in PEM format :param data: data to be checked in binary mode. @@ -132,7 +137,7 @@ class X509Parser: if m is not None: return m.group('type').decode('UTF-8') else: - return "" + return None @staticmethod def check_hex_string(hex_str: str) -> bool: @@ -165,6 +170,7 @@ class Auditor: def __init__(self, verbose): self.verbose = verbose self.default_files = [] + # A list to store the parsed audit_data. self.audit_data = [] self.parser = X509Parser({ DataType.CRT: { @@ -198,12 +204,12 @@ class Auditor: """ with open(filename, 'rb') as f: data = f.read() - result_list = [] result = self.parse_bytes(data) if result is not None: result.location = filename - result_list.append(result) - return result_list + return [result] + else: + return [] def parse_bytes(self, data: bytes): """Parse AuditData from bytes.""" @@ -218,11 +224,11 @@ class Auditor: return audit_data return None - def walk_all(self, file_list): + def walk_all(self, file_list: typing.Optional[typing.List[str]] = None): """ Iterate over all the files in the list and get audit data. """ - if not file_list: + if file_list is None: file_list = self.default_files for filename in file_list: data_list = self.parse_file(filename) @@ -250,11 +256,9 @@ class TestDataAuditor(Auditor): def collect_default_files(self): """Collect all files in tests/data_files/""" test_dir = self.find_test_dir() - test_data_folder = os.path.join(test_dir, 'data_files') - data_files = [] - for (dir_path, _, file_names) in os.walk(test_data_folder): - data_files.extend(os.path.join(dir_path, file_name) - for file_name in file_names) + test_data_glob = os.path.join(test_dir, 'data_files/**') + data_files = [f for f in glob.glob(test_data_glob, recursive=True) + if os.path.isfile(f)] return data_files class FileWrapper(): From 7ff6f7c6c927d9bd4fd9dfc441f59d1d84ff8440 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Tue, 11 Apr 2023 16:42:06 +0100 Subject: [PATCH 378/490] ecp_curves: Added unit-tests for `secp224k1` This patch introduces basic unit-testing for the `ecp_mod_p224k1()`. The method is exposed through the ecp_invasive interface, and the standard testing data is being provided by the python framework. Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 65 ++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 8a711ce1f..94ecdfe9e 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -552,3 +552,68 @@ class EcpP192K1Raw(bignum_common.ModOperationCommon, @property def is_valid(self) -> bool: return True + + +class EcpP224K1Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ECP P224 fast reduction.""" + symbol = "-" + test_function = "ecp_mod_p224k1" + test_name = "ecp_mod_p224k1" + input_style = "fixed" + arity = 1 + dependencies = ["MBEDTLS_ECP_DP_SECP224K1_ENABLED"] + + moduli = ["fffffffffffffffffffffffffffffffffffffffffffffffeffffe56d"] # type: List[str] + + input_values = [ + "0", "1", + + # Modulus - 1 + "fffffffffffffffffffffffffffffffffffffffffffffffeffffe56c", + + # Modulus + 1 + "fffffffffffffffffffffffffffffffffffffffffffffffeffffe56e", + + # 2^224 - 1 + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + + # Maximum canonical P224 multiplication result + ("fffffffffffffffffffffffffffffffffffffffffffffffdffffcad8" + "00000000000000000000000000000000000000010000352802c26590"), + + # First 8 number generated by random.getrandbits(448) - seed(2,2) + ("da94e3e8ab73738fcf1822ffbc6887782b491044d5e341245c6e4337" + "15ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("cdbd47d364be8049a372db8f6e405d93ffed9235288bc781ae662675" + "94c9c9500925e4749b575bd13653f8dd9b1f282e4067c3584ee207f8"), + ("defc044a09325626e6b58de744ab6cce80877b6f71e1f6d2ef8acd12" + "8b4f2fc15f3f57ebf30b94fa82523e86feac7eb7dc38f519b91751da"), + ("2d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71c4a6" + "6148a86fe8624fab5186ee32ee8d7ee9770348a05d300cb90706a045"), + ("8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0829a48d4" + "22fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578"), + ("97eeab64ca2ce6bc5d3fd983c34c769fe89204e2e8168561867e5e15" + "bc01bfce6a27e0dfcbf8754472154e76e4c11ab2fec3f6b32e8d4b8a"), + ("a7a83ee0761ebfd2bd143fa9b714210c665d7435c1066932f4767f26" + "294365b2721dea3bf63f23d0dbe53fcafb2147df5ca495fa5a91c89b"), + ("74667bffe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e" + "80371eb97f81375eecc1cb6347733e847d718d733ff98ff387c56473"), + + # Next 2 number generated by random.getrandbits(224) + ("eb9ac688b9d39cca91551e8259cc60b17604e4b4e73695c3e652c71a"), + ("f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f258ebdbfe3"), + ] + + @property + def arg_a(self) -> str: + hex_digits = bignum_common.hex_digits_for_limb(448 // self.bits_in_limb, self.bits_in_limb) + return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True From 9dfb26f2505f0759fe29ce0e03642b4f85ce8160 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Wed, 19 Apr 2023 15:03:20 +0800 Subject: [PATCH 379/490] New implementation for generate_test_code.FileWrapper We get some performance benefit from the Buffered I/O. Signed-off-by: Pengyu Lv --- scripts/generate_test_code.py | 49 ++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index f19d30b61..347100dbe 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -163,7 +163,6 @@ __MBEDTLS_TEST_TEMPLATE__PLATFORM_CODE """ -import io import os import re import sys @@ -208,43 +207,57 @@ class GeneratorInputError(Exception): pass -class FileWrapper(io.FileIO): +class FileWrapper: """ - This class extends built-in io.FileIO class with attribute line_no, + This class extends the file object with attribute line_no, that indicates line number for the line that is read. """ - def __init__(self, file_name): + def __init__(self, file_name) -> None: """ - Instantiate the base class and initialize the line number to 0. + Instantiate the file object and initialize the line number to 0. :param file_name: File path to open. """ - super().__init__(file_name, 'r') + # private mix-in file object + self._f = open(file_name, 'rb') self._line_no = 0 + def __iter__(self): + return self + def __next__(self): """ - This method overrides base class's __next__ method and extends it - method to count the line numbers as each line is read. + This method makes FileWrapper iterable. + It counts the line numbers as each line is read. :return: Line read from file. """ - line = super().__next__() - if line is not None: - self._line_no += 1 - # Convert byte array to string with correct encoding and - # strip any whitespaces added in the decoding process. - return line.decode(sys.getdefaultencoding()).rstrip() + '\n' - return None + line = self._f.__next__() + self._line_no += 1 + # Convert byte array to string with correct encoding and + # strip any whitespaces added in the decoding process. + return line.decode(sys.getdefaultencoding()).rstrip()+ '\n' - def get_line_no(self): + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._f.__exit__(exc_type, exc_val, exc_tb) + + @property + def line_no(self): """ - Gives current line number. + Property that indicates line number for the line that is read. """ return self._line_no - line_no = property(get_line_no) + @property + def name(self): + """ + Property that indicates name of the file that is read. + """ + return self._f.name def split_dep(dep): From 228d863362e69bfafbf0aeb518d1f19a2a8bdde3 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Wed, 19 Apr 2023 15:07:03 +0800 Subject: [PATCH 380/490] cert_audit: Reuse generate_test_code.FileWrapper Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 59 +-------------------------------- 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 575da12d0..89a6dd4f5 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -39,6 +39,7 @@ from cryptography import x509 #pylint: disable=import-error # reuse the function to parse *.data file in tests/suites/ from generate_test_code import parse_test_data as parse_suite_data +from generate_test_code import FileWrapper class DataType(Enum): CRT = 1 # Certificate @@ -261,64 +262,6 @@ class TestDataAuditor(Auditor): if os.path.isfile(f)] return data_files -class FileWrapper(): - """ - This a stub class of generate_test_code.FileWrapper. - - This class reads the whole file to memory before iterating - over the lines. - """ - - def __init__(self, file_name): - """ - Read the file and initialize the line number to 0. - - :param file_name: File path to open. - """ - with open(file_name, 'rb') as f: - self.buf = f.read() - self.buf_len = len(self.buf) - self._line_no = 0 - self._line_start = 0 - - def __iter__(self): - """Make the class iterable.""" - return self - - def __next__(self): - """ - This method for returning a line of the file per iteration. - - :return: Line read from file. - """ - # If we reach the end of the file. - if not self._line_start < self.buf_len: - raise StopIteration - - line_end = self.buf.find(b'\n', self._line_start) + 1 - if line_end > 0: - # Find the first LF as the end of the new line. - line = self.buf[self._line_start:line_end] - self._line_start = line_end - self._line_no += 1 - else: - # No LF found. We are at the last line without LF. - line = self.buf[self._line_start:] - self._line_start = self.buf_len - self._line_no += 1 - - # Convert byte array to string with correct encoding and - # strip any whitespaces added in the decoding process. - return line.decode(sys.getdefaultencoding()).rstrip() + '\n' - - def get_line_no(self): - """ - Gives current line number. - """ - return self._line_no - - line_no = property(get_line_no) - class SuiteDataAuditor(Auditor): """Class for auditing files in tests/suites/*.data""" def __init__(self, options): From 8f21dd93e4cd5868bc23d2ed97b9d80c6ca2a54b Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 21 Apr 2023 11:04:07 +0800 Subject: [PATCH 381/490] cert_audit: Enable logging module Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 61 +++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 89a6dd4f5..400066840 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -30,6 +30,7 @@ import typing import argparse import datetime import glob +import logging from enum import Enum # The script requires cryptography >= 35.0.0 which is only available @@ -168,8 +169,8 @@ class X509Parser: class Auditor: """A base class for audit.""" - def __init__(self, verbose): - self.verbose = verbose + def __init__(self, logger): + self.logger = logger self.default_files = [] # A list to store the parsed audit_data. self.audit_data = [] @@ -188,14 +189,6 @@ class Auditor: }, }) - def error(self, *args): - #pylint: disable=no-self-use - print("Error: ", *args, file=sys.stderr) - - def warn(self, *args): - if self.verbose: - print("Warn: ", *args, file=sys.stderr) - def parse_file(self, filename: str) -> typing.List[AuditData]: """ Parse a list of AuditData from file. @@ -219,7 +212,7 @@ class Auditor: result = self.parser[data_type](data) except ValueError as val_error: result = None - self.warn(val_error) + self.logger.warning(val_error) if result is not None: audit_data = AuditData(data_type, result) return audit_data @@ -308,6 +301,39 @@ def list_all(audit_data: AuditData): audit_data.data_type.name, audit_data.location)) + +def configure_logger(logger: logging.Logger) -> None: + """ + Configure the logging.Logger instance so that: + - Format is set to "[%(levelname)s]: %(message)s". + - loglevel >= WARNING are printed to stderr. + - loglevel < WARNING are printed to stdout. + """ + class MaxLevelFilter(logging.Filter): + # pylint: disable=too-few-public-methods + def __init__(self, max_level, name=''): + super().__init__(name) + self.max_level = max_level + + def filter(self, record: logging.LogRecord) -> bool: + return record.levelno <= self.max_level + + log_formatter = logging.Formatter("[%(levelname)s]: %(message)s") + + # set loglevel >= WARNING to be printed to stderr + stderr_hdlr = logging.StreamHandler(sys.stderr) + stderr_hdlr.setLevel(logging.WARNING) + stderr_hdlr.setFormatter(log_formatter) + + # set loglevel <= INFO to be printed to stdout + stdout_hdlr = logging.StreamHandler(sys.stdout) + stdout_hdlr.addFilter(MaxLevelFilter(logging.INFO)) + stdout_hdlr.setFormatter(log_formatter) + + logger.addHandler(stderr_hdlr) + logger.addHandler(stdout_hdlr) + + def main(): """ Perform argument parsing. @@ -319,7 +345,7 @@ def main(): help='list the information of all the files') parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', - help='show warnings') + help='show logs') parser.add_argument('--not-before', dest='not_before', help=('not valid before this date (UTC, YYYY-MM-DD). ' 'Default: today'), @@ -334,8 +360,13 @@ def main(): args = parser.parse_args() # start main routine - td_auditor = TestDataAuditor(args.verbose) - sd_auditor = SuiteDataAuditor(args.verbose) + # setup logger + logger = logging.getLogger() + configure_logger(logger) + logger.setLevel(logging.DEBUG if args.verbose else logging.ERROR) + + td_auditor = TestDataAuditor(logger) + sd_auditor = SuiteDataAuditor(logger) if args.files: data_files = args.files @@ -368,7 +399,7 @@ def main(): for d in filter(filter_func, audit_results): list_all(d) - print("\nDone!\n") + logger.debug("Done!") if __name__ == "__main__": main() From fb508f0c313e705da164fab8bcd771f4b1c4499f Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 21 Apr 2023 11:59:25 +0800 Subject: [PATCH 382/490] cert_audit: Add data-files and suite-data-files options The commit adds '--data-files' and '--suite-data-files' options so that we could pass names for the two types of files separately. Additionally, the commit improves the documentation in the script. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 400066840..d74c6f826 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -171,9 +171,9 @@ class Auditor: """A base class for audit.""" def __init__(self, logger): self.logger = logger - self.default_files = [] + self.default_files = [] # type: typing.List[str] # A list to store the parsed audit_data. - self.audit_data = [] + self.audit_data = [] # type: typing.List[AuditData] self.parser = X509Parser({ DataType.CRT: { DataFormat.PEM: x509.load_pem_x509_certificate, @@ -354,7 +354,11 @@ def main(): help=('not valid after this date (UTC, YYYY-MM-DD). ' 'Default: not-before'), metavar='DATE') - parser.add_argument('files', nargs='*', help='files to audit', + parser.add_argument('--data-files', action='append', nargs='*', + help='data files to audit', + metavar='FILE') + parser.add_argument('--suite-data-files', action='append', nargs='*', + help='suite data files to audit', metavar='FILE') args = parser.parse_args() @@ -368,22 +372,29 @@ def main(): td_auditor = TestDataAuditor(logger) sd_auditor = SuiteDataAuditor(logger) - if args.files: - data_files = args.files - suite_data_files = args.files - else: + data_files = [] + suite_data_files = [] + if args.data_files is None and args.suite_data_files is None: data_files = td_auditor.default_files suite_data_files = sd_auditor.default_files + else: + if args.data_files is not None: + data_files = [x for l in args.data_files for x in l] + if args.suite_data_files is not None: + suite_data_files = [x for l in args.suite_data_files for x in l] + # validity period start date if args.not_before: not_before_date = datetime.datetime.fromisoformat(args.not_before) else: not_before_date = datetime.datetime.today() + # validity period end date if args.not_after: not_after_date = datetime.datetime.fromisoformat(args.not_after) else: not_after_date = not_before_date + # go through all the files td_auditor.walk_all(data_files) sd_auditor.walk_all(suite_data_files) audit_results = td_auditor.audit_data + sd_auditor.audit_data @@ -396,6 +407,7 @@ def main(): if args.all: filter_func = None + # filter and output the results for d in filter(filter_func, audit_results): list_all(d) From f948226b5cb4d1d5c2407d547c8cf9021045e9cc Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 21 Apr 2023 12:41:24 +0800 Subject: [PATCH 383/490] cert_audit: Improve the method to find tests folder Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index d74c6f826..09559dc98 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -42,15 +42,20 @@ from cryptography import x509 #pylint: disable=import-error from generate_test_code import parse_test_data as parse_suite_data from generate_test_code import FileWrapper +import scripts_path # pylint: disable=unused-import +from mbedtls_dev import build_tree + class DataType(Enum): CRT = 1 # Certificate CRL = 2 # Certificate Revocation List CSR = 3 # Certificate Signing Request + class DataFormat(Enum): PEM = 1 # Privacy-Enhanced Mail DER = 2 # Distinguished Encoding Rules + class AuditData: """Store data location, type and validity period of X.509 objects.""" #pylint: disable=too-few-public-methods @@ -78,6 +83,7 @@ class AuditData: else: raise ValueError("Unsupported file_type: {}".format(self.data_type)) + class X509Parser: """A parser class to parse crt/crl/csr file or data in PEM/DER format.""" PEM_REGEX = br'-{5}BEGIN (?P.*?)-{5}\n(?P.*?)-{5}END (?P=type)-{5}\n' @@ -167,6 +173,7 @@ class X509Parser: return False return True + class Auditor: """A base class for audit.""" def __init__(self, logger): @@ -231,15 +238,8 @@ class Auditor: @staticmethod def find_test_dir(): """Get the relative path for the MbedTLS test directory.""" - if os.path.isdir('tests'): - tests_dir = 'tests' - elif os.path.isdir('suites'): - tests_dir = '.' - elif os.path.isdir('../suites'): - tests_dir = '..' - else: - raise Exception("Mbed TLS source tree not found") - return tests_dir + return os.path.relpath(build_tree.guess_mbedtls_root() + '/tests') + class TestDataAuditor(Auditor): """Class for auditing files in tests/data_files/""" @@ -255,6 +255,7 @@ class TestDataAuditor(Auditor): if os.path.isfile(f)] return data_files + class SuiteDataAuditor(Auditor): """Class for auditing files in tests/suites/*.data""" def __init__(self, options): @@ -294,6 +295,7 @@ class SuiteDataAuditor(Auditor): return audit_data_list + def list_all(audit_data: AuditData): print("{}\t{}\t{}\t{}".format( audit_data.not_valid_before.isoformat(timespec='seconds'), From 64a5fb5d24696adc5d67d77f7688bed7b5d71ff8 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Sun, 23 Apr 2023 13:56:25 +0800 Subject: [PATCH 384/490] cert_audit: Add simple parser of suite data file Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 09559dc98..ea6795904 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -38,8 +38,6 @@ from enum import Enum # using modern system on our CI. from cryptography import x509 #pylint: disable=import-error -# reuse the function to parse *.data file in tests/suites/ -from generate_test_code import parse_test_data as parse_suite_data from generate_test_code import FileWrapper import scripts_path # pylint: disable=unused-import @@ -256,6 +254,31 @@ class TestDataAuditor(Auditor): return data_files +def parse_suite_data(data_f): + """ + Parses .data file for test arguments that possiblly have a + valid X.509 data. If you need a more precise parser, please + use generate_test_code.parse_test_data instead. + + :param data_f: file object of the data file. + :return: Generator that yields test function argument list. + """ + for line in data_f: + line = line.strip() + # Skip comments + if line.startswith('#'): + continue + + # Check parameters line + match = re.search(r'\A\w+(.*:)?\"', line) + if match: + # Read test vectors + parts = re.split(r'(?[0-9a-fA-F]+)"', test_arg) if not match: From 25d1daef8d5fab2ab78b6b03a15b0c804fe72c5c Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Sun, 23 Apr 2023 14:51:18 +0800 Subject: [PATCH 385/490] cert_audit: Clarify the abstraction of Auditor Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 64 ++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index ea6795904..1517babb8 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -173,10 +173,25 @@ class X509Parser: class Auditor: - """A base class for audit.""" + """ + A base class that uses X509Parser to parse files to a list of AuditData. + + A subclass must implement the following methods: + - collect_default_files: Return a list of file names that are defaultly + used for parsing (auditing). The list will be stored in + Auditor.default_files. + - parse_file: Method that parses a single file to a list of AuditData. + + A subclass may override the following methods: + - parse_bytes: Defaultly, it parses `bytes` that contains only one valid + X.509 data(DER/PEM format) to an X.509 object. + - walk_all: Defaultly, it iterates over all the files in the provided + file name list, calls `parse_file` for each file and stores the results + by extending Auditor.audit_data. + """ def __init__(self, logger): self.logger = logger - self.default_files = [] # type: typing.List[str] + self.default_files = self.collect_default_files() # A list to store the parsed audit_data. self.audit_data = [] # type: typing.List[AuditData] self.parser = X509Parser({ @@ -194,6 +209,10 @@ class Auditor: }, }) + def collect_default_files(self) -> typing.List[str]: + """Collect the default files for parsing.""" + raise NotImplementedError + def parse_file(self, filename: str) -> typing.List[AuditData]: """ Parse a list of AuditData from file. @@ -201,14 +220,7 @@ class Auditor: :param filename: name of the file to parse. :return list of AuditData parsed from the file. """ - with open(filename, 'rb') as f: - data = f.read() - result = self.parse_bytes(data) - if result is not None: - result.location = filename - return [result] - else: - return [] + raise NotImplementedError def parse_bytes(self, data: bytes): """Parse AuditData from bytes.""" @@ -240,19 +252,32 @@ class Auditor: class TestDataAuditor(Auditor): - """Class for auditing files in tests/data_files/""" - def __init__(self, verbose): - super().__init__(verbose) - self.default_files = self.collect_default_files() + """Class for auditing files in `tests/data_files/`""" def collect_default_files(self): - """Collect all files in tests/data_files/""" + """Collect all files in `tests/data_files/`""" test_dir = self.find_test_dir() test_data_glob = os.path.join(test_dir, 'data_files/**') data_files = [f for f in glob.glob(test_data_glob, recursive=True) if os.path.isfile(f)] return data_files + def parse_file(self, filename: str) -> typing.List[AuditData]: + """ + Parse a list of AuditData from data file. + + :param filename: name of the file to parse. + :return list of AuditData parsed from the file. + """ + with open(filename, 'rb') as f: + data = f.read() + result = self.parse_bytes(data) + if result is not None: + result.location = filename + return [result] + else: + return [] + def parse_suite_data(data_f): """ @@ -280,13 +305,10 @@ def parse_suite_data(data_f): class SuiteDataAuditor(Auditor): - """Class for auditing files in tests/suites/*.data""" - def __init__(self, options): - super().__init__(options) - self.default_files = self.collect_default_files() + """Class for auditing files in `tests/suites/*.data`""" def collect_default_files(self): - """Collect all files in tests/suites/*.data""" + """Collect all files in `tests/suites/*.data`""" test_dir = self.find_test_dir() suites_data_folder = os.path.join(test_dir, 'suites') data_files = glob.glob(os.path.join(suites_data_folder, '*.data')) @@ -294,7 +316,7 @@ class SuiteDataAuditor(Auditor): def parse_file(self, filename: str): """ - Parse a list of AuditData from file. + Parse a list of AuditData from test suite data file. :param filename: name of the file to parse. :return list of AuditData parsed from the file. From 831471acfc84d7d37509a4362be42b8b35a3341f Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Tue, 25 Apr 2023 14:55:38 +0800 Subject: [PATCH 386/490] cert_audit: Check the version of cryptography The script requires cryptography >= 35.0.0, we need to check the version and provide meaningful error message when the package version was too old. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 1517babb8..594777408 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -34,15 +34,21 @@ import logging from enum import Enum # The script requires cryptography >= 35.0.0 which is only available -# for Python >= 3.6. Disable the pylint error here until we were -# using modern system on our CI. -from cryptography import x509 #pylint: disable=import-error +# for Python >= 3.6. +import cryptography +from cryptography import x509 from generate_test_code import FileWrapper import scripts_path # pylint: disable=unused-import from mbedtls_dev import build_tree +def check_cryptography_version(): + match = re.match(r'^[0-9]+', cryptography.__version__) + if match is None or int(match[0]) < 35: + raise Exception("audit-validity-dates requires cryptography >= 35.0.0" + + "({} is too old)".format(cryptography.__version__)) + class DataType(Enum): CRT = 1 # Certificate CRL = 2 # Certificate Revocation List @@ -460,5 +466,6 @@ def main(): logger.debug("Done!") +check_cryptography_version() if __name__ == "__main__": main() From 17b7c708f65c0f162cb7cc4ab9948f873f43cff4 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Tue, 25 Apr 2023 15:17:19 +0800 Subject: [PATCH 387/490] cert_audit: Reword the options and their descriptions Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 594777408..1ccfc2188 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -18,8 +18,8 @@ """Audit validity date of X509 crt/crl/csr. This script is used to audit the validity date of crt/crl/csr used for testing. -It would print the information about X.509 data if the validity period of the -X.509 data didn't cover the provided validity period. The data are collected +It prints the information about X.509 objects excluding the objects that +are valid throughout the desired validity period. The data are collected from tests/data_files/ and tests/suites/*.data files by default. """ @@ -399,13 +399,13 @@ def main(): parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', help='show logs') - parser.add_argument('--not-before', dest='not_before', - help=('not valid before this date (UTC, YYYY-MM-DD). ' + parser.add_argument('--from', dest='start_date', + help=('Start of desired validity period (UTC, YYYY-MM-DD). ' 'Default: today'), metavar='DATE') - parser.add_argument('--not-after', dest='not_after', - help=('not valid after this date (UTC, YYYY-MM-DD). ' - 'Default: not-before'), + parser.add_argument('--to', dest='end_date', + help=('End of desired validity period (UTC, YYYY-MM-DD). ' + 'Default: --from'), metavar='DATE') parser.add_argument('--data-files', action='append', nargs='*', help='data files to audit', @@ -437,15 +437,15 @@ def main(): suite_data_files = [x for l in args.suite_data_files for x in l] # validity period start date - if args.not_before: - not_before_date = datetime.datetime.fromisoformat(args.not_before) + if args.start_date: + start_date = datetime.datetime.fromisoformat(args.start_date) else: - not_before_date = datetime.datetime.today() + start_date = datetime.datetime.today() # validity period end date - if args.not_after: - not_after_date = datetime.datetime.fromisoformat(args.not_after) + if args.end_date: + end_date = datetime.datetime.fromisoformat(args.end_date) else: - not_after_date = not_before_date + end_date = start_date # go through all the files td_auditor.walk_all(data_files) @@ -454,8 +454,8 @@ def main(): # we filter out the files whose validity duration covers the provided # duration. - filter_func = lambda d: (not_before_date < d.not_valid_before) or \ - (d.not_valid_after < not_after_date) + filter_func = lambda d: (start_date < d.not_valid_before) or \ + (d.not_valid_after < end_date) if args.all: filter_func = None From f6a6fe8530f018a928112b79fa5619c7030d0d39 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Tue, 11 Apr 2023 17:25:31 +0100 Subject: [PATCH 388/490] ecp_curves: Added unit-tests for `secp256k1` This patch introduces basic unit-testing for the `ecp_mod_p256k1()`. The method is exposed through the ecp_invasive interface, and the standard testing data is being provided by the python framework. Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 65 ++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 94ecdfe9e..b7b66e418 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -617,3 +617,68 @@ class EcpP224K1Raw(bignum_common.ModOperationCommon, @property def is_valid(self) -> bool: return True + + +class EcpP256K1Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ECP P256 fast reduction.""" + symbol = "-" + test_function = "ecp_mod_p256k1" + test_name = "ecp_mod_p256k1" + input_style = "fixed" + arity = 1 + dependencies = ["MBEDTLS_ECP_DP_SECP256K1_ENABLED"] + + moduli = ["fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f"] # type: List[str] + + input_values = [ + "0", "1", + + # Modulus - 1 + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2e", + + # Modulus + 1 + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc30", + + # 2^256 - 1 + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + + # Maximum canonical P256 multiplication result + ("fffffffffffffffffffffffffffffffffffffffffffffffffffffffdfffff85c0" + "00000000000000000000000000000000000000000000001000007a4000e9844"), + + # First 8 number generated by random.getrandbits(512) - seed(2,2) + ("4067c3584ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" + "5c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("82523e86feac7eb7dc38f519b91751dacdbd47d364be8049a372db8f6e405d93" + "ffed9235288bc781ae66267594c9c9500925e4749b575bd13653f8dd9b1f282e"), + ("e8624fab5186ee32ee8d7ee9770348a05d300cb90706a045defc044a09325626" + "e6b58de744ab6cce80877b6f71e1f6d2ef8acd128b4f2fc15f3f57ebf30b94fa"), + ("829a48d422fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578" + "2d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f"), + ("e89204e2e8168561867e5e15bc01bfce6a27e0dfcbf8754472154e76e4c11ab2" + "fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0"), + ("bd143fa9b714210c665d7435c1066932f4767f26294365b2721dea3bf63f23d0" + "dbe53fcafb2147df5ca495fa5a91c89b97eeab64ca2ce6bc5d3fd983c34c769f"), + ("74667bffe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e80371eb9" + "7f81375eecc1cb6347733e847d718d733ff98ff387c56473a7a83ee0761ebfd2"), + ("d08f1bb2531d6460f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f25" + "8ebdbfe3eb9ac688b9d39cca91551e8259cc60b17604e4b4e73695c3e652c71a"), + + # Next 2 number generated by random.getrandbits(256) + ("c5e2486c44a4a8f69dc8db48e86ec9c6e06f291b2a838af8d5c44a4eb3172062"), + ("d4c0dca8b4c9e755cc9c3adcf515a8234da4daeb4f3f87777ad1f45ae9500ec9"), + ] + + @property + def arg_a(self) -> str: + hex_digits = bignum_common.hex_digits_for_limb(448 // self.bits_in_limb, self.bits_in_limb) + return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True From afe5ca739e7fa5d19d33415a2b1081657e4710c2 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Tue, 25 Apr 2023 12:13:25 +0100 Subject: [PATCH 389/490] bignum_core.py: Add "BignumCoreShiftL()" This patch introduces automatic test input generation for `mpi_core_shift_l()` function. It also adds two utility functions in bignum_common. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_common.py | 17 +++++++ scripts/framework_dev/bignum_core.py | 62 ++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index d8ef4a84f..20f7ff88b 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -80,6 +80,23 @@ def hex_digits_for_limb(limbs: int, bits_in_limb: int) -> int: """ Retrun the hex digits need for a number of limbs. """ return 2 * (limbs * bits_in_limb // 8) +def hex_digits_max_int(val: str, bits_in_limb: int) -> int: + """ Return the first number exceeding maximum the limb space + required to store the input hex-string value. This method + weights on the input str_len rather than numerical value + and works with zero-padded inputs""" + n = ((1 << (len(val) * 4)) - 1) + l = limbs_mpi(n, bits_in_limb) + return bound_mpi_limbs(l, bits_in_limb) + +def zfill_match(reference: str, target: str) -> str: + """ Zero pad target hex-string the match the limb size of + the refference input """ + lt = len(target) + lr = len(reference) + targen_len = lr if lt < lr else lt + return "{:x}".format(int(target, 16)).zfill(targen_len) + class OperationCommon(test_data_generation.BaseTest): """Common features for bignum binary operations. diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 5801caef5..2abf77ac8 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -68,6 +68,68 @@ class BignumCoreShiftR(BignumCoreTarget, test_data_generation.BaseTest): for count in counts: yield cls(input_hex, descr, count).create_test_case() + +class BignumCoreShiftL(BignumCoreTarget, bignum_common.ModOperationCommon): + """Test cases for mbedtls_bignum_core_shift_l().""" + + BIT_SHIFT_VALUES = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', + '1f', '20', '21', '3f', '40', '41', '47', '48', '4f', + '50', '51', '58', '80', '81', '88'] + DATA = ["0", "1", "40", "dee5ca1a7ef10a75", "a1055eb0bb1efa1150ff", + "002e7ab0070ad57001", "020100000000000000001011121314151617", + "1946e2958a85d8863ae21f4904fcc49478412534ed53eaf321f63f2a222" + "7a3c63acbf50b6305595f90cfa8327f6db80d986fe96080bcbb5df1bdbe" + "9b74fb8dedf2bddb3f8215b54dffd66409323bcc473e45a8fe9d08e77a51" + "1698b5dad0416305db7fcf"] + arity = 1 + test_function = "mpi_core_shift_l" + test_name = "Core shift(L)" + input_style = "arch_split" + symbol = "<<" + input_values = BIT_SHIFT_VALUES + moduli = DATA + + @property + def val_n_max_limbs(self) -> int: + """ Return the limb count required to store the maximum number that can + fit in a the number of digits used by val_n """ + m = bignum_common.hex_digits_max_int(self.val_n, self.bits_in_limb) - 1 + return bignum_common.limbs_mpi(m, self.bits_in_limb) + + def arguments(self) -> List[str]: + return [bignum_common.quote_str(self.val_n), + str(self.int_a) + ] + self.result() + + def description(self) -> str: + """ Format the output as: + #{count} {hex input} ({input bits} {limbs capacity}) << {bit shift} """ + bits = "({} bits in {} limbs)".format(self.int_n.bit_length(), self.val_n_max_limbs) + return "{} #{} {} {} {} {}".format(self.test_name, + self.count, + self.val_n, + bits, + self.symbol, + self.int_a) + + def format_result(self, res: int) -> str: + # Override to match zero-pading for leading digits between the output and input. + res_str = bignum_common.zfill_match(self.val_n, "{:x}".format(res)) + return bignum_common.quote_str(res_str) + + def result(self) -> List[str]: + result = (self.int_n << self.int_a) + # Calculate if there is space for shifting to the left(leading zero limbs) + mx = bignum_common.hex_digits_max_int(self.val_n, self.bits_in_limb) + # If there are empty limbs ahead, adjust the bitmask accordingly + result = result & (self.r - 1) if mx == self.r else result & (mx - 1) + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True + + class BignumCoreCTLookup(BignumCoreTarget, test_data_generation.BaseTest): """Test cases for mbedtls_mpi_core_ct_uint_table_lookup().""" test_function = "mpi_core_ct_uint_table_lookup" From f33ba7be0c149b81018a30d89d8dbc9e4bedb590 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 3 Dec 2022 22:58:52 +0100 Subject: [PATCH 390/490] Add line number to a few error messages This is just a quick improvement, not meant to tackle the problem as a whole. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 14 ++++----- scripts/test_generate_test_code.py | 50 +++++++++++++++--------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index f19d30b61..be08fbcfd 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -705,7 +705,7 @@ def parse_test_data(data_f): execution. :param data_f: file object of the data file. - :return: Generator that yields test name, function name, + :return: Generator that yields line number, test name, function name, dependency list and function argument list. """ __state_read_name = 0 @@ -748,7 +748,7 @@ def parse_test_data(data_f): parts = escaped_split(line, ':') test_function = parts[0] args = parts[1:] - yield name, test_function, dependencies, args + yield data_f.line_no, name, test_function, dependencies, args dependencies = [] state = __state_read_name if state == __state_read_args: @@ -931,7 +931,7 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_dependencies): unique_expressions = [] dep_check_code = '' expression_code = '' - for test_name, function_name, test_dependencies, test_args in \ + for line_no, test_name, function_name, test_dependencies, test_args in \ parse_test_data(data_f): out_data_f.write(test_name + '\n') @@ -942,16 +942,16 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_dependencies): # Write test function name test_function_name = 'test_' + function_name if test_function_name not in func_info: - raise GeneratorInputError("Function %s not found!" % - test_function_name) + raise GeneratorInputError("%d: Function %s not found!" % + (line_no, test_function_name)) func_id, func_args = func_info[test_function_name] out_data_f.write(str(func_id)) # Write parameters if len(test_args) != len(func_args): - raise GeneratorInputError("Invalid number of arguments in test " + raise GeneratorInputError("%d: Invalid number of arguments in test " "%s. See function %s signature." % - (test_name, function_name)) + (line_no, test_name, function_name)) expression_code += write_parameters(out_data_f, test_args, func_args, unique_expressions) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index d23d74219..9a78ca113 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -1264,29 +1264,29 @@ dhm_selftest: # List of (name, function_name, dependencies, args) tests = list(parse_test_data(stream)) test1, test2, test3, test4 = tests - self.assertEqual(test1[0], 'Diffie-Hellman full exchange #1') - self.assertEqual(test1[1], 'dhm_do_dhm') - self.assertEqual(test1[2], []) - self.assertEqual(test1[3], ['10', '"23"', '10', '"5"']) + self.assertEqual(test1[1], 'Diffie-Hellman full exchange #1') + self.assertEqual(test1[2], 'dhm_do_dhm') + self.assertEqual(test1[3], []) + self.assertEqual(test1[4], ['10', '"23"', '10', '"5"']) - self.assertEqual(test2[0], 'Diffie-Hellman full exchange #2') - self.assertEqual(test2[1], 'dhm_do_dhm') - self.assertEqual(test2[2], []) - self.assertEqual(test2[3], ['10', '"93450983094850938450983409623"', + self.assertEqual(test2[1], 'Diffie-Hellman full exchange #2') + self.assertEqual(test2[2], 'dhm_do_dhm') + self.assertEqual(test2[3], []) + self.assertEqual(test2[4], ['10', '"93450983094850938450983409623"', '10', '"9345098304850938450983409622"']) - self.assertEqual(test3[0], 'Diffie-Hellman full exchange #3') - self.assertEqual(test3[1], 'dhm_do_dhm') - self.assertEqual(test3[2], []) - self.assertEqual(test3[3], ['10', + self.assertEqual(test3[1], 'Diffie-Hellman full exchange #3') + self.assertEqual(test3[2], 'dhm_do_dhm') + self.assertEqual(test3[3], []) + self.assertEqual(test3[4], ['10', '"9345098382739712938719287391879381271"', '10', '"9345098792137312973297123912791271"']) - self.assertEqual(test4[0], 'Diffie-Hellman selftest') - self.assertEqual(test4[1], 'dhm_selftest') - self.assertEqual(test4[2], []) + self.assertEqual(test4[1], 'Diffie-Hellman selftest') + self.assertEqual(test4[2], 'dhm_selftest') self.assertEqual(test4[3], []) + self.assertEqual(test4[4], []) def test_with_dependencies(self): """ @@ -1306,15 +1306,15 @@ dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" # List of (name, function_name, dependencies, args) tests = list(parse_test_data(stream)) test1, test2 = tests - self.assertEqual(test1[0], 'Diffie-Hellman full exchange #1') - self.assertEqual(test1[1], 'dhm_do_dhm') - self.assertEqual(test1[2], ['YAHOO']) - self.assertEqual(test1[3], ['10', '"23"', '10', '"5"']) + self.assertEqual(test1[1], 'Diffie-Hellman full exchange #1') + self.assertEqual(test1[2], 'dhm_do_dhm') + self.assertEqual(test1[3], ['YAHOO']) + self.assertEqual(test1[4], ['10', '"23"', '10', '"5"']) - self.assertEqual(test2[0], 'Diffie-Hellman full exchange #2') - self.assertEqual(test2[1], 'dhm_do_dhm') - self.assertEqual(test2[2], []) - self.assertEqual(test2[3], ['10', '"93450983094850938450983409623"', + self.assertEqual(test2[1], 'Diffie-Hellman full exchange #2') + self.assertEqual(test2[2], 'dhm_do_dhm') + self.assertEqual(test2[3], []) + self.assertEqual(test2[4], ['10', '"93450983094850938450983409623"', '10', '"9345098304850938450983409622"']) def test_no_args(self): @@ -1335,7 +1335,7 @@ dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" stream = StringIOWrapper('test_suite_ut.function', data) err = None try: - for _, _, _, _ in parse_test_data(stream): + for _, _, _, _, _ in parse_test_data(stream): pass except GeneratorInputError as err: self.assertEqual(type(err), GeneratorInputError) @@ -1353,7 +1353,7 @@ depends_on:YAHOO stream = StringIOWrapper('test_suite_ut.function', data) err = None try: - for _, _, _, _ in parse_test_data(stream): + for _, _, _, _, _ in parse_test_data(stream): pass except GeneratorInputError as err: self.assertEqual(type(err), GeneratorInputError) From e74801e1842560e4d9a2379b68f46d53a93594f0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 4 Dec 2022 17:27:25 +0100 Subject: [PATCH 391/490] Factor get_function_info out of gen_from_test_data No intended behavior change. This commit is mainly to satisfy pylint, which complains that gen_from_test_data now has too many variables. But it's a good thing anyway to make the function a little more readable. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index be08fbcfd..9759f2ab2 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -909,6 +909,24 @@ def gen_suite_dep_checks(suite_dependencies, dep_check_code, expression_code): return dep_check_code, expression_code +def get_function_info(func_info, function_name, line_no): + """Look up information about a test function by name. + + Raise an informative expression if function_name is not found. + + :param func_info: dictionary mapping function names to their information. + :param function_name: the function name as written in the .function and + .data files. + :param line_no: line number for error messages. + :return Function information (id, args). + """ + test_function_name = 'test_' + function_name + if test_function_name not in func_info: + raise GeneratorInputError("%d: Function %s not found!" % + (line_no, test_function_name)) + return func_info[test_function_name] + + def gen_from_test_data(data_f, out_data_f, func_info, suite_dependencies): """ This function reads test case name, dependencies and test vectors @@ -940,11 +958,8 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_dependencies): unique_dependencies) # Write test function name - test_function_name = 'test_' + function_name - if test_function_name not in func_info: - raise GeneratorInputError("%d: Function %s not found!" % - (line_no, test_function_name)) - func_id, func_args = func_info[test_function_name] + func_id, func_args = \ + get_function_info(func_info, function_name, line_no) out_data_f.write(str(func_id)) # Write parameters From 2d07e7f05b3baee837e1f8afcc63fe59c6bdf75b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 4 Dec 2022 00:28:56 +0100 Subject: [PATCH 392/490] Simplify parsing of integers in .datax files In the .datax parser, since we're calling strtol() anyway, rely on it for verification. This makes the .datax parser very slightly more liberal (leading spaces and '+' are now accepted), and changes the interpretation of numbers with leading zeros to octal. Before, an argument like :0123: was parsed as decimal, but an argument like :0123+1: was parsed as a C expression and hence the leading zero marked an octal representation. Now, a leading zero is always interpreted according to C syntax, namely indicating octal. There are no nonzero integer constants with a leading zero in a .data file, so this does not affect existing test cases. In the .datax generator, allow negative arguments to be 'int' (before, they were systematically treated as 'exp' even though they didn't need to be). In the .datax parser, validate the range of integer constants. They have to fit in int32_t. In the .datax generator, use 'exp' instead of 'int' for integer constants that are out of range. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 9759f2ab2..6b5f11bb4 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -846,6 +846,14 @@ def write_dependencies(out_data_f, test_dependencies, unique_dependencies): return dep_check_code +INT_VAL_REGEX = re.compile(r'-?(\d+|0x[0-9a-f]+)$', re.I) +def val_is_int(val: str) -> bool: + """Whether val is suitable as an 'int' parameter in the .datax file.""" + if not INT_VAL_REGEX.match(val): + return False + # Limit the range to what is guaranteed to get through strtol() + return abs(int(val, 0)) <= 0x7fffffff + def write_parameters(out_data_f, test_args, func_args, unique_expressions): """ Writes test parameters to the intermediate data file, replacing @@ -864,9 +872,9 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): typ = func_args[i] val = test_args[i] - # check if val is a non literal int val (i.e. an expression) - if typ == 'int' and not re.match(r'(\d+|0x[0-9a-f]+)$', - val, re.I): + # Pass small integer constants literally. This reduces the size of + # the C code. Register anything else as an expression. + if typ == 'int' and not val_is_int(val): typ = 'exp' if val not in unique_expressions: unique_expressions.append(val) From 8f35c81f5714f67f6ef92c3ff0866b910d622013 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 4 Dec 2022 13:10:55 +0100 Subject: [PATCH 393/490] Support different types in the parameter store The test framework stores size_t and int32_t values in the parameter store by converting them all to int. This is ok in practice, since we assume int covers int32_t and we don't have test data larger than 2GB. But it's confusing and error-prone. So make the parameter store a union, which allows size_t values not to be potentially truncated and makes the code a little clearer. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 4 ++-- scripts/test_generate_test_code.py | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 6b5f11bb4..54c7e3a6f 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -471,7 +471,7 @@ def parse_function_arguments(line): continue if re.search(INT_CHECK_REGEX, arg.strip()): args.append('int') - args_dispatch.append('*( (int *) params[%d] )' % arg_idx) + args_dispatch.append('((mbedtls_test_argument_t*)params[%d])->s32' % arg_idx) elif re.search(CHAR_CHECK_REGEX, arg.strip()): args.append('char*') args_dispatch.append('(char *) params[%d]' % arg_idx) @@ -479,7 +479,7 @@ def parse_function_arguments(line): args.append('hex') # create a structure pointer_initializer = '(uint8_t *) params[%d]' % arg_idx - len_initializer = '*( (uint32_t *) params[%d] )' % (arg_idx+1) + len_initializer = '((mbedtls_test_argument_t*)params[%d])->len' % (arg_idx+1) local_vars += """ data_t data%d = {%s, %s}; """ % (arg_idx, pointer_initializer, len_initializer) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 9a78ca113..c198ad0e3 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -485,9 +485,10 @@ class ParseFuncSignature(TestCase): args, local, arg_dispatch = parse_function_arguments(line) self.assertEqual(args, ['char*', 'int', 'int']) self.assertEqual(local, '') - self.assertEqual(arg_dispatch, ['(char *) params[0]', - '*( (int *) params[1] )', - '*( (int *) params[2] )']) + self.assertEqual(arg_dispatch, + ['(char *) params[0]', + '((mbedtls_test_argument_t*)params[1])->s32', + '((mbedtls_test_argument_t*)params[2])->s32']) def test_hex_params(self): """ @@ -499,10 +500,10 @@ class ParseFuncSignature(TestCase): self.assertEqual(args, ['char*', 'hex', 'int']) self.assertEqual(local, ' data_t data1 = {(uint8_t *) params[1], ' - '*( (uint32_t *) params[2] )};\n') + '((mbedtls_test_argument_t*)params[2])->len};\n') self.assertEqual(arg_dispatch, ['(char *) params[0]', '&data1', - '*( (int *) params[3] )']) + '((mbedtls_test_argument_t*)params[3])->s32']) def test_unsupported_arg(self): """ From 4bd2ad93887edae9c2e4522b6863a0079cf82b15 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 4 Dec 2022 14:00:32 +0100 Subject: [PATCH 394/490] parse_function_arguments: make local_vars a list Internal refactoring only, no behavior change. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 54c7e3a6f..5066a57b7 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -456,7 +456,7 @@ def parse_function_arguments(line): wrapper function and argument dispatch code. """ args = [] - local_vars = '' + local_vars = [] args_dispatch = [] arg_idx = 0 # Remove characters before arguments @@ -480,9 +480,8 @@ def parse_function_arguments(line): # create a structure pointer_initializer = '(uint8_t *) params[%d]' % arg_idx len_initializer = '((mbedtls_test_argument_t*)params[%d])->len' % (arg_idx+1) - local_vars += """ data_t data%d = {%s, %s}; -""" % (arg_idx, pointer_initializer, len_initializer) - + local_vars.append(' data_t data%d = {%s, %s};\n' % + (arg_idx, pointer_initializer, len_initializer)) args_dispatch.append('&data%d' % arg_idx) arg_idx += 1 else: @@ -490,7 +489,7 @@ def parse_function_arguments(line): "'char *' or 'data_t'\n%s" % line) arg_idx += 1 - return args, local_vars, args_dispatch + return args, ''.join(local_vars), args_dispatch def generate_function_code(name, code, local_vars, args_dispatch, From 02fc566e2bb86175fc73346e682f019fabae35c3 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 4 Dec 2022 14:10:39 +0100 Subject: [PATCH 395/490] parse_function_arguments: extract per-argument function Internal refactoring only, no behavior change. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 58 +++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 5066a57b7..53efcea84 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -444,6 +444,40 @@ def parse_function_dependencies(line): return dependencies +def parse_function_argument(arg, arg_idx, args, local_vars, args_dispatch): + """ + Parses one test function's argument declaration. + + :param arg: argument declaration. + :param arg_idx: current wrapper argument index. + :param args: accumulator of arguments' internal types. + :param local_vars: accumulator of internal variable declarations. + :param args_dispatch: accumulator of argument usage expressions. + :return: the number of new wrapper arguments, + or None if the argument declaration is invalid. + """ + arg = arg.strip() + if arg == '': + return 0 + if re.search(INT_CHECK_REGEX, arg.strip()): + args.append('int') + args_dispatch.append('((mbedtls_test_argument_t*)params[%d])->s32' % arg_idx) + return 1 + if re.search(CHAR_CHECK_REGEX, arg.strip()): + args.append('char*') + args_dispatch.append('(char *) params[%d]' % arg_idx) + return 1 + if re.search(DATA_T_CHECK_REGEX, arg.strip()): + args.append('hex') + # create a structure + pointer_initializer = '(uint8_t *) params[%d]' % arg_idx + len_initializer = '((mbedtls_test_argument_t*)params[%d])->len' % (arg_idx+1) + local_vars.append(' data_t data%d = {%s, %s};\n' % + (arg_idx, pointer_initializer, len_initializer)) + args_dispatch.append('&data%d' % arg_idx) + return 2 + return None + def parse_function_arguments(line): """ Parses test function signature for validation and generates @@ -466,28 +500,12 @@ def parse_function_arguments(line): # i.e. the test functions will not have a function pointer # argument. for arg in line[:line.find(')')].split(','): - arg = arg.strip() - if arg == '': - continue - if re.search(INT_CHECK_REGEX, arg.strip()): - args.append('int') - args_dispatch.append('((mbedtls_test_argument_t*)params[%d])->s32' % arg_idx) - elif re.search(CHAR_CHECK_REGEX, arg.strip()): - args.append('char*') - args_dispatch.append('(char *) params[%d]' % arg_idx) - elif re.search(DATA_T_CHECK_REGEX, arg.strip()): - args.append('hex') - # create a structure - pointer_initializer = '(uint8_t *) params[%d]' % arg_idx - len_initializer = '((mbedtls_test_argument_t*)params[%d])->len' % (arg_idx+1) - local_vars.append(' data_t data%d = {%s, %s};\n' % - (arg_idx, pointer_initializer, len_initializer)) - args_dispatch.append('&data%d' % arg_idx) - arg_idx += 1 - else: + indexes = parse_function_argument(arg, arg_idx, + args, local_vars, args_dispatch) + if indexes is None: raise ValueError("Test function arguments can only be 'int', " "'char *' or 'data_t'\n%s" % line) - arg_idx += 1 + arg_idx += indexes return args, ''.join(local_vars), args_dispatch From 84fd79953b0300d6e1ed85106f804eda78019762 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 4 Dec 2022 14:29:06 +0100 Subject: [PATCH 396/490] Support (void) as an argument list of a test function Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 19 +++++++-------- scripts/test_generate_test_code.py | 37 ++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 53efcea84..b72e2cb00 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -457,8 +457,6 @@ def parse_function_argument(arg, arg_idx, args, local_vars, args_dispatch): or None if the argument declaration is invalid. """ arg = arg.strip() - if arg == '': - return 0 if re.search(INT_CHECK_REGEX, arg.strip()): args.append('int') args_dispatch.append('((mbedtls_test_argument_t*)params[%d])->s32' % arg_idx) @@ -478,6 +476,7 @@ def parse_function_argument(arg, arg_idx, args, local_vars, args_dispatch): return 2 return None +ARGUMENT_LIST_REGEX = re.compile(r'\((.*?)\)', re.S) def parse_function_arguments(line): """ Parses test function signature for validation and generates @@ -489,17 +488,19 @@ def parse_function_arguments(line): :return: argument list, local variables for wrapper function and argument dispatch code. """ - args = [] - local_vars = [] - args_dispatch = [] - arg_idx = 0 - # Remove characters before arguments - line = line[line.find('(') + 1:] # Process arguments, ex: arg1, arg2 ) # This script assumes that the argument list is terminated by ')' # i.e. the test functions will not have a function pointer # argument. - for arg in line[:line.find(')')].split(','): + m = ARGUMENT_LIST_REGEX.search(line) + arg_list = m.group(1).strip() + if arg_list in ['', 'void']: + return [], '', [] + args = [] + local_vars = [] + args_dispatch = [] + arg_idx = 0 + for arg in arg_list.split(','): indexes = parse_function_argument(arg, arg_idx, args, local_vars, args_dispatch) if indexes is None: diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index c198ad0e3..889b96299 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -513,9 +513,9 @@ class ParseFuncSignature(TestCase): line = 'void entropy_threshold( char * a, data_t * h, char result )' self.assertRaises(ValueError, parse_function_arguments, line) - def test_no_params(self): + def test_empty_params(self): """ - Test no parameters. + Test no parameters (nothing between parentheses). :return: """ line = 'void entropy_threshold()' @@ -524,6 +524,39 @@ class ParseFuncSignature(TestCase): self.assertEqual(local, '') self.assertEqual(arg_dispatch, []) + def test_blank_params(self): + """ + Test no parameters (space between parentheses). + :return: + """ + line = 'void entropy_threshold( )' + args, local, arg_dispatch = parse_function_arguments(line) + self.assertEqual(args, []) + self.assertEqual(local, '') + self.assertEqual(arg_dispatch, []) + + def test_void_params(self): + """ + Test no parameters (void keyword). + :return: + """ + line = 'void entropy_threshold(void)' + args, local, arg_dispatch = parse_function_arguments(line) + self.assertEqual(args, []) + self.assertEqual(local, '') + self.assertEqual(arg_dispatch, []) + + def test_void_space_params(self): + """ + Test no parameters (void with spaces). + :return: + """ + line = 'void entropy_threshold( void )' + args, local, arg_dispatch = parse_function_arguments(line) + self.assertEqual(args, []) + self.assertEqual(local, '') + self.assertEqual(arg_dispatch, []) + class ParseFunctionCode(TestCase): """ From 88d56deae68db67ed7cf8263d8ca8d4720ba0804 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 4 Dec 2022 15:11:00 +0100 Subject: [PATCH 397/490] parse_function_arguments: stricter type parsing Use normalization the equality comparisons instead of loose regular expressions to determine the type of an argument of a test function. Now declarations are parsed in a stricter way: there can't be ignored junk at the beginning or at the end. For example, `long long unsigned int x` was accepted as a test function argument (but not `long long unsigned x`), although this was misleading since the value was truncated to the range of int. Now only recognized types are accepted. The new code is slightly looser in that it accepts `char const*` as well as `const char*`. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index b72e2cb00..263cd90eb 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -171,6 +171,13 @@ import string import argparse +# Types regognized as integer arguments in test functions. +INTEGER_TYPES = frozenset(['int']) +# Types recognized as string arguments in test functions. +STRING_TYPES = frozenset(['char*', 'const char*', 'char const*']) +# Types recognized as hex data arguments in test functions. +DATA_TYPES = frozenset(['data_t*', 'const data_t*', 'data_t const*']) + BEGIN_HEADER_REGEX = r'/\*\s*BEGIN_HEADER\s*\*/' END_HEADER_REGEX = r'/\*\s*END_HEADER\s*\*/' @@ -192,9 +199,6 @@ CONDITION_REGEX = r'({})(?:\s*({})\s*({}))?$'.format(C_IDENTIFIER_REGEX, CONDITION_OPERATOR_REGEX, CONDITION_VALUE_REGEX) TEST_FUNCTION_VALIDATION_REGEX = r'\s*void\s+(?P\w+)\s*\(' -INT_CHECK_REGEX = r'int\s+.*' -CHAR_CHECK_REGEX = r'char\s*\*\s*.*' -DATA_T_CHECK_REGEX = r'data_t\s*\*\s*.*' FUNCTION_ARG_LIST_END_REGEX = r'.*\)' EXIT_LABEL_REGEX = r'^exit:' @@ -444,6 +448,7 @@ def parse_function_dependencies(line): return dependencies +ARGUMENT_DECLARATION_REGEX = re.compile(r'(.+?) ?(?:\bconst\b)? ?(\w+)\Z', re.S) def parse_function_argument(arg, arg_idx, args, local_vars, args_dispatch): """ Parses one test function's argument declaration. @@ -456,16 +461,25 @@ def parse_function_argument(arg, arg_idx, args, local_vars, args_dispatch): :return: the number of new wrapper arguments, or None if the argument declaration is invalid. """ + # Normalize whitespace arg = arg.strip() - if re.search(INT_CHECK_REGEX, arg.strip()): + arg = re.sub(r'\s*\*\s*', r'*', arg) + arg = re.sub(r'\s+', r' ', arg) + # Extract name and type + m = ARGUMENT_DECLARATION_REGEX.search(arg) + if not m: + # E.g. "int x[42]" + return None + typ, _ = m.groups() + if typ in INTEGER_TYPES: args.append('int') args_dispatch.append('((mbedtls_test_argument_t*)params[%d])->s32' % arg_idx) return 1 - if re.search(CHAR_CHECK_REGEX, arg.strip()): + if typ in STRING_TYPES: args.append('char*') args_dispatch.append('(char *) params[%d]' % arg_idx) return 1 - if re.search(DATA_T_CHECK_REGEX, arg.strip()): + if typ in DATA_TYPES: args.append('hex') # create a structure pointer_initializer = '(uint8_t *) params[%d]' % arg_idx From 0091624f37c137f9f13df992ef98d115f8b129e2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 4 Dec 2022 15:32:54 +0100 Subject: [PATCH 398/490] Support larger integer test arguments: C part Change the type of signed integer arguments from int32_t to intmax_t. This allows the C code to work with test function arguments with a range larger than int32_t. A subsequent commit will change the .datax generator to support larger types. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 2 +- scripts/test_generate_test_code.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 263cd90eb..a3f6937c7 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -473,7 +473,7 @@ def parse_function_argument(arg, arg_idx, args, local_vars, args_dispatch): typ, _ = m.groups() if typ in INTEGER_TYPES: args.append('int') - args_dispatch.append('((mbedtls_test_argument_t*)params[%d])->s32' % arg_idx) + args_dispatch.append('((mbedtls_test_argument_t*)params[%d])->sint' % arg_idx) return 1 if typ in STRING_TYPES: args.append('char*') diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 889b96299..7c0ac0c31 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -487,8 +487,8 @@ class ParseFuncSignature(TestCase): self.assertEqual(local, '') self.assertEqual(arg_dispatch, ['(char *) params[0]', - '((mbedtls_test_argument_t*)params[1])->s32', - '((mbedtls_test_argument_t*)params[2])->s32']) + '((mbedtls_test_argument_t*)params[1])->sint', + '((mbedtls_test_argument_t*)params[2])->sint']) def test_hex_params(self): """ @@ -503,7 +503,7 @@ class ParseFuncSignature(TestCase): '((mbedtls_test_argument_t*)params[2])->len};\n') self.assertEqual(arg_dispatch, ['(char *) params[0]', '&data1', - '((mbedtls_test_argument_t*)params[3])->s32']) + '((mbedtls_test_argument_t*)params[3])->sint']) def test_unsupported_arg(self): """ From 99be1733d124a0f01c03c2741cb1978e385f1a16 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 4 Dec 2022 15:57:49 +0100 Subject: [PATCH 399/490] Allow more signed integer types in test function arguments Now that the C code supports the full range of intmax_t, allow any size of signed integer type in the .data file parser. Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 21 ++++++++++++++++++--- scripts/test_generate_test_code.py | 4 ++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index a3f6937c7..f900fd2ba 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -171,8 +171,23 @@ import string import argparse -# Types regognized as integer arguments in test functions. -INTEGER_TYPES = frozenset(['int']) +# Types regognized as signed integer arguments in test functions. +SIGNED_INTEGER_TYPES = frozenset([ + 'char', + 'short', + 'short int', + 'int', + 'int8_t', + 'int16_t', + 'int32_t', + 'int64_t', + 'intmax_t', + 'long', + 'long int', + 'long long int', + 'mbedtls_mpi_sint', + 'psa_status_t', +]) # Types recognized as string arguments in test functions. STRING_TYPES = frozenset(['char*', 'const char*', 'char const*']) # Types recognized as hex data arguments in test functions. @@ -471,7 +486,7 @@ def parse_function_argument(arg, arg_idx, args, local_vars, args_dispatch): # E.g. "int x[42]" return None typ, _ = m.groups() - if typ in INTEGER_TYPES: + if typ in SIGNED_INTEGER_TYPES: args.append('int') args_dispatch.append('((mbedtls_test_argument_t*)params[%d])->sint' % arg_idx) return 1 diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 7c0ac0c31..effa46b58 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -507,10 +507,10 @@ class ParseFuncSignature(TestCase): def test_unsupported_arg(self): """ - Test unsupported arguments (not among int, char * and data_t) + Test unsupported argument type :return: """ - line = 'void entropy_threshold( char * a, data_t * h, char result )' + line = 'void entropy_threshold( char * a, data_t * h, unknown_t result )' self.assertRaises(ValueError, parse_function_arguments, line) def test_empty_params(self): From 15018cfa115319a278ddaa2c14bf698a19861478 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 23 Feb 2023 20:47:24 +0100 Subject: [PATCH 400/490] Test the line number returned by parse_test_data Signed-off-by: Gilles Peskine --- scripts/test_generate_test_code.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index effa46b58..48d3e53af 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -1298,17 +1298,20 @@ dhm_selftest: # List of (name, function_name, dependencies, args) tests = list(parse_test_data(stream)) test1, test2, test3, test4 = tests + self.assertEqual(test1[0], 3) self.assertEqual(test1[1], 'Diffie-Hellman full exchange #1') self.assertEqual(test1[2], 'dhm_do_dhm') self.assertEqual(test1[3], []) self.assertEqual(test1[4], ['10', '"23"', '10', '"5"']) + self.assertEqual(test2[0], 6) self.assertEqual(test2[1], 'Diffie-Hellman full exchange #2') self.assertEqual(test2[2], 'dhm_do_dhm') self.assertEqual(test2[3], []) self.assertEqual(test2[4], ['10', '"93450983094850938450983409623"', '10', '"9345098304850938450983409622"']) + self.assertEqual(test3[0], 9) self.assertEqual(test3[1], 'Diffie-Hellman full exchange #3') self.assertEqual(test3[2], 'dhm_do_dhm') self.assertEqual(test3[3], []) @@ -1317,6 +1320,7 @@ dhm_selftest: '10', '"9345098792137312973297123912791271"']) + self.assertEqual(test4[0], 12) self.assertEqual(test4[1], 'Diffie-Hellman selftest') self.assertEqual(test4[2], 'dhm_selftest') self.assertEqual(test4[3], []) @@ -1340,11 +1344,13 @@ dhm_do_dhm:10:"93450983094850938450983409623":10:"9345098304850938450983409622" # List of (name, function_name, dependencies, args) tests = list(parse_test_data(stream)) test1, test2 = tests + self.assertEqual(test1[0], 4) self.assertEqual(test1[1], 'Diffie-Hellman full exchange #1') self.assertEqual(test1[2], 'dhm_do_dhm') self.assertEqual(test1[3], ['YAHOO']) self.assertEqual(test1[4], ['10', '"23"', '10', '"5"']) + self.assertEqual(test2[0], 7) self.assertEqual(test2[1], 'Diffie-Hellman full exchange #2') self.assertEqual(test2[2], 'dhm_do_dhm') self.assertEqual(test2[3], []) From d8476e8723276d711f0ab000d02a8b749d43e6fe Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 26 Apr 2023 19:57:46 +0200 Subject: [PATCH 401/490] typo Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index f900fd2ba..18a29d1e6 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -171,7 +171,7 @@ import string import argparse -# Types regognized as signed integer arguments in test functions. +# Types recognized as signed integer arguments in test functions. SIGNED_INTEGER_TYPES = frozenset([ 'char', 'short', From ad5629dbb5cd02047559e698b6e089549d02dde8 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 26 Apr 2023 19:59:28 +0200 Subject: [PATCH 402/490] Adjust code style for pointer types and casts Signed-off-by: Gilles Peskine --- scripts/generate_test_code.py | 6 +++--- scripts/test_generate_test_code.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 18a29d1e6..839fccddd 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -322,7 +322,7 @@ def gen_function_wrapper(name, local_vars, args_dispatch): :param name: Test function name :param local_vars: Local variables declaration code :param args_dispatch: List of dispatch arguments. - Ex: ['(char *)params[0]', '*((int *)params[1])'] + Ex: ['(char *) params[0]', '*((int *) params[1])'] :return: Test function wrapper. """ # Then create the wrapper @@ -488,7 +488,7 @@ def parse_function_argument(arg, arg_idx, args, local_vars, args_dispatch): typ, _ = m.groups() if typ in SIGNED_INTEGER_TYPES: args.append('int') - args_dispatch.append('((mbedtls_test_argument_t*)params[%d])->sint' % arg_idx) + args_dispatch.append('((mbedtls_test_argument_t *) params[%d])->sint' % arg_idx) return 1 if typ in STRING_TYPES: args.append('char*') @@ -498,7 +498,7 @@ def parse_function_argument(arg, arg_idx, args, local_vars, args_dispatch): args.append('hex') # create a structure pointer_initializer = '(uint8_t *) params[%d]' % arg_idx - len_initializer = '((mbedtls_test_argument_t*)params[%d])->len' % (arg_idx+1) + len_initializer = '((mbedtls_test_argument_t *) params[%d])->len' % (arg_idx+1) local_vars.append(' data_t data%d = {%s, %s};\n' % (arg_idx, pointer_initializer, len_initializer)) args_dispatch.append('&data%d' % arg_idx) diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index 48d3e53af..fe748aeb4 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -487,8 +487,8 @@ class ParseFuncSignature(TestCase): self.assertEqual(local, '') self.assertEqual(arg_dispatch, ['(char *) params[0]', - '((mbedtls_test_argument_t*)params[1])->sint', - '((mbedtls_test_argument_t*)params[2])->sint']) + '((mbedtls_test_argument_t *) params[1])->sint', + '((mbedtls_test_argument_t *) params[2])->sint']) def test_hex_params(self): """ @@ -500,10 +500,10 @@ class ParseFuncSignature(TestCase): self.assertEqual(args, ['char*', 'hex', 'int']) self.assertEqual(local, ' data_t data1 = {(uint8_t *) params[1], ' - '((mbedtls_test_argument_t*)params[2])->len};\n') + '((mbedtls_test_argument_t *) params[2])->len};\n') self.assertEqual(arg_dispatch, ['(char *) params[0]', '&data1', - '((mbedtls_test_argument_t*)params[3])->sint']) + '((mbedtls_test_argument_t *) params[3])->sint']) def test_unsupported_arg(self): """ From b70de439da30bce2f0ed7eb1644a8154f161e975 Mon Sep 17 00:00:00 2001 From: Paul Elliott Date: Sun, 23 Apr 2023 23:19:21 +0100 Subject: [PATCH 403/490] Add Curve 448 tests Signed-off-by: Paul Elliott --- scripts/framework_dev/ecp.py | 90 ++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index b7b66e418..d4bfa85a3 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -682,3 +682,93 @@ class EcpP256K1Raw(bignum_common.ModOperationCommon, @property def is_valid(self) -> bool: return True + + +class EcpP448Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ECP P448 fast reduction.""" + symbol = "-" + test_function = "ecp_mod_p448" + test_name = "ecp_mod_p448" + input_style = "fixed" + arity = 1 + dependencies = ["MBEDTLS_ECP_DP_CURVE448_ENABLED"] + + moduli = [("fffffffffffffffffffffffffffffffffffffffffffffffffffffffe" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff")] # type: List[str] + + input_values = [ + "0", "1", + + # Modulus - 1 + ("fffffffffffffffffffffffffffffffffffffffffffffffffffffffe" + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffe"), + + # Modulus + 1 + ("ffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "00000000000000000000000000000000000000000000000000000000"), + + # 2^448 - 1 + ("ffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + + # Maximum canonical P448 multiplication result + ("fffffffffffffffffffffffffffffffffffffffffffffffffffffffd" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "00000000000000000000000000000000000000000000000000000002" + "00000000000000000000000000000000000000000000000000000001"), + + # First 8 number generated by random.getrandbits(896) - seed(2,2) + ("74667bffe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e" + "80371eb97f81375eecc1cb6347733e847d718d733ff98ff387c56473" + "a7a83ee0761ebfd2bd143fa9b714210c665d7435c1066932f4767f26" + "294365b2721dea3bf63f23d0dbe53fcafb2147df5ca495fa5a91c89b"), + ("4da4daeb4f3f87777ad1f45ae9500ec9c5e2486c44a4a8f69dc8db48" + "e86ec9c6e06f291b2a838af8d5c44a4eb3172062d08f1bb2531d6460" + "f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f258ebdbfe3" + "eb9ac688b9d39cca91551e8259cc60b17604e4b4e73695c3e652c71a"), + ("bc1b00d92838e766ef9b6bf2d037fe2e20b6a8464174e75a5f834da7" + "0569c018eb2b5693babb7fbb0a76c196067cfdcb11457d9cf45e2fa0" + "1d7f4275153924800600571fac3a5b263fdf57cd2c0064975c374746" + "5cc36c270e8a35b10828d569c268a20eb78ac332e5e138e26c4454b9"), + ("8d2f527e72daf0a54ef25c0707e338687d1f71575653a45c49390aa5" + "1cf5192bbf67da14be11d56ba0b4a2969d8055a9f03f2d71581d8e83" + "0112ff0f0948eccaf8877acf26c377c13f719726fd70bddacb4deeec" + "0b0c995e96e6bc4d62b47204007ee4fab105d83e85e951862f0981ae"), + ("84ae65e920a63ac1f2b64df6dff07870c9d531ae72a47403063238da" + "1a1fe3f9d6a179fa50f96cd4aff9261aa92c0e6f17ec940639bc2ccd" + "f572df00790813e32748dd1db4917fc09f20dbb0dcc93f0e66dfe717" + "c17313394391b6e2e6eacb0f0bb7be72bd6d25009aeb7fa0c4169b14"), + ("2bb3b36f29421c4021b7379f0897246a40c270b00e893302aba9e7b8" + "23fc5ad2f58105748ed5d1b7b310b730049dd332a73fa0b26b75196c" + "f87eb8a09b27ec714307c68c425424a1574f1eedf5b0f16cdfdb8394" + "24d201e653f53d6883ca1c107ca6e706649889c0c7f3860895bfa813"), + ("af3f5d7841b1256d5c1dc12fb5a1ae519fb8883accda6559caa538a0" + "9fc9370d3a6b86a7975b54a31497024640332b0612d4050771d7b14e" + "b6c004cc3b8367dc3f2bb31efe9934ad0809eae3ef232a32b5459d83" + "fbc46f1aea990e94821d46063b4dbf2ca294523d74115c86188b1044"), + ("7430051376e31f5aab63ad02854efa600641b4fa37a47ce41aeffafc" + "3b45402ac02659fe2e87d4150511baeb198ababb1a16daff3da95cd2" + "167b75dfb948f82a8317cba01c75f67e290535d868a24b7f627f2855" + "09167d4126af8090013c3273c02c6b9586b4625b475b51096c4ad652"), + + # Next 2 number generated by random.getrandbits(448) + ("8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0829a48d4" + "22fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578"), + ("97eeab64ca2ce6bc5d3fd983c34c769fe89204e2e8168561867e5e15" + "bc01bfce6a27e0dfcbf8754472154e76e4c11ab2fec3f6b32e8d4b8a"), + + ] + + @property + def arg_a(self) -> str: + hex_digits = bignum_common.hex_digits_for_limb(448 // self.bits_in_limb, self.bits_in_limb) + return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True From 379f6714dc293ca55f3966ccd772016249be146f Mon Sep 17 00:00:00 2001 From: Paul Elliott Date: Wed, 3 May 2023 14:13:42 +0100 Subject: [PATCH 404/490] Correct max canonical multiplication result Signed-off-by: Paul Elliott --- scripts/framework_dev/ecp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index d4bfa85a3..37c67a64d 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -714,9 +714,9 @@ class EcpP448Raw(bignum_common.ModOperationCommon, # Maximum canonical P448 multiplication result ("fffffffffffffffffffffffffffffffffffffffffffffffffffffffd" - "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "00000000000000000000000000000000000000000000000000000002" - "00000000000000000000000000000000000000000000000000000001"), + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffd" + "00000000000000000000000000000000000000000000000000000004" + "00000000000000000000000000000000000000000000000000000004"), # First 8 number generated by random.getrandbits(896) - seed(2,2) ("74667bffe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e" From 61052598f74069f32a47737115f068cc48ead516 Mon Sep 17 00:00:00 2001 From: Paul Elliott Date: Wed, 3 May 2023 14:14:55 +0100 Subject: [PATCH 405/490] Remove unrequired limb size calculation Signed-off-by: Paul Elliott --- scripts/framework_dev/ecp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 37c67a64d..f9f27faaa 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -762,8 +762,7 @@ class EcpP448Raw(bignum_common.ModOperationCommon, @property def arg_a(self) -> str: - hex_digits = bignum_common.hex_digits_for_limb(448 // self.bits_in_limb, self.bits_in_limb) - return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) + return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits) def result(self) -> List[str]: result = self.int_a % self.int_n From ebef055bdb081ebf6c83f4c47294fdc3a974a2f6 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Fri, 5 May 2023 16:32:28 +0200 Subject: [PATCH 406/490] Fix input parameter alignment in P256K1 test cases Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index b7b66e418..5d0de2654 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -672,8 +672,7 @@ class EcpP256K1Raw(bignum_common.ModOperationCommon, @property def arg_a(self) -> str: - hex_digits = bignum_common.hex_digits_for_limb(448 // self.bits_in_limb, self.bits_in_limb) - return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) + return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits) def result(self) -> List[str]: result = self.int_a % self.int_n From 26f023954008272fcbd5d8550c0dac27f8770f93 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 28 Apr 2023 10:46:18 +0800 Subject: [PATCH 407/490] cert_audit: Support parsing file with multiple PEMs Previously, if a file had multiple PEM objects, only the first one would be parsed. This commit add the support so that we could parse all the PEM objects in the file. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 1ccfc2188..d6e73fffb 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -90,7 +90,7 @@ class AuditData: class X509Parser: """A parser class to parse crt/crl/csr file or data in PEM/DER format.""" - PEM_REGEX = br'-{5}BEGIN (?P.*?)-{5}\n(?P.*?)-{5}END (?P=type)-{5}\n' + PEM_REGEX = br'-{5}BEGIN (?P.*?)-{5}(?P.*?)-{5}END (?P=type)-{5}' PEM_TAG_REGEX = br'-{5}BEGIN (?P.*?)-{5}\n' PEM_TAGS = { DataType.CRT: 'CERTIFICATE', @@ -277,12 +277,15 @@ class TestDataAuditor(Auditor): """ with open(filename, 'rb') as f: data = f.read() - result = self.parse_bytes(data) - if result is not None: - result.location = filename - return [result] - else: - return [] + + results = [] + for idx, m in enumerate(re.finditer(X509Parser.PEM_REGEX, data, flags=re.S), 1): + result = self.parse_bytes(data[m.start():m.end()]) + if result is not None: + result.location = "{}#{}".format(filename, idx) + results.append(result) + + return results def parse_suite_data(data_f): From 12157ba251d0a90f906d40e7f85f93a5f5aa0ccc Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 28 Apr 2023 10:58:38 +0800 Subject: [PATCH 408/490] cert_audit: Merge audit_data for identical X.509 objects Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 41 +++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index d6e73fffb..5729ee988 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -65,8 +65,13 @@ class AuditData: #pylint: disable=too-few-public-methods def __init__(self, data_type: DataType, x509_obj): self.data_type = data_type - self.location = "" + # the locations that the x509 object could be found + self.locations = [] # type: typing.List[str] self.fill_validity_duration(x509_obj) + self._obj = x509_obj + + def __eq__(self, __value) -> bool: + return self._obj == __value._obj def fill_validity_duration(self, x509_obj): """Read validity period from an X.509 object.""" @@ -282,7 +287,7 @@ class TestDataAuditor(Auditor): for idx, m in enumerate(re.finditer(X509Parser.PEM_REGEX, data, flags=re.S), 1): result = self.parse_bytes(data[m.start():m.end()]) if result is not None: - result.location = "{}#{}".format(filename, idx) + result.locations.append("{}#{}".format(filename, idx)) results.append(result) return results @@ -342,20 +347,38 @@ class SuiteDataAuditor(Auditor): audit_data = self.parse_bytes(bytes.fromhex(match.group('data'))) if audit_data is None: continue - audit_data.location = "{}:{}:#{}".format(filename, - data_f.line_no, - idx + 1) + audit_data.locations.append("{}:{}:#{}".format(filename, + data_f.line_no, + idx + 1)) audit_data_list.append(audit_data) return audit_data_list +def merge_auditdata(original: typing.List[AuditData]) \ + -> typing.List[AuditData]: + """ + Multiple AuditData might be extracted from different locations for + an identical X.509 object. Merge them into one entry in the list. + """ + results = [] + for x in original: + if x not in results: + results.append(x) + else: + idx = results.index(x) + results[idx].locations.extend(x.locations) + return results + + def list_all(audit_data: AuditData): - print("{}\t{}\t{}\t{}".format( + print("{:20}\t{:20}\t{:3}\t{}".format( audit_data.not_valid_before.isoformat(timespec='seconds'), audit_data.not_valid_after.isoformat(timespec='seconds'), audit_data.data_type.name, - audit_data.location)) + audit_data.locations[0])) + for loc in audit_data.locations[1:]: + print("{:20}\t{:20}\t{:3}\t{}".format('', '', '', loc)) def configure_logger(logger: logging.Logger) -> None: @@ -455,6 +478,10 @@ def main(): sd_auditor.walk_all(suite_data_files) audit_results = td_auditor.audit_data + sd_auditor.audit_data + audit_results = merge_auditdata(audit_results) + + logger.info("Total: {} objects found!".format(len(audit_results))) + # we filter out the files whose validity duration covers the provided # duration. filter_func = lambda d: (start_date < d.not_valid_before) or \ From 318d6964b5f663437a2a9041d2f60b7935bc84e1 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 28 Apr 2023 11:14:28 +0800 Subject: [PATCH 409/490] cert_audit: Sort the outputs by not_valid_after date Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 5729ee988..81c69d370 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -487,11 +487,13 @@ def main(): filter_func = lambda d: (start_date < d.not_valid_before) or \ (d.not_valid_after < end_date) + sortby_end = lambda d: d.not_valid_after + if args.all: filter_func = None # filter and output the results - for d in filter(filter_func, audit_results): + for d in sorted(filter(filter_func, audit_results), key=sortby_end): list_all(d) logger.debug("Done!") From fbe2939421e79e9f6fa492a852cbf91e9e8ef1ef Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 28 Apr 2023 11:17:24 +0800 Subject: [PATCH 410/490] cert_audit: Fix bug in check_cryptography_version check_cryptography_version didn't provide helpful message with Python < 3.6, because re.Match object is not subscriptable. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 81c69d370..35ea93c0d 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -45,7 +45,7 @@ from mbedtls_dev import build_tree def check_cryptography_version(): match = re.match(r'^[0-9]+', cryptography.__version__) - if match is None or int(match[0]) < 35: + if match is None or int(match.group(0)) < 35: raise Exception("audit-validity-dates requires cryptography >= 35.0.0" + "({} is too old)".format(cryptography.__version__)) From 2f52f0518bf1f50540dcd8b10e0fa4de8b7688df Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 5 May 2023 16:53:37 +0800 Subject: [PATCH 411/490] cert_audit: Calculate identifier for X.509 objects The identifier is calculated SHA1 hex string from the DER encoding of each X.509 objects. It's useful for finding out the identical X.509 objects. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 35ea93c0d..73509e154 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -31,6 +31,7 @@ import argparse import datetime import glob import logging +import hashlib from enum import Enum # The script requires cryptography >= 35.0.0 which is only available @@ -69,10 +70,20 @@ class AuditData: self.locations = [] # type: typing.List[str] self.fill_validity_duration(x509_obj) self._obj = x509_obj + encoding = cryptography.hazmat.primitives.serialization.Encoding.DER + self._identifier = hashlib.sha1(self._obj.public_bytes(encoding)).hexdigest() def __eq__(self, __value) -> bool: return self._obj == __value._obj + @property + def identifier(self): + """ + Identifier of the underlying X.509 object, which is consistent across + different runs. + """ + return self._identifier + def fill_validity_duration(self, x509_obj): """Read validity period from an X.509 object.""" # Certificate expires after "not_valid_after" From f2bb1eb5e9f972334fbbe7f14faab7070eeca146 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 5 May 2023 17:01:49 +0800 Subject: [PATCH 412/490] cert_audit: Output format improvement We should print all the information for each objects found every line. This makes it easy to analyze the output. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 73509e154..6c8a4e81f 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -383,13 +383,13 @@ def merge_auditdata(original: typing.List[AuditData]) \ def list_all(audit_data: AuditData): - print("{:20}\t{:20}\t{:3}\t{}".format( - audit_data.not_valid_before.isoformat(timespec='seconds'), - audit_data.not_valid_after.isoformat(timespec='seconds'), - audit_data.data_type.name, - audit_data.locations[0])) - for loc in audit_data.locations[1:]: - print("{:20}\t{:20}\t{:3}\t{}".format('', '', '', loc)) + for loc in audit_data.locations: + print("{}\t{:20}\t{:20}\t{:3}\t{}".format( + audit_data.identifier, + audit_data.not_valid_before.isoformat(timespec='seconds'), + audit_data.not_valid_after.isoformat(timespec='seconds'), + audit_data.data_type.name, + loc)) def configure_logger(logger: logging.Logger) -> None: From beac0bb85a93034cb2f472b7f5b0d08b60c930d8 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 5 May 2023 17:29:12 +0800 Subject: [PATCH 413/490] cert_audit: Use dictionary to store parsed AuditData Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 6c8a4e81f..127c0a0fe 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -209,13 +209,11 @@ class Auditor: X.509 data(DER/PEM format) to an X.509 object. - walk_all: Defaultly, it iterates over all the files in the provided file name list, calls `parse_file` for each file and stores the results - by extending Auditor.audit_data. + by extending the `results` passed to the function. """ def __init__(self, logger): self.logger = logger self.default_files = self.collect_default_files() - # A list to store the parsed audit_data. - self.audit_data = [] # type: typing.List[AuditData] self.parser = X509Parser({ DataType.CRT: { DataFormat.PEM: x509.load_pem_x509_certificate, @@ -257,15 +255,27 @@ class Auditor: return audit_data return None - def walk_all(self, file_list: typing.Optional[typing.List[str]] = None): + def walk_all(self, + results: typing.Dict[str, AuditData], + file_list: typing.Optional[typing.List[str]] = None) \ + -> None: """ - Iterate over all the files in the list and get audit data. + Iterate over all the files in the list and get audit data. The + results will be written to `results` passed to this function. + + :param results: The dictionary used to store the parsed + AuditData. The keys of this dictionary should + be the identifier of the AuditData. """ if file_list is None: file_list = self.default_files for filename in file_list: data_list = self.parse_file(filename) - self.audit_data.extend(data_list) + for d in data_list: + if d.identifier in results: + results[d.identifier].locations.extend(d.locations) + else: + results[d.identifier] = d @staticmethod def find_test_dir(): @@ -485,11 +495,9 @@ def main(): end_date = start_date # go through all the files - td_auditor.walk_all(data_files) - sd_auditor.walk_all(suite_data_files) - audit_results = td_auditor.audit_data + sd_auditor.audit_data - - audit_results = merge_auditdata(audit_results) + audit_results = {} + td_auditor.walk_all(audit_results, data_files) + sd_auditor.walk_all(audit_results, suite_data_files) logger.info("Total: {} objects found!".format(len(audit_results))) @@ -504,7 +512,7 @@ def main(): filter_func = None # filter and output the results - for d in sorted(filter(filter_func, audit_results), key=sortby_end): + for d in sorted(filter(filter_func, audit_results.values()), key=sortby_end): list_all(d) logger.debug("Done!") From 02178084877c729467b702468c23ddecc338f1a7 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Sat, 6 May 2023 10:06:19 +0800 Subject: [PATCH 414/490] cert_audit: Remove merge_auditdata We maintain a dict with unique AudiData objects (AuditData with unique underlying X.509 objects). We don't need merge_auditdata anymore. Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 127c0a0fe..ecde42845 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -73,9 +73,6 @@ class AuditData: encoding = cryptography.hazmat.primitives.serialization.Encoding.DER self._identifier = hashlib.sha1(self._obj.public_bytes(encoding)).hexdigest() - def __eq__(self, __value) -> bool: - return self._obj == __value._obj - @property def identifier(self): """ @@ -263,7 +260,7 @@ class Auditor: Iterate over all the files in the list and get audit data. The results will be written to `results` passed to this function. - :param results: The dictionary used to store the parsed + :param results: The dictionary used to store the parsed AuditData. The keys of this dictionary should be the identifier of the AuditData. """ @@ -376,22 +373,6 @@ class SuiteDataAuditor(Auditor): return audit_data_list -def merge_auditdata(original: typing.List[AuditData]) \ - -> typing.List[AuditData]: - """ - Multiple AuditData might be extracted from different locations for - an identical X.509 object. Merge them into one entry in the list. - """ - results = [] - for x in original: - if x not in results: - results.append(x) - else: - idx = results.index(x) - results[idx].locations.extend(x.locations) - return results - - def list_all(audit_data: AuditData): for loc in audit_data.locations: print("{}\t{:20}\t{:20}\t{:3}\t{}".format( From 876d93329168c057ea902e156490e5a1629641ff Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Mon, 8 May 2023 18:07:28 +0800 Subject: [PATCH 415/490] cert_audit: Fix DER files missed from parsing Signed-off-by: Pengyu Lv --- scripts/audit-validity-dates.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index ecde42845..5506e40e7 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -302,12 +302,22 @@ class TestDataAuditor(Auditor): data = f.read() results = [] + # Try to parse all PEM blocks. + is_pem = False for idx, m in enumerate(re.finditer(X509Parser.PEM_REGEX, data, flags=re.S), 1): + is_pem = True result = self.parse_bytes(data[m.start():m.end()]) if result is not None: result.locations.append("{}#{}".format(filename, idx)) results.append(result) + # Might be DER format. + if not is_pem: + result = self.parse_bytes(data) + if result is not None: + result.locations.append("{}".format(filename)) + results.append(result) + return results From 6a1358e030f8cd9a09a25bdb74a409e9fae81634 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Thu, 11 May 2023 10:54:44 +0100 Subject: [PATCH 416/490] bignum_common.py: Addressed minor typos Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_common.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 20f7ff88b..51b25a371 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -90,12 +90,12 @@ def hex_digits_max_int(val: str, bits_in_limb: int) -> int: return bound_mpi_limbs(l, bits_in_limb) def zfill_match(reference: str, target: str) -> str: - """ Zero pad target hex-string the match the limb size of - the refference input """ + """ Zero pad target hex-string to match the limb size of + the reference input """ lt = len(target) lr = len(reference) - targen_len = lr if lt < lr else lt - return "{:x}".format(int(target, 16)).zfill(targen_len) + target_len = lr if lt < lr else lt + return "{:x}".format(int(target, 16)).zfill(target_len) class OperationCommon(test_data_generation.BaseTest): """Common features for bignum binary operations. From 872a5fa7537c2d1bd6a21a8a941dd6e0e69dd7f6 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Mon, 8 May 2023 17:28:21 +0200 Subject: [PATCH 417/490] Add test cases to test overflow in the Kobltz reduction Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 2dae703d8..5f0efcf1c 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -518,6 +518,10 @@ class EcpP192K1Raw(bignum_common.ModOperationCommon, ("fffffffffffffffffffffffffffffffffffffffdffffdc6c" "0000000000000000000000000000000100002394013c7364"), + # Test case for overflow during addition + ("00000007ffff71b809e27dd832cfd5e04d9d2dbb9f8da217" + "0000000000000000000000000000000000000000520834f0"), + # First 8 number generated by random.getrandbits(384) - seed(2,2) ("cf1822ffbc6887782b491044d5e341245c6e433715ba2bdd" "177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), @@ -582,6 +586,10 @@ class EcpP224K1Raw(bignum_common.ModOperationCommon, ("fffffffffffffffffffffffffffffffffffffffffffffffdffffcad8" "00000000000000000000000000000000000000010000352802c26590"), + # Test case for overflow during addition + ("0000007ffff2b68161180fd8cd92e1a109be158a19a99b1809db8032" + "0000000000000000000000000000000000000000000000000bf04f49"), + # First 8 number generated by random.getrandbits(448) - seed(2,2) ("da94e3e8ab73738fcf1822ffbc6887782b491044d5e341245c6e4337" "15ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), @@ -647,6 +655,10 @@ class EcpP256K1Raw(bignum_common.ModOperationCommon, ("fffffffffffffffffffffffffffffffffffffffffffffffffffffffdfffff85c0" "00000000000000000000000000000000000000000000001000007a4000e9844"), + # Test case for overflow during addition + ("0000fffffc2f000e90a0c86a0a63234e5ba641f43a7e4aecc4040e67ec850562" + "00000000000000000000000000000000000000000000000000000000585674fd"), + # First 8 number generated by random.getrandbits(512) - seed(2,2) ("4067c3584ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" "5c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), From ef01247a362f9374f17af4eb84befcea59a38013 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Tue, 16 May 2023 15:26:06 +0100 Subject: [PATCH 418/490] bignum_core.py: Simplified result calculation for `BignumCoreShiftL` Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 2abf77ac8..ff3fd23e6 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -122,7 +122,7 @@ class BignumCoreShiftL(BignumCoreTarget, bignum_common.ModOperationCommon): # Calculate if there is space for shifting to the left(leading zero limbs) mx = bignum_common.hex_digits_max_int(self.val_n, self.bits_in_limb) # If there are empty limbs ahead, adjust the bitmask accordingly - result = result & (self.r - 1) if mx == self.r else result & (mx - 1) + result = result & (mx - 1) return [self.format_result(result)] @property From cf8826caaaad77414b71bcdb083cbd6ce32260f3 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 2 May 2023 14:05:13 +0200 Subject: [PATCH 419/490] Add `_raw` function to P192K1 Modified the testing to use the generic fast reduction test function. Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 5f0efcf1c..76a369701 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -494,8 +494,8 @@ class EcpP192K1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ECP P192K1 fast reduction.""" symbol = "-" - test_function = "ecp_mod_p192k1" - test_name = "ecp_mod_p192k1" + test_function = "ecp_mod_p_generic_raw" + test_name = "ecp_mod_p192k1_raw" input_style = "fixed" arity = 1 dependencies = ["MBEDTLS_ECP_DP_SECP192K1_ENABLED"] @@ -557,6 +557,10 @@ class EcpP192K1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True + def arguments(self): + args = super().arguments() + return ["MBEDTLS_ECP_DP_SECP192K1"] + args + class EcpP224K1Raw(bignum_common.ModOperationCommon, EcpTarget): From 9c82445d6f2847df85641c2f4e40de0fa47d59be Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 2 May 2023 14:10:57 +0200 Subject: [PATCH 420/490] Add `_raw` function to P224K1 Modified the testing to use the generic fast reduction test function. Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 76a369701..7efb32d89 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -566,8 +566,8 @@ class EcpP224K1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ECP P224 fast reduction.""" symbol = "-" - test_function = "ecp_mod_p224k1" - test_name = "ecp_mod_p224k1" + test_function = "ecp_mod_p_generic_raw" + test_name = "ecp_mod_p224k1_raw" input_style = "fixed" arity = 1 dependencies = ["MBEDTLS_ECP_DP_SECP224K1_ENABLED"] @@ -586,7 +586,7 @@ class EcpP224K1Raw(bignum_common.ModOperationCommon, # 2^224 - 1 "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - # Maximum canonical P224 multiplication result + # Maximum canonical P224K1 multiplication result ("fffffffffffffffffffffffffffffffffffffffffffffffdffffcad8" "00000000000000000000000000000000000000010000352802c26590"), @@ -630,6 +630,10 @@ class EcpP224K1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True + def arguments(self): + args = super().arguments() + return ["MBEDTLS_ECP_DP_SECP224K1"] + args + class EcpP256K1Raw(bignum_common.ModOperationCommon, EcpTarget): From 1e9020a6c48824b7e8245da70a4e5c0d514a1a30 Mon Sep 17 00:00:00 2001 From: Gabor Mezei Date: Tue, 2 May 2023 14:12:25 +0200 Subject: [PATCH 421/490] Add `_raw` function to P256K1 Modified the testing to use the generic fast reduction test function. Signed-off-by: Gabor Mezei --- scripts/framework_dev/ecp.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 7efb32d89..c9fb5e55e 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -639,8 +639,8 @@ class EcpP256K1Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ECP P256 fast reduction.""" symbol = "-" - test_function = "ecp_mod_p256k1" - test_name = "ecp_mod_p256k1" + test_function = "ecp_mod_p_generic_raw" + test_name = "ecp_mod_p256k1_raw" input_style = "fixed" arity = 1 dependencies = ["MBEDTLS_ECP_DP_SECP256K1_ENABLED"] @@ -659,9 +659,13 @@ class EcpP256K1Raw(bignum_common.ModOperationCommon, # 2^256 - 1 "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - # Maximum canonical P256 multiplication result - ("fffffffffffffffffffffffffffffffffffffffffffffffffffffffdfffff85c0" - "00000000000000000000000000000000000000000000001000007a4000e9844"), + # Maximum canonical P256K1 multiplication result + ("fffffffffffffffffffffffffffffffffffffffffffffffffffffffdfffff85c" + "000000000000000000000000000000000000000000000001000007a4000e9844"), + + # Test case for overflow during addition + ("0000fffffc2f000e90a0c86a0a63234e5ba641f43a7e4aecc4040e67ec850562" + "00000000000000000000000000000000000000000000000000000000585674fd"), # Test case for overflow during addition ("0000fffffc2f000e90a0c86a0a63234e5ba641f43a7e4aecc4040e67ec850562" @@ -702,6 +706,10 @@ class EcpP256K1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True + def arguments(self): + args = super().arguments() + return ["MBEDTLS_ECP_DP_SECP256K1"] + args + class EcpP448Raw(bignum_common.ModOperationCommon, EcpTarget): From b93e2ee95926ddefbb9192d07e310aa828752d44 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Thu, 11 May 2023 09:59:05 +0100 Subject: [PATCH 422/490] ecp.py: Added tests for `mbedtls_ecp_mod_p255_raw` This patch introduces the `EcpP255Raw` test class for testing the curve using the preestablished `ecp_mod_p_generic_raw()` test. The test's logic has been updated accordingly. Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 82 ++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index c9fb5e55e..9d882819c 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -711,6 +711,88 @@ class EcpP256K1Raw(bignum_common.ModOperationCommon, return ["MBEDTLS_ECP_DP_SECP256K1"] + args +class EcpP255Raw(bignum_common.ModOperationCommon, + EcpTarget): + """Test cases for ECP 25519 fast reduction.""" + symbol = "-" + test_function = "ecp_mod_p_generic_raw" + test_name = "mbedtls_ecp_mod_p255_raw" + input_style = "fixed" + arity = 1 + dependencies = ["MBEDTLS_ECP_DP_CURVE25519_ENABLED"] + + moduli = [("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed")] # type: List[str] + + input_values = [ + "0", "1", + + # Modulus - 1 + ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffec"), + + # Modulus + 1 + ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffee"), + + # 2^255 - 1 + ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffff"), + + # Maximum canonical P255 multiplication result + ("3fffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffec000000000000000000000000000000000000000000000000" + "0000000000000190"), + + # First 8 number generated by random.getrandbits(510) - seed(2,2) + ("1019f0d64ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5" + "e341245c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4" + "bb99f4bea973"), + ("20948fa1feac7eb7dc38f519b91751dacdbd47d364be8049a372db8f6e" + "405d93ffed9235288bc781ae66267594c9c9500925e4749b575bd13653" + "f8dd9b1f282e"), + ("3a1893ea5186ee32ee8d7ee9770348a05d300cb90706a045defc044a09" + "325626e6b58de744ab6cce80877b6f71e1f6d2ef8acd128b4f2fc15f3f" + "57ebf30b94fa"), + ("20a6923522fe99a22c70501e533c91352d3d854e061b90303b08c6e33c" + "7295782d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71" + "c4a66148a86f"), + ("3a248138e8168561867e5e15bc01bfce6a27e0dfcbf8754472154e76e4" + "c11ab2fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f15c14" + "bc4a829e07b0"), + ("2f450feab714210c665d7435c1066932f4767f26294365b2721dea3bf6" + "3f23d0dbe53fcafb2147df5ca495fa5a91c89b97eeab64ca2ce6bc5d3f" + "d983c34c769f"), + ("1d199effe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e80" + "371eb97f81375eecc1cb6347733e847d718d733ff98ff387c56473a7a8" + "3ee0761ebfd2"), + ("3423c6ec531d6460f0caeef038c89b38a8acb5137c9260dc74e088a9b9" + "492f258ebdbfe3eb9ac688b9d39cca91551e8259cc60b17604e4b4e736" + "95c3e652c71a"), + + # Next 2 number generated by random.getrandbits(255) + ("62f1243644a4a8f69dc8db48e86ec9c6e06f291b2a838af8d5c44a4eb3" + "172062"), + ("6a606e54b4c9e755cc9c3adcf515a8234da4daeb4f3f87777ad1f45ae9" + "500ec9"), + ] + + @property + def arg_a(self) -> str: + return super().format_arg('{:x}'.format(self.int_a)).zfill(2 * self.hex_digits) + + def result(self) -> List[str]: + result = self.int_a % self.int_n + return [self.format_result(result)] + + @property + def is_valid(self) -> bool: + return True + + def arguments(self): + args = super().arguments() + return ["MBEDTLS_ECP_DP_CURVE25519"] + args + + class EcpP448Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ECP P448 fast reduction.""" From 9fd6d1ed8f0320d1d4e1428630e281f8b5aecf0d Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Fri, 12 May 2023 17:10:16 +0100 Subject: [PATCH 423/490] ecp.py: Fixed types for `arguments()` overrides. Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 9d882819c..7626eda40 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -97,7 +97,7 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True - def arguments(self): + def arguments(self)-> List[str]: args = super().arguments() return ["MBEDTLS_ECP_DP_SECP192R1"] + args @@ -174,7 +174,7 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True - def arguments(self): + def arguments(self)-> List[str]: args = super().arguments() return ["MBEDTLS_ECP_DP_SECP224R1"] + args @@ -258,7 +258,7 @@ class EcpP256R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True - def arguments(self): + def arguments(self)-> List[str]: args = super().arguments() return ["MBEDTLS_ECP_DP_SECP256R1"] + args @@ -380,7 +380,7 @@ class EcpP384R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True - def arguments(self): + def arguments(self)-> List[str]: args = super().arguments() return ["MBEDTLS_ECP_DP_SECP384R1"] + args @@ -485,7 +485,7 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True - def arguments(self): + def arguments(self)-> List[str]: args = super().arguments() return ["MBEDTLS_ECP_DP_SECP521R1"] + args @@ -721,18 +721,19 @@ class EcpP255Raw(bignum_common.ModOperationCommon, arity = 1 dependencies = ["MBEDTLS_ECP_DP_CURVE25519_ENABLED"] - moduli = [("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed")] # type: List[str] + moduli = [("7fffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffffffffed")] # type: List[str] input_values = [ "0", "1", # Modulus - 1 ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "ffffffec"), + "ffffffec"), # Modulus + 1 ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "ffffffee"), + "ffffffee"), # 2^255 - 1 ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffff" @@ -788,7 +789,7 @@ class EcpP255Raw(bignum_common.ModOperationCommon, def is_valid(self) -> bool: return True - def arguments(self): + def arguments(self)-> List[str]: args = super().arguments() return ["MBEDTLS_ECP_DP_CURVE25519"] + args From 7bb80193441ebf31c20aa26ab5850d377f27dd8f Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Wed, 17 May 2023 15:01:08 +0100 Subject: [PATCH 424/490] ecp_curves: Minor refactoring of `mbedtls_ecp_mod_p255_raw()` * Fixed whitespace issues. * Renamed variables to align with bignum conventions. * Updated alignment on test input data. Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 60 ++++++++++++++---------------------- 1 file changed, 23 insertions(+), 37 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 7626eda40..02db438a7 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -728,53 +728,39 @@ class EcpP255Raw(bignum_common.ModOperationCommon, "0", "1", # Modulus - 1 - ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "ffffffec"), + ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffec"), # Modulus + 1 - ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "ffffffee"), + ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffee"), # 2^255 - 1 - ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "ffffffff"), + ("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), # Maximum canonical P255 multiplication result - ("3fffffffffffffffffffffffffffffffffffffffffffffffffffffff" - "ffffffec000000000000000000000000000000000000000000000000" - "0000000000000190"), + ("3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffec" + "0000000000000000000000000000000000000000000000000000000000000190"), # First 8 number generated by random.getrandbits(510) - seed(2,2) - ("1019f0d64ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5" - "e341245c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4" - "bb99f4bea973"), - ("20948fa1feac7eb7dc38f519b91751dacdbd47d364be8049a372db8f6e" - "405d93ffed9235288bc781ae66267594c9c9500925e4749b575bd13653" - "f8dd9b1f282e"), - ("3a1893ea5186ee32ee8d7ee9770348a05d300cb90706a045defc044a09" - "325626e6b58de744ab6cce80877b6f71e1f6d2ef8acd128b4f2fc15f3f" - "57ebf30b94fa"), - ("20a6923522fe99a22c70501e533c91352d3d854e061b90303b08c6e33c" - "7295782d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71" - "c4a66148a86f"), - ("3a248138e8168561867e5e15bc01bfce6a27e0dfcbf8754472154e76e4" - "c11ab2fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f15c14" - "bc4a829e07b0"), - ("2f450feab714210c665d7435c1066932f4767f26294365b2721dea3bf6" - "3f23d0dbe53fcafb2147df5ca495fa5a91c89b97eeab64ca2ce6bc5d3f" - "d983c34c769f"), - ("1d199effe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e80" - "371eb97f81375eecc1cb6347733e847d718d733ff98ff387c56473a7a8" - "3ee0761ebfd2"), - ("3423c6ec531d6460f0caeef038c89b38a8acb5137c9260dc74e088a9b9" - "492f258ebdbfe3eb9ac688b9d39cca91551e8259cc60b17604e4b4e736" - "95c3e652c71a"), + ("1019f0d64ee207f8da94e3e8ab73738fcf1822ffbc6887782b491044d5e34124" + "5c6e433715ba2bdd177219d30e7a269fd95bafc8f2a4d27bdcf4bb99f4bea973"), + ("20948fa1feac7eb7dc38f519b91751dacdbd47d364be8049a372db8f6e405d93" + "ffed9235288bc781ae66267594c9c9500925e4749b575bd13653f8dd9b1f282e"), + ("3a1893ea5186ee32ee8d7ee9770348a05d300cb90706a045defc044a09325626" + "e6b58de744ab6cce80877b6f71e1f6d2ef8acd128b4f2fc15f3f57ebf30b94fa"), + ("20a6923522fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578" + "2d6c797f8f7d9b782a1be9cd8697bbd0e2520e33e44c50556c71c4a66148a86f"), + ("3a248138e8168561867e5e15bc01bfce6a27e0dfcbf8754472154e76e4c11ab2" + "fec3f6b32e8d4b8a8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0"), + ("2f450feab714210c665d7435c1066932f4767f26294365b2721dea3bf63f23d0" + "dbe53fcafb2147df5ca495fa5a91c89b97eeab64ca2ce6bc5d3fd983c34c769f"), + ("1d199effe202849da9643a295a9ac6decbd4d3e2d4dec9ef83f0be4e80371eb9" + "7f81375eecc1cb6347733e847d718d733ff98ff387c56473a7a83ee0761ebfd2"), + ("3423c6ec531d6460f0caeef038c89b38a8acb5137c9260dc74e088a9b9492f25" + "8ebdbfe3eb9ac688b9d39cca91551e8259cc60b17604e4b4e73695c3e652c71a"), # Next 2 number generated by random.getrandbits(255) - ("62f1243644a4a8f69dc8db48e86ec9c6e06f291b2a838af8d5c44a4eb3" - "172062"), - ("6a606e54b4c9e755cc9c3adcf515a8234da4daeb4f3f87777ad1f45ae9" - "500ec9"), + ("62f1243644a4a8f69dc8db48e86ec9c6e06f291b2a838af8d5c44a4eb3172062"), + ("6a606e54b4c9e755cc9c3adcf515a8234da4daeb4f3f87777ad1f45ae9500ec9"), ] @property From ba37450c4c2a61c72818b4a667c60ea434f31610 Mon Sep 17 00:00:00 2001 From: Paul Elliott Date: Tue, 13 Jun 2023 17:42:01 +0100 Subject: [PATCH 425/490] Move corner test case into python framework Signed-off-by: Paul Elliott --- scripts/framework_dev/ecp.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index c9fb5e55e..722e244fe 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -779,6 +779,12 @@ class EcpP448Raw(bignum_common.ModOperationCommon, "167b75dfb948f82a8317cba01c75f67e290535d868a24b7f627f2855" "09167d4126af8090013c3273c02c6b9586b4625b475b51096c4ad652"), + # Corner case which causes maximum overflow + ("f4ae65e920a63ac1f2b64df6dff07870c9d531ae72a47403063238da1" + "a1fe3f9d6a179fa50f96cd4aff9261aa92c0e6f17ec940639bc2ccd0B" + "519A16DF59C53E0D49B209200F878F362ACE518D5B8BFCF9CDC725E5E" + "01C06295E8605AF06932B5006D9E556D3F190E8136BF9C643D332"), + # Next 2 number generated by random.getrandbits(448) ("8f54f8ceacaab39e83844b40ffa9b9f15c14bc4a829e07b0829a48d4" "22fe99a22c70501e533c91352d3d854e061b90303b08c6e33c729578"), From de1a0284d09e473f0f3f1832f6f60478670539e9 Mon Sep 17 00:00:00 2001 From: Paul Elliott Date: Fri, 2 Jun 2023 16:00:05 +0100 Subject: [PATCH 426/490] Split out mbedtls_ecp_mod_p448_raw() Switch testing over to using the generic raw functions. Signed-off-by: Paul Elliott --- scripts/framework_dev/ecp.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index bed4d56ac..e5dd4d9bd 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -784,8 +784,8 @@ class EcpP448Raw(bignum_common.ModOperationCommon, EcpTarget): """Test cases for ECP P448 fast reduction.""" symbol = "-" - test_function = "ecp_mod_p448" - test_name = "ecp_mod_p448" + test_function = "ecp_mod_p_generic_raw" + test_name = "ecp_mod_p448_raw" input_style = "fixed" arity = 1 dependencies = ["MBEDTLS_ECP_DP_CURVE448_ENABLED"] @@ -873,3 +873,7 @@ class EcpP448Raw(bignum_common.ModOperationCommon, @property def is_valid(self) -> bool: return True + + def arguments(self): + args = super().arguments() + return ["MBEDTLS_ECP_DP_CURVE448"] + args From e46cde917b38a83dec40be0a32f8f0b5f18f7b58 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Fri, 9 Jun 2023 14:23:55 +0100 Subject: [PATCH 427/490] test_suite_ecp: Added `MBEDTLS_ECP_NIST_OPTIM` define guards. This patch updates `ecp_mod_p_generic_raw` and corresponding curve test methods, that depend on the NIST optimisation parameter to not run when it is not included. The following curves are affected: * SECP192R1 * SECP224R1 * SECP256R1 * SECP384R1 * SECP521R1 Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index e5dd4d9bd..8a3ab281f 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -34,7 +34,8 @@ class EcpP192R1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p192_raw" input_style = "fixed" arity = 1 - dependencies = ["MBEDTLS_ECP_DP_SECP192R1_ENABLED"] + dependencies = ["MBEDTLS_ECP_DP_SECP192R1_ENABLED", + "MBEDTLS_ECP_NIST_OPTIM"] moduli = ["fffffffffffffffffffffffffffffffeffffffffffffffff"] # type: List[str] @@ -110,7 +111,8 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p224_raw" input_style = "arch_split" arity = 1 - dependencies = ["MBEDTLS_ECP_DP_SECP224R1_ENABLED"] + dependencies = ["MBEDTLS_ECP_DP_SECP224R1_ENABLED", + "MBEDTLS_ECP_NIST_OPTIM"] moduli = ["ffffffffffffffffffffffffffffffff000000000000000000000001"] # type: List[str] @@ -187,7 +189,8 @@ class EcpP256R1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p256_raw" input_style = "fixed" arity = 1 - dependencies = ["MBEDTLS_ECP_DP_SECP256R1_ENABLED"] + dependencies = ["MBEDTLS_ECP_DP_SECP256R1_ENABLED", + "MBEDTLS_ECP_NIST_OPTIM"] moduli = ["ffffffff00000001000000000000000000000000ffffffffffffffffffffffff"] # type: List[str] @@ -270,7 +273,8 @@ class EcpP384R1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p384_raw" input_style = "fixed" arity = 1 - dependencies = ["MBEDTLS_ECP_DP_SECP384R1_ENABLED"] + dependencies = ["MBEDTLS_ECP_DP_SECP384R1_ENABLED", + "MBEDTLS_ECP_NIST_OPTIM"] moduli = [("ffffffffffffffffffffffffffffffffffffffffffffffff" "fffffffffffffffeffffffff0000000000000000ffffffff") @@ -392,7 +396,8 @@ class EcpP521R1Raw(bignum_common.ModOperationCommon, test_name = "ecp_mod_p521_raw" input_style = "arch_split" arity = 1 - dependencies = ["MBEDTLS_ECP_DP_SECP521R1_ENABLED"] + dependencies = ["MBEDTLS_ECP_DP_SECP521R1_ENABLED", + "MBEDTLS_ECP_NIST_OPTIM"] moduli = [("01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") From 61af299bd981431bac07c20f7c9e98288c9d3025 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 22 Jun 2023 19:45:01 +0200 Subject: [PATCH 428/490] Clean up subprocess invocation in get_src_files Signed-off-by: Gilles Peskine --- scripts/code_style.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index c31fb2949..cf50c8d42 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -65,29 +65,27 @@ def list_generated_files() -> FrozenSet[str]: def get_src_files() -> List[str]: """ - Use git ls-files to get a list of the source files + Use git to get a list of the source files. + + Only C files are included, and certain files (generated, or 3rdparty) + are excluded. """ git_ls_files_cmd = ["git", "ls-files", "*.[hc]", "tests/suites/*.function", "scripts/data_files/*.fmt"] + output = subprocess.check_output(git_ls_files_cmd, + universal_newlines=True) + src_files = output.split() - result = subprocess.run(git_ls_files_cmd, stdout=subprocess.PIPE, - check=False) - - if result.returncode != 0: - print_err("git ls-files returned: " + str(result.returncode)) - return [] - else: - generated_files = list_generated_files() - src_files = str(result.stdout, "utf-8").split() - # Don't correct style for third-party files (and, for simplicity, - # companion files in the same subtree), or for automatically - # generated files (we're correcting the templates instead). - src_files = [filename for filename in src_files - if not (filename.startswith("3rdparty/") or - filename in generated_files)] - return src_files + generated_files = list_generated_files() + # Don't correct style for third-party files (and, for simplicity, + # companion files in the same subtree), or for automatically + # generated files (we're correcting the templates instead). + src_files = [filename for filename in src_files + if not (filename.startswith("3rdparty/") or + filename in generated_files)] + return src_files def get_uncrustify_version() -> str: """ From 35de652fb8f6c716327b49c44d1646d11006050b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 22 Jun 2023 20:29:41 +0200 Subject: [PATCH 429/490] Add --since option to check files modified since a given commit Signed-off-by: Gilles Peskine --- scripts/code_style.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index cf50c8d42..4cb58babb 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -22,7 +22,7 @@ import os import re import subprocess import sys -from typing import FrozenSet, List +from typing import FrozenSet, List, Optional UNCRUSTIFY_SUPPORTED_VERSION = "0.75.1" CONFIG_FILE = ".uncrustify.cfg" @@ -63,19 +63,31 @@ def list_generated_files() -> FrozenSet[str]: checks = re.findall(CHECK_CALL_RE, content) return frozenset(word for s in checks for word in s.split()) -def get_src_files() -> List[str]: +def get_src_files(since: Optional[str]) -> List[str]: """ Use git to get a list of the source files. + The optional argument since is a commit, indicating to only list files + that have changed since that commit. Without this argument, list all + files known to git. + Only C files are included, and certain files (generated, or 3rdparty) are excluded. """ - git_ls_files_cmd = ["git", "ls-files", - "*.[hc]", - "tests/suites/*.function", - "scripts/data_files/*.fmt"] - output = subprocess.check_output(git_ls_files_cmd, - universal_newlines=True) + if since is None: + git_ls_files_cmd = ["git", "ls-files", + "*.[hc]", + "tests/suites/*.function", + "scripts/data_files/*.fmt"] + output = subprocess.check_output(git_ls_files_cmd, + universal_newlines=True) + else: + git_ls_files_cmd = ["git", "diff", "--name-only", since, "--", + "*.[hc]", + "tests/suites/*.function", + "scripts/data_files/*.fmt"] + output = subprocess.check_output(git_ls_files_cmd, + universal_newlines=True) src_files = output.split() generated_files = list_generated_files() @@ -180,6 +192,9 @@ def main() -> int: parser.add_argument('-f', '--fix', action='store_true', help=('modify source files to fix the code style ' '(default: print diff, do not modify files)')) + parser.add_argument('-s', '--since', metavar='COMMIT', + help=('only check files modified since the specified commit' + ' (e.g. --since=HEAD~3 or --since=development)')) # --subset is almost useless: it only matters if there are no files # ('code_style.py' without arguments checks all files known to Git, # 'code_style.py --subset' does nothing). In particular, @@ -192,7 +207,7 @@ def main() -> int: args = parser.parse_args() - covered = frozenset(get_src_files()) + covered = frozenset(get_src_files(args.since)) # We only check files that are known to git if args.subset or args.operands: src_files = [f for f in args.operands if f in covered] From 31299ca50cc014d9a266b49088d4dfa7ac1049d9 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 25 Jun 2023 22:18:40 +0200 Subject: [PATCH 430/490] Handle deleted files correctly Don't attempt to run on a file that isn't present now. Signed-off-by: Gilles Peskine --- scripts/code_style.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 4cb58babb..7de93b085 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -74,21 +74,18 @@ def get_src_files(since: Optional[str]) -> List[str]: Only C files are included, and certain files (generated, or 3rdparty) are excluded. """ - if since is None: - git_ls_files_cmd = ["git", "ls-files", - "*.[hc]", - "tests/suites/*.function", - "scripts/data_files/*.fmt"] - output = subprocess.check_output(git_ls_files_cmd, - universal_newlines=True) - else: - git_ls_files_cmd = ["git", "diff", "--name-only", since, "--", - "*.[hc]", - "tests/suites/*.function", - "scripts/data_files/*.fmt"] - output = subprocess.check_output(git_ls_files_cmd, - universal_newlines=True) + file_patterns = ["*.[hc]", + "tests/suites/*.function", + "scripts/data_files/*.fmt"] + output = subprocess.check_output(["git", "ls-files"] + file_patterns, + universal_newlines=True) src_files = output.split() + if since: + output = subprocess.check_output(["git", "diff", "--name-only", + since, "--"] + + src_files, + universal_newlines=True) + src_files = output.split() generated_files = list_generated_files() # Don't correct style for third-party files (and, for simplicity, From c69a549e81dc804988855512461dee177a3c8b08 Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Tue, 27 Jun 2023 16:34:59 +0100 Subject: [PATCH 431/490] bignum_common.py: Added `bits_to_limbs` method. This patch introduces a rounding-error-resiliant method to calculate bits_to_limbs, and is updating `SECP224R1` and `SECP224K1` to use it. Signed-off-by: Minos Galanakis --- scripts/framework_dev/bignum_common.py | 10 ++++++++-- scripts/framework_dev/ecp.py | 6 ++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 51b25a371..3bef16db6 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -19,6 +19,7 @@ import enum from typing import Iterator, List, Tuple, TypeVar, Any from copy import deepcopy from itertools import chain +from math import ceil from . import test_case from . import test_data_generation @@ -76,9 +77,14 @@ def combination_pairs(values: List[T]) -> List[Tuple[T, T]]: """Return all pair combinations from input values.""" return [(x, y) for x in values for y in values] +def bits_to_limbs(bits: int, bits_in_limb: int) -> int: + """ Return the appropriate ammount of limbs needed to store + a number contained in input bits""" + return ceil(bits / bits_in_limb) + def hex_digits_for_limb(limbs: int, bits_in_limb: int) -> int: - """ Retrun the hex digits need for a number of limbs. """ - return 2 * (limbs * bits_in_limb // 8) + """ Return the hex digits need for a number of limbs. """ + return 2 * ((limbs * bits_in_limb) // 8) def hex_digits_max_int(val: str, bits_in_limb: int) -> int: """ Return the first number exceeding maximum the limb space diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 8a3ab281f..ed79a073c 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -165,7 +165,8 @@ class EcpP224R1Raw(bignum_common.ModOperationCommon, @property def arg_a(self) -> str: - hex_digits = bignum_common.hex_digits_for_limb(448 // self.bits_in_limb, self.bits_in_limb) + limbs = 2 * bignum_common.bits_to_limbs(224, self.bits_in_limb) + hex_digits = bignum_common.hex_digits_for_limb(limbs, self.bits_in_limb) return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) def result(self) -> List[str]: @@ -624,7 +625,8 @@ class EcpP224K1Raw(bignum_common.ModOperationCommon, @property def arg_a(self) -> str: - hex_digits = bignum_common.hex_digits_for_limb(448 // self.bits_in_limb, self.bits_in_limb) + limbs = 2 * bignum_common.bits_to_limbs(224, self.bits_in_limb) + hex_digits = bignum_common.hex_digits_for_limb(limbs, self.bits_in_limb) return super().format_arg('{:x}'.format(self.int_a)).zfill(hex_digits) def result(self) -> List[str]: From 7fe083e070e37666e97087560e9a728bbcb4f57c Mon Sep 17 00:00:00 2001 From: Minos Galanakis Date: Tue, 27 Jun 2023 18:54:53 +0100 Subject: [PATCH 432/490] ecp.py: Extended EcpP224K1Raw tests for 32/64 bit architectures. Signed-off-by: Minos Galanakis --- scripts/framework_dev/ecp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index ed79a073c..410c77e11 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -574,7 +574,7 @@ class EcpP224K1Raw(bignum_common.ModOperationCommon, symbol = "-" test_function = "ecp_mod_p_generic_raw" test_name = "ecp_mod_p224k1_raw" - input_style = "fixed" + input_style = "arch_split" arity = 1 dependencies = ["MBEDTLS_ECP_DP_SECP224K1_ENABLED"] From f877ce6e967f8b210109e2e7294f186d97fce431 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Wed, 19 Jul 2023 12:09:45 +0800 Subject: [PATCH 433/490] code_size_compare: add logging module and tweak prompt message Signed-off-by: Yanray Wang --- scripts/audit-validity-dates.py | 36 +----------------- scripts/framework_dev/logging_util.py | 55 +++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 34 deletions(-) create mode 100644 scripts/framework_dev/logging_util.py diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 5506e40e7..623fd2352 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -24,7 +24,6 @@ from tests/data_files/ and tests/suites/*.data files by default. """ import os -import sys import re import typing import argparse @@ -43,6 +42,7 @@ from generate_test_code import FileWrapper import scripts_path # pylint: disable=unused-import from mbedtls_dev import build_tree +from mbedtls_dev import logging_util def check_cryptography_version(): match = re.match(r'^[0-9]+', cryptography.__version__) @@ -393,38 +393,6 @@ def list_all(audit_data: AuditData): loc)) -def configure_logger(logger: logging.Logger) -> None: - """ - Configure the logging.Logger instance so that: - - Format is set to "[%(levelname)s]: %(message)s". - - loglevel >= WARNING are printed to stderr. - - loglevel < WARNING are printed to stdout. - """ - class MaxLevelFilter(logging.Filter): - # pylint: disable=too-few-public-methods - def __init__(self, max_level, name=''): - super().__init__(name) - self.max_level = max_level - - def filter(self, record: logging.LogRecord) -> bool: - return record.levelno <= self.max_level - - log_formatter = logging.Formatter("[%(levelname)s]: %(message)s") - - # set loglevel >= WARNING to be printed to stderr - stderr_hdlr = logging.StreamHandler(sys.stderr) - stderr_hdlr.setLevel(logging.WARNING) - stderr_hdlr.setFormatter(log_formatter) - - # set loglevel <= INFO to be printed to stdout - stdout_hdlr = logging.StreamHandler(sys.stdout) - stdout_hdlr.addFilter(MaxLevelFilter(logging.INFO)) - stdout_hdlr.setFormatter(log_formatter) - - logger.addHandler(stderr_hdlr) - logger.addHandler(stdout_hdlr) - - def main(): """ Perform argument parsing. @@ -457,7 +425,7 @@ def main(): # start main routine # setup logger logger = logging.getLogger() - configure_logger(logger) + logging_util.configure_logger(logger) logger.setLevel(logging.DEBUG if args.verbose else logging.ERROR) td_auditor = TestDataAuditor(logger) diff --git a/scripts/framework_dev/logging_util.py b/scripts/framework_dev/logging_util.py new file mode 100644 index 000000000..962361a49 --- /dev/null +++ b/scripts/framework_dev/logging_util.py @@ -0,0 +1,55 @@ +"""Auxiliary functions used for logging module. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys + +def configure_logger( + logger: logging.Logger, + logger_format="[%(levelname)s]: %(message)s" + ) -> None: + """ + Configure the logging.Logger instance so that: + - Format is set to any logger_format. + Default: "[%(levelname)s]: %(message)s" + - loglevel >= WARNING are printed to stderr. + - loglevel < WARNING are printed to stdout. + """ + class MaxLevelFilter(logging.Filter): + # pylint: disable=too-few-public-methods + def __init__(self, max_level, name=''): + super().__init__(name) + self.max_level = max_level + + def filter(self, record: logging.LogRecord) -> bool: + return record.levelno <= self.max_level + + log_formatter = logging.Formatter(logger_format) + + # set loglevel >= WARNING to be printed to stderr + stderr_hdlr = logging.StreamHandler(sys.stderr) + stderr_hdlr.setLevel(logging.WARNING) + stderr_hdlr.setFormatter(log_formatter) + + # set loglevel <= INFO to be printed to stdout + stdout_hdlr = logging.StreamHandler(sys.stdout) + stdout_hdlr.addFilter(MaxLevelFilter(logging.INFO)) + stdout_hdlr.setFormatter(log_formatter) + + logger.addHandler(stderr_hdlr) + logger.addHandler(stdout_hdlr) From bd4f3cc8f37d421b314873031e612b3437468597 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Wed, 26 Jul 2023 14:48:08 +0800 Subject: [PATCH 434/490] logging_util: rename argument Signed-off-by: Yanray Wang --- scripts/framework_dev/logging_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/logging_util.py b/scripts/framework_dev/logging_util.py index 962361a49..85a3f19ac 100644 --- a/scripts/framework_dev/logging_util.py +++ b/scripts/framework_dev/logging_util.py @@ -21,11 +21,11 @@ import sys def configure_logger( logger: logging.Logger, - logger_format="[%(levelname)s]: %(message)s" + log_format="[%(levelname)s]: %(message)s" ) -> None: """ Configure the logging.Logger instance so that: - - Format is set to any logger_format. + - Format is set to any log_format. Default: "[%(levelname)s]: %(message)s" - loglevel >= WARNING are printed to stderr. - loglevel < WARNING are printed to stdout. @@ -39,7 +39,7 @@ def configure_logger( def filter(self, record: logging.LogRecord) -> bool: return record.levelno <= self.max_level - log_formatter = logging.Formatter(logger_format) + log_formatter = logging.Formatter(log_format) # set loglevel >= WARNING to be printed to stderr stderr_hdlr = logging.StreamHandler(sys.stderr) From 537fd3e6512aaa438e37e390c5169996ac7a5bb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Tue, 11 Jul 2023 10:23:02 +0200 Subject: [PATCH 435/490] Enable DH in generate_psa_tests.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- scripts/framework_dev/crypto_knowledge.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 819d92afb..eab6f5660 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -138,6 +138,9 @@ class KeyType: """Whether the key type is for public keys.""" return self.name.endswith('_PUBLIC_KEY') + DH_KEY_SIZES = { + 'PSA_DH_FAMILY_RFC7919': (2048, 3072, 4096, 6144, 8192), + } # type: Dict[str, Tuple[int, ...]] ECC_KEY_SIZES = { 'PSA_ECC_FAMILY_SECP_K1': (192, 224, 256), 'PSA_ECC_FAMILY_SECP_R1': (225, 256, 384, 521), @@ -175,6 +178,9 @@ class KeyType: if self.private_type == 'PSA_KEY_TYPE_ECC_KEY_PAIR': assert self.params is not None return self.ECC_KEY_SIZES[self.params[0]] + if self.private_type == 'PSA_KEY_TYPE_DH_KEY_PAIR': + assert self.params is not None + return self.DH_KEY_SIZES[self.params[0]] return self.KEY_TYPE_SIZES[self.private_type] # "48657265006973206b6579a064617461" From aa454f6cc074126f43e42864c924f071f7b7500d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Tue, 18 Jul 2023 11:00:36 +0200 Subject: [PATCH 436/490] Shorten DH_FAMILY just like ECC_FAMILY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- scripts/framework_dev/crypto_knowledge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index eab6f5660..3230a005d 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -34,7 +34,7 @@ def short_expression(original: str, level: int = 0) -> str: unambiguous, but ad hoc way. """ short = original - short = re.sub(r'\bPSA_(?:ALG|ECC_FAMILY|KEY_[A-Z]+)_', r'', short) + short = re.sub(r'\bPSA_(?:ALG|DH_FAMILY|ECC_FAMILY|KEY_[A-Z]+)_', r'', short) short = re.sub(r' +', r'', short) if level >= 1: short = re.sub(r'PUBLIC_KEY\b', r'PUB', short) From b8f8019a935ffe3db75f53b3d1d19e92dd5901e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Tue, 18 Jul 2023 17:58:09 +0200 Subject: [PATCH 437/490] Fix KeyType.can_do() for DH+FFDH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Pégourié-Gonnard --- scripts/framework_dev/crypto_knowledge.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 3230a005d..45d253b9b 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -267,6 +267,8 @@ class KeyType: if alg.head in {'PURE_EDDSA', 'EDDSA_PREHASH'} and \ eccc == EllipticCurveCategory.TWISTED_EDWARDS: return True + if self.head == 'DH' and alg.head == 'FFDH': + return True return False From 56bab4e6ea29a7f1a4772a8ef10b3c9978bb1702 Mon Sep 17 00:00:00 2001 From: Gowtham Suresh Kumar Date: Wed, 26 Jul 2023 15:47:45 +0100 Subject: [PATCH 438/490] Fix warnings from clang-16 Running clang-16 on mbedtls reports warnings of type "-Wstrict-prototypes". This patch fixes these warnings by adding void to functions with no arguments. The generate_test_code.py is modified to insert void into test functions with no arguments in *.function files. Signed-off-by: Gowtham Suresh Kumar --- scripts/generate_test_code.py | 5 +++++ scripts/test_generate_test_code.py | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index ff7f9b997..c42f9a84b 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -667,6 +667,11 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): code = code.replace(name, 'test_' + name, 1) name = 'test_' + name + # If a test function has no arguments then add 'void' argument to + # avoid "-Wstrict-prototypes" warnings from clang-16 + if len(args) == 0: + code = code.replace('()', '(void)', 1) + for line in funcs_f: if re.search(END_CASE_REGEX, line): break diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index fe748aeb4..b32d18423 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -647,7 +647,7 @@ void func() self.assertEqual(arg, []) expected = '''#line 1 "test_suite_ut.function" -void test_func() +void test_func(void) { ba ba black sheep have you any wool @@ -690,7 +690,7 @@ exit: expected = '''#line 1 "test_suite_ut.function" -void test_func() +void test_func(void) { ba ba black sheep have you any wool @@ -750,7 +750,7 @@ exit: void -test_func() +test_func(void) { ba ba black sheep have you any wool @@ -803,7 +803,7 @@ exit: -void test_func() +void test_func(void) { ba ba black sheep have you any wool @@ -1139,7 +1139,7 @@ void func2() #if defined(MBEDTLS_ENTROPY_NV_SEED) #if defined(MBEDTLS_FS_IO) #line 13 "test_suite_ut.function" -void test_func1() +void test_func1(void) { exit: ; @@ -1156,7 +1156,7 @@ void test_func1_wrapper( void ** params ) #if defined(MBEDTLS_ENTROPY_NV_SEED) #if defined(MBEDTLS_FS_IO) #line 19 "test_suite_ut.function" -void test_func2() +void test_func2(void) { exit: ; From 12cd806a6090ec0ce2beff018713abaca72fa91a Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Thu, 27 Jul 2023 14:22:34 +0100 Subject: [PATCH 439/490] Make code_style.py -s more precise Signed-off-by: Dave Rodgman --- scripts/code_style.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 7de93b085..89263cec0 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -81,11 +81,15 @@ def get_src_files(since: Optional[str]) -> List[str]: universal_newlines=True) src_files = output.split() if since: - output = subprocess.check_output(["git", "diff", "--name-only", - since, "--"] + - src_files, - universal_newlines=True) - src_files = output.split() + # get all files changed in commits since the starting point + cmd = ["git", "log", since + "..HEAD", "--name-only", "--pretty=", "--" ] + src_files + output = subprocess.check_output(cmd, universal_newlines=True) + committed_changed_files = output.split() + # and also get all files with uncommitted changes + cmd = ["git", "diff", "--name-only", "--" ] + src_files + output = subprocess.check_output(cmd, universal_newlines=True) + uncommitted_changed_files = output.split() + src_files = set(committed_changed_files + uncommitted_changed_files) generated_files = list_generated_files() # Don't correct style for third-party files (and, for simplicity, From 7853f4e498eb0f2531dc5c454e2b7b674dc4bff6 Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Thu, 27 Jul 2023 14:22:55 +0100 Subject: [PATCH 440/490] Make code_style.py -s default to -s=development Signed-off-by: Dave Rodgman --- scripts/code_style.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 89263cec0..664222df5 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -193,9 +193,10 @@ def main() -> int: parser.add_argument('-f', '--fix', action='store_true', help=('modify source files to fix the code style ' '(default: print diff, do not modify files)')) - parser.add_argument('-s', '--since', metavar='COMMIT', + parser.add_argument('-s', '--since', metavar='COMMIT', const='development', nargs='?', help=('only check files modified since the specified commit' - ' (e.g. --since=HEAD~3 or --since=development)')) + ' (e.g. --since=HEAD~3 or --since=development). If no' + ' commit is specified, default to development.')) # --subset is almost useless: it only matters if there are no files # ('code_style.py' without arguments checks all files known to Git, # 'code_style.py --subset' does nothing). In particular, From 34b89f9c3d96885c1861bb47345bd9e88526069a Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Thu, 27 Jul 2023 18:50:50 +0100 Subject: [PATCH 441/490] pylint tidy-up Signed-off-by: Dave Rodgman --- scripts/code_style.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 664222df5..d6e56b29d 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -82,14 +82,14 @@ def get_src_files(since: Optional[str]) -> List[str]: src_files = output.split() if since: # get all files changed in commits since the starting point - cmd = ["git", "log", since + "..HEAD", "--name-only", "--pretty=", "--" ] + src_files + cmd = ["git", "log", since + "..HEAD", "--name-only", "--pretty=", "--"] + src_files output = subprocess.check_output(cmd, universal_newlines=True) committed_changed_files = output.split() # and also get all files with uncommitted changes cmd = ["git", "diff", "--name-only", "--" ] + src_files output = subprocess.check_output(cmd, universal_newlines=True) uncommitted_changed_files = output.split() - src_files = set(committed_changed_files + uncommitted_changed_files) + src_files = list(set(committed_changed_files + uncommitted_changed_files)) generated_files = list_generated_files() # Don't correct style for third-party files (and, for simplicity, From bb2f4d631757231a3a904d1323de6d7b4d06e920 Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Thu, 27 Jul 2023 20:00:41 +0100 Subject: [PATCH 442/490] whitespace fix Signed-off-by: Dave Rodgman --- scripts/code_style.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index d6e56b29d..ddd0a9800 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -86,7 +86,7 @@ def get_src_files(since: Optional[str]) -> List[str]: output = subprocess.check_output(cmd, universal_newlines=True) committed_changed_files = output.split() # and also get all files with uncommitted changes - cmd = ["git", "diff", "--name-only", "--" ] + src_files + cmd = ["git", "diff", "--name-only", "--"] + src_files output = subprocess.check_output(cmd, universal_newlines=True) uncommitted_changed_files = output.split() src_files = list(set(committed_changed_files + uncommitted_changed_files)) From 86fd07c75433ab881870920171b343108a22d141 Mon Sep 17 00:00:00 2001 From: Gowtham Suresh Kumar Date: Mon, 31 Jul 2023 16:38:10 +0100 Subject: [PATCH 443/490] Update default variable values for compilers Signed-off-by: Gowtham Suresh Kumar --- scripts/generate_test_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index c42f9a84b..76806de95 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -668,7 +668,7 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): name = 'test_' + name # If a test function has no arguments then add 'void' argument to - # avoid "-Wstrict-prototypes" warnings from clang-16 + # avoid "-Wstrict-prototypes" warnings from clang if len(args) == 0: code = code.replace('()', '(void)', 1) From 08984c07d93a87be7fa363d80797950611217f91 Mon Sep 17 00:00:00 2001 From: Agathiyan Bragadeesh Date: Tue, 1 Aug 2023 16:30:51 +0100 Subject: [PATCH 444/490] Add default test cases for add/subtract in bignum Signed-off-by: Agathiyan Bragadeesh --- scripts/framework_dev/bignum_data.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py index 0a48e538d..28076a49b 100644 --- a/scripts/framework_dev/bignum_data.py +++ b/scripts/framework_dev/bignum_data.py @@ -106,6 +106,29 @@ INPUTS_DEFAULT = [ RANDOM_1024_BIT_SEED_4_NO2, # largest (not a prime) ] +ADD_SUB_DEFAULT = [ + "0", "1", "3", "f", "fe", "ff", "100", "ff00", + "fffe", "ffff", "10000", # 2^16 - 1, 2^16, 2^16 + 1 + "fffffffe", "ffffffff", "100000000", # 2^32 - 1, 2^32, 2^32 + 1 + "1f7f7f7f7f7f7f", + "8000000000000000", "fefefefefefefefe", + "fffffffffffffffe", "ffffffffffffffff", "10000000000000000", # 2^64 - 1, 2^64, 2^64 + 1 + "1234567890abcdef0", + "fffffffffffffffffffffffe", + "ffffffffffffffffffffffff", + "1000000000000000000000000", + "fffffffffffffffffefefefefefefefe", + "fffffffffffffffffffffffffffffffe", + "ffffffffffffffffffffffffffffffff", + "100000000000000000000000000000000", + "1234567890abcdef01234567890abcdef0", + "fffffffffffffffffffffffffffffffffffffffffffffffffefefefefefefefe", + "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "10000000000000000000000000000000000000000000000000000000000000000", + "1234567890abcdef01234567890abcdef01234567890abcdef01234567890abcdef0", + ] + # Only odd moduli are present as in the new bignum code only odd moduli are # supported for now. MODULI_DEFAULT = [ From f00f5e40674df92b0cb4039f75de9e61ade60a51 Mon Sep 17 00:00:00 2001 From: Agathiyan Bragadeesh Date: Tue, 1 Aug 2023 16:32:36 +0100 Subject: [PATCH 445/490] Use ADD_SUB_DEFAULT as test input for bignum tests In BignumCoreAddAndAddIf and BignumCoreSub we use the new dedicated test inputs. Signed-off-by: Agathiyan Bragadeesh --- scripts/framework_dev/bignum_core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index ff3fd23e6..eecf94a11 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -21,6 +21,7 @@ from typing import Dict, Iterator, List, Tuple from . import test_case from . import test_data_generation from . import bignum_common +from .bignum_data import ADD_SUB_DEFAULT class BignumCoreTarget(test_data_generation.BaseTarget): #pylint: disable=abstract-method, too-few-public-methods @@ -176,6 +177,7 @@ class BignumCoreAddAndAddIf(BignumCoreTarget, bignum_common.OperationCommon): test_function = "mpi_core_add_and_add_if" test_name = "mpi_core_add_and_add_if" input_style = "arch_split" + input_values = ADD_SUB_DEFAULT unique_combinations_only = True def result(self) -> List[str]: @@ -196,6 +198,7 @@ class BignumCoreSub(BignumCoreTarget, bignum_common.OperationCommon): symbol = "-" test_function = "mpi_core_sub" test_name = "mbedtls_mpi_core_sub" + input_values = ADD_SUB_DEFAULT def result(self) -> List[str]: if self.int_a >= self.int_b: From f43c186334bb86a686c03f3cf3043b577cccd976 Mon Sep 17 00:00:00 2001 From: Agathiyan Bragadeesh Date: Tue, 1 Aug 2023 17:18:31 +0100 Subject: [PATCH 446/490] Rename ADD_SUB_DEFAULT to ADD_SUB_DATA Signed-off-by: Agathiyan Bragadeesh --- scripts/framework_dev/bignum_core.py | 6 +++--- scripts/framework_dev/bignum_data.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index eecf94a11..563492b29 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -21,7 +21,7 @@ from typing import Dict, Iterator, List, Tuple from . import test_case from . import test_data_generation from . import bignum_common -from .bignum_data import ADD_SUB_DEFAULT +from .bignum_data import ADD_SUB_DATA class BignumCoreTarget(test_data_generation.BaseTarget): #pylint: disable=abstract-method, too-few-public-methods @@ -177,7 +177,7 @@ class BignumCoreAddAndAddIf(BignumCoreTarget, bignum_common.OperationCommon): test_function = "mpi_core_add_and_add_if" test_name = "mpi_core_add_and_add_if" input_style = "arch_split" - input_values = ADD_SUB_DEFAULT + input_values = ADD_SUB_DATA unique_combinations_only = True def result(self) -> List[str]: @@ -198,7 +198,7 @@ class BignumCoreSub(BignumCoreTarget, bignum_common.OperationCommon): symbol = "-" test_function = "mpi_core_sub" test_name = "mbedtls_mpi_core_sub" - input_values = ADD_SUB_DEFAULT + input_values = ADD_SUB_DATA def result(self) -> List[str]: if self.int_a >= self.int_b: diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py index 28076a49b..6f132a6b3 100644 --- a/scripts/framework_dev/bignum_data.py +++ b/scripts/framework_dev/bignum_data.py @@ -106,7 +106,7 @@ INPUTS_DEFAULT = [ RANDOM_1024_BIT_SEED_4_NO2, # largest (not a prime) ] -ADD_SUB_DEFAULT = [ +ADD_SUB_DATA = [ "0", "1", "3", "f", "fe", "ff", "100", "ff00", "fffe", "ffff", "10000", # 2^16 - 1, 2^16, 2^16 + 1 "fffffffe", "ffffffff", "100000000", # 2^32 - 1, 2^32, 2^32 + 1 From 24d635d8952b5e60c98462ac515992378eef3fcf Mon Sep 17 00:00:00 2001 From: Agathiyan Bragadeesh Date: Thu, 3 Aug 2023 12:32:09 +0100 Subject: [PATCH 447/490] Remove trailing whitespace Signed-off-by: Agathiyan Bragadeesh --- scripts/framework_dev/bignum_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py index 6f132a6b3..897e31989 100644 --- a/scripts/framework_dev/bignum_data.py +++ b/scripts/framework_dev/bignum_data.py @@ -107,7 +107,7 @@ INPUTS_DEFAULT = [ ] ADD_SUB_DATA = [ - "0", "1", "3", "f", "fe", "ff", "100", "ff00", + "0", "1", "3", "f", "fe", "ff", "100", "ff00", "fffe", "ffff", "10000", # 2^16 - 1, 2^16, 2^16 + 1 "fffffffe", "ffffffff", "100000000", # 2^32 - 1, 2^32, 2^32 + 1 "1f7f7f7f7f7f7f", From f335772861744635182671531af29f475ec5538e Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Mon, 14 Aug 2023 10:33:37 +0800 Subject: [PATCH 448/490] logging_util: support to tweak loglevel directed to stderr/stdout Previously we set loglevel >= WARNING printed to stderr and loglevel < WARNING printed to stdout. To be more flexible, we replace this `WARNING` value with an argument: split_level and leave `WARNING` as default split_level if not set. Signed-off-by: Yanray Wang --- scripts/framework_dev/logging_util.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/framework_dev/logging_util.py b/scripts/framework_dev/logging_util.py index 85a3f19ac..db1ebfe5c 100644 --- a/scripts/framework_dev/logging_util.py +++ b/scripts/framework_dev/logging_util.py @@ -21,14 +21,16 @@ import sys def configure_logger( logger: logging.Logger, - log_format="[%(levelname)s]: %(message)s" + log_format="[%(levelname)s]: %(message)s", + split_level=logging.WARNING ) -> None: """ Configure the logging.Logger instance so that: - Format is set to any log_format. Default: "[%(levelname)s]: %(message)s" - - loglevel >= WARNING are printed to stderr. - - loglevel < WARNING are printed to stdout. + - loglevel >= split_level are printed to stderr. + - loglevel < split_level are printed to stdout. + Default: logging.WARNING """ class MaxLevelFilter(logging.Filter): # pylint: disable=too-few-public-methods @@ -41,14 +43,14 @@ def configure_logger( log_formatter = logging.Formatter(log_format) - # set loglevel >= WARNING to be printed to stderr + # set loglevel >= split_level to be printed to stderr stderr_hdlr = logging.StreamHandler(sys.stderr) - stderr_hdlr.setLevel(logging.WARNING) + stderr_hdlr.setLevel(split_level) stderr_hdlr.setFormatter(log_formatter) - # set loglevel <= INFO to be printed to stdout + # set loglevel < split_level to be printed to stdout stdout_hdlr = logging.StreamHandler(sys.stdout) - stdout_hdlr.addFilter(MaxLevelFilter(logging.INFO)) + stdout_hdlr.addFilter(MaxLevelFilter(split_level - 1)) stdout_hdlr.setFormatter(log_formatter) logger.addHandler(stderr_hdlr) From 7a19bbc9067ff5ee4b5c663d1fe2d903e52416c0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 19 Jun 2023 20:46:47 +0200 Subject: [PATCH 449/490] Move PSA information and dependency automation into their own module This will let us use these features from other modules (yet to be created). Signed-off-by: Gilles Peskine --- scripts/framework_dev/psa_information.py | 162 +++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 scripts/framework_dev/psa_information.py diff --git a/scripts/framework_dev/psa_information.py b/scripts/framework_dev/psa_information.py new file mode 100644 index 000000000..a82df41df --- /dev/null +++ b/scripts/framework_dev/psa_information.py @@ -0,0 +1,162 @@ +"""Collect information about PSA cryptographic mechanisms. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, FrozenSet, List, Optional + +from . import macro_collector + + +class Information: + """Gather information about PSA constructors.""" + + def __init__(self) -> None: + self.constructors = self.read_psa_interface() + + @staticmethod + def remove_unwanted_macros( + constructors: macro_collector.PSAMacroEnumerator + ) -> None: + # Mbed TLS does not support finite-field DSA. + # Don't attempt to generate any related test case. + constructors.key_types.discard('PSA_KEY_TYPE_DSA_KEY_PAIR') + constructors.key_types.discard('PSA_KEY_TYPE_DSA_PUBLIC_KEY') + + def read_psa_interface(self) -> macro_collector.PSAMacroEnumerator: + """Return the list of known key types, algorithms, etc.""" + constructors = macro_collector.InputsForTest() + header_file_names = ['include/psa/crypto_values.h', + 'include/psa/crypto_extra.h'] + test_suites = ['tests/suites/test_suite_psa_crypto_metadata.data'] + for header_file_name in header_file_names: + constructors.parse_header(header_file_name) + for test_cases in test_suites: + constructors.parse_test_cases(test_cases) + self.remove_unwanted_macros(constructors) + constructors.gather_arguments() + return constructors + + +def psa_want_symbol(name: str) -> str: + """Return the PSA_WANT_xxx symbol associated with a PSA crypto feature.""" + if name.startswith('PSA_'): + return name[:4] + 'WANT_' + name[4:] + else: + raise ValueError('Unable to determine the PSA_WANT_ symbol for ' + name) + +def finish_family_dependency(dep: str, bits: int) -> str: + """Finish dep if it's a family dependency symbol prefix. + + A family dependency symbol prefix is a PSA_WANT_ symbol that needs to be + qualified by the key size. If dep is such a symbol, finish it by adjusting + the prefix and appending the key size. Other symbols are left unchanged. + """ + return re.sub(r'_FAMILY_(.*)', r'_\1_' + str(bits), dep) + +def finish_family_dependencies(dependencies: List[str], bits: int) -> List[str]: + """Finish any family dependency symbol prefixes. + + Apply `finish_family_dependency` to each element of `dependencies`. + """ + return [finish_family_dependency(dep, bits) for dep in dependencies] + +SYMBOLS_WITHOUT_DEPENDENCY = frozenset([ + 'PSA_ALG_AEAD_WITH_AT_LEAST_THIS_LENGTH_TAG', # modifier, only in policies + 'PSA_ALG_AEAD_WITH_SHORTENED_TAG', # modifier + 'PSA_ALG_ANY_HASH', # only in policies + 'PSA_ALG_AT_LEAST_THIS_LENGTH_MAC', # modifier, only in policies + 'PSA_ALG_KEY_AGREEMENT', # chaining + 'PSA_ALG_TRUNCATED_MAC', # modifier +]) +def automatic_dependencies(*expressions: str) -> List[str]: + """Infer dependencies of a test case by looking for PSA_xxx symbols. + + The arguments are strings which should be C expressions. Do not use + string literals or comments as this function is not smart enough to + skip them. + """ + used = set() + for expr in expressions: + used.update(re.findall(r'PSA_(?:ALG|ECC_FAMILY|KEY_TYPE)_\w+', expr)) + used.difference_update(SYMBOLS_WITHOUT_DEPENDENCY) + return sorted(psa_want_symbol(name) for name in used) + +# Define set of regular expressions and dependencies to optionally append +# extra dependencies for test case. +AES_128BIT_ONLY_DEP_REGEX = r'AES\s(192|256)' +AES_128BIT_ONLY_DEP = ["!MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH"] + +DEPENDENCY_FROM_KEY = { + AES_128BIT_ONLY_DEP_REGEX: AES_128BIT_ONLY_DEP +}#type: Dict[str, List[str]] +def generate_key_dependencies(description: str) -> List[str]: + """Return additional dependencies based on pairs of REGEX and dependencies. + """ + deps = [] + for regex, dep in DEPENDENCY_FROM_KEY.items(): + if re.search(regex, description): + deps += dep + + return deps + +# A temporary hack: at the time of writing, not all dependency symbols +# are implemented yet. Skip test cases for which the dependency symbols are +# not available. Once all dependency symbols are available, this hack must +# be removed so that a bug in the dependency symbols properly leads to a test +# failure. +def read_implemented_dependencies(filename: str) -> FrozenSet[str]: + return frozenset(symbol + for line in open(filename) + for symbol in re.findall(r'\bPSA_WANT_\w+\b', line)) +_implemented_dependencies = None #type: Optional[FrozenSet[str]] #pylint: disable=invalid-name +def hack_dependencies_not_implemented(dependencies: List[str]) -> None: + global _implemented_dependencies #pylint: disable=global-statement,invalid-name + if _implemented_dependencies is None: + _implemented_dependencies = \ + read_implemented_dependencies('include/psa/crypto_config.h') + if not all((dep.lstrip('!') in _implemented_dependencies or + not dep.lstrip('!').startswith('PSA_WANT')) + for dep in dependencies): + dependencies.append('DEPENDENCY_NOT_IMPLEMENTED_YET') + +def tweak_key_pair_dependency(dep: str, usage: str): + """ + This helper function add the proper suffix to PSA_WANT_KEY_TYPE_xxx_KEY_PAIR + symbols according to the required usage. + """ + ret_list = list() + if dep.endswith('KEY_PAIR'): + if usage == "BASIC": + # BASIC automatically includes IMPORT and EXPORT for test purposes (see + # config_psa.h). + ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_BASIC', dep)) + ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_IMPORT', dep)) + ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_EXPORT', dep)) + elif usage == "GENERATE": + ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_GENERATE', dep)) + else: + # No replacement to do in this case + ret_list.append(dep) + return ret_list + +def fix_key_pair_dependencies(dep_list: List[str], usage: str): + new_list = [new_deps + for dep in dep_list + for new_deps in tweak_key_pair_dependency(dep, usage)] + + return new_list From 496313d34d9dfeaca9a67704e6b45ca087d7c8e2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 20 Jun 2023 15:22:53 +0200 Subject: [PATCH 450/490] New test suite for the low-level hash interface Some basic test coverage for now: * Nominal operation. * Larger output buffer. * Clone an operation and use it after the original operation stops. Generate test data automatically. For the time being, only do that for hashes that Python supports natively. Supporting all algorithms is future work. Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_data_tests.py | 123 +++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 scripts/framework_dev/crypto_data_tests.py diff --git a/scripts/framework_dev/crypto_data_tests.py b/scripts/framework_dev/crypto_data_tests.py new file mode 100644 index 000000000..80051fa43 --- /dev/null +++ b/scripts/framework_dev/crypto_data_tests.py @@ -0,0 +1,123 @@ +"""Generate test data for cryptographic mechanisms. + +This module is a work in progress, only implementing a few cases for now. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +from typing import Callable, Dict, Iterator, List, Optional #pylint: disable=unused-import + +from . import crypto_knowledge +from . import psa_information +from . import test_case + + +def psa_low_level_dependencies(*expressions: str) -> List[str]: + """Infer dependencies of a PSA low-level test case by looking for PSA_xxx symbols. + + This function generates MBEDTLS_PSA_BUILTIN_xxx symbols. + """ + high_level = psa_information.automatic_dependencies(*expressions) + for dep in high_level: + assert dep.startswith('PSA_WANT_') + return ['MBEDTLS_PSA_BUILTIN_' + dep[9:] for dep in high_level] + + +class HashPSALowLevel: + """Generate test cases for the PSA low-level hash interface.""" + + def __init__(self, info: psa_information.Information) -> None: + self.info = info + base_algorithms = sorted(info.constructors.algorithms) + all_algorithms = \ + [crypto_knowledge.Algorithm(expr) + for expr in info.constructors.generate_expressions(base_algorithms)] + self.algorithms = \ + [alg + for alg in all_algorithms + if (not alg.is_wildcard and + alg.can_do(crypto_knowledge.AlgorithmCategory.HASH))] + + # CALCULATE[alg] = function to return the hash of its argument in hex + # TO-DO: implement the None entries with a third-party library, because + # hashlib might not have everything, depending on the Python version and + # the underlying OpenSSL. On Ubuntu 16.04, truncated sha512 and sha3/shake + # are not available. On Ubuntu 22.04, md2, md4 and ripemd160 are not + # available. + CALCULATE = { + 'PSA_ALG_MD5': lambda data: hashlib.md5(data).hexdigest(), + 'PSA_ALG_RIPEMD160': None, #lambda data: hashlib.new('ripdemd160').hexdigest() + 'PSA_ALG_SHA_1': lambda data: hashlib.sha1(data).hexdigest(), + 'PSA_ALG_SHA_224': lambda data: hashlib.sha224(data).hexdigest(), + 'PSA_ALG_SHA_256': lambda data: hashlib.sha256(data).hexdigest(), + 'PSA_ALG_SHA_384': lambda data: hashlib.sha384(data).hexdigest(), + 'PSA_ALG_SHA_512': lambda data: hashlib.sha512(data).hexdigest(), + 'PSA_ALG_SHA_512_224': None, #lambda data: hashlib.new('sha512_224').hexdigest() + 'PSA_ALG_SHA_512_256': None, #lambda data: hashlib.new('sha512_256').hexdigest() + 'PSA_ALG_SHA3_224': None, #lambda data: hashlib.sha3_224(data).hexdigest(), + 'PSA_ALG_SHA3_256': None, #lambda data: hashlib.sha3_256(data).hexdigest(), + 'PSA_ALG_SHA3_384': None, #lambda data: hashlib.sha3_384(data).hexdigest(), + 'PSA_ALG_SHA3_512': None, #lambda data: hashlib.sha3_512(data).hexdigest(), + 'PSA_ALG_SHAKE256_512': None, #lambda data: hashlib.shake_256(data).hexdigest(64), + } #typing: Optional[Dict[str, Callable[[bytes], str]]] + + @staticmethod + def one_test_case(alg: crypto_knowledge.Algorithm, + function: str, note: str, + arguments: List[str]) -> test_case.TestCase: + """Construct one test case involving a hash.""" + tc = test_case.TestCase() + tc.set_description('{}{} {}' + .format(function, + ' ' + note if note else '', + alg.short_expression())) + tc.set_dependencies(psa_low_level_dependencies(alg.expression)) + tc.set_function(function) + tc.set_arguments([alg.expression] + + ['"{}"'.format(arg) for arg in arguments]) + return tc + + def test_cases_for_hash(self, + alg: crypto_knowledge.Algorithm + ) -> Iterator[test_case.TestCase]: + """Enumerate all test cases for one hash algorithm.""" + calc = self.CALCULATE[alg.expression] + if calc is None: + return # not implemented yet + + short = b'abc' + hash_short = calc(short) + long = (b'Hello, world. Here are 16 unprintable bytes: [' + b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a' + b'\x80\x81\x82\x83\xfe\xff]. ' + b' This message was brought to you by a natural intelligence. ' + b' If you can read this, good luck with your debugging!') + hash_long = calc(long) + + yield self.one_test_case(alg, 'hash_empty', '', [calc(b'')]) + yield self.one_test_case(alg, 'hash_valid_one_shot', '', + [short.hex(), hash_short]) + for n in [0, 1, 64, len(long) - 1, len(long)]: + yield self.one_test_case(alg, 'hash_valid_multipart', + '{} + {}'.format(n, len(long) - n), + [long[:n].hex(), calc(long[:n]), + long[n:].hex(), hash_long]) + + def all_test_cases(self) -> Iterator[test_case.TestCase]: + """Enumerate all test cases for all hash algorithms.""" + for alg in self.algorithms: + yield from self.test_cases_for_hash(alg) From 6ef458a7d3180241417e1e7edc5ef7e96944ff5d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 9 Aug 2023 10:50:58 +0200 Subject: [PATCH 451/490] Fix type annotation Signed-off-by: Gilles Peskine --- scripts/framework_dev/crypto_data_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/crypto_data_tests.py b/scripts/framework_dev/crypto_data_tests.py index 80051fa43..7593952da 100644 --- a/scripts/framework_dev/crypto_data_tests.py +++ b/scripts/framework_dev/crypto_data_tests.py @@ -73,7 +73,7 @@ class HashPSALowLevel: 'PSA_ALG_SHA3_384': None, #lambda data: hashlib.sha3_384(data).hexdigest(), 'PSA_ALG_SHA3_512': None, #lambda data: hashlib.sha3_512(data).hexdigest(), 'PSA_ALG_SHAKE256_512': None, #lambda data: hashlib.shake_256(data).hexdigest(64), - } #typing: Optional[Dict[str, Callable[[bytes], str]]] + } #type: Dict[str, Optional[Callable[[bytes], str]]] @staticmethod def one_test_case(alg: crypto_knowledge.Algorithm, From 567a8289c46c6d4492fa2d24ffdd6beeb474ad13 Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Tue, 18 Jul 2023 17:03:03 +0100 Subject: [PATCH 452/490] Modify build_tree.py for the PSA Crypto repo When detecting the root dir, look both for PSA Crypto and Mbed TLS directories. Signed-off-by: David Horstmann --- scripts/framework_dev/build_tree.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index f52b785d9..864085748 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -19,12 +19,19 @@ import os import inspect +def looks_like_psa_crypto_root(path: str) -> bool: + """Whether the given directory looks like the root of the PSA Crypto source tree.""" + return all(os.path.isdir(os.path.join(path, subdir)) + for subdir in ['include', 'core', 'tests']) def looks_like_mbedtls_root(path: str) -> bool: """Whether the given directory looks like the root of the Mbed TLS source tree.""" return all(os.path.isdir(os.path.join(path, subdir)) for subdir in ['include', 'library', 'programs', 'tests']) +def looks_like_root(path: str) -> bool: + return looks_like_psa_crypto_root(path) or looks_like_mbedtls_root(path) + def check_repo_path(): """ Check that the current working directory is the project root, and throw @@ -42,7 +49,7 @@ def chdir_to_root() -> None: for d in [os.path.curdir, os.path.pardir, os.path.join(os.path.pardir, os.path.pardir)]: - if looks_like_mbedtls_root(d): + if looks_like_root(d): os.chdir(d) return raise Exception('Mbed TLS source tree not found') @@ -62,6 +69,6 @@ def guess_mbedtls_root(): if d in dirs: continue dirs.add(d) - if looks_like_mbedtls_root(d): + if looks_like_root(d): return d raise Exception('Mbed TLS source tree not found') From 7f35b6bcbf250bfc8945a2960cc71b1ce0e5bf5b Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Tue, 18 Jul 2023 17:53:01 +0100 Subject: [PATCH 453/490] Support psa-crypto repo in psa_storage.py Signed-off-by: David Horstmann --- scripts/framework_dev/psa_storage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index bae99383d..a2e4c74a4 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -27,6 +27,7 @@ from typing import Dict, List, Optional, Set, Union import unittest from . import c_build_helper +from . import build_tree class Expr: @@ -51,13 +52,16 @@ class Expr: def update_cache(self) -> None: """Update `value_cache` for expressions registered in `unknown_values`.""" expressions = sorted(self.unknown_values) + includes = ['include'] + if build_tree.looks_like_psa_crypto_root('.'): + includes.append('drivers/builtin/include') values = c_build_helper.get_c_expression_values( 'unsigned long', '%lu', expressions, header=""" #include """, - include_path=['include']) #type: List[str] + include_path=includes) #type: List[str] for e, v in zip(expressions, values): self.value_cache[e] = int(v, 0) self.unknown_values.clear() From 81e8e304d1eaa139e8fcc11e5dd046b32062af1c Mon Sep 17 00:00:00 2001 From: David Horstmann Date: Tue, 29 Aug 2023 09:48:39 +0100 Subject: [PATCH 454/490] Improve directory coverage in PSA repo detection Check for the 'drivers' and 'programs' directories additionally to the ones that are already there. Signed-off-by: David Horstmann --- scripts/framework_dev/build_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index 864085748..b48a27711 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -22,7 +22,7 @@ import inspect def looks_like_psa_crypto_root(path: str) -> bool: """Whether the given directory looks like the root of the PSA Crypto source tree.""" return all(os.path.isdir(os.path.join(path, subdir)) - for subdir in ['include', 'core', 'tests']) + for subdir in ['include', 'core', 'drivers', 'programs', 'tests']) def looks_like_mbedtls_root(path: str) -> bool: """Whether the given directory looks like the root of the Mbed TLS source tree.""" From 4ab66b8de541be734221d74fc5e9c4b78bffd216 Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Fri, 6 Oct 2023 17:07:24 +0100 Subject: [PATCH 455/490] Correct styling of Mbed TLS in documentation Several bits of documentation were incorrectly styling Mbed TLS as MbedTLS. Signed-off-by: Thomas Daubney --- scripts/audit-validity-dates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 623fd2352..5128dc788 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -276,7 +276,7 @@ class Auditor: @staticmethod def find_test_dir(): - """Get the relative path for the MbedTLS test directory.""" + """Get the relative path for the Mbed TLS test directory.""" return os.path.relpath(build_tree.guess_mbedtls_root() + '/tests') From 7e479ed2c9209b59bacda7c37834491752c5ba52 Mon Sep 17 00:00:00 2001 From: Ronald Cron Date: Mon, 9 Oct 2023 10:25:45 +0200 Subject: [PATCH 456/490] Adapt to new PSA Crypto repo name Patterns I looked for: grep -i "psa-crypto" grep -i "psa.*crypto.*repo" grep -i "psa.*crypto.*root" Signed-off-by: Ronald Cron --- scripts/framework_dev/build_tree.py | 4 ++-- scripts/framework_dev/psa_storage.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index b48a27711..2e10c88e2 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -19,7 +19,7 @@ import os import inspect -def looks_like_psa_crypto_root(path: str) -> bool: +def looks_like_tf_psa_crypto_root(path: str) -> bool: """Whether the given directory looks like the root of the PSA Crypto source tree.""" return all(os.path.isdir(os.path.join(path, subdir)) for subdir in ['include', 'core', 'drivers', 'programs', 'tests']) @@ -30,7 +30,7 @@ def looks_like_mbedtls_root(path: str) -> bool: for subdir in ['include', 'library', 'programs', 'tests']) def looks_like_root(path: str) -> bool: - return looks_like_psa_crypto_root(path) or looks_like_mbedtls_root(path) + return looks_like_tf_psa_crypto_root(path) or looks_like_mbedtls_root(path) def check_repo_path(): """ diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index a2e4c74a4..737760fd4 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -53,7 +53,7 @@ class Expr: """Update `value_cache` for expressions registered in `unknown_values`.""" expressions = sorted(self.unknown_values) includes = ['include'] - if build_tree.looks_like_psa_crypto_root('.'): + if build_tree.looks_like_tf_psa_crypto_root('.'): includes.append('drivers/builtin/include') values = c_build_helper.get_c_expression_values( 'unsigned long', '%lu', From 2515cfae13a623f01a0bb0ae331b4bc2dedb9d48 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Tue, 23 May 2023 17:21:52 +0800 Subject: [PATCH 457/490] add script for server9_bad_saltlen Signed-off-by: Jerry Yu --- .../generate_server9_bad_saltlen.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100755 scripts/framework_dev/generate_server9_bad_saltlen.py diff --git a/scripts/framework_dev/generate_server9_bad_saltlen.py b/scripts/framework_dev/generate_server9_bad_saltlen.py new file mode 100755 index 000000000..68a8a5f15 --- /dev/null +++ b/scripts/framework_dev/generate_server9_bad_saltlen.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Generate server9-bad-saltlen.crt + +`server9-bad-saltlen.crt (announcing saltlen = 0xDE, signed with another len)`. It can not generate +with normal command. This script is to generate the file. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import argparse +from asn1crypto import pem, x509, core #type: ignore #pylint: disable=import-error + +OPENSSL_RSA_PSS_CERT_COMMAND = r''' +openssl x509 -req -CA {ca_name}.crt -CAkey {ca_name}.key -set_serial 24 {ca_password} \ + {openssl_extfile} -days 3650 -outform DER -in {csr} \ + -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:{anounce_saltlen} \ + -sigopt rsa_mgf1_md:sha256 +''' +SIG_OPT = \ + r'-sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:{saltlen} -sigopt rsa_mgf1_md:sha256' +OPENSSL_RSA_PSS_DGST_COMMAND = r'''openssl dgst -sign {ca_name}.key {ca_password} \ + -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:{actual_saltlen} \ + -sigopt rsa_mgf1_md:sha256''' + + +def auto_int(x): + return int(x, 0) + + +def build_argparser(parser): + """Build argument parser""" + parser.description = __doc__ + parser.add_argument('--ca-name', type=str, required=True, + help='Basename of CA files') + parser.add_argument('--ca-password', type=str, + required=True, help='CA key file password') + parser.add_argument('--csr', type=str, required=True, + help='CSR file for generating certificate') + parser.add_argument('--openssl-extfile', type=str, + required=True, help='X905 v3 extension config file') + parser.add_argument('--anounce_saltlen', type=auto_int, + required=True, help='Announced salt length') + parser.add_argument('--actual_saltlen', type=auto_int, + required=True, help='Actual salt length') + parser.add_argument('--output', type=str, required=True) + + +def main(): + parser = argparse.ArgumentParser() + build_argparser(parser) + args = parser.parse_args() + + return generate(**vars(args)) + +def generate(**kwargs): + """Generate different slt length certificate file.""" + ca_password = kwargs.get('ca_password', '') + if ca_password: + kwargs['ca_password'] = r'-passin "pass:{ca_password}"'.format( + **kwargs) + else: + kwargs['ca_password'] = '' + extfile = kwargs.get('openssl_extfile', '') + if extfile: + kwargs['openssl_extfile'] = '-extfile {openssl_extfile}'.format( + **kwargs) + else: + kwargs['openssl_extfile'] = '' + + cmd = OPENSSL_RSA_PSS_CERT_COMMAND.format(**kwargs) + der_bytes = subprocess.check_output(cmd, shell=True) + target_certificate = x509.Certificate.load(der_bytes) + + cmd = OPENSSL_RSA_PSS_DGST_COMMAND.format(**kwargs) + #pylint: disable=unexpected-keyword-arg + der_bytes = subprocess.check_output(cmd, + input=target_certificate['tbs_certificate'].dump(), + shell=True) + + with open(kwargs.get('output'), 'wb') as f: + target_certificate['signature_value'] = core.OctetBitString(der_bytes) + f.write(pem.armor('CERTIFICATE', target_certificate.dump())) + + +if __name__ == '__main__': + main() From 432e00ebc96f6f18ade876a47858d824128a5275 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Wed, 18 Oct 2023 15:06:54 +0800 Subject: [PATCH 458/490] fix wrong typo and indent issue Signed-off-by: Jerry Yu --- scripts/framework_dev/generate_server9_bad_saltlen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/generate_server9_bad_saltlen.py b/scripts/framework_dev/generate_server9_bad_saltlen.py index 68a8a5f15..813e6dc0f 100755 --- a/scripts/framework_dev/generate_server9_bad_saltlen.py +++ b/scripts/framework_dev/generate_server9_bad_saltlen.py @@ -67,7 +67,7 @@ def main(): return generate(**vars(args)) def generate(**kwargs): - """Generate different slt length certificate file.""" + """Generate different salt length certificate file.""" ca_password = kwargs.get('ca_password', '') if ca_password: kwargs['ca_password'] = r'-passin "pass:{ca_password}"'.format( From 20c91d577a81ee773324a7ea06ef16b58c277ece Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Tue, 24 Oct 2023 15:44:00 +0800 Subject: [PATCH 459/490] improve document Signed-off-by: Jerry Yu --- scripts/framework_dev/generate_server9_bad_saltlen.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/framework_dev/generate_server9_bad_saltlen.py b/scripts/framework_dev/generate_server9_bad_saltlen.py index 813e6dc0f..36682152a 100755 --- a/scripts/framework_dev/generate_server9_bad_saltlen.py +++ b/scripts/framework_dev/generate_server9_bad_saltlen.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 """Generate server9-bad-saltlen.crt -`server9-bad-saltlen.crt (announcing saltlen = 0xDE, signed with another len)`. It can not generate -with normal command. This script is to generate the file. +Generate a certificate signed with RSA-PSS, with an incorrect salt length. """ # Copyright The Mbed TLS Contributors From 07bc4a8043c13b04bdab4875e6aebea3c3d14616 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Tue, 24 Oct 2023 15:45:41 +0800 Subject: [PATCH 460/490] move script to `tests/scripts` Signed-off-by: Jerry Yu --- .../generate_server9_bad_saltlen.py | 99 ------------------- 1 file changed, 99 deletions(-) delete mode 100755 scripts/framework_dev/generate_server9_bad_saltlen.py diff --git a/scripts/framework_dev/generate_server9_bad_saltlen.py b/scripts/framework_dev/generate_server9_bad_saltlen.py deleted file mode 100755 index 36682152a..000000000 --- a/scripts/framework_dev/generate_server9_bad_saltlen.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -"""Generate server9-bad-saltlen.crt - -Generate a certificate signed with RSA-PSS, with an incorrect salt length. -""" - -# Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess -import argparse -from asn1crypto import pem, x509, core #type: ignore #pylint: disable=import-error - -OPENSSL_RSA_PSS_CERT_COMMAND = r''' -openssl x509 -req -CA {ca_name}.crt -CAkey {ca_name}.key -set_serial 24 {ca_password} \ - {openssl_extfile} -days 3650 -outform DER -in {csr} \ - -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:{anounce_saltlen} \ - -sigopt rsa_mgf1_md:sha256 -''' -SIG_OPT = \ - r'-sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:{saltlen} -sigopt rsa_mgf1_md:sha256' -OPENSSL_RSA_PSS_DGST_COMMAND = r'''openssl dgst -sign {ca_name}.key {ca_password} \ - -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:{actual_saltlen} \ - -sigopt rsa_mgf1_md:sha256''' - - -def auto_int(x): - return int(x, 0) - - -def build_argparser(parser): - """Build argument parser""" - parser.description = __doc__ - parser.add_argument('--ca-name', type=str, required=True, - help='Basename of CA files') - parser.add_argument('--ca-password', type=str, - required=True, help='CA key file password') - parser.add_argument('--csr', type=str, required=True, - help='CSR file for generating certificate') - parser.add_argument('--openssl-extfile', type=str, - required=True, help='X905 v3 extension config file') - parser.add_argument('--anounce_saltlen', type=auto_int, - required=True, help='Announced salt length') - parser.add_argument('--actual_saltlen', type=auto_int, - required=True, help='Actual salt length') - parser.add_argument('--output', type=str, required=True) - - -def main(): - parser = argparse.ArgumentParser() - build_argparser(parser) - args = parser.parse_args() - - return generate(**vars(args)) - -def generate(**kwargs): - """Generate different salt length certificate file.""" - ca_password = kwargs.get('ca_password', '') - if ca_password: - kwargs['ca_password'] = r'-passin "pass:{ca_password}"'.format( - **kwargs) - else: - kwargs['ca_password'] = '' - extfile = kwargs.get('openssl_extfile', '') - if extfile: - kwargs['openssl_extfile'] = '-extfile {openssl_extfile}'.format( - **kwargs) - else: - kwargs['openssl_extfile'] = '' - - cmd = OPENSSL_RSA_PSS_CERT_COMMAND.format(**kwargs) - der_bytes = subprocess.check_output(cmd, shell=True) - target_certificate = x509.Certificate.load(der_bytes) - - cmd = OPENSSL_RSA_PSS_DGST_COMMAND.format(**kwargs) - #pylint: disable=unexpected-keyword-arg - der_bytes = subprocess.check_output(cmd, - input=target_certificate['tbs_certificate'].dump(), - shell=True) - - with open(kwargs.get('output'), 'wb') as f: - target_certificate['signature_value'] = core.OctetBitString(der_bytes) - f.write(pem.armor('CERTIFICATE', target_certificate.dump())) - - -if __name__ == '__main__': - main() From 09146d73f9f505b3e32ae7994e8a22332a8d297a Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Thu, 2 Nov 2023 19:47:20 +0000 Subject: [PATCH 461/490] update headers Signed-off-by: Dave Rodgman --- scripts/audit-validity-dates.py | 14 +------------- scripts/code_style.py | 14 +------------- scripts/framework_dev/asymmetric_key_data.py | 13 +------------ scripts/framework_dev/bignum_common.py | 13 +------------ scripts/framework_dev/bignum_core.py | 13 +------------ scripts/framework_dev/bignum_data.py | 13 +------------ scripts/framework_dev/bignum_mod.py | 13 +------------ scripts/framework_dev/bignum_mod_raw.py | 13 +------------ scripts/framework_dev/build_tree.py | 13 +------------ scripts/framework_dev/c_build_helper.py | 13 +------------ scripts/framework_dev/crypto_data_tests.py | 13 +------------ scripts/framework_dev/crypto_knowledge.py | 13 +------------ scripts/framework_dev/ecp.py | 13 +------------ scripts/framework_dev/logging_util.py | 13 +------------ scripts/framework_dev/macro_collector.py | 13 +------------ scripts/framework_dev/psa_information.py | 13 +------------ scripts/framework_dev/psa_storage.py | 13 +------------ scripts/framework_dev/test_case.py | 13 +------------ scripts/framework_dev/test_data_generation.py | 13 +------------ scripts/framework_dev/typing_util.py | 13 +------------ scripts/generate_test_code.py | 14 +------------- scripts/test_generate_test_code.py | 14 +------------- 22 files changed, 22 insertions(+), 268 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 5128dc788..96b705a28 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -1,19 +1,7 @@ #!/usr/bin/env python3 # # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later """Audit validity date of X509 crt/crl/csr. diff --git a/scripts/code_style.py b/scripts/code_style.py index ddd0a9800..08ec4af42 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -4,19 +4,7 @@ This script must be run from the root of a Git work tree containing Mbed TLS. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later import argparse import os import re diff --git a/scripts/framework_dev/asymmetric_key_data.py b/scripts/framework_dev/asymmetric_key_data.py index 6fd6223f3..ef3e3a05e 100644 --- a/scripts/framework_dev/asymmetric_key_data.py +++ b/scripts/framework_dev/asymmetric_key_data.py @@ -4,19 +4,8 @@ Meant for use in crypto_knowledge.py. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import binascii import re diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 3bef16db6..eebc858b2 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -1,18 +1,7 @@ """Common features for bignum in test generation framework.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from abc import abstractmethod import enum diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 563492b29..909f6a306 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -1,18 +1,7 @@ """Framework classes for generation of bignum core test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import random diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py index 897e31989..5c6c2c81e 100644 --- a/scripts/framework_dev/bignum_data.py +++ b/scripts/framework_dev/bignum_data.py @@ -1,19 +1,8 @@ """Base values and datasets for bignum generated tests and helper functions that produced them.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import random diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 77c7b1bbd..f554001ec 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -1,18 +1,7 @@ """Framework classes for generation of bignum mod test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from typing import Dict, List diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 7121f2f49..37ad27a11 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -1,18 +1,7 @@ """Framework classes for generation of bignum mod_raw test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from typing import Iterator, List diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index 2e10c88e2..a657a5138 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import os import inspect diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 9bd17d608..f2cbbe4af 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import os import platform diff --git a/scripts/framework_dev/crypto_data_tests.py b/scripts/framework_dev/crypto_data_tests.py index 7593952da..a36de692e 100644 --- a/scripts/framework_dev/crypto_data_tests.py +++ b/scripts/framework_dev/crypto_data_tests.py @@ -4,19 +4,8 @@ This module is a work in progress, only implementing a few cases for now. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import hashlib from typing import Callable, Dict, Iterator, List, Optional #pylint: disable=unused-import diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 45d253b9b..285d6c638 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -4,19 +4,8 @@ This module is entirely based on the PSA API. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import enum import re diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 410c77e11..b40f3b126 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -1,18 +1,7 @@ """Framework classes for generation of ecp test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from typing import List diff --git a/scripts/framework_dev/logging_util.py b/scripts/framework_dev/logging_util.py index db1ebfe5c..ddd7c7fd6 100644 --- a/scripts/framework_dev/logging_util.py +++ b/scripts/framework_dev/logging_util.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import logging import sys diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 3cad2a3f6..d68be00bd 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import itertools import re diff --git a/scripts/framework_dev/psa_information.py b/scripts/framework_dev/psa_information.py index a82df41df..32e500977 100644 --- a/scripts/framework_dev/psa_information.py +++ b/scripts/framework_dev/psa_information.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import re from typing import Dict, FrozenSet, List, Optional diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index 737760fd4..b1fc37710 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -7,19 +7,8 @@ before changing how test data is constructed or validated. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import re import struct diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index 8f0870367..6ed5e849d 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import binascii import os diff --git a/scripts/framework_dev/test_data_generation.py b/scripts/framework_dev/test_data_generation.py index 02aa51051..a84f7dd2f 100644 --- a/scripts/framework_dev/test_data_generation.py +++ b/scripts/framework_dev/test_data_generation.py @@ -7,19 +7,8 @@ These are used both by generate_psa_tests.py and generate_bignum_tests.py. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import argparse import os diff --git a/scripts/framework_dev/typing_util.py b/scripts/framework_dev/typing_util.py index 4c344492c..2ec448d00 100644 --- a/scripts/framework_dev/typing_util.py +++ b/scripts/framework_dev/typing_util.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from typing import Any diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 76806de95..5f711bfb1 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -2,19 +2,7 @@ # Test suites code generator. # # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later """ This script is a key part of Mbed TLS test suites framework. For diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index b32d18423..abc46a729 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -2,19 +2,7 @@ # Unit test for generate_test_code.py # # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later """ Unit tests for generate_test_code.py From b2329744bf17da011e6f86eec627871c6827817b Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Wed, 1 Nov 2023 13:44:14 +0800 Subject: [PATCH 462/490] psa_information.py: generate dep for AES/ARIA/CAMELLIA ECB test case Signed-off-by: Yanray Wang --- scripts/framework_dev/psa_information.py | 34 +++++++++++++++--------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/scripts/framework_dev/psa_information.py b/scripts/framework_dev/psa_information.py index a82df41df..3c51ee150 100644 --- a/scripts/framework_dev/psa_information.py +++ b/scripts/framework_dev/psa_information.py @@ -17,7 +17,8 @@ # limitations under the License. import re -from typing import Dict, FrozenSet, List, Optional +from collections import OrderedDict +from typing import FrozenSet, List, Optional from . import macro_collector @@ -97,22 +98,31 @@ def automatic_dependencies(*expressions: str) -> List[str]: return sorted(psa_want_symbol(name) for name in used) # Define set of regular expressions and dependencies to optionally append -# extra dependencies for test case. +# extra dependencies for test case based on key description. + +# Skip AES test cases which require 192- or 256-bit key +# if MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH defined AES_128BIT_ONLY_DEP_REGEX = r'AES\s(192|256)' -AES_128BIT_ONLY_DEP = ["!MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH"] +AES_128BIT_ONLY_DEP = ['!MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH'] +# Skip AES/ARIA/CAMELLIA test cases which require decrypt operation in ECB mode +# if MBEDTLS_BLOCK_CIPHER_NO_DECRYPT enabled. +ECB_NO_PADDING_DEP_REGEX = r'(AES|ARIA|CAMELLIA).*ECB_NO_PADDING' +ECB_NO_PADDING_DEP = ['!MBEDTLS_BLOCK_CIPHER_NO_DECRYPT'] -DEPENDENCY_FROM_KEY = { - AES_128BIT_ONLY_DEP_REGEX: AES_128BIT_ONLY_DEP -}#type: Dict[str, List[str]] -def generate_key_dependencies(description: str) -> List[str]: - """Return additional dependencies based on pairs of REGEX and dependencies. +DEPENDENCY_FROM_DESCRIPTION = OrderedDict() +DEPENDENCY_FROM_DESCRIPTION[AES_128BIT_ONLY_DEP_REGEX] = AES_128BIT_ONLY_DEP +DEPENDENCY_FROM_DESCRIPTION[ECB_NO_PADDING_DEP_REGEX] = ECB_NO_PADDING_DEP +def generate_description_dependencies( + dep_list: List[str], + description: str + ) -> List[str]: + """Return additional dependencies based on test case description and REGEX. """ - deps = [] - for regex, dep in DEPENDENCY_FROM_KEY.items(): + for regex, deps in DEPENDENCY_FROM_DESCRIPTION.items(): if re.search(regex, description): - deps += dep + dep_list += deps - return deps + return dep_list # A temporary hack: at the time of writing, not all dependency symbols # are implemented yet. Skip test cases for which the dependency symbols are From 3652581dc323a2d740fb66b1bd5bafa7483b8bc9 Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Fri, 3 Nov 2023 12:21:36 +0000 Subject: [PATCH 463/490] Header updates Signed-off-by: Dave Rodgman --- scripts/audit-validity-dates.py | 14 +------------- scripts/code_style.py | 14 +------------- scripts/framework_dev/asymmetric_key_data.py | 13 +------------ scripts/framework_dev/bignum_common.py | 13 +------------ scripts/framework_dev/bignum_core.py | 13 +------------ scripts/framework_dev/bignum_data.py | 13 +------------ scripts/framework_dev/bignum_mod.py | 13 +------------ scripts/framework_dev/bignum_mod_raw.py | 13 +------------ scripts/framework_dev/build_tree.py | 13 +------------ scripts/framework_dev/c_build_helper.py | 13 +------------ scripts/framework_dev/crypto_data_tests.py | 13 +------------ scripts/framework_dev/crypto_knowledge.py | 13 +------------ scripts/framework_dev/ecp.py | 13 +------------ scripts/framework_dev/logging_util.py | 13 +------------ scripts/framework_dev/macro_collector.py | 13 +------------ scripts/framework_dev/psa_information.py | 13 +------------ scripts/framework_dev/psa_storage.py | 13 +------------ scripts/framework_dev/test_case.py | 13 +------------ scripts/framework_dev/test_data_generation.py | 13 +------------ scripts/framework_dev/typing_util.py | 13 +------------ scripts/generate_test_code.py | 14 +------------- scripts/test_generate_test_code.py | 14 +------------- 22 files changed, 22 insertions(+), 268 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 623fd2352..b2230dd47 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -1,19 +1,7 @@ #!/usr/bin/env python3 # # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later """Audit validity date of X509 crt/crl/csr. diff --git a/scripts/code_style.py b/scripts/code_style.py index ddd0a9800..08ec4af42 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -4,19 +4,7 @@ This script must be run from the root of a Git work tree containing Mbed TLS. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later import argparse import os import re diff --git a/scripts/framework_dev/asymmetric_key_data.py b/scripts/framework_dev/asymmetric_key_data.py index 6fd6223f3..ef3e3a05e 100644 --- a/scripts/framework_dev/asymmetric_key_data.py +++ b/scripts/framework_dev/asymmetric_key_data.py @@ -4,19 +4,8 @@ Meant for use in crypto_knowledge.py. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import binascii import re diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index 3bef16db6..eebc858b2 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -1,18 +1,7 @@ """Common features for bignum in test generation framework.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from abc import abstractmethod import enum diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 563492b29..909f6a306 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -1,18 +1,7 @@ """Framework classes for generation of bignum core test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import random diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py index 897e31989..5c6c2c81e 100644 --- a/scripts/framework_dev/bignum_data.py +++ b/scripts/framework_dev/bignum_data.py @@ -1,19 +1,8 @@ """Base values and datasets for bignum generated tests and helper functions that produced them.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import random diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index 77c7b1bbd..f554001ec 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -1,18 +1,7 @@ """Framework classes for generation of bignum mod test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from typing import Dict, List diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 7121f2f49..37ad27a11 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -1,18 +1,7 @@ """Framework classes for generation of bignum mod_raw test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from typing import Iterator, List diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index b48a27711..f63c3c828 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import os import inspect diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index 9bd17d608..f2cbbe4af 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import os import platform diff --git a/scripts/framework_dev/crypto_data_tests.py b/scripts/framework_dev/crypto_data_tests.py index 7593952da..a36de692e 100644 --- a/scripts/framework_dev/crypto_data_tests.py +++ b/scripts/framework_dev/crypto_data_tests.py @@ -4,19 +4,8 @@ This module is a work in progress, only implementing a few cases for now. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import hashlib from typing import Callable, Dict, Iterator, List, Optional #pylint: disable=unused-import diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 45d253b9b..285d6c638 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -4,19 +4,8 @@ This module is entirely based on the PSA API. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import enum import re diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index 410c77e11..b40f3b126 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -1,18 +1,7 @@ """Framework classes for generation of ecp test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from typing import List diff --git a/scripts/framework_dev/logging_util.py b/scripts/framework_dev/logging_util.py index db1ebfe5c..ddd7c7fd6 100644 --- a/scripts/framework_dev/logging_util.py +++ b/scripts/framework_dev/logging_util.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import logging import sys diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index 3cad2a3f6..d68be00bd 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import itertools import re diff --git a/scripts/framework_dev/psa_information.py b/scripts/framework_dev/psa_information.py index a82df41df..32e500977 100644 --- a/scripts/framework_dev/psa_information.py +++ b/scripts/framework_dev/psa_information.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import re from typing import Dict, FrozenSet, List, Optional diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index a2e4c74a4..00a8beaab 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -7,19 +7,8 @@ before changing how test data is constructed or validated. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import re import struct diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index 8f0870367..6ed5e849d 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import binascii import os diff --git a/scripts/framework_dev/test_data_generation.py b/scripts/framework_dev/test_data_generation.py index 02aa51051..a84f7dd2f 100644 --- a/scripts/framework_dev/test_data_generation.py +++ b/scripts/framework_dev/test_data_generation.py @@ -7,19 +7,8 @@ These are used both by generate_psa_tests.py and generate_bignum_tests.py. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import argparse import os diff --git a/scripts/framework_dev/typing_util.py b/scripts/framework_dev/typing_util.py index 4c344492c..2ec448d00 100644 --- a/scripts/framework_dev/typing_util.py +++ b/scripts/framework_dev/typing_util.py @@ -2,19 +2,8 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from typing import Any diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 76806de95..5f711bfb1 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -2,19 +2,7 @@ # Test suites code generator. # # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later """ This script is a key part of Mbed TLS test suites framework. For diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index b32d18423..abc46a729 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -2,19 +2,7 @@ # Unit test for generate_test_code.py # # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later """ Unit tests for generate_test_code.py From b56cc5353fde20f76bcab4fa1497585e2411753e Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Wed, 8 Nov 2023 11:37:56 +0000 Subject: [PATCH 464/490] Revert back to v3.5.0 git revert v3.5.0..v3.5.1 git rebase to combine the resulting revert commits Signed-off-by: Dave Rodgman --- scripts/audit-validity-dates.py | 14 +++++++++++++- scripts/code_style.py | 14 +++++++++++++- scripts/framework_dev/asymmetric_key_data.py | 13 ++++++++++++- scripts/framework_dev/bignum_common.py | 13 ++++++++++++- scripts/framework_dev/bignum_core.py | 13 ++++++++++++- scripts/framework_dev/bignum_data.py | 13 ++++++++++++- scripts/framework_dev/bignum_mod.py | 13 ++++++++++++- scripts/framework_dev/bignum_mod_raw.py | 13 ++++++++++++- scripts/framework_dev/build_tree.py | 13 ++++++++++++- scripts/framework_dev/c_build_helper.py | 13 ++++++++++++- scripts/framework_dev/crypto_data_tests.py | 13 ++++++++++++- scripts/framework_dev/crypto_knowledge.py | 13 ++++++++++++- scripts/framework_dev/ecp.py | 13 ++++++++++++- scripts/framework_dev/logging_util.py | 13 ++++++++++++- scripts/framework_dev/macro_collector.py | 13 ++++++++++++- scripts/framework_dev/psa_information.py | 13 ++++++++++++- scripts/framework_dev/psa_storage.py | 13 ++++++++++++- scripts/framework_dev/test_case.py | 13 ++++++++++++- scripts/framework_dev/test_data_generation.py | 13 ++++++++++++- scripts/framework_dev/typing_util.py | 13 ++++++++++++- scripts/generate_test_code.py | 14 +++++++++++++- scripts/test_generate_test_code.py | 14 +++++++++++++- 22 files changed, 268 insertions(+), 22 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index b2230dd47..623fd2352 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -1,7 +1,19 @@ #!/usr/bin/env python3 # # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Audit validity date of X509 crt/crl/csr. diff --git a/scripts/code_style.py b/scripts/code_style.py index 08ec4af42..ddd0a9800 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -4,7 +4,19 @@ This script must be run from the root of a Git work tree containing Mbed TLS. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import argparse import os import re diff --git a/scripts/framework_dev/asymmetric_key_data.py b/scripts/framework_dev/asymmetric_key_data.py index ef3e3a05e..6fd6223f3 100644 --- a/scripts/framework_dev/asymmetric_key_data.py +++ b/scripts/framework_dev/asymmetric_key_data.py @@ -4,8 +4,19 @@ Meant for use in crypto_knowledge.py. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import binascii import re diff --git a/scripts/framework_dev/bignum_common.py b/scripts/framework_dev/bignum_common.py index eebc858b2..3bef16db6 100644 --- a/scripts/framework_dev/bignum_common.py +++ b/scripts/framework_dev/bignum_common.py @@ -1,7 +1,18 @@ """Common features for bignum in test generation framework.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from abc import abstractmethod import enum diff --git a/scripts/framework_dev/bignum_core.py b/scripts/framework_dev/bignum_core.py index 909f6a306..563492b29 100644 --- a/scripts/framework_dev/bignum_core.py +++ b/scripts/framework_dev/bignum_core.py @@ -1,7 +1,18 @@ """Framework classes for generation of bignum core test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import random diff --git a/scripts/framework_dev/bignum_data.py b/scripts/framework_dev/bignum_data.py index 5c6c2c81e..897e31989 100644 --- a/scripts/framework_dev/bignum_data.py +++ b/scripts/framework_dev/bignum_data.py @@ -1,8 +1,19 @@ """Base values and datasets for bignum generated tests and helper functions that produced them.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import random diff --git a/scripts/framework_dev/bignum_mod.py b/scripts/framework_dev/bignum_mod.py index f554001ec..77c7b1bbd 100644 --- a/scripts/framework_dev/bignum_mod.py +++ b/scripts/framework_dev/bignum_mod.py @@ -1,7 +1,18 @@ """Framework classes for generation of bignum mod test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import Dict, List diff --git a/scripts/framework_dev/bignum_mod_raw.py b/scripts/framework_dev/bignum_mod_raw.py index 37ad27a11..7121f2f49 100644 --- a/scripts/framework_dev/bignum_mod_raw.py +++ b/scripts/framework_dev/bignum_mod_raw.py @@ -1,7 +1,18 @@ """Framework classes for generation of bignum mod_raw test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import Iterator, List diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index f63c3c828..b48a27711 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -2,8 +2,19 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import os import inspect diff --git a/scripts/framework_dev/c_build_helper.py b/scripts/framework_dev/c_build_helper.py index f2cbbe4af..9bd17d608 100644 --- a/scripts/framework_dev/c_build_helper.py +++ b/scripts/framework_dev/c_build_helper.py @@ -2,8 +2,19 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import os import platform diff --git a/scripts/framework_dev/crypto_data_tests.py b/scripts/framework_dev/crypto_data_tests.py index a36de692e..7593952da 100644 --- a/scripts/framework_dev/crypto_data_tests.py +++ b/scripts/framework_dev/crypto_data_tests.py @@ -4,8 +4,19 @@ This module is a work in progress, only implementing a few cases for now. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import hashlib from typing import Callable, Dict, Iterator, List, Optional #pylint: disable=unused-import diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 285d6c638..45d253b9b 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -4,8 +4,19 @@ This module is entirely based on the PSA API. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import enum import re diff --git a/scripts/framework_dev/ecp.py b/scripts/framework_dev/ecp.py index b40f3b126..410c77e11 100644 --- a/scripts/framework_dev/ecp.py +++ b/scripts/framework_dev/ecp.py @@ -1,7 +1,18 @@ """Framework classes for generation of ecp test cases.""" # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import List diff --git a/scripts/framework_dev/logging_util.py b/scripts/framework_dev/logging_util.py index ddd7c7fd6..db1ebfe5c 100644 --- a/scripts/framework_dev/logging_util.py +++ b/scripts/framework_dev/logging_util.py @@ -2,8 +2,19 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import logging import sys diff --git a/scripts/framework_dev/macro_collector.py b/scripts/framework_dev/macro_collector.py index d68be00bd..3cad2a3f6 100644 --- a/scripts/framework_dev/macro_collector.py +++ b/scripts/framework_dev/macro_collector.py @@ -2,8 +2,19 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import itertools import re diff --git a/scripts/framework_dev/psa_information.py b/scripts/framework_dev/psa_information.py index 32e500977..a82df41df 100644 --- a/scripts/framework_dev/psa_information.py +++ b/scripts/framework_dev/psa_information.py @@ -2,8 +2,19 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import re from typing import Dict, FrozenSet, List, Optional diff --git a/scripts/framework_dev/psa_storage.py b/scripts/framework_dev/psa_storage.py index 00a8beaab..a2e4c74a4 100644 --- a/scripts/framework_dev/psa_storage.py +++ b/scripts/framework_dev/psa_storage.py @@ -7,8 +7,19 @@ before changing how test data is constructed or validated. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import re import struct diff --git a/scripts/framework_dev/test_case.py b/scripts/framework_dev/test_case.py index 6ed5e849d..8f0870367 100644 --- a/scripts/framework_dev/test_case.py +++ b/scripts/framework_dev/test_case.py @@ -2,8 +2,19 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import binascii import os diff --git a/scripts/framework_dev/test_data_generation.py b/scripts/framework_dev/test_data_generation.py index a84f7dd2f..02aa51051 100644 --- a/scripts/framework_dev/test_data_generation.py +++ b/scripts/framework_dev/test_data_generation.py @@ -7,8 +7,19 @@ These are used both by generate_psa_tests.py and generate_bignum_tests.py. """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import argparse import os diff --git a/scripts/framework_dev/typing_util.py b/scripts/framework_dev/typing_util.py index 2ec448d00..4c344492c 100644 --- a/scripts/framework_dev/typing_util.py +++ b/scripts/framework_dev/typing_util.py @@ -2,8 +2,19 @@ """ # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 # +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import Any diff --git a/scripts/generate_test_code.py b/scripts/generate_test_code.py index 5f711bfb1..76806de95 100755 --- a/scripts/generate_test_code.py +++ b/scripts/generate_test_code.py @@ -2,7 +2,19 @@ # Test suites code generator. # # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ This script is a key part of Mbed TLS test suites framework. For diff --git a/scripts/test_generate_test_code.py b/scripts/test_generate_test_code.py index abc46a729..b32d18423 100755 --- a/scripts/test_generate_test_code.py +++ b/scripts/test_generate_test_code.py @@ -2,7 +2,19 @@ # Unit test for generate_test_code.py # # Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Unit tests for generate_test_code.py From e803233b0f5e5a3a103b738d65b73c063fdd3498 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Thu, 9 Nov 2023 16:13:53 +0800 Subject: [PATCH 465/490] psa_information: compile a regex instead of using string directly Compiling a regex improves performance and avoids accidentally combining it with a string. This commit makes this change. Signed-off-by: Yanray Wang --- scripts/framework_dev/psa_information.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/psa_information.py b/scripts/framework_dev/psa_information.py index 3c51ee150..2287ae13e 100644 --- a/scripts/framework_dev/psa_information.py +++ b/scripts/framework_dev/psa_information.py @@ -102,11 +102,11 @@ def automatic_dependencies(*expressions: str) -> List[str]: # Skip AES test cases which require 192- or 256-bit key # if MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH defined -AES_128BIT_ONLY_DEP_REGEX = r'AES\s(192|256)' +AES_128BIT_ONLY_DEP_REGEX = re.compile(r'AES\s(192|256)') AES_128BIT_ONLY_DEP = ['!MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH'] # Skip AES/ARIA/CAMELLIA test cases which require decrypt operation in ECB mode # if MBEDTLS_BLOCK_CIPHER_NO_DECRYPT enabled. -ECB_NO_PADDING_DEP_REGEX = r'(AES|ARIA|CAMELLIA).*ECB_NO_PADDING' +ECB_NO_PADDING_DEP_REGEX = re.compile(r'(AES|ARIA|CAMELLIA).*ECB_NO_PADDING') ECB_NO_PADDING_DEP = ['!MBEDTLS_BLOCK_CIPHER_NO_DECRYPT'] DEPENDENCY_FROM_DESCRIPTION = OrderedDict() From 5fd78db5690ff3b801f420d5cdb6acd14ef8b173 Mon Sep 17 00:00:00 2001 From: Yanray Wang Date: Mon, 13 Nov 2023 17:39:32 +0800 Subject: [PATCH 466/490] psa_information: improve code readability Signed-off-by: Yanray Wang --- scripts/framework_dev/psa_information.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/psa_information.py b/scripts/framework_dev/psa_information.py index 2287ae13e..ea8c12341 100644 --- a/scripts/framework_dev/psa_information.py +++ b/scripts/framework_dev/psa_information.py @@ -112,12 +112,12 @@ ECB_NO_PADDING_DEP = ['!MBEDTLS_BLOCK_CIPHER_NO_DECRYPT'] DEPENDENCY_FROM_DESCRIPTION = OrderedDict() DEPENDENCY_FROM_DESCRIPTION[AES_128BIT_ONLY_DEP_REGEX] = AES_128BIT_ONLY_DEP DEPENDENCY_FROM_DESCRIPTION[ECB_NO_PADDING_DEP_REGEX] = ECB_NO_PADDING_DEP -def generate_description_dependencies( - dep_list: List[str], +def generate_deps_from_description( description: str ) -> List[str]: """Return additional dependencies based on test case description and REGEX. """ + dep_list = [] for regex, deps in DEPENDENCY_FROM_DESCRIPTION.items(): if re.search(regex, description): dep_list += deps From 0231c7d145929b6ad9d0f08abb532d6dca82135a Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Thu, 16 Nov 2023 18:34:58 +0000 Subject: [PATCH 467/490] Introduce function to return library/core directory Add crypto_core_directory in build_tree.py so that the libary/core directory can be returned based on what repository we are in. Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index a657a5138..4a42d9a2b 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -21,6 +21,14 @@ def looks_like_mbedtls_root(path: str) -> bool: def looks_like_root(path: str) -> bool: return looks_like_tf_psa_crypto_root(path) or looks_like_mbedtls_root(path) +def crypto_core_directory() -> str: + if looks_like_tf_psa_crypto_root(os.path.curdir): + return "core" + elif looks_like_mbedtls_root(os.path.curdir): + return "library" + else: + raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') + def check_repo_path(): """ Check that the current working directory is the project root, and throw From 372e532d3e46dabeb61c776f0add8f3e2f1a7ab8 Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Wed, 22 Nov 2023 17:00:34 +0000 Subject: [PATCH 468/490] Improve implementation of crypto_core_directory Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index 4a42d9a2b..9c0b5ba96 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -7,6 +7,7 @@ import os import inspect +from typing import Optional def looks_like_tf_psa_crypto_root(path: str) -> bool: """Whether the given directory looks like the root of the PSA Crypto source tree.""" @@ -21,10 +22,12 @@ def looks_like_mbedtls_root(path: str) -> bool: def looks_like_root(path: str) -> bool: return looks_like_tf_psa_crypto_root(path) or looks_like_mbedtls_root(path) -def crypto_core_directory() -> str: - if looks_like_tf_psa_crypto_root(os.path.curdir): +def crypto_core_directory(root: Optional[str] = None) -> str: + if root is None: + root = guess_mbedtls_root() + if looks_like_tf_psa_crypto_root(root): return "core" - elif looks_like_mbedtls_root(os.path.curdir): + elif looks_like_mbedtls_root(root): return "library" else: raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') From d9903421d8ee4c751d9f944b327faebc606e3aaa Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Wed, 22 Nov 2023 17:18:22 +0000 Subject: [PATCH 469/490] Rename guess_mbedtls_root to guess_project_root Rename for consistency. Also, replace all calls to this function with correct name. Signed-off-by: Thomas Daubney --- scripts/audit-validity-dates.py | 2 +- scripts/framework_dev/build_tree.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index 96b705a28..ab09b4a1e 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -265,7 +265,7 @@ class Auditor: @staticmethod def find_test_dir(): """Get the relative path for the Mbed TLS test directory.""" - return os.path.relpath(build_tree.guess_mbedtls_root() + '/tests') + return os.path.relpath(build_tree.guess_project_root() + '/tests') class TestDataAuditor(Auditor): diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index 9c0b5ba96..da455a7f2 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -24,7 +24,7 @@ def looks_like_root(path: str) -> bool: def crypto_core_directory(root: Optional[str] = None) -> str: if root is None: - root = guess_mbedtls_root() + root = guess_project_root() if looks_like_tf_psa_crypto_root(root): return "core" elif looks_like_mbedtls_root(root): @@ -55,10 +55,10 @@ def chdir_to_root() -> None: raise Exception('Mbed TLS source tree not found') -def guess_mbedtls_root(): - """Guess mbedTLS source code directory. +def guess_project_root(): + """Guess project source code directory. - Return the first possible mbedTLS root directory + Return the first possible project root directory. """ dirs = set({}) for frame in inspect.stack(): @@ -71,4 +71,4 @@ def guess_mbedtls_root(): dirs.add(d) if looks_like_root(d): return d - raise Exception('Mbed TLS source tree not found') + raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') From acf4569625dd505bf09ba8502fb40506e87f0309 Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Thu, 23 Nov 2023 10:14:12 +0000 Subject: [PATCH 470/490] Introduce project_crypto_name in build_tree.py Add new function to build_tree.py to return the crypto name for the project; either tfpsacrypto or mbedcrypto. Deploy this function where needed. Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index da455a7f2..f506c4142 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -32,6 +32,16 @@ def crypto_core_directory(root: Optional[str] = None) -> str: else: raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') +def project_crypto_name(root: Optional[str] = None) -> str: + if root is None: + root = guess_project_root() + if looks_like_tf_psa_crypto_root(root): + return "tfpsacrypto" + elif looks_like_mbedtls_root(root): + return "mbedcrypto" + else: + raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') + def check_repo_path(): """ Check that the current working directory is the project root, and throw From 415486a63b76be43b3e1c2e7da05a77d615b900e Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Fri, 24 Nov 2023 10:48:44 +0000 Subject: [PATCH 471/490] Use os.path.join in crypto_core_directory Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index f506c4142..fa309d3e6 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -26,9 +26,9 @@ def crypto_core_directory(root: Optional[str] = None) -> str: if root is None: root = guess_project_root() if looks_like_tf_psa_crypto_root(root): - return "core" + return os.path.join(root, "core") elif looks_like_mbedtls_root(root): - return "library" + return os.path.join(root, "library") else: raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') From ebae927a747f2959dd5df3b3bfabd855f08b4903 Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Fri, 24 Nov 2023 10:54:56 +0000 Subject: [PATCH 472/490] Add documentation for new public functions Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index fa309d3e6..4e0ae1953 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -23,6 +23,7 @@ def looks_like_root(path: str) -> bool: return looks_like_tf_psa_crypto_root(path) or looks_like_mbedtls_root(path) def crypto_core_directory(root: Optional[str] = None) -> str: + """Return the path of the library code for either TF-PSA-Crypto or Mbed TLS.""" if root is None: root = guess_project_root() if looks_like_tf_psa_crypto_root(root): @@ -33,6 +34,7 @@ def crypto_core_directory(root: Optional[str] = None) -> str: raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') def project_crypto_name(root: Optional[str] = None) -> str: + """Return the crypto library filename for either TF-PSA-Crypto or Mbed TLS.""" if root is None: root = guess_project_root() if looks_like_tf_psa_crypto_root(root): From f880d037c03d81c5d417fc033929a5d1ff606c71 Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Thu, 30 Nov 2023 13:56:09 +0000 Subject: [PATCH 473/490] Rename project_crypto_name Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index 4e0ae1953..c2a370d7e 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -33,7 +33,7 @@ def crypto_core_directory(root: Optional[str] = None) -> str: else: raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') -def project_crypto_name(root: Optional[str] = None) -> str: +def crypto_library_filename(root: Optional[str] = None) -> str: """Return the crypto library filename for either TF-PSA-Crypto or Mbed TLS.""" if root is None: root = guess_project_root() From 176d3a6f90698a1880d43aee7f19e2290867f6df Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Thu, 30 Nov 2023 13:59:30 +0000 Subject: [PATCH 474/490] Improve documentation of crypto_core_directory Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index c2a370d7e..ff8d57594 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -23,7 +23,10 @@ def looks_like_root(path: str) -> bool: return looks_like_tf_psa_crypto_root(path) or looks_like_mbedtls_root(path) def crypto_core_directory(root: Optional[str] = None) -> str: - """Return the path of the library code for either TF-PSA-Crypto or Mbed TLS.""" + """ + Return the path of the directory containing the PSA crypto core + for either TF-PSA-Crypto or Mbed TLS. + """ if root is None: root = guess_project_root() if looks_like_tf_psa_crypto_root(root): From d2eedbd27d4a509e2db13da41a2efc57334a30b1 Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Thu, 30 Nov 2023 17:25:55 +0000 Subject: [PATCH 475/490] Introduce guess_mbedtls_root Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index ff8d57594..1868a0f89 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -48,8 +48,7 @@ def crypto_library_filename(root: Optional[str] = None) -> str: raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') def check_repo_path(): - """ - Check that the current working directory is the project root, and throw + """Check that the current working directory is the project root, and throw an exception if not. """ if not all(os.path.isdir(d) for d in ["include", "library", "tests"]): @@ -69,7 +68,6 @@ def chdir_to_root() -> None: return raise Exception('Mbed TLS source tree not found') - def guess_project_root(): """Guess project source code directory. @@ -87,3 +85,16 @@ def guess_project_root(): if looks_like_root(d): return d raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') + +def guess_mbedtls_root(root: Optional[str] = None) -> str: + """Guess Mbed TLS source code directory. + + Return the first possible Mbed TLS root directory. + Raise an exception if we are not in Mbed TLS. + """ + if root is None: + root = guess_project_root() + if looks_like_mbedtls_root(root): + return root + else: + raise Exception('Mbed TLS source tree not found') From 19da414938b5db0ea560ee4b458264c79241e321 Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Thu, 30 Nov 2023 17:33:54 +0000 Subject: [PATCH 476/490] Introduce guess_tf_psa_crypto_root Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index 1868a0f89..86c838900 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -98,3 +98,16 @@ def guess_mbedtls_root(root: Optional[str] = None) -> str: return root else: raise Exception('Mbed TLS source tree not found') + +def guess_tf_psa_crypto_root(root: Optional[str] = None) -> str: + """Guess TF-PSA-Crypto source code directory. + + Return the first possible TF-PSA-Crypto root directory. + Raise an exception if we are not in TF-PSA-Crypto. + """ + if root is None: + root = guess_project_root() + if looks_like_tf_psa_crypto_root(root): + return root + else: + raise Exception('TF-PSA-Crypto source tree not found') From a62109a82213cf50ff4e15e21ccfbdb5dcbbee68 Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Fri, 1 Dec 2023 09:52:35 +0000 Subject: [PATCH 477/490] Remove trailing whitespace Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index 86c838900..14790fc9c 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -24,7 +24,7 @@ def looks_like_root(path: str) -> bool: def crypto_core_directory(root: Optional[str] = None) -> str: """ - Return the path of the directory containing the PSA crypto core + Return the path of the directory containing the PSA crypto core for either TF-PSA-Crypto or Mbed TLS. """ if root is None: From f8d03139d41bf41178559dcb63ed057ea35ea109 Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Fri, 1 Dec 2023 17:18:38 +0000 Subject: [PATCH 478/490] Modify crypto_core_directory to also return a relative path Signed-off-by: Thomas Daubney --- scripts/framework_dev/build_tree.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/framework_dev/build_tree.py b/scripts/framework_dev/build_tree.py index 14790fc9c..ec67e4cdf 100644 --- a/scripts/framework_dev/build_tree.py +++ b/scripts/framework_dev/build_tree.py @@ -22,16 +22,23 @@ def looks_like_mbedtls_root(path: str) -> bool: def looks_like_root(path: str) -> bool: return looks_like_tf_psa_crypto_root(path) or looks_like_mbedtls_root(path) -def crypto_core_directory(root: Optional[str] = None) -> str: +def crypto_core_directory(root: Optional[str] = None, relative: Optional[bool] = False) -> str: """ Return the path of the directory containing the PSA crypto core for either TF-PSA-Crypto or Mbed TLS. + + Returns either the full path or relative path depending on the + "relative" boolean argument. """ if root is None: root = guess_project_root() if looks_like_tf_psa_crypto_root(root): + if relative: + return "core" return os.path.join(root, "core") elif looks_like_mbedtls_root(root): + if relative: + return "library" return os.path.join(root, "library") else: raise Exception('Neither Mbed TLS nor TF-PSA-Crypto source tree found') From ebd330453ea1e4471a38cb5c3d2275913d1a48c6 Mon Sep 17 00:00:00 2001 From: Thomas Daubney Date: Fri, 1 Dec 2023 18:27:25 +0000 Subject: [PATCH 479/490] Use guess_mbedtls_root in Mbed-TLS-only script Signed-off-by: Thomas Daubney --- scripts/audit-validity-dates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/audit-validity-dates.py b/scripts/audit-validity-dates.py index ab09b4a1e..96b705a28 100755 --- a/scripts/audit-validity-dates.py +++ b/scripts/audit-validity-dates.py @@ -265,7 +265,7 @@ class Auditor: @staticmethod def find_test_dir(): """Get the relative path for the Mbed TLS test directory.""" - return os.path.relpath(build_tree.guess_project_root() + '/tests') + return os.path.relpath(build_tree.guess_mbedtls_root() + '/tests') class TestDataAuditor(Auditor): From 4bcd1023d3448cbf81522d88cd0d8f5688fbc4c2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 3 Jan 2024 20:50:56 +0100 Subject: [PATCH 480/490] Fix mixup between secp224r1 and secp224k1 in test scripts secp224k1 is the one with 225-bit private keys. The consequences of this mistake were: * We emitted positive test cases for hypothetical SECP_R1_225 and SECP_K1_224 curves, which were never executed. * We emitted useless not-supported test cases for SECP_R1_225 and SECP_K1_224. * We were missing positive test cases for SECP_R1_224 in automatically generated tests. * We were missing not-supported test cases for SECP_R1_224 and SECP_K1_225. Thus this didn't cause test failures, but it caused missing test coverage and some never-executed test cases. Signed-off-by: Gilles Peskine --- scripts/framework_dev/asymmetric_key_data.py | 4 ++-- scripts/framework_dev/crypto_knowledge.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/framework_dev/asymmetric_key_data.py b/scripts/framework_dev/asymmetric_key_data.py index ef3e3a05e..29d95d0e1 100644 --- a/scripts/framework_dev/asymmetric_key_data.py +++ b/scripts/framework_dev/asymmetric_key_data.py @@ -41,13 +41,13 @@ ASYMMETRIC_KEY_DATA = construct_asymmetric_key_data({ 'ECC(PSA_ECC_FAMILY_SECP_K1)': { 192: ("297ac1722ccac7589ecb240dc719842538ca974beb79f228", "0426b7bb38da649ac2138fc050c6548b32553dab68afebc36105d325b75538c12323cb0764789ecb992671beb2b6bef2f5"), - 224: ("0024122bf020fa113f6c0ac978dfbd41f749257a9468febdbe0dc9f7e8", + 225: ("0024122bf020fa113f6c0ac978dfbd41f749257a9468febdbe0dc9f7e8", "042cc7335f4b76042bed44ef45959a62aa215f7a5ff0c8111b8c44ed654ee71c1918326ad485b2d599fe2a6eab096ee26d977334d2bac6d61d"), 256: ("7fa06fa02d0e911b9a47fdc17d2d962ca01e2f31d60c6212d0ed7e3bba23a7b9", "045c39154579efd667adc73a81015a797d2c8682cdfbd3c3553c4a185d481cdc50e42a0e1cbc3ca29a32a645e927f54beaed14c9dbbf8279d725f5495ca924b24d"), }, 'ECC(PSA_ECC_FAMILY_SECP_R1)': { - 225: ("872f203b3ad35b7f2ecc803c3a0e1e0b1ed61cc1afe71b189cd4c995", + 224: ("872f203b3ad35b7f2ecc803c3a0e1e0b1ed61cc1afe71b189cd4c995", "046f00eadaa949fee3e9e1c7fa1247eecec86a0dce46418b9bd3117b981d4bd0ae7a990de912f9d060d6cb531a42d22e394ac29e81804bf160"), 256: ("49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee", "047772656f814b399279d5e1f1781fac6f099a3c5ca1b0e35351834b08b65e0b572590cdaf8f769361bcf34acfc11e5e074e8426bdde04be6e653945449617de45"), diff --git a/scripts/framework_dev/crypto_knowledge.py b/scripts/framework_dev/crypto_knowledge.py index 285d6c638..ebfd55cdb 100644 --- a/scripts/framework_dev/crypto_knowledge.py +++ b/scripts/framework_dev/crypto_knowledge.py @@ -131,8 +131,8 @@ class KeyType: 'PSA_DH_FAMILY_RFC7919': (2048, 3072, 4096, 6144, 8192), } # type: Dict[str, Tuple[int, ...]] ECC_KEY_SIZES = { - 'PSA_ECC_FAMILY_SECP_K1': (192, 224, 256), - 'PSA_ECC_FAMILY_SECP_R1': (225, 256, 384, 521), + 'PSA_ECC_FAMILY_SECP_K1': (192, 225, 256), + 'PSA_ECC_FAMILY_SECP_R1': (224, 256, 384, 521), 'PSA_ECC_FAMILY_SECP_R2': (160,), 'PSA_ECC_FAMILY_SECT_K1': (163, 233, 239, 283, 409, 571), 'PSA_ECC_FAMILY_SECT_R1': (163, 233, 283, 409, 571), From cee6a0669db86c09d0b12688820db32732b3d08f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 3 Jan 2024 20:57:52 +0100 Subject: [PATCH 481/490] Add test data for secp192r1 Same generation methodology as b68a3588ac108799ece69faea399598c082e227c: ``` openssl genpkey -algorithm ec -pkeyopt ec_paramgen_curve:P-192 -text |perl -0777 -pe 's/.*\npriv:([\n 0-9a-f:]*)pub:([\n 0-9a-f:]*).*/"$1","$2"/s or die; y/\n ://d; s/,/,\n /;' ``` Signed-off-by: Gilles Peskine --- scripts/framework_dev/asymmetric_key_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/framework_dev/asymmetric_key_data.py b/scripts/framework_dev/asymmetric_key_data.py index 29d95d0e1..8ca675878 100644 --- a/scripts/framework_dev/asymmetric_key_data.py +++ b/scripts/framework_dev/asymmetric_key_data.py @@ -47,6 +47,8 @@ ASYMMETRIC_KEY_DATA = construct_asymmetric_key_data({ "045c39154579efd667adc73a81015a797d2c8682cdfbd3c3553c4a185d481cdc50e42a0e1cbc3ca29a32a645e927f54beaed14c9dbbf8279d725f5495ca924b24d"), }, 'ECC(PSA_ECC_FAMILY_SECP_R1)': { + 192: ("d83b57a59c51358d9c8bbb898aff507f44dd14cf16917190", + "04e35fcbee11cec3154f80a1a61df7d7612de4f2fd70c5608d0ee3a4a1a5719471adb33966dd9b035fdb774feeba94b04c"), 224: ("872f203b3ad35b7f2ecc803c3a0e1e0b1ed61cc1afe71b189cd4c995", "046f00eadaa949fee3e9e1c7fa1247eecec86a0dce46418b9bd3117b981d4bd0ae7a990de912f9d060d6cb531a42d22e394ac29e81804bf160"), 256: ("49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee", From 903f3aa2cb5d5beb754dd01d1a567af9f2ed7a6a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 22 Nov 2023 19:24:31 +0100 Subject: [PATCH 482/490] Python module to parse function declarations from a header file Signed-off-by: Gilles Peskine --- scripts/framework_dev/c_parsing_helper.py | 127 ++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 scripts/framework_dev/c_parsing_helper.py diff --git a/scripts/framework_dev/c_parsing_helper.py b/scripts/framework_dev/c_parsing_helper.py new file mode 100644 index 000000000..3bb6f0405 --- /dev/null +++ b/scripts/framework_dev/c_parsing_helper.py @@ -0,0 +1,127 @@ +"""Helper functions to parse C code in heavily constrained scenarios. + +Currently supported functionality: + +* read_function_declarations: read function declarations from a header file. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +import re +from typing import Dict, Iterable, Iterator, List, Optional, Tuple + + +class ArgumentInfo: + """Information about an argument to an API function.""" + #pylint: disable=too-few-public-methods + + _KEYWORDS = [ + 'const', 'register', 'restrict', + 'int', 'long', 'short', 'signed', 'unsigned', + ] + _DECLARATION_RE = re.compile( + r'(?P\w[\w\s*]*?)\s*' + + r'(?!(?:' + r'|'.join(_KEYWORDS) + r'))(?P\b\w+\b)?' + + r'\s*(?P\[[^][]*\])?\Z', + re.A | re.S) + + @classmethod + def normalize_type(cls, typ: str) -> str: + """Normalize whitespace in a type.""" + typ = re.sub(r'\s+', r' ', typ) + typ = re.sub(r'\s*\*', r' *', typ) + return typ + + def __init__(self, decl: str) -> None: + self.decl = decl.strip() + m = self._DECLARATION_RE.match(self.decl) + if not m: + raise ValueError(self.decl) + self.type = self.normalize_type(m.group('type')) #type: str + self.name = m.group('name') #type: Optional[str] + self.suffix = m.group('suffix') if m.group('suffix') else '' #type: str + + +class FunctionInfo: + """Information about an API function.""" + #pylint: disable=too-few-public-methods + + # Regex matching the declaration of a function that returns void. + VOID_RE = re.compile(r'\s*\bvoid\s*\Z', re.A) + + def __init__(self, #pylint: disable=too-many-arguments + filename: str, + line_number: int, + qualifiers: Iterable[str], + return_type: str, + name: str, + arguments: List[str]) -> None: + self.filename = filename + self.line_number = line_number + self.qualifiers = frozenset(qualifiers) + self.return_type = return_type + self.name = name + self.arguments = [ArgumentInfo(arg) for arg in arguments] + + def returns_void(self) -> bool: + """Whether the function returns void.""" + return bool(self.VOID_RE.search(self.return_type)) + + +# Match one C comment. +# Note that we match both comment types, so things like // in a /*...*/ +# comment are handled correctly. +_C_COMMENT_RE = re.compile(r'//[^n]*|/\*.*?\*/', re.S) +_NOT_NEWLINES_RE = re.compile(r'[^\n]+') + +def read_logical_lines(filename: str) -> Iterator[Tuple[int, str]]: + """Read logical lines from a file. + + Logical lines are one or more physical line, with balanced parentheses. + """ + with open(filename, encoding='utf-8') as inp: + content = inp.read() + # Strip comments, but keep newlines for line numbering + content = re.sub(_C_COMMENT_RE, + lambda m: re.sub(_NOT_NEWLINES_RE, "", m.group(0)), + content) + lines = enumerate(content.splitlines(), 1) + for line_number, line in lines: + # Read a logical line, containing balanced parentheses. + # We assume that parentheses are balanced (this should be ok + # since comments have been stripped), otherwise there will be + # a gigantic logical line at the end. + paren_level = line.count('(') - line.count(')') + while paren_level > 0: + _, more = next(lines) #pylint: disable=stop-iteration-return + paren_level += more.count('(') - more.count(')') + line += '\n' + more + yield line_number, line + +_C_FUNCTION_DECLARATION_RE = re.compile( + r'(?P(?:(?:extern|inline|static)\b\s*)*)' + r'(?P\w[\w\s*]*?)\s*' + + r'\b(?P\w+)' + + r'\s*\((?P.*)\)\s*;', + re.A | re.S) + +def read_function_declarations(functions: Dict[str, FunctionInfo], + filename: str) -> None: + """Collect function declarations from a C header file.""" + for line_number, line in read_logical_lines(filename): + m = _C_FUNCTION_DECLARATION_RE.match(line) + if not m: + continue + qualifiers = m.group('qualifiers').split() + return_type = m.group('return_type') + name = m.group('name') + arguments = m.group('arguments').split(',') + if len(arguments) == 1 and re.match(FunctionInfo.VOID_RE, arguments[0]): + arguments = [] + # Note: we replace any existing declaration for the same name. + functions[name] = FunctionInfo(filename, line_number, + qualifiers, + return_type, + name, + arguments) From 3621060e3ed18808bfcc45818187e48ed03e4c0d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 22 Nov 2023 19:24:59 +0100 Subject: [PATCH 483/490] C function wrapper generator The Base class generates trivial wrappers that just call the underlying function. It is meant as a base class to construct useful wrapper generators. The Logging class generates wrappers that can log the inputs and outputs to a function. Signed-off-by: Gilles Peskine --- scripts/framework_dev/c_wrapper_generator.py | 459 +++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 scripts/framework_dev/c_wrapper_generator.py diff --git a/scripts/framework_dev/c_wrapper_generator.py b/scripts/framework_dev/c_wrapper_generator.py new file mode 100644 index 000000000..26da9b2f2 --- /dev/null +++ b/scripts/framework_dev/c_wrapper_generator.py @@ -0,0 +1,459 @@ +"""Generate C wrapper functions. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +import os +import re +import sys +import typing +from typing import Dict, List, Optional, Tuple + +from .c_parsing_helper import ArgumentInfo, FunctionInfo +from . import typing_util + + +def c_declare(prefix: str, name: str, suffix: str) -> str: + """Format a declaration of name with the given type prefix and suffix.""" + if not prefix.endswith('*'): + prefix += ' ' + return prefix + name + suffix + + +WrapperInfo = typing.NamedTuple('WrapperInfo', [ + ('argument_names', List[str]), + ('guard', Optional[str]), + ('wrapper_name', str), +]) + + +class Base: + """Generate a C source file containing wrapper functions.""" + + # This class is designed to have many methods potentially overloaded. + # Tell pylint not to complain about methods that have unused arguments: + # child classes are likely to override those methods and need the + # arguments in question. + #pylint: disable=no-self-use,unused-argument + + # Prefix prepended to the function's name to form the wrapper name. + _WRAPPER_NAME_PREFIX = '' + # Suffix appended to the function's name to form the wrapper name. + _WRAPPER_NAME_SUFFIX = '_wrap' + + # Functions with one of these qualifiers are skipped. + _SKIP_FUNCTION_WITH_QUALIFIERS = frozenset(['inline', 'static']) + + def __init__(self): + """Construct a wrapper generator object. + """ + self.program_name = os.path.basename(sys.argv[0]) + # To be populated in a derived class + self.functions = {} #type: Dict[str, FunctionInfo] + # Preprocessor symbol used as a guard against multiple inclusion in the + # header. Must be set before writing output to a header. + # Not used when writing .c output. + self.header_guard = None #type: Optional[str] + + def _write_prologue(self, out: typing_util.Writable, header: bool) -> None: + """Write the prologue of a C file. + + This includes a description comment and some include directives. + """ + out.write("""/* Automatically generated by {}, do not edit! */ + +/* Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ +""" + .format(self.program_name)) + if header: + out.write(""" +#ifndef {guard} +#define {guard} + +#ifdef __cplusplus +extern "C" {{ +#endif +""" + .format(guard=self.header_guard)) + out.write(""" +#include +""") + + def _write_epilogue(self, out: typing_util.Writable, header: bool) -> None: + """Write the epilogue of a C file. + """ + if header: + out.write(""" +#ifdef __cplusplus +}} +#endif + +#endif /* {guard} */ +""" + .format(guard=self.header_guard)) + out.write(""" +/* End of automatically generated file. */ +""") + + def _wrapper_function_name(self, original_name: str) -> str: + """The name of the wrapper function. + + By default, this adds a suffix. + """ + return (self._WRAPPER_NAME_PREFIX + + original_name + + self._WRAPPER_NAME_SUFFIX) + + def _wrapper_declaration_start(self, + function: FunctionInfo, + wrapper_name: str) -> str: + """The beginning of the wrapper function declaration. + + This ends just before the opening parenthesis of the argument list. + + This is a string containing at least the return type and the + function name. It may start with additional qualifiers or attributes + such as `static`, `__attribute__((...))`, etc. + """ + return c_declare(function.return_type, wrapper_name, '') + + def _argument_name(self, + function_name: str, + num: int, + arg: ArgumentInfo) -> str: + """Name to use for the given argument in the wrapper function. + + Argument numbers count from 0. + """ + name = 'arg' + str(num) + if arg.name: + name += '_' + arg.name + return name + + def _wrapper_declaration_argument(self, + function_name: str, + num: int, name: str, + arg: ArgumentInfo) -> str: + """One argument definition in the wrapper function declaration. + + Argument numbers count from 0. + """ + return c_declare(arg.type, name, arg.suffix) + + def _underlying_function_name(self, function: FunctionInfo) -> str: + """The name of the underlying function. + + By default, this is the name of the wrapped function. + """ + return function.name + + def _return_variable_name(self, function: FunctionInfo) -> str: + """The name of the variable that will contain the return value.""" + return 'retval' + + def _write_function_call(self, out: typing_util.Writable, + function: FunctionInfo, + argument_names: List[str]) -> None: + """Write the call to the underlying function. + """ + # Note that the function name is in parentheses, to avoid calling + # a function-like macro with the same name, since in typical usage + # there is a function-like macro with the same name which is the + # wrapper. + call = '({})({})'.format(self._underlying_function_name(function), + ', '.join(argument_names)) + if function.returns_void(): + out.write(' {};\n'.format(call)) + else: + ret_name = self._return_variable_name(function) + ret_decl = c_declare(function.return_type, ret_name, '') + out.write(' {} = {};\n'.format(ret_decl, call)) + + def _write_function_return(self, out: typing_util.Writable, + function: FunctionInfo, + if_void: bool = False) -> None: + """Write a return statement. + + If the function returns void, only write a statement if if_void is true. + """ + if function.returns_void(): + if if_void: + out.write(' return;\n') + else: + ret_name = self._return_variable_name(function) + out.write(' return {};\n'.format(ret_name)) + + def _write_function_body(self, out: typing_util.Writable, + function: FunctionInfo, + argument_names: List[str]) -> None: + """Write the body of the wrapper code for the specified function. + """ + self._write_function_call(out, function, argument_names) + self._write_function_return(out, function) + + def _skip_function(self, function: FunctionInfo) -> bool: + """Whether to skip this function. + + By default, static or inline functions are skipped. + """ + if not self._SKIP_FUNCTION_WITH_QUALIFIERS.isdisjoint(function.qualifiers): + return True + return False + + _FUNCTION_GUARDS = { + } #type: Dict[str, str] + + def _function_guard(self, function: FunctionInfo) -> Optional[str]: + """A preprocessor condition for this function. + + The wrapper will be guarded with `#if` on this condition, if not None. + """ + return self._FUNCTION_GUARDS.get(function.name) + + def _wrapper_info(self, function: FunctionInfo) -> Optional[WrapperInfo]: + """Information about the wrapper for one function. + + Return None if the function should be skipped. + """ + if self._skip_function(function): + return None + argument_names = [self._argument_name(function.name, num, arg) + for num, arg in enumerate(function.arguments)] + return WrapperInfo( + argument_names=argument_names, + guard=self._function_guard(function), + wrapper_name=self._wrapper_function_name(function.name), + ) + + def _write_function_prototype(self, out: typing_util.Writable, + function: FunctionInfo, + wrapper: WrapperInfo, + header: bool) -> None: + """Write the prototype of a wrapper function. + + If header is true, write a function declaration, with a semicolon at + the end. Otherwise just write the prototype, intended to be followed + by the function's body. + """ + declaration_start = self._wrapper_declaration_start(function, + wrapper.wrapper_name) + arg_indent = ' ' + terminator = ';\n' if header else '\n' + if function.arguments: + out.write(declaration_start + '(\n') + for num in range(len(function.arguments)): + arg_def = self._wrapper_declaration_argument( + function.name, + num, wrapper.argument_names[num], function.arguments[num]) + arg_terminator = \ + (')' + terminator if num == len(function.arguments) - 1 else + ',\n') + out.write(arg_indent + arg_def + arg_terminator) + else: + out.write(declaration_start + '(void)' + terminator) + + def _write_c_function(self, out: typing_util.Writable, + function: FunctionInfo) -> None: + """Write wrapper code for one function. + + Do nothing if the function is skipped. + """ + wrapper = self._wrapper_info(function) + if wrapper is None: + return + out.write(""" +/* Wrapper for {} */ +""" + .format(function.name)) + if wrapper.guard is not None: + out.write('#if {}\n'.format(wrapper.guard)) + self._write_function_prototype(out, function, wrapper, False) + out.write('{\n') + self._write_function_body(out, function, wrapper.argument_names) + out.write('}\n') + if wrapper.guard is not None: + out.write('#endif /* {} */\n'.format(wrapper.guard)) + + def _write_h_function(self, out: typing_util.Writable, + function: FunctionInfo, + wrapper: WrapperInfo) -> None: + """Write the declaration of one wrapper function. + """ + out.write('\n') + if wrapper.guard is not None: + out.write('#if {}\n'.format(wrapper.guard)) + self._write_function_prototype(out, function, wrapper, True) + if wrapper.guard is not None: + out.write('#endif /* {} */\n'.format(wrapper.guard)) + + def _write_h_macro(self, out: typing_util.Writable, + function: FunctionInfo, + wrapper: WrapperInfo) -> None: + """Write the macro definition for one wrapper. + """ + arg_list = ', '.join(wrapper.argument_names) + out.write('#define {function_name}({args}) \\\n {wrapper_name}({args})\n' + .format(function_name=function.name, + wrapper_name=wrapper.wrapper_name, + args=arg_list)) + + def write_c_file(self, filename: str) -> None: + """Output a whole C file containing function wrapper definitions.""" + with open(filename, 'w', encoding='utf-8') as out: + self._write_prologue(out, False) + for name in sorted(self.functions): + self._write_c_function(out, self.functions[name]) + self._write_epilogue(out, False) + + def _header_guard_from_file_name(self, filename: str) -> str: + """Preprocessor symbol used as a guard against multiple inclusion.""" + # Heuristic to strip irrelevant leading directories + filename = re.sub(r'.*include[\\/]', r'', filename) + return re.sub(r'[^0-9A-Za-z]', r'_', filename, re.A).upper() + + def write_h_file(self, filename: str) -> None: + """Output a header file with function wrapper declarations and macro definitions.""" + self.header_guard = self._header_guard_from_file_name(filename) + with open(filename, 'w', encoding='utf-8') as out: + self._write_prologue(out, True) + for name in sorted(self.functions.keys()): + function = self.functions[name] + wrapper = self._wrapper_info(function) + if wrapper is None: + continue + self._write_h_function(out, function, wrapper) + self._write_h_macro(out, function, wrapper) + self._write_epilogue(out, True) + + +class UnknownTypeForPrintf(Exception): + """Exception raised when attempting to generate code that logs a value of an unknown type.""" + + def __init__(self, typ: str) -> None: + super().__init__("Unknown type for printf format generation: " + typ) + + +class Logging(Base): + """Generate wrapper functions that log the inputs and outputs.""" + + def __init__(self) -> None: + """Construct a wrapper generator including logging of inputs and outputs. + + Log to stdout by default. Call `set_stream` to change this. + """ + super().__init__() + self.stream = 'stdout' + + def set_stream(self, stream: str) -> None: + """Set the stdio stream to log to. + + Call this method before calling `write_c_output` or `write_h_output`. + """ + self.stream = stream + + def _write_prologue(self, out: typing_util.Writable, header: bool) -> None: + super()._write_prologue(out, header) + if not header: + out.write(""" +#if defined(MBEDTLS_FS_IO) && defined(MBEDTLS_TEST_HOOKS) +#include +#include +#include // for MBEDTLS_PRINTF_SIZET +#include // for mbedtls_fprintf +#endif /* defined(MBEDTLS_FS_IO) && defined(MBEDTLS_TEST_HOOKS) */ +""") + + _PRINTF_SIMPLE_FORMAT = { + 'int': '%d', + 'long': '%ld', + 'long long': '%lld', + 'size_t': '%"MBEDTLS_PRINTF_SIZET"', + 'unsigned': '0x%08x', + 'unsigned int': '0x%08x', + 'unsigned long': '0x%08lx', + 'unsigned long long': '0x%016llx', + } + + def _printf_simple_format(self, typ: str) -> Optional[str]: + """Use this printf format for a value of typ. + + Return None if values of typ need more complex handling. + """ + return self._PRINTF_SIMPLE_FORMAT.get(typ) + + _PRINTF_TYPE_CAST = { + 'int32_t': 'int', + 'uint32_t': 'unsigned', + 'uint64_t': 'unsigned long long', + } #type: Dict[str, str] + + def _printf_type_cast(self, typ: str) -> Optional[str]: + """Cast values of typ to this type before passing them to printf. + + Return None if values of the given type do not need a cast. + """ + return self._PRINTF_TYPE_CAST.get(typ) + + _POINTER_TYPE_RE = re.compile(r'\s*\*\Z') + + def _printf_parameters(self, typ: str, var: str) -> Tuple[str, List[str]]: + """The printf format and arguments for a value of type typ stored in var. + """ + expr = var + base_type = typ + # For outputs via a pointer, get the value that has been written. + # Note: we don't support pointers to pointers here. + pointer_match = self._POINTER_TYPE_RE.search(base_type) + if pointer_match: + base_type = base_type[:pointer_match.start(0)] + expr = '*({})'.format(expr) + # Maybe cast the value to a standard type. + cast_to = self._printf_type_cast(base_type) + if cast_to is not None: + expr = '({}) {}'.format(cast_to, expr) + base_type = cast_to + # Try standard types. + fmt = self._printf_simple_format(base_type) + if fmt is not None: + return '{}={}'.format(var, fmt), [expr] + raise UnknownTypeForPrintf(typ) + + def _write_function_logging(self, out: typing_util.Writable, + function: FunctionInfo, + argument_names: List[str]) -> None: + """Write code to log the function's inputs and outputs.""" + formats, values = '%s', ['"' + function.name + '"'] + for arg_info, arg_name in zip(function.arguments, argument_names): + fmt, vals = self._printf_parameters(arg_info.type, arg_name) + if fmt: + formats += ' ' + fmt + values += vals + if not function.returns_void(): + ret_name = self._return_variable_name(function) + fmt, vals = self._printf_parameters(function.return_type, ret_name) + if fmt: + formats += ' ' + fmt + values += vals + out.write("""\ +#if defined(MBEDTLS_FS_IO) && defined(MBEDTLS_TEST_HOOKS) + if ({stream}) {{ + mbedtls_fprintf({stream}, "{formats}\\n", + {values}); + }} +#endif /* defined(MBEDTLS_FS_IO) && defined(MBEDTLS_TEST_HOOKS) */ +""" + .format(stream=self.stream, + formats=formats, + values=', '.join(values))) + + def _write_function_body(self, out: typing_util.Writable, + function: FunctionInfo, + argument_names: List[str]) -> None: + """Write the body of the wrapper code for the specified function. + """ + self._write_function_call(out, function, argument_names) + self._write_function_logging(out, function, argument_names) + self._write_function_return(out, function) From 3e623730316c396674686454df13a6705d29731c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 23 Nov 2023 14:12:29 +0100 Subject: [PATCH 484/490] Guard the macro definition It doesn't make sense to define a macro expanding to a non-existent function. Signed-off-by: Gilles Peskine --- scripts/framework_dev/c_wrapper_generator.py | 46 ++++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/scripts/framework_dev/c_wrapper_generator.py b/scripts/framework_dev/c_wrapper_generator.py index 26da9b2f2..de99ddb9a 100644 --- a/scripts/framework_dev/c_wrapper_generator.py +++ b/scripts/framework_dev/c_wrapper_generator.py @@ -277,21 +277,16 @@ extern "C" {{ if wrapper.guard is not None: out.write('#endif /* {} */\n'.format(wrapper.guard)) - def _write_h_function(self, out: typing_util.Writable, - function: FunctionInfo, - wrapper: WrapperInfo) -> None: + def _write_h_function_declaration(self, out: typing_util.Writable, + function: FunctionInfo, + wrapper: WrapperInfo) -> None: """Write the declaration of one wrapper function. """ - out.write('\n') - if wrapper.guard is not None: - out.write('#if {}\n'.format(wrapper.guard)) self._write_function_prototype(out, function, wrapper, True) - if wrapper.guard is not None: - out.write('#endif /* {} */\n'.format(wrapper.guard)) - def _write_h_macro(self, out: typing_util.Writable, - function: FunctionInfo, - wrapper: WrapperInfo) -> None: + def _write_h_macro_definition(self, out: typing_util.Writable, + function: FunctionInfo, + wrapper: WrapperInfo) -> None: """Write the macro definition for one wrapper. """ arg_list = ', '.join(wrapper.argument_names) @@ -300,6 +295,26 @@ extern "C" {{ wrapper_name=wrapper.wrapper_name, args=arg_list)) + def _write_h_function(self, out: typing_util.Writable, + function: FunctionInfo) -> None: + """Write the complete header content for one wrapper. + + This is the declaration of the wrapper function, and the + definition of a function-like macro that calls the wrapper function. + + Do nothing if the function is skipped. + """ + wrapper = self._wrapper_info(function) + if wrapper is None: + return + out.write('\n') + if wrapper.guard is not None: + out.write('#if {}\n'.format(wrapper.guard)) + self._write_h_function_declaration(out, function, wrapper) + self._write_h_macro_definition(out, function, wrapper) + if wrapper.guard is not None: + out.write('#endif /* {} */\n'.format(wrapper.guard)) + def write_c_file(self, filename: str) -> None: """Output a whole C file containing function wrapper definitions.""" with open(filename, 'w', encoding='utf-8') as out: @@ -319,13 +334,8 @@ extern "C" {{ self.header_guard = self._header_guard_from_file_name(filename) with open(filename, 'w', encoding='utf-8') as out: self._write_prologue(out, True) - for name in sorted(self.functions.keys()): - function = self.functions[name] - wrapper = self._wrapper_info(function) - if wrapper is None: - continue - self._write_h_function(out, function, wrapper) - self._write_h_macro(out, function, wrapper) + for name in sorted(self.functions): + self._write_h_function(out, self.functions[name]) self._write_epilogue(out, True) From c4bf74c72146ebc91f9e6df919364a623fe2005f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 4 Jan 2024 17:28:59 +0100 Subject: [PATCH 485/490] Add review exception warning Signed-off-by: Gilles Peskine --- scripts/framework_dev/c_parsing_helper.py | 4 ++++ scripts/framework_dev/c_wrapper_generator.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/scripts/framework_dev/c_parsing_helper.py b/scripts/framework_dev/c_parsing_helper.py index 3bb6f0405..72214d51c 100644 --- a/scripts/framework_dev/c_parsing_helper.py +++ b/scripts/framework_dev/c_parsing_helper.py @@ -8,6 +8,10 @@ Currently supported functionality: # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +### WARNING: the code in this file has not been extensively reviewed yet. +### We do not think it is harmful, but it may be below our normal standards +### for robustness and maintainability. + import re from typing import Dict, Iterable, Iterator, List, Optional, Tuple diff --git a/scripts/framework_dev/c_wrapper_generator.py b/scripts/framework_dev/c_wrapper_generator.py index de99ddb9a..3cf1e05eb 100644 --- a/scripts/framework_dev/c_wrapper_generator.py +++ b/scripts/framework_dev/c_wrapper_generator.py @@ -4,6 +4,10 @@ # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later +### WARNING: the code in this file has not been extensively reviewed yet. +### We do not think it is harmful, but it may be below our normal standards +### for robustness and maintainability. + import os import re import sys From d4d4fb628f96eb2659938a5d141ae8dc2cc77813 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 8 Jan 2024 21:05:42 +0100 Subject: [PATCH 486/490] Fix parsing of C line comments Fix // comments stopping on 'n' instead of newlines. Also allow backslash-newline in // comments. Signed-off-by: Gilles Peskine --- scripts/framework_dev/c_parsing_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/c_parsing_helper.py b/scripts/framework_dev/c_parsing_helper.py index 72214d51c..2657b7d23 100644 --- a/scripts/framework_dev/c_parsing_helper.py +++ b/scripts/framework_dev/c_parsing_helper.py @@ -76,7 +76,7 @@ class FunctionInfo: # Match one C comment. # Note that we match both comment types, so things like // in a /*...*/ # comment are handled correctly. -_C_COMMENT_RE = re.compile(r'//[^n]*|/\*.*?\*/', re.S) +_C_COMMENT_RE = re.compile(r'//(?:[^\n]|\\\n)*|/\*.*?\*/', re.S) _NOT_NEWLINES_RE = re.compile(r'[^\n]+') def read_logical_lines(filename: str) -> Iterator[Tuple[int, str]]: From 00a325d7c13e729b4eb88a614644099694013f85 Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Wed, 17 Jan 2024 15:22:47 +0100 Subject: [PATCH 487/490] tests: add guards for DH groups Signed-off-by: Valerio Setti --- scripts/framework_dev/psa_information.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_dev/psa_information.py b/scripts/framework_dev/psa_information.py index b21a0cfc2..60803864f 100644 --- a/scripts/framework_dev/psa_information.py +++ b/scripts/framework_dev/psa_information.py @@ -82,7 +82,7 @@ def automatic_dependencies(*expressions: str) -> List[str]: """ used = set() for expr in expressions: - used.update(re.findall(r'PSA_(?:ALG|ECC_FAMILY|KEY_TYPE)_\w+', expr)) + used.update(re.findall(r'PSA_(?:ALG|ECC_FAMILY|DH_FAMILY|KEY_TYPE)_\w+', expr)) used.difference_update(SYMBOLS_WITHOUT_DEPENDENCY) return sorted(psa_want_symbol(name) for name in used) From aeb4fb50229d010eda02af238dccef12dad9f45d Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Mon, 18 Mar 2024 11:15:06 +0000 Subject: [PATCH 488/490] Check file content to see if it looks auto-generated Signed-off-by: Dave Rodgman --- scripts/code_style.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 08ec4af42..71e3edf03 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -51,6 +51,12 @@ def list_generated_files() -> FrozenSet[str]: checks = re.findall(CHECK_CALL_RE, content) return frozenset(word for s in checks for word in s.split()) +# Check for comment string indicating an auto-generated file +AUTOGEN_RE = re.compile(r"Warning[ :-]+This file is (now )?auto[ -]generated", re.ASCII | re.IGNORECASE) +def is_file_autogenerated(filename): + content = open(filename, encoding="utf-8").read() + return AUTOGEN_RE.search(content) is not None + def get_src_files(since: Optional[str]) -> List[str]: """ Use git to get a list of the source files. @@ -85,7 +91,8 @@ def get_src_files(since: Optional[str]) -> List[str]: # generated files (we're correcting the templates instead). src_files = [filename for filename in src_files if not (filename.startswith("3rdparty/") or - filename in generated_files)] + filename in generated_files or + is_file_autogenerated(filename))] return src_files def get_uncrustify_version() -> str: From d2f82ca837092bf1d2e9c342745f6df4638baacb Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Mon, 18 Mar 2024 11:55:39 +0000 Subject: [PATCH 489/490] line length fix Signed-off-by: Dave Rodgman --- scripts/code_style.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index 71e3edf03..b12198d38 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -52,7 +52,8 @@ def list_generated_files() -> FrozenSet[str]: return frozenset(word for s in checks for word in s.split()) # Check for comment string indicating an auto-generated file -AUTOGEN_RE = re.compile(r"Warning[ :-]+This file is (now )?auto[ -]generated", re.ASCII | re.IGNORECASE) +AUTOGEN_RE = re.compile(r"Warning[ :-]+This file is (now )?auto[ -]generated", + re.ASCII | re.IGNORECASE) def is_file_autogenerated(filename): content = open(filename, encoding="utf-8").read() return AUTOGEN_RE.search(content) is not None From d9f386c93cab2caadf154bb06c0065608a1ac0b1 Mon Sep 17 00:00:00 2001 From: Dave Rodgman Date: Mon, 18 Mar 2024 12:32:49 +0000 Subject: [PATCH 490/490] Minor relaxation to auto-gen regex Signed-off-by: Dave Rodgman --- scripts/code_style.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/code_style.py b/scripts/code_style.py index b12198d38..07952b6cb 100755 --- a/scripts/code_style.py +++ b/scripts/code_style.py @@ -52,7 +52,7 @@ def list_generated_files() -> FrozenSet[str]: return frozenset(word for s in checks for word in s.split()) # Check for comment string indicating an auto-generated file -AUTOGEN_RE = re.compile(r"Warning[ :-]+This file is (now )?auto[ -]generated", +AUTOGEN_RE = re.compile(r"Warning[ :-]+This file is (now )?auto[ -]?generated", re.ASCII | re.IGNORECASE) def is_file_autogenerated(filename): content = open(filename, encoding="utf-8").read()