From 8c23ac8520befe08a057351424502e2152adca8f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Dec 2024 17:56:05 +0100 Subject: [PATCH 1/9] Be more explicit about key pair usage dependencies Make the code that generates the test case be explicit about which usage(s) will be needed for key pairs (`PSA_WANT_KEY_TYPE_xxx_KEY_PAIR_uuu`). Allow more than one usage specifier. Do not systematically generalize BASIC to also include IMPORT and EXPORT: not all tests actually need this, and our test configurations don't try to have BASIC without IMPORT and EXPORT at the moment because we don't track those dependencies accurately in manually written tests anyway. Fix a bug whereby any usage other than BASIC or GENERATE led to the dependency being silently dropped. No change to the generated output. Signed-off-by: Gilles Peskine --- scripts/generate_psa_tests.py | 15 ++++++++------ scripts/mbedtls_framework/psa_information.py | 21 +++++--------------- scripts/mbedtls_framework/psa_test_case.py | 6 +++--- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/scripts/generate_psa_tests.py b/scripts/generate_psa_tests.py index 40c701217..46b40e52a 100755 --- a/scripts/generate_psa_tests.py +++ b/scripts/generate_psa_tests.py @@ -42,7 +42,7 @@ def test_case_for_key_type_not_supported( .format(verb, short_key_type, bits, adverb)) tc.set_function(verb + '_not_supported') tc.set_key_bits(bits) - tc.set_key_pair_usage(verb.upper()) + tc.set_key_pair_usage([verb.upper()]) tc.set_arguments([key_type] + list(args)) tc.set_dependencies(dependencies) tc.skip_if_any_not_implemented(dependencies) @@ -87,9 +87,11 @@ class KeyTypeNotSupported: generate_dependencies = [] else: generate_dependencies = \ - psa_information.fix_key_pair_dependencies(import_dependencies, 'GENERATE') + psa_information.fix_key_pair_dependencies(import_dependencies, + ['GENERATE']) import_dependencies = \ - psa_information.fix_key_pair_dependencies(import_dependencies, 'BASIC') + psa_information.fix_key_pair_dependencies(import_dependencies, + ['BASIC', 'IMPORT', 'EXPORT']) for bits in kt.sizes_to_test(): yield test_case_for_key_type_not_supported( 'import', kt.expression, bits, @@ -155,7 +157,7 @@ def test_case_for_key_generation( .format(short_key_type, bits)) tc.set_function('generate_key') tc.set_key_bits(bits) - tc.set_key_pair_usage('GENERATE') + tc.set_key_pair_usage(['GENERATE']) tc.set_arguments([key_type] + list(args) + [result]) return tc @@ -258,7 +260,8 @@ class OpFail: pretty_reason, ' with ' + pretty_type if pretty_type else '')) dependencies = psa_information.automatic_dependencies(alg.base_expression, key_type) - dependencies = psa_information.fix_key_pair_dependencies(dependencies, 'BASIC') + dependencies = psa_information.fix_key_pair_dependencies(dependencies, + ['BASIC', 'IMPORT', 'EXPORT']) for i, dep in enumerate(dependencies): if dep in not_deps: dependencies[i] = '!' + dep @@ -481,7 +484,7 @@ class StorageFormat: tc.add_dependencies(psa_information.generate_deps_from_description(key.description)) tc.set_function('key_storage_' + verb) tc.set_key_bits(key.bits) - tc.set_key_pair_usage('BASIC') + tc.set_key_pair_usage(['BASIC', 'EXPORT', 'IMPORT']) if self.forward: extra_arguments = [] else: diff --git a/scripts/mbedtls_framework/psa_information.py b/scripts/mbedtls_framework/psa_information.py index 1ff02da61..015dfe388 100644 --- a/scripts/mbedtls_framework/psa_information.py +++ b/scripts/mbedtls_framework/psa_information.py @@ -139,29 +139,18 @@ def generate_deps_from_description( return dep_list -def tweak_key_pair_dependency(dep: str, usage: str): +def tweak_key_pair_dependency(dep: str, usages: List[str]) -> List[str]: """ This helper function add the proper suffix to PSA_WANT_KEY_TYPE_xxx_KEY_PAIR symbols according to the required usage. """ - ret_list = list() if dep.endswith('KEY_PAIR'): - if usage == "BASIC": - # BASIC automatically includes IMPORT and EXPORT for test purposes (see - # config_psa.h). - ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_BASIC', dep)) - ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_IMPORT', dep)) - ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_EXPORT', dep)) - elif usage == "GENERATE": - ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_GENERATE', dep)) - else: - # No replacement to do in this case - ret_list.append(dep) - return ret_list + return [dep + '_' + usage for usage in usages] + return [dep] -def fix_key_pair_dependencies(dep_list: List[str], usage: str): +def fix_key_pair_dependencies(dep_list: List[str], usages: List[str]) -> List[str]: new_list = [new_deps for dep in dep_list - for new_deps in tweak_key_pair_dependency(dep, usage)] + for new_deps in tweak_key_pair_dependency(dep, usages)] return new_list diff --git a/scripts/mbedtls_framework/psa_test_case.py b/scripts/mbedtls_framework/psa_test_case.py index eea969f7a..2c730a7f8 100644 --- a/scripts/mbedtls_framework/psa_test_case.py +++ b/scripts/mbedtls_framework/psa_test_case.py @@ -71,7 +71,7 @@ class TestCase(test_case.TestCase): self.automatic_dependencies = set() #type: Set[str] self.dependency_prefix = dependency_prefix #type: Optional[str] self.key_bits = None #type: Optional[int] - self.key_pair_usage = None #type: Optional[str] + self.key_pair_usage = None #type: Optional[List[str]] def set_key_bits(self, key_bits: Optional[int]) -> None: """Use the given key size for automatic dependency generation. @@ -83,8 +83,8 @@ class TestCase(test_case.TestCase): """ self.key_bits = key_bits - def set_key_pair_usage(self, key_pair_usage: Optional[str]) -> None: - """Use the given suffix for key pair dependencies. + def set_key_pair_usage(self, key_pair_usage: Optional[List[str]]) -> None: + """Use the given suffixes for key pair dependencies. Call this function before set_arguments() if relevant. From 5dcf16ad7b66a859712a8b6e46044a3e6ba34495 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Dec 2024 18:06:05 +0100 Subject: [PATCH 2/9] Be more precise about key pair usage dependencies Don't always require all of BASIC, IMPORT and EXPORT. BASIC is always implied by any of the creation methods. * `KeyTypeNotSupported`: only does an IMPORT (or GENERATE) attempt. EXPORT is not needed. This reduces dependencies in `test_suite_psa_crypto_not_supported.generated.data`. * `OpFail`: only does an IMPORT, followed by a BASIC attempt. EXPORT is not needed. This reduces dependencies in `test_suite_psa_crypto_op_fail.generated.data`. * `StorageFormat`: only does an IMPORT for save (forward compatibility) tests, and only does an EXPORT for read (backward compatibility) tests. This reduces dependencies in `test_suite_psa_crypto_storage_format.current.data` and `test_suite_psa_crypto_storage_format.v0.data` respectively. Positive test cases that create and exercise a key are still potentially missing BASIC (which is implied) and EXPORT (which isn't) for exercising the key, but this is out of scope of this commit. The generated output has fewer test case dependencies as described above, with BASIC+IMPORT+EXPORT replaced by only one of IMPORT or EXPORT. Since we never test partial support for a key type with import or export disabled, this doesn't change which test cases are executed in each tested configuration. Signed-off-by: Gilles Peskine --- scripts/generate_psa_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/generate_psa_tests.py b/scripts/generate_psa_tests.py index 46b40e52a..482c827db 100755 --- a/scripts/generate_psa_tests.py +++ b/scripts/generate_psa_tests.py @@ -91,7 +91,7 @@ class KeyTypeNotSupported: ['GENERATE']) import_dependencies = \ psa_information.fix_key_pair_dependencies(import_dependencies, - ['BASIC', 'IMPORT', 'EXPORT']) + ['IMPORT']) for bits in kt.sizes_to_test(): yield test_case_for_key_type_not_supported( 'import', kt.expression, bits, @@ -261,7 +261,7 @@ class OpFail: ' with ' + pretty_type if pretty_type else '')) dependencies = psa_information.automatic_dependencies(alg.base_expression, key_type) dependencies = psa_information.fix_key_pair_dependencies(dependencies, - ['BASIC', 'IMPORT', 'EXPORT']) + ['IMPORT']) for i, dep in enumerate(dependencies): if dep in not_deps: dependencies[i] = '!' + dep @@ -484,7 +484,7 @@ class StorageFormat: tc.add_dependencies(psa_information.generate_deps_from_description(key.description)) tc.set_function('key_storage_' + verb) tc.set_key_bits(key.bits) - tc.set_key_pair_usage(['BASIC', 'EXPORT', 'IMPORT']) + tc.set_key_pair_usage(['IMPORT'] if self.forward else ['EXPORT']) if self.forward: extra_arguments = [] else: From 91a3626ab78e8595918b52a51ba74a306c25bc5a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Dec 2024 18:58:30 +0100 Subject: [PATCH 3/9] PSA test case generation: dependency inference class: key not supported In `psa_test_case.TestCase`, add a method `assumes_not_supported` which allows using the automatic dependency calculation framework when the test case intends to run in configurations where one mechanism is not supported. Use `psa_test_case.TestCase` for not-supported test cases for key import and generation. No change to the generated output. Signed-off-by: Gilles Peskine --- scripts/generate_psa_tests.py | 49 ++++++++-------------- scripts/mbedtls_framework/psa_test_case.py | 41 ++++++++++++++++++ 2 files changed, 59 insertions(+), 31 deletions(-) diff --git a/scripts/generate_psa_tests.py b/scripts/generate_psa_tests.py index 482c827db..0f22272f9 100755 --- a/scripts/generate_psa_tests.py +++ b/scripts/generate_psa_tests.py @@ -26,7 +26,7 @@ from mbedtls_framework import test_data_generation def test_case_for_key_type_not_supported( verb: str, key_type: str, bits: int, - dependencies: List[str], + not_supported_mechanism: str, *args: str, param_descr: str = '' ) -> test_case.TestCase: @@ -35,17 +35,16 @@ def test_case_for_key_type_not_supported( """ tc = psa_test_case.TestCase() short_key_type = crypto_knowledge.short_expression(key_type) - adverb = 'not' if dependencies else 'never' - if param_descr: - adverb = param_descr + ' ' + adverb - tc.set_description('PSA {} {} {}-bit {} supported' - .format(verb, short_key_type, bits, adverb)) + tc.set_description('PSA {} {} {}-bit{} not supported' + .format(verb, short_key_type, bits, + ' ' + param_descr if param_descr else '')) + # if tc.description == 'PSA import RSA_KEY_PAIR 1024-bit not supported': + # import pdb; pdb.set_trace() tc.set_function(verb + '_not_supported') tc.set_key_bits(bits) tc.set_key_pair_usage([verb.upper()]) + tc.assumes_not_supported(not_supported_mechanism) tc.set_arguments([key_type] + list(args)) - tc.set_dependencies(dependencies) - tc.skip_if_any_not_implemented(dependencies) return tc class KeyTypeNotSupported: @@ -77,39 +76,27 @@ class KeyTypeNotSupported: # Don't generate test cases for key types that are always supported. # They would be skipped in all configurations, which is noise. return - import_dependencies = [('!' if param is None else '') + - psa_information.psa_want_symbol(kt.name)] - if kt.params is not None: - import_dependencies += [('!' if param == i else '') + - psa_information.psa_want_symbol(sym) - for i, sym in enumerate(kt.params)] - if kt.name.endswith('_PUBLIC_KEY'): - generate_dependencies = [] + if param is None: + not_supported_mechanism = kt.name else: - generate_dependencies = \ - psa_information.fix_key_pair_dependencies(import_dependencies, - ['GENERATE']) - import_dependencies = \ - psa_information.fix_key_pair_dependencies(import_dependencies, - ['IMPORT']) + assert kt.params is not None + not_supported_mechanism = kt.params[param] for bits in kt.sizes_to_test(): yield test_case_for_key_type_not_supported( 'import', kt.expression, bits, - psa_information.finish_family_dependencies(import_dependencies, bits), + not_supported_mechanism, test_case.hex_string(kt.key_material(bits)), param_descr=param_descr, ) - if not generate_dependencies and param is not None: - # If generation is impossible for this key type, rather than - # supported or not depending on implementation capabilities, - # only generate the test case once. - continue - # For public key we expect that key generation fails with - # INVALID_ARGUMENT. It is handled by KeyGenerate class. + # Don't generate not-supported test cases for key generation of + # public keys. Our implementation always returns + # PSA_ERROR_INVALID_ARGUMENT when attempting to generate a + # public key, so we cover this together with the positive cases + # in the KeyGenerate class. if not kt.is_public(): yield test_case_for_key_type_not_supported( 'generate', kt.expression, bits, - psa_information.finish_family_dependencies(generate_dependencies, bits), + not_supported_mechanism, str(bits), param_descr=param_descr, ) diff --git a/scripts/mbedtls_framework/psa_test_case.py b/scripts/mbedtls_framework/psa_test_case.py index 2c730a7f8..39b0ab37e 100644 --- a/scripts/mbedtls_framework/psa_test_case.py +++ b/scripts/mbedtls_framework/psa_test_case.py @@ -70,6 +70,7 @@ class TestCase(test_case.TestCase): self.manual_dependencies = [] #type: List[str] self.automatic_dependencies = set() #type: Set[str] self.dependency_prefix = dependency_prefix #type: Optional[str] + self.negated_dependencies = set() #type: Set[str] self.key_bits = None #type: Optional[int] self.key_pair_usage = None #type: Optional[List[str]] @@ -104,16 +105,56 @@ class TestCase(test_case.TestCase): dependencies = psa_information.fix_key_pair_dependencies(dependencies, self.key_pair_usage) if 'PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE' in dependencies and \ + 'PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE' not in self.negated_dependencies and \ self.key_bits is not None: size_dependency = ('PSA_VENDOR_RSA_GENERATE_MIN_KEY_BITS <= ' + str(self.key_bits)) dependencies.append(size_dependency) return dependencies + def assumes_not_supported(self, name: str) -> None: + """Negate the given mechanism for automatic dependency generation. + + Call this function before set_arguments() for a test case that should + run if the given mechanism is not supported. + + Call modifiers such as set_key_bits() and set_key_pair_usage() before + calling this method, if applicable. + + A mechanism is a PSA_XXX symbol, e.g. PSA_KEY_TYPE_AES, PSA_ALG_HMAC, + etc. For mechanisms like ECC curves where the support status includes + the key bit-size, this class assumes that only one bit-size is + involved in a given test case. + """ + if name == 'PSA_KEY_TYPE_RSA_KEY_PAIR' and \ + self.key_bits is not None and \ + self.key_pair_usage == ['GENERATE']: + # When RSA key pair generation is not supported, it could be + # due to the specific key size is out of range, or because + # RSA key pair generation itself is not supported. Assume the + # latter. + dep = psa_information.psa_want_symbol(name, prefix=self.dependency_prefix) + + self.negated_dependencies.add(dep + '_GENERATE') + return + dependencies = self.infer_dependencies([name]) + # * If we have more than one dependency to negate, the result would + # say that all of the dependencies are disabled, which is not + # a desirable outcome: the negation of (A and B) is (!A or !B), + # not (!A and !B). + # * If we have no dependency to negate, the result wouldn't be a + # not-supported case. + # Assert that we don't reach either such case. + assert len(dependencies) == 1 + self.negated_dependencies.add(dependencies[0]) + def set_arguments(self, arguments: List[str]) -> None: """Set test case arguments and automatically infer dependencies.""" super().set_arguments(arguments) dependencies = self.infer_dependencies(arguments) + for i in range(len(dependencies)): #pylint: disable=consider-using-enumerate + if dependencies[i] in self.negated_dependencies: + dependencies[i] = '!' + dependencies[i] self.skip_if_any_not_implemented(dependencies) self.automatic_dependencies.update(dependencies) From 96da26ec6510e124ad5208b2106bab34f0c550b5 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Dec 2024 19:22:52 +0100 Subject: [PATCH 4/9] PSA test case generation: operation fail: fix family dependencies In operation failure test cases, fix dependencies on DH or ECC groups, which were not spelled correctly and were missing the size suffix. This changes the dependencies of many test cases in `test_suite_psa_crypto_op_fail.generated.data` to no longer have a never-implemented symbol as a dependency. Thus more test cases will run. Signed-off-by: Gilles Peskine --- scripts/generate_psa_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/generate_psa_tests.py b/scripts/generate_psa_tests.py index 0f22272f9..a87ec81ad 100755 --- a/scripts/generate_psa_tests.py +++ b/scripts/generate_psa_tests.py @@ -257,6 +257,7 @@ class OpFail: if kt: bits = kt.sizes_to_test()[0] tc.set_key_bits(bits) + dependencies = psa_information.finish_family_dependencies(dependencies, bits) key_material = kt.key_material(bits) arguments += [key_type, test_case.hex_string(key_material)] arguments.append(alg.expression) From 58cf9e90b82d29566aa5af1a4ddf56ce59cb3aaf Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Dec 2024 19:38:36 +0100 Subject: [PATCH 5/9] PSA test case generation: operation fail: dependency inference class Use the automatic dependency generation mechanism from `psa_test_case.TestCase` for operation failure test cases. But tweak them explicitly to preserve the same set of (not-quite-right) dependencies, to facilitate understanding and reviewing how the current series of commits gradually changes the generated dependencies. No changes to the generated output. Signed-off-by: Gilles Peskine --- scripts/generate_psa_tests.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/generate_psa_tests.py b/scripts/generate_psa_tests.py index a87ec81ad..793596d15 100755 --- a/scripts/generate_psa_tests.py +++ b/scripts/generate_psa_tests.py @@ -246,18 +246,12 @@ class OpFail: pretty_alg, pretty_reason, ' with ' + pretty_type if pretty_type else '')) - dependencies = psa_information.automatic_dependencies(alg.base_expression, key_type) - dependencies = psa_information.fix_key_pair_dependencies(dependencies, - ['IMPORT']) - for i, dep in enumerate(dependencies): - if dep in not_deps: - dependencies[i] = '!' + dep tc.set_function(category.name.lower() + '_fail') arguments = [] # type: List[str] if kt: bits = kt.sizes_to_test()[0] tc.set_key_bits(bits) - dependencies = psa_information.finish_family_dependencies(dependencies, bits) + tc.set_key_pair_usage(['IMPORT']) key_material = kt.key_material(bits) arguments += [key_type, test_case.hex_string(key_material)] arguments.append(alg.expression) @@ -266,8 +260,17 @@ class OpFail: error = ('NOT_SUPPORTED' if reason == self.Reason.NOT_SUPPORTED else 'INVALID_ARGUMENT') arguments.append('PSA_ERROR_' + error) + if reason == self.Reason.NOT_SUPPORTED: + # It isn't nice that we access the field directly. We should + # call tc.assumes_not_supported() instead, but that requires + # further refactoring here because that method assumes a + # mechanism symbol (e.g. PSA_KEY_TYPE_xxx), not a dependency + # symbol (e.g. PSA_WANT_KEY_TYPE_xxx) like we have here. + tc.negated_dependencies.update(not_deps) tc.set_arguments(arguments) - tc.set_dependencies(dependencies) + # Temporarily preserve the former behavior where operation failure + # test cases were executed when they shouldn't have been. + tc.skip_reasons = [] return tc def no_key_test_cases( From fd0130576c0151027c299181813dc15e1e4b1d30 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Dec 2024 20:02:45 +0100 Subject: [PATCH 6/9] Do run not-supported test cases on not-implemented mechanisms In automatically generated PSA test cases with automatically inferred dependencies, we were systematically skipping test cases when a dependency mentions a mechanism that is not supported, even when that dependency is negated. Fix this. This causes more not-supported test cases to run. Signed-off-by: Gilles Peskine --- scripts/mbedtls_framework/psa_test_case.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/mbedtls_framework/psa_test_case.py b/scripts/mbedtls_framework/psa_test_case.py index 39b0ab37e..7084bada8 100644 --- a/scripts/mbedtls_framework/psa_test_case.py +++ b/scripts/mbedtls_framework/psa_test_case.py @@ -13,11 +13,14 @@ from . import psa_information from . import test_case -# A temporary hack: at the time of writing, not all dependency symbols -# are implemented yet. Skip test cases for which the dependency symbols are -# not available. Once all dependency symbols are available, this hack must -# be removed so that a bug in the dependency symbols properly leads to a test -# failure. +# Skip test cases for which the dependency symbols are not defined. +# We assume that this means that a required mechanism is not implemented. +# Note that if we erroneously skip generating test cases for +# mechanisms that are not implemented, this should be caught +# by the NOT_SUPPORTED test cases generated by generate_psa_tests.py +# in test_suite_psa_crypto_not_supported and test_suite_psa_crypto_op_fail: +# those emit tests with negative dependencies, which will not be skipped here. + def read_implemented_dependencies(acc: Set[str], filename: str) -> None: with open(filename) as input_stream: for line in input_stream: @@ -46,8 +49,8 @@ def find_dependencies_not_implemented(dependencies: List[str]) -> List[str]: _implemented_dependencies = frozenset(acc) return [dep for dep in dependencies - if (dep.lstrip('!') not in _implemented_dependencies and - dep.lstrip('!').startswith('PSA_WANT'))] + if (dep not in _implemented_dependencies and + dep.startswith('PSA_WANT'))] class TestCase(test_case.TestCase): From bc6f8ac30431b9eaf37b6c7270b24e6ceb472c28 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Dec 2024 19:46:29 +0100 Subject: [PATCH 7/9] PSA test case generation: operation fail: skip never-implemented mechanisms In `OpFail` test cases, remove the temporary hack whereby test cases were not skipped when they should be due to a mechanism being never implemented. This changes many test cases in `test_suite_psa_crypto_op_fail.generated.data` to be commented out with a "skipped because" reason instead of having a dependency on an algorithm or an ECC/DH group that is not implemented. Signed-off-by: Gilles Peskine --- scripts/generate_psa_tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/generate_psa_tests.py b/scripts/generate_psa_tests.py index 793596d15..4e165cbf0 100755 --- a/scripts/generate_psa_tests.py +++ b/scripts/generate_psa_tests.py @@ -268,9 +268,6 @@ class OpFail: # symbol (e.g. PSA_WANT_KEY_TYPE_xxx) like we have here. tc.negated_dependencies.update(not_deps) tc.set_arguments(arguments) - # Temporarily preserve the former behavior where operation failure - # test cases were executed when they shouldn't have been. - tc.skip_reasons = [] return tc def no_key_test_cases( From 78e4c8a73ba7373d6c693015dffdc283b3c7391d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Dec 2024 20:25:07 +0100 Subject: [PATCH 8/9] PSA test case generation: operation fail: simplify NOT_SUPPORTED In `generate_psa_tests.py, `OpFail.make_test_case()` is only ever used with a single mechanism being not supported. Take advantage of that to simplify parts of the function. Call `psa_test_case.TestCase.assumes_not_supported()` instead of partly reinventing that wheel. No change to the generated output. Signed-off-by: Gilles Peskine --- scripts/generate_psa_tests.py | 27 +++++++++++----------- scripts/mbedtls_framework/psa_test_case.py | 6 +++++ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/scripts/generate_psa_tests.py b/scripts/generate_psa_tests.py index 4e165cbf0..3771859ea 100755 --- a/scripts/generate_psa_tests.py +++ b/scripts/generate_psa_tests.py @@ -11,7 +11,7 @@ generate only the specified files. import enum import re import sys -from typing import Callable, Dict, FrozenSet, Iterable, Iterator, List, Optional +from typing import Callable, Dict, Iterable, Iterator, List, Optional from mbedtls_framework import crypto_data_tests from mbedtls_framework import crypto_knowledge @@ -223,16 +223,19 @@ class OpFail: category: crypto_knowledge.AlgorithmCategory, reason: 'Reason', kt: Optional[crypto_knowledge.KeyType] = None, - not_deps: FrozenSet[str] = frozenset(), + not_supported: Optional[str] = None, ) -> test_case.TestCase: - """Construct a failure test case for a one-key or keyless operation.""" + """Construct a failure test case for a one-key or keyless operation. + + If `reason` is `Reason.NOT_SUPPORTED`, pass the not-supported + dependency symbol as the `not_supported` argument. + """ #pylint: disable=too-many-arguments,too-many-locals tc = psa_test_case.TestCase() pretty_alg = alg.short_expression() if reason == self.Reason.NOT_SUPPORTED: - short_deps = [re.sub(r'PSA_WANT_ALG_', r'', dep) - for dep in not_deps] - pretty_reason = '!' + '&'.join(sorted(short_deps)) + assert not_supported is not None + pretty_reason = '!' + re.sub(r'PSA_WANT_[A-Z]+_', r'', not_supported) else: pretty_reason = reason.name.lower() if kt: @@ -261,12 +264,8 @@ class OpFail: 'INVALID_ARGUMENT') arguments.append('PSA_ERROR_' + error) if reason == self.Reason.NOT_SUPPORTED: - # It isn't nice that we access the field directly. We should - # call tc.assumes_not_supported() instead, but that requires - # further refactoring here because that method assumes a - # mechanism symbol (e.g. PSA_KEY_TYPE_xxx), not a dependency - # symbol (e.g. PSA_WANT_KEY_TYPE_xxx) like we have here. - tc.negated_dependencies.update(not_deps) + assert not_supported is not None + tc.assumes_not_supported(not_supported) tc.set_arguments(arguments) return tc @@ -281,7 +280,7 @@ class OpFail: for dep in psa_information.automatic_dependencies(alg.base_expression): yield self.make_test_case(alg, category, self.Reason.NOT_SUPPORTED, - not_deps=frozenset([dep])) + not_supported=dep) else: # Incompatible operation, supported algorithm yield self.make_test_case(alg, category, self.Reason.INVALID) @@ -299,7 +298,7 @@ class OpFail: for dep in psa_information.automatic_dependencies(alg.base_expression): yield self.make_test_case(alg, category, self.Reason.NOT_SUPPORTED, - kt=kt, not_deps=frozenset([dep])) + kt=kt, not_supported=dep) # Public key for a private-key operation if category.is_asymmetric() and kt.is_public(): yield self.make_test_case(alg, category, diff --git a/scripts/mbedtls_framework/psa_test_case.py b/scripts/mbedtls_framework/psa_test_case.py index 7084bada8..77ba31b39 100644 --- a/scripts/mbedtls_framework/psa_test_case.py +++ b/scripts/mbedtls_framework/psa_test_case.py @@ -118,6 +118,9 @@ class TestCase(test_case.TestCase): def assumes_not_supported(self, name: str) -> None: """Negate the given mechanism for automatic dependency generation. + `name` can be either a dependency symbol (``PSA_WANT_xxx``) or + a mechanism name (``PSA_KEY_TYPE_xxx``, etc.). + Call this function before set_arguments() for a test case that should run if the given mechanism is not supported. @@ -129,6 +132,9 @@ class TestCase(test_case.TestCase): the key bit-size, this class assumes that only one bit-size is involved in a given test case. """ + if name.startswith('PSA_WANT_'): + self.negated_dependencies.add(name) + return if name == 'PSA_KEY_TYPE_RSA_KEY_PAIR' and \ self.key_bits is not None and \ self.key_pair_usage == ['GENERATE']: From 814196869328accc75151a882dc3f4bd6c6765cd Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Dec 2024 22:37:57 +0100 Subject: [PATCH 9/9] Fix edge case with half-supported ECDSA: automatic test cases ECDSA has two variants: deterministic (PSA_ALG_DETERMINISTIC_ECDSA) and randomized (PSA_ALG_ECDSA). The two variants are different for signature but identical for verification. Mbed TLS accepts either variant as the algorithm parameter for verification even when only the other variant is supported, so we need to handle this as a special case when generating not-supported test cases. In this commit, suppress generated test cases for operation failures due to unsupported ECDSA when exactly one of the two ECDSA variants is supported. This edge case will only be tested manually (done in mbedtls or TF-PSA-Crypto in the commit "Fix edge case with half-supported ECDSA (manual test cases)"). Changes to the generated output: in `test_suite_psa_crypto_op_fail.generated.data`, wherever one of `!PSA_WANT_ALG_DETERMINISTIC_ECDSA` or `!PSA_WANT_ALG_ECDSA` appears as a dependency, add the other one. Signed-off-by: Gilles Peskine --- scripts/generate_psa_tests.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/generate_psa_tests.py b/scripts/generate_psa_tests.py index 3771859ea..9e628abc6 100755 --- a/scripts/generate_psa_tests.py +++ b/scripts/generate_psa_tests.py @@ -266,6 +266,21 @@ class OpFail: if reason == self.Reason.NOT_SUPPORTED: assert not_supported is not None tc.assumes_not_supported(not_supported) + # Special case: if one of deterministic/randomized + # ECDSA is supported but not the other, then the one + # that is not supported in the signature direction is + # still supported in the verification direction, + # because the two verification algorithms are + # identical. This property is how Mbed TLS chooses to + # behave, the specification would also allow it to + # reject the algorithm. In the generated test cases, + # we avoid this difficulty by not running the + # not-supported test case when exactly one of the + # two variants is supported. + if not_supported == 'PSA_WANT_ALG_ECDSA': + tc.add_dependencies(['!PSA_WANT_ALG_DETERMINISTIC_ECDSA']) + if not_supported == 'PSA_WANT_ALG_DETERMINISTIC_ECDSA': + tc.add_dependencies(['!PSA_WANT_ALG_ECDSA']) tc.set_arguments(arguments) return tc