Mbed-TLS/mbedtls-framework
1
0
Fork 0
mirror of https://github.com/Mbed-TLS/mbedtls-framework.git synced 2026-07-28 14:57:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #83 from Mbed-TLS/dev/gilles-peskine-arm/psa-storage-test-cases-never-supported-positive-framework

Browse Source 
Switch generate_psa_test.py to automatic dependencies for positive test cases
This commit is contained in:
Janos Follath
2024-12-20 08:05:31 +00:00
committed by GitHub
parent d9d14c4d30 e1f38eb599
commit 4e3d7d86ac
5 changed files with 229 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/generate_psa_tests.py
+26 -43
View File
@@ -18,6 +18,7 @@ from mbedtls_framework import crypto_knowledge
from mbedtls_framework import macro_collector #pylint: disable=unused-import
from mbedtls_framework import psa_information
from mbedtls_framework import psa_storage
from mbedtls_framework import psa_test_case
from mbedtls_framework import test_case
from mbedtls_framework import test_data_generation
@@ -32,17 +33,19 @@ def test_case_for_key_type_not_supported(
"""Return one test case exercising a key creation method
for an unsupported key type or size.
"""
psa_information.hack_dependencies_not_implemented(dependencies)
tc = test_case.TestCase()
tc = psa_test_case.TestCase()
short_key_type = crypto_knowledge.short_expression(key_type)
adverb = 'not' if dependencies else 'never'
if param_descr:
adverb = param_descr + ' ' + adverb
tc.set_description('PSA {} {} {}-bit {} supported'
.format(verb, short_key_type, bits, adverb))
tc.set_dependencies(dependencies)
tc.set_function(verb + '_not_supported')
tc.set_key_bits(bits)
tc.set_key_pair_usage(verb.upper())
tc.set_arguments([key_type] + list(args))
tc.set_dependencies(dependencies)
tc.skip_if_any_not_implemented(dependencies)
return tc
class KeyTypeNotSupported:
@@ -141,21 +144,19 @@ class KeyTypeNotSupported:
def test_case_for_key_generation(
key_type: str, bits: int,
dependencies: List[str],
*args: str,
result: str = ''
) -> test_case.TestCase:
"""Return one test case exercising a key generation.
"""
psa_information.hack_dependencies_not_implemented(dependencies)
tc = test_case.TestCase()
tc = psa_test_case.TestCase()
short_key_type = crypto_knowledge.short_expression(key_type)
tc.set_description('PSA {} {}-bit'
.format(short_key_type, bits))
tc.set_dependencies(dependencies)
tc.set_function('generate_key')
tc.set_key_bits(bits)
tc.set_key_pair_usage('GENERATE')
tc.set_arguments([key_type] + list(args) + [result])
return tc
class KeyGenerate:
@@ -178,33 +179,18 @@ class KeyGenerate:
All key types can be generated except for public keys. For public key
PSA_ERROR_INVALID_ARGUMENT status is expected.
"""
result = 'PSA_SUCCESS'
import_dependencies = [psa_information.psa_want_symbol(kt.name)]
if kt.params is not None:
import_dependencies += [psa_information.psa_want_symbol(sym)
for i, sym in enumerate(kt.params)]
if kt.name.endswith('_PUBLIC_KEY'):
# The library checks whether the key type is a public key generically,
# before it reaches a point where it needs support for the specific key
# type, so it returns INVALID_ARGUMENT for unsupported public key types.
generate_dependencies = []
result = 'PSA_ERROR_INVALID_ARGUMENT'
else:
generate_dependencies = \
psa_information.fix_key_pair_dependencies(import_dependencies, 'GENERATE')
for bits in kt.sizes_to_test():
if kt.name == 'PSA_KEY_TYPE_RSA_KEY_PAIR':
size_dependency = "PSA_VENDOR_RSA_GENERATE_MIN_KEY_BITS <= " + str(bits)
test_dependencies = generate_dependencies + [size_dependency]
else:
test_dependencies = generate_dependencies
yield test_case_for_key_generation(
tc = test_case_for_key_generation(
kt.expression, bits,
psa_information.finish_family_dependencies(test_dependencies, bits),
str(bits),
result
'PSA_ERROR_INVALID_ARGUMENT' if kt.is_public() else 'PSA_SUCCESS'
)
if kt.is_public():
# The library checks whether the key type is a public key generically,
# before it reaches a point where it needs support for the specific key
# type, so it returns INVALID_ARGUMENT for unsupported public key types.
tc.set_dependencies([])
yield tc
def test_cases_for_key_generation(self) -> Iterator[test_case.TestCase]:
"""Generate test cases that exercise the generation of keys."""
@@ -252,7 +238,7 @@ class OpFail:
) -> test_case.TestCase:
"""Construct a failure test case for a one-key or keyless operation."""
#pylint: disable=too-many-arguments,too-many-locals
tc = test_case.TestCase()
tc = psa_test_case.TestCase()
pretty_alg = alg.short_expression()
if reason == self.Reason.NOT_SUPPORTED:
short_deps = [re.sub(r'PSA_WANT_ALG_', r'', dep)
@@ -276,11 +262,12 @@ class OpFail:
for i, dep in enumerate(dependencies):
if dep in not_deps:
dependencies[i] = '!' + dep
tc.set_dependencies(dependencies)
tc.set_function(category.name.lower() + '_fail')
arguments = [] # type: List[str]
if kt:
key_material = kt.key_material(kt.sizes_to_test()[0])
bits = kt.sizes_to_test()[0]
tc.set_key_bits(bits)
key_material = kt.key_material(bits)
arguments += [key_type, test_case.hex_string(key_material)]
arguments.append(alg.expression)
if category.is_asymmetric():
@@ -289,6 +276,7 @@ class OpFail:
'INVALID_ARGUMENT')
arguments.append('PSA_ERROR_' + error)
tc.set_arguments(arguments)
tc.set_dependencies(dependencies)
return tc
def no_key_test_cases(
@@ -488,17 +476,12 @@ class StorageFormat:
correctly.
"""
verb = 'save' if self.forward else 'read'
tc = test_case.TestCase()
tc = psa_test_case.TestCase()
tc.set_description(verb + ' ' + key.description)
dependencies = psa_information.automatic_dependencies(
key.lifetime.string, key.type.string,
key.alg.string, key.alg2.string,
)
dependencies = psa_information.finish_family_dependencies(dependencies, key.bits)
dependencies += psa_information.generate_deps_from_description(key.description)
dependencies = psa_information.fix_key_pair_dependencies(dependencies, 'BASIC')
tc.set_dependencies(dependencies)
tc.add_dependencies(psa_information.generate_deps_from_description(key.description))
tc.set_function('key_storage_' + verb)
tc.set_key_bits(key.bits)
tc.set_key_pair_usage('BASIC')
if self.forward:
extra_arguments = []
else:
scripts/mbedtls_framework/crypto_data_tests.py
+2 -13
View File
@@ -12,20 +12,10 @@ from typing import Callable, Dict, Iterator, List, Optional #pylint: disable=unu
from . import crypto_knowledge
from . import psa_information
from . import psa_test_case
from . import test_case
def psa_low_level_dependencies(*expressions: str) -> List[str]:
"""Infer dependencies of a PSA low-level test case by looking for PSA_xxx symbols.
This function generates MBEDTLS_PSA_BUILTIN_xxx symbols.
"""
high_level = psa_information.automatic_dependencies(*expressions)
for dep in high_level:
assert dep.startswith('PSA_WANT_')
return ['MBEDTLS_PSA_BUILTIN_' + dep[9:] for dep in high_level]
class HashPSALowLevel:
"""Generate test cases for the PSA low-level hash interface."""
@@ -69,12 +59,11 @@ class HashPSALowLevel:
function: str, note: str,
arguments: List[str]) -> test_case.TestCase:
"""Construct one test case involving a hash."""
tc = test_case.TestCase()
tc = psa_test_case.TestCase(dependency_prefix='MBEDTLS_PSA_BUILTIN_')
tc.set_description('{}{} {}'
.format(function,
' ' + note if note else '',
alg.short_expression()))
tc.set_dependencies(psa_low_level_dependencies(alg.expression))
tc.set_function(function)
tc.set_arguments([alg.expression] +
['"{}"'.format(arg) for arg in arguments])
scripts/mbedtls_framework/psa_information.py
+23 -39
View File
@@ -8,7 +8,7 @@
import os
import re
from collections import OrderedDict
from typing import FrozenSet, List, Optional
from typing import List, Optional
from . import macro_collector
@@ -23,8 +23,13 @@ class Information:
def remove_unwanted_macros(
constructors: macro_collector.PSAMacroEnumerator
) -> None:
# Mbed TLS does not support finite-field DSA.
"""Remove constructors that should be exckuded from systematic testing."""
# Mbed TLS does not support finite-field DSA, but 3.6 defines DSA
# identifiers for historical reasons.
# Don't attempt to generate any related test case.
# The corresponding test cases would be commented out anyway,
# but for DSA, we don't have enough support in the test scripts
# to generate these test cases.
constructors.key_types.discard('PSA_KEY_TYPE_DSA_KEY_PAIR')
constructors.key_types.discard('PSA_KEY_TYPE_DSA_PUBLIC_KEY')
@@ -52,10 +57,16 @@ class Information:
return constructors
def psa_want_symbol(name: str) -> str:
"""Return the PSA_WANT_xxx symbol associated with a PSA crypto feature."""
def psa_want_symbol(name: str, prefix: Optional[str] = None) -> str:
"""Return the PSA_WANT_xxx symbol associated with a PSA crypto feature.
You can use an altenative `prefix`, e.g. 'MBEDTLS_PSA_BUILTIN_'
when specifically testing builtin implementations.
"""
if prefix is None:
prefix = 'PSA_WANT_'
if name.startswith('PSA_'):
return name[:4] + 'WANT_' + name[4:]
return prefix + name[4:]
else:
raise ValueError('Unable to determine the PSA_WANT_ symbol for ' + name)
@@ -83,18 +94,23 @@ SYMBOLS_WITHOUT_DEPENDENCY = frozenset([
'PSA_ALG_KEY_AGREEMENT', # chaining
'PSA_ALG_TRUNCATED_MAC', # modifier
])
def automatic_dependencies(*expressions: str) -> List[str]:
def automatic_dependencies(*expressions: str,
prefix: Optional[str] = None) -> List[str]:
"""Infer dependencies of a test case by looking for PSA_xxx symbols.
The arguments are strings which should be C expressions. Do not use
string literals or comments as this function is not smart enough to
skip them.
`prefix`: prefix to use in dependencies. Defaults to ``'PSA_WANT_'``.
Use ``'MBEDTLS_PSA_BUILTIN_'`` when specifically testing
builtin implementations.
"""
used = set()
for expr in expressions:
used.update(re.findall(r'PSA_(?:ALG|ECC_FAMILY|DH_FAMILY|KEY_TYPE)_\w+', expr))
used.difference_update(SYMBOLS_WITHOUT_DEPENDENCY)
return sorted(psa_want_symbol(name) for name in used)
return sorted(psa_want_symbol(name, prefix=prefix) for name in used)
# Define set of regular expressions and dependencies to optionally append
# extra dependencies for test case based on key description.
@@ -123,38 +139,6 @@ def generate_deps_from_description(
return dep_list
# A temporary hack: at the time of writing, not all dependency symbols
# are implemented yet. Skip test cases for which the dependency symbols are
# not available. Once all dependency symbols are available, this hack must
# be removed so that a bug in the dependency symbols properly leads to a test
# failure.
def read_implemented_dependencies(filename: str) -> FrozenSet[str]:
return frozenset(symbol
for line in open(filename)
for symbol in re.findall(r'\bPSA_WANT_\w+\b', line))
_implemented_dependencies = None #type: Optional[FrozenSet[str]] #pylint: disable=invalid-name
def hack_dependencies_not_implemented(dependencies: List[str]) -> None:
"""
Hack dependencies to skip test cases for which at least one dependency
symbol is not available yet.
"""
global _implemented_dependencies #pylint: disable=global-statement,invalid-name
if _implemented_dependencies is None:
# Temporary, while Mbed TLS does not just rely on the TF-PSA-Crypto
# build system to build its crypto library. When it does, the first
# case can just be removed.
if os.path.isdir('tf-psa-crypto'):
_implemented_dependencies = \
read_implemented_dependencies('tf-psa-crypto/include/psa/crypto_config.h')
else:
_implemented_dependencies = \
read_implemented_dependencies('include/psa/crypto_config.h')
if not all((dep.lstrip('!') in _implemented_dependencies or
not dep.lstrip('!').startswith('PSA_WANT'))
for dep in dependencies):
dependencies.append('DEPENDENCY_NOT_IMPLEMENTED_YET')
def tweak_key_pair_dependency(dep: str, usage: str):
"""
This helper function add the proper suffix to PSA_WANT_KEY_TYPE_xxx_KEY_PAIR
scripts/mbedtls_framework/psa_test_case.py
+145
View File
@@ -0,0 +1,145 @@
"""Generate test cases for PSA API calls, with automatic dependencies.
"""
# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
#
import os
import re
from typing import FrozenSet, List, Optional, Set
from . import psa_information
from . import test_case
# A temporary hack: at the time of writing, not all dependency symbols
# are implemented yet. Skip test cases for which the dependency symbols are
# not available. Once all dependency symbols are available, this hack must
# be removed so that a bug in the dependency symbols properly leads to a test
# failure.
def read_implemented_dependencies(acc: Set[str], filename: str) -> None:
with open(filename) as input_stream:
for line in input_stream:
for symbol in re.findall(r'\bPSA_WANT_\w+\b', line):
acc.add(symbol)
_implemented_dependencies = None #type: Optional[FrozenSet[str]] #pylint: disable=invalid-name
def find_dependencies_not_implemented(dependencies: List[str]) -> List[str]:
"""List the dependencies that are not implemented."""
global _implemented_dependencies #pylint: disable=global-statement,invalid-name
if _implemented_dependencies is None:
# Temporary, while Mbed TLS does not just rely on the TF-PSA-Crypto
# build system to build its crypto library. When it does, the first
# case can just be removed.
if os.path.isdir('tf-psa-crypto'):
include_dir = 'tf-psa-crypto/include'
else:
include_dir = 'include'
acc = set() #type: Set[str]
for filename in [
os.path.join(include_dir, 'psa/crypto_config.h'),
os.path.join(include_dir, 'psa/crypto_adjust_config_synonyms.h'),
]:
read_implemented_dependencies(acc, filename)
_implemented_dependencies = frozenset(acc)
return [dep
for dep in dependencies
if (dep.lstrip('!') not in _implemented_dependencies and
dep.lstrip('!').startswith('PSA_WANT'))]
class TestCase(test_case.TestCase):
"""A PSA test case with automatically inferred dependencies.
For mechanisms like ECC curves where the support status includes
the key bit-size, this class assumes that only one bit-size is
involved in a given test case.
"""
def __init__(self, dependency_prefix: Optional[str] = None) -> None:
"""Construct a test case for a PSA Crypto API call.
`dependency_prefix`: prefix to use in dependencies. Defaults to
``'PSA_WANT_'``. Use ``'MBEDTLS_PSA_BUILTIN_'``
when specifically testing builtin implementations.
"""
super().__init__()
del self.dependencies
self.manual_dependencies = [] #type: List[str]
self.automatic_dependencies = set() #type: Set[str]
self.dependency_prefix = dependency_prefix #type: Optional[str]
self.key_bits = None #type: Optional[int]
self.key_pair_usage = None #type: Optional[str]
def set_key_bits(self, key_bits: Optional[int]) -> None:
"""Use the given key size for automatic dependency generation.
Call this function before set_arguments() if relevant.
This is only relevant for ECC and DH keys. For other key types,
this information is ignored.
"""
self.key_bits = key_bits
def set_key_pair_usage(self, key_pair_usage: Optional[str]) -> None:
"""Use the given suffix for key pair dependencies.
Call this function before set_arguments() if relevant.
This is only relevant for key pair types. For other key types,
this information is ignored.
"""
self.key_pair_usage = key_pair_usage
def infer_dependencies(self, arguments: List[str]) -> List[str]:
"""Infer dependencies based on the test case arguments."""
dependencies = psa_information.automatic_dependencies(*arguments,
prefix=self.dependency_prefix)
if self.key_bits is not None:
dependencies = psa_information.finish_family_dependencies(dependencies,
self.key_bits)
if self.key_pair_usage is not None:
dependencies = psa_information.fix_key_pair_dependencies(dependencies,
self.key_pair_usage)
if 'PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE' in dependencies and \
self.key_bits is not None:
size_dependency = ('PSA_VENDOR_RSA_GENERATE_MIN_KEY_BITS <= ' +
str(self.key_bits))
dependencies.append(size_dependency)
return dependencies
def set_arguments(self, arguments: List[str]) -> None:
"""Set test case arguments and automatically infer dependencies."""
super().set_arguments(arguments)
dependencies = self.infer_dependencies(arguments)
self.skip_if_any_not_implemented(dependencies)
self.automatic_dependencies.update(dependencies)
def set_dependencies(self, dependencies: List[str]) -> None:
"""Override any previously added automatic or manual dependencies.
Also override any previous instruction to skip the test case.
"""
self.manual_dependencies = dependencies
self.automatic_dependencies.clear()
self.skip_reasons = []
def add_dependencies(self, dependencies: List[str]) -> None:
"""Add manual dependencies."""
self.manual_dependencies += dependencies
def get_dependencies(self) -> List[str]:
# Make the output independent of the order in which the dependencies
# are calculated by the script. Also avoid duplicates. This makes
# the output robust with respect to refactoring of the scripts.
dependencies = set(self.manual_dependencies)
dependencies.update(self.automatic_dependencies)
return sorted(dependencies)
def skip_if_any_not_implemented(self, dependencies: List[str]) -> None:
"""Skip the test case if any of the given dependencies is not implemented."""
not_implemented = find_dependencies_not_implemented(dependencies)
for dep in not_implemented:
self.skip_because('not implemented: ' + dep)
scripts/mbedtls_framework/test_case.py
+33 -4
View File
@@ -57,6 +57,7 @@ class TestCase:
self.dependencies = [] #type: List[str]
self.function = None #type: Optional[str]
self.arguments = [] #type: List[str]
self.skip_reasons = [] #type: List[str]
def add_comment(self, *lines: str) -> None:
self.comments += lines
@@ -64,6 +65,9 @@ class TestCase:
def set_description(self, description: str) -> None:
self.description = description
def get_dependencies(self) -> List[str]:
return self.dependencies
def set_dependencies(self, dependencies: List[str]) -> None:
self.dependencies = dependencies
@@ -73,6 +77,23 @@ class TestCase:
def set_arguments(self, arguments: List[str]) -> None:
self.arguments = arguments
def skip_because(self, reason: str) -> None:
"""Skip this test case.
It will be included in the output, but commented out.
This is intended for test cases that are obtained from a
systematic enumeration, but that have dependencies that cannot
be fulfilled. Since we don't want to have test cases that are
never executed, we arrange not to have actual test cases. But
we do include comments to make it easier to understand the output
of test case generation.
reason must be a non-empty string explaining to humans why this
test case is skipped.
"""
self.skip_reasons.append(reason)
def check_completeness(self) -> None:
if self.description is None:
raise MissingDescription
@@ -93,10 +114,18 @@ class TestCase:
out.write('\n')
for line in self.comments:
out.write('# ' + line + '\n')
out.write(self.description + '\n')
if self.dependencies:
out.write('depends_on:' + ':'.join(self.dependencies) + '\n')
out.write(self.function + ':' + ':'.join(self.arguments) + '\n')
prefix = ''
if self.skip_reasons:
prefix = '## '
for reason in self.skip_reasons:
out.write('## # skipped because: ' + reason + '\n')
out.write(prefix + self.description + '\n')
dependencies = self.get_dependencies()
if dependencies:
out.write(prefix + 'depends_on:' +
':'.join(dependencies) + '\n')
out.write(prefix + self.function + ':' +
':'.join(self.arguments) + '\n')
def write_data_file(filename: str,
test_cases: Iterable[TestCase],