mirror of
https://github.com/Mbed-TLS/mbedtls-framework.git
synced 2026-06-06 05:25:18 +00:00
Merge pull request #20 from gilles-peskine-arm/search_outcomes_config-create
Script to search the outcome file for configurations with given requirements
This commit is contained in:
Gilles Peskine
Executable
+225
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python
|
||||
"""Search an outcome file for configurations with given settings.
|
||||
|
||||
Read an outcome file and report the configurations in which test_suite_config
|
||||
runs with the required settings (compilation option enabled or disabled).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from typing import Dict, FrozenSet, Iterator, List, Set
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from mbedtls_framework import build_tree
|
||||
|
||||
|
||||
def make_regexp_for_settings(settings: List[str]) -> str:
|
||||
"""Construct a regexp matching the interesting outcome lines.
|
||||
|
||||
Interesting outcome lines are from test_suite_config where the given
|
||||
setting is passing.
|
||||
|
||||
We assume that the elements of settings don't contain regexp special
|
||||
characters.
|
||||
"""
|
||||
return (r';test_suite_config[^;]*;Config: (' +
|
||||
'|'.join(settings) +
|
||||
r');PASS;')
|
||||
|
||||
def run_grep(regexp: str, outcome_file: str) -> List[str]:
|
||||
"""Run grep on the outcome file and return the matching lines."""
|
||||
env = os.environ.copy()
|
||||
env['LC_ALL'] = 'C' # Speeds up some versions of GNU grep
|
||||
try:
|
||||
return subprocess.check_output(['grep', '-E', regexp, outcome_file],
|
||||
encoding='ascii',
|
||||
env=env).splitlines()
|
||||
except subprocess.CalledProcessError as exn:
|
||||
if exn.returncode == 1:
|
||||
return [] # No results. We don't consider this an error.
|
||||
raise
|
||||
|
||||
OUTCOME_LINE_RE = re.compile(r'[^;]*;'
|
||||
r'([^;]*);'
|
||||
r'test_suite_config\.(?:[^;]*);'
|
||||
r'Config: ([^;]*);'
|
||||
r'PASS;')
|
||||
|
||||
def extract_configuration_data(outcome_lines: List[str]) -> Dict[str, FrozenSet[str]]:
|
||||
"""Extract the configuration data from outcome lines.
|
||||
|
||||
The result maps a configuration name to the list of passing settings
|
||||
in that configuration.
|
||||
"""
|
||||
config_data = {} #type: Dict[str, Set[str]]
|
||||
for line in outcome_lines:
|
||||
m = OUTCOME_LINE_RE.match(line)
|
||||
# Assuming a well-formed outcome file, make_regexp_for_settings()
|
||||
# arranges to only return lines that should match OUTCOME_LINE_RE.
|
||||
# So this assertion can't fail unless there is an unexpected
|
||||
# divergence between OUTCOME_LINE_RE, make_regexp_for_settings()
|
||||
# and the format of the given outcome file
|
||||
assert m is not None
|
||||
config_name, setting = m.groups()
|
||||
if config_name not in config_data:
|
||||
config_data[config_name] = set()
|
||||
config_data[config_name].add(setting)
|
||||
return dict((name, frozenset(settings))
|
||||
for name, settings in config_data.items())
|
||||
|
||||
|
||||
def matching_configurations(config_data: Dict[str, FrozenSet[str]],
|
||||
required: List[str]) -> Iterator[str]:
|
||||
"""Search configurations with the given passing settings.
|
||||
|
||||
config_data maps a configuration name to the list of passing settings
|
||||
in that configuration.
|
||||
|
||||
Each setting should be an Mbed TLS compile setting (MBEDTLS_xxx or
|
||||
PSA_xxx), optionally prefixed with "!".
|
||||
"""
|
||||
required_set = frozenset(required)
|
||||
for config, observed in config_data.items():
|
||||
if required_set.issubset(observed):
|
||||
yield config
|
||||
|
||||
def search_config_outcomes(outcome_file: str, settings: List[str]) -> List[str]:
|
||||
"""Search the given outcome file for reports of the given settings.
|
||||
|
||||
Each setting should be an Mbed TLS compile setting (MBEDTLS_xxx or
|
||||
PSA_xxx), optionally prefixed with "!".
|
||||
"""
|
||||
# The outcome file is large enough (hundreds of MB) that parsing it
|
||||
# in Python is slow. Use grep to speed this up considerably.
|
||||
regexp = make_regexp_for_settings(settings)
|
||||
outcome_lines = run_grep(regexp, outcome_file)
|
||||
config_data = extract_configuration_data(outcome_lines)
|
||||
return sorted(matching_configurations(config_data, settings))
|
||||
|
||||
|
||||
class TestSearch(unittest.TestCase):
|
||||
"""Tests of search functionality."""
|
||||
|
||||
OUTCOME_FILE_CONTENT = """\
|
||||
whatever;foobar;test_suite_config.part;Config: MBEDTLS_FOO;PASS;
|
||||
whatever;foobar;test_suite_config.part;Config: !MBEDTLS_FOO;SKIP;
|
||||
whatever;foobar;test_suite_config.part;Config: MBEDTLS_BAR;PASS;
|
||||
whatever;foobar;test_suite_config.part;Config: !MBEDTLS_BAR;SKIP;
|
||||
whatever;foobar;test_suite_config.part;Config: MBEDTLS_QUX;SKIP;
|
||||
whatever;foobar;test_suite_config.part;Config: !MBEDTLS_QUX;PASS;
|
||||
whatever;fooqux;test_suite_config.part;Config: MBEDTLS_FOO;PASS;
|
||||
whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_FOO;SKIP;
|
||||
whatever;fooqux;test_suite_config.part;Config: MBEDTLS_BAR;SKIP;
|
||||
whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_BAR;PASS;
|
||||
whatever;fooqux;test_suite_config.part;Config: MBEDTLS_QUX;PASS;
|
||||
whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_QUX;SKIP;
|
||||
whatever;fooqux;test_suite_something.else;Config: MBEDTLS_BAR;PASS;
|
||||
whatever;boring;test_suite_config.part;Config: BORING;PASS;
|
||||
whatever;parasite;not_test_suite_config.not;Config: MBEDTLS_FOO;PASS;
|
||||
whatever;parasite;test_suite_config.but;Config: MBEDTLS_QUX with bells on;PASS;
|
||||
whatever;parasite;test_suite_config.but;Not Config: MBEDTLS_QUX;PASS;
|
||||
"""
|
||||
|
||||
def search(self, settings: List[str], expected: List[str]) -> None:
|
||||
"""Test the search functionality.
|
||||
|
||||
* settings: settings to search.
|
||||
* expected: expected search results.
|
||||
"""
|
||||
with tempfile.NamedTemporaryFile() as tmp:
|
||||
tmp.write(self.OUTCOME_FILE_CONTENT.encode())
|
||||
tmp.flush()
|
||||
actual = search_config_outcomes(tmp.name, settings)
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_foo(self) -> None:
|
||||
self.search(['MBEDTLS_FOO'], ['foobar', 'fooqux'])
|
||||
|
||||
def test_bar(self) -> None:
|
||||
self.search(['MBEDTLS_BAR'], ['foobar'])
|
||||
|
||||
def test_foo_bar(self) -> None:
|
||||
self.search(['MBEDTLS_FOO', 'MBEDTLS_BAR'], ['foobar'])
|
||||
|
||||
def test_foo_notbar(self) -> None:
|
||||
self.search(['MBEDTLS_FOO', '!MBEDTLS_BAR'], ['fooqux'])
|
||||
|
||||
|
||||
class TestOutcome(unittest.TestCase):
|
||||
"""Tests of outcome file format expectations.
|
||||
|
||||
This class builds and runs the config tests in the current configuration.
|
||||
The configuration must have at least one feature enabled and at least
|
||||
one feature disabled in each category: MBEDTLS_xxx and PSA_WANT_xxx.
|
||||
It needs a C compiler.
|
||||
"""
|
||||
|
||||
outcome_content = '' # Let mypy know this field can be used in test case methods
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
"""Generate, build and run the config tests."""
|
||||
root_dir = build_tree.guess_project_root()
|
||||
tests_dir = os.path.join(root_dir, 'tests')
|
||||
suites = ['test_suite_config.mbedtls_boolean',
|
||||
'test_suite_config.psa_boolean']
|
||||
_output = subprocess.check_output(['make'] + suites,
|
||||
cwd=tests_dir,
|
||||
stderr=subprocess.STDOUT)
|
||||
with tempfile.NamedTemporaryFile(dir=tests_dir) as outcome_file:
|
||||
env = os.environ.copy()
|
||||
env['MBEDTLS_TEST_PLATFORM'] = 'some_platform'
|
||||
env['MBEDTLS_TEST_CONFIGURATION'] = 'some_configuration'
|
||||
env['MBEDTLS_TEST_OUTCOME_FILE'] = outcome_file.name
|
||||
for suite in suites:
|
||||
_output = subprocess.check_output([os.path.join(os.path.curdir, suite)],
|
||||
cwd=tests_dir,
|
||||
env=env,
|
||||
stderr=subprocess.STDOUT)
|
||||
cls.outcome_content = outcome_file.read().decode('ascii')
|
||||
|
||||
def test_outcome_format(self) -> None:
|
||||
"""Check that there are outcome lines matching the expected general format."""
|
||||
def regex(prefix: str, result: str) -> str:
|
||||
return (r'(?:\A|\n)some_platform;some_configuration;'
|
||||
r'test_suite_config\.\w+;Config: {}_\w+;{};'
|
||||
.format(prefix, result))
|
||||
self.assertRegex(self.outcome_content, regex('MBEDTLS', 'PASS'))
|
||||
self.assertRegex(self.outcome_content, regex('MBEDTLS', 'SKIP'))
|
||||
self.assertRegex(self.outcome_content, regex('!MBEDTLS', 'PASS'))
|
||||
self.assertRegex(self.outcome_content, regex('!MBEDTLS', 'SKIP'))
|
||||
self.assertRegex(self.outcome_content, regex('PSA_WANT', 'PASS'))
|
||||
self.assertRegex(self.outcome_content, regex('PSA_WANT', 'SKIP'))
|
||||
self.assertRegex(self.outcome_content, regex('!PSA_WANT', 'PASS'))
|
||||
self.assertRegex(self.outcome_content, regex('!PSA_WANT', 'SKIP'))
|
||||
|
||||
def test_outcome_lines(self) -> None:
|
||||
"""Look for some sample outcome lines."""
|
||||
def regex(setting: str) -> str:
|
||||
return (r'(?:\A|\n)some_platform;some_configuration;'
|
||||
r'test_suite_config\.\w+;Config: {};(PASS|SKIP);'
|
||||
.format(setting))
|
||||
self.assertRegex(self.outcome_content, regex('MBEDTLS_AES_C'))
|
||||
self.assertRegex(self.outcome_content, regex('MBEDTLS_AES_ROM_TABLES'))
|
||||
self.assertRegex(self.outcome_content, regex('MBEDTLS_SSL_CLI_C'))
|
||||
self.assertRegex(self.outcome_content, regex('MBEDTLS_X509_CRT_PARSE_C'))
|
||||
self.assertRegex(self.outcome_content, regex('PSA_WANT_ALG_HMAC'))
|
||||
self.assertRegex(self.outcome_content, regex('PSA_WANT_KEY_TYPE_AES'))
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--outcome-file', '-f', metavar='FILE',
|
||||
default='outcomes.csv',
|
||||
help='Outcome file to read (default: outcomes.csv)')
|
||||
parser.add_argument('settings', metavar='SETTING', nargs='+',
|
||||
help='Required setting (e.g. "MBEDTLS_RSA_C" or "!PSA_WANT_ALG_SHA256")')
|
||||
options = parser.parse_args()
|
||||
found = search_config_outcomes(options.outcome_file, options.settings)
|
||||
for name in found:
|
||||
print(name)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user