diff --git a/.github/workflows/simulation-1.2.yml b/.github/workflows/simulation-1.2.yml index 31f3ff70c..bca95fd0f 100644 --- a/.github/workflows/simulation-1.2.yml +++ b/.github/workflows/simulation-1.2.yml @@ -271,7 +271,7 @@ jobs: run: | export CI_ENV="$(bash <(curl -s https://codecov.io/env)) -e GITHUB_ACTIONS -e COVERAGE" echo "CI_ENV=${CI_ENV}" - sudo -E ./script/test cert_bbr ./tests/scripts/thread-cert/backbone/*.py || (sudo chmod a+r *.log *.json *.pcap && false) + sudo -E ./script/test cert_suite ./tests/scripts/thread-cert/backbone/*.py || (sudo chmod a+r *.log *.json *.pcap && false) - uses: actions/upload-artifact@v2 with: name: cov-thread-1-2-backbone-docker diff --git a/script/test b/script/test index d967bde2b..7dc18b919 100755 --- a/script/test +++ b/script/test @@ -201,61 +201,11 @@ do_cert_suite() fi fi - local pass_count=0 - local fail_count=0 - - [[ ! -f fail.log ]] || rm fail.log - - for test_case in "$@"; do - rm -rf tmp || sudo rm -rf tmp - if TEST_NAME="$(basename "${test_case}" .py)" "${test_case}" &>test.log; then - echo -e "${COLOR_PASS}PASS${COLOR_NONE} ${test_case}" - pass_count=$((pass_count + 1)) - else - echo -e "${COLOR_FAIL}FAIL${COLOR_NONE} ${test_case}" - fail_count=$((fail_count + 1)) - { - echo "==============================" - echo "!!! FAIL:${test_case}" - echo "==============================" - cat test.log - } >>fail.log - fi - done - - echo "==================================" - echo " Test Summary" - echo "==================================" - echo "# TOTAL: $((pass_count + fail_count))" - echo "# PASS: ${pass_count}" - echo "# FAIL: ${fail_count}" - - if [[ ${fail_count} -gt 0 ]]; then - tr -dc '[:print:]\r\n\t' /dev/null || true') bash('pkill dumpcap 2>/dev/null || true') @@ -84,8 +92,15 @@ def cleanup_env(): bash(f'docker network rm $(docker network ls -q -f "name=backbone") 2>/dev/null || true') -def setup_env(): +def setup_backbone_env(): bash('sudo modprobe ip6table_filter') + + if THREAD_VERSION != '1.2': + raise RuntimeError('Backbone tests only work with THREAD_VERSION=1.2') + + if VIRTUAL_TIME: + raise RuntimeError('Backbone tests only work with VIRTUAL_TIME=0') + bash(f'docker image inspect {config.OTBR_DOCKER_IMAGE} >/dev/null') @@ -96,40 +111,90 @@ def parse_args(): parser.add_argument("scripts", nargs='+', type=str, help='specify Backbone test scripts') args = parser.parse_args() + logging.info("Max jobs: %d", MAX_JOBS) logging.info("Multiply: %d", args.multiply) - logging.info("Test scripts: %s", args.scripts) + logging.info("Test scripts: %d", len(args.scripts)) return args +def check_has_backbone_tests(scripts): + for script in scripts: + relpath = os.path.relpath(script, _BACKBONE_TESTS_DIR) + if not relpath.startswith('..'): + return True + + return False + + +class PortOffsetPool: + + def __init__(self, size: int): + self._size = size + self._pool = queue.Queue(maxsize=size) + for port_offset in range(1, size + 1): + self.release(port_offset) + + def allocate(self) -> int: + return self._pool.get() + + def release(self, port_offset: int): + assert 1 <= port_offset <= self._size, port_offset + self._pool.put_nowait(port_offset) + + +def run_tests(scripts: List[str], multiply: int = 1): + script_fail_count = Counter() + script_succ_count = Counter() + + # Run each script for multiple times + scripts = [script for script in scripts for _ in range(multiply)] + port_offset_pool = PortOffsetPool(MAX_JOBS) + + def error_callback(port_offset, script, err): + port_offset_pool.release(port_offset) + + script_fail_count[script] += 1 + if script_succ_count[script] + script_fail_count[script] == multiply: + color = _COLOR_PASS if script_fail_count[script] == 0 else _COLOR_FAIL + print(f'{color}PASS {script_succ_count[script]} FAIL {script_fail_count[script]}{_COLOR_NONE} {script}') + + def pass_callback(port_offset, script): + port_offset_pool.release(port_offset) + + script_succ_count[script] += 1 + if script_succ_count[script] + script_fail_count[script] == multiply: + color = _COLOR_PASS if script_fail_count[script] == 0 else _COLOR_FAIL + print(f'{color}PASS {script_succ_count[script]} FAIL {script_fail_count[script]}{_COLOR_NONE} {script}') + + for i, script in enumerate(scripts): + port_offset = port_offset_pool.allocate() + pool.apply_async( + run_cert, [port_offset, script], + callback=lambda ret, port_offset=port_offset, script=script: pass_callback(port_offset, script), + error_callback=lambda err, port_offset=port_offset, script=script: error_callback( + port_offset, script, err)) + + pool.close() + pool.join() + return sum(script_fail_count.values()) + + def main(): args = parse_args() - cleanup_env() - setup_env() - script_fail_count = Counter() + has_backbone_tests = check_has_backbone_tests(args.scripts) + logging.info('Has Backbone tests: %s', has_backbone_tests) - def error_callback(script, err): - logging.error("Test %s failed: %s", script, err) - script_fail_count[script] += 1 + if has_backbone_tests: + cleanup_backbone_env() + setup_backbone_env() - # Run each script for multiple times - scripts = args.scripts * args.multiply - - for i, script in enumerate(scripts): - pool.apply_async(run_bbr_test, [i, script], - error_callback=lambda err, script=script: error_callback(script, err)) - - pool.close() - - logging.info("Waiting for tests to complete ...") - pool.join() - - cleanup_env() - - for script in args.scripts: - logging.info("Test %s: %d PASS/%d TOTAL", script, args.multiply - script_fail_count[script], args.multiply) - - exit(len(script_fail_count)) + try: + fail_count = run_tests(args.scripts, args.multiply) + exit(fail_count) + finally: + if has_backbone_tests: + cleanup_backbone_env() if __name__ == '__main__':