From d8b5a3ee515e40fd9d9c56e11f2d6d849937c342 Mon Sep 17 00:00:00 2001 From: Jintao Lin Date: Fri, 19 Feb 2021 01:05:29 +0800 Subject: [PATCH] [test] add posix verification for OT 1.2 (#6070) This change: 1. Adds new github action to verify posix app for OT 1.2. 2. Only enable compile flag OT_SIMULATION_VIRTUAL_TIME_UART in script/test for ot-rcp build target. 3. Allow mix use of posix node and ot-cli-mtd node by changing python test script to accept node type as parameter. 4. Remove CSL test from expect as it is covered by thread-1-2-posix tests. --- .github/workflows/simulation-1.2.yml | 59 +++++++++++++++++++ script/test | 14 +++-- tests/scripts/expect/v1_2-rcp.exp | 48 --------------- tests/scripts/thread-cert/node.py | 12 +++- tests/scripts/thread-cert/simulator.py | 3 +- .../v1_2_LowPower_5_3_01_SSEDAttachment.py | 1 + .../thread-cert/v1_2_test_csl_transmission.py | 1 + 7 files changed, 82 insertions(+), 56 deletions(-) delete mode 100755 tests/scripts/expect/v1_2-rcp.exp diff --git a/.github/workflows/simulation-1.2.yml b/.github/workflows/simulation-1.2.yml index a3b76c918..39f10b0e3 100644 --- a/.github/workflows/simulation-1.2.yml +++ b/.github/workflows/simulation-1.2.yml @@ -358,6 +358,64 @@ jobs: name: cov-thread-border-router path: tmp/coverage.info + thread-1-2-posix: + runs-on: ubuntu-20.04 + env: + COVERAGE: 1 + PYTHONUNBUFFERED: 1 + READLINE: readline + THREAD_VERSION: 1.2 + OT_NODE_TYPE: rcp + USE_MTD: 1 + VIRTUAL_TIME: 1 + INTER_OP: 1 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Bootstrap + run: | + sudo rm /etc/apt/sources.list.d/* && sudo apt-get update + sudo apt-get --no-install-recommends install -y libreadline6-dev g++-multilib ninja-build python3-setuptools python3-wheel llvm lcov + python3 -m pip install -r tests/scripts/thread-cert/requirements.txt + - name: Build + run: | + ./script/test build + - name: Run + run: | + ulimit -c unlimited + ./script/test prepare_coredump_upload + ./script/test cert tests/scripts/thread-cert/v1_2_LowPower_5_3_01_SSEDAttachment.py + ./script/test cert tests/scripts/thread-cert/v1_2_LowPower_6_1_07_PreferringARouterOverAReed.py + ./script/test cert tests/scripts/thread-cert/v1_2_router_5_1_1.py + ./script/test cert tests/scripts/thread-cert/v1_2_test_csl_transmission.py + ./script/test cert tests/scripts/thread-cert/v1_2_test_enhanced_frame_pending.py + ./script/test cert tests/scripts/thread-cert/v1_2_test_parent_selection.py + - name: Check Crash + if: ${{ failure() }} + run: | + CRASHED=$(./script/test check_crash | tail -1) + [[ $CRASHED -eq "1" ]] && echo "Crashed!" || echo "Not crashed." + echo "CRASHED=$CRASHED" >> $GITHUB_ENV + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: thread-1-2-posix-pcaps + path: "*.pcap" + - uses: actions/upload-artifact@v2 + if: ${{ failure() && env.CRASHED == '1' }} + with: + name: core-thread-1-2-posix + path: | + ./ot-core-dump/* + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 + with: + name: cov-thread-1-2-posix + path: tmp/coverage.info + upload-coverage: needs: - thread-1-2 @@ -366,6 +424,7 @@ jobs: - expects - thread-1-2-backbone - thread-border-router + - thread-1-2-posix runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 diff --git a/script/test b/script/test index 431dfcdfe..ae9b1f0b9 100755 --- a/script/test +++ b/script/test @@ -78,11 +78,6 @@ build_simulation() if [[ ${VIRTUAL_TIME} == 1 ]]; then options+=("-DOT_SIMULATION_VIRTUAL_TIME=ON") - - if [[ ${OT_NODE_TYPE} == rcp* ]]; then - options+=("-DOT_SIMULATION_VIRTUAL_TIME_UART=ON") - fi - fi if [[ ${version} == "1.2" ]]; then @@ -96,12 +91,20 @@ build_simulation() OT_CMAKE_BUILD_DIR="${OT_BUILDDIR}/openthread-simulation-${version}" "${OT_SRCDIR}"/script/cmake-build simulation "${options[@]}" + if [[ ${VIRTUAL_TIME} == 1 ]] && [[ ${OT_NODE_TYPE} == rcp* ]]; then + OT_CMAKE_NINJA_TARGET=ot-rcp OT_CMAKE_BUILD_DIR="${OT_BUILDDIR}/openthread-simulation-${version}" "${OT_SRCDIR}"/script/cmake-build simulation "${options[@]}" "-DOT_SIMULATION_VIRTUAL_TIME_UART=ON" + fi + if [[ ${version} == "1.2" && ${INTER_OP_BBR} == 1 ]]; then options+=("-DOT_BACKBONE_ROUTER=ON") OT_CMAKE_BUILD_DIR="${OT_BUILDDIR}/openthread-simulation-${version}-bbr" "${OT_SRCDIR}"/script/cmake-build simulation "${options[@]}" + if [[ ${VIRTUAL_TIME} == 1 ]] && [[ ${OT_NODE_TYPE} == rcp* ]]; then + OT_CMAKE_NINJA_TARGET=ot-rcp OT_CMAKE_BUILD_DIR="${OT_BUILDDIR}/openthread-simulation-${version}-bbr" "${OT_SRCDIR}"/script/cmake-build simulation "${options[@]}" "-DOT_SIMULATION_VIRTUAL_TIME_UART=ON" + fi + fi } @@ -217,6 +220,7 @@ do_cert() [[ ! -d tmp ]] || rm -rvf tmp PYTHONUNBUFFERED=1 "$@" + exit 0 } do_cert_suite() diff --git a/tests/scripts/expect/v1_2-rcp.exp b/tests/scripts/expect/v1_2-rcp.exp deleted file mode 100755 index 6815baa12..000000000 --- a/tests/scripts/expect/v1_2-rcp.exp +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/expect -f -# -# Copyright (c) 2020, The OpenThread Authors. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. Neither the name of the copyright holder nor the -# names of its contributors may be used to endorse or promote products -# derived from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# - -source "tests/scripts/expect/_common.exp" -source "tests/scripts/expect/_multinode.exp" - -spawn_node 1 - -spawn_node 2 mtd -send "csl period 5000\n" -expect_line "Done" - -setup_two_nodes "-" false - -switch_node 1 -set addr [get_ipaddr "mleid"] - -switch_node 2 -send "ping $addr\n" -expect "16 bytes from $addr: icmp_seq=1" - -dispose_all diff --git a/tests/scripts/thread-cert/node.py b/tests/scripts/thread-cert/node.py index 394095b88..351034c04 100755 --- a/tests/scripts/thread-cert/node.py +++ b/tests/scripts/thread-cert/node.py @@ -345,8 +345,13 @@ class OtCli: # Default command if no match below, will be overridden if below conditions are met. cmd = './ot-cli-%s' % (mode) + # For Thread 1.2 MTD node, use ot-cli-mtd build regardless of OT_CLI_PATH + if self.version == '1.2' and mode == 'mtd' and 'top_builddir' in os.environ: + srcdir = os.environ['top_builddir'] + cmd = '%s/examples/apps/cli/ot-cli-%s %d' % (srcdir, mode, nodeid) + # If Thread version of node matches the testing environment version. - if self.version == self.env_version: + elif self.version == self.env_version: # Load Thread 1.2 BBR device when testing Thread 1.2 scenarios # which requires device with Backbone functionality. if self.version == '1.2' and self.is_bbr: @@ -367,6 +372,7 @@ class OtCli: if 'RADIO_DEVICE' in os.environ: cmd += ' --real-time-signal=+1 -v spinel+hdlc+uart://%s?forkpty-arg=%d' % (os.environ['RADIO_DEVICE'], nodeid) + self.is_posix = True else: cmd += ' %d' % nodeid @@ -382,6 +388,7 @@ class OtCli: if 'RADIO_DEVICE_1_1' in os.environ: cmd += ' --real-time-signal=+1 -v spinel+hdlc+uart://%s?forkpty-arg=%d' % ( os.environ['RADIO_DEVICE_1_1'], nodeid) + self.is_posix = True else: cmd += ' %d' % nodeid @@ -410,6 +417,7 @@ class OtCli: if 'RADIO_DEVICE' in os.environ: args = ' --real-time-signal=+1 spinel+hdlc+uart://%s?forkpty-arg=%d' % (os.environ['RADIO_DEVICE'], nodeid) + self.is_posix = True else: args = '' @@ -449,6 +457,7 @@ class OtCli: if 'RADIO_DEVICE_1_1' in os.environ: args = ' --real-time-signal=+1 spinel+hdlc+uart://%s?forkpty-arg=%d' % (os.environ['RADIO_DEVICE_1_1'], nodeid) + self.is_posix = True else: args = '' @@ -502,6 +511,7 @@ class NodeImpl: def __init__(self, nodeid, name=None, simulator=None, **kwargs): self.nodeid = nodeid self.name = name or ('Node%d' % nodeid) + self.is_posix = False self.simulator = simulator if self.simulator: diff --git a/tests/scripts/thread-cert/simulator.py b/tests/scripts/thread-cert/simulator.py index d6d0caa63..a07fcb939 100644 --- a/tests/scripts/thread-cert/simulator.py +++ b/tests/scripts/thread-cert/simulator.py @@ -144,7 +144,6 @@ class VirtualTime(BaseSimulator): BLOCK_TIMEOUT = 10 - RADIO_ONLY = os.getenv('RADIO_DEVICE') is not None NCP_SIM = os.getenv('NODE_TYPE', 'sim') == 'ncp-sim' _message_factory = None @@ -246,7 +245,7 @@ class VirtualTime(BaseSimulator): return (addr[0], addr[1] - self.BASE_PORT) def _core_addr_from(self, nodeid): - if self.RADIO_ONLY: + if self._nodes[nodeid].is_posix: return ('127.0.0.1', self.BASE_PORT + self.port + nodeid) else: return ('127.0.0.1', self.port + nodeid) diff --git a/tests/scripts/thread-cert/v1_2_LowPower_5_3_01_SSEDAttachment.py b/tests/scripts/thread-cert/v1_2_LowPower_5_3_01_SSEDAttachment.py index cb8fd8d5a..15bafe913 100755 --- a/tests/scripts/thread-cert/v1_2_LowPower_5_3_01_SSEDAttachment.py +++ b/tests/scripts/thread-cert/v1_2_LowPower_5_3_01_SSEDAttachment.py @@ -61,6 +61,7 @@ class LowPower_5_3_01_SSEDAttachment(thread_cert.TestCase): SSED_1: { 'version': '1.2', 'name': 'SSED_1', + 'is_mtd': True, 'mode': '-', 'panid': 0xface, 'allowlist': [LEADER], diff --git a/tests/scripts/thread-cert/v1_2_test_csl_transmission.py b/tests/scripts/thread-cert/v1_2_test_csl_transmission.py index 352ad4777..1b99ef162 100755 --- a/tests/scripts/thread-cert/v1_2_test_csl_transmission.py +++ b/tests/scripts/thread-cert/v1_2_test_csl_transmission.py @@ -47,6 +47,7 @@ class SSED_CslTransmission(thread_cert.TestCase): }, SSED_1: { 'version': '1.2', + 'is_mtd': True, 'mode': '-', }, }