From 354387c931811582476385afd52e20c6b77f0dcf Mon Sep 17 00:00:00 2001 From: Yakun Xu Date: Tue, 3 Aug 2021 11:26:32 +0800 Subject: [PATCH] [test] fix reporting expect failure (#6869) This commit fixes reporting expect failures by adding fail on error flag to the ot_exec_expect_script function in script/test script. This commit also fixes failing expect tests and removes expect tests on macOS for unstable issues. --- .github/workflows/posix.yml | 25 ------ script/test | 83 ++++++++++--------- src/cli/cli.cpp | 15 ++-- src/cli/cli_coap_secure.cpp | 11 ++- src/cli/cli_joiner.cpp | 1 + src/core/net/dns_client.cpp | 2 +- src/core/net/sntp_client.cpp | 2 +- tests/scripts/expect/cli-channel.exp | 4 +- tests/scripts/expect/cli-coaps.exp | 3 - tests/scripts/expect/cli-counters.exp | 2 +- tests/scripts/expect/cli-dataset.exp | 8 +- tests/scripts/expect/cli-ipmaddr.exp | 2 +- tests/scripts/expect/cli-mac.exp | 2 +- tests/scripts/expect/cli-misc.exp | 2 +- tests/scripts/expect/cli-ping.exp | 2 +- tests/scripts/expect/cli-promiscuous.exp | 3 +- tests/scripts/expect/cli-routereligible.exp | 2 +- tests/scripts/expect/cli-scan-discover.exp | 4 +- tests/scripts/expect/cli-udp.exp | 2 +- tests/scripts/expect/tun-netstat.exp | 6 +- .../expect/tun-realm-local-multicast.exp | 2 +- tests/scripts/expect/tun-udp.exp | 2 +- 22 files changed, 83 insertions(+), 102 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index c54eebc04..09b50d56b 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -40,31 +40,6 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" if: "github.ref != 'refs/heads/main'" - expects-macos: - runs-on: macos-10.15 - env: - CFLAGS: -DCLI_COAP_SECURE_USE_COAP_DEFAULT_HANDLER=1 -DOPENTHREAD_CONFIG_MLE_MAX_CHILDREN=15 -DOPENTHREAD_CONFIG_MAC_SCAN_DURATION=500 - CXXFLAGS: -DCLI_COAP_SECURE_USE_COAP_DEFAULT_HANDLER=1 -DOPENTHREAD_CONFIG_MLE_MAX_CHILDREN=15 -DOPENTHREAD_CONFIG_MAC_SCAN_DURATION=500 - CC: clang - CXX: clang++ - THREAD_VERSION: 1.1 - steps: - - uses: actions/checkout@v2 - - name: Bootstrap - run: | - brew update - brew install ninja - - name: Run RCP Mode - run: | - OT_OPTIONS='-DOT_READLINE=OFF -DOT_APP_NCP=OFF' OT_NODE_TYPE=rcp-cli ./script/test build expect - - name: Run Native IP Mode - run: | - brew install dnsmasq - echo 'listen-address=::1' | sudo tee $(brew --prefix)/etc/dnsmasq.conf - sudo brew services start dnsmasq - host ipv6.google.com ::1 - OT_OPTIONS=-DOT_READLINE=OFF OT_NATIVE_IP=1 OT_NODE_TYPE=rcp-cli ./script/test clean build expect - expects-linux: runs-on: ubuntu-18.04 env: diff --git a/script/test b/script/test index 6f502d003..65533870a 100755 --- a/script/test +++ b/script/test @@ -35,10 +35,10 @@ set -euo pipefail readonly OT_BUILDDIR="${OT_BUILDDIR:-${PWD}/build}" readonly OT_SRCDIR="${PWD}" -readonly COLOR_PASS='\033[0;32m' -readonly COLOR_FAIL='\033[0;31m' -readonly COLOR_SKIP='\033[0;33m' -readonly COLOR_NONE='\033[0m' +readonly OT_COLOR_PASS='\033[0;32m' +readonly OT_COLOR_FAIL='\033[0;31m' +readonly OT_COLOR_SKIP='\033[0;33m' +readonly OT_COLOR_NONE='\033[0m' readonly OT_NODE_TYPE="${OT_NODE_TYPE:-cli}" readonly OT_NATIVE_IP="${OT_NATIVE_IP:-0}" @@ -309,38 +309,39 @@ do_pktverify() ot_exec_expect_script() { local log_file="tmp/log_expect" - local script="$1" - echo -e "\n${COLOR_PASS}EXEC${COLOR_NONE} ${script}" - sudo killall ot-rcp || true - sudo killall ot-cli || true - sudo killall ot-cli-ftd || true - sudo killall ot-cli-mtd || true - sudo rm -rf tmp - mkdir tmp - { - if [[ ${OT_NATIVE_IP} == 1 ]]; then - sudo -E expect -df "${script}" 2>"${log_file}" - else - expect -df "${script}" 2>"${log_file}" - fi - } || { - local EXIT_CODE=$? + for script in "$@"; do + echo -e "\n${OT_COLOR_PASS}EXEC${OT_COLOR_NONE} ${script}" + sudo killall ot-rcp || true + sudo killall ot-cli || true + sudo killall ot-cli-ftd || true + sudo killall ot-cli-mtd || true + sudo rm -rf tmp + mkdir tmp + { + if [[ ${OT_NATIVE_IP} == 1 ]]; then + sudo -E expect -df "${script}" 2>"${log_file}" + else + expect -df "${script}" 2>"${log_file}" + fi + } || { + local EXIT_CODE=$? - # The exit status 77 for skipping is inherited from automake's test driver for script-based testsuites - if [[ ${EXIT_CODE} == 77 ]]; then - echo -e "\n${COLOR_SKIP}SKIP${COLOR_NONE} ${script}" - return 0 - else - echo -e "\n${COLOR_FAIL}FAIL${COLOR_NONE} ${script}" + # The exit status 77 for skipping is inherited from automake's test driver for script-based testsuites + if [[ ${EXIT_CODE} == 77 ]]; then + echo -e "\n${OT_COLOR_SKIP}SKIP${OT_COLOR_NONE} ${script}" + return 0 + else + echo -e "\n${OT_COLOR_FAIL}FAIL${OT_COLOR_NONE} ${script}" + cat "${log_file}" >&2 + return "${EXIT_CODE}" + fi + } + echo -e "\n${OT_COLOR_PASS}PASS${OT_COLOR_NONE} ${script}" + if [[ ${VERBOSE} == 1 ]]; then cat "${log_file}" >&2 - return "${EXIT_CODE}" fi - } - echo -e "\n${COLOR_PASS}PASS${COLOR_NONE} ${script}" - if [[ ${VERBOSE} == 1 ]]; then - cat "${log_file}" >&2 - fi + done } do_expect() @@ -348,23 +349,25 @@ do_expect() local test_patterns if [[ ${OT_NODE_TYPE} == rcp* ]]; then - if [[ ${THREAD_VERSION} == "1.2" ]]; then - test_patterns=(-name 'v1_2-*.exp') - elif [[ ${OT_NATIVE_IP} == 1 ]]; then + if [[ ${OT_NATIVE_IP} == 1 ]]; then test_patterns=(-name 'tun-*.exp') else test_patterns=(-name 'posix-*.exp' -o -name 'cli-*.exp') + if [[ ${THREAD_VERSION} == "1.2" ]]; then + test_patterns+=(-o -name 'v1_2-*.exp') + fi fi else test_patterns=(-name 'cli-*.exp' -o -name 'simulation-*.exp') fi - export -f ot_exec_expect_script - if [[ $# != 0 ]]; then - for script in "$@"; do bash -c "ot_exec_expect_script ${script}"; done + ot_exec_expect_script "$@" else - find tests/scripts/expect -type f -perm "$([[ $OSTYPE == darwin* ]] && echo '+' || echo '/')"111 \( "${test_patterns[@]}" \) -exec bash -c 'ot_exec_expect_script "$1"' _ {} \; + export OT_COLOR_PASS OT_COLOR_FAIL OT_COLOR_SKIP OT_COLOR_NONE OT_NATIVE_IP VERBOSE + export -f ot_exec_expect_script + + find tests/scripts/expect -type f -perm "$([[ $OSTYPE == darwin* ]] && echo '+' || echo '/')"111 \( "${test_patterns[@]}" \) -exec bash -c 'set -euo pipefail;ot_exec_expect_script "$@"' _ {} + fi exit 0 @@ -628,7 +631,7 @@ main() ;; *) echo - echo -e "${COLOR_FAIL}Warning:${COLOR_NONE} Ignoring: '$1'" + echo -e "${OT_COLOR_FAIL}Warning:${OT_COLOR_NONE} Ignoring: '$1'" ;; esac shift diff --git a/src/cli/cli.cpp b/src/cli/cli.cpp index 60391bb6c..51105ecad 100644 --- a/src/cli/cli.cpp +++ b/src/cli/cli.cpp @@ -707,11 +707,7 @@ exit: otError Interpreter::ProcessChannel(Arg aArgs[]) { - otError error; - - error = ProcessGetSet(aArgs, otLinkGetChannel, otLinkSetChannel); - - VerifyOrExit(error != OT_ERROR_NONE); + otError error = OT_ERROR_NONE; if (aArgs[0] == "supported") { @@ -830,8 +826,16 @@ otError Interpreter::ProcessChannel(Arg aArgs[]) { error = ProcessSet(aArgs + 2, otChannelManagerSetCcaFailureRateThreshold); } + else + { + ExitNow(error = OT_ERROR_INVALID_ARGS); + } } #endif // OPENTHREAD_CONFIG_CHANNEL_MANAGER_ENABLE && OPENTHREAD_FTD + else + { + ExitNow(error = ProcessGetSet(aArgs, otLinkGetChannel, otLinkSetChannel)); + } exit: return error; @@ -1767,6 +1771,7 @@ otError Interpreter::ProcessLog(Arg aArgs[]) #if OPENTHREAD_CONFIG_LOG_LEVEL_DYNAMIC_ENABLE uint8_t level; + VerifyOrExit(aArgs[2].IsEmpty(), error = OT_ERROR_INVALID_ARGS); SuccessOrExit(error = aArgs[1].ParseAsUint8(level)); error = otLoggingSetLevel(static_cast(level)); #else diff --git a/src/cli/cli_coap_secure.cpp b/src/cli/cli_coap_secure.cpp index 2b2505a42..ad37468cc 100644 --- a/src/cli/cli_coap_secure.cpp +++ b/src/cli/cli_coap_secure.cpp @@ -240,18 +240,17 @@ otError CoapSecure::ProcessRequest(Arg aArgs[], otCoapCode aCoapCode) uint16_t payloadLength = 0; // Default parameters - char coapUri[kMaxUriLength] = "test"; - otCoapType coapType = OT_COAP_TYPE_NON_CONFIRMABLE; + char coapUri[kMaxUriLength]; + otCoapType coapType = OT_COAP_TYPE_NON_CONFIRMABLE; #if OPENTHREAD_CONFIG_COAP_BLOCKWISE_TRANSFER_ENABLE bool coapBlock = false; otCoapBlockSzx coapBlockSize = OT_COAP_OPTION_BLOCK_SZX_16; BlockType coapBlockType = (aCoapCode == OT_COAP_CODE_GET) ? kBlockType2 : kBlockType1; #endif - if (!aArgs[0].IsEmpty()) - { - strncpy(coapUri, aArgs[0].GetCString(), sizeof(coapUri) - 1); - } + VerifyOrExit(!aArgs[0].IsEmpty(), error = OT_ERROR_INVALID_ARGS); + + strncpy(coapUri, aArgs[0].GetCString(), sizeof(coapUri) - 1); if (!aArgs[1].IsEmpty()) { diff --git a/src/cli/cli_joiner.cpp b/src/cli/cli_joiner.cpp index ea0a351c1..435fedc13 100644 --- a/src/cli/cli_joiner.cpp +++ b/src/cli/cli_joiner.cpp @@ -69,6 +69,7 @@ otError Joiner::ProcessDiscerner(Arg aArgs[]) } else { + VerifyOrExit(aArgs[1].IsEmpty()); SuccessOrExit(Interpreter::ParseJoinerDiscerner(aArgs[0], discerner)); error = otJoinerSetDiscerner(mInterpreter.mInstance, &discerner); } diff --git a/src/core/net/dns_client.cpp b/src/core/net/dns_client.cpp index 94980c0b5..bcb8c40ed 100644 --- a/src/core/net/dns_client.cpp +++ b/src/core/net/dns_client.cpp @@ -579,7 +579,7 @@ Error Client::Start(void) Error error; SuccessOrExit(error = mSocket.Open(&Client::HandleUdpReceive, this)); - SuccessOrExit(error = mSocket.Bind()); + SuccessOrExit(error = mSocket.Bind(0, OT_NETIF_UNSPECIFIED)); exit: return error; diff --git a/src/core/net/sntp_client.cpp b/src/core/net/sntp_client.cpp index 2199c54d8..1269cbe82 100644 --- a/src/core/net/sntp_client.cpp +++ b/src/core/net/sntp_client.cpp @@ -102,7 +102,7 @@ Error Client::Start(void) Error error; SuccessOrExit(error = mSocket.Open(&Client::HandleUdpReceive, this)); - SuccessOrExit(error = mSocket.Bind()); + SuccessOrExit(error = mSocket.Bind(0, OT_NETIF_UNSPECIFIED)); exit: return error; diff --git a/tests/scripts/expect/cli-channel.exp b/tests/scripts/expect/cli-channel.exp index 238bac98d..8250a6ed7 100755 --- a/tests/scripts/expect/cli-channel.exp +++ b/tests/scripts/expect/cli-channel.exp @@ -80,8 +80,8 @@ expect "channel: 15" expect "auto: 1" expect "delay: 200" expect "interval: 20000" -expect "supported: { 11-26}" -expect "favored: { 11-26}" +expect "supported: { 11-26 }" +expect "favored: { 11-26 }" expect_line "Done" send "channel manager something_invalid\n" expect "Error 7: InvalidArgs" diff --git a/tests/scripts/expect/cli-coaps.exp b/tests/scripts/expect/cli-coaps.exp index 1629da33c..1b0c3db86 100755 --- a/tests/scripts/expect/cli-coaps.exp +++ b/tests/scripts/expect/cli-coaps.exp @@ -66,9 +66,6 @@ expect_line "Done" expect "coaps response from $addr_1" send "coaps post test/resource none Testing123\n" expect_line "Done" -send "coaps get default\n" -expect_line "Done" -expect "coaps response from $addr_1" set addr_2 [get_ipaddr mleid] switch_node 1 diff --git a/tests/scripts/expect/cli-counters.exp b/tests/scripts/expect/cli-counters.exp index 6c6f33e65..bb2d35857 100755 --- a/tests/scripts/expect/cli-counters.exp +++ b/tests/scripts/expect/cli-counters.exp @@ -32,9 +32,9 @@ source "tests/scripts/expect/_common.exp" spawn_node 1 send "counters\n" +expect "ip" expect "mac" expect "mle" -expect "ip" expect_line "Done" send "counters mac\n" expect_line "Done" diff --git a/tests/scripts/expect/cli-dataset.exp b/tests/scripts/expect/cli-dataset.exp index 5075f870c..19d8ba185 100755 --- a/tests/scripts/expect/cli-dataset.exp +++ b/tests/scripts/expect/cli-dataset.exp @@ -44,7 +44,7 @@ expect -re {Network Key: [0-9a-f]{32}} expect -re {Network Name: [^\r\n]+} expect -re {PAN ID: 0x[0-9a-f]{4}} expect -re {PSKc: [0-9a-f]{32}} -expect -re {Security Policy: \d+, o?n?r?c?b?} +expect -re {Security Policy: \d+ o?n?r?c?b?} send "dataset pending\n" expect "Error 23: NotFound" send "dataset init active\n" @@ -135,7 +135,7 @@ expect "Network Key: aabbccddeeff00112233445566778899" expect "Network Name: OT-network" expect "PAN ID: 0xface" expect "PSKc: 00112233445566778899aabbccddeeff" -expect "Security Policy: 678, onrcb" +expect "Security Policy: 678 onrcb" expect_line "Done" sleep 30 @@ -157,7 +157,7 @@ expect "Network Key: aabbccddeeff00112233445566778899" expect "Network Name: OT-network" expect "PAN ID: 0xface" expect "PSKc: 00112233445566778899aabbccddeeff" -expect "Security Policy: 678, onrcb" +expect "Security Policy: 678 onrcb" expect_line "Done" send "dataset clear\n" expect_line "Done" @@ -177,7 +177,7 @@ expect "Network Key: aabbccddeeff00112233445566778899" expect "Network Name: OT-network" expect "PAN ID: 0xface" expect "PSKc: 00112233445566778899aabbccddeeff" -expect "Security Policy: 678, onrcb" +expect "Security Policy: 678 onrcb" expect_line "Done" send "dataset init pending\n" expect "Error 23: NotFound" diff --git a/tests/scripts/expect/cli-ipmaddr.exp b/tests/scripts/expect/cli-ipmaddr.exp index d44013d3b..0c9babd6d 100755 --- a/tests/scripts/expect/cli-ipmaddr.exp +++ b/tests/scripts/expect/cli-ipmaddr.exp @@ -51,8 +51,8 @@ expect_line "Done" switch_node 2 send "ping ff0e::1\n" -expect_line "Done" expect "16 bytes from $addr: icmp_seq=1" +expect_line "Done" switch_node 1 send "ipmaddr del ff0e::1\n" diff --git a/tests/scripts/expect/cli-mac.exp b/tests/scripts/expect/cli-mac.exp index 0d3054410..66af468d5 100755 --- a/tests/scripts/expect/cli-mac.exp +++ b/tests/scripts/expect/cli-mac.exp @@ -44,7 +44,7 @@ expect "2" expect_line "Done" send "mac\n" -expect "Error 7: InvalidArgs" +expect "Error 35: InvalidCommand" send "mac something_invalid\n" expect "Error 35: InvalidCommand" send "mac retries\n" diff --git a/tests/scripts/expect/cli-misc.exp b/tests/scripts/expect/cli-misc.exp index 66174a70c..89da8075e 100755 --- a/tests/scripts/expect/cli-misc.exp +++ b/tests/scripts/expect/cli-misc.exp @@ -167,7 +167,7 @@ expect_line "Done" send "route add ::/0 s low\n" expect_line "Done" send "route\n" -expect "0:0:0:0::/0 s low" +expect "::/0 s low" send "route remove ::/0\n" expect_line "Done" diff --git a/tests/scripts/expect/cli-ping.exp b/tests/scripts/expect/cli-ping.exp index db24d4728..edef8f23a 100755 --- a/tests/scripts/expect/cli-ping.exp +++ b/tests/scripts/expect/cli-ping.exp @@ -41,7 +41,7 @@ send "ping ::1 1 2 0.12345 1\n" expect_line "Done" send "ping stop\n" expect_line "Done" -send "ping ::1 1 2 1 1 1\n" +send "ping ::1 1 2 1 1 1 1\n" expect "Error 7: InvalidArgs" dispose_all diff --git a/tests/scripts/expect/cli-promiscuous.exp b/tests/scripts/expect/cli-promiscuous.exp index 61acf4987..228c21cf6 100755 --- a/tests/scripts/expect/cli-promiscuous.exp +++ b/tests/scripts/expect/cli-promiscuous.exp @@ -44,7 +44,7 @@ expect_line "Done" send "promiscuous disable\n" expect_line "Done" send "promiscuous a\n" -expect "Error 7: InvalidArgs" +expect "Error 35: InvalidCommand" dispose_all @@ -73,6 +73,7 @@ expect_line "Done" switch_node 1 send "ping $addr\n" expect "16 bytes from $addr: icmp_seq=1" +expect_line "Done" switch_node 3 expect -re {============================================\[len = +\d+]============================} diff --git a/tests/scripts/expect/cli-routereligible.exp b/tests/scripts/expect/cli-routereligible.exp index 821b1272b..f30dfb0fd 100755 --- a/tests/scripts/expect/cli-routereligible.exp +++ b/tests/scripts/expect/cli-routereligible.exp @@ -42,7 +42,7 @@ send "routereligible\n" expect "Enabled" expect_line "Done" send "routereligible something_invalid\n" -expect "Error 7: InvalidArgs" +expect "Error 35: InvalidCommand" send "mode r\n" expect_line "Done" send "routereligible\n" diff --git a/tests/scripts/expect/cli-scan-discover.exp b/tests/scripts/expect/cli-scan-discover.exp index 812b88298..54a4ce106 100755 --- a/tests/scripts/expect/cli-scan-discover.exp +++ b/tests/scripts/expect/cli-scan-discover.exp @@ -89,7 +89,7 @@ send "discover something_invalid\n" expect "Error 7: InvalidArgs" switch_node 1 -expect "version=2,joiner=0" +expect -re {version=\d,joiner=0} switch_node 3 send "joiner start 123456\n" @@ -97,6 +97,6 @@ set timeout 10 expect "NotFound" switch_node 1 -expect "version=2,joiner=1" +expect -re {version=\d,joiner=1} dispose_all diff --git a/tests/scripts/expect/cli-udp.exp b/tests/scripts/expect/cli-udp.exp index 2a78f7ed9..071117252 100755 --- a/tests/scripts/expect/cli-udp.exp +++ b/tests/scripts/expect/cli-udp.exp @@ -64,7 +64,7 @@ switch_node 1 send "udp help\n" expect_line "Done" send "udp connect something_invalid\n" -expect "Error 7: InvalidArgs" +expect "Error 6: Parse" send "udp send -x something_invalid\n" expect "Error 7: InvalidArgs" send "udp\n" diff --git a/tests/scripts/expect/tun-netstat.exp b/tests/scripts/expect/tun-netstat.exp index 0a6cee979..50895b5c9 100755 --- a/tests/scripts/expect/tun-netstat.exp +++ b/tests/scripts/expect/tun-netstat.exp @@ -37,14 +37,14 @@ expect_line "Done" send "netstat\n" expect "| Local Address | Peer Address |" expect "+-------------------------------------------------+-------------------------------------------------+" -expect "| [0:0:0:0:0:0:0:0]:0 | [0:0:0:0:0:0:0:0]:0 |" +expect -ex "| \[0:0:0:0:0:0:0:0]:0 | \[0:0:0:0:0:0:0:0]:0 |" expect_line "Done" send "udp bind :: 10001\n" expect_line "Done" send "netstat\n" expect "| Local Address | Peer Address |" expect "+-------------------------------------------------+-------------------------------------------------+" -expect "| [0:0:0:0:0:0:0:0]:1001 | [0:0:0:0:0:0:0:0]:0 |" +expect -ex "| \[0:0:0:0:0:0:0:0]:10001 | \[0:0:0:0:0:0:0:0]:0 |" expect_line "Done" set addr [get_ipaddr mleid] send "udp connect $addr 10001\n" @@ -52,7 +52,7 @@ expect_line "Done" send "netstat\n" expect "| Local Address | Peer Address |" expect "+-------------------------------------------------+-------------------------------------------------+" -expect -re "\\| \\[0:0:0:0:0:0:0:0\\]:10001 +\\| \\[$addr\\]:10001 +\\|" +expect -re "\\| \\\[0:0:0:0:0:0:0:0]:10001 +\\| \\\[$addr]:10001 +\\|" expect_line "Done" dispose_all diff --git a/tests/scripts/expect/tun-realm-local-multicast.exp b/tests/scripts/expect/tun-realm-local-multicast.exp index e8c054347..b18ffba22 100755 --- a/tests/scripts/expect/tun-realm-local-multicast.exp +++ b/tests/scripts/expect/tun-realm-local-multicast.exp @@ -107,7 +107,7 @@ set mleid4 [get_ipaddr "mleid"] switch_node 1 send "ping ${mleid4}\n" -expect_line "Done" expect "16 bytes from $mleid4: icmp_seq=1" +expect_line "Done" dispose_all diff --git a/tests/scripts/expect/tun-udp.exp b/tests/scripts/expect/tun-udp.exp index 187bda48a..03ba12f15 100755 --- a/tests/scripts/expect/tun-udp.exp +++ b/tests/scripts/expect/tun-udp.exp @@ -61,7 +61,7 @@ expect_line "Done" send "udp connect $prefix:bb1:ebd6:ad10:f33 1234\n" expect_line "Done" -send "udp connect :: 1\n" +send "udp connect $prefix:1:2:3:4 4321\n" expect_line "Done" send "udp close\n"