[test] fix reporting expect failure (#6869)

This commit fixes reporting expect failures by adding fail on error
flag to the ot_exec_expect_script function in script/test script.

This commit also fixes failing expect tests and removes expect tests
on macOS for unstable issues.

.github/workflows/posix.yml

@@ -40,31 +40,6 @@ jobs:
         GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
       if: "github.ref != 'refs/heads/main'"
 
-  expects-macos:
-    runs-on: macos-10.15
-    env:
-      CFLAGS: -DCLI_COAP_SECURE_USE_COAP_DEFAULT_HANDLER=1 -DOPENTHREAD_CONFIG_MLE_MAX_CHILDREN=15 -DOPENTHREAD_CONFIG_MAC_SCAN_DURATION=500
-      CXXFLAGS: -DCLI_COAP_SECURE_USE_COAP_DEFAULT_HANDLER=1 -DOPENTHREAD_CONFIG_MLE_MAX_CHILDREN=15 -DOPENTHREAD_CONFIG_MAC_SCAN_DURATION=500
-      CC: clang
-      CXX: clang++
-      THREAD_VERSION: 1.1
-    steps:
-    - uses: actions/checkout@v2
-    - name: Bootstrap
-      run: |
-        brew update
-        brew install ninja
-    - name: Run RCP Mode
-      run: |
-        OT_OPTIONS='-DOT_READLINE=OFF -DOT_APP_NCP=OFF' OT_NODE_TYPE=rcp-cli ./script/test build expect
-    - name: Run Native IP Mode
-      run: |
-        brew install dnsmasq
-        echo 'listen-address=::1' | sudo tee $(brew --prefix)/etc/dnsmasq.conf
-        sudo brew services start dnsmasq
-        host ipv6.google.com ::1
-        OT_OPTIONS=-DOT_READLINE=OFF OT_NATIVE_IP=1 OT_NODE_TYPE=rcp-cli ./script/test clean build expect
-
   expects-linux:
     runs-on: ubuntu-18.04
     env:

script/test

@@ -35,10 +35,10 @@ set -euo pipefail
 readonly OT_BUILDDIR="${OT_BUILDDIR:-${PWD}/build}"
 readonly OT_SRCDIR="${PWD}"
 
-readonly COLOR_PASS='\033[0;32m'
-readonly COLOR_FAIL='\033[0;31m'
-readonly COLOR_SKIP='\033[0;33m'
-readonly COLOR_NONE='\033[0m'
+readonly OT_COLOR_PASS='\033[0;32m'
+readonly OT_COLOR_FAIL='\033[0;31m'
+readonly OT_COLOR_SKIP='\033[0;33m'
+readonly OT_COLOR_NONE='\033[0m'
 
 readonly OT_NODE_TYPE="${OT_NODE_TYPE:-cli}"
 readonly OT_NATIVE_IP="${OT_NATIVE_IP:-0}"
@@ -309,38 +309,39 @@ do_pktverify()
 ot_exec_expect_script()
 {
     local log_file="tmp/log_expect"
-    local script="$1"
 
-    echo -e "\n${COLOR_PASS}EXEC${COLOR_NONE} ${script}"
-    sudo killall ot-rcp || true
-    sudo killall ot-cli || true
-    sudo killall ot-cli-ftd || true
-    sudo killall ot-cli-mtd || true
-    sudo rm -rf tmp
-    mkdir tmp
-    {
-        if [[ ${OT_NATIVE_IP} == 1 ]]; then
-            sudo -E expect -df "${script}" 2>"${log_file}"
-        else
-            expect -df "${script}" 2>"${log_file}"
-        fi
-    } || {
-        local EXIT_CODE=$?
+    for script in "$@"; do
+        echo -e "\n${OT_COLOR_PASS}EXEC${OT_COLOR_NONE} ${script}"
+        sudo killall ot-rcp || true
+        sudo killall ot-cli || true
+        sudo killall ot-cli-ftd || true
+        sudo killall ot-cli-mtd || true
+        sudo rm -rf tmp
+        mkdir tmp
+        {
+            if [[ ${OT_NATIVE_IP} == 1 ]]; then
+                sudo -E expect -df "${script}" 2>"${log_file}"
+            else
+                expect -df "${script}" 2>"${log_file}"
+            fi
+        } || {
+            local EXIT_CODE=$?
 
-        # The exit status 77 for skipping is inherited from automake's test driver for script-based testsuites
-        if [[ ${EXIT_CODE} == 77 ]]; then
-            echo -e "\n${COLOR_SKIP}SKIP${COLOR_NONE} ${script}"
-            return 0
-        else
-            echo -e "\n${COLOR_FAIL}FAIL${COLOR_NONE} ${script}"
+            # The exit status 77 for skipping is inherited from automake's test driver for script-based testsuites
+            if [[ ${EXIT_CODE} == 77 ]]; then
+                echo -e "\n${OT_COLOR_SKIP}SKIP${OT_COLOR_NONE} ${script}"
+                return 0
+            else
+                echo -e "\n${OT_COLOR_FAIL}FAIL${OT_COLOR_NONE} ${script}"
+                cat "${log_file}" >&2
+                return "${EXIT_CODE}"
+            fi
+        }
+        echo -e "\n${OT_COLOR_PASS}PASS${OT_COLOR_NONE} ${script}"
+        if [[ ${VERBOSE} == 1 ]]; then
             cat "${log_file}" >&2
-            return "${EXIT_CODE}"
         fi
-    }
-    echo -e "\n${COLOR_PASS}PASS${COLOR_NONE} ${script}"
-    if [[ ${VERBOSE} == 1 ]]; then
-        cat "${log_file}" >&2
-    fi
+    done
 }
 
 do_expect()
@@ -348,23 +349,25 @@ do_expect()
     local test_patterns
 
     if [[ ${OT_NODE_TYPE} == rcp* ]]; then
-        if [[ ${THREAD_VERSION} == "1.2" ]]; then
-            test_patterns=(-name 'v1_2-*.exp')
-        elif [[ ${OT_NATIVE_IP} == 1 ]]; then
+        if [[ ${OT_NATIVE_IP} == 1 ]]; then
             test_patterns=(-name 'tun-*.exp')
         else
             test_patterns=(-name 'posix-*.exp' -o -name 'cli-*.exp')
+            if [[ ${THREAD_VERSION} == "1.2" ]]; then
+                test_patterns+=(-o -name 'v1_2-*.exp')
+            fi
         fi
     else
         test_patterns=(-name 'cli-*.exp' -o -name 'simulation-*.exp')
     fi
 
-    export -f ot_exec_expect_script
-
     if [[ $# != 0 ]]; then
-        for script in "$@"; do bash -c "ot_exec_expect_script ${script}"; done
+        ot_exec_expect_script "$@"
     else
-        find tests/scripts/expect -type f -perm "$([[ $OSTYPE == darwin* ]] && echo '+' || echo '/')"111 \( "${test_patterns[@]}" \) -exec bash -c 'ot_exec_expect_script "$1"' _ {} \;
+        export OT_COLOR_PASS OT_COLOR_FAIL OT_COLOR_SKIP OT_COLOR_NONE OT_NATIVE_IP VERBOSE
+        export -f ot_exec_expect_script
+
+        find tests/scripts/expect -type f -perm "$([[ $OSTYPE == darwin* ]] && echo '+' || echo '/')"111 \( "${test_patterns[@]}" \) -exec bash -c 'set -euo pipefail;ot_exec_expect_script "$@"' _ {} +
     fi
 
     exit 0
@@ -628,7 +631,7 @@ main()
                 ;;
             *)
                 echo
-                echo -e "${COLOR_FAIL}Warning:${COLOR_NONE} Ignoring: '$1'"
+                echo -e "${OT_COLOR_FAIL}Warning:${OT_COLOR_NONE} Ignoring: '$1'"
                 ;;
         esac
         shift

src/cli/cli.cpp

@@ -707,11 +707,7 @@ exit:
 
 otError Interpreter::ProcessChannel(Arg aArgs[])
 {
-    otError error;
-
-    error = ProcessGetSet(aArgs, otLinkGetChannel, otLinkSetChannel);
-
-    VerifyOrExit(error != OT_ERROR_NONE);
+    otError error = OT_ERROR_NONE;
 
     if (aArgs[0] == "supported")
     {
@@ -830,8 +826,16 @@ otError Interpreter::ProcessChannel(Arg aArgs[])
         {
             error = ProcessSet(aArgs + 2, otChannelManagerSetCcaFailureRateThreshold);
         }
+        else
+        {
+            ExitNow(error = OT_ERROR_INVALID_ARGS);
+        }
     }
 #endif // OPENTHREAD_CONFIG_CHANNEL_MANAGER_ENABLE && OPENTHREAD_FTD
+    else
+    {
+        ExitNow(error = ProcessGetSet(aArgs, otLinkGetChannel, otLinkSetChannel));
+    }
 
 exit:
     return error;
@@ -1767,6 +1771,7 @@ otError Interpreter::ProcessLog(Arg aArgs[])
 #if OPENTHREAD_CONFIG_LOG_LEVEL_DYNAMIC_ENABLE
             uint8_t level;
 
+            VerifyOrExit(aArgs[2].IsEmpty(), error = OT_ERROR_INVALID_ARGS);
             SuccessOrExit(error = aArgs[1].ParseAsUint8(level));
             error = otLoggingSetLevel(static_cast<otLogLevel>(level));
 #else

src/cli/cli_coap_secure.cpp

@@ -240,18 +240,17 @@ otError CoapSecure::ProcessRequest(Arg aArgs[], otCoapCode aCoapCode)
     uint16_t   payloadLength = 0;
 
     // Default parameters
-    char       coapUri[kMaxUriLength] = "test";
-    otCoapType coapType               = OT_COAP_TYPE_NON_CONFIRMABLE;
+    char       coapUri[kMaxUriLength];
+    otCoapType coapType = OT_COAP_TYPE_NON_CONFIRMABLE;
 #if OPENTHREAD_CONFIG_COAP_BLOCKWISE_TRANSFER_ENABLE
     bool           coapBlock     = false;
     otCoapBlockSzx coapBlockSize = OT_COAP_OPTION_BLOCK_SZX_16;
     BlockType      coapBlockType = (aCoapCode == OT_COAP_CODE_GET) ? kBlockType2 : kBlockType1;
 #endif
 
-    if (!aArgs[0].IsEmpty())
-    {
-        strncpy(coapUri, aArgs[0].GetCString(), sizeof(coapUri) - 1);
-    }
+    VerifyOrExit(!aArgs[0].IsEmpty(), error = OT_ERROR_INVALID_ARGS);
+
+    strncpy(coapUri, aArgs[0].GetCString(), sizeof(coapUri) - 1);
 
     if (!aArgs[1].IsEmpty())
     {

src/cli/cli_joiner.cpp

@@ -69,6 +69,7 @@ otError Joiner::ProcessDiscerner(Arg aArgs[])
         }
         else
         {
+            VerifyOrExit(aArgs[1].IsEmpty());
             SuccessOrExit(Interpreter::ParseJoinerDiscerner(aArgs[0], discerner));
             error = otJoinerSetDiscerner(mInterpreter.mInstance, &discerner);
         }

src/core/net/dns_client.cpp

@@ -579,7 +579,7 @@ Error Client::Start(void)
     Error error;
 
     SuccessOrExit(error = mSocket.Open(&Client::HandleUdpReceive, this));
-    SuccessOrExit(error = mSocket.Bind());
+    SuccessOrExit(error = mSocket.Bind(0, OT_NETIF_UNSPECIFIED));
 
 exit:
     return error;

src/core/net/sntp_client.cpp

@@ -102,7 +102,7 @@ Error Client::Start(void)
     Error error;
 
     SuccessOrExit(error = mSocket.Open(&Client::HandleUdpReceive, this));
-    SuccessOrExit(error = mSocket.Bind());
+    SuccessOrExit(error = mSocket.Bind(0, OT_NETIF_UNSPECIFIED));
 
 exit:
     return error;

tests/scripts/expect/cli-channel.exp

@@ -80,8 +80,8 @@ expect "channel: 15"
 expect "auto: 1"
 expect "delay: 200"
 expect "interval: 20000"
-expect "supported: { 11-26}"
-expect "favored: { 11-26}"
+expect "supported: { 11-26 }"
+expect "favored: { 11-26 }"
 expect_line "Done"
 send "channel manager something_invalid\n"
 expect "Error 7: InvalidArgs"

tests/scripts/expect/cli-coaps.exp

@@ -66,9 +66,6 @@ expect_line "Done"
 expect "coaps response from $addr_1"
 send "coaps post test/resource none Testing123\n"
 expect_line "Done"
-send "coaps get default\n"
-expect_line "Done"
-expect "coaps response from $addr_1"
 set addr_2 [get_ipaddr mleid]
 
 switch_node 1

tests/scripts/expect/cli-counters.exp

@@ -32,9 +32,9 @@ source "tests/scripts/expect/_common.exp"
 spawn_node 1
 
 send "counters\n"
+expect "ip"
 expect "mac"
 expect "mle"
-expect "ip"
 expect_line "Done"
 send "counters mac\n"
 expect_line "Done"

tests/scripts/expect/cli-dataset.exp

@@ -44,7 +44,7 @@ expect -re {Network Key: [0-9a-f]{32}}
 expect -re {Network Name: [^\r\n]+}
 expect -re {PAN ID: 0x[0-9a-f]{4}}
 expect -re {PSKc: [0-9a-f]{32}}
-expect -re {Security Policy: \d+, o?n?r?c?b?}
+expect -re {Security Policy: \d+ o?n?r?c?b?}
 send "dataset pending\n"
 expect "Error 23: NotFound"
 send "dataset init active\n"
@@ -135,7 +135,7 @@ expect "Network Key: aabbccddeeff00112233445566778899"
 expect "Network Name: OT-network"
 expect "PAN ID: 0xface"
 expect "PSKc: 00112233445566778899aabbccddeeff"
-expect "Security Policy: 678, onrcb"
+expect "Security Policy: 678 onrcb"
 expect_line "Done"
 
 sleep 30
@@ -157,7 +157,7 @@ expect "Network Key: aabbccddeeff00112233445566778899"
 expect "Network Name: OT-network"
 expect "PAN ID: 0xface"
 expect "PSKc: 00112233445566778899aabbccddeeff"
-expect "Security Policy: 678, onrcb"
+expect "Security Policy: 678 onrcb"
 expect_line "Done"
 send "dataset clear\n"
 expect_line "Done"
@@ -177,7 +177,7 @@ expect "Network Key: aabbccddeeff00112233445566778899"
 expect "Network Name: OT-network"
 expect "PAN ID: 0xface"
 expect "PSKc: 00112233445566778899aabbccddeeff"
-expect "Security Policy: 678, onrcb"
+expect "Security Policy: 678 onrcb"
 expect_line "Done"
 send "dataset init pending\n"
 expect "Error 23: NotFound"

tests/scripts/expect/cli-ipmaddr.exp

@@ -51,8 +51,8 @@ expect_line "Done"
 
 switch_node 2
 send "ping ff0e::1\n"
-expect_line "Done"
 expect "16 bytes from $addr: icmp_seq=1"
+expect_line "Done"
 
 switch_node 1
 send "ipmaddr del ff0e::1\n"

tests/scripts/expect/cli-mac.exp

@@ -44,7 +44,7 @@ expect "2"
 expect_line "Done"
 
 send "mac\n"
-expect "Error 7: InvalidArgs"
+expect "Error 35: InvalidCommand"
 send "mac something_invalid\n"
 expect "Error 35: InvalidCommand"
 send "mac retries\n"

tests/scripts/expect/cli-misc.exp

@@ -167,7 +167,7 @@ expect_line "Done"
 send "route add ::/0 s low\n"
 expect_line "Done"
 send "route\n"
-expect "0:0:0:0::/0 s low"
+expect "::/0 s low"
 send "route remove ::/0\n"
 expect_line "Done"
 

tests/scripts/expect/cli-ping.exp

@@ -41,7 +41,7 @@ send "ping ::1 1 2 0.12345 1\n"
 expect_line "Done"
 send "ping stop\n"
 expect_line "Done"
-send "ping ::1 1 2 1 1 1\n"
+send "ping ::1 1 2 1 1 1 1\n"
 expect "Error 7: InvalidArgs"
 
 dispose_all

tests/scripts/expect/cli-promiscuous.exp

@@ -44,7 +44,7 @@ expect_line "Done"
 send "promiscuous disable\n"
 expect_line "Done"
 send "promiscuous a\n"
-expect "Error 7: InvalidArgs"
+expect "Error 35: InvalidCommand"
 
 dispose_all
 
@@ -73,6 +73,7 @@ expect_line "Done"
 switch_node 1
 send "ping $addr\n"
 expect "16 bytes from $addr: icmp_seq=1"
+expect_line "Done"
 
 switch_node 3
 expect -re {============================================\[len = +\d+]============================}

tests/scripts/expect/cli-routereligible.exp

@@ -42,7 +42,7 @@ send "routereligible\n"
 expect "Enabled"
 expect_line "Done"
 send "routereligible something_invalid\n"
-expect "Error 7: InvalidArgs"
+expect "Error 35: InvalidCommand"
 send "mode r\n"
 expect_line "Done"
 send "routereligible\n"

tests/scripts/expect/cli-scan-discover.exp

@@ -89,7 +89,7 @@ send "discover something_invalid\n"
 expect "Error 7: InvalidArgs"
 
 switch_node 1
-expect "version=2,joiner=0"
+expect -re {version=\d,joiner=0}
 
 switch_node 3
 send "joiner start 123456\n"
@@ -97,6 +97,6 @@ set timeout 10
 expect "NotFound"
 
 switch_node 1
-expect "version=2,joiner=1"
+expect -re {version=\d,joiner=1}
 
 dispose_all

tests/scripts/expect/cli-udp.exp

@@ -64,7 +64,7 @@ switch_node 1
 send "udp help\n"
 expect_line "Done"
 send "udp connect something_invalid\n"
-expect "Error 7: InvalidArgs"
+expect "Error 6: Parse"
 send "udp send -x something_invalid\n"
 expect "Error 7: InvalidArgs"
 send "udp\n"

tests/scripts/expect/tun-netstat.exp

@@ -37,14 +37,14 @@ expect_line "Done"
 send "netstat\n"
 expect "| Local Address                                   | Peer Address                                    |"
 expect "+-------------------------------------------------+-------------------------------------------------+"
-expect "| [0:0:0:0:0:0:0:0]:0                             | [0:0:0:0:0:0:0:0]:0                             |"
+expect -ex "| \[0:0:0:0:0:0:0:0]:0                             | \[0:0:0:0:0:0:0:0]:0                             |"
 expect_line "Done"
 send "udp bind :: 10001\n"
 expect_line "Done"
 send "netstat\n"
 expect "| Local Address                                   | Peer Address                                    |"
 expect "+-------------------------------------------------+-------------------------------------------------+"
-expect "| [0:0:0:0:0:0:0:0]:1001                          | [0:0:0:0:0:0:0:0]:0                             |"
+expect -ex "| \[0:0:0:0:0:0:0:0]:10001                         | \[0:0:0:0:0:0:0:0]:0                             |"
 expect_line "Done"
 set addr [get_ipaddr mleid]
 send "udp connect $addr 10001\n"
@@ -52,7 +52,7 @@ expect_line "Done"
 send "netstat\n"
 expect "| Local Address                                   | Peer Address                                    |"
 expect "+-------------------------------------------------+-------------------------------------------------+"
-expect -re "\\| \\[0:0:0:0:0:0:0:0\\]:10001 +\\| \\[$addr\\]:10001 +\\|"
+expect -re "\\| \\\[0:0:0:0:0:0:0:0]:10001 +\\| \\\[$addr]:10001 +\\|"
 expect_line "Done"
 
 dispose_all

tests/scripts/expect/tun-realm-local-multicast.exp

@@ -107,7 +107,7 @@ set mleid4 [get_ipaddr "mleid"]
 switch_node 1
 
 send "ping ${mleid4}\n"
-expect_line "Done"
 expect "16 bytes from $mleid4: icmp_seq=1"
+expect_line "Done"
 
 dispose_all

tests/scripts/expect/tun-udp.exp

@@ -61,7 +61,7 @@ expect_line "Done"
 send "udp connect $prefix:bb1:ebd6:ad10:f33 1234\n"
 expect_line "Done"
 
-send "udp connect :: 1\n"
+send "udp connect $prefix:1:2:3:4 4321\n"
 expect_line "Done"
 
 send "udp close\n"