[spinel] RCP restoration on failures (#5694)

This commit implements the RCP restoration on failure feature. It
keeps the host process running while resetting and initializing the
RCP.
This commit is contained in:
Moandor
2020-11-26 08:57:13 -08:00
committed by GitHub
parent f4554e8ff7
commit 9d9646afaa
25 changed files with 1022 additions and 132 deletions
+19 -5
View File
@@ -40,7 +40,7 @@ proc wait_for {command success {failure {[\r\n]FAILURE_NOT_EXPECTED[\r\n]}}} {
set result 1
}
-re $failure {
exit 1
error "Got failure in wait_for"
}
timeout {
# Do nothing
@@ -51,11 +51,11 @@ proc wait_for {command success {failure {[\r\n]FAILURE_NOT_EXPECTED[\r\n]}}} {
}
}
if {$result == 0} {
exit 1
error "Timed out in wait_for"
}
}
proc spawn_node {id {type ""}} {
proc spawn_node {id {type ""} {radio_url ""}} {
global spawn_id
global spawn_ids
@@ -63,26 +63,39 @@ proc spawn_node {id {type ""}} {
set type $::env(OT_NODE_TYPE)
}
if {${radio_url} == ""} {
set radio_url "spinel+hdlc+uart://$::env(OT_SIMULATION_APPS)/ncp/ot-rcp?forkpty-arg=$id"
}
send_user "\n# ${id} ${type}\n"
switch -regexp ${type} {
{rcp|rcp-cli} {
spawn $::env(OT_POSIX_APPS)/ot-cli "spinel+hdlc+uart://$::env(OT_SIMULATION_APPS)/ncp/ot-rcp?forkpty-arg=$id"
spawn $::env(OT_POSIX_APPS)/ot-cli $radio_url
send "factoryreset\n"
wait_for "state" "disabled"
expect "Done"
send "routerselectionjitter 1\n"
expect "Done"
}
cli {
spawn $::env(OT_SIMULATION_APPS)/cli/ot-cli-ftd $id
send "factoryreset\n"
wait_for "state" "disabled"
expect "Done"
send "routerselectionjitter 1\n"
expect "Done"
}
mtd {
spawn $::env(OT_SIMULATION_APPS)/cli/ot-cli-mtd $id
send "factoryreset\n"
wait_for "state" "disabled"
expect "Done"
}
}
expect_after {
timeout { exit 1 }
timeout { error "Timed out" }
}
set spawn_ids($id) $spawn_id
@@ -123,6 +136,7 @@ proc dispose_all {} {
for {set i 1} {$i <= $max_node} {incr i} {
dispose_node $i
}
array unset spawn_ids
}
proc get_ipaddr {type} {
+11 -3
View File
@@ -29,19 +29,27 @@
source "tests/scripts/expect/_common.exp"
proc setup_two_nodes {{childmode {r}}} {
proc setup_two_nodes {{childmode {r}} {should_spawn_node true}} {
# Sets up a Thread network with 2 nodes, spawn_1 as the leader and spawn_2
# as a child.
set psk "J01NME"
spawn_node 2
if {$should_spawn_node} {
spawn_node 2
} else {
switch_node 2
}
send "eui64\n"
expect -re {([0-9a-f]{16})}
set eui64 $expect_out(1,string)
expect "Done"
# Sets up a Thread network with node 1 as the leader.
spawn_node 1
if {$should_spawn_node} {
spawn_node 1
} else {
switch_node 1
}
setup_leader
send "commissioner start\n"
-9
View File
@@ -31,9 +31,6 @@ source "tests/scripts/expect/_common.exp"
set max_node 15
spawn_node 1
expect_after {
timeout { exit 1 }
}
send "panid 0xface\n"
expect "Done"
send "ifconfig up\n"
@@ -45,9 +42,6 @@ expect "Done"
for {set i 2} {$i <= $max_node} {incr i} {
spawn_node $i
expect_after {
timeout { exit 1 }
}
send "mode r\n"
expect "Done"
send "panid 0xface\n"
@@ -61,9 +55,6 @@ for {set i 2} {$i <= $max_node} {incr i} {
}
set spawn_id $spawn_ids(1)
expect_after {
timeout { exit 1 }
}
send "child table\n"
expect "Done"
@@ -30,10 +30,7 @@
source "tests/scripts/expect/_common.exp"
# allows 11-25 and forbidden 26
spawn $env(OT_POSIX_APPS)/ot-cli "spinel+hdlc+uart://$env(OT_SIMULATION_APPS)/ncp/ot-rcp?max-power-table=11,12,13,14,15,16,17,18,19,20,21,22,23,24,-1,0x7f&forkpty-arg=1"
expect_after {
timeout { exit 1 }
}
spawn_node 1 "rcp" "spinel+hdlc+uart://$env(OT_SIMULATION_APPS)/ncp/ot-rcp?max-power-table=11,12,13,14,15,16,17,18,19,20,21,22,23,24,-1,0x7f&forkpty-arg=1"
send "channel supported\n"
expect "0x3fff800"
expect "Done"
@@ -43,10 +40,7 @@ expect "Done"
send "\x04"
expect eof
# allows all channels by default
spawn $env(OT_POSIX_APPS)/ot-cli "spinel+hdlc+uart://$env(OT_SIMULATION_APPS)/ncp/ot-rcp?forkpty-arg=1"
expect_after {
timeout { exit 1 }
}
spawn_node 1 "rcp" "spinel+hdlc+uart://$env(OT_SIMULATION_APPS)/ncp/ot-rcp?forkpty-arg=1"
send "channel supported\n"
expect "0x7fff800"
expect "Done"
+306
View File
@@ -0,0 +1,306 @@
#!/usr/bin/expect -f
#
# Copyright (c) 2020, The OpenThread Authors.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 3. Neither the name of the copyright holder nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
source "tests/scripts/expect/_common.exp"
source "tests/scripts/expect/_multinode.exp"
file tempfile socat_out
set socat_pid [exec socat -d -d pty,raw,echo=0 pty,raw,echo=0 >/dev/null 2>$socat_out &]
while {true} {
if {[exec head -n2 $socat_out | wc -l] == 2} {
set radio_pty [exec head -n1 $socat_out | grep -o {/dev/.\+}]
set host_pty [exec head -n2 $socat_out | tail -n1 | grep -o {/dev/.\+}]
break
}
sleep 1
}
puts "Radio PTY: $radio_pty"
puts "Host PTY: $host_pty"
set rcp_pid [exec $::env(OT_SIMULATION_APPS)/ncp/ot-rcp 1 < $radio_pty > $radio_pty &]
puts "RCP PID: $rcp_pid"
try {
puts "Before enabling"
spawn_node 1 "rcp" "spinel+hdlc_uart://$host_pty"
exec kill $rcp_pid
puts "Killed $rcp_pid"
sleep 1
set rcp_pid [exec $::env(OT_SIMULATION_APPS)/ncp/ot-rcp 1 < $radio_pty > $radio_pty &]
puts "RCP PID: $rcp_pid"
expect eof
puts "Queued parent-to-child packets, as parent"
spawn_node 1 "rcp" "spinel+hdlc_uart://$host_pty"
spawn_node 2
setup_two_nodes "-" false
switch_node 2
send "ipaddr mleid\n"
expect "ipaddr mleid"
expect -re {(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4})}
set addr_2 $expect_out(1,string)
send "udp open\n"
expect "Done"
send "udp bind :: 11003\n"
expect "Done"
send "pollperiod 100000\n"
expect "Done"
sleep 1
switch_node 1
send "ipaddr mleid\n"
expect "ipaddr mleid"
expect -re {(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4})}
set addr_1 $expect_out(1,string)
send "udp open\n"
expect "Done"
send "udp bind :: 11004\n"
expect "Done"
send "udp connect $addr_2 11003\n"
expect "Done"
send "udp send hello\n"
expect "Done"
send "udp send there\n"
expect "Done"
sleep 1
exec kill $rcp_pid
puts "Killed $rcp_pid"
sleep 1
set rcp_pid [exec $::env(OT_SIMULATION_APPS)/ncp/ot-rcp 1 < $radio_pty > $radio_pty &]
puts "RCP PID: $rcp_pid"
switch_node 2
send "pollperiod 1000\n"
expect "Done"
expect "5 bytes from $addr_1 11004 hello"
expect "5 bytes from $addr_1 11004 there"
dispose_all
puts "Queued parent-to-child packets, as child"
exec kill $rcp_pid
puts "Killed $rcp_pid"
sleep 1
set rcp_pid [exec $::env(OT_SIMULATION_APPS)/ncp/ot-rcp 2 < $radio_pty > $radio_pty &]
puts "RCP PID: $rcp_pid"
spawn_node 1
spawn_node 2 "rcp" "spinel+hdlc_uart://$host_pty"
setup_two_nodes "-" false
switch_node 2
send "ipaddr mleid\n"
expect "ipaddr mleid"
expect -re {(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4})}
set addr_2 $expect_out(1,string)
send "udp open\n"
expect "Done"
send "udp bind :: 11003\n"
expect "Done"
send "pollperiod 100000\n"
expect "Done"
sleep 1
switch_node 1
send "ipaddr mleid\n"
expect "ipaddr mleid"
expect -re {(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4})}
set addr_1 $expect_out(1,string)
send "udp open\n"
expect "Done"
send "udp bind :: 11004\n"
expect "Done"
send "udp connect $addr_2 11003\n"
expect "Done"
send "udp send hello\n"
expect "Done"
sleep 1
exec kill $rcp_pid
puts "Killed $rcp_pid"
sleep 1
set rcp_pid [exec $::env(OT_SIMULATION_APPS)/ncp/ot-rcp 2 < $radio_pty > $radio_pty &]
puts "RCP PID: $rcp_pid"
sleep 1
switch_node 2
send "pollperiod 1000\n"
expect "Done"
expect "5 bytes from $addr_1 11004 hello"
dispose_all
exec kill $rcp_pid
puts "Killed $rcp_pid"
sleep 1
set rcp_pid [exec $::env(OT_SIMULATION_APPS)/ncp/ot-rcp 1 < $radio_pty > $radio_pty &]
puts "RCP PID: $rcp_pid"
puts "Many children, queued child-to-child packets"
spawn_node 1 "rcp" "spinel+hdlc_uart://$host_pty"
send "panid 0xface\n"
expect "Done"
send "ifconfig up\n"
expect "Done"
send "thread start\n"
expect "Done"
wait_for "state" "leader"
expect "Done"
send "ipaddr mleid\n"
expect "ipaddr mleid"
expect -re {(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4})}
set addr(1) $expect_out(1,string)
send "udp open\n"
expect "Done"
send "udp bind :: 11004\n"
expect "Done"
set max_children 15
for {set i 2} {$i <= $max_children + 1} {incr i} {
spawn_node $i
send "mode -\n"
expect "Done"
send "panid 0xface\n"
expect "Done"
send "ifconfig up\n"
expect "Done"
send "thread start\n"
expect "Done"
wait_for "state" "child"
send "ipaddr mleid\n"
expect "ipaddr mleid"
expect -re {(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4})}
set addr($i) $expect_out(1,string)
expect "Done"
send "udp open\n"
expect "Done"
send "udp bind :: 11003\n"
expect "Done"
send "pollperiod 100000\n"
expect "Done"
}
sleep 1
switch_node 1
for {set i 2} {$i <= $max_children + 1} {incr i} {
send "udp send $addr($i) 11003 hello\n"
expect "Done"
}
sleep 1
exec kill $rcp_pid
puts "Killed $rcp_pid"
sleep 1
set rcp_pid [exec $::env(OT_SIMULATION_APPS)/ncp/ot-rcp 1 < $radio_pty > $radio_pty &]
puts "RCP PID: $rcp_pid"
for {set i 7} {$i <= 9} {incr i} {
switch_node $i
send "pollperiod 1000\n"
expect "Done"
expect "5 bytes from $addr(1) 11004 hello"
}
switch_node 4
send "udp send $addr(5) 11003 hello_from_node_4\n"
expect "Done"
if {$::env(THREAD_VERSION) != "1.1"} {
expect "5 bytes from $addr(1) 11004 hello"
}
sleep 1
exec kill $rcp_pid
puts "Killed $rcp_pid"
sleep 1
set rcp_pid [exec $::env(OT_SIMULATION_APPS)/ncp/ot-rcp 1 < $radio_pty > $radio_pty &]
puts "RCP PID: $rcp_pid"
switch_node 5
send "pollperiod 1000\n"
expect "Done"
expect "17 bytes from $addr(4) 11003 hello_from_node_4"
dispose_all
puts "While energy scanning"
spawn_node 1 "rcp" "spinel+hdlc_uart://$host_pty"
send "ifconfig up\n"
expect "Done"
send "thread start\n"
expect "Done"
wait_for "state" "leader"
expect "Done"
send "scan energy 100\n"
expect "| Ch | RSSI |"
expect "+----+------+"
sleep 1
exec kill $rcp_pid
puts "Killed $rcp_pid"
sleep 1
set rcp_pid [exec $::env(OT_SIMULATION_APPS)/ncp/ot-rcp 1 < $radio_pty > $radio_pty &]
puts "RCP PID: $rcp_pid"
for {set i 11} {$i <= 26} {incr i} {
expect -re "\\| +$i \\| +-?\\d+ \\|"
}
expect "Done"
dispose_all
} finally {
exec kill $rcp_pid
exec kill $socat_pid
}
@@ -29,12 +29,7 @@
source "tests/scripts/expect/_common.exp"
spawn $env(OT_POSIX_APPS)/ot-cli "spinel+hdlc+uart://$env(OT_SIMULATION_APPS)/ncp/ot-rcp?forkpty-arg=--sleep-to-tx&forkpty-arg=1"
set node_1 $spawn_id
expect_after {
timeout { exit 1 }
}
spawn_node 1 "rcp" "spinel+hdlc+uart://$env(OT_SIMULATION_APPS)/ncp/ot-rcp?forkpty-arg=--sleep-to-tx&forkpty-arg=1"
send "dataset init new\n"
expect "Done"
send "dataset panid 0xface\n"
@@ -75,12 +70,7 @@ expect -re {(\d+)}
set channel $expect_out(1,string)
expect "Done"
spawn $env(OT_POSIX_APPS)/ot-cli "spinel+hdlc+uart://$env(OT_SIMULATION_APPS)/ncp/ot-rcp?forkpty-arg=--sleep-to-tx&forkpty-arg=2"
set node_2 $spawn_id
expect_after {
timeout { exit 1 }
}
spawn_node 2 "rcp" "spinel+hdlc+uart://$env(OT_SIMULATION_APPS)/ncp/ot-rcp?forkpty-arg=--sleep-to-tx&forkpty-arg=2"
send "scan\n"
expect "| J | Network Name | Extended PAN | PAN | MAC Address | Ch | dBm | LQI |"
expect "+---+------------------+------------------+------+------------------+----+-----+-----+"
@@ -0,0 +1,30 @@
#!/usr/bin/expect -f
#
# Copyright (c) 2020, The OpenThread Authors.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 3. Neither the name of the copyright holder nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
source "tests/scripts/expect/posix-rcp-restoration.exp"