[tests] Update toranj test script to retry failed tests (#2615)

This commit updates the `start.sh` script under `toranj` test
framework so that failed tested are tried again up to 3 times.
It also modifies how wpantund project is built and installed
under `toranj`.

.travis/before_install.sh

@@ -116,7 +116,8 @@ cd /tmp || die
         cd wpantund || die
         ./bootstrap.sh || die
         ./configure || die
-        sudo make install -j 8 || die
+        sudo make -j 8 || die
+        sudo make install || die
         cd .. || die
     }
 

tests/toranj/start.sh

@@ -32,15 +32,29 @@ die() {
     exit 1
 }
 
-failed() {
-    echo " *** TEST FAILED: ERROR: " $*
-    tail -n 30 wpantund-logs*.log
-    exit 1
-}
+run() {
+    counter=0
+    while true; do
+        # Clear logs and flash files
+        sudo rm tmp/*.flash > /dev/null 2>&1
+        sudo rm *.log > /dev/null 2>&1
 
-clean() {
-    sudo rm tmp/*.flash > /dev/null 2>&1
-    sudo rm *.log > /dev/null 2>&1
+        sudo python $1 && return
+
+        # On Travis, we allow a failed test to be retried up to 3 attempts.
+        if [ "$BUILD_TARGET" = "toranj-test-framework" ]; then
+            if [ "$counter" -lt 2 ]; then
+                counter=$((counter+1))
+                echo Attempt $counter running "$1" failed. Trying again.
+                sleep 10
+                continue
+            fi
+        fi
+
+        echo " *** TEST FAILED"
+        tail -n 40 wpantund-logs*.log
+        exit 1
+    done
 }
 
 cd $(dirname $0)
@@ -71,11 +85,11 @@ make -j 8 || die
 
 cd tests/toranj
 
-clean; sudo python test-001-get-set.py                                   || failed
-clean; sudo python test-002-form.py                                      || failed
-clean; sudo python test-003-join.py                                      || failed
-clean; sudo python test-004-scan.py                                      || failed
-clean; sudo python test-005-discover-scan.py                             || failed
-clean; sudo python test-006-traffic-router-end-device.py                 || failed
-clean; sudo python test-007-traffic-router-sleepy.py                     || failed
-clean; sudo python test-008-permit-join.py                               || failed
+run test-001-get-set.py
+run test-002-form.py
+run test-003-join.py
+run test-004-scan.py
+run test-005-discover-scan.py
+run test-006-traffic-router-end-device.py
+run test-007-traffic-router-sleepy.py
+run test-008-permit-join.py

tests/toranj/wpan.py

@@ -404,7 +404,7 @@ class Node(object):
     # class methods
 
     @classmethod
-    def init_all_nodes(cls, wait_time=20):
+    def init_all_nodes(cls, wait_time=15):
         """Issues a `wpanctl.leave` on all `Node` objects and waits for them to be ready"""
         random.seed(12345)
         time.sleep(0.5)
@@ -422,7 +422,7 @@ class Node(object):
                     raise
                 else:
                     break
-                time.sleep(0.1)
+                time.sleep(0.4)
 
     #------------------------------------------------------------------------------------------------------------------
     # IPv6 message Sender and Receiver class