From 36e51132c882ebae0fb277a89e8d137fcc96506f Mon Sep 17 00:00:00 2001 From: Jonathan Hui Date: Wed, 4 Mar 2020 20:17:56 -0800 Subject: [PATCH] [tests] fix 1.2 test env and travis (#4614) --- .travis/script.sh | 6 +-- script/test | 96 +++++++++++++++++++++++++++++++++++++---------- 2 files changed, 78 insertions(+), 24 deletions(-) diff --git a/.travis/script.sh b/.travis/script.sh index 163026abb..2c52f21e4 100755 --- a/.travis/script.sh +++ b/.travis/script.sh @@ -584,11 +584,7 @@ build_samr21() { [ $BUILD_TARGET != v1.2 ] || { ./bootstrap || die ./script/test build || die - - ls tests/scripts/thread-cert/v1_2_*.py | while read test_case; do - [[ ! -d tmp ]] || rm -rvf tmp - PYTHONUNBUFFERED=1 "${test_case}" - done + ./script/test cert_suite tests/scripts/thread-cert/v1_2_* || die } [ $BUILD_TARGET != posix-app-pty ] || { diff --git a/script/test b/script/test index a2f559f55..a3b44c520 100755 --- a/script/test +++ b/script/test @@ -38,26 +38,29 @@ readonly OT_SYSTEM_TRIPLET="$(third_party/nlbuild-autotools/repo/third_party/aut readonly OT_BUILDDIR="$(pwd)/build" readonly OT_SRCDIR="$(pwd)" +readonly COLOR_PASS='\033[0;32m' +readonly COLOR_FAIL='\033[0;31m' +readonly COLOR_NONE='\033[0m' + + do_build() { - ./bootstrap - if [ "${VIRTUAL_TIME}" = 1 ]; then + if [[ "${NODE_MODE}" == "transceiver" && "${VIRTUAL_TIME}" == 1 ]]; then OPTIONS="VIRTUAL_TIME_UART=1" fi make -f examples/Makefile-simulation ${OPTIONS} if [[ -n "${RADIO_DEVICE}" ]]; then - make -f src/posix/Makefile-posix + make -f src/posix/Makefile-posix ${OPTIONS} fi # Extra 1.1 build for 1.2 env - if [ "${THREAD_VERSION}" = "1.2" ]; then - ./bootstrap - OPTIONS="TargetTuple=${SYSTEM_TRIPLET}-1.1 THREAD_VERSION=1.1 ${OPTIONS}" + if [[ "${THREAD_VERSION}" == "1.2" ]]; then + OPTIONS="TargetTuple=${OT_SYSTEM_TRIPLET}-1.1 THREAD_VERSION=1.1 ${OPTIONS}" make -f examples/Makefile-simulation ${OPTIONS} if [[ -n "${RADIO_DEVICE}" ]]; then - make -f src/posix/Makefile-posix + make -f src/posix/Makefile-posix ${OPTIONS} fi fi } @@ -68,10 +71,63 @@ do_clean() { do_cert() { export top_builddir="${OT_BUILDDIR}/${OT_SYSTEM_TRIPLET}" + + if [[ "${THREAD_VERSION}" == "1.2" ]]; then + export top_builddir_1_1="${OT_BUILDDIR}/${OT_SYSTEM_TRIPLET}-1.1" + fi + [[ ! -d tmp ]] || rm -rvf tmp PYTHONUNBUFFERED=1 "$1" } +do_cert_suite() { + export top_builddir="${OT_BUILDDIR}/${OT_SYSTEM_TRIPLET}" + + if [[ "${THREAD_VERSION}" == "1.2" ]]; then + export top_builddir_1_1="${OT_BUILDDIR}/${OT_SYSTEM_TRIPLET}-1.1" + fi + + local pass_count=0 + local fail_count=0 + + [[ ! -f fail.log ]] || rm fail.log + + for test_case in "$@"; do + [[ ! -d tmp ]] || rm -rf tmp + exit_code=0 + "${test_case}" &> test.log || exit_code=$? + if [[ "${exit_code}" == 0 ]]; then + echo -e "${COLOR_PASS}PASS${COLOR_NONE} ${test_case}" + pass_count=$(( pass_count + 1 )) + else + echo -e "${COLOR_FAIL}FAIL${COLOR_NONE} ${test_case}" + fail_count=$(( fail_count + 1 )) + { + echo "==============================" + echo "!!! FAIL:${test_case}" + echo "==============================" + cat test.log + } >> fail.log + fi + done + + echo "==================================" + echo " Test Summary" + echo "==================================" + echo "# TOTAL: $((pass_count + fail_count))" + echo "# PASS: ${pass_count}" + echo "# FAIL: ${fail_count}" + + if [[ "${fail_count}" -gt 0 ]]; then + while read -r line; do + echo "${line}" + done < fail.log + exit 1 + else + exit 0 + fi +} + print_usage() { echo "USAGE: [ENVIRONMENTS] $0 COMMANDS @@ -85,6 +141,7 @@ COMMANDS: clean Clean built files to prepare for new build. build Build project for running tests. This can be used to rebuild the project for changes. cert Run a single thread-cert test. ENVIRONMENTS should be the same as those given to build or update. + cert_suite Run a batch of thread-cert tests and summarize the test results. Only echo logs for failing tests. help Print this help. EXAMPLES: @@ -106,10 +163,10 @@ EXAMPLES: # Test Thread 1.2 with real time THREAD_VERSION=1.2 VIRTUAL_TIME=0 $0 clean build cert tests/scripts/thread-cert/v1_2_router_5_1_1.py - THREAD_VERSION=1.2 VIRTUAL_TIME=0 $0 clean build cert tests/scripts/thread-cert/v1_2_test_parent_selection.py + THREAD_VERSION=1.2 VIRTUAL_TIME=0 $0 clean build cert_suite tests/scripts/thread-cert/v1_2_* " - exit $1 + exit "$1" } do_package() { @@ -117,21 +174,21 @@ do_package() { (mkdir -p "${builddir}" \ && cd "${builddir}" \ && cmake "${OT_SRCDIR}" -DOT_PLATFORM="simulation" \ - && make -j${OT_BUILD_JOBS} package \ + && make -j"${OT_BUILD_JOBS}" package \ && ls "${builddir}"/openthread-simulation-*.deb) builddir="${OT_BUILDDIR}/cmake/openthread-host" (mkdir -p "${builddir}" \ && cd "${builddir}" \ && cmake "${OT_SRCDIR}" -DOT_PLATFORM="posix-host" \ - && make -j${OT_BUILD_JOBS} package \ + && make -j"${OT_BUILD_JOBS}" package \ && ls "${builddir}"/openthread-standalone-*.deb) builddir="${OT_BUILDDIR}/cmake/openthread-daemon" (mkdir -p "${builddir}" \ && cd "${builddir}" \ && cmake "${OT_SRCDIR}" -DOT_PLATFORM="posix-host" -DOT_DAEMON=1 -DOT_PLATFORM_NETIF=1 -DOT_PLATFORM_UDP=1 \ - && make -j${OT_BUILD_JOBS} package \ + && make -j"${OT_BUILD_JOBS}" package \ && ls "${builddir}"/openthread-daemon-*.deb) } @@ -141,19 +198,16 @@ main() export VERBOSE="${VERBOSE:-1}" export VIRTUAL_TIME="${VIRTUAL_TIME:-1}" export THREAD_VERSION="${THREAD_VERSION:-1.1}" - if [ "${THREAD_VERSION}" = "1.2" ]; then - export top_builddir_1_1="build/${SYSTEM_TRIPLET}-1.1" - fi if [[ "${NODE_MODE}" = 'transceiver' ]] ; then export RADIO_DEVICE="${OT_BUILDDIR}/${OT_SYSTEM_TRIPLET}/examples/apps/ncp/ot-rcp" export OT_CLI_PATH="${OT_BUILDDIR}/posix/${OT_SYSTEM_TRIPLET}/src/posix/ot-cli" export OT_NCP_PATH="${OT_BUILDDIR}/posix/${OT_SYSTEM_TRIPLET}/src/posix/ot-ncp" - if [ "${THREAD_VERSION}" = "1.2" ]; then - export RADIO_DEVICE_1_1="${OT_BUILDDIR}/${SYSTEM_TRIPLET}-1.1/examples/apps/ncp/ot-rcp" - export OT_CLI_PATH_1_1="${OT_BUILDDIR}/posix/${SYSTEM_TRIPLET}-1.1/src/posix/ot-cli" - export OT_NCP_PATH_1_1="${OT_BUILDDIR}/posix/${SYSTEM_TRIPLET}-1.1/src/posix/ot-ncp" + if [[ "${THREAD_VERSION}" == "1.2" ]]; then + export RADIO_DEVICE_1_1="${OT_BUILDDIR}/${OT_SYSTEM_TRIPLET}-1.1/examples/apps/ncp/ot-rcp" + export OT_CLI_PATH_1_1="${OT_BUILDDIR}/posix/${OT_SYSTEM_TRIPLET}-1.1/src/posix/ot-cli" + export OT_NCP_PATH_1_1="${OT_BUILDDIR}/posix/${OT_SYSTEM_TRIPLET}-1.1/src/posix/ot-ncp" fi fi @@ -178,6 +232,10 @@ main() shift do_cert "$1" ;; + cert_suite) + shift + do_cert_suite "$@" + ;; help) print_usage ;;