openthread/script/test

#!/bin/bash
#
#  Copyright (c) 2018, The OpenThread Authors.
#  All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are met:
#  1. Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#  2. Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#  3. Neither the name of the copyright holder nor the
#     names of its contributors may be used to endorse or promote products
#     derived from this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
#  POSSIBILITY OF SUCH DAMAGE.
#
#    Description:
#      This file runs various tests of OpenThread.
#

set -euo pipefail

readonly OT_BUILDDIR="$(pwd)/build"
readonly OT_SRCDIR="$(pwd)"
readonly OT_TEST_COMMON_OPTIONS=(
    "-DOT_BORDER_AGENT=ON"
    "-DOT_BORDER_ROUTER=ON"
    "-DOT_CHANNEL_MANAGER=ON"
    "-DOT_CHANNEL_MONITOR=ON"
    "-DOT_CHILD_SUPERVISION=ON"
    "-DOT_COAP=ON"
    "-DOT_COAPS=ON"
    "-DOT_COMMISSIONER=ON"
    "-DOT_COVERAGE=ON"
    "-DOT_DHCP6_CLIENT=ON"
    "-DOT_DHCP6_SERVER=ON"
    "-DOT_DIAGNOSTIC=ON"
    "-DOT_IP6_FRAGM=ON"
    "-DOT_JAM_DETECTION=ON"
    "-DOT_JOINER=ON"
    "-DOT_LOG_LEVEL_DYNAMIC=ON"
    "-DOT_MAC_FILTER=ON"
    "-DOT_REFERENCE_DEVICE=ON"
    "-DOT_SERVICE=ON"
    "-DOT_SLAAC=ON"
)

readonly COLOR_PASS='\033[0;32m'
readonly COLOR_FAIL='\033[0;31m'
readonly COLOR_NONE='\033[0m'

readonly NODE_MODE="${NODE_MODE:-standalone}"
readonly NODE_TYPE="${NODE_TYPE:-sim}"
readonly THREAD_VERSION="${THREAD_VERSION:-1.1}"
readonly VERBOSE="${VERBOSE:-0}"
readonly VIRTUAL_TIME="${VIRTUAL_TIME:-1}"

build_simulation()
{
    local version="$1"
    local options=("${OT_TEST_COMMON_OPTIONS[@]}")

    options+=(
        "-DOT_PLATFORM=simulation"
        "-DOT_THREAD_VERSION=${version}"
    )

    if [[ "${VIRTUAL_TIME}" == 1 ]]; then
        options+=("-DOT_SIMULATION_VIRTUAL_TIME=ON")

        if [[ "${NODE_MODE}" == "rcp" ]]; then
            options+=("-DOT_SIMULATION_VIRTUAL_TIME_UART=ON")
        fi

    fi

    local builddir="${OT_BUILDDIR}/cmake/openthread-simulation-${version}"
    (mkdir -p "${builddir}" \
        && cd "${builddir}" \
        && cmake -GNinja "${OT_SRCDIR}" "${options[@]}" \
        && ninja)

    if [[ "${version}" == "1.2" ]]; then

        options+=(
        "-DOT_BACKBONE_ROUTER=ON"
        "-DOT_BORDER_ROUTER=ON"
        "-DOT_SERVICE=ON"
        )

        local builddir="${OT_BUILDDIR}/cmake/openthread-simulation-${version}-bbr"
        (mkdir -p "${builddir}" \
            && cd "${builddir}" \
            && cmake -GNinja "${OT_SRCDIR}" "${options[@]}" \
            && ninja)

    fi
}

build_posix()
{
    local version="$1"
    local options=("${OT_TEST_COMMON_OPTIONS[@]}")

    options+=(
        "-DOT_PLATFORM=posix"
        "-DOT_READLINE=readline"
        "-DOT_THREAD_VERSION=${version}"
    )

    if [[ "${VIRTUAL_TIME}" == 1 ]]; then
        options+=("-DOT_POSIX_VIRTUAL_TIME=ON")
    fi

    local builddir="${OT_BUILDDIR}/cmake/openthread-posix-${version}"
    (mkdir -p "${builddir}" \
        && cd "${builddir}" \
        && cmake -GNinja "${OT_SRCDIR}" "${options[@]}" \
        && ninja)

    if [[ "${version}" == "1.2" ]]; then

        options+=(
        "-DOT_BACKBONE_ROUTER=ON"
        "-DOT_BORDER_ROUTER=ON"
        "-DOT_SERVICE=ON"
        )

        local builddir="${OT_BUILDDIR}/cmake/openthread-posix-${version}-bbr"
        (mkdir -p "${builddir}" \
            && cd "${builddir}" \
            && cmake -GNinja "${OT_SRCDIR}" "${options[@]}" \
            && ninja)
    fi
}

do_build() {
    build_simulation "${THREAD_VERSION}"

    if [[ "${NODE_MODE}" == "rcp" ]]; then
        build_posix "${THREAD_VERSION}"
    fi

    # Extra 1.1 build for 1.2 tests
    if [[ "${THREAD_VERSION}" == "1.2" ]]; then
        build_simulation 1.1

        if [[ "${NODE_MODE}" == "rcp" ]]; then
            build_posix 1.1
        fi
    fi
}

do_clean() {
    rm -rfv "${OT_BUILDDIR}"
}

do_unit() {
    local builddir="${OT_BUILDDIR}/cmake/openthread-simulation-${THREAD_VERSION}"
    if [[ ! -d "${builddir}" ]]; then
        echo "Cannot find build directory!"
        exit -1
    fi

    cd "${builddir}"
    ninja test
}

do_cert() {
    export top_builddir="${OT_BUILDDIR}/cmake/openthread-simulation-${THREAD_VERSION}"

    if [[ "${THREAD_VERSION}" == "1.2" ]]; then
        export top_builddir_1_1="${OT_BUILDDIR}/cmake/openthread-simulation-1.1"
        export top_builddir_1_2_bbr="${OT_BUILDDIR}/cmake/openthread-simulation-1.2-bbr"
    fi

    [[ ! -d tmp ]] || rm -rvf tmp
    PYTHONUNBUFFERED=1 "$1"
}

do_cert_suite() {
    export top_builddir="${OT_BUILDDIR}/cmake/openthread-simulation-${THREAD_VERSION}"

    if [[ "${THREAD_VERSION}" == "1.2" ]]; then
        export top_builddir_1_1="${OT_BUILDDIR}/cmake/openthread-simulation-1.1"
        export top_builddir_1_2_bbr="${OT_BUILDDIR}/cmake/openthread-simulation-1.2-bbr"
    fi

    local pass_count=0
    local fail_count=0

    [[ ! -f fail.log ]] || rm fail.log

    for test_case in "$@"; do
        rm -rf tmp
        if "${test_case}" &> test.log; then
            echo -e "${COLOR_PASS}PASS${COLOR_NONE} ${test_case}"
            pass_count=$(( pass_count + 1 ))
        else
            echo -e "${COLOR_FAIL}FAIL${COLOR_NONE} ${test_case}"
            fail_count=$(( fail_count + 1 ))
            {
                echo "=============================="
                echo "!!! FAIL:${test_case}"
                echo "=============================="
                cat test.log
            } >> fail.log
        fi
    done

    echo "=================================="
    echo "         Test Summary"
    echo "=================================="
    echo "# TOTAL: $((pass_count + fail_count))"
    echo "# PASS: ${pass_count}"
    echo "# FAIL: ${fail_count}"

    if [[ "${fail_count}" -gt 0 ]]; then
        tr -dc '[:print:]\r\n\t' < fail.log
        exit 1
    else
        exit 0
    fi
}

do_expect()
{
    local ot_command

    if [[ "${NODE_MODE}" == rcp ]]; then
        ot_command="${OT_CLI_PATH} ${RADIO_DEVICE}"
    else
        ot_command="${OT_BUILDDIR}/cmake/openthread-simulation-${THREAD_VERSION}/examples/apps/cli/ot-cli-ftd"
    fi

    while read -r script; do
        rm -rf tmp
        OT_COMMAND="${ot_command}" "${script}" || {
            local exit_code=$?
            echo -e "${COLOR_FAIL}FAIL${COLOR_NONE} ${script}"
            exit "${exit_code}"
        }
        echo -e "${COLOR_PASS}PASS${COLOR_NONE} ${script}"
    done < <(
        if [[ $# != 0 ]]; then
            for script in "$@"; do echo ${script}; done
        else
            find tests/scripts/expect -type f -executable
        fi
    )

    exit 0
}

print_usage() {
    echo "USAGE: [ENVIRONMENTS] $0 COMMANDS

ENVIRONMENTS:
    NODE_TYPE       'sim' for CLI, 'ncp-sim' for NCP. The default is 'sim'.
    NODE_MODE       'rcp' for RCP mode, otherwise for standalone mode. The default is standalone mode.
    VERBOSE         1 to build or test verbosely. The default is 0.
    VIRTUAL_TIME    1 for virtual time, otherwise real time. The default is 1.
    THREAD_VERSION  1.1 for Thread 1.1 stack, 1.2 for Thread 1.2 stack. The default is 1.1.

COMMANDS:
    clean           Clean built files to prepare for new build.
    build           Build project for running tests. This can be used to rebuild the project for changes.
    cert            Run a single thread-cert test. ENVIRONMENTS should be the same as those given to build or update.
    cert_suite      Run a batch of thread-cert tests and summarize the test results. Only echo logs for failing tests.
    unit            Run all the unit tests. This should be called after simulation is built.
    expect          Run expect tests.
    help            Print this help.

EXAMPLES:
    # Test CLI with default settings
    $0 clean build cert tests/scripts/thread-cert/Cert_5_1_01_RouterAttach.py
    $0 cert tests/scripts/thread-cert/Cert_5_1_02_ChildAddressTimeout.py

    # Test NCP with default settings
    NODE_TYPE=ncp-sim $0 clean build cert tests/scripts/thread-cert/Cert_5_1_01_RouterAttach.py
    NODE_TYPE=ncp-sim $0 cert tests/scripts/thread-cert/Cert_5_1_02_ChildAddressTimeout.py

    # Test CLI with radio only
    NODE_MODE=rcp $0 clean build cert tests/scripts/thread-cert/Cert_5_1_01_RouterAttach.py
    NODE_MODE=rcp $0 cert tests/scripts/thread-cert/Cert_5_1_02_ChildAddressTimeout.py

    # Test CLI with real time
    VIRTUAL_TIME=0 $0 clean build cert tests/scripts/thread-cert/Cert_5_1_01_RouterAttach.py
    VIRTUAL_TIME=0 $0 cert tests/scripts/thread-cert/Cert_5_1_02_ChildAddressTimeout.py

    # Test Thread 1.2 with real time
    THREAD_VERSION=1.2 VIRTUAL_TIME=0 $0 clean build cert tests/scripts/thread-cert/v1_2_router_5_1_1.py
    THREAD_VERSION=1.2 VIRTUAL_TIME=0 $0 clean build cert_suite tests/scripts/thread-cert/v1_2_*

    # Run a single expect test
    $0 clean build expect tests/scripts/expect/log-level.exp

    # Run all expect tests
    $0 clean build expect
    "

    exit "$1"
}

do_package() {
    local builddir="${OT_BUILDDIR}/cmake/openthread-sim"
    local options=(
        "-DOT_BORDER_AGENT=ON"
        "-DOT_BORDER_ROUTER=ON"
        "-DOT_CHILD_SUPERVISION=ON"
        "-DOT_COMMISSIONER=ON"
        "-DOT_DIAGNOSTIC=ON"
        "-DOT_IP6_FRAGM=ON"
        "-DOT_JAM_DETECTION=ON"
        "-DOT_JOINER=ON"
        "-DOT_LOG_LEVEL_DYNAMIC=ON"
        "-DOT_MAC_FILTER=ON"
        "-DOT_SERVICE=ON"
        "-DOT_SLAAC=ON"
    )
    (mkdir -p "${builddir}" \
        && cd "${builddir}" \
        && cmake -GNinja "${OT_SRCDIR}" -DOT_PLATFORM="simulation" "${options[@]}" \
        && ninja package \
        && ls "${builddir}"/openthread-simulation-*.deb)

    builddir="${OT_BUILDDIR}/cmake/openthread-host"
    (mkdir -p "${builddir}" \
        && cd "${builddir}" \
        && cmake -GNinja "${OT_SRCDIR}" -DOT_PLATFORM="posix" "${options[@]}" \
        && ninja package \
        && ls "${builddir}"/openthread-standalone-*.deb)

    builddir="${OT_BUILDDIR}/cmake/openthread-daemon"
    (mkdir -p "${builddir}" \
        && cd "${builddir}" \
        && cmake -GNinja "${OT_SRCDIR}" -DOT_PLATFORM="posix" \
        -DOT_DAEMON=1 -DOT_PLATFORM_NETIF=1 -DOT_PLATFORM_UDP=1 "${options[@]}" \
        && ninja package \
        && ls "${builddir}"/openthread-daemon-*.deb)
}

envsetup()
{
    if [[ "${NODE_MODE}" == 'rcp' ]] ; then
        export RADIO_DEVICE="${OT_BUILDDIR}/cmake/openthread-simulation-${THREAD_VERSION}/examples/apps/ncp/ot-rcp"
        export OT_CLI_PATH="${OT_BUILDDIR}/cmake/openthread-posix-${THREAD_VERSION}/src/posix/ot-cli"
        export OT_NCP_PATH="${OT_BUILDDIR}/cmake/openthread-posix-${THREAD_VERSION}/src/posix/ot-ncp"

        if [[ "${THREAD_VERSION}" == "1.2" ]]; then
            export RADIO_DEVICE_1_1="${OT_BUILDDIR}/cmake/openthread-simulation-1.1/examples/apps/ncp/ot-rcp"
            export OT_CLI_PATH_1_1="${OT_BUILDDIR}/cmake/openthread-posix-1.1/src/posix/ot-cli"
            export OT_NCP_PATH_1_1="${OT_BUILDDIR}/cmake/openthread-posix-1.1/src/posix/ot-ncp"
            export OT_CLI_PATH_1_2_BBR="${OT_BUILDDIR}/cmake/openthread-posix-1.2-bbr/src/posix/ot-cli"
            export OT_NCP_PATH_1_2_BBR="${OT_BUILDDIR}/cmake/openthread-posix-1.2-bbr/src/posix/ot-ncp"
        fi
    fi

    export NODE_TYPE VIRTUAL_TIME

    # CMake always works in verbose mode if VERBOSE exists in environments.
    if [[ "${VERBOSE}" == 1 ]]; then
        export VERBOSE
    else
        export -n VERBOSE
    fi
}

main()
{
    envsetup

    if [[ -z "$1" ]]; then
        print_usage 1
    fi

    [[ "${NODE_MODE}" == "rcp" ]] && echo "Using rcp mode" || echo "Using standalone mode"
    [[ "${NODE_TYPE}" == "ncp-sim" ]] && echo "Using NCP node" || echo "Using CLI node"
    [[ "${VIRTUAL_TIME}" == 1 ]] && echo "Using virtual time" || echo "Using real time"
    [[ "${THREAD_VERSION}" == "1.2" ]] && echo "Using Thread 1.2 stack" || echo "Using Thread 1.1 stack"

    while [[ $# != 0 ]]; do
        case "$1" in
            clean)
                do_clean
                ;;
            build)
                do_build
                ;;
            cert)
                shift
                do_cert "$1"
                ;;
            cert_suite)
                shift
                do_cert_suite "$@"
                ;;
            unit)
                do_unit
                ;;
            help)
                print_usage
                ;;
            package)
                do_package
                ;;
            expect)
                shift
                do_expect "$@"
                ;;
        esac
        shift
    done
}

main "$@"