From 713662bf805462664226abb3ea5ae100d7d1e8db Mon Sep 17 00:00:00 2001 From: Moandor Date: Thu, 29 Oct 2020 23:34:37 +0800 Subject: [PATCH] [continuous-integration] use lcov to generate coverage data (#5635) --- .codecov.yml | 2 +- .github/workflows/posix.yml | 120 ++++++--- .github/workflows/simulation-1.2.yml | 278 ++++++++++++--------- .github/workflows/simulation.yml | 260 ++++++++++++------- .github/workflows/toranj.yml | 37 ++- script/test | 40 ++- tests/scripts/thread-cert/node.py | 13 +- tests/scripts/thread-cert/run_bbr_tests.py | 2 - 8 files changed, 492 insertions(+), 260 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index c7310f3e0..d8f757083 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -9,4 +9,4 @@ ignore: comment: layout: "diff, flags, files" - after_n_builds: 16 + after_n_builds: 4 diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 3acf48a00..8fd18d30d 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -55,7 +55,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y libreadline6-dev python3-setuptools python3-wheel + sudo apt-get --no-install-recommends install -y libreadline6-dev python3-setuptools python3-wheel lcov python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - name: Build run: | @@ -74,18 +74,24 @@ jobs: run: | tar -cf build/posix.tar -C build/ posix/ rm -rf build/posix/ - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-posix-cli-1 + path: tmp/coverage.info - name: Keep POSIX Only run: | rm -rf build/x86_64-unknown-linux-gnu/ tar -xf build/posix.tar -C build/ - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-posix-cli-2 + path: tmp/coverage.info posix-ncp: runs-on: ubuntu-18.04 @@ -102,7 +108,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y libreadline6-dev python3-setuptools python3-wheel + sudo apt-get --no-install-recommends install -y libreadline6-dev python3-setuptools python3-wheel lcov python3 -m pip install -r tests/scripts/thread-cert/requirements.txt sudo python3 -m pip install git+https://github.com/openthread/pyspinel - name: Build @@ -122,18 +128,24 @@ jobs: run: | tar -cf build/posix.tar -C build/ posix/ rm -rf build/posix/ - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-posix-ncp-1 + path: tmp/coverage.info - name: Keep POSIX Only run: | rm -rf build/x86_64-unknown-linux-gnu/ tar -xf build/posix.tar -C build/ - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-posix-ncp-2 + path: tmp/coverage.info posix-ncp-rcp-migrate: runs-on: ubuntu-18.04 @@ -144,7 +156,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y expect + sudo apt-get --no-install-recommends install -y expect lcov - name: Build run: | ./bootstrap @@ -156,18 +168,24 @@ jobs: run: | tar -cf build/posix.tar -C build/ posix/ rm -rf build/posix/ - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-posix-ncp-rcp-migrate-1 + path: tmp/coverage.info - name: Keep POSIX Only run: | rm -rf build/x86_64-unknown-linux-gnu/ tar -xf build/posix.tar -C build/ - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-posix-ncp-rcp-migrate-2 + path: tmp/coverage.info posix-pty: runs-on: ubuntu-18.04 @@ -178,7 +196,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y socat expect + sudo apt-get --no-install-recommends install -y socat expect lcov cd /tmp wget https://github.com/obgm/libcoap/archive/bsd-licensed.tar.gz tar xvf bsd-licensed.tar.gz @@ -198,18 +216,24 @@ jobs: run: | tar -cf build/posix.tar -C build/ posix/ rm -rf build/posix/ - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-posix-pty-1 + path: tmp/coverage.info - name: Keep POSIX Only run: | rm -rf build/x86_64-unknown-linux-gnu/ tar -xf build/posix.tar -C build/ - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-posix-pty-2 + path: tmp/coverage.info posix-pty-daemon: runs-on: ubuntu-18.04 @@ -221,7 +245,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y socat expect + sudo apt-get --no-install-recommends install -y socat expect lcov cd /tmp wget https://github.com/obgm/libcoap/archive/bsd-licensed.tar.gz tar xvf bsd-licensed.tar.gz @@ -241,15 +265,41 @@ jobs: run: | tar -cf build/posix.tar -C build/ posix/ rm -rf build/posix/ - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-posix-pty-daemon-1 + path: tmp/coverage.info - name: Keep POSIX Only run: | rm -rf build/x86_64-unknown-linux-gnu/ tar -xf build/posix.tar -C build/ - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-posix-pty-daemon-2 + path: tmp/coverage.info + + upload-coverage: + needs: + - posix-cli + - posix-ncp + - posix-ncp-rcp-migrate + - posix-pty + - posix-pty-daemon + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Bootstrap + run: | + sudo apt-get --no-install-recommends install -y lcov + - uses: actions/download-artifact@v2 + with: + path: coverage/ + - name: Upload Coverage + run: | + script/test upload_codecov diff --git a/.github/workflows/simulation-1.2.yml b/.github/workflows/simulation-1.2.yml index 5975c60a6..87a7ca3ea 100644 --- a/.github/workflows/simulation-1.2.yml +++ b/.github/workflows/simulation-1.2.yml @@ -55,14 +55,14 @@ jobs: CXX: ${{ matrix.compiler.cxx }} strategy: matrix: - compiler: [{c: "gcc", cxx: "g++", gcov: "gcov"}, { c: "clang", cxx: "clang++", gcov: "llvm-cov gcov"}] + compiler: [{c: "gcc", cxx: "g++", gcov: "gcc"}, { c: "clang", cxx: "clang++", gcov: "llvm"}] arch: ["m32", "m64"] steps: - uses: actions/checkout@v2 - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y g++-multilib ninja-build python3-setuptools python3-wheel llvm + sudo apt-get --no-install-recommends install -y g++-multilib ninja-build python3-setuptools python3-wheel llvm lcov python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - name: Build run: | @@ -81,23 +81,35 @@ jobs: run: | ./script/test tar simulation 1.1 ./script/test tar simulation 1.2-bbr - - name: Codecov + - name: Generate Coverage run: | - bash <(curl -s https://codecov.io/bash) -x "${{ matrix.compiler.gcov }}" -Z + ./script/test generate_coverage "${{ matrix.compiler.gcov }}" + - uses: actions/upload-artifact@v2 + with: + name: cov-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-1 + path: tmp/coverage.info - name: Keep-1-2-bbr-only run: | ./script/test tar simulation 1.2 ./script/test untar simulation 1.2-bbr - - name: Codecov + - name: Generate Coverage run: | - bash <(curl -s https://codecov.io/bash) -x "${{ matrix.compiler.gcov }}" -Z + ./script/test generate_coverage "${{ matrix.compiler.gcov }}" + - uses: actions/upload-artifact@v2 + with: + name: cov-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-2 + path: tmp/coverage.info - name: Keep-1-1-only run: | ./script/test tar simulation 1.2-bbr ./script/test untar simulation 1.1 - - name: Codecov + - name: Generate Coverage run: | - bash <(curl -s https://codecov.io/bash) -x "${{ matrix.compiler.gcov }}" -Z + ./script/test generate_coverage "${{ matrix.compiler.gcov }}" + - uses: actions/upload-artifact@v2 + with: + name: cov-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-3 + path: tmp/coverage.info packet-verification-low-power: runs-on: ubuntu-18.04 @@ -110,46 +122,52 @@ jobs: MAC_FILTER: 1 INTER_OP: 1 steps: - - uses: actions/checkout@v2 - - name: Bootstrap - run: | - sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build - python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - - name: Build - run: | - ./script/test build - - name: Get Thread-Wireshark - run: | - ./script/test get_thread_wireshark - - name: Run - run: | - for i in {1..10} - do - ./script/test cert_suite ./tests/scripts/thread-cert/v1_2_LowPower*.py - done - - uses: actions/upload-artifact@v2 - if: ${{ failure() }} - with: - name: packet-verification-low-power-pcaps - path: | - *.pcap - *.json - - name: Keep-1-2-only - run: | - ./script/test tar simulation 1.2-bbr - - name: Codecov - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: true - - name: Keep-1-2-bbr-only - run: | - ./script/test tar simulation 1.2 - ./script/test untar simulation 1.2-bbr - - name: Codecov - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: true + - uses: actions/checkout@v2 + - name: Bootstrap + run: | + sudo rm /etc/apt/sources.list.d/* && sudo apt-get update + sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build lcov + python3 -m pip install -r tests/scripts/thread-cert/requirements.txt + - name: Build + run: | + ./script/test build + - name: Get Thread-Wireshark + run: | + ./script/test get_thread_wireshark + - name: Run + run: | + for i in {1..10} + do + ./script/test cert_suite ./tests/scripts/thread-cert/v1_2_LowPower*.py + done + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: packet-verification-low-power-pcaps + path: | + *.pcap + *.json + - name: Keep-1-2-only + run: | + ./script/test tar simulation 1.2-bbr + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 + with: + name: cov-packet-verification-low-power-1 + path: tmp/coverage.info + - name: Keep-1-2-bbr-only + run: | + ./script/test tar simulation 1.2 + ./script/test untar simulation 1.2-bbr + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 + with: + name: cov-packet-verification-low-power-2 + path: tmp/coverage.info packet-verification-1-1-on-1-2: runs-on: ubuntu-18.04 @@ -159,36 +177,38 @@ jobs: PACKET_VERIFICATION: 1 THREAD_VERSION: 1.2 steps: - - uses: actions/checkout@v2 - - name: Bootstrap - run: | - sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build - python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - - name: Build - run: | - ./script/test build - - name: Get Thread-Wireshark - run: | - ./script/test get_thread_wireshark - - name: Run - run: | - for i in {1..3} - do - ./script/test cert_suite ./tests/scripts/thread-cert/Cert_*.py - done - - uses: actions/upload-artifact@v2 - if: ${{ failure() }} - with: - name: packet-verification-1.1-on-1.2-pcaps - path: | - *.pcap - *.json - - name: Codecov - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: true - + - uses: actions/checkout@v2 + - name: Bootstrap + run: | + sudo rm /etc/apt/sources.list.d/* && sudo apt-get update + sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build lcov + python3 -m pip install -r tests/scripts/thread-cert/requirements.txt + - name: Build + run: | + ./script/test build + - name: Get Thread-Wireshark + run: | + ./script/test get_thread_wireshark + - name: Run + run: | + for i in {1..3} + do + ./script/test cert_suite ./tests/scripts/thread-cert/Cert_*.py + done + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: packet-verification-1.1-on-1.2-pcaps + path: | + *.pcap + *.json + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 + with: + name: cov-packet-verification-1-1-on-1-2 + path: tmp/coverage.info expects: runs-on: ubuntu-18.04 @@ -200,7 +220,7 @@ jobs: - uses: actions/checkout@v2 - name: Bootstrap run: | - sudo apt-get --no-install-recommends install -y expect ninja-build + sudo apt-get --no-install-recommends install -y expect ninja-build lcov - name: Run RCP Mode run: | OT_OPTIONS=-DOT_READLINE=OFF NODE_MODE=rcp ./script/test build expect @@ -211,10 +231,13 @@ jobs: - name: Keep POSIX Only run: | ./script/test tar simulation 1.2 - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-expects + path: tmp/coverage.info thread-1-2-backbone: runs-on: ubuntu-18.04 @@ -230,35 +253,62 @@ jobs: VERBOSE: 1 OTBR_COMMIT: "3cf6ff1126dca96803252514691f40d0554046bf" # Date: Mon Oct 26 15:27:04 2020 +0800 steps: - - uses: actions/checkout@v2 - - name: Build OTBR Docker - run: | - ./script/test build_otbr_docker - - name: Bootstrap - run: | - sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel ninja-build socat - python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - - name: Build - run: | - ./script/test build - - name: Get Thread-Wireshark - run: | - ./script/test get_thread_wireshark - - name: Run - run: | - export CI_ENV="$(bash <(curl -s https://codecov.io/env)) -e GITHUB_ACTIONS -e COVERAGE" - echo "CI_ENV=${CI_ENV}" - sudo -E ./script/test cert_bbr ./tests/scripts/thread-cert/backbone/*.py || (sudo chmod a+r *.log *.json *.pcap && false) - - uses: actions/upload-artifact@v2 - if: ${{ failure() }} - with: - name: thread-1-2-backbone-results - path: | - *.pcap - *.json - *.log - - name: Codecov - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: true + - uses: actions/checkout@v2 + - name: Build OTBR Docker + run: | + ./script/test build_otbr_docker + - name: Bootstrap + run: | + sudo rm /etc/apt/sources.list.d/* && sudo apt-get update + sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel ninja-build socat lcov + python3 -m pip install -r tests/scripts/thread-cert/requirements.txt + - name: Build + run: | + ./script/test build + - name: Get Thread-Wireshark + run: | + ./script/test get_thread_wireshark + - name: Run + run: | + export CI_ENV="$(bash <(curl -s https://codecov.io/env)) -e GITHUB_ACTIONS -e COVERAGE" + echo "CI_ENV=${CI_ENV}" + sudo -E ./script/test cert_bbr ./tests/scripts/thread-cert/backbone/*.py || (sudo chmod a+r *.log *.json *.pcap && false) + - uses: actions/upload-artifact@v2 + with: + name: cov-thread-1-2-backbone-docker + path: /tmp/coverage/ + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: thread-1-2-backbone-results + path: | + *.pcap + *.json + *.log + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 + with: + name: cov-thread-1-2-backbone + path: tmp/coverage.info + + upload-coverage: + needs: + - thread-1-2 + - packet-verification-low-power + - packet-verification-1-1-on-1-2 + - expects + - thread-1-2-backbone + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Bootstrap + run: | + sudo apt-get --no-install-recommends install -y lcov + - uses: actions/download-artifact@v2 + with: + path: coverage/ + - name: Upload Coverage + run: | + script/test upload_codecov diff --git a/.github/workflows/simulation.yml b/.github/workflows/simulation.yml index 3edb53d77..79b6132e3 100644 --- a/.github/workflows/simulation.yml +++ b/.github/workflows/simulation.yml @@ -85,7 +85,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y g++-multilib + sudo apt-get --no-install-recommends install -y g++-multilib lcov ./bootstrap - name: Run OTNS Tests env: @@ -108,11 +108,13 @@ jobs: path: | ./output/*/bin/*.pcap ./output/*/bin/*.replay - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true - + name: cov-cli-ftd-otns + path: tmp/coverage.info packet-verification: runs-on: ubuntu-18.04 @@ -121,36 +123,38 @@ jobs: VIRTUAL_TIME: 1 PACKET_VERIFICATION: 1 steps: - - uses: actions/checkout@v2 - - name: Bootstrap - run: | - sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build - python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - - name: Build - run: | - ./script/test build - - name: Get Thread-Wireshark - run: | - ./script/test get_thread_wireshark - - name: Run - run: | - for i in {1..10} - do - ./script/test cert_suite ./tests/scripts/thread-cert/Cert_*.py - done - - uses: actions/upload-artifact@v2 - if: ${{ failure() }} - with: - name: packet-verification-pcaps - path: | - *.pcap - *.json - - name: Codecov - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: true - + - uses: actions/checkout@v2 + - name: Bootstrap + run: | + sudo rm /etc/apt/sources.list.d/* && sudo apt-get update + sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build lcov + python3 -m pip install -r tests/scripts/thread-cert/requirements.txt + - name: Build + run: | + ./script/test build + - name: Get Thread-Wireshark + run: | + ./script/test get_thread_wireshark + - name: Run + run: | + for i in {1..10} + do + ./script/test cert_suite ./tests/scripts/thread-cert/Cert_*.py + done + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: packet-verification-pcaps + path: | + *.pcap + *.json + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 + with: + name: cov-packet-verification + path: tmp/coverage.info cli-ftd: runs-on: ubuntu-18.04 @@ -166,7 +170,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel + sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel lcov python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - name: Build run: | @@ -180,10 +184,13 @@ jobs: with: name: cli-ftd-thread-cert path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-cli-ftd + path: tmp/coverage.info cli-mtd: runs-on: ubuntu-18.04 @@ -200,7 +207,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel + sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel lcov python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - name: Build run: | @@ -214,10 +221,13 @@ jobs: with: name: cli-mtd-thread-cert path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-cli-mtd + path: tmp/coverage.info cli-time-sync: runs-on: ubuntu-18.04 @@ -234,7 +244,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel + sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel lcov python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - name: Build run: | @@ -248,10 +258,13 @@ jobs: with: name: cli-time-sync-thread-cert path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-cli-time-sync + path: tmp/coverage.info expects: runs-on: ubuntu-18.04 @@ -262,32 +275,41 @@ jobs: - uses: actions/checkout@v2 - name: Bootstrap run: | - OT_OPTIONS=-DOT_READLINE=OFF sudo apt-get --no-install-recommends install -y expect ninja-build + OT_OPTIONS=-DOT_READLINE=OFF sudo apt-get --no-install-recommends install -y expect ninja-build lcov - name: Run run: | OT_OPTIONS=-DOT_TIME_SYNC=ON VIRTUAL_TIME=0 ./script/test build expect - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-expects-1 + path: tmp/coverage.info - name: Run RCP Mode run: | OT_OPTIONS=-DOT_READLINE=OFF VIRTUAL_TIME=0 NODE_MODE=rcp ./script/test clean build expect - name: Keep POSIX Only run: | ./script/test tar simulation 1.1 - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-expects-2 + path: tmp/coverage.info - name: Keep Simulation Only run: | ./script/test tar posix 1.1 ./script/test untar simulation 1.1 - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-expects-3 + path: tmp/coverage.info - name: Run TUN Mode run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update @@ -302,18 +324,24 @@ jobs: - name: Keep POSIX Only run: | ./script/test tar simulation 1.1 - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-expects-4 + path: tmp/coverage.info - name: Keep Simulation Only run: | ./script/test tar posix 1.1 ./script/test untar simulation 1.1 - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-expects-5 + path: tmp/coverage.info external-commissioner: runs-on: ubuntu-18.04 @@ -322,7 +350,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* - sudo apt-get install avahi-daemon avahi-utils -y + sudo apt-get install -y avahi-daemon avahi-utils lcov git clone https://github.com/openthread/ot-commissioner.git /tmp/ot-commissioner --depth 1 --branch master - name: Build run: | @@ -344,10 +372,13 @@ jobs: cd /tmp/ot-commissioner/tests/integration ./bootstrap.sh ./run_tests.sh - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-external-commissioner + path: tmp/coverage.info multiple-instance: runs-on: ubuntu-18.04 @@ -360,7 +391,7 @@ jobs: - uses: actions/checkout@v2 - name: Bootstrap run: | - sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel + sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel lcov python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - name: Build run: | @@ -374,10 +405,13 @@ jobs: with: name: multiple-instance-thread-cert path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-multiple-instance + path: tmp/coverage.info ncp-gcc-m32: runs-on: ubuntu-18.04 @@ -395,7 +429,7 @@ jobs: - name: Bootstrap run: | sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel + sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel lcov python3 -m pip install -r tests/scripts/thread-cert/requirements.txt sudo python3 -m pip install git+https://github.com/openthread/pyspinel - name: Build @@ -410,10 +444,13 @@ jobs: with: name: ncp-gcc-m32-thread-cert path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-ncp-gcc-m32 + path: tmp/coverage.info ncp-clang: runs-on: ubuntu-18.04 @@ -426,25 +463,54 @@ jobs: CC: clang CXX: clang++ steps: - - uses: actions/checkout@v2 - - name: Bootstrap - run: | - sudo rm /etc/apt/sources.list.d/* && sudo apt-get update - sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel llvm - python3 -m pip install -r tests/scripts/thread-cert/requirements.txt - sudo python3 -m pip install git+https://github.com/openthread/pyspinel - - name: Build - run: | - ./bootstrap - make -f examples/Makefile-simulation - - name: Run - run: | - VERBOSE=1 make -f examples/Makefile-simulation check - - uses: actions/upload-artifact@v2 - if: ${{ failure() }} - with: - name: cli-clang-thread-cert - path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert - - name: Codecov - run: | - bash <(curl -s https://codecov.io/bash) -x "llvm-cov gcov" -Z + - uses: actions/checkout@v2 + - name: Bootstrap + run: | + sudo rm /etc/apt/sources.list.d/* && sudo apt-get update + sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel llvm lcov + python3 -m pip install -r tests/scripts/thread-cert/requirements.txt + sudo python3 -m pip install git+https://github.com/openthread/pyspinel + - name: Build + run: | + ./bootstrap + make -f examples/Makefile-simulation + - name: Run + run: | + VERBOSE=1 make -f examples/Makefile-simulation check + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: cli-clang-thread-cert + path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert + - name: Generate Coverage + run: | + ./script/test generate_coverage llvm + - uses: actions/upload-artifact@v2 + with: + name: cov-ncp-clang + path: tmp/coverage.info + + upload-coverage: + needs: + - cli-ftd-otns + - packet-verification + - cli-ftd + - cli-mtd + - cli-time-sync + - expects + - external-commissioner + - multiple-instance + - ncp-gcc-m32 + - ncp-clang + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Bootstrap + run: | + sudo apt-get --no-install-recommends install -y lcov + - uses: actions/download-artifact@v2 + with: + path: coverage/ + - name: Upload Coverage + run: | + script/test upload_codecov diff --git a/.github/workflows/toranj.yml b/.github/workflows/toranj.yml index 08c45a25c..6547ba00a 100644 --- a/.github/workflows/toranj.yml +++ b/.github/workflows/toranj.yml @@ -55,6 +55,7 @@ jobs: sudo apt-get --no-install-recommends install -y libtool sudo apt-get --no-install-recommends install -y libglib2.0-dev sudo apt-get --no-install-recommends install -y libboost-dev libboost-signals-dev + sudo apt-get --no-install-recommends install -y lcov git clone --depth=1 --branch=master https://github.com/openthread/wpantund.git cd wpantund @@ -65,10 +66,13 @@ jobs: - name: Build & Run run: | top_builddir=$(pwd)/build/toranj ./tests/toranj/start.sh - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-toranj-ncp + path: tmp/coverage.info toranj-rcp: runs-on: ubuntu-18.04 @@ -86,6 +90,7 @@ jobs: sudo apt-get --no-install-recommends install -y libtool sudo apt-get --no-install-recommends install -y libglib2.0-dev sudo apt-get --no-install-recommends install -y libboost-dev libboost-signals-dev + sudo apt-get --no-install-recommends install -y lcov git clone --depth=1 --branch=master https://github.com/openthread/wpantund.git cd wpantund @@ -96,7 +101,27 @@ jobs: - name: Build & Run run: | top_builddir=$(pwd)/build/toranj ./tests/toranj/start.sh - - name: Codecov - uses: codecov/codecov-action@v1 + - name: Generate Coverage + run: | + ./script/test generate_coverage gcc + - uses: actions/upload-artifact@v2 with: - fail_ci_if_error: true + name: cov-toranj-rcp + path: tmp/coverage.info + + upload-coverage: + needs: + - toranj-ncp + - toranj-rcp + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Bootstrap + run: | + sudo apt-get --no-install-recommends install -y lcov + - uses: actions/download-artifact@v2 + with: + path: coverage/ + - name: Upload Coverage + run: | + script/test upload_codecov diff --git a/script/test b/script/test index fbe1c1855..7bb9e6341 100755 --- a/script/test +++ b/script/test @@ -272,7 +272,7 @@ do_build_otbr_docker() local otdir local otbrdir local otbr_options="-DOT_DUA=ON -DOT_MLR=ON -DOT_COVERAGE=ON" - local otbr_commit=${OTBR_COMMIT:-ce1cc3f9141571505bc9a8b705d2eb775849c92c} # ot-br-posix #532 + local otbr_commit=${OTBR_COMMIT:-master} local otbr_docker_image=${OTBR_DOCKER_IMAGE:-otbr-ot12-backbone-ci} otbrdir=$(mktemp -d -t otbr_XXXXXX) @@ -450,6 +450,37 @@ do_untar() rm "${build_dir:?}/${target_name}.tar" } +do_generate_coverage() +{ + mkdir -p tmp/ + sudo chmod 777 tmp/ + rm -f tmp/coverage.lcov + if [[ $1 == "llvm" ]]; then + local llvm_gcov + llvm_gcov="$(mktemp -d)/llvm-gcov" + echo '#!/bin/bash' >>"$llvm_gcov" + echo 'exec llvm-cov gcov "$@"' >>"$llvm_gcov" + chmod +x "$llvm_gcov" + lcov --gcov-tool "$llvm_gcov" --directory . --capture --output-file tmp/coverage.info + else + lcov --directory . --capture --output-file tmp/coverage.info + fi + lcov --list tmp/coverage.info +} + +do_upload_codecov() +{ + ls -R coverage/ + readarray -d '' files < <(find coverage/ -type f -name 'coverage*.info' -print0) + local args=() + for i in "${files[@]}"; do + args+=('-a') + args+=("$i") + done + lcov "${args[@]}" -o final.info + bash <(curl -s https://codecov.io/bash) -f final.info -Z +} + envsetup() { if [[ ${NODE_MODE} == 'rcp' ]]; then @@ -558,6 +589,13 @@ main() shift do_untar "$@" ;; + generate_coverage) + shift + do_generate_coverage "$1" + ;; + upload_codecov) + do_upload_codecov + ;; *) echo echo -e "${COLOR_FAIL}Warning:${COLOR_NONE} Ignoring: '$1'" diff --git a/tests/scripts/thread-cert/node.py b/tests/scripts/thread-cert/node.py index afbe8b89a..4f9e31a43 100755 --- a/tests/scripts/thread-cert/node.py +++ b/tests/scripts/thread-cert/node.py @@ -93,6 +93,7 @@ class OtbrDocker: def _launch_docker(self): subprocess.check_call(f"docker rm -f {self._docker_name} || true", shell=True) CI_ENV = os.getenv('CI_ENV', '').split() + os.makedirs('/tmp/coverage/', exist_ok=True) self._docker_proc = subprocess.Popen(['docker', 'run'] + CI_ENV + [ '--rm', '--name', @@ -107,7 +108,7 @@ class OtbrDocker: '--volume', f'{self._rcp_device}:/dev/ttyUSB0', '-v', - '/tmp/codecov.bash:/tmp/codecov.bash', + '/tmp/coverage/:/tmp/coverage/', config.OTBR_DOCKER_IMAGE, '-B', config.BACKBONE_IFNAME, @@ -160,10 +161,14 @@ class OtbrDocker: if COVERAGE or OTBR_COVERAGE: self.bash('service otbr-agent stop') - codecov_cmd = 'bash /tmp/codecov.bash -Z' + test_name = os.getenv('TEST_NAME') + cov_file_path = f'/tmp/coverage/coverage-{test_name}-{PORT_OFFSET}-{self.nodeid}.info' # Upload OTBR code coverage if OTBR_COVERAGE=1, otherwise OpenThread code coverage. - if not OTBR_COVERAGE: - codecov_cmd += ' -R third_party/openthread/repo' + if OTBR_COVERAGE: + codecov_cmd = f'lcov --directory . --capture --output-file {cov_file_path}' + else: + codecov_cmd = ('lcov --directory build/otbr/third_party/openthread/repo --capture ' + f'--output-file {cov_file_path}') self.bash(codecov_cmd) diff --git a/tests/scripts/thread-cert/run_bbr_tests.py b/tests/scripts/thread-cert/run_bbr_tests.py index 4ac5914a6..784175627 100644 --- a/tests/scripts/thread-cert/run_bbr_tests.py +++ b/tests/scripts/thread-cert/run_bbr_tests.py @@ -86,8 +86,6 @@ def cleanup_env(): def setup_env(): bash(f'docker image inspect {config.OTBR_DOCKER_IMAGE} >/dev/null') - # Download codecov bash to be used for OTBR Dockers - bash('curl -L https://codecov.io/bash -o /tmp/codecov.bash --retry 5') def parse_args():