[continuous-integration] use lcov to generate coverage data (#5635)

.codecov.yml

@@ -9,4 +9,4 @@ ignore:
 
 comment:
   layout: "diff, flags, files"
-  after_n_builds: 16
+  after_n_builds: 4

.github/workflows/posix.yml

@@ -55,7 +55,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y libreadline6-dev python3-setuptools python3-wheel
+        sudo apt-get --no-install-recommends install -y libreadline6-dev python3-setuptools python3-wheel lcov
         python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
     - name: Build
       run: |
@@ -74,18 +74,24 @@ jobs:
       run: |
         tar -cf build/posix.tar -C build/ posix/
         rm -rf build/posix/
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-posix-cli-1
+        path: tmp/coverage.info
     - name: Keep POSIX Only
       run: |
         rm -rf build/x86_64-unknown-linux-gnu/
         tar -xf build/posix.tar -C build/
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-posix-cli-2
+        path: tmp/coverage.info
 
   posix-ncp:
     runs-on: ubuntu-18.04
@@ -102,7 +108,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y libreadline6-dev python3-setuptools python3-wheel
+        sudo apt-get --no-install-recommends install -y libreadline6-dev python3-setuptools python3-wheel lcov
         python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
         sudo python3 -m pip install git+https://github.com/openthread/pyspinel
     - name: Build
@@ -122,18 +128,24 @@ jobs:
       run: |
         tar -cf build/posix.tar -C build/ posix/
         rm -rf build/posix/
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-posix-ncp-1
+        path: tmp/coverage.info
     - name: Keep POSIX Only
       run: |
         rm -rf build/x86_64-unknown-linux-gnu/
         tar -xf build/posix.tar -C build/
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-posix-ncp-2
+        path: tmp/coverage.info
 
   posix-ncp-rcp-migrate:
     runs-on: ubuntu-18.04
@@ -144,7 +156,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y expect
+        sudo apt-get --no-install-recommends install -y expect lcov
     - name: Build
       run: |
         ./bootstrap
@@ -156,18 +168,24 @@ jobs:
       run: |
         tar -cf build/posix.tar -C build/ posix/
         rm -rf build/posix/
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-posix-ncp-rcp-migrate-1
+        path: tmp/coverage.info
     - name: Keep POSIX Only
       run: |
         rm -rf build/x86_64-unknown-linux-gnu/
         tar -xf build/posix.tar -C build/
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-posix-ncp-rcp-migrate-2
+        path: tmp/coverage.info
 
   posix-pty:
     runs-on: ubuntu-18.04
@@ -178,7 +196,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y socat expect
+        sudo apt-get --no-install-recommends install -y socat expect lcov
         cd /tmp
         wget https://github.com/obgm/libcoap/archive/bsd-licensed.tar.gz
         tar xvf bsd-licensed.tar.gz
@@ -198,18 +216,24 @@ jobs:
       run: |
         tar -cf build/posix.tar -C build/ posix/
         rm -rf build/posix/
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-posix-pty-1
+        path: tmp/coverage.info
     - name: Keep POSIX Only
       run: |
         rm -rf build/x86_64-unknown-linux-gnu/
         tar -xf build/posix.tar -C build/
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-posix-pty-2
+        path: tmp/coverage.info
 
   posix-pty-daemon:
     runs-on: ubuntu-18.04
@@ -221,7 +245,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y socat expect
+        sudo apt-get --no-install-recommends install -y socat expect lcov
         cd /tmp
         wget https://github.com/obgm/libcoap/archive/bsd-licensed.tar.gz
         tar xvf bsd-licensed.tar.gz
@@ -241,15 +265,41 @@ jobs:
       run: |
         tar -cf build/posix.tar -C build/ posix/
         rm -rf build/posix/
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-posix-pty-daemon-1
+        path: tmp/coverage.info
     - name: Keep POSIX Only
       run: |
         rm -rf build/x86_64-unknown-linux-gnu/
         tar -xf build/posix.tar -C build/
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-posix-pty-daemon-2
+        path: tmp/coverage.info
+
+  upload-coverage:
+    needs:
+    - posix-cli
+    - posix-ncp
+    - posix-ncp-rcp-migrate
+    - posix-pty
+    - posix-pty-daemon
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Bootstrap
+      run: |
+        sudo apt-get --no-install-recommends install -y lcov
+    - uses: actions/download-artifact@v2
+      with:
+        path: coverage/
+    - name: Upload Coverage
+      run: |
+        script/test upload_codecov

.github/workflows/simulation-1.2.yml

@@ -55,14 +55,14 @@ jobs:
       CXX: ${{ matrix.compiler.cxx }}
     strategy:
       matrix:
-        compiler: [{c: "gcc", cxx: "g++", gcov: "gcov"}, { c: "clang", cxx: "clang++", gcov: "llvm-cov gcov"}]
+        compiler: [{c: "gcc", cxx: "g++", gcov: "gcc"}, { c: "clang", cxx: "clang++", gcov: "llvm"}]
         arch: ["m32", "m64"]
     steps:
     - uses: actions/checkout@v2
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y g++-multilib ninja-build python3-setuptools python3-wheel llvm
+        sudo apt-get --no-install-recommends install -y g++-multilib ninja-build python3-setuptools python3-wheel llvm lcov
         python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
     - name: Build
       run: |
@@ -81,23 +81,35 @@ jobs:
       run: |
         ./script/test tar simulation 1.1
         ./script/test tar simulation 1.2-bbr
-    - name: Codecov
+    - name: Generate Coverage
       run: |
-        bash <(curl -s https://codecov.io/bash) -x "${{ matrix.compiler.gcov }}" -Z
+        ./script/test generate_coverage "${{ matrix.compiler.gcov }}"
+    - uses: actions/upload-artifact@v2
+      with:
+        name: cov-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-1
+        path: tmp/coverage.info
     - name: Keep-1-2-bbr-only
       run: |
         ./script/test tar simulation 1.2
         ./script/test untar simulation 1.2-bbr
-    - name: Codecov
+    - name: Generate Coverage
       run: |
-        bash <(curl -s https://codecov.io/bash) -x "${{ matrix.compiler.gcov }}" -Z
+        ./script/test generate_coverage "${{ matrix.compiler.gcov }}"
+    - uses: actions/upload-artifact@v2
+      with:
+        name: cov-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-2
+        path: tmp/coverage.info
     - name: Keep-1-1-only
       run: |
         ./script/test tar simulation 1.2-bbr
         ./script/test untar simulation 1.1
-    - name: Codecov
+    - name: Generate Coverage
       run: |
-        bash <(curl -s https://codecov.io/bash) -x "${{ matrix.compiler.gcov }}" -Z
+        ./script/test generate_coverage "${{ matrix.compiler.gcov }}"
+    - uses: actions/upload-artifact@v2
+      with:
+        name: cov-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-3
+        path: tmp/coverage.info
 
   packet-verification-low-power:
     runs-on: ubuntu-18.04
@@ -110,46 +122,52 @@ jobs:
       MAC_FILTER: 1
       INTER_OP: 1
     steps:
-      - uses: actions/checkout@v2
-      - name: Bootstrap
-        run: |
-          sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-          sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build
-          python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
-      - name: Build
-        run: |
-          ./script/test build
-      - name: Get Thread-Wireshark
-        run: |
-          ./script/test get_thread_wireshark
-      - name: Run
-        run: |
-          for i in {1..10}
-          do
-            ./script/test cert_suite ./tests/scripts/thread-cert/v1_2_LowPower*.py
-          done
-      - uses: actions/upload-artifact@v2
-        if: ${{ failure() }}
-        with:
-          name: packet-verification-low-power-pcaps
-          path: |
-            *.pcap
-            *.json
-      - name: Keep-1-2-only
-        run: |
-          ./script/test tar simulation 1.2-bbr
-      - name: Codecov
-        uses: codecov/codecov-action@v1
-        with:
-          fail_ci_if_error: true
-      - name: Keep-1-2-bbr-only
-        run: |
-          ./script/test tar simulation 1.2
-          ./script/test untar simulation 1.2-bbr
-      - name: Codecov
-        uses: codecov/codecov-action@v1
-        with:
-          fail_ci_if_error: true
+    - uses: actions/checkout@v2
+    - name: Bootstrap
+      run: |
+        sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
+        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build lcov
+        python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
+    - name: Build
+      run: |
+        ./script/test build
+    - name: Get Thread-Wireshark
+      run: |
+        ./script/test get_thread_wireshark
+    - name: Run
+      run: |
+        for i in {1..10}
+        do
+          ./script/test cert_suite ./tests/scripts/thread-cert/v1_2_LowPower*.py
+        done
+    - uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: packet-verification-low-power-pcaps
+        path: |
+          *.pcap
+          *.json
+    - name: Keep-1-2-only
+      run: |
+        ./script/test tar simulation 1.2-bbr
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
+      with:
+        name: cov-packet-verification-low-power-1
+        path: tmp/coverage.info
+    - name: Keep-1-2-bbr-only
+      run: |
+        ./script/test tar simulation 1.2
+        ./script/test untar simulation 1.2-bbr
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
+      with:
+        name: cov-packet-verification-low-power-2
+        path: tmp/coverage.info
 
   packet-verification-1-1-on-1-2:
     runs-on: ubuntu-18.04
@@ -159,36 +177,38 @@ jobs:
       PACKET_VERIFICATION: 1
       THREAD_VERSION: 1.2
     steps:
-      - uses: actions/checkout@v2
-      - name: Bootstrap
-        run: |
-          sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-          sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build
-          python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
-      - name: Build
-        run: |
-          ./script/test build
-      - name: Get Thread-Wireshark
-        run: |
-          ./script/test get_thread_wireshark
-      - name: Run
-        run: |
-          for i in {1..3}
-          do
-            ./script/test cert_suite ./tests/scripts/thread-cert/Cert_*.py
-          done
-      - uses: actions/upload-artifact@v2
-        if: ${{ failure() }}
-        with:
-          name: packet-verification-1.1-on-1.2-pcaps
-          path: |
-            *.pcap
-            *.json
-      - name: Codecov
-        uses: codecov/codecov-action@v1
-        with:
-          fail_ci_if_error: true
-
+    - uses: actions/checkout@v2
+    - name: Bootstrap
+      run: |
+        sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
+        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build lcov
+        python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
+    - name: Build
+      run: |
+        ./script/test build
+    - name: Get Thread-Wireshark
+      run: |
+        ./script/test get_thread_wireshark
+    - name: Run
+      run: |
+        for i in {1..3}
+        do
+          ./script/test cert_suite ./tests/scripts/thread-cert/Cert_*.py
+        done
+    - uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: packet-verification-1.1-on-1.2-pcaps
+        path: |
+          *.pcap
+          *.json
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
+      with:
+        name: cov-packet-verification-1-1-on-1-2
+        path: tmp/coverage.info
 
   expects:
     runs-on: ubuntu-18.04
@@ -200,7 +220,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Bootstrap
       run: |
-        sudo apt-get --no-install-recommends install -y expect ninja-build
+        sudo apt-get --no-install-recommends install -y expect ninja-build lcov
     - name: Run RCP Mode
       run: |
         OT_OPTIONS=-DOT_READLINE=OFF NODE_MODE=rcp ./script/test build expect
@@ -211,10 +231,13 @@ jobs:
     - name: Keep POSIX Only
       run: |
         ./script/test tar simulation 1.2
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-expects
+        path: tmp/coverage.info
 
   thread-1-2-backbone:
     runs-on: ubuntu-18.04
@@ -230,35 +253,62 @@ jobs:
       VERBOSE: 1
       OTBR_COMMIT: "3cf6ff1126dca96803252514691f40d0554046bf" # Date:   Mon Oct 26 15:27:04 2020 +0800
     steps:
-      - uses: actions/checkout@v2
-      - name: Build OTBR Docker
-        run: |
-          ./script/test build_otbr_docker
-      - name: Bootstrap
-        run: |
-          sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-          sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel ninja-build socat
-          python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
-      - name: Build
-        run: |
-          ./script/test build
-      - name: Get Thread-Wireshark
-        run: |
-          ./script/test get_thread_wireshark
-      - name: Run
-        run: |
-          export CI_ENV="$(bash <(curl -s https://codecov.io/env)) -e GITHUB_ACTIONS -e COVERAGE"
-          echo "CI_ENV=${CI_ENV}"
-          sudo -E ./script/test cert_bbr ./tests/scripts/thread-cert/backbone/*.py || (sudo chmod a+r *.log *.json *.pcap && false)
-      - uses: actions/upload-artifact@v2
-        if: ${{ failure() }}
-        with:
-          name: thread-1-2-backbone-results
-          path: |
-            *.pcap
-            *.json
-            *.log
-      - name: Codecov
-        uses: codecov/codecov-action@v1
-        with:
-          fail_ci_if_error: true
+    - uses: actions/checkout@v2
+    - name: Build OTBR Docker
+      run: |
+        ./script/test build_otbr_docker
+    - name: Bootstrap
+      run: |
+        sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
+        sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel ninja-build socat lcov
+        python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
+    - name: Build
+      run: |
+        ./script/test build
+    - name: Get Thread-Wireshark
+      run: |
+        ./script/test get_thread_wireshark
+    - name: Run
+      run: |
+        export CI_ENV="$(bash <(curl -s https://codecov.io/env)) -e GITHUB_ACTIONS -e COVERAGE"
+        echo "CI_ENV=${CI_ENV}"
+        sudo -E ./script/test cert_bbr ./tests/scripts/thread-cert/backbone/*.py || (sudo chmod a+r *.log *.json *.pcap && false)
+    - uses: actions/upload-artifact@v2
+      with:
+        name: cov-thread-1-2-backbone-docker
+        path: /tmp/coverage/
+    - uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: thread-1-2-backbone-results
+        path: |
+          *.pcap
+          *.json
+          *.log
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
+      with:
+        name: cov-thread-1-2-backbone
+        path: tmp/coverage.info
+
+  upload-coverage:
+    needs:
+    - thread-1-2
+    - packet-verification-low-power
+    - packet-verification-1-1-on-1-2
+    - expects
+    - thread-1-2-backbone
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Bootstrap
+      run: |
+        sudo apt-get --no-install-recommends install -y lcov
+    - uses: actions/download-artifact@v2
+      with:
+        path: coverage/
+    - name: Upload Coverage
+      run: |
+        script/test upload_codecov

.github/workflows/simulation.yml

@@ -85,7 +85,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y g++-multilib
+        sudo apt-get --no-install-recommends install -y g++-multilib lcov
         ./bootstrap
     - name: Run OTNS Tests
       env:
@@ -108,11 +108,13 @@ jobs:
         path: |
           ./output/*/bin/*.pcap
           ./output/*/bin/*.replay
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
-
+        name: cov-cli-ftd-otns
+        path: tmp/coverage.info
 
   packet-verification:
     runs-on: ubuntu-18.04
@@ -121,36 +123,38 @@ jobs:
       VIRTUAL_TIME: 1
       PACKET_VERIFICATION: 1
     steps:
-      - uses: actions/checkout@v2
-      - name: Bootstrap
-        run: |
-          sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-          sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build
-          python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
-      - name: Build
-        run: |
-          ./script/test build
-      - name: Get Thread-Wireshark
-        run: |
-          ./script/test get_thread_wireshark
-      - name: Run
-        run: |
-          for i in {1..10}
-          do
-            ./script/test cert_suite ./tests/scripts/thread-cert/Cert_*.py
-          done
-      - uses: actions/upload-artifact@v2
-        if: ${{ failure() }}
-        with:
-          name: packet-verification-pcaps
-          path: |
-            *.pcap
-            *.json
-      - name: Codecov
-        uses: codecov/codecov-action@v1
-        with:
-          fail_ci_if_error: true
-
+    - uses: actions/checkout@v2
+    - name: Bootstrap
+      run: |
+        sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
+        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel ninja-build lcov
+        python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
+    - name: Build
+      run: |
+        ./script/test build
+    - name: Get Thread-Wireshark
+      run: |
+        ./script/test get_thread_wireshark
+    - name: Run
+      run: |
+        for i in {1..10}
+        do
+          ./script/test cert_suite ./tests/scripts/thread-cert/Cert_*.py
+        done
+    - uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: packet-verification-pcaps
+        path: |
+          *.pcap
+          *.json
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
+      with:
+        name: cov-packet-verification
+        path: tmp/coverage.info
 
   cli-ftd:
     runs-on: ubuntu-18.04
@@ -166,7 +170,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel
+        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel lcov
         python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
     - name: Build
       run: |
@@ -180,10 +184,13 @@ jobs:
       with:
         name: cli-ftd-thread-cert
         path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-cli-ftd
+        path: tmp/coverage.info
 
   cli-mtd:
     runs-on: ubuntu-18.04
@@ -200,7 +207,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel
+        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel lcov
         python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
     - name: Build
       run: |
@@ -214,10 +221,13 @@ jobs:
       with:
         name: cli-mtd-thread-cert
         path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-cli-mtd
+        path: tmp/coverage.info
 
   cli-time-sync:
     runs-on: ubuntu-18.04
@@ -234,7 +244,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel
+        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel lcov
         python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
     - name: Build
       run: |
@@ -248,10 +258,13 @@ jobs:
       with:
         name: cli-time-sync-thread-cert
         path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-cli-time-sync
+        path: tmp/coverage.info
 
   expects:
     runs-on: ubuntu-18.04
@@ -262,32 +275,41 @@ jobs:
     - uses: actions/checkout@v2
     - name: Bootstrap
       run: |
-        OT_OPTIONS=-DOT_READLINE=OFF sudo apt-get --no-install-recommends install -y expect ninja-build
+        OT_OPTIONS=-DOT_READLINE=OFF sudo apt-get --no-install-recommends install -y expect ninja-build lcov
     - name: Run
       run: |
         OT_OPTIONS=-DOT_TIME_SYNC=ON VIRTUAL_TIME=0 ./script/test build expect
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-expects-1
+        path: tmp/coverage.info
     - name: Run RCP Mode
       run: |
         OT_OPTIONS=-DOT_READLINE=OFF VIRTUAL_TIME=0 NODE_MODE=rcp ./script/test clean build expect
     - name: Keep POSIX Only
       run: |
         ./script/test tar simulation 1.1
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-expects-2
+        path: tmp/coverage.info
     - name: Keep Simulation Only
       run: |
         ./script/test tar posix 1.1
         ./script/test untar simulation 1.1
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-expects-3
+        path: tmp/coverage.info
     - name: Run TUN Mode
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
@@ -302,18 +324,24 @@ jobs:
     - name: Keep POSIX Only
       run: |
         ./script/test tar simulation 1.1
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-expects-4
+        path: tmp/coverage.info
     - name: Keep Simulation Only
       run: |
         ./script/test tar posix 1.1
         ./script/test untar simulation 1.1
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-expects-5
+        path: tmp/coverage.info
 
   external-commissioner:
     runs-on: ubuntu-18.04
@@ -322,7 +350,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/*
-        sudo apt-get install avahi-daemon avahi-utils -y
+        sudo apt-get install -y avahi-daemon avahi-utils lcov
         git clone https://github.com/openthread/ot-commissioner.git /tmp/ot-commissioner --depth 1 --branch master
     - name: Build
       run: |
@@ -344,10 +372,13 @@ jobs:
         cd /tmp/ot-commissioner/tests/integration
         ./bootstrap.sh
         ./run_tests.sh
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-external-commissioner
+        path: tmp/coverage.info
 
   multiple-instance:
     runs-on: ubuntu-18.04
@@ -360,7 +391,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Bootstrap
       run: |
-        sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel
+        sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel lcov
         python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
     - name: Build
       run: |
@@ -374,10 +405,13 @@ jobs:
       with:
         name: multiple-instance-thread-cert
         path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-multiple-instance
+        path: tmp/coverage.info
 
   ncp-gcc-m32:
     runs-on: ubuntu-18.04
@@ -395,7 +429,7 @@ jobs:
     - name: Bootstrap
       run: |
         sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel
+        sudo apt-get --no-install-recommends install -y g++-multilib python3-setuptools python3-wheel lcov
         python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
         sudo python3 -m pip install git+https://github.com/openthread/pyspinel
     - name: Build
@@ -410,10 +444,13 @@ jobs:
       with:
         name: ncp-gcc-m32-thread-cert
         path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-ncp-gcc-m32
+        path: tmp/coverage.info
 
   ncp-clang:
     runs-on: ubuntu-18.04
@@ -426,25 +463,54 @@ jobs:
       CC: clang
       CXX: clang++
     steps:
-      - uses: actions/checkout@v2
-      - name: Bootstrap
-        run: |
-          sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
-          sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel llvm
-          python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
-          sudo python3 -m pip install git+https://github.com/openthread/pyspinel
-      - name: Build
-        run: |
-          ./bootstrap
-          make -f examples/Makefile-simulation
-      - name: Run
-        run: |
-          VERBOSE=1 make -f examples/Makefile-simulation check
-      - uses: actions/upload-artifact@v2
-        if: ${{ failure() }}
-        with:
-          name: cli-clang-thread-cert
-          path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert
-      - name: Codecov
-        run: |
-          bash <(curl -s https://codecov.io/bash) -x "llvm-cov gcov" -Z
+    - uses: actions/checkout@v2
+    - name: Bootstrap
+      run: |
+        sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
+        sudo apt-get --no-install-recommends install -y python3-setuptools python3-wheel llvm lcov
+        python3 -m pip install -r tests/scripts/thread-cert/requirements.txt
+        sudo python3 -m pip install git+https://github.com/openthread/pyspinel
+    - name: Build
+      run: |
+        ./bootstrap
+        make -f examples/Makefile-simulation
+    - name: Run
+      run: |
+        VERBOSE=1 make -f examples/Makefile-simulation check
+    - uses: actions/upload-artifact@v2
+      if: ${{ failure() }}
+      with:
+        name: cli-clang-thread-cert
+        path: build/x86_64-unknown-linux-gnu/tests/scripts/thread-cert
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage llvm
+    - uses: actions/upload-artifact@v2
+      with:
+        name: cov-ncp-clang
+        path: tmp/coverage.info
+
+  upload-coverage:
+    needs:
+    - cli-ftd-otns
+    - packet-verification
+    - cli-ftd
+    - cli-mtd
+    - cli-time-sync
+    - expects
+    - external-commissioner
+    - multiple-instance
+    - ncp-gcc-m32
+    - ncp-clang
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Bootstrap
+      run: |
+        sudo apt-get --no-install-recommends install -y lcov
+    - uses: actions/download-artifact@v2
+      with:
+        path: coverage/
+    - name: Upload Coverage
+      run: |
+        script/test upload_codecov

.github/workflows/toranj.yml

@@ -55,6 +55,7 @@ jobs:
         sudo apt-get --no-install-recommends install -y libtool
         sudo apt-get --no-install-recommends install -y libglib2.0-dev
         sudo apt-get --no-install-recommends install -y libboost-dev libboost-signals-dev
+        sudo apt-get --no-install-recommends install -y lcov
 
         git clone --depth=1 --branch=master https://github.com/openthread/wpantund.git
         cd wpantund
@@ -65,10 +66,13 @@ jobs:
     - name: Build & Run
       run: |
         top_builddir=$(pwd)/build/toranj ./tests/toranj/start.sh
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-toranj-ncp
+        path: tmp/coverage.info
 
   toranj-rcp:
     runs-on: ubuntu-18.04
@@ -86,6 +90,7 @@ jobs:
         sudo apt-get --no-install-recommends install -y libtool
         sudo apt-get --no-install-recommends install -y libglib2.0-dev
         sudo apt-get --no-install-recommends install -y libboost-dev libboost-signals-dev
+        sudo apt-get --no-install-recommends install -y lcov
 
         git clone --depth=1 --branch=master https://github.com/openthread/wpantund.git
         cd wpantund
@@ -96,7 +101,27 @@ jobs:
     - name: Build & Run
       run: |
         top_builddir=$(pwd)/build/toranj ./tests/toranj/start.sh
-    - name: Codecov
-      uses: codecov/codecov-action@v1
+    - name: Generate Coverage
+      run: |
+        ./script/test generate_coverage gcc
+    - uses: actions/upload-artifact@v2
       with:
-        fail_ci_if_error: true
+        name: cov-toranj-rcp
+        path: tmp/coverage.info
+
+  upload-coverage:
+    needs:
+    - toranj-ncp
+    - toranj-rcp
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Bootstrap
+      run: |
+        sudo apt-get --no-install-recommends install -y lcov
+    - uses: actions/download-artifact@v2
+      with:
+        path: coverage/
+    - name: Upload Coverage
+      run: |
+        script/test upload_codecov

script/test

@@ -272,7 +272,7 @@ do_build_otbr_docker()
     local otdir
     local otbrdir
     local otbr_options="-DOT_DUA=ON -DOT_MLR=ON -DOT_COVERAGE=ON"
-    local otbr_commit=${OTBR_COMMIT:-ce1cc3f9141571505bc9a8b705d2eb775849c92c} # ot-br-posix #532
+    local otbr_commit=${OTBR_COMMIT:-master}
     local otbr_docker_image=${OTBR_DOCKER_IMAGE:-otbr-ot12-backbone-ci}
 
     otbrdir=$(mktemp -d -t otbr_XXXXXX)
@@ -450,6 +450,37 @@ do_untar()
     rm "${build_dir:?}/${target_name}.tar"
 }
 
+do_generate_coverage()
+{
+    mkdir -p tmp/
+    sudo chmod 777 tmp/
+    rm -f tmp/coverage.lcov
+    if [[ $1 == "llvm" ]]; then
+        local llvm_gcov
+        llvm_gcov="$(mktemp -d)/llvm-gcov"
+        echo '#!/bin/bash' >>"$llvm_gcov"
+        echo 'exec llvm-cov gcov "$@"' >>"$llvm_gcov"
+        chmod +x "$llvm_gcov"
+        lcov --gcov-tool "$llvm_gcov" --directory . --capture --output-file tmp/coverage.info
+    else
+        lcov --directory . --capture --output-file tmp/coverage.info
+    fi
+    lcov --list tmp/coverage.info
+}
+
+do_upload_codecov()
+{
+    ls -R coverage/
+    readarray -d '' files < <(find coverage/ -type f -name 'coverage*.info' -print0)
+    local args=()
+    for i in "${files[@]}"; do
+        args+=('-a')
+        args+=("$i")
+    done
+    lcov "${args[@]}" -o final.info
+    bash <(curl -s https://codecov.io/bash) -f final.info -Z
+}
+
 envsetup()
 {
     if [[ ${NODE_MODE} == 'rcp' ]]; then
@@ -558,6 +589,13 @@ main()
                 shift
                 do_untar "$@"
                 ;;
+            generate_coverage)
+                shift
+                do_generate_coverage "$1"
+                ;;
+            upload_codecov)
+                do_upload_codecov
+                ;;
             *)
                 echo
                 echo -e "${COLOR_FAIL}Warning:${COLOR_NONE} Ignoring: '$1'"

tests/scripts/thread-cert/node.py

@@ -93,6 +93,7 @@ class OtbrDocker:
     def _launch_docker(self):
         subprocess.check_call(f"docker rm -f {self._docker_name} || true", shell=True)
         CI_ENV = os.getenv('CI_ENV', '').split()
+        os.makedirs('/tmp/coverage/', exist_ok=True)
         self._docker_proc = subprocess.Popen(['docker', 'run'] + CI_ENV + [
             '--rm',
             '--name',
@@ -107,7 +108,7 @@ class OtbrDocker:
             '--volume',
             f'{self._rcp_device}:/dev/ttyUSB0',
             '-v',
-            '/tmp/codecov.bash:/tmp/codecov.bash',
+            '/tmp/coverage/:/tmp/coverage/',
             config.OTBR_DOCKER_IMAGE,
             '-B',
             config.BACKBONE_IFNAME,
@@ -160,10 +161,14 @@ class OtbrDocker:
             if COVERAGE or OTBR_COVERAGE:
                 self.bash('service otbr-agent stop')
 
-                codecov_cmd = 'bash /tmp/codecov.bash -Z'
+                test_name = os.getenv('TEST_NAME')
+                cov_file_path = f'/tmp/coverage/coverage-{test_name}-{PORT_OFFSET}-{self.nodeid}.info'
                 # Upload OTBR code coverage if OTBR_COVERAGE=1, otherwise OpenThread code coverage.
-                if not OTBR_COVERAGE:
-                    codecov_cmd += ' -R third_party/openthread/repo'
+                if OTBR_COVERAGE:
+                    codecov_cmd = f'lcov --directory . --capture --output-file {cov_file_path}'
+                else:
+                    codecov_cmd = ('lcov --directory build/otbr/third_party/openthread/repo --capture '
+                                   f'--output-file {cov_file_path}')
 
                 self.bash(codecov_cmd)
 

tests/scripts/thread-cert/run_bbr_tests.py

@@ -86,8 +86,6 @@ def cleanup_env():
 
 def setup_env():
     bash(f'docker image inspect {config.OTBR_DOCKER_IMAGE} >/dev/null')
-    # Download codecov bash to be used for OTBR Dockers
-    bash('curl -L https://codecov.io/bash -o /tmp/codecov.bash --retry 5')
 
 
 def parse_args():