[test] separate code coverage data files (#5920)

This commit ensures each node create its own profile data, so that they can be merged after tests to produce more accurate profile data. This commit also prints the coverage data of message.cpp after all profile data are merged.
2026-06-05 21:14:49 +00:00 · 2020-12-08 05:23:26 +08:00
parent dc5c25799d
commit 7a7390b9ee
8 changed files with 141 additions and 198 deletions
@@ -70,27 +70,12 @@ jobs:
      with:
        name: posix-cli-thread-cert
        path: build/posix/tests/scripts/thread-cert
-    - name: Keep Simulation Only
-      run: |
-        tar -cf build/posix.tar -C build/ posix/
-        rm -rf build/posix/
    - name: Generate Coverage
      run: |
        ./script/test generate_coverage gcc
    - uses: actions/upload-artifact@v2
      with:
-        name: cov-posix-cli-1
-        path: tmp/coverage.info
-    - name: Keep POSIX Only
-      run: |
-        rm -rf build/simulation/
-        tar -xf build/posix.tar -C build/
-    - name: Generate Coverage
-      run: |
-        ./script/test generate_coverage gcc
-    - uses: actions/upload-artifact@v2
-      with:
-        name: cov-posix-cli-2
+        name: cov-posix-cli
        path: tmp/coverage.info

  posix-ncp:
@@ -124,27 +109,12 @@ jobs:
      with:
        name: posix-ncp-thread-cert
        path: build/posix/tests/scripts/thread-cert
-    - name: Keep Simulation Only
-      run: |
-        tar -cf build/posix.tar -C build/ posix/
-        rm -rf build/posix/
    - name: Generate Coverage
      run: |
        ./script/test generate_coverage gcc
    - uses: actions/upload-artifact@v2
      with:
-        name: cov-posix-ncp-1
-        path: tmp/coverage.info
-    - name: Keep POSIX Only
-      run: |
-        rm -rf build/simulation/
-        tar -xf build/posix.tar -C build/
-    - name: Generate Coverage
-      run: |
-        ./script/test generate_coverage gcc
-    - uses: actions/upload-artifact@v2
-      with:
-        name: cov-posix-ncp-2
+        name: cov-posix-ncp
        path: tmp/coverage.info

  posix-ncp-rcp-migrate:
@@ -164,27 +134,12 @@ jobs:
    - name: Run
      run: |
        script/check-ncp-rcp-migrate check
-    - name: Keep Simulation Only
-      run: |
-        tar -cf build/posix.tar -C build/ posix/
-        rm -rf build/posix/
    - name: Generate Coverage
      run: |
        ./script/test generate_coverage gcc
    - uses: actions/upload-artifact@v2
      with:
-        name: cov-posix-ncp-rcp-migrate-1
-        path: tmp/coverage.info
-    - name: Keep POSIX Only
-      run: |
-        rm -rf build/simulation/
-        tar -xf build/posix.tar -C build/
-    - name: Generate Coverage
-      run: |
-        ./script/test generate_coverage gcc
-    - uses: actions/upload-artifact@v2
-      with:
-        name: cov-posix-ncp-rcp-migrate-2
+        name: cov-posix-ncp-rcp-migrate
        path: tmp/coverage.info

  posix-pty:
@@ -212,27 +167,12 @@ jobs:
    - name: Run
      run: |
        script/check-posix-pty check
-    - name: Keep Simulation Only
-      run: |
-        tar -cf build/posix.tar -C build/ posix/
-        rm -rf build/posix/
    - name: Generate Coverage
      run: |
        ./script/test generate_coverage gcc
    - uses: actions/upload-artifact@v2
      with:
-        name: cov-posix-pty-1
-        path: tmp/coverage.info
-    - name: Keep POSIX Only
-      run: |
-        rm -rf build/simulation/
-        tar -xf build/posix.tar -C build/
-    - name: Generate Coverage
-      run: |
-        ./script/test generate_coverage gcc
-    - uses: actions/upload-artifact@v2
-      with:
-        name: cov-posix-pty-2
+        name: cov-posix-pty
        path: tmp/coverage.info

  posix-pty-daemon:
@@ -261,27 +201,12 @@ jobs:
    - name: Run
      run: |
        script/check-posix-pty check
-    - name: Keep Simulation Only
-      run: |
-        tar -cf build/posix.tar -C build/ posix/
-        rm -rf build/posix/
    - name: Generate Coverage
      run: |
        ./script/test generate_coverage gcc
    - uses: actions/upload-artifact@v2
      with:
-        name: cov-posix-pty-daemon-1
-        path: tmp/coverage.info
-    - name: Keep POSIX Only
-      run: |
-        rm -rf build/simulation/
-        tar -xf build/posix.tar -C build/
-    - name: Generate Coverage
-      run: |
-        ./script/test generate_coverage gcc
-    - uses: actions/upload-artifact@v2
-      with:
-        name: cov-posix-pty-daemon-2
+        name: cov-posix-pty-daemon
        path: tmp/coverage.info

  upload-coverage:
@@ -83,46 +83,12 @@ jobs:
      with:
        name: thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-pcaps
        path: "*.pcap"
-    - uses: actions/upload-artifact@v2
-      if: ${{ failure() }}
-      with:
-        name: core-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}
-        path: |
-          ./build/openthread-simulation-1.2/examples/apps/cli/ot-cli-*
-          ./ot-core-dump/*
-          ./so-lib/*
-    - name: Keep-1-2-only
-      run: |
-        ./script/test tar simulation 1.1
-        ./script/test tar simulation 1.2-bbr
    - name: Generate Coverage
      run: |
        ./script/test generate_coverage "${{ matrix.compiler.gcov }}"
    - uses: actions/upload-artifact@v2
      with:
-        name: cov-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-1
-        path: tmp/coverage.info
-    - name: Keep-1-2-bbr-only
-      run: |
-        ./script/test tar simulation 1.2
-        ./script/test untar simulation 1.2-bbr
-    - name: Generate Coverage
-      run: |
-        ./script/test generate_coverage "${{ matrix.compiler.gcov }}"
-    - uses: actions/upload-artifact@v2
-      with:
-        name: cov-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-2
-        path: tmp/coverage.info
-    - name: Keep-1-1-only
-      run: |
-        ./script/test tar simulation 1.2-bbr
-        ./script/test untar simulation 1.1
-    - name: Generate Coverage
-      run: |
-        ./script/test generate_coverage "${{ matrix.compiler.gcov }}"
-    - uses: actions/upload-artifact@v2
-      with:
-        name: cov-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}-3
+        name: cov-thread-1-2-${{ matrix.compiler.c }}-${{ matrix.arch }}
        path: tmp/coverage.info

  packet-verification-low-power:
@@ -176,26 +142,12 @@ jobs:
          ./build/openthread-simulation-1.1/examples/apps/cli/ot-cli-*
          ./ot-core-dump/*
          ./so-lib/*
-    - name: Keep-1-2-only
-      run: |
-        ./script/test tar simulation 1.2-bbr
    - name: Generate Coverage
      run: |
        ./script/test generate_coverage gcc
    - uses: actions/upload-artifact@v2
      with:
-        name: cov-packet-verification-low-power-1
-        path: tmp/coverage.info
-    - name: Keep-1-2-bbr-only
-      run: |
-        ./script/test tar simulation 1.2
-        ./script/test untar simulation 1.2-bbr
-    - name: Generate Coverage
-      run: |
-        ./script/test generate_coverage gcc
-    - uses: actions/upload-artifact@v2
-      with:
-        name: cov-packet-verification-low-power-2
+        name: cov-packet-verification-low-power
        path: tmp/coverage.info

  packet-verification-1-1-on-1-2:
@@ -253,13 +205,6 @@ jobs:
    - name: Run RCP Mode
      run: |
        OT_OPTIONS=-DOT_READLINE=OFF OT_NODE_TYPE=rcp ./script/test build expect
-    - name: Keep-1-2-only
-      run: |
-        ./script/test tar posix 1.2-bbr
-        ./script/test tar simulation 1.2-bbr
-    - name: Keep POSIX Only
-      run: |
-        ./script/test tar simulation 1.2
    - name: Generate Coverage
      run: |
        ./script/test generate_coverage gcc
@@ -290,9 +290,6 @@ jobs:
    - name: Run RCP Mode
      run: |
        OT_OPTIONS=-DOT_READLINE=OFF VIRTUAL_TIME=0 OT_NODE_TYPE=rcp ./script/test clean build expect
-    - name: Keep POSIX Only
-      run: |
-        ./script/test tar simulation 1.1
    - name: Generate Coverage
      run: |
        ./script/test generate_coverage gcc
@@ -300,17 +297,6 @@ jobs:
      with:
        name: cov-expects-2
        path: tmp/coverage.info
-    - name: Keep Simulation Only
-      run: |
-        ./script/test tar posix 1.1
-        ./script/test untar simulation 1.1
-    - name: Generate Coverage
-      run: |
-        ./script/test generate_coverage gcc
-    - uses: actions/upload-artifact@v2
-      with:
-        name: cov-expects-3
-        path: tmp/coverage.info
    - name: Run TUN Mode
      run: |
        sudo rm /etc/apt/sources.list.d/* && sudo apt-get update
@@ -322,26 +308,12 @@ jobs:
        sudo systemctl restart dnsmasq
        host ipv6.google.com ::1
        OT_OPTIONS=-DOT_READLINE=OFF OT_NATIVE_IP=1 VIRTUAL_TIME=0 OT_NODE_TYPE=rcp ./script/test clean build expect
-    - name: Keep POSIX Only
-      run: |
-        ./script/test tar simulation 1.1
    - name: Generate Coverage
      run: |
        ./script/test generate_coverage gcc
    - uses: actions/upload-artifact@v2
      with:
-        name: cov-expects-4
-        path: tmp/coverage.info
-    - name: Keep Simulation Only
-      run: |
-        ./script/test tar posix 1.1
-        ./script/test untar simulation 1.1
-    - name: Generate Coverage
-      run: |
-        ./script/test generate_coverage gcc
-    - uses: actions/upload-artifact@v2
-      with:
-        name: cov-expects-5
+        name: cov-expects-3
        path: tmp/coverage.info

  external-commissioner:
@@ -0,0 +1,107 @@
+#!/bin/bash
+#
+#  Copyright (c) 2020, The OpenThread Authors.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#  3. Neither the name of the copyright holder nor the
+#     names of its contributors may be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+#  POSSIBILITY OF SUCH DAMAGE.
+#
+
+set -euxo pipefail
+
+readonly OT_MERGED_PROFILES=merged_profiles
+readonly OT_GCOV_PREFIX_BASE=ot-run
+
+merge_profiles()
+{
+    local profile_current="$1"
+    local profile_merged="$2"
+
+    [[ -d ${profile_current} ]] || return 0
+
+    local profile_temporary="${profile_merged}.tmp"
+
+    gcov-tool merge "${profile_current}" "${profile_merged}" -o "${profile_temporary}" || return 0
+    cp -r "${profile_temporary}"/* "${profile_merged}"
+    rm -rf "${profile_temporary}"
+}
+
+do_clean()
+{
+    rm -rfv "${OT_MERGED_PROFILES}" "${OT_GCOV_PREFIX_BASE}" || sudo rm -rfv "${OT_MERGED_PROFILES}" "${OT_GCOV_PREFIX_BASE}"
+}
+
+do_collect()
+{
+    [[ -d ${OT_GCOV_PREFIX_BASE} ]] || return 0
+    sudo chown -R "$USER" "${OT_GCOV_PREFIX_BASE}"
+
+    while read -r node_gcda; do
+        while read -r build_name; do
+            local profile_current="${node_gcda}/$PWD/build/${build_name}"
+            local profile_merged="${OT_MERGED_PROFILES}/build/${build_name}"
+            if [[ -d ${profile_merged} ]]; then
+                merge_profiles "${profile_current}" "${profile_merged}"
+                rm -rf "${profile_current}"
+            else
+                mkdir -p "$(dirname "${profile_merged}")"
+                mv "${profile_current}" "${profile_merged}"
+            fi
+        done < <(ls "${node_gcda}/$PWD/build")
+    done < <(find "${OT_GCOV_PREFIX_BASE}" -type d -name 'ot-gcda.*')
+}
+
+do_install()
+{
+    [[ -d ${OT_MERGED_PROFILES} ]] || return 0
+
+    while read -r build_name; do
+        local profile_current="build/${build_name}"
+        local profile_merged="${OT_MERGED_PROFILES}/build/${build_name}"
+        merge_profiles "${profile_current}" "${profile_merged}"
+    done < <(ls "build")
+
+    sudo chown -R "$USER" build
+    cp -r "${OT_MERGED_PROFILES}"/build/* build
+    rm -rf "${OT_MERGED_PROFILES}" "${OT_GCOV_PREFIX_BASE}"
+}
+
+main()
+{
+    case $1 in
+        collect)
+            do_collect
+            ;;
+        install)
+            do_install
+            ;;
+        clean)
+            do_clean
+            ;;
+        *)
+            false
+            ;;
+    esac
+}
+
+main "$@"
@@ -32,8 +32,8 @@

 set -euo pipefail

-readonly OT_BUILDDIR="$(pwd)/build"
-readonly OT_SRCDIR="$(pwd)"
+readonly OT_BUILDDIR="${PWD}/build"
+readonly OT_SRCDIR="${PWD}"

 readonly COLOR_PASS='\033[0;32m'
 readonly COLOR_FAIL='\033[0;31m'
@@ -138,6 +138,7 @@ do_build()

 do_clean()
 {
+    ./script/gcda-tool clean
    rm -rfv "${OT_BUILDDIR}" || sudo rm -rfv "${OT_BUILDDIR}"
 }

@@ -169,6 +170,7 @@ do_unit()
 do_cert()
 {
    export top_builddir="${OT_BUILDDIR}/openthread-simulation-${THREAD_VERSION}"
+    export top_srcdir="${OT_SRCDIR}"

    case "${OT_NODE_TYPE}" in
        rcp | rcp-cli | cli)
@@ -382,23 +384,6 @@ do_package()
    ls "${builddir}"/openthread-daemon-*.deb
 }

-do_tar()
-{
-    local target_name="openthread-$1-$2"
-    local build_dir="${OT_BUILDDIR}"
-    tar -cf "${target_name}.tar" -C "${build_dir}" "${target_name}"
-    mv "${target_name}.tar" "${build_dir}/"
-    rm -rf "${build_dir:?}/${target_name}"
-}
-
-do_untar()
-{
-    local target_name="openthread-$1-$2"
-    local build_dir="${OT_BUILDDIR}"
-    tar -xf "${build_dir}/${target_name}.tar" -C "${build_dir}"
-    rm "${build_dir:?}/${target_name}.tar"
-}
-
 do_prepare_coredump_upload()
 {
    echo "$PWD/ot-core-dump/corefile-%e-%p-%t" | sudo tee /proc/sys/kernel/core_pattern
@@ -426,9 +411,13 @@ do_generate_coverage()
        chmod +x "$llvm_gcov"
        lcov --gcov-tool "$llvm_gcov" --directory . --capture --output-file tmp/coverage.info
    else
+        ./script/gcda-tool collect
+        ./script/gcda-tool install
+
        lcov --directory . --capture --output-file tmp/coverage.info
    fi
    lcov --list tmp/coverage.info
+    lcov --extract tmp/coverage.info "$PWD/src/core/common/message.cpp" | c++filt
 }

 do_upload_codecov()
@@ -557,14 +546,6 @@ main()
                shift
                do_expect "$@"
                ;;
-            tar)
-                shift
-                do_tar "$@"
-                ;;
-            untar)
-                shift
-                do_untar "$@"
-                ;;
            prepare_coredump_upload)
                do_prepare_coredump_upload
                ;;
@@ -58,6 +58,7 @@ proc wait_for {command success {failure {[\r\n]FAILURE_NOT_EXPECTED[\r\n]}}} {
 proc spawn_node {id {type ""} {radio_url ""}} {
    global spawn_id
    global spawn_ids
+    global argv0

    if {${type} == ""} {
        set type $::env(OT_NODE_TYPE)
@@ -69,9 +70,15 @@ proc spawn_node {id {type ""} {radio_url ""}} {

    send_user "\n# ${id} ${type}\n"

+    if {[info exists ::env(CC)] && $::env(CC) == "clang"} {
+        set gcov_prefix ""
+    } else {
+        set gcov_prefix "ot-run/$argv0/ot-gcda.$id"
+    }
+
    switch -regexp ${type} {
        {rcp|rcp-cli} {
-            spawn $::env(OT_POSIX_APPS)/ot-cli $radio_url
+            spawn /usr/bin/env GCOV_PREFIX=$gcov_prefix $::env(OT_POSIX_APPS)/ot-cli $radio_url
            send "factoryreset\n"
            wait_for "state" "disabled"
            expect "Done"
@@ -79,7 +86,7 @@ proc spawn_node {id {type ""} {radio_url ""}} {
            expect "Done"
        }
        cli {
-            spawn $::env(OT_SIMULATION_APPS)/cli/ot-cli-ftd $id
+            spawn /usr/bin/env GCOV_PREFIX=$gcov_prefix $::env(OT_SIMULATION_APPS)/cli/ot-cli-ftd $id
            send "factoryreset\n"
            wait_for "state" "disabled"
            expect "Done"
@@ -87,7 +94,7 @@ proc spawn_node {id {type ""} {radio_url ""}} {
            expect "Done"
        }
        mtd {
-            spawn $::env(OT_SIMULATION_APPS)/cli/ot-cli-mtd $id
+            spawn /usr/bin/env GCOV_PREFIX=$gcov_prefix $::env(OT_SIMULATION_APPS)/cli/ot-cli-mtd $id
            send "factoryreset\n"
            wait_for "state" "disabled"
            expect "Done"
@@ -302,6 +302,7 @@ check_SCRIPTS                                                      = \
 TESTS_ENVIRONMENT                                                  = \
    export                                                           \
    top_builddir='$(top_builddir)'                                   \
+    top_srcdir='$(top_srcdir)'                                       \
    VERBOSE=1;                                                       \
    $(NULL)

@@ -229,6 +229,11 @@ class OtCli:
        self.env_version = os.getenv('THREAD_VERSION', '1.1')
        self.is_bbr = is_bbr
        self._initialized = False
+        if os.getenv('COVERAGE', 0) and os.getenv('CC', 'gcc') == 'gcc':
+            self._cmd_prefix = '/usr/bin/env GCOV_PREFIX=%s/ot-run/%s/ot-gcda.%d ' % (os.getenv(
+                'top_srcdir', '.'), sys.argv[0], nodeid)
+        else:
+            self._cmd_prefix = ''

        if version is not None:
            self.version = version
@@ -298,7 +303,7 @@ class OtCli:

        print("%s" % cmd)

-        self.pexpect = pexpect.popen_spawn.PopenSpawn(cmd, timeout=10)
+        self.pexpect = pexpect.popen_spawn.PopenSpawn(self._cmd_prefix + cmd, timeout=10)

        # Add delay to ensure that the process is ready to receive commands.
        timeout = 0.4
@@ -379,7 +384,7 @@ class OtCli:
        cmd += ' %d' % nodeid
        print("%s" % cmd)

-        self.pexpect = pexpect.spawn(cmd, timeout=10)
+        self.pexpect = pexpect.spawn(self._cmd_prefix + cmd, timeout=10)

        # Add delay to ensure that the process is ready to receive commands.
        time.sleep(0.2)