ci: Adds target test coverage in the conformance app

.build-test-rules.yml

@@ -103,6 +103,13 @@ test/apps/mqtt_conformance:
     enable:
         - if: IDF_TARGET in ["esp32"]
           reason: Integration test for conformance scenarios
+    disable:
+        # On-target coverage (esp_gcov) is captured once, on the pinned latest IDF line.
+        # The default config is instrumented, so this is also the single functional run.
+        # PINNED: bump the minor when the idf-latest image advances (e.g. 6.2 -> 6.3).
+        - if: IDF_VERSION_MAJOR != 6 or IDF_VERSION_MINOR != 2
+          temporary: false
+          reason: Conformance and coverage run once on the pinned latest IDF (6.2)
     disable_test:
         - if: IDF_TARGET != "esp32"
           temporary: false

.gitignore

@@ -40,13 +40,7 @@ dependencies.lock
 managed_components/
 
 # ESP-IDF build directories (but not source directories like build_test)
-**/build/
-**/build_esp32*/
-**/build_esp32c*/
-**/build_esp32s*/
-**/build_esp32h*/
-**/build_esp32p*/
-**/build_linux*/
+**/build*
 
 # Examples and test app builds
 examples/**/build/
@@ -82,6 +76,8 @@ test/host/managed_components/
 # idf-ci generated files
 app_info_*.txt
 size_info_*.txt
+non_test_related_apps.txt
+test_related_apps.txt
 test_child_pipeline.yml
 compile_commands.json
 *.log
@@ -94,9 +90,11 @@ __pycache__/
 .cache/
 
 # Coverage and test results
+**/test_logs/
 **/coverage.xml
 **/coverage.html
 **/junit.xml
+**/XUNIT_RESULT_*.xml
 **/pytest_*.xml
 **/test_results_*.xml
 

.idf_ci.toml

@@ -1,4 +1,9 @@
 # ESP-MQTT idf-ci Configuration
+#
+# Requires idf-ci >= 1.2.2. That release forwards nested pipeline images and
+# job-name suffixes, so this file does not copy gitlab.build_pipeline.yaml_jinja.
+# idf-dynamic-pipeline@main sets both build and test job_name_suffix from job-suffix.
+
 [gitlab.artifacts.native]
 build_job_filepatterns = [
     "**/build*/partition_table/*.bin",
@@ -6,65 +11,29 @@ build_job_filepatterns = [
     "**/build*/config/sdkconfig.json",
     "**/build*/*.bin",
     "**/build*/flasher_args.json",
+    "**/build*/**/*.gcno",
     "**/build.log",
 ]
 
-test_job_filepatterns = ["**/test_logs", "**/XUNIT_RESULT_*.xml"]
+test_job_filepatterns = [
+    "**/test_logs",
+    "**/XUNIT_RESULT_*.xml",
+    "**/build*/**/*.gcno",
+    "**/build*/**/*.gcda",
+    "gcov_shards/",
+]
 
 [gitlab.build_pipeline]
-# Forward the build job suffix to test jobs so each IDF version has unique names.
-yaml_jinja = """
-{{ settings.gitlab.build_pipeline.pre_yaml_jinja }}
-
-workflow:
-  name: "{{ settings.gitlab.build_pipeline.workflow_name }}"
-  rules:
-    - when: "always"
-
-{% if settings.gitlab.build_pipeline.job_template_jinja %}
-{{ job_template }}
-{%- endif %}
-
-{{ jobs }}
-
-{%- if test_related_apps_count > 0 %}
-generate_test_child_pipeline{{ settings.gitlab.build_pipeline.job_name_suffix }}:
-  extends: "{{ settings.gitlab.build_pipeline.job_template_name }}"
-  needs:
-    - "build_test_related_apps{{ settings.gitlab.build_pipeline.job_name_suffix }}"
-  artifacts:
-    paths:
-    {%- for path in settings.gitlab.artifacts.native.build_job_filepatterns %}
-      - "{{ path }}"
-    {%- endfor %}
-      - "{{ settings.gitlab.test_pipeline.yaml_filename }}"
-  script:
-    - idf-ci
-      --config 'gitlab.build_pipeline.job_name_suffix="{{ settings.gitlab.build_pipeline.job_name_suffix }}"'
-      --config 'gitlab.test_pipeline.job_name_suffix="{{ settings.gitlab.build_pipeline.job_name_suffix }}"'
-      --config 'gitlab.test_pipeline.job_image="{{ settings.gitlab.test_pipeline.job_image }}"'
-      gitlab test-child-pipeline
-
-test-child-pipeline{{ settings.gitlab.build_pipeline.job_name_suffix }}:
-  stage: ".post"
-  needs:
-    - "generate_test_child_pipeline{{ settings.gitlab.build_pipeline.job_name_suffix }}"
-  variables:
-    PARENT_PIPELINE_ID: "$CI_PIPELINE_ID"
-  trigger:
-    include:
-      - artifact: "{{ settings.gitlab.test_pipeline.yaml_filename }}"
-        job: "generate_test_child_pipeline{{ settings.gitlab.build_pipeline.job_name_suffix }}"
-    strategy: "depend"
-{%- endif %}
-"""
-
 job_template_jinja = """
 {{ settings.gitlab.build_pipeline.job_template_name }}:
   stage: "{{ settings.gitlab.build_pipeline.job_stage }}"
   tags: {{ settings.gitlab.build_pipeline.job_tags }}
   image: "{{ settings.gitlab.build_pipeline.job_image }}"
   timeout: "1h"
+{% if settings.gitlab.build_pipeline.job_name_suffix != ":idf-latest" %}
+  variables:
+    MQTT_CONFORMANCE_ENABLED: "0"
+{% endif %}
   artifacts:
     paths:
     {%- for path in settings.gitlab.artifacts.native.build_job_filepatterns %}
@@ -73,7 +42,7 @@ job_template_jinja = """
     expire_in: "1 week"
     when: "always"
   before_script:
-    - pip install -U 'idf-ci>=1,<2'
+    - pip install -U 'idf-ci>=1.2.2,<2'
   script: |
     set -euo pipefail
     ORIG_DIR="${CI_PROJECT_DIR:-$(pwd)}"
@@ -99,7 +68,109 @@ job_template_jinja = """
 runs_per_job = 15
 
 [gitlab.test_pipeline]
+job_before_script_extra = [
+  """'OPENOCD_BIN="$(find "${IDF_TOOLS_PATH:-${HOME}/.espressif}/tools/openocd-esp32" -type f -path "*/bin/openocd" -print -quit 2>/dev/null || true)"; if [ -n "${OPENOCD_BIN}" ] && [ -x "${OPENOCD_BIN}" ]; then export PATH="$(dirname "${OPENOCD_BIN}"):${PATH}"; export OPENOCD_SCRIPTS="$(dirname "$(dirname "${OPENOCD_BIN}")")/share/openocd/scripts"; test -d "${OPENOCD_SCRIPTS}"; openocd --version; fi'""",
+]
 pre_yaml_jinja = """
 variables:
   GIT_SUBMODULE_STRATEGY: recursive
 """
+
+# jobs_jinja is Jinja-rendered; pre_yaml_jinja is inserted as plain YAML.
+jobs_jinja = """
+{% for job in jobs %}
+{{ job['name'] }}{{ settings.gitlab.test_pipeline.job_name_suffix }}:
+  extends:
+    - "{{ settings.gitlab.test_pipeline.job_template_name }}"
+    {%- for extra_extend in job.get('extra_extends', []) %}
+    - "{{ extra_extend }}"
+    {%- endfor %}
+  tags: {{ job['tags'] }}
+{%- if job['parallel_count'] > 1 %}
+  parallel: {{ job['parallel_count'] }}
+{%- endif %}
+  variables:
+    nodes: {{ job['nodes'] }}
+  after_script:
+    - |
+      {%- raw %}
+      shard="gcov_shards/${CI_NODE_INDEX:-${CI_JOB_ID}}"
+      mkdir -p "${shard}"
+      if [ -d test/apps/mqtt_conformance ]; then
+        find test/apps/mqtt_conformance -name '*.gcda' -print0 2>/dev/null |
+        while IFS= read -r -d '' f; do
+          rel="${f#./}"
+          dest="${shard}/${rel}"
+          mkdir -p "$(dirname "${dest}")"
+          cp "${f}" "${dest}"
+        done
+      fi
+      {%- endraw %}
+{% endfor %}
+
+{% if settings.gitlab.test_pipeline.job_name_suffix == ":idf-latest" %}
+mqtt_conformance_coverage_report{{ settings.gitlab.test_pipeline.job_name_suffix }}:
+  stage: "{{ settings.gitlab.test_pipeline.job_stage }}"
+  image: "{{ settings.gitlab.build_pipeline.job_image }}"
+  tags: [build]
+  needs:
+    - job: "esp32 - eth_ip101{{ settings.gitlab.test_pipeline.job_name_suffix }}"
+      artifacts: true
+  before_script:
+    - pip install -U gcovr
+  coverage: '/^lines:\\s+(\\d+\\.\\d+)%/'
+  artifacts:
+    paths:
+      - "test/apps/mqtt_conformance/build_esp32_default/coverage_report"
+    expire_in: "1 week"
+    when: "always"
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: test/apps/mqtt_conformance/build_esp32_default/coverage_report/coverage.xml
+  script: |
+    {%- raw %}
+    set -euo pipefail
+    report_dir="test/apps/mqtt_conformance/build_esp32_default/coverage_report"
+    build_dir="test/apps/mqtt_conformance/build_esp32_default"
+    mkdir -p "${report_dir}/html"
+    gcovr_filters=(
+      --filter="${CI_PROJECT_DIR}/mqtt_client.c"
+      --filter="${CI_PROJECT_DIR}/mqtt5_client.c"
+      --filter="${CI_PROJECT_DIR}/lib/mqtt_msg.c"
+      --filter="${CI_PROJECT_DIR}/lib/mqtt5_msg.c"
+      --filter="${CI_PROJECT_DIR}/lib/platform.c"
+    )
+    shopt -s nullglob
+    shards=(gcov_shards/*)
+    gcovr_search=(--object-directory="${build_dir}" --exclude-directories=gcov_shards)
+    if [ "${#shards[@]}" -eq 0 ]; then
+      gcovr -r . --gcov-executable xtensa-esp32-elf-gcov "${gcovr_search[@]}" "${gcovr_filters[@]}" --xml="${report_dir}/coverage.xml" --html-details="${report_dir}/html/index.html" --print-summary --fail-under-line 50
+    else
+      traces=()
+      i=0
+      for s in "${shards[@]}"; do
+        gcda_count="$(find "${s}" -name '*.gcda' | wc -l)"
+        if [ "${gcda_count}" -eq 0 ]; then
+          continue
+        fi
+        find "${build_dir}" -name '*.gcda' -delete
+        while IFS= read -r -d '' f; do
+          rel="${f#${s}/}"
+          mkdir -p "$(dirname "${rel}")"
+          cp "${f}" "${rel}"
+        done < <(find "${s}" -name '*.gcda' -print0)
+        json="${report_dir}/shard_${i}.json"
+        gcovr -r . --gcov-executable xtensa-esp32-elf-gcov "${gcovr_search[@]}" "${gcovr_filters[@]}" --json="${json}"
+        traces+=(--add-tracefile "${json}")
+        i=$((i + 1))
+      done
+      if [ "${#traces[@]}" -eq 0 ]; then
+        echo "No per-shard .gcda found under gcov_shards/" >&2
+        exit 1
+      fi
+      gcovr "${traces[@]}" --xml="${report_dir}/coverage.xml" --html-details="${report_dir}/html/index.html" --print-summary --fail-under-line 50
+    fi
+    {%- endraw %}
+{% endif %}
+"""

conftest.py

@@ -1,10 +1,160 @@
-# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2025-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
-import socket
-import pytest
 import logging
+import os
+import shlex
+import shutil
+import socket
+import subprocess
+import time
 import typing as t
+import warnings
+from pathlib import Path
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", DeprecationWarning)
+    from telnetlib import Telnet
+
+import pytest
+from pytest_embedded_idf import IdfDut
+
+
+_MQTT_CONFORMANCE_DIR = Path(__file__).parent / "test" / "apps" / "mqtt_conformance"
+_coverage_prepared_builds: set[Path] = set()
+
+collect_ignore = ["test/apps/mqtt_conformance"] if os.getenv("MQTT_CONFORMANCE_ENABLED") == "0" else []
+
+
+def _is_mqtt_conformance_test(request: pytest.FixtureRequest) -> bool:
+    try:
+        request.node.path.relative_to(_MQTT_CONFORMANCE_DIR)
+    except ValueError:
+        return False
+    return True
+
+
+def _gcov_enabled(dut: IdfDut) -> bool:
+    return dut.app.sdkconfig.get("ESP_GCOV_ENABLE") is True
+
+
+def _flatten_gcov_data(binary_path: Path) -> None:
+    for gcda in binary_path.rglob("*.gcda"):
+        if gcda.parent == binary_path:
+            continue
+        flattened = binary_path / "#".join(gcda.relative_to(binary_path).parts)
+        shutil.copy2(gcda, flattened)
+
+
+def _restore_gcov_data(binary_path: Path) -> None:
+    for gcda in binary_path.glob("*.gcda"):
+        destination = binary_path.joinpath(*gcda.name.split("#"))
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        gcda.replace(destination)
+
+
+class OpenOCD:
+    def __init__(self, dut: IdfDut) -> None:
+        self.binary_path = dut.app.binary_path
+        self.log_file = os.path.join(dut.logdir, "ocd.txt")
+        self.log_stream: t.Optional[t.TextIO] = None
+        self.proc: t.Optional[subprocess.Popen] = None
+        self.telnet: t.Optional[Telnet] = None
+
+    def run(self) -> None:
+        scripts = os.getenv("OPENOCD_SCRIPTS")
+        if not scripts:
+            raise RuntimeError("OPENOCD_SCRIPTS is not set; export the ESP-IDF environment")
+        args = shlex.split(
+            os.getenv(
+                "MQTT_CONFORMANCE_OPENOCD_ARGS",
+                "-f board/esp32-ethernet-kit-3.3v.cfg",
+            )
+        )
+        self.log_stream = open(self.log_file, "w", encoding="utf-8")
+        self.proc = subprocess.Popen(
+            ["openocd", "-s", scripts, *args],
+            stdout=self.log_stream,
+            stderr=subprocess.STDOUT,
+            cwd=self.binary_path,
+        )
+        for _ in range(20):
+            if self.proc.poll() is not None:
+                raise RuntimeError(f"OpenOCD exited; see {self.log_file}")
+            try:
+                self.telnet = Telnet("127.0.0.1", 4444, 1)
+                greeting = self.telnet.read_until(b">", timeout=5)
+                if greeting.endswith(b">"):
+                    return
+            except (ConnectionRefusedError, TimeoutError):
+                pass
+            time.sleep(0.25)
+        raise RuntimeError(f"OpenOCD did not start; see {self.log_file}")
+
+    def write(self, command: str, timeout: int = 120) -> str:
+        if self.telnet is None:
+            raise RuntimeError("OpenOCD console is not connected")
+        self.telnet.write((command + "\n").encode())
+        response = self.telnet.read_until(b">", timeout=timeout)
+        if not response.endswith(b">"):
+            raise TimeoutError(f"OpenOCD command timed out: {command}")
+        return response.decode(errors="replace")
+
+    def gcov_dump(self) -> str:
+        output = self.write("esp gcov dump")
+        if "Targets connected." not in output or "Targets disconnected." not in output:
+            raise RuntimeError(f"Incomplete gcov dump: {output}")
+        return output
+
+    def kill(self) -> None:
+        if self.telnet is not None:
+            self.telnet.close()
+            self.telnet = None
+        if self.proc is not None and self.proc.poll() is None:
+            self.proc.terminate()
+            try:
+                self.proc.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                self.proc.kill()
+                self.proc.wait(timeout=5)
+        if self.log_stream is not None:
+            self.log_stream.close()
+
+
+@pytest.fixture(autouse=True)
+def dump_mqtt_conformance_gcov(request: pytest.FixtureRequest):
+    if not _is_mqtt_conformance_test(request):
+        yield
+        return
+
+    dut = request.getfixturevalue("dut")
+    if not _gcov_enabled(dut):
+        yield
+        return
+
+    binary_path = Path(dut.app.binary_path)
+    if binary_path not in _coverage_prepared_builds:
+        logging.info("Removing stale gcov data from %s", binary_path)
+        for gcda in binary_path.rglob("*.gcda"):
+            gcda.unlink()
+        _coverage_prepared_builds.add(binary_path)
+
+    yield
+
+    relocatable_gcov = os.getenv("CI") is not None
+    if relocatable_gcov:
+        _flatten_gcov_data(binary_path)
+    ocd = OpenOCD(dut)
+    try:
+        ocd.run()
+        dut.write("gcov")
+        dut.expect("GCOV dump waiting for host", timeout=10)
+        logging.info("gcov dump:\n%s", ocd.gcov_dump())
+        if relocatable_gcov:
+            _restore_gcov_data(binary_path)
+        dut.expect("GCOV dump complete", timeout=10)
+    finally:
+        ocd.kill()
 
 
 def get_host_ip4_by_dest_ip(dest_ip: str = "") -> str:

test/apps/mqtt_conformance/CMakeLists.txt

@@ -6,3 +6,33 @@ set(EXTRA_COMPONENT_DIRS "../common")
 idf_build_set_property(MINIMAL_BUILD ON)
 
 project(mqtt_conformance)
+
+if(CONFIG_ESP_GCOV_ENABLE)
+    get_filename_component(mqtt_root "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE)
+    idf_component_get_property(mqtt mqtt COMPONENT_LIB)
+    target_compile_options(${mqtt} PRIVATE --coverage)
+    if(DEFINED ENV{CI})
+        get_filename_component(ci_artifact_root "${mqtt_root}/.." ABSOLUTE)
+        target_compile_options(
+            ${mqtt}
+            PRIVATE
+            "-fprofile-dir=."
+            "-fprofile-prefix-path=${CMAKE_BINARY_DIR}"
+            "-fprofile-prefix-map=${mqtt_root}=${ci_artifact_root}")
+    endif()
+    target_link_options(${mqtt} PUBLIC --coverage)
+endif()
+
+if(CONFIG_ESP_GCOV_ENABLE AND COMMAND idf_create_coverage_report)
+    idf_create_coverage_report(
+        ${CMAKE_CURRENT_BINARY_DIR}/coverage_report
+        SOURCE_DIR ${mqtt_root}
+        GCOV_OPTIONS
+            "--filter=${mqtt_root}/mqtt_client.c"
+            "--filter=${mqtt_root}/mqtt5_client.c"
+            "--filter=${mqtt_root}/lib/mqtt_msg.c"
+            "--filter=${mqtt_root}/lib/mqtt5_msg.c"
+            "--filter=${mqtt_root}/lib/platform.c"
+            "--xml=${CMAKE_CURRENT_BINARY_DIR}/coverage_report/coverage.xml")
+    idf_clean_coverage_report(${CMAKE_CURRENT_BINARY_DIR}/coverage_report)
+endif()

test/apps/mqtt_conformance/README.md

@@ -131,6 +131,26 @@ between tests and there's no port to configure/coordinate.
 - `MQTT_CONFORMANCE_RETRY_BACKOFF_SEC` — backoff between connect retries, in seconds
   (default: 2).
 
+## On-target coverage
+
+The default sdkconfig enables gcov collection automatically on ESP-IDF 6.0 or newer.
+Use an ESP32-Ethernet-Kit with JTAG enabled (on-board FT2232H channel A):
+
+```bash
+cd test/apps/mqtt_conformance
+idf.py -B build_esp32_default \
+    -DIDF_TARGET=esp32 \
+    -DSDKCONFIG=build_esp32_default/sdkconfig \
+    -DSDKCONFIG_DEFAULTS=sdkconfig.ci.default \
+    build flash
+pytest --target esp32 --build-dir build_esp32_default --timeout 180
+idf.py -B build_esp32_default gcovr-report
+```
+
+After each test, the application calls `esp_gcov_dump()` and OpenOCD retrieves the data
+with `esp gcov dump`. Repeated dumps merge coverage even when the application is reflashed
+between tests.
+
 ### Timeouts
 
 Tests use **operation-based timeouts** (not a flat 60 s wait): the budget is computed

test/apps/mqtt_conformance/main/Kconfig.projbuild

@@ -0,0 +1,11 @@
+menu "MQTT conformance configuration"
+
+    config MQTT_CONFORMANCE_GCOV
+        bool "Enable on-target GCOV collection"
+        imply ESP_GCOV_ENABLE
+        default n
+        help
+            Add the esp_gcov component for GCOV data transfer from the target
+            through OpenOCD. This option requires ESP-IDF 6.0 or newer.
+
+endmenu

test/apps/mqtt_conformance/main/idf_component.yml

@@ -5,3 +5,8 @@ dependencies:
     version: '*'
     override_path: ../../../..
   mittelab/nlohmann-json: ^3.11.3
+  espressif/esp_gcov:
+    version: ^1.0.5
+    rules:
+      - if: "idf_version >=6.0"
+      - if: "$CONFIG{MQTT_CONFORMANCE_GCOV} == True"

test/apps/mqtt_conformance/main/mqtt_conformance_console.cpp

@@ -21,6 +21,12 @@
 #include "esp_log.h"
 #include "mqtt_conformance.hpp"
 
+#if CONFIG_ESP_GCOV_ENABLE
+extern "C" {
+#include "esp_gcov.h"
+}
+#endif
+
 namespace
 {
 
@@ -297,6 +303,18 @@ int do_publish(int argc, char **argv)
     return 0;
 }
 
+#if CONFIG_ESP_GCOV_ENABLE
+int do_gcov_dump(int argc, char **argv)
+{
+    (void)argc;
+    (void)argv;
+    ESP_LOGI(TAG, "GCOV dump waiting for host");
+    esp_gcov_dump();
+    ESP_LOGI(TAG, "GCOV dump complete");
+    return 0;
+}
+#endif
+
 void register_commands()
 {
     init_args.b64 = arg_str1(nullptr, nullptr, "<base64_json>", "JSON config blob (base64-encoded)");
@@ -414,6 +432,17 @@ void register_commands()
         .func_w_context = nullptr,
         .context = nullptr,
     };
+#if CONFIG_ESP_GCOV_ENABLE
+    const esp_console_cmd_t gcov_dump = {
+        .command = "gcov",
+        .help = "Dump coverage data through OpenOCD",
+        .hint = nullptr,
+        .func = &do_gcov_dump,
+        .argtable = nullptr,
+        .func_w_context = nullptr,
+        .context = nullptr,
+    };
+#endif
     ESP_ERROR_CHECK(esp_console_cmd_register(&init));
     ESP_ERROR_CHECK(esp_console_cmd_register(&config_cmd));
     ESP_ERROR_CHECK(esp_console_cmd_register(&start));
@@ -425,6 +454,9 @@ void register_commands()
     ESP_ERROR_CHECK(esp_console_cmd_register(&subscribe));
     ESP_ERROR_CHECK(esp_console_cmd_register(&unsubscribe));
     ESP_ERROR_CHECK(esp_console_cmd_register(&publish));
+#if CONFIG_ESP_GCOV_ENABLE
+    ESP_ERROR_CHECK(esp_console_cmd_register(&gcov_dump));
+#endif
 }
 
 } // namespace

test/apps/mqtt_conformance/sdkconfig.ci.default

@@ -11,4 +11,9 @@ CONFIG_EXAMPLE_ETH_MDIO_GPIO=18
 CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5
 CONFIG_EXAMPLE_ETH_PHY_ADDR=1
 CONFIG_EXAMPLE_CONNECT_IPV6=y
-CONFIG_LOG_MAXIMUM_LEVEL_DEBUG=y
+CONFIG_ESP_TRACE_ENABLE=y
+CONFIG_ESP_TRACE_LIB_NONE=y
+CONFIG_ESP_TRACE_TRANSPORT_APPTRACE=y
+CONFIG_APPTRACE_DEST_JTAG=y
+CONFIG_APPTRACE_LOCK_ENABLE=y
+CONFIG_MQTT_CONFORMANCE_GCOV=y