From 37a2e555c57dc1a82ea474031a2b3790556d0eb4 Mon Sep 17 00:00:00 2001 From: Euripedes Rocha Filho Date: Mon, 9 Feb 2026 15:15:23 +0100 Subject: [PATCH] feat: add mqtt conformance test app Adds a conformance test app based on paho test suite. This introduce the basis infrastructure and initial tests. --- .build-test-rules.yml | 10 + .gitignore | 1 + .gitlab/ci/ignore_build_warnings.txt | 7 + .gitmodules | 3 + .idf_ci.toml | 6 + test/apps/mqtt_conformance/CMakeLists.txt | 8 + test/apps/mqtt_conformance/README.md | 38 +++ .../apps/mqtt_conformance/main/CMakeLists.txt | 3 + .../mqtt_conformance/main/idf_component.yml | 6 + .../mqtt_conformance/main/mqtt_conformance.c | 125 ++++++++ .../mqtt_conformance/main/mqtt_conformance.h | 42 +++ .../main/mqtt_conformance_console.c | 297 ++++++++++++++++++ .../pytest_mqtt_conformance.py | 194 ++++++++++++ .../mqtt_conformance/sdkconfig.ci.default | 13 + test/tools/paho.mqtt.testing | 1 + 15 files changed, 754 insertions(+) create mode 100644 .gitmodules create mode 100644 test/apps/mqtt_conformance/CMakeLists.txt create mode 100644 test/apps/mqtt_conformance/README.md create mode 100644 test/apps/mqtt_conformance/main/CMakeLists.txt create mode 100644 test/apps/mqtt_conformance/main/idf_component.yml create mode 100644 test/apps/mqtt_conformance/main/mqtt_conformance.c create mode 100644 test/apps/mqtt_conformance/main/mqtt_conformance.h create mode 100644 test/apps/mqtt_conformance/main/mqtt_conformance_console.c create mode 100644 test/apps/mqtt_conformance/pytest_mqtt_conformance.py create mode 100644 test/apps/mqtt_conformance/sdkconfig.ci.default create mode 160000 test/tools/paho.mqtt.testing diff --git a/.build-test-rules.yml b/.build-test-rules.yml index 2f8060b..d7bfbfd 100644 --- a/.build-test-rules.yml +++ b/.build-test-rules.yml @@ -98,6 +98,16 @@ test/apps/publish_connect_test: temporary: false reason: Only esp32 target has ethernet runners for integration tests +# MQTT conformance integration test +test/apps/mqtt_conformance: + enable: + - if: IDF_TARGET in ["esp32"] + reason: Integration test for conformance scenarios + disable_test: + - if: IDF_TARGET != "esp32" + temporary: false + reason: Only esp32 target has ethernet runners for integration tests + # Host tests (unit tests with mocks) test/host: enable: diff --git a/.gitignore b/.gitignore index 73a4e7a..2435538 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ test/host/managed_components/ # idf-ci generated files app_info_*.txt size_info_*.txt +test_child_pipeline.yml compile_commands.json *.log diff --git a/.gitlab/ci/ignore_build_warnings.txt b/.gitlab/ci/ignore_build_warnings.txt index 61f6c5e..b6141c1 100644 --- a/.gitlab/ci/ignore_build_warnings.txt +++ b/.gitlab/ci/ignore_build_warnings.txt @@ -4,5 +4,12 @@ Warning: Deprecated: Option '--flash_freq' is deprecated. Use '--flash-freq' ins Warning: Deprecated: Command 'sign_data' is deprecated. Use 'sign-data' instead. Warning: Deprecated: Command 'extract_public_key' is deprecated. Use 'extract-public-key' instead. CryptographyDeprecationWarning +warning: unknown kconfig symbol 'EXAMPLE_USE_INTERNAL_ETHERNET' assigned to 'y' warning: unknown kconfig symbol 'EXAMPLE_ETH_PHY_IP101' assigned to 'y' +warning: unknown kconfig symbol 'EXAMPLE_ETH_MDC_GPIO' assigned +warning: unknown kconfig symbol 'EXAMPLE_ETH_MDIO_GPIO' assigned +warning: unknown kconfig symbol 'EXAMPLE_ETH_PHY_RST_GPIO' assigned +warning: unknown kconfig symbol 'EXAMPLE_ETH_PHY_ADDR' assigned WARNING: The following Kconfig variables were used in "if" clauses +Kconfig variables were used in .* "if" clauses +Missing kconfig option diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..104deb0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "test/tools/paho.mqtt.testing"] + path = test/tools/paho.mqtt.testing + url = https://github.com/eclipse-paho/paho.mqtt.testing.git diff --git a/.idf_ci.toml b/.idf_ci.toml index 2068a46..783a24a 100644 --- a/.idf_ci.toml +++ b/.idf_ci.toml @@ -50,3 +50,9 @@ job_template_jinja = """ git worktree remove -f mqtt || rm -rf mqtt """ runs_per_job = 15 + +[gitlab.test_pipeline] +pre_yaml_jinja = """ +variables: + GIT_SUBMODULE_STRATEGY: recursive +""" diff --git a/test/apps/mqtt_conformance/CMakeLists.txt b/test/apps/mqtt_conformance/CMakeLists.txt new file mode 100644 index 0000000..47e5c92 --- /dev/null +++ b/test/apps/mqtt_conformance/CMakeLists.txt @@ -0,0 +1,8 @@ +# The following boilerplate must appear in this order. +cmake_minimum_required(VERSION 3.16) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +set(EXTRA_COMPONENT_DIRS "../common") +idf_build_set_property(MINIMAL_BUILD ON) + +project(mqtt_conformance) diff --git a/test/apps/mqtt_conformance/README.md b/test/apps/mqtt_conformance/README.md new file mode 100644 index 0000000..62b059c --- /dev/null +++ b/test/apps/mqtt_conformance/README.md @@ -0,0 +1,38 @@ +# MQTT conformance app (HIL) + +This app exposes a console API for pytest-embedded HIL tests that target MQTT conformance behavior. + +## Console commands + +- `init`: Create and configure MQTT client +- `set_uri `: Override broker URI before `start` +- `start`: Start MQTT client +- `stop`: Stop MQTT client +- `destroy`: Destroy MQTT client +- `subscribe `: Subscribe to topic +- `publish `: Publish payload + +## Conformance mapping + +Each pytest case should document the MQTT specification section it validates where practical. + +The paho reference suite is integrated as git submodule at: + +`test/tools/paho.mqtt.testing` + +## Running tests locally + +From the repository root (or the mqtt worktree root if using worktrees): + +1. Ensure the environment is active (e.g. `direnv allow` at repo root so IDF and pytest-embedded are available). + +2. Initialize the paho.mqtt.testing submodule: + ```bash + git submodule update --init --recursive test/tools/paho.mqtt.testing + ``` + +3. Run the conformance tests (connect a board with Ethernet, or use the same target/port as in CI): + ```bash + pytest test/apps/mqtt_conformance/ -v + ``` + To run a single test or filter by keyword, add e.g. `-k test_mqtt_v311` or the test path. diff --git a/test/apps/mqtt_conformance/main/CMakeLists.txt b/test/apps/mqtt_conformance/main/CMakeLists.txt new file mode 100644 index 0000000..48be37b --- /dev/null +++ b/test/apps/mqtt_conformance/main/CMakeLists.txt @@ -0,0 +1,3 @@ +idf_component_register(SRCS "mqtt_conformance.c" "mqtt_conformance_console.c" + INCLUDE_DIRS "." + REQUIRES mqtt nvs_flash console esp_netif) diff --git a/test/apps/mqtt_conformance/main/idf_component.yml b/test/apps/mqtt_conformance/main/idf_component.yml new file mode 100644 index 0000000..fe64949 --- /dev/null +++ b/test/apps/mqtt_conformance/main/idf_component.yml @@ -0,0 +1,6 @@ +dependencies: + protocol_examples_common: + path: ${IDF_PATH}/examples/common_components/protocol_examples_common + mqtt: + version: "*" + override_path: "../../../.." diff --git a/test/apps/mqtt_conformance/main/mqtt_conformance.c b/test/apps/mqtt_conformance/main/mqtt_conformance.c new file mode 100644 index 0000000..4cfd114 --- /dev/null +++ b/test/apps/mqtt_conformance/main/mqtt_conformance.c @@ -0,0 +1,125 @@ +/* + * SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ +#include +#include +#include +#include +#include "esp_event.h" +#include "esp_random.h" +#include "esp_system.h" +#include "esp_log.h" +#include "mqtt_client.h" +#if CONFIG_MQTT_PROTOCOL_5 +#include "mqtt5_client.h" +#endif +#include "mqtt_conformance.h" + +static const char *TAG = "mqtt_conformance"; + +#define CLIENT_ID_SIZE 20 + +static void mqtt_event_handler(void *handler_args, esp_event_base_t base, int32_t event_id, void *event_data) +{ + esp_mqtt_event_handle_t event = event_data; + + switch (event->event_id) { + case MQTT_EVENT_CONNECTED: + ESP_LOGI(TAG, "MQTT_EVENT_CONNECTED"); + break; + + case MQTT_EVENT_DISCONNECTED: +#if CONFIG_MQTT_PROTOCOL_5 + if (event->error_handle) { + ESP_LOGW(TAG, "DISCONNECT_REASON=%d", event->error_handle->disconnect_return_code); + } + +#endif + ESP_LOGI(TAG, "MQTT_EVENT_DISCONNECTED"); + break; + + case MQTT_EVENT_SUBSCRIBED: + ESP_LOGI(TAG, "MQTT_EVENT_SUBSCRIBED, msg_id=%d", event->msg_id); + + if (event->data_len > 0 && event->data) { + ESP_LOGI(TAG, "MQTT_EVENT_SUBSCRIBED data_len=%d return_code=0x%02x", + event->data_len, (unsigned int)(uint8_t)event->data[0]); + } + + break; + + case MQTT_EVENT_UNSUBSCRIBED: + ESP_LOGI(TAG, "MQTT_EVENT_UNSUBSCRIBED, msg_id=%d", event->msg_id); + + if (event->data_len > 0 && event->data) { + ESP_LOGI(TAG, "MQTT_EVENT_UNSUBSCRIBED data_len=%d reason_code=0x%02x", + event->data_len, (unsigned int)(uint8_t)event->data[0]); + } + + break; + + case MQTT_EVENT_PUBLISHED: + ESP_LOGI(TAG, "MQTT_EVENT_PUBLISHED, msg_id=%d", event->msg_id); + break; + + case MQTT_EVENT_DATA: + ESP_LOGI(TAG, "MQTT_EVENT_DATA topic=%.*s qos=%d len=%d offset=%d total=%d", event->topic_len, event->topic, + event->qos, event->data_len, event->current_data_offset, event->total_data_len); + ESP_LOGI(TAG, "MQTT_EVENT_DATA_PAYLOAD %.*s", event->data_len, event->data ? event->data : ""); + + if (event->current_data_offset + event->data_len == event->total_data_len) { + ESP_LOGI(TAG, "MQTT_EVENT_DATA_COMPLETE msg_id=%d total=%d", event->msg_id, event->total_data_len); + } + + break; + + case MQTT_EVENT_ERROR: + ESP_LOGE(TAG, "MQTT_EVENT_ERROR"); + + if (event->error_handle) { + ESP_LOGE(TAG, "error_type=%" PRId32 " connect_return_code=%" PRId32, + (int32_t)event->error_handle->error_type, + (int32_t)event->error_handle->connect_return_code); + } + + if (event->data_len > 0 && event->data) { + ESP_LOGE(TAG, "MQTT_EVENT_ERROR data_len=%d data=%.*s", + event->data_len, event->data_len, event->data); + } + + break; + + default: + ESP_LOGI(TAG, "Other event id:%d", event->event_id); + break; + } +} + +void conformance_register_event_handlers(command_context_t *ctx) +{ + esp_mqtt_client_register_event(ctx->mqtt_client, ESP_EVENT_ANY_ID, mqtt_event_handler, NULL); +} + +void conformance_unregister_event_handlers(command_context_t *ctx) +{ + esp_mqtt_client_unregister_event(ctx->mqtt_client, ESP_EVENT_ANY_ID, mqtt_event_handler); +} + +void conformance_set_broker_uri(command_context_t *ctx, const char *uri) +{ + esp_mqtt_client_config_t config = {0}; + config.broker.address.uri = uri; + esp_mqtt_set_config(ctx->mqtt_client, &config); +} + +void conformance_configure_client(command_context_t *ctx) +{ + static char client_id[CLIENT_ID_SIZE]; + snprintf(client_id, sizeof(client_id), "esp-%08" PRIx32, esp_random()); + esp_mqtt_client_config_t config = {0}; + config.credentials.client_id = client_id; + ESP_LOGI(TAG, "Client configured, client_id=%s (broker URI set via set_uri command)", client_id); + esp_mqtt_set_config(ctx->mqtt_client, &config); +} diff --git a/test/apps/mqtt_conformance/main/mqtt_conformance.h b/test/apps/mqtt_conformance/main/mqtt_conformance.h new file mode 100644 index 0000000..10b7d22 --- /dev/null +++ b/test/apps/mqtt_conformance/main/mqtt_conformance.h @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ +#pragma once + +#include "mqtt_client.h" + +struct arg_int; +struct arg_str; +struct arg_end; + +typedef struct { + esp_mqtt_client_handle_t mqtt_client; +} command_context_t; + +typedef struct { + struct arg_str *uri; + struct arg_end *end; +} set_uri_args_t; + +typedef struct { + struct arg_str *topic; + struct arg_int *qos; + struct arg_end *end; +} subscribe_args_t; + +typedef struct { + struct arg_str *topic; + struct arg_str *pattern; + struct arg_int *pattern_repetitions; + struct arg_int *qos; + struct arg_int *retain; + struct arg_int *enqueue; + struct arg_end *end; +} publish_args_t; + +void conformance_register_event_handlers(command_context_t *ctx); +void conformance_unregister_event_handlers(command_context_t *ctx); +void conformance_configure_client(command_context_t *ctx); +void conformance_set_broker_uri(command_context_t *ctx, const char *uri); diff --git a/test/apps/mqtt_conformance/main/mqtt_conformance_console.c b/test/apps/mqtt_conformance/main/mqtt_conformance_console.c new file mode 100644 index 0000000..8d34401 --- /dev/null +++ b/test/apps/mqtt_conformance/main/mqtt_conformance_console.c @@ -0,0 +1,297 @@ +/* + * SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Unlicense OR CC0-1.0 + */ +#include +#include +#include +#include +#include +#include "esp_system.h" +#include "mqtt_client.h" +#include "nvs_flash.h" +#include "esp_event.h" +#include "esp_netif.h" +#include "protocol_examples_common.h" +#include "esp_console.h" +#include "argtable3/argtable3.h" +#include "esp_log.h" +#include "mqtt_conformance.h" + +static const char *TAG = "mqtt_conformance"; + +static command_context_t command_context; +static set_uri_args_t set_uri_args; +static subscribe_args_t subscribe_args; +static publish_args_t publish_args; + +#define RETURN_ON_PARSE_ERROR(args) do { \ + int nerrors = arg_parse(argc, argv, (void **) &(args)); \ + if (nerrors != 0) { \ + arg_print_errors(stderr, (args).end, argv[0]); \ + return 1; \ + }} while(0) + +static int require_client(void) +{ + if (!command_context.mqtt_client) { + ESP_LOGE(TAG, "MQTT client not initialized, call init first"); + return 1; + } + + return 0; +} + +static int do_init(int argc, char **argv) +{ + if (command_context.mqtt_client) { + ESP_LOGW(TAG, "MQTT client already initialized"); + return 0; + } + + const esp_mqtt_client_config_t mqtt_cfg = { + .broker.address.uri = "mqtt://127.0.0.1:1234", + .network.disable_auto_reconnect = true, +#if CONFIG_MQTT_PROTOCOL_5 + .session.protocol_ver = MQTT_PROTOCOL_V_5, +#endif + }; + command_context.mqtt_client = esp_mqtt_client_init(&mqtt_cfg); + + if (!command_context.mqtt_client) { + ESP_LOGE(TAG, "Failed to initialize client"); + return 1; + } + + conformance_configure_client(&command_context); + conformance_register_event_handlers(&command_context); + ESP_LOGI(TAG, "Mqtt client initialized"); + return 0; +} + +static int do_set_uri(int argc, char **argv) +{ + RETURN_ON_PARSE_ERROR(set_uri_args); + + if (require_client() != 0) { + return 1; + } + + conformance_set_broker_uri(&command_context, set_uri_args.uri->sval[0]); + ESP_LOGI(TAG, "Broker URI updated to %s", set_uri_args.uri->sval[0]); + return 0; +} + +static int do_start(int argc, char **argv) +{ + if (require_client() != 0) { + return 1; + } + + if (esp_mqtt_client_start(command_context.mqtt_client) != ESP_OK) { + ESP_LOGE(TAG, "Failed to start mqtt client task"); + return 1; + } + + ESP_LOGI(TAG, "Mqtt client started"); + return 0; +} + +static int do_stop(int argc, char **argv) +{ + if (require_client() != 0) { + return 1; + } + + if (esp_mqtt_client_stop(command_context.mqtt_client) != ESP_OK) { + ESP_LOGE(TAG, "Failed to stop mqtt client task"); + return 1; + } + + ESP_LOGI(TAG, "Mqtt client stopped"); + return 0; +} + +static int do_destroy(int argc, char **argv) +{ + if (!command_context.mqtt_client) { + return 0; + } + + conformance_unregister_event_handlers(&command_context); + esp_mqtt_client_destroy(command_context.mqtt_client); + command_context.mqtt_client = NULL; + ESP_LOGI(TAG, "mqtt client for tests destroyed"); + return 0; +} + +static int do_subscribe(int argc, char **argv) +{ + RETURN_ON_PARSE_ERROR(subscribe_args); + + if (require_client() != 0) { + return 1; + } + + int msg_id = esp_mqtt_client_subscribe(command_context.mqtt_client, subscribe_args.topic->sval[0], + subscribe_args.qos->ival[0]); + + if (msg_id < 0) { + ESP_LOGE(TAG, "Subscribe failed, msg_id=%d", msg_id); + return 1; + } + + ESP_LOGI(TAG, "Subscribe requested, msg_id=%d", msg_id); + return 0; +} + +static int do_publish(int argc, char **argv) +{ + RETURN_ON_PARSE_ERROR(publish_args); + + if (require_client() != 0) { + return 1; + } + + const char *pattern = publish_args.pattern->sval[0]; + int repetitions = publish_args.pattern_repetitions->ival[0]; + size_t pattern_len = strlen(pattern); + size_t payload_len = pattern_len * (size_t)repetitions; + char *payload = NULL; + + if (repetitions < 0) { + ESP_LOGE(TAG, "Invalid pattern repetitions"); + return 1; + } + + if (payload_len > 0) { + payload = malloc(payload_len); + + if (!payload) { + ESP_LOGE(TAG, "Failed to allocate payload"); + return 1; + } + + for (int i = 0; i < repetitions; i++) { + memcpy(payload + (size_t)i * pattern_len, pattern, pattern_len); + } + } + + int msg_id; + + if (publish_args.enqueue->ival[0]) { + msg_id = esp_mqtt_client_enqueue(command_context.mqtt_client, publish_args.topic->sval[0], payload, payload_len, + publish_args.qos->ival[0], publish_args.retain->ival[0], true); + } else { + msg_id = esp_mqtt_client_publish(command_context.mqtt_client, publish_args.topic->sval[0], payload, payload_len, + publish_args.qos->ival[0], publish_args.retain->ival[0]); + } + + free(payload); + + if (msg_id < 0) { + ESP_LOGE(TAG, "Publish failed, msg_id=%d", msg_id); + return 1; + } + + ESP_LOGI(TAG, "Publish requested, msg_id=%d", msg_id); + return 0; +} + +static void register_common_commands(void) +{ + const esp_console_cmd_t init = { + .command = "init", + .help = "Initialize mqtt client", + .hint = NULL, + .func = &do_init, + }; + const esp_console_cmd_t set_uri = { + .command = "set_uri", + .help = "Set broker URI", + .hint = NULL, + .func = &do_set_uri, + .argtable = &set_uri_args, + }; + const esp_console_cmd_t start = { + .command = "start", + .help = "Start mqtt client", + .hint = NULL, + .func = &do_start, + }; + const esp_console_cmd_t stop = { + .command = "stop", + .help = "Stop mqtt client", + .hint = NULL, + .func = &do_stop, + }; + const esp_console_cmd_t destroy = { + .command = "destroy", + .help = "Destroy mqtt client", + .hint = NULL, + .func = &do_destroy, + }; + ESP_ERROR_CHECK(esp_console_cmd_register(&init)); + ESP_ERROR_CHECK(esp_console_cmd_register(&set_uri)); + ESP_ERROR_CHECK(esp_console_cmd_register(&start)); + ESP_ERROR_CHECK(esp_console_cmd_register(&stop)); + ESP_ERROR_CHECK(esp_console_cmd_register(&destroy)); +} + +static void register_pubsub_commands(void) +{ + set_uri_args.uri = arg_str1(NULL, NULL, "", "Broker URI"); + set_uri_args.end = arg_end(1); + subscribe_args.topic = arg_str1(NULL, NULL, "", "Subscribe topic"); + subscribe_args.qos = arg_int1(NULL, NULL, "", "Subscribe qos"); + subscribe_args.end = arg_end(1); + publish_args.topic = arg_str1(NULL, NULL, "", "Publish topic"); + publish_args.pattern = arg_str1(NULL, NULL, "", "Payload pattern"); + publish_args.pattern_repetitions = arg_int1(NULL, NULL, "", "Number of pattern repetitions"); + publish_args.qos = arg_int1(NULL, NULL, "", "Publish qos"); + publish_args.retain = arg_int1(NULL, NULL, "", "Publish retain flag"); + publish_args.enqueue = arg_int1(NULL, NULL, "", "0=publish,1=enqueue"); + publish_args.end = arg_end(1); + const esp_console_cmd_t subscribe = { + .command = "subscribe", + .help = "Subscribe to a topic", + .hint = NULL, + .func = &do_subscribe, + .argtable = &subscribe_args, + }; + const esp_console_cmd_t publish = { + .command = "publish", + .help = "Publish a message", + .hint = NULL, + .func = &do_publish, + .argtable = &publish_args, + }; + ESP_ERROR_CHECK(esp_console_cmd_register(&subscribe)); + ESP_ERROR_CHECK(esp_console_cmd_register(&publish)); +} + +void app_main(void) +{ + static const size_t max_line = 256; + ESP_LOGI(TAG, "[APP] Free memory: %" PRIu32 " bytes", esp_get_free_heap_size()); + ESP_LOGI(TAG, "[APP] IDF version: %s", esp_get_idf_version()); + esp_log_level_set("*", ESP_LOG_INFO); + esp_log_level_set("wifi", ESP_LOG_ERROR); + esp_log_level_set("mqtt_client", ESP_LOG_INFO); + esp_log_level_set("outbox", ESP_LOG_INFO); + ESP_ERROR_CHECK(nvs_flash_init()); + ESP_ERROR_CHECK(esp_netif_init()); + ESP_ERROR_CHECK(esp_event_loop_create_default()); + ESP_ERROR_CHECK(example_connect()); + esp_console_repl_t *repl = NULL; + esp_console_repl_config_t repl_config = ESP_CONSOLE_REPL_CONFIG_DEFAULT(); + repl_config.prompt = "mqtt>"; + repl_config.max_cmdline_length = max_line; + esp_console_register_help_command(); + register_pubsub_commands(); + register_common_commands(); + esp_console_dev_uart_config_t hw_config = ESP_CONSOLE_DEV_UART_CONFIG_DEFAULT(); + ESP_ERROR_CHECK(esp_console_new_repl_uart(&hw_config, &repl_config, &repl)); + ESP_ERROR_CHECK(esp_console_start_repl(repl)); +} diff --git a/test/apps/mqtt_conformance/pytest_mqtt_conformance.py b/test/apps/mqtt_conformance/pytest_mqtt_conformance.py new file mode 100644 index 0000000..d3eefec --- /dev/null +++ b/test/apps/mqtt_conformance/pytest_mqtt_conformance.py @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Unlicense OR CC0-1.0 +from __future__ import annotations + +import contextlib +import os +import random +import re +import socket +import string +import sys +import threading +import time +from pathlib import Path +from typing import Generator, Protocol + +import pexpect +import pytest +from pytest_embedded import Dut +from pytest_embedded_idf.utils import idf_parametrize + +TOPIC_SIZE = 16 +DUT_READY_TIMEOUT = 30 +DUT_CONNECT_TIMEOUT = 30 +DUT_SUBSCRIBE_TIMEOUT = 60 +DUT_TEST_TIMEOUT = 60 +PAHO_BROKER_PORT = int(os.getenv("MQTT_CONFORMANCE_PAHO_BROKER_PORT", "18883")) +CONNECT_RETRIES = int(os.getenv("MQTT_CONFORMANCE_CONNECT_RETRIES", "3")) +RETRY_BACKOFF_SEC = float(os.getenv("MQTT_CONFORMANCE_RETRY_BACKOFF_SEC", "2")) + +PAHO_SPEC_FILE = ( + Path(__file__).resolve().parents[3] + / "test" + / "tools" + / "paho.mqtt.testing" + / "interoperability" + / "specifications" + / "MQTTV311.py" +) +PAHO_INTEROP_DIR = PAHO_SPEC_FILE.parent.parent + +# Add paho interoperability directory so we can import the broker at fixture time. +if str(PAHO_INTEROP_DIR) not in sys.path: + sys.path.insert(0, str(PAHO_INTEROP_DIR)) + + +def build_topic() -> str: + suffix = "".join(random.choice(string.ascii_letters) for _ in range(TOPIC_SIZE)) + return f"test/conformance/{suffix}" + + +def require_paho_testing_checked_out() -> None: + """Hard requirement: fail the test if the paho.mqtt.testing submodule is not available.""" + if not PAHO_SPEC_FILE.exists(): + pytest.fail( + "paho.mqtt.testing submodule is not available (required for mqtt conformance tests). " + "Run: git submodule update --init --recursive test/tools/paho.mqtt.testing" + ) + + +def get_host_ip4_by_dest_ip(dest_ip: str = "8.8.8.8") -> str: + """Return the primary host IPv4 used to reach dest_ip (e.g. for DUT to reach host broker).""" + with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock: + sock.connect((dest_ip, 80)) + return sock.getsockname()[0] + + +class _BrokerHandle(Protocol): + uri: str + + def shutdown(self) -> None: ... + + +def _start_paho_broker(port: int, host_ip: str) -> _BrokerHandle: + """Start paho V311+V5 broker in-process; return object with .uri and .shutdown(). + Imports deferred so idf-ci collection-time mocking does not replace paho. + """ + from mqtt.brokers.V311 import MQTTBrokers as MQTTV3Brokers + from mqtt.brokers.V5 import MQTTBrokers as MQTTV5Brokers + from mqtt.brokers.listeners import TCPListeners + + lock = threading.RLock() + shared_data: dict = {} + options = { + "visual": False, + "persistence": False, + "overlapping_single": True, + "dropQoS0": True, + "zero_length_clientids": True, + "publish_on_pubrel": False, + "topicAliasMaximum": 2, + "maximumPacketSize": 16384, + "receiveMaximum": 2, + "serverKeepAlive": 60, + "maximum_qos": 2, + "retain_available": True, + "subscription_identifier_available": True, + "shared_subscription_available": True, + "server_keep_alive": None, + } + broker3 = MQTTV3Brokers(options=options.copy(), lock=lock, sharedData=shared_data) + broker5 = MQTTV5Brokers(options=options.copy(), lock=lock, sharedData=shared_data) + broker3.setBroker5(broker5) + broker5.setBroker3(broker3) + TCPListeners.setBrokers(broker3, broker5) + server = TCPListeners.create(port=port, host="", serve_forever=False) + + class _Broker: + def __init__(self) -> None: + self.uri = f"mqtt://{host_ip}:{port}" + self._broker3 = broker3 + self._broker5 = broker5 + self._server = server + + def shutdown(self) -> None: + self._broker3.shutdown() + self._broker5.shutdown() + if self._server: + self._server.shutdown() + + return _Broker() + + +@pytest.fixture(scope="module") +def broker() -> Generator[_BrokerHandle, None, None]: + """Start paho MQTT broker in-process for the smoke test. No subclass, just V311+V5 + TCP listener.""" + require_paho_testing_checked_out() + host_ip = os.getenv("MQTT_CONFORMANCE_HOST_IP", "").strip() or get_host_ip4_by_dest_ip() + b = _start_paho_broker(port=PAHO_BROKER_PORT, host_ip=host_ip) + yield b + b.shutdown() + + +@pytest.fixture(scope="module") +def broker_uri(broker: _BrokerHandle) -> str: + return broker.uri + + +@pytest.fixture +def mqtt_client(dut: Dut, broker_uri: str): + require_paho_testing_checked_out() + dut.expect(re.compile(rb"mqtt>"), timeout=DUT_READY_TIMEOUT) + dut.write("init") + dut.write(f"set_uri {broker_uri}") + yield dut + dut.write("destroy") + + +def start_client(dut: Dut) -> None: + for attempt in range(1, CONNECT_RETRIES + 1): + dut.write("start") + try: + dut.expect(re.compile(rb"MQTT_EVENT_CONNECTED"), timeout=DUT_CONNECT_TIMEOUT) + return + except pexpect.TIMEOUT: + dut.write("stop") + if attempt == CONNECT_RETRIES: + raise + time.sleep(RETRY_BACKOFF_SEC) + + +def stop_client(dut: Dut) -> None: + dut.write("stop") + + +@pytest.mark.eth_ip101 +@idf_parametrize("target", ["esp32"], indirect=["target"]) +def test_mqtt_v311_subscribe_and_qos1_publish__sec_3_8_4_and_4_3(mqtt_client: Dut) -> None: + """ + MQTT v3.1.1 conformance smoke case: + - section 3.8.4: SUBSCRIBE/SUBACK interaction + - section 4.3: QoS 1 publish flow (at least once semantics) + + Reference suite integrated from: + test/tools/paho.mqtt.testing/interoperability/specifications/MQTTV311.py + """ + topic = build_topic() + + start_client(mqtt_client) + mqtt_client.write(f"subscribe {topic} 1") + mqtt_client.expect(re.compile(rb"MQTT_EVENT_SUBSCRIBED"), timeout=DUT_SUBSCRIBE_TIMEOUT) + + mqtt_client.write(f"publish {topic} qos1 4 1 0 1") + # DUT may emit DATA_COMPLETE (incoming) before PUBLISHED (outgoing ack); accept either order. + mqtt_client.expect( + [re.compile(rb"MQTT_EVENT_PUBLISHED"), re.compile(rb"MQTT_EVENT_DATA_COMPLETE")], + timeout=DUT_TEST_TIMEOUT, + ) + mqtt_client.expect( + [re.compile(rb"MQTT_EVENT_PUBLISHED"), re.compile(rb"MQTT_EVENT_DATA_COMPLETE")], + timeout=DUT_TEST_TIMEOUT, + ) + + stop_client(mqtt_client) diff --git a/test/apps/mqtt_conformance/sdkconfig.ci.default b/test/apps/mqtt_conformance/sdkconfig.ci.default new file mode 100644 index 0000000..b039d7d --- /dev/null +++ b/test/apps/mqtt_conformance/sdkconfig.ci.default @@ -0,0 +1,13 @@ +CONFIG_MQTT_PROTOCOL_5=y +CONFIG_EXAMPLE_CONNECT_ETHERNET=y +CONFIG_EXAMPLE_CONNECT_WIFI=n +CONFIG_ESP_NETIF_RECEIVE_REPORT_ERRORS=y +# CONFIG_EXAMPLE_* names work on IDF 5.x; IDF 6.x maps them via protocol_examples_common sdkconfig.rename. +CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y +CONFIG_EXAMPLE_ETH_PHY_IP101=y +CONFIG_EXAMPLE_ETH_MDC_GPIO=23 +CONFIG_EXAMPLE_ETH_MDIO_GPIO=18 +CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5 +CONFIG_EXAMPLE_ETH_PHY_ADDR=1 +CONFIG_EXAMPLE_CONNECT_IPV6=y +CONFIG_LOG_MAXIMUM_LEVEL_DEBUG=y diff --git a/test/tools/paho.mqtt.testing b/test/tools/paho.mqtt.testing new file mode 160000 index 0000000..9d7bb80 --- /dev/null +++ b/test/tools/paho.mqtt.testing @@ -0,0 +1 @@ +Subproject commit 9d7bb80bb8b9d9cfc0b52f8cb4c1916401281103