From 08ed9d7305721a311f98738535ceb33b3525a11a Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 5 Aug 2025 17:02:17 +0800 Subject: [PATCH] [None][doc] add introduction doc on qa test (#6535) Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- tests/integration/test_lists/qa/README.md | 108 ++++++++++++++++++ ...se_digits_func.txt => llm_digits_func.txt} | 0 ...se_digits_perf.txt => llm_digits_perf.txt} | 0 ...es_test_list.txt => llm_function_full.txt} | 0 ...lease_gb20x.txt => llm_function_gb20x.txt} | 0 ...on_test.txt => llm_function_multinode.txt} | 0 ...ark_test_list.txt => llm_function_nim.txt} | 0 ...x_pro_6000.txt => llm_function_rtx6kd.txt} | 0 ...anity_test.txt => llm_function_sanity.txt} | 0 ..._cluster_test.yml => llm_perf_cluster.yml} | 0 ...elease_perf_test.yml => llm_perf_full.yml} | 0 ...ease_perf_l2_test.yml => llm_perf_nim.yml} | 0 ...rf_sanity_test.yml => llm_perf_sanity.yml} | 0 ...on_test.txt => llm_triton_integration.txt} | 0 ..._test.yml => llm_trt_integration_perf.yml} | 0 ...ml => llm_trt_integration_perf_sanity.yml} | 0 16 files changed, 108 insertions(+) create mode 100644 tests/integration/test_lists/qa/README.md rename tests/integration/test_lists/qa/{llm_release_digits_func.txt => llm_digits_func.txt} (100%) rename tests/integration/test_lists/qa/{llm_release_digits_perf.txt => llm_digits_perf.txt} (100%) rename tests/integration/test_lists/qa/{examples_test_list.txt => llm_function_full.txt} (100%) rename tests/integration/test_lists/qa/{llm_release_gb20x.txt => llm_function_gb20x.txt} (100%) rename tests/integration/test_lists/qa/{llm_multinodes_function_test.txt => llm_function_multinode.txt} (100%) rename tests/integration/test_lists/qa/{benchmark_test_list.txt => llm_function_nim.txt} (100%) rename tests/integration/test_lists/qa/{llm_release_rtx_pro_6000.txt => llm_function_rtx6kd.txt} (100%) rename tests/integration/test_lists/qa/{llm_sanity_test.txt => llm_function_sanity.txt} (100%) rename tests/integration/test_lists/qa/{trt_llm_release_perf_cluster_test.yml => llm_perf_cluster.yml} (100%) rename tests/integration/test_lists/qa/{trt_llm_release_perf_test.yml => llm_perf_full.yml} (100%) rename tests/integration/test_lists/qa/{trt_llm_release_perf_l2_test.yml => llm_perf_nim.yml} (100%) rename tests/integration/test_lists/qa/{trt_llm_release_perf_sanity_test.yml => llm_perf_sanity.yml} (100%) rename tests/integration/test_lists/qa/{llm_triton_integration_test.txt => llm_triton_integration.txt} (100%) rename tests/integration/test_lists/qa/{trt_llm_integration_perf_test.yml => llm_trt_integration_perf.yml} (100%) rename tests/integration/test_lists/qa/{trt_llm_integration_perf_sanity_test.yml => llm_trt_integration_perf_sanity.yml} (100%) diff --git a/tests/integration/test_lists/qa/README.md b/tests/integration/test_lists/qa/README.md new file mode 100644 index 0000000000..7e1a093229 --- /dev/null +++ b/tests/integration/test_lists/qa/README.md @@ -0,0 +1,108 @@ +# Description + +This folder contains QA test definitions for TensorRT-LLM, which are executed on a daily/release schedule. These tests focus on end-to-end validation, accuracy verification, disaggregated testing, and performance benchmarking. + +## Test Categories + +QA tests are organized into three main categories: + +### 1. Functional Tests +Functional tests include E2E (end-to-end), accuracy, and disaggregated test cases: + +- **E2E Tests**: Complete workflow validation from model loading to inference output +- **Accuracy Tests**: Model accuracy verification against reference implementations +- **Disaggregated Tests**: Distributed deployment and multi-node scenario validation + +### 2. Performance Tests +Performance tests focus on benchmarking and performance validation: +- Baseline performance measurements +- Performance regression detection +- Throughput and latency benchmarking +- Resource utilization analysis + +### 3. Triton Backend Tests +Triton backend tests validate the integration with NVIDIA Triton Inference Server: +- Backend functionality validation +- Model serving capabilities +- API compatibility testing +- Integration performance testing + +## Dependencies + +The following Python packages are required for running QA tests: + +```bash +pip install mako oyaml rouge_score lm_eval +``` + +### Dependency Details + +- **mako**: Template engine for test generation and configuration +- **oyaml**: YAML parser with ordered dictionary support +- **rouge_score**: ROUGE evaluation metrics for text generation quality assessment +- **lm_eval**: Language model evaluation framework + +## Test Files + +This directory contains various test configuration files: + +### Functional Test Lists +- `llm_function_full.txt` - Primary test list for single node multi-GPU scenarios (all new test cases should be added here) +- `llm_function_sanity.txt` - Subset of examples for quick torch flow validation +- `llm_function_nim.txt` - NIM-specific functional test cases +- `llm_function_multinode.txt` - Multi-node functional test cases +- `llm_function_gb20x.txt` - GB20X release test cases +- `llm_function_rtx6kd.txt` - RTX 6000 Ada specific tests + +### Performance Test Files +- `llm_perf_full.yml` - Main performance test configuration +- `llm_perf_cluster.yml` - Cluster-based performance tests +- `llm_perf_sanity.yml` - Performance sanity checks +- `llm_perf_nim.yml` - NIM-specific performance tests +- `llm_trt_integration_perf.yml` - Integration performance tests +- `llm_trt_integration_perf_sanity.yml` - Integration performance sanity checks + +### Triton Backend Tests +- `llm_triton_integration.txt` - Triton backend integration tests + +### Release-Specific Tests +- `llm_digits_func.txt` - Functional tests for DIGITS release +- `llm_digits_perf.txt` - Performance tests for DIGITS release + +## Test Execution Schedule + +QA tests are executed on a regular schedule: + +- **Weekly**: Automated regression testing +- **Release**: Comprehensive validation before each release +- **On-demand**: Manual execution for specific validation needs + +## Running Tests + +### Manual Execution + +To run specific test categories: + +```bash +# direct to defs folder +cd tests/integration/defs +# Run all fp8 functional test +pytest --no-header -vs --test-list=../test_lists/qa/llm_function_full.txt -k fp8 +# Run a single test case +pytest -vs accuracy/test_cli_flow.py::TestLlama3_1_8B::test_auto_dtype +``` + +### Automated Execution + +QA tests are typically executed through CI/CD pipelines with appropriate test selection based on: + +- Release requirements +- Hardware availability +- Test priority and scope + +## Test Guidelines + +### Adding New Test Cases +- **Primary Location**: For functional testing, new test cases should be added to `llm_function_full.txt` first +- **Categorization**: Test cases should be categorized based on their scope and execution time +- **Validation**: Ensure test cases are properly validated before adding to any test list diff --git a/tests/integration/test_lists/qa/llm_release_digits_func.txt b/tests/integration/test_lists/qa/llm_digits_func.txt similarity index 100% rename from tests/integration/test_lists/qa/llm_release_digits_func.txt rename to tests/integration/test_lists/qa/llm_digits_func.txt diff --git a/tests/integration/test_lists/qa/llm_release_digits_perf.txt b/tests/integration/test_lists/qa/llm_digits_perf.txt similarity index 100% rename from tests/integration/test_lists/qa/llm_release_digits_perf.txt rename to tests/integration/test_lists/qa/llm_digits_perf.txt diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/llm_function_full.txt similarity index 100% rename from tests/integration/test_lists/qa/examples_test_list.txt rename to tests/integration/test_lists/qa/llm_function_full.txt diff --git a/tests/integration/test_lists/qa/llm_release_gb20x.txt b/tests/integration/test_lists/qa/llm_function_gb20x.txt similarity index 100% rename from tests/integration/test_lists/qa/llm_release_gb20x.txt rename to tests/integration/test_lists/qa/llm_function_gb20x.txt diff --git a/tests/integration/test_lists/qa/llm_multinodes_function_test.txt b/tests/integration/test_lists/qa/llm_function_multinode.txt similarity index 100% rename from tests/integration/test_lists/qa/llm_multinodes_function_test.txt rename to tests/integration/test_lists/qa/llm_function_multinode.txt diff --git a/tests/integration/test_lists/qa/benchmark_test_list.txt b/tests/integration/test_lists/qa/llm_function_nim.txt similarity index 100% rename from tests/integration/test_lists/qa/benchmark_test_list.txt rename to tests/integration/test_lists/qa/llm_function_nim.txt diff --git a/tests/integration/test_lists/qa/llm_release_rtx_pro_6000.txt b/tests/integration/test_lists/qa/llm_function_rtx6kd.txt similarity index 100% rename from tests/integration/test_lists/qa/llm_release_rtx_pro_6000.txt rename to tests/integration/test_lists/qa/llm_function_rtx6kd.txt diff --git a/tests/integration/test_lists/qa/llm_sanity_test.txt b/tests/integration/test_lists/qa/llm_function_sanity.txt similarity index 100% rename from tests/integration/test_lists/qa/llm_sanity_test.txt rename to tests/integration/test_lists/qa/llm_function_sanity.txt diff --git a/tests/integration/test_lists/qa/trt_llm_release_perf_cluster_test.yml b/tests/integration/test_lists/qa/llm_perf_cluster.yml similarity index 100% rename from tests/integration/test_lists/qa/trt_llm_release_perf_cluster_test.yml rename to tests/integration/test_lists/qa/llm_perf_cluster.yml diff --git a/tests/integration/test_lists/qa/trt_llm_release_perf_test.yml b/tests/integration/test_lists/qa/llm_perf_full.yml similarity index 100% rename from tests/integration/test_lists/qa/trt_llm_release_perf_test.yml rename to tests/integration/test_lists/qa/llm_perf_full.yml diff --git a/tests/integration/test_lists/qa/trt_llm_release_perf_l2_test.yml b/tests/integration/test_lists/qa/llm_perf_nim.yml similarity index 100% rename from tests/integration/test_lists/qa/trt_llm_release_perf_l2_test.yml rename to tests/integration/test_lists/qa/llm_perf_nim.yml diff --git a/tests/integration/test_lists/qa/trt_llm_release_perf_sanity_test.yml b/tests/integration/test_lists/qa/llm_perf_sanity.yml similarity index 100% rename from tests/integration/test_lists/qa/trt_llm_release_perf_sanity_test.yml rename to tests/integration/test_lists/qa/llm_perf_sanity.yml diff --git a/tests/integration/test_lists/qa/llm_triton_integration_test.txt b/tests/integration/test_lists/qa/llm_triton_integration.txt similarity index 100% rename from tests/integration/test_lists/qa/llm_triton_integration_test.txt rename to tests/integration/test_lists/qa/llm_triton_integration.txt diff --git a/tests/integration/test_lists/qa/trt_llm_integration_perf_test.yml b/tests/integration/test_lists/qa/llm_trt_integration_perf.yml similarity index 100% rename from tests/integration/test_lists/qa/trt_llm_integration_perf_test.yml rename to tests/integration/test_lists/qa/llm_trt_integration_perf.yml diff --git a/tests/integration/test_lists/qa/trt_llm_integration_perf_sanity_test.yml b/tests/integration/test_lists/qa/llm_trt_integration_perf_sanity.yml similarity index 100% rename from tests/integration/test_lists/qa/trt_llm_integration_perf_sanity_test.yml rename to tests/integration/test_lists/qa/llm_trt_integration_perf_sanity.yml