diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index c393f775c3..a79f225023 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -129,15 +129,8 @@ def uploadResults(def pipeline, SlurmCluster cluster, String nodeName, String st
def downloadTimeoutTestSucceed = Utils.exec(pipeline, script: "sshpass -p '${remote.passwd}' scp -P ${remote.port} -r -p ${COMMON_SSH_OPTIONS} ${remote.user}@${remote.host}:${timeoutTestFilePath} ${stageName}/", returnStatus: true, numRetries: 3) == 0
if (downloadTimeoutTestSucceed) {
sh "ls -al ${stageName}/"
- def timeoutTestXml = generateTimeoutTestResultXml(stageName, "unfinished_test.txt")
- if (timeoutTestXml != null) {
- sh """
-cat > ${stageName}/results-timeout.xml << 'EOF_TIMEOUT_XML'
-${timeoutTestXml}
-EOF_TIMEOUT_XML
- """
- hasTimeoutTest = true
- }
+ // Generate timeout test result xml if there are terminated unexpectedly tests
+ hasTimeoutTest = generateTimeoutTestResultXml(pipeline, stageName)
}
// Download normal test results
def resultsFilePath = "/home/svc_tensorrt/bloom/scripts/${nodeName}/results*.xml"
@@ -1621,14 +1614,9 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu
sh "mkdir -p ${stageName}"
finallyRunner()
if (stageIsFailed) {
- def timeoutTestXml = generateTimeoutTestResultXml(stageName, "unfinished_test.txt")
- if (timeoutTestXml != null) {
- sh """
-cat > ${stageName}/results-timeout.xml << 'EOF_TIMEOUT_XML'
-${timeoutTestXml}
-EOF_TIMEOUT_XML
- """
- }
+ // Generate timeout test result xml if there are terminated unexpectedly tests
+ generateTimeoutTestResultXml(pipeline, stageName)
+ // Generate stage fail test result xml if the stage failed and there is no result*.xml
def stageXml = generateStageFailTestResultXml(stageName, "Stage Failed", "Stage run failed without result", "results*.xml")
if (stageXml != null) {
sh "echo '${stageXml}' > ${stageName}/results-stage.xml"
@@ -2023,27 +2011,18 @@ def launchTestListCheck(pipeline)
})
}
-def generateTimeoutTestResultXml(stageName, testFilePath) {
- if (!fileExists("${stageName}/${testFilePath}")) {
- echo "No ${testFilePath} found in ${stageName}, skipping timeout XML generation"
- return null
+def generateTimeoutTestResultXml(pipeline, stageName) {
+ def scriptPath = sh(
+ script: "find . -name generate_timeout_xml.py | head -n 1 | xargs realpath",
+ returnStdout: true
+ ).trim()
+ def curPath = sh(script: "realpath .", returnStdout: true).trim()
+ def outputFilePath = "${curPath}/${stageName}/results-timeout.xml"
+ sh """python3 ${scriptPath} --stage-name '${stageName}' --test-file-path 'unfinished_test.txt' --output-file '${outputFilePath}'"""
+ if (fileExists(outputFilePath)) {
+ return true
}
- String timeoutTests = sh(script: "cd ${stageName} && cat ${testFilePath}", returnStdout: true).trim()
- echo "timeoutTests: ${timeoutTests}"
-
- if (timeoutTests == null || timeoutTests == "") {
- return null
- }
- def testList = timeoutTests.split("\n")
- String xmlContent = """
- """
- testList.each { test ->
- xmlContent += """
- Test terminated unexpectedly
- """
- }
- xmlContent += ""
- return xmlContent
+ return false
}
def generateStageFailTestResultXml(stageName, subName, failureLog, resultPath) {
diff --git a/jenkins/scripts/generate_timeout_xml.py b/jenkins/scripts/generate_timeout_xml.py
new file mode 100644
index 0000000000..ea5a4203d6
--- /dev/null
+++ b/jenkins/scripts/generate_timeout_xml.py
@@ -0,0 +1,147 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import sys
+from html import escape
+
+
+def parse_xml_classname_name_file_from_testname(testname, stage_name):
+ """Parse XML attributes from a test name.
+
+ Args:
+ testname: Test identifier, may be prefixed with stage_name and can have
+ different formats (e.g., "unittest/...", "file.py::class::test")
+ stage_name: Name of the test stage, used for classname construction
+
+ Returns:
+ Tuple of (classname, name, file) where:
+ - classname: Fully qualified class name for the test
+ - name: Test method or case name
+ - file: Source file containing the test
+ """
+ classname, name, file = "", "", ""
+
+ # Remove stage_name prefix if present
+ if testname.startswith(stage_name + "/"):
+ testname = testname[len(stage_name) + 1 :]
+
+ # Get file name
+ if testname.startswith("unittest/"):
+ file = "test_unittests.py"
+ else:
+ file = testname.split("::")[0]
+
+ # Get test name
+ if testname.startswith("unittest/"):
+ name = "test_unittests_v2[" + testname + "]"
+ else:
+ name = testname.split("::")[-1]
+
+ # Get class name
+ if testname.startswith("unittest/"):
+ classname = stage_name + ".test_unittests"
+ elif len(testname.split("::")) == 3:
+ classname = (
+ stage_name
+ + "."
+ + testname.split("::")[0].replace(".py", "").replace("/", ".")
+ + "."
+ + testname.split("::")[1]
+ )
+ else:
+ classname = stage_name + "." + testname.split("::")[0].replace(".py", "").replace("/", ".")
+ if testname.startswith("accuracy/") or (
+ testname.startswith("examples/") and "[" not in testname
+ ):
+ classname = ""
+
+ return classname, name, file
+
+
+def generate_timeout_xml(stage_name, testList, outputFilePath):
+ """Generate JUnit XML report for timed-out tests.
+
+ Args:
+ stage_name: Name of the test stage
+ testList: List of test names that timed out
+ outputFilePath: Path where the XML report will be written
+ """
+ num_tests = len(testList)
+ # Escape stage_name for XML safety
+ stage_name_escaped = escape(stage_name, quote=True)
+ xmlContent = (
+ f'\n'
+ f' '
+ )
+
+ for test in testList:
+ classname, name, file = parse_xml_classname_name_file_from_testname(test, stage_name)
+ # Escape all XML attribute values
+ classname_escaped = escape(classname, quote=True)
+ name_escaped = escape(name, quote=True)
+ file_escaped = escape(file, quote=True)
+ xmlContent += (
+ f'\n'
+ f' '
+ f" Test terminated unexpectedly\n"
+ f" "
+ )
+ xmlContent += ""
+
+ with open(outputFilePath, "w", encoding="utf-8") as f:
+ f.write(xmlContent)
+
+
+def main():
+ """Parse arguments and generate timeout test XML report.
+
+ Reads a list of timed-out tests from a file and generates a JUnit-compatible
+ XML report marking each test with an error status.
+ """
+ # Parse command-line arguments
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--stage-name", required=True, help="Stage name")
+ parser.add_argument("--test-file-path", required=True, help="Test list file path")
+ parser.add_argument("--output-file", required=True, help="Output file path")
+ args = parser.parse_args(sys.argv[1:])
+ stageName = args.stage_name
+ testFilePath = args.test_file_path
+ outputFilePath = args.output_file
+
+ full_path = os.path.join(stageName, testFilePath)
+ if not os.path.exists(full_path):
+ print(f"No {full_path} found, skipping timeout XML generation")
+ return
+
+ try:
+ with open(full_path, "r", encoding="utf-8") as f:
+ timeoutTests = [line.strip() for line in f if line.strip()]
+ except IOError as e:
+ print(f"Error reading {full_path}: {e}")
+ return
+
+ if len(timeoutTests) == 0:
+ print(f"No timeout tests found in {full_path}, skipping timeout XML generation")
+ return
+
+ generate_timeout_xml(stageName, timeoutTests, outputFilePath)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/integration/defs/utils/periodic_junit.py b/tests/integration/defs/utils/periodic_junit.py
index 26ff6ba8e5..68dfedfa17 100644
--- a/tests/integration/defs/utils/periodic_junit.py
+++ b/tests/integration/defs/utils/periodic_junit.py
@@ -207,6 +207,7 @@ class PeriodicJUnitXML:
if should_flush_by_batch or should_flush_by_time:
if should_flush_by_batch:
+ print() # Add blank line before info message
self._log_info(
f"Completed {self.completed_tests} cases in the last "
f"{current_time - self.last_save_time:.0f} seconds")