mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-01-13 22:18:36 +08:00
[Infra][TRTLLM-4374] Upgrade TRT 10.10.0 GA, CUDA 12.9 GA and DLFW 25.04 (#4049)
* [TRTLLM-4374] Upgrade TRT 10.10.0 GA, CUDA 12.9 GA and DLFW 25.04 Signed-off-by: Yiqing Yan <yiqingy@nvidia.com> * fix review Signed-off-by: Yiqing Yan <yiqingy@nvidia.com> * update images Signed-off-by: Yiqing Yan <yiqingy@nvidia.com> * Update jenkins/L0_Test.groovy Co-authored-by: Yanchao Lu <yanchaol@nvidia.com> Signed-off-by: Yiqing Yan <yiqingy@nvidia.com> * update image name Signed-off-by: Yiqing Yan <yiqingy@nvidia.com> --------- Signed-off-by: Yiqing Yan <yiqingy@nvidia.com> Co-authored-by: Yanchao Lu <yanchaol@nvidia.com>
This commit is contained in:
Yiqing Yan
GitHub
parent
e8d7834c50
commit
fda8b0277a
@ -1,7 +1,7 @@
|
||||
version: "3.9"
|
||||
services:
|
||||
tensorrt_llm-dev:
|
||||
image: urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.03-py3-x86_64-ubuntu24.04-trt10.9.0.34-skip-devel-202504250100-3759
|
||||
image: urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.04-py3-x86_64-ubuntu24.04-trt10.10.0.31-skip-tritondevel-202505121727-4049
|
||||
network_mode: host
|
||||
ipc: host
|
||||
|
||||
|
||||
@ -7,8 +7,8 @@ TensorRT-LLM
|
||||
[](https://nvidia.github.io/TensorRT-LLM/)
|
||||
[](https://www.python.org/downloads/release/python-3123/)
|
||||
[](https://www.python.org/downloads/release/python-31012/)
|
||||
[](https://developer.nvidia.com/cuda-downloads)
|
||||
[](https://developer.nvidia.com/tensorrt)
|
||||
[](https://developer.nvidia.com/cuda-downloads)
|
||||
[](https://developer.nvidia.com/tensorrt)
|
||||
[](./tensorrt_llm/version.py)
|
||||
[](./LICENSE)
|
||||
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
# Multi-stage Dockerfile
|
||||
ARG BASE_IMAGE=nvcr.io/nvidia/pytorch
|
||||
ARG TRITON_IMAGE=nvcr.io/nvidia/tritonserver
|
||||
ARG BASE_TAG=25.03-py3
|
||||
ARG TRITON_BASE_TAG=25.03-py3
|
||||
ARG BASE_TAG=25.04-py3
|
||||
ARG TRITON_BASE_TAG=25.04-py3
|
||||
ARG DEVEL_IMAGE=devel
|
||||
|
||||
FROM ${BASE_IMAGE}:${BASE_TAG} AS base
|
||||
|
||||
@ -165,16 +165,16 @@ jenkins-aarch64_%: STAGE = devel
|
||||
jenkins-rockylinux8_%: IMAGE_WITH_TAG = $(shell grep 'LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE = ' ../jenkins/L0_MergeRequest.groovy | grep -o '".*"' | tr -d '"')
|
||||
jenkins-rockylinux8_%: STAGE = devel
|
||||
jenkins-rockylinux8_%: BASE_IMAGE = nvidia/cuda
|
||||
jenkins-rockylinux8_%: BASE_TAG = 12.8.1-devel-rockylinux8
|
||||
jenkins-rockylinux8_%: BASE_TAG = 12.9.0-devel-rockylinux8
|
||||
|
||||
rockylinux8_%: STAGE = devel
|
||||
rockylinux8_%: BASE_IMAGE = nvidia/cuda
|
||||
rockylinux8_%: BASE_TAG = 12.8.1-devel-rockylinux8
|
||||
rockylinux8_%: BASE_TAG = 12.9.0-devel-rockylinux8
|
||||
|
||||
# For x86_64 and aarch64
|
||||
ubuntu22_%: STAGE = devel
|
||||
ubuntu22_%: BASE_IMAGE = nvidia/cuda
|
||||
ubuntu22_%: BASE_TAG = 12.8.1-devel-ubuntu22.04
|
||||
ubuntu22_%: BASE_TAG = 12.9.0-devel-ubuntu22.04
|
||||
|
||||
trtllm_%: STAGE = release
|
||||
trtllm_%: PUSH_TO_STAGING := 0
|
||||
|
||||
@ -5,7 +5,7 @@ set -ex
|
||||
# This script is used for reinstalling CUDA on Rocky Linux 8 with the run file.
|
||||
# CUDA version is usually aligned with the latest NGC CUDA image tag.
|
||||
# Only use when public CUDA image is not ready.
|
||||
CUDA_VER="12.8.1_570.124.06"
|
||||
CUDA_VER="12.9.0_575.51.03"
|
||||
CUDA_VER_SHORT="${CUDA_VER%_*}"
|
||||
|
||||
NVCC_VERSION_OUTPUT=$(nvcc --version)
|
||||
|
||||
@ -2,20 +2,24 @@
|
||||
|
||||
set -ex
|
||||
|
||||
TRT_VER="10.9.0.34"
|
||||
TRT_VER="10.10.0.31"
|
||||
# Align with the pre-installed cuDNN / cuBLAS / NCCL versions from
|
||||
# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-25-03.html#rel-25-03
|
||||
CUDA_VER="12.8" # 12.8.1
|
||||
# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-25-04.html#rel-25-04
|
||||
CUDA_VER="12.9" # 12.9.0
|
||||
# Keep the installation for cuDNN if users want to install PyTorch with source codes.
|
||||
# PyTorch 2.x can compile with cuDNN v9.
|
||||
CUDNN_VER="9.8.0.87-1"
|
||||
NCCL_VER="2.25.1-1+cuda12.8"
|
||||
CUBLAS_VER="12.8.4.1-1"
|
||||
CUDNN_VER="9.9.0.52-1"
|
||||
# NCCL version 2.26.3 used in the NGC PyTorch 25.04 image but not existing in public.
|
||||
# Use NCCL version 2.26.5 instead.
|
||||
NCCL_VER="2.26.5-1+cuda12.9"
|
||||
# cuBLAS version 12.9.0.2 used in the NGC PyTorch 25.04 image but not existing in public.
|
||||
# Use cuBLAS version 12.9.0.13 instead.
|
||||
CUBLAS_VER="12.9.0.13-1"
|
||||
# Align with the pre-installed CUDA / NVCC / NVRTC versions from
|
||||
# https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html
|
||||
NVRTC_VER="12.8.93-1"
|
||||
CUDA_RUNTIME="12.8.90-1"
|
||||
CUDA_DRIVER_VERSION="570.124.06-1.el8"
|
||||
NVRTC_VER="12.9.41-1"
|
||||
CUDA_RUNTIME="12.9.37-1"
|
||||
CUDA_DRIVER_VERSION="575.51.03-1.el8"
|
||||
|
||||
for i in "$@"; do
|
||||
case $i in
|
||||
@ -86,7 +90,7 @@ install_rockylinux_requirements() {
|
||||
"libnccl-${NCCL_VER}.${ARCH1}" \
|
||||
"libnccl-devel-${NCCL_VER}.${ARCH1}" \
|
||||
"cuda-compat-${CUBLAS_CUDA_VERSION}-${CUDA_DRIVER_VERSION}.${ARCH1}" \
|
||||
"cuda-toolkit-12-8-config-common-${CUDA_RUNTIME}.noarch" \
|
||||
"cuda-toolkit-${CUBLAS_CUDA_VERSION}-config-common-${CUDA_RUNTIME}.noarch" \
|
||||
"cuda-toolkit-12-config-common-${CUDA_RUNTIME}.noarch" \
|
||||
"cuda-toolkit-config-common-${CUDA_RUNTIME}.noarch" \
|
||||
"libcublas-${CUBLAS_CUDA_VERSION}-${CUBLAS_VER}.${ARCH1}" \
|
||||
@ -102,7 +106,7 @@ install_rockylinux_requirements() {
|
||||
libnccl-${NCCL_VER}.${ARCH1}.rpm \
|
||||
libnccl-devel-${NCCL_VER}.${ARCH1}.rpm \
|
||||
cuda-compat-${CUBLAS_CUDA_VERSION}-${CUDA_DRIVER_VERSION}.${ARCH1}.rpm \
|
||||
cuda-toolkit-12-8-config-common-${CUDA_RUNTIME}.noarch.rpm \
|
||||
cuda-toolkit-${CUBLAS_CUDA_VERSION}-config-common-${CUDA_RUNTIME}.noarch.rpm \
|
||||
cuda-toolkit-12-config-common-${CUDA_RUNTIME}.noarch.rpm \
|
||||
cuda-toolkit-config-common-${CUDA_RUNTIME}.noarch.rpm \
|
||||
libcublas-${CUBLAS_CUDA_VERSION}-${CUBLAS_VER}.${ARCH1}.rpm \
|
||||
@ -117,14 +121,15 @@ install_rockylinux_requirements() {
|
||||
install_tensorrt() {
|
||||
PY_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[0:2])))')
|
||||
PARSED_PY_VERSION=$(echo "${PY_VERSION//./}")
|
||||
TRT_CUDA_VERSION="12.8"
|
||||
TRT_CUDA_VERSION=${CUDA_VER}
|
||||
TRT_VER_SHORT=$(echo $TRT_VER | cut -d. -f1-3)
|
||||
|
||||
if [ -z "$RELEASE_URL_TRT" ];then
|
||||
ARCH=${TRT_TARGETARCH}
|
||||
if [ -z "$ARCH" ];then ARCH=$(uname -m);fi
|
||||
if [ "$ARCH" = "arm64" ];then ARCH="aarch64";fi
|
||||
if [ "$ARCH" = "amd64" ];then ARCH="x86_64";fi
|
||||
RELEASE_URL_TRT="https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.9.0/tars/TensorRT-${TRT_VER}.Linux.${ARCH}-gnu.cuda-${TRT_CUDA_VERSION}.tar.gz"
|
||||
RELEASE_URL_TRT="https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/${TRT_VER_SHORT}/tars/TensorRT-${TRT_VER}.Linux.${ARCH}-gnu.cuda-${TRT_CUDA_VERSION}.tar.gz"
|
||||
fi
|
||||
|
||||
wget --no-verbose ${RELEASE_URL_TRT} -O /tmp/TensorRT.tar
|
||||
|
||||
@ -115,9 +115,9 @@ The following table shows the supported software for TensorRT-LLM.
|
||||
* -
|
||||
- Software Compatibility
|
||||
* - Container
|
||||
- [25.03](https://docs.nvidia.com/deeplearning/frameworks/support-matrix/index.html)
|
||||
- [25.04](https://docs.nvidia.com/deeplearning/frameworks/support-matrix/index.html)
|
||||
* - TensorRT
|
||||
- [10.9](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html)
|
||||
- [10.10](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html)
|
||||
* - Precision
|
||||
-
|
||||
- Hopper (SM90) - FP32, FP16, BF16, FP8, INT8, INT4
|
||||
|
||||
@ -16,7 +16,7 @@ AARCH64_TRIPLE = "aarch64-linux-gnu"
|
||||
|
||||
LLM_DOCKER_IMAGE = env.dockerImage
|
||||
|
||||
AGENT_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.03-py3-x86_64-ubuntu24.04-trt10.9.0.34-skip-tritondevel-202505110947-4191"
|
||||
AGENT_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.04-py3-x86_64-ubuntu24.04-trt10.10.0.31-skip-tritondevel-202505121727-4049"
|
||||
|
||||
POD_TIMEOUT_SECONDS = env.podTimeoutSeconds ? env.podTimeoutSeconds : "21600"
|
||||
|
||||
|
||||
@ -21,10 +21,10 @@ UPLOAD_PATH = env.uploadPath ? env.uploadPath : "sw-tensorrt-generic/llm-artifac
|
||||
// Container configuration
|
||||
// available tags can be found in: https://urm.nvidia.com/artifactory/sw-tensorrt-docker/tensorrt-llm/
|
||||
// [base_image_name]-[arch]-[os](-[python_version])-[trt_version]-[torch_install_type]-[stage]-[date]-[mr_id]
|
||||
LLM_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.03-py3-x86_64-ubuntu24.04-trt10.9.0.34-skip-tritondevel-202505110947-4191"
|
||||
LLM_SBSA_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.03-py3-aarch64-ubuntu24.04-trt10.9.0.34-skip-tritondevel-202505110947-4191"
|
||||
LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.8.1-devel-rocky8-x86_64-rocky8-py310-trt10.9.0.34-skip-tritondevel-202505110947-4191"
|
||||
LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.8.1-devel-rocky8-x86_64-rocky8-py312-trt10.9.0.34-skip-tritondevel-202505110947-4191"
|
||||
LLM_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.04-py3-x86_64-ubuntu24.04-trt10.10.0.31-skip-tritondevel-202505121727-4049"
|
||||
LLM_SBSA_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.04-py3-aarch64-ubuntu24.04-trt10.10.0.31-skip-tritondevel-202505121727-4049"
|
||||
LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.9.0-devel-rocky8-x86_64-rocky8-py310-trt10.10.0.31-skip-tritondevel-202505121727-4049"
|
||||
LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.9.0-devel-rocky8-x86_64-rocky8-py312-trt10.10.0.31-skip-tritondevel-202505121727-4049"
|
||||
|
||||
LLM_ROCKYLINUX8_DOCKER_IMAGE = LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE
|
||||
|
||||
|
||||
@ -35,11 +35,11 @@ linuxPkgName = ( env.targetArch == AARCH64_TRIPLE ? "tensorrt-llm-sbsa-release-s
|
||||
// available tags can be found in: https://urm.nvidia.com/artifactory/sw-tensorrt-docker/tensorrt-llm/
|
||||
// [base_image_name]-[arch]-[os](-[python_version])-[trt_version]-[torch_install_type]-[stage]-[date]-[mr_id]
|
||||
LLM_DOCKER_IMAGE = env.dockerImage
|
||||
LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.8.1-devel-rocky8-x86_64-rocky8-py310-trt10.9.0.34-skip-tritondevel-202505110947-4191"
|
||||
LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.8.1-devel-rocky8-x86_64-rocky8-py312-trt10.9.0.34-skip-tritondevel-202505110947-4191"
|
||||
LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.9.0-devel-rocky8-x86_64-rocky8-py310-trt10.10.0.31-skip-tritondevel-202505121727-4049"
|
||||
LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.9.0-devel-rocky8-x86_64-rocky8-py312-trt10.10.0.31-skip-tritondevel-202505121727-4049"
|
||||
|
||||
// DLFW torch image
|
||||
DLFW_IMAGE = "nvcr.io/nvidia/pytorch:25.03-py3"
|
||||
DLFW_IMAGE = "nvcr.io/nvidia/pytorch:25.04-py3"
|
||||
|
||||
//Ubuntu base image
|
||||
UBUNTU_22_04_IMAGE = "urm.nvidia.com/docker/ubuntu:22.04"
|
||||
@ -1580,9 +1580,10 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null)
|
||||
|
||||
def fullWheelPath = "${cpu_arch}/${wheelPath}${wheelName}"
|
||||
|
||||
sanityRunner("Sanity check") {
|
||||
runPackageSanityCheck(pipeline, fullWheelPath, values[3], cpver)
|
||||
}
|
||||
// TODO: Re-enable the sanity check after updating GPU testers' driver version.
|
||||
// sanityRunner("Sanity check") {
|
||||
// runPackageSanityCheck(pipeline, fullWheelPath, values[3], cpver)
|
||||
// }
|
||||
|
||||
def checkPipStage = false
|
||||
if (cpu_arch == X86_64_TRIPLE) {
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
|
||||
import java.lang.InterruptedException
|
||||
|
||||
DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.03-py3-x86_64-ubuntu24.04-trt10.9.0.34-skip-tritondevel-202505110947-4191"
|
||||
DOCKER_IMAGE = "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.04-py3-x86_64-ubuntu24.04-trt10.10.0.31-skip-tritondevel-202505121727-4049"
|
||||
|
||||
def createKubernetesPodConfig(image)
|
||||
{
|
||||
|
||||
@ -19,8 +19,8 @@ pandas
|
||||
h5py==3.12.1
|
||||
StrEnum
|
||||
sentencepiece>=0.1.99
|
||||
tensorrt~=10.9.0
|
||||
# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-25-03.html#rel-25-03 uses 2.7.0a0.
|
||||
tensorrt~=10.10.0
|
||||
# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-25-04.html#rel-25-04 uses 2.7.0a0.
|
||||
torch>=2.6.0,<=2.7.0a0
|
||||
torchvision
|
||||
nvidia-modelopt[torch]~=0.29.0
|
||||
|
||||
@ -157,6 +157,43 @@ def setup_conan(scripts_dir, venv_python):
|
||||
return venv_conan
|
||||
|
||||
|
||||
def apply_torch_nvtx3_workaround(venv_python: Path):
|
||||
"""Workaround for nvtx3 path detection in PyTorch's CMake files."""
|
||||
try:
|
||||
# Get site-packages directory
|
||||
result = check_output(
|
||||
f'"{venv_python}" -c "import site; print(site.getsitepackages()[0])"',
|
||||
shell=True,
|
||||
text=True)
|
||||
site_packages = Path(result.strip())
|
||||
torch_dir = site_packages / "torch"
|
||||
|
||||
if not torch_dir.exists():
|
||||
print(f"Not found torch installation for patching NVTX3 workaround")
|
||||
return
|
||||
|
||||
# Define patterns and their corresponding messages
|
||||
replacement_patterns = [
|
||||
("find_path(nvtx3_dir NAMES nvtx3)",
|
||||
"Applying NVTX3 workaround to {cmake_file}"),
|
||||
('find_path(nvtx3_dir NAMES nvtx3 PATHS "${PROJECT_SOURCE_DIR}/third_party/NVTX/c/include" NO_DEFAULT_PATH)',
|
||||
"Applying additional NVTX3 workaround to {cmake_file}")
|
||||
]
|
||||
|
||||
replacement = "find_path(nvtx3_dir NAMES nvtx3 PATHS ${CUDA_INCLUDE_DIRS})"
|
||||
|
||||
for search_pattern, message_template in replacement_patterns:
|
||||
for cmake_file in torch_dir.rglob("*.cmake"):
|
||||
content = cmake_file.read_text()
|
||||
if search_pattern in content:
|
||||
print(message_template.format(cmake_file=cmake_file))
|
||||
new_content = content.replace(search_pattern, replacement)
|
||||
cmake_file.write_text(new_content)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Failed to apply NVTX3 workaround: {e}")
|
||||
|
||||
|
||||
def main(*,
|
||||
build_type: str = "Release",
|
||||
generator: str = "",
|
||||
@ -207,6 +244,11 @@ def main(*,
|
||||
venv_python, venv_conan = setup_venv(project_dir,
|
||||
project_dir / requirements_filename)
|
||||
|
||||
# Workaround for torch nvtx3 find_path not work issue with CUDA 12.9.
|
||||
# See https://github.com/pytorch/pytorch/pull/147418.
|
||||
apply_torch_nvtx3_workaround(Path(sys.executable))
|
||||
apply_torch_nvtx3_workaround(venv_python)
|
||||
|
||||
# Ensure base TRT is installed (check inside the venv)
|
||||
reqs = check_output([str(venv_python), "-m", "pip", "freeze"])
|
||||
installed_packages = [r.decode().split("==")[0] for r in reqs.split()]
|
||||
|
||||
@ -487,3 +487,4 @@ accuracy/test_llm_api_pytorch.py::TestLlama4MaverickInstruct::test_auto_dtype[tp
|
||||
test_e2e.py::test_ptp_quickstart_advanced_8gpus[Nemotron-Ultra-253B-nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1] SKIP (https://nvbugs/5273697)
|
||||
examples/test_gpt.py::test_starcoder_fp8_quantization_2gpu[starcoder] SKIP (https://nvbugs/5144931)
|
||||
examples/test_gpt.py::test_starcoder_fp8_quantization_2gpu[starcoderplus] SKIP (https://nvbugs/5144931)
|
||||
unittest/_torch -k "not (modeling or multi_gpu or auto_deploy)" SKIP (https://nvbugs/5271015)
|
||||
|
||||
@ -123,6 +123,7 @@ from utils.llm_data import llm_models_root
|
||||
],
|
||||
)
|
||||
def test_build_ad(world_size: Optional[int], config: Dict):
|
||||
pytest.skip("https://nvbugs/5271004")
|
||||
simple_config = SimpleConfig(**config)
|
||||
simple_config.world_size = world_size
|
||||
main(simple_config)
|
||||
|
||||
@ -57,6 +57,7 @@ from utils.llm_data import llm_models_root
|
||||
],
|
||||
)
|
||||
def test_build_ad(world_size: Optional[int], config: Dict):
|
||||
pytest.skip("https://nvbugs/5271004")
|
||||
simple_config = SimpleConfig(**config)
|
||||
simple_config.world_size = world_size
|
||||
main(simple_config)
|
||||
|
||||
@ -71,6 +71,7 @@ from utils.llm_data import llm_models_root
|
||||
],
|
||||
)
|
||||
def test_build_ad(world_size: Optional[int], config: Dict):
|
||||
pytest.skip("https://nvbugs/5271004")
|
||||
simple_config = SimpleConfig(**config)
|
||||
simple_config.world_size = world_size
|
||||
main(simple_config)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user