From 0a0dcf2e493fa0e4696cd30f01645e4bb14d2c90 Mon Sep 17 00:00:00 2001 From: David Addison Date: Mon, 21 Apr 2025 09:53:12 -0700 Subject: [PATCH] Make nccl-tests verifiable a DSO and add NAME_SUFFIX support --- README.md | 18 ++++++-- src/Makefile | 93 +++++++++++----------------------------- src/common.cu | 4 +- src/common.mk | 69 +++++++++++++++++++++++++++++ verifiable/Makefile | 17 +++++--- verifiable/main.cu | 14 ++++++ verifiable/verifiable.h | 4 ++ verifiable/verifiable.mk | 15 +++++-- 8 files changed, 150 insertions(+), 84 deletions(-) create mode 100644 src/common.mk create mode 100644 verifiable/main.cu diff --git a/README.md b/README.md index 957f6af..deb6ddc 100644 --- a/README.md +++ b/README.md @@ -4,33 +4,43 @@ These tests check both the performance and the correctness of [NCCL](http://gith ## Build -To build the tests, just type `make`. +To build the tests, just type `make` or `make -j` -If CUDA is not installed in /usr/local/cuda, you may specify CUDA\_HOME. Similarly, if NCCL is not installed in /usr, you may specify NCCL\_HOME. +If CUDA is not installed in `/usr/local/cuda`, you may specify `CUDA_HOME`. Similarly, if NCCL is not installed in `/usr`, you may specify `NCCL_HOME`. ```shell $ make CUDA_HOME=/path/to/cuda NCCL_HOME=/path/to/nccl ``` -NCCL tests rely on MPI to work on multiple processes, hence multiple nodes. If you want to compile the tests with MPI support, you need to set MPI=1 and set MPI\_HOME to the path where MPI is installed. +NCCL tests rely on MPI to work on multiple processes, hence multiple nodes. If you want to compile the tests with MPI support, you need to set `MPI=1` and set `MPI_HOME` to the path where MPI is installed. ```shell $ make MPI=1 MPI_HOME=/path/to/mpi CUDA_HOME=/path/to/cuda NCCL_HOME=/path/to/nccl ``` +You can also add a suffix to the name of the generated binaries with `NAME_SUFFIX`. For example when compiling with the MPI versions you could use: + +```shell +$ make MPI=1 NAME_SUFFIX=_mpi MPI_HOME=/path/to/mpi CUDA_HOME=/path/to/cuda NCCL_HOME=/path/to/nccl +``` + +This will generate test binaries with names such as `all_reduce_perf_mpi`. + ## Usage -NCCL tests can run on multiple processes, multiple threads, and multiple CUDA devices per thread. The number of process is managed by MPI and is therefore not passed to the tests as argument. The total number of ranks (=CUDA devices) will be equal to (number of processes)\*(number of threads)\*(number of GPUs per thread). +NCCL tests can run on multiple processes, multiple threads, and multiple CUDA devices per thread. The number of process is managed by MPI and is therefore not passed to the tests as argument. The total number of ranks (=CUDA devices) will be equal to `(number of processes)*(number of threads)*(number of GPUs per thread)`. ### Quick examples Run on single node with 8 GPUs (`-g 8`), scanning from 8 Bytes to 128MBytes : + ```shell $ ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 8 ``` Run 64 MPI processes on nodes with 8 GPUs each, for a total of 64 GPUs spread across 8 nodes : (NB: The nccl-tests binaries must be compiled with `MPI=1` for this case) + ```shell $ mpirun -np 64 -N 8 ./build/all_reduce_perf -b 8 -e 8G -f 2 -g 1 ``` diff --git a/src/Makefile b/src/Makefile index 5737092..612395f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,73 +1,13 @@ # -# Copyright (c) 2015-2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2015-2025, NVIDIA CORPORATION. All rights reserved. # # See LICENSE.txt for license information # +include common.mk -CUDA_HOME ?= /usr/local/cuda -PREFIX ?= /usr/local -VERBOSE ?= 0 -DEBUG ?= 0 - -CUDA_LIB ?= $(CUDA_HOME)/lib64 -CUDA_INC ?= $(CUDA_HOME)/include -NVCC ?= $(CUDA_HOME)/bin/nvcc -CUDARTLIB ?= cudart - -CUDA_VERSION = $(strip $(shell which $(NVCC) >/dev/null && $(NVCC) --version | grep release | sed 's/.*release //' | sed 's/\,.*//')) -CUDA_MAJOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 1) -CUDA_MINOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 2) - -# Better define NVCC_GENCODE in your environment to the minimal set -# of archs to reduce compile time. -ifeq ($(shell test "0$(CUDA_MAJOR)" -eq 12 -a "0$(CUDA_MINOR)" -ge 8 -o "0$(CUDA_MAJOR)" -ge 13; echo $$?),0) -# Include Blackwell support if we're using CUDA12.8 or above -NVCC_GENCODE ?= -gencode=arch=compute_80,code=sm_80 \ - -gencode=arch=compute_90,code=sm_90 \ - -gencode=arch=compute_100,code=sm_100 \ - -gencode=arch=compute_120,code=sm_120 \ - -gencode=arch=compute_120,code=compute_120 -else ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 12; echo $$?),0) -NVCC_GENCODE ?= -gencode=arch=compute_60,code=sm_60 \ - -gencode=arch=compute_61,code=sm_61 \ - -gencode=arch=compute_70,code=sm_70 \ - -gencode=arch=compute_80,code=sm_80 \ - -gencode=arch=compute_90,code=sm_90 \ - -gencode=arch=compute_90,code=compute_90 -else ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 11; echo $$?),0) -NVCC_GENCODE ?= -gencode=arch=compute_60,code=sm_60 \ - -gencode=arch=compute_61,code=sm_61 \ - -gencode=arch=compute_70,code=sm_70 \ - -gencode=arch=compute_80,code=sm_80 \ - -gencode=arch=compute_80,code=compute_80 -else -NVCC_GENCODE ?= -gencode=arch=compute_35,code=sm_35 \ - -gencode=arch=compute_50,code=sm_50 \ - -gencode=arch=compute_60,code=sm_60 \ - -gencode=arch=compute_61,code=sm_61 \ - -gencode=arch=compute_70,code=sm_70 \ - -gencode=arch=compute_70,code=compute_70 -endif - -NVCUFLAGS := -ccbin $(CXX) $(NVCC_GENCODE) -std=c++11 -CXXFLAGS := -std=c++11 - -LDFLAGS := -L${CUDA_LIB} -lcudart -lrt -NVLDFLAGS := -L${CUDA_LIB} -l${CUDARTLIB} -lrt - -ifeq ($(DEBUG), 0) -NVCUFLAGS += -O3 -g -CXXFLAGS += -O3 -g -else -NVCUFLAGS += -O0 -G -g -CXXFLAGS += -O0 -g -ggdb3 -endif - -ifneq ($(VERBOSE), 0) -NVCUFLAGS += -Xcompiler -Wall,-Wextra,-Wno-unused-parameter -else -.SILENT: -endif +MPI ?= 0 # Set to 1 to enable MPI support (multi-process/multi-node) +NAME_SUFFIX ?= # e.g. _mpi when using MPI=1 +DSO ?= 0 # Set to 1 to create and use libverifiable.so to reduce binary size .PHONY: build clean @@ -92,7 +32,7 @@ DST_DIR := $(BUILDDIR) SRC_FILES := $(wildcard *.cu) OBJ_FILES := $(SRC_FILES:%.cu=${DST_DIR}/%.o) BIN_FILES_LIST := all_reduce all_gather broadcast reduce_scatter reduce alltoall scatter gather sendrecv hypercube -BIN_FILES := $(BIN_FILES_LIST:%=${DST_DIR}/%_perf) +BIN_FILES := $(BIN_FILES_LIST:%=${DST_DIR}/%_perf${NAME_SUFFIX}) build: ${BIN_FILES} @@ -103,18 +43,35 @@ TEST_VERIFIABLE_SRCDIR := ../verifiable TEST_VERIFIABLE_BUILDDIR := $(BUILDDIR)/verifiable include ../verifiable/verifiable.mk +.PRECIOUS: ${DST_DIR}/%.o + ${DST_DIR}/%.o: %.cu common.h $(TEST_VERIFIABLE_HDRS) @printf "Compiling %-35s > %s\n" $< $@ @mkdir -p ${DST_DIR} $(NVCC) -o $@ $(NVCUFLAGS) -c $< +${DST_DIR}/%$(NAME_SUFFIX).o: %.cu common.h $(TEST_VERIFIABLE_HDRS) + @printf "Compiling %-35s > %s\n" $< $@ + @mkdir -p ${DST_DIR} + $(NVCC) -o $@ $(NVCUFLAGS) -c $< + ${DST_DIR}/timer.o: timer.cc timer.h @printf "Compiling %-35s > %s\n" $< $@ @mkdir -p ${DST_DIR} - $(CXX) $(CXXFLAGS) -o $@ -c timer.cc + $(CXX) $(CXXFLAGS) -o $@ -c $< -${DST_DIR}/%_perf:${DST_DIR}/%.o ${DST_DIR}/common.o ${DST_DIR}/timer.o $(TEST_VERIFIABLE_OBJS) +ifeq ($(DSO), 1) +${DST_DIR}/%_perf$(NAME_SUFFIX): ${DST_DIR}/%.o ${DST_DIR}/common$(NAME_SUFFIX).o ${DST_DIR}/timer.o $(TEST_VERIFIABLE_LIBS) + @printf "Linking %-35s > %s\n" $< $@ + @mkdir -p ${DST_DIR} + $(NVCC) -o $@ $(NVCUFLAGS) $^ -L$(TEST_VERIFIABLE_BUILDDIR) -lverifiable ${NVLDFLAGS} -Xlinker "--enable-new-dtags" -Xlinker "-rpath,\$$ORIGIN:\$$ORIGIN/verifiable" +else +${DST_DIR}/%_perf$(NAME_SUFFIX):${DST_DIR}/%.o ${DST_DIR}/common$(NAME_SUFFIX).o ${DST_DIR}/timer.o $(TEST_VERIFIABLE_OBJS) @printf "Linking %-35s > %s\n" $< $@ @mkdir -p ${DST_DIR} $(NVCC) -o $@ $(NVCUFLAGS) $^ ${NVLDFLAGS} +endif + +clean_intermediates: + rm -f ${DST_DIR}/*.o $(TEST_VERIFIABLE_OBJS) diff --git a/src/common.cu b/src/common.cu index def0586..df4c366 100644 --- a/src/common.cu +++ b/src/common.cu @@ -1260,8 +1260,8 @@ testResult_t run() { int rank = proc*nThreads*nGpus+i; cudaDeviceProp prop; CUDACHECK(cudaGetDeviceProperties(&prop, cudaDev)); - len += snprintf(line+len, MAX_LINE-len, "# Rank %2d Group %2d Pid %6d on %10s device %2d [0x%02x] %s\n", - rank, color, getpid(), hostname, cudaDev, prop.pciBusID, prop.name); + len += snprintf(line+len, MAX_LINE-len, "# Rank %2d Group %2d Pid %6d on %10s device %2d [%04x:%02x:%02x] %s\n", + rank, color, getpid(), hostname, cudaDev, prop.pciDomainID, prop.pciBusID, prop.pciDeviceID, prop.name); maxMem = std::min(maxMem, prop.totalGlobalMem); } diff --git a/src/common.mk b/src/common.mk new file mode 100644 index 0000000..2bc7e35 --- /dev/null +++ b/src/common.mk @@ -0,0 +1,69 @@ +# +# Copyright (c) 2015-2025, NVIDIA CORPORATION. All rights reserved. +# +# See LICENSE.txt for license information +# +CUDA_HOME ?= /usr/local/cuda +PREFIX ?= /usr/local +VERBOSE ?= 0 +DEBUG ?= 0 + +CUDA_LIB ?= $(CUDA_HOME)/lib64 +CUDA_INC ?= $(CUDA_HOME)/include +NVCC ?= $(CUDA_HOME)/bin/nvcc +CUDARTLIB ?= cudart + +CUDA_VERSION = $(strip $(shell which $(NVCC) >/dev/null && $(NVCC) --version | grep release | sed 's/.*release //' | sed 's/\,.*//')) +CUDA_MAJOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 1) +CUDA_MINOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 2) + +# Better define NVCC_GENCODE in your environment to the minimal set +# of archs to reduce compile time. +ifeq ($(shell test "0$(CUDA_MAJOR)" -eq 12 -a "0$(CUDA_MINOR)" -ge 8 -o "0$(CUDA_MAJOR)" -ge 13; echo $$?),0) +# Include Blackwell support if we're using CUDA12.8 or above +NVCC_GENCODE ?= -gencode=arch=compute_80,code=sm_80 \ + -gencode=arch=compute_90,code=sm_90 \ + -gencode=arch=compute_100,code=sm_100 \ + -gencode=arch=compute_120,code=sm_120 \ + -gencode=arch=compute_120,code=compute_120 +else ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 12; echo $$?),0) +NVCC_GENCODE ?= -gencode=arch=compute_60,code=sm_60 \ + -gencode=arch=compute_61,code=sm_61 \ + -gencode=arch=compute_70,code=sm_70 \ + -gencode=arch=compute_80,code=sm_80 \ + -gencode=arch=compute_90,code=sm_90 \ + -gencode=arch=compute_90,code=compute_90 +else ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 11; echo $$?),0) +NVCC_GENCODE ?= -gencode=arch=compute_60,code=sm_60 \ + -gencode=arch=compute_61,code=sm_61 \ + -gencode=arch=compute_70,code=sm_70 \ + -gencode=arch=compute_80,code=sm_80 \ + -gencode=arch=compute_80,code=compute_80 +else +NVCC_GENCODE ?= -gencode=arch=compute_35,code=sm_35 \ + -gencode=arch=compute_50,code=sm_50 \ + -gencode=arch=compute_60,code=sm_60 \ + -gencode=arch=compute_61,code=sm_61 \ + -gencode=arch=compute_70,code=sm_70 \ + -gencode=arch=compute_70,code=compute_70 +endif + +NVCUFLAGS := -ccbin $(CXX) $(NVCC_GENCODE) -std=c++11 +CXXFLAGS := -std=c++11 + +LDFLAGS := -L${CUDA_LIB} -lcudart -lrt +NVLDFLAGS := -L${CUDA_LIB} -l${CUDARTLIB} -lrt + +ifeq ($(DEBUG), 0) +NVCUFLAGS += -O3 -g +CXXFLAGS += -O3 -g +else +NVCUFLAGS += -O0 -G -g +CXXFLAGS += -O0 -g -ggdb3 +endif + +ifneq ($(VERBOSE), 0) +NVCUFLAGS += -Xcompiler -Wall,-Wextra,-Wno-unused-parameter +else +.SILENT: +endif diff --git a/verifiable/Makefile b/verifiable/Makefile index b141a2a..bb90001 100644 --- a/verifiable/Makefile +++ b/verifiable/Makefile @@ -1,13 +1,18 @@ -include ../../makefiles/common.mk +# +# Copyright (c) 2015-2025, NVIDIA CORPORATION. All rights reserved. +# +# See LICENSE.txt for license information +# +include ../src/common.mk .PHONY: all clean -BUILDDIR := $(abspath ../../build) +BUILDDIR := $(abspath ../build) NCCLDIR := $(BUILDDIR) NVCUFLAGS += -I$(NCCLDIR)/include/ -I../include -DST_DIR := $(BUILDDIR)/test/verifiable +DST_DIR := $(BUILDDIR)/verifiable -all: $(DST_DIR)/self_test $(DST_DIR)/verifiable.o +all: $(DST_DIR)/self_test clean: rm -rf $(DST_DIR) @@ -18,7 +23,7 @@ include verifiable.mk self_test: $(DST_DIR)/self_test -$(DST_DIR)/self_test: verifiable.cu verifiable.h +$(DST_DIR)/self_test: main.cu $(TEST_VERIFIABLE_LIBS) @printf "Linking %s\n" $@ @mkdir -p $(DST_DIR) - $(NVCC) -o $@ $(NVCUFLAGS) -DSELF_TEST=1 verifiable.cu $(NVLDFLAGS) + $(NVCC) -o $@ $(NVCUFLAGS) $< -L$(TEST_VERIFIABLE_BUILDDIR) -lverifiable $(NVLDFLAGS) -Xlinker "-rpath=\$$ORIGIN" diff --git a/verifiable/main.cu b/verifiable/main.cu new file mode 100644 index 0000000..4e4aef6 --- /dev/null +++ b/verifiable/main.cu @@ -0,0 +1,14 @@ + +#include +#include + +#define NCCL_VERIFIABLE_SELF_TEST 1 +#include "verifiable.h" + +int main(int arg_n, char **args) { + std::cerr<<"You are hoping to see no output beyond this line."< -# TEST_VERIFIABLE_BUILDDIR = +# TEST_VERIFIABLE_BUILDDIR = TEST_VERIFIABLE_HDRS = $(TEST_VERIFIABLE_SRCDIR)/verifiable.h TEST_VERIFIABLE_OBJS = $(TEST_VERIFIABLE_BUILDDIR)/verifiable.o +TEST_VERIFIABLE_LIBS = $(TEST_VERIFIABLE_BUILDDIR)/libverifiable.so -$(TEST_VERIFIABLE_BUILDDIR)/verifiable.o: $(TEST_VERIFIABLE_SRCDIR)/verifiable.cu $(TEST_VERIFY_REDUCE_HDRS) +$(TEST_VERIFIABLE_BUILDDIR)/verifiable.o: $(TEST_VERIFIABLE_SRCDIR)/verifiable.cu $(TEST_VERIFIABLE_HDRS) @printf "Compiling %s\n" $@ @mkdir -p $(TEST_VERIFIABLE_BUILDDIR) - $(NVCC) -o $@ $(NVCUFLAGS) -c $(TEST_VERIFIABLE_SRCDIR)/verifiable.cu + $(NVCC) -Xcompiler "-fPIC" -o $@ $(NVCUFLAGS) -c $(TEST_VERIFIABLE_SRCDIR)/verifiable.cu + +$(TEST_VERIFIABLE_BUILDDIR)/libverifiable.so: $(TEST_VERIFIABLE_OBJS) + @printf "Creating DSO %s\n" $@ + @mkdir -p $(TEST_VERIFIABLE_BUILDDIR) + $(CC) -shared -o $@.0 $^ -Wl,-soname,$(notdir $@).0 + ln -sf $(notdir $@).0 $@