/* * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "envUtils.h" #include "tensorrt_llm/common/logger.h" #include namespace tensorrt_llm::common { static std::optional getIntEnv(char const* name) { char const* const env = std::getenv(name); if (env == nullptr) { return std::nullopt; } int32_t const val = std::stoi(env); if (val <= 0) { return std::nullopt; } return {val}; }; // XQA kernels (optimized kernels for generation phase). bool forceXQAKernels() { static bool const forceXQA = (getIntEnv("TRTLLM_FORCE_XQA").value_or(0) != 0); return forceXQA; } int32_t xqaMaxNbCtaPerKVHeadFactor() { return envXqaNbCtaPerKVHead().value_or(8); } std::optional envXqaNbCtaPerKVHead() { static std::optional const ret = getIntEnv("TRTLLM_XQA_BLOCKS_PER_SEQUENCE"); return ret; } bool getEnvDisableXQAJIT() { static bool init = false; static bool disableXQAJIT = false; if (!init) { init = true; char const* disable_xqa_jit_var = std::getenv("TRTLLM_DISABLE_XQA_JIT"); if (disable_xqa_jit_var) { if (disable_xqa_jit_var[0] == '1' && disable_xqa_jit_var[1] == '\0') { disableXQAJIT = true; } } } return disableXQAJIT; } // Tune the number of blocks per sequence for accuracy/performance purpose. bool getEnvMmhaMultiblockDebug() { static bool init = false; static bool forceMmhaMaxSeqLenTile = false; if (!init) { init = true; char const* enable_mmha_debug_var = std::getenv("TRTLLM_ENABLE_MMHA_MULTI_BLOCK_DEBUG"); if (enable_mmha_debug_var) { if (enable_mmha_debug_var[0] == '1' && enable_mmha_debug_var[1] == '\0') { forceMmhaMaxSeqLenTile = true; } } } return forceMmhaMaxSeqLenTile; } int getEnvMmhaBlocksPerSequence() { static bool init = false; static int mmhaBlocksPerSequence = 0; if (!init) { init = true; char const* mmhaBlocksPerSequenceEnv = std::getenv("TRTLLM_MMHA_BLOCKS_PER_SEQUENCE"); if (mmhaBlocksPerSequenceEnv) { mmhaBlocksPerSequence = std::atoi(mmhaBlocksPerSequenceEnv); if (mmhaBlocksPerSequence <= 0) { TLLM_LOG_WARNING("Invalid value for TRTLLM_MMHA_BLOCKS_PER_SEQUENCE. Will use default values instead!"); } } } return mmhaBlocksPerSequence; } int getEnvMmhaKernelBlockSize() { static bool init = false; static int mmhaKernelBlockSize = 0; if (!init) { init = true; char const* mmhaKernelBlockSizeEnv = std::getenv("TRTLLM_MMHA_KERNEL_BLOCK_SIZE"); if (mmhaKernelBlockSizeEnv) { mmhaKernelBlockSize = std::atoi(mmhaKernelBlockSizeEnv); if (mmhaKernelBlockSize <= 0) { TLLM_LOG_WARNING("Invalid value for TRTLLM_MMHA_KERNEL_BLOCK_SIZE. Will use default values instead!"); } } } return mmhaKernelBlockSize; } } // namespace tensorrt_llm::common