/* * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/executor/types.h" namespace tensorrt_llm::executor { SamplingConfig::SamplingConfig(SizeType32 beamWidth, std::optional const& topK, std::optional const& topP, std::optional const& topPMin, std::optional const& topPResetIds, std::optional const& topPDecay, std::optional const& seed, std::optional const& temperature, std::optional const& minTokens, std::optional const& beamSearchDiversityRate, std::optional const& repetitionPenalty, std::optional const& presencePenalty, std::optional const& frequencyPenalty, std::optional const& lengthPenalty, std::optional const& earlyStopping, std::optional const& noRepeatNgramSize, std::optional const& numReturnSequences, std::optional const& minP) : mBeamWidth(checkBeamWidth(beamWidth)) , mTopK(checkTopK(topK)) , mTopP(checkTopP(topP)) , mTopPMin(checkTopPMin(topPMin)) , mTopPResetIds(checkTopPResetIds(topPResetIds)) , mTopPDecay(checkTopPDecay(topPDecay)) , mSeed(seed) , mTemperature(checkTemperature(temperature)) , mMinTokens(checkMinTokens(minTokens)) , mBeamSearchDiversityRate(checkBeamSearchDiversityRate(beamSearchDiversityRate)) , mRepetitionPenalty(checkRepetitionPenalty(repetitionPenalty)) , mPresencePenalty(presencePenalty) , mFrequencyPenalty(frequencyPenalty) , mLengthPenalty(lengthPenalty) , mEarlyStopping(earlyStopping) , mNoRepeatNgramSize(checkNoRepeatNgramSize(noRepeatNgramSize)) , mNumReturnSequences(checkNumReturnSequences(numReturnSequences, beamWidth)) , mMinP(checkMinP(minP)) { updateNumReturnBeams(); } bool SamplingConfig::operator==(SamplingConfig const& other) const { return mBeamWidth == other.mBeamWidth && mTopK == other.mTopK && mTopP == other.mTopP && mTopPMin == other.mTopPMin && mTopPResetIds == other.mTopPResetIds && mTopPDecay == other.mTopPDecay && mSeed == other.mSeed && mTemperature == other.mTemperature && mMinTokens == other.mMinTokens && mBeamSearchDiversityRate == other.mBeamSearchDiversityRate && mRepetitionPenalty == other.mRepetitionPenalty && mPresencePenalty == other.mPresencePenalty && mFrequencyPenalty == other.mFrequencyPenalty && mLengthPenalty == other.mLengthPenalty && mEarlyStopping == other.mEarlyStopping && mNoRepeatNgramSize == other.mNoRepeatNgramSize && mNumReturnSequences == other.mNumReturnSequences && mMinP == other.mMinP; } SizeType32 SamplingConfig::getBeamWidth() const { return mBeamWidth; } SizeType32 SamplingConfig::getNumReturnBeams() const { return mNumReturnBeams; } std::optional SamplingConfig::getTopK() const { return mTopK; } std::optional SamplingConfig::getTopP() const { return mTopP; } std::optional SamplingConfig::getTopPMin() const { return mTopPMin; } std::optional SamplingConfig::getTopPResetIds() const { return mTopPResetIds; } std::optional SamplingConfig::getTopPDecay() const { return mTopPDecay; } std::optional SamplingConfig::getSeed() const { return mSeed; } std::optional SamplingConfig::getRandomSeed() const { TLLM_LOG_WARNING("getRandomSeed is being deprecated; please use getSeed instead."); return mSeed; } std::optional SamplingConfig::getTemperature() const { return mTemperature; } std::optional SamplingConfig::getMinTokens() const { return mMinTokens; } std::optional SamplingConfig::getMinLength() const { TLLM_LOG_WARNING("getMinLength is being deprecated; please use getMinTokens instead."); return mMinTokens; } std::optional SamplingConfig::getBeamSearchDiversityRate() const { return mBeamSearchDiversityRate; } std::optional SamplingConfig::getRepetitionPenalty() const { return mRepetitionPenalty; } std::optional SamplingConfig::getPresencePenalty() const { return mPresencePenalty; } std::optional SamplingConfig::getFrequencyPenalty() const { return mFrequencyPenalty; } std::optional SamplingConfig::getLengthPenalty() const { return mLengthPenalty; } std::optional SamplingConfig::getEarlyStopping() const { return mEarlyStopping; } std::optional SamplingConfig::getNoRepeatNgramSize() const { return mNoRepeatNgramSize; } std::optional SamplingConfig::getNumReturnSequences() const { return mNumReturnSequences; } std::optional SamplingConfig::getMinP() const { return mMinP; } // the setters void SamplingConfig::setBeamWidth(SizeType32 beamWidth) { mBeamWidth = checkBeamWidth(beamWidth); updateNumReturnBeams(); } void SamplingConfig::setTopK(std::optional const& topK) { mTopK = checkTopK(topK); } void SamplingConfig::setTopP(std::optional const& topP) { mTopP = checkTopP(topP); } void SamplingConfig::setTopPMin(std::optional const& topPMin) { mTopPMin = checkTopPMin(topPMin); } void SamplingConfig::setTopPResetIds(std::optional const& topPResetIds) { mTopPResetIds = checkTopPResetIds(topPResetIds); } void SamplingConfig::setTopPDecay(std::optional const& topPDecay) { mTopPDecay = checkTopPDecay(topPDecay); } void SamplingConfig::setSeed(std::optional const& seed) { mSeed = seed; } void SamplingConfig::setRandomSeed(std::optional const& randomSeed) { TLLM_LOG_WARNING("setRandomSeed is being deprecated; please use setSeed instead."); mSeed = randomSeed; } void SamplingConfig::setTemperature(std::optional const& temperature) { mTemperature = checkTemperature(temperature); } void SamplingConfig::setMinTokens(std::optional const& minTokens) { mMinTokens = checkMinTokens(minTokens); } void SamplingConfig::setMinLength(std::optional const& minLength) { TLLM_LOG_WARNING("setMinLength is being deprecated; please use setMinTokens instead."); mMinTokens = checkMinTokens(minLength); } void SamplingConfig::setBeamSearchDiversityRate(std::optional const& beamSearchDiversityRate) { mBeamSearchDiversityRate = checkBeamSearchDiversityRate(beamSearchDiversityRate); } void SamplingConfig::setRepetitionPenalty(std::optional const& repetitionPenalty) { mRepetitionPenalty = checkRepetitionPenalty(repetitionPenalty); } void SamplingConfig::setPresencePenalty(std::optional const& presencePenalty) { mPresencePenalty = presencePenalty; } void SamplingConfig::setFrequencyPenalty(std::optional const& frequencyPenalty) { mFrequencyPenalty = frequencyPenalty; } void SamplingConfig::setLengthPenalty(std::optional const& lengthPenalty) { mLengthPenalty = lengthPenalty; } void SamplingConfig::setEarlyStopping(std::optional const& earlyStopping) { mEarlyStopping = earlyStopping; } void SamplingConfig::setNoRepeatNgramSize(std::optional const& noRepeatNgramSize) { mNoRepeatNgramSize = checkNoRepeatNgramSize(noRepeatNgramSize); } void SamplingConfig::setNumReturnSequences(std::optional const& numReturnSequences) { mNumReturnSequences = checkNumReturnSequences(numReturnSequences, mBeamWidth); updateNumReturnBeams(); } void SamplingConfig::setMinP(std::optional const& minP) { mMinP = checkMinP(minP); } SizeType32 SamplingConfig::checkBeamWidth(SizeType32 beamWidth) { TLLM_CHECK(beamWidth > 0); return beamWidth; } std::optional const& SamplingConfig::checkTopK(std::optional const& topK) { if (topK.has_value()) { TLLM_CHECK(topK.value() >= 0); } return topK; } std::optional const& SamplingConfig::checkTopP(std::optional const& topP) { if (topP.has_value()) { TLLM_CHECK(topP.value() > 0.f); TLLM_CHECK(topP.value() <= 1.f); } return topP; } std::optional const& SamplingConfig::checkTopPMin(std::optional const& topPMin) { if (topPMin.has_value()) { TLLM_CHECK(topPMin.value() >= 0.f); TLLM_CHECK(topPMin.value() <= 1.f); } return topPMin; } std::optional const& SamplingConfig::checkTopPResetIds(std::optional const& topPResetIds) { if (topPResetIds.has_value()) { TLLM_CHECK(topPResetIds.value() >= 0); } return topPResetIds; } std::optional const& SamplingConfig::checkTopPDecay(std::optional const& topPDecay) { if (topPDecay.has_value()) { TLLM_CHECK(topPDecay.value() > 0.f); TLLM_CHECK(topPDecay.value() <= 1.f); } return topPDecay; } std::optional const& SamplingConfig::checkTemperature(std::optional const& temperature) { if (temperature.has_value()) { TLLM_CHECK(temperature.value() >= 0.f); } return temperature; } std::optional const& SamplingConfig::checkMinTokens(std::optional const& minTokens) { if (minTokens.has_value()) { TLLM_CHECK(minTokens.value() >= 0); } return minTokens; } std::optional const& SamplingConfig::checkRepetitionPenalty(std::optional const& penalty) { if (penalty.has_value()) { TLLM_CHECK_WITH_INFO(penalty.value() > 0.F, "Repetition penalty should be strictly greater than zero. Provided value was %f", penalty.value()); } return penalty; } std::optional const& SamplingConfig::checkNoRepeatNgramSize( std::optional const& noRepeatNgramSize) { if (noRepeatNgramSize.has_value()) { TLLM_CHECK(noRepeatNgramSize.value() > 0); } return noRepeatNgramSize; } std::optional const& SamplingConfig::checkBeamSearchDiversityRate( std::optional const& beamSearchDiversityRate) { if (beamSearchDiversityRate.has_value()) { TLLM_CHECK(beamSearchDiversityRate.value() >= 0.f); } return beamSearchDiversityRate; } std::optional const& SamplingConfig::checkNumReturnSequences( std::optional const& numReturnSequences, SizeType32 beamWidth) { if (numReturnSequences.has_value()) { TLLM_CHECK(numReturnSequences.value() > 0); TLLM_CHECK(beamWidth == 1 || numReturnSequences.value() <= beamWidth); } return numReturnSequences; } std::optional const& SamplingConfig::checkMinP(std::optional const& minP) { if (minP.has_value()) { TLLM_CHECK(minP.value() >= 0.f && minP.value() <= 1.0f); } return minP; } void SamplingConfig::updateNumReturnBeams() { mNumReturnBeams = (mNumReturnSequences && mBeamWidth > 1) ? std::min(mNumReturnSequences.value(), mBeamWidth) : mBeamWidth; } } // namespace tensorrt_llm::executor