/* * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include "tensorrt_llm/batch_manager/common.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/promptTuningParams.h" #include "tensorrt_llm/runtime/worldConfig.h" namespace tensorrt_llm::batch_manager { class PromptTuningBuffers { public: using SizeType32 = tensorrt_llm::runtime::SizeType32; using ITensor = tensorrt_llm::runtime::ITensor; using TensorPtr = runtime::ITensor::SharedPtr; runtime::PromptTuningParams mPromptTuningParams; SizeType32 mMaxPromptVocabSize; PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); void validate(std::optional const& optReqPromptEmbeddingTable, std::optional const& optReqPromptVocabSize); void fill(RequestVector const& contextRequests, RequestVector const& genRequests, runtime::BufferManager const& manager, bool packed); }; } // namespace tensorrt_llm::batch_manager