kanshan/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-10 13:10:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

server : respect min-step when splitting prompt batches (#25420)

Browse Source
This commit is contained in:
Aldehir Rojas
2026-07-09 01:23:30 -05:00
committed by GitHub
parent f2d1c2f398
commit 64c8b7db72
1 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+8 -3
View File
@@ -3435,9 +3435,14 @@ private:
slot.n_prompt_tokens_processed++;
// stop the prompt batch exactly before a user message
if (spans.is_user_start(slot.prompt.n_tokens())) {
break;
// break at the last user message, or at user messages at least min step past the last checkpoint
if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) {
const auto pos = slot.prompt.n_tokens();
const auto & checkpoints = slot.prompt.checkpoints;
if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) {
break;
}
}
// process the last few tokens of the prompt separately in order to allow for a checkpoint to be created.