[ModelRunnerV2] Various model/config compatibility fixes (#45868)

Signed-off-by: Nick Hill <[email protected]>

vllm/config/vllm.py

@@ -2026,6 +2026,14 @@ class VllmConfig:
         ):
             unsupported.append("sequence parallelism")
 
+        # V2 does not implement the external_launcher (torchrun) PP-output
+        # broadcast that V1 uses to keep all ranks in sync (broadcast_pp_output).
+        if (
+            self.parallel_config.distributed_executor_backend == "external_launcher"
+            and self.parallel_config.pipeline_parallel_size > 1
+        ):
+            unsupported.append("pipeline parallelism with external_launcher")
+
         if speculative_config is not None:
             # TODO: ngram / ngram_gpu are not supported by the v2 model runner yet
             if speculative_config.method in ("ngram", "ngram_gpu"):

vllm/v1/worker/gpu/mm/encoder_runner.py

@@ -124,7 +124,7 @@ class EncoderRunner:
                     mm_embeds_item = encoder_output[start_idx:end_idx]
 
                 req_start_pos = query_start_loc[i] + start_pos - query_start[i]
-                is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] = (
+                is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] |= (
                     True if is_embed is None else is_embed
                 )
                 mm_embeds.append(mm_embeds_item)

vllm/v1/worker/gpu/model_runner.py

@@ -1204,6 +1204,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
                 self.kv_cache_config,
             )
 
+        input_ids = input_batch.input_ids
         inputs_embeds = None
         if self.supports_mm_inputs and self.is_first_pp_rank:
             # Run MM encoder (if needed) and get multimodal embeddings.
@@ -1222,11 +1223,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
             inputs_embeds = self.model_state.get_mm_embeddings(
                 scheduler_output.scheduled_encoder_inputs, input_batch
             )
+            if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
+                input_ids = None
 
         model_inputs = {
-            "input_ids": input_batch.input_ids,
+            "input_ids": input_ids,
             "positions": input_batch.positions,
             "inputs_embeds": inputs_embeds,
+            "intermediate_tensors": None,
             # NOTE: Values returned by `prepare_inputs` will override the default
             # values above.
             **self.model_state.prepare_inputs(input_batch, self.req_states),

vllm/v1/worker/gpu/model_states/whisper.py

@@ -132,7 +132,9 @@ class WhisperModelState(ModelState):
             num_reqs = input_batch.num_reqs
             num_tokens = input_batch.num_tokens
         whisper_attn_metadata = WhisperAttnMetadata(
-            self._get_encoder_seq_lens(input_batch.req_ids, attn_groups, for_capture)
+            self._get_encoder_seq_lens(
+                input_batch.req_ids, attn_groups, for_capture, num_reqs
+            )
         )
 
         query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
@@ -166,8 +168,8 @@ class WhisperModelState(ModelState):
         req_ids: list[str],
         attn_groups: list[list[AttentionGroup]],
         for_capture: bool,
+        num_reqs: int,
     ) -> dict[int, tuple[torch.Tensor, np.ndarray]]:
-        num_reqs = len(req_ids)
         encoder_seq_lens_np = np.zeros(num_reqs, dtype=np.int32)
         if not for_capture:
             # During normal execution, use actual encoder lengths.