A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 03:00 UTC
[Bugfix] Add early validation to reject incompatible runner types for embedding models (#43079)
Signed-off-by: anish <anishesg@users.noreply.github.com> Signed-off-by: Your Name <ak8686@princeton.edu> Signed-off-by: anish <145943060+anishesg@users.noreply.github.com> Co-authored-by: anish <anishesg@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
anish committed
May 21, 2026 at 11:07 UTC
1c78f76c29a642379ad0ec953a77af9bc44376b6
1 file changed
+9
vllm/config/model.py
+9
@@ -584,6 +584,15 @@ class ModelConfig:
584
architectures, self.runner_type, self.convert
585
)
586
587
+ if (
588
+ is_pooling_model
589
+ and not is_generative_model
590
+ and self.runner_type in ("draft", "generate")
591
+ ):
592
+ raise ValueError(
593
+ f"Embedding models do not support `--runner {self.runner_type}`. "
594
+ "Use `--runner pooling` or `--runner auto` for embedding models."
595
+ )
596
if self.runner_type == "generate" and not is_generative_model:
597
generate_converts = _RUNNER_CONVERTS["generate"]
598
if self.convert_type not in generate_converts: