A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 19:00 UTC
[Misc] Update code for encoder-decoder models (#33900)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 6, 2026 at 11:38 UTC
035a6cb09a3f7076d2e79db1af3070325230bb59
2 files changed
+9
-3
vllm/multimodal/inputs.py
+1
-1
@@ -1098,7 +1098,7 @@ class MultiModalEncDecInputs(MultiModalInputs):
1098
1099
Note: Even text-only encoder-decoder models are currently implemented
1100
as multi-modal models for convenience.
1101
- (Example: https://github.com/neuralmagic/bart-plugin)
1101
+ (Example: https://github.com/vllm-project/bart-plugin)
1102
"""
1103
1104
encoder_prompt_token_ids: list[int]
vllm/v1/core/sched/scheduler.py
+8
-2
@@ -185,7 +185,13 @@ class Scheduler(SchedulerInterface):
185
186
# NOTE: Text-only encoder-decoder models are implemented as
187
# multi-modal models for convenience
188
- # Example: https://github.com/neuralmagic/bart-plugin
188
+ # Example: https://github.com/vllm-project/bart-plugin
189
+ if self.is_encoder_decoder:
190
+ assert mm_budget and len(mm_budget.mm_max_toks_per_item) <= 1, (
191
+ "Encoder-decoder models are expected to implement the "
192
+ "multimodal interface with at most one modality."
193
+ )
194
+
195
self.max_num_encoder_input_tokens = (
196
mm_budget.encoder_compute_budget if mm_budget else 0
197
)
@@ -200,7 +206,7 @@ class Scheduler(SchedulerInterface):
206
# TODO (NickLucche): Generalize to models with variable-length encoder inputs.
207
self._num_encoder_max_input_tokens = (
208
mm_budget.mm_max_toks_per_item[mm_budget.get_modality_with_max_tokens()]
203
- if mm_budget
209
+ if mm_budget and mm_budget.mm_max_toks_per_item
210
else 0
211
)
212