A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Bugfix] Fix EAGLE vocab embedding for multimodal target model (#19570)
Signed-off-by: qizixi <qizixi@meta.com>
qizixi committed
Jun 12, 2025 at 20:09 UTC
c68698b3264cbb05011a020ae654d38d155d9dcd
1 file changed
+12
-7
vllm/v1/spec_decode/eagle.py
+12
-7
@@ -329,16 +329,24 @@ class EagleProposer:
329
330
self.attn_layer_names = list(draft_attn_layer_names)
331
332
+ if supports_multimodal(target_model):
333
+ # handle multimodality
334
+ self.model.config.image_token_index = (
335
+ target_model.config.image_token_index)
336
+ target_language_model = target_model.get_language_model()
337
+ else:
338
+ target_language_model = target_model
339
# share embed_tokens with the target model if needed
340
if get_pp_group().world_size == 1 \
341
and self.model.model.embed_tokens.weight.shape \
335
- == target_model.model.embed_tokens.weight.shape:
342
+ == target_language_model.model.embed_tokens.weight.shape:
343
logger.info(
344
"Assuming the EAGLE head shares the same vocab embedding" \
345
" with the target model."
346
)
347
del self.model.model.embed_tokens
341
- self.model.model.embed_tokens = target_model.model.embed_tokens
348
+ self.model.model.embed_tokens = (
349
+ target_language_model.model.embed_tokens)
350
else:
351
logger.info(
352
"The EAGLE head's vocab embedding will be loaded separately" \
@@ -349,12 +357,9 @@ class EagleProposer:
357
# some model definition do not define lm_head explicitly
358
# and reuse embed_tokens for lm_head, e.g., CohereForCausalLM
359
if self.vllm_config.speculative_config.method != "eagle3" and \
352
- hasattr(target_model, "lm_head"):
360
+ hasattr(target_language_model, "lm_head"):
361
logger.info("Loading EAGLE LM head weights from the target model.")
354
- if supports_multimodal(target_model):
355
- self.model.lm_head = target_model.get_language_model().lm_head
356
- else:
357
- self.model.lm_head = target_model.lm_head
362
+ self.model.lm_head = target_language_model.lm_head
363
364
@torch.inference_mode()
365
def dummy_run(