@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7727ce35c2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 21:48 UTC

[Model] Add Eagle2.5-8B Vision-Language Model support (#32456)

Signed-off-by: kimheesu <wlskaka4@gmail.com>

Kim Hee Su committed Jan 21, 2026 at 18:39 UTC 7727ce35c26629dfdcc7aaa9f27077d110a314ae
5 files changed +518
docs/models/supported_models.md
+1
@@ -666,6 +666,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
666 | `Cohere2VisionForConditionalGeneration` | Command A Vision | T + I<sup>+</sup> | `CohereLabs/command-a-vision-07-2025`, etc. | | ✅︎ |
667 | `DeepseekVLV2ForCausalLM`<sup>^</sup> | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
668 | `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
669 +| `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ |
670 | `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ |
671 | `FuyuForCausalLM` | Fuyu | T + I | `adept/fuyu-8b`, etc. | | ✅︎ |
672 | `Gemma3ForConditionalGeneration` | Gemma 3 | T + I<sup>E+</sup> | `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc. | ✅︎ | ✅︎ |
examples/offline_inference/vision_language.py
+35
@@ -287,6 +287,40 @@ def run_dots_ocr(questions: list[str], modality: str) -> ModelRequestData:
287 )
288
289
290 +# Eagle2.5-VL
291 +def run_eagle2_5(questions: list[str], modality: str) -> ModelRequestData:
292 + assert modality == "image"
293 +
294 + model_name = "nvidia/Eagle2.5-8B"
295 +
296 + engine_args = EngineArgs(
297 + model=model_name,
298 + max_model_len=4096,
299 + max_num_seqs=2,
300 + trust_remote_code=True,
301 + limit_mm_per_prompt={modality: 1},
302 + )
303 +
304 + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
305 + messages = [
306 + [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
307 + ]
308 + prompts = tokenizer.apply_chat_template(
309 + messages, tokenize=False, add_generation_prompt=True
310 + )
311 +
312 + # Stop tokens for Eagle2.5 (Qwen2 based)
313 + stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
314 + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
315 + stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
316 +
317 + return ModelRequestData(
318 + engine_args=engine_args,
319 + prompts=prompts,
320 + stop_token_ids=stop_token_ids,
321 + )
322 +
323 +
324 # Ernie4.5-VL
325 def run_ernie45_vl(questions: list[str], modality: str) -> ModelRequestData:
326 model_name = "baidu/ERNIE-4.5-VL-28B-A3B-PT"
@@ -1919,6 +1953,7 @@ model_example_map = {
1953 "deepseek_vl_v2": run_deepseek_vl2,
1954 "deepseek_ocr": run_deepseek_ocr,
1955 "dots_ocr": run_dots_ocr,
1956 + "eagle2_5": run_eagle2_5,
1957 "ernie45_vl": run_ernie45_vl,
1958 "fuyu": run_fuyu,
1959 "gemma3": run_gemma3,
tests/models/registry.py
+3
@@ -679,6 +679,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
679 "DotsOCRForCausalLM": _HfExamplesInfo(
680 "rednote-hilab/dots.ocr", trust_remote_code=True
681 ),
682 + "Eagle2_5_VLForConditionalGeneration": _HfExamplesInfo(
683 + "nvidia/Eagle2.5-8B", trust_remote_code=True, is_available_online=False
684 + ),
685 "Emu3ForConditionalGeneration": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
686 "Ernie4_5_VLMoeForConditionalGeneration": _HfExamplesInfo(
687 "baidu/ERNIE-4.5-VL-28B-A3B-PT",
vllm/model_executor/models/eagle2_5_vl.py new
+475
@@ -0,0 +1,475 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# Adapted from NVIDIA Eagle2.5-VL model
4 +# https://huggingface.co/nvidia/Eagle2.5-8B
5 +
6 +from collections.abc import Iterable
7 +from typing import Annotated, Literal, TypeAlias
8 +
9 +import torch
10 +import torch.nn as nn
11 +from transformers import PretrainedConfig
12 +
13 +from vllm.config import VllmConfig
14 +from vllm.model_executor.layers.quantization import QuantizationConfig
15 +from vllm.model_executor.models.module_mapping import MultiModelKeys
16 +from vllm.model_executor.models.siglip import SiglipVisionModel
17 +from vllm.multimodal import MULTIMODAL_REGISTRY
18 +from vllm.multimodal.processing import PromptUpdateDetails
19 +from vllm.sequence import IntermediateTensors
20 +from vllm.tokenizers import TokenizerLike
21 +from vllm.utils.tensor_schema import TensorSchema, TensorShape
22 +
23 +from .interfaces import (
24 + MultiModalEmbeddings,
25 + SupportsLoRA,
26 + SupportsMultiModal,
27 + SupportsPP,
28 +)
29 +from .internvl import (
30 + IMG_CONTEXT,
31 + IMG_END,
32 + IMG_START,
33 + BaseInternVLDummyInputsBuilder,
34 + BaseInternVLMultiModalProcessor,
35 + BaseInternVLProcessingInfo,
36 + BaseInternVLProcessor,
37 +)
38 +from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
39 +
40 +
41 +class Eagle2_5_VLImagePixelInputs(TensorSchema):
42 + """
43 + Dimensions:
44 + - bn: Batch size * number of images
45 + - bnp: Batch size * number of images * (1 + num_patches)
46 + - c: Number of channels (3)
47 + - h: Height of each image patch
48 + - w: Width of each image patch
49 + """
50 +
51 + type: Literal["pixel_values"]
52 + pixel_values_flat: Annotated[torch.Tensor, TensorShape("bnp", 3, "h", "w")]
53 + num_patches: Annotated[torch.Tensor, TensorShape("bn")]
54 +
55 +
56 +class Eagle2_5_VLImageEmbeddingInputs(TensorSchema):
57 + """
58 + Dimensions:
59 + - n: Number of images
60 + - f: Total image feature size
61 + - h: Hidden size (must match the hidden size of language model backbone)
62 + """
63 +
64 + type: Literal["image_embeds"]
65 + data: Annotated[torch.Tensor | list[torch.Tensor], TensorShape("n", "f", "h")]
66 +
67 +
68 +Eagle2_5_VLImageInputs: TypeAlias = (
69 + Eagle2_5_VLImagePixelInputs | Eagle2_5_VLImageEmbeddingInputs
70 +)
71 +
72 +
73 +class Eagle2_5_VLProcessor(BaseInternVLProcessor):
74 + """
75 + Custom processor for Eagle2.5-VL model.
76 + Extends BaseInternVLProcessor with Eagle-specific token handling.
77 + """
78 +
79 + def __init__(
80 + self,
81 + config: PretrainedConfig,
82 + tokenizer: TokenizerLike,
83 + *,
84 + min_dynamic_patch: int | None = None,
85 + max_dynamic_patch: int | None = None,
86 + dynamic_image_size: bool | None = None,
87 + ) -> None:
88 + # Skip super().__init__() to avoid config manipulation
89 + # Directly initialize all required attributes
90 + self.config = config
91 + self.tokenizer = tokenizer
92 +
93 + # Image size with force_image_size override
94 + image_size: int = config.vision_config.image_size
95 + if hasattr(config, "force_image_size") and config.force_image_size:
96 + image_size = config.force_image_size
97 +
98 + patch_size: int = config.vision_config.patch_size
99 + downsample_ratio: float = getattr(config, "downsample_ratio", 0.5)
100 +
101 + # Compute num_image_token
102 + self.num_image_token = int(
103 + (image_size // patch_size) ** 2 * (downsample_ratio**2)
104 + )
105 + self.image_size = image_size
106 +
107 + # Dynamic patch settings with defaults
108 + self.min_dynamic_patch = (
109 + min_dynamic_patch
110 + if min_dynamic_patch is not None
111 + else getattr(config, "min_dynamic_patch", 1)
112 + )
113 + self.max_dynamic_patch = (
114 + max_dynamic_patch
115 + if max_dynamic_patch is not None
116 + else getattr(config, "max_dynamic_patch", 12)
117 + )
118 + self.dynamic_image_size = (
119 + dynamic_image_size
120 + if dynamic_image_size is not None
121 + else getattr(config, "dynamic_image_size", True)
122 + )
123 + self.use_thumbnail: bool = getattr(config, "use_thumbnail", True)
124 +
125 + @property
126 + def image_token_id(self) -> int:
127 + """Get the image token ID from config or tokenizer."""
128 + if hasattr(self.config, "image_token_index"):
129 + return self.config.image_token_index
130 + # Fallback to tokenizer vocab - use <IMG_CONTEXT> (ID: 151667)
131 + vocab = self.tokenizer.get_vocab()
132 + if IMG_CONTEXT in vocab:
133 + return vocab[IMG_CONTEXT]
134 + raise ValueError(f"Cannot find image token '{IMG_CONTEXT}' in vocabulary")
135 +
136 + def get_image_repl(
137 + self,
138 + feature_size: int,
139 + num_patches: int | None,
140 + ) -> PromptUpdateDetails[str]:
141 + """Get image replacement string for prompt."""
142 + repl_features = IMG_CONTEXT * feature_size
143 + repl_full = IMG_START + repl_features + IMG_END
144 +
145 + return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
146 +
147 +
148 +class Eagle2_5_VLProcessingInfo(BaseInternVLProcessingInfo):
149 + """Processing info for Eagle2.5-VL model."""
150 +
151 + def get_hf_processor(self, **kwargs) -> Eagle2_5_VLProcessor:
152 + return self.ctx.init_processor(
153 + Eagle2_5_VLProcessor,
154 + config=self.ctx.get_hf_config(),
155 + tokenizer=self.get_tokenizer(),
156 + **kwargs,
157 + )
158 +
159 +
160 +class Eagle2_5_VLDummyInputsBuilder(
161 + BaseInternVLDummyInputsBuilder[Eagle2_5_VLProcessingInfo]
162 +):
163 + """Dummy inputs builder for Eagle2.5-VL model."""
164 +
165 + pass
166 +
167 +
168 +class Eagle2_5_VLMultiModalProcessor(
169 + BaseInternVLMultiModalProcessor[Eagle2_5_VLProcessingInfo]
170 +):
171 + """Multi-modal processor for Eagle2.5-VL model."""
172 +
173 + pass
174 +
175 +
176 +@MULTIMODAL_REGISTRY.register_processor(
177 + Eagle2_5_VLMultiModalProcessor,
178 + info=Eagle2_5_VLProcessingInfo,
179 + dummy_inputs=Eagle2_5_VLDummyInputsBuilder,
180 +)
181 +class Eagle2_5_VLForConditionalGeneration(
182 + nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
183 +):
184 + """
185 + Eagle2.5-VL model for conditional generation.
186 +
187 + Architecture:
188 + - Vision Encoder: SigLIP
189 + - Language Model: Qwen2
190 + - Projection: MLP with pixel shuffle downsampling
191 + """
192 +
193 + supports_encoder_tp_data = True
194 +
195 + @classmethod
196 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
197 + if modality.startswith("image"):
198 + return "<image>"
199 + raise ValueError("Only image modality is supported")
200 +
201 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
202 + super().__init__()
203 +
204 + config = vllm_config.model_config.hf_config
205 + quant_config = vllm_config.quant_config
206 + multimodal_config = vllm_config.model_config.multimodal_config
207 +
208 + self.config = config
209 + self.multimodal_config = multimodal_config
210 + self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
211 +
212 + # Image configuration
213 + image_size = (
214 + getattr(config, "force_image_size", None) or config.vision_config.image_size
215 + )
216 + patch_size = config.vision_config.patch_size
217 + self.patch_size = patch_size
218 + self.downsample_ratio = getattr(config, "downsample_ratio", 0.5)
219 + self.num_image_token = int(
220 + (image_size // patch_size) ** 2 * (self.downsample_ratio**2)
221 + )
222 +
223 + self.select_layer = getattr(config, "select_layer", -1)
224 +
225 + # Vision encoder (SigLIP)
226 + self.vision_model = self._init_vision_model(
227 + config,
228 + quant_config=quant_config,
229 + prefix=maybe_prefix(prefix, "vision_model"),
230 + )
231 +
232 + # Language model (Qwen2)
233 + self.language_model = init_vllm_registered_model(
234 + vllm_config=vllm_config,
235 + hf_config=config.text_config,
236 + prefix=maybe_prefix(prefix, "language_model"),
237 + )
238 +
239 + # MLP projection
240 + self.mlp1 = self._init_mlp1(config)
241 +
242 + self.img_context_token_id = None
243 +
244 + self.make_empty_intermediate_tensors = (
245 + self.language_model.make_empty_intermediate_tensors
246 + )
247 +
248 + def _init_vision_model(
249 + self,
250 + config: PretrainedConfig,
251 + quant_config: QuantizationConfig | None,
252 + prefix: str,
253 + ):
254 + """Initialize SigLIP vision model."""
255 + vision_config = config.vision_config
256 +
257 + # Determine number of hidden layers based on select_layer
258 + vision_feature_layer = self.select_layer
259 + if vision_feature_layer < 0:
260 + num_hidden_layers = (
261 + vision_config.num_hidden_layers + vision_feature_layer + 1
262 + )
263 + else:
264 + num_hidden_layers = vision_feature_layer + 1
265 +
266 + # Disable the pooling head - Eagle2.5 needs all patch tokens,
267 + # not a single pooled output
268 + vision_config.vision_use_head = False
269 +
270 + return SiglipVisionModel(
271 + vision_config,
272 + quant_config=quant_config,
273 + multimodal_config=self.multimodal_config,
274 + num_hidden_layers_override=num_hidden_layers,
275 + prefix=prefix,
276 + )
277 +
278 + def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
279 + """Initialize MLP projection layer."""
280 + vit_hidden_size = config.vision_config.hidden_size
281 + llm_hidden_size = config.text_config.hidden_size
282 +
283 + return nn.Sequential(
284 + nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio) ** 2),
285 + nn.Linear(
286 + vit_hidden_size * int(1 / self.downsample_ratio) ** 2, llm_hidden_size
287 + ),
288 + nn.GELU(),
289 + nn.Linear(llm_hidden_size, llm_hidden_size),
290 + )
291 +
292 + def pixel_shuffle(self, x: torch.Tensor, scale_factor: float = 0.5) -> torch.Tensor:
293 + """
294 + Pixel shuffle operation for downsampling vision features.
295 +
296 + Args:
297 + x: Input tensor of shape (n, w, h, c)
298 + scale_factor: Downsampling factor
299 +
300 + Returns:
301 + Downsampled tensor
302 + """
303 + n, w, h, c = x.size()
304 + # N, W, H, C --> N, W, H * scale, C // scale
305 + x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
306 + # N, W, H * scale, C // scale --> N, H * scale, W, C // scale
307 + x = x.permute(0, 2, 1, 3).contiguous()
308 + # N, H * scale, W, C // scale --> N, H * scale, W * scale, C // (scale ** 2)
309 + x = x.view(
310 + n,
311 + int(h * scale_factor),
312 + int(w * scale_factor),
313 + int(c / (scale_factor * scale_factor)),
314 + )
315 + x = x.permute(0, 2, 1, 3).contiguous()
316 + return x
317 +
318 + def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor:
319 + """
320 + Extract visual features from pixel values.
321 +
322 + Args:
323 + pixel_values: Input pixel values of shape (batch, channels, height, width)
324 +
325 + Returns:
326 + Visual embeddings
327 + """
328 + vit_embeds = self.vision_model(pixel_values=pixel_values)
329 +
330 + h = w = int(vit_embeds.shape[1] ** 0.5)
331 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
332 + vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio)
333 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
334 + vit_embeds = self.mlp1(vit_embeds)
335 + return vit_embeds
336 +
337 + def _parse_and_validate_image_input(
338 + self, **kwargs: object
339 + ) -> Eagle2_5_VLImageInputs | None:
340 + """Parse and validate image inputs."""
341 + pixel_values_flat = kwargs.pop("pixel_values_flat", None)
342 + image_num_patches = kwargs.pop("image_num_patches", None)
343 + image_embeds = kwargs.pop("image_embeds", None)
344 +
345 + if pixel_values_flat is None and image_embeds is None:
346 + return None
347 +
348 + if image_embeds is not None:
349 + return Eagle2_5_VLImageEmbeddingInputs(
350 + type="image_embeds",
351 + data=image_embeds,
352 + )
353 +
354 + image_token_id = kwargs.get("image_token_id")
355 + if image_token_id is not None:
356 + if isinstance(image_token_id, torch.Tensor):
357 + image_token_id = image_token_id.flatten().unique().item()
358 + assert isinstance(image_token_id, int)
359 + self.img_context_token_id = image_token_id
360 +
361 + if pixel_values_flat is not None:
362 + image_size = getattr(self.config, "force_image_size", None)
363 + if image_size is None:
364 + image_size = self.config.vision_config.image_size
365 + expected_h = expected_w = image_size
366 + resolve_bindings = {"h": expected_h, "w": expected_w}
367 +
368 + return Eagle2_5_VLImagePixelInputs(
369 + type="pixel_values",
370 + pixel_values_flat=pixel_values_flat,
371 + num_patches=image_num_patches,
372 + resolve_bindings=resolve_bindings,
373 + )
374 +
375 + raise AssertionError("This line should be unreachable.")
376 +
377 + def _process_image_input(
378 + self,
379 + image_input: Eagle2_5_VLImageInputs,
380 + ) -> tuple[torch.Tensor, ...]:
381 + """Process image input to get embeddings."""
382 + if image_input["type"] == "image_embeds":
383 + return image_input["data"]
384 +
385 + assert self.vision_model is not None
386 +
387 + image_embeds = self.extract_feature(image_input["pixel_values_flat"])
388 +
389 + num_patches = image_input["num_patches"]
390 +
391 + # Only one image in the current batch
392 + if len(num_patches) == 1:
393 + return (image_embeds.view(-1, self.config.text_config.hidden_size),)
394 +
395 + # Split embeddings by image
396 + feature_size = image_embeds.shape[1]
397 + image_embeds = image_embeds.view(-1, self.config.text_config.hidden_size)
398 + image_feature_sizes = [
399 + num_patches * feature_size for num_patches in num_patches
400 + ]
401 + return image_embeds.split(image_feature_sizes)
402 +
403 + def get_language_model(self) -> torch.nn.Module:
404 + return self.language_model
405 +
406 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
407 + """Embed multimodal inputs."""
408 + image_input = self._parse_and_validate_image_input(**kwargs)
409 + if image_input is None:
410 + return []
411 +
412 + image_embeddings = self._process_image_input(image_input)
413 + return tuple(image_embeddings)
414 +
415 + def embed_input_ids(
416 + self,
417 + input_ids: torch.Tensor,
418 + multimodal_embeddings: MultiModalEmbeddings | None = None,
419 + *,
420 + is_multimodal: torch.Tensor | None = None,
421 + handle_oov_mm_token: bool = False,
422 + ) -> torch.Tensor:
423 + """Embed input IDs with optional multimodal embeddings."""
424 + if multimodal_embeddings is None or is_multimodal is None:
425 + return super().embed_input_ids(input_ids)
426 +
427 + return super().embed_input_ids(
428 + input_ids,
429 + multimodal_embeddings=multimodal_embeddings,
430 + is_multimodal=is_multimodal,
431 + handle_oov_mm_token=handle_oov_mm_token,
432 + )
433 +
434 + def forward(
435 + self,
436 + input_ids: torch.Tensor,
437 + positions: torch.Tensor,
438 + intermediate_tensors: IntermediateTensors | None = None,
439 + inputs_embeds: torch.Tensor | None = None,
440 + **kwargs: object,
441 + ) -> IntermediateTensors:
442 + """Forward pass through the model."""
443 + if intermediate_tensors is not None:
444 + input_ids = None
445 + inputs_embeds = None
446 +
447 + forward_kwargs = {
448 + "input_ids": input_ids,
449 + "positions": positions,
450 + "intermediate_tensors": intermediate_tensors,
451 + "inputs_embeds": inputs_embeds,
452 + }
453 +
454 + hidden_states = self.language_model.model(**forward_kwargs)
455 + return hidden_states
456 +
457 + def compute_logits(
458 + self,
459 + hidden_states: torch.Tensor,
460 + ) -> torch.Tensor | None:
461 + """Compute logits from hidden states."""
462 + return self.language_model.compute_logits(hidden_states)
463 +
464 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
465 + """Load model weights."""
466 + loader = AutoWeightsLoader(self)
467 + return loader.load_weights(weights)
468 +
469 + def get_mm_mapping(self) -> MultiModelKeys:
470 + """Get the module prefix mapping for multimodal models."""
471 + return MultiModelKeys.from_string_field(
472 + language_model="language_model",
473 + connector="mlp1",
474 + tower_model="vision_model",
475 + )
vllm/model_executor/models/registry.py
+4
@@ -302,6 +302,10 @@ _MULTIMODAL_MODELS = {
302 "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
303 "DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
304 "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
305 + "Eagle2_5_VLForConditionalGeneration": (
306 + "eagle2_5_vl",
307 + "Eagle2_5_VLForConditionalGeneration",
308 + ),
309 "Ernie4_5_VLMoeForConditionalGeneration": (
310 "ernie45_vl",
311 "Ernie4_5_VLMoeForConditionalGeneration",