A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[VLM] Add video support for Intern-S1 (#21671)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Jul 27, 2025 at 19:49 UTC
3d847a3125cd640c1f31a1ba28c0b24c0fd201a8
5 files changed
+173
-50
docs/models/supported_models.md
+1
-1
@@ -593,7 +593,7 @@ Specified using `--task generate`.
593
| `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ | ✅︎ |
594
| `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | | ✅︎ | ✅︎ |
595
| `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3`, etc. | ✅︎ | | ✅︎ |
596
-| `InternS1ForConditionalGeneration` | Intern-S1 | T + I<sup>E+</sup> | `internlm/Intern-S1`, etc. | | ✅︎ | ✅︎ |
596
+| `InternS1ForConditionalGeneration` | Intern-S1 | T + I<sup>E+</sup> + V<sup>E+</sup> | `internlm/Intern-S1`, etc. | | ✅︎ | ✅︎ |
597
| `InternVLChatModel` | InternVL 3.0, InternVideo 2.5, InternVL 2.5, Mono-InternVL, InternVL 2.0 | T + I<sup>E+</sup> + (V<sup>E+</sup>) | `OpenGVLab/InternVL3-9B`, `OpenGVLab/InternVideo2_5_Chat_8B`, `OpenGVLab/InternVL2_5-4B`, `OpenGVLab/Mono-InternVL-2B`, `OpenGVLab/InternVL2-4B`, etc. | ✅︎ | ✅︎ | ✅︎ |
598
| `KeyeForConditionalGeneration` | Keye-VL-8B-Preview | T + I<sup>E+</sup> + V<sup>E+</sup> | `Kwai-Keye/Keye-VL-8B-Preview` | | | ✅︎ |
599
| `KimiVLForConditionalGeneration` | Kimi-VL-A3B-Instruct, Kimi-VL-A3B-Thinking | T + I<sup>+</sup> | `moonshotai/Kimi-VL-A3B-Instruct`, `moonshotai/Kimi-VL-A3B-Thinking` | | | ✅︎ |
examples/offline_inference/vision_language.py
+5
-3
@@ -470,8 +470,6 @@ def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
470
471
# Intern-S1
472
def run_interns1(questions: list[str], modality: str) -> ModelRequestData:
473
- assert modality == "image"
474
-
473
model_name = "internlm/Intern-S1"
474
475
engine_args = EngineArgs(
@@ -483,7 +481,11 @@ def run_interns1(questions: list[str], modality: str) -> ModelRequestData:
481
enforce_eager=True,
482
)
483
486
- placeholder = "<IMG_CONTEXT>"
484
+ if modality == "image":
485
+ placeholder = "<IMG_CONTEXT>"
486
+ elif modality == "video":
487
+ placeholder = "<video>"
488
+
489
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
490
messages = [
491
[{"role": "user", "content": f"{placeholder}\n{question}"}]
tests/models/multimodal/processing/test_common.py
+1
@@ -278,6 +278,7 @@ def _test_processing_correctness_one(
278
"THUDM/GLM-4.1V-9B-Thinking",
279
"ibm-granite/granite-speech-3.3-2b",
280
"h2oai/h2ovl-mississippi-800m",
281
+ "internlm/Intern-S1",
282
"OpenGVLab/InternVL2-1B",
283
"OpenGVLab/InternVL3-1B",
284
"HuggingFaceM4/Idefics3-8B-Llama3",
vllm/model_executor/models/interns1.py
+166
-45
@@ -9,9 +9,10 @@
9
from collections.abc import Iterable, Mapping, Sequence
10
from typing import Literal, Optional, TypedDict, Union
11
12
+import regex as re
13
import torch
14
import torch.nn as nn
14
-from transformers import InternVLProcessor, PretrainedConfig
15
+from transformers import BatchFeature, InternVLProcessor, PretrainedConfig
16
from transformers.activations import ACT2FN
17
from transformers.models.got_ocr2.image_processing_got_ocr2_fast import (
18
GotOcr2ImageProcessorFast)
@@ -139,13 +140,13 @@ def get_interns1_target_ratios(
140
141
142
class InternS1ProcessingInfo(BaseProcessingInfo):
142
- """Basic image-only ProcessingInfo for InternS1-style models."""
143
+ """ProcessingInfo for InternS1-style models."""
144
145
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
146
return self.ctx.get_hf_processor(InternVLProcessor, **kwargs)
147
148
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
148
- return {"image": None}
149
+ return {"image": None, "video": None}
150
151
def get_num_image_tokens(
152
self,
@@ -218,16 +219,35 @@ class InternS1ProcessingInfo(BaseProcessingInfo):
219
processor=processor.image_processor,
220
)
221
222
+ def get_num_frames_with_most_features(
223
+ self,
224
+ seq_len: int,
225
+ mm_counts: Mapping[str, int],
226
+ ) -> int:
227
+ max_images = mm_counts.get("image", 0)
228
+ max_videos = mm_counts.get("video", 0)
229
+
230
+ processor = self.get_hf_processor()
231
+
232
+ max_image_tokens = self.get_max_image_tokens() * max_images
233
+ max_total_frames = (seq_len -
234
+ max_image_tokens) // processor.image_seq_length
235
+ max_frames_per_video = max_total_frames // max(max_videos, 1)
236
+
237
+ return max(max_frames_per_video, 1)
238
+
239
240
class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo]
241
):
224
- """Basic image-only DummyInputsBuilder for InternS1-style models."""
242
+ """DummyInputsBuilder for InternS1-style models."""
243
244
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
245
num_images = mm_counts.get("image", 0)
246
+ num_videos = mm_counts.get("video", 0)
247
image_token = self.info.get_hf_processor().image_token
248
+ video_token = self.info.get_hf_processor().video_token
249
230
- return image_token * num_images
250
+ return image_token * num_images + video_token * num_videos
251
252
def get_dummy_mm_data(
253
self,
@@ -236,13 +256,24 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo]
256
) -> MultiModalDataDict:
257
target_width, target_height = \
258
self.info.get_image_size_with_most_features()
259
+ target_num_frames = \
260
+ self.info.get_num_frames_with_most_features(seq_len, mm_counts)
261
num_images = mm_counts.get("image", 0)
262
+ num_videos = mm_counts.get("video", 0)
263
+
264
+ config = self.info.get_hf_config()
265
+ image_size_h, image_size_w = config.vision_config.image_size
266
267
return {
268
"image":
269
self._get_dummy_images(width=target_width,
270
height=target_height,
245
- num_images=num_images)
271
+ num_images=num_images),
272
+ "video":
273
+ self._get_dummy_videos(width=image_size_w,
274
+ height=image_size_h,
275
+ num_frames=target_num_frames,
276
+ num_videos=num_videos),
277
}
278
279
@@ -257,33 +288,89 @@ class InternS1MultiModalProcessor(
288
mm_kwargs: Mapping[str, object],
289
tok_kwargs: Mapping[str, object],
290
) -> Mapping[str, NestedTensors]:
260
- processed_outputs = super()._call_hf_processor(
261
- prompt=prompt,
262
- mm_data=mm_data,
263
- mm_kwargs=mm_kwargs,
264
- tok_kwargs=tok_kwargs,
265
- )
291
+ mm_data = dict(mm_data)
292
+ videos = mm_data.pop("videos", [])
293
+ images = mm_data.pop("images", [])
294
+ assert isinstance(videos, list)
295
+ assert isinstance(images, list)
296
297
hf_processor = self.info.get_hf_processor(**mm_kwargs)
268
- image_token_id = hf_processor.image_token_id
269
-
270
- # Since there may be extra tokens in the feature placeholders,
271
- # we need to pass the image token ID to the model to select the
272
- # tokens to merge from the vision encoder outputs
273
- processed_outputs["image_token_id"] = torch.tensor(image_token_id)
274
- images = mm_data.get('images', None)
275
- image_processor = self.info.get_hf_processor().image_processor
276
- if images is not None:
277
- image_inputs = image_processor(images=images)
278
- image_num_patches = image_inputs.pop("num_patches")
279
- if not isinstance(image_num_patches, list):
280
- raise ValueError(
281
- f'num_patches is supposed to be list, but got '
282
- f'{type(image_num_patches)}')
283
- image_num_patches = torch.tensor(image_num_patches)
284
- processed_outputs['image_num_patches'] = image_num_patches
285
-
286
- return processed_outputs
298
+ tokenizer = hf_processor.tokenizer
299
+ video_token_id = tokenizer.encode(hf_processor.video_token,
300
+ add_special_tokens=False)
301
+ assert len(video_token_id) == 1
302
+ video_token_id = video_token_id[0]
303
+
304
+ prompt = re.sub(hf_processor.image_token, "<image_placeholder>",
305
+ prompt)
306
+ prompt = re.sub(hf_processor.video_token, "<video_placeholder>",
307
+ prompt)
308
+
309
+ image_outputs = {}
310
+ if images:
311
+ image_pixel_values = []
312
+ for image in images:
313
+ processed_outputs = super()._call_hf_processor(
314
+ prompt=hf_processor.image_token,
315
+ mm_data={"images": image},
316
+ mm_kwargs=mm_kwargs,
317
+ tok_kwargs=tok_kwargs,
318
+ )
319
+ image_pixel_values.append(
320
+ processed_outputs.pop("pixel_values"))
321
+
322
+ input_ids = processed_outputs.pop("input_ids")
323
+ image_placeholder = tokenizer.batch_decode(input_ids)[0]
324
+ prompt = prompt.replace("<image_placeholder>",
325
+ image_placeholder, 1)
326
+
327
+ num_patches = [len(item) for item in image_pixel_values]
328
+ image_outputs: dict[str, NestedTensors] = {
329
+ "pixel_values": torch.concat(image_pixel_values),
330
+ "image_num_patches": torch.tensor(num_patches),
331
+ "image_token_id": torch.tensor(hf_processor.image_token_id),
332
+ }
333
+
334
+ video_outputs = {}
335
+ if videos:
336
+ video_pixel_values = []
337
+ for video in videos:
338
+ processed_outputs = super()._call_hf_processor(
339
+ prompt=hf_processor.video_token,
340
+ mm_data={"videos": video},
341
+ mm_kwargs=mm_kwargs,
342
+ tok_kwargs=tok_kwargs,
343
+ )
344
+ video_pixel_values.append(
345
+ processed_outputs.pop("pixel_values"))
346
+
347
+ input_ids = processed_outputs.pop("input_ids")
348
+ input_ids[input_ids ==
349
+ hf_processor.image_token_id] = video_token_id
350
+
351
+ video_placeholder = tokenizer.batch_decode(input_ids)[0]
352
+ prompt = prompt.replace("<video_placeholder>",
353
+ video_placeholder, 1)
354
+
355
+ num_frames = [len(item) for item in video_pixel_values]
356
+ video_outputs: dict[str, NestedTensors] = {
357
+ "pixel_values_videos": torch.concat(video_pixel_values),
358
+ "video_num_patches": torch.tensor(num_frames),
359
+ "video_token_id": torch.tensor(video_token_id),
360
+ }
361
+
362
+ prompt = re.sub("<image_placeholder>", hf_processor.image_token,
363
+ prompt)
364
+ prompt = re.sub("<video_placeholder>", hf_processor.video_token,
365
+ prompt)
366
+ text_outputs = tokenizer(prompt, **tok_kwargs, return_tensors="pt")
367
+
368
+ combined_outputs = dict(
369
+ **text_outputs,
370
+ **image_outputs,
371
+ **video_outputs,
372
+ )
373
+ return BatchFeature(combined_outputs)
374
375
def _get_mm_fields_config(
376
self,
@@ -292,7 +379,9 @@ class InternS1MultiModalProcessor(
379
) -> Mapping[str, MultiModalFieldConfig]:
380
381
image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
382
+ video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0))
383
num_images = len(image_num_patches)
384
+ num_videos = len(video_num_patches)
385
386
return dict(
387
pixel_values=MultiModalFieldConfig.flat_from_sizes(
@@ -300,6 +389,10 @@ class InternS1MultiModalProcessor(
389
image_num_patches=MultiModalFieldConfig.batched("image"),
390
image_embeds=MultiModalFieldConfig.batched("image"),
391
image_token_id=MultiModalFieldConfig.shared("image", num_images),
392
+ pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
393
+ "video", video_num_patches),
394
+ video_num_patches=MultiModalFieldConfig.batched("video"),
395
+ video_token_id=MultiModalFieldConfig.shared("video", num_videos),
396
)
397
398
def _get_prompt_updates(
@@ -312,32 +405,61 @@ class InternS1MultiModalProcessor(
405
img_context_token = hf_processor.image_token
406
start_image_token = hf_processor.start_image_token
407
end_image_token = hf_processor.end_image_token
315
-
316
- def get_replacement(item_idx: int):
408
+ video_token = hf_processor.video_token
409
+
410
+ if "video_num_patches" in out_mm_kwargs:
411
+ video_num_patches = out_mm_kwargs["video_num_patches"]
412
+ assert isinstance(video_num_patches, torch.Tensor)
413
+ video_num_patches = video_num_patches.tolist()
414
+ else:
415
+ video_num_patches = []
416
+
417
+ if "image_num_patches" in out_mm_kwargs:
418
+ image_num_patches = out_mm_kwargs["image_num_patches"]
419
+ assert isinstance(image_num_patches, torch.Tensor)
420
+ image_num_patches = image_num_patches.tolist()
421
+ else:
422
+ image_num_patches = []
423
+
424
+ def get_replacement_interns1_image(item_idx: int):
425
images = mm_items.get_items(
426
"image", (ImageEmbeddingItems, ImageProcessorItems))
427
428
if isinstance(images, ImageEmbeddingItems):
429
feature_size = images.get_feature_size(item_idx)
430
else:
323
- image_size = images.get_image_size(item_idx)
324
- feature_size = self.info.get_num_image_tokens(
325
- image_width=image_size.width,
326
- image_height=image_size.height,
327
- processor=hf_processor.image_processor,
328
- )
431
+ num_patches = image_num_patches[item_idx]
432
+ feature_size = num_patches * hf_processor.image_seq_length
433
434
repl_features = img_context_token * feature_size
435
repl_full = start_image_token + repl_features + end_image_token
436
return PromptUpdateDetails.select_text(repl_full,
437
img_context_token)
438
439
+ def get_replacement_interns1_video(item_idx: int):
440
+ num_patches = video_num_patches[item_idx]
441
+ repl_features = video_token * hf_processor.image_seq_length
442
+ repl_features_with_sep = (start_image_token + repl_features +
443
+ end_image_token)
444
+ # num_patches is equal to num_frames
445
+ repl_full = '\n'.join([
446
+ f'Frame{i+1}: {repl_features_with_sep}'
447
+ for i in range(num_patches)
448
+ ])
449
+
450
+ return PromptUpdateDetails.select_text(repl_full, video_token)
451
+
452
return [
453
PromptReplacement(
454
modality="image",
455
target=img_context_token,
339
- replacement=get_replacement,
340
- )
456
+ replacement=get_replacement_interns1_image,
457
+ ),
458
+ PromptReplacement(
459
+ modality="video",
460
+ target=video_token,
461
+ replacement=get_replacement_interns1_video,
462
+ ),
463
]
464
465
@@ -514,7 +636,7 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
636
637
def _parse_and_validate_video_input(
638
self, **kwargs: object) -> Optional[InternS1VideoPixelInputs]:
517
- pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)
639
+ pixel_values_flat_video = kwargs.pop("pixel_values_videos", None)
640
video_num_patches = kwargs.pop("video_num_patches", None)
641
video_embeds = kwargs.pop("video_embeds", None)
642
@@ -595,8 +717,8 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
717
"image_embeds") and "images" not in modalities:
718
modalities["images"] = self._parse_and_validate_image_input(
719
**kwargs)
598
- if input_key in ("pixel_values_flat_video",
599
- ) and "videos" not in modalities:
720
+ if input_key in (
721
+ "pixel_values_videos", ) and "videos" not in modalities:
722
modalities["videos"] = self._parse_and_validate_video_input(
723
**kwargs)
724
@@ -614,7 +736,6 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
736
modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
737
if not modalities:
738
return []
617
- return None
739
740
# The result multimodal_embeddings is tuple of tensors, with each
741
# tensor correspoending to a multimodal data item (image or video).
vllm/model_executor/models/internvl.py
-1
@@ -1322,7 +1322,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1322
modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
1323
if not modalities:
1324
return []
1325
- return None
1325
1326
# The result multimodal_embeddings is tuple of tensors, with each
1327
# tensor correspoending to a multimodal data item (image or video).