A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 07:00 UTC
[Model] Update Kimi-K25 and Isaac processors to fit HF-style (#37693)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 21, 2026 at 02:30 UTC
37aadf623786a0fb22a29051b8084168f18db1c9
5 files changed
+128
-95
vllm/model_executor/models/isaac.py
+3
-4
@@ -334,15 +334,14 @@ class IsaacProcessingInfo(BaseProcessingInfo):
334
return IsaacConfig()
335
336
def get_image_processor(self, **kwargs) -> IsaacImageProcessor:
337
- return IsaacImageProcessor(kwargs)
337
+ return IsaacImageProcessor(**kwargs)
338
339
def get_hf_processor(self, **kwargs) -> IsaacProcessor:
340
hf_config = self.get_hf_config()
341
342
- return self.ctx.init_processor(
343
- IsaacProcessor,
342
+ return IsaacProcessor(
343
tokenizer=self.get_tokenizer(),
345
- image_processor=self.get_image_processor(),
344
+ image_processor=self.get_image_processor(**kwargs),
345
image_token=hf_config.vision_token,
346
)
347
vllm/model_executor/models/kimi_k25.py
+16
-18
@@ -104,19 +104,25 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
104
105
def __init__(self, ctx: InputProcessingContext) -> None:
106
super().__init__(ctx)
107
- self.hf_config = self.get_hf_config()
108
- self.media_token_id = self.hf_config.media_placeholder_token_id
109
- media_processor = cached_get_image_processor(
107
+
108
+ self.hf_config = hf_config = self.get_hf_config()
109
+
110
+ tokenizer = self.get_tokenizer()
111
+ image_processor = cached_get_image_processor(
112
self.ctx.model_config.model,
113
trust_remote_code=self.ctx.model_config.trust_remote_code,
114
)
113
- self.media_processor = media_processor
115
+
116
+ self.media_token_id = media_token_id = hf_config.media_placeholder_token_id
117
+ self.media_token = tokenizer.decode(media_token_id)
118
+
119
+ self.image_processor = image_processor
120
self.hf_processor = KimiK25Processor(
115
- media_processor=self.media_processor,
116
- tokenizer=self.get_tokenizer(),
117
- media_token_id=self.media_token_id,
121
+ tokenizer=tokenizer,
122
+ image_processor=image_processor,
123
+ media_token_id=media_token_id,
124
)
119
- self.media_tokens_calculator = self.media_processor.media_tokens_calculator
125
+ self.media_tokens_calculator = image_processor.media_tokens_calculator
126
127
def get_hf_processor(self):
128
return self.hf_processor
@@ -132,20 +138,15 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
138
class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):
139
"""Builds dummy inputs for Kimi-K2.5 model profiling."""
140
135
- def __init__(self, info: KimiK25ProcessingInfo) -> None:
136
- super().__init__(info)
137
- self.media_token_id = self.info.media_token_id
138
- self.frame_per_chunk = self.info.media_processor.num_frames_per_chunk
139
-
141
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
142
num_media = mm_counts.get("vision_chunk", 0)
142
- return "<|media_pad|>" * num_media
143
+ return self.info.media_token * num_media
144
145
def get_dummy_mm_items(self):
146
dummy_videos = self._get_dummy_images(
147
height=MaxImageTokenMeta.height,
148
width=MaxImageTokenMeta.width,
148
- num_images=self.frame_per_chunk,
149
+ num_images=self.info.image_processor.num_frames_per_chunk,
150
)
151
152
video_chunk_dummy_item = VisionChunkVideo(
@@ -236,9 +237,6 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
237
),
238
]
239
239
- def split_video_chunks(self, video):
240
- return self.info.media_processor.split_video_chunks(video)
241
-
240
241
@MULTIMODAL_REGISTRY.register_processor(
242
KimiK25MultiModalProcessor,
vllm/transformers_utils/processors/isaac.py
+54
-32
@@ -6,12 +6,14 @@ import math
6
from typing import Any
7
8
import numpy as np
9
-import PIL.Image
9
import torch
10
import torch.nn.functional as F
11
+from PIL import Image
12
from transformers import BatchFeature, ProcessorMixin, TensorType
13
from typing_extensions import TypedDict, Unpack
14
15
+from vllm.tokenizers.hf import HfTokenizer
16
+
17
MAX_PIXELS = 60_000_000 # 60-megapixel ceiling ≈ 8200 × 7300 px
18
19
# Vision preprocessing constants
@@ -39,7 +41,7 @@ def _make_writeable(arr: np.ndarray) -> np.ndarray:
41
return arr.copy()
42
43
42
-def extract_image_pil(image: PIL.Image.Image) -> torch.Tensor | None:
44
+def extract_image_pil(image: Image.Image) -> torch.Tensor:
45
if image.width * image.height > MAX_PIXELS:
46
raise ValueError(
47
f"Image (w={image.width}, h={image.height}) > MAX=`{MAX_PIXELS}`"
@@ -314,31 +316,30 @@ class IsaacImageProcessorKwargs(TypedDict, total=False):
316
317
318
class IsaacImageProcessor:
317
- patch_size = 16
318
- max_num_patches = 6144
319
- min_num_patches = 256
320
- pixel_shuffle_scale = 2
321
-
319
valid_kwargs = IsaacImageProcessorKwargs
320
model_input_names = ["pixel_values", "image_grid_thw"]
321
325
- def __init__(self, kwargs):
326
- self.patch_size = kwargs.pop("patch_size", self.patch_size)
327
- self.vision_max_num_patches = kwargs.pop(
328
- "vision_max_num_patches", self.max_num_patches
329
- )
330
- self.vision_min_num_patches = kwargs.pop(
331
- "vision_min_num_patches", self.min_num_patches
332
- )
333
- self.pixel_shuffle_scale = kwargs.pop("pixel_shuffle_scale", 2)
334
-
335
- def preprocess(
322
+ def __init__(
323
self,
337
- images: list[torch.Tensor],
338
- return_tensors: str | TensorType | None,
324
+ patch_size: int = 16,
325
+ vision_max_num_patches: int = 6144,
326
+ vision_min_num_patches: int = 256,
327
+ pixel_shuffle_scale: int = 2,
328
+ ) -> None:
329
+ self.patch_size = patch_size
330
+ self.vision_max_num_patches = vision_max_num_patches
331
+ self.vision_min_num_patches = vision_min_num_patches
332
+ self.pixel_shuffle_scale = pixel_shuffle_scale
333
+
334
+ def __call__(
335
+ self,
336
+ images: Image.Image | list[Image.Image],
337
+ return_tensors: str | TensorType | None = None,
338
**kwargs: Unpack[IsaacImageProcessorKwargs],
339
) -> BatchFeature:
340
"""Preprocess images into format compatible with vLLM input processing."""
341
+ if not isinstance(images, list):
342
+ images = [images]
343
344
all_pixel_values: list[torch.Tensor] = []
345
all_image_grids: list[torch.Tensor] = []
@@ -388,23 +389,40 @@ class IsaacImageProcessor:
389
class IsaacProcessor(ProcessorMixin):
390
attributes = ["image_processor", "tokenizer"]
391
391
- def __init__(self, image_processor=None, tokenizer=None, **kwargs):
392
- self.image_token = kwargs.pop("image_token", "<image>")
392
+ def __init__(
393
+ self,
394
+ image_processor: IsaacImageProcessor,
395
+ tokenizer: HfTokenizer,
396
+ image_token: str = "<image>",
397
+ ):
398
self.image_processor = image_processor
399
self.tokenizer = tokenizer
400
396
- def __call__(self, text=None, images=None, **kwargs) -> BatchFeature:
397
- result = {}
401
+ self.image_token = image_token
402
403
+ def __call__(
404
+ self,
405
+ text: str | list[str] | None = None,
406
+ images: Image.Image | list[Image.Image] | None = None,
407
+ return_tensors: str | TensorType | None = None,
408
+ **kwargs,
409
+ ) -> BatchFeature:
410
if images is not None:
400
- image_inputs = self.image_processor.preprocess(images, **kwargs)
411
+ image_inputs = self.image_processor(
412
+ images,
413
+ return_tensors=return_tensors,
414
+ **kwargs,
415
+ )
416
image_grid_thw = image_inputs["image_grid_thw"]
402
- result.update(image_inputs)
417
+ else:
418
+ image_inputs = {}
419
+ image_grid_thw = []
420
404
- if text is not None:
405
- if not isinstance(text, list):
406
- text = [text]
421
+ if text is not None:
422
+ if not isinstance(text, list):
423
+ text = [text]
424
425
+ if image_inputs:
426
text = text.copy() # below lines change text in-place
427
merge_length = self.image_processor.pixel_shuffle_scale**2
428
index = 0
@@ -417,10 +435,14 @@ class IsaacProcessor(ProcessorMixin):
435
index += 1
436
text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>")
437
420
- if text is not None:
421
- result.update(self.tokenizer(text, **kwargs))
438
+ text_inputs = self.tokenizer(text, return_tensors=return_tensors)
439
+ else:
440
+ text_inputs = {}
441
423
- return BatchFeature(result)
442
+ return BatchFeature(
443
+ data={**text_inputs, **image_inputs},
444
+ tensor_type=return_tensors,
445
+ )
446
447
def apply_chat_template(
448
self,
vllm/transformers_utils/processors/kimi_k25.py
+54
-38
@@ -1,38 +1,41 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import torch
4
-from transformers import BatchFeature
3
+from transformers import BaseImageProcessor, BatchFeature, TensorType
4
from transformers.processing_utils import ProcessorMixin
5
6
from vllm.multimodal.inputs import VisionChunk
7
+from vllm.tokenizers.hf import HfTokenizer
8
9
10
class KimiK25Processor(ProcessorMixin):
11
- attributes = ["tokenizer"]
12
- tokenizer_class = "AutoTokenizer"
11
+ attributes = ["image_processor", "tokenizer"]
12
13
def __init__(
15
- self, media_processor=None, tokenizer=None, media_token_id: int | None = None
16
- ):
17
- super().__init__(tokenizer)
18
- self.media_processor = media_processor
14
+ self,
15
+ image_processor: BaseImageProcessor,
16
+ tokenizer: HfTokenizer,
17
+ media_token_id: int,
18
+ ) -> None:
19
+ self.image_processor = image_processor
20
+ self.tokenizer = tokenizer
21
+
22
self.media_token_id = media_token_id
20
- assert self.media_token_id is not None
23
24
def __call__(
25
self,
26
+ text: str | list[str] | None = None,
27
vision_chunks: list[VisionChunk] | None = None,
25
- *,
26
- text: list[int] | str,
28
+ return_tensors: str | TensorType | None = None,
29
**kwargs,
30
) -> BatchFeature:
31
"""
32
Args:
31
- vision_chunks: List of VisionChunk items to be processed.
32
- For image: VisionChunkImage with type='image', image=PIL.Image
33
- For video_chunk: VisionChunkVideo with type='video_chunk',
34
- video_chunk=list[PIL.Image]
35
- text: The token ids to be fed to a model (required).
33
+ text: The text to be field to the model.
34
+ vision_chunks: List of `VisionChunk` items to be processed.
35
+ For image: `VisionChunkImage` with
36
+ `type='image', image=PIL.Image`
37
+ For video_chunk: `VisionChunkVideo` with
38
+ `type='video_chunk', video_chunk=list[PIL.Image]`
39
Returns:
40
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
41
@@ -42,31 +45,44 @@ class KimiK25Processor(ProcessorMixin):
45
- **grid_thws** -- list of image 3D grid in LLM.
46
Returned when `vision_chunks` is not `None`.
47
"""
45
- mm_inputs = {}
46
- input_ids = self.tokenizer.encode(text) if isinstance(text, str) else text
48
if vision_chunks is not None:
48
- assert isinstance(vision_chunks, list)
49
- mm_inputs = self.media_processor.preprocess(vision_chunks)
49
+ mm_inputs = self.image_processor.preprocess(
50
+ vision_chunks,
51
+ return_tensors=return_tensors,
52
+ )
53
+ else:
54
+ mm_inputs = {}
55
+
56
+ if text is not None:
57
+ if not isinstance(text, list):
58
+ text = [text]
59
+
60
+ text_inputs = self.tokenizer(text)
61
+
62
+ # Note: Modify in-place
63
+ input_ids: list[list[int]] = text_inputs["input_ids"] # type: ignore
64
+
65
+ if vision_chunks is not None:
66
+ num_tokens_per_chunk = [
67
+ self.image_processor.media_tokens_calculator(chunk)
68
+ for chunk in vision_chunks
69
+ ]
70
51
- num_tokens_per_chunk = [
52
- self.media_processor.media_tokens_calculator(chunk)
53
- for chunk in vision_chunks
54
- ]
71
+ for i in range(len(input_ids)):
72
+ new_input_ids = []
73
+ for token in input_ids[i]:
74
+ if token == self.media_token_id:
75
+ new_input_ids.extend(
76
+ [self.media_token_id] * num_tokens_per_chunk.pop(0)
77
+ )
78
+ else:
79
+ new_input_ids.append(token)
80
56
- new_input_ids = []
57
- for token in input_ids:
58
- if token == self.media_token_id:
59
- new_input_ids.extend(
60
- [self.media_token_id] * num_tokens_per_chunk.pop(0)
61
- )
62
- else:
63
- new_input_ids.append(token)
64
- input_ids = new_input_ids
81
+ input_ids[i] = new_input_ids
82
+ else:
83
+ text_inputs = {}
84
66
- # XXX: _apply_hf_processor_text_mm will call tolist() on input_ids
85
return BatchFeature(
68
- data={
69
- "input_ids": torch.tensor([input_ids]),
70
- **mm_inputs,
71
- }
86
+ data={**text_inputs, **mm_inputs},
87
+ tensor_type=return_tensors,
88
)
vllm/transformers_utils/processors/step3_vl.py
+1
-3
@@ -286,11 +286,9 @@ class Step3VLImageProcessor:
286
287
def __call__(
288
self,
289
- images: Image.Image | list[Image.Image] | None = None,
289
+ images: Image.Image | list[Image.Image],
290
return_tensors: str | TensorType | None = None,
291
) -> BatchFeature:
292
- if images is None:
293
- images = []
292
if not isinstance(images, list):
293
images = [images]
294