A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Misc] Clean up Kimi-VL (#16833)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Apr 18, 2025 at 20:15 UTC
aadb6565628cbc3b56cfc20df245ad36de675262
3 files changed
+20
-44
examples/offline_inference/vision_language.py
+2
-2
@@ -376,9 +376,9 @@ def run_kimi_vl(questions: list[str], modality: str) -> ModelRequestData:
376
377
engine_args = EngineArgs(
378
model="moonshotai/Kimi-VL-A3B-Instruct",
379
- max_model_len=4096,
380
- disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
379
trust_remote_code=True,
380
+ max_model_len=4096,
381
+ limit_mm_per_prompt={"image": 1},
382
)
383
384
return ModelRequestData(
examples/offline_inference/vision_language_multi_image.py
+1
-2
@@ -331,11 +331,10 @@ def load_kimi_vl(question: str, image_urls: list[str]) -> ModelRequestData:
331
332
engine_args = EngineArgs(
333
model=model_name,
334
+ trust_remote_code=True,
335
max_model_len=4096,
336
max_num_seqs=4,
336
- tensor_parallel_size=1,
337
limit_mm_per_prompt={"image": len(image_urls)},
338
- trust_remote_code=True,
338
)
339
340
placeholders = [{"type": "image", "image": url} for url in image_urls]
vllm/model_executor/models/kimi_vl.py
+17
-40
@@ -56,7 +56,6 @@ from transformers.activations import GELUActivation
56
from vllm.config import VllmConfig
57
from vllm.distributed import (get_tensor_model_parallel_rank,
58
get_tensor_model_parallel_world_size)
59
-from vllm.logger import init_logger
59
from vllm.model_executor.layers.fused_moe import FusedMoE
60
from vllm.model_executor.layers.logits_processor import LogitsProcessor
61
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -70,22 +69,20 @@ from vllm.model_executor.models.moonvit import MoonVitPretrainedModel
69
from vllm.model_executor.models.utils import merge_multimodal_embeddings
70
from vllm.model_executor.sampling_metadata import SamplingMetadata
71
from vllm.multimodal import MULTIMODAL_REGISTRY
73
-from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
74
- NestedTensors)
72
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
73
+ MultiModalKwargs, NestedTensors)
74
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
75
MultiModalDataItems)
76
from vllm.multimodal.processing import (BaseMultiModalProcessor,
77
BaseProcessingInfo, PromptReplacement,
78
PromptUpdate)
80
-from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
79
+from vllm.multimodal.profiling import BaseDummyInputsBuilder
80
from vllm.sequence import IntermediateTensors
81
from vllm.transformers_utils.configs import KimiVLConfig, MoonViTConfig
82
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekV2Config
83
84
from .utils import is_pp_missing_parameter, maybe_prefix
85
87
-logger = init_logger(__name__)
88
-
86
87
# For dummy input only
88
@dataclass
@@ -143,6 +140,9 @@ class KimiVLProcessingInfo(BaseProcessingInfo):
140
def get_hf_config(self):
141
return self.ctx.get_hf_config(KimiVLConfig)
142
143
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
144
+ return {"image": None}
145
+
146
def get_num_image_tokens(
147
self,
148
*,
@@ -180,23 +180,6 @@ class KimiVLProcessingInfo(BaseProcessingInfo):
180
token_width = (width + pad_width) // (kernel_size[1] * patch_size)
181
return int(token_height * token_width)
182
183
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
184
- # None means unlimited
185
- return {"image": None}
186
-
187
- def get_mm_max_tokens_per_item(
188
- self,
189
- seq_len: int,
190
- mm_counts: Mapping[str, int],
191
- ) -> Mapping[str, int]:
192
- return {
193
- "image":
194
- self.get_num_image_tokens(
195
- image_width=MaxImageTokenMeta.width,
196
- image_height=MaxImageTokenMeta.height,
197
- ),
198
- }
199
-
183
@property
184
def image_token_id(self) -> int:
185
return self.get_hf_config().media_placeholder_token_id
@@ -204,34 +187,28 @@ class KimiVLProcessingInfo(BaseProcessingInfo):
187
188
class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
189
207
- def __init__(self, info: KimiVLProcessingInfo) -> None:
208
- super().__init__(info)
190
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
191
+ num_images = mm_counts.get("image", 0)
192
+
193
+ processor = self.info.get_hf_processor()
194
+ image_token = processor.image_token
195
210
- self.image_token_id = self.info.image_token_id
211
- self.image_token = self.info.get_tokenizer().decode(
212
- self.image_token_id)
196
+ return image_token * num_images
197
214
- def get_dummy_processor_inputs(
198
+ def get_dummy_mm_data(
199
self,
200
seq_len: int,
201
mm_counts: Mapping[str, int],
218
- ) -> ProcessorInputs:
202
+ ) -> MultiModalDataDict:
203
num_images = mm_counts.get("image", 0)
204
221
- width = MaxImageTokenMeta.width
222
- height = MaxImageTokenMeta.height
223
- mm_data = {
205
+ return {
206
"image":
225
- self._get_dummy_images(width=width,
226
- height=height,
207
+ self._get_dummy_images(width=MaxImageTokenMeta.width,
208
+ height=MaxImageTokenMeta.height,
209
num_images=num_images)
210
}
211
230
- return ProcessorInputs(
231
- prompt_text=self.image_token * num_images,
232
- mm_data=mm_data,
233
- )
234
-
212
213
class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
214