@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4061dcf4c5

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 02:30 UTC

[Bugfix] Enable Kimi k25 processor test (#33562)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Isotr0py committed Feb 2, 2026 at 22:25 UTC 4061dcf4c51ae33aee4bc73096f82da29b580c57

4 files changed +96 -12

examples/offline_inference/vision_language.py

+41 -2

@@ -995,6 +995,31 @@ def run_kimi_vl(questions: list[str], modality: str) -> ModelRequestData:

995 )

996

997

998 +# Kimi-VL

999 +def run_kimi_k25(questions: list[str], modality: str) -> ModelRequestData:

1000 + assert modality == "vision_chunk"

1001 +

1002 + prompts = [

1004 + f"<|media_pad|><|media_end|>{question}<|im_end|>"

1005 + "<|im_assistant|>assistant<|im_middle|>"

1006 + for question in questions

1007 + ]

1008 +

1009 + engine_args = EngineArgs(

1010 + model="moonshotai/Kimi-K2.5",

1011 + trust_remote_code=True,

1012 + max_model_len=4096,

1013 + limit_mm_per_prompt={modality: 1},

1014 + tensor_parallel_size=4,

1015 + )

1016 +

1017 + return ModelRequestData(

1018 + engine_args=engine_args,

1019 + prompts=prompts,

1020 + )

1021 +

1022 +

1023 # LightOnOCR

1024 def run_lightonocr(questions: list[str], modality: str) -> ModelRequestData:

1025 assert modality == "image"

@@ -2110,6 +2135,7 @@ model_example_map = {

2135 "keye_vl": run_keye_vl,

2136 "keye_vl1_5": run_keye_vl1_5,

2137 "kimi_vl": run_kimi_vl,

2138 + "kimi_k25": run_kimi_k25,

2139 "lightonocr": run_lightonocr,

2140 "lfm2_vl": run_lfm2_vl,

2141 "llama4": run_llama4,

@@ -2196,6 +2222,19 @@ def get_multi_modal_input(args):

2222 "questions": vid_questions,

2223 }

2224

2225 + if args.modality == "vision_chunk":

2226 + # Input vision chunks and question

2227 + image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")

2228 + vision_chunk_questions = [

2229 + "What is the content of this image chunk?",

2230 + "Describe the content of this image chunk in detail.",

2231 + ]

2232 +

2233 + return {

2234 + "data": {"type": "image", "image": image},

2235 + "questions": vision_chunk_questions,

2236 + }

2237 +

2238 msg = f"Modality {args.modality} is not supported."

2239 raise ValueError(msg)

2240

@@ -2278,7 +2317,7 @@ def parse_args():

2317 "--modality",

2318 type=str,

2319 default="image",

2281 - choices=["image", "video"],

2320 + choices=["image", "video", "vision_chunk"],

2321 help="Modality of the input.",

2322 )

2323 parser.add_argument(

@@ -2355,7 +2394,7 @@ def main(args):

2394 req_data = model_example_map[model](questions, modality)

2395

2396 # Disable other modalities to save memory

2358 - default_limits = {"image": 0, "video": 0, "audio": 0}

2397 + default_limits = {"image": 0, "video": 0, "audio": 0, "vision_chunk": 0}

2398 req_data.engine_args.limit_mm_per_prompt = default_limits | dict(

2399 req_data.engine_args.limit_mm_per_prompt or {}

2400 )

tests/models/multimodal/processing/test_common.py

+26 -5

@@ -214,6 +214,28 @@ def get_text_token_prompts(

214 return text_prompt, token_prompt

215

216

217 +def random_vision_chunk(

218 + rng: np.random.RandomState,

219 + min_wh: int,

220 + max_wh: int,

221 + min_frames: int,

222 + max_frames: int,

223 +) -> dict:

224 + num_frames = rng.randint(min_frames, max_frames + 1)

225 + if num_frames == 1:

226 + # Single image chunk

227 + wh = rng.randint(min_wh, max_wh + 1)

228 + image = random_image(rng, wh, wh + 1)

229 + return {"type": "image", "image": image}

230 + frames = []

231 + for _ in range(num_frames):

232 + wh = rng.randint(min_wh, max_wh + 1)

233 + frame = rng.randint(0, 256, size=(wh, wh, 3), dtype=np.uint8)

234 + frames.append(frame)

235 + video_array = np.stack(frames, axis=0)

236 + return {"type": "video_chunk", "video_chunk": video_array}

237 +

238 +

239 def _test_processing_correctness(

240 model_id_or_arch: str,

241 hit_rate: float,

@@ -291,6 +313,7 @@ def _test_processing_correctness(

313 "image": Image.new("RGB", size=(128, 128)),

314 "video": np.zeros((4, 128, 128, 3), dtype=np.uint8),

315 "audio": (np.zeros((512,)), 16000),

316 + "vision_chunk": {"type": "image", "image": Image.new("RGB", size=(128, 128))},

317 }

318 input_factory = {

319 "image": partial(random_image, rng, min_wh=128, max_wh=256),

@@ -298,6 +321,9 @@ def _test_processing_correctness(

321 random_video, rng, min_frames=2, max_frames=16, min_wh=128, max_wh=256

322 ),

323 "audio": partial(random_audio, rng, min_len=512, max_len=1024, sr=16000),

324 + "vision_chunk": partial(

325 + random_vision_chunk, rng, min_wh=128, max_wh=256, min_frames=1, max_frames=1

326 + ),

327 }

328

329 for batch_idx in range(num_batches):

@@ -413,11 +439,6 @@ def test_processing_correctness(

439 "Qwen-VL tokenizer requires downloading a font file from "

440 "servers that often refuse connections in CI"

441 )

416 - if model_id == "moonshotai/Kimi-K2.5":

417 - # FIXME(Isaac): Fix Kimi-K2.5's offline inference about vision chunks.

418 - pytest.skip(

419 - "Kimi-K2.5's offline inference has issues about vision chunks. Fix later."

420 - )

442

443 _test_processing_correctness(

444 model_id,

vllm/model_executor/models/kimi_k25.py

+27 -5

@@ -96,16 +96,20 @@ class MoonshotKimiVAutoProcessor(ProcessorMixin):

96 attributes = ["tokenizer"]

97 tokenizer_class = "AutoTokenizer"

99 - def __init__(self, media_processor=None, tokenizer=None):

99 + def __init__(

100 + self, media_processor=None, tokenizer=None, media_token_id: int | None = None

101 + ):

102 super().__init__(tokenizer)

103 self.media_processor = media_processor

104 + self.media_token_id = media_token_id

105 + assert self.media_token_id is not None

106

107 # We do not support str input for text here

108 def __call__(

109 self,

110 vision_chunks: list[VisionChunk] | None = None,

111 *,

108 - text: list[int],

112 + text: list[int] | str,

113 **kwargs,

114 ) -> BatchFeature:

115 """

@@ -122,13 +126,30 @@ class MoonshotKimiVAutoProcessor(ProcessorMixin):

126 - **grid_thws** -- list of image 3D grid in LLM. Returned when `vision_chunks` is not `None`.

127 """

128 mm_inputs = {}

129 + input_ids = self.tokenizer.encode(text) if isinstance(text, str) else text

130 if vision_chunks is not None:

131 assert isinstance(vision_chunks, list)

132 mm_inputs = self.media_processor.preprocess(vision_chunks)

133 +

134 + num_tokens_per_chunk = [

135 + self.media_processor.media_tokens_calculator(chunk)

136 + for chunk in vision_chunks

137 + ]

138 +

139 + new_input_ids = []

140 + for token in input_ids:

141 + if token == self.media_token_id:

142 + new_input_ids.extend(

143 + [self.media_token_id] * num_tokens_per_chunk.pop(0)

144 + )

145 + else:

146 + new_input_ids.append(token)

147 + input_ids = new_input_ids

148 +

149 # XXX: _apply_hf_processor_text_mm will call tolist() on input_ids

150 return BatchFeature(

151 data={

131 - "input_ids": torch.tensor([text]),

152 + "input_ids": torch.tensor([input_ids]),

153 **mm_inputs,

154 }

155 )

@@ -152,6 +173,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):

173 self.hf_processor = MoonshotKimiVAutoProcessor(

174 media_processor=self.media_processor,

175 tokenizer=self.get_tokenizer(),

176 + media_token_id=self.media_token_id,

177 )

178 self.media_tokens_calculator = self.media_processor.media_tokens_calculator

179

@@ -174,9 +196,9 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):

196 self.media_token_id = self.info.media_token_id

197 self.frame_per_chunk = self.info.media_processor.num_frames_per_chunk

198

177 - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> list[int]:

199 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

200 num_media = mm_counts.get("vision_chunk", 0)

179 - return [self.media_token_id] * num_media

201 + return "<|media_pad|>" * num_media

202

203 def get_dummy_mm_items(self):

204 dummy_videos = self._get_dummy_images(

vllm/multimodal/parse.py

@@ -668,6 +668,8 @@ class MultiModalDataParser:

668 return None

669 if self.is_embeddings(data):

670 raise ValueError("Do not support embedding data for vision_chunk right now")

671 + if isinstance(data, dict):

672 + data = [data]

673 return VisionChunkProcessorItems(data)

674

675 def _get_subparsers(self) -> Mapping[str, ModalityDataParser]: