@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3a92c6f3b5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 10:00 UTC

[Misc] Cleanup Kimi-K2.5's vision chunk modality entrypoints (#33157)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Isotr0py committed Jan 29, 2026 at 17:46 UTC 3a92c6f3b5f010453d81f871f642839c15402cda
7 files changed +733 -204
tests/entrypoints/test_chat_utils.py
+531
@@ -24,12 +24,25 @@ from vllm.multimodal.utils import (
24 )
25 from vllm.utils.serial_utils import tensor2base64
26
27 +KIMI_K2_5_MODEL_ID = "moonshotai/Kimi-K2.5"
28 PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
29 QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct"
30 QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
31 MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
32
33
34 +@pytest.fixture(scope="function")
35 +def kimi_k2_5_model_config():
36 + return ModelConfig(
37 + KIMI_K2_5_MODEL_ID,
38 + runner="generate",
39 + trust_remote_code=True,
40 + limit_mm_per_prompt={
41 + "image": 2,
42 + },
43 + )
44 +
45 +
46 @pytest.fixture(scope="function")
47 def phi3v_model_config():
48 return ModelConfig(
@@ -163,6 +176,22 @@ def _assert_mm_data_is_image_input(
176 assert image_data[i] is None
177
178
179 +def _assert_mm_data_is_vision_chunk_input(
180 + mm_data: MultiModalDataDict | None,
181 + vision_chunk_count: int,
182 +) -> None:
183 + assert mm_data is not None
184 + assert set(mm_data.keys()) == {"vision_chunk"}
185 +
186 + vision_chunk_data = mm_data.get("vision_chunk")
187 + assert vision_chunk_data is not None
188 +
189 + assert (
190 + isinstance(vision_chunk_data, list)
191 + and len(vision_chunk_data) == vision_chunk_count
192 + )
193 +
194 +
195 def _assert_mm_uuids(
196 mm_uuids: MultiModalUUIDDict | None,
197 media_count: int,
@@ -2151,3 +2180,505 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
2180 ]
2181 _assert_mm_data_inputs(mm_data, {"audio": 1})
2182 _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
2183 +
2184 +
2185 +def test_parse_chat_messages_image_vision_chunk(
2186 + kimi_k2_5_model_config,
2187 + image_url,
2188 +):
2189 + messages = [
2190 + {
2191 + "role": "user",
2192 + "content": [
2193 + {"type": "text", "text": "Analyze this image."},
2194 + {
2195 + "type": "image_url",
2196 + "image_url": {"url": image_url},
2197 + },
2198 + ],
2199 + }
2200 + ]
2201 +
2202 + conversation, mm_data, mm_uuids = parse_chat_messages(
2203 + messages,
2204 + kimi_k2_5_model_config,
2205 + content_format="string",
2206 + )
2207 +
2208 + placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
2209 + expected_conversation = [
2210 + {
2211 + "role": "user",
2212 + "content": f"{placeholder}\nAnalyze this image.",
2213 + }
2214 + ]
2215 +
2216 + assert conversation == expected_conversation
2217 + _assert_mm_data_is_vision_chunk_input(mm_data, 1)
2218 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
2219 +
2220 +
2221 +def test_parse_chat_messages_video_vision_chunk(
2222 + kimi_k2_5_model_config,
2223 + video_url,
2224 +):
2225 + messages = [
2226 + {
2227 + "role": "user",
2228 + "content": [
2229 + {"type": "text", "text": "Analyze this video."},
2230 + {
2231 + "type": "video_url",
2232 + "video_url": {"url": video_url},
2233 + },
2234 + ],
2235 + }
2236 + ]
2237 +
2238 + conversation, mm_data, mm_uuids = parse_chat_messages(
2239 + messages,
2240 + kimi_k2_5_model_config,
2241 + content_format="string",
2242 + )
2243 +
2244 + placeholder = "<|kimi_k25_video_placeholder|>"
2245 + expected_conversation = [
2246 + {
2247 + "role": "user",
2248 + "content": f"{placeholder}\nAnalyze this video.",
2249 + }
2250 + ]
2251 +
2252 + assert conversation == expected_conversation
2253 + _assert_mm_data_is_vision_chunk_input(mm_data, 1)
2254 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
2255 +
2256 +
2257 +def test_parse_chat_messages_image_vision_chunk_with_uuid(
2258 + kimi_k2_5_model_config,
2259 + image_url,
2260 +):
2261 + image_uuid = "image_123"
2262 + messages = [
2263 + {
2264 + "role": "user",
2265 + "content": [
2266 + {"type": "text", "text": "Analyze this image."},
2267 + {
2268 + "type": "image_url",
2269 + "image_url": {"url": image_url},
2270 + "uuid": image_uuid,
2271 + },
2272 + ],
2273 + }
2274 + ]
2275 +
2276 + conversation, mm_data, mm_uuids = parse_chat_messages(
2277 + messages,
2278 + kimi_k2_5_model_config,
2279 + content_format="string",
2280 + )
2281 +
2282 + placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
2283 + expected_conversation = [
2284 + {
2285 + "role": "user",
2286 + "content": f"{placeholder}\nAnalyze this image.",
2287 + }
2288 + ]
2289 +
2290 + assert conversation == expected_conversation
2291 + _assert_mm_data_is_vision_chunk_input(mm_data, 1)
2292 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid], modality="vision_chunk")
2293 +
2294 +
2295 +def test_parse_chat_messages_video_vision_chunk_with_uuid(
2296 + kimi_k2_5_model_config,
2297 + video_url,
2298 +):
2299 + video_uuid = "video_456"
2300 + messages = [
2301 + {
2302 + "role": "user",
2303 + "content": [
2304 + {"type": "text", "text": "Analyze this video."},
2305 + {
2306 + "type": "video_url",
2307 + "video_url": {"url": video_url},
2308 + "uuid": video_uuid,
2309 + },
2310 + ],
2311 + }
2312 + ]
2313 +
2314 + conversation, mm_data, mm_uuids = parse_chat_messages(
2315 + messages,
2316 + kimi_k2_5_model_config,
2317 + content_format="string",
2318 + )
2319 +
2320 + placeholder = "<|kimi_k25_video_placeholder|>"
2321 + expected_conversation = [
2322 + {
2323 + "role": "user",
2324 + "content": f"{placeholder}\nAnalyze this video.",
2325 + }
2326 + ]
2327 +
2328 + assert conversation == expected_conversation
2329 + _assert_mm_data_is_vision_chunk_input(mm_data, 1)
2330 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[video_uuid], modality="vision_chunk")
2331 +
2332 +
2333 +def test_parse_chat_messages_mixed_vision_chunk(
2334 + kimi_k2_5_model_config,
2335 + image_url,
2336 + video_url,
2337 +):
2338 + messages = [
2339 + {
2340 + "role": "user",
2341 + "content": [
2342 + {"type": "text", "text": "Analyze this image and video."},
2343 + {
2344 + "type": "image_url",
2345 + "image_url": {"url": image_url},
2346 + },
2347 + {
2348 + "type": "video_url",
2349 + "video_url": {"url": video_url},
2350 + },
2351 + ],
2352 + }
2353 + ]
2354 +
2355 + conversation, mm_data, mm_uuids = parse_chat_messages(
2356 + messages,
2357 + kimi_k2_5_model_config,
2358 + content_format="string",
2359 + )
2360 +
2361 + image_placeholder = (
2362 + "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
2363 + )
2364 + video_placeholder = "<|kimi_k25_video_placeholder|>"
2365 + expected_conversation = [
2366 + {
2367 + "role": "user",
2368 + "content": (
2369 + f"{image_placeholder}\n{video_placeholder}\n"
2370 + "Analyze this image and video."
2371 + ),
2372 + }
2373 + ]
2374 +
2375 + assert conversation == expected_conversation
2376 + _assert_mm_data_is_vision_chunk_input(mm_data, 2)
2377 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None], modality="vision_chunk")
2378 +
2379 +
2380 +def test_parse_chat_messages_mixed_vision_chunk_with_uuid(
2381 + kimi_k2_5_model_config,
2382 + image_url,
2383 + video_url,
2384 +):
2385 + image_uuid = "image_123"
2386 + video_uuid = "video_456"
2387 + messages = [
2388 + {
2389 + "role": "user",
2390 + "content": [
2391 + {"type": "text", "text": "Analyze this image and video."},
2392 + {
2393 + "type": "image_url",
2394 + "image_url": {"url": image_url},
2395 + "uuid": image_uuid,
2396 + },
2397 + {
2398 + "type": "video_url",
2399 + "video_url": {"url": video_url},
2400 + "uuid": video_uuid,
2401 + },
2402 + ],
2403 + }
2404 + ]
2405 +
2406 + conversation, mm_data, mm_uuids = parse_chat_messages(
2407 + messages,
2408 + kimi_k2_5_model_config,
2409 + content_format="string",
2410 + )
2411 +
2412 + image_placeholder = (
2413 + "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
2414 + )
2415 + video_placeholder = "<|kimi_k25_video_placeholder|>"
2416 + expected_conversation = [
2417 + {
2418 + "role": "user",
2419 + "content": (
2420 + f"{image_placeholder}\n{video_placeholder}\n"
2421 + "Analyze this image and video."
2422 + ),
2423 + }
2424 + ]
2425 +
2426 + assert conversation == expected_conversation
2427 + _assert_mm_data_is_vision_chunk_input(mm_data, 2)
2428 + _assert_mm_uuids(
2429 + mm_uuids, 2, expected_uuids=[image_uuid, video_uuid], modality="vision_chunk"
2430 + )
2431 +
2432 +
2433 +@pytest.mark.asyncio
2434 +async def test_parse_chat_messages_mixed_vision_chunk_async(
2435 + kimi_k2_5_model_config,
2436 + image_url,
2437 + video_url,
2438 +):
2439 + messages = [
2440 + {
2441 + "role": "user",
2442 + "content": [
2443 + {"type": "text", "text": "Analyze this image and video."},
2444 + {
2445 + "type": "image_url",
2446 + "image_url": {"url": image_url},
2447 + },
2448 + {
2449 + "type": "video_url",
2450 + "video_url": {"url": video_url},
2451 + },
2452 + ],
2453 + }
2454 + ]
2455 +
2456 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
2457 + messages,
2458 + kimi_k2_5_model_config,
2459 + content_format="string",
2460 + )
2461 +
2462 + image_placeholder = (
2463 + "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
2464 + )
2465 + video_placeholder = "<|kimi_k25_video_placeholder|>"
2466 + expected_conversation = [
2467 + {
2468 + "role": "user",
2469 + "content": (
2470 + f"{image_placeholder}\n{video_placeholder}\n"
2471 + "Analyze this image and video."
2472 + ),
2473 + }
2474 + ]
2475 +
2476 + assert conversation == expected_conversation
2477 + _assert_mm_data_is_vision_chunk_input(mm_data, 2)
2478 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None], modality="vision_chunk")
2479 +
2480 +
2481 +@pytest.mark.asyncio
2482 +async def test_parse_chat_messages_mixed_vision_chunk_with_uuid_async(
2483 + kimi_k2_5_model_config,
2484 + image_url,
2485 + video_url,
2486 +):
2487 + image_uuid = "image_123"
2488 + video_uuid = "video_456"
2489 + messages = [
2490 + {
2491 + "role": "user",
2492 + "content": [
2493 + {"type": "text", "text": "Analyze this image and video."},
2494 + {
2495 + "type": "image_url",
2496 + "image_url": {"url": image_url},
2497 + "uuid": image_uuid,
2498 + },
2499 + {
2500 + "type": "video_url",
2501 + "video_url": {"url": video_url},
2502 + "uuid": video_uuid,
2503 + },
2504 + ],
2505 + }
2506 + ]
2507 +
2508 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
2509 + messages,
2510 + kimi_k2_5_model_config,
2511 + content_format="string",
2512 + )
2513 +
2514 + image_placeholder = (
2515 + "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
2516 + )
2517 + video_placeholder = "<|kimi_k25_video_placeholder|>"
2518 + expected_conversation = [
2519 + {
2520 + "role": "user",
2521 + "content": (
2522 + f"{image_placeholder}\n{video_placeholder}\n"
2523 + "Analyze this image and video."
2524 + ),
2525 + }
2526 + ]
2527 +
2528 + assert conversation == expected_conversation
2529 + _assert_mm_data_is_vision_chunk_input(mm_data, 2)
2530 + _assert_mm_uuids(
2531 + mm_uuids, 2, expected_uuids=[image_uuid, video_uuid], modality="vision_chunk"
2532 + )
2533 +
2534 +
2535 +@pytest.mark.asyncio
2536 +async def test_parse_chat_messages_image_vision_chunk_async(
2537 + kimi_k2_5_model_config,
2538 + image_url,
2539 +):
2540 + messages = [
2541 + {
2542 + "role": "user",
2543 + "content": [
2544 + {"type": "text", "text": "Analyze this image."},
2545 + {
2546 + "type": "image_url",
2547 + "image_url": {"url": image_url},
2548 + },
2549 + ],
2550 + }
2551 + ]
2552 +
2553 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
2554 + messages,
2555 + kimi_k2_5_model_config,
2556 + content_format="string",
2557 + )
2558 +
2559 + placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
2560 + expected_conversation = [
2561 + {
2562 + "role": "user",
2563 + "content": f"{placeholder}\nAnalyze this image.",
2564 + }
2565 + ]
2566 +
2567 + assert conversation == expected_conversation
2568 + _assert_mm_data_is_vision_chunk_input(mm_data, 1)
2569 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
2570 +
2571 +
2572 +@pytest.mark.asyncio
2573 +async def test_parse_chat_messages_video_vision_chunk_async(
2574 + kimi_k2_5_model_config,
2575 + video_url,
2576 +):
2577 + messages = [
2578 + {
2579 + "role": "user",
2580 + "content": [
2581 + {"type": "text", "text": "Analyze this video."},
2582 + {
2583 + "type": "video_url",
2584 + "video_url": {"url": video_url},
2585 + },
2586 + ],
2587 + }
2588 + ]
2589 +
2590 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
2591 + messages,
2592 + kimi_k2_5_model_config,
2593 + content_format="string",
2594 + )
2595 +
2596 + placeholder = "<|kimi_k25_video_placeholder|>"
2597 + expected_conversation = [
2598 + {
2599 + "role": "user",
2600 + "content": f"{placeholder}\nAnalyze this video.",
2601 + }
2602 + ]
2603 +
2604 + assert conversation == expected_conversation
2605 + _assert_mm_data_is_vision_chunk_input(mm_data, 1)
2606 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None], modality="vision_chunk")
2607 +
2608 +
2609 +@pytest.mark.asyncio
2610 +async def test_parse_chat_messages_image_vision_chunk_with_uuid_async(
2611 + kimi_k2_5_model_config,
2612 + image_url,
2613 +):
2614 + image_uuid = "image_123"
2615 + messages = [
2616 + {
2617 + "role": "user",
2618 + "content": [
2619 + {"type": "text", "text": "Analyze this image."},
2620 + {
2621 + "type": "image_url",
2622 + "image_url": {"url": image_url},
2623 + "uuid": image_uuid,
2624 + },
2625 + ],
2626 + }
2627 + ]
2628 +
2629 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
2630 + messages,
2631 + kimi_k2_5_model_config,
2632 + content_format="string",
2633 + )
2634 +
2635 + placeholder = "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
2636 + expected_conversation = [
2637 + {
2638 + "role": "user",
2639 + "content": f"{placeholder}\nAnalyze this image.",
2640 + }
2641 + ]
2642 +
2643 + assert conversation == expected_conversation
2644 + _assert_mm_data_is_vision_chunk_input(mm_data, 1)
2645 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid], modality="vision_chunk")
2646 +
2647 +
2648 +@pytest.mark.asyncio
2649 +async def test_parse_chat_messages_video_vision_chunk_with_uuid_async(
2650 + kimi_k2_5_model_config,
2651 + video_url,
2652 +):
2653 + video_uuid = "video_456"
2654 + messages = [
2655 + {
2656 + "role": "user",
2657 + "content": [
2658 + {"type": "text", "text": "Analyze this video."},
2659 + {
2660 + "type": "video_url",
2661 + "video_url": {"url": video_url},
2662 + "uuid": video_uuid,
2663 + },
2664 + ],
2665 + }
2666 + ]
2667 +
2668 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
2669 + messages,
2670 + kimi_k2_5_model_config,
2671 + content_format="string",
2672 + )
2673 +
2674 + placeholder = "<|kimi_k25_video_placeholder|>"
2675 + expected_conversation = [
2676 + {
2677 + "role": "user",
2678 + "content": f"{placeholder}\nAnalyze this video.",
2679 + }
2680 + ]
2681 +
2682 + assert conversation == expected_conversation
2683 + _assert_mm_data_is_vision_chunk_input(mm_data, 1)
2684 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[video_uuid], modality="vision_chunk")
tests/models/multimodal/processing/test_common.py
+5
@@ -411,6 +411,11 @@ def test_processing_correctness(
411 "Qwen-VL tokenizer requires downloading a font file from "
412 "servers that often refuse connections in CI"
413 )
414 + if model_id == "moonshotai/Kimi-K2.5":
415 + # FIXME(Isaac): Fix Kimi-K2.5's offline inference about vision chunks.
416 + pytest.skip(
417 + "Kimi-K2.5's offline inference has issues about vision chunks. Fix later."
418 + )
419
420 _test_processing_correctness(
421 model_id,
tests/models/multimodal/processing/test_tensor_schema.py
+6
@@ -155,6 +155,12 @@ def initialize_dummy_model(
155 @create_new_process_for_each_test()
156 @pytest.mark.parametrize("model_id", get_model_ids_to_test())
157 def test_model_tensor_schema(model_id: str):
158 + if model_id == "moonshotai/Kimi-K2.5":
159 + # FIXME(Isotr0py): Fix Kimi-K2.5's offline inference about vision chunks.
160 + pytest.skip(
161 + "Kimi-K2.5's offline inference has issues about vision chunks. Fix later."
162 + )
163 +
164 model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
165 model_info.check_available_online(on_fail="skip")
166 model_info.check_transformers_version(
tests/models/registry.py
-1
@@ -786,7 +786,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
786 "KimiK25ForConditionalGeneration": _HfExamplesInfo(
787 "moonshotai/Kimi-K2.5",
788 trust_remote_code=True,
789 - is_available_online=False,
789 ),
790 "LightOnOCRForConditionalGeneration": _HfExamplesInfo(
791 "lightonai/LightOnOCR-1B-1025"
vllm/entrypoints/chat_utils.py
+68 -136
@@ -454,78 +454,6 @@ def _get_embeds_data(
454 raise NotImplementedError(type(data_items))
455
456
457 -def rebuild_mm_uuids_from_mm_data(
458 - mm_uuids: MultiModalUUIDDict,
459 - mm_data: MultiModalDataDict,
460 -) -> MultiModalUUIDDict:
461 - """Rebuild mm_uuids after vision_chunk processing.
462 -
463 - When videos are split into chunks, the original UUIDs need to be updated
464 - to reflect the new UUIDs generated for each chunk.
465 -
466 - Args:
467 - mm_uuids: Original UUIDs dictionary
468 - mm_data: Processed multimodal data with vision_chunk items
469 -
470 - Returns:
471 - Updated UUIDs dictionary with chunk UUIDs
472 - """
473 - vision_chunks = mm_data.get("vision_chunk")
474 - if vision_chunks is None:
475 - return mm_uuids
476 -
477 - new_uuids = dict(mm_uuids)
478 - vision_chunk_uuids = []
479 -
480 - for item in vision_chunks:
481 - # vision_chunk items are always dicts (VisionChunkImage/VisionChunkVideo)
482 - assert isinstance(item, dict)
483 - uuid_val = item.get("uuid")
484 - if uuid_val is not None:
485 - vision_chunk_uuids.append(uuid_val)
486 -
487 - if vision_chunk_uuids:
488 - new_uuids["vision_chunk"] = vision_chunk_uuids
489 -
490 - return new_uuids
491 -
492 -
493 -def build_video_prompts_from_mm_data(
494 - mm_data: MultiModalDataDict,
495 -) -> list[str]:
496 - """Build video prompts from vision_chunk data.
497 -
498 - Collects prompts from video chunks and groups them by video_idx.
499 -
500 - Args:
501 - mm_data: Processed multimodal data with vision_chunk items
502 -
503 - Returns:
504 - List of video prompts, one per video.
505 - """
506 - vision_chunks = mm_data.get("vision_chunk")
507 - if vision_chunks is None:
508 - return []
509 -
510 - # Group chunks by video_idx
511 - video_prompts_dict: dict[int, list[str]] = defaultdict(list)
512 -
513 - for item in vision_chunks:
514 - # vision_chunk items are always dicts (VisionChunkImage/VisionChunkVideo)
515 - assert isinstance(item, dict)
516 - if item.get("type") == "video_chunk":
517 - video_idx = item.get("video_idx", 0)
518 - prompt = item.get("prompt", "")
519 - video_prompts_dict[video_idx].append(prompt)
520 -
521 - # Build prompts in video order
522 - video_prompts = []
523 - for video_idx in sorted(video_prompts_dict.keys()):
524 - video_prompts.append("".join(video_prompts_dict[video_idx]))
525 -
526 - return video_prompts
527 -
528 -
457 class BaseMultiModalItemTracker(ABC, Generic[_T]):
458 """
459 Tracks multi-modal items in a given request and ensures that the number
@@ -616,10 +544,72 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
544 raise NotImplementedError
545
546
547 +def _resolve_vision_chunk_items(
548 + vision_chunk_items: list[tuple[object, str | None]],
549 + mm_processor: BaseMultiModalProcessor,
550 + vision_chunks_modality_order: list[str],
551 +):
552 + # Process vision_chunk items - extract from (data, modality) tuples
553 + # and convert to VisionChunk types with proper UUID handling
554 + vision_chunks_uuids = [uuid for data, uuid in vision_chunk_items]
555 +
556 + assert len(vision_chunk_items) == len(vision_chunks_modality_order), (
557 + f"vision_chunk items ({len(vision_chunk_items)}) and "
558 + f"modality_order ({len(vision_chunks_modality_order)}) must have same length"
559 + )
560 +
561 + processed_chunks: list[VisionChunk] = []
562 + video_idx = 0
563 + for inner_modality, (data, uuid) in zip(
564 + vision_chunks_modality_order, vision_chunk_items
565 + ):
566 + if inner_modality == "image":
567 + # Cast data to proper type for image
568 + # Use .media (PIL.Image) directly to avoid redundant
569 + # bytes→PIL conversion in media_processor
570 + if hasattr(data, "media"):
571 + image_data = data.media # type: ignore[union-attr]
572 + processed_chunks.append(
573 + VisionChunkImage(type="image", image=image_data, uuid=uuid)
574 + )
575 + else:
576 + processed_chunks.append(data) # type: ignore[arg-type]
577 + elif inner_modality == "video":
578 + # For video, we may need to split into chunks
579 + # if processor supports it
580 + # For now, just wrap as a video chunk placeholder
581 + if hasattr(mm_processor, "split_video_chunks") and data is not None:
582 + try:
583 + video_uuid = uuid or random_uuid()
584 + # video await result is (video_data, video_meta) tuple
585 + if isinstance(data, tuple) and len(data) >= 1:
586 + video_data = data[0]
587 + else:
588 + video_data = data
589 + video_chunks = mm_processor.split_video_chunks(video_data)
590 + for i, vc in enumerate(video_chunks):
591 + processed_chunks.append(
592 + VisionChunkVideo(
593 + type="video_chunk",
594 + video_chunk=vc["video_chunk"],
595 + uuid=f"{video_uuid}-{i}",
596 + video_idx=video_idx,
597 + prompt=vc["prompt"],
598 + )
599 + )
600 + video_idx += 1
601 + except Exception as e:
602 + logger.warning("Failed to split video chunks: %s", e)
603 + processed_chunks.append(data) # type: ignore[arg-type]
604 + else:
605 + processed_chunks.append(data) # type: ignore[arg-type]
606 + return processed_chunks, vision_chunks_uuids
607 +
608 +
609 def _resolve_items(
610 items_by_modality: dict[str, list[tuple[object, str | None]]],
611 mm_processor: BaseMultiModalProcessor,
622 - vision_chunk_modality_order: dict[str, list[str]],
612 + modality_order: dict[str, list[str]],
613 ) -> tuple[MultiModalDataDict, MultiModalUUIDDict]:
614 if "image" in items_by_modality and "image_embeds" in items_by_modality:
615 raise ValueError("Mixing raw image and embedding inputs is not allowed")
@@ -654,71 +644,13 @@ def _resolve_items(
644 if "vision_chunk" in items_by_modality:
645 # Process vision_chunk items - extract from (data, modality) tuples
646 # and convert to VisionChunk types with proper UUID handling
657 - vision_chunk_items = items_by_modality["vision_chunk"]
658 - modality_order = vision_chunk_modality_order.get("vision_chunk", [])
659 - mm_uuids["vision_chunk"] = [
660 - uuid for data, uuid in items_by_modality["vision_chunk"]
661 - ]
662 -
663 - # Filter out None items (from asyncio.sleep(0) placeholders)
664 - filtered_items = [
665 - (idx, item)
666 - for idx, item in enumerate(vision_chunk_items)
667 - if item is not None
668 - ]
669 -
670 - assert len(filtered_items) == len(modality_order), (
671 - f"vision_chunk items ({len(filtered_items)}) and "
672 - f"modality_order ({len(modality_order)}) must have same length"
647 + processed_chunks, vision_chunk_uuids = _resolve_vision_chunk_items(
648 + items_by_modality["vision_chunk"],
649 + mm_processor,
650 + modality_order.get("vision_chunk", []),
651 )
674 -
675 - processed_chunks: list[VisionChunk] = []
676 - video_idx = 0
677 - for i, (idx, item) in enumerate(filtered_items):
678 - inner_modality = modality_order[i]
679 - data, uuid = item
680 - uuid_val = uuid if idx < len(mm_uuids["vision_chunk"]) else None
681 - if inner_modality == "image":
682 - # Cast data to proper type for image
683 - # Use .media (PIL.Image) directly to avoid redundant
684 - # bytes→PIL conversion in media_processor
685 - if hasattr(data, "media"):
686 - image_data = data.media # type: ignore[union-attr]
687 - processed_chunks.append(
688 - VisionChunkImage(type="image", image=image_data, uuid=uuid_val)
689 - )
690 - else:
691 - processed_chunks.append(data) # type: ignore[arg-type]
692 - elif inner_modality == "video":
693 - # For video, we may need to split into chunks
694 - # if processor supports it
695 - # For now, just wrap as a video chunk placeholder
696 - if hasattr(mm_processor, "split_video_chunks") and data is not None:
697 - try:
698 - video_uuid = uuid_val or random_uuid()
699 - # video await result is (video_data, video_meta) tuple
700 - if isinstance(data, tuple) and len(data) >= 1:
701 - video_data = data[0]
702 - else:
703 - video_data = data
704 - video_chunks = mm_processor.split_video_chunks(video_data)
705 - for i, vc in enumerate(video_chunks):
706 - processed_chunks.append(
707 - VisionChunkVideo(
708 - type="video_chunk",
709 - video_chunk=vc["video_chunk"],
710 - uuid=f"{video_uuid}-{i}",
711 - video_idx=video_idx,
712 - prompt=vc["prompt"],
713 - )
714 - )
715 - video_idx += 1
716 - except Exception as e:
717 - logger.warning("Failed to split video chunks: %s", e)
718 - processed_chunks.append(data) # type: ignore[arg-type]
719 - else:
720 - processed_chunks.append(data) # type: ignore[arg-type]
652 mm_data["vision_chunk"] = processed_chunks
653 + mm_uuids["vision_chunk"] = vision_chunk_uuids
654
655 return mm_data, mm_uuids
656
vllm/multimodal/video.py
-21
@@ -235,27 +235,6 @@ class VideoLoader:
235 VIDEO_LOADER_REGISTRY = ExtensionManager()
236
237
238 -@VIDEO_LOADER_REGISTRY.register("identity")
239 -class IdentityVideoLoader(VideoLoader):
240 - """IdentityVideoLoader returns raw video bytes without decoding.
241 -
242 - This allows the model processor to handle video decoding and
243 - is required for models like Kimi-K2.5 that need custom video chunk splitting.
244 -
245 - NOTE: This is temporary for Kimi-K2.5 testing. Remember to change back
246 - to opencv before release if needed.
247 - """
248 -
249 - @classmethod
250 - def load_bytes(
251 - cls,
252 - data: bytes,
253 - num_frames: int = -1,
254 - **kwargs: Any,
255 - ) -> tuple[Any, Any]:
256 - return data, None
257 -
258 -
238 @VIDEO_LOADER_REGISTRY.register("opencv")
239 class OpenCVVideoBackend(VideoLoader):
240 def get_cv2_video_api(self):
vllm/renderers/hf.py
+123 -46
@@ -1,10 +1,11 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import inspect
4 -from collections import deque
4 +import itertools
5 +from collections import defaultdict, deque
6 from collections.abc import Set
7 from functools import lru_cache
7 -from typing import Any, cast
8 +from typing import TYPE_CHECKING, Any, cast
9
10 import jinja2
11 import jinja2.ext
@@ -20,11 +21,9 @@ from vllm.entrypoints.chat_utils import (
21 ChatTemplateContentFormatOption,
22 ChatTemplateResolutionError,
23 ConversationMessage,
23 - build_video_prompts_from_mm_data,
24 load_chat_template,
25 parse_chat_messages,
26 parse_chat_messages_async,
27 - rebuild_mm_uuids_from_mm_data,
27 )
28 from vllm.inputs import TextPrompt, TokensPrompt
29 from vllm.logger import init_logger
@@ -36,6 +35,13 @@ from vllm.utils.func_utils import supports_kw
35
36 from .protocol import RendererLike
37
38 +if TYPE_CHECKING:
39 + from vllm.multimodal.inputs import MultiModalDataDict, MultiModalUUIDDict
40 +else:
41 + MultiModalDataDict = dict[str, Any]
42 + MultiModalUUIDDict = dict[str, Any]
43 +
44 +
45 logger = init_logger(__name__)
46
47
@@ -479,6 +485,104 @@ def safe_apply_chat_template(
485 raise ValueError(str(e)) from e
486
487
488 +def rebuild_mm_uuids_from_mm_data(
489 + mm_uuids: "MultiModalUUIDDict",
490 + mm_data: "MultiModalDataDict",
491 +) -> "MultiModalUUIDDict":
492 + """Rebuild mm_uuids after vision_chunk processing.
493 +
494 + When videos are split into chunks, the original UUIDs need to be updated
495 + to reflect the new UUIDs generated for each chunk.
496 +
497 + Args:
498 + mm_uuids: Original UUIDs dictionary
499 + mm_data: Processed multimodal data with vision_chunk items
500 +
501 + Returns:
502 + Updated UUIDs dictionary with chunk UUIDs
503 + """
504 + vision_chunks = mm_data.get("vision_chunk")
505 + if vision_chunks is None:
506 + return mm_uuids
507 +
508 + assert all(isinstance(item, dict) for item in vision_chunks), (
509 + "Expected all vision_chunk items to be dicts"
510 + )
511 + vision_chunks = cast(list[dict[str, Any]], vision_chunks)
512 + vision_chunk_uuids = [
513 + uuid_val for item in vision_chunks if (uuid_val := item.get("uuid")) is not None
514 + ]
515 +
516 + if vision_chunk_uuids:
517 + mm_uuids = dict(mm_uuids)
518 + mm_uuids["vision_chunk"] = vision_chunk_uuids
519 +
520 + return mm_uuids
521 +
522 +
523 +def build_video_prompts_from_mm_data(
524 + mm_data: "MultiModalDataDict",
525 +) -> list[str]:
526 + """Build video prompts from vision_chunk data.
527 +
528 + Collects prompts from video chunks and groups them by video_idx.
529 +
530 + Args:
531 + mm_data: Processed multimodal data with vision_chunk items
532 +
533 + Returns:
534 + List of video prompts, one per video.
535 + """
536 + vision_chunks = mm_data.get("vision_chunk")
537 + if vision_chunks is None:
538 + return []
539 +
540 + # Group chunks by video_idx
541 + video_prompts_dict: dict[int, list[str]] = defaultdict(list)
542 +
543 + for item in vision_chunks:
544 + # vision_chunk items are always dicts (VisionChunkImage/VisionChunkVideo)
545 + assert isinstance(item, dict)
546 + if item.get("type") == "video_chunk":
547 + video_idx = item.get("video_idx", 0)
548 + prompt = item.get("prompt", "")
549 + video_prompts_dict[video_idx].append(prompt)
550 +
551 + # Build prompts in video order
552 + video_prompts = [
553 + "".join(video_prompts_dict[video_idx])
554 + for video_idx in sorted(video_prompts_dict.keys())
555 + ]
556 +
557 + return video_prompts
558 +
559 +
560 +def replace_vision_chunk_video_placeholder(
561 + prompt_raw: str | list[int],
562 + mm_data: "MultiModalDataDict",
563 + video_placeholder: str | None,
564 +) -> str | list[int]:
565 + # get video placehoder, replace it with runtime video-chunk prompts
566 + if video_placeholder and isinstance(prompt_raw, str):
567 + video_prompts = build_video_prompts_from_mm_data(mm_data)
568 +
569 + # replace in order
570 + prompt_raw_parts = prompt_raw.split(video_placeholder)
571 + if len(prompt_raw_parts) == len(video_prompts) + 1:
572 + prompt_raw = "".join(
573 + itertools.chain.from_iterable(zip(prompt_raw_parts, video_prompts))
574 + )
575 + prompt_raw += prompt_raw_parts[-1]
576 + else:
577 + logger.warning(
578 + "Number of video placeholders (%d) does not match "
579 + "number of videos (%d) in the request.",
580 + len(prompt_raw_parts) - 1,
581 + len(video_prompts),
582 + )
583 + return prompt_raw
584 +
585 +
586 class HfRenderer(RendererLike):
587 @classmethod
588 def from_config(
@@ -496,6 +600,9 @@ class HfRenderer(RendererLike):
600 super().__init__()
601
602 self.config = config
603 + self.use_unified_vision_chunk = getattr(
604 + config.hf_config, "use_unified_vision_chunk", False
605 + )
606
607 if config.skip_tokenizer_init:
608 tokenizer = None
@@ -552,7 +659,7 @@ class HfRenderer(RendererLike):
659 # NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
660 # model which uses unified vision chunks for both images and videos.
661 if (
555 - getattr(model_config.hf_config, "use_unified_vision_chunk", False)
662 + self.use_unified_vision_chunk
663 and mm_uuids is not None
664 and mm_data is not None
665 ):
@@ -562,26 +669,11 @@ class HfRenderer(RendererLike):
669 video_placeholder = getattr(
670 model_config.hf_config, "video_placeholder", None
671 )
565 - if video_placeholder and isinstance(prompt_raw, str):
566 - video_prompts = build_video_prompts_from_mm_data(mm_data)
567 -
568 - # replace in order
569 - prompt_raw_parts = prompt_raw.split(video_placeholder)
570 - if len(prompt_raw_parts) == len(video_prompts) + 1:
571 - prompt_raw = "".join(
572 - [
573 - prompt_raw_parts[i] + video_prompts[i]
574 - for i in range(len(video_prompts))
575 - ]
576 - )
577 - prompt_raw += prompt_raw_parts[-1]
578 - else:
579 - logger.warning(
580 - "Number of video placeholders (%d) does not match "
581 - "number of videos (%d) in the request.",
582 - len(prompt_raw_parts) - 1,
583 - len(video_prompts),
584 - )
672 + prompt_raw = replace_vision_chunk_video_placeholder(
673 + prompt_raw,
674 + mm_data,
675 + video_placeholder,
676 + )
677
678 prompt = (
679 TextPrompt(prompt=prompt_raw)
@@ -626,7 +718,7 @@ class HfRenderer(RendererLike):
718 # NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
719 # model which uses unified vision chunks for both images and videos.
720 if (
629 - getattr(model_config.hf_config, "use_unified_vision_chunk", False)
721 + self.use_unified_vision_chunk
722 and mm_uuids is not None
723 and mm_data is not None
724 ):
@@ -636,26 +728,11 @@ class HfRenderer(RendererLike):
728 video_placeholder = getattr(
729 model_config.hf_config, "video_placeholder", None
730 )
639 - if video_placeholder and isinstance(prompt_raw, str):
640 - video_prompts = build_video_prompts_from_mm_data(mm_data)
641 -
642 - # replace in order
643 - prompt_raw_parts = prompt_raw.split(video_placeholder)
644 - if len(prompt_raw_parts) == len(video_prompts) + 1:
645 - prompt_raw = "".join(
646 - [
647 - prompt_raw_parts[i] + video_prompts[i]
648 - for i in range(len(video_prompts))
649 - ]
650 - )
651 - prompt_raw += prompt_raw_parts[-1]
652 - else:
653 - logger.warning(
654 - "Number of video placeholders (%d) does not match "
655 - "number of videos (%d) in the request.",
656 - len(prompt_raw_parts) - 1,
657 - len(video_prompts),
658 - )
731 + prompt_raw = replace_vision_chunk_video_placeholder(
732 + prompt_raw,
733 + mm_data,
734 + video_placeholder,
735 + )
736
737 prompt = (
738 TextPrompt(prompt=prompt_raw)