@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 361c29e174
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Fix M-RoPE position calculation when chunked prefill is enabled (#10388)

Signed-off-by: imkero <kerorek@outlook.com>

电脑星人 committed Nov 17, 2024 at 02:10 UTC 361c29e1740e0b2186f8cca3ed96ad235a8a960a
3 files changed +135 -5
tests/models/decoder_only/vision_language/test_qwen2_vl.py
+132 -4
@@ -18,6 +18,7 @@ target_dtype = "half"
18
19 IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
20 VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
21 +MODEL_HIDDEN_SIZE = 1536
22
23
24 def qwen2_vl_chat_template(*query):
@@ -230,7 +231,7 @@ def batch_make_video_embeddings(
231 return result
232
233
233 -def run_test(
234 +def run_embedding_input_test(
235 vllm_runner: Type[VllmRunner],
236 inputs: List[Tuple[List[str], PromptImageInput, PromptVideoInput]],
237 model: str,
@@ -326,7 +327,7 @@ def test_qwen2_vl_image_embeddings_input(vllm_runner, image_assets, model,
327 [],
328 ) for image, prompt in zip(images, IMAGE_PROMPTS)]
329
329 - run_test(
330 + run_embedding_input_test(
331 vllm_runner,
332 inputs_per_case,
333 model,
@@ -371,7 +372,7 @@ def test_qwen2_vl_multiple_image_embeddings_input(vllm_runner, image_assets,
372 [],
373 )]
374
374 - run_test(
375 + run_embedding_input_test(
376 vllm_runner,
377 inputs_per_case,
378 model,
@@ -416,7 +417,134 @@ def test_qwen2_vl_video_embeddings_input(vllm_runner, video_assets, model,
417 [rescale_video_size(video, factor) for factor in size_factors],
418 ) for video, prompt in zip(sampled_vids, VIDEO_PROMPTS)]
419
419 - run_test(
420 + run_embedding_input_test(
421 + vllm_runner,
422 + inputs_per_case,
423 + model,
424 + dtype=dtype,
425 + max_tokens=max_tokens,
426 + num_logprobs=num_logprobs,
427 + mm_limit=1,
428 + tensor_parallel_size=1,
429 + )
430 +
431 +
432 +def run_chunked_prefill_test(
433 + vllm_runner: Type[VllmRunner],
434 + inputs: List[Tuple[List[str], PromptImageInput, PromptVideoInput]],
435 + model: str,
436 + *,
437 + dtype: str,
438 + max_tokens: int,
439 + num_logprobs: int,
440 + mm_limit: int,
441 + tensor_parallel_size: int,
442 + distributed_executor_backend: Optional[str] = None,
443 +):
444 + """Compare inference result between
445 + chunked prefill disabled and chunked prefill enabled
446 + """
447 +
448 + # NOTE:
449 + # max_model_len should be greater than image_feature_size
450 + with vllm_runner(model,
451 + task="generate",
452 + max_model_len=4000,
453 + max_num_seqs=4,
454 + dtype=dtype,
455 + limit_mm_per_prompt={
456 + "image": mm_limit,
457 + "video": mm_limit
458 + },
459 + tensor_parallel_size=tensor_parallel_size,
460 + distributed_executor_backend=distributed_executor_backend
461 + ) as vllm_model:
462 +
463 + outputs_per_case = [
464 + vllm_model.generate_greedy_logprobs(prompts,
465 + max_tokens,
466 + num_logprobs=num_logprobs,
467 + images=images or None,
468 + videos=videos or None)
469 + for prompts, images, videos in inputs
470 + ]
471 +
472 + with vllm_runner(
473 + model,
474 + task="generate",
475 + max_model_len=4000,
476 + max_num_seqs=4,
477 + dtype=dtype,
478 + limit_mm_per_prompt={
479 + "image": mm_limit,
480 + "video": mm_limit
481 + },
482 + tensor_parallel_size=tensor_parallel_size,
483 + distributed_executor_backend=distributed_executor_backend,
484 + enable_chunked_prefill=True,
485 + # should be small enough to ensure prefilling is chunked
486 + max_num_batched_tokens=32,
487 + mm_processor_kwargs={
488 + "max_pixels": 16 * 28 * 28,
489 + }) as vllm_model_chunked:
490 + outputs_per_case_chunked = [
491 + vllm_model_chunked.generate_greedy_logprobs(
492 + prompts,
493 + max_tokens,
494 + num_logprobs=num_logprobs,
495 + images=images or None,
496 + videos=videos or None) for prompts, images, videos in inputs
497 + ]
498 +
499 + for outputs, \
500 + outputs_chunked \
501 + in zip(outputs_per_case,
502 + outputs_per_case_chunked):
503 + check_logprobs_close(
504 + outputs_0_lst=outputs,
505 + outputs_1_lst=outputs_chunked,
506 + name_0="non_chunked",
507 + name_1="chunked",
508 + )
509 +
510 +
511 +@pytest.mark.core_model
512 +@pytest.mark.parametrize("model", models)
513 +@pytest.mark.parametrize("dtype", [target_dtype])
514 +@pytest.mark.parametrize("max_tokens", [1])
515 +@pytest.mark.parametrize("num_logprobs", [10])
516 +def test_qwen2_vl_mrope_chunked_prefill(vllm_runner, example_prompts,
517 + model: str, dtype: str,
518 + max_tokens: int,
519 + num_logprobs: int) -> None:
520 + """
521 + Test Qwen2-VL's chunked prefill with M-RoPE
522 + """
523 + prompts = [
524 + qwen2_vl_chat_template(IMAGE_PLACEHOLDER, prompt)
525 + for prompt in example_prompts[:1]
526 + ]
527 +
528 + # 1. Qwen2-VL's M-RoPE works only when there are some multi-modal inputs,
529 + # so an image is included in the inputs
530 + # 2. however, Qwen2-VL currently won't work properly
531 + # when chunked prefill is enabled and there are some multi-modal inputs,
532 + # here use a hacky way: provide a **zero-length** image to make it happy
533 + #
534 + # and finally we achieved:
535 + # (1) chunked_prefill enabled; (2) M-RoPE works; to continue our tests
536 + zero_len_image = {
537 + "image_embeds": torch.empty((0, MODEL_HIDDEN_SIZE)),
538 + "image_grid_thw": torch.tensor([[0, 0, 0]])
539 + }
540 + images = [zero_len_image] * len(prompts)
541 +
542 + inputs_per_case: List[Tuple[List[str], PromptImageInput,
543 + PromptVideoInput]] = [
544 + (prompts, images, []),
545 + ]
546 +
547 + run_chunked_prefill_test(
548 vllm_runner,
549 inputs_per_case,
550 model,
vllm/model_executor/layers/rotary_embedding.py
+2 -1
@@ -847,6 +847,7 @@ class MRotaryEmbedding(RotaryEmbedding):
847 vision_end_token_id: int,
848 spatial_merge_size: int,
849 context_len: int = 0,
850 + seq_len: Optional[int] = None,
851 ) -> Tuple[List[List[int]], int]:
852 """Get mrope input positions and delta value."""
853
@@ -921,7 +922,7 @@ class MRotaryEmbedding(RotaryEmbedding):
922 torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
923
924 llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
924 - llm_positions = llm_positions[:, context_len:]
925 + llm_positions = llm_positions[:, context_len:seq_len]
926 mrope_position_delta = (llm_positions.max() + 1 -
927 len(input_tokens)).item()
928
vllm/worker/model_runner.py
+1
@@ -700,6 +700,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
700 spatial_merge_size=hf_config.vision_config.
701 spatial_merge_size,
702 context_len=inter_data.context_lens[seq_idx],
703 + seq_len=inter_data.seq_lens[seq_idx],
704 )
705
706 seq_data.mrope_position_delta = mrope_position_delta