@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 036ca94c25
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix] handle alignment of arguments in convert_sparse_cross_attention_mask_to_dense (#12347)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Signed-off-by: Wallas Santos <wallashss@ibm.com> Co-authored-by: Wallas Santos <wallashss@ibm.com>

Travis Johnson committed Jan 29, 2025 at 01:54 UTC 036ca94c25fa07391016aa1b4f93a8ac5d74f296
2 files changed +222 -4
tests/models/encoder_decoder/vision_language/test_mllama.py
+208
@@ -1,11 +1,15 @@
1 from typing import List, Optional, Tuple, Type, overload
2
3 import pytest
4 +import torch
5 from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
6 BatchEncoding)
7
8 +from vllm.attention.backends.flash_attn import FlashAttentionMetadata
9 from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
10 global_force_attn_backend_context_manager)
11 +from vllm.model_executor.models.mllama import (MLLAMA_IMAGE_TOKEN_ID,
12 + MllamaForConditionalGeneration)
13 from vllm.multimodal.image import rescale_image_size
14 from vllm.sequence import SampleLogprobs
15
@@ -33,6 +37,29 @@ models = [
37 "meta-llama/Llama-3.2-11B-Vision-Instruct",
38 ]
39
40 +# Indices for inputs
41 +TEXT_ONLY = '0'
42 +IMAGE_AT_BEG = '1'
43 +IMAGE_AT_MIDDLE = '2'
44 +TWO_IMAGES = '3'
45 +
46 +# Input tokenized
47 +prompt_data = {
48 + # Tell me a story
49 + TEXT_ONLY: [41551, 757, 264, 3446],
50 + # <|image|> What's the content of this image
51 + IMAGE_AT_BEG:
52 + [MLLAMA_IMAGE_TOKEN_ID, 3639, 596, 279, 2262, 315, 420, 2217, 220],
53 + # Hello <|image|>What' the content of this image
54 + IMAGE_AT_MIDDLE:
55 + [9906, 220, MLLAMA_IMAGE_TOKEN_ID, 3923, 6, 279, 2262, 315, 420, 2217],
56 + #<|image|>Is there a duck in this image?<|image|>What's the animal in this image? # noqa: E501
57 + TWO_IMAGES: [
58 + MLLAMA_IMAGE_TOKEN_ID, 3957, 1070, 264, 37085, 304, 420, 2217, 30,
59 + MLLAMA_IMAGE_TOKEN_ID, 3923, 596, 279, 10065, 304, 420, 2217, 30
60 + ]
61 +}
62 +
63
64 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
65 Optional[SampleLogprobs]],
@@ -365,3 +392,184 @@ def test_models_interleaved_images(hf_runner, vllm_runner, image_assets, model,
392 num_logprobs=num_logprobs,
393 tensor_parallel_size=1,
394 )
395 +
396 +
397 +@large_gpu_test(min_gb=48)
398 +@pytest.mark.core_model
399 +@pytest.mark.parametrize("model", models)
400 +@pytest.mark.parametrize("dtype", ["bfloat16"])
401 +@pytest.mark.parametrize("max_tokens", [128])
402 +@pytest.mark.parametrize("num_logprobs", [5])
403 +@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
404 +def test_regression(vllm_runner, image_assets, model, dtype, max_tokens,
405 + num_logprobs, attn_backend: _Backend) -> None:
406 +
407 + stop_sign = image_assets[0].pil_image
408 +
409 + with global_force_attn_backend_context_manager(attn_backend), vllm_runner(
410 + model,
411 + dtype=dtype,
412 + max_model_len=4096,
413 + max_num_seqs=2,
414 + tensor_parallel_size=1,
415 + enforce_eager=True,
416 + limit_mm_per_prompt={"image":
417 + _LIMIT_IMAGE_PER_PROMPT}) as vllm_model:
418 +
419 + # Regression tests for https://github.com/vllm-project/vllm/issues/10648
420 +
421 + # Number of image tags is greater than the number of images provided
422 + prompt = "<|begin_of_text|><|image|><|image|> Compare the two images" # noqa: E501
423 + image = stop_sign
424 + with pytest.raises(ValueError):
425 + vllm_model.generate_greedy_logprobs([prompt],
426 + max_tokens,
427 + num_logprobs,
428 + images=[image])
429 +
430 + # Batch of a text-only and image request that requires cross-attention
431 + prompts = [
432 + "What is the capital of spain?",
433 + "Text before the image...<|image|>What is in the image?", # noqa: E501
434 + ]
435 + images = [
436 + None,
437 + [stop_sign],
438 + ]
439 + vllm_model.generate_greedy_logprobs(prompts,
440 + max_tokens,
441 + num_logprobs,
442 + images=images)
443 +
444 + # Test the reverse order too for good measure
445 + prompts = [
446 + "<|begin_of_text|>Text before the image...<|image|>What is in the image?", # noqa: E501
447 + "<|begin_of_text|>Hello!",
448 + ]
449 + images = [
450 + [stop_sign],
451 + None,
452 + ]
453 + vllm_model.generate_greedy_logprobs(prompts,
454 + max_tokens,
455 + num_logprobs,
456 + images=images)
457 +
458 +
459 +@pytest.mark.core_model
460 +@pytest.mark.parametrize(
461 + "input_indices_and_output",
462 + # inputs, (cross_attention_mask, kv_range_for_decode)
463 + [([TEXT_ONLY], (None, None)), ([IMAGE_AT_BEG], (None, None)),
464 + ([TEXT_ONLY, IMAGE_AT_BEG], (None, None)),
465 + ([IMAGE_AT_MIDDLE], ((10, 12), [[0, 6]])),
466 + ([TEXT_ONLY, IMAGE_AT_MIDDLE], ((14, 12), [[0, 6]])),
467 + ([TEXT_ONLY, IMAGE_AT_BEG, IMAGE_AT_MIDDLE],
468 + ((23, 24), [[0, 6], [6, 12]])),
469 + ([IMAGE_AT_MIDDLE, TEXT_ONLY], ((14, 12), [[0, 6]])),
470 + ([TWO_IMAGES], ((18, 12), [[6, 12]])),
471 + ([TEXT_ONLY, TWO_IMAGES], ((22, 12), [[6, 12]]))])
472 +def test_get_cross_attention_mask(input_indices_and_output) -> None:
473 +
474 + input_indices, expected_output = input_indices_and_output
475 +
476 + sequences = [torch.tensor(prompt_data[i]) for i in input_indices]
477 + num_tiles = [[2, 2] if i != TEXT_ONLY else [] for i in input_indices
478 + if i != TEXT_ONLY]
479 + input = torch.cat(sequences)
480 +
481 + seq_lens = [len(s) for s in sequences]
482 +
483 + attn_data = FlashAttentionMetadata(
484 + seq_lens=seq_lens,
485 + # Dummy values
486 + enable_kv_scales_calculation=False,
487 + num_prefills=0,
488 + num_prefill_tokens=0,
489 + num_decode_tokens=0,
490 + slot_mapping=0,
491 + multi_modal_placeholder_index_maps=None,
492 + seq_lens_tensor=0,
493 + max_prefill_seq_len=0,
494 + max_decode_seq_len=0,
495 + context_lens_tensor=None,
496 + block_tables=None,
497 + use_cuda_graph=False,
498 + )
499 +
500 + dummy: dict[str, str] = {}
501 +
502 + cross_attention_mask, kv_range_for_decode = MllamaForConditionalGeneration\
503 + .get_cross_attention_mask(dummy,
504 + input,
505 + attn_data,
506 + num_tiles=num_tiles,
507 + num_tokens_per_tile=3,
508 + dtype=torch.bfloat16)
509 +
510 + expected_cross_attention_mask, expected_kv_range_for_decode = \
511 + expected_output
512 +
513 + assert kv_range_for_decode == expected_kv_range_for_decode
514 + if expected_cross_attention_mask is not None:
515 + assert cross_attention_mask is not None
516 + assert cross_attention_mask.shape == expected_cross_attention_mask
517 + else:
518 + assert cross_attention_mask is None
519 +
520 +
521 +@pytest.mark.core_model
522 +@pytest.mark.parametrize(
523 + "input_indices",
524 + [[TEXT_ONLY], [IMAGE_AT_BEG], [TEXT_ONLY, IMAGE_AT_BEG], [IMAGE_AT_MIDDLE],
525 + [TEXT_ONLY, IMAGE_AT_MIDDLE], [TEXT_ONLY, IMAGE_AT_BEG, IMAGE_AT_MIDDLE],
526 + [IMAGE_AT_MIDDLE, TEXT_ONLY], [TWO_IMAGES], [TEXT_ONLY, TWO_IMAGES]])
527 +def test_get_full_text_row_masked_out_mask(input_indices) -> None:
528 +
529 + sequences = [torch.tensor(prompt_data[i]) for i in input_indices]
530 +
531 + seq_lens = [len(s) for s in sequences]
532 +
533 + num_prefill_tokens = sum(seq_lens)
534 +
535 + # TEXT_ONLY is zero, so it will be masked out,
536 + # other instances should not be.
537 + encoder_seq_lens = [int(i) for i in input_indices]
538 +
539 + attn_data = FlashAttentionMetadata(
540 + seq_lens=seq_lens,
541 + encoder_seq_lens=encoder_seq_lens,
542 + num_prefill_tokens=num_prefill_tokens,
543 + # Dummy values
544 + enable_kv_scales_calculation=False,
545 + num_prefills=0,
546 + num_decode_tokens=0,
547 + slot_mapping=0,
548 + multi_modal_placeholder_index_maps=None,
549 + seq_lens_tensor=0,
550 + max_prefill_seq_len=0,
551 + max_decode_seq_len=0,
552 + context_lens_tensor=None,
553 + block_tables=None,
554 + use_cuda_graph=False,
555 + )
556 +
557 + dummy: dict[str, str] = {}
558 +
559 + full_text_row_masked_out_mask = MllamaForConditionalGeneration\
560 + .get_full_text_row_masked_out_mask(dummy,
561 + attn_data,
562 + torch.get_default_device())
563 +
564 + full_text_row_masked_out_mask = full_text_row_masked_out_mask.squeeze()
565 + full_text_row_masked_out_mask = full_text_row_masked_out_mask.tolist()
566 +
567 + idx = 0
568 + assert len(full_text_row_masked_out_mask) == num_prefill_tokens
569 + for i, seq_len in enumerate(seq_lens):
570 + must_be_masked = input_indices[i] != TEXT_ONLY
571 + for _ in range(seq_len):
572 + assert full_text_row_masked_out_mask[idx] == must_be_masked, \
573 + f"full_text_row_masked_out_mask[{idx}] must be " \
574 + f"'{must_be_masked}' "
575 + idx += 1
vllm/model_executor/models/mllama.py
+14 -4
@@ -1485,14 +1485,23 @@ def convert_sparse_cross_attention_mask_to_dense(
1485 total_length = sum(lengths)
1486 total_tiles = sum([sum(tiles) for tiles in num_tiles])
1487 dense_mask = np.zeros(shape=(total_length, total_tiles), dtype=np.int64)
1488 - # A list of ranges, range[i] = [start, end] means
1489 - # if the i-th sample has N tiles in total, the tiles[start, end]
1490 - # will be used for cross-attention decoding.
1488 + # A list of ranges, range[i] = [start, end] means that the i-th image will
1489 + # use tiles[start, end] for cross-attention decoding.
1490 tile_range_for_decode = []
1491
1492 seq_start = 0
1493 tile_start = 0
1495 - for masks, tiles, length in zip(sparse_mask, num_tiles, lengths):
1494 +
1495 + # sparse_mask has an [] entry for each sequence that does not have images,
1496 + # but num_tiles does not have these entries...
1497 + num_tiles_idx = 0
1498 + for masks, length in zip(sparse_mask, lengths):
1499 + if len(masks) == 0:
1500 + # Text only
1501 + continue
1502 +
1503 + tiles = num_tiles[num_tiles_idx]
1504 + num_tiles_idx += 1
1505 ts, td = -1, 0
1506 for mask, tile in zip(masks, tiles):
1507 if len(mask) != 2:
@@ -1512,6 +1521,7 @@ def convert_sparse_cross_attention_mask_to_dense(
1521 assert td != 0
1522 tile_range_for_decode.append((ts, ts + td))
1523 seq_start += length
1524 + assert num_tiles_idx == len(num_tiles)
1525
1526 return dense_mask, tile_range_for_decode
1527