A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] [Encoder-Decoder] Bugfix for encoder specific metadata construction during decode of encoder-decoder models. (#8545)
sroy745 committed
Sep 18, 2024 at 19:24 UTC
3118f63385c0d767fba8b6d2039fc35440678da9
2 files changed
+69
-31
tests/worker/test_encoder_decoder_model_runner.py
+63
-25
@@ -273,7 +273,8 @@ def test_prepare_prompt(batch_size):
273
"unsupported for encoder/ "
274
"decoder models")
275
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
276
-def test_prepare_decode(batch_size):
276
+@pytest.mark.parametrize("multiple_seqs_per_seq_group", [True, False])
277
+def test_prepare_decode(batch_size, multiple_seqs_per_seq_group):
278
'''
279
Test the ability of the encoder/decoder model runner subclass to
280
produce decode-phase model inputs & attention metadata.
@@ -288,6 +289,7 @@ def test_prepare_decode(batch_size):
289
Arguments:
290
291
* batch_size
292
+ * multiple_seqs_per_seq_group
293
* backend_name: The attention backend under test
294
* enforce_eager: Enforce eager mode if True (i.e. no CUDAGraph)
295
'''
@@ -305,22 +307,29 @@ def test_prepare_decode(batch_size):
307
seq_lens: List[int] = []
308
encoder_seq_lens: List[int] = []
309
seq_group_metadata_list: List[SequenceGroupMetadata] = []
308
- block_tables = {0: [1]}
310
+ block_tables = {
311
+ 0: [1],
312
+ 1: [3]
313
+ } if multiple_seqs_per_seq_group else {
314
+ 0: [1]
315
+ }
316
cross_block_table = [2]
317
for i in range(batch_size):
318
# make sure all tokens fit into one block
319
seq_len = i % (model_runner.block_size - 1) + 1
313
- seq_lens.append(seq_len)
320
seq_data = SequenceData(
321
array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
322
encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
317
- encoder_seq_lens.append(encoder_seq_len)
323
encoder_seq_data = SequenceData(
324
array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
325
+
326
seq_group_metadata = SequenceGroupMetadata(
327
request_id=f"test_{i}",
328
is_prompt=False,
323
- seq_data={0: seq_data},
329
+ seq_data={
330
+ 0: seq_data,
331
+ 1: seq_data
332
+ } if multiple_seqs_per_seq_group else {0: seq_data},
333
sampling_params=SamplingParams(temperature=0),
334
block_tables=block_tables,
335
encoder_seq_data=encoder_seq_data,
@@ -328,6 +337,10 @@ def test_prepare_decode(batch_size):
337
)
338
assert seq_group_metadata.token_chunk_size == 1
339
seq_group_metadata_list.append(seq_group_metadata)
340
+ seq_lens.extend(
341
+ [seq_len for _ in range(len(seq_group_metadata.seq_data))])
342
+ encoder_seq_lens.extend(
343
+ [encoder_seq_len for _ in range(len(seq_group_metadata.seq_data))])
344
345
# Build
346
# * Decoder model inputs
@@ -398,19 +411,24 @@ def test_prepare_decode(batch_size):
411
412
# Verify block tables are correct for prompts
413
# - Decoder self-attention
401
- expected = torch.tensor(
402
- [block_tables[0] for _ in range(len(seq_group_metadata_list))],
403
- dtype=torch.int32,
404
- device=model_runner.device)
414
+ flattened_block_tables = [
415
+ block_table for block_table in block_tables.values()
416
+ ]
417
+ expected = torch.tensor(flattened_block_tables *
418
+ len(seq_group_metadata_list),
419
+ dtype=torch.int32,
420
+ device=model_runner.device)
421
assert torch.equal(
422
attn_metadata.block_tables,
423
expected,
424
)
425
# - Encoder/decoder cross-attention
410
- expected = torch.tensor(
411
- [cross_block_table for _ in range(len(seq_group_metadata_list))],
412
- dtype=torch.int32,
413
- device=model_runner.device)
426
+ expected = torch.tensor([
427
+ cross_block_table for seq_group_metadata in seq_group_metadata_list
428
+ for _ in range(len(seq_group_metadata.seq_data))
429
+ ],
430
+ dtype=torch.int32,
431
+ device=model_runner.device)
432
assert torch.equal(
433
attn_metadata.cross_block_tables,
434
expected,
@@ -474,7 +492,8 @@ def test_prepare_decode(batch_size):
492
493
494
@pytest.mark.parametrize("batch_size", list(range(1, 257)))
477
-def test_prepare_decode_cuda_graph(batch_size):
495
+@pytest.mark.parametrize("multiple_seqs_per_seq_group", [True, False])
496
+def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
497
"""
498
Tests that for encoder-decoder models with CUDA Graph capture and replay
499
enabled, the tensors used during the decode phase are correctly padded
@@ -489,32 +508,45 @@ def test_prepare_decode_cuda_graph(batch_size):
508
enable_chunked_prefill=False,
509
enforce_eager=False,
510
)
492
-
511
+ block_tables = {
512
+ 0: [1],
513
+ 1: [3]
514
+ } if multiple_seqs_per_seq_group else {
515
+ 0: [1]
516
+ }
517
seq_lens: List[int] = []
518
encoder_seq_lens: List[int] = []
519
seq_group_metadata_list: List[SequenceGroupMetadata] = []
496
- block_tables = {0: [1]}
520
+
521
cross_block_table = [2]
522
+ expanded_batch_size = 0
523
for i in range(batch_size):
524
# make sure all tokens fit into one block
525
seq_len = i % (model_runner.block_size - 1) + 1
501
- seq_lens.append(seq_len)
526
seq_data = SequenceData(
527
array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
528
encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
505
- encoder_seq_lens.append(encoder_seq_len)
529
encoder_seq_data = SequenceData(
530
array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
531
seq_group_metadata = SequenceGroupMetadata(
532
request_id=f"test_{i}",
533
is_prompt=False,
511
- seq_data={0: seq_data},
534
+ seq_data={
535
+ 0: seq_data,
536
+ 1: seq_data
537
+ } if multiple_seqs_per_seq_group else {0: seq_data},
538
sampling_params=SamplingParams(temperature=0),
539
block_tables=block_tables,
540
encoder_seq_data=encoder_seq_data,
541
cross_block_table=cross_block_table,
542
)
543
assert seq_group_metadata.token_chunk_size == 1
544
+ seq_lens.extend(
545
+ [seq_len for _ in range(len(seq_group_metadata.seq_data))])
546
+ encoder_seq_lens.extend(
547
+ [encoder_seq_len for _ in range(len(seq_group_metadata.seq_data))])
548
+ expanded_batch_size = expanded_batch_size + len(
549
+ seq_group_metadata.seq_data)
550
seq_group_metadata_list.append(seq_group_metadata)
551
552
model_input = model_runner.prepare_model_input(seq_group_metadata_list)
@@ -530,8 +562,8 @@ def test_prepare_decode_cuda_graph(batch_size):
562
# With CUDA Graph capture and replay enabled, the decoder and encoder
563
# input sequences will be padded. Create the expected padded tensors
564
# accordingly.
533
- graph_batch_size = _get_graph_batch_size(batch_size)
534
- cuda_graph_pad_size = graph_batch_size - batch_size
565
+ graph_batch_size = _get_graph_batch_size(expanded_batch_size)
566
+ cuda_graph_pad_size = graph_batch_size - expanded_batch_size
567
padded_seq_lens = seq_lens + list(itertools.repeat(1, cuda_graph_pad_size))
568
padded_encoder_seq_lens = encoder_seq_lens + list(
569
itertools.repeat(1, cuda_graph_pad_size))
@@ -560,10 +592,13 @@ def test_prepare_decode_cuda_graph(batch_size):
592
593
# Verify block tables are correct for prompts
594
# - Decoder self-attention. Pad the block tables as expected.
563
- expected = [block_tables[0] for _ in range(batch_size)]
564
- expected.extend([[] for _ in range(cuda_graph_pad_size)])
595
+ flattened_block_tables = [
596
+ block_table for _ in range(len(seq_group_metadata_list))
597
+ for block_table in block_tables.values()
598
+ ]
599
+ flattened_block_tables.extend([[] for _ in range(cuda_graph_pad_size)])
600
expected = make_tensor_with_pad(
566
- expected,
601
+ flattened_block_tables,
602
max_len=64,
603
pad=0,
604
dtype=torch.int32,
@@ -575,7 +610,10 @@ def test_prepare_decode_cuda_graph(batch_size):
610
)
611
# - Encoder/decoder cross-attention. Pad the cross-attention block tables
612
# as expected.
578
- expected = [cross_block_table for _ in range(len(seq_group_metadata_list))]
613
+ expected = [
614
+ cross_block_table for seq_group_metadata in seq_group_metadata_list
615
+ for _ in range(len(seq_group_metadata.seq_data))
616
+ ]
617
expected.extend([[] for _ in range(cuda_graph_pad_size)])
618
expected = make_tensor_with_pad(
619
expected,
vllm/worker/enc_dec_model_runner.py
+6
-6
@@ -435,18 +435,18 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
435
encoder_input_tokens_tensor = self._empty_long_tensor()
436
encoder_input_positions_tensor = self._empty_long_tensor()
437
cross_slot_mapping_tensor = self._empty_long_tensor()
438
-
438
# Extract cross-attention block tables &
439
# seq len from each sequence group metadata.
440
# Cross-attention block tables are empty
441
# during vLLM memory profiling.
442
cross_block_tables = []
443
for seq_group_metadata in seq_group_metadata_list:
445
- encoder_seq_lens.append(
446
- seq_group_metadata.encoder_seq_data.get_len())
447
- cross_block_table = seq_group_metadata.cross_block_table
448
- cross_block_tables.append([] if (
449
- cross_block_table is None) else cross_block_table)
444
+ for _ in range(len(seq_group_metadata.seq_data)):
445
+ encoder_seq_lens.append(
446
+ seq_group_metadata.encoder_seq_data.get_len())
447
+ cross_block_table = seq_group_metadata.cross_block_table
448
+ cross_block_tables.append([] if (
449
+ cross_block_table is None) else cross_block_table)
450
451
if (model_input.attn_metadata is not None
452
and model_input.attn_metadata.use_cuda_graph):