@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3118f63385
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] [Encoder-Decoder] Bugfix for encoder specific metadata construction during decode of encoder-decoder models. (#8545)

sroy745 committed Sep 18, 2024 at 19:24 UTC 3118f63385c0d767fba8b6d2039fc35440678da9
2 files changed +69 -31
tests/worker/test_encoder_decoder_model_runner.py
+63 -25
@@ -273,7 +273,8 @@ def test_prepare_prompt(batch_size):
273 "unsupported for encoder/ "
274 "decoder models")
275 @pytest.mark.parametrize("batch_size", BATCH_SIZES)
276 -def test_prepare_decode(batch_size):
276 +@pytest.mark.parametrize("multiple_seqs_per_seq_group", [True, False])
277 +def test_prepare_decode(batch_size, multiple_seqs_per_seq_group):
278 '''
279 Test the ability of the encoder/decoder model runner subclass to
280 produce decode-phase model inputs & attention metadata.
@@ -288,6 +289,7 @@ def test_prepare_decode(batch_size):
289 Arguments:
290
291 * batch_size
292 + * multiple_seqs_per_seq_group
293 * backend_name: The attention backend under test
294 * enforce_eager: Enforce eager mode if True (i.e. no CUDAGraph)
295 '''
@@ -305,22 +307,29 @@ def test_prepare_decode(batch_size):
307 seq_lens: List[int] = []
308 encoder_seq_lens: List[int] = []
309 seq_group_metadata_list: List[SequenceGroupMetadata] = []
308 - block_tables = {0: [1]}
310 + block_tables = {
311 + 0: [1],
312 + 1: [3]
313 + } if multiple_seqs_per_seq_group else {
314 + 0: [1]
315 + }
316 cross_block_table = [2]
317 for i in range(batch_size):
318 # make sure all tokens fit into one block
319 seq_len = i % (model_runner.block_size - 1) + 1
313 - seq_lens.append(seq_len)
320 seq_data = SequenceData(
321 array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
322 encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
317 - encoder_seq_lens.append(encoder_seq_len)
323 encoder_seq_data = SequenceData(
324 array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
325 +
326 seq_group_metadata = SequenceGroupMetadata(
327 request_id=f"test_{i}",
328 is_prompt=False,
323 - seq_data={0: seq_data},
329 + seq_data={
330 + 0: seq_data,
331 + 1: seq_data
332 + } if multiple_seqs_per_seq_group else {0: seq_data},
333 sampling_params=SamplingParams(temperature=0),
334 block_tables=block_tables,
335 encoder_seq_data=encoder_seq_data,
@@ -328,6 +337,10 @@ def test_prepare_decode(batch_size):
337 )
338 assert seq_group_metadata.token_chunk_size == 1
339 seq_group_metadata_list.append(seq_group_metadata)
340 + seq_lens.extend(
341 + [seq_len for _ in range(len(seq_group_metadata.seq_data))])
342 + encoder_seq_lens.extend(
343 + [encoder_seq_len for _ in range(len(seq_group_metadata.seq_data))])
344
345 # Build
346 # * Decoder model inputs
@@ -398,19 +411,24 @@ def test_prepare_decode(batch_size):
411
412 # Verify block tables are correct for prompts
413 # - Decoder self-attention
401 - expected = torch.tensor(
402 - [block_tables[0] for _ in range(len(seq_group_metadata_list))],
403 - dtype=torch.int32,
404 - device=model_runner.device)
414 + flattened_block_tables = [
415 + block_table for block_table in block_tables.values()
416 + ]
417 + expected = torch.tensor(flattened_block_tables *
418 + len(seq_group_metadata_list),
419 + dtype=torch.int32,
420 + device=model_runner.device)
421 assert torch.equal(
422 attn_metadata.block_tables,
423 expected,
424 )
425 # - Encoder/decoder cross-attention
410 - expected = torch.tensor(
411 - [cross_block_table for _ in range(len(seq_group_metadata_list))],
412 - dtype=torch.int32,
413 - device=model_runner.device)
426 + expected = torch.tensor([
427 + cross_block_table for seq_group_metadata in seq_group_metadata_list
428 + for _ in range(len(seq_group_metadata.seq_data))
429 + ],
430 + dtype=torch.int32,
431 + device=model_runner.device)
432 assert torch.equal(
433 attn_metadata.cross_block_tables,
434 expected,
@@ -474,7 +492,8 @@ def test_prepare_decode(batch_size):
492
493
494 @pytest.mark.parametrize("batch_size", list(range(1, 257)))
477 -def test_prepare_decode_cuda_graph(batch_size):
495 +@pytest.mark.parametrize("multiple_seqs_per_seq_group", [True, False])
496 +def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
497 """
498 Tests that for encoder-decoder models with CUDA Graph capture and replay
499 enabled, the tensors used during the decode phase are correctly padded
@@ -489,32 +508,45 @@ def test_prepare_decode_cuda_graph(batch_size):
508 enable_chunked_prefill=False,
509 enforce_eager=False,
510 )
492 -
511 + block_tables = {
512 + 0: [1],
513 + 1: [3]
514 + } if multiple_seqs_per_seq_group else {
515 + 0: [1]
516 + }
517 seq_lens: List[int] = []
518 encoder_seq_lens: List[int] = []
519 seq_group_metadata_list: List[SequenceGroupMetadata] = []
496 - block_tables = {0: [1]}
520 +
521 cross_block_table = [2]
522 + expanded_batch_size = 0
523 for i in range(batch_size):
524 # make sure all tokens fit into one block
525 seq_len = i % (model_runner.block_size - 1) + 1
501 - seq_lens.append(seq_len)
526 seq_data = SequenceData(
527 array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
528 encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
505 - encoder_seq_lens.append(encoder_seq_len)
529 encoder_seq_data = SequenceData(
530 array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
531 seq_group_metadata = SequenceGroupMetadata(
532 request_id=f"test_{i}",
533 is_prompt=False,
511 - seq_data={0: seq_data},
534 + seq_data={
535 + 0: seq_data,
536 + 1: seq_data
537 + } if multiple_seqs_per_seq_group else {0: seq_data},
538 sampling_params=SamplingParams(temperature=0),
539 block_tables=block_tables,
540 encoder_seq_data=encoder_seq_data,
541 cross_block_table=cross_block_table,
542 )
543 assert seq_group_metadata.token_chunk_size == 1
544 + seq_lens.extend(
545 + [seq_len for _ in range(len(seq_group_metadata.seq_data))])
546 + encoder_seq_lens.extend(
547 + [encoder_seq_len for _ in range(len(seq_group_metadata.seq_data))])
548 + expanded_batch_size = expanded_batch_size + len(
549 + seq_group_metadata.seq_data)
550 seq_group_metadata_list.append(seq_group_metadata)
551
552 model_input = model_runner.prepare_model_input(seq_group_metadata_list)
@@ -530,8 +562,8 @@ def test_prepare_decode_cuda_graph(batch_size):
562 # With CUDA Graph capture and replay enabled, the decoder and encoder
563 # input sequences will be padded. Create the expected padded tensors
564 # accordingly.
533 - graph_batch_size = _get_graph_batch_size(batch_size)
534 - cuda_graph_pad_size = graph_batch_size - batch_size
565 + graph_batch_size = _get_graph_batch_size(expanded_batch_size)
566 + cuda_graph_pad_size = graph_batch_size - expanded_batch_size
567 padded_seq_lens = seq_lens + list(itertools.repeat(1, cuda_graph_pad_size))
568 padded_encoder_seq_lens = encoder_seq_lens + list(
569 itertools.repeat(1, cuda_graph_pad_size))
@@ -560,10 +592,13 @@ def test_prepare_decode_cuda_graph(batch_size):
592
593 # Verify block tables are correct for prompts
594 # - Decoder self-attention. Pad the block tables as expected.
563 - expected = [block_tables[0] for _ in range(batch_size)]
564 - expected.extend([[] for _ in range(cuda_graph_pad_size)])
595 + flattened_block_tables = [
596 + block_table for _ in range(len(seq_group_metadata_list))
597 + for block_table in block_tables.values()
598 + ]
599 + flattened_block_tables.extend([[] for _ in range(cuda_graph_pad_size)])
600 expected = make_tensor_with_pad(
566 - expected,
601 + flattened_block_tables,
602 max_len=64,
603 pad=0,
604 dtype=torch.int32,
@@ -575,7 +610,10 @@ def test_prepare_decode_cuda_graph(batch_size):
610 )
611 # - Encoder/decoder cross-attention. Pad the cross-attention block tables
612 # as expected.
578 - expected = [cross_block_table for _ in range(len(seq_group_metadata_list))]
613 + expected = [
614 + cross_block_table for seq_group_metadata in seq_group_metadata_list
615 + for _ in range(len(seq_group_metadata.seq_data))
616 + ]
617 expected.extend([[] for _ in range(cuda_graph_pad_size)])
618 expected = make_tensor_with_pad(
619 expected,
vllm/worker/enc_dec_model_runner.py
+6 -6
@@ -435,18 +435,18 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
435 encoder_input_tokens_tensor = self._empty_long_tensor()
436 encoder_input_positions_tensor = self._empty_long_tensor()
437 cross_slot_mapping_tensor = self._empty_long_tensor()
438 -
438 # Extract cross-attention block tables &
439 # seq len from each sequence group metadata.
440 # Cross-attention block tables are empty
441 # during vLLM memory profiling.
442 cross_block_tables = []
443 for seq_group_metadata in seq_group_metadata_list:
445 - encoder_seq_lens.append(
446 - seq_group_metadata.encoder_seq_data.get_len())
447 - cross_block_table = seq_group_metadata.cross_block_table
448 - cross_block_tables.append([] if (
449 - cross_block_table is None) else cross_block_table)
444 + for _ in range(len(seq_group_metadata.seq_data)):
445 + encoder_seq_lens.append(
446 + seq_group_metadata.encoder_seq_data.get_len())
447 + cross_block_table = seq_group_metadata.cross_block_table
448 + cross_block_tables.append([] if (
449 + cross_block_table is None) else cross_block_table)
450
451 if (model_input.attn_metadata is not None
452 and model_input.attn_metadata.use_cuda_graph):