A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Encoder decoder] Add cuda graph support during decoding for encoder-decoder models (#7631)
sroy745 committed
Sep 17, 2024 at 07:35 UTC
1009e93c5d634c724eeff3d4e453369337f502d4
15 files changed
+526
-112
.buildkite/test-pipeline.yaml
+7
@@ -252,6 +252,13 @@ steps:
252
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
253
- bash ./run-tests.sh -c configs/models-small.txt -t 1
254
255
+- label: Encoder Decoder tests # 5min
256
+ source_file_dependencies:
257
+ - vllm/
258
+ - tests/encoder_decoder
259
+ commands:
260
+ - pytest -v -s encoder_decoder
261
+
262
- label: OpenAI-Compatible Tool Use # 20 min
263
fast_check: false
264
mirror_hardwares: [ amd ]
tests/encoder_decoder/__init__.py
tests/encoder_decoder/test_e2e_correctness.py
new
+98
@@ -0,0 +1,98 @@
1
+"""E2E tests to verify the correctness of the encoder-decoder framework
2
+
3
+Run `pytest tests/encoder_decoder/test_e2e_correctness.py`.
4
+"""
5
+from typing import List, Optional, Tuple
6
+
7
+import pytest
8
+from transformers import AutoModelForSeq2SeqLM
9
+
10
+from vllm.sequence import SampleLogprobs
11
+from vllm.utils import is_cpu
12
+
13
+from ..conftest import DecoderPromptType
14
+from ..models.utils import check_logprobs_close
15
+
16
+
17
+def vllm_to_hf_output(
18
+ vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
19
+ decoder_prompt_type: DecoderPromptType,
20
+):
21
+ """Sanitize vllm output to be comparable with hf output."""
22
+ output_ids, output_str, out_logprobs = vllm_output
23
+
24
+ hf_output_str = output_str + "</s>"
25
+ if decoder_prompt_type == DecoderPromptType.NONE:
26
+ hf_output_str = "<s>" + hf_output_str
27
+
28
+ return output_ids, hf_output_str, out_logprobs
29
+
30
+
31
+@pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
32
+@pytest.mark.parametrize("dtype", ["bfloat16"])
33
+@pytest.mark.parametrize("max_tokens", [128])
34
+@pytest.mark.parametrize("num_logprobs", [5])
35
+@pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
36
+@pytest.mark.parametrize("enforce_eager", [True, False])
37
+@pytest.mark.skipif(
38
+ is_cpu(),
39
+ reason="CPU backend is not currently supported with encoder/decoder models"
40
+)
41
+def test_encoder_decoder_e2e(
42
+ hf_runner,
43
+ vllm_runner,
44
+ example_encoder_decoder_prompts,
45
+ model: str,
46
+ dtype: str,
47
+ max_tokens: int,
48
+ num_logprobs: int,
49
+ decoder_prompt_type: DecoderPromptType,
50
+ enforce_eager: bool,
51
+) -> None:
52
+ '''
53
+ End-to-End (E2E) test for the encoder-decoder framework.
54
+ This test evaluates the encoder-decoder functionality using the BART
55
+ model. We compare the outputs of the Hugging Face and vLLM
56
+ implementations to ensure that both implementations produce consistent
57
+ and correct results.
58
+ '''
59
+ test_case_prompts = example_encoder_decoder_prompts[decoder_prompt_type]
60
+
61
+ # Configuration settings for HF baseline
62
+ hf_kwargs = {
63
+ "top_k": None,
64
+ "num_beams": 1,
65
+ "repetition_penalty": 1.0,
66
+ "top_p": 1.0,
67
+ "length_penalty": 1.0,
68
+ "early_stopping": False,
69
+ "no_repeat_ngram_size": None,
70
+ "min_length": 0
71
+ }
72
+
73
+ with hf_runner(model, dtype=dtype,
74
+ auto_cls=AutoModelForSeq2SeqLM) as hf_model:
75
+ hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
76
+ test_case_prompts,
77
+ max_tokens,
78
+ num_logprobs,
79
+ **hf_kwargs,
80
+ ))
81
+ with vllm_runner(model, dtype=dtype,
82
+ enforce_eager=enforce_eager) as vllm_model:
83
+ vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
84
+ test_case_prompts, max_tokens, num_logprobs)
85
+
86
+ hf_skip_tokens = (1
87
+ if decoder_prompt_type == DecoderPromptType.NONE else 0)
88
+
89
+ check_logprobs_close(
90
+ outputs_0_lst=hf_outputs,
91
+ outputs_1_lst=[
92
+ vllm_to_hf_output(vllm_output, decoder_prompt_type)
93
+ for vllm_output in vllm_outputs
94
+ ],
95
+ name_0="hf",
96
+ name_1="vllm",
97
+ num_outputs_0_skip_tokens=hf_skip_tokens,
98
+ )
tests/worker/test_encoder_decoder_model_runner.py
+160
-22
@@ -1,3 +1,4 @@
1
+import itertools
2
from array import array
3
from typing import List
4
@@ -7,13 +8,9 @@ import torch
8
from vllm.engine.arg_utils import EngineArgs
9
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
10
SequenceData, SequenceGroupMetadata)
10
-from vllm.utils import is_cpu
11
+from vllm.utils import is_cpu, make_tensor_with_pad
12
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
12
-
13
-# CUDA graph scenarios to test
14
-#
15
-# Currently CUDA graph is not supported
16
-ENFORCE_EAGER = [True]
13
+from vllm.worker.model_runner import _get_graph_batch_size
14
15
BATCH_SIZES = [1, 4, 16, 64, 256]
16
@@ -40,8 +37,7 @@ def _create_model_runner(model: str, *args,
37
reason="CPU backend is currently "
38
"unsupported for encoder/ "
39
"decoder models")
43
-@pytest.mark.parametrize("enforce_eager", ENFORCE_EAGER)
44
-def test_empty_seq_group(enforce_eager, ):
40
+def test_empty_seq_group():
41
"""Verify prepare prompt and decode returns empty output
42
for empty seq group list"""
43
@@ -52,7 +48,7 @@ def test_empty_seq_group(enforce_eager, ):
48
max_num_batched_tokens=100000,
49
max_num_seqs=100000,
50
enable_chunked_prefill=False,
55
- enforce_eager=enforce_eager,
51
+ enforce_eager=True,
52
)
53
seq_group_metadata_list: List[SequenceGroupMetadata] = []
54
model_input = model_runner._prepare_model_input_tensors(
@@ -85,11 +81,7 @@ def test_empty_seq_group(enforce_eager, ):
81
"unsupported for encoder/ "
82
"decoder models")
83
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
88
-@pytest.mark.parametrize("enforce_eager", ENFORCE_EAGER)
89
-def test_prepare_prompt(
90
- batch_size,
91
- enforce_eager,
92
-):
84
+def test_prepare_prompt(batch_size):
85
'''
86
Test the ability of the encoder/decoder model runner subclass to
87
produce prefill-phase model inputs & attention metadata.
@@ -115,7 +107,7 @@ def test_prepare_prompt(
107
max_num_batched_tokens=100000,
108
max_num_seqs=100000,
109
enable_chunked_prefill=False,
118
- enforce_eager=enforce_eager,
110
+ enforce_eager=True,
111
)
112
113
seq_lens: List[int] = []
@@ -281,11 +273,7 @@ def test_prepare_prompt(
273
"unsupported for encoder/ "
274
"decoder models")
275
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
284
-@pytest.mark.parametrize("enforce_eager", ENFORCE_EAGER)
285
-def test_prepare_decode(
286
- batch_size,
287
- enforce_eager,
288
-):
276
+def test_prepare_decode(batch_size):
277
'''
278
Test the ability of the encoder/decoder model runner subclass to
279
produce decode-phase model inputs & attention metadata.
@@ -311,7 +299,7 @@ def test_prepare_decode(
299
max_num_batched_tokens=100000,
300
max_num_seqs=100000,
301
enable_chunked_prefill=False,
314
- enforce_eager=enforce_eager,
302
+ enforce_eager=True,
303
)
304
305
seq_lens: List[int] = []
@@ -428,7 +416,8 @@ def test_prepare_decode(
416
expected,
417
)
418
431
- # Cuda graph should is currently not supported for encoder/decoer.
419
+ # Model runner's CUDAGraph setting should be propagated to attention
420
+ # metadata.
421
assert attn_metadata.use_cuda_graph is False
422
423
# Verify the lengths of input tokens & positions
@@ -484,3 +473,152 @@ def test_prepare_decode(
473
dtype=actual.dtype,
474
)
475
assert torch.equal(actual, expected)
476
+
477
+
478
+@pytest.mark.parametrize("batch_size", list(range(1, 257)))
479
+def test_prepare_decode_cuda_graph(batch_size):
480
+ """
481
+ Tests that for encoder-decoder models with CUDA Graph capture and replay
482
+ enabled, the tensors used during the decode phase are correctly padded
483
+ for varying input batch sizes.
484
+ """
485
+ model_runner = _create_model_runner(
486
+ "facebook/bart-base",
487
+ seed=0,
488
+ dtype="float16",
489
+ max_num_batched_tokens=100000,
490
+ max_num_seqs=100000,
491
+ enable_chunked_prefill=False,
492
+ enforce_eager=False,
493
+ )
494
+
495
+ seq_lens: List[int] = []
496
+ encoder_seq_lens: List[int] = []
497
+ seq_group_metadata_list: List[SequenceGroupMetadata] = []
498
+ block_tables = {0: [1]}
499
+ cross_block_table = [2]
500
+ for i in range(batch_size):
501
+ # make sure all tokens fit into one block
502
+ seq_len = i % (model_runner.block_size - 1) + 1
503
+ seq_lens.append(seq_len)
504
+ seq_data = SequenceData(
505
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
506
+ encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
507
+ encoder_seq_lens.append(encoder_seq_len)
508
+ encoder_seq_data = SequenceData(
509
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
510
+ seq_group_metadata = SequenceGroupMetadata(
511
+ request_id=f"test_{i}",
512
+ is_prompt=False,
513
+ seq_data={0: seq_data},
514
+ sampling_params=SamplingParams(temperature=0),
515
+ block_tables=block_tables,
516
+ encoder_seq_data=encoder_seq_data,
517
+ cross_block_table=cross_block_table,
518
+ )
519
+ assert seq_group_metadata.token_chunk_size == 1
520
+ seq_group_metadata_list.append(seq_group_metadata)
521
+
522
+ model_input = model_runner.prepare_model_input(seq_group_metadata_list)
523
+ input_tokens = model_input.input_tokens
524
+ input_positions = model_input.input_positions
525
+ attn_metadata = model_input.attn_metadata
526
+ return_seq_lens = model_input.seq_lens
527
+ slot_mapping = attn_metadata.slot_mapping
528
+ encoder_input_tokens = model_input.encoder_input_tokens
529
+ encoder_input_positions = model_input.encoder_input_positions
530
+ cross_slot_mapping = attn_metadata.cross_slot_mapping
531
+
532
+ # With CUDA Graph capture and replay enabled, the decoder and encoder
533
+ # input sequences will be padded. Create the expected padded tensors
534
+ # accordingly.
535
+ graph_batch_size = _get_graph_batch_size(batch_size)
536
+ cuda_graph_pad_size = graph_batch_size - batch_size
537
+ padded_seq_lens = seq_lens + list(itertools.repeat(1, cuda_graph_pad_size))
538
+ padded_encoder_seq_lens = encoder_seq_lens + list(
539
+ itertools.repeat(1, cuda_graph_pad_size))
540
+
541
+ assert return_seq_lens == padded_seq_lens
542
+ assert len(slot_mapping) == len(input_tokens)
543
+ assert len(cross_slot_mapping) == len(encoder_input_tokens)
544
+
545
+ # Verify attention metadata
546
+ device = model_runner.device
547
+ assert attn_metadata.num_prefills == 0
548
+ assert attn_metadata.num_decode_tokens > 0
549
+ assert torch.equal(
550
+ attn_metadata.seq_lens_tensor,
551
+ torch.tensor(padded_seq_lens, device=device, dtype=torch.int))
552
+ assert attn_metadata.seq_lens == padded_seq_lens
553
+ assert attn_metadata.max_prefill_seq_len == 0
554
+ assert attn_metadata.max_decode_seq_len == max(seq_lens)
555
+ # - Encoder attention metadata
556
+ assert attn_metadata.encoder_seq_lens == padded_encoder_seq_lens
557
+ assert torch.equal(
558
+ attn_metadata.encoder_seq_lens_tensor,
559
+ torch.tensor(padded_encoder_seq_lens, device=device, dtype=torch.int))
560
+ assert attn_metadata.max_encoder_seq_len == max(padded_encoder_seq_lens)
561
+ assert attn_metadata.num_encoder_tokens == sum(padded_encoder_seq_lens)
562
+
563
+ # Verify block tables are correct for prompts
564
+ # - Decoder self-attention. Pad the block tables as expected.
565
+ expected = [block_tables[0] for _ in range(batch_size)]
566
+ expected.extend([[] for _ in range(cuda_graph_pad_size)])
567
+ expected = make_tensor_with_pad(
568
+ expected,
569
+ max_len=64,
570
+ pad=0,
571
+ dtype=torch.int32,
572
+ device=model_runner.device,
573
+ )
574
+ assert torch.equal(
575
+ attn_metadata.block_tables,
576
+ expected,
577
+ )
578
+ # - Encoder/decoder cross-attention. Pad the cross-attention block tables
579
+ # as expected.
580
+ expected = [cross_block_table for _ in range(len(seq_group_metadata_list))]
581
+ expected.extend([[] for _ in range(cuda_graph_pad_size)])
582
+ expected = make_tensor_with_pad(
583
+ expected,
584
+ max_len=64,
585
+ pad=0,
586
+ dtype=torch.int32,
587
+ device=model_runner.device,
588
+ )
589
+ assert torch.equal(
590
+ attn_metadata.cross_block_tables,
591
+ expected,
592
+ )
593
+
594
+ # Model runner's CUDAGraph setting should be propagated to attention
595
+ # metadata.
596
+ assert attn_metadata.use_cuda_graph is True
597
+
598
+ # Verify the lengths of input tokens & positions
599
+ # - Decoder
600
+ assert len(input_tokens) == len(padded_seq_lens)
601
+ assert len(input_positions) == len(padded_seq_lens)
602
+ # -- An indirect check that model_input.input_tokens
603
+ # and model_input.input_positions are correct -
604
+ # by design of the test, the input tokens are
605
+ # equal to the input position values, so if
606
+ # the model_input data structure has the correct
607
+ # values then these two should be equal
608
+ assert torch.equal(
609
+ input_tokens,
610
+ input_positions,
611
+ )
612
+ # - Encoder
613
+ assert len(encoder_input_tokens) == 0
614
+ assert len(encoder_input_tokens) == 0
615
+ # -- An indirect check that model_input.encoder_input_tokens
616
+ # and model_input.encoder_input_positions are correct -
617
+ # by design of the test, the input tokens are
618
+ # equal to the input position values, so if
619
+ # the model_input data structure has the correct
620
+ # values then these two should be equal
621
+ assert torch.equal(
622
+ encoder_input_tokens,
623
+ encoder_input_positions,
624
+ )
vllm/attention/backends/abstract.py
+13
-4
@@ -156,18 +156,27 @@ class AttentionState(ABC, Generic[T]):
156
...
157
158
@abstractmethod
159
- def graph_capture_get_metadata_for_batch(self, batch_size: int) -> T:
159
+ def graph_capture_get_metadata_for_batch(
160
+ self,
161
+ batch_size: int,
162
+ is_encoder_decoder_model: bool = False) -> T:
163
"""Get attention metadata for CUDA graph capture of batch_size."""
164
...
165
166
@abstractmethod
164
- def get_graph_input_buffers(self, attn_metadata: T) -> Dict[str, Any]:
167
+ def get_graph_input_buffers(
168
+ self,
169
+ attn_metadata: T,
170
+ is_encoder_decoder_model: bool = False) -> Dict[str, Any]:
171
"""Get attention-specific input buffers for CUDA graph capture."""
172
...
173
174
@abstractmethod
169
- def prepare_graph_input_buffers(self, input_buffers: Dict[str, Any],
170
- attn_metadata: T) -> None:
175
+ def prepare_graph_input_buffers(
176
+ self,
177
+ input_buffers: Dict[str, Any],
178
+ attn_metadata: T,
179
+ is_encoder_decoder_model: bool = False) -> None:
180
"""In-place modify input buffers dict for CUDA graph replay."""
181
...
182
vllm/attention/backends/flashinfer.py
+9
-3
@@ -172,7 +172,8 @@ class FlashInferState(AttentionState):
172
state._prefill_wrapper = self._get_prefill_wrapper()
173
return state
174
175
- def graph_capture_get_metadata_for_batch(self, batch_size: int):
175
+ def graph_capture_get_metadata_for_batch(
176
+ self, batch_size: int, is_encoder_decoder_model: bool = False):
177
assert self._is_graph_capturing
178
_indptr_buffer = self._graph_indptr_buffer[:batch_size + 1]
179
_last_page_len_buffer = self._graph_last_page_len_buffer[:batch_size]
@@ -232,12 +233,17 @@ class FlashInferState(AttentionState):
233
attn_metadata.begin_forward()
234
return attn_metadata
235
235
- def get_graph_input_buffers(self, attn_metadata):
236
+ def get_graph_input_buffers(self,
237
+ attn_metadata,
238
+ is_encoder_decoder_model: bool = False):
239
return {
240
"slot_mapping": attn_metadata.slot_mapping,
241
}
242
240
- def prepare_graph_input_buffers(self, input_buffers, attn_metadata):
243
+ def prepare_graph_input_buffers(self,
244
+ input_buffers,
245
+ attn_metadata,
246
+ is_encoder_decoder_model: bool = False):
247
return
248
249
def begin_forward(self, model_input):
vllm/attention/backends/utils.py
+107
-6
@@ -304,7 +304,8 @@ class CommonAttentionState(AttentionState):
304
assert self._is_graph_capturing
305
return self.__class__(self.runner)
306
307
- def graph_capture_get_metadata_for_batch(self, batch_size: int):
307
+ def graph_capture_get_metadata_for_batch(
308
+ self, batch_size: int, is_encoder_decoder_model: bool = False):
309
assert self._is_graph_capturing
310
attn_metadata = self.runner.attn_backend.make_metadata(
311
num_prefills=0,
@@ -322,21 +323,121 @@ class CommonAttentionState(AttentionState):
323
block_tables=self._graph_block_tables[:batch_size],
324
use_cuda_graph=True,
325
)
326
+ if is_encoder_decoder_model:
327
+ # The encoder decoder model works only with XFormers backend.
328
+ # Assert the same.
329
+ assert self.runner.attn_backend.get_name() == "xformers", \
330
+ f"Expected attn_backend name to be 'xformers', but "\
331
+ f" got '{self.runner.attn_backend.get_name()}'"
332
+ self._update_captured_metadata_for_enc_dec_model(
333
+ batch_size=batch_size, attn_metadata=attn_metadata)
334
+
335
return attn_metadata
336
327
- def get_graph_input_buffers(self, attn_metadata) -> Dict[str, Any]:
328
- return {
337
+ def get_graph_input_buffers(
338
+ self,
339
+ attn_metadata,
340
+ is_encoder_decoder_model: bool = False) -> Dict[str, Any]:
341
+ input_buffers = {
342
"slot_mapping": attn_metadata.slot_mapping,
343
"seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor,
344
"block_tables": attn_metadata.decode_metadata.block_tables,
345
}
333
-
334
- def prepare_graph_input_buffers(self, input_buffers,
335
- attn_metadata) -> None:
346
+ if is_encoder_decoder_model:
347
+ # The encoder decoder model works only with XFormers backend.
348
+ # Assert the same.
349
+ assert self.runner.attn_backend.get_name() == "xformers", \
350
+ f"Expected attn_backend name to be 'xformers', but "\
351
+ f" got '{self.runner.attn_backend.get_name()}'"
352
+ self._add_additonal_input_buffers_for_enc_dec_model(
353
+ attn_metadata=attn_metadata, input_buffers=input_buffers)
354
+ return input_buffers
355
+
356
+ def prepare_graph_input_buffers(
357
+ self,
358
+ input_buffers,
359
+ attn_metadata,
360
+ is_encoder_decoder_model: bool = False) -> None:
361
input_buffers["seq_lens_tensor"].copy_(
362
attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True)
363
input_buffers["block_tables"].copy_(
364
attn_metadata.decode_metadata.block_tables, non_blocking=True)
365
+ if is_encoder_decoder_model:
366
+ # The encoder decoder model works only with XFormers backend.
367
+ # Assert the same.
368
+ assert self.runner.attn_backend.get_name() == "xformers", \
369
+ f"Expected attn_backend name to be 'xformers', but "\
370
+ f" got '{self.runner.attn_backend.get_name()}'"
371
+ self._prepare_input_buffers_for_enc_dec_model(
372
+ attn_metadata, input_buffers)
373
374
def begin_forward(self, model_input) -> None:
375
return
376
+
377
+ def _update_captured_metadata_for_enc_dec_model(self, batch_size: int,
378
+ attn_metadata):
379
+ """
380
+ Updates the attention metadata parameters for CUDA graph capture in an
381
+ encoder-decoder model.
382
+
383
+ This method modifies attention-related tensors and metadata required
384
+ for CUDA graph capture in encoder-decoder models. Specifically, it
385
+ updates the cross-attention and encoder sequence tensors in the
386
+ AttentionMetadata object.
387
+ """
388
+ # During decode phase the cross_slot_mapping will be empty. Hence set
389
+ # an empty tensor for CUDA Graph capture.
390
+ attn_metadata.cross_slot_mapping = torch.tensor(
391
+ [], dtype=torch.int).cuda()
392
+ attn_metadata.cross_block_tables = torch.full(
393
+ (batch_size, self.runner.get_max_block_per_batch()),
394
+ 1,
395
+ dtype=torch.int).cuda()
396
+ attn_metadata.encoder_seq_lens = torch.full((batch_size, ),
397
+ 1,
398
+ dtype=torch.int).cuda()
399
+ attn_metadata.encoder_seq_lens_tensor = torch.full(
400
+ (batch_size, ), 1, dtype=torch.int).cuda()
401
+ attn_metadata.max_encoder_seq_len = self.runner.max_seq_len_to_capture
402
+
403
+ def _add_additonal_input_buffers_for_enc_dec_model(
404
+ self, attn_metadata, input_buffers: Dict[str, Any]):
405
+ """
406
+ Saves additional input buffers specific to the encoder-decoder model
407
+ from the attention metadata.
408
+
409
+ This method extracts and stores encoder-decoder related input buffers
410
+ from the `attn_metadata` into the `input_buffers` dictionary. The
411
+ buffers include encoder sequence lengths, cross-slot mappings, and
412
+ cross-block tables, which are essential for the encoder-decoder model
413
+ during CUDA graph replay.
414
+ """
415
+ input_buffers["encoder_seq_lens_tensor"] = (
416
+ attn_metadata.decode_metadata.encoder_seq_lens_tensor)
417
+ input_buffers["cross_slot_mapping"] = (
418
+ attn_metadata.decode_metadata.cross_slot_mapping)
419
+ input_buffers["cross_block_tables"] = (
420
+ attn_metadata.decode_metadata.cross_block_tables)
421
+
422
+ def _prepare_input_buffers_for_enc_dec_model(self, attn_metadata,
423
+ input_buffers: Dict[str,
424
+ Any]):
425
+ """
426
+ Populates input buffers with data from the encoder-decoder model's
427
+ attention metadata.
428
+
429
+ This method fills the input buffers with encoder-decoder specific
430
+ tensors. It copies data from the `attn_metadata` and keyword arguments
431
+ (`kwargs`) into corresponding buffers in the `input_buffers` dictionary.
432
+ The copied data includes attention-related metadata as well as input
433
+ IDs and positional information for the encoder.
434
+ """
435
+ input_buffers["encoder_seq_lens_tensor"].copy_(
436
+ attn_metadata.decode_metadata.encoder_seq_lens_tensor,
437
+ non_blocking=True)
438
+ input_buffers["cross_slot_mapping"].copy_(
439
+ attn_metadata.decode_metadata.cross_slot_mapping,
440
+ non_blocking=True)
441
+ input_buffers["cross_block_tables"].copy_(
442
+ attn_metadata.decode_metadata.cross_block_tables,
443
+ non_blocking=True)
vllm/config.py
+8
-33
@@ -16,9 +16,8 @@ from vllm.tracing import is_otel_available, otel_import_error_traceback
16
from vllm.transformers_utils.config import (ConfigFormat, get_config,
17
get_hf_image_processor_config,
18
get_hf_text_config)
19
-from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH, GiB_bytes,
20
- cuda_device_count_stateless, get_cpu_memory, is_cpu,
21
- is_hip, is_neuron, is_openvino, is_xpu,
19
+from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
20
+ is_cpu, is_hip, is_neuron, is_openvino, is_xpu,
21
print_warning_once)
22
23
if TYPE_CHECKING:
@@ -96,15 +95,15 @@ class ModelConfig:
95
enforce_eager: Whether to enforce eager execution. If True, we will
96
disable CUDA graph and always execute the model in eager mode.
97
If False, we will use CUDA graph and eager execution in hybrid.
99
- If None, the user did not specify, so default to False -
100
- except for encoder/decoder models, which currently require
101
- eager mode.
98
+ If None, the user did not specify, so default to False.
99
max_context_len_to_capture: Maximum context len covered by CUDA graphs.
100
When a sequence has context length larger than this, we fall back
101
to eager mode (DEPRECATED. Use max_seq_len_to_capture instead).
102
max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
103
When a sequence has context length larger than this, we fall back
107
- to eager mode
104
+ to eager mode. Additionally for encoder-decoder models, if the
105
+ sequence length of the encoder input is larger than this, we fall
106
+ back to the eager mode.
107
disable_sliding_window: Whether to disable sliding window. If True,
108
we will disable the sliding window functionality of the model.
109
If the model does not support sliding window, this argument is
@@ -186,32 +185,8 @@ class ModelConfig:
185
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
186
self.use_async_output_proc = use_async_output_proc
187
189
- # Choose a default enforce_eager value if the user did not specify
190
- # a value (enforce_eager is None)
191
- if getattr(self.hf_config, 'is_encoder_decoder', False):
192
- if self.enforce_eager is None:
193
- # *Only for encoder/decoder models* and
194
- # *only if enforce_eager is unset*, override
195
- # to enforce_eager=True
196
- #
197
- # Add a logger message since it is *somewhat* non-intuitive that
198
- # enforce_eager is True when the user has not specified its
199
- # value.
200
- logger.info("Forcing enforce_eager == True because "
201
- "enforce_eager setting was unspecified and "
202
- "CUDAGraph is not supported with encoder/ "
203
- "decoder models.")
204
- self.enforce_eager = True
205
-
206
- if not self.enforce_eager:
207
- # Eager mode explicitly disabled by user for an encoder/
208
- # decoder model; however CUDAGRAPH + encoder/decoder is
209
- # not currently supported
210
- raise ValueError(STR_NOT_IMPL_ENC_DEC_CUDAGRAPH)
211
- elif self.enforce_eager is None:
212
- # *Only for decoder-only models*, enforce_eager
213
- # defaults to False if unset. This is intuitive
214
- # so no logging message needed.
188
+ # Set enforce_eager to False if the value is unset.
189
+ if self.enforce_eager is None:
190
self.enforce_eager = False
191
192
if (not self.disable_sliding_window
vllm/engine/arg_utils.py
+4
-1
@@ -472,7 +472,10 @@ class EngineArgs:
472
default=EngineArgs.max_seq_len_to_capture,
473
help='Maximum sequence length covered by CUDA '
474
'graphs. When a sequence has context length '
475
- 'larger than this, we fall back to eager mode.')
475
+ 'larger than this, we fall back to eager mode. '
476
+ 'Additionally for encoder-decoder models, if the '
477
+ 'sequence length of the encoder input is larger '
478
+ 'than this, we fall back to the eager mode.')
479
parser.add_argument('--disable-custom-all-reduce',
480
action='store_true',
481
default=EngineArgs.disable_custom_all_reduce,
vllm/entrypoints/llm.py
+4
-4
@@ -88,7 +88,9 @@ class LLM:
88
to eager mode (DEPRECATED. Use `max_seq_len_to_capture` instead).
89
max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
90
When a sequence has context length larger than this, we fall back
91
- to eager mode.
91
+ to eager mode. Additionally for encoder-decoder models, if the
92
+ sequence length of the encoder input is larger than this, we fall
93
+ back to the eager mode.
94
disable_custom_all_reduce: See ParallelConfig
95
**kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
96
:ref:`engine_args`)
@@ -137,9 +139,7 @@ class LLM:
139
LLM constructor.
140
141
Note: if enforce_eager is unset (enforce_eager is None)
140
- it defaults to False for decoder-only models and True
141
- for encoder/decoder models, since encoder/decoder models
142
- do not currently support CUDAGraph.
142
+ it defaults to False.
143
'''
144
145
if "disable_log_stats" not in kwargs:
vllm/model_executor/models/bart.py
+4
-2
@@ -848,11 +848,13 @@ class BartForConditionalGeneration(nn.Module):
848
self,
849
input_ids: torch.Tensor,
850
positions: torch.Tensor,
851
- encoder_input_ids: torch.Tensor,
852
- encoder_positions: torch.Tensor,
851
kv_caches: List[torch.Tensor],
852
attn_metadata: AttentionMetadata,
853
intermediate_tensors: Optional[IntermediateTensors] = None,
854
+ *,
855
+ encoder_input_ids: torch.Tensor,
856
+ encoder_positions: torch.Tensor,
857
+ **kwargs,
858
) -> torch.Tensor:
859
r"""
860
Args:
vllm/utils.py
-5
@@ -71,10 +71,6 @@ STR_NOT_IMPL_ENC_DEC_SPEC_DEC = ("Speculative decoding is not "
71
"currently supported with encoder/"
72
"decoder models.")
73
74
-STR_NOT_IMPL_ENC_DEC_CUDAGRAPH = ("CUDAGraph is not "
75
- "currently supported with encoder/"
76
- "decoder models.")
77
-
74
STR_NOT_IMPL_ENC_DEC_BACKEND = ("XFormers is the only backend "
75
"currently supported with encoder/"
76
"decoder models.")
@@ -98,7 +94,6 @@ STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
94
"STR_NOT_IMPL_ENC_DEC_PP": STR_NOT_IMPL_ENC_DEC_PP,
95
"STR_NOT_IMPL_ENC_DEC_MM": STR_NOT_IMPL_ENC_DEC_MM,
96
"STR_NOT_IMPL_ENC_DEC_SPEC_DEC": STR_NOT_IMPL_ENC_DEC_SPEC_DEC,
101
- "STR_NOT_IMPL_ENC_DEC_CUDA_GRAPH": STR_NOT_IMPL_ENC_DEC_CUDAGRAPH,
97
"STR_NOT_IMPL_ENC_DEC_BACKEND": STR_NOT_IMPL_ENC_DEC_BACKEND,
98
"STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER": STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER,
99
"STR_NOT_IMPL_ENC_DEC_CPU": STR_NOT_IMPL_ENC_DEC_CPU
vllm/worker/enc_dec_model_runner.py
+36
-7
@@ -1,4 +1,5 @@
1
import dataclasses
2
+import itertools
3
from typing import Any, Dict, List, Optional, Tuple, Type, cast
4
5
import torch
@@ -24,7 +25,8 @@ from vllm.sequence import (IntermediateTensors, PoolerOutput,
25
from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad
26
from vllm.worker.model_runner import (GPUModelRunnerBase,
27
ModelInputForGPUBuilder,
27
- ModelInputForGPUWithSamplingMetadata)
28
+ ModelInputForGPUWithSamplingMetadata,
29
+ _get_graph_batch_size)
30
from vllm.worker.model_runner_base import (
31
_add_attn_metadata_broadcastable_dict,
32
_add_sampling_metadata_broadcastable_dict)
@@ -178,7 +180,15 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
180
raise ValueError("num_steps > 1 is not supported in "
181
"EncoderDecoderModelRunner")
182
181
- model_executable = self.model
183
+ if (model_input.attn_metadata is not None
184
+ and model_input.attn_metadata.prefill_metadata is None
185
+ and model_input.attn_metadata.decode_metadata.use_cuda_graph):
186
+ assert model_input.input_tokens is not None
187
+ graph_batch_size = model_input.input_tokens.shape[0]
188
+ model_executable = self.graph_runners[
189
+ model_input.virtual_engine][graph_batch_size]
190
+ else:
191
+ model_executable = self.model
192
193
seqlen_agnostic_kwargs = {
194
"finished_requests_ids": model_input.finished_requests_ids,
@@ -200,6 +210,9 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
210
if not self.is_driver_worker:
211
return []
212
213
+ if model_input.async_callback is not None:
214
+ model_input.async_callback()
215
+
216
# Sample the next token.
217
output: SamplerOutput = self.model.sample(
218
logits=logits,
@@ -231,14 +244,12 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
244
"""
245
model_input = self._prepare_model_input_tensors(
246
seq_group_metadata_list, finished_requests_ids)
234
-
247
(
248
attn_metadata,
249
encoder_input_tokens_tensor,
250
encoder_input_positions_tensor,
251
) = (self._prepare_encoder_model_input_tensors(seq_group_metadata_list,
252
model_input))
241
-
253
# Inject attn_metadata encoder/cross-attention fields &
254
# encoder input tokens/positions into model_input.
255
# Frozen dataclass fields cannot be modified, so use
@@ -437,11 +448,29 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
448
cross_block_tables.append([] if (
449
cross_block_table is None) else cross_block_table)
450
440
- # Convert cross-attention block tables to encoder input tensor
451
+ if (model_input.attn_metadata is not None
452
+ and model_input.attn_metadata.use_cuda_graph):
453
+ # We will be using CUDA graph replay for this decode.
454
+ max_len_of_block_table = self.get_max_block_per_batch()
455
+ batch_size = len(encoder_seq_lens)
456
+ graph_batch_size = _get_graph_batch_size(batch_size)
457
+ assert graph_batch_size >= batch_size
458
+ cuda_graph_pad_size = graph_batch_size - batch_size
459
+ # extend the cross_block_tables and encoder_seq_lens to match
460
+ # the graph_batch_size.
461
+ cross_block_tables.extend([[]
462
+ for _ in range(cuda_graph_pad_size)
463
+ ])
464
+ encoder_seq_lens.extend(
465
+ itertools.repeat(1, cuda_graph_pad_size))
466
+
467
+ else:
468
+ max_len_of_block_table = max(
469
+ len(block_table) for block_table in cross_block_tables)
470
+
471
cross_block_tables = make_tensor_with_pad(
472
cross_block_tables,
443
- max_len=max(
444
- len(block_table) for block_table in cross_block_tables),
473
+ max_len=max_len_of_block_table,
474
pad=0,
475
dtype=torch.int32,
476
device=self.device,
vllm/worker/model_runner.py
+76
-21
@@ -243,6 +243,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
243
prefix_cache_hit: bool = False,
244
reinit: bool = False,
245
reinit_use_defaults: bool = False,
246
+ encoder_seq_len: int = 0,
247
):
248
if reinit:
249
assert len(self.seq_ids) == len(seq_ids) # type: ignore
@@ -256,6 +257,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
257
self.block_tables = block_tables
258
self.computed_block_nums = computed_block_nums
259
self.n_seqs = n_seqs
260
+ self.encoder_seq_len = encoder_seq_len
261
262
if reinit:
263
if len(self.seq_ids) == 1 and reinit_use_defaults:
@@ -702,6 +704,11 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
704
assert n_seqs == 1
705
self.decode_only = False
706
707
+ encoder_seq_len = 0
708
+
709
+ if self.runner.model_config.is_encoder_decoder_model:
710
+ encoder_seq_len = seq_group_metadata.encoder_seq_data.get_len()
711
+
712
inter_data = self.init_cached_inter_data(
713
request_id=seq_group_metadata.request_id,
714
seq_ids=seq_ids,
@@ -709,7 +716,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
716
block_tables=seq_group_metadata.block_tables,
717
computed_block_nums=seq_group_metadata.computed_block_nums,
718
reinit=True,
712
- reinit_use_defaults=True)
719
+ reinit_use_defaults=True,
720
+ encoder_seq_len=encoder_seq_len)
721
722
self.inter_data_list.append(inter_data)
723
@@ -719,11 +727,15 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
727
for per_seq_group_fn in self.per_seq_group_compute_fns:
728
per_seq_group_fn(inter_data, seq_group_metadata)
729
722
- def _use_captured_graph(self, batch_size: int,
723
- max_decode_seq_len: int) -> bool:
730
+ def _use_captured_graph(self,
731
+ batch_size: int,
732
+ max_decode_seq_len: int,
733
+ max_encoder_seq_len: int = 0) -> bool:
734
return (self.decode_only and not self.runner.model_config.enforce_eager
725
- and batch_size <= self.runner.max_batchsize_to_capture
726
- and max_decode_seq_len <= self.runner.max_seq_len_to_capture)
735
+ and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
736
+ and max_decode_seq_len <= self.runner.max_seq_len_to_capture
737
+ and max_encoder_seq_len <= self.runner.max_seq_len_to_capture
738
+ and batch_size <= self.runner.max_batchsize_to_capture)
739
740
def build(self) -> ModelInputForGPU:
741
"""Finalize the builder intermediate data and
@@ -763,15 +775,18 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
775
input_positions.extend(cur_input_positions)
776
777
seq_lens = []
778
+ query_lens = []
779
max_decode_seq_len = 0
780
+ max_encoder_seq_len = 0
781
for inter_data in self.inter_data_list:
782
seq_lens.extend(inter_data.seq_lens)
783
+ query_lens.extend(inter_data.query_lens)
784
if not inter_data.is_prompt:
785
max_decode_seq_len = max(max_decode_seq_len,
786
max(inter_data.seq_lens))
772
- query_lens = []
773
- for inter_data in self.inter_data_list:
774
- query_lens.extend(inter_data.query_lens)
787
+ if self.runner.model_config.is_encoder_decoder_model:
788
+ max_encoder_seq_len = max(max_encoder_seq_len,
789
+ inter_data.encoder_seq_len)
790
791
# Mapping from request IDs to sequence IDs. Used for Jamba models
792
# that manages the cache by itself.
@@ -781,8 +796,10 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
796
}
797
798
batch_size = len(input_tokens)
784
- use_captured_graph = self._use_captured_graph(batch_size,
785
- max_decode_seq_len)
799
+ use_captured_graph = self._use_captured_graph(
800
+ batch_size,
801
+ max_decode_seq_len,
802
+ max_encoder_seq_len=max_encoder_seq_len)
803
804
# If cuda graph can be used, pad tensors accordingly.
805
# See `capture_model` API for more details.
@@ -1364,7 +1381,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1381
for batch_size in reversed(batch_size_capture_list):
1382
attn_metadata = (
1383
self.attn_state.graph_capture_get_metadata_for_batch(
1367
- batch_size))
1384
+ batch_size,
1385
+ is_encoder_decoder_model=self.model_config.
1386
+ is_encoder_decoder_model))
1387
1388
if self.lora_config:
1389
lora_mapping = LoRAMapping(
@@ -1380,10 +1399,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1399
)
1400
self.set_active_prompt_adapters(
1401
set(), prompt_adapter_mapping)
1383
-
1402
graph_runner = CUDAGraphRunner(
1403
self.model, self.attn_backend.get_name(),
1386
- self.attn_state.graph_clone(batch_size))
1404
+ self.attn_state.graph_clone(batch_size),
1405
+ self.model_config.is_encoder_decoder_model)
1406
1407
capture_inputs = {
1408
"input_ids":
@@ -1420,6 +1439,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1439
self.model.get_seqlen_agnostic_capture_inputs(
1440
batch_size)
1441
})
1442
+ if self.model_config.is_encoder_decoder_model:
1443
+ # add the additional inputs to capture for
1444
+ # encoder-decoder models.
1445
+ self._update_inputs_to_capture_for_enc_dec_model(
1446
+ capture_inputs)
1447
+
1448
graph_runner.capture(**capture_inputs)
1449
self.graph_memory_pool = graph_runner.graph.pool()
1450
self.graph_runners[virtual_engine][batch_size] = (
@@ -1430,6 +1455,24 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1455
# This usually takes < 10 seconds.
1456
logger.info("Graph capturing finished in %.0f secs.", elapsed_time)
1457
1458
+ def _update_inputs_to_capture_for_enc_dec_model(self,
1459
+ capture_inputs: Dict[str,
1460
+ Any]):
1461
+ """
1462
+ Updates the set of input tensors needed for CUDA graph capture in an
1463
+ encoder-decoder model.
1464
+
1465
+ This method modifies the provided `capture_inputs` dictionary by
1466
+ adding tensors specific to encoder-decoder specific models that
1467
+ need to be captured for CUDA Graph replay.
1468
+ """
1469
+ # During the decode phase encoder_input_ids and encoder_positions are
1470
+ # unset. Do the same thing for graph capture.
1471
+ capture_inputs["encoder_input_ids"] = torch.tensor(
1472
+ [], dtype=torch.long).cuda()
1473
+ capture_inputs["encoder_positions"] = torch.tensor(
1474
+ [], dtype=torch.long).cuda()
1475
+
1476
@property
1477
def vocab_size(self) -> int:
1478
return self.model_config.get_vocab_size()
@@ -1629,7 +1672,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1672
class CUDAGraphRunner:
1673
1674
def __init__(self, model: nn.Module, backend_name: str,
1632
- attn_state: AttentionState):
1675
+ attn_state: AttentionState, is_encoder_decoder_model: bool):
1676
self.model = model
1677
self.backend_name = backend_name
1678
self.attn_state = attn_state
@@ -1638,6 +1681,7 @@ class CUDAGraphRunner:
1681
self.output_buffers: Dict[str, torch.Tensor] = {}
1682
1683
self._graph: Optional[torch.cuda.CUDAGraph] = None
1684
+ self._is_encoder_decoder_model = is_encoder_decoder_model
1685
1686
@property
1687
def graph(self):
@@ -1671,8 +1715,9 @@ class CUDAGraphRunner:
1715
intermediate_tensors=intermediate_inputs,
1716
**kwargs,
1717
)
1718
+ # Wait for the warm up operations to finish before proceeding with
1719
+ # Graph Capture.
1720
torch.cuda.synchronize()
1675
-
1721
# Capture the graph.
1722
self._graph = torch.cuda.CUDAGraph()
1723
with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream):
@@ -1704,10 +1749,14 @@ class CUDAGraphRunner:
1749
1750
# Save the input and output buffers.
1751
self.input_buffers = {
1707
- "input_ids": input_ids,
1708
- "positions": positions,
1709
- "kv_caches": kv_caches,
1710
- **self.attn_state.get_graph_input_buffers(attn_metadata),
1752
+ "input_ids":
1753
+ input_ids,
1754
+ "positions":
1755
+ positions,
1756
+ "kv_caches":
1757
+ kv_caches,
1758
+ **self.attn_state.get_graph_input_buffers(
1759
+ attn_metadata, self._is_encoder_decoder_model),
1760
**kwargs,
1761
}
1762
if intermediate_inputs is not None:
@@ -1737,8 +1786,8 @@ class CUDAGraphRunner:
1786
self.input_buffers["positions"].copy_(positions, non_blocking=True)
1787
self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1788
non_blocking=True)
1740
- self.attn_state.prepare_graph_input_buffers(self.input_buffers,
1741
- attn_metadata)
1789
+ self.attn_state.prepare_graph_input_buffers(
1790
+ self.input_buffers, attn_metadata, self._is_encoder_decoder_model)
1791
if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1792
self.model.copy_inputs_before_cuda_graphs(self.input_buffers,
1793
**kwargs)
@@ -1752,6 +1801,12 @@ class CUDAGraphRunner:
1801
if key != "model_execute_time" and key != "model_forward_time":
1802
self.input_buffers[key].copy_(intermediate_tensors[key],
1803
non_blocking=True)
1804
+ if self._is_encoder_decoder_model:
1805
+ self.input_buffers["encoder_input_ids"].copy_(
1806
+ kwargs['encoder_input_ids'], non_blocking=True)
1807
+ self.input_buffers["encoder_positions"].copy_(
1808
+ kwargs['encoder_positions'], non_blocking=True)
1809
+
1810
# Run the graph.
1811
self.graph.replay()
1812
# Return the output tensor.
vllm/worker/utils.py
-4
@@ -47,10 +47,6 @@ def assert_enc_dec_mr_supported_scenario(
47
raise NotImplementedError(
48
STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_SPEC_DEC'])
49
50
- if not enc_dec_mr.model_config.enforce_eager:
51
- raise NotImplementedError(
52
- STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CUDA_GRAPH'])
53
-
50
if enc_dec_mr.prompt_adapter_config is not None:
51
raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_ERR_STRS[
52
'STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER'])