@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1009e93c5d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Encoder decoder] Add cuda graph support during decoding for encoder-decoder models (#7631)

sroy745 committed Sep 17, 2024 at 07:35 UTC 1009e93c5d634c724eeff3d4e453369337f502d4
15 files changed +526 -112
.buildkite/test-pipeline.yaml
+7
@@ -252,6 +252,13 @@ steps:
252 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
253 - bash ./run-tests.sh -c configs/models-small.txt -t 1
254
255 +- label: Encoder Decoder tests # 5min
256 + source_file_dependencies:
257 + - vllm/
258 + - tests/encoder_decoder
259 + commands:
260 + - pytest -v -s encoder_decoder
261 +
262 - label: OpenAI-Compatible Tool Use # 20 min
263 fast_check: false
264 mirror_hardwares: [ amd ]
tests/encoder_decoder/__init__.py
tests/encoder_decoder/test_e2e_correctness.py new
+98
@@ -0,0 +1,98 @@
1 +"""E2E tests to verify the correctness of the encoder-decoder framework
2 +
3 +Run `pytest tests/encoder_decoder/test_e2e_correctness.py`.
4 +"""
5 +from typing import List, Optional, Tuple
6 +
7 +import pytest
8 +from transformers import AutoModelForSeq2SeqLM
9 +
10 +from vllm.sequence import SampleLogprobs
11 +from vllm.utils import is_cpu
12 +
13 +from ..conftest import DecoderPromptType
14 +from ..models.utils import check_logprobs_close
15 +
16 +
17 +def vllm_to_hf_output(
18 + vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
19 + decoder_prompt_type: DecoderPromptType,
20 +):
21 + """Sanitize vllm output to be comparable with hf output."""
22 + output_ids, output_str, out_logprobs = vllm_output
23 +
24 + hf_output_str = output_str + "</s>"
25 + if decoder_prompt_type == DecoderPromptType.NONE:
26 + hf_output_str = "<s>" + hf_output_str
27 +
28 + return output_ids, hf_output_str, out_logprobs
29 +
30 +
31 +@pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
32 +@pytest.mark.parametrize("dtype", ["bfloat16"])
33 +@pytest.mark.parametrize("max_tokens", [128])
34 +@pytest.mark.parametrize("num_logprobs", [5])
35 +@pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
36 +@pytest.mark.parametrize("enforce_eager", [True, False])
37 +@pytest.mark.skipif(
38 + is_cpu(),
39 + reason="CPU backend is not currently supported with encoder/decoder models"
40 +)
41 +def test_encoder_decoder_e2e(
42 + hf_runner,
43 + vllm_runner,
44 + example_encoder_decoder_prompts,
45 + model: str,
46 + dtype: str,
47 + max_tokens: int,
48 + num_logprobs: int,
49 + decoder_prompt_type: DecoderPromptType,
50 + enforce_eager: bool,
51 +) -> None:
52 + '''
53 + End-to-End (E2E) test for the encoder-decoder framework.
54 + This test evaluates the encoder-decoder functionality using the BART
55 + model. We compare the outputs of the Hugging Face and vLLM
56 + implementations to ensure that both implementations produce consistent
57 + and correct results.
58 + '''
59 + test_case_prompts = example_encoder_decoder_prompts[decoder_prompt_type]
60 +
61 + # Configuration settings for HF baseline
62 + hf_kwargs = {
63 + "top_k": None,
64 + "num_beams": 1,
65 + "repetition_penalty": 1.0,
66 + "top_p": 1.0,
67 + "length_penalty": 1.0,
68 + "early_stopping": False,
69 + "no_repeat_ngram_size": None,
70 + "min_length": 0
71 + }
72 +
73 + with hf_runner(model, dtype=dtype,
74 + auto_cls=AutoModelForSeq2SeqLM) as hf_model:
75 + hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
76 + test_case_prompts,
77 + max_tokens,
78 + num_logprobs,
79 + **hf_kwargs,
80 + ))
81 + with vllm_runner(model, dtype=dtype,
82 + enforce_eager=enforce_eager) as vllm_model:
83 + vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
84 + test_case_prompts, max_tokens, num_logprobs)
85 +
86 + hf_skip_tokens = (1
87 + if decoder_prompt_type == DecoderPromptType.NONE else 0)
88 +
89 + check_logprobs_close(
90 + outputs_0_lst=hf_outputs,
91 + outputs_1_lst=[
92 + vllm_to_hf_output(vllm_output, decoder_prompt_type)
93 + for vllm_output in vllm_outputs
94 + ],
95 + name_0="hf",
96 + name_1="vllm",
97 + num_outputs_0_skip_tokens=hf_skip_tokens,
98 + )
tests/worker/test_encoder_decoder_model_runner.py
+160 -22
@@ -1,3 +1,4 @@
1 +import itertools
2 from array import array
3 from typing import List
4
@@ -7,13 +8,9 @@ import torch
8 from vllm.engine.arg_utils import EngineArgs
9 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
10 SequenceData, SequenceGroupMetadata)
10 -from vllm.utils import is_cpu
11 +from vllm.utils import is_cpu, make_tensor_with_pad
12 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
12 -
13 -# CUDA graph scenarios to test
14 -#
15 -# Currently CUDA graph is not supported
16 -ENFORCE_EAGER = [True]
13 +from vllm.worker.model_runner import _get_graph_batch_size
14
15 BATCH_SIZES = [1, 4, 16, 64, 256]
16
@@ -40,8 +37,7 @@ def _create_model_runner(model: str, *args,
37 reason="CPU backend is currently "
38 "unsupported for encoder/ "
39 "decoder models")
43 -@pytest.mark.parametrize("enforce_eager", ENFORCE_EAGER)
44 -def test_empty_seq_group(enforce_eager, ):
40 +def test_empty_seq_group():
41 """Verify prepare prompt and decode returns empty output
42 for empty seq group list"""
43
@@ -52,7 +48,7 @@ def test_empty_seq_group(enforce_eager, ):
48 max_num_batched_tokens=100000,
49 max_num_seqs=100000,
50 enable_chunked_prefill=False,
55 - enforce_eager=enforce_eager,
51 + enforce_eager=True,
52 )
53 seq_group_metadata_list: List[SequenceGroupMetadata] = []
54 model_input = model_runner._prepare_model_input_tensors(
@@ -85,11 +81,7 @@ def test_empty_seq_group(enforce_eager, ):
81 "unsupported for encoder/ "
82 "decoder models")
83 @pytest.mark.parametrize("batch_size", BATCH_SIZES)
88 -@pytest.mark.parametrize("enforce_eager", ENFORCE_EAGER)
89 -def test_prepare_prompt(
90 - batch_size,
91 - enforce_eager,
92 -):
84 +def test_prepare_prompt(batch_size):
85 '''
86 Test the ability of the encoder/decoder model runner subclass to
87 produce prefill-phase model inputs & attention metadata.
@@ -115,7 +107,7 @@ def test_prepare_prompt(
107 max_num_batched_tokens=100000,
108 max_num_seqs=100000,
109 enable_chunked_prefill=False,
118 - enforce_eager=enforce_eager,
110 + enforce_eager=True,
111 )
112
113 seq_lens: List[int] = []
@@ -281,11 +273,7 @@ def test_prepare_prompt(
273 "unsupported for encoder/ "
274 "decoder models")
275 @pytest.mark.parametrize("batch_size", BATCH_SIZES)
284 -@pytest.mark.parametrize("enforce_eager", ENFORCE_EAGER)
285 -def test_prepare_decode(
286 - batch_size,
287 - enforce_eager,
288 -):
276 +def test_prepare_decode(batch_size):
277 '''
278 Test the ability of the encoder/decoder model runner subclass to
279 produce decode-phase model inputs & attention metadata.
@@ -311,7 +299,7 @@ def test_prepare_decode(
299 max_num_batched_tokens=100000,
300 max_num_seqs=100000,
301 enable_chunked_prefill=False,
314 - enforce_eager=enforce_eager,
302 + enforce_eager=True,
303 )
304
305 seq_lens: List[int] = []
@@ -428,7 +416,8 @@ def test_prepare_decode(
416 expected,
417 )
418
431 - # Cuda graph should is currently not supported for encoder/decoer.
419 + # Model runner's CUDAGraph setting should be propagated to attention
420 + # metadata.
421 assert attn_metadata.use_cuda_graph is False
422
423 # Verify the lengths of input tokens & positions
@@ -484,3 +473,152 @@ def test_prepare_decode(
473 dtype=actual.dtype,
474 )
475 assert torch.equal(actual, expected)
476 +
477 +
478 +@pytest.mark.parametrize("batch_size", list(range(1, 257)))
479 +def test_prepare_decode_cuda_graph(batch_size):
480 + """
481 + Tests that for encoder-decoder models with CUDA Graph capture and replay
482 + enabled, the tensors used during the decode phase are correctly padded
483 + for varying input batch sizes.
484 + """
485 + model_runner = _create_model_runner(
486 + "facebook/bart-base",
487 + seed=0,
488 + dtype="float16",
489 + max_num_batched_tokens=100000,
490 + max_num_seqs=100000,
491 + enable_chunked_prefill=False,
492 + enforce_eager=False,
493 + )
494 +
495 + seq_lens: List[int] = []
496 + encoder_seq_lens: List[int] = []
497 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
498 + block_tables = {0: [1]}
499 + cross_block_table = [2]
500 + for i in range(batch_size):
501 + # make sure all tokens fit into one block
502 + seq_len = i % (model_runner.block_size - 1) + 1
503 + seq_lens.append(seq_len)
504 + seq_data = SequenceData(
505 + array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
506 + encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
507 + encoder_seq_lens.append(encoder_seq_len)
508 + encoder_seq_data = SequenceData(
509 + array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
510 + seq_group_metadata = SequenceGroupMetadata(
511 + request_id=f"test_{i}",
512 + is_prompt=False,
513 + seq_data={0: seq_data},
514 + sampling_params=SamplingParams(temperature=0),
515 + block_tables=block_tables,
516 + encoder_seq_data=encoder_seq_data,
517 + cross_block_table=cross_block_table,
518 + )
519 + assert seq_group_metadata.token_chunk_size == 1
520 + seq_group_metadata_list.append(seq_group_metadata)
521 +
522 + model_input = model_runner.prepare_model_input(seq_group_metadata_list)
523 + input_tokens = model_input.input_tokens
524 + input_positions = model_input.input_positions
525 + attn_metadata = model_input.attn_metadata
526 + return_seq_lens = model_input.seq_lens
527 + slot_mapping = attn_metadata.slot_mapping
528 + encoder_input_tokens = model_input.encoder_input_tokens
529 + encoder_input_positions = model_input.encoder_input_positions
530 + cross_slot_mapping = attn_metadata.cross_slot_mapping
531 +
532 + # With CUDA Graph capture and replay enabled, the decoder and encoder
533 + # input sequences will be padded. Create the expected padded tensors
534 + # accordingly.
535 + graph_batch_size = _get_graph_batch_size(batch_size)
536 + cuda_graph_pad_size = graph_batch_size - batch_size
537 + padded_seq_lens = seq_lens + list(itertools.repeat(1, cuda_graph_pad_size))
538 + padded_encoder_seq_lens = encoder_seq_lens + list(
539 + itertools.repeat(1, cuda_graph_pad_size))
540 +
541 + assert return_seq_lens == padded_seq_lens
542 + assert len(slot_mapping) == len(input_tokens)
543 + assert len(cross_slot_mapping) == len(encoder_input_tokens)
544 +
545 + # Verify attention metadata
546 + device = model_runner.device
547 + assert attn_metadata.num_prefills == 0
548 + assert attn_metadata.num_decode_tokens > 0
549 + assert torch.equal(
550 + attn_metadata.seq_lens_tensor,
551 + torch.tensor(padded_seq_lens, device=device, dtype=torch.int))
552 + assert attn_metadata.seq_lens == padded_seq_lens
553 + assert attn_metadata.max_prefill_seq_len == 0
554 + assert attn_metadata.max_decode_seq_len == max(seq_lens)
555 + # - Encoder attention metadata
556 + assert attn_metadata.encoder_seq_lens == padded_encoder_seq_lens
557 + assert torch.equal(
558 + attn_metadata.encoder_seq_lens_tensor,
559 + torch.tensor(padded_encoder_seq_lens, device=device, dtype=torch.int))
560 + assert attn_metadata.max_encoder_seq_len == max(padded_encoder_seq_lens)
561 + assert attn_metadata.num_encoder_tokens == sum(padded_encoder_seq_lens)
562 +
563 + # Verify block tables are correct for prompts
564 + # - Decoder self-attention. Pad the block tables as expected.
565 + expected = [block_tables[0] for _ in range(batch_size)]
566 + expected.extend([[] for _ in range(cuda_graph_pad_size)])
567 + expected = make_tensor_with_pad(
568 + expected,
569 + max_len=64,
570 + pad=0,
571 + dtype=torch.int32,
572 + device=model_runner.device,
573 + )
574 + assert torch.equal(
575 + attn_metadata.block_tables,
576 + expected,
577 + )
578 + # - Encoder/decoder cross-attention. Pad the cross-attention block tables
579 + # as expected.
580 + expected = [cross_block_table for _ in range(len(seq_group_metadata_list))]
581 + expected.extend([[] for _ in range(cuda_graph_pad_size)])
582 + expected = make_tensor_with_pad(
583 + expected,
584 + max_len=64,
585 + pad=0,
586 + dtype=torch.int32,
587 + device=model_runner.device,
588 + )
589 + assert torch.equal(
590 + attn_metadata.cross_block_tables,
591 + expected,
592 + )
593 +
594 + # Model runner's CUDAGraph setting should be propagated to attention
595 + # metadata.
596 + assert attn_metadata.use_cuda_graph is True
597 +
598 + # Verify the lengths of input tokens & positions
599 + # - Decoder
600 + assert len(input_tokens) == len(padded_seq_lens)
601 + assert len(input_positions) == len(padded_seq_lens)
602 + # -- An indirect check that model_input.input_tokens
603 + # and model_input.input_positions are correct -
604 + # by design of the test, the input tokens are
605 + # equal to the input position values, so if
606 + # the model_input data structure has the correct
607 + # values then these two should be equal
608 + assert torch.equal(
609 + input_tokens,
610 + input_positions,
611 + )
612 + # - Encoder
613 + assert len(encoder_input_tokens) == 0
614 + assert len(encoder_input_tokens) == 0
615 + # -- An indirect check that model_input.encoder_input_tokens
616 + # and model_input.encoder_input_positions are correct -
617 + # by design of the test, the input tokens are
618 + # equal to the input position values, so if
619 + # the model_input data structure has the correct
620 + # values then these two should be equal
621 + assert torch.equal(
622 + encoder_input_tokens,
623 + encoder_input_positions,
624 + )
vllm/attention/backends/abstract.py
+13 -4
@@ -156,18 +156,27 @@ class AttentionState(ABC, Generic[T]):
156 ...
157
158 @abstractmethod
159 - def graph_capture_get_metadata_for_batch(self, batch_size: int) -> T:
159 + def graph_capture_get_metadata_for_batch(
160 + self,
161 + batch_size: int,
162 + is_encoder_decoder_model: bool = False) -> T:
163 """Get attention metadata for CUDA graph capture of batch_size."""
164 ...
165
166 @abstractmethod
164 - def get_graph_input_buffers(self, attn_metadata: T) -> Dict[str, Any]:
167 + def get_graph_input_buffers(
168 + self,
169 + attn_metadata: T,
170 + is_encoder_decoder_model: bool = False) -> Dict[str, Any]:
171 """Get attention-specific input buffers for CUDA graph capture."""
172 ...
173
174 @abstractmethod
169 - def prepare_graph_input_buffers(self, input_buffers: Dict[str, Any],
170 - attn_metadata: T) -> None:
175 + def prepare_graph_input_buffers(
176 + self,
177 + input_buffers: Dict[str, Any],
178 + attn_metadata: T,
179 + is_encoder_decoder_model: bool = False) -> None:
180 """In-place modify input buffers dict for CUDA graph replay."""
181 ...
182
vllm/attention/backends/flashinfer.py
+9 -3
@@ -172,7 +172,8 @@ class FlashInferState(AttentionState):
172 state._prefill_wrapper = self._get_prefill_wrapper()
173 return state
174
175 - def graph_capture_get_metadata_for_batch(self, batch_size: int):
175 + def graph_capture_get_metadata_for_batch(
176 + self, batch_size: int, is_encoder_decoder_model: bool = False):
177 assert self._is_graph_capturing
178 _indptr_buffer = self._graph_indptr_buffer[:batch_size + 1]
179 _last_page_len_buffer = self._graph_last_page_len_buffer[:batch_size]
@@ -232,12 +233,17 @@ class FlashInferState(AttentionState):
233 attn_metadata.begin_forward()
234 return attn_metadata
235
235 - def get_graph_input_buffers(self, attn_metadata):
236 + def get_graph_input_buffers(self,
237 + attn_metadata,
238 + is_encoder_decoder_model: bool = False):
239 return {
240 "slot_mapping": attn_metadata.slot_mapping,
241 }
242
240 - def prepare_graph_input_buffers(self, input_buffers, attn_metadata):
243 + def prepare_graph_input_buffers(self,
244 + input_buffers,
245 + attn_metadata,
246 + is_encoder_decoder_model: bool = False):
247 return
248
249 def begin_forward(self, model_input):
vllm/attention/backends/utils.py
+107 -6
@@ -304,7 +304,8 @@ class CommonAttentionState(AttentionState):
304 assert self._is_graph_capturing
305 return self.__class__(self.runner)
306
307 - def graph_capture_get_metadata_for_batch(self, batch_size: int):
307 + def graph_capture_get_metadata_for_batch(
308 + self, batch_size: int, is_encoder_decoder_model: bool = False):
309 assert self._is_graph_capturing
310 attn_metadata = self.runner.attn_backend.make_metadata(
311 num_prefills=0,
@@ -322,21 +323,121 @@ class CommonAttentionState(AttentionState):
323 block_tables=self._graph_block_tables[:batch_size],
324 use_cuda_graph=True,
325 )
326 + if is_encoder_decoder_model:
327 + # The encoder decoder model works only with XFormers backend.
328 + # Assert the same.
329 + assert self.runner.attn_backend.get_name() == "xformers", \
330 + f"Expected attn_backend name to be 'xformers', but "\
331 + f" got '{self.runner.attn_backend.get_name()}'"
332 + self._update_captured_metadata_for_enc_dec_model(
333 + batch_size=batch_size, attn_metadata=attn_metadata)
334 +
335 return attn_metadata
336
327 - def get_graph_input_buffers(self, attn_metadata) -> Dict[str, Any]:
328 - return {
337 + def get_graph_input_buffers(
338 + self,
339 + attn_metadata,
340 + is_encoder_decoder_model: bool = False) -> Dict[str, Any]:
341 + input_buffers = {
342 "slot_mapping": attn_metadata.slot_mapping,
343 "seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor,
344 "block_tables": attn_metadata.decode_metadata.block_tables,
345 }
333 -
334 - def prepare_graph_input_buffers(self, input_buffers,
335 - attn_metadata) -> None:
346 + if is_encoder_decoder_model:
347 + # The encoder decoder model works only with XFormers backend.
348 + # Assert the same.
349 + assert self.runner.attn_backend.get_name() == "xformers", \
350 + f"Expected attn_backend name to be 'xformers', but "\
351 + f" got '{self.runner.attn_backend.get_name()}'"
352 + self._add_additonal_input_buffers_for_enc_dec_model(
353 + attn_metadata=attn_metadata, input_buffers=input_buffers)
354 + return input_buffers
355 +
356 + def prepare_graph_input_buffers(
357 + self,
358 + input_buffers,
359 + attn_metadata,
360 + is_encoder_decoder_model: bool = False) -> None:
361 input_buffers["seq_lens_tensor"].copy_(
362 attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True)
363 input_buffers["block_tables"].copy_(
364 attn_metadata.decode_metadata.block_tables, non_blocking=True)
365 + if is_encoder_decoder_model:
366 + # The encoder decoder model works only with XFormers backend.
367 + # Assert the same.
368 + assert self.runner.attn_backend.get_name() == "xformers", \
369 + f"Expected attn_backend name to be 'xformers', but "\
370 + f" got '{self.runner.attn_backend.get_name()}'"
371 + self._prepare_input_buffers_for_enc_dec_model(
372 + attn_metadata, input_buffers)
373
374 def begin_forward(self, model_input) -> None:
375 return
376 +
377 + def _update_captured_metadata_for_enc_dec_model(self, batch_size: int,
378 + attn_metadata):
379 + """
380 + Updates the attention metadata parameters for CUDA graph capture in an
381 + encoder-decoder model.
382 +
383 + This method modifies attention-related tensors and metadata required
384 + for CUDA graph capture in encoder-decoder models. Specifically, it
385 + updates the cross-attention and encoder sequence tensors in the
386 + AttentionMetadata object.
387 + """
388 + # During decode phase the cross_slot_mapping will be empty. Hence set
389 + # an empty tensor for CUDA Graph capture.
390 + attn_metadata.cross_slot_mapping = torch.tensor(
391 + [], dtype=torch.int).cuda()
392 + attn_metadata.cross_block_tables = torch.full(
393 + (batch_size, self.runner.get_max_block_per_batch()),
394 + 1,
395 + dtype=torch.int).cuda()
396 + attn_metadata.encoder_seq_lens = torch.full((batch_size, ),
397 + 1,
398 + dtype=torch.int).cuda()
399 + attn_metadata.encoder_seq_lens_tensor = torch.full(
400 + (batch_size, ), 1, dtype=torch.int).cuda()
401 + attn_metadata.max_encoder_seq_len = self.runner.max_seq_len_to_capture
402 +
403 + def _add_additonal_input_buffers_for_enc_dec_model(
404 + self, attn_metadata, input_buffers: Dict[str, Any]):
405 + """
406 + Saves additional input buffers specific to the encoder-decoder model
407 + from the attention metadata.
408 +
409 + This method extracts and stores encoder-decoder related input buffers
410 + from the `attn_metadata` into the `input_buffers` dictionary. The
411 + buffers include encoder sequence lengths, cross-slot mappings, and
412 + cross-block tables, which are essential for the encoder-decoder model
413 + during CUDA graph replay.
414 + """
415 + input_buffers["encoder_seq_lens_tensor"] = (
416 + attn_metadata.decode_metadata.encoder_seq_lens_tensor)
417 + input_buffers["cross_slot_mapping"] = (
418 + attn_metadata.decode_metadata.cross_slot_mapping)
419 + input_buffers["cross_block_tables"] = (
420 + attn_metadata.decode_metadata.cross_block_tables)
421 +
422 + def _prepare_input_buffers_for_enc_dec_model(self, attn_metadata,
423 + input_buffers: Dict[str,
424 + Any]):
425 + """
426 + Populates input buffers with data from the encoder-decoder model's
427 + attention metadata.
428 +
429 + This method fills the input buffers with encoder-decoder specific
430 + tensors. It copies data from the `attn_metadata` and keyword arguments
431 + (`kwargs`) into corresponding buffers in the `input_buffers` dictionary.
432 + The copied data includes attention-related metadata as well as input
433 + IDs and positional information for the encoder.
434 + """
435 + input_buffers["encoder_seq_lens_tensor"].copy_(
436 + attn_metadata.decode_metadata.encoder_seq_lens_tensor,
437 + non_blocking=True)
438 + input_buffers["cross_slot_mapping"].copy_(
439 + attn_metadata.decode_metadata.cross_slot_mapping,
440 + non_blocking=True)
441 + input_buffers["cross_block_tables"].copy_(
442 + attn_metadata.decode_metadata.cross_block_tables,
443 + non_blocking=True)
vllm/config.py
+8 -33
@@ -16,9 +16,8 @@ from vllm.tracing import is_otel_available, otel_import_error_traceback
16 from vllm.transformers_utils.config import (ConfigFormat, get_config,
17 get_hf_image_processor_config,
18 get_hf_text_config)
19 -from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH, GiB_bytes,
20 - cuda_device_count_stateless, get_cpu_memory, is_cpu,
21 - is_hip, is_neuron, is_openvino, is_xpu,
19 +from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
20 + is_cpu, is_hip, is_neuron, is_openvino, is_xpu,
21 print_warning_once)
22
23 if TYPE_CHECKING:
@@ -96,15 +95,15 @@ class ModelConfig:
95 enforce_eager: Whether to enforce eager execution. If True, we will
96 disable CUDA graph and always execute the model in eager mode.
97 If False, we will use CUDA graph and eager execution in hybrid.
99 - If None, the user did not specify, so default to False -
100 - except for encoder/decoder models, which currently require
101 - eager mode.
98 + If None, the user did not specify, so default to False.
99 max_context_len_to_capture: Maximum context len covered by CUDA graphs.
100 When a sequence has context length larger than this, we fall back
101 to eager mode (DEPRECATED. Use max_seq_len_to_capture instead).
102 max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
103 When a sequence has context length larger than this, we fall back
107 - to eager mode
104 + to eager mode. Additionally for encoder-decoder models, if the
105 + sequence length of the encoder input is larger than this, we fall
106 + back to the eager mode.
107 disable_sliding_window: Whether to disable sliding window. If True,
108 we will disable the sliding window functionality of the model.
109 If the model does not support sliding window, this argument is
@@ -186,32 +185,8 @@ class ModelConfig:
185 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
186 self.use_async_output_proc = use_async_output_proc
187
189 - # Choose a default enforce_eager value if the user did not specify
190 - # a value (enforce_eager is None)
191 - if getattr(self.hf_config, 'is_encoder_decoder', False):
192 - if self.enforce_eager is None:
193 - # *Only for encoder/decoder models* and
194 - # *only if enforce_eager is unset*, override
195 - # to enforce_eager=True
196 - #
197 - # Add a logger message since it is *somewhat* non-intuitive that
198 - # enforce_eager is True when the user has not specified its
199 - # value.
200 - logger.info("Forcing enforce_eager == True because "
201 - "enforce_eager setting was unspecified and "
202 - "CUDAGraph is not supported with encoder/ "
203 - "decoder models.")
204 - self.enforce_eager = True
205 -
206 - if not self.enforce_eager:
207 - # Eager mode explicitly disabled by user for an encoder/
208 - # decoder model; however CUDAGRAPH + encoder/decoder is
209 - # not currently supported
210 - raise ValueError(STR_NOT_IMPL_ENC_DEC_CUDAGRAPH)
211 - elif self.enforce_eager is None:
212 - # *Only for decoder-only models*, enforce_eager
213 - # defaults to False if unset. This is intuitive
214 - # so no logging message needed.
188 + # Set enforce_eager to False if the value is unset.
189 + if self.enforce_eager is None:
190 self.enforce_eager = False
191
192 if (not self.disable_sliding_window
vllm/engine/arg_utils.py
+4 -1
@@ -472,7 +472,10 @@ class EngineArgs:
472 default=EngineArgs.max_seq_len_to_capture,
473 help='Maximum sequence length covered by CUDA '
474 'graphs. When a sequence has context length '
475 - 'larger than this, we fall back to eager mode.')
475 + 'larger than this, we fall back to eager mode. '
476 + 'Additionally for encoder-decoder models, if the '
477 + 'sequence length of the encoder input is larger '
478 + 'than this, we fall back to the eager mode.')
479 parser.add_argument('--disable-custom-all-reduce',
480 action='store_true',
481 default=EngineArgs.disable_custom_all_reduce,
vllm/entrypoints/llm.py
+4 -4
@@ -88,7 +88,9 @@ class LLM:
88 to eager mode (DEPRECATED. Use `max_seq_len_to_capture` instead).
89 max_seq_len_to_capture: Maximum sequence len covered by CUDA graphs.
90 When a sequence has context length larger than this, we fall back
91 - to eager mode.
91 + to eager mode. Additionally for encoder-decoder models, if the
92 + sequence length of the encoder input is larger than this, we fall
93 + back to the eager mode.
94 disable_custom_all_reduce: See ParallelConfig
95 **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
96 :ref:`engine_args`)
@@ -137,9 +139,7 @@ class LLM:
139 LLM constructor.
140
141 Note: if enforce_eager is unset (enforce_eager is None)
140 - it defaults to False for decoder-only models and True
141 - for encoder/decoder models, since encoder/decoder models
142 - do not currently support CUDAGraph.
142 + it defaults to False.
143 '''
144
145 if "disable_log_stats" not in kwargs:
vllm/model_executor/models/bart.py
+4 -2
@@ -848,11 +848,13 @@ class BartForConditionalGeneration(nn.Module):
848 self,
849 input_ids: torch.Tensor,
850 positions: torch.Tensor,
851 - encoder_input_ids: torch.Tensor,
852 - encoder_positions: torch.Tensor,
851 kv_caches: List[torch.Tensor],
852 attn_metadata: AttentionMetadata,
853 intermediate_tensors: Optional[IntermediateTensors] = None,
854 + *,
855 + encoder_input_ids: torch.Tensor,
856 + encoder_positions: torch.Tensor,
857 + **kwargs,
858 ) -> torch.Tensor:
859 r"""
860 Args:
vllm/utils.py
-5
@@ -71,10 +71,6 @@ STR_NOT_IMPL_ENC_DEC_SPEC_DEC = ("Speculative decoding is not "
71 "currently supported with encoder/"
72 "decoder models.")
73
74 -STR_NOT_IMPL_ENC_DEC_CUDAGRAPH = ("CUDAGraph is not "
75 - "currently supported with encoder/"
76 - "decoder models.")
77 -
74 STR_NOT_IMPL_ENC_DEC_BACKEND = ("XFormers is the only backend "
75 "currently supported with encoder/"
76 "decoder models.")
@@ -98,7 +94,6 @@ STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
94 "STR_NOT_IMPL_ENC_DEC_PP": STR_NOT_IMPL_ENC_DEC_PP,
95 "STR_NOT_IMPL_ENC_DEC_MM": STR_NOT_IMPL_ENC_DEC_MM,
96 "STR_NOT_IMPL_ENC_DEC_SPEC_DEC": STR_NOT_IMPL_ENC_DEC_SPEC_DEC,
101 - "STR_NOT_IMPL_ENC_DEC_CUDA_GRAPH": STR_NOT_IMPL_ENC_DEC_CUDAGRAPH,
97 "STR_NOT_IMPL_ENC_DEC_BACKEND": STR_NOT_IMPL_ENC_DEC_BACKEND,
98 "STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER": STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER,
99 "STR_NOT_IMPL_ENC_DEC_CPU": STR_NOT_IMPL_ENC_DEC_CPU
vllm/worker/enc_dec_model_runner.py
+36 -7
@@ -1,4 +1,5 @@
1 import dataclasses
2 +import itertools
3 from typing import Any, Dict, List, Optional, Tuple, Type, cast
4
5 import torch
@@ -24,7 +25,8 @@ from vllm.sequence import (IntermediateTensors, PoolerOutput,
25 from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad
26 from vllm.worker.model_runner import (GPUModelRunnerBase,
27 ModelInputForGPUBuilder,
27 - ModelInputForGPUWithSamplingMetadata)
28 + ModelInputForGPUWithSamplingMetadata,
29 + _get_graph_batch_size)
30 from vllm.worker.model_runner_base import (
31 _add_attn_metadata_broadcastable_dict,
32 _add_sampling_metadata_broadcastable_dict)
@@ -178,7 +180,15 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
180 raise ValueError("num_steps > 1 is not supported in "
181 "EncoderDecoderModelRunner")
182
181 - model_executable = self.model
183 + if (model_input.attn_metadata is not None
184 + and model_input.attn_metadata.prefill_metadata is None
185 + and model_input.attn_metadata.decode_metadata.use_cuda_graph):
186 + assert model_input.input_tokens is not None
187 + graph_batch_size = model_input.input_tokens.shape[0]
188 + model_executable = self.graph_runners[
189 + model_input.virtual_engine][graph_batch_size]
190 + else:
191 + model_executable = self.model
192
193 seqlen_agnostic_kwargs = {
194 "finished_requests_ids": model_input.finished_requests_ids,
@@ -200,6 +210,9 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
210 if not self.is_driver_worker:
211 return []
212
213 + if model_input.async_callback is not None:
214 + model_input.async_callback()
215 +
216 # Sample the next token.
217 output: SamplerOutput = self.model.sample(
218 logits=logits,
@@ -231,14 +244,12 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
244 """
245 model_input = self._prepare_model_input_tensors(
246 seq_group_metadata_list, finished_requests_ids)
234 -
247 (
248 attn_metadata,
249 encoder_input_tokens_tensor,
250 encoder_input_positions_tensor,
251 ) = (self._prepare_encoder_model_input_tensors(seq_group_metadata_list,
252 model_input))
241 -
253 # Inject attn_metadata encoder/cross-attention fields &
254 # encoder input tokens/positions into model_input.
255 # Frozen dataclass fields cannot be modified, so use
@@ -437,11 +448,29 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
448 cross_block_tables.append([] if (
449 cross_block_table is None) else cross_block_table)
450
440 - # Convert cross-attention block tables to encoder input tensor
451 + if (model_input.attn_metadata is not None
452 + and model_input.attn_metadata.use_cuda_graph):
453 + # We will be using CUDA graph replay for this decode.
454 + max_len_of_block_table = self.get_max_block_per_batch()
455 + batch_size = len(encoder_seq_lens)
456 + graph_batch_size = _get_graph_batch_size(batch_size)
457 + assert graph_batch_size >= batch_size
458 + cuda_graph_pad_size = graph_batch_size - batch_size
459 + # extend the cross_block_tables and encoder_seq_lens to match
460 + # the graph_batch_size.
461 + cross_block_tables.extend([[]
462 + for _ in range(cuda_graph_pad_size)
463 + ])
464 + encoder_seq_lens.extend(
465 + itertools.repeat(1, cuda_graph_pad_size))
466 +
467 + else:
468 + max_len_of_block_table = max(
469 + len(block_table) for block_table in cross_block_tables)
470 +
471 cross_block_tables = make_tensor_with_pad(
472 cross_block_tables,
443 - max_len=max(
444 - len(block_table) for block_table in cross_block_tables),
473 + max_len=max_len_of_block_table,
474 pad=0,
475 dtype=torch.int32,
476 device=self.device,
vllm/worker/model_runner.py
+76 -21
@@ -243,6 +243,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
243 prefix_cache_hit: bool = False,
244 reinit: bool = False,
245 reinit_use_defaults: bool = False,
246 + encoder_seq_len: int = 0,
247 ):
248 if reinit:
249 assert len(self.seq_ids) == len(seq_ids) # type: ignore
@@ -256,6 +257,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
257 self.block_tables = block_tables
258 self.computed_block_nums = computed_block_nums
259 self.n_seqs = n_seqs
260 + self.encoder_seq_len = encoder_seq_len
261
262 if reinit:
263 if len(self.seq_ids) == 1 and reinit_use_defaults:
@@ -702,6 +704,11 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
704 assert n_seqs == 1
705 self.decode_only = False
706
707 + encoder_seq_len = 0
708 +
709 + if self.runner.model_config.is_encoder_decoder_model:
710 + encoder_seq_len = seq_group_metadata.encoder_seq_data.get_len()
711 +
712 inter_data = self.init_cached_inter_data(
713 request_id=seq_group_metadata.request_id,
714 seq_ids=seq_ids,
@@ -709,7 +716,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
716 block_tables=seq_group_metadata.block_tables,
717 computed_block_nums=seq_group_metadata.computed_block_nums,
718 reinit=True,
712 - reinit_use_defaults=True)
719 + reinit_use_defaults=True,
720 + encoder_seq_len=encoder_seq_len)
721
722 self.inter_data_list.append(inter_data)
723
@@ -719,11 +727,15 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
727 for per_seq_group_fn in self.per_seq_group_compute_fns:
728 per_seq_group_fn(inter_data, seq_group_metadata)
729
722 - def _use_captured_graph(self, batch_size: int,
723 - max_decode_seq_len: int) -> bool:
730 + def _use_captured_graph(self,
731 + batch_size: int,
732 + max_decode_seq_len: int,
733 + max_encoder_seq_len: int = 0) -> bool:
734 return (self.decode_only and not self.runner.model_config.enforce_eager
725 - and batch_size <= self.runner.max_batchsize_to_capture
726 - and max_decode_seq_len <= self.runner.max_seq_len_to_capture)
735 + and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
736 + and max_decode_seq_len <= self.runner.max_seq_len_to_capture
737 + and max_encoder_seq_len <= self.runner.max_seq_len_to_capture
738 + and batch_size <= self.runner.max_batchsize_to_capture)
739
740 def build(self) -> ModelInputForGPU:
741 """Finalize the builder intermediate data and
@@ -763,15 +775,18 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
775 input_positions.extend(cur_input_positions)
776
777 seq_lens = []
778 + query_lens = []
779 max_decode_seq_len = 0
780 + max_encoder_seq_len = 0
781 for inter_data in self.inter_data_list:
782 seq_lens.extend(inter_data.seq_lens)
783 + query_lens.extend(inter_data.query_lens)
784 if not inter_data.is_prompt:
785 max_decode_seq_len = max(max_decode_seq_len,
786 max(inter_data.seq_lens))
772 - query_lens = []
773 - for inter_data in self.inter_data_list:
774 - query_lens.extend(inter_data.query_lens)
787 + if self.runner.model_config.is_encoder_decoder_model:
788 + max_encoder_seq_len = max(max_encoder_seq_len,
789 + inter_data.encoder_seq_len)
790
791 # Mapping from request IDs to sequence IDs. Used for Jamba models
792 # that manages the cache by itself.
@@ -781,8 +796,10 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
796 }
797
798 batch_size = len(input_tokens)
784 - use_captured_graph = self._use_captured_graph(batch_size,
785 - max_decode_seq_len)
799 + use_captured_graph = self._use_captured_graph(
800 + batch_size,
801 + max_decode_seq_len,
802 + max_encoder_seq_len=max_encoder_seq_len)
803
804 # If cuda graph can be used, pad tensors accordingly.
805 # See `capture_model` API for more details.
@@ -1364,7 +1381,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1381 for batch_size in reversed(batch_size_capture_list):
1382 attn_metadata = (
1383 self.attn_state.graph_capture_get_metadata_for_batch(
1367 - batch_size))
1384 + batch_size,
1385 + is_encoder_decoder_model=self.model_config.
1386 + is_encoder_decoder_model))
1387
1388 if self.lora_config:
1389 lora_mapping = LoRAMapping(
@@ -1380,10 +1399,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1399 )
1400 self.set_active_prompt_adapters(
1401 set(), prompt_adapter_mapping)
1383 -
1402 graph_runner = CUDAGraphRunner(
1403 self.model, self.attn_backend.get_name(),
1386 - self.attn_state.graph_clone(batch_size))
1404 + self.attn_state.graph_clone(batch_size),
1405 + self.model_config.is_encoder_decoder_model)
1406
1407 capture_inputs = {
1408 "input_ids":
@@ -1420,6 +1439,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1439 self.model.get_seqlen_agnostic_capture_inputs(
1440 batch_size)
1441 })
1442 + if self.model_config.is_encoder_decoder_model:
1443 + # add the additional inputs to capture for
1444 + # encoder-decoder models.
1445 + self._update_inputs_to_capture_for_enc_dec_model(
1446 + capture_inputs)
1447 +
1448 graph_runner.capture(**capture_inputs)
1449 self.graph_memory_pool = graph_runner.graph.pool()
1450 self.graph_runners[virtual_engine][batch_size] = (
@@ -1430,6 +1455,24 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1455 # This usually takes < 10 seconds.
1456 logger.info("Graph capturing finished in %.0f secs.", elapsed_time)
1457
1458 + def _update_inputs_to_capture_for_enc_dec_model(self,
1459 + capture_inputs: Dict[str,
1460 + Any]):
1461 + """
1462 + Updates the set of input tensors needed for CUDA graph capture in an
1463 + encoder-decoder model.
1464 +
1465 + This method modifies the provided `capture_inputs` dictionary by
1466 + adding tensors specific to encoder-decoder specific models that
1467 + need to be captured for CUDA Graph replay.
1468 + """
1469 + # During the decode phase encoder_input_ids and encoder_positions are
1470 + # unset. Do the same thing for graph capture.
1471 + capture_inputs["encoder_input_ids"] = torch.tensor(
1472 + [], dtype=torch.long).cuda()
1473 + capture_inputs["encoder_positions"] = torch.tensor(
1474 + [], dtype=torch.long).cuda()
1475 +
1476 @property
1477 def vocab_size(self) -> int:
1478 return self.model_config.get_vocab_size()
@@ -1629,7 +1672,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1672 class CUDAGraphRunner:
1673
1674 def __init__(self, model: nn.Module, backend_name: str,
1632 - attn_state: AttentionState):
1675 + attn_state: AttentionState, is_encoder_decoder_model: bool):
1676 self.model = model
1677 self.backend_name = backend_name
1678 self.attn_state = attn_state
@@ -1638,6 +1681,7 @@ class CUDAGraphRunner:
1681 self.output_buffers: Dict[str, torch.Tensor] = {}
1682
1683 self._graph: Optional[torch.cuda.CUDAGraph] = None
1684 + self._is_encoder_decoder_model = is_encoder_decoder_model
1685
1686 @property
1687 def graph(self):
@@ -1671,8 +1715,9 @@ class CUDAGraphRunner:
1715 intermediate_tensors=intermediate_inputs,
1716 **kwargs,
1717 )
1718 + # Wait for the warm up operations to finish before proceeding with
1719 + # Graph Capture.
1720 torch.cuda.synchronize()
1675 -
1721 # Capture the graph.
1722 self._graph = torch.cuda.CUDAGraph()
1723 with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream):
@@ -1704,10 +1749,14 @@ class CUDAGraphRunner:
1749
1750 # Save the input and output buffers.
1751 self.input_buffers = {
1707 - "input_ids": input_ids,
1708 - "positions": positions,
1709 - "kv_caches": kv_caches,
1710 - **self.attn_state.get_graph_input_buffers(attn_metadata),
1752 + "input_ids":
1753 + input_ids,
1754 + "positions":
1755 + positions,
1756 + "kv_caches":
1757 + kv_caches,
1758 + **self.attn_state.get_graph_input_buffers(
1759 + attn_metadata, self._is_encoder_decoder_model),
1760 **kwargs,
1761 }
1762 if intermediate_inputs is not None:
@@ -1737,8 +1786,8 @@ class CUDAGraphRunner:
1786 self.input_buffers["positions"].copy_(positions, non_blocking=True)
1787 self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1788 non_blocking=True)
1740 - self.attn_state.prepare_graph_input_buffers(self.input_buffers,
1741 - attn_metadata)
1789 + self.attn_state.prepare_graph_input_buffers(
1790 + self.input_buffers, attn_metadata, self._is_encoder_decoder_model)
1791 if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1792 self.model.copy_inputs_before_cuda_graphs(self.input_buffers,
1793 **kwargs)
@@ -1752,6 +1801,12 @@ class CUDAGraphRunner:
1801 if key != "model_execute_time" and key != "model_forward_time":
1802 self.input_buffers[key].copy_(intermediate_tensors[key],
1803 non_blocking=True)
1804 + if self._is_encoder_decoder_model:
1805 + self.input_buffers["encoder_input_ids"].copy_(
1806 + kwargs['encoder_input_ids'], non_blocking=True)
1807 + self.input_buffers["encoder_positions"].copy_(
1808 + kwargs['encoder_positions'], non_blocking=True)
1809 +
1810 # Run the graph.
1811 self.graph.replay()
1812 # Return the output tensor.
vllm/worker/utils.py
-4
@@ -47,10 +47,6 @@ def assert_enc_dec_mr_supported_scenario(
47 raise NotImplementedError(
48 STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_SPEC_DEC'])
49
50 - if not enc_dec_mr.model_config.enforce_eager:
51 - raise NotImplementedError(
52 - STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CUDA_GRAPH'])
53 -
50 if enc_dec_mr.prompt_adapter_config is not None:
51 raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_ERR_STRS[
52 'STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER'])