@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fd95e026e0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Subclass ModelRunner to support cross-attention & encoder sequences (towards eventual encoder/decoder model support) (#4942)

Co-authored-by: Andrew Feldman <afeld2012@gmail.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>

afeldman-nm committed Aug 6, 2024 at 16:51 UTC fd95e026e0f9f50bacf1a63ef419df8bacfc99c0
33 files changed +3976 -352
.buildkite/test-pipeline.yaml
+3 -1
@@ -148,8 +148,9 @@ steps:
148 - python3 cpu_offload.py
149 - python3 offline_inference_with_prefix.py
150 - python3 llm_engine_example.py
151 - - python3 llava_example.py
151 + - python3 offline_inference_vision_language.py
152 - python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
153 + - python3 offline_inference_encoder_decoder.py
154
155 - label: Models Test # 1hr10min
156 source_file_dependencies:
@@ -289,6 +290,7 @@ steps:
290 commands:
291 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
292 - TARGET_TEST_SUITE=L4 pytest -v -s distributed/test_basic_distributed_correctness.py
293 + - pytest -v -s distributed/test_basic_distributed_correctness_enc_dec.py
294 - pytest -v -s distributed/test_chunked_prefill_distributed.py
295 - pytest -v -s distributed/test_multimodal_broadcast.py
296 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
examples/offline_inference_encoder_decoder.py new
+99
@@ -0,0 +1,99 @@
1 +'''
2 +Demonstrate prompting of text-to-text
3 +encoder/decoder models, specifically BART
4 +'''
5 +
6 +from vllm import LLM, SamplingParams
7 +from vllm.inputs import ExplicitEncoderDecoderPrompt, TextPrompt, TokensPrompt
8 +from vllm.utils import zip_enc_dec_prompt_lists
9 +
10 +dtype = "float"
11 +
12 +# Create a BART encoder/decoder model instance
13 +llm = LLM(
14 + model="facebook/bart-large-cnn",
15 + dtype=dtype,
16 +)
17 +
18 +# Get BART tokenizer
19 +tokenizer = llm.llm_engine.get_tokenizer_group()
20 +
21 +# Test prompts
22 +#
23 +# This section shows all of the valid ways to prompt an
24 +# encoder/decoder model.
25 +#
26 +# - Helpers for building prompts
27 +text_prompt_raw = "Hello, my name is"
28 +text_prompt = TextPrompt(prompt="The president of the United States is")
29 +tokens_prompt = TokensPrompt(prompt_token_ids=tokenizer.encode(
30 + prompt="The capital of France is"))
31 +# - Pass a single prompt to encoder/decoder model
32 +# (implicitly encoder input prompt);
33 +# decoder input prompt is assumed to be None
34 +
35 +single_text_prompt_raw = text_prompt_raw # Pass a string directly
36 +single_text_prompt = text_prompt # Pass a TextPrompt
37 +single_tokens_prompt = tokens_prompt # Pass a TokensPrompt
38 +
39 +# - Pass explicit encoder and decoder input prompts within one data structure.
40 +# Encoder and decoder prompts can both independently be text or tokens, with
41 +# no requirement that they be the same prompt type. Some example prompt-type
42 +# combinations are shown below, note that these are not exhaustive.
43 +
44 +enc_dec_prompt1 = ExplicitEncoderDecoderPrompt(
45 + # Pass encoder prompt string directly, &
46 + # pass decoder prompt tokens
47 + encoder_prompt=single_text_prompt_raw,
48 + decoder_prompt=single_tokens_prompt,
49 +)
50 +enc_dec_prompt2 = ExplicitEncoderDecoderPrompt(
51 + # Pass TextPrompt to encoder, and
52 + # pass decoder prompt string directly
53 + encoder_prompt=single_text_prompt,
54 + decoder_prompt=single_text_prompt_raw,
55 +)
56 +enc_dec_prompt3 = ExplicitEncoderDecoderPrompt(
57 + # Pass encoder prompt tokens directly, and
58 + # pass TextPrompt to decoder
59 + encoder_prompt=single_tokens_prompt,
60 + decoder_prompt=single_text_prompt,
61 +)
62 +
63 +# - Finally, here's a useful helper function for zipping encoder and
64 +# decoder prompt lists together into a list of ExplicitEncoderDecoderPrompt
65 +# instances
66 +zipped_prompt_list = zip_enc_dec_prompt_lists(
67 + ['An encoder prompt', 'Another encoder prompt'],
68 + ['A decoder prompt', 'Another decoder prompt'])
69 +
70 +# - Let's put all of the above example prompts together into one list
71 +# which we will pass to the encoder/decoder LLM.
72 +prompts = [
73 + single_text_prompt_raw, single_text_prompt, single_tokens_prompt,
74 + enc_dec_prompt1, enc_dec_prompt2, enc_dec_prompt3
75 +] + zipped_prompt_list
76 +
77 +print(prompts)
78 +
79 +# Create a sampling params object.
80 +sampling_params = SamplingParams(
81 + temperature=0,
82 + top_p=1.0,
83 + min_tokens=0,
84 + max_tokens=20,
85 +)
86 +
87 +# Generate output tokens from the prompts. The output is a list of
88 +# RequestOutput objects that contain the prompt, generated
89 +# text, and other information.
90 +outputs = llm.generate(prompts, sampling_params)
91 +
92 +# Print the outputs.
93 +for output in outputs:
94 + prompt = output.prompt
95 + encoder_prompt = output.encoder_prompt
96 + generated_text = output.outputs[0].text
97 + print(f"Encoder prompt: {encoder_prompt!r}, "
98 + f"Decoder prompt: {prompt!r}, "
99 + f"Generated text: {generated_text!r}")
tests/conftest.py
+185 -37
@@ -10,9 +10,11 @@ import torch
10 import torch.nn as nn
11 import torch.nn.functional as F
12 from PIL import Image
13 -from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
14 - AutoTokenizer, BatchEncoding, BatchFeature)
13 +from transformers import (AutoModelForCausalLM, AutoModelForSeq2SeqLM,
14 + AutoModelForVision2Seq, AutoTokenizer, BatchEncoding,
15 + BatchFeature)
16
17 +from tests.models.utils import DecoderPromptType
18 from vllm import LLM, SamplingParams
19 from vllm.assets.image import ImageAsset
20 from vllm.config import TokenizerPoolConfig
@@ -21,9 +23,11 @@ from vllm.distributed import (destroy_distributed_environment,
23 destroy_model_parallel)
24 from vllm.inputs import TextPrompt
25 from vllm.logger import init_logger
26 +from vllm.outputs import RequestOutput
27 from vllm.sequence import SampleLogprobs
28 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
26 - is_cpu)
29 + is_cpu, to_enc_dec_tuple_list,
30 + zip_enc_dec_prompt_lists)
31
32 logger = init_logger(__name__)
33
@@ -120,6 +124,40 @@ def example_prompts() -> List[str]:
124 return prompts
125
126
127 +@pytest.fixture
128 +def example_encoder_decoder_prompts() \
129 + -> Dict[DecoderPromptType,
130 + Tuple[List[str], List[Optional[str]]]]:
131 + '''
132 + Returns an encoder prompt list and a decoder prompt list, wherein each pair
133 + of same-index entries in both lists corresponds to an (encoder prompt,
134 + decoder prompt) tuple.
135 +
136 + Returns:
137 +
138 + * Encoder prompt list
139 + * Decoder prompt list (reverse of encoder prompt list)
140 + '''
141 +
142 + encoder_prompts = []
143 + for filename in _TEST_PROMPTS:
144 + encoder_prompts += _read_prompts(filename)
145 +
146 + custom_decoder_prompts = encoder_prompts[::-1]
147 + empty_str_decoder_prompts = [""] * len(encoder_prompts)
148 + none_decoder_prompts = [None] * len(encoder_prompts)
149 +
150 + # NONE decoder prompt type
151 + return {
152 + DecoderPromptType.NONE:
153 + zip_enc_dec_prompt_lists(encoder_prompts, none_decoder_prompts),
154 + DecoderPromptType.EMPTY_STR:
155 + zip_enc_dec_prompt_lists(encoder_prompts, empty_str_decoder_prompts),
156 + DecoderPromptType.CUSTOM:
157 + zip_enc_dec_prompt_lists(encoder_prompts, custom_decoder_prompts),
158 + }
159 +
160 +
161 @pytest.fixture
162 def example_long_prompts() -> List[str]:
163 prompts = []
@@ -152,6 +190,7 @@ class HfRunner:
190 model_kwargs: Optional[Dict[str, Any]] = None,
191 is_embedding_model: bool = False,
192 is_vision_model: bool = False,
193 + is_encoder_decoder_model: bool = False,
194 ) -> None:
195 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
196
@@ -168,6 +207,8 @@ class HfRunner:
207 else:
208 if is_vision_model:
209 auto_cls = AutoModelForVision2Seq
210 + elif is_encoder_decoder_model:
211 + auto_cls = AutoModelForSeq2SeqLM
212 else:
213 auto_cls = AutoModelForCausalLM
214
@@ -314,6 +355,44 @@ class HfRunner:
355 all_logprobs.append(seq_logprobs)
356 return all_logprobs
357
358 + def _hidden_states_to_logprobs(
359 + self,
360 + hidden_states,
361 + num_logprobs,
362 + ) -> Tuple[List[Dict[int, float]], int]:
363 + seq_logprobs: List[torch.Tensor] = []
364 + output_len = len(hidden_states)
365 + for _, hidden_state in enumerate(hidden_states):
366 + last_hidden_states = hidden_state[-1][0]
367 + logits = torch.matmul(
368 + last_hidden_states,
369 + self.model.get_output_embeddings().weight.t(),
370 + )
371 + if getattr(self.model.get_output_embeddings(), "bias",
372 + None) is not None:
373 + logits += self.model.get_output_embeddings().bias.unsqueeze(0)
374 + logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
375 + seq_logprobs.append(logprobs)
376 +
377 + # convert to dict
378 + seq_logprobs_lst: List[Dict[int, float]] = []
379 + for tok_idx, tok_logprobs in enumerate(seq_logprobs):
380 + # drop prompt logprobs
381 + if tok_idx == 0:
382 + tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)
383 + topk = tok_logprobs.topk(num_logprobs)
384 +
385 + tok_logprobs_dct = {}
386 + for token_id, logprob in zip(topk.indices[0], topk.values[0]):
387 + tok_logprobs_dct[token_id.item()] = logprob.item()
388 +
389 + seq_logprobs_lst.append(tok_logprobs_dct)
390 +
391 + return (
392 + seq_logprobs_lst,
393 + output_len,
394 + )
395 +
396 def generate_greedy_logprobs_limit(
397 self,
398 prompts: List[str],
@@ -346,37 +425,66 @@ class HfRunner:
425 **kwargs,
426 )
427
349 - seq_logprobs: List[torch.Tensor] = []
350 - for _, hidden_states in enumerate(output.hidden_states):
351 - last_hidden_states = hidden_states[-1][0]
352 - logits = torch.matmul(
353 - last_hidden_states,
354 - self.model.get_output_embeddings().weight.t(),
355 - )
356 - if getattr(self.model.get_output_embeddings(), "bias",
357 - None) is not None:
358 - logits += self.model.get_output_embeddings(
359 - ).bias.unsqueeze(0)
360 - logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
361 - seq_logprobs.append(logprobs)
428 + (
429 + seq_logprobs_lst,
430 + output_len,
431 + ) = self._hidden_states_to_logprobs(output.hidden_states,
432 + num_logprobs)
433 +
434 + all_logprobs.append(seq_logprobs_lst)
435 + seq_ids = output.sequences[0]
436 + output_len = len(seq_logprobs_lst)
437 + output_ids = seq_ids[-output_len:]
438 + all_output_ids.append(output_ids.tolist())
439 + all_output_strs.append(self.tokenizer.decode(output_ids))
440
363 - # convert to dict
364 - seq_logprobs_lst: List[Dict[int, float]] = []
365 - for tok_idx, tok_logprobs in enumerate(seq_logprobs):
366 - # drop prompt logprobs
367 - if tok_idx == 0:
368 - tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)
369 - topk = tok_logprobs.topk(num_logprobs)
441 + outputs = zip(all_output_ids, all_output_strs, all_logprobs)
442 + return [(output_ids, output_str, output_logprobs)
443 + for output_ids, output_str, output_logprobs in outputs]
444 +
445 + def generate_encoder_decoder_greedy_logprobs_limit(
446 + self,
447 + encoder_decoder_prompts: Tuple[List[str], List[str]],
448 + max_tokens: int,
449 + num_logprobs: int,
450 + **kwargs: Any,
451 + ) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
452 + '''
453 + Greedy logprobs generation for vLLM encoder/decoder models
454 + '''
455
371 - tok_logprobs_dct = {}
372 - for token_id, logprob in zip(topk.indices[0], topk.values[0]):
373 - tok_logprobs_dct[token_id.item()] = logprob.item()
456 + all_logprobs: List[List[Dict[int, float]]] = []
457 + all_output_ids: List[List[int]] = []
458 + all_output_strs: List[str] = []
459
375 - seq_logprobs_lst.append(tok_logprobs_dct)
460 + for (encoder_prompt,
461 + decoder_prompt) in to_enc_dec_tuple_list(encoder_decoder_prompts):
462 + encoder_input_ids = self.wrap_device(
463 + self.tokenizer(encoder_prompt, return_tensors="pt").input_ids)
464 + decoder_input_ids = (
465 + None if decoder_prompt is None else self.wrap_device(
466 + self.tokenizer(decoder_prompt,
467 + return_tensors="pt").input_ids))
468 +
469 + output = self.model.generate(
470 + encoder_input_ids,
471 + decoder_input_ids=decoder_input_ids,
472 + use_cache=True,
473 + do_sample=False,
474 + max_new_tokens=max_tokens,
475 + output_hidden_states=True,
476 + return_dict_in_generate=True,
477 + **kwargs,
478 + )
479 +
480 + (
481 + seq_logprobs_lst,
482 + output_len,
483 + ) = self._hidden_states_to_logprobs(output.decoder_hidden_states,
484 + num_logprobs)
485
486 all_logprobs.append(seq_logprobs_lst)
487 seq_ids = output.sequences[0]
379 - output_len = len(seq_logprobs_lst)
488 output_ids = seq_ids[-output_len:]
489 all_output_ids.append(output_ids.tolist())
490 all_output_strs.append(self.tokenizer.decode(output_ids))
@@ -416,7 +524,7 @@ class VllmRunner:
524 block_size: int = 16,
525 enable_chunked_prefill: bool = False,
526 swap_space: int = 4,
419 - enforce_eager: bool = False,
527 + enforce_eager: Optional[bool] = False,
528 **kwargs,
529 ) -> None:
530 self.model = LLM(
@@ -465,6 +573,19 @@ class VllmRunner:
573 outputs.append((req_sample_output_ids, req_sample_output_strs))
574 return outputs
575
576 + def _final_steps_generate_w_logprobs(
577 + self,
578 + req_outputs: List[RequestOutput],
579 + ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
580 + outputs: List[Tuple[List[int], str, Optional[SampleLogprobs]]] = []
581 + for req_output in req_outputs:
582 + for sample in req_output.outputs:
583 + output_str = sample.text
584 + output_ids = sample.token_ids
585 + output_logprobs = sample.logprobs
586 + outputs.append((output_ids, output_str, output_logprobs))
587 + return outputs
588 +
589 def generate_w_logprobs(
590 self,
591 prompts: List[str],
@@ -483,14 +604,21 @@ class VllmRunner:
604
605 req_outputs = self.model.generate(inputs,
606 sampling_params=sampling_params)
486 - outputs: List[Tuple[List[int], str, Optional[SampleLogprobs]]] = []
487 - for req_output in req_outputs:
488 - for sample in req_output.outputs:
489 - output_str = sample.text
490 - output_ids = sample.token_ids
491 - output_logprobs = sample.logprobs
492 - outputs.append((output_ids, output_str, output_logprobs))
493 - return outputs
607 + return self._final_steps_generate_w_logprobs(req_outputs)
608 +
609 + def generate_encoder_decoder_w_logprobs(
610 + self,
611 + encoder_decoder_prompts: Tuple[List[str], List[str]],
612 + sampling_params: SamplingParams,
613 + ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
614 + '''
615 + Logprobs generation for vLLM encoder/decoder models
616 + '''
617 +
618 + assert sampling_params.logprobs is not None
619 + req_outputs = self.model.generate(encoder_decoder_prompts,
620 + sampling_params=sampling_params)
621 + return self._final_steps_generate_w_logprobs(req_outputs)
622
623 def generate_greedy(
624 self,
@@ -523,6 +651,26 @@ class VllmRunner:
651 return [(output_ids, output_str, output_logprobs)
652 for output_ids, output_str, output_logprobs in outputs]
653
654 + def generate_encoder_decoder_greedy_logprobs(
655 + self,
656 + encoder_decoder_prompts: Tuple[List[str], List[str]],
657 + max_tokens: int,
658 + num_logprobs: int,
659 + ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
660 + greedy_logprobs_params = SamplingParams(temperature=0.0,
661 + use_beam_search=False,
662 + max_tokens=max_tokens,
663 + logprobs=num_logprobs)
664 + '''
665 + Greedy logprobs generation for vLLM encoder/decoder models
666 + '''
667 +
668 + outputs = self.generate_encoder_decoder_w_logprobs(
669 + encoder_decoder_prompts, greedy_logprobs_params)
670 +
671 + return [(output_ids, output_str, output_logprobs)
672 + for output_ids, output_str, output_logprobs in outputs]
673 +
674 def generate_beam_search(
675 self,
676 prompts: List[str],
tests/core/test_scheduler.py
+4 -26
@@ -9,33 +9,11 @@ from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
9 from vllm.core.interfaces import AllocStatus
10 from vllm.core.scheduler import Scheduler, SchedulingBudget
11 from vllm.lora.request import LoRARequest
12 -from vllm.sequence import Logprob, SequenceGroup, SequenceStatus
12 +from vllm.sequence import SequenceGroup, SequenceStatus
13
14 -from .utils import create_dummy_prompt
15 -
16 -
17 -def get_sequence_groups(scheduler_output):
18 - return [s.seq_group for s in scheduler_output.scheduled_seq_groups]
19 -
20 -
21 -def append_new_token(out, token_id: int):
22 - seq_groups = get_sequence_groups(out)
23 - for seq_group in seq_groups:
24 - for seq in seq_group.get_seqs():
25 - seq.append_token_id(token_id, {token_id: Logprob(token_id)})
26 -
27 -
28 -def schedule_and_update_computed_tokens(scheduler):
29 - metas, out = scheduler.schedule()
30 - for s, meta in zip(out.scheduled_seq_groups, metas):
31 - s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
32 - return metas, out
33 -
34 -
35 -def append_new_token_seq_group(token_chunk_size, seq_group, token_id: int):
36 - seq_group.update_num_computed_tokens(token_chunk_size)
37 - for seq in seq_group.get_seqs():
38 - seq.append_token_id(token_id, {token_id: Logprob(token_id)})
14 +from .utils import (append_new_token, append_new_token_seq_group,
15 + create_dummy_prompt, get_sequence_groups,
16 + schedule_and_update_computed_tokens)
17
18
19 def test_scheduler_add_seq_group():
tests/core/test_scheduler_encoder_decoder.py new
+99
@@ -0,0 +1,99 @@
1 +from typing import List
2 +
3 +import pytest # noqa
4 +
5 +from vllm.config import CacheConfig, SchedulerConfig
6 +from vllm.core.scheduler import Scheduler
7 +from vllm.sequence import SequenceGroup
8 +
9 +from .utils import (append_new_token, create_dummy_prompt_encoder_decoder,
10 + get_sequence_groups, schedule_and_update_computed_tokens)
11 +
12 +
13 +def test_scheduler_schedule_simple_encoder_decoder():
14 + '''
15 + Test basic scheduler functionality in the context
16 + of an encoder/decoder model. Focus on testing
17 + enc/dec-specific functionality sense tests already
18 + exist for decoder-only functionality
19 +
20 + Test behavior:
21 + * Construct Scheduler
22 + * Construct dummy encoder/decoder sequence groups
23 + * Add dummy seq groups to scheduler backlog
24 + * Schedule the next seq group & validate:
25 + * Cross-attn block tables
26 + * Updated states of seq groups
27 + * Number of batched tokens
28 + * Number of blocks to copy/swap-in/swap-out
29 + * Number of scheduled seq groups
30 + * Repeat for both prefill- and decode-phase
31 + * Abort scheduled seq groups
32 + * Assert that aborted seq groups no longer appear in
33 + cross-attention block table
34 + '''
35 +
36 + block_size = 4
37 + num_seq_group = 4
38 + max_model_len = 16
39 + scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len)
40 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
41 + cache_config.num_cpu_blocks = 16 # enc and dec prompts per seq_group
42 + cache_config.num_gpu_blocks = 16 # enc and dec prompts per seq_group
43 + scheduler = Scheduler(scheduler_config, cache_config, None)
44 + running: List[SequenceGroup] = []
45 +
46 + # Add seq groups to scheduler.
47 + req_id_list = []
48 + for i in range(num_seq_group):
49 + req_id = str(i)
50 + req_id_list.append(req_id)
51 + _, _, seq_group = create_dummy_prompt_encoder_decoder(
52 + req_id, block_size, block_size, block_size)
53 + scheduler.add_seq_group(seq_group)
54 + running.append(seq_group)
55 +
56 + # Schedule seq groups prefill.
57 + num_tokens = block_size * num_seq_group
58 + seq_group_meta_list, out = schedule_and_update_computed_tokens(scheduler)
59 + # - Verify that sequence group cross-attention block tables are
60 + # registered with the block manager
61 + assert all([(req_id in scheduler.block_manager.cross_block_tables)
62 + for req_id in req_id_list])
63 + # - Validate sequence-group status
64 + assert set(get_sequence_groups(out)) == set(running)
65 + # - Validate number of batched tokens
66 + assert out.num_batched_tokens == num_tokens
67 + # - Validate there are no remaining blocks to swap
68 + assert (not out.blocks_to_copy and not out.blocks_to_swap_in
69 + and not out.blocks_to_swap_out)
70 + # - Validate all seq groups were scheduled
71 + assert len(seq_group_meta_list) == num_seq_group
72 + append_new_token(out, 1)
73 +
74 + # Schedule seq groups decode.
75 + seq_group_meta_list, out = schedule_and_update_computed_tokens(scheduler)
76 + # - Verify that sequence group metadata includes encoder attention
77 + # and cross-attention metadata
78 + assert all([
79 + not ((seq_group_meta.encoder_seq_data is None) or
80 + (seq_group_meta.cross_block_table is None))
81 + for seq_group_meta in seq_group_meta_list
82 + ])
83 + # - Validate sequence-group status
84 + assert set(get_sequence_groups(out)) == set(running)
85 + # - Validate there is one batched token per seq group
86 + assert out.num_batched_tokens == num_seq_group
87 + # - Validate there are no remaining blocks to swap
88 + assert (not out.blocks_to_copy and not out.blocks_to_swap_in
89 + and not out.blocks_to_swap_out)
90 + # - Validate that all seq groups were scheduled
91 + assert len(seq_group_meta_list) == num_seq_group
92 + append_new_token(out, 1)
93 +
94 + # Abort sequences
95 + for req_id in req_id_list:
96 + scheduler.abort_seq_group(req_id)
97 + # - Verify that sequence group cross-attention block tables are
98 + # NO LONGER registered with the block manager
99 + assert req_id not in scheduler.block_manager.cross_block_tables
tests/core/utils.py
+64 -35
@@ -53,27 +53,30 @@ def create_dummy_prompt_encoder_decoder(
53 block_size = decoder_prompt_length
54
55 # Create dummy prompt sequence with tokens 0...block_size-1
56 - # and prompt "0 ... block_size".
56 + # and prompt "0 ... block_size". Note that the prompt string
57 + # doesn't actually match the tokens
58 decoder_prompt_tokens = list(range(decoder_prompt_length))
59 decoder_prompt_str = " ".join([str(t) for t in decoder_prompt_tokens])
60 + encoder_prompt_tokens = list(reversed(list(range(encoder_prompt_length))))
61 + encoder_prompt_str = " ".join([str(t) for t in encoder_prompt_tokens])
62 +
63 + inputs = {
64 + "prompt": decoder_prompt_str,
65 + "prompt_token_ids": decoder_prompt_tokens,
66 + "encoder_prompt": encoder_prompt_str,
67 + "encoder_prompt_token_ids": encoder_prompt_tokens,
68 + "multi_modal_data": None,
69 + }
70
71 decoder_prompt = Sequence(int(request_id),
61 - inputs={
62 - "prompt": decoder_prompt_str,
63 - "prompt_token_ids": decoder_prompt_tokens,
64 - "multi_modal_data": None,
65 - },
66 - block_size=block_size)
72 + inputs=inputs,
73 + block_size=block_size,
74 + from_decoder_prompt=True)
75
68 - encoder_prompt_tokens = list(reversed(list(range(encoder_prompt_length))))
69 - encoder_prompt_str = " ".join([str(t) for t in encoder_prompt_tokens])
76 encoder_prompt = Sequence(int(request_id),
71 - inputs={
72 - "prompt": encoder_prompt_str,
73 - "prompt_token_ids": encoder_prompt_tokens,
74 - "multi_modal_data": None,
75 - },
76 - block_size=block_size)
77 + inputs=inputs,
78 + block_size=block_size,
79 + from_decoder_prompt=False)
80 seq_group = SequenceGroup(request_id=request_id,
81 seqs=[decoder_prompt],
82 sampling_params=SamplingParams(
@@ -139,17 +142,21 @@ def create_seq_group_encoder_decoder(
142
143 prompt_token_ids = [0] * seq_prompt_len
144
145 + inputs = {
146 + "prompt": "",
147 + "prompt_token_ids": prompt_token_ids,
148 + "encoder_prompt": "",
149 + "encoder_prompt_token_ids": prompt_token_ids,
150 + "multi_modal_data": None,
151 + }
152 +
153 seqs = []
154 for seq_id_offset, output_len in enumerate(seq_output_lens):
144 - seq = Sequence(
145 - seq_id=seq_id_start + seq_id_offset,
146 - inputs={
147 - "prompt": "",
148 - "prompt_token_ids": prompt_token_ids,
149 - "multi_modal_data": None,
150 - },
151 - block_size=16,
152 - )
155 + # Construct decoder input sequences
156 + seq = Sequence(seq_id=seq_id_start + seq_id_offset,
157 + inputs=inputs,
158 + block_size=16,
159 + from_decoder_prompt=True)
160
161 for i in range(output_len):
162 seq.append_token_id(
@@ -158,16 +165,11 @@ def create_seq_group_encoder_decoder(
165 )
166 seqs.append(seq)
167
161 - # Encoder sequence
162 - encoder_seq = Sequence(
163 - seq_id=seq_id_start + len(seq_output_lens),
164 - inputs={
165 - "prompt": "",
166 - "prompt_token_ids": prompt_token_ids,
167 - "multi_modal_data": None,
168 - },
169 - block_size=16,
170 - )
168 + # Encoder input sequence
169 + encoder_seq = Sequence(seq_id=seq_id_start + len(seq_output_lens),
170 + inputs=inputs,
171 + block_size=16,
172 + from_decoder_prompt=False)
173
174 return SequenceGroup(request_id=request_id,
175 seqs=seqs,
@@ -177,4 +179,31 @@ def create_seq_group_encoder_decoder(
179
180
181 def round_up_to_next_block(seq_len: int, block_size: int) -> int:
180 - return (seq_len + block_size - 1) // block_size
\ No newline at end of file
182 + return (seq_len + block_size - 1) // block_size
183 +
184 +
185 +# Helper functions for scheduler tests
186 +
187 +
188 +def get_sequence_groups(scheduler_output):
189 + return [s.seq_group for s in scheduler_output.scheduled_seq_groups]
190 +
191 +
192 +def append_new_token(out, token_id: int):
193 + seq_groups = get_sequence_groups(out)
194 + for seq_group in seq_groups:
195 + for seq in seq_group.get_seqs():
196 + seq.append_token_id(token_id, {token_id: Logprob(token_id)})
197 +
198 +
199 +def schedule_and_update_computed_tokens(scheduler):
200 + metas, out = scheduler.schedule()
201 + for s, meta in zip(out.scheduled_seq_groups, metas):
202 + s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
203 + return metas, out
204 +
205 +
206 +def append_new_token_seq_group(token_chunk_size, seq_group, token_id: int):
207 + seq_group.update_num_computed_tokens(token_chunk_size)
208 + for seq in seq_group.get_seqs():
209 + seq.append_token_id(token_id, {token_id: Logprob(token_id)})
tests/distributed/test_basic_distributed_correctness_enc_dec.py new
+101
@@ -0,0 +1,101 @@
1 +"""For encoder/decoder models only:
2 +Compare the outputs of HF and distributed vLLM when using greedy sampling.
3 +
4 +Run:
5 +```sh
6 +cd $VLLM_PATH/tests
7 +
8 +pytest distributed/test_basic_distributed_correctness_enc_dec.py
9 +```
10 +"""
11 +
12 +import pytest
13 +
14 +from tests.models.utils import DecoderPromptType
15 +from vllm.utils import cuda_device_count_stateless
16 +
17 +from ..models.utils import check_logprobs_close
18 +from ..utils import fork_new_process_for_each_test
19 +
20 +
21 +@pytest.mark.skipif(cuda_device_count_stateless() < 2,
22 + reason="Need at least 2 GPUs to run the test.")
23 +@pytest.mark.parametrize("model, distributed_executor_backend", [
24 + ("facebook/bart-large-cnn", "ray"),
25 + ("facebook/bart-large-cnn", "mp"),
26 +])
27 +@fork_new_process_for_each_test
28 +def test_models(
29 + model: str,
30 + distributed_executor_backend: str,
31 + hf_runner,
32 + vllm_runner,
33 + example_encoder_decoder_prompts,
34 +) -> None:
35 + '''
36 + Test vLLM BART inference on more than one GPU, comparing
37 + outputs against HF as a baseline.
38 +
39 + Fork a new process for each test, to prevent CUDA from
40 + being re-initialized by successive tests within the same
41 + process.
42 +
43 + Arguments:
44 +
45 + * model: the HF ID of the specific BART variant under test
46 + * distributed_executor_backend
47 + * hf_runner: HuggingFace (HF) test model runner
48 + * vllm_runner: vLLM test model runner
49 + * example_encoder_decoder_prompts: test fixture which provides a
50 + dictionary of dummy prompts
51 + '''
52 +
53 + dtype = "float"
54 + max_tokens = 64
55 + num_logprobs = 5
56 +
57 + # Example inputs with non-trivial (i.e. not None/empty) encoder &
58 + # decoder prompts.
59 + test_prompts = example_encoder_decoder_prompts[DecoderPromptType.CUSTOM]
60 +
61 + # NOTE: take care of the order. run vLLM first, and then run HF.
62 + # vLLM needs a fresh new process without cuda initialization.
63 + # if we run HF first, the cuda initialization will be done and it
64 + # will hurt multiprocessing backend with fork method (the default method).
65 + with vllm_runner(
66 + model,
67 + dtype=dtype,
68 + tensor_parallel_size=2,
69 + distributed_executor_backend=distributed_executor_backend,
70 + enforce_eager=True,
71 + ) as vllm_model:
72 + vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
73 + test_prompts, max_tokens, num_logprobs)
74 +
75 + # Configuration settings for HF baseline
76 + hf_kwargs = {
77 + "top_k": None,
78 + "num_beams": 1,
79 + "repetition_penalty": 1.0,
80 + "top_p": 1.0,
81 + "length_penalty": 1.0,
82 + "early_stopping": False,
83 + "no_repeat_ngram_size": None,
84 + "min_length": 0
85 + }
86 +
87 + with hf_runner(model, dtype=dtype,
88 + is_encoder_decoder_model=True) as hf_model:
89 + hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
90 + test_prompts,
91 + max_tokens,
92 + num_logprobs,
93 + **hf_kwargs,
94 + ))
95 +
96 + check_logprobs_close(
97 + outputs_0_lst=hf_outputs,
98 + outputs_1_lst=vllm_outputs,
99 + name_0="hf",
100 + name_1="vllm",
101 + )
tests/kernels/test_attention_selector.py
+2 -2
@@ -3,9 +3,9 @@ from unittest.mock import patch
3 import pytest
4 import torch
5
6 -from tests.kernels.utils import (STR_FLASH_ATTN_VAL, STR_INVALID_VAL,
7 - override_backend_env_variable)
6 +from tests.kernels.utils import override_backend_env_variable
7 from vllm.attention.selector import which_attn_to_use
8 +from vllm.utils import STR_FLASH_ATTN_VAL, STR_INVALID_VAL
9
10
11 @pytest.mark.parametrize(
tests/kernels/test_encoder_decoder_attn.py
+207 -159
@@ -4,8 +4,6 @@ Tests:
4 * E2E test of Encoder attention + Decoder self-attention +
5 Encoder/decoder cross-attention (collectively
6 "encoder/decoder attention")
7 -* Confirm enc/dec models will fail for chunked prefill
8 -* Confirm enc/dec models will fail for prefix caching
7
8 """
9
@@ -15,19 +13,22 @@ import pytest
13 import torch
14
15 from tests.kernels.utils import *
18 -from tests.kernels.utils import make_causal_mask, maybe_make_long_tensor
19 -from vllm.attention import Attention, AttentionMetadata
20 -from vllm.attention.backends.abstract import AttentionBackend, AttentionType
16 +from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
17 + AttentionType)
18 from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
19 +from vllm.attention.selector import (_Backend,
20 + global_force_attn_backend_context_manager)
21 from vllm.utils import is_hip
22
23 +# List of support backends for encoder/decoder models
24 +LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS]
25 +
26 HEAD_SIZES = [64, 256]
27
28 NUM_HEADS = [1, 16]
29
30 BATCH_SIZES = [1, 16]
31 BLOCK_SIZES = [16]
30 -BACKEND_NAMES = [STR_XFORMERS_ATTN_VAL]
32 CUDA_DEVICE = "cuda:0"
33
34 MAX_DEC_SEQ_LENS = [128]
@@ -724,57 +725,92 @@ def _run_encoder_decoder_cross_attention_test(
725 @pytest.mark.skipif(is_hip(), reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
726 @pytest.mark.parametrize("num_heads", NUM_HEADS)
727 @pytest.mark.parametrize("head_size", HEAD_SIZES)
727 -@pytest.mark.parametrize("backend_name", BACKEND_NAMES)
728 +@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
729 @pytest.mark.parametrize("batch_size", BATCH_SIZES)
730 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
731 @pytest.mark.parametrize("max_dec_seq_len", MAX_DEC_SEQ_LENS)
732 @pytest.mark.parametrize("max_enc_seq_len", MAX_ENC_SEQ_LENS)
732 -def test_encoder_only(num_heads: int, head_size: int, backend_name: str,
733 - batch_size: int, block_size: int, max_dec_seq_len: int,
734 - max_enc_seq_len: int, monkeypatch):
733 +def test_encoder_only(
734 + num_heads: int,
735 + head_size: int,
736 + attn_backend: _Backend,
737 + batch_size: int,
738 + block_size: int,
739 + max_dec_seq_len: int,
740 + max_enc_seq_len: int,
741 +):
742 + '''
743 + End-to-end encoder-only attention test:
744 +
745 + * Construct fake test vectors for (1) encoder attention
746 + * Construct (1) attention metadata structure with prefill-phase
747 + encoder attention, and (2) an analogous attention metadata
748 + structure but for decode-phase
749 + * Test & validate encoder attention against ideal output
750 +
751 + No KV cache is required for encoder-only attention.
752 +
753 + Note on ROCm/HIP: currently encoder/decoder models are not supported on
754 + AMD GPUs, therefore this test simply is skipped if is_hip().
755 +
756 + This test globally forces an override of the usual backend
757 + auto-selection process, forcing the specific backend-under-test
758 + to be utilized.
759 +
760 + Arguments:
761 +
762 + * num_heads
763 + * head_size,
764 + * attn_backend: The attention backend to employ for testing
765 + * batch_size
766 + * block_size: KV cache block size
767 + * max_dec_seq_len: max length of decoder input sequences
768 + * max_enc_seq_len: max length of encoder input sequences
769 + '''
770
771 # Force Attention wrapper backend
737 - override_backend_env_variable(monkeypatch, backend_name)
772 + with global_force_attn_backend_context_manager(attn_backend):
773
739 - # Note: KV cache size of 4096 is arbitrary & chosen intentionally
740 - # to be more than necessary, since exceeding the kv cache size
741 - # is not part of this test
742 - test_pt = TestPoint(num_heads, head_size, backend_name, batch_size,
743 - block_size, max_dec_seq_len, max_enc_seq_len, 4096)
774 + # Note: KV cache size of 4096 is arbitrary & chosen intentionally
775 + # to be more than necessary, since exceeding the kv cache size
776 + # is not part of this test
777 + test_pt = TestPoint(num_heads, head_size, attn_backend.name,
778 + batch_size, block_size, max_dec_seq_len,
779 + max_enc_seq_len, 4096)
780
745 - # Attention scale factor, attention backend instance, attention wrapper
746 - # instance, KV cache init
747 - test_rsrcs = _make_test_resources(test_pt)
781 + # Attention scale factor, attention backend instance, attention wrapper
782 + # instance, KV cache init
783 + test_rsrcs = _make_test_resources(test_pt)
784
749 - # Construct encoder attention test params (only used
750 - # during prefill)
785 + # Construct encoder attention test params (only used
786 + # during prefill)
787
752 - enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
788 + enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
789
754 - # Shared prefill metadata structure
790 + # Shared prefill metadata structure
791
756 - prephase_attn_metadata: AttentionMetadata = make_test_metadata(
757 - test_rsrcs.attn_backend,
758 - True,
759 - None,
760 - decoder_test_params=None,
761 - encoder_test_params=enc_test_params,
762 - cross_test_params=None,
763 - device=CUDA_DEVICE)
792 + prephase_attn_metadata: AttentionMetadata = make_test_metadata(
793 + test_rsrcs.attn_backend,
794 + True,
795 + None,
796 + decoder_test_params=None,
797 + encoder_test_params=enc_test_params,
798 + cross_test_params=None,
799 + device=CUDA_DEVICE)
800
765 - # PREFILL: encoder attention
801 + # PREFILL: encoder attention
802
767 - enc_pckd_act_out: torch.Tensor = (_run_encoder_attention_test(
768 - test_rsrcs.attn, enc_test_params, prephase_attn_metadata))
803 + enc_pckd_act_out: torch.Tensor = (_run_encoder_attention_test(
804 + test_rsrcs.attn, enc_test_params, prephase_attn_metadata))
805
770 - # - Is encoder attention result correct?
771 - assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
806 + # - Is encoder attention result correct?
807 + assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
808
809
810 @pytest.mark.skipif(is_hip(), reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
811 @pytest.mark.parametrize("num_heads", NUM_HEADS)
812 @pytest.mark.parametrize("head_size", HEAD_SIZES)
777 -@pytest.mark.parametrize("backend_name", BACKEND_NAMES)
813 +@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
814 @pytest.mark.parametrize("batch_size", BATCH_SIZES)
815 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
816 @pytest.mark.parametrize("max_dec_seq_len", MAX_DEC_SEQ_LENS)
@@ -782,12 +818,11 @@ def test_encoder_only(num_heads: int, head_size: int, backend_name: str,
818 def test_e2e_enc_dec_attn(
819 num_heads: int,
820 head_size: int,
785 - backend_name: str,
821 + attn_backend: _Backend,
822 batch_size: int,
823 block_size: int,
824 max_dec_seq_len: int,
825 max_enc_seq_len: int,
790 - monkeypatch,
826 ) -> None:
827 '''
828 End-to-end encoder/decoder test:
@@ -820,8 +855,9 @@ def test_e2e_enc_dec_attn(
855 cross-attention K/Vs are allowed to differ in seq len, as is often the case
856 for cross-attention.
857
823 - This test utilizes PyTest monkey patching to force the attention backend
824 - via an environment variable.
858 + This test globally forces an override of the usual backend
859 + auto-selection process, forcing the specific backend-under-test
860 + to be utilized.
861
862 Note on ROCm/HIP: currently encoder/decoder models are not supported on
863 AMD GPUs, therefore this test simply is skipped if is_hip().
@@ -830,124 +866,136 @@ def test_e2e_enc_dec_attn(
866 all prefill-phase attention operations (encoder, decoder, enc/dec cross),
867 and a single one shared by all decode-phase attention operations
868 (decoder & enc/dec cross.) This is intended to reflect the behavior
833 - of ModelRunner, which constructs a single attention metadata structure for
834 - each prefill or decode run. A realistic scenario would rely on the
835 - attention backend to utilize the appropriate attention metadata fields
836 - according to the value of attn_metadata.attention_type. Thus, this test is
837 - organized so as to confirm that the backend-under-test can handle a
838 - shared prefill attention metadata structure & a shared decode attention
839 - metadata structure.
840 - '''
841 -
842 - # Force Attention wrapper backend
843 - override_backend_env_variable(monkeypatch, backend_name)
844 -
845 - # Note: KV cache size of 4096 is arbitrary & chosen intentionally
846 - # to be more than necessary, since exceeding the kv cache size
847 - # is not part of this test
848 - test_pt = TestPoint(num_heads, head_size, backend_name, batch_size,
849 - block_size, max_dec_seq_len, max_enc_seq_len, 4096)
850 -
851 - # Attention scale factor, attention backend instance, attention wrapper
852 - # instance, KV cache init
853 - test_rsrcs = _make_test_resources(test_pt)
869 + of EncoderDecoderModelRunner, which constructs a single attention metadata
870 + structure for each prefill or decode run. A realistic scenario would rely
871 + on the attention backend to utilize the appropriate attention metadata
872 + fields according to the value of attn_metadata.attention_type. Thus,
873 + this test is organized so as to confirm that the backend-under-test can
874 + handle a shared prefill attention metadata structure & a shared decode\
875 + attention metadata structure.
876
855 - # Construct encoder attention test params (only used
856 - # during prefill)
857 -
858 - enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
859 -
860 - # Construct Decoder self-attention prefill-phase & decode-phase
861 - # test params, including query/key/value tensors, decoder self-attention
862 - # memory-mapping. cross_block_base_addr is the uppermost address in the
863 - # decoder self-attention block-table, i.e. a base address which the
864 - # encoder/decoder cross-attention block-table may build downward toward.
865 -
866 - (
867 - dec_qkv,
868 - prephase_dec_test_params,
869 - decphase_dec_test_params,
870 - cross_block_base_addr,
871 - ) = _decoder_attn_setup(test_pt, test_rsrcs)
877 + Arguments:
878
873 - # Construct encoder/decoder cross-attention prefill-phase & decode-phase
874 - # test params, including key/value tensors, cross-attention memory-mapping
879 + * num_heads
880 + * head_size,
881 + * attn_backend: The attention backend to employ for testing
882 + * batch_size
883 + * block_size: KV cache block size
884 + * max_dec_seq_len: max length of decoder input sequences
885 + * max_enc_seq_len: max length of encoder input sequences
886 + '''
887
876 - (
877 - prephase_cross_test_params,
878 - decphase_cross_test_params,
879 - ) = _enc_dec_cross_attn_setup_reuses_query(
880 - dec_qkv,
881 - enc_test_params,
882 - prephase_dec_test_params,
883 - test_pt,
884 - test_rsrcs,
885 - block_base_addr=cross_block_base_addr)
886 -
887 - # Shared prefill metadata structure
888 - assert prephase_dec_test_params.packed_qkvo.packed_qkv is not None
889 - prephase_attn_metadata: AttentionMetadata = make_test_metadata(
890 - test_rsrcs.attn_backend,
891 - True,
892 - prephase_dec_test_params.packed_qkvo.packed_qkv.q_seq_lens,
893 - decoder_test_params=prephase_dec_test_params,
894 - encoder_test_params=enc_test_params,
895 - cross_test_params=prephase_cross_test_params,
896 - device=CUDA_DEVICE)
897 -
898 - # PREFILL: encoder attention
899 -
900 - enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
901 - enc_test_params,
902 - prephase_attn_metadata)
903 -
904 - # - Is encoder attention result correct?
905 - assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
906 -
907 - # PREFILL: decoder self-attention test
908 -
909 - prephase_dec_pckd_act_out = _run_decoder_self_attention_test(
910 - test_rsrcs, prephase_dec_test_params, prephase_attn_metadata)
911 -
912 - # - Is prefill decoder self-attention correct?
913 - assert_actual_matches_ideal(prephase_dec_test_params,
914 - prephase_dec_pckd_act_out)
915 -
916 - # PREFILL: encoder/decoder cross-attention test
917 -
918 - prephase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
919 - test_rsrcs, prephase_dec_test_params, prephase_cross_test_params,
920 - prephase_attn_metadata)
921 -
922 - # - Is prefill encoder/decoder cross-attention correct?
923 - assert_actual_matches_ideal(prephase_cross_test_params,
924 - prephase_cross_pckd_act_out)
925 -
926 - # DECODE: build decode-phase attention metadata
927 -
928 - decphase_attn_metadata: AttentionMetadata = make_test_metadata(
929 - test_rsrcs.attn_backend,
930 - False,
931 - dec_qkv.q_seq_lens,
932 - decoder_test_params=decphase_dec_test_params,
933 - encoder_test_params=enc_test_params,
934 - cross_test_params=decphase_cross_test_params,
935 - device=CUDA_DEVICE)
936 -
937 - # DECODE: decoder self-attention test
938 -
939 - decphase_dec_pckd_act_out = _run_decoder_self_attention_test(
940 - test_rsrcs, decphase_dec_test_params, decphase_attn_metadata)
941 -
942 - # - Is decode-phase decoder self-attention correct?
943 - assert_actual_matches_ideal(decphase_dec_test_params,
944 - decphase_dec_pckd_act_out)
945 -
946 - # DECODE: encoder/decoder cross-attention test
947 -
948 - decphase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
949 - test_rsrcs, decphase_dec_test_params, None, decphase_attn_metadata)
950 -
951 - # - Is decode-phase encoder/decoder cross-attention correct?
952 - assert_actual_matches_ideal(decphase_cross_test_params,
953 - decphase_cross_pckd_act_out)
888 + # Force Attention wrapper backend
889 + with global_force_attn_backend_context_manager(attn_backend):
890 +
891 + # Note: KV cache size of 4096 is arbitrary & chosen intentionally
892 + # to be more than necessary, since exceeding the kv cache size
893 + # is not part of this test
894 + test_pt = TestPoint(num_heads, head_size, attn_backend.name,
895 + batch_size, block_size, max_dec_seq_len,
896 + max_enc_seq_len, 4096)
897 +
898 + # Attention scale factor, attention backend instance, attention wrapper
899 + # instance, KV cache init
900 + test_rsrcs = _make_test_resources(test_pt)
901 +
902 + # Construct encoder attention test params (only used
903 + # during prefill)
904 +
905 + enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
906 +
907 + # Construct Decoder self-attention prefill-phase & decode-phase
908 + # test params, including query/key/value tensors, decoder self-attention
909 + # memory-mapping. cross_block_base_addr is the uppermost address in the
910 + # decoder self-attention block-table, i.e. a base address which the
911 + # encoder/decoder cross-attention block-table may build downward toward.
912 +
913 + (
914 + dec_qkv,
915 + prephase_dec_test_params,
916 + decphase_dec_test_params,
917 + cross_block_base_addr,
918 + ) = _decoder_attn_setup(test_pt, test_rsrcs)
919 +
920 + # Construct encoder/decoder cross-attention prefill-phase
921 + # & decode-phase test params, including key/value tensors,
922 + # cross-attention memory-mapping
923 +
924 + (
925 + prephase_cross_test_params,
926 + decphase_cross_test_params,
927 + ) = _enc_dec_cross_attn_setup_reuses_query(
928 + dec_qkv,
929 + enc_test_params,
930 + prephase_dec_test_params,
931 + test_pt,
932 + test_rsrcs,
933 + block_base_addr=cross_block_base_addr)
934 +
935 + # Shared prefill metadata structure
936 + assert prephase_dec_test_params.packed_qkvo.packed_qkv is not None
937 + prephase_attn_metadata: AttentionMetadata = make_test_metadata(
938 + test_rsrcs.attn_backend,
939 + True,
940 + prephase_dec_test_params.packed_qkvo.packed_qkv.q_seq_lens,
941 + decoder_test_params=prephase_dec_test_params,
942 + encoder_test_params=enc_test_params,
943 + cross_test_params=prephase_cross_test_params,
944 + device=CUDA_DEVICE)
945 +
946 + # PREFILL: encoder attention
947 +
948 + enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
949 + enc_test_params,
950 + prephase_attn_metadata)
951 +
952 + # - Is encoder attention result correct?
953 + assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
954 +
955 + # PREFILL: decoder self-attention test
956 +
957 + prephase_dec_pckd_act_out = _run_decoder_self_attention_test(
958 + test_rsrcs, prephase_dec_test_params, prephase_attn_metadata)
959 +
960 + # - Is prefill decoder self-attention correct?
961 + assert_actual_matches_ideal(prephase_dec_test_params,
962 + prephase_dec_pckd_act_out)
963 +
964 + # PREFILL: encoder/decoder cross-attention test
965 +
966 + prephase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
967 + test_rsrcs, prephase_dec_test_params, prephase_cross_test_params,
968 + prephase_attn_metadata)
969 +
970 + # - Is prefill encoder/decoder cross-attention correct?
971 + assert_actual_matches_ideal(prephase_cross_test_params,
972 + prephase_cross_pckd_act_out)
973 +
974 + # DECODE: build decode-phase attention metadata
975 +
976 + decphase_attn_metadata: AttentionMetadata = make_test_metadata(
977 + test_rsrcs.attn_backend,
978 + False,
979 + dec_qkv.q_seq_lens,
980 + decoder_test_params=decphase_dec_test_params,
981 + encoder_test_params=enc_test_params,
982 + cross_test_params=decphase_cross_test_params,
983 + device=CUDA_DEVICE)
984 +
985 + # DECODE: decoder self-attention test
986 +
987 + decphase_dec_pckd_act_out = _run_decoder_self_attention_test(
988 + test_rsrcs, decphase_dec_test_params, decphase_attn_metadata)
989 +
990 + # - Is decode-phase decoder self-attention correct?
991 + assert_actual_matches_ideal(decphase_dec_test_params,
992 + decphase_dec_pckd_act_out)
993 +
994 + # DECODE: encoder/decoder cross-attention test
995 +
996 + decphase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
997 + test_rsrcs, decphase_dec_test_params, None, decphase_attn_metadata)
998 +
999 + # - Is decode-phase encoder/decoder cross-attention correct?
1000 + assert_actual_matches_ideal(decphase_cross_test_params,
1001 + decphase_cross_pckd_act_out)
tests/kernels/test_flash_attn.py
+1 -1
@@ -211,5 +211,5 @@ def test_varlen_with_paged_kv(
211 sliding_window=sliding_window,
212 soft_cap=soft_cap,
213 )
214 - assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
214 + assert torch.allclose(output, ref_output, atol=2e-2, rtol=1e-2), \
215 f"{torch.max(torch.abs(output - ref_output))}"
tests/kernels/utils.py
+3 -17
@@ -8,24 +8,10 @@ from typing import Any, List, NamedTuple, Optional, Tuple, Union
8 import pytest
9 import torch
10
11 -from vllm.attention.backends.abstract import (AttentionBackend,
12 - AttentionMetadata, AttentionType)
11 +from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType
12 from vllm.attention.backends.xformers import XFormersBackend
14 -from vllm.utils import make_tensor_with_pad
15 -
16 -# String name of register which may be set in order to
17 -# force auto-selection of attention backend by Attention
18 -# wrapper
19 -STR_BACKEND_ENV_VAR: str = "VLLM_ATTENTION_BACKEND"
20 -
21 -# Possible string values of STR_BACKEND_ENV_VAR
22 -# register, corresponding to possible backends
23 -STR_FLASHINFER_ATTN_VAL: str = "FLASHINFER"
24 -STR_TORCH_SDPA_ATTN_VAL: str = "TORCH_SDPA"
25 -STR_ROCM_FLASH_ATTN_VAL: str = "ROCM_FLASH"
26 -STR_XFORMERS_ATTN_VAL: str = "XFORMERS"
27 -STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
28 -STR_INVALID_VAL: str = "INVALID"
13 +from vllm.utils import (STR_BACKEND_ENV_VAR, STR_XFORMERS_ATTN_VAL,
14 + make_tensor_with_pad)
15
16
17 class QKVInputs(NamedTuple):
tests/models/test_bart.py new
+153
@@ -0,0 +1,153 @@
1 +"""Compare the outputs of HF and vLLM for BART models using greedy sampling.
2 +
3 +Run `pytest tests/models/test_bart.py`.
4 +"""
5 +from vllm.utils import is_cpu
6 +
7 +if not is_cpu():
8 + # CPU backend is not currently supported with encoder/decoder models
9 + # skip test definitions entirely to avoid importing GPU kernel libs
10 + # (xFormers, etc.)
11 +
12 + import pytest
13 +
14 + from tests.models.utils import DecoderPromptType
15 +
16 + from .utils import check_logprobs_close
17 +
18 + MODELS = ["facebook/bart-base", "facebook/bart-large-cnn"]
19 +
20 + DECODER_PROMPT_TYPES = ([
21 + DecoderPromptType.CUSTOM, DecoderPromptType.EMPTY_STR,
22 + DecoderPromptType.NONE
23 + ])
24 +
25 + @pytest.mark.parametrize("model", MODELS)
26 + @pytest.mark.parametrize("dtype", ["float", "bfloat16"])
27 + @pytest.mark.parametrize("max_tokens", [64])
28 + @pytest.mark.parametrize("num_logprobs", [5])
29 + @pytest.mark.parametrize("decoder_prompt_type", DECODER_PROMPT_TYPES)
30 + def test_models(
31 + hf_runner,
32 + vllm_runner,
33 + example_encoder_decoder_prompts,
34 + model: str,
35 + dtype: str,
36 + max_tokens: int,
37 + num_logprobs: int,
38 + decoder_prompt_type: DecoderPromptType,
39 + ) -> None:
40 + '''
41 + Test the vLLM BART model for a variety of encoder/decoder input prompts,
42 + by validating it against HuggingFace (HF) BART.
43 +
44 + Arguments:
45 +
46 + * hf_runner: HuggingFace (HF) test model runner
47 + * vllm_runner: vLLM test model runner
48 + * example_encoder_decoder_prompts: test fixture which provides a
49 + dictionary of dummy prompts
50 + * model: the HF ID of the specific BART variant under test
51 + * dtype: the tensor datatype to employ
52 + * max_tokens
53 + * num_logprobs
54 + * decoder_prompt_type: key into the example_encoder_decoder_prompts
55 + dictionary; selects specific encoder/decoder
56 + prompt scenarios to test
57 +
58 + A note on using HF BART as a baseline for validating vLLM BART,
59 + specifically when the decoder prompt is None.
60 +
61 + The HF GenerationMixin's default behavior is to force the first
62 + decoded token to be <BOS> if the prompt does not already contain
63 + <BOS> (this is accomplished using a logit
64 + processor setting.)
65 +
66 + So when we use HF BART as our baseline for comparison, note that
67 + when the user provides a request with a None decoder prompt
68 + (i.e. a singleton encoder prompt, or else an explicit encoder/
69 + decoder prompt with the decoder sub-prompt set to None), HF and
70 + vLLM handle this in different ways:
71 +
72 + * HF will (1) tokenize the None prompt as an empty token-list,
73 + (2) append <decoder-start-token> to the beginning, yielding
74 + [<decoder-start-token>], (3) pass this token list to the model, and
75 + then (4) after computing logits during prefill, override the model
76 + logits & force <BOS> to be the first generated token.
77 +
78 + * vLLM will (1) tokenize the None prompt as [<BOS>], (2) append decoder-
79 + start-token to the beginning, yielding [<decoder-start-token><BOS>],
80 + (3) pass these tokens to the model & proceed with generation.
81 +
82 + The net effect is that compared to vLLM, the list of HF *decoded* tokens
83 + will contain one more initial <BOS> than the vLLM generated tokens,
84 + because vLLM's <BOS> token is injected into the prompt rather than into
85 + the generated output. This is in spite of the fact that overall, the
86 + complete sequences (prompt + decoded tokens) produced by vLLM will match
87 + HF.
88 +
89 + So when we use HF decoded token output to validate vLLM's decoded token
90 + output, the testing process must account for the difference in decoded
91 + token sequences between vLLM and HF specifically in the
92 + decoder-prompt-is-None case.
93 +
94 + One option is to disable the logit processor feature that forces the
95 + <BOS> token to be decoded (forced_bos_token_id = None), eliminating
96 + the problem entirely. However this is not "normal" BART usage.
97 +
98 + The other option is - only in the decoder-prompt-is-None case - to
99 + discard the first decoded token from the HF output before comparing it
100 + to vLLM.
101 +
102 + To that end, when testing the scenario where the decoder prompt is None
103 + (and only in that one scenario), this test skips the first HF decoded
104 + token during the process of validating the vLLM decoded output.
105 + '''
106 +
107 + test_case_prompts = example_encoder_decoder_prompts[
108 + decoder_prompt_type]
109 +
110 + # Configuration settings for HF baseline
111 + hf_kwargs = {
112 + "top_k": None,
113 + "num_beams": 1,
114 + "repetition_penalty": 1.0,
115 + "top_p": 1.0,
116 + "length_penalty": 1.0,
117 + "early_stopping": False,
118 + "no_repeat_ngram_size": None,
119 + "min_length": 0
120 + }
121 +
122 + with hf_runner(model, dtype=dtype,
123 + is_encoder_decoder_model=True) as hf_model:
124 + hf_outputs = (
125 + hf_model.generate_encoder_decoder_greedy_logprobs_limit(
126 + test_case_prompts,
127 + max_tokens,
128 + num_logprobs,
129 + **hf_kwargs,
130 + ))
131 +
132 + # Note: currently encoder/decoder models are only compatible with
133 + # enforce_eager=True. Normally this is not a problem because
134 + # for encoder/decoder models vLLM will
135 + # default to enforce_eager=True if enforce_eager
136 + # is left unspecified. However, the
137 + # VllmRunner test fixture (which wraps around the LLM class) defaults to
138 + # enforce_eager=False (a behavior which a number of already-exisitng
139 + # decoder-only unit tests expect), so when testing an encoder/decoder
140 + # model we must explicitly specify enforce_eager=True in the VllmRunner
141 + # constructor.
142 + with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
143 + vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
144 + test_case_prompts, max_tokens, num_logprobs)
145 +
146 + hf_skip_tokens = (1 if decoder_prompt_type == DecoderPromptType.NONE
147 + else 0)
148 +
149 + check_logprobs_close(outputs_0_lst=hf_outputs,
150 + outputs_1_lst=vllm_outputs,
151 + name_0="hf",
152 + name_1="vllm",
153 + num_outputs_0_skip_tokens=hf_skip_tokens)
tests/models/utils.py
+36
@@ -1,4 +1,5 @@
1 import warnings
2 +from enum import Enum
3 from typing import Dict, List, Optional, Sequence, Tuple, Union
4
5 from vllm.sequence import SampleLogprobs
@@ -45,11 +46,27 @@ def check_logprobs_close(
46 outputs_1_lst: Sequence[TokensTextLogprobs],
47 name_0: str,
48 name_1: str,
49 + num_outputs_0_skip_tokens: int = 0,
50 warn_on_mismatch: bool = True,
51 ):
52 """
53 Compare the logprobs of two sequences generated by different models,
54 which should be similar but not necessarily equal.
55 +
56 + Arguments:
57 +
58 + * outputs_0_lst: First sequence to compare
59 + * outputs_0_lst: Second sequence to compare
60 + * name_0: sequence #0 name
61 + * name_1: sequence #1 name
62 + * num_outputs_0_skip_tokens: If > 0, specifies the number of initial
63 + sequence #0 tokens & logprobs to discard
64 + before comparison, i.e. all
65 + of sequence #1 will be compared to
66 + sequence #0 beginning at index
67 + num_outputs_0_skip_tokens
68 + * warn_on_mismatch: Issue a warning if there is token-wise or text-wise
69 + mismatch between the two sequences
70 """
71 assert len(outputs_0_lst) == len(outputs_1_lst)
72
@@ -65,6 +82,15 @@ def check_logprobs_close(
82 if logprobs_1 is None:
83 logprobs_1 = [None] * len(output_ids_1)
84
85 + # Skip specified number of initial sequence #0 tokens
86 + # & logprobs, leaving output text as-is for simplicity
87 + # (text mismatches may generate warnings but do not
88 + # cause the test to fail.)
89 + if num_outputs_0_skip_tokens < 0:
90 + raise ValueError("num_outputs_0_skip_tokens must be non-negative")
91 + output_ids_0 = output_ids_0[num_outputs_0_skip_tokens:]
92 + logprobs_0 = logprobs_0[num_outputs_0_skip_tokens:]
93 +
94 # Loop through generated tokens.
95 for idx, (output_id_0,
96 output_id_1) in enumerate(zip(output_ids_0, output_ids_1)):
@@ -110,3 +136,13 @@ def check_logprobs_close(
136 warnings.simplefilter("always")
137
138 warnings.warn(fail_msg, stacklevel=2)
139 +
140 +
141 +class DecoderPromptType(Enum):
142 + '''
143 + For encoder/decoder models only -
144 +
145 + '''
146 + CUSTOM = 1
147 + NONE = 2
148 + EMPTY_STR = 3
tests/worker/test_encoder_decoder_model_runner.py new
+480
@@ -0,0 +1,480 @@
1 +from typing import List
2 +
3 +import pytest
4 +import torch
5 +
6 +from vllm.engine.arg_utils import EngineArgs
7 +from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
8 +from vllm.utils import is_cpu
9 +from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
10 +
11 +# CUDA graph scenarios to test
12 +#
13 +# Currently CUDA graph is not supported
14 +ENFORCE_EAGER = [True]
15 +
16 +BATCH_SIZES = [1, 4, 16, 64, 256]
17 +
18 +
19 +def _create_model_runner(model: str, *args,
20 + **kwargs) -> EncoderDecoderModelRunner:
21 + engine_args = EngineArgs(model, *args, **kwargs)
22 + engine_config = engine_args.create_engine_config()
23 + model_runner = EncoderDecoderModelRunner(
24 + model_config=engine_config.model_config,
25 + parallel_config=engine_config.parallel_config,
26 + scheduler_config=engine_config.scheduler_config,
27 + device_config=engine_config.device_config,
28 + cache_config=engine_config.cache_config,
29 + load_config=engine_config.load_config,
30 + lora_config=engine_config.lora_config,
31 + prompt_adapter_config=engine_config.prompt_adapter_config,
32 + is_driver_worker=True,
33 + )
34 + return model_runner
35 +
36 +
37 +@pytest.mark.skipif(condition=is_cpu(),
38 + reason="CPU backend is currently "
39 + "unsupported for encoder/ "
40 + "decoder models")
41 +@pytest.mark.parametrize("enforce_eager", ENFORCE_EAGER)
42 +def test_empty_seq_group(enforce_eager, ):
43 + """Verify prepare prompt and decode returns empty output
44 + for empty seq group list"""
45 +
46 + model_runner = _create_model_runner(
47 + "facebook/bart-base",
48 + seed=0,
49 + dtype="float16",
50 + max_num_batched_tokens=100000,
51 + max_num_seqs=100000,
52 + enable_chunked_prefill=False,
53 + enforce_eager=enforce_eager,
54 + )
55 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
56 + model_input = model_runner._prepare_model_input_tensors(
57 + seq_group_metadata_list)
58 + (
59 + input_tokens,
60 + input_positions,
61 + encoder_input_tokens,
62 + encoder_input_positions,
63 + attn_metadata,
64 + return_seq_lens,
65 + ) = (
66 + model_input.input_tokens,
67 + model_input.input_positions,
68 + model_input.encoder_input_tokens,
69 + model_input.encoder_input_positions,
70 + model_input.attn_metadata,
71 + model_input.seq_lens,
72 + )
73 + assert input_tokens is None
74 + assert input_positions is None
75 + assert encoder_input_tokens is None
76 + assert encoder_input_positions is None
77 + assert attn_metadata is None
78 + assert return_seq_lens is None
79 +
80 +
81 +@pytest.mark.skipif(condition=is_cpu(),
82 + reason="CPU backend is currently "
83 + "unsupported for encoder/ "
84 + "decoder models")
85 +@pytest.mark.parametrize("batch_size", BATCH_SIZES)
86 +@pytest.mark.parametrize("enforce_eager", ENFORCE_EAGER)
87 +def test_prepare_prompt(
88 + batch_size,
89 + enforce_eager,
90 +):
91 + '''
92 + Test the ability of the encoder/decoder model runner subclass to
93 + produce prefill-phase model inputs & attention metadata.
94 +
95 + Test behavior:
96 +
97 + * Instantiate BART base model & enc/dec model runner
98 + * Construct sequence-group metadata for dummy prompts
99 + * Test that encoder attention, decoder self-attention,
100 + and encoder/decoder cross-attention inputs are correct
101 +
102 + Arguments:
103 +
104 + * batch_size
105 + * backend_name: The attention backend under test
106 + * enforce_eager: Enforce eager mode if True (i.e. no CUDAGraph)
107 + '''
108 +
109 + model_runner = _create_model_runner(
110 + "facebook/bart-base",
111 + seed=0,
112 + dtype="float16",
113 + max_num_batched_tokens=100000,
114 + max_num_seqs=100000,
115 + enable_chunked_prefill=False,
116 + enforce_eager=enforce_eager,
117 + )
118 +
119 + seq_lens: List[int] = []
120 + encoder_seq_lens: List[int] = []
121 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
122 + block_tables = {0: [1]}
123 + cross_block_table = [2]
124 + for i in range(batch_size):
125 + # make sure all tokens fit into one block
126 + seq_len = i % (model_runner.block_size - 1) + 1
127 + seq_lens.append(seq_len)
128 + seq_data = SequenceData(list(range(seq_len)))
129 + encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
130 + encoder_seq_lens.append(encoder_seq_len)
131 + encoder_seq_data = SequenceData(list(range(encoder_seq_len)))
132 + seq_group_metadata = SequenceGroupMetadata(
133 + request_id=f"test_{i}",
134 + is_prompt=True,
135 + seq_data={0: seq_data},
136 + sampling_params=SamplingParams(temperature=0),
137 + block_tables=block_tables,
138 + encoder_seq_data=encoder_seq_data,
139 + cross_block_table=cross_block_table,
140 + )
141 + assert seq_group_metadata.token_chunk_size == seq_data.get_len()
142 + seq_group_metadata_list.append(seq_group_metadata)
143 +
144 + # Build
145 + # * Decoder model inputs
146 + # * Decoder self-attention KV caching data structures
147 + # * Encoder model inputs
148 + # * Encoder/decoder cross-attention KV caching data structures
149 + model_input = model_runner.prepare_model_input(seq_group_metadata_list)
150 +
151 + input_tokens = model_input.input_tokens
152 + input_positions = model_input.input_positions
153 + attn_metadata = model_input.attn_metadata
154 + return_seq_lens = model_input.seq_lens
155 + slot_mapping = attn_metadata.slot_mapping
156 + encoder_input_tokens = model_input.encoder_input_tokens
157 + encoder_input_positions = model_input.encoder_input_positions
158 + cross_slot_mapping = attn_metadata.cross_slot_mapping
159 + assert return_seq_lens == seq_lens
160 + assert len(slot_mapping) == len(input_tokens)
161 + assert len(cross_slot_mapping) == len(encoder_input_tokens)
162 +
163 + # Verify input metadata is correct for prompts.
164 + # - Decoder attention metadata
165 + device = model_runner.device
166 + assert attn_metadata.num_prefills > 0
167 + assert attn_metadata.num_decode_tokens == 0
168 + assert torch.equal(attn_metadata.seq_lens_tensor,
169 + torch.tensor(seq_lens, device=device, dtype=torch.int))
170 + assert attn_metadata.seq_lens == seq_lens
171 + assert attn_metadata.max_prefill_seq_len == max(seq_lens)
172 + assert attn_metadata.max_decode_seq_len == 0
173 + # - Encoder attention metadata
174 + assert attn_metadata.encoder_seq_lens == encoder_seq_lens
175 + assert torch.equal(
176 + attn_metadata.encoder_seq_lens_tensor,
177 + torch.tensor(encoder_seq_lens, device=device, dtype=torch.int))
178 + assert attn_metadata.max_encoder_seq_len == max(encoder_seq_lens)
179 + assert attn_metadata.num_encoder_tokens == sum(encoder_seq_lens)
180 +
181 + # Test decoder subquery start locs.
182 + start_idx = 0
183 + start_loc = [start_idx]
184 + for seq_len in seq_lens:
185 + start_idx += seq_len
186 + start_loc.append(start_idx)
187 + assert torch.equal(
188 + attn_metadata.query_start_loc,
189 + torch.tensor(start_loc, dtype=torch.int32, device=device),
190 + )
191 +
192 + # Test decoder seq start locs & context lengths
193 +
194 + assert torch.equal(
195 + attn_metadata.seq_start_loc,
196 + torch.tensor(start_loc, dtype=torch.int32, device=device),
197 + )
198 + assert torch.equal(
199 + attn_metadata.context_lens_tensor,
200 + torch.zeros(attn_metadata.context_lens_tensor.shape[0],
201 + dtype=torch.int,
202 + device=device),
203 + )
204 +
205 + # Verify block tables are correct for prompts
206 + # - Decoder self-attention
207 + expected = torch.tensor(
208 + [[] for _ in range(len(seq_group_metadata_list))],
209 + dtype=torch.int32,
210 + device=model_runner.device,
211 + )
212 + assert torch.equal(
213 + attn_metadata.block_tables,
214 + expected,
215 + )
216 + # - Encoder/decoder cross-attention
217 + assert torch.equal(
218 + attn_metadata.cross_block_tables,
219 + expected,
220 + )
221 +
222 + # Cuda graph should not be used for prefill.
223 + assert attn_metadata.use_cuda_graph is False
224 +
225 + # Verify the lengths of input tokens & positions
226 + # - Decoder
227 + assert len(input_tokens) == sum(seq_lens)
228 + assert len(input_positions) == sum(seq_lens)
229 + # -- An indirect check that model_input.input_tokens
230 + # and model_input.input_positions are correct -
231 + # by design of the test, the input tokens are
232 + # equal to the input position values, so if
233 + # the model_input data structure has the correct
234 + # values then these two should be equal
235 + assert torch.equal(
236 + input_tokens,
237 + input_positions,
238 + )
239 + # - Encoder
240 + assert len(encoder_input_tokens) == sum(encoder_seq_lens)
241 + # -- An indirect check that model_input.encoder_input_tokens
242 + # and model_input.encoder_input_positions are correct -
243 + # by design of the test, the input tokens are
244 + # equal to the input position values, so if
245 + # the model_input data structure has the correct
246 + # values then these two should be equal
247 + assert torch.equal(
248 + encoder_input_tokens,
249 + encoder_input_positions,
250 + )
251 +
252 + # Test that vLLM sampling infrastructure chooses the correct
253 + # sequence positions at which to sample (i.e. the end of
254 + # each sequence) in the prefill phase
255 +
256 + expected_selected_token_indices = []
257 + selected_token_start_idx = 0
258 + for seq_len in seq_lens:
259 + # Compute the index offset of the final token in each
260 + # prompt (recall that the prompts are concatenated)
261 + expected_selected_token_indices.append(selected_token_start_idx +
262 + seq_len - 1)
263 + selected_token_start_idx += seq_len
264 +
265 + sampling_metadata = model_input.sampling_metadata
266 + actual = sampling_metadata.selected_token_indices
267 + expected = torch.tensor(
268 + expected_selected_token_indices,
269 + device=actual.device,
270 + dtype=actual.dtype,
271 + )
272 + assert torch.equal(actual, expected)
273 +
274 +
275 +@pytest.mark.skipif(condition=is_cpu(),
276 + reason="CPU backend is currently "
277 + "unsupported for encoder/ "
278 + "decoder models")
279 +@pytest.mark.parametrize("batch_size", BATCH_SIZES)
280 +@pytest.mark.parametrize("enforce_eager", ENFORCE_EAGER)
281 +def test_prepare_decode(
282 + batch_size,
283 + enforce_eager,
284 +):
285 + '''
286 + Test the ability of the encoder/decoder model runner subclass to
287 + produce decode-phase model inputs & attention metadata.
288 +
289 + Test behavior:
290 +
291 + * Instantiate BART base model & enc/dec model runner
292 + * Construct sequence-group metadata for dummy prompts
293 + * Test that encoder attention, decoder self-attention,
294 + and encoder/decoder cross-attention inputs are correct
295 +
296 + Arguments:
297 +
298 + * batch_size
299 + * backend_name: The attention backend under test
300 + * enforce_eager: Enforce eager mode if True (i.e. no CUDAGraph)
301 + '''
302 +
303 + model_runner = _create_model_runner(
304 + "facebook/bart-base",
305 + seed=0,
306 + dtype="float16",
307 + max_num_batched_tokens=100000,
308 + max_num_seqs=100000,
309 + enable_chunked_prefill=False,
310 + enforce_eager=enforce_eager,
311 + )
312 +
313 + seq_lens: List[int] = []
314 + encoder_seq_lens: List[int] = []
315 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
316 + block_tables = {0: [1]}
317 + cross_block_table = [2]
318 + for i in range(batch_size):
319 + # make sure all tokens fit into one block
320 + seq_len = i % (model_runner.block_size - 1) + 1
321 + seq_lens.append(seq_len)
322 + seq_data = SequenceData(list(range(seq_len)))
323 + encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
324 + encoder_seq_lens.append(encoder_seq_len)
325 + encoder_seq_data = SequenceData(list(range(encoder_seq_len)))
326 + seq_group_metadata = SequenceGroupMetadata(
327 + request_id=f"test_{i}",
328 + is_prompt=False,
329 + seq_data={0: seq_data},
330 + sampling_params=SamplingParams(temperature=0),
331 + block_tables=block_tables,
332 + encoder_seq_data=encoder_seq_data,
333 + cross_block_table=cross_block_table,
334 + )
335 + assert seq_group_metadata.token_chunk_size == 1
336 + seq_group_metadata_list.append(seq_group_metadata)
337 +
338 + # Build
339 + # * Decoder model inputs
340 + # * Decoder self-attention KV caching data structures
341 + # * Encoder model inputs
342 + # * Encoder/decoder cross-attention KV caching data structures
343 + model_input = model_runner.prepare_model_input(seq_group_metadata_list)
344 + input_tokens = model_input.input_tokens
345 + input_positions = model_input.input_positions
346 + attn_metadata = model_input.attn_metadata
347 + return_seq_lens = model_input.seq_lens
348 + slot_mapping = attn_metadata.slot_mapping
349 + encoder_input_tokens = model_input.encoder_input_tokens
350 + encoder_input_positions = model_input.encoder_input_positions
351 + cross_slot_mapping = attn_metadata.cross_slot_mapping
352 + assert return_seq_lens == seq_lens
353 + assert len(slot_mapping) == len(input_tokens)
354 + assert len(cross_slot_mapping) == len(encoder_input_tokens)
355 +
356 + # Verify input metadata is correct for decode phase.
357 + # - Decoder attention metadata
358 + device = model_runner.device
359 + assert attn_metadata.num_prefills == 0
360 + assert attn_metadata.num_decode_tokens > 0
361 + assert torch.equal(attn_metadata.seq_lens_tensor,
362 + torch.tensor(seq_lens, device=device, dtype=torch.int))
363 + assert attn_metadata.seq_lens == seq_lens
364 + assert attn_metadata.max_prefill_seq_len == 0
365 + assert attn_metadata.max_decode_seq_len == max(seq_lens)
366 + # - Encoder attention metadata
367 + assert attn_metadata.encoder_seq_lens == encoder_seq_lens
368 + assert torch.equal(
369 + attn_metadata.encoder_seq_lens_tensor,
370 + torch.tensor(encoder_seq_lens, device=device, dtype=torch.int))
371 + assert attn_metadata.max_encoder_seq_len == max(encoder_seq_lens)
372 + assert attn_metadata.num_encoder_tokens == sum(encoder_seq_lens)
373 +
374 + # Test decoder subquery start locs.
375 + start_idx = 0
376 + start_loc = [start_idx]
377 + for seq_len in seq_lens:
378 + start_idx += 1
379 + start_loc.append(start_idx)
380 + assert torch.equal(
381 + attn_metadata.query_start_loc,
382 + torch.tensor(start_loc, dtype=torch.int32, device=device),
383 + )
384 +
385 + # Test decoder seq start locs. Note that for normal prefill it is
386 + # equivalent to query_start_loc.
387 + start_idx = 0
388 + seq_start_loc = [start_idx]
389 + for seq_len in seq_lens:
390 + start_idx += seq_len
391 + seq_start_loc.append(start_idx)
392 +
393 + # Test seq_start_loc and context lengths
394 +
395 + assert torch.equal(
396 + attn_metadata.seq_start_loc,
397 + torch.tensor(seq_start_loc, dtype=torch.int32, device=device),
398 + )
399 + assert torch.equal(
400 + attn_metadata.context_lens_tensor,
401 + torch.tensor([seq_len - 1 for seq_len in seq_lens],
402 + dtype=torch.int,
403 + device=device))
404 +
405 + # Verify block tables are correct for prompts
406 + # - Decoder self-attention
407 + expected = torch.tensor(
408 + [block_tables[0] for _ in range(len(seq_group_metadata_list))],
409 + dtype=torch.int32,
410 + device=model_runner.device)
411 + assert torch.equal(
412 + attn_metadata.block_tables,
413 + expected,
414 + )
415 + # - Encoder/decoder cross-attention
416 + expected = torch.tensor(
417 + [cross_block_table for _ in range(len(seq_group_metadata_list))],
418 + dtype=torch.int32,
419 + device=model_runner.device)
420 + assert torch.equal(
421 + attn_metadata.cross_block_tables,
422 + expected,
423 + )
424 +
425 + # Cuda graph should is currently not supported for encoder/decoer.
426 + assert attn_metadata.use_cuda_graph is False
427 +
428 + # Verify the lengths of input tokens & positions
429 + # - Decoder
430 + assert len(input_tokens) == len(seq_lens)
431 + assert len(input_positions) == len(seq_lens)
432 + # -- An indirect check that model_input.input_tokens
433 + # and model_input.input_positions are correct -
434 + # by design of the test, the input tokens are
435 + # equal to the input position values, so if
436 + # the model_input data structure has the correct
437 + # values then these two should be equal
438 + assert torch.equal(
439 + input_tokens,
440 + input_positions,
441 + )
442 + # - Encoder
443 + assert len(encoder_input_tokens) == 0
444 + assert len(encoder_input_tokens) == 0
445 + # -- An indirect check that model_input.encoder_input_tokens
446 + # and model_input.encoder_input_positions are correct -
447 + # by design of the test, the input tokens are
448 + # equal to the input position values, so if
449 + # the model_input data structure has the correct
450 + # values then these two should be equal
451 + assert torch.equal(
452 + encoder_input_tokens,
453 + encoder_input_positions,
454 + )
455 +
456 + # Test that vLLM sampling infrastructure chooses the correct
457 + # sequence positions at which to sample (i.e. the end of
458 + # each sequence) in the decode phase
459 +
460 + expected_selected_token_indices = []
461 + selected_token_start_idx = 0
462 + for seq_len in seq_lens:
463 + # Compute the index offset of the final token in each
464 + # sequence's decoded outputs; since a single token is
465 + # decoded per iteration per sequence, then the length
466 + # of the decoded tokens for a given sequence is 1 and
467 + # the final index offset into a given sequence's
468 + # generated tokens is 0 (i.e. the expected sampling index
469 + # for a given sequence is just `selected_token_start_idx`)
470 + expected_selected_token_indices.append(selected_token_start_idx)
471 + selected_token_start_idx += 1
472 +
473 + sampling_metadata = model_input.sampling_metadata
474 + actual = sampling_metadata.selected_token_indices
475 + expected = torch.tensor(
476 + expected_selected_token_indices,
477 + device=actual.device,
478 + dtype=actual.dtype,
479 + )
480 + assert torch.equal(actual, expected)
vllm/attention/__init__.py
+3 -1
@@ -1,6 +1,7 @@
1 from vllm.attention.backends.abstract import (AttentionBackend,
2 AttentionMetadata,
3 - AttentionMetadataBuilder)
3 + AttentionMetadataBuilder,
4 + AttentionType)
5 from vllm.attention.layer import Attention
6 from vllm.attention.selector import get_attn_backend
7
@@ -8,6 +9,7 @@ __all__ = [
9 "Attention",
10 "AttentionBackend",
11 "AttentionMetadata",
12 + "AttentionType",
13 "AttentionMetadataBuilder",
14 "Attention",
15 "get_attn_backend",
vllm/attention/layer.py
+1 -1
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, Optional
4 import torch
5 import torch.nn as nn
6
7 -from vllm.attention.backends.abstract import AttentionMetadata, AttentionType
7 +from vllm.attention import AttentionMetadata, AttentionType
8 from vllm.attention.selector import get_attn_backend
9 from vllm.config import CacheConfig
10 from vllm.model_executor.layers.quantization.base_config import (
vllm/attention/selector.py
+111 -12
@@ -1,6 +1,8 @@
1 import enum
2 +import os
3 +from contextlib import contextmanager
4 from functools import lru_cache
3 -from typing import Optional, Type
5 +from typing import Generator, Optional, Type
6
7 import torch
8
@@ -8,7 +10,8 @@ import vllm.envs as envs
10 from vllm.attention.backends.abstract import AttentionBackend
11 from vllm.logger import init_logger
12 from vllm.platforms import current_platform
11 -from vllm.utils import is_cpu, is_hip, is_openvino, is_tpu, is_xpu
13 +from vllm.utils import (STR_BACKEND_ENV_VAR, is_cpu, is_hip, is_openvino,
14 + is_tpu, is_xpu)
15
16 logger = init_logger(__name__)
17
@@ -24,6 +27,66 @@ class _Backend(enum.Enum):
27 IPEX = enum.auto()
28
29
30 +def backend_name_to_enum(backend_name: str) -> _Backend:
31 + assert backend_name is not None
32 +
33 + backend_members = _Backend.__members__
34 + if backend_name not in backend_members:
35 + raise ValueError(f"Invalid attention backend '{backend_name}'. "
36 + f"Available backends: {', '.join(backend_members)} "
37 + "(case-sensitive).")
38 +
39 + return _Backend[backend_name]
40 +
41 +
42 +def get_env_variable_attn_backend() -> Optional[_Backend]:
43 + '''
44 + Get the backend override specified by the vLLM attention
45 + backend environment variable, if one is specified.
46 +
47 + Returns:
48 +
49 + * _Backend enum value if an override is specified
50 + * None otherwise
51 + '''
52 + backend_name = os.environ.get(STR_BACKEND_ENV_VAR)
53 + return (None
54 + if backend_name is None else backend_name_to_enum(backend_name))
55 +
56 +
57 +# Global state allows a particular choice of backend
58 +# to be forced, overriding the logic which auto-selects
59 +# a backend based on system & workload configuration
60 +# (default behavior if this variable is None)
61 +#
62 +# THIS SELECTION TAKES PRECEDENCE OVER THE
63 +# VLLM ATTENTION BACKEND ENVIRONMENT VARIABLE
64 +forced_attn_backend: Optional[_Backend] = None
65 +
66 +
67 +def global_force_attn_backend(attn_backend: Optional[_Backend]) -> None:
68 + '''
69 + Force all attention operations to use a specified backend.
70 +
71 + Passing `None` for the argument re-enables automatic
72 + backend selection.,
73 +
74 + Arguments:
75 +
76 + * attn_backend: backend selection (None to revert to auto)
77 + '''
78 + global forced_attn_backend
79 + forced_attn_backend = attn_backend
80 +
81 +
82 +def get_global_forced_attn_backend() -> Optional[_Backend]:
83 + '''
84 + Get the currently-forced choice of attention backend,
85 + or None if auto-selection is currently enabled.
86 + '''
87 + return forced_attn_backend
88 +
89 +
90 @lru_cache(maxsize=None)
91 def get_attn_backend(
92 num_heads: int,
@@ -101,16 +164,20 @@ def which_attn_to_use(
164 # Default case.
165 selected_backend = _Backend.FLASH_ATTN
166
104 - # Check the environment variable and override if specified
105 - backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
106 - if backend_by_env_var is not None:
107 - backend_members = _Backend.__members__
108 - if backend_by_env_var not in backend_members:
109 - raise ValueError(
110 - f"Invalid attention backend '{backend_by_env_var}'. "
111 - f"Available backends: {', '.join(backend_members)} "
112 - "(case-sensitive).")
113 - selected_backend = _Backend[backend_by_env_var]
167 + # Check whether a particular choice of backend was
168 + # previously forced.
169 + #
170 + # THIS SELECTION OVERRIDES THE VLLM_ATTENTION_BACKEND
171 + # ENVIRONMENT VARIABLE.
172 + backend_by_global_setting: Optional[_Backend] = (
173 + get_global_forced_attn_backend())
174 + if backend_by_global_setting is not None:
175 + selected_backend = backend_by_global_setting
176 + else:
177 + # Check the environment variable and override if specified
178 + backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
179 + if backend_by_env_var is not None:
180 + selected_backend = backend_name_to_enum(backend_by_env_var)
181
182 if is_cpu():
183 if selected_backend != _Backend.TORCH_SDPA:
@@ -193,3 +260,35 @@ def which_attn_to_use(
260 selected_backend = _Backend.XFORMERS
261
262 return selected_backend
263 +
264 +
265 +@contextmanager
266 +def global_force_attn_backend_context_manager(
267 + attn_backend: _Backend) -> Generator[None, None, None]:
268 + '''
269 + Globally force a vLLM attention backend override within a
270 + context manager, reverting the global attention backend
271 + override to its prior state upon exiting the context
272 + manager.
273 +
274 + Arguments:
275 +
276 + * attn_backend: attention backend to force
277 +
278 + Returns:
279 +
280 + * Generator
281 + '''
282 +
283 + # Save the current state of the global backend override (if any)
284 + original_value = get_global_forced_attn_backend()
285 +
286 + # Globally force the new backend override
287 + global_force_attn_backend(attn_backend)
288 +
289 + # Yield control back to the enclosed code block
290 + try:
291 + yield
292 + finally:
293 + # Revert the original global backend override, if any
294 + global_force_attn_backend(original_value)
vllm/config.py
+34 -2
@@ -12,7 +12,8 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12 from vllm.model_executor.models import ModelRegistry
13 from vllm.tracing import is_otel_installed
14 from vllm.transformers_utils.config import get_config, get_hf_text_config
15 -from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
15 +from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH,
16 + cuda_device_count_stateless, get_cpu_memory, is_cpu,
17 is_hip, is_neuron, is_openvino, is_tpu, is_xpu,
18 print_warning_once)
19
@@ -87,6 +88,9 @@ class ModelConfig:
88 enforce_eager: Whether to enforce eager execution. If True, we will
89 disable CUDA graph and always execute the model in eager mode.
90 If False, we will use CUDA graph and eager execution in hybrid.
91 + If None, the user did not specify, so default to False -
92 + except for encoder/decoder models, which currently require
93 + eager mode.
94 max_context_len_to_capture: Maximum context len covered by CUDA graphs.
95 When a sequence has context length larger than this, we fall back
96 to eager mode (DEPRECATED. Use max_seq_len_to_capture instead).
@@ -121,7 +125,7 @@ class ModelConfig:
125 max_model_len: Optional[int] = None,
126 quantization: Optional[str] = None,
127 quantization_param_path: Optional[str] = None,
124 - enforce_eager: bool = False,
128 + enforce_eager: Optional[bool] = None,
129 max_context_len_to_capture: Optional[int] = None,
130 max_seq_len_to_capture: Optional[int] = None,
131 max_logprobs: int = 20,
@@ -160,6 +164,34 @@ class ModelConfig:
164 self.hf_text_config = get_hf_text_config(self.hf_config)
165 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
166
167 + # Choose a default enforce_eager value if the user did not specify
168 + # a value (enforce_eager is None)
169 + if getattr(self.hf_config, 'is_encoder_decoder', False):
170 + if self.enforce_eager is None:
171 + # *Only for encoder/decoder models* and
172 + # *only if enforce_eager is unset*, override
173 + # to enforce_eager=True
174 + #
175 + # Add a logger message since it is *somewhat* non-intuitive that
176 + # enforce_eager is True when the user has not specified its
177 + # value.
178 + logger.info("Forcing enforce_eager == True because "
179 + "enforce_eager setting was unspecified and "
180 + "CUDAGraph is not supported with encoder/ "
181 + "decoder models.")
182 + self.enforce_eager = True
183 +
184 + if not self.enforce_eager:
185 + # Eager mode explicitly disabled by user for an encoder/
186 + # decoder model; however CUDAGRAPH + encoder/decoder is
187 + # not currently supported
188 + raise ValueError(STR_NOT_IMPL_ENC_DEC_CUDAGRAPH)
189 + elif self.enforce_eager is None:
190 + # *Only for decoder-only models*, enforce_eager
191 + # defaults to False if unset. This is intuitive
192 + # so no logging message needed.
193 + self.enforce_eager = False
194 +
195 if (not self.disable_sliding_window
196 and self.hf_text_config.model_type == "gemma2"
197 and self.hf_text_config.sliding_window is not None):
vllm/core/block/utils.py
+2 -10
@@ -1,15 +1,7 @@
1 """Block manager utils."""
2 from vllm.sequence import SequenceGroup
3 -
4 -# Exception strings for non-implemented block manager enc/dec scenarios
5 -
6 -STR_NOT_IMPL_ENC_DEC_SWA = \
7 - "Sliding window attention for encoder/decoder models " + \
8 - "is not currently supported."
9 -
10 -STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE = \
11 - "Prefix caching for encoder/decoder models " + \
12 - "is not currently supported."
3 +from vllm.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
4 + STR_NOT_IMPL_ENC_DEC_SWA)
5
6
7 def _get_block_mgr_sliding_window_attr(block_mgr):
vllm/core/scheduler.py
+28
@@ -392,6 +392,19 @@ class Scheduler:
392 seq.status = SequenceStatus.FINISHED_ABORTED
393 self.free_seq(seq)
394
395 + self._free_seq_group_cross_attn_blocks(aborted_group)
396 +
397 + def _free_seq_group_cross_attn_blocks(
398 + self,
399 + seq_group: SequenceGroup,
400 + ) -> None:
401 + """
402 + Free a sequence group from a cross-attention block table.
403 + Has no effect on decoder-only models.
404 + """
405 + if seq_group.is_encoder_decoder():
406 + self.block_manager.free_cross(seq_group)
407 +
408 def has_unfinished_seqs(self) -> bool:
409 return len(self.waiting) != 0 or len(self.running) != 0 or len(
410 self.swapped) != 0
@@ -963,6 +976,17 @@ class Scheduler:
976 # seq_id -> physical block numbers
977 block_tables: Dict[int, List[int]] = {}
978
979 + if seq_group.is_encoder_decoder():
980 + # Encoder associated with SequenceGroup
981 + encoder_seq_data = seq_group.get_encoder_seq().data
982 + # Block table for cross-attention
983 + # Also managed at SequenceGroup level
984 + cross_block_table = self.block_manager.get_cross_block_table(
985 + seq_group)
986 + else:
987 + encoder_seq_data = None
988 + cross_block_table = None
989 +
990 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
991 seq_id = seq.seq_id
992 seq_data[seq_id] = seq.data
@@ -1001,6 +1025,8 @@ class Scheduler:
1025 token_chunk_size=token_chunk_size,
1026 lora_request=seq_group.lora_request,
1027 computed_block_nums=common_computed_block_nums,
1028 + encoder_seq_data=encoder_seq_data,
1029 + cross_block_table=cross_block_table,
1030 # `multi_modal_data` will only be present for the 1st comm
1031 # between engine and worker.
1032 # the subsequent comms can still use delta, but
@@ -1032,6 +1058,8 @@ class Scheduler:
1058 remaining: Deque[SequenceGroup] = deque()
1059 for seq_group in self.running:
1060 if seq_group.is_finished():
1061 + # Free cross-attention block table, if it exists
1062 + self._free_seq_group_cross_attn_blocks(seq_group)
1063 # Add the finished requests to the finished requests list.
1064 # This list will be used to update the Mamba cache in the
1065 # next step.
vllm/engine/arg_utils.py
+1 -1
@@ -69,7 +69,7 @@ class EngineArgs:
69 rope_theta: Optional[float] = None
70 tokenizer_revision: Optional[str] = None
71 quantization: Optional[str] = None
72 - enforce_eager: bool = False
72 + enforce_eager: Optional[bool] = None
73 max_context_len_to_capture: Optional[int] = None
74 max_seq_len_to_capture: int = 8192
75 disable_custom_all_reduce: bool = False
vllm/engine/llm_engine.py
+411 -24
@@ -3,7 +3,7 @@ from contextlib import contextmanager
3 from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
4 Mapping, Optional)
5 from typing import Sequence as GenericSequence
6 -from typing import Set, Type, TypeVar, Union
6 +from typing import Set, Tuple, Type, TypeVar, Union
7
8 import vllm.envs as envs
9 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
@@ -22,7 +22,8 @@ from vllm.engine.output_processor.stop_checker import StopChecker
22 from vllm.engine.output_processor.util import create_output_by_sequence_group
23 from vllm.executor.executor_base import ExecutorBase
24 from vllm.executor.ray_utils import initialize_ray_cluster
25 -from vllm.inputs import INPUT_REGISTRY, LLMInputs, PromptInputs
25 +from vllm.inputs import (INPUT_REGISTRY, LLMInputs, PromptInputs,
26 + get_prompt_type)
27 from vllm.logger import init_logger
28 from vllm.lora.request import LoRARequest
29 from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
@@ -42,7 +43,8 @@ from vllm.transformers_utils.tokenizer_group import (
43 AnyTokenizer, BaseTokenizerGroup, init_tokenizer_from_configs)
44 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
45 usage_message)
45 -from vllm.utils import Counter
46 +from vllm.utils import (Counter, is_embedding_model_config,
47 + is_encoder_decoder_model_config)
48 from vllm.version import __version__ as VLLM_VERSION
49
50 logger = init_logger(__name__)
@@ -502,8 +504,19 @@ class LLMEngine:
504 self.prompt_adapter_config.verify_with_model_config(
505 self.model_config)
506
505 - def _get_eos_token_id(
506 - self, lora_request: Optional[LoRARequest]) -> Optional[int]:
507 + def _get_bos_token_id(self,
508 + lora_request: Optional[LoRARequest] = None
509 + ) -> Optional[int]:
510 + if self.tokenizer is None:
511 + logger.warning("Using None for BOS token id because tokenizer "
512 + "is not initialized")
513 + return None
514 +
515 + return self.tokenizer.get_lora_tokenizer(lora_request).bos_token_id
516 +
517 + def _get_eos_token_id(self,
518 + lora_request: Optional[LoRARequest] = None
519 + ) -> Optional[int]:
520 if self.tokenizer is None:
521 logger.warning("Using None for EOS token id because tokenizer "
522 "is not initialized")
@@ -511,6 +524,32 @@ class LLMEngine:
524
525 return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id
526
527 + def _get_decoder_start_token_id(self, ) -> Optional[int]:
528 + '''
529 + Obtain the decoder start token id employed by an encoder/decoder
530 + model. Returns None for non-encoder/decoder models or if the
531 + model config is unavailable.
532 + '''
533 +
534 + if not self.is_encoder_decoder_model():
535 + logger.warning("Using None for decoder start token id because "
536 + "this is not an encoder/decoder model.")
537 + return None
538 +
539 + if (self.model_config is None or self.model_config.hf_config is None):
540 + logger.warning("Using None for decoder start token id because "
541 + "model config is not available.")
542 + return None
543 +
544 + dec_start_token_id = getattr(self.model_config.hf_config,
545 + 'decoder_start_token_id', None)
546 + if dec_start_token_id is None:
547 + logger.warning("Falling back on <BOS> for decoder start token id "
548 + "because decoder start token id is not available.")
549 + dec_start_token_id = self._get_bos_token_id()
550 +
551 + return dec_start_token_id
552 +
553 def _add_processed_request(
554 self,
555 request_id: str,
@@ -529,6 +568,16 @@ class LLMEngine:
568 seq = Sequence(seq_id, processed_inputs, block_size, eos_token_id,
569 lora_request, prompt_adapter_request)
570
571 + encoder_seq = None
572 + if 'encoder_prompt_token_ids' in processed_inputs:
573 + encoder_seq = Sequence(seq_id,
574 + processed_inputs,
575 + block_size,
576 + eos_token_id,
577 + lora_request,
578 + prompt_adapter_request,
579 + from_decoder_prompt=False)
580 +
581 # Create a SequenceGroup based on SamplingParams or PoolingParams
582 if isinstance(params, SamplingParams):
583 seq_group = self._create_sequence_group_with_sampling(
@@ -538,7 +587,8 @@ class LLMEngine:
587 arrival_time=arrival_time,
588 lora_request=lora_request,
589 trace_headers=trace_headers,
541 - prompt_adapter_request=prompt_adapter_request)
590 + prompt_adapter_request=prompt_adapter_request,
591 + encoder_seq=encoder_seq)
592 elif isinstance(params, PoolingParams):
593 seq_group = self._create_sequence_group_with_pooling(
594 request_id,
@@ -546,7 +596,8 @@ class LLMEngine:
596 params,
597 arrival_time=arrival_time,
598 lora_request=lora_request,
549 - prompt_adapter_request=prompt_adapter_request)
599 + prompt_adapter_request=prompt_adapter_request,
600 + encoder_seq=encoder_seq)
601 else:
602 raise ValueError(
603 "Either SamplingParams or PoolingParams must be provided.")
@@ -562,36 +613,362 @@ class LLMEngine:
613 def stop_remote_worker_execution_loop(self) -> None:
614 self.model_executor.stop_remote_worker_execution_loop()
615
565 - def process_model_inputs(
616 + _LLMInputComponentsType = Tuple[str, List[int], ]
617 +
618 + def _prepare_decoder_input_ids_for_generation(
619 + self,
620 + decoder_input_ids: Optional[List[int]] = None,
621 + ) -> List[int]:
622 + """
623 + Prepares `decoder_input_ids` for generation with encoder-decoder models.
624 +
625 + Based on
626 +
627 + https://github.com/huggingface/transformers/blob/
628 + 4037a2b5b1278736e566aec12e169100275545ea/
629 + src/transformers/generation/utils.py
630 +
631 + specifically GenerationMixin._prepare_decoder_input_ids_for_generation()
632 +
633 + Arguments:
634 +
635 + * decoder_input_ids: input token ids to preprocess
636 +
637 + Returns:
638 +
639 + * Processed token list
640 + """
641 +
642 + decoder_start_token_id: Optional[int] = (
643 + self._get_decoder_start_token_id())
644 + assert decoder_start_token_id is not None
645 +
646 + if decoder_input_ids is None:
647 + # no decoder prompt input ->
648 + # use decoder_start_token_id as decoder_input_ids
649 + (decoder_input_ids) = self._get_default_enc_dec_decoder_prompt()
650 +
651 + if (len(decoder_input_ids) == 0
652 + or decoder_input_ids[0] != decoder_start_token_id):
653 + decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
654 +
655 + return decoder_input_ids
656 +
657 + def _tokenize_prompt(
658 + self,
659 + prompt: str,
660 + request_id: Optional[str] = None,
661 + lora_request: Optional[str] = None,
662 + ) -> List[int]:
663 + '''
664 + Wrapper around application of the model's
665 + tokenizer.
666 +
667 + Arguments:
668 +
669 + * prompt
670 + * request_id
671 + * lora_request
672 +
673 + Returns:
674 +
675 + * prompt token ids
676 + '''
677 +
678 + tokenizer = self.get_tokenizer_group("prompts must be None if "
679 + "skip_tokenizer_init is True")
680 +
681 + prompt_token_ids = tokenizer.encode(request_id=request_id,
682 + prompt=prompt,
683 + lora_request=lora_request)
684 +
685 + return prompt_token_ids
686 +
687 + def _extract_single_prompt_for_enc_dec_input(
688 + self,
689 + inputs: Optional[PromptInputs],
690 + request_id: Optional[str] = None,
691 + ptype: Optional[str] = None,
692 + is_encoder_prompt: bool = False,
693 + ) -> Tuple[Optional[str], List[int]]:
694 + '''
695 + Only for encoder/decoder models:
696 + Extract prompt & prompt_token_ids from any single
697 + encoder or decoder input prompt. For encoder input prompts
698 + in particular, also extract multi-modal data.
699 +
700 + This function handles the following scenarios:
701 + 1. The user supplied a singleton encoder prompt
702 + & the prompt/prompt-token-ids must be extracted.
703 + 2. The user supplied an explicit encoder/decoder
704 + prompt & the prompt/prompt-token-ids must be
705 + extracted from either the encoder and decoder prompts.
706 +
707 + For decoder prompts in particular (scenario 2), special
708 + processing is applied to the returned decoder token ids.
709 +
710 + Arguments:
711 +
712 + * request_id
713 + * ptype: str representation of the input prompt type.
714 + If `ptype` is `None`, assume that the prompt
715 + type is unknown and must be inferred. This is the
716 + case for ExplicitEncoderDecoder sub-prompts.
717 + * inputs: single encoder or decoder input prompt
718 + * is_encoder_prompt: True if encoder input prompt.
719 + If False, decoder prompt tokens
720 + are preprocessed.
721 +
722 + Returns:
723 +
724 + * prompt
725 + * prompt_token_ids
726 + '''
727 + prompt_token_ids = None
728 + ptype = (get_prompt_type(inputs) if ptype is None else ptype)
729 +
730 + if inputs is None:
731 + prompt = None
732 + elif ptype == 'str':
733 + prompt = inputs
734 + prompt_token_ids = self._tokenize_prompt(
735 + prompt,
736 + request_id=request_id,
737 + )
738 + elif ptype == 'TokensPrompt':
739 + prompt = None
740 + prompt_token_ids = inputs['prompt_token_ids']
741 + else:
742 + prompt = inputs['prompt']
743 + prompt_token_ids = self._tokenize_prompt(
744 + prompt,
745 + request_id=request_id,
746 + )
747 +
748 + if not is_encoder_prompt:
749 + # Apply special pre-processing to
750 + # decoder prompts
751 + prompt_token_ids = (self._prepare_decoder_input_ids_for_generation(
752 + prompt_token_ids, ))
753 +
754 + assert prompt_token_ids is not None
755 +
756 + return (
757 + prompt,
758 + prompt_token_ids,
759 + )
760 +
761 + def _get_default_enc_dec_decoder_prompt(self, ) -> List[int]:
762 + '''
763 + Specifically for encoder/decoder models:
764 + generate a default decoder prompt for when
765 + the user specifies only the encoder prompt.
766 +
767 + Encoder/decoder models utilize the decoder
768 + prompt in different ways; as new models are
769 + added, it is intended that this function
770 + will be extended to produce differing
771 + default decoder prompts, depending on the
772 + model variety.
773 +
774 + Absent a special case, the default behavior
775 + of this method is to mirror the behavior of
776 + the HuggingFace (HF) GenerationMixin for a None
777 + decoder prompt, which is to employ a logit processor
778 + setting to force the first decoded token to be <BOS>.
779 + Here, this behavior is approximated by having the
780 + "default" decoder prompt be <BOS>.
781 +
782 + However, it is possible that in the future
783 + other models may have different or more
784 + complex logic for the default decoder prompt.
785 + This motivates having a special helper method
786 + for default decoder prompts.
787 +
788 + Returns:
789 +
790 + * prompt_token_ids
791 + '''
792 +
793 + bos_token_id = self._get_bos_token_id()
794 + assert bos_token_id is not None
795 + prompt_token_ids: List[int] = [bos_token_id]
796 + return prompt_token_ids
797 +
798 + def _process_encoder_decoder_prompt(
799 + self,
800 + inputs: PromptInputs,
801 + request_id: Optional[str] = None,
802 + ) -> LLMInputs:
803 + '''
804 + For encoder/decoder models only:
805 + Process an input prompt
806 + into an `LLMInputs` instance.
807 +
808 + There are two types of input prompts:
809 + singleton prompts which carry only the
810 + encoder prompt, and explicit encoder/decoder
811 + prompts which carry both the encoder and the
812 + decoder prompts as member variables.
813 +
814 + This function handles the following scenarios:
815 + * Singleton encoder prompt: extract encoder prompt
816 + token ids & infer default decoder prompt token ids
817 + * Explicit encoder/decoder prompt: extract encoder
818 + and decoder prompt token ids
819 +
820 + Note that for Explicit encoder/decoder prompts,
821 + each sub-prompt (encoder or decoder prompt) can
822 + have any possible singleton type; thus this
823 + method relies on helper functions to obtain
824 + token ids for the sub-prompts.
825 +
826 + Arguments:
827 +
828 + * inputs: an input prompt
829 + * request_id
830 +
831 + Returns:
832 +
833 + * `LLMInputs` instance
834 + '''
835 +
836 + ptype = get_prompt_type(inputs)
837 +
838 + # Obtain encoder and decoder prompt tokens. Note
839 + # that, no matter what, the decoder
840 + # prompt type is unknown.
841 + if ptype == "ExplicitEncoderDecoder":
842 + # If input is explicit encoder/decoder prompt,
843 + # then it remains to be determined what type
844 + # of encoder prompt we have
845 + extracted_encoder_prompt = inputs.get('encoder_prompt')
846 + encoder_ptype = None
847 + # Extract decoder prompt from explicit
848 + # encoder/decoder prompt
849 + extracted_decoder_prompt = inputs.get('decoder_prompt')
850 + else:
851 + # If input is singleton encoder prompt, then
852 + # we know the encoder prompt type
853 + extracted_encoder_prompt = inputs
854 + encoder_ptype = ptype
855 + # Decoder prompt is always unknown if
856 + # encoder/decoder prompt is not explicit
857 + extracted_decoder_prompt = None
858 +
859 + # Invoke helper function to obtain encoder
860 + # prompt and prompt token ids, either from
861 + # singleton encoder prompt or from the
862 + # encoder sub-prompt of an explicit
863 + # encoder/decode scenario 2), special
864 + # processing is applied to the returned decoder token ids
865 + (
866 + encoder_prompt,
867 + encoder_prompt_token_ids,
868 + ) = self._extract_single_prompt_for_enc_dec_input(
869 + extracted_encoder_prompt,
870 + request_id=request_id,
871 + ptype=encoder_ptype,
872 + is_encoder_prompt=True,
873 + )
874 +
875 + # Invoke helper method to obtain
876 + # decoder prompt and prompt token ids.
877 + #
878 + # The helper method will detect the decoder
879 + # prompt type.
880 + #
881 + # Helper method will also apply special
882 + # preprocessing unique to decoder prompts.
883 + (
884 + decoder_prompt,
885 + decoder_prompt_token_ids,
886 + ) = self._extract_single_prompt_for_enc_dec_input(
887 + extracted_decoder_prompt,
888 + request_id=request_id,
889 + ptype=None,
890 + is_encoder_prompt=False,
891 + )
892 +
893 + return LLMInputs(
894 + prompt_token_ids=decoder_prompt_token_ids,
895 + prompt=decoder_prompt,
896 + encoder_prompt_token_ids=encoder_prompt_token_ids,
897 + encoder_prompt=encoder_prompt,
898 + )
899 +
900 + def _process_decoder_only_prompt(
901 self,
567 - request_id: str,
902 inputs: PromptInputs,
903 lora_request: Optional[LoRARequest] = None,
904 + request_id: Optional[str] = None,
905 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
906 ) -> LLMInputs:
907 + '''
908 + For decoder-only models:
909 + Process an input prompt
910 + into an `LLMInputs` instance.
911 +
912 + Arguments:
913 +
914 + * inputs: input prompt
915 + * lora_request
916 + * request_id
917 + * prompt_adapter_request
918 +
919 + Returns:
920 +
921 + * `LLMInputs` instance
922 + '''
923 +
924 if isinstance(inputs, str):
925 inputs = {"prompt": inputs}
926 + prompt = inputs.get("prompt")
927
928 if "prompt_token_ids" not in inputs:
576 - tokenizer = self.get_tokenizer_group("prompts must be None if "
577 - "skip_tokenizer_init is True")
578 -
579 - prompt_token_ids = tokenizer.encode(request_id=request_id,
580 - prompt=inputs["prompt"],
581 - lora_request=lora_request)
929 + prompt_token_ids = self._tokenize_prompt(
930 + prompt,
931 + request_id=request_id,
932 + lora_request=lora_request,
933 + )
934 else:
935 prompt_token_ids = inputs["prompt_token_ids"]
936
937 if prompt_adapter_request:
586 - prompt_token_ids = \
587 - [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens\
588 - + prompt_token_ids
938 + prompt_token_ids = (
939 + [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens
940 + + prompt_token_ids)
941 +
942 + return LLMInputs(prompt_token_ids=prompt_token_ids,
943 + prompt=prompt,
944 + multi_modal_data=inputs.get("multi_modal_data"))
945 +
946 + def process_model_inputs(
947 + self,
948 + request_id: str,
949 + inputs: PromptInputs,
950 + lora_request: Optional[LoRARequest] = None,
951 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
952 + ) -> LLMInputs:
953
590 - llm_inputs = LLMInputs(prompt_token_ids=prompt_token_ids,
591 - prompt=inputs.get("prompt"),
592 - multi_modal_data=inputs.get("multi_modal_data"))
954 + if self.is_encoder_decoder_model():
955 + # Encoder-decoder model requires special mapping of
956 + # input prompts to encoder & decoder
957
594 - return self.input_processor(llm_inputs)
958 + model_inputs = self._process_encoder_decoder_prompt(
959 + inputs,
960 + request_id=request_id,
961 + )
962 + else:
963 + # Decoder-only operation
964 + model_inputs = self._process_decoder_only_prompt(
965 + inputs,
966 + request_id=request_id,
967 + lora_request=lora_request,
968 + prompt_adapter_request=prompt_adapter_request,
969 + )
970 +
971 + return self.input_processor(model_inputs)
972
973 def add_request(
974 self,
@@ -676,6 +1053,7 @@ class LLMEngine:
1053 lora_request: Optional[LoRARequest],
1054 trace_headers: Optional[Mapping[str, str]] = None,
1055 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1056 + encoder_seq: Optional[Sequence] = None,
1057 ) -> SequenceGroup:
1058 """Creates a SequenceGroup with SamplingParams."""
1059 max_logprobs = self.get_model_config().max_logprobs
@@ -701,7 +1079,8 @@ class LLMEngine:
1079 sampling_params=sampling_params,
1080 lora_request=lora_request,
1081 trace_headers=trace_headers,
704 - prompt_adapter_request=prompt_adapter_request)
1082 + prompt_adapter_request=prompt_adapter_request,
1083 + encoder_seq=encoder_seq)
1084
1085 return seq_group
1086
@@ -713,6 +1092,7 @@ class LLMEngine:
1092 arrival_time: float,
1093 lora_request: Optional[LoRARequest],
1094 prompt_adapter_request: Optional[PromptAdapterRequest],
1095 + encoder_seq: Optional[Sequence] = None,
1096 ) -> SequenceGroup:
1097 """Creates a SequenceGroup with PoolingParams."""
1098 # Defensive copy of PoolingParams, which are used by the pooler
@@ -724,7 +1104,8 @@ class LLMEngine:
1104 arrival_time=arrival_time,
1105 lora_request=lora_request,
1106 pooling_params=pooling_params,
727 - prompt_adapter_request=prompt_adapter_request)
1107 + prompt_adapter_request=prompt_adapter_request,
1108 + encoder_seq=encoder_seq)
1109 return seq_group
1110
1111 def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
@@ -1214,3 +1595,9 @@ class LLMEngine:
1595 seq_span.set_attribute(
1596 SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
1597 seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time)
1598 +
1599 + def is_encoder_decoder_model(self):
1600 + return is_encoder_decoder_model_config(self.model_config)
1601 +
1602 + def is_embedding_model(self):
1603 + return is_embedding_model_config(self.model_config)
vllm/entrypoints/llm.py
+18 -3
@@ -121,12 +121,21 @@ class LLM:
121 gpu_memory_utilization: float = 0.9,
122 swap_space: int = 4,
123 cpu_offload_gb: float = 0,
124 - enforce_eager: bool = False,
124 + enforce_eager: Optional[bool] = None,
125 max_context_len_to_capture: Optional[int] = None,
126 max_seq_len_to_capture: int = 8192,
127 disable_custom_all_reduce: bool = False,
128 **kwargs,
129 ) -> None:
130 + '''
131 + LLM constructor.
132 +
133 + Note: if enforce_eager is unset (enforce_eager is None)
134 + it defaults to False for decoder-only models and True
135 + for encoder/decoder models, since encoder/decoder models
136 + do not currently support CUDAGraph.
137 + '''
138 +
139 if "disable_log_stats" not in kwargs:
140 kwargs["disable_log_stats"] = True
141 removed_vision_keys = ("image_token_id", "image_feature_size",
@@ -297,8 +306,8 @@ class LLM:
306 """
307 if self.llm_engine.model_config.embedding_mode:
308 raise ValueError(
300 - "LLM.generate() is only supported for generation models "
301 - "(XForCausalLM).")
309 + "LLM.generate() is only supported for (conditional) generation "
310 + "models (XForCausalLM, XForConditionalGeneration).")
311
312 if prompt_token_ids is not None:
313 inputs = self._convert_v1_inputs(
@@ -631,3 +640,9 @@ class LLM:
640 # This is necessary because some requests may be finished earlier than
641 # its previous requests.
642 return sorted(outputs, key=lambda x: int(x.request_id))
643 +
644 + def _is_encoder_decoder_model(self):
645 + return self.llm_engine.is_encoder_decoder_model()
646 +
647 + def _is_embedding_model(self):
648 + return self.llm_engine.is_embedding_model()
vllm/inputs/__init__.py
+18 -5
@@ -1,5 +1,7 @@
1 -from .data import (LLMInputs, ParsedText, ParsedTokens, PromptInputs,
2 - TextPrompt, TokensPrompt, parse_and_batch_prompt)
1 +from .data import (ExplicitEncoderDecoderPrompt, LLMInputs, ParsedText,
2 + ParsedTokens, PromptInputs, SingletonPromptInputs,
3 + TextPrompt, TokensPrompt, get_prompt_type,
4 + is_valid_encoder_decoder_llm_inputs, parse_and_batch_prompt)
5 from .registry import InputContext, InputRegistry
6
7 INPUT_REGISTRY = InputRegistry()
@@ -12,7 +14,18 @@ See also:
14 """
15
16 __all__ = [
15 - "ParsedText", "ParsedTokens", "parse_and_batch_prompt", "TextPrompt",
16 - "TokensPrompt", "PromptInputs", "LLMInputs", "INPUT_REGISTRY",
17 - "InputContext", "InputRegistry"
17 + "ParsedText",
18 + "ParsedTokens",
19 + "parse_and_batch_prompt",
20 + "TextPrompt",
21 + "TokensPrompt",
22 + "PromptInputs",
23 + "LLMInputs",
24 + "INPUT_REGISTRY",
25 + "InputContext",
26 + "InputRegistry",
27 + "get_prompt_type",
28 + "is_valid_encoder_decoder_llm_inputs",
29 + "ExplicitEncoderDecoderPrompt",
30 + "SingletonPromptInputs",
31 ]
vllm/inputs/data.py
+122 -2
@@ -92,15 +92,114 @@ class TokensPrompt(TypedDict):
92 """
93
94
95 -PromptInputs = Union[str, TextPrompt, TokensPrompt]
95 +SingletonPromptInputs = Union[str, TextPrompt, TokensPrompt]
96 """
97 -The inputs to the LLM, which can take one of the following forms:
97 +Set of possible schemas for a single LLM input:
98
99 - A text prompt (:class:`str` or :class:`TextPrompt`)
100 - A tokenized prompt (:class:`TokensPrompt`)
101 +
102 +Note that "singleton" is as opposed to a data structure
103 +which encapsulates multiple prompts, i.e. of the sort
104 +which may be utilized for encoder/decoder models when
105 +the user desires to express both the encoder & decoder
106 +prompts explicitly, i.e. ExplicitEncoderDecoderPrompt
107 +
108 +A prompt of type SingletonPromptInputs may be employed
109 +as (1) input to a decoder-only model, (2) input to
110 +the encoder of an encoder/decoder model, in the scenario
111 +where the decoder-prompt is not specified explicitly, or
112 +(3) as a member of a larger data structure encapsulating
113 +more than one prompt, i.e. ExplicitEncoderDecoderPrompt
114 """
115
116
117 +class ExplicitEncoderDecoderPrompt(TypedDict):
118 + """Represents an encoder/decoder model input prompt,
119 + comprising an explicit encoder prompt and a
120 + decoder prompt.
121 +
122 + The encoder and decoder prompts, respectively,
123 + may formatted according to any of the
124 + SingletonPromptInputs schemas, and are not
125 + required to have the same schema.
126 +
127 + Only the encoder prompt may have multi-modal data.
128 +
129 + Note that an ExplicitEncoderDecoderPrompt may not
130 + be used as an input to a decoder-only model,
131 + and that the `encoder_prompt` and `decoder_prompt`
132 + fields of this data structure may not themselves
133 + must be SingletonPromptInputs instances.
134 + """
135 +
136 + encoder_prompt: SingletonPromptInputs
137 +
138 + decoder_prompt: SingletonPromptInputs
139 +
140 +
141 +PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
142 +"""
143 +Set of possible schemas for an LLM input, including
144 +both decoder-only and encoder/decoder input types:
145 +
146 +- A text prompt (:class:`str` or :class:`TextPrompt`)
147 +- A tokenized prompt (:class:`TokensPrompt`)
148 +- A single data structure containing both an encoder and a decoder prompt
149 + (:class:`ExplicitEncoderDecoderPrompt`)
150 +"""
151 +
152 +
153 +def _has_required_keys(
154 + d: dict,
155 + required_keys: set,
156 +) -> bool:
157 + return required_keys.issubset(d.keys())
158 +
159 +
160 +def get_prompt_type(prompt: Optional[PromptInputs]) -> Optional[str]:
161 + """
162 + Get the type-name of the prompt argument instance, given that
163 + isinstance() cannot apply to TypedDict subclasses directly.
164 + If the prompt is None, return 'None' as the type name.
165 +
166 + Arguments:
167 +
168 + * prompt: LLM input prompt or None
169 +
170 + Returns:
171 +
172 + * String representation of prompt type
173 + """
174 +
175 + if prompt is None:
176 + return 'None'
177 +
178 + required_keys_dict = {
179 + 'TextPrompt': {'prompt'},
180 + 'TokensPrompt': {'prompt_token_ids'},
181 + 'ExplicitEncoderDecoder': {'encoder_prompt', 'decoder_prompt'},
182 + }
183 +
184 + if isinstance(prompt, dict):
185 + for (ptype, required_keys) in required_keys_dict.items():
186 + # Ignore type checking in the conditional below because type
187 + # checker does not understand that is_dict(prompt) narrows
188 + # down the possible types
189 + if _has_required_keys(
190 + prompt, # type: ignore
191 + required_keys):
192 + return ptype
193 +
194 + raise ValueError(f"Invalid prompt {prompt}, valid types are "
195 + "required_keys_dict={required_keys_dict}")
196 +
197 + if isinstance(prompt, str):
198 + return "str"
199 +
200 + raise ValueError(f"Invalid prompt {prompt}")
201 +
202 +
203 class LLMInputs(TypedDict):
204 """
205 The inputs in :class:`~vllm.LLMEngine` before they are
@@ -114,8 +213,29 @@ class LLMInputs(TypedDict):
213 The original prompt text corresponding to the token IDs, if available.
214 """
215
216 + encoder_prompt_token_ids: NotRequired[List[int]]
217 + """The token IDs of the encoder prompt."""
218 +
219 + encoder_prompt: NotRequired[Optional[str]]
220 + """
221 + The original encoder prompt text corresponding to the token IDs, if
222 + available.
223 + """
224 +
225 multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
226 """
227 Optional multi-modal data to pass to the model,
228 if the model supports it.
229 """
230 +
231 +
232 +def is_valid_encoder_decoder_llm_inputs(inputs: LLMInputs) -> bool:
233 + """
234 + Return True if the LLMInputs instance has the correct configuration
235 + for encoder/decoder.
236 + """
237 +
238 + # True if encoder prompt token ids field exists &
239 + # is not None
240 + return ('encoder_prompt_token_ids' in inputs
241 + and inputs['encoder_prompt_token_ids'] is not None)
vllm/model_executor/models/__init__.py
+10 -1
@@ -83,7 +83,16 @@ _EMBEDDING_MODELS = {
83 "MistralModel": ("llama_embedding", "LlamaEmbeddingModel"),
84 }
85
86 -_MODELS = {**_GENERATION_MODELS, **_EMBEDDING_MODELS}
86 +_CONDITIONAL_GENERATION_MODELS = {
87 + "BartModel": ("bart", "BartForConditionalGeneration"),
88 + "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
89 +}
90 +
91 +_MODELS = {
92 + **_GENERATION_MODELS,
93 + **_EMBEDDING_MODELS,
94 + **_CONDITIONAL_GENERATION_MODELS
95 +}
96
97 # Architecture -> type.
98 # out of tree models
vllm/model_executor/models/bart.py new
+996
@@ -0,0 +1,996 @@
1 +# Derived from BART implementation posted on HuggingFace; license below:
2 +#
3 +# coding=utf-8
4 +# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team.
5 +# All rights reserved.
6 +#
7 +# Licensed under the Apache License, Version 2.0 (the "License");
8 +# you may not use this file except in compliance with the License.
9 +# You may obtain a copy of the License at
10 +#
11 +# http://www.apache.org/licenses/LICENSE-2.0
12 +#
13 +# Unless required by applicable law or agreed to in writing, software
14 +# distributed under the License is distributed on an "AS IS" BASIS,
15 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 +# See the License for the specific language governing permissions and
17 +# limitations under the License.
18 +"""PyTorch BART model."""
19 +import math
20 +from typing import Iterable, List, Optional, Tuple
21 +
22 +import torch
23 +from torch import nn
24 +from transformers import BartConfig
25 +from transformers.utils import logging
26 +
27 +from vllm.attention import Attention, AttentionMetadata, AttentionType
28 +from vllm.config import CacheConfig, LoRAConfig
29 +from vllm.distributed import get_tensor_model_parallel_world_size
30 +from vllm.model_executor.layers.activation import get_act_fn
31 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 + QKVParallelLinear,
33 + RowParallelLinear)
34 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 +from vllm.model_executor.layers.quantization.base_config import (
36 + QuantizationConfig)
37 +from vllm.model_executor.layers.sampler import Sampler
38 +from vllm.model_executor.layers.vocab_parallel_embedding import (
39 + ParallelLMHead, VocabParallelEmbedding)
40 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 +from vllm.model_executor.sampling_metadata import SamplingMetadata
42 +from vllm.sequence import IntermediateTensors, SamplerOutput
43 +
44 +logger = logging.get_logger(__name__)
45 +
46 +
47 +def get_bsz_seq_len(input_ids):
48 + shp = input_ids.shape
49 + ndim = len(shp)
50 + if ndim == 1:
51 + return 1, input_ids.numel()
52 + else:
53 + return shp[:2]
54 +
55 +
56 +class BartLearnedPositionalEmbedding(VocabParallelEmbedding):
57 + """
58 + This module learns positional embeddings up to a fixed maximum size.
59 + """
60 +
61 + def __init__(self, num_embeddings: int, embedding_dim: int):
62 + # Bart is set up so that if padding_idx is
63 + # specified then offset the embedding ids by 2
64 + # and adjust num_embeddings appropriately.
65 + # Other models don't have this hack
66 + self.offset = 2
67 + super().__init__(num_embeddings + self.offset, embedding_dim)
68 +
69 + def forward(
70 + self,
71 + positions: torch.Tensor,
72 + attn_type: AttentionType,
73 + ) -> torch.Tensor:
74 + """`input_ids' shape is expected to be [bsz x seqlen]."""
75 +
76 + assert attn_type != AttentionType.ENCODER_DECODER
77 +
78 + return super().forward(positions + self.offset)
79 +
80 +
81 +class BartScaledWordEmbedding(VocabParallelEmbedding):
82 + """
83 + This module overrides VocabParallelEmbedding's
84 + forward by multiplying with embeddings scale.
85 + """
86 +
87 + def __init__(self,
88 + num_embeddings: int,
89 + embedding_dim: int,
90 + embed_scale: float = 1.0):
91 + super().__init__(num_embeddings, embedding_dim)
92 + self.embed_scale = embed_scale
93 +
94 + def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
95 + return super().forward(input_ids) * self.embed_scale
96 +
97 +
98 +class BartParallelLMHead(ParallelLMHead):
99 + """
100 + This module overrides ParallelLMHead's
101 + forward by dividing by embeddings scale,
102 + yielding effectively the inverse of
103 + BartScaledWordEmbedding
104 + """
105 +
106 + def __init__(self,
107 + num_embeddings: int,
108 + embedding_dim: int,
109 + embed_scale: float = 1.0):
110 + super().__init__(num_embeddings, embedding_dim)
111 + self.embed_scale = embed_scale
112 +
113 + def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
114 + return super().forward(input_ids) / self.embed_scale
115 +
116 +
117 +class BartEncoderAttention(nn.Module):
118 +
119 + def __init__(
120 + self,
121 + embed_dim: int,
122 + num_heads: int,
123 + bias: bool = True,
124 + config: Optional[BartConfig] = None,
125 + cache_config: Optional[CacheConfig] = None,
126 + quant_config: Optional[QuantizationConfig] = None,
127 + ):
128 + super().__init__()
129 + self.d_model = config.d_model
130 + self.embed_dim = embed_dim
131 + self.total_num_heads = num_heads
132 + self.total_num_kv_heads = self.total_num_heads
133 + self.head_dim = embed_dim // num_heads
134 + self.config = config
135 +
136 + if (self.head_dim * num_heads) != self.embed_dim:
137 + raise ValueError(f"embed_dim must be divisible by num_heads "
138 + f"(got `embed_dim`: {self.embed_dim}"
139 + f" and `num_heads`: {num_heads}).")
140 + self.scaling = self.head_dim**-0.5
141 +
142 + self.qkv_proj = QKVParallelLinear(
143 + self.d_model,
144 + self.d_model // self.total_num_heads,
145 + self.total_num_heads,
146 + self.total_num_kv_heads,
147 + bias=bias,
148 + quant_config=quant_config,
149 + )
150 +
151 + self.out_proj = RowParallelLinear(
152 + embed_dim,
153 + embed_dim,
154 + bias=bias,
155 + quant_config=quant_config,
156 + )
157 +
158 + tp_world_size = get_tensor_model_parallel_world_size()
159 + assert self.total_num_heads % tp_world_size == 0
160 + self.num_heads = self.total_num_heads // tp_world_size
161 +
162 + if self.total_num_kv_heads >= tp_world_size:
163 + # Number of KV heads is greater than TP size, so we partition
164 + # the KV heads across multiple tensor parallel GPUs.
165 + assert self.total_num_kv_heads % tp_world_size == 0
166 + else:
167 + # Number of KV heads is less than TP size, so we replicate
168 + # the KV heads across multiple tensor parallel GPUs.
169 + assert tp_world_size % self.total_num_kv_heads == 0
170 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
171 + self.q_size = self.num_heads * self.head_dim
172 + self.kv_size = self.num_kv_heads * self.head_dim
173 +
174 + self.attn = Attention(self.num_heads,
175 + self.head_dim,
176 + self.scaling,
177 + num_kv_heads=self.num_kv_heads,
178 + cache_config=cache_config,
179 + quant_config=quant_config)
180 +
181 + def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
182 + attn_metadata: AttentionMetadata) -> torch.Tensor:
183 + """Input shape: Batch x Time x Channel"""
184 +
185 + qkv, _ = self.qkv_proj(hidden_states)
186 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
187 +
188 + attn_output = self.attn(q,
189 + k,
190 + v,
191 + kv_cache,
192 + attn_metadata,
193 + attn_type=AttentionType.ENCODER)
194 +
195 + output, _ = self.out_proj(attn_output)
196 + return output
197 +
198 +
199 +class BartDecoderSelfAttention(nn.Module):
200 +
201 + def __init__(
202 + self,
203 + embed_dim: int,
204 + num_heads: int,
205 + bias: bool = True,
206 + config: Optional[BartConfig] = None,
207 + cache_config: Optional[CacheConfig] = None,
208 + quant_config: Optional[QuantizationConfig] = None,
209 + ):
210 + super().__init__()
211 + self.d_model = config.d_model
212 + self.embed_dim = embed_dim
213 + self.total_num_heads = num_heads
214 + self.total_num_kv_heads = self.total_num_heads
215 + self.head_dim = embed_dim // num_heads
216 + self.config = config
217 +
218 + if (self.head_dim * num_heads) != self.embed_dim:
219 + raise ValueError(f"embed_dim must be divisible by num_heads "
220 + f"(got `embed_dim`: {self.embed_dim}"
221 + f" and `num_heads`: {num_heads}).")
222 + self.scaling = self.head_dim**-0.5
223 +
224 + self.qkv_proj = QKVParallelLinear(
225 + self.d_model,
226 + self.d_model // self.total_num_heads,
227 + self.total_num_heads,
228 + self.total_num_kv_heads,
229 + bias=bias,
230 + quant_config=quant_config,
231 + )
232 +
233 + self.out_proj = RowParallelLinear(
234 + embed_dim,
235 + embed_dim,
236 + bias=bias,
237 + quant_config=quant_config,
238 + )
239 +
240 + tp_world_size = get_tensor_model_parallel_world_size()
241 + assert self.total_num_heads % tp_world_size == 0
242 + self.num_heads = self.total_num_heads // tp_world_size
243 +
244 + if self.total_num_kv_heads >= tp_world_size:
245 + # Number of KV heads is greater than TP size, so we partition
246 + # the KV heads across multiple tensor parallel GPUs.
247 + assert self.total_num_kv_heads % tp_world_size == 0
248 + else:
249 + # Number of KV heads is less than TP size, so we replicate
250 + # the KV heads across multiple tensor parallel GPUs.
251 + assert tp_world_size % self.total_num_kv_heads == 0
252 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
253 + self.q_size = self.num_heads * self.head_dim
254 + self.kv_size = self.num_kv_heads * self.head_dim
255 +
256 + self.attn = Attention(self.num_heads,
257 + self.head_dim,
258 + self.scaling,
259 + num_kv_heads=self.num_kv_heads,
260 + cache_config=cache_config,
261 + quant_config=quant_config)
262 +
263 + def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
264 + attn_metadata: AttentionMetadata) -> torch.Tensor:
265 + """Input shape: Batch x Time x Channel"""
266 +
267 + qkv, _ = self.qkv_proj(hidden_states)
268 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
269 +
270 + attn_output = self.attn(q,
271 + k,
272 + v,
273 + kv_cache,
274 + attn_metadata,
275 + attn_type=AttentionType.DECODER)
276 +
277 + output, _ = self.out_proj(attn_output)
278 + return output
279 +
280 +
281 +class BartCrossAttention(nn.Module):
282 +
283 + def __init__(
284 + self,
285 + embed_dim: int,
286 + num_heads: int,
287 + bias: bool = True,
288 + config: Optional[BartConfig] = None,
289 + cache_config: Optional[CacheConfig] = None,
290 + quant_config: Optional[QuantizationConfig] = None,
291 + ):
292 + super().__init__()
293 + self.d_model = config.d_model
294 + self.embed_dim = embed_dim
295 + self.total_num_heads = num_heads
296 + self.total_num_kv_heads = self.total_num_heads
297 + self.head_dim = embed_dim // num_heads
298 + self.config = config
299 +
300 + if (self.head_dim * num_heads) != self.embed_dim:
301 + raise ValueError(f"embed_dim must be divisible by num_heads "
302 + f"(got `embed_dim`: {self.embed_dim}"
303 + f" and `num_heads`: {num_heads}).")
304 + self.scaling = self.head_dim**-0.5
305 +
306 + self.qkv_proj = QKVParallelLinear(
307 + self.d_model,
308 + self.d_model // self.total_num_heads,
309 + self.total_num_heads,
310 + self.total_num_kv_heads,
311 + bias=bias,
312 + quant_config=quant_config,
313 + )
314 +
315 + self.out_proj = RowParallelLinear(
316 + embed_dim,
317 + embed_dim,
318 + bias=bias,
319 + quant_config=quant_config,
320 + )
321 +
322 + tp_world_size = get_tensor_model_parallel_world_size()
323 + assert self.total_num_heads % tp_world_size == 0
324 + self.num_heads = self.total_num_heads // tp_world_size
325 +
326 + if self.total_num_kv_heads >= tp_world_size:
327 + # Number of KV heads is greater than TP size, so we partition
328 + # the KV heads across multiple tensor parallel GPUs.
329 + assert self.total_num_kv_heads % tp_world_size == 0
330 + else:
331 + # Number of KV heads is less than TP size, so we replicate
332 + # the KV heads across multiple tensor parallel GPUs.
333 + assert tp_world_size % self.total_num_kv_heads == 0
334 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
335 + self.q_size = self.num_heads * self.head_dim
336 + self.kv_size = self.num_kv_heads * self.head_dim
337 +
338 + self.attn = Attention(self.num_heads,
339 + self.head_dim,
340 + self.scaling,
341 + num_kv_heads=self.num_kv_heads,
342 + cache_config=cache_config,
343 + quant_config=quant_config)
344 +
345 + def forward(
346 + self,
347 + decoder_hidden_states: torch.Tensor,
348 + kv_cache: torch.Tensor,
349 + attn_metadata: AttentionMetadata,
350 + encoder_hidden_states: Optional[torch.Tensor] = None,
351 + ) -> torch.Tensor:
352 + """Input shape: Batch x Time x Channel"""
353 +
354 + # (afeldman-nm 2024/07/22) TODO:
355 + # Need a more efficient solution for q/k/v
356 + qkv_dec, _ = self.qkv_proj(decoder_hidden_states)
357 + q, _, _ = qkv_dec.split([self.q_size, self.kv_size, self.kv_size],
358 + dim=-1)
359 + if encoder_hidden_states is None:
360 + k = None
361 + v = None
362 + else:
363 + qkv_enc, _ = self.qkv_proj(encoder_hidden_states)
364 + _, k, v = qkv_enc.split([self.q_size, self.kv_size, self.kv_size],
365 + dim=-1)
366 +
367 + attn_output = self.attn(q,
368 + k,
369 + v,
370 + kv_cache,
371 + attn_metadata,
372 + attn_type=AttentionType.ENCODER_DECODER)
373 +
374 + output, _ = self.out_proj(attn_output)
375 + return output
376 +
377 +
378 +class BartEncoderLayer(nn.Module):
379 +
380 + def __init__(
381 + self,
382 + config: BartConfig,
383 + cache_config: Optional[CacheConfig] = None,
384 + quant_config: Optional[QuantizationConfig] = None,
385 + ):
386 + super().__init__()
387 + self.embed_dim = config.d_model
388 +
389 + self.self_attn = BartEncoderAttention(
390 + embed_dim=self.embed_dim,
391 + num_heads=config.encoder_attention_heads,
392 + config=config,
393 + cache_config=cache_config,
394 + quant_config=quant_config)
395 + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
396 + self.activation_fn = get_act_fn(config.activation_function,
397 + quant_config)
398 +
399 + ffn_hidden_size = self.embed_dim
400 + ffn_intermediate_size = config.encoder_ffn_dim
401 + ffn_has_bias = True
402 + self.fc1 = ColumnParallelLinear(
403 + ffn_hidden_size,
404 + ffn_intermediate_size,
405 + bias=ffn_has_bias,
406 + quant_config=quant_config,
407 + )
408 + self.act = get_act_fn("gelu", quant_config, ffn_intermediate_size)
409 + self.fc2 = RowParallelLinear(
410 + ffn_intermediate_size,
411 + ffn_hidden_size,
412 + bias=ffn_has_bias,
413 + quant_config=quant_config,
414 + )
415 +
416 + self.final_layer_norm = nn.LayerNorm(self.embed_dim)
417 +
418 + def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
419 + attn_metadata: AttentionMetadata) -> torch.Tensor:
420 + r"""
421 + Args:
422 + hidden_states
423 + torch.Tensor of *encoder* input embeddings.
424 + kv_cache:
425 + Layer-wise list of KV cache tensors
426 + attn_metadata:
427 + vLLM Attention metadata structure
428 + Returns:
429 + Encoder layer output torch.Tensor
430 + """
431 + residual = hidden_states
432 + hidden_states = self.self_attn(hidden_states=hidden_states,
433 + kv_cache=kv_cache,
434 + attn_metadata=attn_metadata)
435 +
436 + hidden_states = residual + hidden_states
437 + hidden_states = self.self_attn_layer_norm(hidden_states)
438 +
439 + residual = hidden_states
440 + fc1_out, _ = self.fc1(hidden_states)
441 + hidden_states = self.activation_fn(fc1_out)
442 +
443 + hidden_states, _ = self.fc2(hidden_states)
444 +
445 + hidden_states = residual + hidden_states
446 + hidden_states = self.final_layer_norm(hidden_states)
447 +
448 + if hidden_states.dtype == torch.float16 and (
449 + torch.isinf(hidden_states).any()
450 + or torch.isnan(hidden_states).any()):
451 + clamp_value = torch.finfo(hidden_states.dtype).max - 1000
452 + hidden_states = torch.clamp(hidden_states,
453 + min=-clamp_value,
454 + max=clamp_value)
455 +
456 + return hidden_states
457 +
458 +
459 +class BartDecoderLayer(nn.Module):
460 +
461 + def __init__(
462 + self,
463 + config: BartConfig,
464 + cache_config: Optional[CacheConfig] = None,
465 + quant_config: Optional[QuantizationConfig] = None,
466 + ):
467 + super().__init__()
468 + self.embed_dim = config.d_model
469 +
470 + self.self_attn = BartDecoderSelfAttention(
471 + embed_dim=self.embed_dim,
472 + num_heads=config.decoder_attention_heads,
473 + config=config,
474 + cache_config=cache_config,
475 + quant_config=quant_config)
476 + self.activation_fn = get_act_fn(config.activation_function,
477 + quant_config)
478 +
479 + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
480 + '''
481 + afeldman-nm: personally I would call this "cross-attention",
482 + however I left the name as "encoder_attn" to maintain consistency
483 + with the name of the pretrained weights.
484 + '''
485 + self.encoder_attn = BartCrossAttention(
486 + self.embed_dim,
487 + config.decoder_attention_heads,
488 + config=config,
489 + )
490 + self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim)
491 +
492 + ffn_hidden_size = self.embed_dim
493 + ffn_intermediate_size = config.encoder_ffn_dim
494 + ffn_has_bias = True
495 + self.fc1 = ColumnParallelLinear(
496 + ffn_hidden_size,
497 + ffn_intermediate_size,
498 + bias=ffn_has_bias,
499 + quant_config=quant_config,
500 + )
501 + self.fc2 = RowParallelLinear(
502 + ffn_intermediate_size,
503 + ffn_hidden_size,
504 + bias=ffn_has_bias,
505 + quant_config=quant_config,
506 + )
507 +
508 + self.final_layer_norm = nn.LayerNorm(self.embed_dim)
509 +
510 + def forward(
511 + self,
512 + decoder_hidden_states: torch.Tensor,
513 + kv_cache: torch.Tensor,
514 + attn_metadata: AttentionMetadata,
515 + encoder_hidden_states: Optional[torch.Tensor] = None,
516 + ) -> torch.Tensor:
517 + r"""
518 + Args:
519 + decoder_hidden_states
520 + torch.Tensor of *decoder* input embeddings.
521 + kv_cache:
522 + KV cache tensor
523 + attn_metadata:
524 + vLLM Attention metadata structure
525 + encoder_hidden_states
526 + torch.Tensor of *encoder* input embeddings.
527 + Returns:
528 + Decoder layer output torch.Tensor
529 + """
530 + residual = decoder_hidden_states
531 +
532 + # Self Attention
533 + hidden_states = self.self_attn(hidden_states=decoder_hidden_states,
534 + kv_cache=kv_cache,
535 + attn_metadata=attn_metadata)
536 +
537 + hidden_states = residual + hidden_states
538 + hidden_states = self.self_attn_layer_norm(hidden_states)
539 +
540 + # Cross-Attention Block
541 +
542 + residual = hidden_states
543 +
544 + hidden_states = self.encoder_attn(
545 + decoder_hidden_states=hidden_states,
546 + kv_cache=kv_cache,
547 + attn_metadata=attn_metadata,
548 + encoder_hidden_states=encoder_hidden_states,
549 + )
550 +
551 + hidden_states = residual + hidden_states
552 + hidden_states = self.encoder_attn_layer_norm(hidden_states)
553 +
554 + # Fully Connected
555 + residual = hidden_states
556 + fc1_out, _ = self.fc1(hidden_states)
557 + hidden_states = self.activation_fn(fc1_out)
558 +
559 + hidden_states, _ = self.fc2(hidden_states)
560 +
561 + hidden_states = residual + hidden_states
562 + hidden_states = self.final_layer_norm(hidden_states)
563 +
564 + return hidden_states
565 +
566 +
567 +class BartEncoder(nn.Module):
568 + """
569 + Transformer encoder consisting of *config.encoder_layers*
570 + self attention layers. Each layer is a [`BartEncoderLayer`].
571 + Args:
572 + config: BartConfig
573 + embed_tokens (nn.Embedding): output embedding
574 + """
575 +
576 + def __init__(self,
577 + config: BartConfig,
578 + cache_config: Optional[CacheConfig] = None,
579 + quant_config: Optional[QuantizationConfig] = None,
580 + lora_config: Optional[LoRAConfig] = None,
581 + embed_tokens: Optional[nn.Embedding] = None):
582 + super().__init__()
583 +
584 + self.cache_config = cache_config
585 + self.quant_config = quant_config
586 + self.lora_config = lora_config
587 + embed_dim = config.d_model
588 + self.max_source_positions = config.max_position_embeddings
589 + embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
590 +
591 + self.embed_tokens = BartScaledWordEmbedding(config.vocab_size,
592 + embed_dim,
593 + embed_scale=embed_scale)
594 +
595 + if embed_tokens is not None:
596 + self.embed_tokens.weight = embed_tokens.weight
597 +
598 + self.embed_positions = BartLearnedPositionalEmbedding(
599 + config.max_position_embeddings,
600 + embed_dim,
601 + )
602 + self.layers = nn.ModuleList(
603 + [BartEncoderLayer(config,cache_config,quant_config) \
604 + for _ in range(config.encoder_layers)])
605 +
606 + self.layernorm_embedding = nn.LayerNorm(embed_dim)
607 +
608 + def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
609 + kv_caches: List[torch.Tensor],
610 + attn_metadata: AttentionMetadata) -> torch.Tensor:
611 + r"""
612 + Args:
613 + input_ids
614 + Indices of *encoder* input sequence tokens in the vocabulary.
615 + Padding will be ignored by default should you
616 + provide it.
617 + positions
618 + Positions of *encoder* input sequence tokens.
619 + kv_caches:
620 + Layer-wise list of KV cache tensors
621 + attn_metadata:
622 + vLLM Attention metadata structure
623 + Returns:
624 + Decoder output torch.Tensor
625 + """
626 + # retrieve input_ids and inputs_embeds
627 +
628 + input_ids = input_ids.view(-1, input_ids.shape[-1])
629 + inputs_embeds = self.embed_tokens(input_ids)
630 +
631 + embed_pos = self.embed_positions(
632 + positions,
633 + AttentionType.ENCODER,
634 + )
635 + embed_pos = embed_pos.to(inputs_embeds.device)
636 +
637 + hidden_states = inputs_embeds + embed_pos
638 + hidden_states = self.layernorm_embedding(hidden_states)
639 +
640 + for idx, encoder_layer in enumerate(self.layers):
641 + hidden_states = encoder_layer(
642 + hidden_states=hidden_states,
643 + kv_cache=kv_caches[idx],
644 + attn_metadata=attn_metadata,
645 + )
646 +
647 + return hidden_states
648 +
649 +
650 +class BartDecoder(nn.Module):
651 + """
652 + Transformer decoder consisting of *config.decoder_layers* layers.
653 + Each layer is a [`BartDecoderLayer`]
654 + Args:
655 + config: BartConfig
656 + embed_tokens (nn.Embedding): output embedding
657 + """
658 +
659 + def __init__(
660 + self,
661 + config: BartConfig,
662 + cache_config: Optional[CacheConfig] = None,
663 + quant_config: Optional[QuantizationConfig] = None,
664 + lora_config: Optional[LoRAConfig] = None,
665 + embed_tokens: Optional[nn.Embedding] = None,
666 + ):
667 + super().__init__()
668 + self.cache_config = cache_config
669 + self.quant_config = quant_config
670 + self.lora_config = lora_config
671 + self.max_target_positions = config.max_position_embeddings
672 + embed_scale = math.sqrt(
673 + config.d_model) if config.scale_embedding else 1.0
674 +
675 + self.embed_tokens = BartScaledWordEmbedding(config.vocab_size,
676 + config.d_model,
677 + embed_scale=embed_scale)
678 +
679 + if embed_tokens is not None:
680 + self.embed_tokens.weight = embed_tokens.weight
681 +
682 + self.embed_positions = BartLearnedPositionalEmbedding(
683 + config.max_position_embeddings,
684 + config.d_model,
685 + )
686 +
687 + self.layers = nn.ModuleList(
688 + [BartDecoderLayer(config,cache_config,quant_config) \
689 + for _ in range(config.decoder_layers)])
690 +
691 + self.layernorm_embedding = nn.LayerNorm(config.d_model)
692 +
693 + def forward(self, decoder_input_ids: torch.Tensor,
694 + decoder_positions: torch.Tensor,
695 + encoder_hidden_states: Optional[torch.Tensor],
696 + kv_caches: List[torch.Tensor],
697 + attn_metadata: AttentionMetadata) -> torch.Tensor:
698 + r"""
699 + Args:
700 + decoder_input_ids
701 + Indices of *decoder* input sequence tokens in the vocabulary.
702 + Padding will be ignored by default should you
703 + provide it.
704 + decoder_positions
705 + Positions of *decoder* input sequence tokens.
706 + encoder_hidden_states:
707 + Tensor of encoder output embeddings
708 + kv_caches:
709 + Layer-wise list of KV cache tensors
710 + attn_metadata:
711 + vLLM Attention metadata structure
712 + Returns:
713 + Decoder output torch.Tensor
714 + """
715 +
716 + inputs_embeds = self.embed_tokens(decoder_input_ids)
717 +
718 + # embed positions
719 + embed_pos = self.embed_positions(
720 + decoder_positions,
721 + AttentionType.DECODER,
722 + )
723 + embed_pos = embed_pos.to(inputs_embeds.device)
724 +
725 + hidden_states = inputs_embeds + embed_pos
726 + hidden_states = self.layernorm_embedding(hidden_states)
727 +
728 + # decoder layers
729 +
730 + for idx, decoder_layer in enumerate(self.layers):
731 + hidden_states = decoder_layer(
732 + decoder_hidden_states=hidden_states,
733 + kv_cache=kv_caches[idx],
734 + attn_metadata=attn_metadata,
735 + encoder_hidden_states=encoder_hidden_states,
736 + )
737 +
738 + return hidden_states
739 +
740 +
741 +class BartModel(nn.Module):
742 + _tied_weights_keys = [
743 + "encoder.embed_tokens.weight", "decoder.embed_tokens.weight"
744 + ]
745 +
746 + def __init__(self,
747 + config: BartConfig,
748 + cache_config: Optional[CacheConfig] = None,
749 + quant_config: Optional[QuantizationConfig] = None,
750 + lora_config: Optional[LoRAConfig] = None):
751 + super().__init__()
752 +
753 + self.config = config
754 +
755 + self.padding_idx = config.pad_token_id
756 + lora_vocab = (lora_config.lora_extra_vocab_size *
757 + (lora_config.max_loras or 1)) if lora_config else 0
758 + self.vocab_size = config.vocab_size + lora_vocab
759 + self.org_vocab_size = config.vocab_size
760 +
761 + self.encoder = BartEncoder(config,
762 + cache_config,
763 + quant_config=quant_config)
764 + self.decoder = BartDecoder(config,
765 + cache_config,
766 + quant_config=quant_config)
767 +
768 + def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
769 + encoder_input_ids: torch.Tensor,
770 + encoder_positions: torch.Tensor, kv_caches: List[torch.Tensor],
771 + attn_metadata: AttentionMetadata) -> torch.Tensor:
772 + r"""
773 + Args:
774 + input_ids
775 + Indices of *decoder* input sequence tokens in the vocabulary.
776 + Padding will be ignored by default should you
777 + provide it.
778 + positions
779 + Positions of *decoder* input sequence tokens.
780 + encoder_input_ids
781 + Indices of *encoder* input sequence tokens in the vocabulary.
782 + encoder_positions:
783 + Positions of *encoder* input sequence tokens.
784 + kv_caches:
785 + Layer-wise list of KV cache tensors
786 + attn_metadata:
787 + vLLM Attention metadata structure
788 + Returns:
789 + Model output torch.Tensor
790 + """
791 +
792 + encoder_hidden_states = None
793 +
794 + if encoder_input_ids.numel() > 0:
795 + # Run encoder attention if a non-zero number of encoder tokens
796 + # are provided as input
797 + encoder_hidden_states = self.encoder(input_ids=encoder_input_ids,
798 + positions=encoder_positions,
799 + kv_caches=kv_caches,
800 + attn_metadata=attn_metadata)
801 +
802 + # decoder outputs consists of
803 + # (dec_features, past_key_value, dec_hidden, dec_attn)
804 + decoder_outputs = self.decoder(
805 + decoder_input_ids=input_ids,
806 + decoder_positions=positions,
807 + encoder_hidden_states=encoder_hidden_states,
808 + kv_caches=kv_caches,
809 + attn_metadata=attn_metadata)
810 +
811 + return decoder_outputs
812 +
813 +
814 +class BartForConditionalGeneration(nn.Module):
815 + base_model_prefix = "model"
816 +
817 + def __init__(self,
818 + config: BartConfig,
819 + cache_config: Optional[CacheConfig] = None,
820 + quant_config: Optional[QuantizationConfig] = None,
821 + lora_config: Optional[LoRAConfig] = None):
822 +
823 + super().__init__()
824 + self.config = config
825 + self.model = BartModel(config,
826 + cache_config,
827 + quant_config,
828 + lora_config=lora_config)
829 +
830 + self.unpadded_vocab_size = config.vocab_size
831 + if lora_config:
832 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
833 +
834 + embed_scale = math.sqrt(
835 + config.d_model) if config.scale_embedding else 1.0
836 +
837 + self.lm_head = BartParallelLMHead(config.vocab_size,
838 + config.d_model,
839 + embed_scale=embed_scale)
840 +
841 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
842 + config.vocab_size)
843 + self.sampler = Sampler()
844 +
845 + def forward(
846 + self,
847 + input_ids: torch.Tensor,
848 + positions: torch.Tensor,
849 + encoder_input_ids: torch.Tensor,
850 + encoder_positions: torch.Tensor,
851 + kv_caches: List[torch.Tensor],
852 + attn_metadata: AttentionMetadata,
853 + intermediate_tensors: Optional[IntermediateTensors] = None,
854 + ) -> torch.Tensor:
855 + r"""
856 + Args:
857 + input_ids
858 + torch.Tensor of *decoder* input token ids.
859 + positions
860 + torch.Tensor of *decoder* position indices.
861 + encoder_input_ids
862 + torch.Tensor of *encoder* input token ids.
863 + encoder_positions
864 + torch.Tensor of *encoder* position indices
865 + kv_caches:
866 + Layer-wise list of KV cache tensors
867 + attn_metadata:
868 + vLLM Attention metadata structure
869 + Returns:
870 + Output torch.Tensor
871 + """
872 + return self.model(input_ids, positions, encoder_input_ids,
873 + encoder_positions, kv_caches, attn_metadata)
874 +
875 + def compute_logits(self, hidden_states: torch.Tensor,
876 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
877 + logits = self.logits_processor(self.lm_head, hidden_states,
878 + sampling_metadata)
879 + return logits
880 +
881 + def sample(
882 + self,
883 + logits: Optional[torch.Tensor],
884 + sampling_metadata: SamplingMetadata,
885 + ) -> Optional[SamplerOutput]:
886 + next_tokens = self.sampler(logits, sampling_metadata)
887 + return next_tokens
888 +
889 + stacked_params_mapping = {
890 + "q_proj": {
891 + "param_name": "qkv_proj",
892 + "shard_id": "q",
893 + },
894 + "k_proj": {
895 + "param_name": "qkv_proj",
896 + "shard_id": "k",
897 + },
898 + "v_proj": {
899 + "param_name": "qkv_proj",
900 + "shard_id": "v",
901 + },
902 + }
903 +
904 + params_mapping = {
905 + "beta": "bias",
906 + "gamma": "weight",
907 + "LayerNorm": "layernorm",
908 + }
909 +
910 + def _rename_key(self, key: str):
911 + prefix = f"{self.base_model_prefix}."
912 + key = key[len(prefix):] if key.startswith(prefix) else key
913 +
914 + for src, dst in self.params_mapping.items():
915 + key = key.replace(src, dst)
916 +
917 + return key
918 +
919 + def _rename_stacked_param(
920 + self,
921 + name: str,
922 + ) -> Tuple[str, Optional[str]]:
923 + for key, mapping in self.stacked_params_mapping.items():
924 + if key in name:
925 + name = name.replace(key, mapping["param_name"])
926 + return name, mapping["shard_id"]
927 + return name, None
928 +
929 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
930 +
931 + model_params_dict = dict(self.model.named_parameters())
932 + top_params_dict = dict(self.named_parameters())
933 +
934 + weights_tuple_list = list(weights)
935 +
936 + shared_embedding_weight = None
937 + shared_embedding_shard_id = None
938 +
939 + for name, loaded_weight in weights_tuple_list:
940 +
941 + name = self._rename_key(name)
942 + name, shard_id = self._rename_stacked_param(name)
943 +
944 + if ('shared.weight' in name
945 + or 'encoder.embed_tokens.weight' in name
946 + or 'decoder.embed_tokens.weight' in name
947 + or 'lm_head.weight' in name):
948 + assert shared_embedding_weight is None, (
949 + "Conflicting embedding weights.")
950 + shared_embedding_weight = loaded_weight
951 + shared_embedding_shard_id = shard_id
952 + else:
953 + # Skip the specific downstream task weight.
954 + if name.startswith('cls.'):
955 + continue
956 + # use Pooler instead.
957 + if name.startswith('pooler.'):
958 + continue
959 + # Skip loading extra bias for GPTQ models.
960 + if name.endswith(".bias") and name not in model_params_dict:
961 + continue
962 +
963 + param = model_params_dict[name]
964 + weight_loader = getattr(param, "weight_loader",
965 + default_weight_loader)
966 + if shard_id:
967 + weight_loader(param, loaded_weight, shard_id)
968 + else:
969 + weight_loader(param, loaded_weight)
970 +
971 + # Assign shared weight values
972 + encoder_in_param = model_params_dict['encoder.embed_tokens.weight']
973 + encoder_in_weight_loader = getattr(encoder_in_param, "weight_loader",
974 + default_weight_loader)
975 +
976 + decoder_in_param = model_params_dict['decoder.embed_tokens.weight']
977 + decoder_in_weight_loader = getattr(decoder_in_param, "weight_loader",
978 + default_weight_loader)
979 +
980 + lm_head_in_param = top_params_dict['lm_head.weight']
981 + lm_head_in_weight_loader = getattr(lm_head_in_param, "weight_loader",
982 + default_weight_loader)
983 +
984 + assert shared_embedding_weight is not None
985 +
986 + if shared_embedding_shard_id:
987 + encoder_in_weight_loader(encoder_in_param, shared_embedding_weight,
988 + shared_embedding_shard_id)
989 + decoder_in_weight_loader(decoder_in_param, shared_embedding_weight,
990 + shared_embedding_shard_id)
991 + lm_head_in_weight_loader(lm_head_in_param, shared_embedding_weight,
992 + shared_embedding_shard_id)
993 + else:
994 + encoder_in_weight_loader(encoder_in_param, shared_embedding_weight)
995 + decoder_in_weight_loader(decoder_in_param, shared_embedding_weight)
996 + lm_head_in_weight_loader(lm_head_in_param, shared_embedding_weight)
vllm/outputs.py
+19 -1
@@ -70,12 +70,20 @@ class RequestOutput:
70 Args:
71 request_id: The unique ID of the request.
72 prompt: The prompt string of the request.
73 + For encoder/decoder models, this is the
74 + decoder input prompt.
75 prompt_token_ids: The token IDs of the prompt.
76 + For encoder/decoder models, this is the
77 + decoder input prompt token ids.
78 prompt_logprobs: The log probabilities to return per prompt token.
79 outputs: The output sequences of the request.
80 finished: Whether the whole request is finished.
81 metrics: Metrics associated with the request.
82 lora_request: The LoRA request that was used to generate the output.
83 + encoder_prompt: The encoder prompt string of the request;
84 + None if decoder-only
85 + encoder_prompt_token_ids: The token IDs of the encoder prompt;
86 + None if decoder-only
87 """
88
89 def __init__(
@@ -88,6 +96,8 @@ class RequestOutput:
96 finished: bool,
97 metrics: Optional[RequestMetrics] = None,
98 lora_request: Optional[LoRARequest] = None,
99 + encoder_prompt: Optional[str] = None,
100 + encoder_prompt_token_ids: Optional[List[int]] = None,
101 ) -> None:
102 self.request_id = request_id
103 self.prompt = prompt
@@ -97,6 +107,8 @@ class RequestOutput:
107 self.finished = finished
108 self.metrics = metrics
109 self.lora_request = lora_request
110 + self.encoder_prompt = encoder_prompt
111 + self.encoder_prompt_token_ids = encoder_prompt_token_ids
112
113 @classmethod
114 def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
@@ -137,6 +149,8 @@ class RequestOutput:
149 # Every sequence in the sequence group should have the same prompt.
150 prompt = seq_group.prompt
151 prompt_token_ids = seq_group.prompt_token_ids
152 + encoder_prompt = seq_group.encoder_prompt
153 + encoder_prompt_token_ids = seq_group.encoder_prompt_token_ids
154 prompt_logprobs = seq_group.prompt_logprobs
155 finished = seq_group.is_finished()
156 finished_time = time.time() if finished else None
@@ -148,12 +162,16 @@ class RequestOutput:
162 outputs,
163 finished,
164 seq_group.metrics,
151 - lora_request=seq_group.lora_request)
165 + lora_request=seq_group.lora_request,
166 + encoder_prompt=encoder_prompt,
167 + encoder_prompt_token_ids=encoder_prompt_token_ids)
168
169 def __repr__(self) -> str:
170 return (f"RequestOutput(request_id={self.request_id}, "
171 f"prompt={self.prompt!r}, "
172 f"prompt_token_ids={self.prompt_token_ids}, "
173 + f"encoder_prompt={self.encoder_prompt!r}, "
174 + f"encoder_prompt_token_ids={self.encoder_prompt_token_ids}, "
175 f"prompt_logprobs={self.prompt_logprobs}, "
176 f"outputs={self.outputs}, "
177 f"finished={self.finished}, "
vllm/sequence.py
+95 -10
@@ -7,10 +7,11 @@ from array import array
7 from collections import defaultdict
8 from dataclasses import dataclass, field
9 from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
10 - Union)
10 + Union, cast)
11
12 import torch
13
14 +from vllm.inputs import is_valid_encoder_decoder_llm_inputs
15 from vllm.lora.request import LoRARequest
16 from vllm.pooling_params import PoolingParams
17 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -244,24 +245,38 @@ class SequenceData:
245 class Sequence:
246 """Stores the data, status, and block information of a sequence.
247
248 + The sequence is constructed from the LLMInputs instance passed
249 + in through the `inputs` constructor argument.
250 +
251 + For encoder/decoder models, LLMInputs encapsulates both a
252 + decoder and encoder prompt, creating an ambiguity about which
253 + prompt to construct the sequence from. The `from_decoder_prompt`
254 + constructor argument signals whether to construct the Sequence
255 + from the LLMInputs decoder prompt, or encoder prompt.
256 +
257 Args:
258 seq_id: The ID of the sequence.
259 inputs: The inputs of the sequence.
260 block_size: The block size of the sequence. Should be the same as the
261 block size used by the block manager and cache engine.
262 + eos_token_id: The end-of-sequence (EOS) token id recognized by this LLM.
263 lora_request: LoRA request.
264 prompt_adapter_request: Prompt Adapter request.
265 + from_decoder_prompt: Construct Sequence from LLMInputs decoder prompt
266 + (True) or encoder prompt (False.) Must be True
267 + for decoder-only model.
268
269 """
270
271 def __init__(
258 - self,
259 - seq_id: int,
260 - inputs: "LLMInputs",
261 - block_size: int,
262 - eos_token_id: Optional[int] = None,
263 - lora_request: Optional[LoRARequest] = None,
264 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
272 + self,
273 + seq_id: int,
274 + inputs: "LLMInputs",
275 + block_size: int,
276 + eos_token_id: Optional[int] = None,
277 + lora_request: Optional[LoRARequest] = None,
278 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
279 + from_decoder_prompt: bool = True,
280 ) -> None:
281 self.seq_id = seq_id
282 self.inputs = inputs
@@ -269,6 +284,36 @@ class Sequence:
284 self.eos_token_id = eos_token_id
285 self.lora_request = lora_request
286 self.prompt_adapter_request = prompt_adapter_request
287 + self.from_decoder_prompt = from_decoder_prompt
288 + self._prompt: Optional[str] = None
289 + self._prompt_token_ids: Optional[List[int]] = None
290 +
291 + # For decoder-only models, a Sequence is constructed
292 + # from an LLMInputs instance (the `inputs` arg.)
293 + #
294 + # For encoder/decoder models the same `inputs`
295 + # instance could be utilized to construct either an
296 + # encoder sequence or a decoder sequence, because
297 + # `LLMInputs` has both decoder- and encoder-oriented
298 + # member variables (i.e. it encapsulates both an encoder
299 + # and a decoder prompt.) The decision of which type of sequence
300 + # to generate is determined by the `from_decoder_prompt` argument.
301 + #
302 + # When constructing a encoder sequence
303 + # (`from_decoder_prompt` False) it matters that
304 + # the `LLMInputs` instance stored in `inputs` is valid
305 + # in the sense that its encoder-related member variables are
306 + # populated; below, an exception is raised if this is
307 + # not the case.
308 + #
309 + # When constructing a decoder sequence (`from_decoder_prompt` True)
310 + # it does not matter whether `inputs` has its encoder-related
311 + # member variables populated.
312 + if not (from_decoder_prompt
313 + or is_valid_encoder_decoder_llm_inputs(inputs)):
314 + raise ValueError("Cannot extract encoder input prompt from "
315 + f"invalid input {inputs}; did you forget the "
316 + "encoder input prompt fields?")
317
318 self.data = SequenceData(self.prompt_token_ids)
319 self.output_logprobs: SampleLogprobs = []
@@ -289,11 +334,35 @@ class Sequence:
334
335 @property
336 def prompt(self) -> Optional[str]:
292 - return self.inputs.get("prompt")
337 + if self._prompt is not None:
338 + # Reuse precomputed prompt string
339 + return self._prompt
340 +
341 + # Select decoder or encoder input prompt str,
342 + # as appropriate
343 + prompt_key: str = ("prompt"
344 + if self.from_decoder_prompt else "encoder_prompt")
345 +
346 + # Cache prompt
347 + self._prompt = cast(Optional[str], self.inputs.get(prompt_key))
348 + return self._prompt
349
350 @property
351 def prompt_token_ids(self) -> List[int]:
296 - return self.inputs["prompt_token_ids"]
352 + if self._prompt_token_ids is not None:
353 + # Reuse precomputed prompt token ids
354 + return self._prompt_token_ids
355 +
356 + # Select decoder or encoder input prompt
357 + # token ids, as appropriate
358 + prompt_token_ids_key: str = ("prompt_token_ids"
359 + if self.from_decoder_prompt else
360 + "encoder_prompt_token_ids")
361 +
362 + # Cache computed prompt token ids
363 + self._prompt_token_ids = cast(List[int],
364 + self.inputs.get(prompt_token_ids_key))
365 + return self._prompt_token_ids
366
367 @property
368 def multi_modal_data(self) -> "MultiModalDataDict":
@@ -472,6 +541,22 @@ class SequenceGroup:
541 # We use the prompt of an arbitrary sequence.
542 return self.seqs[0].prompt_token_ids
543
544 + @property
545 + def encoder_prompt(self) -> Optional[str]:
546 + # There are either 0 or 1 encoder sequences
547 + # If one is present, its prompt is distinct
548 + # from the decoder's.
549 + return (self.encoder_seq.prompt
550 + if self.encoder_seq is not None else None)
551 +
552 + @property
553 + def encoder_prompt_token_ids(self) -> Optional[List[int]]:
554 + # There are either 0 or 1 encoder sequences
555 + # If one is present, its prompt token ids are
556 + # distinct from the decoder's.
557 + return (self.encoder_seq.prompt_token_ids
558 + if self.encoder_seq is not None else None)
559 +
560 @property
561 def multi_modal_data(self) -> "MultiModalDataDict":
562 # All sequences in the group should have the same multi-modal data.
vllm/utils.py
+130
@@ -27,10 +27,93 @@ from typing_extensions import ParamSpec
27
28 import vllm.envs as envs
29 from vllm import _custom_ops as ops
30 +from vllm.inputs import (ExplicitEncoderDecoderPrompt, PromptInputs,
31 + SingletonPromptInputs)
32 from vllm.logger import enable_trace_function_call, init_logger
33
34 logger = init_logger(__name__)
35
36 +# Exception strings for non-implemented encoder/decoder scenarios
37 +
38 +STR_NOT_IMPL_ENC_DEC_SWA = \
39 + "Sliding window attention for encoder/decoder models " + \
40 + "is not currently supported."
41 +
42 +STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE = \
43 + "Prefix caching for encoder/decoder models " + \
44 + "is not currently supported."
45 +
46 +STR_NOT_IMPL_ENC_DEC_CHUNKED_PREFILL = \
47 + "Chunked prefill for encoder/decoder models " + \
48 + "is not currently supported."
49 +
50 +STR_NOT_IMPL_ENC_DEC_LOGIT_SOFTCAP = (
51 + "Models with logits_soft_cap "
52 + "require FlashInfer backend, which is "
53 + "currently not supported for encoder/decoder "
54 + "models.")
55 +
56 +STR_NOT_IMPL_ENC_DEC_LORA = ("LoRA is currently not currently "
57 + "supported with encoder/decoder "
58 + "models.")
59 +
60 +STR_NOT_IMPL_ENC_DEC_PP = ("Pipeline parallelism is not "
61 + "currently supported with "
62 + "encoder/decoder models.")
63 +
64 +STR_NOT_IMPL_ENC_DEC_MM = ("Multimodal is not currently "
65 + "supported with encoder/decoder "
66 + "models.")
67 +
68 +STR_NOT_IMPL_ENC_DEC_SPEC_DEC = ("Speculative decoding is not "
69 + "currently supported with encoder/"
70 + "decoder models.")
71 +
72 +STR_NOT_IMPL_ENC_DEC_CUDAGRAPH = ("CUDAGraph is not "
73 + "currently supported with encoder/"
74 + "decoder models.")
75 +
76 +STR_NOT_IMPL_ENC_DEC_BACKEND = ("XFormers is the only backend "
77 + "currently supported with encoder/"
78 + "decoder models.")
79 +
80 +STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER = ("Prompt adapters are not "
81 + "currently supported with encoder/"
82 + "decoder models.")
83 +
84 +# Efficiently import all enc/dec error strings
85 +# rather than having to import all of the above
86 +STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
87 + "STR_NOT_IMPL_ENC_DEC_SWA": STR_NOT_IMPL_ENC_DEC_SWA,
88 + "STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE": STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
89 + "STR_NOT_IMPL_ENC_DEC_CHUNKED_PREFILL":
90 + STR_NOT_IMPL_ENC_DEC_CHUNKED_PREFILL,
91 + "STR_NOT_IMPL_ENC_DEC_LOGIT_SOFTCAP": STR_NOT_IMPL_ENC_DEC_LOGIT_SOFTCAP,
92 + "STR_NOT_IMPL_ENC_DEC_LORA": STR_NOT_IMPL_ENC_DEC_LORA,
93 + "STR_NOT_IMPL_ENC_DEC_PP": STR_NOT_IMPL_ENC_DEC_PP,
94 + "STR_NOT_IMPL_ENC_DEC_MM": STR_NOT_IMPL_ENC_DEC_MM,
95 + "STR_NOT_IMPL_ENC_DEC_SPEC_DEC": STR_NOT_IMPL_ENC_DEC_SPEC_DEC,
96 + "STR_NOT_IMPL_ENC_DEC_CUDA_GRAPH": STR_NOT_IMPL_ENC_DEC_CUDAGRAPH,
97 + "STR_NOT_IMPL_ENC_DEC_BACKEND": STR_NOT_IMPL_ENC_DEC_BACKEND,
98 + "STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER": STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER,
99 +}
100 +
101 +# Constants related to forcing the attention backend selection
102 +
103 +# String name of register which may be set in order to
104 +# force auto-selection of attention backend by Attention
105 +# wrapper
106 +STR_BACKEND_ENV_VAR: str = "VLLM_ATTENTION_BACKEND"
107 +
108 +# Possible string values of STR_BACKEND_ENV_VAR
109 +# register, corresponding to possible backends
110 +STR_FLASHINFER_ATTN_VAL: str = "FLASHINFER"
111 +STR_TORCH_SDPA_ATTN_VAL: str = "TORCH_SDPA"
112 +STR_ROCM_FLASH_ATTN_VAL: str = "ROCM_FLASH"
113 +STR_XFORMERS_ATTN_VAL: str = "XFORMERS"
114 +STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
115 +STR_INVALID_VAL: str = "INVALID"
116 +
117 STR_DTYPE_TO_TORCH_DTYPE = {
118 "half": torch.half,
119 "bfloat16": torch.bfloat16,
@@ -1029,3 +1112,50 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1112 """Utility function to run async task in a lock"""
1113 async with lock:
1114 return await task(*args, **kwargs)
1115 +
1116 +
1117 +def is_encoder_decoder_model_config(model_config) -> bool:
1118 + '''
1119 + Extract the HF encoder/decoder model flag from the ModelConfig instance.
1120 + Return False if model_config is None.
1121 + '''
1122 + return model_config is not None and \
1123 + getattr(model_config.hf_config,
1124 + "is_encoder_decoder",
1125 + False)
1126 +
1127 +
1128 +def is_embedding_model_config(model_config) -> bool:
1129 + '''
1130 + Extract the embedding model flag from the ModelConfig instance.
1131 + Return False if model_config is None.
1132 + '''
1133 + return model_config is not None and \
1134 + model_config.embedding_mode
1135 +
1136 +
1137 +def build_explicit_enc_dec_prompt(
1138 + encoder_prompt: SingletonPromptInputs,
1139 + decoder_prompt: SingletonPromptInputs,
1140 +) -> ExplicitEncoderDecoderPrompt:
1141 + return ExplicitEncoderDecoderPrompt(encoder_prompt=encoder_prompt,
1142 + decoder_prompt=decoder_prompt)
1143 +
1144 +
1145 +def zip_enc_dec_prompt_lists(
1146 + enc_prompt_list: List[SingletonPromptInputs],
1147 + dec_prompt_list: List[SingletonPromptInputs],
1148 +) -> List[ExplicitEncoderDecoderPrompt]:
1149 + return [
1150 + build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt)
1151 + for (encoder_prompt,
1152 + decoder_prompt) in zip(enc_prompt_list, dec_prompt_list)
1153 + ]
1154 +
1155 +
1156 +def to_enc_dec_tuple_list(
1157 + enc_dec_prompts: List[ExplicitEncoderDecoderPrompt],
1158 +) -> List[Tuple[PromptInputs, PromptInputs]]:
1159 + return [(enc_dec_prompt['encoder_prompt'],
1160 + enc_dec_prompt['decoder_prompt'])
1161 + for enc_dec_prompt in enc_dec_prompts]
vllm/worker/enc_dec_model_runner.py new
+472
@@ -0,0 +1,472 @@
1 +import dataclasses
2 +from typing import Any, Dict, List, Optional, Tuple, Type, cast
3 +
4 +import torch
5 +import torch.distributed
6 +
7 +from vllm.attention.backends.abstract import (AttentionBackend,
8 + AttentionMetadata)
9 +from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
10 + get_global_forced_attn_backend,
11 + global_force_attn_backend)
12 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13 + ModelConfig, MultiModalConfig, ParallelConfig,
14 + PromptAdapterConfig, SchedulerConfig)
15 +from vllm.inputs import INPUT_REGISTRY
16 +from vllm.logger import init_logger
17 +from vllm.model_executor import SamplingMetadata
18 +from vllm.sampling_params import SamplingParams
19 +from vllm.sequence import (IntermediateTensors, PoolerOutput, SamplerOutput,
20 + SequenceGroupMetadata)
21 +from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad
22 +from vllm.worker.model_runner import (_PAD_SLOT_ID, GPUModelRunnerBase,
23 + ModelInputForGPUBuilder,
24 + ModelInputForGPUWithSamplingMetadata)
25 +from vllm.worker.model_runner_base import (
26 + _add_attn_metadata_broadcastable_dict,
27 + _add_sampling_metadata_broadcastable_dict)
28 +from vllm.worker.utils import assert_enc_dec_mr_supported_scenario
29 +
30 +logger = init_logger(__name__)
31 +
32 +
33 +@dataclasses.dataclass(frozen=True)
34 +class EncoderDecoderModelInput(ModelInputForGPUWithSamplingMetadata):
35 + """
36 + Used by the EncoderDecoderModelRunner.
37 + """
38 + encoder_input_tokens: Optional[torch.Tensor] = None
39 + encoder_input_positions: Optional[torch.Tensor] = None
40 +
41 + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
42 + tensor_dict = {
43 + "input_tokens": self.input_tokens,
44 + "input_positions": self.input_positions,
45 + "encoder_input_tokens": self.encoder_input_tokens,
46 + "encoder_input_positions": self.encoder_input_positions,
47 + "virtual_engine": self.virtual_engine,
48 + "request_ids_to_seq_ids": self.request_ids_to_seq_ids,
49 + "finished_requests_ids": self.finished_requests_ids,
50 + }
51 + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
52 + _add_sampling_metadata_broadcastable_dict(tensor_dict,
53 + self.sampling_metadata)
54 + return tensor_dict
55 +
56 + @classmethod
57 + def from_broadcasted_tensor_dict(
58 + cls,
59 + tensor_dict: Dict[str, Any],
60 + attn_backend: Optional["AttentionBackend"] = None,
61 + ) -> "EncoderDecoderModelInput":
62 + return cast(
63 + EncoderDecoderModelInput,
64 + super().from_broadcasted_tensor_dict(tensor_dict, attn_backend))
65 +
66 +
67 +class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
68 + _model_input_cls: Type[EncoderDecoderModelInput] = (
69 + EncoderDecoderModelInput)
70 + _builder_cls: Type[ModelInputForGPUBuilder] = (ModelInputForGPUBuilder)
71 +
72 + def __init__(
73 + self,
74 + model_config: ModelConfig,
75 + parallel_config: ParallelConfig,
76 + scheduler_config: SchedulerConfig,
77 + device_config: DeviceConfig,
78 + cache_config: CacheConfig,
79 + load_config: LoadConfig,
80 + lora_config: Optional[LoRAConfig],
81 + kv_cache_dtype: Optional[str] = "auto",
82 + is_driver_worker: bool = False,
83 + prompt_adapter_config: Optional[PromptAdapterConfig] = None,
84 + multimodal_config: Optional[MultiModalConfig] = None,
85 + ):
86 + '''
87 + EncoderDecoderModelRunner constructor.
88 +
89 + `lora_config`, `multimodal_config`, and prompt_adapter_config are
90 + unused (since these features are not yet supported for encoder/decoder
91 + models) but these arguments are present here for compatibility with
92 + the base-class constructor.
93 + '''
94 +
95 + self._maybe_force_supported_attention_backend()
96 +
97 + super().__init__(
98 + model_config,
99 + parallel_config,
100 + scheduler_config,
101 + device_config,
102 + cache_config,
103 + load_config,
104 + lora_config=None,
105 + kv_cache_dtype=kv_cache_dtype,
106 + is_driver_worker=is_driver_worker,
107 + )
108 +
109 + # Crash for unsupported encoder/scenarios
110 + assert_enc_dec_mr_supported_scenario(self)
111 +
112 + def _maybe_force_supported_attention_backend(self):
113 + '''
114 + Force vLLM to use the XFormers attention backend,
115 + which is currently the only supported option.
116 + '''
117 +
118 + def raise_backend_err():
119 + # The user has specified an attention backend override
120 + # which is invalid for encoder/decoder models
121 + raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_BACKEND)
122 +
123 + maybe_env_var_forced_backend = get_env_variable_attn_backend()
124 + maybe_global_forced_backend = get_global_forced_attn_backend()
125 + is_forced_by_global = maybe_global_forced_backend is not None
126 + is_forced_by_env_var = maybe_env_var_forced_backend is not None
127 +
128 + if not (is_forced_by_global or is_forced_by_env_var):
129 + # The user has not already specified an attention backend
130 + # override
131 + logger.info("EncoderDecoderModelRunner requires "
132 + "XFormers backend; overriding backend "
133 + "auto-selection and forcing XFormers.")
134 + global_force_attn_backend(_Backend.XFORMERS)
135 + elif is_forced_by_global:
136 + # Backend override enforced by global variable takes
137 + # precedence over vLLM backend environment variable.
138 + if maybe_global_forced_backend != _Backend.XFORMERS:
139 + raise_backend_err()
140 + elif is_forced_by_env_var:
141 + # Backend override enforced by vLLM backend
142 + # environment variable
143 + if maybe_env_var_forced_backend != _Backend.XFORMERS:
144 + raise_backend_err()
145 +
146 + def _list_to_int32_tensor(
147 + self,
148 + _list: List[int],
149 + ) -> torch.Tensor:
150 + return torch.tensor(_list, dtype=torch.int32, device=self.device)
151 +
152 + def _list_to_long_tensor(
153 + self,
154 + _list: List[int],
155 + ) -> torch.Tensor:
156 + return torch.tensor(_list, dtype=torch.long, device=self.device)
157 +
158 + def _empty_int32_tensor(self) -> torch.Tensor:
159 + return self._list_to_int32_tensor([])
160 +
161 + def _empty_long_tensor(self) -> torch.Tensor:
162 + return self._list_to_long_tensor([])
163 +
164 + @torch.inference_mode()
165 + def execute_model(
166 + self,
167 + model_input: EncoderDecoderModelInput,
168 + kv_caches: List[torch.Tensor],
169 + intermediate_tensors: Optional[IntermediateTensors] = None,
170 + num_steps: int = 1,
171 + ) -> Optional[List[PoolerOutput]]:
172 + if num_steps > 1:
173 + raise ValueError("num_steps > 1 is not supported in "
174 + "EncoderDecoderModelRunner")
175 +
176 + model_executable = self.model
177 +
178 + seqlen_agnostic_kwargs = {
179 + "finished_requests_ids": model_input.finished_requests_ids,
180 + "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
181 + } if self.has_seqlen_agnostic else {}
182 + hidden_or_intermediate_states = model_executable(
183 + input_ids=model_input.input_tokens,
184 + positions=model_input.input_positions,
185 + encoder_input_ids=model_input.encoder_input_tokens,
186 + encoder_positions=model_input.encoder_input_positions,
187 + kv_caches=kv_caches,
188 + attn_metadata=model_input.attn_metadata,
189 + intermediate_tensors=intermediate_tensors,
190 + **seqlen_agnostic_kwargs)
191 +
192 + logits = self.model.compute_logits(hidden_or_intermediate_states,
193 + model_input.sampling_metadata)
194 +
195 + if not self.is_driver_worker:
196 + return []
197 +
198 + # Sample the next token.
199 + output: SamplerOutput = self.model.sample(
200 + logits=logits,
201 + sampling_metadata=model_input.sampling_metadata,
202 + )
203 +
204 + return [output]
205 +
206 + def make_model_input_from_broadcasted_tensor_dict(
207 + self, tensor_dict: Dict[str, Any]) -> EncoderDecoderModelInput:
208 + return EncoderDecoderModelInput.from_broadcasted_tensor_dict(
209 + tensor_dict,
210 + attn_backend=self.attn_backend,
211 + )
212 +
213 + def prepare_model_input(
214 + self,
215 + seq_group_metadata_list: List[SequenceGroupMetadata],
216 + virtual_engine: int = 0,
217 + finished_requests_ids: Optional[List[str]] = None
218 + ) -> EncoderDecoderModelInput:
219 + """Prepare the model input based on a given sequence group, including
220 + metadata for the sampling step.
221 +
222 + Since chunked prefill is not supported for encoder/decoder models,
223 + `input_tokens` is assumed to be either entirely prefill tokens or
224 + entirely decode tokens.
225 +
226 + """
227 + model_input = self._prepare_model_input_tensors(
228 + seq_group_metadata_list, finished_requests_ids)
229 +
230 + (
231 + attn_metadata,
232 + encoder_input_tokens_tensor,
233 + encoder_input_positions_tensor,
234 + ) = (self._prepare_encoder_model_input_tensors(seq_group_metadata_list,
235 + model_input))
236 +
237 + # Inject attn_metadata encoder/cross-attention fields &
238 + # encoder input tokens/positions into model_input.
239 + # Frozen dataclass fields cannot be modified, so use
240 + # dataclasses.replace to construct a new model input
241 + # instance.
242 + model_input = dataclasses.replace(
243 + model_input,
244 + attn_metadata=attn_metadata,
245 + encoder_input_tokens=encoder_input_tokens_tensor,
246 + encoder_input_positions=encoder_input_positions_tensor,
247 + )
248 +
249 + sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
250 + model_input.seq_lens,
251 + model_input.query_lens,
252 + self.device,
253 + self.pin_memory)
254 + is_prompt = (seq_group_metadata_list[0].is_prompt
255 + if seq_group_metadata_list else None)
256 + return dataclasses.replace(model_input,
257 + sampling_metadata=sampling_metadata,
258 + is_prompt=is_prompt,
259 + virtual_engine=virtual_engine)
260 +
261 + @torch.inference_mode()
262 + def profile_run(self) -> None:
263 + # Enable top-k sampling to reflect the accurate memory usage.
264 + sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
265 + max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
266 + max_num_seqs = self.scheduler_config.max_num_seqs
267 +
268 + # Profile memory usage with max_num_sequences sequences and the total
269 + # number of tokens equal to max_num_batched_tokens.
270 + seqs: List[SequenceGroupMetadata] = []
271 +
272 + model_config = self.model_config
273 +
274 + batch_size = 0
275 + for group_id in range(max_num_seqs):
276 + seq_len = (max_num_batched_tokens // max_num_seqs +
277 + (group_id < max_num_batched_tokens % max_num_seqs))
278 + batch_size += seq_len
279 +
280 + seq_data, _ = INPUT_REGISTRY \
281 + .dummy_data_for_profiling(model_config, seq_len)
282 +
283 + # Having more tokens is over-conservative but otherwise fine
284 + assert len(seq_data.prompt_token_ids) >= seq_len, (
285 + f"Expected at least {seq_len} dummy tokens for profiling, "
286 + f"but got: {len(seq_data.prompt_token_ids)}")
287 +
288 + seq = SequenceGroupMetadata(
289 + request_id=str(group_id),
290 + is_prompt=True,
291 + seq_data={group_id: seq_data},
292 + sampling_params=sampling_params,
293 + block_tables=None,
294 + encoder_seq_data=seq_data,
295 + cross_block_table=None,
296 + )
297 + seqs.append(seq)
298 +
299 + # Run the model with the dummy inputs.
300 + num_layers = self.model_config.get_num_layers(self.parallel_config)
301 + kv_caches = [None] * num_layers
302 + finished_requests_ids = [seq.request_id for seq in seqs]
303 + model_input = self.prepare_model_input(
304 + seqs, finished_requests_ids=finished_requests_ids)
305 + intermediate_tensors = None
306 + self.execute_model(model_input, kv_caches, intermediate_tensors)
307 + torch.cuda.synchronize()
308 + return
309 +
310 + def _prepare_encoder_model_input_tensors(
311 + self,
312 + seq_group_metadata_list: List[SequenceGroupMetadata],
313 + model_input: EncoderDecoderModelInput,
314 + ) -> Tuple[AttentionMetadata, Optional[torch.Tensor],
315 + Optional[torch.Tensor]]:
316 + """Helper method to prepare the encoder- and cross-attn-related
317 + model inputs based on a given sequence group. These additional inputs
318 + are used to augment an already-computed `EncoderDecoderModelInput`
319 + data structure which already has decoder-related model inputs
320 + populated.
321 +
322 + Sets the following attn_metadata fields:
323 + * `num_encoder_tokens`
324 + * `encoder_seq_lens`
325 + * `encoder_seq_lens_tensor`
326 + * `max_encoder_seq_len`
327 + * `cross_slot_mapping`
328 + * `cross_block_tables`
329 +
330 + Constructs a new model inputs data structure, based on
331 + (1) the existing fields in the `model_inputs` argument,
332 + and (2) the following additional fields which are
333 + computed (or in the case of `attn_metadata`, updated)
334 + by this function:
335 + * attn_metadata
336 + * encoder_input_tokens
337 + * encoder_input_positions
338 +
339 + Arguments:
340 +
341 + * seq_group_metadata_list: list of sequence groups for which to
342 + compute inputs
343 + * model_inputs: model inputs data structure with decoder-oriented
344 + fields already computed.
345 +
346 + Return:
347 +
348 + * Updated model inputs data structure
349 + """
350 +
351 + if len(seq_group_metadata_list) == 0:
352 + return (model_input.attn_metadata, None, None)
353 +
354 + # Since we are not supporting chunked prefill either the entire
355 + # batch is prefill or it is decode
356 + is_prompt = seq_group_metadata_list[0].is_prompt
357 +
358 + # Build encoder inputs
359 + encoder_seq_lens: List[int] = []
360 + if is_prompt:
361 + # Prefill phase.
362 + cross_block_tables = self._empty_int32_tensor().view(
363 + len(seq_group_metadata_list), -1)
364 +
365 + # Extract input tokens/positions, cross-attention slot-mapping,
366 + # & seq len from each sequence group metadata
367 + (
368 + encoder_input_tokens,
369 + encoder_input_positions,
370 + cross_slot_mapping,
371 + ) = (
372 + [],
373 + [],
374 + [],
375 + )
376 + for seq_group_metadata in seq_group_metadata_list:
377 + # Build seq lens
378 + seq_len = seq_group_metadata.encoder_seq_data.get_len()
379 + token_ids = seq_group_metadata.encoder_seq_data.get_token_ids()
380 + encoder_seq_lens.append(seq_len)
381 +
382 + # Build slot mapping
383 + is_profile_run = (seq_group_metadata.block_tables is None)
384 + if is_profile_run:
385 + # During memory profiling, the block tables are not
386 + # initialized yet. In this case, we just use a dummy
387 + # slot mapping.
388 + # In embeddings, the block tables are {seq_id: None}.
389 + cross_slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
390 + else:
391 + for i in range(0, seq_len):
392 + block_number = seq_group_metadata.cross_block_table[
393 + i // self.block_size]
394 + block_offset = i % self.block_size
395 + slot = block_number * self.block_size + block_offset
396 + cross_slot_mapping.append(slot)
397 +
398 + # Build encoder input tokens
399 + encoder_input_tokens.extend(token_ids)
400 + encoder_input_positions.extend(list(range(0, seq_len)))
401 +
402 + # Convert tokens/positions & cross-attention
403 + # slot-mapping to encoder input tensors
404 + encoder_input_tokens_tensor = self._list_to_long_tensor(
405 + encoder_input_tokens)
406 + encoder_input_positions_tensor = self._list_to_long_tensor(
407 + encoder_input_positions)
408 + cross_slot_mapping_tensor = self._list_to_long_tensor(
409 + cross_slot_mapping)
410 +
411 + else:
412 + # Decode phase.
413 + encoder_input_tokens_tensor = self._empty_long_tensor()
414 + encoder_input_positions_tensor = self._empty_long_tensor()
415 + cross_slot_mapping_tensor = self._empty_long_tensor()
416 +
417 + # Extract cross-attention block tables &
418 + # seq len from each sequence group metadata.
419 + # Cross-attention block tables are empty
420 + # during vLLM memory profiling.
421 + cross_block_tables = []
422 + for seq_group_metadata in seq_group_metadata_list:
423 + encoder_seq_lens.append(
424 + seq_group_metadata.encoder_seq_data.get_len())
425 + cross_block_table = seq_group_metadata.cross_block_table
426 + cross_block_tables.append([] if (
427 + cross_block_table is None) else cross_block_table)
428 +
429 + # Convert cross-attention block tables to encoder input tensor
430 + cross_block_tables = make_tensor_with_pad(
431 + cross_block_tables,
432 + max_len=max(
433 + len(block_table) for block_table in cross_block_tables),
434 + pad=0,
435 + dtype=torch.int32,
436 + device=self.device,
437 + )
438 +
439 + # Compute encoder sequence lengths & encoder
440 + # sequence starting offset tensors
441 + max_encoder_seq_len = max(encoder_seq_lens, default=0)
442 + encoder_seq_lens_tensor = self._list_to_int32_tensor(encoder_seq_lens)
443 + encoder_seq_start_loc = torch.zeros(encoder_seq_lens_tensor.shape[0] +
444 + 1,
445 + dtype=torch.int32,
446 + device=self.device)
447 + torch.cumsum(encoder_seq_lens_tensor,
448 + dim=0,
449 + dtype=encoder_seq_start_loc.dtype,
450 + out=encoder_seq_start_loc[1:])
451 +
452 + # Update attention metadata with encoder-oriented attributes
453 + attn_metadata = model_input.attn_metadata
454 + assert attn_metadata is not None
455 + (
456 + attn_metadata.num_encoder_tokens,
457 + attn_metadata.encoder_seq_lens,
458 + attn_metadata.encoder_seq_lens_tensor,
459 + attn_metadata.max_encoder_seq_len,
460 + attn_metadata.cross_slot_mapping,
461 + attn_metadata.cross_block_tables,
462 + ) = (
463 + sum(encoder_seq_lens),
464 + encoder_seq_lens,
465 + encoder_seq_lens_tensor,
466 + max_encoder_seq_len,
467 + cross_slot_mapping_tensor,
468 + cross_block_tables,
469 + )
470 +
471 + return (attn_metadata, encoder_input_tokens_tensor,
472 + encoder_input_positions_tensor)
vllm/worker/utils.py new
+56
@@ -0,0 +1,56 @@
1 +'''
2 +Worker-related helper functions.
3 +'''
4 +
5 +from vllm.utils import STR_NOT_IMPL_ENC_DEC_ERR_STRS
6 +from vllm.worker.model_runner import GPUModelRunnerBase
7 +
8 +
9 +def assert_enc_dec_mr_supported_scenario(
10 + enc_dec_mr: GPUModelRunnerBase) -> None:
11 + '''
12 + Asserted that the provided encoder/decoder model runner instance reflects
13 + a supported scenario.
14 + '''
15 +
16 + if enc_dec_mr.cache_config.enable_prefix_caching:
17 + raise NotImplementedError(
18 + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE'])
19 +
20 + if enc_dec_mr.sliding_window is not None:
21 + raise NotImplementedError(
22 + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_SWA'])
23 +
24 + if enc_dec_mr.scheduler_config.chunked_prefill_enabled:
25 + raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_ERR_STRS[
26 + 'STR_NOT_IMPL_ENC_DEC_CHUNKED_PREFILL'])
27 +
28 + if getattr(enc_dec_mr.model_config.hf_config, 'attn_logit_softcapping',
29 + None) is not None:
30 + raise NotImplementedError(
31 + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_LOGIT_SOFTCAP']
32 + )
33 +
34 + if enc_dec_mr.lora_config is not None:
35 + raise NotImplementedError(
36 + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_LORA'])
37 +
38 + if enc_dec_mr.parallel_config.pipeline_parallel_size > 1:
39 + raise NotImplementedError(
40 + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PP'])
41 +
42 + if enc_dec_mr.multimodal_config is not None:
43 + raise NotImplementedError(
44 + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_MM'])
45 +
46 + if enc_dec_mr.scheduler_config.num_lookahead_slots > 0:
47 + raise NotImplementedError(
48 + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_SPEC_DEC'])
49 +
50 + if not enc_dec_mr.model_config.enforce_eager:
51 + raise NotImplementedError(
52 + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CUDA_GRAPH'])
53 +
54 + if enc_dec_mr.prompt_adapter_config is not None:
55 + raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_ERR_STRS[
56 + 'STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER'])
vllm/worker/worker.py
+12 -1
@@ -19,8 +19,11 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
19 from vllm.platforms import current_platform
20 from vllm.prompt_adapter.request import PromptAdapterRequest
21 from vllm.sequence import ExecuteModelRequest
22 +from vllm.utils import (is_embedding_model_config,
23 + is_encoder_decoder_model_config)
24 from vllm.worker.cache_engine import CacheEngine
25 from vllm.worker.embedding_model_runner import EmbeddingModelRunner
26 +from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
27 from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
28 from vllm.worker.worker_base import LocalOrDistributedWorkerBase, WorkerInput
29
@@ -85,8 +88,10 @@ class Worker(LocalOrDistributedWorkerBase):
88 ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
89 if model_runner_cls is not None:
90 ModelRunnerClass = model_runner_cls
88 - elif self.model_config.embedding_mode:
91 + elif self._is_embedding_model():
92 ModelRunnerClass = EmbeddingModelRunner
93 + elif self._is_encoder_decoder_model():
94 + ModelRunnerClass = EncoderDecoderModelRunner
95 self.model_runner: GPUModelRunnerBase = ModelRunnerClass(
96 model_config,
97 parallel_config,
@@ -107,6 +112,12 @@ class Worker(LocalOrDistributedWorkerBase):
112 # Initialize gpu_cache as embedding models don't initialize kv_caches
113 self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
114
115 + def _is_encoder_decoder_model(self):
116 + return is_encoder_decoder_model_config(self.model_config)
117 +
118 + def _is_embedding_model(self):
119 + return is_embedding_model_config(self.model_config)
120 +
121 def init_device(self) -> None:
122 if self.device_config.device.type == "cuda":
123 # torch.distributed.all_reduce does not free the input tensor until