@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4f95ffee6f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Hardware][CPU] Cross-attention and Encoder-Decoder models support on CPU backend (#9089)

Isotr0py committed Oct 7, 2024 at 14:50 UTC 4f95ffee6f40198911ee824ed06d645fe9678511
6 files changed +834 -287
.buildkite/run-cpu-test.sh
+1
@@ -23,6 +23,7 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23 # Run basic model test
24 docker exec cpu-test bash -c "
25 pip install pytest matplotlib einops transformers_stream_generator datamodel_code_generator
26 + pytest -v -s tests/models/encoder_decoder/language
27 pytest -v -s tests/models/decoder_only/language \
28 --ignore=tests/models/test_fp8.py \
29 --ignore=tests/models/decoder_only/language/test_jamba.py \
tests/models/encoder_decoder/language/test_bart.py
+211 -217
@@ -4,220 +4,214 @@ Run `pytest tests/models/encoder_decoder/language/test_bart.py`.
4 """
5 from typing import List, Optional, Tuple, Type
6
7 -from vllm.utils import is_cpu
8 -
9 -if not is_cpu():
10 - # CPU backend is not currently supported with encoder/decoder models
11 - # skip test definitions entirely to avoid importing GPU kernel libs
12 - # (xFormers, etc.)
13 -
14 - import pytest
15 - from transformers import AutoModelForSeq2SeqLM
16 -
17 - from vllm.sequence import SampleLogprobs
18 -
19 - from ....conftest import (DecoderPromptType, ExplicitEncoderDecoderPrompt,
20 - HfRunner, VllmRunner)
21 - from ....utils import multi_gpu_test
22 - from ...utils import check_logprobs_close
23 -
24 - MODELS = ["facebook/bart-base", "facebook/bart-large-cnn"]
25 -
26 - def vllm_to_hf_output(
27 - vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
28 - decoder_prompt_type: DecoderPromptType,
29 - ):
30 - """Sanitize vllm output to be comparable with hf output."""
31 - output_ids, output_str, out_logprobs = vllm_output
32 -
33 - hf_output_str = output_str + "</s>"
34 - if decoder_prompt_type == DecoderPromptType.NONE:
35 - hf_output_str = "<s>" + hf_output_str
36 -
37 - return output_ids, hf_output_str, out_logprobs
38 -
39 - def run_test(
40 - hf_runner: Type[HfRunner],
41 - vllm_runner: Type[VllmRunner],
42 - prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
43 - decoder_prompt_type: DecoderPromptType,
44 - model: str,
45 - *,
46 - dtype: str,
47 - max_tokens: int,
48 - num_logprobs: int,
49 - tensor_parallel_size: int,
50 - distributed_executor_backend: Optional[str] = None,
51 - ) -> None:
52 - '''
53 - Test the vLLM BART model for a variety of encoder/decoder input prompts,
54 - by validating it against HuggingFace (HF) BART.
55 -
56 - Arguments:
57 -
58 - * hf_runner: HuggingFace (HF) test model runner
59 - * vllm_runner: vLLM test model runner
60 - * example_encoder_decoder_prompts: test fixture which provides a
61 - dictionary of dummy prompts
62 - * model: the HF ID of the specific BART variant under test
63 - * dtype: the tensor datatype to employ
64 - * max_tokens
65 - * num_logprobs
66 - * decoder_prompt_type: key into the example_encoder_decoder_prompts
67 - dictionary; selects specific encoder/decoder
68 - prompt scenarios to test
69 -
70 - A note on using HF BART as a baseline for validating vLLM BART,
71 - specifically when the decoder prompt is None.
72 -
73 - The HF GenerationMixin's default behavior is to force the first
74 - decoded token to be <BOS> if the prompt does not already contain
75 - <BOS> (this is accomplished using a logit
76 - processor setting.)
77 -
78 - So when we use HF BART as our baseline for comparison, note that
79 - when the user provides a request with a None decoder prompt
80 - (i.e. a singleton encoder prompt, or else an explicit encoder/
81 - decoder prompt with the decoder sub-prompt set to None), HF and
82 - vLLM handle this in different ways:
83 -
84 - * HF will (1) tokenize the None prompt as an empty token-list,
85 - (2) append <decoder-start-token> to the beginning, yielding
86 - [<decoder-start-token>], (3) pass this token list to the model, and
87 - then (4) after computing logits during prefill, override the model
88 - logits & force <BOS> to be the first generated token.
89 -
90 - * vLLM will (1) tokenize the None prompt as [<BOS>], (2) append decoder-
91 - start-token to the beginning, yielding [<decoder-start-token><BOS>],
92 - (3) pass these tokens to the model & proceed with generation.
93 -
94 - The net effect is that compared to vLLM, the list of HF *decoded* tokens
95 - will contain one more initial <BOS> than the vLLM generated tokens,
96 - because vLLM's <BOS> token is injected into the prompt rather than into
97 - the generated output. This is in spite of the fact that overall, the
98 - complete sequences (prompt + decoded tokens) produced by vLLM will match
99 - HF.
100 -
101 - So when we use HF decoded token output to validate vLLM's decoded token
102 - output, the testing process must account for the difference in decoded
103 - token sequences between vLLM and HF specifically in the
104 - decoder-prompt-is-None case.
105 -
106 - One option is to disable the logit processor feature that forces the
107 - <BOS> token to be decoded (forced_bos_token_id = None), eliminating
108 - the problem entirely. However this is not "normal" BART usage.
109 -
110 - The other option is - only in the decoder-prompt-is-None case - to
111 - discard the first decoded token from the HF output before comparing it
112 - to vLLM.
113 -
114 - To that end, when testing the scenario where the decoder prompt is None
115 - (and only in that one scenario), this test skips the first HF decoded
116 - token during the process of validating the vLLM decoded output.
117 - '''
118 -
119 - # NOTE: take care of the order. run vLLM first, and then run HF.
120 - # vLLM needs a fresh new process without cuda initialization.
121 - # if we run HF first, the cuda initialization will be done and it
122 - # will hurt multiprocessing backend with fork method (the default).
123 -
124 - # Note: currently encoder/decoder models are only compatible with
125 - # enforce_eager=True. Normally this is not a problem because
126 - # for encoder/decoder models vLLM will
127 - # default to enforce_eager=True if enforce_eager
128 - # is left unspecified. However, the
129 - # VllmRunner test fixture (which wraps around the LLM class) defaults to
130 - # enforce_eager=False (a behavior which a number of already-exisitng
131 - # decoder-only unit tests expect), so when testing an encoder/decoder
132 - # model we must explicitly specify enforce_eager=True in the VllmRunner
133 - # constructor.
134 - with vllm_runner(
135 - model,
136 - dtype=dtype,
137 - tensor_parallel_size=tensor_parallel_size,
138 - distributed_executor_backend=distributed_executor_backend,
139 - enforce_eager=True) as vllm_model:
140 - vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
141 - prompts, max_tokens, num_logprobs)
142 -
143 - # Configuration settings for HF baseline
144 - hf_kwargs = {
145 - "top_k": None,
146 - "num_beams": 1,
147 - "repetition_penalty": 1.0,
148 - "top_p": 1.0,
149 - "length_penalty": 1.0,
150 - "early_stopping": False,
151 - "no_repeat_ngram_size": None,
152 - "min_length": 0
153 - }
154 -
155 - with hf_runner(model, dtype=dtype,
156 - auto_cls=AutoModelForSeq2SeqLM) as hf_model:
157 - hf_outputs = (
158 - hf_model.generate_encoder_decoder_greedy_logprobs_limit(
159 - prompts,
160 - max_tokens,
161 - num_logprobs,
162 - **hf_kwargs,
163 - ))
164 -
165 - hf_skip_tokens = (1 if decoder_prompt_type == DecoderPromptType.NONE
166 - else 0)
167 -
168 - check_logprobs_close(
169 - outputs_0_lst=hf_outputs,
170 - outputs_1_lst=[
171 - vllm_to_hf_output(vllm_output, decoder_prompt_type)
172 - for vllm_output in vllm_outputs
173 - ],
174 - name_0="hf",
175 - name_1="vllm",
176 - num_outputs_0_skip_tokens=hf_skip_tokens,
177 - )
178 -
179 - @pytest.mark.parametrize("model", MODELS)
180 - @pytest.mark.parametrize("dtype", ["float", "bfloat16"])
181 - @pytest.mark.parametrize("max_tokens", [64])
182 - @pytest.mark.parametrize("num_logprobs", [5])
183 - @pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
184 - def test_models(hf_runner, vllm_runner, example_encoder_decoder_prompts,
185 - model, dtype, max_tokens, num_logprobs,
186 - decoder_prompt_type) -> None:
187 -
188 - run_test(
189 - hf_runner,
190 - vllm_runner,
191 - example_encoder_decoder_prompts[decoder_prompt_type],
192 - decoder_prompt_type,
193 - model,
194 - dtype=dtype,
195 - max_tokens=max_tokens,
196 - num_logprobs=num_logprobs,
197 - tensor_parallel_size=1,
198 - )
199 -
200 - @multi_gpu_test(num_gpus=2)
201 - @pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
202 - @pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
203 - @pytest.mark.parametrize("dtype", ["float"])
204 - @pytest.mark.parametrize("max_tokens", [64])
205 - @pytest.mark.parametrize("num_logprobs", [5])
206 - @pytest.mark.parametrize("decoder_prompt_type", [DecoderPromptType.CUSTOM])
207 - def test_models_distributed(hf_runner, vllm_runner,
208 - example_encoder_decoder_prompts,
209 - distributed_executor_backend, model, dtype,
210 - max_tokens, num_logprobs,
211 - decoder_prompt_type) -> None:
212 - run_test(
213 - hf_runner,
214 - vllm_runner,
215 - example_encoder_decoder_prompts[decoder_prompt_type],
216 - decoder_prompt_type,
217 - model,
218 - dtype=dtype,
219 - max_tokens=max_tokens,
220 - num_logprobs=num_logprobs,
221 - tensor_parallel_size=2,
222 - distributed_executor_backend=distributed_executor_backend,
223 - )
7 +import pytest
8 +from transformers import AutoModelForSeq2SeqLM
9 +
10 +from vllm.sequence import SampleLogprobs
11 +
12 +from ....conftest import (DecoderPromptType, ExplicitEncoderDecoderPrompt,
13 + HfRunner, VllmRunner)
14 +from ....utils import multi_gpu_test
15 +from ...utils import check_logprobs_close
16 +
17 +MODELS = ["facebook/bart-base", "facebook/bart-large-cnn"]
18 +
19 +
20 +def vllm_to_hf_output(
21 + vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
22 + decoder_prompt_type: DecoderPromptType,
23 +):
24 + """Sanitize vllm output to be comparable with hf output."""
25 + output_ids, output_str, out_logprobs = vllm_output
26 +
27 + hf_output_str = output_str + "</s>"
28 + if decoder_prompt_type == DecoderPromptType.NONE:
29 + hf_output_str = "<s>" + hf_output_str
30 +
31 + return output_ids, hf_output_str, out_logprobs
32 +
33 +
34 +def run_test(
35 + hf_runner: Type[HfRunner],
36 + vllm_runner: Type[VllmRunner],
37 + prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
38 + decoder_prompt_type: DecoderPromptType,
39 + model: str,
40 + *,
41 + dtype: str,
42 + max_tokens: int,
43 + num_logprobs: int,
44 + tensor_parallel_size: int,
45 + distributed_executor_backend: Optional[str] = None,
46 +) -> None:
47 + '''
48 + Test the vLLM BART model for a variety of encoder/decoder input prompts,
49 + by validating it against HuggingFace (HF) BART.
50 +
51 + Arguments:
52 +
53 + * hf_runner: HuggingFace (HF) test model runner
54 + * vllm_runner: vLLM test model runner
55 + * example_encoder_decoder_prompts: test fixture which provides a
56 + dictionary of dummy prompts
57 + * model: the HF ID of the specific BART variant under test
58 + * dtype: the tensor datatype to employ
59 + * max_tokens
60 + * num_logprobs
61 + * decoder_prompt_type: key into the example_encoder_decoder_prompts
62 + dictionary; selects specific encoder/decoder
63 + prompt scenarios to test
64 +
65 + A note on using HF BART as a baseline for validating vLLM BART,
66 + specifically when the decoder prompt is None.
67 +
68 + The HF GenerationMixin's default behavior is to force the first
69 + decoded token to be <BOS> if the prompt does not already contain
70 + <BOS> (this is accomplished using a logit
71 + processor setting.)
72 +
73 + So when we use HF BART as our baseline for comparison, note that
74 + when the user provides a request with a None decoder prompt
75 + (i.e. a singleton encoder prompt, or else an explicit encoder/
76 + decoder prompt with the decoder sub-prompt set to None), HF and
77 + vLLM handle this in different ways:
78 +
79 + * HF will (1) tokenize the None prompt as an empty token-list,
80 + (2) append <decoder-start-token> to the beginning, yielding
81 + [<decoder-start-token>], (3) pass this token list to the model, and
82 + then (4) after computing logits during prefill, override the model
83 + logits & force <BOS> to be the first generated token.
84 +
85 + * vLLM will (1) tokenize the None prompt as [<BOS>], (2) append decoder-
86 + start-token to the beginning, yielding [<decoder-start-token><BOS>],
87 + (3) pass these tokens to the model & proceed with generation.
88 +
89 + The net effect is that compared to vLLM, the list of HF *decoded* tokens
90 + will contain one more initial <BOS> than the vLLM generated tokens,
91 + because vLLM's <BOS> token is injected into the prompt rather than into
92 + the generated output. This is in spite of the fact that overall, the
93 + complete sequences (prompt + decoded tokens) produced by vLLM will match
94 + HF.
95 +
96 + So when we use HF decoded token output to validate vLLM's decoded token
97 + output, the testing process must account for the difference in decoded
98 + token sequences between vLLM and HF specifically in the
99 + decoder-prompt-is-None case.
100 +
101 + One option is to disable the logit processor feature that forces the
102 + <BOS> token to be decoded (forced_bos_token_id = None), eliminating
103 + the problem entirely. However this is not "normal" BART usage.
104 +
105 + The other option is - only in the decoder-prompt-is-None case - to
106 + discard the first decoded token from the HF output before comparing it
107 + to vLLM.
108 +
109 + To that end, when testing the scenario where the decoder prompt is None
110 + (and only in that one scenario), this test skips the first HF decoded
111 + token during the process of validating the vLLM decoded output.
112 + '''
113 +
114 + # NOTE: take care of the order. run vLLM first, and then run HF.
115 + # vLLM needs a fresh new process without cuda initialization.
116 + # if we run HF first, the cuda initialization will be done and it
117 + # will hurt multiprocessing backend with fork method (the default).
118 +
119 + # Note: currently encoder/decoder models are only compatible with
120 + # enforce_eager=True. Normally this is not a problem because
121 + # for encoder/decoder models vLLM will
122 + # default to enforce_eager=True if enforce_eager
123 + # is left unspecified. However, the
124 + # VllmRunner test fixture (which wraps around the LLM class) defaults to
125 + # enforce_eager=False (a behavior which a number of already-exisitng
126 + # decoder-only unit tests expect), so when testing an encoder/decoder
127 + # model we must explicitly specify enforce_eager=True in the VllmRunner
128 + # constructor.
129 + with vllm_runner(model,
130 + dtype=dtype,
131 + tensor_parallel_size=tensor_parallel_size,
132 + distributed_executor_backend=distributed_executor_backend,
133 + enforce_eager=True) as vllm_model:
134 + vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
135 + prompts, max_tokens, num_logprobs)
136 +
137 + # Configuration settings for HF baseline
138 + hf_kwargs = {
139 + "top_k": None,
140 + "num_beams": 1,
141 + "repetition_penalty": 1.0,
142 + "top_p": 1.0,
143 + "length_penalty": 1.0,
144 + "early_stopping": False,
145 + "no_repeat_ngram_size": None,
146 + "min_length": 0
147 + }
148 +
149 + with hf_runner(model, dtype=dtype,
150 + auto_cls=AutoModelForSeq2SeqLM) as hf_model:
151 + hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
152 + prompts,
153 + max_tokens,
154 + num_logprobs,
155 + **hf_kwargs,
156 + ))
157 +
158 + hf_skip_tokens = (1
159 + if decoder_prompt_type == DecoderPromptType.NONE else 0)
160 +
161 + check_logprobs_close(
162 + outputs_0_lst=hf_outputs,
163 + outputs_1_lst=[
164 + vllm_to_hf_output(vllm_output, decoder_prompt_type)
165 + for vllm_output in vllm_outputs
166 + ],
167 + name_0="hf",
168 + name_1="vllm",
169 + num_outputs_0_skip_tokens=hf_skip_tokens,
170 + )
171 +
172 +
173 +@pytest.mark.parametrize("model", MODELS)
174 +@pytest.mark.parametrize("dtype", ["float", "bfloat16"])
175 +@pytest.mark.parametrize("max_tokens", [64])
176 +@pytest.mark.parametrize("num_logprobs", [5])
177 +@pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
178 +def test_models(hf_runner, vllm_runner, example_encoder_decoder_prompts, model,
179 + dtype, max_tokens, num_logprobs, decoder_prompt_type) -> None:
180 +
181 + run_test(
182 + hf_runner,
183 + vllm_runner,
184 + example_encoder_decoder_prompts[decoder_prompt_type],
185 + decoder_prompt_type,
186 + model,
187 + dtype=dtype,
188 + max_tokens=max_tokens,
189 + num_logprobs=num_logprobs,
190 + tensor_parallel_size=1,
191 + )
192 +
193 +
194 +@multi_gpu_test(num_gpus=2)
195 +@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
196 +@pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
197 +@pytest.mark.parametrize("dtype", ["float"])
198 +@pytest.mark.parametrize("max_tokens", [64])
199 +@pytest.mark.parametrize("num_logprobs", [5])
200 +@pytest.mark.parametrize("decoder_prompt_type", [DecoderPromptType.CUSTOM])
201 +def test_models_distributed(hf_runner, vllm_runner,
202 + example_encoder_decoder_prompts,
203 + distributed_executor_backend, model, dtype,
204 + max_tokens, num_logprobs,
205 + decoder_prompt_type) -> None:
206 + run_test(
207 + hf_runner,
208 + vllm_runner,
209 + example_encoder_decoder_prompts[decoder_prompt_type],
210 + decoder_prompt_type,
211 + model,
212 + dtype=dtype,
213 + max_tokens=max_tokens,
214 + num_logprobs=num_logprobs,
215 + tensor_parallel_size=2,
216 + distributed_executor_backend=distributed_executor_backend,
217 + )
vllm/attention/backends/torch_sdpa.py
+299 -61
@@ -75,6 +75,22 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
75 slot_mapping: torch.Tensor
76 seq_lens: Optional[List[int]]
77
78 + # Begin encoder attn & enc/dec cross-attn fields...
79 + # Encoder sequence lengths representation
80 + encoder_seq_lens: Optional[List[int]] = None
81 + encoder_seq_lens_tensor: Optional[torch.Tensor] = None
82 +
83 + # Maximum sequence length among encoder sequences
84 + max_encoder_seq_len: Optional[int] = None
85 +
86 + # Number of tokens input to encoder
87 + num_encoder_tokens: Optional[int] = None
88 +
89 + # Cross-attention memory-mapping data structures: slot mapping
90 + # and block tables
91 + cross_slot_mapping: Optional[torch.Tensor] = None
92 + cross_block_tables: Optional[torch.Tensor] = None
93 +
94 def __post_init__(self):
95 # Set during the execution of the first attention op.
96 # It is a list because it is needed to set per prompt
@@ -82,6 +98,28 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
98 # from xformer API.
99 # will not appear in the __repr__ and __init__
100 self.attn_bias: Optional[List[torch.Tensor]] = None
101 + self.encoder_attn_bias: Optional[List[torch.Tensor]] = None
102 + self.cross_attn_bias: Optional[List[torch.Tensor]] = None
103 +
104 + @property
105 + def is_all_encoder_attn_metadata_set(self):
106 + '''
107 + All attention metadata required for encoder attention is set.
108 + '''
109 + return ((self.encoder_seq_lens is not None)
110 + and (self.encoder_seq_lens_tensor is not None)
111 + and (self.max_encoder_seq_len is not None))
112 +
113 + @property
114 + def is_all_cross_attn_metadata_set(self):
115 + '''
116 + All attention metadata required for enc/dec cross-attention is set.
117 +
118 + Superset of encoder attention required metadata.
119 + '''
120 + return (self.is_all_encoder_attn_metadata_set
121 + and (self.cross_slot_mapping is not None)
122 + and (self.cross_block_tables is not None))
123
124 @property
125 def prefill_metadata(self) -> Optional["TorchSDPAMetadata"]:
@@ -101,6 +139,136 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
139
140 return self
141
142 + def get_seq_lens(
143 + self,
144 + attn_type: AttentionType,
145 + ):
146 + '''
147 + Extract appropriate sequence lengths from attention metadata
148 + according to attention type.
149 +
150 + Arguments:
151 +
152 + * attn_metadata: Attention metadata structure associated with attention
153 + * attn_type: encoder attention, decoder self-attention,
154 + encoder/decoder cross-attention
155 +
156 + Returns:
157 + * Appropriate sequence lengths tensor for query
158 + * Appropriate sequence lengths tensor for key & value
159 + '''
160 +
161 + if attn_type == AttentionType.DECODER:
162 + seq_lens_q = self.seq_lens
163 + seq_lens_kv = self.seq_lens
164 + elif attn_type == AttentionType.ENCODER:
165 + seq_lens_q = self.encoder_seq_lens
166 + seq_lens_kv = self.encoder_seq_lens
167 + elif attn_type == AttentionType.ENCODER_DECODER:
168 + seq_lens_q = self.seq_lens
169 + seq_lens_kv = self.encoder_seq_lens
170 + else:
171 + raise AttributeError(f"Invalid attention type {str(attn_type)}")
172 + return seq_lens_q, seq_lens_kv
173 +
174 + def get_attn_bias(
175 + self,
176 + attn_type: AttentionType,
177 + ) -> Optional[List[torch.Tensor]]:
178 + '''
179 + Extract appropriate attention bias from attention metadata
180 + according to attention type.
181 +
182 + Arguments:
183 +
184 + * attn_metadata: Attention metadata structure associated with attention
185 + * attn_type: encoder attention, decoder self-attention,
186 + encoder/decoder cross-attention
187 +
188 + Returns:
189 + * Appropriate attention bias value given the attention type
190 + '''
191 +
192 + if attn_type == AttentionType.DECODER:
193 + return self.attn_bias
194 + elif attn_type == AttentionType.ENCODER:
195 + return self.encoder_attn_bias
196 + elif attn_type == AttentionType.ENCODER_DECODER:
197 + return self.cross_attn_bias
198 + else:
199 + raise AttributeError(f"Invalid attention type {str(attn_type)}")
200 +
201 + def set_attn_bias(
202 + self,
203 + attn_bias: List[torch.Tensor],
204 + attn_type: AttentionType,
205 + ) -> None:
206 + '''
207 + Update appropriate attention bias field of attention metadata,
208 + according to attention type.
209 +
210 + Arguments:
211 +
212 + * attn_metadata: Attention metadata structure associated with attention
213 + * attn_bias: The desired attention bias value
214 + * attn_type: encoder attention, decoder self-attention,
215 + encoder/decoder cross-attention
216 + '''
217 +
218 + if attn_type == AttentionType.DECODER:
219 + self.attn_bias = attn_bias
220 + elif attn_type == AttentionType.ENCODER:
221 + self.encoder_attn_bias = attn_bias
222 + elif attn_type == AttentionType.ENCODER_DECODER:
223 + self.cross_attn_bias = attn_bias
224 + else:
225 + raise AttributeError(f"Invalid attention type {str(attn_type)}")
226 +
227 + def get_seq_len_block_table_args(
228 + self,
229 + attn_type: AttentionType,
230 + ) -> tuple:
231 + '''
232 + The particular choice of sequence-length- and block-table-related
233 + attributes which should be extracted from attn_metadata is dependent
234 + on the type of attention operation.
235 +
236 + Decoder attn -> select entirely decoder self-attention-related fields
237 + Encoder/decoder cross-attn -> select encoder sequence lengths &
238 + cross-attn block-tables fields
239 + Encoder attn -> select encoder sequence lengths fields & no block tables
240 +
241 + Arguments:
242 +
243 + * attn_metadata: Attention metadata structure associated with attention
244 + * is_prompt: True if prefill, False otherwise
245 + * attn_type: encoder attention, decoder self-attention,
246 + encoder/decoder cross-attention
247 +
248 + Returns:
249 +
250 + * Appropriate sequence-lengths tensor
251 + * Appropriate max sequence-length scalar
252 + * Appropriate block tables (or None)
253 + '''
254 +
255 + if attn_type == AttentionType.DECODER:
256 + # Decoder self-attention
257 + # Choose max_seq_len based on whether we are in prompt_run
258 + return (self.seq_lens_tensor, self.max_decode_seq_len,
259 + self.block_tables)
260 + elif attn_type == AttentionType.ENCODER_DECODER:
261 + # Enc/dec cross-attention KVs match encoder sequence length;
262 + # cross-attention utilizes special "cross" block tables
263 + return (self.encoder_seq_lens_tensor, self.max_encoder_seq_len,
264 + self.cross_block_tables)
265 + elif attn_type == AttentionType.ENCODER:
266 + # No block tables associated with encoder attention
267 + return (self.encoder_seq_lens_tensor, self.max_encoder_seq_len,
268 + None)
269 + else:
270 + raise AttributeError(f"Invalid attention type {str(attn_type)}")
271 +
272
273 class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
274
@@ -171,84 +339,101 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
339 shape = [num_tokens, num_heads * head_size]
340 """
341 assert k_scale == 1.0 and v_scale == 1.0
174 - if attn_type != AttentionType.DECODER:
175 - raise NotImplementedError("Encoder self-attention and "
176 - "encoder/decoder cross-attention "
177 - "are not implemented for "
178 - "TorchSDPABackendImpl")
179 - num_tokens, hidden_size = query.shape
342 + if (attn_type == AttentionType.ENCODER
343 + and (not attn_metadata.is_all_encoder_attn_metadata_set)):
344 + raise AttributeError("Encoder attention requires setting "
345 + "encoder metadata attributes.")
346 + elif (attn_type == AttentionType.ENCODER_DECODER
347 + and (not attn_metadata.is_all_cross_attn_metadata_set)):
348 + raise AttributeError("Encoder/decoder cross-attention "
349 + "requires setting cross-attention "
350 + "metadata attributes.")
351 +
352 # Reshape the query, key, and value tensors.
353 query = query.view(-1, self.num_heads, self.head_size)
182 - key = key.view(-1, self.num_kv_heads, self.head_size)
183 - value = value.view(-1, self.num_kv_heads, self.head_size)
184 -
185 - if kv_cache.numel() > 0:
354 + if key is not None:
355 + assert value is not None
356 + key = key.view(-1, self.num_kv_heads, self.head_size)
357 + value = value.view(-1, self.num_kv_heads, self.head_size)
358 + else:
359 + assert value is None
360 +
361 + if (attn_type != AttentionType.ENCODER and kv_cache.numel() > 0):
362 + # KV-cache during decoder-self- or
363 + # encoder-decoder-cross-attention, but not
364 + # during encoder attention.
365 + #
366 + # Even if there are no new key/value pairs to cache,
367 + # we still need to break out key_cache and value_cache
368 + # i.e. for later use by paged attention
369 key_cache, value_cache = PagedAttention.split_kv_cache(
370 kv_cache, self.num_kv_heads, self.head_size)
188 - PagedAttention.write_to_paged_cache(key, value, key_cache,
189 - value_cache,
190 - attn_metadata.slot_mapping,
191 - self.kv_cache_dtype, k_scale,
192 - v_scale)
371
194 - if attn_metadata.is_prompt:
372 + if (key is not None) and (value is not None):
373 + if attn_type == AttentionType.ENCODER_DECODER:
374 + # Update cross-attention KV cache (prefill-only)
375 + # During cross-attention decode, key & value will be None,
376 + # preventing this IF-statement branch from running
377 + updated_slot_mapping = attn_metadata.cross_slot_mapping
378 + else:
379 + # Update self-attention KV cache (prefill/decode)
380 + updated_slot_mapping = attn_metadata.slot_mapping
381 +
382 + PagedAttention.write_to_paged_cache(key, value, key_cache,
383 + value_cache,
384 + updated_slot_mapping,
385 + self.kv_cache_dtype,
386 + k_scale, v_scale)
387 +
388 + if attn_type != AttentionType.ENCODER:
389 + # Decoder self-attention supports chunked prefill.
390 + # Encoder/decoder cross-attention requires no chunked
391 + # prefill (100% prefill or 100% decode tokens, no mix)
392 + num_prefill_tokens = attn_metadata.num_prefill_tokens
393 + num_decode_tokens = attn_metadata.num_decode_tokens
394 + else:
395 + # Encoder attention - chunked prefill is not applicable;
396 + # derive token-count from query shape & and treat them
397 + # as 100% prefill tokens
398 + assert attn_metadata.num_encoder_tokens is not None
399 + num_prefill_tokens = attn_metadata.num_encoder_tokens
400 + num_decode_tokens = 0
401 +
402 + if attn_type == AttentionType.DECODER:
403 + # Only enforce this shape-constraint for decoder
404 + # self-attention
405 + assert key.shape[0] == num_prefill_tokens + num_decode_tokens
406 + assert value.shape[0] == num_prefill_tokens + num_decode_tokens
407 +
408 + if prefill_meta := attn_metadata.prefill_metadata:
409 assert attn_metadata.seq_lens is not None
410 if (kv_cache.numel() == 0
197 - or attn_metadata.block_tables.numel() == 0):
198 - if self.num_kv_heads != self.num_heads:
199 - key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
200 - value = value.repeat_interleave(self.num_queries_per_kv,
201 - dim=1)
202 -
203 - if attn_metadata.attn_bias is None:
204 - if self.alibi_slopes is not None:
205 - att_masks = _make_alibi_bias(
206 - self.alibi_slopes, query.dtype,
207 - attn_metadata.seq_lens) # type: ignore
208 - elif self.sliding_window is not None:
209 - att_masks = _make_sliding_window_bias(
210 - attn_metadata.seq_lens, self.sliding_window,
211 - query.dtype) # type: ignore
212 - else:
213 - att_masks = [None] * len(attn_metadata.seq_lens)
214 - attn_metadata.attn_bias = att_masks
215 -
216 - query = query.movedim(0, query.dim() - 2)
217 - key = key.movedim(0, key.dim() - 2)
218 - value = value.movedim(0, value.dim() - 2)
219 -
220 - start = 0
221 - output = torch.empty(
222 - (num_tokens, self.num_heads, self.head_size),
223 - dtype=query.dtype)
224 - for seq_len, mask in zip(attn_metadata.seq_lens,
225 - attn_metadata.attn_bias):
226 - end = start + seq_len
227 - sub_out = scaled_dot_product_attention(
228 - query[None, :, start:end, :],
229 - key[None, :, start:end, :],
230 - value[None, :, start:end, :],
231 - attn_mask=mask,
232 - dropout_p=0.0,
233 - is_causal=not self.need_mask,
234 - scale=self.scale).squeeze(0).movedim(
235 - query.dim() - 2, 0)
236 - output[start:end, :, :] = sub_out
237 - start = end
411 + or prefill_meta.block_tables.numel() == 0):
412 + output = self._run_sdpa_forward(query,
413 + key,
414 + value,
415 + prefill_meta,
416 + attn_type=attn_type)
417 else:
418 # prefix-enabled attention
419 raise RuntimeError(
420 "Torch SDPA backend doesn't support prefix decoding.")
421
243 - else:
422 + if decode_meta := attn_metadata.decode_metadata:
423 # Decoding run.
424 + (
425 + seq_lens_arg,
426 + max_seq_len_arg,
427 + block_tables_arg,
428 + ) = decode_meta.get_seq_len_block_table_args(attn_type)
429 +
430 output = PagedAttention.forward_decode(
431 query,
432 key_cache,
433 value_cache,
249 - attn_metadata.block_tables,
250 - attn_metadata.seq_lens_tensor,
251 - attn_metadata.max_decode_seq_len,
434 + block_tables_arg,
435 + seq_lens_arg,
436 + max_seq_len_arg,
437 self.kv_cache_dtype,
438 self.num_kv_heads,
439 self.scale,
@@ -260,6 +445,59 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
445 # Reshape the output tensor.
446 return output.view(-1, self.num_heads * self.head_size)
447
448 + def _run_sdpa_forward(
449 + self,
450 + query: torch.Tensor,
451 + key: torch.Tensor,
452 + value: torch.Tensor,
453 + attn_metadata: TorchSDPAMetadata,
454 + attn_type: AttentionType = AttentionType.DECODER,
455 + ):
456 + if self.num_kv_heads != self.num_heads:
457 + key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
458 + value = value.repeat_interleave(self.num_queries_per_kv, dim=1)
459 +
460 + attn_masks = attn_metadata.get_attn_bias(attn_type)
461 + if attn_masks is None:
462 + if self.alibi_slopes is not None:
463 + attn_masks = _make_alibi_bias(
464 + self.alibi_slopes, query.dtype,
465 + attn_metadata.seq_lens) # type: ignore
466 + elif self.sliding_window is not None:
467 + assert attn_metadata.seq_lens is not None
468 + attn_masks = _make_sliding_window_bias(
469 + attn_metadata.seq_lens, self.sliding_window,
470 + query.dtype) # type: ignore
471 + else:
472 + seq_lens, _ = attn_metadata.get_seq_lens(attn_type)
473 + attn_masks = [None] * len(seq_lens)
474 + attn_metadata.set_attn_bias(attn_masks, attn_type)
475 +
476 + output = torch.empty_like(query)
477 + query = query.movedim(0, query.dim() - 2)
478 + key = key.movedim(0, key.dim() - 2)
479 + value = value.movedim(0, value.dim() - 2)
480 +
481 + causal_attn = (attn_type == AttentionType.DECODER)
482 +
483 + seq_lens_q, seq_lens_kv = attn_metadata.get_seq_lens(attn_type)
484 + start_q, start_kv = 0, 0
485 + for seq_len_q, seq_len_kv, mask in zip(seq_lens_q, seq_lens_kv,
486 + attn_masks):
487 + end_q = start_q + seq_len_q
488 + end_kv = start_kv + seq_len_kv
489 + sub_out = scaled_dot_product_attention(
490 + query[None, :, start_q:end_q, :],
491 + key[None, :, start_kv:end_kv, :],
492 + value[None, :, start_kv:end_kv, :],
493 + attn_mask=mask,
494 + dropout_p=0.0,
495 + is_causal=causal_attn and not self.need_mask,
496 + scale=self.scale).squeeze(0).movedim(query.dim() - 2, 0)
497 + output[start_q:end_q, :, :] = sub_out
498 + start_q, start_kv = end_q, end_kv
499 + return output
500 +
501
502 def _make_alibi_bias(
503 alibi_slopes: torch.Tensor,
vllm/worker/cpu_enc_dec_model_runner.py new
+311
@@ -0,0 +1,311 @@
1 +import dataclasses
2 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
3 +
4 +import torch
5 +
6 +from vllm.attention import AttentionMetadata
7 +from vllm.model_executor.layers.sampler import SamplerOutput
8 +from vllm.multimodal import MultiModalInputs
9 +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
10 +from vllm.utils import make_tensor_with_pad
11 +from vllm.worker.cpu_model_runner import (CPUModelRunner,
12 + ModelInputForCPUBuilder,
13 + ModelInputForCPUWithSamplingMetadata)
14 +from vllm.worker.model_runner_base import (
15 + _add_attn_metadata_broadcastable_dict,
16 + _add_sampling_metadata_broadcastable_dict)
17 +
18 +if TYPE_CHECKING:
19 + from vllm.attention.backends.abstract import AttentionBackend
20 +
21 +
22 +@dataclasses.dataclass(frozen=True)
23 +class EncoderDecoderModelInputForCPU(ModelInputForCPUWithSamplingMetadata):
24 + """
25 + Used by the EncoderDecoderModelRunner.
26 + """
27 + encoder_input_tokens: Optional[torch.Tensor] = None
28 + encoder_input_positions: Optional[torch.Tensor] = None
29 +
30 + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
31 + tensor_dict = {
32 + "input_tokens": self.input_tokens,
33 + "input_positions": self.input_positions,
34 + "encoder_input_tokens": self.encoder_input_tokens,
35 + "encoder_input_positions": self.encoder_input_positions,
36 + }
37 + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
38 + _add_sampling_metadata_broadcastable_dict(tensor_dict,
39 + self.sampling_metadata)
40 + return tensor_dict
41 +
42 + @classmethod
43 + def from_broadcasted_tensor_dict(
44 + cls,
45 + tensor_dict: Dict[str, Any],
46 + attn_backend: Optional["AttentionBackend"] = None,
47 + ) -> "EncoderDecoderModelInputForCPU":
48 + return cast(
49 + EncoderDecoderModelInputForCPU,
50 + super().from_broadcasted_tensor_dict(tensor_dict, attn_backend))
51 +
52 +
53 +class CPUEncoderDecoderModelRunner(CPUModelRunner):
54 + _model_input_cls: Type[EncoderDecoderModelInputForCPU] = (
55 + EncoderDecoderModelInputForCPU)
56 + _builder_cls: Type[ModelInputForCPUBuilder] = ModelInputForCPUBuilder
57 +
58 + def _list_to_int32_tensor(
59 + self,
60 + _list: List[int],
61 + ) -> torch.Tensor:
62 + return torch.tensor(_list, dtype=torch.int32, device=self.device)
63 +
64 + def _list_to_long_tensor(
65 + self,
66 + _list: List[int],
67 + ) -> torch.Tensor:
68 + return torch.tensor(_list, dtype=torch.long, device=self.device)
69 +
70 + def _empty_int32_tensor(self) -> torch.Tensor:
71 + return self._list_to_int32_tensor([])
72 +
73 + def _empty_long_tensor(self) -> torch.Tensor:
74 + return self._list_to_long_tensor([])
75 +
76 + def make_model_input_from_broadcasted_tensor_dict(
77 + self, tensor_dict: Dict[str,
78 + Any]) -> EncoderDecoderModelInputForCPU:
79 + return EncoderDecoderModelInputForCPU.from_broadcasted_tensor_dict(
80 + tensor_dict,
81 + attn_backend=self.attn_backend,
82 + )
83 +
84 + def prepare_model_input(
85 + self,
86 + seq_group_metadata_list: List[SequenceGroupMetadata],
87 + virtual_engine: int = 0,
88 + finished_requests_ids: Optional[List[str]] = None
89 + ) -> EncoderDecoderModelInputForCPU:
90 + model_input = super().prepare_model_input(seq_group_metadata_list,
91 + virtual_engine,
92 + finished_requests_ids)
93 + model_input = cast(EncoderDecoderModelInputForCPU, model_input)
94 + (
95 + attn_metadata,
96 + encoder_input_tokens_tensor,
97 + encoder_input_positions_tensor,
98 + ) = self._prepare_encoder_model_input_tensors(seq_group_metadata_list,
99 + model_input)
100 + return dataclasses.replace(
101 + model_input,
102 + attn_metadata=attn_metadata,
103 + encoder_input_tokens=encoder_input_tokens_tensor,
104 + encoder_input_positions=encoder_input_positions_tensor,
105 + )
106 +
107 + def _prepare_encoder_model_input_tensors(
108 + self,
109 + seq_group_metadata_list: List[SequenceGroupMetadata],
110 + model_input: EncoderDecoderModelInputForCPU,
111 + ) -> Tuple[AttentionMetadata, Optional[torch.Tensor],
112 + Optional[torch.Tensor]]:
113 + """Helper method to prepare the encoder- and cross-attn-related
114 + model inputs based on a given sequence group. These additional inputs
115 + are used to augment an already-computed `EncoderDecoderModelInput`
116 + data structure which already has decoder-related model inputs
117 + populated.
118 +
119 + Sets the following attn_metadata fields:
120 + * `num_encoder_tokens`
121 + * `encoder_seq_lens`
122 + * `encoder_seq_lens_tensor`
123 + * `max_encoder_seq_len`
124 + * `cross_slot_mapping`
125 + * `cross_block_tables`
126 +
127 + Constructs a new model inputs data structure, based on
128 + (1) the existing fields in the `model_inputs` argument,
129 + and (2) the following additional fields which are
130 + computed (or in the case of `attn_metadata`, updated)
131 + by this function:
132 + * attn_metadata
133 + * encoder_input_tokens
134 + * encoder_input_positions
135 +
136 + Arguments:
137 +
138 + * seq_group_metadata_list: list of sequence groups for which to
139 + compute inputs
140 + * model_inputs: model inputs data structure with decoder-oriented
141 + fields already computed.
142 +
143 + Return:
144 +
145 + * Updated model inputs data structure
146 + """
147 +
148 + if len(seq_group_metadata_list) == 0:
149 + return (model_input.attn_metadata, None, None)
150 +
151 + # Since we are not supporting chunked prefill either the entire
152 + # batch is prefill or it is decode
153 + is_prompt = seq_group_metadata_list[0].is_prompt
154 +
155 + # Build encoder inputs
156 + encoder_seq_lens: List[int] = []
157 + if is_prompt:
158 + # Prefill phase.
159 + cross_block_tables = self._empty_int32_tensor().view(
160 + len(seq_group_metadata_list), -1)
161 +
162 + # Extract input tokens/positions, cross-attention slot-mapping,
163 + # & seq len from each sequence group metadata
164 + (
165 + encoder_input_tokens,
166 + encoder_input_positions,
167 + cross_slot_mapping,
168 + ) = (
169 + [],
170 + [],
171 + [],
172 + )
173 + for seq_group_metadata in seq_group_metadata_list:
174 + # Build seq lens
175 + seq_len = seq_group_metadata.encoder_seq_data.get_len()
176 + token_ids = seq_group_metadata.encoder_seq_data.get_token_ids()
177 + encoder_seq_lens.append(seq_len)
178 +
179 + # Build slot mapping
180 + for i in range(0, seq_len):
181 + block_number = seq_group_metadata.cross_block_table[
182 + i // self.block_size]
183 + block_offset = i % self.block_size
184 + slot = block_number * self.block_size + block_offset
185 + cross_slot_mapping.append(slot)
186 +
187 + # Build encoder input tokens
188 + encoder_input_tokens.extend(token_ids)
189 + encoder_input_positions.extend(list(range(0, seq_len)))
190 +
191 + # Convert tokens/positions & cross-attention
192 + # slot-mapping to encoder input tensors
193 + encoder_input_tokens_tensor = self._list_to_long_tensor(
194 + encoder_input_tokens)
195 + encoder_input_positions_tensor = self._list_to_long_tensor(
196 + encoder_input_positions)
197 + cross_slot_mapping_tensor = self._list_to_long_tensor(
198 + cross_slot_mapping)
199 +
200 + else:
201 + # Decode phase.
202 + encoder_input_tokens_tensor = self._empty_long_tensor()
203 + encoder_input_positions_tensor = self._empty_long_tensor()
204 + cross_slot_mapping_tensor = self._empty_long_tensor()
205 + # Extract cross-attention block tables &
206 + # seq len from each sequence group metadata.
207 + # Cross-attention block tables are empty
208 + # during vLLM memory profiling.
209 + cross_block_tables = []
210 + for seq_group_metadata in seq_group_metadata_list:
211 + for _ in range(len(seq_group_metadata.seq_data)):
212 + encoder_seq_lens.append(
213 + seq_group_metadata.encoder_seq_data.get_len())
214 + cross_block_table = seq_group_metadata.cross_block_table
215 + cross_block_tables.append([] if (
216 + cross_block_table is None) else cross_block_table)
217 +
218 + max_len_of_block_table = max(
219 + len(block_table) for block_table in cross_block_tables)
220 +
221 + cross_block_tables = make_tensor_with_pad(
222 + cross_block_tables,
223 + max_len=max_len_of_block_table,
224 + pad=0,
225 + dtype=torch.int32,
226 + device=self.device,
227 + )
228 +
229 + # Compute encoder sequence lengths & encoder
230 + # sequence starting offset tensors
231 + max_encoder_seq_len = max(encoder_seq_lens, default=0)
232 + encoder_seq_lens_tensor = self._list_to_int32_tensor(encoder_seq_lens)
233 + encoder_seq_start_loc = torch.zeros(encoder_seq_lens_tensor.shape[0] +
234 + 1,
235 + dtype=torch.int32,
236 + device=self.device)
237 + torch.cumsum(encoder_seq_lens_tensor,
238 + dim=0,
239 + dtype=encoder_seq_start_loc.dtype,
240 + out=encoder_seq_start_loc[1:])
241 +
242 + # Update attention metadata with encoder-oriented attributes
243 + attn_metadata = model_input.attn_metadata
244 + assert attn_metadata is not None
245 + (
246 + attn_metadata.num_encoder_tokens,
247 + attn_metadata.encoder_seq_lens,
248 + attn_metadata.encoder_seq_lens_tensor,
249 + attn_metadata.max_encoder_seq_len,
250 + attn_metadata.cross_slot_mapping,
251 + attn_metadata.cross_block_tables,
252 + ) = (
253 + sum(encoder_seq_lens),
254 + encoder_seq_lens,
255 + encoder_seq_lens_tensor,
256 + max_encoder_seq_len,
257 + cross_slot_mapping_tensor,
258 + cross_block_tables,
259 + )
260 +
261 + return (attn_metadata, encoder_input_tokens_tensor,
262 + encoder_input_positions_tensor)
263 +
264 + @torch.no_grad()
265 + def execute_model(
266 + self,
267 + model_input: EncoderDecoderModelInputForCPU,
268 + kv_caches: List[torch.Tensor],
269 + intermediate_tensors: Optional[IntermediateTensors] = None,
270 + num_steps: int = 1,
271 + ) -> Optional[List[SamplerOutput]]:
272 + if num_steps > 1:
273 + raise ValueError(
274 + "CPU worker does not support multi-step execution.")
275 +
276 + model_executable = self.model
277 + execute_model_kwargs = {
278 + "input_ids":
279 + model_input.input_tokens,
280 + "positions":
281 + model_input.input_positions,
282 + "encoder_input_ids":
283 + model_input.encoder_input_tokens,
284 + "encoder_positions":
285 + model_input.encoder_input_positions,
286 + "kv_caches":
287 + kv_caches,
288 + "attn_metadata":
289 + model_input.attn_metadata,
290 + **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
291 + device=self.device),
292 + "intermediate_tensors":
293 + intermediate_tensors,
294 + }
295 +
296 + hidden_states = model_executable(**execute_model_kwargs)
297 +
298 + # Compute the logits.
299 + logits = self.model.compute_logits(hidden_states,
300 + model_input.sampling_metadata)
301 +
302 + # Only perform sampling in the driver worker.
303 + if not self.is_driver_worker:
304 + return []
305 +
306 + # Sample the next token.
307 + output = self.model.sample(
308 + logits=logits,
309 + sampling_metadata=model_input.sampling_metadata,
310 + )
311 + return [output]
vllm/worker/cpu_model_runner.py
+3 -7
@@ -19,7 +19,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19 MultiModalInputs)
20 from vllm.sequence import (IntermediateTensors, SequenceData,
21 SequenceGroupMetadata)
22 -from vllm.utils import STR_NOT_IMPL_ENC_DEC_ERR_STRS, make_tensor_with_pad
22 +from vllm.utils import make_tensor_with_pad
23 from vllm.worker.model_runner_base import (
24 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
25 _add_attn_metadata_broadcastable_dict,
@@ -434,10 +434,6 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
434 # Lazy initialization.
435 self.model: nn.Module # Set after init_Model
436
437 - if self.model_config.is_encoder_decoder_model:
438 - raise NotImplementedError(
439 - STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CPU'])
440 -
437 @property
438 def model_is_mrope(self) -> bool:
439 """Detect if the model has "mrope" rope_scaling type.
@@ -459,8 +455,8 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
455 def make_model_input_from_broadcasted_tensor_dict(
456 self,
457 tensor_dict: Dict[str, Any],
462 - ) -> ModelInputForCPU:
463 - return ModelInputForCPU.from_broadcasted_tensor_dict(
458 + ) -> ModelInputForCPUWithSamplingMetadata:
459 + return ModelInputForCPUWithSamplingMetadata.from_broadcasted_tensor_dict( # noqa: E501
460 tensor_dict,
461 attn_backend=self.attn_backend,
462 )
vllm/worker/cpu_worker.py
+9 -2
@@ -1,5 +1,5 @@
1 """A CPU worker class."""
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import Dict, List, Optional, Tuple, Type
3
4 import torch
5 import torch.distributed
@@ -15,6 +15,7 @@ from vllm.logger import init_logger
15 from vllm.model_executor import set_random_seed
16 from vllm.sequence import ExecuteModelRequest
17 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18 +from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
19 from vllm.worker.cpu_model_runner import CPUModelRunner
20 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
21 LoraNotSupportedWorkerBase, WorkerInput)
@@ -163,7 +164,10 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
164 else:
165 self.local_omp_cpuid = omp_cpuids.split("|")[rank]
166
166 - self.model_runner: CPUModelRunner = CPUModelRunner(
167 + ModelRunnerClass: Type[CPUModelRunner] = CPUModelRunner
168 + if self._is_encoder_decoder_model():
169 + ModelRunnerClass = CPUEncoderDecoderModelRunner
170 + self.model_runner: CPUModelRunner = ModelRunnerClass(
171 model_config,
172 parallel_config,
173 scheduler_config,
@@ -205,6 +209,9 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
209 raise RuntimeError("Profiler is not enabled.")
210 self.profiler.stop()
211
212 + def _is_encoder_decoder_model(self):
213 + return self.model_config.is_encoder_decoder_model
214 +
215 def init_device(self) -> None:
216 if self.local_omp_cpuid != "all":
217 ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)