A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Hardware][CPU] Cross-attention and Encoder-Decoder models support on CPU backend (#9089)
Isotr0py committed
Oct 7, 2024 at 14:50 UTC
4f95ffee6f40198911ee824ed06d645fe9678511
6 files changed
+834
-287
.buildkite/run-cpu-test.sh
+1
@@ -23,6 +23,7 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23
# Run basic model test
24
docker exec cpu-test bash -c "
25
pip install pytest matplotlib einops transformers_stream_generator datamodel_code_generator
26
+ pytest -v -s tests/models/encoder_decoder/language
27
pytest -v -s tests/models/decoder_only/language \
28
--ignore=tests/models/test_fp8.py \
29
--ignore=tests/models/decoder_only/language/test_jamba.py \
tests/models/encoder_decoder/language/test_bart.py
+211
-217
@@ -4,220 +4,214 @@ Run `pytest tests/models/encoder_decoder/language/test_bart.py`.
4
"""
5
from typing import List, Optional, Tuple, Type
6
7
-from vllm.utils import is_cpu
8
-
9
-if not is_cpu():
10
- # CPU backend is not currently supported with encoder/decoder models
11
- # skip test definitions entirely to avoid importing GPU kernel libs
12
- # (xFormers, etc.)
13
-
14
- import pytest
15
- from transformers import AutoModelForSeq2SeqLM
16
-
17
- from vllm.sequence import SampleLogprobs
18
-
19
- from ....conftest import (DecoderPromptType, ExplicitEncoderDecoderPrompt,
20
- HfRunner, VllmRunner)
21
- from ....utils import multi_gpu_test
22
- from ...utils import check_logprobs_close
23
-
24
- MODELS = ["facebook/bart-base", "facebook/bart-large-cnn"]
25
-
26
- def vllm_to_hf_output(
27
- vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
28
- decoder_prompt_type: DecoderPromptType,
29
- ):
30
- """Sanitize vllm output to be comparable with hf output."""
31
- output_ids, output_str, out_logprobs = vllm_output
32
-
33
- hf_output_str = output_str + "</s>"
34
- if decoder_prompt_type == DecoderPromptType.NONE:
35
- hf_output_str = "<s>" + hf_output_str
36
-
37
- return output_ids, hf_output_str, out_logprobs
38
-
39
- def run_test(
40
- hf_runner: Type[HfRunner],
41
- vllm_runner: Type[VllmRunner],
42
- prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
43
- decoder_prompt_type: DecoderPromptType,
44
- model: str,
45
- *,
46
- dtype: str,
47
- max_tokens: int,
48
- num_logprobs: int,
49
- tensor_parallel_size: int,
50
- distributed_executor_backend: Optional[str] = None,
51
- ) -> None:
52
- '''
53
- Test the vLLM BART model for a variety of encoder/decoder input prompts,
54
- by validating it against HuggingFace (HF) BART.
55
-
56
- Arguments:
57
-
58
- * hf_runner: HuggingFace (HF) test model runner
59
- * vllm_runner: vLLM test model runner
60
- * example_encoder_decoder_prompts: test fixture which provides a
61
- dictionary of dummy prompts
62
- * model: the HF ID of the specific BART variant under test
63
- * dtype: the tensor datatype to employ
64
- * max_tokens
65
- * num_logprobs
66
- * decoder_prompt_type: key into the example_encoder_decoder_prompts
67
- dictionary; selects specific encoder/decoder
68
- prompt scenarios to test
69
-
70
- A note on using HF BART as a baseline for validating vLLM BART,
71
- specifically when the decoder prompt is None.
72
-
73
- The HF GenerationMixin's default behavior is to force the first
74
- decoded token to be <BOS> if the prompt does not already contain
75
- <BOS> (this is accomplished using a logit
76
- processor setting.)
77
-
78
- So when we use HF BART as our baseline for comparison, note that
79
- when the user provides a request with a None decoder prompt
80
- (i.e. a singleton encoder prompt, or else an explicit encoder/
81
- decoder prompt with the decoder sub-prompt set to None), HF and
82
- vLLM handle this in different ways:
83
-
84
- * HF will (1) tokenize the None prompt as an empty token-list,
85
- (2) append <decoder-start-token> to the beginning, yielding
86
- [<decoder-start-token>], (3) pass this token list to the model, and
87
- then (4) after computing logits during prefill, override the model
88
- logits & force <BOS> to be the first generated token.
89
-
90
- * vLLM will (1) tokenize the None prompt as [<BOS>], (2) append decoder-
91
- start-token to the beginning, yielding [<decoder-start-token><BOS>],
92
- (3) pass these tokens to the model & proceed with generation.
93
-
94
- The net effect is that compared to vLLM, the list of HF *decoded* tokens
95
- will contain one more initial <BOS> than the vLLM generated tokens,
96
- because vLLM's <BOS> token is injected into the prompt rather than into
97
- the generated output. This is in spite of the fact that overall, the
98
- complete sequences (prompt + decoded tokens) produced by vLLM will match
99
- HF.
100
-
101
- So when we use HF decoded token output to validate vLLM's decoded token
102
- output, the testing process must account for the difference in decoded
103
- token sequences between vLLM and HF specifically in the
104
- decoder-prompt-is-None case.
105
-
106
- One option is to disable the logit processor feature that forces the
107
- <BOS> token to be decoded (forced_bos_token_id = None), eliminating
108
- the problem entirely. However this is not "normal" BART usage.
109
-
110
- The other option is - only in the decoder-prompt-is-None case - to
111
- discard the first decoded token from the HF output before comparing it
112
- to vLLM.
113
-
114
- To that end, when testing the scenario where the decoder prompt is None
115
- (and only in that one scenario), this test skips the first HF decoded
116
- token during the process of validating the vLLM decoded output.
117
- '''
118
-
119
- # NOTE: take care of the order. run vLLM first, and then run HF.
120
- # vLLM needs a fresh new process without cuda initialization.
121
- # if we run HF first, the cuda initialization will be done and it
122
- # will hurt multiprocessing backend with fork method (the default).
123
-
124
- # Note: currently encoder/decoder models are only compatible with
125
- # enforce_eager=True. Normally this is not a problem because
126
- # for encoder/decoder models vLLM will
127
- # default to enforce_eager=True if enforce_eager
128
- # is left unspecified. However, the
129
- # VllmRunner test fixture (which wraps around the LLM class) defaults to
130
- # enforce_eager=False (a behavior which a number of already-exisitng
131
- # decoder-only unit tests expect), so when testing an encoder/decoder
132
- # model we must explicitly specify enforce_eager=True in the VllmRunner
133
- # constructor.
134
- with vllm_runner(
135
- model,
136
- dtype=dtype,
137
- tensor_parallel_size=tensor_parallel_size,
138
- distributed_executor_backend=distributed_executor_backend,
139
- enforce_eager=True) as vllm_model:
140
- vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
141
- prompts, max_tokens, num_logprobs)
142
-
143
- # Configuration settings for HF baseline
144
- hf_kwargs = {
145
- "top_k": None,
146
- "num_beams": 1,
147
- "repetition_penalty": 1.0,
148
- "top_p": 1.0,
149
- "length_penalty": 1.0,
150
- "early_stopping": False,
151
- "no_repeat_ngram_size": None,
152
- "min_length": 0
153
- }
154
-
155
- with hf_runner(model, dtype=dtype,
156
- auto_cls=AutoModelForSeq2SeqLM) as hf_model:
157
- hf_outputs = (
158
- hf_model.generate_encoder_decoder_greedy_logprobs_limit(
159
- prompts,
160
- max_tokens,
161
- num_logprobs,
162
- **hf_kwargs,
163
- ))
164
-
165
- hf_skip_tokens = (1 if decoder_prompt_type == DecoderPromptType.NONE
166
- else 0)
167
-
168
- check_logprobs_close(
169
- outputs_0_lst=hf_outputs,
170
- outputs_1_lst=[
171
- vllm_to_hf_output(vllm_output, decoder_prompt_type)
172
- for vllm_output in vllm_outputs
173
- ],
174
- name_0="hf",
175
- name_1="vllm",
176
- num_outputs_0_skip_tokens=hf_skip_tokens,
177
- )
178
-
179
- @pytest.mark.parametrize("model", MODELS)
180
- @pytest.mark.parametrize("dtype", ["float", "bfloat16"])
181
- @pytest.mark.parametrize("max_tokens", [64])
182
- @pytest.mark.parametrize("num_logprobs", [5])
183
- @pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
184
- def test_models(hf_runner, vllm_runner, example_encoder_decoder_prompts,
185
- model, dtype, max_tokens, num_logprobs,
186
- decoder_prompt_type) -> None:
187
-
188
- run_test(
189
- hf_runner,
190
- vllm_runner,
191
- example_encoder_decoder_prompts[decoder_prompt_type],
192
- decoder_prompt_type,
193
- model,
194
- dtype=dtype,
195
- max_tokens=max_tokens,
196
- num_logprobs=num_logprobs,
197
- tensor_parallel_size=1,
198
- )
199
-
200
- @multi_gpu_test(num_gpus=2)
201
- @pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
202
- @pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
203
- @pytest.mark.parametrize("dtype", ["float"])
204
- @pytest.mark.parametrize("max_tokens", [64])
205
- @pytest.mark.parametrize("num_logprobs", [5])
206
- @pytest.mark.parametrize("decoder_prompt_type", [DecoderPromptType.CUSTOM])
207
- def test_models_distributed(hf_runner, vllm_runner,
208
- example_encoder_decoder_prompts,
209
- distributed_executor_backend, model, dtype,
210
- max_tokens, num_logprobs,
211
- decoder_prompt_type) -> None:
212
- run_test(
213
- hf_runner,
214
- vllm_runner,
215
- example_encoder_decoder_prompts[decoder_prompt_type],
216
- decoder_prompt_type,
217
- model,
218
- dtype=dtype,
219
- max_tokens=max_tokens,
220
- num_logprobs=num_logprobs,
221
- tensor_parallel_size=2,
222
- distributed_executor_backend=distributed_executor_backend,
223
- )
7
+import pytest
8
+from transformers import AutoModelForSeq2SeqLM
9
+
10
+from vllm.sequence import SampleLogprobs
11
+
12
+from ....conftest import (DecoderPromptType, ExplicitEncoderDecoderPrompt,
13
+ HfRunner, VllmRunner)
14
+from ....utils import multi_gpu_test
15
+from ...utils import check_logprobs_close
16
+
17
+MODELS = ["facebook/bart-base", "facebook/bart-large-cnn"]
18
+
19
+
20
+def vllm_to_hf_output(
21
+ vllm_output: Tuple[List[int], str, Optional[SampleLogprobs]],
22
+ decoder_prompt_type: DecoderPromptType,
23
+):
24
+ """Sanitize vllm output to be comparable with hf output."""
25
+ output_ids, output_str, out_logprobs = vllm_output
26
+
27
+ hf_output_str = output_str + "</s>"
28
+ if decoder_prompt_type == DecoderPromptType.NONE:
29
+ hf_output_str = "<s>" + hf_output_str
30
+
31
+ return output_ids, hf_output_str, out_logprobs
32
+
33
+
34
+def run_test(
35
+ hf_runner: Type[HfRunner],
36
+ vllm_runner: Type[VllmRunner],
37
+ prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
38
+ decoder_prompt_type: DecoderPromptType,
39
+ model: str,
40
+ *,
41
+ dtype: str,
42
+ max_tokens: int,
43
+ num_logprobs: int,
44
+ tensor_parallel_size: int,
45
+ distributed_executor_backend: Optional[str] = None,
46
+) -> None:
47
+ '''
48
+ Test the vLLM BART model for a variety of encoder/decoder input prompts,
49
+ by validating it against HuggingFace (HF) BART.
50
+
51
+ Arguments:
52
+
53
+ * hf_runner: HuggingFace (HF) test model runner
54
+ * vllm_runner: vLLM test model runner
55
+ * example_encoder_decoder_prompts: test fixture which provides a
56
+ dictionary of dummy prompts
57
+ * model: the HF ID of the specific BART variant under test
58
+ * dtype: the tensor datatype to employ
59
+ * max_tokens
60
+ * num_logprobs
61
+ * decoder_prompt_type: key into the example_encoder_decoder_prompts
62
+ dictionary; selects specific encoder/decoder
63
+ prompt scenarios to test
64
+
65
+ A note on using HF BART as a baseline for validating vLLM BART,
66
+ specifically when the decoder prompt is None.
67
+
68
+ The HF GenerationMixin's default behavior is to force the first
69
+ decoded token to be <BOS> if the prompt does not already contain
70
+ <BOS> (this is accomplished using a logit
71
+ processor setting.)
72
+
73
+ So when we use HF BART as our baseline for comparison, note that
74
+ when the user provides a request with a None decoder prompt
75
+ (i.e. a singleton encoder prompt, or else an explicit encoder/
76
+ decoder prompt with the decoder sub-prompt set to None), HF and
77
+ vLLM handle this in different ways:
78
+
79
+ * HF will (1) tokenize the None prompt as an empty token-list,
80
+ (2) append <decoder-start-token> to the beginning, yielding
81
+ [<decoder-start-token>], (3) pass this token list to the model, and
82
+ then (4) after computing logits during prefill, override the model
83
+ logits & force <BOS> to be the first generated token.
84
+
85
+ * vLLM will (1) tokenize the None prompt as [<BOS>], (2) append decoder-
86
+ start-token to the beginning, yielding [<decoder-start-token><BOS>],
87
+ (3) pass these tokens to the model & proceed with generation.
88
+
89
+ The net effect is that compared to vLLM, the list of HF *decoded* tokens
90
+ will contain one more initial <BOS> than the vLLM generated tokens,
91
+ because vLLM's <BOS> token is injected into the prompt rather than into
92
+ the generated output. This is in spite of the fact that overall, the
93
+ complete sequences (prompt + decoded tokens) produced by vLLM will match
94
+ HF.
95
+
96
+ So when we use HF decoded token output to validate vLLM's decoded token
97
+ output, the testing process must account for the difference in decoded
98
+ token sequences between vLLM and HF specifically in the
99
+ decoder-prompt-is-None case.
100
+
101
+ One option is to disable the logit processor feature that forces the
102
+ <BOS> token to be decoded (forced_bos_token_id = None), eliminating
103
+ the problem entirely. However this is not "normal" BART usage.
104
+
105
+ The other option is - only in the decoder-prompt-is-None case - to
106
+ discard the first decoded token from the HF output before comparing it
107
+ to vLLM.
108
+
109
+ To that end, when testing the scenario where the decoder prompt is None
110
+ (and only in that one scenario), this test skips the first HF decoded
111
+ token during the process of validating the vLLM decoded output.
112
+ '''
113
+
114
+ # NOTE: take care of the order. run vLLM first, and then run HF.
115
+ # vLLM needs a fresh new process without cuda initialization.
116
+ # if we run HF first, the cuda initialization will be done and it
117
+ # will hurt multiprocessing backend with fork method (the default).
118
+
119
+ # Note: currently encoder/decoder models are only compatible with
120
+ # enforce_eager=True. Normally this is not a problem because
121
+ # for encoder/decoder models vLLM will
122
+ # default to enforce_eager=True if enforce_eager
123
+ # is left unspecified. However, the
124
+ # VllmRunner test fixture (which wraps around the LLM class) defaults to
125
+ # enforce_eager=False (a behavior which a number of already-exisitng
126
+ # decoder-only unit tests expect), so when testing an encoder/decoder
127
+ # model we must explicitly specify enforce_eager=True in the VllmRunner
128
+ # constructor.
129
+ with vllm_runner(model,
130
+ dtype=dtype,
131
+ tensor_parallel_size=tensor_parallel_size,
132
+ distributed_executor_backend=distributed_executor_backend,
133
+ enforce_eager=True) as vllm_model:
134
+ vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
135
+ prompts, max_tokens, num_logprobs)
136
+
137
+ # Configuration settings for HF baseline
138
+ hf_kwargs = {
139
+ "top_k": None,
140
+ "num_beams": 1,
141
+ "repetition_penalty": 1.0,
142
+ "top_p": 1.0,
143
+ "length_penalty": 1.0,
144
+ "early_stopping": False,
145
+ "no_repeat_ngram_size": None,
146
+ "min_length": 0
147
+ }
148
+
149
+ with hf_runner(model, dtype=dtype,
150
+ auto_cls=AutoModelForSeq2SeqLM) as hf_model:
151
+ hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
152
+ prompts,
153
+ max_tokens,
154
+ num_logprobs,
155
+ **hf_kwargs,
156
+ ))
157
+
158
+ hf_skip_tokens = (1
159
+ if decoder_prompt_type == DecoderPromptType.NONE else 0)
160
+
161
+ check_logprobs_close(
162
+ outputs_0_lst=hf_outputs,
163
+ outputs_1_lst=[
164
+ vllm_to_hf_output(vllm_output, decoder_prompt_type)
165
+ for vllm_output in vllm_outputs
166
+ ],
167
+ name_0="hf",
168
+ name_1="vllm",
169
+ num_outputs_0_skip_tokens=hf_skip_tokens,
170
+ )
171
+
172
+
173
+@pytest.mark.parametrize("model", MODELS)
174
+@pytest.mark.parametrize("dtype", ["float", "bfloat16"])
175
+@pytest.mark.parametrize("max_tokens", [64])
176
+@pytest.mark.parametrize("num_logprobs", [5])
177
+@pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
178
+def test_models(hf_runner, vllm_runner, example_encoder_decoder_prompts, model,
179
+ dtype, max_tokens, num_logprobs, decoder_prompt_type) -> None:
180
+
181
+ run_test(
182
+ hf_runner,
183
+ vllm_runner,
184
+ example_encoder_decoder_prompts[decoder_prompt_type],
185
+ decoder_prompt_type,
186
+ model,
187
+ dtype=dtype,
188
+ max_tokens=max_tokens,
189
+ num_logprobs=num_logprobs,
190
+ tensor_parallel_size=1,
191
+ )
192
+
193
+
194
+@multi_gpu_test(num_gpus=2)
195
+@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
196
+@pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
197
+@pytest.mark.parametrize("dtype", ["float"])
198
+@pytest.mark.parametrize("max_tokens", [64])
199
+@pytest.mark.parametrize("num_logprobs", [5])
200
+@pytest.mark.parametrize("decoder_prompt_type", [DecoderPromptType.CUSTOM])
201
+def test_models_distributed(hf_runner, vllm_runner,
202
+ example_encoder_decoder_prompts,
203
+ distributed_executor_backend, model, dtype,
204
+ max_tokens, num_logprobs,
205
+ decoder_prompt_type) -> None:
206
+ run_test(
207
+ hf_runner,
208
+ vllm_runner,
209
+ example_encoder_decoder_prompts[decoder_prompt_type],
210
+ decoder_prompt_type,
211
+ model,
212
+ dtype=dtype,
213
+ max_tokens=max_tokens,
214
+ num_logprobs=num_logprobs,
215
+ tensor_parallel_size=2,
216
+ distributed_executor_backend=distributed_executor_backend,
217
+ )
vllm/attention/backends/torch_sdpa.py
+299
-61
@@ -75,6 +75,22 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
75
slot_mapping: torch.Tensor
76
seq_lens: Optional[List[int]]
77
78
+ # Begin encoder attn & enc/dec cross-attn fields...
79
+ # Encoder sequence lengths representation
80
+ encoder_seq_lens: Optional[List[int]] = None
81
+ encoder_seq_lens_tensor: Optional[torch.Tensor] = None
82
+
83
+ # Maximum sequence length among encoder sequences
84
+ max_encoder_seq_len: Optional[int] = None
85
+
86
+ # Number of tokens input to encoder
87
+ num_encoder_tokens: Optional[int] = None
88
+
89
+ # Cross-attention memory-mapping data structures: slot mapping
90
+ # and block tables
91
+ cross_slot_mapping: Optional[torch.Tensor] = None
92
+ cross_block_tables: Optional[torch.Tensor] = None
93
+
94
def __post_init__(self):
95
# Set during the execution of the first attention op.
96
# It is a list because it is needed to set per prompt
@@ -82,6 +98,28 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
98
# from xformer API.
99
# will not appear in the __repr__ and __init__
100
self.attn_bias: Optional[List[torch.Tensor]] = None
101
+ self.encoder_attn_bias: Optional[List[torch.Tensor]] = None
102
+ self.cross_attn_bias: Optional[List[torch.Tensor]] = None
103
+
104
+ @property
105
+ def is_all_encoder_attn_metadata_set(self):
106
+ '''
107
+ All attention metadata required for encoder attention is set.
108
+ '''
109
+ return ((self.encoder_seq_lens is not None)
110
+ and (self.encoder_seq_lens_tensor is not None)
111
+ and (self.max_encoder_seq_len is not None))
112
+
113
+ @property
114
+ def is_all_cross_attn_metadata_set(self):
115
+ '''
116
+ All attention metadata required for enc/dec cross-attention is set.
117
+
118
+ Superset of encoder attention required metadata.
119
+ '''
120
+ return (self.is_all_encoder_attn_metadata_set
121
+ and (self.cross_slot_mapping is not None)
122
+ and (self.cross_block_tables is not None))
123
124
@property
125
def prefill_metadata(self) -> Optional["TorchSDPAMetadata"]:
@@ -101,6 +139,136 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
139
140
return self
141
142
+ def get_seq_lens(
143
+ self,
144
+ attn_type: AttentionType,
145
+ ):
146
+ '''
147
+ Extract appropriate sequence lengths from attention metadata
148
+ according to attention type.
149
+
150
+ Arguments:
151
+
152
+ * attn_metadata: Attention metadata structure associated with attention
153
+ * attn_type: encoder attention, decoder self-attention,
154
+ encoder/decoder cross-attention
155
+
156
+ Returns:
157
+ * Appropriate sequence lengths tensor for query
158
+ * Appropriate sequence lengths tensor for key & value
159
+ '''
160
+
161
+ if attn_type == AttentionType.DECODER:
162
+ seq_lens_q = self.seq_lens
163
+ seq_lens_kv = self.seq_lens
164
+ elif attn_type == AttentionType.ENCODER:
165
+ seq_lens_q = self.encoder_seq_lens
166
+ seq_lens_kv = self.encoder_seq_lens
167
+ elif attn_type == AttentionType.ENCODER_DECODER:
168
+ seq_lens_q = self.seq_lens
169
+ seq_lens_kv = self.encoder_seq_lens
170
+ else:
171
+ raise AttributeError(f"Invalid attention type {str(attn_type)}")
172
+ return seq_lens_q, seq_lens_kv
173
+
174
+ def get_attn_bias(
175
+ self,
176
+ attn_type: AttentionType,
177
+ ) -> Optional[List[torch.Tensor]]:
178
+ '''
179
+ Extract appropriate attention bias from attention metadata
180
+ according to attention type.
181
+
182
+ Arguments:
183
+
184
+ * attn_metadata: Attention metadata structure associated with attention
185
+ * attn_type: encoder attention, decoder self-attention,
186
+ encoder/decoder cross-attention
187
+
188
+ Returns:
189
+ * Appropriate attention bias value given the attention type
190
+ '''
191
+
192
+ if attn_type == AttentionType.DECODER:
193
+ return self.attn_bias
194
+ elif attn_type == AttentionType.ENCODER:
195
+ return self.encoder_attn_bias
196
+ elif attn_type == AttentionType.ENCODER_DECODER:
197
+ return self.cross_attn_bias
198
+ else:
199
+ raise AttributeError(f"Invalid attention type {str(attn_type)}")
200
+
201
+ def set_attn_bias(
202
+ self,
203
+ attn_bias: List[torch.Tensor],
204
+ attn_type: AttentionType,
205
+ ) -> None:
206
+ '''
207
+ Update appropriate attention bias field of attention metadata,
208
+ according to attention type.
209
+
210
+ Arguments:
211
+
212
+ * attn_metadata: Attention metadata structure associated with attention
213
+ * attn_bias: The desired attention bias value
214
+ * attn_type: encoder attention, decoder self-attention,
215
+ encoder/decoder cross-attention
216
+ '''
217
+
218
+ if attn_type == AttentionType.DECODER:
219
+ self.attn_bias = attn_bias
220
+ elif attn_type == AttentionType.ENCODER:
221
+ self.encoder_attn_bias = attn_bias
222
+ elif attn_type == AttentionType.ENCODER_DECODER:
223
+ self.cross_attn_bias = attn_bias
224
+ else:
225
+ raise AttributeError(f"Invalid attention type {str(attn_type)}")
226
+
227
+ def get_seq_len_block_table_args(
228
+ self,
229
+ attn_type: AttentionType,
230
+ ) -> tuple:
231
+ '''
232
+ The particular choice of sequence-length- and block-table-related
233
+ attributes which should be extracted from attn_metadata is dependent
234
+ on the type of attention operation.
235
+
236
+ Decoder attn -> select entirely decoder self-attention-related fields
237
+ Encoder/decoder cross-attn -> select encoder sequence lengths &
238
+ cross-attn block-tables fields
239
+ Encoder attn -> select encoder sequence lengths fields & no block tables
240
+
241
+ Arguments:
242
+
243
+ * attn_metadata: Attention metadata structure associated with attention
244
+ * is_prompt: True if prefill, False otherwise
245
+ * attn_type: encoder attention, decoder self-attention,
246
+ encoder/decoder cross-attention
247
+
248
+ Returns:
249
+
250
+ * Appropriate sequence-lengths tensor
251
+ * Appropriate max sequence-length scalar
252
+ * Appropriate block tables (or None)
253
+ '''
254
+
255
+ if attn_type == AttentionType.DECODER:
256
+ # Decoder self-attention
257
+ # Choose max_seq_len based on whether we are in prompt_run
258
+ return (self.seq_lens_tensor, self.max_decode_seq_len,
259
+ self.block_tables)
260
+ elif attn_type == AttentionType.ENCODER_DECODER:
261
+ # Enc/dec cross-attention KVs match encoder sequence length;
262
+ # cross-attention utilizes special "cross" block tables
263
+ return (self.encoder_seq_lens_tensor, self.max_encoder_seq_len,
264
+ self.cross_block_tables)
265
+ elif attn_type == AttentionType.ENCODER:
266
+ # No block tables associated with encoder attention
267
+ return (self.encoder_seq_lens_tensor, self.max_encoder_seq_len,
268
+ None)
269
+ else:
270
+ raise AttributeError(f"Invalid attention type {str(attn_type)}")
271
+
272
273
class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
274
@@ -171,84 +339,101 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
339
shape = [num_tokens, num_heads * head_size]
340
"""
341
assert k_scale == 1.0 and v_scale == 1.0
174
- if attn_type != AttentionType.DECODER:
175
- raise NotImplementedError("Encoder self-attention and "
176
- "encoder/decoder cross-attention "
177
- "are not implemented for "
178
- "TorchSDPABackendImpl")
179
- num_tokens, hidden_size = query.shape
342
+ if (attn_type == AttentionType.ENCODER
343
+ and (not attn_metadata.is_all_encoder_attn_metadata_set)):
344
+ raise AttributeError("Encoder attention requires setting "
345
+ "encoder metadata attributes.")
346
+ elif (attn_type == AttentionType.ENCODER_DECODER
347
+ and (not attn_metadata.is_all_cross_attn_metadata_set)):
348
+ raise AttributeError("Encoder/decoder cross-attention "
349
+ "requires setting cross-attention "
350
+ "metadata attributes.")
351
+
352
# Reshape the query, key, and value tensors.
353
query = query.view(-1, self.num_heads, self.head_size)
182
- key = key.view(-1, self.num_kv_heads, self.head_size)
183
- value = value.view(-1, self.num_kv_heads, self.head_size)
184
-
185
- if kv_cache.numel() > 0:
354
+ if key is not None:
355
+ assert value is not None
356
+ key = key.view(-1, self.num_kv_heads, self.head_size)
357
+ value = value.view(-1, self.num_kv_heads, self.head_size)
358
+ else:
359
+ assert value is None
360
+
361
+ if (attn_type != AttentionType.ENCODER and kv_cache.numel() > 0):
362
+ # KV-cache during decoder-self- or
363
+ # encoder-decoder-cross-attention, but not
364
+ # during encoder attention.
365
+ #
366
+ # Even if there are no new key/value pairs to cache,
367
+ # we still need to break out key_cache and value_cache
368
+ # i.e. for later use by paged attention
369
key_cache, value_cache = PagedAttention.split_kv_cache(
370
kv_cache, self.num_kv_heads, self.head_size)
188
- PagedAttention.write_to_paged_cache(key, value, key_cache,
189
- value_cache,
190
- attn_metadata.slot_mapping,
191
- self.kv_cache_dtype, k_scale,
192
- v_scale)
371
194
- if attn_metadata.is_prompt:
372
+ if (key is not None) and (value is not None):
373
+ if attn_type == AttentionType.ENCODER_DECODER:
374
+ # Update cross-attention KV cache (prefill-only)
375
+ # During cross-attention decode, key & value will be None,
376
+ # preventing this IF-statement branch from running
377
+ updated_slot_mapping = attn_metadata.cross_slot_mapping
378
+ else:
379
+ # Update self-attention KV cache (prefill/decode)
380
+ updated_slot_mapping = attn_metadata.slot_mapping
381
+
382
+ PagedAttention.write_to_paged_cache(key, value, key_cache,
383
+ value_cache,
384
+ updated_slot_mapping,
385
+ self.kv_cache_dtype,
386
+ k_scale, v_scale)
387
+
388
+ if attn_type != AttentionType.ENCODER:
389
+ # Decoder self-attention supports chunked prefill.
390
+ # Encoder/decoder cross-attention requires no chunked
391
+ # prefill (100% prefill or 100% decode tokens, no mix)
392
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
393
+ num_decode_tokens = attn_metadata.num_decode_tokens
394
+ else:
395
+ # Encoder attention - chunked prefill is not applicable;
396
+ # derive token-count from query shape & and treat them
397
+ # as 100% prefill tokens
398
+ assert attn_metadata.num_encoder_tokens is not None
399
+ num_prefill_tokens = attn_metadata.num_encoder_tokens
400
+ num_decode_tokens = 0
401
+
402
+ if attn_type == AttentionType.DECODER:
403
+ # Only enforce this shape-constraint for decoder
404
+ # self-attention
405
+ assert key.shape[0] == num_prefill_tokens + num_decode_tokens
406
+ assert value.shape[0] == num_prefill_tokens + num_decode_tokens
407
+
408
+ if prefill_meta := attn_metadata.prefill_metadata:
409
assert attn_metadata.seq_lens is not None
410
if (kv_cache.numel() == 0
197
- or attn_metadata.block_tables.numel() == 0):
198
- if self.num_kv_heads != self.num_heads:
199
- key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
200
- value = value.repeat_interleave(self.num_queries_per_kv,
201
- dim=1)
202
-
203
- if attn_metadata.attn_bias is None:
204
- if self.alibi_slopes is not None:
205
- att_masks = _make_alibi_bias(
206
- self.alibi_slopes, query.dtype,
207
- attn_metadata.seq_lens) # type: ignore
208
- elif self.sliding_window is not None:
209
- att_masks = _make_sliding_window_bias(
210
- attn_metadata.seq_lens, self.sliding_window,
211
- query.dtype) # type: ignore
212
- else:
213
- att_masks = [None] * len(attn_metadata.seq_lens)
214
- attn_metadata.attn_bias = att_masks
215
-
216
- query = query.movedim(0, query.dim() - 2)
217
- key = key.movedim(0, key.dim() - 2)
218
- value = value.movedim(0, value.dim() - 2)
219
-
220
- start = 0
221
- output = torch.empty(
222
- (num_tokens, self.num_heads, self.head_size),
223
- dtype=query.dtype)
224
- for seq_len, mask in zip(attn_metadata.seq_lens,
225
- attn_metadata.attn_bias):
226
- end = start + seq_len
227
- sub_out = scaled_dot_product_attention(
228
- query[None, :, start:end, :],
229
- key[None, :, start:end, :],
230
- value[None, :, start:end, :],
231
- attn_mask=mask,
232
- dropout_p=0.0,
233
- is_causal=not self.need_mask,
234
- scale=self.scale).squeeze(0).movedim(
235
- query.dim() - 2, 0)
236
- output[start:end, :, :] = sub_out
237
- start = end
411
+ or prefill_meta.block_tables.numel() == 0):
412
+ output = self._run_sdpa_forward(query,
413
+ key,
414
+ value,
415
+ prefill_meta,
416
+ attn_type=attn_type)
417
else:
418
# prefix-enabled attention
419
raise RuntimeError(
420
"Torch SDPA backend doesn't support prefix decoding.")
421
243
- else:
422
+ if decode_meta := attn_metadata.decode_metadata:
423
# Decoding run.
424
+ (
425
+ seq_lens_arg,
426
+ max_seq_len_arg,
427
+ block_tables_arg,
428
+ ) = decode_meta.get_seq_len_block_table_args(attn_type)
429
+
430
output = PagedAttention.forward_decode(
431
query,
432
key_cache,
433
value_cache,
249
- attn_metadata.block_tables,
250
- attn_metadata.seq_lens_tensor,
251
- attn_metadata.max_decode_seq_len,
434
+ block_tables_arg,
435
+ seq_lens_arg,
436
+ max_seq_len_arg,
437
self.kv_cache_dtype,
438
self.num_kv_heads,
439
self.scale,
@@ -260,6 +445,59 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
445
# Reshape the output tensor.
446
return output.view(-1, self.num_heads * self.head_size)
447
448
+ def _run_sdpa_forward(
449
+ self,
450
+ query: torch.Tensor,
451
+ key: torch.Tensor,
452
+ value: torch.Tensor,
453
+ attn_metadata: TorchSDPAMetadata,
454
+ attn_type: AttentionType = AttentionType.DECODER,
455
+ ):
456
+ if self.num_kv_heads != self.num_heads:
457
+ key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
458
+ value = value.repeat_interleave(self.num_queries_per_kv, dim=1)
459
+
460
+ attn_masks = attn_metadata.get_attn_bias(attn_type)
461
+ if attn_masks is None:
462
+ if self.alibi_slopes is not None:
463
+ attn_masks = _make_alibi_bias(
464
+ self.alibi_slopes, query.dtype,
465
+ attn_metadata.seq_lens) # type: ignore
466
+ elif self.sliding_window is not None:
467
+ assert attn_metadata.seq_lens is not None
468
+ attn_masks = _make_sliding_window_bias(
469
+ attn_metadata.seq_lens, self.sliding_window,
470
+ query.dtype) # type: ignore
471
+ else:
472
+ seq_lens, _ = attn_metadata.get_seq_lens(attn_type)
473
+ attn_masks = [None] * len(seq_lens)
474
+ attn_metadata.set_attn_bias(attn_masks, attn_type)
475
+
476
+ output = torch.empty_like(query)
477
+ query = query.movedim(0, query.dim() - 2)
478
+ key = key.movedim(0, key.dim() - 2)
479
+ value = value.movedim(0, value.dim() - 2)
480
+
481
+ causal_attn = (attn_type == AttentionType.DECODER)
482
+
483
+ seq_lens_q, seq_lens_kv = attn_metadata.get_seq_lens(attn_type)
484
+ start_q, start_kv = 0, 0
485
+ for seq_len_q, seq_len_kv, mask in zip(seq_lens_q, seq_lens_kv,
486
+ attn_masks):
487
+ end_q = start_q + seq_len_q
488
+ end_kv = start_kv + seq_len_kv
489
+ sub_out = scaled_dot_product_attention(
490
+ query[None, :, start_q:end_q, :],
491
+ key[None, :, start_kv:end_kv, :],
492
+ value[None, :, start_kv:end_kv, :],
493
+ attn_mask=mask,
494
+ dropout_p=0.0,
495
+ is_causal=causal_attn and not self.need_mask,
496
+ scale=self.scale).squeeze(0).movedim(query.dim() - 2, 0)
497
+ output[start_q:end_q, :, :] = sub_out
498
+ start_q, start_kv = end_q, end_kv
499
+ return output
500
+
501
502
def _make_alibi_bias(
503
alibi_slopes: torch.Tensor,
vllm/worker/cpu_enc_dec_model_runner.py
new
+311
@@ -0,0 +1,311 @@
1
+import dataclasses
2
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
3
+
4
+import torch
5
+
6
+from vllm.attention import AttentionMetadata
7
+from vllm.model_executor.layers.sampler import SamplerOutput
8
+from vllm.multimodal import MultiModalInputs
9
+from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
10
+from vllm.utils import make_tensor_with_pad
11
+from vllm.worker.cpu_model_runner import (CPUModelRunner,
12
+ ModelInputForCPUBuilder,
13
+ ModelInputForCPUWithSamplingMetadata)
14
+from vllm.worker.model_runner_base import (
15
+ _add_attn_metadata_broadcastable_dict,
16
+ _add_sampling_metadata_broadcastable_dict)
17
+
18
+if TYPE_CHECKING:
19
+ from vllm.attention.backends.abstract import AttentionBackend
20
+
21
+
22
+@dataclasses.dataclass(frozen=True)
23
+class EncoderDecoderModelInputForCPU(ModelInputForCPUWithSamplingMetadata):
24
+ """
25
+ Used by the EncoderDecoderModelRunner.
26
+ """
27
+ encoder_input_tokens: Optional[torch.Tensor] = None
28
+ encoder_input_positions: Optional[torch.Tensor] = None
29
+
30
+ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
31
+ tensor_dict = {
32
+ "input_tokens": self.input_tokens,
33
+ "input_positions": self.input_positions,
34
+ "encoder_input_tokens": self.encoder_input_tokens,
35
+ "encoder_input_positions": self.encoder_input_positions,
36
+ }
37
+ _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
38
+ _add_sampling_metadata_broadcastable_dict(tensor_dict,
39
+ self.sampling_metadata)
40
+ return tensor_dict
41
+
42
+ @classmethod
43
+ def from_broadcasted_tensor_dict(
44
+ cls,
45
+ tensor_dict: Dict[str, Any],
46
+ attn_backend: Optional["AttentionBackend"] = None,
47
+ ) -> "EncoderDecoderModelInputForCPU":
48
+ return cast(
49
+ EncoderDecoderModelInputForCPU,
50
+ super().from_broadcasted_tensor_dict(tensor_dict, attn_backend))
51
+
52
+
53
+class CPUEncoderDecoderModelRunner(CPUModelRunner):
54
+ _model_input_cls: Type[EncoderDecoderModelInputForCPU] = (
55
+ EncoderDecoderModelInputForCPU)
56
+ _builder_cls: Type[ModelInputForCPUBuilder] = ModelInputForCPUBuilder
57
+
58
+ def _list_to_int32_tensor(
59
+ self,
60
+ _list: List[int],
61
+ ) -> torch.Tensor:
62
+ return torch.tensor(_list, dtype=torch.int32, device=self.device)
63
+
64
+ def _list_to_long_tensor(
65
+ self,
66
+ _list: List[int],
67
+ ) -> torch.Tensor:
68
+ return torch.tensor(_list, dtype=torch.long, device=self.device)
69
+
70
+ def _empty_int32_tensor(self) -> torch.Tensor:
71
+ return self._list_to_int32_tensor([])
72
+
73
+ def _empty_long_tensor(self) -> torch.Tensor:
74
+ return self._list_to_long_tensor([])
75
+
76
+ def make_model_input_from_broadcasted_tensor_dict(
77
+ self, tensor_dict: Dict[str,
78
+ Any]) -> EncoderDecoderModelInputForCPU:
79
+ return EncoderDecoderModelInputForCPU.from_broadcasted_tensor_dict(
80
+ tensor_dict,
81
+ attn_backend=self.attn_backend,
82
+ )
83
+
84
+ def prepare_model_input(
85
+ self,
86
+ seq_group_metadata_list: List[SequenceGroupMetadata],
87
+ virtual_engine: int = 0,
88
+ finished_requests_ids: Optional[List[str]] = None
89
+ ) -> EncoderDecoderModelInputForCPU:
90
+ model_input = super().prepare_model_input(seq_group_metadata_list,
91
+ virtual_engine,
92
+ finished_requests_ids)
93
+ model_input = cast(EncoderDecoderModelInputForCPU, model_input)
94
+ (
95
+ attn_metadata,
96
+ encoder_input_tokens_tensor,
97
+ encoder_input_positions_tensor,
98
+ ) = self._prepare_encoder_model_input_tensors(seq_group_metadata_list,
99
+ model_input)
100
+ return dataclasses.replace(
101
+ model_input,
102
+ attn_metadata=attn_metadata,
103
+ encoder_input_tokens=encoder_input_tokens_tensor,
104
+ encoder_input_positions=encoder_input_positions_tensor,
105
+ )
106
+
107
+ def _prepare_encoder_model_input_tensors(
108
+ self,
109
+ seq_group_metadata_list: List[SequenceGroupMetadata],
110
+ model_input: EncoderDecoderModelInputForCPU,
111
+ ) -> Tuple[AttentionMetadata, Optional[torch.Tensor],
112
+ Optional[torch.Tensor]]:
113
+ """Helper method to prepare the encoder- and cross-attn-related
114
+ model inputs based on a given sequence group. These additional inputs
115
+ are used to augment an already-computed `EncoderDecoderModelInput`
116
+ data structure which already has decoder-related model inputs
117
+ populated.
118
+
119
+ Sets the following attn_metadata fields:
120
+ * `num_encoder_tokens`
121
+ * `encoder_seq_lens`
122
+ * `encoder_seq_lens_tensor`
123
+ * `max_encoder_seq_len`
124
+ * `cross_slot_mapping`
125
+ * `cross_block_tables`
126
+
127
+ Constructs a new model inputs data structure, based on
128
+ (1) the existing fields in the `model_inputs` argument,
129
+ and (2) the following additional fields which are
130
+ computed (or in the case of `attn_metadata`, updated)
131
+ by this function:
132
+ * attn_metadata
133
+ * encoder_input_tokens
134
+ * encoder_input_positions
135
+
136
+ Arguments:
137
+
138
+ * seq_group_metadata_list: list of sequence groups for which to
139
+ compute inputs
140
+ * model_inputs: model inputs data structure with decoder-oriented
141
+ fields already computed.
142
+
143
+ Return:
144
+
145
+ * Updated model inputs data structure
146
+ """
147
+
148
+ if len(seq_group_metadata_list) == 0:
149
+ return (model_input.attn_metadata, None, None)
150
+
151
+ # Since we are not supporting chunked prefill either the entire
152
+ # batch is prefill or it is decode
153
+ is_prompt = seq_group_metadata_list[0].is_prompt
154
+
155
+ # Build encoder inputs
156
+ encoder_seq_lens: List[int] = []
157
+ if is_prompt:
158
+ # Prefill phase.
159
+ cross_block_tables = self._empty_int32_tensor().view(
160
+ len(seq_group_metadata_list), -1)
161
+
162
+ # Extract input tokens/positions, cross-attention slot-mapping,
163
+ # & seq len from each sequence group metadata
164
+ (
165
+ encoder_input_tokens,
166
+ encoder_input_positions,
167
+ cross_slot_mapping,
168
+ ) = (
169
+ [],
170
+ [],
171
+ [],
172
+ )
173
+ for seq_group_metadata in seq_group_metadata_list:
174
+ # Build seq lens
175
+ seq_len = seq_group_metadata.encoder_seq_data.get_len()
176
+ token_ids = seq_group_metadata.encoder_seq_data.get_token_ids()
177
+ encoder_seq_lens.append(seq_len)
178
+
179
+ # Build slot mapping
180
+ for i in range(0, seq_len):
181
+ block_number = seq_group_metadata.cross_block_table[
182
+ i // self.block_size]
183
+ block_offset = i % self.block_size
184
+ slot = block_number * self.block_size + block_offset
185
+ cross_slot_mapping.append(slot)
186
+
187
+ # Build encoder input tokens
188
+ encoder_input_tokens.extend(token_ids)
189
+ encoder_input_positions.extend(list(range(0, seq_len)))
190
+
191
+ # Convert tokens/positions & cross-attention
192
+ # slot-mapping to encoder input tensors
193
+ encoder_input_tokens_tensor = self._list_to_long_tensor(
194
+ encoder_input_tokens)
195
+ encoder_input_positions_tensor = self._list_to_long_tensor(
196
+ encoder_input_positions)
197
+ cross_slot_mapping_tensor = self._list_to_long_tensor(
198
+ cross_slot_mapping)
199
+
200
+ else:
201
+ # Decode phase.
202
+ encoder_input_tokens_tensor = self._empty_long_tensor()
203
+ encoder_input_positions_tensor = self._empty_long_tensor()
204
+ cross_slot_mapping_tensor = self._empty_long_tensor()
205
+ # Extract cross-attention block tables &
206
+ # seq len from each sequence group metadata.
207
+ # Cross-attention block tables are empty
208
+ # during vLLM memory profiling.
209
+ cross_block_tables = []
210
+ for seq_group_metadata in seq_group_metadata_list:
211
+ for _ in range(len(seq_group_metadata.seq_data)):
212
+ encoder_seq_lens.append(
213
+ seq_group_metadata.encoder_seq_data.get_len())
214
+ cross_block_table = seq_group_metadata.cross_block_table
215
+ cross_block_tables.append([] if (
216
+ cross_block_table is None) else cross_block_table)
217
+
218
+ max_len_of_block_table = max(
219
+ len(block_table) for block_table in cross_block_tables)
220
+
221
+ cross_block_tables = make_tensor_with_pad(
222
+ cross_block_tables,
223
+ max_len=max_len_of_block_table,
224
+ pad=0,
225
+ dtype=torch.int32,
226
+ device=self.device,
227
+ )
228
+
229
+ # Compute encoder sequence lengths & encoder
230
+ # sequence starting offset tensors
231
+ max_encoder_seq_len = max(encoder_seq_lens, default=0)
232
+ encoder_seq_lens_tensor = self._list_to_int32_tensor(encoder_seq_lens)
233
+ encoder_seq_start_loc = torch.zeros(encoder_seq_lens_tensor.shape[0] +
234
+ 1,
235
+ dtype=torch.int32,
236
+ device=self.device)
237
+ torch.cumsum(encoder_seq_lens_tensor,
238
+ dim=0,
239
+ dtype=encoder_seq_start_loc.dtype,
240
+ out=encoder_seq_start_loc[1:])
241
+
242
+ # Update attention metadata with encoder-oriented attributes
243
+ attn_metadata = model_input.attn_metadata
244
+ assert attn_metadata is not None
245
+ (
246
+ attn_metadata.num_encoder_tokens,
247
+ attn_metadata.encoder_seq_lens,
248
+ attn_metadata.encoder_seq_lens_tensor,
249
+ attn_metadata.max_encoder_seq_len,
250
+ attn_metadata.cross_slot_mapping,
251
+ attn_metadata.cross_block_tables,
252
+ ) = (
253
+ sum(encoder_seq_lens),
254
+ encoder_seq_lens,
255
+ encoder_seq_lens_tensor,
256
+ max_encoder_seq_len,
257
+ cross_slot_mapping_tensor,
258
+ cross_block_tables,
259
+ )
260
+
261
+ return (attn_metadata, encoder_input_tokens_tensor,
262
+ encoder_input_positions_tensor)
263
+
264
+ @torch.no_grad()
265
+ def execute_model(
266
+ self,
267
+ model_input: EncoderDecoderModelInputForCPU,
268
+ kv_caches: List[torch.Tensor],
269
+ intermediate_tensors: Optional[IntermediateTensors] = None,
270
+ num_steps: int = 1,
271
+ ) -> Optional[List[SamplerOutput]]:
272
+ if num_steps > 1:
273
+ raise ValueError(
274
+ "CPU worker does not support multi-step execution.")
275
+
276
+ model_executable = self.model
277
+ execute_model_kwargs = {
278
+ "input_ids":
279
+ model_input.input_tokens,
280
+ "positions":
281
+ model_input.input_positions,
282
+ "encoder_input_ids":
283
+ model_input.encoder_input_tokens,
284
+ "encoder_positions":
285
+ model_input.encoder_input_positions,
286
+ "kv_caches":
287
+ kv_caches,
288
+ "attn_metadata":
289
+ model_input.attn_metadata,
290
+ **MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
291
+ device=self.device),
292
+ "intermediate_tensors":
293
+ intermediate_tensors,
294
+ }
295
+
296
+ hidden_states = model_executable(**execute_model_kwargs)
297
+
298
+ # Compute the logits.
299
+ logits = self.model.compute_logits(hidden_states,
300
+ model_input.sampling_metadata)
301
+
302
+ # Only perform sampling in the driver worker.
303
+ if not self.is_driver_worker:
304
+ return []
305
+
306
+ # Sample the next token.
307
+ output = self.model.sample(
308
+ logits=logits,
309
+ sampling_metadata=model_input.sampling_metadata,
310
+ )
311
+ return [output]
vllm/worker/cpu_model_runner.py
+3
-7
@@ -19,7 +19,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19
MultiModalInputs)
20
from vllm.sequence import (IntermediateTensors, SequenceData,
21
SequenceGroupMetadata)
22
-from vllm.utils import STR_NOT_IMPL_ENC_DEC_ERR_STRS, make_tensor_with_pad
22
+from vllm.utils import make_tensor_with_pad
23
from vllm.worker.model_runner_base import (
24
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
25
_add_attn_metadata_broadcastable_dict,
@@ -434,10 +434,6 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
434
# Lazy initialization.
435
self.model: nn.Module # Set after init_Model
436
437
- if self.model_config.is_encoder_decoder_model:
438
- raise NotImplementedError(
439
- STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CPU'])
440
-
437
@property
438
def model_is_mrope(self) -> bool:
439
"""Detect if the model has "mrope" rope_scaling type.
@@ -459,8 +455,8 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
455
def make_model_input_from_broadcasted_tensor_dict(
456
self,
457
tensor_dict: Dict[str, Any],
462
- ) -> ModelInputForCPU:
463
- return ModelInputForCPU.from_broadcasted_tensor_dict(
458
+ ) -> ModelInputForCPUWithSamplingMetadata:
459
+ return ModelInputForCPUWithSamplingMetadata.from_broadcasted_tensor_dict( # noqa: E501
460
tensor_dict,
461
attn_backend=self.attn_backend,
462
)
vllm/worker/cpu_worker.py
+9
-2
@@ -1,5 +1,5 @@
1
"""A CPU worker class."""
2
-from typing import Dict, List, Optional, Tuple
2
+from typing import Dict, List, Optional, Tuple, Type
3
4
import torch
5
import torch.distributed
@@ -15,6 +15,7 @@ from vllm.logger import init_logger
15
from vllm.model_executor import set_random_seed
16
from vllm.sequence import ExecuteModelRequest
17
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18
+from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
19
from vllm.worker.cpu_model_runner import CPUModelRunner
20
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
21
LoraNotSupportedWorkerBase, WorkerInput)
@@ -163,7 +164,10 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
164
else:
165
self.local_omp_cpuid = omp_cpuids.split("|")[rank]
166
166
- self.model_runner: CPUModelRunner = CPUModelRunner(
167
+ ModelRunnerClass: Type[CPUModelRunner] = CPUModelRunner
168
+ if self._is_encoder_decoder_model():
169
+ ModelRunnerClass = CPUEncoderDecoderModelRunner
170
+ self.model_runner: CPUModelRunner = ModelRunnerClass(
171
model_config,
172
parallel_config,
173
scheduler_config,
@@ -205,6 +209,9 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
209
raise RuntimeError("Profiler is not enabled.")
210
self.profiler.stop()
211
212
+ def _is_encoder_decoder_model(self):
213
+ return self.model_config.is_encoder_decoder_model
214
+
215
def init_device(self) -> None:
216
if self.local_omp_cpuid != "all":
217
ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)