A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Hardware][Intel] Isolate CPUModelRunner and ModelRunner for better maintenance (#3824)
bigPYJ1151 committed
Apr 12, 2024 at 02:56 UTC
8afca50889bad6ad987c523c48c31fc52fcb72e4
5 files changed
+443
-61
vllm/attention/backends/torch_sdpa.py
+24
-48
@@ -50,20 +50,15 @@ class TorchSDPABackend(AttentionBackend):
50
51
52
@dataclass
53
-class TorchSDPAMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
53
+class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata,
54
+ AttentionMetadataPerStage):
55
"""Metadata for TorchSDPABackend.
56
"""
57
# Currently, input sequences can only contain all prompts
58
# or all decoding. True if all sequences are prompts.
59
is_prompt: bool
60
+ slot_mapping: torch.Tensor
61
prompt_lens: Optional[List[int]]
60
- prompt_lens_tensor: Optional[torch.Tensor]
61
-
62
- max_subquery_len: Optional[int] = None
63
- max_prompt_len: Optional[int] = None
64
- subquery_start_loc: Optional[torch.Tensor] = None
65
- seq_start_loc: Optional[torch.Tensor] = None
66
- use_cuda_graph: bool = False
62
63
def __post_init__(self):
64
# Set during the execution of the first attention op.
@@ -111,7 +106,7 @@ class TorchSDPABackendImpl(AttentionImpl):
106
key: torch.Tensor,
107
value: torch.Tensor,
108
kv_cache: Optional[torch.Tensor],
114
- attn_metadata: AttentionMetadata[TorchSDPAMetadata],
109
+ attn_metadata: TorchSDPAMetadata,
110
kv_scale: float,
111
) -> torch.Tensor:
112
"""Forward pass with torch SDPA and PagedAttention.
@@ -140,51 +135,36 @@ class TorchSDPABackendImpl(AttentionImpl):
135
attn_metadata.kv_cache_dtype,
136
kv_scale)
137
143
- num_prefill_tokens = attn_metadata.num_prefill_tokens
144
- num_decode_tokens = attn_metadata.num_decode_tokens
145
- assert key.shape[0] == num_prefill_tokens + num_decode_tokens
146
- assert value.shape[0] == num_prefill_tokens + num_decode_tokens
147
-
148
- output = torch.empty_like(query)
149
- # Query for decode. KV is not needed because it is already cached.
150
- decode_query = query[num_prefill_tokens:]
151
- # QKV for prefill.
152
- query = query[:num_prefill_tokens]
153
- key = key[:num_prefill_tokens]
154
- value = value[:num_prefill_tokens]
155
-
156
- assert query.shape[0] == num_prefill_tokens
157
- assert decode_query.shape[0] == num_decode_tokens
158
-
159
- if prefill_meta := attn_metadata.prefill_metadata:
160
- if (kv_cache is None or prefill_meta.block_tables.numel() == 0):
138
+ if attn_metadata.is_prompt:
139
+ if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
140
if self.num_kv_heads != self.num_heads:
141
key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
142
value = value.repeat_interleave(self.num_queries_per_kv,
143
dim=1)
144
166
- if prefill_meta.attn_bias is None:
145
+ if attn_metadata.attn_bias is None:
146
if self.alibi_slopes is not None:
147
att_masks = _make_alibi_bias(
148
self.alibi_slopes, query.dtype,
170
- prefill_meta.prompt_lens) # type: ignore
149
+ attn_metadata.prompt_lens) # type: ignore
150
elif self.sliding_window is not None:
151
att_masks = _make_sliding_window_bias(
173
- prefill_meta.prompt_lens, self.sliding_window,
152
+ attn_metadata.prompt_lens, self.sliding_window,
153
query.dtype) # type: ignore
154
else:
176
- att_masks = [None] * len(prefill_meta.prompt_lens)
177
- prefill_meta.attn_bias = att_masks
155
+ att_masks = [None] * len(attn_metadata.prompt_lens)
156
+ attn_metadata.attn_bias = att_masks
157
158
query = query.movedim(0, query.dim() - 2)
159
key = key.movedim(0, key.dim() - 2)
160
value = value.movedim(0, value.dim() - 2)
161
162
start = 0
184
- out = torch.empty((num_tokens, self.num_heads, self.head_size),
185
- dtype=query.dtype)
186
- for prompt_len, mask in zip(prefill_meta.prompt_lens,
187
- prefill_meta.attn_bias):
163
+ output = torch.empty(
164
+ (num_tokens, self.num_heads, self.head_size),
165
+ dtype=query.dtype)
166
+ for prompt_len, mask in zip(attn_metadata.prompt_lens,
167
+ attn_metadata.attn_bias):
168
end = start + prompt_len
169
sub_out = scaled_dot_product_attention(
170
query[:, start:end, :],
@@ -194,32 +174,28 @@ class TorchSDPABackendImpl(AttentionImpl):
174
dropout_p=0.0,
175
is_causal=not self.need_mask,
176
scale=self.scale).movedim(query.dim() - 2, 0)
197
- out[start:end, :, :] = sub_out
177
+ output[start:end, :, :] = sub_out
178
start = end
199
- assert out.shape == output[:num_prefill_tokens].shape
200
- output[:num_prefill_tokens] = out
179
else:
180
# prefix-enabled attention
181
raise RuntimeError(
182
"Torch SDPA backend doesn't support prefix decoding.")
183
206
- if decode_meta := attn_metadata.decode_metadata:
184
+ else:
185
# Decoding run.
208
- out = PagedAttention.forward_decode(
209
- decode_query,
186
+ output = PagedAttention.forward_decode(
187
+ query,
188
key_cache,
189
value_cache,
212
- decode_meta.block_tables,
213
- decode_meta.context_lens,
214
- decode_meta.max_context_len,
190
+ attn_metadata.block_tables,
191
+ attn_metadata.context_lens,
192
+ attn_metadata.max_context_len,
193
attn_metadata.kv_cache_dtype,
194
self.num_kv_heads,
195
self.scale,
196
self.alibi_slopes,
197
kv_scale,
198
)
221
- assert out.shape == output[num_prefill_tokens:].shape
222
- output[num_prefill_tokens:]
199
200
# Reshape the output tensor.
201
return output.view(-1, self.num_heads * self.head_size)
@@ -241,7 +217,7 @@ def _make_alibi_bias(
217
bias = bias[None, :] - bias[:, None]
218
219
num_heads = alibi_slopes.shape[0]
244
- bias = bias[None, :].expand(num_heads, prompt_len, prompt_len)
220
+ bias = bias[None, :].repeat((num_heads, 1, 1))
221
bias.mul_(alibi_slopes[:, None, None])
222
inf_mask = torch.empty(
223
(1, prompt_len, prompt_len),
vllm/executor/cpu_executor.py
+10
@@ -25,6 +25,7 @@ class CPUExecutor(ExecutorBase):
25
assert lora_config is None, "cpu backend doesn't support LoRA"
26
model_config = _verify_and_get_model_config(model_config)
27
cache_config = _verify_and_get_cache_config(cache_config)
28
+ scheduler_config = _verify_and_get_scheduler_config(scheduler_config)
29
30
self.model_config = model_config
31
self.cache_config = cache_config
@@ -116,6 +117,15 @@ def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
117
return config
118
119
120
+def _verify_and_get_scheduler_config(
121
+ config: SchedulerConfig) -> SchedulerConfig:
122
+ if config.chunked_prefill_enabled:
123
+ logger.warning("Chunked prefill is not supported on CPU, disable it.")
124
+ config.chunked_prefill_enabled = False
125
+
126
+ return config
127
+
128
+
129
def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
130
_GB = 1 << 30
131
if config.enable_prefix_caching:
vllm/utils.py
-1
@@ -372,7 +372,6 @@ def is_pin_memory_available() -> bool:
372
print_warning_once("Pin memory is not supported on Neuron.")
373
return False
374
elif is_cpu():
375
- print_warning_once("Pin memory is not supported on CPU.")
375
return False
376
return True
377
vllm/worker/cpu_model_runner.py
new
+408
@@ -0,0 +1,408 @@
1
+from typing import Dict, List, Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm.attention import AttentionMetadata, get_attn_backend
6
+from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
7
+ SchedulerConfig)
8
+from vllm.distributed import broadcast_tensor_dict
9
+from vllm.logger import init_logger
10
+from vllm.model_executor import SamplingMetadata
11
+from vllm.model_executor.model_loader import get_model
12
+from vllm.sampling_params import SamplingParams, SamplingType
13
+from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
14
+from vllm.utils import make_tensor_with_pad, maybe_expand_dim
15
+
16
+logger = init_logger(__name__)
17
+
18
+_PAD_SLOT_ID = -1
19
+
20
+
21
+class CPUModelRunner:
22
+
23
+ def __init__(
24
+ self,
25
+ model_config: ModelConfig,
26
+ parallel_config: ParallelConfig,
27
+ scheduler_config: SchedulerConfig,
28
+ device_config: DeviceConfig,
29
+ lora_config: Optional[LoRAConfig],
30
+ kv_cache_dtype: Optional[str] = "auto",
31
+ is_driver_worker: bool = False,
32
+ *args,
33
+ **kwargs,
34
+ ):
35
+ self.model_config = model_config
36
+ self.parallel_config = parallel_config
37
+ self.scheduler_config = scheduler_config
38
+ self.lora_config = lora_config
39
+ self.is_driver_worker = is_driver_worker
40
+
41
+ # model_config can be None in tests/samplers/test_sampler.py.
42
+ # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
43
+ self.sliding_window = (model_config.get_sliding_window()
44
+ if model_config is not None else None)
45
+ self.device_config = (device_config
46
+ if device_config is not None else DeviceConfig())
47
+ self.device = self.device_config.device
48
+
49
+ self.model = None
50
+ self.block_size = None # Set after initial profiling.
51
+
52
+ self.kv_cache_dtype = kv_cache_dtype
53
+
54
+ self.attn_backend = get_attn_backend(
55
+ self.model_config.dtype if model_config is not None else None)
56
+
57
+ def load_model(self) -> None:
58
+ self.model = get_model(self.model_config,
59
+ self.device_config,
60
+ lora_config=self.lora_config,
61
+ parallel_config=self.parallel_config,
62
+ scheduler_config=self.scheduler_config)
63
+
64
+ def _prepare_prompt(
65
+ self,
66
+ seq_group_metadata_list: List[SequenceGroupMetadata],
67
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int]]:
68
+ assert len(seq_group_metadata_list) > 0
69
+ input_tokens: List[int] = []
70
+ input_positions: List[int] = []
71
+ slot_mapping: List[int] = []
72
+ prompt_lens: List[int] = []
73
+
74
+ for seq_group_metadata in seq_group_metadata_list:
75
+ assert seq_group_metadata.is_prompt
76
+ seq_ids = list(seq_group_metadata.seq_data.keys())
77
+ assert len(seq_ids) == 1
78
+ seq_id = seq_ids[0]
79
+
80
+ seq_data = seq_group_metadata.seq_data[seq_id]
81
+ prompt_tokens = seq_data.get_token_ids()
82
+ computed_len = seq_data.get_num_computed_tokens()
83
+ prompt_len = len(prompt_tokens)
84
+
85
+ prompt_lens.append(prompt_len) # Prompt token num
86
+ input_tokens.extend(prompt_tokens) # Token ids
87
+
88
+ # Token position ids
89
+ # NOTE(woosuk): Here we assume that the first token in the prompt
90
+ # is always the first token in the sequence.
91
+ input_positions.extend(list(range(computed_len, prompt_len)))
92
+
93
+ # Compute the slot mapping.
94
+ block_table = seq_group_metadata.block_tables[seq_id]
95
+ # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
96
+ # where start_idx is max(0, prompt_len - sliding_window).
97
+ # For example, if the prompt len is 10, sliding window is 8, and
98
+ # block size is 4, the first two tokens are masked and the slot
99
+ # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
100
+ start_idx = 0
101
+ if self.sliding_window is not None:
102
+ start_idx = max(0, prompt_len - self.sliding_window)
103
+
104
+ for i in range(computed_len, prompt_len):
105
+ if i < start_idx:
106
+ slot_mapping.append(_PAD_SLOT_ID)
107
+ continue
108
+
109
+ block_number = block_table[i //
110
+ self.block_size] # type: ignore
111
+ block_offset = i % self.block_size # type: ignore
112
+ slot = block_number * self.block_size + block_offset
113
+ slot_mapping.append(slot)
114
+
115
+ num_prompt_tokens = len(input_tokens)
116
+
117
+ input_tokens = torch.tensor(input_tokens,
118
+ dtype=torch.long,
119
+ device=self.device) # type: ignore
120
+ input_positions = torch.tensor(input_positions,
121
+ dtype=torch.long,
122
+ device=self.device) # type: ignore
123
+ slot_mapping = torch.tensor(slot_mapping,
124
+ dtype=torch.long,
125
+ device=self.device) # type: ignore
126
+
127
+ attn_metadata = self.attn_backend.make_metadata(
128
+ is_prompt=True,
129
+ prompt_lens=prompt_lens,
130
+ num_prefills=len(prompt_lens),
131
+ num_prefill_tokens=num_prompt_tokens,
132
+ num_decode_tokens=0,
133
+ prefill_metadata=None,
134
+ decode_metadata=None,
135
+ max_context_len=None,
136
+ context_lens=None,
137
+ block_tables=torch.tensor([]),
138
+ slot_mapping=slot_mapping,
139
+ kv_cache_dtype=self.kv_cache_dtype,
140
+ )
141
+ return (
142
+ input_tokens,
143
+ input_positions,
144
+ attn_metadata,
145
+ prompt_lens,
146
+ )
147
+
148
+ def _prepare_decode(
149
+ self,
150
+ seq_group_metadata_list: List[SequenceGroupMetadata],
151
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
152
+ assert len(seq_group_metadata_list) > 0
153
+ input_tokens: List[int] = []
154
+ input_positions: List[int] = []
155
+ slot_mapping: List[int] = []
156
+ context_lens: List[int] = []
157
+ block_tables: List[List[int]] = []
158
+
159
+ for seq_group_metadata in seq_group_metadata_list:
160
+ assert not seq_group_metadata.is_prompt
161
+ assert seq_group_metadata.token_chunk_size == 1
162
+
163
+ seq_ids = list(seq_group_metadata.seq_data.keys())
164
+
165
+ for seq_id in seq_ids:
166
+ seq_data = seq_group_metadata.seq_data[seq_id]
167
+ generation_token = seq_data.get_last_token_id()
168
+ input_tokens.append(generation_token)
169
+
170
+ seq_len = seq_data.get_len()
171
+ position = seq_len - 1
172
+ input_positions.append(position)
173
+
174
+ context_len = seq_len if self.sliding_window is None else min(
175
+ seq_len, self.sliding_window)
176
+ context_lens.append(context_len)
177
+
178
+ block_table = seq_group_metadata.block_tables[seq_id]
179
+ block_number = block_table[position // self.block_size]
180
+ block_offset = position % self.block_size
181
+ slot = block_number * self.block_size + block_offset
182
+ slot_mapping.append(slot)
183
+
184
+ if self.sliding_window is not None:
185
+ sliding_window_blocks = (self.sliding_window //
186
+ self.block_size)
187
+ block_table = block_table[-sliding_window_blocks:]
188
+ block_tables.append(block_table)
189
+
190
+ max_context_len = max(context_lens)
191
+
192
+ input_tokens = torch.tensor(input_tokens,
193
+ dtype=torch.long,
194
+ device=self.device)
195
+ input_positions = torch.tensor(input_positions,
196
+ dtype=torch.long,
197
+ device=self.device)
198
+ slot_mapping = torch.tensor(slot_mapping,
199
+ dtype=torch.long,
200
+ device=self.device)
201
+ context_lens = torch.tensor(context_lens,
202
+ dtype=torch.int,
203
+ device=self.device)
204
+
205
+ max_block_table_len = max(
206
+ len(block_table) for block_table in block_tables)
207
+ block_tables = make_tensor_with_pad(
208
+ block_tables,
209
+ max_len=max_block_table_len,
210
+ pad=0,
211
+ dtype=torch.int,
212
+ device=self.device,
213
+ )
214
+
215
+ attn_metadata = self.attn_backend.make_metadata(
216
+ is_prompt=False,
217
+ slot_mapping=slot_mapping,
218
+ prompt_lens=None,
219
+ num_prefill_tokens=0,
220
+ num_decode_tokens=len(input_tokens),
221
+ max_context_len=max_context_len,
222
+ num_prefills=0,
223
+ prefill_metadata=None,
224
+ decode_metadata=None,
225
+ context_lens=context_lens,
226
+ block_tables=block_tables,
227
+ kv_cache_dtype=self.kv_cache_dtype,
228
+ )
229
+ return (
230
+ input_tokens,
231
+ input_positions,
232
+ attn_metadata,
233
+ )
234
+
235
+ def _prepare_sample(
236
+ self,
237
+ seq_group_metadata_list: List[SequenceGroupMetadata],
238
+ prompt_lens: List[int],
239
+ ) -> SamplingMetadata:
240
+ seq_groups: List[Tuple[List[int], SamplingParams]] = []
241
+ selected_token_indices: List[int] = []
242
+ generators: List[torch.Generator] = []
243
+ selected_token_start_idx = 0
244
+ categorized_sample_indices = {t: [] for t in SamplingType}
245
+ categorized_sample_indices_start_idx = 0
246
+ categorized_sampled_token_indices_start_idx = 0
247
+
248
+ for i, seq_group_metadata in enumerate(seq_group_metadata_list):
249
+ seq_ids = list(seq_group_metadata.seq_data.keys())
250
+ sampling_params = seq_group_metadata.sampling_params
251
+ seq_groups.append((seq_ids, sampling_params))
252
+
253
+ if seq_group_metadata.is_prompt:
254
+ assert len(seq_ids) == 1
255
+ subquery_len = prompt_lens[i]
256
+ if sampling_params.prompt_logprobs is not None:
257
+ # NOTE: prompt token positions do not need sample, skip
258
+ categorized_sample_indices_start_idx += subquery_len - 1
259
+
260
+ categorized_sample_indices[
261
+ sampling_params.sampling_type].append([
262
+ categorized_sample_indices_start_idx,
263
+ categorized_sampled_token_indices_start_idx
264
+ ])
265
+ categorized_sample_indices_start_idx += 1
266
+ categorized_sampled_token_indices_start_idx += 1
267
+
268
+ if sampling_params.prompt_logprobs is not None:
269
+ selected_token_indices.extend(
270
+ range(selected_token_start_idx,
271
+ selected_token_start_idx + subquery_len - 1))
272
+ selected_token_indices.append(selected_token_start_idx +
273
+ subquery_len - 1)
274
+ selected_token_start_idx += subquery_len
275
+
276
+ if sampling_params.seed is not None:
277
+ seq_group_metadata.state.generator = torch.Generator(
278
+ device=self.device).manual_seed(sampling_params.seed)
279
+ else:
280
+ num_seqs = len(seq_ids)
281
+ selected_token_indices.extend(
282
+ range(selected_token_start_idx,
283
+ selected_token_start_idx + num_seqs))
284
+ selected_token_start_idx += num_seqs
285
+
286
+ categorized_sample_indices[
287
+ sampling_params.sampling_type].extend(
288
+ zip(
289
+ range(
290
+ categorized_sample_indices_start_idx,
291
+ categorized_sample_indices_start_idx +
292
+ num_seqs),
293
+ range(
294
+ categorized_sampled_token_indices_start_idx,
295
+ categorized_sampled_token_indices_start_idx +
296
+ num_seqs)))
297
+ categorized_sample_indices_start_idx += num_seqs
298
+ categorized_sampled_token_indices_start_idx += num_seqs
299
+
300
+ if sampling_params.seed is not None:
301
+ generators.append(seq_group_metadata.state.generator)
302
+
303
+ selected_token_indices = torch.tensor(selected_token_indices,
304
+ dtype=torch.long)
305
+
306
+ categorized_sample_indices = {
307
+ t: maybe_expand_dim(torch.tensor(seq_ids, dtype=torch.int), 2, 2)
308
+ for t, seq_ids in categorized_sample_indices.items()
309
+ }
310
+
311
+ seq_data: Dict[int, SequenceData] = {}
312
+ for seq_group_metadata in seq_group_metadata_list:
313
+ seq_data.update(seq_group_metadata.seq_data)
314
+
315
+ sampling_metadata = SamplingMetadata(
316
+ seq_groups=seq_groups,
317
+ seq_data=seq_data,
318
+ prompt_lens=prompt_lens,
319
+ selected_token_indices=selected_token_indices,
320
+ categorized_sample_indices=categorized_sample_indices,
321
+ generators=generators,
322
+ )
323
+ return sampling_metadata
324
+
325
+ def prepare_input_tensors(
326
+ self,
327
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
328
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata,
329
+ SamplingMetadata]:
330
+ if self.is_driver_worker:
331
+ # NOTE: We assume that all sequences in the group are all prompts or
332
+ # all decodes.
333
+ is_prompt = seq_group_metadata_list[0].is_prompt
334
+ # Prepare input tensors.
335
+ if is_prompt:
336
+ (input_tokens, input_positions, attn_metadata,
337
+ prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
338
+ else:
339
+ (input_tokens, input_positions,
340
+ attn_metadata) = self._prepare_decode(seq_group_metadata_list)
341
+ prompt_lens = []
342
+ sampling_metadata = self._prepare_sample(seq_group_metadata_list,
343
+ prompt_lens)
344
+ # Broadcast the metadata.
345
+ metadata_dict = {
346
+ "input_tokens": input_tokens,
347
+ "input_positions": input_positions,
348
+ "selected_token_indices":
349
+ sampling_metadata.selected_token_indices,
350
+ }
351
+ metadata_dict.update(attn_metadata.asdict_zerocopy())
352
+ broadcast_tensor_dict(metadata_dict, src=0)
353
+ else:
354
+ metadata_dict = broadcast_tensor_dict(src=0)
355
+ input_tokens = metadata_dict.pop("input_tokens")
356
+ input_positions = metadata_dict.pop("input_positions")
357
+ selected_token_indices = metadata_dict.pop(
358
+ "selected_token_indices")
359
+ attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
360
+ sampling_metadata = SamplingMetadata(
361
+ seq_groups=None,
362
+ seq_data=None,
363
+ prompt_lens=None,
364
+ selected_token_indices=selected_token_indices,
365
+ categorized_sample_indices=None,
366
+ generators=None,
367
+ perform_sampling=False,
368
+ )
369
+
370
+ return (
371
+ input_tokens,
372
+ input_positions,
373
+ attn_metadata,
374
+ sampling_metadata,
375
+ )
376
+
377
+ @torch.inference_mode()
378
+ def execute_model(
379
+ self,
380
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
381
+ kv_caches: List[torch.Tensor],
382
+ ) -> Optional[SamplerOutput]:
383
+ (input_tokens, input_positions, attn_metadata, sampling_metadata
384
+ ) = self.prepare_input_tensors(seq_group_metadata_list)
385
+
386
+ model_executable = self.model
387
+ execute_model_kwargs = {
388
+ "input_ids": input_tokens,
389
+ "positions": input_positions,
390
+ "kv_caches": kv_caches,
391
+ "attn_metadata": attn_metadata,
392
+ }
393
+
394
+ hidden_states = model_executable(**execute_model_kwargs)
395
+
396
+ # Compute the logits.
397
+ logits = self.model.compute_logits(hidden_states, sampling_metadata)
398
+
399
+ # Only perform sampling in the driver worker.
400
+ if not sampling_metadata.perform_sampling:
401
+ return None
402
+
403
+ # Sample the next token.
404
+ output = self.model.sample(
405
+ logits=logits,
406
+ sampling_metadata=sampling_metadata,
407
+ )
408
+ return output
vllm/worker/cpu_worker.py
+1
-12
@@ -12,25 +12,14 @@ from vllm.distributed import (broadcast_tensor_dict,
12
init_distributed_environment)
13
from vllm.logger import init_logger
14
from vllm.model_executor import set_random_seed
15
-from vllm.model_executor.model_loader import get_model
15
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
16
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18
-from vllm.worker.model_runner import ModelRunner
17
+from vllm.worker.cpu_model_runner import CPUModelRunner
18
from vllm.worker.worker_base import LoraNotSupportedWorkerBase
19
20
logger = init_logger(__name__)
21
22
24
-class CPUModelRunner(ModelRunner):
25
-
26
- def load_model(self) -> None:
27
- self.model = get_model(self.model_config,
28
- self.device_config,
29
- lora_config=self.lora_config,
30
- parallel_config=self.parallel_config,
31
- scheduler_config=self.scheduler_config)
32
-
33
-
23
class CPUCacheEngine:
24
"""Manages the KV cache for CPU backend.
25