@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8afca50889
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Hardware][Intel] Isolate CPUModelRunner and ModelRunner for better maintenance (#3824)

bigPYJ1151 committed Apr 12, 2024 at 02:56 UTC 8afca50889bad6ad987c523c48c31fc52fcb72e4
5 files changed +443 -61
vllm/attention/backends/torch_sdpa.py
+24 -48
@@ -50,20 +50,15 @@ class TorchSDPABackend(AttentionBackend):
50
51
52 @dataclass
53 -class TorchSDPAMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
53 +class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata,
54 + AttentionMetadataPerStage):
55 """Metadata for TorchSDPABackend.
56 """
57 # Currently, input sequences can only contain all prompts
58 # or all decoding. True if all sequences are prompts.
59 is_prompt: bool
60 + slot_mapping: torch.Tensor
61 prompt_lens: Optional[List[int]]
60 - prompt_lens_tensor: Optional[torch.Tensor]
61 -
62 - max_subquery_len: Optional[int] = None
63 - max_prompt_len: Optional[int] = None
64 - subquery_start_loc: Optional[torch.Tensor] = None
65 - seq_start_loc: Optional[torch.Tensor] = None
66 - use_cuda_graph: bool = False
62
63 def __post_init__(self):
64 # Set during the execution of the first attention op.
@@ -111,7 +106,7 @@ class TorchSDPABackendImpl(AttentionImpl):
106 key: torch.Tensor,
107 value: torch.Tensor,
108 kv_cache: Optional[torch.Tensor],
114 - attn_metadata: AttentionMetadata[TorchSDPAMetadata],
109 + attn_metadata: TorchSDPAMetadata,
110 kv_scale: float,
111 ) -> torch.Tensor:
112 """Forward pass with torch SDPA and PagedAttention.
@@ -140,51 +135,36 @@ class TorchSDPABackendImpl(AttentionImpl):
135 attn_metadata.kv_cache_dtype,
136 kv_scale)
137
143 - num_prefill_tokens = attn_metadata.num_prefill_tokens
144 - num_decode_tokens = attn_metadata.num_decode_tokens
145 - assert key.shape[0] == num_prefill_tokens + num_decode_tokens
146 - assert value.shape[0] == num_prefill_tokens + num_decode_tokens
147 -
148 - output = torch.empty_like(query)
149 - # Query for decode. KV is not needed because it is already cached.
150 - decode_query = query[num_prefill_tokens:]
151 - # QKV for prefill.
152 - query = query[:num_prefill_tokens]
153 - key = key[:num_prefill_tokens]
154 - value = value[:num_prefill_tokens]
155 -
156 - assert query.shape[0] == num_prefill_tokens
157 - assert decode_query.shape[0] == num_decode_tokens
158 -
159 - if prefill_meta := attn_metadata.prefill_metadata:
160 - if (kv_cache is None or prefill_meta.block_tables.numel() == 0):
138 + if attn_metadata.is_prompt:
139 + if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
140 if self.num_kv_heads != self.num_heads:
141 key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
142 value = value.repeat_interleave(self.num_queries_per_kv,
143 dim=1)
144
166 - if prefill_meta.attn_bias is None:
145 + if attn_metadata.attn_bias is None:
146 if self.alibi_slopes is not None:
147 att_masks = _make_alibi_bias(
148 self.alibi_slopes, query.dtype,
170 - prefill_meta.prompt_lens) # type: ignore
149 + attn_metadata.prompt_lens) # type: ignore
150 elif self.sliding_window is not None:
151 att_masks = _make_sliding_window_bias(
173 - prefill_meta.prompt_lens, self.sliding_window,
152 + attn_metadata.prompt_lens, self.sliding_window,
153 query.dtype) # type: ignore
154 else:
176 - att_masks = [None] * len(prefill_meta.prompt_lens)
177 - prefill_meta.attn_bias = att_masks
155 + att_masks = [None] * len(attn_metadata.prompt_lens)
156 + attn_metadata.attn_bias = att_masks
157
158 query = query.movedim(0, query.dim() - 2)
159 key = key.movedim(0, key.dim() - 2)
160 value = value.movedim(0, value.dim() - 2)
161
162 start = 0
184 - out = torch.empty((num_tokens, self.num_heads, self.head_size),
185 - dtype=query.dtype)
186 - for prompt_len, mask in zip(prefill_meta.prompt_lens,
187 - prefill_meta.attn_bias):
163 + output = torch.empty(
164 + (num_tokens, self.num_heads, self.head_size),
165 + dtype=query.dtype)
166 + for prompt_len, mask in zip(attn_metadata.prompt_lens,
167 + attn_metadata.attn_bias):
168 end = start + prompt_len
169 sub_out = scaled_dot_product_attention(
170 query[:, start:end, :],
@@ -194,32 +174,28 @@ class TorchSDPABackendImpl(AttentionImpl):
174 dropout_p=0.0,
175 is_causal=not self.need_mask,
176 scale=self.scale).movedim(query.dim() - 2, 0)
197 - out[start:end, :, :] = sub_out
177 + output[start:end, :, :] = sub_out
178 start = end
199 - assert out.shape == output[:num_prefill_tokens].shape
200 - output[:num_prefill_tokens] = out
179 else:
180 # prefix-enabled attention
181 raise RuntimeError(
182 "Torch SDPA backend doesn't support prefix decoding.")
183
206 - if decode_meta := attn_metadata.decode_metadata:
184 + else:
185 # Decoding run.
208 - out = PagedAttention.forward_decode(
209 - decode_query,
186 + output = PagedAttention.forward_decode(
187 + query,
188 key_cache,
189 value_cache,
212 - decode_meta.block_tables,
213 - decode_meta.context_lens,
214 - decode_meta.max_context_len,
190 + attn_metadata.block_tables,
191 + attn_metadata.context_lens,
192 + attn_metadata.max_context_len,
193 attn_metadata.kv_cache_dtype,
194 self.num_kv_heads,
195 self.scale,
196 self.alibi_slopes,
197 kv_scale,
198 )
221 - assert out.shape == output[num_prefill_tokens:].shape
222 - output[num_prefill_tokens:]
199
200 # Reshape the output tensor.
201 return output.view(-1, self.num_heads * self.head_size)
@@ -241,7 +217,7 @@ def _make_alibi_bias(
217 bias = bias[None, :] - bias[:, None]
218
219 num_heads = alibi_slopes.shape[0]
244 - bias = bias[None, :].expand(num_heads, prompt_len, prompt_len)
220 + bias = bias[None, :].repeat((num_heads, 1, 1))
221 bias.mul_(alibi_slopes[:, None, None])
222 inf_mask = torch.empty(
223 (1, prompt_len, prompt_len),
vllm/executor/cpu_executor.py
+10
@@ -25,6 +25,7 @@ class CPUExecutor(ExecutorBase):
25 assert lora_config is None, "cpu backend doesn't support LoRA"
26 model_config = _verify_and_get_model_config(model_config)
27 cache_config = _verify_and_get_cache_config(cache_config)
28 + scheduler_config = _verify_and_get_scheduler_config(scheduler_config)
29
30 self.model_config = model_config
31 self.cache_config = cache_config
@@ -116,6 +117,15 @@ def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
117 return config
118
119
120 +def _verify_and_get_scheduler_config(
121 + config: SchedulerConfig) -> SchedulerConfig:
122 + if config.chunked_prefill_enabled:
123 + logger.warning("Chunked prefill is not supported on CPU, disable it.")
124 + config.chunked_prefill_enabled = False
125 +
126 + return config
127 +
128 +
129 def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
130 _GB = 1 << 30
131 if config.enable_prefix_caching:
vllm/utils.py
-1
@@ -372,7 +372,6 @@ def is_pin_memory_available() -> bool:
372 print_warning_once("Pin memory is not supported on Neuron.")
373 return False
374 elif is_cpu():
375 - print_warning_once("Pin memory is not supported on CPU.")
375 return False
376 return True
377
vllm/worker/cpu_model_runner.py new
+408
@@ -0,0 +1,408 @@
1 +from typing import Dict, List, Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm.attention import AttentionMetadata, get_attn_backend
6 +from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
7 + SchedulerConfig)
8 +from vllm.distributed import broadcast_tensor_dict
9 +from vllm.logger import init_logger
10 +from vllm.model_executor import SamplingMetadata
11 +from vllm.model_executor.model_loader import get_model
12 +from vllm.sampling_params import SamplingParams, SamplingType
13 +from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
14 +from vllm.utils import make_tensor_with_pad, maybe_expand_dim
15 +
16 +logger = init_logger(__name__)
17 +
18 +_PAD_SLOT_ID = -1
19 +
20 +
21 +class CPUModelRunner:
22 +
23 + def __init__(
24 + self,
25 + model_config: ModelConfig,
26 + parallel_config: ParallelConfig,
27 + scheduler_config: SchedulerConfig,
28 + device_config: DeviceConfig,
29 + lora_config: Optional[LoRAConfig],
30 + kv_cache_dtype: Optional[str] = "auto",
31 + is_driver_worker: bool = False,
32 + *args,
33 + **kwargs,
34 + ):
35 + self.model_config = model_config
36 + self.parallel_config = parallel_config
37 + self.scheduler_config = scheduler_config
38 + self.lora_config = lora_config
39 + self.is_driver_worker = is_driver_worker
40 +
41 + # model_config can be None in tests/samplers/test_sampler.py.
42 + # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
43 + self.sliding_window = (model_config.get_sliding_window()
44 + if model_config is not None else None)
45 + self.device_config = (device_config
46 + if device_config is not None else DeviceConfig())
47 + self.device = self.device_config.device
48 +
49 + self.model = None
50 + self.block_size = None # Set after initial profiling.
51 +
52 + self.kv_cache_dtype = kv_cache_dtype
53 +
54 + self.attn_backend = get_attn_backend(
55 + self.model_config.dtype if model_config is not None else None)
56 +
57 + def load_model(self) -> None:
58 + self.model = get_model(self.model_config,
59 + self.device_config,
60 + lora_config=self.lora_config,
61 + parallel_config=self.parallel_config,
62 + scheduler_config=self.scheduler_config)
63 +
64 + def _prepare_prompt(
65 + self,
66 + seq_group_metadata_list: List[SequenceGroupMetadata],
67 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int]]:
68 + assert len(seq_group_metadata_list) > 0
69 + input_tokens: List[int] = []
70 + input_positions: List[int] = []
71 + slot_mapping: List[int] = []
72 + prompt_lens: List[int] = []
73 +
74 + for seq_group_metadata in seq_group_metadata_list:
75 + assert seq_group_metadata.is_prompt
76 + seq_ids = list(seq_group_metadata.seq_data.keys())
77 + assert len(seq_ids) == 1
78 + seq_id = seq_ids[0]
79 +
80 + seq_data = seq_group_metadata.seq_data[seq_id]
81 + prompt_tokens = seq_data.get_token_ids()
82 + computed_len = seq_data.get_num_computed_tokens()
83 + prompt_len = len(prompt_tokens)
84 +
85 + prompt_lens.append(prompt_len) # Prompt token num
86 + input_tokens.extend(prompt_tokens) # Token ids
87 +
88 + # Token position ids
89 + # NOTE(woosuk): Here we assume that the first token in the prompt
90 + # is always the first token in the sequence.
91 + input_positions.extend(list(range(computed_len, prompt_len)))
92 +
93 + # Compute the slot mapping.
94 + block_table = seq_group_metadata.block_tables[seq_id]
95 + # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
96 + # where start_idx is max(0, prompt_len - sliding_window).
97 + # For example, if the prompt len is 10, sliding window is 8, and
98 + # block size is 4, the first two tokens are masked and the slot
99 + # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
100 + start_idx = 0
101 + if self.sliding_window is not None:
102 + start_idx = max(0, prompt_len - self.sliding_window)
103 +
104 + for i in range(computed_len, prompt_len):
105 + if i < start_idx:
106 + slot_mapping.append(_PAD_SLOT_ID)
107 + continue
108 +
109 + block_number = block_table[i //
110 + self.block_size] # type: ignore
111 + block_offset = i % self.block_size # type: ignore
112 + slot = block_number * self.block_size + block_offset
113 + slot_mapping.append(slot)
114 +
115 + num_prompt_tokens = len(input_tokens)
116 +
117 + input_tokens = torch.tensor(input_tokens,
118 + dtype=torch.long,
119 + device=self.device) # type: ignore
120 + input_positions = torch.tensor(input_positions,
121 + dtype=torch.long,
122 + device=self.device) # type: ignore
123 + slot_mapping = torch.tensor(slot_mapping,
124 + dtype=torch.long,
125 + device=self.device) # type: ignore
126 +
127 + attn_metadata = self.attn_backend.make_metadata(
128 + is_prompt=True,
129 + prompt_lens=prompt_lens,
130 + num_prefills=len(prompt_lens),
131 + num_prefill_tokens=num_prompt_tokens,
132 + num_decode_tokens=0,
133 + prefill_metadata=None,
134 + decode_metadata=None,
135 + max_context_len=None,
136 + context_lens=None,
137 + block_tables=torch.tensor([]),
138 + slot_mapping=slot_mapping,
139 + kv_cache_dtype=self.kv_cache_dtype,
140 + )
141 + return (
142 + input_tokens,
143 + input_positions,
144 + attn_metadata,
145 + prompt_lens,
146 + )
147 +
148 + def _prepare_decode(
149 + self,
150 + seq_group_metadata_list: List[SequenceGroupMetadata],
151 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
152 + assert len(seq_group_metadata_list) > 0
153 + input_tokens: List[int] = []
154 + input_positions: List[int] = []
155 + slot_mapping: List[int] = []
156 + context_lens: List[int] = []
157 + block_tables: List[List[int]] = []
158 +
159 + for seq_group_metadata in seq_group_metadata_list:
160 + assert not seq_group_metadata.is_prompt
161 + assert seq_group_metadata.token_chunk_size == 1
162 +
163 + seq_ids = list(seq_group_metadata.seq_data.keys())
164 +
165 + for seq_id in seq_ids:
166 + seq_data = seq_group_metadata.seq_data[seq_id]
167 + generation_token = seq_data.get_last_token_id()
168 + input_tokens.append(generation_token)
169 +
170 + seq_len = seq_data.get_len()
171 + position = seq_len - 1
172 + input_positions.append(position)
173 +
174 + context_len = seq_len if self.sliding_window is None else min(
175 + seq_len, self.sliding_window)
176 + context_lens.append(context_len)
177 +
178 + block_table = seq_group_metadata.block_tables[seq_id]
179 + block_number = block_table[position // self.block_size]
180 + block_offset = position % self.block_size
181 + slot = block_number * self.block_size + block_offset
182 + slot_mapping.append(slot)
183 +
184 + if self.sliding_window is not None:
185 + sliding_window_blocks = (self.sliding_window //
186 + self.block_size)
187 + block_table = block_table[-sliding_window_blocks:]
188 + block_tables.append(block_table)
189 +
190 + max_context_len = max(context_lens)
191 +
192 + input_tokens = torch.tensor(input_tokens,
193 + dtype=torch.long,
194 + device=self.device)
195 + input_positions = torch.tensor(input_positions,
196 + dtype=torch.long,
197 + device=self.device)
198 + slot_mapping = torch.tensor(slot_mapping,
199 + dtype=torch.long,
200 + device=self.device)
201 + context_lens = torch.tensor(context_lens,
202 + dtype=torch.int,
203 + device=self.device)
204 +
205 + max_block_table_len = max(
206 + len(block_table) for block_table in block_tables)
207 + block_tables = make_tensor_with_pad(
208 + block_tables,
209 + max_len=max_block_table_len,
210 + pad=0,
211 + dtype=torch.int,
212 + device=self.device,
213 + )
214 +
215 + attn_metadata = self.attn_backend.make_metadata(
216 + is_prompt=False,
217 + slot_mapping=slot_mapping,
218 + prompt_lens=None,
219 + num_prefill_tokens=0,
220 + num_decode_tokens=len(input_tokens),
221 + max_context_len=max_context_len,
222 + num_prefills=0,
223 + prefill_metadata=None,
224 + decode_metadata=None,
225 + context_lens=context_lens,
226 + block_tables=block_tables,
227 + kv_cache_dtype=self.kv_cache_dtype,
228 + )
229 + return (
230 + input_tokens,
231 + input_positions,
232 + attn_metadata,
233 + )
234 +
235 + def _prepare_sample(
236 + self,
237 + seq_group_metadata_list: List[SequenceGroupMetadata],
238 + prompt_lens: List[int],
239 + ) -> SamplingMetadata:
240 + seq_groups: List[Tuple[List[int], SamplingParams]] = []
241 + selected_token_indices: List[int] = []
242 + generators: List[torch.Generator] = []
243 + selected_token_start_idx = 0
244 + categorized_sample_indices = {t: [] for t in SamplingType}
245 + categorized_sample_indices_start_idx = 0
246 + categorized_sampled_token_indices_start_idx = 0
247 +
248 + for i, seq_group_metadata in enumerate(seq_group_metadata_list):
249 + seq_ids = list(seq_group_metadata.seq_data.keys())
250 + sampling_params = seq_group_metadata.sampling_params
251 + seq_groups.append((seq_ids, sampling_params))
252 +
253 + if seq_group_metadata.is_prompt:
254 + assert len(seq_ids) == 1
255 + subquery_len = prompt_lens[i]
256 + if sampling_params.prompt_logprobs is not None:
257 + # NOTE: prompt token positions do not need sample, skip
258 + categorized_sample_indices_start_idx += subquery_len - 1
259 +
260 + categorized_sample_indices[
261 + sampling_params.sampling_type].append([
262 + categorized_sample_indices_start_idx,
263 + categorized_sampled_token_indices_start_idx
264 + ])
265 + categorized_sample_indices_start_idx += 1
266 + categorized_sampled_token_indices_start_idx += 1
267 +
268 + if sampling_params.prompt_logprobs is not None:
269 + selected_token_indices.extend(
270 + range(selected_token_start_idx,
271 + selected_token_start_idx + subquery_len - 1))
272 + selected_token_indices.append(selected_token_start_idx +
273 + subquery_len - 1)
274 + selected_token_start_idx += subquery_len
275 +
276 + if sampling_params.seed is not None:
277 + seq_group_metadata.state.generator = torch.Generator(
278 + device=self.device).manual_seed(sampling_params.seed)
279 + else:
280 + num_seqs = len(seq_ids)
281 + selected_token_indices.extend(
282 + range(selected_token_start_idx,
283 + selected_token_start_idx + num_seqs))
284 + selected_token_start_idx += num_seqs
285 +
286 + categorized_sample_indices[
287 + sampling_params.sampling_type].extend(
288 + zip(
289 + range(
290 + categorized_sample_indices_start_idx,
291 + categorized_sample_indices_start_idx +
292 + num_seqs),
293 + range(
294 + categorized_sampled_token_indices_start_idx,
295 + categorized_sampled_token_indices_start_idx +
296 + num_seqs)))
297 + categorized_sample_indices_start_idx += num_seqs
298 + categorized_sampled_token_indices_start_idx += num_seqs
299 +
300 + if sampling_params.seed is not None:
301 + generators.append(seq_group_metadata.state.generator)
302 +
303 + selected_token_indices = torch.tensor(selected_token_indices,
304 + dtype=torch.long)
305 +
306 + categorized_sample_indices = {
307 + t: maybe_expand_dim(torch.tensor(seq_ids, dtype=torch.int), 2, 2)
308 + for t, seq_ids in categorized_sample_indices.items()
309 + }
310 +
311 + seq_data: Dict[int, SequenceData] = {}
312 + for seq_group_metadata in seq_group_metadata_list:
313 + seq_data.update(seq_group_metadata.seq_data)
314 +
315 + sampling_metadata = SamplingMetadata(
316 + seq_groups=seq_groups,
317 + seq_data=seq_data,
318 + prompt_lens=prompt_lens,
319 + selected_token_indices=selected_token_indices,
320 + categorized_sample_indices=categorized_sample_indices,
321 + generators=generators,
322 + )
323 + return sampling_metadata
324 +
325 + def prepare_input_tensors(
326 + self,
327 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
328 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata,
329 + SamplingMetadata]:
330 + if self.is_driver_worker:
331 + # NOTE: We assume that all sequences in the group are all prompts or
332 + # all decodes.
333 + is_prompt = seq_group_metadata_list[0].is_prompt
334 + # Prepare input tensors.
335 + if is_prompt:
336 + (input_tokens, input_positions, attn_metadata,
337 + prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
338 + else:
339 + (input_tokens, input_positions,
340 + attn_metadata) = self._prepare_decode(seq_group_metadata_list)
341 + prompt_lens = []
342 + sampling_metadata = self._prepare_sample(seq_group_metadata_list,
343 + prompt_lens)
344 + # Broadcast the metadata.
345 + metadata_dict = {
346 + "input_tokens": input_tokens,
347 + "input_positions": input_positions,
348 + "selected_token_indices":
349 + sampling_metadata.selected_token_indices,
350 + }
351 + metadata_dict.update(attn_metadata.asdict_zerocopy())
352 + broadcast_tensor_dict(metadata_dict, src=0)
353 + else:
354 + metadata_dict = broadcast_tensor_dict(src=0)
355 + input_tokens = metadata_dict.pop("input_tokens")
356 + input_positions = metadata_dict.pop("input_positions")
357 + selected_token_indices = metadata_dict.pop(
358 + "selected_token_indices")
359 + attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
360 + sampling_metadata = SamplingMetadata(
361 + seq_groups=None,
362 + seq_data=None,
363 + prompt_lens=None,
364 + selected_token_indices=selected_token_indices,
365 + categorized_sample_indices=None,
366 + generators=None,
367 + perform_sampling=False,
368 + )
369 +
370 + return (
371 + input_tokens,
372 + input_positions,
373 + attn_metadata,
374 + sampling_metadata,
375 + )
376 +
377 + @torch.inference_mode()
378 + def execute_model(
379 + self,
380 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
381 + kv_caches: List[torch.Tensor],
382 + ) -> Optional[SamplerOutput]:
383 + (input_tokens, input_positions, attn_metadata, sampling_metadata
384 + ) = self.prepare_input_tensors(seq_group_metadata_list)
385 +
386 + model_executable = self.model
387 + execute_model_kwargs = {
388 + "input_ids": input_tokens,
389 + "positions": input_positions,
390 + "kv_caches": kv_caches,
391 + "attn_metadata": attn_metadata,
392 + }
393 +
394 + hidden_states = model_executable(**execute_model_kwargs)
395 +
396 + # Compute the logits.
397 + logits = self.model.compute_logits(hidden_states, sampling_metadata)
398 +
399 + # Only perform sampling in the driver worker.
400 + if not sampling_metadata.perform_sampling:
401 + return None
402 +
403 + # Sample the next token.
404 + output = self.model.sample(
405 + logits=logits,
406 + sampling_metadata=sampling_metadata,
407 + )
408 + return output
vllm/worker/cpu_worker.py
+1 -12
@@ -12,25 +12,14 @@ from vllm.distributed import (broadcast_tensor_dict,
12 init_distributed_environment)
13 from vllm.logger import init_logger
14 from vllm.model_executor import set_random_seed
15 -from vllm.model_executor.model_loader import get_model
15 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
16 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18 -from vllm.worker.model_runner import ModelRunner
17 +from vllm.worker.cpu_model_runner import CPUModelRunner
18 from vllm.worker.worker_base import LoraNotSupportedWorkerBase
19
20 logger = init_logger(__name__)
21
22
24 -class CPUModelRunner(ModelRunner):
25 -
26 - def load_model(self) -> None:
27 - self.model = get_model(self.model_config,
28 - self.device_config,
29 - lora_config=self.lora_config,
30 - parallel_config=self.parallel_config,
31 - scheduler_config=self.scheduler_config)
32 -
33 -
23 class CPUCacheEngine:
24 """Manages the KV cache for CPU backend.
25