@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4634c8728b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[TPU] Refactor TPU worker & model runner (#6506)

Woosuk Kwon committed Jul 18, 2024 at 01:34 UTC 4634c8728b0fe30a0b2da22dd4a4c8d8a1d9213b
2 files changed +272 -166
vllm/worker/tpu_model_runner.py
+200 -97
@@ -1,5 +1,6 @@
1 import time
2 -from typing import List, Optional, Tuple
2 +from dataclasses import dataclass
3 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
4
5 import numpy as np
6 import torch
@@ -12,10 +13,16 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
13 from vllm.logger import init_logger
14 from vllm.model_executor.model_loader import get_model
15 from vllm.model_executor.sampling_metadata import SamplingMetadata
15 -from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
16 - SamplerOutput, SequenceGroupMetadata,
16 +from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
17 + Logprob, SamplerOutput, SequenceGroupMetadata,
18 SequenceOutput)
18 -from vllm.utils import make_tensor_with_pad
19 +from vllm.worker.model_runner_base import (
20 + ModelRunnerBase, ModelRunnerInputBase,
21 + _add_attn_metadata_broadcastable_dict,
22 + _init_attn_metadata_from_tensor_dict)
23 +
24 +if TYPE_CHECKING:
25 + from vllm.attention.backends.abstract import AttentionBackend
26
27 logger = init_logger(__name__)
28
@@ -27,7 +34,44 @@ _ENABLE_TOP_P = False
34 _MAX_NUM_SAMPLES = 128
35
36
30 -class TPUModelRunner:
37 +@dataclass(frozen=True)
38 +class ModelInputForTPU(ModelRunnerInputBase):
39 + token_ids: torch.Tensor
40 + position_ids: torch.Tensor
41 + attn_metadata: AttentionMetadata
42 + input_lens: torch.Tensor
43 + t: torch.Tensor
44 + p: torch.Tensor
45 + num_samples: int
46 + best_of: List[int]
47 + seq_groups: List[List[int]]
48 +
49 + def as_broadcastable_tensor_dict(
50 + self) -> Dict[str, Union[int, torch.Tensor]]:
51 + tensor_dict = {
52 + "token_ids": self.token_ids,
53 + "position_ids": self.position_ids,
54 + "input_lens": self.input_lens,
55 + "t": self.t,
56 + "p": self.p,
57 + "num_samples": self.num_samples,
58 + }
59 + _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
60 + return tensor_dict
61 +
62 + @classmethod
63 + def from_broadcasted_tensor_dict(
64 + cls: Type["ModelInputForTPU"],
65 + tensor_dict: Dict[str, Any],
66 + attn_backend: Optional["AttentionBackend"] = None,
67 + ) -> "ModelInputForTPU":
68 + if attn_backend is not None:
69 + tensor_dict = _init_attn_metadata_from_tensor_dict(
70 + attn_backend, tensor_dict)
71 + return cls(**tensor_dict)
72 +
73 +
74 +class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
75
76 def __init__(
77 self,
@@ -79,6 +123,7 @@ class TPUModelRunner:
123 multimodal_config=self.multimodal_config,
124 lora_config=None,
125 )
126 + model = model.eval()
127 xm.wait_device_ops()
128
129 model = ModelWrapper(model)
@@ -147,8 +192,8 @@ class TPUModelRunner:
192
193 # Dummy run.
194 num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
150 - self.model(token_ids, position_ids, kv_caches, attn_metadata,
151 - input_lens, t, p, num_samples)
195 + self.model(token_ids, position_ids, attn_metadata, input_lens, t, p,
196 + num_samples, kv_caches)
197
198 def warmup_model(
199 self,
@@ -177,7 +222,7 @@ class TPUModelRunner:
222 # Decode
223 start = time.time()
224 seq_len = 1
180 - batch_size = 1
225 + batch_size = 8 # Must be in sync with _get_padded_batch_size()
226 while True:
227 self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=False)
228 xm.wait_device_ops()
@@ -195,10 +240,10 @@ class TPUModelRunner:
240 seq_group_metadata_list: List[SequenceGroupMetadata],
241 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, torch.Tensor]:
242 assert len(seq_group_metadata_list) > 0
198 - input_tokens: List[List[int]] = []
199 - input_positions: List[List[int]] = []
243 + input_tokens: List[int] = []
244 + input_positions: List[int] = []
245 prompt_lens: List[int] = []
201 - slot_mapping: List[List[int]] = []
246 + slot_mapping: List[int] = []
247
248 for seq_group_metadata in seq_group_metadata_list:
249 assert seq_group_metadata.is_prompt
@@ -212,50 +257,46 @@ class TPUModelRunner:
257 prompt_len = len(prompt_tokens)
258 prompt_lens.append(prompt_len)
259
215 - input_tokens.append(prompt_tokens)
216 - input_positions.append(list(range(prompt_len)))
260 + input_tokens.extend(prompt_tokens)
261 + input_positions.extend(list(range(prompt_len)))
262
263 assert seq_group_metadata.block_tables is not None
264 block_table = seq_group_metadata.block_tables[seq_id]
220 - slot_mapping.append([])
265 for i in range(prompt_len):
266 block_number = block_table[i // self.block_size]
267 block_offset = i % self.block_size
268 slot = block_number * self.block_size + block_offset
225 - slot_mapping[-1].append(slot)
269 + slot_mapping.append(slot)
270 +
271 + # Add paddings to EACH prompt to the smallest power of 2 that is
272 + # greater than or equal to the prompt length.
273 + # We pad the seq_len to reduce the compilation overhead.
274 + # We execute each prompt individually (i.e., with batch_size 1)
275 + # because the FlashAttention kernel does not support ragged inputs.
276 + # TODO(woosuk): Use SplashAttention to support ragged inputs.
277 + padded_prompt_len = _get_padded_prefill_len(prompt_len)
278 + num_paddings = padded_prompt_len - prompt_len
279 + input_tokens += [0] * num_paddings
280 + input_positions += [0] * num_paddings
281 + slot_mapping += [_PAD_SLOT_ID] * num_paddings
282
283 assert len(prompt_lens) > 0
284 num_prefills = len(prompt_lens)
229 - num_prefill_tokens = sum(prompt_lens)
230 -
231 - # Add paddings to make the shape [batch_size, max_prompt_len] where
232 - # max_prompt_len is smallest power of 2 that is greater than or equal
233 - # to the maximum prompt length.
234 - # We need the 2D input shape because the Pallas FlashAttention kernel
235 - # does not support packed 1D inputs.
236 - # We pad the seq_len to powers of 2 to reduce the compilation overhead.
237 - max_prompt_len = _get_padded_prefill_len(max(prompt_lens))
238 - input_tokens = make_tensor_with_pad(input_tokens,
239 - max_prompt_len,
240 - pad=0,
241 - dtype=torch.int32,
242 - device=self.device)
243 - input_positions = make_tensor_with_pad(input_positions,
244 - max_prompt_len,
245 - pad=0,
246 - dtype=torch.int32,
247 - device=self.device)
248 - slot_mapping = make_tensor_with_pad(slot_mapping,
249 - max_prompt_len,
250 - pad=_PAD_SLOT_ID,
251 - dtype=torch.int64,
252 - device=self.device)
285 + input_tokens = torch.tensor(input_tokens,
286 + dtype=torch.int32,
287 + device="cpu")
288 + input_positions = torch.tensor(input_positions,
289 + dtype=torch.int32,
290 + device="cpu")
291 + slot_mapping = torch.tensor(slot_mapping,
292 + dtype=torch.int64,
293 + device="cpu")
294 prompt_lens = torch.tensor(prompt_lens,
295 dtype=torch.int32,
255 - device=self.device)
296 + device="cpu")
297 attn_metadata = self.attn_backend.make_metadata(
298 num_prefills=num_prefills,
258 - num_prefill_tokens=num_prefill_tokens, # NOTE: This is not used.
299 + num_prefill_tokens=0, # NOTE: This is not used.
300 num_decode_tokens=0,
301 slot_mapping=slot_mapping,
302 block_tables=None,
@@ -306,22 +347,22 @@ class TPUModelRunner:
347
348 input_tokens = torch.tensor(input_tokens,
349 dtype=torch.int32,
309 - device=self.device)
350 + device="cpu")
351 input_positions = torch.tensor(input_positions,
352 dtype=torch.int32,
312 - device=self.device)
353 + device="cpu")
354 slot_mapping = torch.tensor(slot_mapping,
355 dtype=torch.int64,
315 - device=self.device)
356 + device="cpu")
357 context_lens = torch.tensor(context_lens,
358 dtype=torch.int32,
318 - device=self.device)
359 + device="cpu")
360 block_tables = torch.tensor(self.block_tables[:batch_size],
361 dtype=torch.int32,
321 - device=self.device)
362 + device="cpu")
363 input_lens = torch.tensor([1] * batch_size,
364 dtype=torch.int32,
324 - device=self.device)
365 + device="cpu")
366 attn_metadata = self.attn_backend.make_metadata(
367 num_prefills=0,
368 num_prefill_tokens=0,
@@ -382,16 +423,18 @@ class TPUModelRunner:
423 t += [1.0] * num_paddings
424 p += [1.0] * num_paddings
425
385 - t = torch.tensor(t, dtype=torch.float32, device=self.device)
386 - p = torch.tensor(p, dtype=torch.float32, device=self.device)
426 + t = torch.tensor(t, dtype=torch.float32, device="cpu")
427 + p = torch.tensor(p, dtype=torch.float32, device="cpu")
428 return t, p, best_of
429
389 - def _execute_model(
430 + def prepare_model_input(
431 self,
432 seq_group_metadata_list: List[SequenceGroupMetadata],
392 - kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
393 - ) -> List[CompletionSequenceGroupOutput]:
394 - # Prepare inputs.
433 + virtual_engine: int = 0,
434 + finished_requests_ids: Optional[List[str]] = None,
435 + ) -> ModelInputForTPU:
436 + del finished_requests_ids # Unused.
437 + assert virtual_engine == 0
438 assert len(seq_group_metadata_list) > 0
439 # NOTE: We assume that all sequences in the group are all prompts or
440 # all decodes.
@@ -400,16 +443,104 @@ class TPUModelRunner:
443 inputs = self._prepare_prompt(seq_group_metadata_list)
444 else:
445 inputs = self._prepare_decode(seq_group_metadata_list)
403 - padded_batch_size = inputs[0].shape[0]
446 + input_tokens, input_positions, attn_metadata, input_lens = inputs
447 + padded_batch_size = input_tokens.shape[0]
448 t, p, best_of = self._prepare_sample(seq_group_metadata_list,
449 padded_batch_size)
450 num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
451
408 - # Execute the model.
409 - next_token_ids = self.model(inputs[0], inputs[1], kv_caches,
410 - *inputs[2:], t, p, num_samples)
411 - # Retrieve the outputs to CPU.
412 - next_token_ids = next_token_ids.cpu().tolist()
452 + seq_groups = [
453 + list(metadata.seq_data.keys())
454 + for metadata in seq_group_metadata_list
455 + ]
456 + return ModelInputForTPU(input_tokens, input_positions, attn_metadata,
457 + input_lens, t, p, num_samples, best_of,
458 + seq_groups)
459 +
460 + def make_model_input_from_broadcasted_tensor_dict(
461 + self, tensor_dict: Dict[str, Any]) -> ModelInputForTPU:
462 + model_input = ModelInputForTPU.from_broadcasted_tensor_dict(
463 + tensor_dict, attn_backend=self.attn_backend)
464 + return model_input
465 +
466 + def execute_model(
467 + self,
468 + model_input: ModelInputForTPU,
469 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
470 + intermediate_tensors: Optional[IntermediateTensors] = None,
471 + num_steps: int = 1,
472 + ) -> List[SamplerOutput]:
473 + assert intermediate_tensors is None
474 + if num_steps > 1:
475 + raise ValueError(
476 + "TPUModelRunner does not support multi-step execution.")
477 +
478 + def _execute_model(*args, clone: bool = False) -> torch.Tensor:
479 + """Move input args from CPU to device and execute the model."""
480 +
481 + def _copy_to_device(x: torch.Tensor) -> torch.Tensor:
482 + if clone:
483 + # When x is a slice of a CPU tensor, XLA may copy the whole
484 + # original tensor to TPU instead of only copying x.
485 + # To avoid this, we copy x after cloning.
486 + x = x.clone()
487 + return x.to(self.device)
488 +
489 + new_args = []
490 + for arg in args:
491 + if isinstance(arg, torch.Tensor):
492 + arg = _copy_to_device(arg)
493 + elif isinstance(arg, AttentionMetadata):
494 + arg.slot_mapping = _copy_to_device(arg.slot_mapping)
495 + if getattr(arg, "block_tables", None) is not None:
496 + arg.block_tables = _copy_to_device(arg.block_tables)
497 + if getattr(arg, "context_lens", None) is not None:
498 + arg.context_lens = _copy_to_device(arg.context_lens)
499 + new_args.append(arg)
500 + return self.model(*new_args)
501 +
502 + num_prefills = model_input.attn_metadata.num_prefills
503 + is_prompt = num_prefills > 0
504 + if is_prompt:
505 + # NOTE(woosuk): Since the FlashAttention kernel does not support
506 + # ragged inputs, we split the prompts into different batches and
507 + # process them separately. This is a temporary hack that should be
508 + # optimized by using SplashAttention.
509 + next_token_ids = []
510 + orig_slot_mapping = model_input.attn_metadata.slot_mapping
511 + batch_size = model_input.input_lens.shape[0]
512 + start_idx = 0
513 + for i in range(batch_size):
514 + # Get the actual prefill_len.
515 + prefill_len = model_input.input_lens[i:i + 1].item()
516 + prefill_len = _get_padded_prefill_len(prefill_len)
517 + end_idx = start_idx + prefill_len
518 +
519 + model_input.attn_metadata.slot_mapping = orig_slot_mapping[
520 + None, start_idx:end_idx]
521 + model_input.attn_metadata.num_prefills = 1
522 + output_token_ids = _execute_model(
523 + model_input.token_ids[None, start_idx:end_idx],
524 + model_input.position_ids[None, start_idx:end_idx],
525 + model_input.attn_metadata,
526 + model_input.input_lens[i:i + 1],
527 + model_input.t[i:i + 1],
528 + model_input.p[i:i + 1],
529 + model_input.num_samples,
530 + kv_caches,
531 + clone=True)
532 + # Retrieve the outputs to CPU.
533 + next_token_ids += output_token_ids.cpu().tolist()
534 + start_idx = end_idx
535 + else:
536 + # Execute the model.
537 + output_token_ids = _execute_model(
538 + model_input.token_ids, model_input.position_ids,
539 + model_input.attn_metadata, model_input.input_lens,
540 + model_input.t, model_input.p, model_input.num_samples,
541 + kv_caches)
542 + # Retrieve the outputs to CPU.
543 + next_token_ids = output_token_ids.cpu().tolist()
544
545 # NOTE(woosuk): Minimal code to construct the sampler outputs.
546 # The TPU backend does not reuse the sampler, since the TPU backend
@@ -417,13 +548,13 @@ class TPUModelRunner:
548 zero_logprob = Logprob(0.0)
549 batch_idx = 0
550 sampler_outputs = []
420 - for seq_group_metadata in seq_group_metadata_list:
551 + for seq_group in model_input.seq_groups:
552 + seq_ids = seq_group
553 seq_outputs = []
422 - seq_ids = list(seq_group_metadata.seq_data.keys())
554 if is_prompt:
555 assert len(seq_ids) == 1
556 seq_id = seq_ids[0]
426 - for i in range(best_of[batch_idx]):
557 + for i in range(model_input.best_of[batch_idx]):
558 next_token_id = next_token_ids[batch_idx][i]
559 seq_outputs.append(
560 SequenceOutput(seq_id, next_token_id,
@@ -438,35 +569,6 @@ class TPUModelRunner:
569 batch_idx += 1
570 sampler_outputs.append(
571 CompletionSequenceGroupOutput(seq_outputs, None))
441 - return sampler_outputs
442 -
443 - def execute_model(
444 - self,
445 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
446 - kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
447 - num_steps: int = 1,
448 - ) -> List[SamplerOutput]:
449 - if num_steps > 1:
450 - raise ValueError(
451 - "TPUModelRunner does not support multi-step execution.")
452 -
453 - assert seq_group_metadata_list is not None
454 - assert len(seq_group_metadata_list) > 0
455 - if seq_group_metadata_list[0].is_prompt:
456 - # NOTE(woosuk): To reduce the compilation time, we only compile the
457 - # prefill inputs with batch size 1. Because the scheduler is not
458 - # aware of this limitation, we need to handle batch size > 1
459 - # internally by calling the model multiple times and concatenating
460 - # the outputs.
461 - # FIXME(woosuk): This is a temporary hack to not change the existing
462 - # scheduler. We need to fix this in the future.
463 - sampler_outputs = []
464 - for seq_group_metadata in seq_group_metadata_list:
465 - sampler_outputs += self._execute_model([seq_group_metadata],
466 - kv_caches)
467 - else:
468 - sampler_outputs = self._execute_model(seq_group_metadata_list,
469 - kv_caches)
572 return [SamplerOutput(sampler_outputs)]
573
574
@@ -474,36 +576,37 @@ class ModelWrapper(nn.Module):
576
577 def __init__(self, model: nn.Module):
578 super().__init__()
477 - self.model = model.eval()
579 + self.model = model
580
581 def forward(
582 self,
583 token_ids: torch.Tensor,
584 position_ids: torch.Tensor,
483 - kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
585 attn_metadata: AttentionMetadata,
586 input_lens: torch.Tensor,
587 t: torch.Tensor,
588 p: torch.Tensor,
589 num_samples: int,
590 + kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
591 ) -> torch.Tensor:
592 """Executes the forward pass of the model and samples the next token.
593
594 Args:
595 token_ids: The input token IDs of shape [batch_size, seq_len].
596 position_ids: The input position IDs of shape [batch_size, seq_len].
495 - kv_caches: The key and value caches. They can be None during the
496 - memory profiling at initialization.
597 attn_metadata: The Pallas attention metadata.
598 input_lens: The actual input lengths of shape [batch_size].
599 t: The sampling temperature of shape [batch_size].
600 p: The top-p probability of shape [batch_size].
601 + num_samples: Number of samples to draw from each logits vector.
602 + kv_caches: The key and value caches. They can be None during the
603 + memory profiling at initialization.
604 """
605 batch_size, seq_len = token_ids.shape
606 # Calculate the positions to sample from.
504 - base_indicies = torch.arange(
607 + start_indicies = torch.arange(
608 batch_size, dtype=torch.int32, device=input_lens.device) * seq_len
506 - logits_indices = base_indicies + input_lens - 1
609 + logits_indices = start_indicies + input_lens - 1
610
611 # FIXME(woosuk): This is a temporary hack to avoid using the existing
612 # sampler and sampling metadata.
vllm/worker/tpu_worker.py
+72 -69
@@ -13,15 +13,16 @@ from vllm.distributed import (ensure_model_parallel_initialized,
13 init_distributed_environment)
14 from vllm.logger import init_logger
15 from vllm.model_executor import set_random_seed
16 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
16 +from vllm.sequence import ExecuteModelRequest
17 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
18 from vllm.worker.tpu_model_runner import TPUModelRunner
19 -from vllm.worker.worker_base import LoraNotSupportedWorkerBase
19 +from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
20 + LoraNotSupportedWorkerBase, WorkerInput)
21
22 logger = init_logger(__name__)
23
24
24 -class TPUWorker(LoraNotSupportedWorkerBase):
25 +class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
26
27 def __init__(
28 self,
@@ -57,14 +58,15 @@ class TPUWorker(LoraNotSupportedWorkerBase):
58 self.cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
59 self.cache_config.cache_dtype]
60
60 - self.model_runner = TPUModelRunner(model_config,
61 - parallel_config,
62 - scheduler_config,
63 - device_config,
64 - cache_config,
65 - load_config,
66 - multimodal_config,
67 - is_driver_worker=is_driver_worker)
61 + self.model_runner: TPUModelRunner = TPUModelRunner(
62 + model_config,
63 + parallel_config,
64 + scheduler_config,
65 + device_config,
66 + cache_config,
67 + load_config,
68 + multimodal_config,
69 + is_driver_worker=is_driver_worker)
70
71 def init_device(self) -> None:
72 os.environ["PJRT_DEVICE"] = "TPU"
@@ -196,69 +198,70 @@ class TPUWorker(LoraNotSupportedWorkerBase):
198 dtype_size = get_dtype_size(self.cache_dtype)
199 return dtype_size * total
200
199 - def execute_model(
201 + @property
202 + def do_metadata_broadcast(self) -> bool:
203 + # TODO(woosuk): Support TP.
204 + return False
205 +
206 + @property
207 + def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
208 + # NOTE(woosuk): This assumes virtual_engine == 0, i.e., no pipeline
209 + # parallelism.
210 + return [self.tpu_cache]
211 +
212 + def prepare_worker_input(
213 self,
201 - execute_model_req: Optional[ExecuteModelRequest] = None,
202 - ) -> List[SamplerOutput]:
203 - if not self.is_driver_worker:
204 - self._execute_model_non_driver()
205 - return []
206 - assert execute_model_req is not None
207 - # Issue cache operations.
208 - self.cache_swap(
209 - execute_model_req.blocks_to_swap_in,
210 - execute_model_req.blocks_to_swap_out,
211 - execute_model_req.blocks_to_copy,
214 + execute_model_req: ExecuteModelRequest,
215 + ) -> WorkerInput:
216 + virtual_engine = execute_model_req.virtual_engine
217 + num_seq_groups = len(execute_model_req.seq_group_metadata_list)
218 + blocks_to_swap_in = _make_src_to_dst(
219 + execute_model_req.blocks_to_swap_in, "cpu", self.device)
220 + blocks_to_swap_out = _make_src_to_dst(
221 + execute_model_req.blocks_to_swap_out, self.device, "cpu")
222 + blocks_to_copy = _make_src_to_dst(execute_model_req.blocks_to_copy,
223 + self.device, self.device)
224 + return WorkerInput(
225 + num_seq_groups=num_seq_groups,
226 + blocks_to_swap_in=blocks_to_swap_in,
227 + blocks_to_swap_out=blocks_to_swap_out,
228 + blocks_to_copy=blocks_to_copy,
229 + virtual_engine=virtual_engine,
230 )
213 - # Run the model.
214 - seq_group_metadata_list = execute_model_req.seq_group_metadata_list
215 - assert len(seq_group_metadata_list) > 0
216 - output = self.model_runner.execute_model(seq_group_metadata_list,
217 - self.tpu_cache)
218 - return output
219 -
220 - def cache_swap(
221 - self,
222 - blocks_to_swap_in: List[Tuple[int, int]],
223 - blocks_to_swap_out: List[Tuple[int, int]],
224 - blocks_to_copy: List[Tuple[int, int]],
225 - ) -> None:
231 +
232 + def execute_worker(self, worker_input: WorkerInput) -> None:
233 + virtual_engine = worker_input.virtual_engine
234 + assert virtual_engine == 0
235 attn_backend = self.model_runner.attn_backend
236 num_layers = self.model_config.get_num_layers(self.parallel_config)
237
229 - if blocks_to_swap_in:
230 - # Swap from CPU to TPU.
231 - src_indices, dst_indices = _make_src_to_dst(
232 - blocks_to_swap_in, "cpu", self.device)
233 - for i in range(num_layers):
234 - tpu_k_cache, tpu_v_cache = self.tpu_cache[i]
235 - cpu_k_cache, cpu_v_cache = self.cpu_cache[i]
236 - k = cpu_k_cache[:, src_indices].to(self.device)
237 - v = cpu_v_cache[:, src_indices].to(self.device)
238 - _insert_kv(k, v, dst_indices, tpu_k_cache, tpu_v_cache)
239 -
240 - if blocks_to_swap_out:
241 - # Swap from TPU to CPU.
242 - src_indices, dst_indices = _make_src_to_dst(
243 - blocks_to_swap_out, self.device, "cpu")
244 - for i in range(num_layers):
245 - tpu_k_cache, tpu_v_cache = self.tpu_cache[i]
246 - cpu_k_cache, cpu_v_cache = self.cpu_cache[i]
247 - cpu_k_cache[:, dst_indices] = tpu_k_cache[:, src_indices].cpu()
248 - cpu_v_cache[:, dst_indices] = tpu_v_cache[:, src_indices].cpu()
249 -
250 - if blocks_to_copy:
251 - src_to_dst = _make_src_to_dst(blocks_to_copy, self.device,
252 - self.device)
253 - attn_backend.copy_blocks(self.tpu_cache, src_to_dst)
254 -
255 - def start_worker_execution_loop(self) -> None:
256 - while self._execute_model_non_driver():
257 - pass
258 -
259 - def _execute_model_non_driver(self) -> bool:
260 - self.model_runner.execute_model(None, self.tpu_cache)
261 - return True
238 + # Issue cache operations.
239 + if worker_input.blocks_to_swap_in is not None:
240 + src_indices, dst_indices = worker_input.blocks_to_swap_in
241 + if src_indices.numel() > 0:
242 + # Swap from CPU to TPU.
243 + for i in range(num_layers):
244 + tpu_k_cache, tpu_v_cache = self.tpu_cache[i]
245 + cpu_k_cache, cpu_v_cache = self.cpu_cache[i]
246 + k = cpu_k_cache[:, src_indices].to(self.device)
247 + v = cpu_v_cache[:, src_indices].to(self.device)
248 + _insert_kv(k, v, dst_indices, tpu_k_cache, tpu_v_cache)
249 +
250 + if worker_input.blocks_to_swap_out is not None:
251 + src_indices, dst_indices = worker_input.blocks_to_swap_out
252 + if src_indices.numel() > 0:
253 + # Swap from TPU to CPU.
254 + for i in range(num_layers):
255 + tpu_k_cache, tpu_v_cache = self.tpu_cache[i]
256 + cpu_k_cache, cpu_v_cache = self.cpu_cache[i]
257 + cpu_k_cache[:, dst_indices] = tpu_k_cache[:, src_indices]
258 + cpu_v_cache[:, dst_indices] = tpu_v_cache[:, src_indices]
259 +
260 + if worker_input.blocks_to_copy is not None:
261 + src_indices, dst_indices = worker_input.blocks_to_copy
262 + if src_indices.numel() > 0:
263 + attn_backend.copy_blocks(self.tpu_cache,
264 + (src_indices, dst_indices))
265
266
267 def _make_src_to_dst(