A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[TPU] Refactor TPU worker & model runner (#6506)
Woosuk Kwon committed
Jul 18, 2024 at 01:34 UTC
4634c8728b0fe30a0b2da22dd4a4c8d8a1d9213b
2 files changed
+272
-166
vllm/worker/tpu_model_runner.py
+200
-97
@@ -1,5 +1,6 @@
1
import time
2
-from typing import List, Optional, Tuple
2
+from dataclasses import dataclass
3
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
4
5
import numpy as np
6
import torch
@@ -12,10 +13,16 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
13
from vllm.logger import init_logger
14
from vllm.model_executor.model_loader import get_model
15
from vllm.model_executor.sampling_metadata import SamplingMetadata
15
-from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
16
- SamplerOutput, SequenceGroupMetadata,
16
+from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
17
+ Logprob, SamplerOutput, SequenceGroupMetadata,
18
SequenceOutput)
18
-from vllm.utils import make_tensor_with_pad
19
+from vllm.worker.model_runner_base import (
20
+ ModelRunnerBase, ModelRunnerInputBase,
21
+ _add_attn_metadata_broadcastable_dict,
22
+ _init_attn_metadata_from_tensor_dict)
23
+
24
+if TYPE_CHECKING:
25
+ from vllm.attention.backends.abstract import AttentionBackend
26
27
logger = init_logger(__name__)
28
@@ -27,7 +34,44 @@ _ENABLE_TOP_P = False
34
_MAX_NUM_SAMPLES = 128
35
36
30
-class TPUModelRunner:
37
+@dataclass(frozen=True)
38
+class ModelInputForTPU(ModelRunnerInputBase):
39
+ token_ids: torch.Tensor
40
+ position_ids: torch.Tensor
41
+ attn_metadata: AttentionMetadata
42
+ input_lens: torch.Tensor
43
+ t: torch.Tensor
44
+ p: torch.Tensor
45
+ num_samples: int
46
+ best_of: List[int]
47
+ seq_groups: List[List[int]]
48
+
49
+ def as_broadcastable_tensor_dict(
50
+ self) -> Dict[str, Union[int, torch.Tensor]]:
51
+ tensor_dict = {
52
+ "token_ids": self.token_ids,
53
+ "position_ids": self.position_ids,
54
+ "input_lens": self.input_lens,
55
+ "t": self.t,
56
+ "p": self.p,
57
+ "num_samples": self.num_samples,
58
+ }
59
+ _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
60
+ return tensor_dict
61
+
62
+ @classmethod
63
+ def from_broadcasted_tensor_dict(
64
+ cls: Type["ModelInputForTPU"],
65
+ tensor_dict: Dict[str, Any],
66
+ attn_backend: Optional["AttentionBackend"] = None,
67
+ ) -> "ModelInputForTPU":
68
+ if attn_backend is not None:
69
+ tensor_dict = _init_attn_metadata_from_tensor_dict(
70
+ attn_backend, tensor_dict)
71
+ return cls(**tensor_dict)
72
+
73
+
74
+class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
75
76
def __init__(
77
self,
@@ -79,6 +123,7 @@ class TPUModelRunner:
123
multimodal_config=self.multimodal_config,
124
lora_config=None,
125
)
126
+ model = model.eval()
127
xm.wait_device_ops()
128
129
model = ModelWrapper(model)
@@ -147,8 +192,8 @@ class TPUModelRunner:
192
193
# Dummy run.
194
num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
150
- self.model(token_ids, position_ids, kv_caches, attn_metadata,
151
- input_lens, t, p, num_samples)
195
+ self.model(token_ids, position_ids, attn_metadata, input_lens, t, p,
196
+ num_samples, kv_caches)
197
198
def warmup_model(
199
self,
@@ -177,7 +222,7 @@ class TPUModelRunner:
222
# Decode
223
start = time.time()
224
seq_len = 1
180
- batch_size = 1
225
+ batch_size = 8 # Must be in sync with _get_padded_batch_size()
226
while True:
227
self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=False)
228
xm.wait_device_ops()
@@ -195,10 +240,10 @@ class TPUModelRunner:
240
seq_group_metadata_list: List[SequenceGroupMetadata],
241
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, torch.Tensor]:
242
assert len(seq_group_metadata_list) > 0
198
- input_tokens: List[List[int]] = []
199
- input_positions: List[List[int]] = []
243
+ input_tokens: List[int] = []
244
+ input_positions: List[int] = []
245
prompt_lens: List[int] = []
201
- slot_mapping: List[List[int]] = []
246
+ slot_mapping: List[int] = []
247
248
for seq_group_metadata in seq_group_metadata_list:
249
assert seq_group_metadata.is_prompt
@@ -212,50 +257,46 @@ class TPUModelRunner:
257
prompt_len = len(prompt_tokens)
258
prompt_lens.append(prompt_len)
259
215
- input_tokens.append(prompt_tokens)
216
- input_positions.append(list(range(prompt_len)))
260
+ input_tokens.extend(prompt_tokens)
261
+ input_positions.extend(list(range(prompt_len)))
262
263
assert seq_group_metadata.block_tables is not None
264
block_table = seq_group_metadata.block_tables[seq_id]
220
- slot_mapping.append([])
265
for i in range(prompt_len):
266
block_number = block_table[i // self.block_size]
267
block_offset = i % self.block_size
268
slot = block_number * self.block_size + block_offset
225
- slot_mapping[-1].append(slot)
269
+ slot_mapping.append(slot)
270
+
271
+ # Add paddings to EACH prompt to the smallest power of 2 that is
272
+ # greater than or equal to the prompt length.
273
+ # We pad the seq_len to reduce the compilation overhead.
274
+ # We execute each prompt individually (i.e., with batch_size 1)
275
+ # because the FlashAttention kernel does not support ragged inputs.
276
+ # TODO(woosuk): Use SplashAttention to support ragged inputs.
277
+ padded_prompt_len = _get_padded_prefill_len(prompt_len)
278
+ num_paddings = padded_prompt_len - prompt_len
279
+ input_tokens += [0] * num_paddings
280
+ input_positions += [0] * num_paddings
281
+ slot_mapping += [_PAD_SLOT_ID] * num_paddings
282
283
assert len(prompt_lens) > 0
284
num_prefills = len(prompt_lens)
229
- num_prefill_tokens = sum(prompt_lens)
230
-
231
- # Add paddings to make the shape [batch_size, max_prompt_len] where
232
- # max_prompt_len is smallest power of 2 that is greater than or equal
233
- # to the maximum prompt length.
234
- # We need the 2D input shape because the Pallas FlashAttention kernel
235
- # does not support packed 1D inputs.
236
- # We pad the seq_len to powers of 2 to reduce the compilation overhead.
237
- max_prompt_len = _get_padded_prefill_len(max(prompt_lens))
238
- input_tokens = make_tensor_with_pad(input_tokens,
239
- max_prompt_len,
240
- pad=0,
241
- dtype=torch.int32,
242
- device=self.device)
243
- input_positions = make_tensor_with_pad(input_positions,
244
- max_prompt_len,
245
- pad=0,
246
- dtype=torch.int32,
247
- device=self.device)
248
- slot_mapping = make_tensor_with_pad(slot_mapping,
249
- max_prompt_len,
250
- pad=_PAD_SLOT_ID,
251
- dtype=torch.int64,
252
- device=self.device)
285
+ input_tokens = torch.tensor(input_tokens,
286
+ dtype=torch.int32,
287
+ device="cpu")
288
+ input_positions = torch.tensor(input_positions,
289
+ dtype=torch.int32,
290
+ device="cpu")
291
+ slot_mapping = torch.tensor(slot_mapping,
292
+ dtype=torch.int64,
293
+ device="cpu")
294
prompt_lens = torch.tensor(prompt_lens,
295
dtype=torch.int32,
255
- device=self.device)
296
+ device="cpu")
297
attn_metadata = self.attn_backend.make_metadata(
298
num_prefills=num_prefills,
258
- num_prefill_tokens=num_prefill_tokens, # NOTE: This is not used.
299
+ num_prefill_tokens=0, # NOTE: This is not used.
300
num_decode_tokens=0,
301
slot_mapping=slot_mapping,
302
block_tables=None,
@@ -306,22 +347,22 @@ class TPUModelRunner:
347
348
input_tokens = torch.tensor(input_tokens,
349
dtype=torch.int32,
309
- device=self.device)
350
+ device="cpu")
351
input_positions = torch.tensor(input_positions,
352
dtype=torch.int32,
312
- device=self.device)
353
+ device="cpu")
354
slot_mapping = torch.tensor(slot_mapping,
355
dtype=torch.int64,
315
- device=self.device)
356
+ device="cpu")
357
context_lens = torch.tensor(context_lens,
358
dtype=torch.int32,
318
- device=self.device)
359
+ device="cpu")
360
block_tables = torch.tensor(self.block_tables[:batch_size],
361
dtype=torch.int32,
321
- device=self.device)
362
+ device="cpu")
363
input_lens = torch.tensor([1] * batch_size,
364
dtype=torch.int32,
324
- device=self.device)
365
+ device="cpu")
366
attn_metadata = self.attn_backend.make_metadata(
367
num_prefills=0,
368
num_prefill_tokens=0,
@@ -382,16 +423,18 @@ class TPUModelRunner:
423
t += [1.0] * num_paddings
424
p += [1.0] * num_paddings
425
385
- t = torch.tensor(t, dtype=torch.float32, device=self.device)
386
- p = torch.tensor(p, dtype=torch.float32, device=self.device)
426
+ t = torch.tensor(t, dtype=torch.float32, device="cpu")
427
+ p = torch.tensor(p, dtype=torch.float32, device="cpu")
428
return t, p, best_of
429
389
- def _execute_model(
430
+ def prepare_model_input(
431
self,
432
seq_group_metadata_list: List[SequenceGroupMetadata],
392
- kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
393
- ) -> List[CompletionSequenceGroupOutput]:
394
- # Prepare inputs.
433
+ virtual_engine: int = 0,
434
+ finished_requests_ids: Optional[List[str]] = None,
435
+ ) -> ModelInputForTPU:
436
+ del finished_requests_ids # Unused.
437
+ assert virtual_engine == 0
438
assert len(seq_group_metadata_list) > 0
439
# NOTE: We assume that all sequences in the group are all prompts or
440
# all decodes.
@@ -400,16 +443,104 @@ class TPUModelRunner:
443
inputs = self._prepare_prompt(seq_group_metadata_list)
444
else:
445
inputs = self._prepare_decode(seq_group_metadata_list)
403
- padded_batch_size = inputs[0].shape[0]
446
+ input_tokens, input_positions, attn_metadata, input_lens = inputs
447
+ padded_batch_size = input_tokens.shape[0]
448
t, p, best_of = self._prepare_sample(seq_group_metadata_list,
449
padded_batch_size)
450
num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
451
408
- # Execute the model.
409
- next_token_ids = self.model(inputs[0], inputs[1], kv_caches,
410
- *inputs[2:], t, p, num_samples)
411
- # Retrieve the outputs to CPU.
412
- next_token_ids = next_token_ids.cpu().tolist()
452
+ seq_groups = [
453
+ list(metadata.seq_data.keys())
454
+ for metadata in seq_group_metadata_list
455
+ ]
456
+ return ModelInputForTPU(input_tokens, input_positions, attn_metadata,
457
+ input_lens, t, p, num_samples, best_of,
458
+ seq_groups)
459
+
460
+ def make_model_input_from_broadcasted_tensor_dict(
461
+ self, tensor_dict: Dict[str, Any]) -> ModelInputForTPU:
462
+ model_input = ModelInputForTPU.from_broadcasted_tensor_dict(
463
+ tensor_dict, attn_backend=self.attn_backend)
464
+ return model_input
465
+
466
+ def execute_model(
467
+ self,
468
+ model_input: ModelInputForTPU,
469
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
470
+ intermediate_tensors: Optional[IntermediateTensors] = None,
471
+ num_steps: int = 1,
472
+ ) -> List[SamplerOutput]:
473
+ assert intermediate_tensors is None
474
+ if num_steps > 1:
475
+ raise ValueError(
476
+ "TPUModelRunner does not support multi-step execution.")
477
+
478
+ def _execute_model(*args, clone: bool = False) -> torch.Tensor:
479
+ """Move input args from CPU to device and execute the model."""
480
+
481
+ def _copy_to_device(x: torch.Tensor) -> torch.Tensor:
482
+ if clone:
483
+ # When x is a slice of a CPU tensor, XLA may copy the whole
484
+ # original tensor to TPU instead of only copying x.
485
+ # To avoid this, we copy x after cloning.
486
+ x = x.clone()
487
+ return x.to(self.device)
488
+
489
+ new_args = []
490
+ for arg in args:
491
+ if isinstance(arg, torch.Tensor):
492
+ arg = _copy_to_device(arg)
493
+ elif isinstance(arg, AttentionMetadata):
494
+ arg.slot_mapping = _copy_to_device(arg.slot_mapping)
495
+ if getattr(arg, "block_tables", None) is not None:
496
+ arg.block_tables = _copy_to_device(arg.block_tables)
497
+ if getattr(arg, "context_lens", None) is not None:
498
+ arg.context_lens = _copy_to_device(arg.context_lens)
499
+ new_args.append(arg)
500
+ return self.model(*new_args)
501
+
502
+ num_prefills = model_input.attn_metadata.num_prefills
503
+ is_prompt = num_prefills > 0
504
+ if is_prompt:
505
+ # NOTE(woosuk): Since the FlashAttention kernel does not support
506
+ # ragged inputs, we split the prompts into different batches and
507
+ # process them separately. This is a temporary hack that should be
508
+ # optimized by using SplashAttention.
509
+ next_token_ids = []
510
+ orig_slot_mapping = model_input.attn_metadata.slot_mapping
511
+ batch_size = model_input.input_lens.shape[0]
512
+ start_idx = 0
513
+ for i in range(batch_size):
514
+ # Get the actual prefill_len.
515
+ prefill_len = model_input.input_lens[i:i + 1].item()
516
+ prefill_len = _get_padded_prefill_len(prefill_len)
517
+ end_idx = start_idx + prefill_len
518
+
519
+ model_input.attn_metadata.slot_mapping = orig_slot_mapping[
520
+ None, start_idx:end_idx]
521
+ model_input.attn_metadata.num_prefills = 1
522
+ output_token_ids = _execute_model(
523
+ model_input.token_ids[None, start_idx:end_idx],
524
+ model_input.position_ids[None, start_idx:end_idx],
525
+ model_input.attn_metadata,
526
+ model_input.input_lens[i:i + 1],
527
+ model_input.t[i:i + 1],
528
+ model_input.p[i:i + 1],
529
+ model_input.num_samples,
530
+ kv_caches,
531
+ clone=True)
532
+ # Retrieve the outputs to CPU.
533
+ next_token_ids += output_token_ids.cpu().tolist()
534
+ start_idx = end_idx
535
+ else:
536
+ # Execute the model.
537
+ output_token_ids = _execute_model(
538
+ model_input.token_ids, model_input.position_ids,
539
+ model_input.attn_metadata, model_input.input_lens,
540
+ model_input.t, model_input.p, model_input.num_samples,
541
+ kv_caches)
542
+ # Retrieve the outputs to CPU.
543
+ next_token_ids = output_token_ids.cpu().tolist()
544
545
# NOTE(woosuk): Minimal code to construct the sampler outputs.
546
# The TPU backend does not reuse the sampler, since the TPU backend
@@ -417,13 +548,13 @@ class TPUModelRunner:
548
zero_logprob = Logprob(0.0)
549
batch_idx = 0
550
sampler_outputs = []
420
- for seq_group_metadata in seq_group_metadata_list:
551
+ for seq_group in model_input.seq_groups:
552
+ seq_ids = seq_group
553
seq_outputs = []
422
- seq_ids = list(seq_group_metadata.seq_data.keys())
554
if is_prompt:
555
assert len(seq_ids) == 1
556
seq_id = seq_ids[0]
426
- for i in range(best_of[batch_idx]):
557
+ for i in range(model_input.best_of[batch_idx]):
558
next_token_id = next_token_ids[batch_idx][i]
559
seq_outputs.append(
560
SequenceOutput(seq_id, next_token_id,
@@ -438,35 +569,6 @@ class TPUModelRunner:
569
batch_idx += 1
570
sampler_outputs.append(
571
CompletionSequenceGroupOutput(seq_outputs, None))
441
- return sampler_outputs
442
-
443
- def execute_model(
444
- self,
445
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
446
- kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
447
- num_steps: int = 1,
448
- ) -> List[SamplerOutput]:
449
- if num_steps > 1:
450
- raise ValueError(
451
- "TPUModelRunner does not support multi-step execution.")
452
-
453
- assert seq_group_metadata_list is not None
454
- assert len(seq_group_metadata_list) > 0
455
- if seq_group_metadata_list[0].is_prompt:
456
- # NOTE(woosuk): To reduce the compilation time, we only compile the
457
- # prefill inputs with batch size 1. Because the scheduler is not
458
- # aware of this limitation, we need to handle batch size > 1
459
- # internally by calling the model multiple times and concatenating
460
- # the outputs.
461
- # FIXME(woosuk): This is a temporary hack to not change the existing
462
- # scheduler. We need to fix this in the future.
463
- sampler_outputs = []
464
- for seq_group_metadata in seq_group_metadata_list:
465
- sampler_outputs += self._execute_model([seq_group_metadata],
466
- kv_caches)
467
- else:
468
- sampler_outputs = self._execute_model(seq_group_metadata_list,
469
- kv_caches)
572
return [SamplerOutput(sampler_outputs)]
573
574
@@ -474,36 +576,37 @@ class ModelWrapper(nn.Module):
576
577
def __init__(self, model: nn.Module):
578
super().__init__()
477
- self.model = model.eval()
579
+ self.model = model
580
581
def forward(
582
self,
583
token_ids: torch.Tensor,
584
position_ids: torch.Tensor,
483
- kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
585
attn_metadata: AttentionMetadata,
586
input_lens: torch.Tensor,
587
t: torch.Tensor,
588
p: torch.Tensor,
589
num_samples: int,
590
+ kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
591
) -> torch.Tensor:
592
"""Executes the forward pass of the model and samples the next token.
593
594
Args:
595
token_ids: The input token IDs of shape [batch_size, seq_len].
596
position_ids: The input position IDs of shape [batch_size, seq_len].
495
- kv_caches: The key and value caches. They can be None during the
496
- memory profiling at initialization.
597
attn_metadata: The Pallas attention metadata.
598
input_lens: The actual input lengths of shape [batch_size].
599
t: The sampling temperature of shape [batch_size].
600
p: The top-p probability of shape [batch_size].
601
+ num_samples: Number of samples to draw from each logits vector.
602
+ kv_caches: The key and value caches. They can be None during the
603
+ memory profiling at initialization.
604
"""
605
batch_size, seq_len = token_ids.shape
606
# Calculate the positions to sample from.
504
- base_indicies = torch.arange(
607
+ start_indicies = torch.arange(
608
batch_size, dtype=torch.int32, device=input_lens.device) * seq_len
506
- logits_indices = base_indicies + input_lens - 1
609
+ logits_indices = start_indicies + input_lens - 1
610
611
# FIXME(woosuk): This is a temporary hack to avoid using the existing
612
# sampler and sampling metadata.
vllm/worker/tpu_worker.py
+72
-69
@@ -13,15 +13,16 @@ from vllm.distributed import (ensure_model_parallel_initialized,
13
init_distributed_environment)
14
from vllm.logger import init_logger
15
from vllm.model_executor import set_random_seed
16
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
16
+from vllm.sequence import ExecuteModelRequest
17
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
18
from vllm.worker.tpu_model_runner import TPUModelRunner
19
-from vllm.worker.worker_base import LoraNotSupportedWorkerBase
19
+from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
20
+ LoraNotSupportedWorkerBase, WorkerInput)
21
22
logger = init_logger(__name__)
23
24
24
-class TPUWorker(LoraNotSupportedWorkerBase):
25
+class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
26
27
def __init__(
28
self,
@@ -57,14 +58,15 @@ class TPUWorker(LoraNotSupportedWorkerBase):
58
self.cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
59
self.cache_config.cache_dtype]
60
60
- self.model_runner = TPUModelRunner(model_config,
61
- parallel_config,
62
- scheduler_config,
63
- device_config,
64
- cache_config,
65
- load_config,
66
- multimodal_config,
67
- is_driver_worker=is_driver_worker)
61
+ self.model_runner: TPUModelRunner = TPUModelRunner(
62
+ model_config,
63
+ parallel_config,
64
+ scheduler_config,
65
+ device_config,
66
+ cache_config,
67
+ load_config,
68
+ multimodal_config,
69
+ is_driver_worker=is_driver_worker)
70
71
def init_device(self) -> None:
72
os.environ["PJRT_DEVICE"] = "TPU"
@@ -196,69 +198,70 @@ class TPUWorker(LoraNotSupportedWorkerBase):
198
dtype_size = get_dtype_size(self.cache_dtype)
199
return dtype_size * total
200
199
- def execute_model(
201
+ @property
202
+ def do_metadata_broadcast(self) -> bool:
203
+ # TODO(woosuk): Support TP.
204
+ return False
205
+
206
+ @property
207
+ def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
208
+ # NOTE(woosuk): This assumes virtual_engine == 0, i.e., no pipeline
209
+ # parallelism.
210
+ return [self.tpu_cache]
211
+
212
+ def prepare_worker_input(
213
self,
201
- execute_model_req: Optional[ExecuteModelRequest] = None,
202
- ) -> List[SamplerOutput]:
203
- if not self.is_driver_worker:
204
- self._execute_model_non_driver()
205
- return []
206
- assert execute_model_req is not None
207
- # Issue cache operations.
208
- self.cache_swap(
209
- execute_model_req.blocks_to_swap_in,
210
- execute_model_req.blocks_to_swap_out,
211
- execute_model_req.blocks_to_copy,
214
+ execute_model_req: ExecuteModelRequest,
215
+ ) -> WorkerInput:
216
+ virtual_engine = execute_model_req.virtual_engine
217
+ num_seq_groups = len(execute_model_req.seq_group_metadata_list)
218
+ blocks_to_swap_in = _make_src_to_dst(
219
+ execute_model_req.blocks_to_swap_in, "cpu", self.device)
220
+ blocks_to_swap_out = _make_src_to_dst(
221
+ execute_model_req.blocks_to_swap_out, self.device, "cpu")
222
+ blocks_to_copy = _make_src_to_dst(execute_model_req.blocks_to_copy,
223
+ self.device, self.device)
224
+ return WorkerInput(
225
+ num_seq_groups=num_seq_groups,
226
+ blocks_to_swap_in=blocks_to_swap_in,
227
+ blocks_to_swap_out=blocks_to_swap_out,
228
+ blocks_to_copy=blocks_to_copy,
229
+ virtual_engine=virtual_engine,
230
)
213
- # Run the model.
214
- seq_group_metadata_list = execute_model_req.seq_group_metadata_list
215
- assert len(seq_group_metadata_list) > 0
216
- output = self.model_runner.execute_model(seq_group_metadata_list,
217
- self.tpu_cache)
218
- return output
219
-
220
- def cache_swap(
221
- self,
222
- blocks_to_swap_in: List[Tuple[int, int]],
223
- blocks_to_swap_out: List[Tuple[int, int]],
224
- blocks_to_copy: List[Tuple[int, int]],
225
- ) -> None:
231
+
232
+ def execute_worker(self, worker_input: WorkerInput) -> None:
233
+ virtual_engine = worker_input.virtual_engine
234
+ assert virtual_engine == 0
235
attn_backend = self.model_runner.attn_backend
236
num_layers = self.model_config.get_num_layers(self.parallel_config)
237
229
- if blocks_to_swap_in:
230
- # Swap from CPU to TPU.
231
- src_indices, dst_indices = _make_src_to_dst(
232
- blocks_to_swap_in, "cpu", self.device)
233
- for i in range(num_layers):
234
- tpu_k_cache, tpu_v_cache = self.tpu_cache[i]
235
- cpu_k_cache, cpu_v_cache = self.cpu_cache[i]
236
- k = cpu_k_cache[:, src_indices].to(self.device)
237
- v = cpu_v_cache[:, src_indices].to(self.device)
238
- _insert_kv(k, v, dst_indices, tpu_k_cache, tpu_v_cache)
239
-
240
- if blocks_to_swap_out:
241
- # Swap from TPU to CPU.
242
- src_indices, dst_indices = _make_src_to_dst(
243
- blocks_to_swap_out, self.device, "cpu")
244
- for i in range(num_layers):
245
- tpu_k_cache, tpu_v_cache = self.tpu_cache[i]
246
- cpu_k_cache, cpu_v_cache = self.cpu_cache[i]
247
- cpu_k_cache[:, dst_indices] = tpu_k_cache[:, src_indices].cpu()
248
- cpu_v_cache[:, dst_indices] = tpu_v_cache[:, src_indices].cpu()
249
-
250
- if blocks_to_copy:
251
- src_to_dst = _make_src_to_dst(blocks_to_copy, self.device,
252
- self.device)
253
- attn_backend.copy_blocks(self.tpu_cache, src_to_dst)
254
-
255
- def start_worker_execution_loop(self) -> None:
256
- while self._execute_model_non_driver():
257
- pass
258
-
259
- def _execute_model_non_driver(self) -> bool:
260
- self.model_runner.execute_model(None, self.tpu_cache)
261
- return True
238
+ # Issue cache operations.
239
+ if worker_input.blocks_to_swap_in is not None:
240
+ src_indices, dst_indices = worker_input.blocks_to_swap_in
241
+ if src_indices.numel() > 0:
242
+ # Swap from CPU to TPU.
243
+ for i in range(num_layers):
244
+ tpu_k_cache, tpu_v_cache = self.tpu_cache[i]
245
+ cpu_k_cache, cpu_v_cache = self.cpu_cache[i]
246
+ k = cpu_k_cache[:, src_indices].to(self.device)
247
+ v = cpu_v_cache[:, src_indices].to(self.device)
248
+ _insert_kv(k, v, dst_indices, tpu_k_cache, tpu_v_cache)
249
+
250
+ if worker_input.blocks_to_swap_out is not None:
251
+ src_indices, dst_indices = worker_input.blocks_to_swap_out
252
+ if src_indices.numel() > 0:
253
+ # Swap from TPU to CPU.
254
+ for i in range(num_layers):
255
+ tpu_k_cache, tpu_v_cache = self.tpu_cache[i]
256
+ cpu_k_cache, cpu_v_cache = self.cpu_cache[i]
257
+ cpu_k_cache[:, dst_indices] = tpu_k_cache[:, src_indices]
258
+ cpu_v_cache[:, dst_indices] = tpu_v_cache[:, src_indices]
259
+
260
+ if worker_input.blocks_to_copy is not None:
261
+ src_indices, dst_indices = worker_input.blocks_to_copy
262
+ if src_indices.numel() > 0:
263
+ attn_backend.copy_blocks(self.tpu_cache,
264
+ (src_indices, dst_indices))
265
266
267
def _make_src_to_dst(