A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[TPU] Remove redundant input tensor cloning (#7660)
Woosuk Kwon committed
Aug 19, 2024 at 15:55 UTC
43735bf5e19eaf243b6edaa5af4c7561a14fc2f6
1 file changed
+8
-20
vllm/worker/tpu_model_runner.py
+8
-20
@@ -516,27 +516,19 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
516
raise ValueError(
517
"TPUModelRunner does not support multi-step execution.")
518
519
- def _execute_model(*args, clone: bool = False) -> torch.Tensor:
519
+ def _execute_model(*args):
520
"""Move input args from CPU to device and execute the model."""
521
522
- def _copy_to_device(x: torch.Tensor) -> torch.Tensor:
523
- if clone:
524
- # When x is a slice of a CPU tensor, XLA may copy the whole
525
- # original tensor to TPU instead of only copying x.
526
- # To avoid this, we copy x after cloning.
527
- x = x.clone()
528
- return x.to(self.device)
529
-
522
new_args = []
523
for arg in args:
524
if isinstance(arg, torch.Tensor):
533
- arg = _copy_to_device(arg)
525
+ arg = arg.to(self.device)
526
elif isinstance(arg, AttentionMetadata):
535
- arg.slot_mapping = _copy_to_device(arg.slot_mapping)
527
+ arg.slot_mapping = arg.slot_mapping.to(self.device)
528
if getattr(arg, "block_tables", None) is not None:
537
- arg.block_tables = _copy_to_device(arg.block_tables)
529
+ arg.block_tables = arg.block_tables.to(self.device)
530
if getattr(arg, "context_lens", None) is not None:
539
- arg.context_lens = _copy_to_device(arg.context_lens)
531
+ arg.context_lens = arg.context_lens.to(self.device)
532
new_args.append(arg)
533
return self.model(*new_args)
534
@@ -563,13 +555,9 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
555
output_token_ids = _execute_model(
556
model_input.token_ids[None, start_idx:end_idx],
557
model_input.position_ids[None, start_idx:end_idx],
566
- model_input.attn_metadata,
567
- model_input.input_lens[i:i + 1],
568
- model_input.t[i:i + 1],
569
- model_input.p[i:i + 1],
570
- model_input.num_samples,
571
- kv_caches,
572
- clone=True)
558
+ model_input.attn_metadata, model_input.input_lens[i:i + 1],
559
+ model_input.t[i:i + 1], model_input.p[i:i + 1],
560
+ model_input.num_samples, kv_caches)
561
# Retrieve the outputs to CPU.
562
next_token_ids += output_token_ids.cpu().tolist()
563
start_idx = end_idx