@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 43735bf5e1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[TPU] Remove redundant input tensor cloning (#7660)

Woosuk Kwon committed Aug 19, 2024 at 15:55 UTC 43735bf5e19eaf243b6edaa5af4c7561a14fc2f6
1 file changed +8 -20
vllm/worker/tpu_model_runner.py
+8 -20
@@ -516,27 +516,19 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
516 raise ValueError(
517 "TPUModelRunner does not support multi-step execution.")
518
519 - def _execute_model(*args, clone: bool = False) -> torch.Tensor:
519 + def _execute_model(*args):
520 """Move input args from CPU to device and execute the model."""
521
522 - def _copy_to_device(x: torch.Tensor) -> torch.Tensor:
523 - if clone:
524 - # When x is a slice of a CPU tensor, XLA may copy the whole
525 - # original tensor to TPU instead of only copying x.
526 - # To avoid this, we copy x after cloning.
527 - x = x.clone()
528 - return x.to(self.device)
529 -
522 new_args = []
523 for arg in args:
524 if isinstance(arg, torch.Tensor):
533 - arg = _copy_to_device(arg)
525 + arg = arg.to(self.device)
526 elif isinstance(arg, AttentionMetadata):
535 - arg.slot_mapping = _copy_to_device(arg.slot_mapping)
527 + arg.slot_mapping = arg.slot_mapping.to(self.device)
528 if getattr(arg, "block_tables", None) is not None:
537 - arg.block_tables = _copy_to_device(arg.block_tables)
529 + arg.block_tables = arg.block_tables.to(self.device)
530 if getattr(arg, "context_lens", None) is not None:
539 - arg.context_lens = _copy_to_device(arg.context_lens)
531 + arg.context_lens = arg.context_lens.to(self.device)
532 new_args.append(arg)
533 return self.model(*new_args)
534
@@ -563,13 +555,9 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
555 output_token_ids = _execute_model(
556 model_input.token_ids[None, start_idx:end_idx],
557 model_input.position_ids[None, start_idx:end_idx],
566 - model_input.attn_metadata,
567 - model_input.input_lens[i:i + 1],
568 - model_input.t[i:i + 1],
569 - model_input.p[i:i + 1],
570 - model_input.num_samples,
571 - kv_caches,
572 - clone=True)
558 + model_input.attn_metadata, model_input.input_lens[i:i + 1],
559 + model_input.t[i:i + 1], model_input.p[i:i + 1],
560 + model_input.num_samples, kv_caches)
561 # Retrieve the outputs to CPU.
562 next_token_ids += output_token_ids.cpu().tolist()
563 start_idx = end_idx