A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[TPU] Skip creating empty tensor (#7630)
Woosuk Kwon committed
Aug 17, 2024 at 14:22 UTC
ce143353c622318a9abf113bebee1cfebc274e0f
1 file changed
+4
-1
vllm/worker/tpu_worker.py
+4
-1
@@ -271,7 +271,10 @@ def _make_src_to_dst(
271
mapping: List[Tuple[int, int]],
272
src_device: Union[torch.device, str],
273
dst_device: Union[torch.device, str],
274
-) -> Tuple[torch.Tensor, torch.Tensor]:
274
+) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
275
+ if not mapping:
276
+ return None
277
+
278
src_indices = [i for i, _ in mapping]
279
dst_indices = [i for _, i in mapping]
280
src_indices = torch.tensor(src_indices,