@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ce143353c6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[TPU] Skip creating empty tensor (#7630)

Woosuk Kwon committed Aug 17, 2024 at 14:22 UTC ce143353c622318a9abf113bebee1cfebc274e0f
1 file changed +4 -1
vllm/worker/tpu_worker.py
+4 -1
@@ -271,7 +271,10 @@ def _make_src_to_dst(
271 mapping: List[Tuple[int, int]],
272 src_device: Union[torch.device, str],
273 dst_device: Union[torch.device, str],
274 -) -> Tuple[torch.Tensor, torch.Tensor]:
274 +) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
275 + if not mapping:
276 + return None
277 +
278 src_indices = [i for i, _ in mapping]
279 dst_indices = [i for _, i in mapping]
280 src_indices = torch.tensor(src_indices,