A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[torch.compile] fix sym_tensor_indices (#12191)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 20, 2025 at 11:37 UTC
51ef828f10acddbe941c38255c5de7f61738abad
1 file changed
+5
-1
vllm/compilation/backends.py
+5
-1
@@ -624,9 +624,13 @@ class VllmBackend:
624
]
625
626
# index of tensors that have symbolic shapes (batch size)
627
+ # for weights and static buffers, they will have concrete shapes.
628
+ # symbolic shape only happens for input tensors.
629
+ from torch.fx.experimental.symbolic_shapes import is_symbolic
630
self.sym_tensor_indices = [
631
i for i, x in enumerate(fake_args)
629
- if isinstance(x, torch._subclasses.fake_tensor.FakeTensor)
632
+ if isinstance(x, torch._subclasses.fake_tensor.FakeTensor) and \
633
+ any(is_symbolic(d) for d in x.size())
634
]
635
636
# compiler managed cudagraph input buffers