@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51ef828f10
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[torch.compile] fix sym_tensor_indices (#12191)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 20, 2025 at 11:37 UTC 51ef828f10acddbe941c38255c5de7f61738abad
1 file changed +5 -1
vllm/compilation/backends.py
+5 -1
@@ -624,9 +624,13 @@ class VllmBackend:
624 ]
625
626 # index of tensors that have symbolic shapes (batch size)
627 + # for weights and static buffers, they will have concrete shapes.
628 + # symbolic shape only happens for input tensors.
629 + from torch.fx.experimental.symbolic_shapes import is_symbolic
630 self.sym_tensor_indices = [
631 i for i, x in enumerate(fake_args)
629 - if isinstance(x, torch._subclasses.fake_tensor.FakeTensor)
632 + if isinstance(x, torch._subclasses.fake_tensor.FakeTensor) and \
633 + any(is_symbolic(d) for d in x.size())
634 ]
635
636 # compiler managed cudagraph input buffers