@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8ca8954841
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix][Misc]: fix graph capture for decoder (#9549)

yudian0504 committed Oct 22, 2024 at 01:33 UTC 8ca895484117e55c66c8b5643929866e634e5ce3
1 file changed +1 -1
vllm/worker/model_runner.py
+1 -1
@@ -828,7 +828,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
828
829 cuda_graph_pad_size = self._get_cuda_graph_pad_size(
830 num_seqs=len(seq_lens),
831 - max_decode_seq_len=max_encoder_seq_len,
831 + max_decode_seq_len=max_decode_seq_len,
832 max_encoder_seq_len=max_encoder_seq_len)
833
834 batch_size = len(input_tokens)