A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix][Misc]: fix graph capture for decoder (#9549)
yudian0504 committed
Oct 22, 2024 at 01:33 UTC
8ca895484117e55c66c8b5643929866e634e5ce3
1 file changed
+1
-1
vllm/worker/model_runner.py
+1
-1
@@ -828,7 +828,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
828
829
cuda_graph_pad_size = self._get_cuda_graph_pad_size(
830
num_seqs=len(seq_lens),
831
- max_decode_seq_len=max_encoder_seq_len,
831
+ max_decode_seq_len=max_decode_seq_len,
832
max_encoder_seq_len=max_encoder_seq_len)
833
834
batch_size = len(input_tokens)