A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix reinit procedure in ModelInputForGPUBuilder (#7360)
Alexander Matveev committed
Aug 9, 2024 at 12:35 UTC
74af2bbd901d82c6bc2583515c4388722d451f07
1 file changed
+2
vllm/worker/model_runner.py
+2
@@ -348,6 +348,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
348
self.multi_modal_inputs = multi_modal_inputs
349
self.prefix_cache_hit = prefix_cache_hit
350
351
+ self.n_seqs = len(self.seq_ids)
352
+
353
if not reinit:
354
self.__post_init__()
355