@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 74af2bbd90
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix reinit procedure in ModelInputForGPUBuilder (#7360)

Alexander Matveev committed Aug 9, 2024 at 12:35 UTC 74af2bbd901d82c6bc2583515c4388722d451f07
1 file changed +2
vllm/worker/model_runner.py
+2
@@ -348,6 +348,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
348 self.multi_modal_inputs = multi_modal_inputs
349 self.prefix_cache_hit = prefix_cache_hit
350
351 + self.n_seqs = len(self.seq_ids)
352 +
353 if not reinit:
354 self.__post_init__()
355