A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 14:04 UTC
[Model Runner v2] Support reload weights (sleep mode) (#42673)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Wentao Ye committed
May 15, 2026 at 12:41 UTC
6147c70224394a620ff3a56190a16539bdebdc3d
1 file changed
+8
vllm/v1/worker/gpu/model_runner.py
+8
@@ -329,6 +329,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
329
def get_model(self) -> nn.Module:
330
return self.model
331
332
+ def reload_weights(self, *args, **kwargs) -> None:
333
+ # TODO(Wentao): Use full version instead of import when fully migrated to v2
334
+ from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1
335
+
336
+ GPUModelRunnerV1.reload_weights(self, *args, **kwargs) # type: ignore[arg-type]
337
+ self.reset_encoder_cache()
338
+ self.reset_mm_cache()
339
+
340
@functools.cached_property
341
def main_stream(self) -> torch.cuda.Stream:
342
# Cache the default CUDA stream to avoid lookup overhead.