A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[TPU][V1] Use `language_model` interface for getting text backbone in MM (#16410)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Apr 10, 2025 at 19:32 UTC
c1b57855ecf257ba620fad022d429332d50f289f
1 file changed
+1
-2
vllm/v1/worker/tpu_model_runner.py
+1
-2
@@ -972,8 +972,7 @@ class TPUModelRunner:
972
973
def reset_dynamo_cache(self):
974
if self.is_multimodal_model:
975
- assert hasattr(self.model, "language_model")
976
- compiled_model = self.model.language_model.model
975
+ compiled_model = self.model.get_language_model().model
976
else:
977
compiled_model = self.model.model
978
if isinstance(compiled_model, TorchCompileWrapperWithCustomDispatcher):