A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[TPU][V1] Make `--disable_chunked_mm_input` mandatory for serving MM models (#16483)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Apr 11, 2025 at 19:06 UTC
4d022cbc757e89b2296daf3b33303eeea4ba4a62
1 file changed
+7
vllm/platforms/tpu.py
+7
@@ -120,6 +120,13 @@ class TpuPlatform(Platform):
120
assert not vllm_config.speculative_config, (
121
"Speculative decoding is not yet supported for TPU backend")
122
123
+ if scheduler_config.is_multimodal_model and not \
124
+ scheduler_config.disable_chunked_mm_input:
125
+ logger.warning("TPU does not support running Multimodal models"\
126
+ " without setting `--disable_chunked_mm_input`. " \
127
+ "Forcing --disable_chunked_mm_input.")
128
+ scheduler_config.disable_chunked_mm_input = True
129
+
130
@classmethod
131
def is_pin_memory_available(cls):
132
logger.warning("Pin memory is not supported on TPU.")