A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] when use s3 model cannot use default load_format (#24435)
Signed-off-by: rongfu.leng <rongfu.leng@daocloud.io> Co-authored-by: 22quinn <33176974+22quinn@users.noreply.github.com>
rongfu.leng committed
Sep 18, 2025 at 15:47 UTC
350c94deb30747f84536ee34d91c6fca564667ce
2 files changed
+12
-1
vllm/config/__init__.py
+12
@@ -3029,6 +3029,18 @@ class VllmConfig:
3029
SequenceClassificationConfig)
3030
SequenceClassificationConfig.verify_and_update_config(self)
3031
3032
+ if hasattr(self.model_config, "model_weights") and is_runai_obj_uri(
3033
+ self.model_config.model_weights):
3034
+ if self.load_config.load_format == "auto":
3035
+ logger.info("Detected Run:ai model config. "
3036
+ "Overriding `load_format` to 'runai_streamer'")
3037
+ self.load_config.load_format = "runai_streamer"
3038
+ elif self.load_config.load_format != "runai_streamer":
3039
+ raise ValueError(f"To load a model from S3, 'load_format' "
3040
+ f"must be 'runai_streamer', "
3041
+ f"but got '{self.load_config.load_format}'. "
3042
+ f"Model: {self.model_config.model}")
3043
+
3044
def __str__(self):
3045
return (
3046
f"model={self.model_config.model!r}, "
vllm/engine/arg_utils.py
-1
@@ -959,7 +959,6 @@ class EngineArgs:
959
if (not isinstance(self, AsyncEngineArgs) and envs.VLLM_CI_USE_S3
960
and self.model in MODELS_ON_S3 and self.load_format == "auto"):
961
self.model = f"{MODEL_WEIGHTS_S3_BUCKET}/{self.model}"
962
- self.load_format = "runai_streamer"
962
963
if self.disable_mm_preprocessor_cache:
964
logger.warning(