A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 12:45 UTC
RayLLM Bugfix - Preserve obj store URL for multi engine_config creation (#30803)
Signed-off-by: Omer Dayan <omdayan@nvidia.com> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
omer-dayan committed
Jan 8, 2026 at 12:00 UTC
04a49669d1be26b2a83441c1c5a968cf3131f0e4
4 files changed
+15
-4
vllm/config/model.py
+8
@@ -106,6 +106,10 @@ class ModelConfig:
106
"""Name or path of the Hugging Face model to use. It is also used as the
107
content for `model_name` tag in metrics output when `served_model_name` is
108
not specified."""
109
+ model_weights: str = ""
110
+ """Original model weights path. Used when the model is pulled from object
111
+ storage (e.g., RunAI) to preserve the original URI while `model` points to
112
+ the local directory."""
113
runner: RunnerOption = "auto"
114
"""The type of model runner to use. Each vLLM instance only supports one
115
model runner, even if the same model can be used for multiple types."""
@@ -705,6 +709,10 @@ class ModelConfig:
709
tokenizer: Tokenizer name or path
710
"""
711
712
+ # Skip if model_weights is already set (model already pulled)
713
+ if self.model_weights:
714
+ return
715
+
716
if not (is_runai_obj_uri(model) or is_runai_obj_uri(tokenizer)):
717
return
718
vllm/engine/arg_utils.py
+3
@@ -354,6 +354,7 @@ class EngineArgs:
354
"""Arguments for vLLM engine."""
355
356
model: str = ModelConfig.model
357
+ model_weights: str = ModelConfig.model_weights
358
served_model_name: str | list[str] | None = ModelConfig.served_model_name
359
tokenizer: str | None = ModelConfig.tokenizer
360
hf_config_path: str | None = ModelConfig.hf_config_path
@@ -1206,6 +1207,7 @@ class EngineArgs:
1207
1208
return ModelConfig(
1209
model=self.model,
1210
+ model_weights=self.model_weights,
1211
hf_config_path=self.hf_config_path,
1212
runner=self.runner,
1213
convert=self.convert,
@@ -1349,6 +1351,7 @@ class EngineArgs:
1351
1352
model_config = self.create_model_config()
1353
self.model = model_config.model
1354
+ self.model_weights = model_config.model_weights
1355
self.tokenizer = model_config.tokenizer
1356
1357
self._check_feature_supported(model_config)
vllm/model_executor/model_loader/runai_streamer_loader.py
+2
-2
@@ -108,8 +108,8 @@ class RunaiModelStreamerLoader(BaseModelLoader):
108
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
109
"""Load weights into a model."""
110
model_weights = model_config.model
111
- if hasattr(model_config, "model_weights"):
112
- model_weights = model_config.model_weights
111
+ if model_weights_override := model_config.model_weights:
112
+ model_weights = model_weights_override
113
model.load_weights(
114
self._get_weights_iterator(model_weights, model_config.revision)
115
)
vllm/model_executor/model_loader/sharded_state_loader.py
+2
-2
@@ -110,8 +110,8 @@ class ShardedStateLoader(BaseModelLoader):
110
from vllm.distributed import get_tensor_model_parallel_rank
111
112
model_weights = model_config.model
113
- if hasattr(model_config, "model_weights"):
114
- model_weights = model_config.model_weights
113
+ if model_weights_override := model_config.model_weights:
114
+ model_weights = model_weights_override
115
local_model_path = model_weights
116
117
rank = get_tensor_model_parallel_rank()