A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 20:30 UTC
[Tests] Skip model weight download for render-only test server (#36813)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
Sage committed
Mar 12, 2026 at 08:24 UTC
802f306cd116c575dd1db6d8dd9c37751916017b
1 file changed
+25
-9
tests/utils.py
+25
-9
@@ -144,6 +144,17 @@ class RemoteVLLMServer:
144
"""Subclasses override this method to customize server process launch"""
145
raise NotImplementedError
146
147
+ def _pre_download_model(self, model: str, args) -> None:
148
+ """Download model weights before starting the server to avoid timeout."""
149
+ is_local = os.path.isdir(model)
150
+ if not is_local:
151
+ engine_args = AsyncEngineArgs.from_cli_args(args)
152
+ model_config = engine_args.create_model_config()
153
+ load_config = engine_args.create_load_config()
154
+
155
+ model_loader = get_model_loader(load_config)
156
+ model_loader.download_model(model_config)
157
+
158
def __init__(
159
self,
160
model: str,
@@ -195,15 +206,7 @@ class RemoteVLLMServer:
206
getattr(args, "show_hidden_metrics_for_version", None) is not None
207
)
208
198
- # download the model before starting the server to avoid timeout
199
- is_local = os.path.isdir(model)
200
- if not is_local:
201
- engine_args = AsyncEngineArgs.from_cli_args(args)
202
- model_config = engine_args.create_model_config()
203
- load_config = engine_args.create_load_config()
204
-
205
- model_loader = get_model_loader(load_config)
206
- model_loader.download_model(model_config)
209
+ self._pre_download_model(model, args)
210
211
# Record GPU memory before server start so we know what
212
# "released" looks like.
@@ -515,6 +518,19 @@ class RemoteLaunchRenderServer(RemoteVLLMServer):
518
start_new_session=True,
519
)
520
521
+ def _pre_download_model(self, model: str, args) -> None:
522
+ """Download only the tokenizer files (no model weights needed)."""
523
+ is_local = os.path.isdir(model)
524
+ if not is_local:
525
+ engine_args = AsyncEngineArgs.from_cli_args(args)
526
+ model_config = engine_args.create_model_config()
527
+ get_tokenizer(
528
+ model_config.tokenizer,
529
+ tokenizer_mode=model_config.tokenizer_mode,
530
+ trust_remote_code=model_config.trust_remote_code,
531
+ revision=model_config.tokenizer_revision,
532
+ )
533
+
534
def _wait_for_gpu_memory_release(self, timeout: float = 30.0):
535
pass # No GPU used
536