A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 20:45 UTC
[Bugfix] Skip gs:// model paths for speculator detection (#27846)
Signed-off-by: Peter Schuurman <psch@google.com>
pwschuurman committed
Nov 3, 2025 at 06:31 UTC
f7d2946e996f656b5f831fe2003f3b95a91fb367
3 files changed
+39
-5
tests/transformers_utils/test_utils.py
new
+26
@@ -0,0 +1,26 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from vllm.transformers_utils.utils import is_cloud_storage, is_gcs, is_s3
6
+
7
+
8
+def test_is_gcs():
9
+ assert is_gcs("gs://model-path")
10
+ assert not is_gcs("s3://model-path/path-to-model")
11
+ assert not is_gcs("/unix/local/path")
12
+ assert not is_gcs("nfs://nfs-fqdn.local")
13
+
14
+
15
+def test_is_s3():
16
+ assert is_s3("s3://model-path/path-to-model")
17
+ assert not is_s3("gs://model-path")
18
+ assert not is_s3("/unix/local/path")
19
+ assert not is_s3("nfs://nfs-fqdn.local")
20
+
21
+
22
+def test_is_cloud_storage():
23
+ assert is_cloud_storage("gs://model-path")
24
+ assert is_cloud_storage("s3://model-path/path-to-model")
25
+ assert not is_cloud_storage("/unix/local/path")
26
+ assert not is_cloud_storage("nfs://nfs-fqdn.local")
vllm/engine/arg_utils.py
+5
-5
@@ -86,7 +86,7 @@ from vllm.transformers_utils.config import (
86
is_interleaved,
87
maybe_override_with_speculators,
88
)
89
-from vllm.transformers_utils.utils import check_gguf_file, is_s3
89
+from vllm.transformers_utils.utils import check_gguf_file, is_cloud_storage
90
from vllm.utils.argparse_utils import FlexibleArgumentParser
91
from vllm.utils.mem_constants import GiB_bytes
92
from vllm.utils.network_utils import get_ip
@@ -1310,10 +1310,10 @@ class EngineArgs:
1310
1311
# Check if the model is a speculator and override model/tokenizer/config
1312
# BEFORE creating ModelConfig, so the config is created with the target model
1313
- # Skip speculator detection for S3 models since HuggingFace cannot load
1314
- # configs directly from S3 URLs. S3 models can still use speculators with
1315
- # explicit --speculative-config.
1316
- if not is_s3(self.model):
1313
+ # Skip speculator detection for cloud storage models (eg: S3, GCS) since
1314
+ # HuggingFace cannot load configs directly from S3 URLs. S3 models can still
1315
+ # use speculators with explicit --speculative-config.
1316
+ if not is_cloud_storage(self.model):
1317
(self.model, self.tokenizer, self.speculative_config) = (
1318
maybe_override_with_speculators(
1319
model=self.model,
vllm/transformers_utils/utils.py
+8
@@ -19,6 +19,14 @@ def is_s3(model_or_path: str) -> bool:
19
return model_or_path.lower().startswith("s3://")
20
21
22
+def is_gcs(model_or_path: str) -> bool:
23
+ return model_or_path.lower().startswith("gs://")
24
+
25
+
26
+def is_cloud_storage(model_or_path: str) -> bool:
27
+ return is_s3(model_or_path) or is_gcs(model_or_path)
28
+
29
+
30
def check_gguf_file(model: str | PathLike) -> bool:
31
"""Check if the file is a GGUF model."""
32
model = Path(model)