A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Run:ai model streamer add GCS package support (#24909)
Signed-off-by: Peter Schuurman <psch@google.com>
pwschuurman committed
Oct 1, 2025 at 20:59 UTC
be22bb6f3dd7aaf8559a4a0a1beb98a37a5a8138
6 files changed
+54
-9
docs/models/extensions/runai_model_streamer.md
+7
@@ -24,6 +24,13 @@ vllm serve s3://core-llm/Llama-3-8b \
24
--load-format runai_streamer
25
```
26
27
+To run model from Google Cloud Storage run:
28
+
29
+```bash
30
+vllm serve gs://core-llm/Llama-3-8b \
31
+ --load-format runai_streamer
32
+```
33
+
34
To run model from a S3 compatible object store run:
35
36
```bash
requirements/nightly_torch_test.txt
+1
-1
@@ -43,6 +43,6 @@ tritonclient==2.51.0
43
numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding
44
numba == 0.61.2; python_version > '3.9'
45
numpy
46
-runai-model-streamer[s3]==0.14.0
46
+runai-model-streamer[s3,gcs]==0.14.0
47
fastsafetensors>=0.1.10
48
pydantic>=2.10 # 2.9 leads to error on python 3.10
requirements/rocm.txt
+2
-2
@@ -13,6 +13,6 @@ tensorizer==2.10.1
13
packaging>=24.2
14
setuptools>=77.0.3,<80.0.0
15
setuptools-scm>=8
16
-runai-model-streamer[s3]==0.14.0
16
+runai-model-streamer[s3,gcs]==0.14.0
17
conch-triton-kernels==1.2.1
18
-timm>=1.0.17
\ No newline at end of file
18
+timm>=1.0.17
requirements/test.in
+1
-1
@@ -51,7 +51,7 @@ tritonclient==2.51.0
51
numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding
52
numba == 0.61.2; python_version > '3.9'
53
numpy
54
-runai-model-streamer[s3]==0.14.0
54
+runai-model-streamer[s3,gcs]==0.14.0
55
fastsafetensors>=0.1.10
56
pydantic>=2.10 # 2.9 leads to error on python 3.10
57
decord==0.6.0
requirements/test.txt
+20
-1
@@ -251,11 +251,27 @@ gitdb==4.0.12
251
gitpython==3.1.44
252
# via mlflow-skinny
253
google-api-core==2.24.2
254
- # via opencensus
254
+ # via
255
+ # google-cloud-core
256
+ # google-cloud-storage
257
+ # opencensus
258
google-auth==2.40.2
259
# via
260
# databricks-sdk
261
# google-api-core
262
+ # google-cloud-core
263
+ # google-cloud-storage
264
+ # runai-model-streamer-gcs
265
+google-cloud-core==2.4.3
266
+ # via google-cloud-storage
267
+google-cloud-storage==3.4.0
268
+ # via runai-model-streamer-gcs
269
+google-crc32c==1.7.1
270
+ # via
271
+ # google-cloud-storage
272
+ # google-resumable-media
273
+google-resumable-media==2.7.2
274
+ # via google-cloud-storage
275
googleapis-common-protos==1.70.0
276
# via google-api-core
277
graphene==3.4.3
@@ -890,6 +906,7 @@ requests==2.32.3
906
# docker
907
# evaluate
908
# google-api-core
909
+ # google-cloud-storage
910
# huggingface-hub
911
# lightly
912
# lm-eval
@@ -929,6 +946,8 @@ rtree==1.4.0
946
# via torchgeo
947
runai-model-streamer==0.14.0
948
# via -r requirements/test.in
949
+runai-model-streamer-gcs==0.14.0
950
+ # via runai-model-streamer
951
runai-model-streamer-s3==0.14.0
952
# via runai-model-streamer
953
s3transfer==0.10.3
tests/model_executor/model_loader/runai_model_streamer/test_runai_utils.py
+23
-4
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import glob
5
+import hashlib
6
import os
7
import tempfile
8
@@ -9,7 +10,8 @@ import huggingface_hub.constants
10
11
from vllm.model_executor.model_loader.weight_utils import (
12
download_weights_from_hf)
12
-from vllm.transformers_utils.runai_utils import (is_runai_obj_uri,
13
+from vllm.transformers_utils.runai_utils import (ObjectStorageModel,
14
+ is_runai_obj_uri,
15
list_safetensors)
16
17
@@ -34,6 +36,23 @@ def test_runai_list_safetensors_local():
36
assert len(safetensors) == len(files)
37
38
37
-if __name__ == "__main__":
38
- test_is_runai_obj_uri()
39
- test_runai_list_safetensors_local()
39
+def test_runai_pull_files_gcs(monkeypatch):
40
+ monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true")
41
+ # Bypass default project lookup by setting GOOGLE_CLOUD_PROJECT
42
+ monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project")
43
+ filename = "LT08_L1GT_074061_20130309_20170505_01_T2_MTL.txt"
44
+ gcs_bucket = "gs://gcp-public-data-landsat/LT08/01/074/061/LT08_L1GT_074061_20130309_20170505_01_T2/"
45
+ gcs_url = f"{gcs_bucket}/{filename}"
46
+ model = ObjectStorageModel(gcs_url)
47
+ model.pull_files(gcs_bucket, allow_pattern=[f"*{filename}"])
48
+ # To re-generate / change URLs:
49
+ # gsutil ls -L gs://<gcs-url> | grep "Hash (md5)" | tr -d ' ' \
50
+ # | cut -d":" -f2 | base64 -d | xxd -p
51
+ expected_checksum = "f60dea775da1392434275b311b31a431"
52
+ hasher = hashlib.new("md5")
53
+ with open(os.path.join(model.dir, filename), 'rb') as f:
54
+ # Read the file in chunks to handle large files efficiently
55
+ for chunk in iter(lambda: f.read(4096), b''):
56
+ hasher.update(chunk)
57
+ actual_checksum = hasher.hexdigest()
58
+ assert actual_checksum == expected_checksum