A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[1/n][CI] Load models in CI from S3 instead of HF (#13205)
Signed-off-by: <> Co-authored-by: EC2 Default User <ec2-user@ip-172-31-20-117.us-west-2.compute.internal>
Kevin H. Luu committed
Feb 18, 2025 at 23:34 UTC
d5d214ac7f379c75265c973fe4f2047e22bda54d
43 files changed
+225
-76
requirements-test.in
+2
@@ -37,3 +37,5 @@ genai_perf==0.0.8
37
tritonclient==2.51.0
38
39
numpy < 2.0.0
40
+runai-model-streamer==0.11.0
41
+runai-model-streamer-s3==0.11.0
\ No newline at end of file
requirements-test.txt
+8
@@ -171,6 +171,8 @@ huggingface-hub==0.26.2
171
# tokenizers
172
# transformers
173
# vocos
174
+humanize==4.11.0
175
+ # via runai-model-streamer
176
idna==3.10
177
# via
178
# anyio
@@ -290,6 +292,7 @@ numpy==1.26.4
292
# patsy
293
# peft
294
# rouge-score
295
+ # runai-model-streamer
296
# sacrebleu
297
# scikit-learn
298
# scipy
@@ -514,6 +517,10 @@ rpds-py==0.20.1
517
# referencing
518
rsa==4.7.2
519
# via awscli
520
+runai-model-streamer==0.11.0
521
+ # via -r requirements-test.in
522
+runai-model-streamer-s3==0.11.0
523
+ # via -r requirements-test.in
524
s3transfer==0.10.3
525
# via
526
# awscli
@@ -594,6 +601,7 @@ torch==2.5.1
601
# encodec
602
# lm-eval
603
# peft
604
+ # runai-model-streamer
605
# sentence-transformers
606
# tensorizer
607
# timm
tests/basic_correctness/test_basic_correctness.py
+10
-9
@@ -9,6 +9,7 @@ import weakref
9
import pytest
10
11
from vllm import LLM
12
+from vllm.config import LoadFormat
13
from vllm.platforms import current_platform
14
15
from ..conftest import VllmRunner
@@ -33,7 +34,7 @@ def v1(run_with_both_engines):
34
35
def test_vllm_gc_ed():
36
"""Verify vllm instance is GC'ed when it is deleted"""
36
- llm = LLM("facebook/opt-125m")
37
+ llm = LLM("distilbert/distilgpt2", load_format=LoadFormat.RUNAI_STREAMER)
38
weak_llm = weakref.ref(llm)
39
del llm
40
# If there's any circular reference to vllm, this fails
@@ -94,14 +95,14 @@ def test_models(
95
@pytest.mark.parametrize(
96
"model, distributed_executor_backend, attention_backend, "
97
"test_suite", [
97
- ("facebook/opt-125m", "ray", "", "L4"),
98
- ("facebook/opt-125m", "mp", "", "L4"),
99
- ("meta-llama/Llama-3.2-1B-Instruct", "ray", "", "L4"),
100
- ("meta-llama/Llama-3.2-1B-Instruct", "mp", "", "L4"),
101
- ("facebook/opt-125m", "ray", "", "A100"),
102
- ("facebook/opt-125m", "mp", "", "A100"),
103
- ("facebook/opt-125m", "mp", "FLASHINFER", "A100"),
104
- ("meta-llama/Llama-3.2-1B-Instruct", "ray", "FLASHINFER", "A100"),
98
+ ("distilbert/distilgpt2", "ray", "", "L4"),
99
+ ("distilbert/distilgpt2", "mp", "", "L4"),
100
+ ("meta-llama/Llama-2-7b-hf", "ray", "", "L4"),
101
+ ("meta-llama/Llama-2-7b-hf", "mp", "", "L4"),
102
+ ("distilbert/distilgpt2", "ray", "", "A100"),
103
+ ("distilbert/distilgpt2", "mp", "", "A100"),
104
+ ("distilbert/distilgpt2", "mp", "FLASHINFER", "A100"),
105
+ ("meta-llama/Meta-Llama-3-8B", "ray", "FLASHINFER", "A100"),
106
])
107
def test_models_distributed(
108
hf_runner,
tests/basic_correctness/test_cumem.py
+10
-3
@@ -4,9 +4,11 @@ import pytest
4
import torch
5
6
from vllm import LLM, SamplingParams
7
+from vllm.config import LoadFormat
8
from vllm.device_allocator.cumem import CuMemAllocator
9
from vllm.utils import GiB_bytes
10
11
+from ..conftest import MODEL_WEIGHTS_S3_BUCKET
12
from ..utils import fork_new_process_for_each_test
13
14
@@ -118,13 +120,18 @@ def test_cumem_with_cudagraph():
120
@pytest.mark.parametrize(
121
"model",
122
[
121
- "meta-llama/Llama-3.2-1B-Instruct", # sleep mode with safetensors
122
- "facebook/opt-125m" # sleep mode with pytorch checkpoint
123
+ # sleep mode with safetensors
124
+ f"{MODEL_WEIGHTS_S3_BUCKET}/Llama-3.2-1B",
125
+ # sleep mode with pytorch checkpoint
126
+ "facebook/opt-125m"
127
])
128
def test_end_to_end(model):
129
free, total = torch.cuda.mem_get_info()
130
used_bytes_baseline = total - free # in case other process is running
127
- llm = LLM(model, enable_sleep_mode=True)
131
+ load_format = LoadFormat.AUTO
132
+ if "Llama" in model:
133
+ load_format = LoadFormat.RUNAI_STREAMER
134
+ llm = LLM(model, load_format=load_format, enable_sleep_mode=True)
135
prompt = "How are you?"
136
sampling_params = SamplingParams(temperature=0, max_tokens=10)
137
output = llm.generate(prompt, sampling_params)
tests/basic_correctness/test_preemption.py
+1
-1
@@ -17,7 +17,7 @@ from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
17
from ..models.utils import check_outputs_equal
18
19
MODELS = [
20
- "facebook/opt-125m",
20
+ "distilbert/distilgpt2",
21
]
22
23
tests/conftest.py
+24
-1
@@ -24,7 +24,7 @@ from tests.models.utils import (TokensTextLogprobs,
24
from vllm import LLM, SamplingParams
25
from vllm.assets.image import ImageAsset
26
from vllm.assets.video import VideoAsset
27
-from vllm.config import TaskOption, TokenizerPoolConfig
27
+from vllm.config import LoadFormat, TaskOption, TokenizerPoolConfig
28
from vllm.connections import global_http_connection
29
from vllm.distributed import (cleanup_dist_env_and_memory,
30
init_distributed_environment,
@@ -46,6 +46,21 @@ _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
46
_SYS_MSG = os.path.join(_TEST_DIR, "system_messages", "sonnet3.5_nov2024.txt")
47
48
_M = TypeVar("_M")
49
+
50
+MODELS_ON_S3 = [
51
+ "distilbert/distilgpt2",
52
+ "meta-llama/Llama-2-7b-hf",
53
+ "meta-llama/Meta-Llama-3-8B",
54
+ "meta-llama/Llama-3.2-1B",
55
+ "meta-llama/Llama-3.2-1B-Instruct",
56
+ "openai-community/gpt2",
57
+ "ArthurZ/Ilama-3.2-1B",
58
+ "llava-hf/llava-1.5-7b-hf",
59
+ "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
60
+]
61
+
62
+MODEL_WEIGHTS_S3_BUCKET = "s3://vllm-ci-model-weights"
63
+
64
_PromptMultiModalInput = Union[List[_M], List[List[_M]]]
65
66
PromptImageInput = _PromptMultiModalInput[Image.Image]
@@ -677,8 +692,15 @@ class VllmRunner:
692
enable_chunked_prefill: bool = False,
693
swap_space: int = 4,
694
enforce_eager: Optional[bool] = False,
695
+ load_format: Optional[LoadFormat] = None,
696
**kwargs,
697
) -> None:
698
+ if model_name in MODELS_ON_S3 and not load_format:
699
+ model_name = (f"s3://vllm-ci-model-weights/"
700
+ f"{model_name.split('/')[-1]}")
701
+ load_format = LoadFormat.RUNAI_STREAMER
702
+ if not load_format:
703
+ load_format = LoadFormat.AUTO
704
self.model = LLM(
705
model=model_name,
706
task=task,
@@ -693,6 +715,7 @@ class VllmRunner:
715
max_model_len=max_model_len,
716
block_size=block_size,
717
enable_chunked_prefill=enable_chunked_prefill,
718
+ load_format=load_format,
719
**kwargs,
720
)
721
tests/engine/test_computed_prefix_blocks.py
+5
-1
@@ -2,12 +2,15 @@
2
3
import pytest
4
5
+from vllm.config import LoadFormat
6
from vllm.engine.arg_utils import EngineArgs
7
from vllm.engine.llm_engine import LLMEngine
8
from vllm.sampling_params import SamplingParams
9
10
+from ..conftest import MODEL_WEIGHTS_S3_BUCKET
11
10
-@pytest.mark.parametrize("model", ["facebook/opt-125m"])
12
+
13
+@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
14
@pytest.mark.parametrize("block_size", [16])
15
def test_computed_prefix_blocks(model: str, block_size: int):
16
# This test checks if we are able to run the engine to completion
@@ -24,6 +27,7 @@ def test_computed_prefix_blocks(model: str, block_size: int):
27
"decoration.")
28
29
engine_args = EngineArgs(model=model,
30
+ load_format=LoadFormat.RUNAI_STREAMER,
31
block_size=block_size,
32
enable_prefix_caching=True)
33
tests/engine/test_detokenization.py
+5
-2
@@ -2,11 +2,14 @@
2
3
import pytest
4
5
+from vllm.config import LoadFormat
6
from vllm.entrypoints.llm import LLM
7
from vllm.sampling_params import SamplingParams
8
9
+from ..conftest import MODEL_WEIGHTS_S3_BUCKET
10
9
-@pytest.mark.parametrize("model", ["facebook/opt-125m"])
11
+
12
+@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
13
def test_computed_prefix_blocks(model: str):
14
# This test checks if the engine generates completions both with and
15
# without optional detokenization, that detokenization includes text
@@ -17,7 +20,7 @@ def test_computed_prefix_blocks(model: str):
20
"paper clips? Is there an easy to follow video tutorial available "
21
"online for free?")
22
20
- llm = LLM(model=model)
23
+ llm = LLM(model=model, load_format=LoadFormat.RUNAI_STREAMER)
24
sampling_params = SamplingParams(max_tokens=10,
25
temperature=0.0,
26
detokenize=False)
tests/engine/test_executor.py
+13
-4
@@ -6,12 +6,17 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
6
7
import pytest
8
9
+from vllm.config import LoadFormat
10
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
12
from vllm.engine.llm_engine import LLMEngine
13
from vllm.executor.uniproc_executor import UniProcExecutor
14
from vllm.sampling_params import SamplingParams
15
16
+from ..conftest import MODEL_WEIGHTS_S3_BUCKET
17
+
18
+RUNAI_STREAMER_LOAD_FORMAT = LoadFormat.RUNAI_STREAMER
19
+
20
21
class Mock:
22
...
@@ -33,10 +38,11 @@ class CustomUniExecutor(UniProcExecutor):
38
CustomUniExecutorAsync = CustomUniExecutor
39
40
36
-@pytest.mark.parametrize("model", ["facebook/opt-125m"])
41
+@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
42
def test_custom_executor_type_checking(model):
43
with pytest.raises(ValueError):
44
engine_args = EngineArgs(model=model,
45
+ load_format=RUNAI_STREAMER_LOAD_FORMAT,
46
distributed_executor_backend=Mock)
47
LLMEngine.from_engine_args(engine_args)
48
with pytest.raises(ValueError):
@@ -45,7 +51,7 @@ def test_custom_executor_type_checking(model):
51
AsyncLLMEngine.from_engine_args(engine_args)
52
53
48
-@pytest.mark.parametrize("model", ["facebook/opt-125m"])
54
+@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
55
def test_custom_executor(model, tmp_path):
56
cwd = os.path.abspath(".")
57
os.chdir(tmp_path)
@@ -54,6 +60,7 @@ def test_custom_executor(model, tmp_path):
60
61
engine_args = EngineArgs(
62
model=model,
63
+ load_format=RUNAI_STREAMER_LOAD_FORMAT,
64
distributed_executor_backend=CustomUniExecutor,
65
enforce_eager=True, # reduce test time
66
)
@@ -68,7 +75,7 @@ def test_custom_executor(model, tmp_path):
75
os.chdir(cwd)
76
77
71
-@pytest.mark.parametrize("model", ["facebook/opt-125m"])
78
+@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
79
def test_custom_executor_async(model, tmp_path):
80
cwd = os.path.abspath(".")
81
os.chdir(tmp_path)
@@ -77,6 +84,7 @@ def test_custom_executor_async(model, tmp_path):
84
85
engine_args = AsyncEngineArgs(
86
model=model,
87
+ load_format=RUNAI_STREAMER_LOAD_FORMAT,
88
distributed_executor_backend=CustomUniExecutorAsync,
89
enforce_eager=True, # reduce test time
90
)
@@ -95,7 +103,7 @@ def test_custom_executor_async(model, tmp_path):
103
os.chdir(cwd)
104
105
98
-@pytest.mark.parametrize("model", ["facebook/opt-125m"])
106
+@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
107
def test_respect_ray(model):
108
# even for TP=1 and PP=1,
109
# if users specify ray, we should use ray.
@@ -104,6 +112,7 @@ def test_respect_ray(model):
112
engine_args = EngineArgs(
113
model=model,
114
distributed_executor_backend="ray",
115
+ load_format=RUNAI_STREAMER_LOAD_FORMAT,
116
enforce_eager=True, # reduce test time
117
)
118
engine = LLMEngine.from_engine_args(engine_args)
tests/engine/test_skip_tokenizer_init.py
+7
-2
@@ -2,16 +2,21 @@
2
3
import pytest
4
5
+from vllm.config import LoadFormat
6
from vllm.entrypoints.llm import LLM
7
from vllm.sampling_params import SamplingParams
8
9
+from ..conftest import MODEL_WEIGHTS_S3_BUCKET
10
9
-@pytest.mark.parametrize("model", ["facebook/opt-125m"])
11
+
12
+@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
13
def test_skip_tokenizer_initialization(model: str):
14
# This test checks if the flag skip_tokenizer_init skips the initialization
15
# of tokenizer and detokenizer. The generated output is expected to contain
16
# token ids.
14
- llm = LLM(model=model, skip_tokenizer_init=True)
17
+ llm = LLM(model=model,
18
+ skip_tokenizer_init=True,
19
+ load_format=LoadFormat.RUNAI_STREAMER)
20
sampling_params = SamplingParams(prompt_logprobs=True, detokenize=True)
21
22
with pytest.raises(ValueError, match="cannot pass text prompts when"):
tests/engine/test_stop_reason.py
+1
-1
@@ -12,7 +12,7 @@ import transformers
12
13
from vllm import SamplingParams
14
15
-MODEL = "facebook/opt-350m"
15
+MODEL = "distilbert/distilgpt2"
16
STOP_STR = "."
17
SEED = 42
18
MAX_TOKENS = 1024
tests/entrypoints/llm/test_chat.py
+10
-3
@@ -5,12 +5,17 @@ from typing import List
5
import pytest
6
7
from vllm import LLM
8
+from vllm.config import LoadFormat
9
10
+from ...conftest import MODEL_WEIGHTS_S3_BUCKET
11
from ..openai.test_vision import TEST_IMAGE_URLS
12
13
+RUNAI_STREAMER_LOAD_FORMAT = LoadFormat.RUNAI_STREAMER
14
+
15
16
def test_chat():
13
- llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct")
17
+ llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/Llama-3.2-1B-Instruct",
18
+ load_format=RUNAI_STREAMER_LOAD_FORMAT)
19
20
prompt1 = "Explain the concept of entropy."
21
messages = [
@@ -28,7 +33,8 @@ def test_chat():
33
34
35
def test_multi_chat():
31
- llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct")
36
+ llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/Llama-3.2-1B-Instruct",
37
+ load_format=RUNAI_STREAMER_LOAD_FORMAT)
38
39
prompt1 = "Explain the concept of entropy."
40
prompt2 = "Explain what among us is."
@@ -65,7 +71,8 @@ def test_multi_chat():
71
[[TEST_IMAGE_URLS[0], TEST_IMAGE_URLS[1]]])
72
def test_chat_multi_image(image_urls: List[str]):
73
llm = LLM(
68
- model="microsoft/Phi-3.5-vision-instruct",
74
+ model=f"{MODEL_WEIGHTS_S3_BUCKET}/Phi-3.5-vision-instruct",
75
+ load_format=RUNAI_STREAMER_LOAD_FORMAT,
76
dtype="bfloat16",
77
max_model_len=4096,
78
max_num_seqs=5,
tests/entrypoints/llm/test_collective_rpc.py
+1
-1
@@ -28,7 +28,7 @@ def test_collective_rpc(tp_size, backend):
28
def echo_rank(self):
29
return self.rank
30
31
- llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct",
31
+ llm = LLM(model="s3://vllm-ci-model-weights/Llama-3.2-1B-Instruct",
32
enforce_eager=True,
33
load_format="dummy",
34
tensor_parallel_size=tp_size,
tests/entrypoints/llm/test_encode.py
+3
-1
@@ -6,9 +6,10 @@ from typing import List
6
import pytest
7
8
from vllm import LLM, PoolingParams, PoolingRequestOutput
9
+from vllm.config import LoadFormat
10
from vllm.distributed import cleanup_dist_env_and_memory
11
11
-MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
12
+MODEL_NAME = "s3://vllm-ci-model-weights/e5-mistral-7b-instruct"
13
14
PROMPTS = [
15
"Hello, my name is",
@@ -32,6 +33,7 @@ def llm():
33
# pytest caches the fixture so we use weakref.proxy to
34
# enable garbage collection
35
llm = LLM(model=MODEL_NAME,
36
+ load_format=LoadFormat.RUNAI_STREAMER,
37
max_num_batched_tokens=32768,
38
tensor_parallel_size=1,
39
gpu_memory_utilization=0.75,
tests/entrypoints/llm/test_generate.py
+3
-1
@@ -6,9 +6,10 @@ from typing import List
6
import pytest
7
8
from vllm import LLM, RequestOutput, SamplingParams
9
+from vllm.config import LoadFormat
10
from vllm.distributed import cleanup_dist_env_and_memory
11
11
-MODEL_NAME = "facebook/opt-125m"
12
+MODEL_NAME = "s3://vllm-ci-model-weights/distilgpt2"
13
14
PROMPTS = [
15
"Hello, my name is",
@@ -30,6 +31,7 @@ def llm():
31
# pytest caches the fixture so we use weakref.proxy to
32
# enable garbage collection
33
llm = LLM(model=MODEL_NAME,
34
+ load_format=LoadFormat.RUNAI_STREAMER,
35
max_num_batched_tokens=4096,
36
tensor_parallel_size=1,
37
gpu_memory_utilization=0.10,
tests/entrypoints/llm/test_generate_multiple_loras.py
+3
-1
@@ -7,10 +7,11 @@ import pytest
7
from huggingface_hub import snapshot_download
8
9
from vllm import LLM
10
+from vllm.config import LoadFormat
11
from vllm.distributed import cleanup_dist_env_and_memory
12
from vllm.lora.request import LoRARequest
13
13
-MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
14
+MODEL_NAME = "s3://vllm-ci-model-weights/zephyr-7b-beta"
15
16
PROMPTS = [
17
"Hello, my name is",
@@ -27,6 +28,7 @@ def llm():
28
# pytest caches the fixture so we use weakref.proxy to
29
# enable garbage collection
30
llm = LLM(model=MODEL_NAME,
31
+ load_format=LoadFormat.RUNAI_STREAMER,
32
tensor_parallel_size=1,
33
max_model_len=8192,
34
enable_lora=True,
tests/entrypoints/llm/test_guided_generate.py
+5
-2
@@ -7,12 +7,13 @@ import weakref
7
import jsonschema
8
import pytest
9
10
+from vllm.config import LoadFormat
11
from vllm.distributed import cleanup_dist_env_and_memory
12
from vllm.entrypoints.llm import LLM
13
from vllm.outputs import RequestOutput
14
from vllm.sampling_params import GuidedDecodingParams, SamplingParams
15
15
-MODEL_NAME = "Qwen/Qwen2.5-7B-Instruct"
16
+MODEL_NAME = "s3://vllm-ci-model-weights/Qwen2.5-1.5B-Instruct"
17
GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
18
19
@@ -20,7 +21,9 @@ GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
21
def llm():
22
# pytest caches the fixture so we use weakref.proxy to
23
# enable garbage collection
23
- llm = LLM(model=MODEL_NAME, max_model_len=1024)
24
+ llm = LLM(model=MODEL_NAME,
25
+ load_format=LoadFormat.RUNAI_STREAMER,
26
+ max_model_len=1024)
27
28
with llm.deprecate_legacy_api():
29
yield weakref.proxy(llm)
tests/entrypoints/llm/test_lazy_outlines.py
+29
-2
@@ -6,10 +6,11 @@ from contextlib import nullcontext
6
from vllm_test_utils import BlameResult, blame
7
8
from vllm import LLM, SamplingParams
9
+from vllm.config import LoadFormat
10
from vllm.distributed import cleanup_dist_env_and_memory
11
12
12
-def run_normal():
13
+def run_normal_opt125m():
14
prompts = [
15
"Hello, my name is",
16
"The president of the United States is",
@@ -33,9 +34,35 @@ def run_normal():
34
cleanup_dist_env_and_memory()
35
36
37
+def run_normal():
38
+ prompts = [
39
+ "Hello, my name is",
40
+ "The president of the United States is",
41
+ "The capital of France is",
42
+ "The future of AI is",
43
+ ]
44
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
45
+
46
+ # Create an LLM without guided decoding as a baseline.
47
+ llm = LLM(model="s3://vllm-ci-model-weights/distilgpt2",
48
+ load_format=LoadFormat.RUNAI_STREAMER,
49
+ enforce_eager=True,
50
+ gpu_memory_utilization=0.3)
51
+ outputs = llm.generate(prompts, sampling_params)
52
+ for output in outputs:
53
+ prompt = output.prompt
54
+ generated_text = output.outputs[0].text
55
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
56
+
57
+ # Destroy the LLM object and free up the GPU memory.
58
+ del llm
59
+ cleanup_dist_env_and_memory()
60
+
61
+
62
def run_lmfe(sample_regex):
63
# Create an LLM with guided decoding enabled.
38
- llm = LLM(model="facebook/opt-125m",
64
+ llm = LLM(model="s3://vllm-ci-model-weights/distilgpt2",
65
+ load_format=LoadFormat.RUNAI_STREAMER,
66
enforce_eager=True,
67
guided_decoding_backend="lm-format-enforcer",
68
gpu_memory_utilization=0.3)
tests/entrypoints/llm/test_prompt_validation.py
+7
-2
@@ -3,6 +3,7 @@
3
import pytest
4
5
from vllm import LLM
6
+from vllm.config import LoadFormat
7
8
9
@pytest.fixture(autouse=True)
@@ -14,13 +15,17 @@ def v1(run_with_both_engines):
15
16
17
def test_empty_prompt():
17
- llm = LLM(model="gpt2", enforce_eager=True)
18
+ llm = LLM(model="s3://vllm-ci-model-weights/gpt2",
19
+ load_format=LoadFormat.RUNAI_STREAMER,
20
+ enforce_eager=True)
21
with pytest.raises(ValueError, match='Prompt cannot be empty'):
22
llm.generate([""])
23
24
25
@pytest.mark.skip_v1
26
def test_out_of_vocab_token():
24
- llm = LLM(model="gpt2", enforce_eager=True)
27
+ llm = LLM(model="s3://vllm-ci-model-weights/gpt2",
28
+ load_format=LoadFormat.RUNAI_STREAMER,
29
+ enforce_eager=True)
30
with pytest.raises(ValueError, match='out of vocabulary'):
31
llm.generate({"prompt_token_ids": [999999]})
tests/entrypoints/openai/test_rerank.py
+1
-1
@@ -86,4 +86,4 @@ def test_rerank_max_model_len(server: RemoteOpenAIServer, model_name: str):
86
assert rerank_response.status_code == 400
87
# Assert just a small fragments of the response
88
assert "Please reduce the length of the input." in \
89
- rerank_response.text
\ No newline at end of file
89
+ rerank_response.text
tests/metrics/test_metrics.py
+15
-6
@@ -8,16 +8,21 @@ import ray
8
from prometheus_client import REGISTRY
9
10
from vllm import EngineArgs, LLMEngine
11
+from vllm.config import LoadFormat
12
from vllm.distributed import cleanup_dist_env_and_memory
13
from vllm.engine.arg_utils import AsyncEngineArgs
14
from vllm.engine.async_llm_engine import AsyncLLMEngine
15
from vllm.engine.metrics import RayPrometheusStatLogger
16
from vllm.sampling_params import SamplingParams
17
18
+from ..conftest import MODEL_WEIGHTS_S3_BUCKET
19
+
20
MODELS = [
18
- "facebook/opt-125m",
21
+ "distilbert/distilgpt2",
22
]
23
24
+RUNAI_STREAMER_LOAD_FORMAT = LoadFormat.RUNAI_STREAMER
25
+
26
27
@pytest.mark.parametrize("model", MODELS)
28
@pytest.mark.parametrize("dtype", ["float"])
@@ -141,8 +146,9 @@ def test_metric_set_tag_model_name(vllm_runner, model: str, dtype: str,
146
metrics_tag_content = stat_logger.labels["model_name"]
147
148
if served_model_name is None or served_model_name == []:
144
- assert metrics_tag_content == model, (
145
- f"Metrics tag model_name is wrong! expect: {model!r}\n"
149
+ actual_model_name = f"{MODEL_WEIGHTS_S3_BUCKET}/{model.split('/')[-1]}"
150
+ assert metrics_tag_content == actual_model_name, (
151
+ f"Metrics tag model_name is wrong! expect: {actual_model_name!r}\n"
152
f"actual: {metrics_tag_content!r}")
153
else:
154
assert metrics_tag_content == served_model_name[0], (
@@ -170,7 +176,8 @@ async def test_async_engine_log_metrics_regression(
176
"""
177
engine_args = AsyncEngineArgs(model=model,
178
dtype=dtype,
173
- disable_log_stats=disable_log_stats)
179
+ disable_log_stats=disable_log_stats,
180
+ load_format=RUNAI_STREAMER_LOAD_FORMAT)
181
async_engine = AsyncLLMEngine.from_engine_args(engine_args)
182
for i, prompt in enumerate(example_prompts):
183
results = async_engine.generate(
@@ -199,7 +206,8 @@ def test_engine_log_metrics_regression(
206
) -> None:
207
engine_args = EngineArgs(model=model,
208
dtype=dtype,
202
- disable_log_stats=disable_log_stats)
209
+ disable_log_stats=disable_log_stats,
210
+ load_format=RUNAI_STREAMER_LOAD_FORMAT)
211
engine = LLMEngine.from_engine_args(engine_args)
212
for i, prompt in enumerate(example_prompts):
213
engine.add_request(
@@ -283,7 +291,8 @@ def test_metric_spec_decode_interval(
291
gpu_memory_utilization=0.4,
292
speculative_model=model,
293
num_speculative_tokens=k,
286
- enforce_eager=True)
294
+ enforce_eager=True,
295
+ load_format=RUNAI_STREAMER_LOAD_FORMAT)
296
297
engine = LLMEngine.from_engine_args(engine_args)
298
tests/models/registry.py
+2
-1
@@ -173,7 +173,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
173
trust_remote_code=True),
174
"QWenLMHeadModel": _HfExamplesInfo("Qwen/Qwen-7B-Chat",
175
trust_remote_code=True),
176
- "Qwen2ForCausalLM": _HfExamplesInfo("Qwen/Qwen2-7B-Instruct"),
176
+ "Qwen2ForCausalLM": _HfExamplesInfo("Qwen/Qwen2-7B-Instruct",
177
+ extras={"2.5": "Qwen/Qwen2.5-7B-Instruct"}), # noqa: E501
178
"Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
179
"RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b",
180
is_available_online=False),
tests/models/test_initialization.py
+5
-1
@@ -7,6 +7,7 @@ from transformers import PretrainedConfig
7
8
from vllm import LLM
9
10
+from ..conftest import MODELS_ON_S3
11
from .registry import HF_EXAMPLE_MODELS
12
13
@@ -42,8 +43,11 @@ def test_can_initialize(model_arch):
43
44
with patch.object(LLM.get_engine_class(), "_initialize_kv_caches",
45
_initialize_kv_caches):
46
+ model_name = model_info.default
47
+ if model_name in MODELS_ON_S3:
48
+ model_name = f"s3://vllm-ci-model-weights/{model_name.split('/')[-1]}"
49
LLM(
46
- model_info.default,
50
+ model_name,
51
tokenizer=model_info.tokenizer,
52
tokenizer_mode=model_info.tokenizer_mode,
53
speculative_model=model_info.speculative_model,
tests/mq_llm_engine/test_abort.py
+2
-2
@@ -10,8 +10,8 @@ import pytest
10
from tests.mq_llm_engine.utils import RemoteMQLLMEngine, generate
11
from vllm.engine.arg_utils import AsyncEngineArgs
12
13
-MODEL = "google/gemma-1.1-2b-it"
14
-ENGINE_ARGS = AsyncEngineArgs(model=MODEL)
13
+MODEL = "s3://vllm-ci-model-weights/gemma-1.1-2b-it"
14
+ENGINE_ARGS = AsyncEngineArgs(model=MODEL, load_format="runai_streamer")
15
RAISED_ERROR = KeyError
16
RAISED_VALUE = "foo"
17
EXPECTED_TOKENS = 250
tests/mq_llm_engine/test_error_handling.py
+4
-2
@@ -21,8 +21,10 @@ from vllm.lora.request import LoRARequest
21
from vllm.usage.usage_lib import UsageContext
22
from vllm.utils import FlexibleArgumentParser
23
24
-MODEL = "google/gemma-1.1-2b-it"
25
-ENGINE_ARGS = AsyncEngineArgs(model=MODEL, enforce_eager=True)
24
+MODEL = "s3://vllm-ci-model-weights/gemma-1.1-2b-it"
25
+ENGINE_ARGS = AsyncEngineArgs(model=MODEL,
26
+ load_format="runai_streamer",
27
+ enforce_eager=True)
28
RAISED_ERROR = KeyError
29
RAISED_VALUE = "foo"
30
tests/mq_llm_engine/test_load.py
+4
-2
@@ -10,12 +10,14 @@ import pytest
10
from tests.mq_llm_engine.utils import RemoteMQLLMEngine, generate
11
from vllm.engine.arg_utils import AsyncEngineArgs
12
13
-MODEL = "google/gemma-1.1-2b-it"
13
+MODEL = "s3://vllm-ci-model-weights/gemma-1.1-2b-it"
14
NUM_EXPECTED_TOKENS = 10
15
NUM_REQUESTS = 10000
16
17
# Scenarios to test for num generated token.
18
-ENGINE_ARGS = AsyncEngineArgs(model=MODEL, disable_log_requests=True)
18
+ENGINE_ARGS = AsyncEngineArgs(model=MODEL,
19
+ load_format="runai_streamer",
20
+ disable_log_requests=True)
21
22
23
@pytest.fixture(scope="function")
tests/multimodal/test_processing.py
+5
-3
@@ -553,7 +553,8 @@ def test_find_mm_placeholders(
553
assert result == expected
554
555
556
-@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
556
+@pytest.mark.parametrize(
557
+ "model_id", ["s3://vllm-ci-model-weights/llava-v1.6-mistral-7b-hf"])
558
@pytest.mark.parametrize(
559
("limit", "num_supported", "is_valid"),
560
[(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
@@ -592,7 +593,8 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
593
profiler.get_dummy_data(model_config.max_model_len)
594
595
595
-@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
596
+@pytest.mark.parametrize(
597
+ "model_id", ["s3://vllm-ci-model-weights/llava-v1.6-mistral-7b-hf"])
598
@pytest.mark.parametrize(
599
("num_images", "limit", "is_valid"),
600
[(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
@@ -661,7 +663,7 @@ class _ProcessorProxy:
663
return dict(exists=exists)
664
665
664
-@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-7B-Instruct"]) # Dummy
666
+@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"]) # Dummy
667
# yapf: disable
668
@pytest.mark.parametrize(
669
("call_kwargs", "expected_kwargs"),
tests/runai_model_streamer_test/__init__.py
renamed
tests/runai_model_streamer_test/test_runai_model_streamer_loader.py
renamed
tests/runai_model_streamer_test/test_weight_utils.py
renamed
tests/samplers/test_ignore_eos.py
+1
-1
@@ -10,7 +10,7 @@ from vllm import SamplingParams
10
11
# We also test with llama because it has generation_config to specify EOS
12
# (past regression).
13
-MODELS = ["facebook/opt-125m", "meta-llama/Llama-3.2-1B-Instruct"]
13
+MODELS = ["distilbert/distilgpt2", "meta-llama/Llama-3.2-1B"]
14
15
16
@pytest.mark.parametrize("model", MODELS)
tests/samplers/test_logits_processor.py
+1
-1
@@ -5,7 +5,7 @@ import torch
5
6
from vllm import SamplingParams
7
8
-MODELS = ["facebook/opt-125m"]
8
+MODELS = ["distilbert/distilgpt2"]
9
10
11
@pytest.mark.parametrize("model", MODELS)
tests/samplers/test_logprobs.py
+1
-1
@@ -9,7 +9,7 @@ from vllm import SamplingParams
9
10
from ..conftest import VllmRunner
11
12
-MODELS = ["facebook/opt-125m"]
12
+MODELS = ["distilbert/distilgpt2"]
13
14
15
@pytest.mark.parametrize("model", MODELS)
tests/samplers/test_no_bad_words.py
+1
-1
@@ -76,7 +76,7 @@ class TestOneTokenBadWord:
76
77
class TestTwoTokenBadWord:
78
# Another model (with a different tokenizer behaviour)
79
- MODEL = "openai-community/gpt2"
79
+ MODEL = "distilbert/distilgpt2"
80
81
PROMPT = "How old are you? I am 10"
82
TARGET_TOKEN1 = "years"
tests/samplers/test_ranks.py
+1
-1
@@ -4,7 +4,7 @@ import pytest
4
5
from vllm import SamplingParams
6
7
-MODELS = ["facebook/opt-125m"]
7
+MODELS = ["distilbert/distilgpt2"]
8
9
10
@pytest.mark.parametrize("model", MODELS)
tests/test_config.py
+9
-4
@@ -8,14 +8,19 @@ from vllm.config import ModelConfig, PoolerConfig
8
from vllm.model_executor.layers.pooler import PoolingType
9
from vllm.platforms import current_platform
10
11
+from .conftest import MODEL_WEIGHTS_S3_BUCKET
12
+
13
14
@pytest.mark.parametrize(
15
("model_id", "expected_runner_type", "expected_task"),
16
[
15
- ("facebook/opt-125m", "generate", "generate"),
16
- ("intfloat/e5-mistral-7b-instruct", "pooling", "embed"),
17
- ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
18
- ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "score"),
17
+ (f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2", "generate", "generate"),
18
+ (f"{MODEL_WEIGHTS_S3_BUCKET}/e5-mistral-7b-instruct", "pooling",
19
+ "embed"),
20
+ (f"{MODEL_WEIGHTS_S3_BUCKET}/Qwen2.5-1.5B-apeach", "pooling",
21
+ "classify"),
22
+ (f"{MODEL_WEIGHTS_S3_BUCKET}/ms-marco-MiniLM-L-6-v2", "pooling",
23
+ "score"),
24
("Qwen/Qwen2.5-Math-RM-72B", "pooling", "reward"),
25
("openai/whisper-small", "transcription", "transcription"),
26
],
tests/test_regression.py
+10
-3
@@ -10,6 +10,9 @@ import gc
10
import torch
11
12
from vllm import LLM, SamplingParams
13
+from vllm.config import LoadFormat
14
+
15
+from .conftest import MODEL_WEIGHTS_S3_BUCKET
16
17
18
def test_duplicated_ignored_sequence_group():
@@ -18,7 +21,8 @@ def test_duplicated_ignored_sequence_group():
21
sampling_params = SamplingParams(temperature=0.01,
22
top_p=0.1,
23
max_tokens=256)
21
- llm = LLM(model="facebook/opt-125m",
24
+ llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2",
25
+ load_format=LoadFormat.RUNAI_STREAMER,
26
max_num_batched_tokens=4096,
27
tensor_parallel_size=1)
28
prompts = ["This is a short prompt", "This is a very long prompt " * 1000]
@@ -31,7 +35,8 @@ def test_max_tokens_none():
35
sampling_params = SamplingParams(temperature=0.01,
36
top_p=0.1,
37
max_tokens=None)
34
- llm = LLM(model="facebook/opt-125m",
38
+ llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2",
39
+ load_format=LoadFormat.RUNAI_STREAMER,
40
max_num_batched_tokens=4096,
41
tensor_parallel_size=1)
42
prompts = ["Just say hello!"]
@@ -41,7 +46,9 @@ def test_max_tokens_none():
46
47
48
def test_gc():
44
- llm = LLM("facebook/opt-125m", enforce_eager=True)
49
+ llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2",
50
+ load_format=LoadFormat.RUNAI_STREAMER,
51
+ enforce_eager=True)
52
del llm
53
54
gc.collect()
tests/worker/test_swap.py
+1
-1
@@ -10,7 +10,7 @@ from vllm.worker.worker import Worker
10
11
def test_swap() -> None:
12
# Configure the engine.
13
- engine_args = EngineArgs(model="facebook/opt-125m",
13
+ engine_args = EngineArgs(model="s3://vllm-ci-model-weights/distilgpt2",
14
dtype="half",
15
load_format="dummy")
16
engine_config = engine_args.create_engine_config()
vllm/config.py
+2
-1
@@ -409,7 +409,8 @@ class ModelConfig:
409
if is_s3(model) or is_s3(tokenizer):
410
if is_s3(model):
411
s3_model = S3Model()
412
- s3_model.pull_files(model, allow_pattern=["*config.json"])
412
+ s3_model.pull_files(
413
+ model, allow_pattern=["*.model", "*.py", "*.json"])
414
self.model_weights = self.model
415
self.model = s3_model.dir
416
vllm/model_executor/model_loader/loader.py
+1
-1
@@ -1327,6 +1327,7 @@ class RunaiModelStreamerLoader(BaseModelLoader):
1327
"""Prepare weights for the model.
1328
1329
If the model is not local, it will be downloaded."""
1330
+
1331
is_s3_path = is_s3(model_name_or_path)
1332
is_local = os.path.isdir(model_name_or_path)
1333
safetensors_pattern = "*.safetensors"
@@ -1340,7 +1341,6 @@ class RunaiModelStreamerLoader(BaseModelLoader):
1341
revision,
1342
ignore_patterns=self.load_config.ignore_patterns,
1343
))
1343
-
1344
if is_s3_path:
1345
hf_weights_files = s3_glob(path=hf_folder,
1346
allow_pattern=[safetensors_pattern])
vllm/model_executor/model_loader/weight_utils.py
+2
-2
@@ -27,6 +27,8 @@ from vllm.model_executor.layers.quantization import (QuantizationConfig,
27
from vllm.platforms import current_platform
28
from vllm.utils import PlaceholderModule
29
30
+logger = init_logger(__name__)
31
+
32
try:
33
from runai_model_streamer import SafetensorsStreamer
34
except (ImportError, OSError):
@@ -37,8 +39,6 @@ except (ImportError, OSError):
39
SafetensorsStreamer = runai_model_streamer.placeholder_attr(
40
"SafetensorsStreamer")
41
40
-logger = init_logger(__name__)
41
-
42
# use system-level temp directory for file locks, so that multiple users
43
# can share the same lock without error.
44
# lock files in the temp directory will be automatically deleted when the
vllm/transformers_utils/config.py
+1
-2
@@ -144,7 +144,6 @@ def file_exists(
144
revision: Optional[str] = None,
145
token: Union[str, bool, None] = None,
146
) -> bool:
147
-
147
file_list = list_repo_files(repo_id,
148
repo_type=repo_type,
149
revision=revision,
@@ -498,7 +497,7 @@ def get_sentence_transformer_tokenizer_config(model: str,
497
if encoder_dict:
498
break
499
501
- if not encoder_dict:
500
+ if not encoder_dict and not model.startswith("/"):
501
try:
502
# If model is on HuggingfaceHub, get the repo files
503
repo_files = list_repo_files(model,
vllm/transformers_utils/s3_utils.py
+9
-2
@@ -46,6 +46,8 @@ def glob(s3=None,
46
"""
47
if s3 is None:
48
s3 = boto3.client("s3")
49
+ if not path.endswith("/"):
50
+ path = path + "/"
51
bucket_name, _, paths = list_files(s3,
52
path=path,
53
allow_pattern=allow_pattern)
@@ -109,6 +111,7 @@ class S3Model:
111
for sig in (signal.SIGINT, signal.SIGTERM):
112
existing_handler = signal.getsignal(sig)
113
signal.signal(sig, self._close_by_signal(existing_handler))
114
+
115
self.dir = tempfile.mkdtemp()
116
117
def __del__(self):
@@ -140,6 +143,9 @@ class S3Model:
143
ignore_pattern: A list of patterns of which files not to pull.
144
145
"""
146
+ if not s3_model_path.endswith("/"):
147
+ s3_model_path = s3_model_path + "/"
148
+
149
bucket_name, base_dir, files = list_files(self.s3, s3_model_path,
150
allow_pattern,
151
ignore_pattern)
@@ -147,8 +153,9 @@ class S3Model:
153
return
154
155
for file in files:
150
- destination_file = os.path.join(self.dir,
151
- file.removeprefix(base_dir))
156
+ destination_file = os.path.join(
157
+ self.dir,
158
+ file.removeprefix(base_dir).lstrip("/"))
159
local_dir = Path(destination_file).parent
160
os.makedirs(local_dir, exist_ok=True)
161
self.s3.download_file(bucket_name, file, destination_file)