A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[2/n][ci] S3: Use full model path (#13564)
Signed-off-by: <>
Kevin H. Luu committed
Feb 20, 2025 at 01:20 UTC
a64a84433d6d1162290bbcd48b161c8ddff2c801
8 files changed
+26
-19
tests/basic_correctness/test_cumem.py
+1
-1
@@ -121,7 +121,7 @@ def test_cumem_with_cudagraph():
121
"model, use_v1",
122
[
123
# sleep mode with safetensors
124
- (f"{MODEL_WEIGHTS_S3_BUCKET}/Llama-3.2-1B", True),
124
+ (f"{MODEL_WEIGHTS_S3_BUCKET}/meta-llama/Llama-3.2-1B", True),
125
# sleep mode with pytorch checkpoint
126
("facebook/opt-125m", False),
127
])
tests/conftest.py
+1
-2
@@ -746,8 +746,7 @@ class VllmRunner:
746
**kwargs,
747
) -> None:
748
if model_name in MODELS_ON_S3 and not load_format:
749
- model_name = (f"s3://vllm-ci-model-weights/"
750
- f"{model_name.split('/')[-1]}")
749
+ model_name = (f"{MODEL_WEIGHTS_S3_BUCKET}/{model_name}")
750
load_format = LoadFormat.RUNAI_STREAMER
751
if not load_format:
752
load_format = LoadFormat.AUTO
tests/engine/test_computed_prefix_blocks.py
+2
-1
@@ -10,7 +10,8 @@ from vllm.sampling_params import SamplingParams
10
from ..conftest import MODEL_WEIGHTS_S3_BUCKET
11
12
13
-@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
13
+@pytest.mark.parametrize("model",
14
+ [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
15
@pytest.mark.parametrize("block_size", [16])
16
def test_computed_prefix_blocks(model: str, block_size: int):
17
# This test checks if we are able to run the engine to completion
tests/engine/test_detokenization.py
+2
-1
@@ -9,7 +9,8 @@ from vllm.sampling_params import SamplingParams
9
from ..conftest import MODEL_WEIGHTS_S3_BUCKET
10
11
12
-@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
12
+@pytest.mark.parametrize("model",
13
+ [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
14
def test_computed_prefix_blocks(model: str):
15
# This test checks if the engine generates completions both with and
16
# without optional detokenization, that detokenization includes text
tests/engine/test_executor.py
+8
-4
@@ -38,7 +38,8 @@ class CustomUniExecutor(UniProcExecutor):
38
CustomUniExecutorAsync = CustomUniExecutor
39
40
41
-@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
41
+@pytest.mark.parametrize("model",
42
+ [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
43
def test_custom_executor_type_checking(model):
44
with pytest.raises(ValueError):
45
engine_args = EngineArgs(model=model,
@@ -51,7 +52,8 @@ def test_custom_executor_type_checking(model):
52
AsyncLLMEngine.from_engine_args(engine_args)
53
54
54
-@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
55
+@pytest.mark.parametrize("model",
56
+ [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
57
def test_custom_executor(model, tmp_path):
58
cwd = os.path.abspath(".")
59
os.chdir(tmp_path)
@@ -75,7 +77,8 @@ def test_custom_executor(model, tmp_path):
77
os.chdir(cwd)
78
79
78
-@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
80
+@pytest.mark.parametrize("model",
81
+ [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
82
def test_custom_executor_async(model, tmp_path):
83
cwd = os.path.abspath(".")
84
os.chdir(tmp_path)
@@ -103,7 +106,8 @@ def test_custom_executor_async(model, tmp_path):
106
os.chdir(cwd)
107
108
106
-@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
109
+@pytest.mark.parametrize("model",
110
+ [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
111
def test_respect_ray(model):
112
# even for TP=1 and PP=1,
113
# if users specify ray, we should use ray.
tests/engine/test_skip_tokenizer_init.py
+2
-1
@@ -9,7 +9,8 @@ from vllm.sampling_params import SamplingParams
9
from ..conftest import MODEL_WEIGHTS_S3_BUCKET
10
11
12
-@pytest.mark.parametrize("model", [f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2"])
12
+@pytest.mark.parametrize("model",
13
+ [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
14
def test_skip_tokenizer_initialization(model: str):
15
# This test checks if the flag skip_tokenizer_init skips the initialization
16
# of tokenizer and detokenizer. The generated output is expected to contain
tests/test_config.py
+7
-6
@@ -14,13 +14,14 @@ from .conftest import MODEL_WEIGHTS_S3_BUCKET
14
@pytest.mark.parametrize(
15
("model_id", "expected_runner_type", "expected_task"),
16
[
17
- (f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2", "generate", "generate"),
18
- (f"{MODEL_WEIGHTS_S3_BUCKET}/e5-mistral-7b-instruct", "pooling",
19
- "embed"),
20
- (f"{MODEL_WEIGHTS_S3_BUCKET}/Qwen2.5-1.5B-apeach", "pooling",
17
+ (f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2", "generate",
18
+ "generate"),
19
+ (f"{MODEL_WEIGHTS_S3_BUCKET}/intfloat/e5-mistral-7b-instruct",
20
+ "pooling", "embed"),
21
+ (f"{MODEL_WEIGHTS_S3_BUCKET}/jason9693/Qwen2.5-1.5B-apeach", "pooling",
22
"classify"),
22
- (f"{MODEL_WEIGHTS_S3_BUCKET}/ms-marco-MiniLM-L-6-v2", "pooling",
23
- "score"),
23
+ (f"{MODEL_WEIGHTS_S3_BUCKET}/cross-encoder/ms-marco-MiniLM-L-6-v2",
24
+ "pooling", "score"),
25
("Qwen/Qwen2.5-Math-RM-72B", "pooling", "reward"),
26
("openai/whisper-small", "transcription", "transcription"),
27
],
tests/test_regression.py
+3
-3
@@ -21,7 +21,7 @@ def test_duplicated_ignored_sequence_group():
21
sampling_params = SamplingParams(temperature=0.01,
22
top_p=0.1,
23
max_tokens=256)
24
- llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2",
24
+ llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2",
25
load_format=LoadFormat.RUNAI_STREAMER,
26
max_num_batched_tokens=4096,
27
tensor_parallel_size=1)
@@ -35,7 +35,7 @@ def test_max_tokens_none():
35
sampling_params = SamplingParams(temperature=0.01,
36
top_p=0.1,
37
max_tokens=None)
38
- llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2",
38
+ llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2",
39
load_format=LoadFormat.RUNAI_STREAMER,
40
max_num_batched_tokens=4096,
41
tensor_parallel_size=1)
@@ -46,7 +46,7 @@ def test_max_tokens_none():
46
47
48
def test_gc():
49
- llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilgpt2",
49
+ llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2",
50
load_format=LoadFormat.RUNAI_STREAMER,
51
enforce_eager=True)
52
del llm