@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2c5e637b57
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 22:15 UTC

[ci] Use env var to control whether to use S3 bucket in CI (#13634)

Kevin H. Luu committed Feb 22, 2025 at 19:19 UTC 2c5e637b57bcb2d5b3c7d992fd8b75a8bbeafcc3
30 files changed +222 -231
.buildkite/test-pipeline.yaml
+2 -2
@@ -278,7 +278,7 @@ steps:
278 command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_minicpmv_tp.py
279 parallelism: 4
280
281 -- label: "PyTorch Fullgraph Smoke Test" # 9min
281 +- label: PyTorch Fullgraph Smoke Test # 9min
282 fast_check: true
283 source_file_dependencies:
284 - vllm/
@@ -289,7 +289,7 @@ steps:
289 - pytest -v -s compile/piecewise/test_simple.py
290 - pytest -v -s compile/piecewise/test_toy_llama.py
291
292 -- label: "PyTorch Fullgraph Test" # 18min
292 +- label: PyTorch Fullgraph Test # 18min
293 source_file_dependencies:
294 - vllm/
295 - tests/compile
tests/basic_correctness/test_basic_correctness.py
+5 -6
@@ -9,7 +9,6 @@ import weakref
9 import pytest
10
11 from vllm import LLM
12 -from vllm.config import LoadFormat
12 from vllm.platforms import current_platform
13
14 from ..conftest import VllmRunner
@@ -34,7 +33,7 @@ def v1(run_with_both_engines):
33
34 def test_vllm_gc_ed():
35 """Verify vllm instance is GC'ed when it is deleted"""
37 - llm = LLM("distilbert/distilgpt2", load_format=LoadFormat.RUNAI_STREAMER)
36 + llm = LLM("distilbert/distilgpt2")
37 weak_llm = weakref.ref(llm)
38 del llm
39 # If there's any circular reference to vllm, this fails
@@ -43,10 +42,10 @@ def test_vllm_gc_ed():
42
43
44 @pytest.mark.parametrize("model", MODELS)
46 -@pytest.mark.parametrize("backend", ["FLASH_ATTN", "XFORMERS", "FLASHINFER"])
45 +@pytest.mark.parametrize("backend", ["FLASH_ATTN"])
46 @pytest.mark.parametrize("dtype", ["half"])
47 @pytest.mark.parametrize("max_tokens", [5])
49 -@pytest.mark.parametrize("enforce_eager", [False, True])
48 +@pytest.mark.parametrize("enforce_eager", [False])
49 def test_models(
50 hf_runner,
51 model: str,
@@ -97,8 +96,8 @@ def test_models(
96 "test_suite", [
97 ("distilbert/distilgpt2", "ray", "", "L4"),
98 ("distilbert/distilgpt2", "mp", "", "L4"),
100 - ("meta-llama/Llama-2-7b-hf", "ray", "", "L4"),
101 - ("meta-llama/Llama-2-7b-hf", "mp", "", "L4"),
99 + ("meta-llama/Llama-3.2-1B-Instruct", "ray", "", "L4"),
100 + ("meta-llama/Llama-3.2-1B-Instruct", "mp", "", "L4"),
101 ("distilbert/distilgpt2", "ray", "", "A100"),
102 ("distilbert/distilgpt2", "mp", "", "A100"),
103 ("distilbert/distilgpt2", "mp", "FLASHINFER", "A100"),
tests/basic_correctness/test_cumem.py
+2 -7
@@ -4,11 +4,9 @@ import pytest
4 import torch
5
6 from vllm import LLM, SamplingParams
7 -from vllm.config import LoadFormat
7 from vllm.device_allocator.cumem import CuMemAllocator
8 from vllm.utils import GiB_bytes
9
11 -from ..conftest import MODEL_WEIGHTS_S3_BUCKET
10 from ..utils import fork_new_process_for_each_test
11
12
@@ -121,7 +119,7 @@ def test_cumem_with_cudagraph():
119 "model, use_v1",
120 [
121 # sleep mode with safetensors
124 - (f"{MODEL_WEIGHTS_S3_BUCKET}/meta-llama/Llama-3.2-1B", True),
122 + ("meta-llama/Llama-3.2-1B", True),
123 # sleep mode with pytorch checkpoint
124 ("facebook/opt-125m", False),
125 ])
@@ -130,10 +128,7 @@ def test_end_to_end(model: str, use_v1: bool):
128 os.environ["VLLM_USE_V1"] = "1" if use_v1 else "0"
129 free, total = torch.cuda.mem_get_info()
130 used_bytes_baseline = total - free # in case other process is running
133 - load_format = LoadFormat.AUTO
134 - if "Llama" in model:
135 - load_format = LoadFormat.RUNAI_STREAMER
136 - llm = LLM(model, load_format=load_format, enable_sleep_mode=True)
131 + llm = LLM(model, enable_sleep_mode=True)
132 prompt = "How are you?"
133 sampling_params = SamplingParams(temperature=0, max_tokens=10)
134 output = llm.generate(prompt, sampling_params)
tests/conftest.py
+1 -72
@@ -24,7 +24,7 @@ from tests.models.utils import (TokensTextLogprobs,
24 from vllm import LLM, SamplingParams
25 from vllm.assets.image import ImageAsset
26 from vllm.assets.video import VideoAsset
27 -from vllm.config import LoadFormat, TaskOption, TokenizerPoolConfig
27 +from vllm.config import TaskOption, TokenizerPoolConfig
28 from vllm.connections import global_http_connection
29 from vllm.distributed import (cleanup_dist_env_and_memory,
30 init_distributed_environment,
@@ -47,70 +47,6 @@ _SYS_MSG = os.path.join(_TEST_DIR, "system_messages", "sonnet3.5_nov2024.txt")
47
48 _M = TypeVar("_M")
49
50 -MODELS_ON_S3 = [
51 - "distilbert/distilgpt2",
52 - "meta-llama/Llama-2-7b-hf",
53 - "meta-llama/Meta-Llama-3-8B",
54 - "meta-llama/Llama-3.2-1B",
55 - "meta-llama/Llama-3.2-1B-Instruct",
56 - "openai-community/gpt2",
57 - "ArthurZ/Ilama-3.2-1B",
58 - "llava-hf/llava-1.5-7b-hf",
59 - "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
60 - "ai21labs/Jamba-tiny-random",
61 - "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
62 - "nm-testing/Phi-3-mini-128k-instruct-FP8",
63 - "nm-testing/Qwen2-0.5B-Instruct-FP8-SkipQKV",
64 - "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
65 - "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V",
66 - "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
67 - "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse",
68 - "AMead10/Llama-3.2-1B-Instruct-AWQ",
69 - "shuyuej/Llama-3.2-1B-Instruct-GPTQ",
70 - "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head",
71 - "ModelCloud/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit-10-25-2024",
72 - "TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",
73 - "neuralmagic/Meta-Llama-3-8B-Instruct-FP8",
74 - "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test",
75 - "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change",
76 - "nm-testing/tinyllama-oneshot-w8-channel-a8-tensor",
77 - "nm-testing/asym-w8w8-int8-static-per-tensor-tiny-llama",
78 - "neuralmagic/Llama-3.2-1B-quantized.w8a8",
79 - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Asym",
80 - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym",
81 - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym",
82 - "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change",
83 - "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2",
84 - "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2-asym",
85 - "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2",
86 - "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2-asym",
87 - "nm-testing/tinyllama-oneshot-w4a16-channel-v2",
88 - "nm-testing/tinyllama-oneshot-w4a16-group128-v2",
89 - "nm-testing/tinyllama-oneshot-w8a16-per-channel",
90 - "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
91 - "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test",
92 - "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme",
93 - "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-2of4-testing",
94 - "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-Per-Tensor-testing",
95 - "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-testing",
96 - "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-IA-Per-Tensor-Weight-testing",
97 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM",
98 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_tensor_act_fp8-BitM",
99 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_per_tok_dyn_act_fp8-BitM",
100 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_tensor_act_fp8-BitM",
101 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_int8-BitM",
102 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_tensor_act_int8-BitM",
103 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_per_tok_dyn_act_int8-BitM",
104 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_tensor_act_int8-BitM",
105 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing",
106 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Static-testing",
107 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Tensor-Weight-testing",
108 - "nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor",
109 - "nm-testing/llama2.c-stories42M-pruned2.4-compressed",
110 -]
111 -
112 -MODEL_WEIGHTS_S3_BUCKET = "s3://vllm-ci-model-weights"
113 -
50 _PromptMultiModalInput = Union[List[_M], List[List[_M]]]
51
52 PromptImageInput = _PromptMultiModalInput[Image.Image]
@@ -742,14 +678,8 @@ class VllmRunner:
678 enable_chunked_prefill: bool = False,
679 swap_space: int = 4,
680 enforce_eager: Optional[bool] = False,
745 - load_format: Optional[LoadFormat] = None,
681 **kwargs,
682 ) -> None:
748 - if model_name in MODELS_ON_S3 and not load_format:
749 - model_name = (f"{MODEL_WEIGHTS_S3_BUCKET}/{model_name}")
750 - load_format = LoadFormat.RUNAI_STREAMER
751 - if not load_format:
752 - load_format = LoadFormat.AUTO
683 self.model = LLM(
684 model=model_name,
685 task=task,
@@ -764,7 +694,6 @@ class VllmRunner:
694 max_model_len=max_model_len,
695 block_size=block_size,
696 enable_chunked_prefill=enable_chunked_prefill,
767 - load_format=load_format,
697 **kwargs,
698 )
699
tests/engine/test_computed_prefix_blocks.py
+1 -6
@@ -2,16 +2,12 @@
2
3 import pytest
4
5 -from vllm.config import LoadFormat
5 from vllm.engine.arg_utils import EngineArgs
6 from vllm.engine.llm_engine import LLMEngine
7 from vllm.sampling_params import SamplingParams
8
10 -from ..conftest import MODEL_WEIGHTS_S3_BUCKET
9
12 -
13 -@pytest.mark.parametrize("model",
14 - [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
10 +@pytest.mark.parametrize("model", ["distilbert/distilgpt2"])
11 @pytest.mark.parametrize("block_size", [16])
12 def test_computed_prefix_blocks(model: str, block_size: int):
13 # This test checks if we are able to run the engine to completion
@@ -28,7 +24,6 @@ def test_computed_prefix_blocks(model: str, block_size: int):
24 "decoration.")
25
26 engine_args = EngineArgs(model=model,
31 - load_format=LoadFormat.RUNAI_STREAMER,
27 block_size=block_size,
28 enable_prefix_caching=True)
29
tests/engine/test_detokenization.py
+2 -6
@@ -2,15 +2,11 @@
2
3 import pytest
4
5 -from vllm.config import LoadFormat
5 from vllm.entrypoints.llm import LLM
6 from vllm.sampling_params import SamplingParams
7
9 -from ..conftest import MODEL_WEIGHTS_S3_BUCKET
8
11 -
12 -@pytest.mark.parametrize("model",
13 - [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
9 +@pytest.mark.parametrize("model", ["distilbert/distilgpt2"])
10 def test_computed_prefix_blocks(model: str):
11 # This test checks if the engine generates completions both with and
12 # without optional detokenization, that detokenization includes text
@@ -21,7 +17,7 @@ def test_computed_prefix_blocks(model: str):
17 "paper clips? Is there an easy to follow video tutorial available "
18 "online for free?")
19
24 - llm = LLM(model=model, load_format=LoadFormat.RUNAI_STREAMER)
20 + llm = LLM(model=model)
21 sampling_params = SamplingParams(max_tokens=10,
22 temperature=0.0,
23 detokenize=False)
tests/engine/test_executor.py
+4 -17
@@ -6,17 +6,12 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
6
7 import pytest
8
9 -from vllm.config import LoadFormat
9 from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
10 from vllm.engine.async_llm_engine import AsyncLLMEngine
11 from vllm.engine.llm_engine import LLMEngine
12 from vllm.executor.uniproc_executor import UniProcExecutor
13 from vllm.sampling_params import SamplingParams
14
16 -from ..conftest import MODEL_WEIGHTS_S3_BUCKET
17 -
18 -RUNAI_STREAMER_LOAD_FORMAT = LoadFormat.RUNAI_STREAMER
19 -
15
16 class Mock:
17 ...
@@ -38,12 +33,10 @@ class CustomUniExecutor(UniProcExecutor):
33 CustomUniExecutorAsync = CustomUniExecutor
34
35
41 -@pytest.mark.parametrize("model",
42 - [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
36 +@pytest.mark.parametrize("model", ["distilbert/distilgpt2"])
37 def test_custom_executor_type_checking(model):
38 with pytest.raises(ValueError):
39 engine_args = EngineArgs(model=model,
46 - load_format=RUNAI_STREAMER_LOAD_FORMAT,
40 distributed_executor_backend=Mock)
41 LLMEngine.from_engine_args(engine_args)
42 with pytest.raises(ValueError):
@@ -52,8 +45,7 @@ def test_custom_executor_type_checking(model):
45 AsyncLLMEngine.from_engine_args(engine_args)
46
47
55 -@pytest.mark.parametrize("model",
56 - [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
48 +@pytest.mark.parametrize("model", ["distilbert/distilgpt2"])
49 def test_custom_executor(model, tmp_path):
50 cwd = os.path.abspath(".")
51 os.chdir(tmp_path)
@@ -62,7 +54,6 @@ def test_custom_executor(model, tmp_path):
54
55 engine_args = EngineArgs(
56 model=model,
65 - load_format=RUNAI_STREAMER_LOAD_FORMAT,
57 distributed_executor_backend=CustomUniExecutor,
58 enforce_eager=True, # reduce test time
59 )
@@ -77,8 +68,7 @@ def test_custom_executor(model, tmp_path):
68 os.chdir(cwd)
69
70
80 -@pytest.mark.parametrize("model",
81 - [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
71 +@pytest.mark.parametrize("model", ["distilbert/distilgpt2"])
72 def test_custom_executor_async(model, tmp_path):
73 cwd = os.path.abspath(".")
74 os.chdir(tmp_path)
@@ -87,7 +77,6 @@ def test_custom_executor_async(model, tmp_path):
77
78 engine_args = AsyncEngineArgs(
79 model=model,
90 - load_format=RUNAI_STREAMER_LOAD_FORMAT,
80 distributed_executor_backend=CustomUniExecutorAsync,
81 enforce_eager=True, # reduce test time
82 )
@@ -106,8 +95,7 @@ def test_custom_executor_async(model, tmp_path):
95 os.chdir(cwd)
96
97
109 -@pytest.mark.parametrize("model",
110 - [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
98 +@pytest.mark.parametrize("model", ["distilbert/distilgpt2"])
99 def test_respect_ray(model):
100 # even for TP=1 and PP=1,
101 # if users specify ray, we should use ray.
@@ -116,7 +104,6 @@ def test_respect_ray(model):
104 engine_args = EngineArgs(
105 model=model,
106 distributed_executor_backend="ray",
119 - load_format=RUNAI_STREAMER_LOAD_FORMAT,
107 enforce_eager=True, # reduce test time
108 )
109 engine = LLMEngine.from_engine_args(engine_args)
tests/engine/test_skip_tokenizer_init.py
+5 -8
@@ -2,22 +2,19 @@
2
3 import pytest
4
5 -from vllm.config import LoadFormat
5 from vllm.entrypoints.llm import LLM
6 from vllm.sampling_params import SamplingParams
7
9 -from ..conftest import MODEL_WEIGHTS_S3_BUCKET
8
11 -
12 -@pytest.mark.parametrize("model",
13 - [f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2"])
9 +@pytest.mark.parametrize("model", ["distilbert/distilgpt2"])
10 def test_skip_tokenizer_initialization(model: str):
11 # This test checks if the flag skip_tokenizer_init skips the initialization
12 # of tokenizer and detokenizer. The generated output is expected to contain
13 # token ids.
18 - llm = LLM(model=model,
19 - skip_tokenizer_init=True,
20 - load_format=LoadFormat.RUNAI_STREAMER)
14 + llm = LLM(
15 + model=model,
16 + skip_tokenizer_init=True,
17 + )
18 sampling_params = SamplingParams(prompt_logprobs=True, detokenize=True)
19
20 with pytest.raises(ValueError, match="cannot pass text prompts when"):
tests/entrypoints/llm/test_chat.py
+3 -10
@@ -5,17 +5,12 @@ from typing import List
5 import pytest
6
7 from vllm import LLM
8 -from vllm.config import LoadFormat
8
10 -from ...conftest import MODEL_WEIGHTS_S3_BUCKET
9 from ..openai.test_vision import TEST_IMAGE_URLS
10
13 -RUNAI_STREAMER_LOAD_FORMAT = LoadFormat.RUNAI_STREAMER
14 -
11
12 def test_chat():
17 - llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/Llama-3.2-1B-Instruct",
18 - load_format=RUNAI_STREAMER_LOAD_FORMAT)
13 + llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct")
14
15 prompt1 = "Explain the concept of entropy."
16 messages = [
@@ -33,8 +28,7 @@ def test_chat():
28
29
30 def test_multi_chat():
36 - llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/Llama-3.2-1B-Instruct",
37 - load_format=RUNAI_STREAMER_LOAD_FORMAT)
31 + llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct")
32
33 prompt1 = "Explain the concept of entropy."
34 prompt2 = "Explain what among us is."
@@ -71,8 +65,7 @@ def test_multi_chat():
65 [[TEST_IMAGE_URLS[0], TEST_IMAGE_URLS[1]]])
66 def test_chat_multi_image(image_urls: List[str]):
67 llm = LLM(
74 - model=f"{MODEL_WEIGHTS_S3_BUCKET}/Phi-3.5-vision-instruct",
75 - load_format=RUNAI_STREAMER_LOAD_FORMAT,
68 + model="microsoft/Phi-3.5-vision-instruct",
69 dtype="bfloat16",
70 max_model_len=4096,
71 max_num_seqs=5,
tests/entrypoints/llm/test_collective_rpc.py
+1 -1
@@ -28,7 +28,7 @@ def test_collective_rpc(tp_size, backend):
28 def echo_rank(self):
29 return self.rank
30
31 - llm = LLM(model="s3://vllm-ci-model-weights/Llama-3.2-1B-Instruct",
31 + llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct",
32 enforce_eager=True,
33 load_format="dummy",
34 tensor_parallel_size=tp_size,
tests/entrypoints/llm/test_encode.py
+1 -3
@@ -6,10 +6,9 @@ from typing import List
6 import pytest
7
8 from vllm import LLM, PoolingParams, PoolingRequestOutput
9 -from vllm.config import LoadFormat
9 from vllm.distributed import cleanup_dist_env_and_memory
10
12 -MODEL_NAME = "s3://vllm-ci-model-weights/e5-mistral-7b-instruct"
11 +MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
12
13 PROMPTS = [
14 "Hello, my name is",
@@ -33,7 +32,6 @@ def llm():
32 # pytest caches the fixture so we use weakref.proxy to
33 # enable garbage collection
34 llm = LLM(model=MODEL_NAME,
36 - load_format=LoadFormat.RUNAI_STREAMER,
35 max_num_batched_tokens=32768,
36 tensor_parallel_size=1,
37 gpu_memory_utilization=0.75,
tests/entrypoints/llm/test_generate.py
+1 -3
@@ -6,10 +6,9 @@ from typing import List
6 import pytest
7
8 from vllm import LLM, RequestOutput, SamplingParams
9 -from vllm.config import LoadFormat
9 from vllm.distributed import cleanup_dist_env_and_memory
10
12 -MODEL_NAME = "s3://vllm-ci-model-weights/distilgpt2"
11 +MODEL_NAME = "distilbert/distilgpt2"
12
13 PROMPTS = [
14 "Hello, my name is",
@@ -31,7 +30,6 @@ def llm():
30 # pytest caches the fixture so we use weakref.proxy to
31 # enable garbage collection
32 llm = LLM(model=MODEL_NAME,
34 - load_format=LoadFormat.RUNAI_STREAMER,
33 max_num_batched_tokens=4096,
34 tensor_parallel_size=1,
35 gpu_memory_utilization=0.10,
tests/entrypoints/llm/test_generate_multiple_loras.py
+1 -3
@@ -7,11 +7,10 @@ import pytest
7 from huggingface_hub import snapshot_download
8
9 from vllm import LLM
10 -from vllm.config import LoadFormat
10 from vllm.distributed import cleanup_dist_env_and_memory
11 from vllm.lora.request import LoRARequest
12
14 -MODEL_NAME = "s3://vllm-ci-model-weights/zephyr-7b-beta"
13 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
14
15 PROMPTS = [
16 "Hello, my name is",
@@ -28,7 +27,6 @@ def llm():
27 # pytest caches the fixture so we use weakref.proxy to
28 # enable garbage collection
29 llm = LLM(model=MODEL_NAME,
31 - load_format=LoadFormat.RUNAI_STREAMER,
30 tensor_parallel_size=1,
31 max_model_len=8192,
32 enable_lora=True,
tests/entrypoints/llm/test_guided_generate.py
+2 -5
@@ -7,13 +7,12 @@ import weakref
7 import jsonschema
8 import pytest
9
10 -from vllm.config import LoadFormat
10 from vllm.distributed import cleanup_dist_env_and_memory
11 from vllm.entrypoints.llm import LLM
12 from vllm.outputs import RequestOutput
13 from vllm.sampling_params import GuidedDecodingParams, SamplingParams
14
16 -MODEL_NAME = "s3://vllm-ci-model-weights/Qwen2.5-1.5B-Instruct"
15 +MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
16 GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
17
18
@@ -21,9 +20,7 @@ GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
20 def llm():
21 # pytest caches the fixture so we use weakref.proxy to
22 # enable garbage collection
24 - llm = LLM(model=MODEL_NAME,
25 - load_format=LoadFormat.RUNAI_STREAMER,
26 - max_model_len=1024)
23 + llm = LLM(model=MODEL_NAME, max_model_len=1024)
24
25 with llm.deprecate_legacy_api():
26 yield weakref.proxy(llm)
tests/entrypoints/llm/test_lazy_outlines.py
+2 -5
@@ -6,7 +6,6 @@ from contextlib import nullcontext
6 from vllm_test_utils import BlameResult, blame
7
8 from vllm import LLM, SamplingParams
9 -from vllm.config import LoadFormat
9 from vllm.distributed import cleanup_dist_env_and_memory
10
11
@@ -44,8 +43,7 @@ def run_normal():
43 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
44
45 # Create an LLM without guided decoding as a baseline.
47 - llm = LLM(model="s3://vllm-ci-model-weights/distilgpt2",
48 - load_format=LoadFormat.RUNAI_STREAMER,
46 + llm = LLM(model="distilbert/distilgpt2",
47 enforce_eager=True,
48 gpu_memory_utilization=0.3)
49 outputs = llm.generate(prompts, sampling_params)
@@ -61,8 +59,7 @@ def run_normal():
59
60 def run_lmfe(sample_regex):
61 # Create an LLM with guided decoding enabled.
64 - llm = LLM(model="s3://vllm-ci-model-weights/distilgpt2",
65 - load_format=LoadFormat.RUNAI_STREAMER,
62 + llm = LLM(model="distilbert/distilgpt2",
63 enforce_eager=True,
64 guided_decoding_backend="lm-format-enforcer",
65 gpu_memory_utilization=0.3)
tests/entrypoints/llm/test_prompt_validation.py
+2 -7
@@ -3,7 +3,6 @@
3 import pytest
4
5 from vllm import LLM
6 -from vllm.config import LoadFormat
6
7
8 @pytest.fixture(autouse=True)
@@ -15,17 +14,13 @@ def v1(run_with_both_engines):
14
15
16 def test_empty_prompt():
18 - llm = LLM(model="s3://vllm-ci-model-weights/gpt2",
19 - load_format=LoadFormat.RUNAI_STREAMER,
20 - enforce_eager=True)
17 + llm = LLM(model="openai-community/gpt2", enforce_eager=True)
18 with pytest.raises(ValueError, match='Prompt cannot be empty'):
19 llm.generate([""])
20
21
22 @pytest.mark.skip_v1
23 def test_out_of_vocab_token():
27 - llm = LLM(model="s3://vllm-ci-model-weights/gpt2",
28 - load_format=LoadFormat.RUNAI_STREAMER,
29 - enforce_eager=True)
24 + llm = LLM(model="openai-community/gpt2", enforce_eager=True)
25 with pytest.raises(ValueError, match='out of vocabulary'):
26 llm.generate({"prompt_token_ids": [999999]})
tests/metrics/test_metrics.py
+27 -28
@@ -8,21 +8,17 @@ import ray
8 from prometheus_client import REGISTRY
9
10 from vllm import EngineArgs, LLMEngine
11 -from vllm.config import LoadFormat
11 from vllm.distributed import cleanup_dist_env_and_memory
12 from vllm.engine.arg_utils import AsyncEngineArgs
13 from vllm.engine.async_llm_engine import AsyncLLMEngine
14 from vllm.engine.metrics import RayPrometheusStatLogger
15 from vllm.sampling_params import SamplingParams
17 -
18 -from ..conftest import MODEL_WEIGHTS_S3_BUCKET
16 +from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET
17
18 MODELS = [
19 "distilbert/distilgpt2",
20 ]
21
24 -RUNAI_STREAMER_LOAD_FORMAT = LoadFormat.RUNAI_STREAMER
25 -
22
23 @pytest.mark.parametrize("model", MODELS)
24 @pytest.mark.parametrize("dtype", ["float"])
@@ -146,9 +142,8 @@ def test_metric_set_tag_model_name(vllm_runner, model: str, dtype: str,
142 metrics_tag_content = stat_logger.labels["model_name"]
143
144 if served_model_name is None or served_model_name == []:
149 - actual_model_name = f"{MODEL_WEIGHTS_S3_BUCKET}/{model}"
150 - assert metrics_tag_content == actual_model_name, (
151 - f"Metrics tag model_name is wrong! expect: {actual_model_name!r}\n"
145 + assert metrics_tag_content == f"{MODEL_WEIGHTS_S3_BUCKET}/{model}", (
146 + f"Metrics tag model_name is wrong! expect: {model!r}\n"
147 f"actual: {metrics_tag_content!r}")
148 else:
149 assert metrics_tag_content == served_model_name[0], (
@@ -174,10 +169,11 @@ async def test_async_engine_log_metrics_regression(
169 when disable_log_stats=False
170 (see: https://github.com/vllm-project/vllm/pull/4150#pullrequestreview-2008176678)
171 """
177 - engine_args = AsyncEngineArgs(model=model,
178 - dtype=dtype,
179 - disable_log_stats=disable_log_stats,
180 - load_format=RUNAI_STREAMER_LOAD_FORMAT)
172 + engine_args = AsyncEngineArgs(
173 + model=model,
174 + dtype=dtype,
175 + disable_log_stats=disable_log_stats,
176 + )
177 async_engine = AsyncLLMEngine.from_engine_args(engine_args)
178 for i, prompt in enumerate(example_prompts):
179 results = async_engine.generate(
@@ -189,7 +185,7 @@ async def test_async_engine_log_metrics_regression(
185 async for _ in results:
186 pass
187
192 - assert_metrics(async_engine.engine, disable_log_stats,
188 + assert_metrics(model, async_engine.engine, disable_log_stats,
189 len(example_prompts))
190
191
@@ -204,10 +200,11 @@ def test_engine_log_metrics_regression(
200 max_tokens: int,
201 disable_log_stats: bool,
202 ) -> None:
207 - engine_args = EngineArgs(model=model,
208 - dtype=dtype,
209 - disable_log_stats=disable_log_stats,
210 - load_format=RUNAI_STREAMER_LOAD_FORMAT)
203 + engine_args = EngineArgs(
204 + model=model,
205 + dtype=dtype,
206 + disable_log_stats=disable_log_stats,
207 + )
208 engine = LLMEngine.from_engine_args(engine_args)
209 for i, prompt in enumerate(example_prompts):
210 engine.add_request(
@@ -218,7 +215,8 @@ def test_engine_log_metrics_regression(
215 while engine.has_unfinished_requests():
216 engine.step()
217
221 - assert_metrics(engine, disable_log_stats, len(example_prompts))
218 + assert_metrics(f"{MODEL_WEIGHTS_S3_BUCKET}/{model}", engine,
219 + disable_log_stats, len(example_prompts))
220
221
222 @pytest.mark.parametrize("model", MODELS)
@@ -285,14 +283,15 @@ def test_metric_spec_decode_interval(
283 ) -> None:
284 k = 5
285
288 - engine_args = EngineArgs(model=model,
289 - dtype=dtype,
290 - disable_log_stats=False,
291 - gpu_memory_utilization=0.4,
292 - speculative_model=model,
293 - num_speculative_tokens=k,
294 - enforce_eager=True,
295 - load_format=RUNAI_STREAMER_LOAD_FORMAT)
286 + engine_args = EngineArgs(
287 + model=model,
288 + dtype=dtype,
289 + disable_log_stats=False,
290 + gpu_memory_utilization=0.4,
291 + speculative_model=model,
292 + num_speculative_tokens=k,
293 + enforce_eager=True,
294 + )
295
296 engine = LLMEngine.from_engine_args(engine_args)
297
@@ -359,7 +358,7 @@ def test_metric_spec_decode_interval(
358 cleanup_dist_env_and_memory()
359
360
362 -def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
361 +def assert_metrics(model: str, engine: LLMEngine, disable_log_stats: bool,
362 num_requests: int) -> None:
363 if disable_log_stats:
364 with pytest.raises(AttributeError):
@@ -370,7 +369,7 @@ def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
369 # Ensure the count bucket of request-level histogram metrics matches
370 # the number of requests as a simple sanity check to ensure metrics are
371 # generated
373 - labels = {'model_name': engine.model_config.model}
372 + labels = {'model_name': model}
373 request_histogram_metrics = [
374 "vllm:e2e_request_latency_seconds",
375 "vllm:request_prompt_tokens",
tests/models/test_initialization.py
+1 -5
@@ -7,7 +7,6 @@ from transformers import PretrainedConfig
7
8 from vllm import LLM
9
10 -from ..conftest import MODELS_ON_S3
10 from .registry import HF_EXAMPLE_MODELS
11
12
@@ -43,11 +42,8 @@ def test_can_initialize(model_arch):
42
43 with patch.object(LLM.get_engine_class(), "_initialize_kv_caches",
44 _initialize_kv_caches):
46 - model_name = model_info.default
47 - if model_name in MODELS_ON_S3:
48 - model_name = f"s3://vllm-ci-model-weights/{model_name.split('/')[-1]}"
45 LLM(
50 - model_name,
46 + model_info.default,
47 tokenizer=model_info.tokenizer,
48 tokenizer_mode=model_info.tokenizer_mode,
49 speculative_model=model_info.speculative_model,
tests/mq_llm_engine/test_abort.py
+2 -2
@@ -10,8 +10,8 @@ import pytest
10 from tests.mq_llm_engine.utils import RemoteMQLLMEngine, generate
11 from vllm.engine.arg_utils import AsyncEngineArgs
12
13 -MODEL = "s3://vllm-ci-model-weights/gemma-1.1-2b-it"
14 -ENGINE_ARGS = AsyncEngineArgs(model=MODEL, load_format="runai_streamer")
13 +MODEL = "google/gemma-1.1-2b-it"
14 +ENGINE_ARGS = AsyncEngineArgs(model=MODEL)
15 RAISED_ERROR = KeyError
16 RAISED_VALUE = "foo"
17 EXPECTED_TOKENS = 250
tests/mq_llm_engine/test_error_handling.py
+2 -4
@@ -21,10 +21,8 @@ from vllm.lora.request import LoRARequest
21 from vllm.usage.usage_lib import UsageContext
22 from vllm.utils import FlexibleArgumentParser
23
24 -MODEL = "s3://vllm-ci-model-weights/gemma-1.1-2b-it"
25 -ENGINE_ARGS = AsyncEngineArgs(model=MODEL,
26 - load_format="runai_streamer",
27 - enforce_eager=True)
24 +MODEL = "google/gemma-1.1-2b-it"
25 +ENGINE_ARGS = AsyncEngineArgs(model=MODEL, enforce_eager=True)
26 RAISED_ERROR = KeyError
27 RAISED_VALUE = "foo"
28
tests/mq_llm_engine/test_load.py
+2 -4
@@ -10,14 +10,12 @@ import pytest
10 from tests.mq_llm_engine.utils import RemoteMQLLMEngine, generate
11 from vllm.engine.arg_utils import AsyncEngineArgs
12
13 -MODEL = "s3://vllm-ci-model-weights/gemma-1.1-2b-it"
13 +MODEL = "google/gemma-1.1-2b-it"
14 NUM_EXPECTED_TOKENS = 10
15 NUM_REQUESTS = 10000
16
17 # Scenarios to test for num generated token.
18 -ENGINE_ARGS = AsyncEngineArgs(model=MODEL,
19 - load_format="runai_streamer",
20 - disable_log_requests=True)
18 +ENGINE_ARGS = AsyncEngineArgs(model=MODEL, disable_log_requests=True)
19
20
21 @pytest.fixture(scope="function")
tests/multimodal/test_processing.py
+2 -4
@@ -553,8 +553,7 @@ def test_find_mm_placeholders(
553 assert result == expected
554
555
556 -@pytest.mark.parametrize(
557 - "model_id", ["s3://vllm-ci-model-weights/llava-v1.6-mistral-7b-hf"])
556 +@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
557 @pytest.mark.parametrize(
558 ("limit", "num_supported", "is_valid"),
559 [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
@@ -593,8 +592,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
592 profiler.get_dummy_data(model_config.max_model_len)
593
594
596 -@pytest.mark.parametrize(
597 - "model_id", ["s3://vllm-ci-model-weights/llava-v1.6-mistral-7b-hf"])
595 +@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
596 @pytest.mark.parametrize(
597 ("num_images", "limit", "is_valid"),
598 [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
tests/prefix_caching/test_prefix_caching.py
+1 -1
@@ -16,7 +16,7 @@ from vllm.engine.llm_engine import LLMEngine
16 from ..models.utils import check_outputs_equal
17
18 MODELS = [
19 - "facebook/opt-125m",
19 + "distilbert/distilgpt2",
20 ]
21
22 UNSTABLE_PROMPT_SEQUENCE = [
tests/test_config.py
+4 -10
@@ -8,20 +8,14 @@ from vllm.config import ModelConfig, PoolerConfig
8 from vllm.model_executor.layers.pooler import PoolingType
9 from vllm.platforms import current_platform
10
11 -from .conftest import MODEL_WEIGHTS_S3_BUCKET
12 -
11
12 @pytest.mark.parametrize(
13 ("model_id", "expected_runner_type", "expected_task"),
14 [
17 - (f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2", "generate",
18 - "generate"),
19 - (f"{MODEL_WEIGHTS_S3_BUCKET}/intfloat/e5-mistral-7b-instruct",
20 - "pooling", "embed"),
21 - (f"{MODEL_WEIGHTS_S3_BUCKET}/jason9693/Qwen2.5-1.5B-apeach", "pooling",
22 - "classify"),
23 - (f"{MODEL_WEIGHTS_S3_BUCKET}/cross-encoder/ms-marco-MiniLM-L-6-v2",
24 - "pooling", "score"),
15 + ("distilbert/distilgpt2", "generate", "generate"),
16 + ("intfloat/e5-mistral-7b-instruct", "pooling", "embed"),
17 + ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),
18 + ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "score"),
19 ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "reward"),
20 ("openai/whisper-small", "transcription", "transcription"),
21 ],
tests/test_regression.py
+3 -10
@@ -10,9 +10,6 @@ import gc
10 import torch
11
12 from vllm import LLM, SamplingParams
13 -from vllm.config import LoadFormat
14 -
15 -from .conftest import MODEL_WEIGHTS_S3_BUCKET
13
14
15 def test_duplicated_ignored_sequence_group():
@@ -21,8 +18,7 @@ def test_duplicated_ignored_sequence_group():
18 sampling_params = SamplingParams(temperature=0.01,
19 top_p=0.1,
20 max_tokens=256)
24 - llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2",
25 - load_format=LoadFormat.RUNAI_STREAMER,
21 + llm = LLM(model="distilbert/distilgpt2",
22 max_num_batched_tokens=4096,
23 tensor_parallel_size=1)
24 prompts = ["This is a short prompt", "This is a very long prompt " * 1000]
@@ -35,8 +31,7 @@ def test_max_tokens_none():
31 sampling_params = SamplingParams(temperature=0.01,
32 top_p=0.1,
33 max_tokens=None)
38 - llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2",
39 - load_format=LoadFormat.RUNAI_STREAMER,
34 + llm = LLM(model="distilbert/distilgpt2",
35 max_num_batched_tokens=4096,
36 tensor_parallel_size=1)
37 prompts = ["Just say hello!"]
@@ -46,9 +41,7 @@ def test_max_tokens_none():
41
42
43 def test_gc():
49 - llm = LLM(model=f"{MODEL_WEIGHTS_S3_BUCKET}/distilbert/distilgpt2",
50 - load_format=LoadFormat.RUNAI_STREAMER,
51 - enforce_eager=True)
44 + llm = LLM(model="distilbert/distilgpt2", enforce_eager=True)
45 del llm
46
47 gc.collect()
tests/worker/test_swap.py
+1 -1
@@ -10,7 +10,7 @@ from vllm.worker.worker import Worker
10
11 def test_swap() -> None:
12 # Configure the engine.
13 - engine_args = EngineArgs(model="s3://vllm-ci-model-weights/distilgpt2",
13 + engine_args = EngineArgs(model="distilbert/distilgpt2",
14 dtype="half",
15 load_format="dummy")
16 engine_config = engine_args.create_engine_config()
vllm/engine/arg_utils.py
+9
@@ -22,6 +22,7 @@ from vllm.executor.executor_base import ExecutorBase
22 from vllm.logger import init_logger
23 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
24 from vllm.plugins import load_general_plugins
25 +from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3
26 from vllm.transformers_utils.utils import check_gguf_file
27 from vllm.usage.usage_lib import UsageContext
28 from vllm.utils import FlexibleArgumentParser, StoreBoolean
@@ -1141,6 +1142,14 @@ class EngineArgs:
1142 f", but got {self.cpu_offload_gb}")
1143
1144 device_config = DeviceConfig(device=self.device)
1145 +
1146 + # NOTE: This is to allow model loading from S3 in CI
1147 + if (not isinstance(self, AsyncEngineArgs) and envs.VLLM_CI_USE_S3
1148 + and self.model in MODELS_ON_S3
1149 + and self.load_format == LoadFormat.AUTO): # noqa: E501
1150 + self.model = f"{MODEL_WEIGHTS_S3_BUCKET}/{self.model}"
1151 + self.load_format = LoadFormat.RUNAI_STREAMER
1152 +
1153 model_config = self.create_model_config()
1154
1155 if (model_config.is_multimodal_model and not envs.VLLM_USE_V1
vllm/envs.py
+4
@@ -618,6 +618,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
618 # Port of the master node in the data parallel setting
619 "VLLM_DP_MASTER_PORT":
620 lambda: int(os.getenv("VLLM_DP_MASTER_PORT", "0")),
621 +
622 + # Whether to use S3 path for model loading in CI via RunAI Streamer
623 + "VLLM_CI_USE_S3":
624 + lambda: os.environ.get("VLLM_CI_USE_S3", "0") == "1",
625 }
626
627 # end-env-vars-definition
vllm/model_executor/model_loader/loader.py
-1
@@ -1394,7 +1394,6 @@ class RunaiModelStreamerLoader(BaseModelLoader):
1394
1395 def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
1396 """Get a model loader based on the load format."""
1397 -
1397 if isinstance(load_config.load_format, type):
1398 return load_config.load_format(load_config)
1399
vllm/test_utils.py new
+129
@@ -0,0 +1,129 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +MODELS_ON_S3 = [
3 + "adept/fuyu-8b",
4 + "ai21labs/AI21-Jamba-1.5-Mini",
5 + "ai21labs/Jamba-tiny-random",
6 + "ai21labs/Jamba-tiny-reward-dev",
7 + "allenai/Molmo-7B-D-0924",
8 + "allenai/OLMo-1B-hf",
9 + "allenai/OLMoE-1B-7B-0924-Instruct",
10 + "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test",
11 + "AMead10/Llama-3.2-1B-Instruct-AWQ",
12 + "ArthurZ/Ilama-3.2-1B",
13 + "BAAI/bge-base-en-v1.5",
14 + "BAAI/bge-multilingual-gemma2",
15 + "BAAI/bge-reranker-v2-m3",
16 + "bigcode/starcoder2-3b",
17 + "cross-encoder/ms-marco-MiniLM-L-6-v2",
18 + "cross-encoder/quora-roberta-base",
19 + "deepseek-ai/deepseek-vl2-tiny",
20 + "distilbert/distilgpt2",
21 + "facebook/bart-base",
22 + "facebook/bart-large-cnn",
23 + # "fixie-ai/ultravox-v0_5-llama-3_2-1b",
24 + "google/gemma-1.1-2b-it",
25 + "google/gemma-2-2b-it",
26 + "google/paligemma-3b-pt-224",
27 + "h2oai/h2ovl-mississippi-800m",
28 + "HuggingFaceM4/Idefics3-8B-Llama3",
29 + "internlm/internlm2-1_8b-reward",
30 + "intfloat/e5-mistral-7b-instruct",
31 + "intfloat/multilingual-e5-large",
32 + "jason9693/Qwen2.5-1.5B-apeach",
33 + "llava-hf/llava-1.5-7b-hf",
34 + "llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
35 + "llava-hf/llava-v1.6-mistral-7b-hf",
36 + "llava-hf/LLaVA-NeXT-Video-7B-hf",
37 + # "meta-llama/Llama-2-7b-hf",
38 + "meta-llama/Llama-3.2-11B-Vision-Instruct",
39 + "meta-llama/Llama-3.2-1B",
40 + "meta-llama/Llama-3.2-1B-Instruct",
41 + "meta-llama/Meta-Llama-3-8B",
42 + "microsoft/phi-2",
43 + "microsoft/Phi-3-mini-4k-instruct",
44 + "microsoft/Phi-3-small-8k-instruct",
45 + "microsoft/Phi-3-vision-128k-instruct",
46 + "microsoft/Phi-3.5-MoE-instruct",
47 + "microsoft/Phi-3.5-vision-instruct",
48 + # "mistralai/Mistral-7B-Instruct-v0.1",
49 + "mistralai/Mixtral-8x7B-Instruct-v0.1",
50 + "mistralai/Pixtral-12B-2409",
51 + "mistral-community/Mixtral-8x22B-v0.1-AWQ",
52 + "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head",
53 + "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse",
54 + "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
55 + "ModelCloud/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit-10-25-2024",
56 + "neuralmagic/Llama-3.2-1B-quantized.w8a8",
57 + "neuralmagic/Meta-Llama-3-8B-Instruct-FP8",
58 + "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
59 + "nm-testing/asym-w8w8-int8-static-per-tensor-tiny-llama",
60 + "nm-testing/llama2.c-stories42M-pruned2.4-compressed",
61 + "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
62 + "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test",
63 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-2of4-testing",
64 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-IA-Per-Tensor-Weight-testing",
65 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-Per-Tensor-testing",
66 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-testing",
67 + "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Asym",
68 + "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym",
69 + "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym",
70 + "nm-testing/Phi-3-mini-128k-instruct-FP8",
71 + "nm-testing/Qwen2-0.5B-Instruct-FP8-SkipQKV",
72 + "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V",
73 + "nm-testing/tinyllama-oneshot-w4a16-channel-v2",
74 + "nm-testing/tinyllama-oneshot-w4a16-group128-v2",
75 + "nm-testing/tinyllama-oneshot-w8-channel-a8-tensor",
76 + "nm-testing/tinyllama-oneshot-w8a16-per-channel",
77 + "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2",
78 + "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2-asym",
79 + "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2",
80 + "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2-asym",
81 + "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change",
82 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor",
83 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM",
84 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_int8-BitM",
85 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_tensor_act_fp8-BitM",
86 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_tensor_act_int8-BitM",
87 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_per_tok_dyn_act_fp8-BitM",
88 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_per_tok_dyn_act_int8-BitM",
89 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_tensor_act_fp8-BitM",
90 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_tensor_act_int8-BitM",
91 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing",
92 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Tensor-Weight-testing",
93 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Static-testing",
94 + "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme",
95 + "nvidia/NVLM-D-72B",
96 + "openai-community/gpt2",
97 + # "openai/whisper-large-v3",
98 + "openbmb/MiniCPM-o-2_6",
99 + "openbmb/MiniCPM-V-2_6",
100 + "OpenGVLab/InternVL2-1B",
101 + "parasail-ai/GritLM-7B-vllm",
102 + "Qwen/Qwen1.5-MoE-A2.7B-Chat",
103 + "Qwen/Qwen2-7B-Instruct",
104 + "Qwen/Qwen2-Audio-7B-Instruct",
105 + "Qwen/Qwen2-VL-2B-Instruct",
106 + "Qwen/Qwen2.5-1.5B-Instruct",
107 + "Qwen/Qwen2.5-Math-PRM-7B",
108 + "Qwen/Qwen2.5-Math-RM-72B",
109 + "Qwen/Qwen2.5-VL-3B-Instruct",
110 + "royokong/e5-v",
111 + "sentence-transformers/all-roberta-large-v1",
112 + "sentence-transformers/stsb-roberta-base-v2",
113 + "shanearora/OLMo-7B-1124-hf",
114 + "shuyuej/Llama-3.2-1B-Instruct-GPTQ",
115 + "ssmits/Qwen2-7B-Instruct-embed-base",
116 + "stabilityai/stablelm-3b-4e1t",
117 + "stabilityai/stablelm-zephyr-3b",
118 + "state-spaces/mamba-130m-hf",
119 + "TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",
120 + "THUDM/glm-4v-9b",
121 + "TIGER-Lab/Mantis-8B-siglip-llama3",
122 + "TIGER-Lab/VLM2Vec-Full",
123 + "tiiuae/falcon-40b",
124 + "tiiuae/falcon-mamba-7b-instruct",
125 + "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
126 + "upstage/solar-pro-preview-instruct",
127 +]
128 +
129 +MODEL_WEIGHTS_S3_BUCKET = "s3://vllm-ci-model-weights"