A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI] Replace large models with tiny alternatives in tests (#24057)
Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Nick Hill <nhill@redhat.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Tahsin Tunan committed
Oct 16, 2025 at 20:51 UTC
43721bc67f9f30dcd1c893af370727b835d3d6dd
17 files changed
+119
-60
tests/basic_correctness/test_basic_correctness.py
+8
-8
@@ -20,7 +20,7 @@ from ..models.utils import check_outputs_equal
20
from ..utils import multi_gpu_test
21
22
MODELS = [
23
- "google/gemma-2-2b-it",
23
+ "hmellor/tiny-random-Gemma2ForCausalLM",
24
"meta-llama/Llama-3.2-1B-Instruct",
25
]
26
@@ -29,7 +29,7 @@ TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
29
30
def test_vllm_gc_ed():
31
"""Verify vllm instance is GC'ed when it is deleted"""
32
- llm = LLM("distilbert/distilgpt2")
32
+ llm = LLM("hmellor/tiny-random-LlamaForCausalLM")
33
weak_llm = weakref.ref(llm)
34
del llm
35
# If there's any circular reference to vllm, this fails
@@ -125,14 +125,14 @@ def test_models(
125
@pytest.mark.parametrize(
126
"model, distributed_executor_backend, attention_backend, test_suite, extra_env",
127
[
128
- ("distilbert/distilgpt2", "ray", "", "L4", {}),
129
- ("distilbert/distilgpt2", "mp", "", "L4", {}),
130
- ("distilbert/distilgpt2", "ray", "", "L4", {"VLLM_SLEEP_WHEN_IDLE": "1"}),
131
- ("distilbert/distilgpt2", "mp", "", "L4", {"VLLM_SLEEP_WHEN_IDLE": "1"}),
128
+ ("facebook/opt-125m", "ray", "", "L4", {}),
129
+ ("facebook/opt-125m", "mp", "", "L4", {}),
130
+ ("facebook/opt-125m", "ray", "", "L4", {"VLLM_SLEEP_WHEN_IDLE": "1"}),
131
+ ("facebook/opt-125m", "mp", "", "L4", {"VLLM_SLEEP_WHEN_IDLE": "1"}),
132
("meta-llama/Llama-3.2-1B-Instruct", "ray", "", "L4", {}),
133
("meta-llama/Llama-3.2-1B-Instruct", "mp", "", "L4", {}),
134
- ("distilbert/distilgpt2", "ray", "", "A100", {}),
135
- ("distilbert/distilgpt2", "mp", "", "A100", {}),
134
+ ("facebook/opt-125m", "ray", "", "A100", {}),
135
+ ("facebook/opt-125m", "mp", "", "A100", {}),
136
],
137
)
138
@pytest.mark.parametrize("enable_prompt_embeds", [True, False])
tests/basic_correctness/test_cpu_offload.py
+1
-1
@@ -6,5 +6,5 @@ from ..utils import compare_two_settings
6
7
def test_cpu_offload():
8
compare_two_settings(
9
- "meta-llama/Llama-3.2-1B-Instruct", [], ["--cpu-offload-gb", "1"]
9
+ "hmellor/tiny-random-LlamaForCausalLM", [], ["--cpu-offload-gb", "1"]
10
)
tests/basic_correctness/test_cumem.py
+2
-2
@@ -120,7 +120,7 @@ def test_cumem_with_cudagraph():
120
"model",
121
[
122
# sleep mode with safetensors
123
- "meta-llama/Llama-3.2-1B",
123
+ "hmellor/tiny-random-LlamaForCausalLM",
124
# sleep mode with pytorch checkpoint
125
"facebook/opt-125m",
126
],
@@ -174,7 +174,7 @@ def test_end_to_end(model: str):
174
175
@create_new_process_for_each_test()
176
def test_deep_sleep():
177
- model = "Qwen/Qwen3-0.6B"
177
+ model = "hmellor/tiny-random-LlamaForCausalLM"
178
free, total = torch.cuda.mem_get_info()
179
used_bytes_baseline = total - free # in case other process is running
180
llm = LLM(model, enable_sleep_mode=True)
tests/distributed/test_sequence_parallel.py
+2
-2
@@ -273,14 +273,14 @@ def _compare_sp(
273
274
SP_TEXT_GENERATION_MODELS = {
275
# [Decoder-only]
276
- "meta-llama/Llama-3.2-1B-Instruct": SPTestSettings.fast(),
276
+ "hmellor/tiny-random-LlamaForCausalLM": SPTestSettings.fast(),
277
"RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8": SPTestSettings.fp8_quant(),
278
}
279
280
SP_TEST_MODELS = [
281
# TODO support other models
282
# [LANGUAGE GENERATION]
283
- "meta-llama/Llama-3.2-1B-Instruct",
283
+ "hmellor/tiny-random-LlamaForCausalLM",
284
"RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
285
]
286
tests/entrypoints/llm/test_collective_rpc.py
+4
-1
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import pytest
5
+import torch
6
7
from vllm import LLM
8
@@ -12,6 +13,8 @@ from ...utils import create_new_process_for_each_test
13
@pytest.mark.parametrize("backend", ["mp", "ray"])
14
@create_new_process_for_each_test()
15
def test_collective_rpc(tp_size, backend, monkeypatch):
16
+ if torch.cuda.device_count() < tp_size:
17
+ pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
18
if tp_size == 1 and backend == "ray":
19
pytest.skip("Skip duplicate test case")
20
if tp_size == 1:
@@ -24,7 +27,7 @@ def test_collective_rpc(tp_size, backend, monkeypatch):
27
28
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
29
llm = LLM(
27
- model="meta-llama/Llama-3.2-1B-Instruct",
30
+ model="hmellor/tiny-random-LlamaForCausalLM",
31
enforce_eager=True,
32
load_format="dummy",
33
tensor_parallel_size=tp_size,
tests/entrypoints/openai/test_run_batch.py
+1
-1
@@ -9,7 +9,7 @@ import pytest
9
10
from vllm.entrypoints.openai.protocol import BatchRequestOutput
11
12
-MODEL_NAME = "Qwen/Qwen3-0.6B"
12
+MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
13
14
# ruff: noqa: E501
15
INPUT_BATCH = (
tests/entrypoints/openai/test_serving_models.py
+1
-1
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.protocol import (
16
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
17
from vllm.lora.request import LoRARequest
18
19
-MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
19
+MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
20
BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
21
LORA_LOADING_SUCCESS_MESSAGE = "Success: LoRA adapter '{lora_name}' added successfully."
22
LORA_UNLOADING_SUCCESS_MESSAGE = (
tests/entrypoints/openai/test_shutdown.py
+80
-24
@@ -1,37 +1,93 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import signal
5
+import subprocess
6
+import sys
7
+import time
8
+
9
import openai
10
import pytest
11
7
-from ...utils import RemoteOpenAIServer
12
+from ...utils import get_open_port
13
9
-MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
14
+MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
15
16
17
@pytest.mark.asyncio
18
async def test_shutdown_on_engine_failure():
14
- # dtype, max-len etc set so that this can run in CI
15
- args = [
16
- "--dtype",
17
- "bfloat16",
18
- "--max-model-len",
19
- "8192",
20
- "--enforce-eager",
21
- "--max-num-seqs",
22
- "128",
23
- ]
24
-
25
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
26
- async with remote_server.get_async_client() as client:
27
- with pytest.raises((openai.APIConnectionError, openai.InternalServerError)):
28
- # Asking for lots of prompt logprobs will currently crash the
29
- # engine. This may change in the future when that bug is fixed
30
- prompt = "Hello " * 4000
31
- await client.completions.create(
32
- model=MODEL_NAME, prompt=prompt, extra_body={"prompt_logprobs": 10}
19
+ """Verify that API returns connection error when server process is killed.
20
+
21
+ Starts a vLLM server, kills it to simulate a crash, then verifies that
22
+ subsequent API calls fail appropriately.
23
+ """
24
+
25
+ port = get_open_port()
26
+
27
+ proc = subprocess.Popen(
28
+ [
29
+ # dtype, max-len etc set so that this can run in CI
30
+ sys.executable,
31
+ "-m",
32
+ "vllm.entrypoints.openai.api_server",
33
+ "--model",
34
+ MODEL_NAME,
35
+ "--dtype",
36
+ "bfloat16",
37
+ "--max-model-len",
38
+ "128",
39
+ "--enforce-eager",
40
+ "--port",
41
+ str(port),
42
+ "--gpu-memory-utilization",
43
+ "0.05",
44
+ "--max-num-seqs",
45
+ "2",
46
+ "--disable-frontend-multiprocessing",
47
+ ],
48
+ stdout=subprocess.PIPE,
49
+ stderr=subprocess.PIPE,
50
+ text=True,
51
+ preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN),
52
+ )
53
+
54
+ # Wait for server startup
55
+ start_time = time.time()
56
+ client = openai.AsyncOpenAI(
57
+ base_url=f"http://localhost:{port}/v1",
58
+ api_key="dummy",
59
+ max_retries=0,
60
+ timeout=10,
61
+ )
62
+
63
+ # Poll until server is ready
64
+ while time.time() - start_time < 30:
65
+ try:
66
+ await client.completions.create(
67
+ model=MODEL_NAME, prompt="Hello", max_tokens=1
68
+ )
69
+ break
70
+ except Exception:
71
+ time.sleep(0.5)
72
+ if proc.poll() is not None:
73
+ stdout, stderr = proc.communicate(timeout=1)
74
+ pytest.fail(
75
+ f"Server died during startup. stdout: {stdout}, stderr: {stderr}"
76
)
77
+ else:
78
+ proc.terminate()
79
+ proc.wait(timeout=5)
80
+ pytest.fail("Server failed to start in 30 seconds")
81
+
82
+ # Kill server to simulate crash
83
+ proc.terminate()
84
+ time.sleep(1)
85
+
86
+ # Verify API calls now fail
87
+ with pytest.raises((openai.APIConnectionError, openai.APIStatusError)):
88
+ await client.completions.create(
89
+ model=MODEL_NAME, prompt="This should fail", max_tokens=1
90
+ )
91
35
- # Now the server should shut down
36
- return_code = remote_server.proc.wait(timeout=8)
37
- assert return_code is not None
92
+ return_code = proc.wait(timeout=5)
93
+ assert return_code is not None
tests/models/registry.py
+1
@@ -330,6 +330,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
330
"guard": "meta-llama/Llama-Guard-3-1B",
331
"hermes": "NousResearch/Hermes-3-Llama-3.1-8B",
332
"fp8": "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
333
+ "tiny": "hmellor/tiny-random-LlamaForCausalLM",
334
},
335
),
336
"LLaMAForCausalLM": _HfExamplesInfo(
tests/samplers/test_no_bad_words.py
+4
-6
@@ -35,15 +35,13 @@ def _generate(
35
36
37
class TestOneTokenBadWord:
38
- MODEL = "TheBloke/Llama-2-7B-fp16"
38
+ MODEL = "hmellor/tiny-random-LlamaForCausalLM"
39
40
- PROMPT = "Hi! How are"
41
- TARGET_TOKEN = "you"
40
+ PROMPT = "How old are "
41
+ TARGET_TOKEN = "mn"
42
43
def setup_method(self, method):
44
- self.tokenizer = AutoTokenizer.from_pretrained(
45
- self.MODEL, add_prefix_space=True
46
- )
44
+ self.tokenizer = AutoTokenizer.from_pretrained(self.MODEL)
45
46
self.num_prompt_tokens = len(self._encode(self.PROMPT))
47
self.target_token_id = self._encode(
tests/v1/core/test_scheduler_e2e.py
+1
-1
@@ -5,7 +5,7 @@ import pytest
5
6
from vllm import LLM
7
8
-MODEL = "meta-llama/Llama-3.2-1B"
8
+MODEL = "hmellor/tiny-random-LlamaForCausalLM"
9
PROMPT = "Hello my name is Robert and I"
10
11
tests/v1/engine/test_engine_core.py
+4
-2
@@ -24,9 +24,11 @@ from ...utils import create_new_process_for_each_test, multi_gpu_test
24
if not current_platform.is_cuda():
25
pytest.skip(reason="V1 currently only supported on CUDA.", allow_module_level=True)
26
27
-MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
27
+MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
28
TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME)
29
-PROMPT = "Hello my name is Robert and I love quantization kernels"
29
+# test_engine_core_concurrent_batches assumes exactly 12 tokens per prompt.
30
+# Adjust prompt if changing model to maintain 12-token length.
31
+PROMPT = "I am Gyoubu Masataka Oniwa"
32
PROMPT_TOKENS = TOKENIZER(PROMPT).input_ids
33
34
tests/v1/entrypoints/openai/test_multi_api_servers.py
+1
-1
@@ -10,7 +10,7 @@ import pytest_asyncio
10
from tests.utils import RemoteOpenAIServer
11
from tests.v1.utils import check_request_balancing
12
13
-MODEL_NAME = "ibm-research/PowerMoE-3b"
13
+MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
14
15
DP_SIZE = os.getenv("DP_SIZE", "1")
16
tests/v1/sample/test_sampling_params_e2e.py
+2
-5
@@ -5,16 +5,13 @@ import pytest
5
6
from vllm import LLM, SamplingParams
7
8
-MODEL = "meta-llama/Llama-3.2-1B"
8
+MODEL = "hmellor/tiny-random-LlamaForCausalLM"
9
PROMPT = "Hello my name is Robert and I"
10
11
12
@pytest.fixture(scope="module")
13
def llm() -> LLM:
14
- # Disable prefix caching so that we can test prompt logprobs.
15
- # TODO remove this after https://github.com/vllm-project/vllm/pull/13949
16
- # is merged
17
- return LLM(MODEL, enforce_eager=True, enable_prefix_caching=False)
14
+ return LLM(MODEL, enforce_eager=True)
15
16
17
def test_n_gt_1(llm):
tests/v1/shutdown/test_delete.py
+1
-1
@@ -15,7 +15,7 @@ from vllm.sampling_params import RequestOutputKind
15
from vllm.utils import cuda_device_count_stateless
16
from vllm.v1.engine.async_llm import AsyncLLM
17
18
-MODELS = ["meta-llama/Llama-3.2-1B"]
18
+MODELS = ["hmellor/tiny-random-LlamaForCausalLM"]
19
20
21
@pytest.mark.asyncio
tests/v1/shutdown/test_forward_error.py
+1
-1
@@ -18,7 +18,7 @@ from vllm.utils import cuda_device_count_stateless
18
from vllm.v1.engine.async_llm import AsyncLLM
19
from vllm.v1.engine.exceptions import EngineDeadError
20
21
-MODELS = ["meta-llama/Llama-3.2-1B"]
21
+MODELS = ["hmellor/tiny-random-LlamaForCausalLM"]
22
23
24
def evil_forward(self, *args, **kwargs):
tests/v1/shutdown/test_startup_error.py
+5
-3
@@ -16,7 +16,7 @@ from vllm.model_executor.models.llama import LlamaForCausalLM
16
from vllm.utils import cuda_device_count_stateless
17
from vllm.v1.engine.async_llm import AsyncLLM
18
19
-MODELS = ["meta-llama/Llama-3.2-1B"]
19
+MODELS = ["hmellor/tiny-random-LlamaForCausalLM"]
20
21
22
def evil_method(self, *args, **kwargs):
@@ -76,8 +76,10 @@ def test_llm_startup_error(
76
Test profiling (forward()) and load weights failures.
77
TODO(andy) - LLM without multiprocessing.
78
"""
79
- if model != "meta-llama/Llama-3.2-1B":
80
- pytest.skip(reason="Only test meta-llama/Llama-3.2-1B")
79
+ # Skip non-Llama models since we monkeypatch LlamaForCausalLM specifically.
80
+ # If MODELS list grows, each architecture needs its own test variant.
81
+ if model != "JackFram/llama-68m":
82
+ pytest.skip(reason="Only test JackFram/llama-68m")
83
if cuda_device_count_stateless() < tensor_parallel_size:
84
pytest.skip(reason="Not enough CUDA devices")
85