A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
Consolidate Llama model usage in tests (#13094)
Harry Mellor committed
Feb 14, 2025 at 06:18 UTC
f2b20fe491073dad7e3c3fe8ab0303b97fb50643
22 files changed
+44
-52
tests/basic_correctness/test_basic_correctness.py
+5
-5
@@ -17,7 +17,7 @@ from ..utils import multi_gpu_test
17
18
MODELS = [
19
"google/gemma-2-2b-it",
20
- "meta-llama/Llama-3.2-1B",
20
+ "meta-llama/Llama-3.2-1B-Instruct",
21
]
22
23
TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
@@ -96,12 +96,12 @@ def test_models(
96
"test_suite", [
97
("facebook/opt-125m", "ray", "", "L4"),
98
("facebook/opt-125m", "mp", "", "L4"),
99
- ("meta-llama/Llama-2-7b-hf", "ray", "", "L4"),
100
- ("meta-llama/Llama-2-7b-hf", "mp", "", "L4"),
99
+ ("meta-llama/Llama-3.2-1B-Instruct", "ray", "", "L4"),
100
+ ("meta-llama/Llama-3.2-1B-Instruct", "mp", "", "L4"),
101
("facebook/opt-125m", "ray", "", "A100"),
102
("facebook/opt-125m", "mp", "", "A100"),
103
("facebook/opt-125m", "mp", "FLASHINFER", "A100"),
104
- ("meta-llama/Meta-Llama-3-8B", "ray", "FLASHINFER", "A100"),
104
+ ("meta-llama/Llama-3.2-1B-Instruct", "ray", "FLASHINFER", "A100"),
105
])
106
def test_models_distributed(
107
hf_runner,
@@ -116,7 +116,7 @@ def test_models_distributed(
116
if test_suite != TARGET_TEST_SUITE:
117
pytest.skip(f"Skip test for {test_suite}")
118
119
- if model == "meta-llama/Llama-2-7b-hf" and distributed_executor_backend == "ray" and attention_backend == "" and test_suite == "L4": # noqa
119
+ if model == "meta-llama/Llama-3.2-1B-Instruct" and distributed_executor_backend == "ray" and attention_backend == "" and test_suite == "L4": # noqa
120
# test ray adag
121
os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
122
os.environ['VLLM_USE_RAY_COMPILED_DAG'] = "1"
tests/basic_correctness/test_chunked_prefill.py
+3
-3
@@ -20,7 +20,7 @@ from ..utils import multi_gpu_test
20
21
MODELS = [
22
"facebook/opt-125m",
23
- "meta-llama/Llama-3.2-1B",
23
+ "meta-llama/Llama-3.2-1B-Instruct",
24
]
25
26
@@ -92,7 +92,7 @@ def test_models_distributed(
92
) -> None:
93
override_backend_env_variable(monkeypatch, attention_backend)
94
95
- if (model == "meta-llama/Llama-2-7b-hf"
95
+ if (model == "meta-llama/Llama-3.2-1B-Instruct"
96
and distributed_executor_backend == "ray"):
97
# test ray adag
98
os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
@@ -221,7 +221,7 @@ def test_with_prefix_caching(
221
Checks exact match decode with and without prefix caching
222
with chunked prefill enabled.
223
"""
224
- model = "meta-llama/Llama-2-7b-chat-hf"
224
+ model = "meta-llama/Llama-3.2-1B-Instruct"
225
# The common prompt has 142 tokens with Llama-2 tokenizer.
226
common_prompt = "You are a helpful AI assistant " * 20
227
unique_prompts = [
tests/basic_correctness/test_cpu_offload.py
+1
-1
@@ -4,5 +4,5 @@ from ..utils import compare_two_settings
4
5
6
def test_cpu_offload():
7
- compare_two_settings("meta-llama/Llama-3.2-1B", [],
7
+ compare_two_settings("meta-llama/Llama-3.2-1B-Instruct", [],
8
["--cpu-offload-gb", "1"])
tests/basic_correctness/test_cumem.py
+1
-1
@@ -118,7 +118,7 @@ def test_cumem_with_cudagraph():
118
@pytest.mark.parametrize(
119
"model",
120
[
121
- "meta-llama/Llama-3.2-1B", # sleep mode with safetensors
121
+ "meta-llama/Llama-3.2-1B-Instruct", # sleep mode with safetensors
122
"facebook/opt-125m" # sleep mode with pytorch checkpoint
123
])
124
def test_end_to_end(model):
tests/compile/test_basic_correctness.py
+1
-1
@@ -26,7 +26,7 @@ class TestSetting:
26
test_settings = [
27
# basic llama model
28
TestSetting(
29
- model="meta-llama/Llama-3.2-1B",
29
+ model="meta-llama/Llama-3.2-1B-Instruct",
30
model_args=[],
31
pp_size=2,
32
tp_size=2,
tests/compile/utils.py
+4
-10
@@ -6,7 +6,6 @@ import torch
6
7
from tests.quantization.utils import is_quant_method_supported
8
from vllm import LLM, SamplingParams
9
-from vllm.config import CompilationLevel
9
from vllm.platforms import current_platform
10
11
TEST_MODELS = [
@@ -15,14 +14,14 @@ TEST_MODELS = [
14
"dtype": torch.float16,
15
"quantization": "compressed-tensors"
16
}),
18
- ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", {
17
+ ("neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic", {
18
"dtype": torch.float16,
20
- "quantization": "fp8"
19
+ "quantization": "compressed-tensors"
20
}),
22
- ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
21
+ ("neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8", {
22
"quantization": "compressed-tensors"
23
}),
25
- ("meta-llama/Meta-Llama-3-8B", {}),
24
+ ("meta-llama/Llama-3.2-1B-Instruct", {}),
25
]
26
27
if is_quant_method_supported("aqlm"):
@@ -69,11 +68,6 @@ def check_full_graph_support(model,
68
# make sure these models can be captured in full graph mode
69
os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
70
72
- # The base meta llama uses too much memory.
73
- if (model == "meta-llama/Meta-Llama-3-8B"
74
- and optimization_level >= CompilationLevel.PIECEWISE):
75
- return
76
-
71
print(f"MODEL={model}")
72
73
prompts = [
tests/distributed/test_pipeline_parallel.py
+2
-2
@@ -162,7 +162,7 @@ TEXT_GENERATION_MODELS = {
162
"internlm/internlm2-chat-7b": PPTestSettings.fast(),
163
"inceptionai/jais-13b-chat": PPTestSettings.fast(),
164
"ai21labs/Jamba-tiny-dev": PPTestSettings.fast(),
165
- "meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
165
+ "meta-llama/Llama-3.2-1B-Instruct": PPTestSettings.detailed(),
166
"openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(),
167
"openbmb/MiniCPM3-4B": PPTestSettings.fast(),
168
# Uses Llama
@@ -230,7 +230,7 @@ MULTIMODAL_MODELS = {
230
TEST_MODELS = [
231
# [LANGUAGE GENERATION]
232
"microsoft/Phi-3.5-MoE-instruct",
233
- "meta-llama/Meta-Llama-3-8B",
233
+ "meta-llama/Llama-3.2-1B-Instruct",
234
"ibm/PowerLM-3b",
235
# [LANGUAGE EMBEDDING]
236
"intfloat/e5-mistral-7b-instruct",
tests/entrypoints/openai/test_serving_models.py
+1
-1
@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.serving_models import (BaseModelPath,
14
OpenAIServingModels)
15
from vllm.lora.request import LoRARequest
16
17
-MODEL_NAME = "meta-llama/Llama-2-7b"
17
+MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
18
BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
19
LORA_LOADING_SUCCESS_MESSAGE = (
20
"Success: LoRA adapter '{lora_name}' added successfully.")
tests/entrypoints/openai/test_shutdown.py
+1
-1
@@ -5,7 +5,7 @@ import pytest
5
6
from ...utils import RemoteOpenAIServer
7
8
-MODEL_NAME = "meta-llama/Llama-3.2-1B"
8
+MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
9
10
11
@pytest.mark.asyncio
tests/kv_transfer/disagg_test.py
+4
-6
@@ -28,7 +28,7 @@ def setup_servers():
28
"-m",
29
"vllm.entrypoints.openai.api_server",
30
"--model",
31
- "meta-llama/Meta-Llama-3.1-8B-Instruct",
31
+ "meta-llama/Llama-3.2-1B-Instruct",
32
"--port",
33
"8100",
34
"--gpu-memory-utilization",
@@ -49,7 +49,7 @@ def setup_servers():
49
"-m",
50
"vllm.entrypoints.openai.api_server",
51
"--model",
52
- "meta-llama/Meta-Llama-3.1-8B-Instruct",
52
+ "meta-llama/Llama-3.2-1B-Instruct",
53
"--port",
54
"8200",
55
"--gpu-memory-utilization",
@@ -100,8 +100,7 @@ def test_disaggregated_prefilling(prompt):
100
response = requests.post("http://localhost:8100/v1/completions",
101
headers={"Content-Type": "application/json"},
102
json={
103
- "model":
104
- "meta-llama/Meta-Llama-3.1-8B-Instruct",
103
+ "model": "meta-llama/Llama-3.2-1B-Instruct",
104
"prompt": prompt,
105
"max_tokens": 1,
106
"temperature": 0
@@ -112,8 +111,7 @@ def test_disaggregated_prefilling(prompt):
111
response = requests.post("http://localhost:8200/v1/completions",
112
headers={"Content-Type": "application/json"},
113
json={
115
- "model":
116
- "meta-llama/Meta-Llama-3.1-8B-Instruct",
114
+ "model": "meta-llama/Llama-3.2-1B-Instruct",
115
"prompt": prompt,
116
"max_tokens": 10,
117
"temperature": 0
tests/models/decoder_only/language/test_fp8.py
+4
-4
@@ -26,12 +26,12 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
26
# Test FP8 checkpoint w. fp8_e4m3 kv-cache scaling factors.
27
("fp8_e4m3", "meta-llama/Llama-3.2-1B-Instruct",
28
"nm-testing/Llama-3.2-1B-Instruct-FP8-KV"),
29
- # Test FP16 checkpoint w. fp8_e5m2 kv-cache.
29
+ # Test BF16 checkpoint w. fp8_e5m2 kv-cache.
30
("fp8_e5m2", "meta-llama/Llama-3.2-1B-Instruct",
31
"meta-llama/Llama-3.2-1B-Instruct"),
32
- # Test FP16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
33
- ("fp8_e4m3", "meta-llama/Llama-2-7b-chat-hf",
34
- "meta-llama/Llama-2-7b-chat-hf")
32
+ # Test BF16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
33
+ ("fp8_e4m3", "meta-llama/Llama-3.2-1B-Instruct",
34
+ "meta-llama/Llama-3.2-1B-Instruct")
35
])
36
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
37
@pytest.mark.parametrize("max_tokens", [4])
tests/models/registry.py
+1
-1
@@ -141,7 +141,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
141
"JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
142
"JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini",
143
extras={"tiny": "ai21labs/Jamba-tiny-dev"}), # noqa: E501
144
- "LlamaForCausalLM": _HfExamplesInfo("meta-llama/Meta-Llama-3-8B"),
144
+ "LlamaForCausalLM": _HfExamplesInfo("meta-llama/Llama-3.2-1B-Instruct"),
145
"LLaMAForCausalLM": _HfExamplesInfo("decapoda-research/llama-7b-hf",
146
is_available_online=False),
147
"MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
tests/quantization/test_register_quantization_config.py
+1
-1
@@ -99,7 +99,7 @@ def test_register_quantization_config():
99
100
@pytest.mark.parametrize(argnames="model",
101
argvalues=[
102
- "meta-llama/Meta-Llama-3-8B-Instruct",
102
+ "meta-llama/Llama-3.2-1B-Instruct",
103
])
104
def test_custom_quant(vllm_runner, model):
105
"""Test infer with the custom quantization method."""
tests/samplers/test_ignore_eos.py
+1
-1
@@ -10,7 +10,7 @@ from vllm import SamplingParams
10
11
# We also test with llama because it has generation_config to specify EOS
12
# (past regression).
13
-MODELS = ["facebook/opt-125m", "meta-llama/Llama-2-7b-hf"]
13
+MODELS = ["facebook/opt-125m", "meta-llama/Llama-3.2-1B-Instruct"]
14
15
16
@pytest.mark.parametrize("model", MODELS)
tests/spec_decode/e2e/test_compatibility.py
+3
-3
@@ -8,7 +8,7 @@ from .conftest import get_output_from_llm_generator
8
9
10
@pytest.mark.parametrize("common_llm_kwargs", [{
11
- "model": "meta-llama/Llama-2-7b-chat-hf",
11
+ "model": "meta-llama/Llama-3.2-1B-Instruct",
12
"speculative_model": "JackFram/llama-68m",
13
"num_speculative_tokens": 5,
14
}])
@@ -27,8 +27,8 @@ from .conftest import get_output_from_llm_generator
27
},
28
{
29
# Speculative max model len > target max model len should raise.
30
- # https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/blob/f5db02db724555f92da89c216ac04704f23d4590/config.json#L12
31
- "speculative_max_model_len": 4096 + 1,
30
+ # https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/blob/9213176726f574b556790deb65791e0c5aa438b6/config.json#L18
31
+ "speculative_max_model_len": 131072 + 1,
32
},
33
])
34
@pytest.mark.parametrize("test_llm_kwargs", [{}])
tests/test_config.py
+1
-1
@@ -251,7 +251,7 @@ def test_rope_customization():
251
@pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
252
("facebook/opt-125m", False),
253
("facebook/bart-base", True),
254
- ("meta-llama/Llama-3.2-1B", False),
254
+ ("meta-llama/Llama-3.2-1B-Instruct", False),
255
("meta-llama/Llama-3.2-11B-Vision", True),
256
])
257
def test_is_encoder_decoder(model_id, is_encoder_decoder):
tests/test_sharded_state_loader.py
+4
-4
@@ -46,9 +46,9 @@ def test_filter_subtensors():
46
47
48
@pytest.fixture(scope="module")
49
-def llama_2_7b_files():
49
+def llama_3p2_1b_files():
50
with TemporaryDirectory() as cache_dir:
51
- input_dir = snapshot_download("meta-llama/Llama-3.2-1B",
51
+ input_dir = snapshot_download("meta-llama/Llama-3.2-1B-Instruct",
52
cache_dir=cache_dir,
53
ignore_patterns=["*.bin*", "original/*"])
54
@@ -81,13 +81,13 @@ def _run_generate(input_dir, queue: mp.Queue, **kwargs):
81
@pytest.mark.parametrize("enable_lora", [False, True])
82
@pytest.mark.parametrize("tp_size", [1, 2])
83
def test_sharded_state_loader(enable_lora, tp_size, num_gpus_available,
84
- llama_2_7b_files):
84
+ llama_3p2_1b_files):
85
if num_gpus_available < tp_size:
86
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
87
88
weights_patterns = ("*.safetensors", )
89
gpu_memory_utilization = 0.8
90
- input_dir = llama_2_7b_files
90
+ input_dir = llama_3p2_1b_files
91
ctx = mp.get_context("spawn")
92
93
# Run in separate processes for memory & CUDA isolation
tests/tokenization/test_detokenize.py
+1
-1
@@ -31,7 +31,7 @@ TOKENIZERS = [
31
"bigscience/bloom-560m",
32
"mosaicml/mpt-7b",
33
"tiiuae/falcon-7b",
34
- "meta-llama/Llama-2-7b-hf",
34
+ "meta-llama/Llama-3.2-1B-Instruct",
35
"codellama/CodeLlama-7b-hf",
36
"mistralai/Pixtral-12B-2409",
37
]
tests/tokenization/test_get_eos.py
+2
-2
@@ -9,7 +9,7 @@ from vllm.transformers_utils.tokenizer import get_tokenizer
9
10
11
def test_get_llama3_eos_token():
12
- model_name = "meta-llama/Meta-Llama-3-8B-Instruct"
12
+ model_name = "meta-llama/Llama-3.2-1B-Instruct"
13
14
tokenizer = get_tokenizer(model_name)
15
assert tokenizer.eos_token_id == 128009
@@ -17,7 +17,7 @@ def test_get_llama3_eos_token():
17
generation_config = try_get_generation_config(model_name,
18
trust_remote_code=False)
19
assert generation_config is not None
20
- assert generation_config.eos_token_id == [128001, 128009]
20
+ assert generation_config.eos_token_id == [128001, 128008, 128009]
21
22
23
def test_get_blip2_eos_token():
tests/v1/engine/test_async_llm.py
+1
-1
@@ -17,7 +17,7 @@ if not current_platform.is_cuda():
17
pytest.skip(reason="V1 currently only supported on CUDA.",
18
allow_module_level=True)
19
20
-ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B",
20
+ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B-Instruct",
21
enforce_eager=True,
22
disable_log_requests=True)
23
tests/v1/sample/test_logprobs.py
+1
-1
@@ -14,7 +14,7 @@ from vllm import SamplingParams
14
15
from ...conftest import VllmRunner
16
17
-MODEL = "meta-llama/Llama-3.2-1B"
17
+MODEL = "meta-llama/Llama-3.2-1B-Instruct"
18
DTYPE = "half"
19
20
tests/v1/sample/test_logprobs_e2e.py
+1
-1
@@ -11,7 +11,7 @@ RTOL = 0.03
11
EXPECTED_VALUE = 0.62
12
13
# FIXME(rob): enable prefix caching once supported.
14
-MODEL = "meta-llama/Llama-3.2-1B"
14
+MODEL = "meta-llama/Llama-3.2-1B-Instruct"
15
MODEL_ARGS = f"pretrained={MODEL},enforce_eager=True,enable_prefix_caching=False" # noqa: E501
16
SERVER_ARGS = [
17
"--enforce_eager", "--no_enable_prefix_caching", "--disable-log-requests"