@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f2b20fe491
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

Consolidate Llama model usage in tests (#13094)

Harry Mellor committed Feb 14, 2025 at 06:18 UTC f2b20fe491073dad7e3c3fe8ab0303b97fb50643
22 files changed +44 -52
tests/basic_correctness/test_basic_correctness.py
+5 -5
@@ -17,7 +17,7 @@ from ..utils import multi_gpu_test
17
18 MODELS = [
19 "google/gemma-2-2b-it",
20 - "meta-llama/Llama-3.2-1B",
20 + "meta-llama/Llama-3.2-1B-Instruct",
21 ]
22
23 TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
@@ -96,12 +96,12 @@ def test_models(
96 "test_suite", [
97 ("facebook/opt-125m", "ray", "", "L4"),
98 ("facebook/opt-125m", "mp", "", "L4"),
99 - ("meta-llama/Llama-2-7b-hf", "ray", "", "L4"),
100 - ("meta-llama/Llama-2-7b-hf", "mp", "", "L4"),
99 + ("meta-llama/Llama-3.2-1B-Instruct", "ray", "", "L4"),
100 + ("meta-llama/Llama-3.2-1B-Instruct", "mp", "", "L4"),
101 ("facebook/opt-125m", "ray", "", "A100"),
102 ("facebook/opt-125m", "mp", "", "A100"),
103 ("facebook/opt-125m", "mp", "FLASHINFER", "A100"),
104 - ("meta-llama/Meta-Llama-3-8B", "ray", "FLASHINFER", "A100"),
104 + ("meta-llama/Llama-3.2-1B-Instruct", "ray", "FLASHINFER", "A100"),
105 ])
106 def test_models_distributed(
107 hf_runner,
@@ -116,7 +116,7 @@ def test_models_distributed(
116 if test_suite != TARGET_TEST_SUITE:
117 pytest.skip(f"Skip test for {test_suite}")
118
119 - if model == "meta-llama/Llama-2-7b-hf" and distributed_executor_backend == "ray" and attention_backend == "" and test_suite == "L4": # noqa
119 + if model == "meta-llama/Llama-3.2-1B-Instruct" and distributed_executor_backend == "ray" and attention_backend == "" and test_suite == "L4": # noqa
120 # test ray adag
121 os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
122 os.environ['VLLM_USE_RAY_COMPILED_DAG'] = "1"
tests/basic_correctness/test_chunked_prefill.py
+3 -3
@@ -20,7 +20,7 @@ from ..utils import multi_gpu_test
20
21 MODELS = [
22 "facebook/opt-125m",
23 - "meta-llama/Llama-3.2-1B",
23 + "meta-llama/Llama-3.2-1B-Instruct",
24 ]
25
26
@@ -92,7 +92,7 @@ def test_models_distributed(
92 ) -> None:
93 override_backend_env_variable(monkeypatch, attention_backend)
94
95 - if (model == "meta-llama/Llama-2-7b-hf"
95 + if (model == "meta-llama/Llama-3.2-1B-Instruct"
96 and distributed_executor_backend == "ray"):
97 # test ray adag
98 os.environ['VLLM_USE_RAY_SPMD_WORKER'] = "1"
@@ -221,7 +221,7 @@ def test_with_prefix_caching(
221 Checks exact match decode with and without prefix caching
222 with chunked prefill enabled.
223 """
224 - model = "meta-llama/Llama-2-7b-chat-hf"
224 + model = "meta-llama/Llama-3.2-1B-Instruct"
225 # The common prompt has 142 tokens with Llama-2 tokenizer.
226 common_prompt = "You are a helpful AI assistant " * 20
227 unique_prompts = [
tests/basic_correctness/test_cpu_offload.py
+1 -1
@@ -4,5 +4,5 @@ from ..utils import compare_two_settings
4
5
6 def test_cpu_offload():
7 - compare_two_settings("meta-llama/Llama-3.2-1B", [],
7 + compare_two_settings("meta-llama/Llama-3.2-1B-Instruct", [],
8 ["--cpu-offload-gb", "1"])
tests/basic_correctness/test_cumem.py
+1 -1
@@ -118,7 +118,7 @@ def test_cumem_with_cudagraph():
118 @pytest.mark.parametrize(
119 "model",
120 [
121 - "meta-llama/Llama-3.2-1B", # sleep mode with safetensors
121 + "meta-llama/Llama-3.2-1B-Instruct", # sleep mode with safetensors
122 "facebook/opt-125m" # sleep mode with pytorch checkpoint
123 ])
124 def test_end_to_end(model):
tests/compile/test_basic_correctness.py
+1 -1
@@ -26,7 +26,7 @@ class TestSetting:
26 test_settings = [
27 # basic llama model
28 TestSetting(
29 - model="meta-llama/Llama-3.2-1B",
29 + model="meta-llama/Llama-3.2-1B-Instruct",
30 model_args=[],
31 pp_size=2,
32 tp_size=2,
tests/compile/utils.py
+4 -10
@@ -6,7 +6,6 @@ import torch
6
7 from tests.quantization.utils import is_quant_method_supported
8 from vllm import LLM, SamplingParams
9 -from vllm.config import CompilationLevel
9 from vllm.platforms import current_platform
10
11 TEST_MODELS = [
@@ -15,14 +14,14 @@ TEST_MODELS = [
14 "dtype": torch.float16,
15 "quantization": "compressed-tensors"
16 }),
18 - ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", {
17 + ("neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic", {
18 "dtype": torch.float16,
20 - "quantization": "fp8"
19 + "quantization": "compressed-tensors"
20 }),
22 - ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
21 + ("neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8", {
22 "quantization": "compressed-tensors"
23 }),
25 - ("meta-llama/Meta-Llama-3-8B", {}),
24 + ("meta-llama/Llama-3.2-1B-Instruct", {}),
25 ]
26
27 if is_quant_method_supported("aqlm"):
@@ -69,11 +68,6 @@ def check_full_graph_support(model,
68 # make sure these models can be captured in full graph mode
69 os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
70
72 - # The base meta llama uses too much memory.
73 - if (model == "meta-llama/Meta-Llama-3-8B"
74 - and optimization_level >= CompilationLevel.PIECEWISE):
75 - return
76 -
71 print(f"MODEL={model}")
72
73 prompts = [
tests/distributed/test_pipeline_parallel.py
+2 -2
@@ -162,7 +162,7 @@ TEXT_GENERATION_MODELS = {
162 "internlm/internlm2-chat-7b": PPTestSettings.fast(),
163 "inceptionai/jais-13b-chat": PPTestSettings.fast(),
164 "ai21labs/Jamba-tiny-dev": PPTestSettings.fast(),
165 - "meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
165 + "meta-llama/Llama-3.2-1B-Instruct": PPTestSettings.detailed(),
166 "openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(),
167 "openbmb/MiniCPM3-4B": PPTestSettings.fast(),
168 # Uses Llama
@@ -230,7 +230,7 @@ MULTIMODAL_MODELS = {
230 TEST_MODELS = [
231 # [LANGUAGE GENERATION]
232 "microsoft/Phi-3.5-MoE-instruct",
233 - "meta-llama/Meta-Llama-3-8B",
233 + "meta-llama/Llama-3.2-1B-Instruct",
234 "ibm/PowerLM-3b",
235 # [LANGUAGE EMBEDDING]
236 "intfloat/e5-mistral-7b-instruct",
tests/entrypoints/openai/test_serving_models.py
+1 -1
@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.serving_models import (BaseModelPath,
14 OpenAIServingModels)
15 from vllm.lora.request import LoRARequest
16
17 -MODEL_NAME = "meta-llama/Llama-2-7b"
17 +MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
18 BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
19 LORA_LOADING_SUCCESS_MESSAGE = (
20 "Success: LoRA adapter '{lora_name}' added successfully.")
tests/entrypoints/openai/test_shutdown.py
+1 -1
@@ -5,7 +5,7 @@ import pytest
5
6 from ...utils import RemoteOpenAIServer
7
8 -MODEL_NAME = "meta-llama/Llama-3.2-1B"
8 +MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
9
10
11 @pytest.mark.asyncio
tests/kv_transfer/disagg_test.py
+4 -6
@@ -28,7 +28,7 @@ def setup_servers():
28 "-m",
29 "vllm.entrypoints.openai.api_server",
30 "--model",
31 - "meta-llama/Meta-Llama-3.1-8B-Instruct",
31 + "meta-llama/Llama-3.2-1B-Instruct",
32 "--port",
33 "8100",
34 "--gpu-memory-utilization",
@@ -49,7 +49,7 @@ def setup_servers():
49 "-m",
50 "vllm.entrypoints.openai.api_server",
51 "--model",
52 - "meta-llama/Meta-Llama-3.1-8B-Instruct",
52 + "meta-llama/Llama-3.2-1B-Instruct",
53 "--port",
54 "8200",
55 "--gpu-memory-utilization",
@@ -100,8 +100,7 @@ def test_disaggregated_prefilling(prompt):
100 response = requests.post("http://localhost:8100/v1/completions",
101 headers={"Content-Type": "application/json"},
102 json={
103 - "model":
104 - "meta-llama/Meta-Llama-3.1-8B-Instruct",
103 + "model": "meta-llama/Llama-3.2-1B-Instruct",
104 "prompt": prompt,
105 "max_tokens": 1,
106 "temperature": 0
@@ -112,8 +111,7 @@ def test_disaggregated_prefilling(prompt):
111 response = requests.post("http://localhost:8200/v1/completions",
112 headers={"Content-Type": "application/json"},
113 json={
115 - "model":
116 - "meta-llama/Meta-Llama-3.1-8B-Instruct",
114 + "model": "meta-llama/Llama-3.2-1B-Instruct",
115 "prompt": prompt,
116 "max_tokens": 10,
117 "temperature": 0
tests/models/decoder_only/language/test_fp8.py
+4 -4
@@ -26,12 +26,12 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
26 # Test FP8 checkpoint w. fp8_e4m3 kv-cache scaling factors.
27 ("fp8_e4m3", "meta-llama/Llama-3.2-1B-Instruct",
28 "nm-testing/Llama-3.2-1B-Instruct-FP8-KV"),
29 - # Test FP16 checkpoint w. fp8_e5m2 kv-cache.
29 + # Test BF16 checkpoint w. fp8_e5m2 kv-cache.
30 ("fp8_e5m2", "meta-llama/Llama-3.2-1B-Instruct",
31 "meta-llama/Llama-3.2-1B-Instruct"),
32 - # Test FP16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
33 - ("fp8_e4m3", "meta-llama/Llama-2-7b-chat-hf",
34 - "meta-llama/Llama-2-7b-chat-hf")
32 + # Test BF16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
33 + ("fp8_e4m3", "meta-llama/Llama-3.2-1B-Instruct",
34 + "meta-llama/Llama-3.2-1B-Instruct")
35 ])
36 # Due to low-precision numerical divergence, we only test logprob of 4 tokens
37 @pytest.mark.parametrize("max_tokens", [4])
tests/models/registry.py
+1 -1
@@ -141,7 +141,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
141 "JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
142 "JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini",
143 extras={"tiny": "ai21labs/Jamba-tiny-dev"}), # noqa: E501
144 - "LlamaForCausalLM": _HfExamplesInfo("meta-llama/Meta-Llama-3-8B"),
144 + "LlamaForCausalLM": _HfExamplesInfo("meta-llama/Llama-3.2-1B-Instruct"),
145 "LLaMAForCausalLM": _HfExamplesInfo("decapoda-research/llama-7b-hf",
146 is_available_online=False),
147 "MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
tests/quantization/test_register_quantization_config.py
+1 -1
@@ -99,7 +99,7 @@ def test_register_quantization_config():
99
100 @pytest.mark.parametrize(argnames="model",
101 argvalues=[
102 - "meta-llama/Meta-Llama-3-8B-Instruct",
102 + "meta-llama/Llama-3.2-1B-Instruct",
103 ])
104 def test_custom_quant(vllm_runner, model):
105 """Test infer with the custom quantization method."""
tests/samplers/test_ignore_eos.py
+1 -1
@@ -10,7 +10,7 @@ from vllm import SamplingParams
10
11 # We also test with llama because it has generation_config to specify EOS
12 # (past regression).
13 -MODELS = ["facebook/opt-125m", "meta-llama/Llama-2-7b-hf"]
13 +MODELS = ["facebook/opt-125m", "meta-llama/Llama-3.2-1B-Instruct"]
14
15
16 @pytest.mark.parametrize("model", MODELS)
tests/spec_decode/e2e/test_compatibility.py
+3 -3
@@ -8,7 +8,7 @@ from .conftest import get_output_from_llm_generator
8
9
10 @pytest.mark.parametrize("common_llm_kwargs", [{
11 - "model": "meta-llama/Llama-2-7b-chat-hf",
11 + "model": "meta-llama/Llama-3.2-1B-Instruct",
12 "speculative_model": "JackFram/llama-68m",
13 "num_speculative_tokens": 5,
14 }])
@@ -27,8 +27,8 @@ from .conftest import get_output_from_llm_generator
27 },
28 {
29 # Speculative max model len > target max model len should raise.
30 - # https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/blob/f5db02db724555f92da89c216ac04704f23d4590/config.json#L12
31 - "speculative_max_model_len": 4096 + 1,
30 + # https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/blob/9213176726f574b556790deb65791e0c5aa438b6/config.json#L18
31 + "speculative_max_model_len": 131072 + 1,
32 },
33 ])
34 @pytest.mark.parametrize("test_llm_kwargs", [{}])
tests/test_config.py
+1 -1
@@ -251,7 +251,7 @@ def test_rope_customization():
251 @pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
252 ("facebook/opt-125m", False),
253 ("facebook/bart-base", True),
254 - ("meta-llama/Llama-3.2-1B", False),
254 + ("meta-llama/Llama-3.2-1B-Instruct", False),
255 ("meta-llama/Llama-3.2-11B-Vision", True),
256 ])
257 def test_is_encoder_decoder(model_id, is_encoder_decoder):
tests/test_sharded_state_loader.py
+4 -4
@@ -46,9 +46,9 @@ def test_filter_subtensors():
46
47
48 @pytest.fixture(scope="module")
49 -def llama_2_7b_files():
49 +def llama_3p2_1b_files():
50 with TemporaryDirectory() as cache_dir:
51 - input_dir = snapshot_download("meta-llama/Llama-3.2-1B",
51 + input_dir = snapshot_download("meta-llama/Llama-3.2-1B-Instruct",
52 cache_dir=cache_dir,
53 ignore_patterns=["*.bin*", "original/*"])
54
@@ -81,13 +81,13 @@ def _run_generate(input_dir, queue: mp.Queue, **kwargs):
81 @pytest.mark.parametrize("enable_lora", [False, True])
82 @pytest.mark.parametrize("tp_size", [1, 2])
83 def test_sharded_state_loader(enable_lora, tp_size, num_gpus_available,
84 - llama_2_7b_files):
84 + llama_3p2_1b_files):
85 if num_gpus_available < tp_size:
86 pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
87
88 weights_patterns = ("*.safetensors", )
89 gpu_memory_utilization = 0.8
90 - input_dir = llama_2_7b_files
90 + input_dir = llama_3p2_1b_files
91 ctx = mp.get_context("spawn")
92
93 # Run in separate processes for memory & CUDA isolation
tests/tokenization/test_detokenize.py
+1 -1
@@ -31,7 +31,7 @@ TOKENIZERS = [
31 "bigscience/bloom-560m",
32 "mosaicml/mpt-7b",
33 "tiiuae/falcon-7b",
34 - "meta-llama/Llama-2-7b-hf",
34 + "meta-llama/Llama-3.2-1B-Instruct",
35 "codellama/CodeLlama-7b-hf",
36 "mistralai/Pixtral-12B-2409",
37 ]
tests/tokenization/test_get_eos.py
+2 -2
@@ -9,7 +9,7 @@ from vllm.transformers_utils.tokenizer import get_tokenizer
9
10
11 def test_get_llama3_eos_token():
12 - model_name = "meta-llama/Meta-Llama-3-8B-Instruct"
12 + model_name = "meta-llama/Llama-3.2-1B-Instruct"
13
14 tokenizer = get_tokenizer(model_name)
15 assert tokenizer.eos_token_id == 128009
@@ -17,7 +17,7 @@ def test_get_llama3_eos_token():
17 generation_config = try_get_generation_config(model_name,
18 trust_remote_code=False)
19 assert generation_config is not None
20 - assert generation_config.eos_token_id == [128001, 128009]
20 + assert generation_config.eos_token_id == [128001, 128008, 128009]
21
22
23 def test_get_blip2_eos_token():
tests/v1/engine/test_async_llm.py
+1 -1
@@ -17,7 +17,7 @@ if not current_platform.is_cuda():
17 pytest.skip(reason="V1 currently only supported on CUDA.",
18 allow_module_level=True)
19
20 -ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B",
20 +ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B-Instruct",
21 enforce_eager=True,
22 disable_log_requests=True)
23
tests/v1/sample/test_logprobs.py
+1 -1
@@ -14,7 +14,7 @@ from vllm import SamplingParams
14
15 from ...conftest import VllmRunner
16
17 -MODEL = "meta-llama/Llama-3.2-1B"
17 +MODEL = "meta-llama/Llama-3.2-1B-Instruct"
18 DTYPE = "half"
19
20
tests/v1/sample/test_logprobs_e2e.py
+1 -1
@@ -11,7 +11,7 @@ RTOL = 0.03
11 EXPECTED_VALUE = 0.62
12
13 # FIXME(rob): enable prefix caching once supported.
14 -MODEL = "meta-llama/Llama-3.2-1B"
14 +MODEL = "meta-llama/Llama-3.2-1B-Instruct"
15 MODEL_ARGS = f"pretrained={MODEL},enforce_eager=True,enable_prefix_caching=False" # noqa: E501
16 SERVER_ARGS = [
17 "--enforce_eager", "--no_enable_prefix_caching", "--disable-log-requests"