A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI] Prune tests/models/decoder_only/language/* tests (#9940)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Nov 5, 2024 at 16:02 UTC
02462465ea1c45163fde632fb94e0e4939ee8a59
9 files changed
+70
-270
.buildkite/test-pipeline.yaml
+1
-2
@@ -321,7 +321,6 @@ steps:
321
- tests/models/decoder_only/language
322
commands:
323
- pytest -v -s models/decoder_only/language/test_models.py
324
- - pytest -v -s models/decoder_only/language/test_big_models.py
324
325
- label: Decoder-only Language Models Test (Extended) # 1h20min
326
nightly: true
@@ -329,7 +328,7 @@ steps:
328
- vllm/
329
- tests/models/decoder_only/language
330
commands:
332
- - pytest -v -s models/decoder_only/language --ignore=models/decoder_only/language/test_models.py --ignore=models/decoder_only/language/test_big_models.py
331
+ - pytest -v -s models/decoder_only/language --ignore=models/decoder_only/language/test_models.py
332
333
- label: Decoder-only Multi-Modal Models Test (Standard)
334
#mirror_hardwares: [amd]
tests/models/decoder_only/language/test_big_models.py
deleted
-93
@@ -1,93 +0,0 @@
1
-"""Compare the outputs of HF and vLLM when using greedy sampling.
2
-
3
-This tests bigger models and use half precision.
4
-
5
-Run `pytest tests/models/test_big_models.py`.
6
-"""
7
-import pytest
8
-
9
-from vllm.platforms import current_platform
10
-
11
-from ...utils import check_logprobs_close, check_outputs_equal
12
-
13
-MODELS = [
14
- "meta-llama/Llama-2-7b-hf",
15
- # "mistralai/Mistral-7B-v0.1", # Tested by test_mistral.py
16
- # "Deci/DeciLM-7b", # Broken
17
- # "tiiuae/falcon-7b", # Broken
18
- "EleutherAI/gpt-j-6b",
19
- # "mosaicml/mpt-7b", # Broken
20
- # "Qwen/Qwen1.5-0.5B" # Broken,
21
-]
22
-
23
-if not current_platform.is_cpu():
24
- MODELS += [
25
- # fused_moe which not supported on CPU
26
- "openbmb/MiniCPM3-4B",
27
- # Head size isn't supported on CPU
28
- "h2oai/h2o-danube3-4b-base",
29
- ]
30
-
31
-# TODO: remove this after CPU float16 support ready
32
-target_dtype = "float" if current_platform.is_cpu() else "half"
33
-
34
-
35
-@pytest.mark.parametrize("model", MODELS)
36
-@pytest.mark.parametrize("dtype", [target_dtype])
37
-@pytest.mark.parametrize("max_tokens", [32])
38
-def test_models(
39
- hf_runner,
40
- vllm_runner,
41
- example_prompts,
42
- model: str,
43
- dtype: str,
44
- max_tokens: int,
45
-) -> None:
46
-
47
- if model == "openbmb/MiniCPM3-4B":
48
- # the output becomes slightly different when upgrading to
49
- # pytorch 2.5 . Changing to logprobs checks instead of exact
50
- # output checks.
51
- NUM_LOG_PROBS = 8
52
- with hf_runner(model, dtype=dtype) as hf_model:
53
- hf_outputs = hf_model.generate_greedy_logprobs_limit(
54
- example_prompts, max_tokens, NUM_LOG_PROBS)
55
-
56
- with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
57
- vllm_outputs = vllm_model.generate_greedy_logprobs(
58
- example_prompts, max_tokens, NUM_LOG_PROBS)
59
-
60
- check_logprobs_close(
61
- outputs_0_lst=hf_outputs,
62
- outputs_1_lst=vllm_outputs,
63
- name_0="hf",
64
- name_1="vllm",
65
- )
66
- else:
67
- with hf_runner(model, dtype=dtype) as hf_model:
68
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
69
-
70
- with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
71
- vllm_outputs = vllm_model.generate_greedy(example_prompts,
72
- max_tokens)
73
-
74
- check_outputs_equal(
75
- outputs_0_lst=hf_outputs,
76
- outputs_1_lst=vllm_outputs,
77
- name_0="hf",
78
- name_1="vllm",
79
- )
80
-
81
-
82
-@pytest.mark.parametrize("model", MODELS)
83
-@pytest.mark.parametrize("dtype", [target_dtype])
84
-def test_model_print(
85
- vllm_runner,
86
- model: str,
87
- dtype: str,
88
-) -> None:
89
- with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
90
- # This test is for verifying whether the model's extra_repr
91
- # can be printed correctly.
92
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
93
- model_runner.model)
tests/models/decoder_only/language/test_fp8.py
+5
-5
@@ -21,11 +21,11 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
21
"kv_cache_dtype,base_model,test_model,scale_path",
22
[
23
# Test FP8 checkpoint w. fp8_e4m3 kv-cache scaling factors.
24
- ("fp8_e4m3", "meta-llama/Meta-Llama-3-8B-Instruct",
25
- "nm-testing/Meta-Llama-3-8B-Instruct-FP8-KV", None),
24
+ ("fp8_e4m3", "meta-llama/Llama-3.2-1B-Instruct",
25
+ "nm-testing/Llama-3.2-1B-Instruct-FP8-KV", None),
26
# Test FP16 checkpoint w. fp8_e5m2 kv-cache.
27
- ("fp8_e5m2", "meta-llama/Meta-Llama-3-8B-Instruct",
28
- "meta-llama/Meta-Llama-3-8B-Instruct", None),
27
+ ("fp8_e5m2", "meta-llama/Llama-3.2-1B-Instruct",
28
+ "meta-llama/Llama-3.2-1B-Instruct", None),
29
# Test FP16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
30
("fp8_e4m3", "meta-llama/Llama-2-7b-chat-hf",
31
"meta-llama/Llama-2-7b-chat-hf",
@@ -33,7 +33,7 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
33
])
34
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
35
@pytest.mark.parametrize("max_tokens", [4])
36
-@pytest.mark.parametrize("enforce_eager", [False, True])
36
+@pytest.mark.parametrize("enforce_eager", [True])
37
@pytest.mark.parametrize("backend", ["FLASH_ATTN", "XFORMERS", "FLASHINFER"])
38
# NOTE: Increasing this in this suite will fail CI because we currently cannot
39
# reset distributed env properly. Use a value > 1 just when you test.
tests/models/decoder_only/language/test_gptq_marlin.py
-13
@@ -22,24 +22,11 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
22
MAX_MODEL_LEN = 1024
23
24
MODELS = [
25
- # act_order==False, group_size=channelwise
26
- ("robertgshaw2/zephyr-7b-beta-channelwise-gptq", "main"),
27
- # act_order==False, group_size=128
28
- ("TheBloke/Llama-2-7B-GPTQ", "main"),
29
-
25
# act_order==True, group_size=128
26
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "main"),
32
- # act_order==True, group_size=64
33
- ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-4bit-64g-actorder_True"),
34
- # act_order==True, group_size=32
35
- ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-4bit-32g-actorder_True"),
27
28
# 8-bit, act_order==True, group_size=channelwise
29
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit--1g-actorder_True"),
39
- # 8-bit, act_order==True, group_size=128
40
- ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-128g-actorder_True"),
41
- # 8-bit, act_order==True, group_size=32
42
- ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-32g-actorder_True"),
30
31
# 4-bit, act_order==True, group_size=128
32
("TechxGenus/gemma-1.1-2b-it-GPTQ", "main")
tests/models/decoder_only/language/test_gptq_marlin_24.py
+6
-6
@@ -25,16 +25,16 @@ model_pairs = [
25
# 4-bit, group_size == 128
26
ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-4bit-g128",
27
model_gptq="alexm-nm/tinyllama-24-gptq-4bit-g128"),
28
- # 4-bit, group_size == channelwise
29
- ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-4bit-channelwise",
30
- model_gptq="alexm-nm/tinyllama-24-gptq-4bit-channelwise"),
28
+ # # 4-bit, group_size == channelwise
29
+ # ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-4bit-channelwise",
30
+ # model_gptq="alexm-nm/tinyllama-24-gptq-4bit-channelwise"),
31
32
# 8-bit, group_size == 128
33
ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-8bit-g128",
34
model_gptq="alexm-nm/tinyllama-24-gptq-8bit-g128"),
35
- # 8-bit, group_size == channelwise
36
- ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-8bit-channelwise",
37
- model_gptq="alexm-nm/tinyllama-24-gptq-8bit-channelwise"),
35
+ # # 8-bit, group_size == channelwise
36
+ # ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-8bit-channelwise",
37
+ # model_gptq="alexm-nm/tinyllama-24-gptq-8bit-channelwise"),
38
]
39
40
tests/models/decoder_only/language/test_marlin.py
deleted
-69
@@ -1,69 +0,0 @@
1
-"""Compare the outputs of a GPTQ model to a Marlin model.
2
-
3
-Note: GPTQ and Marlin do not have bitwise correctness.
4
-As a result, in this test, we just confirm that the top selected tokens of the
5
-Marlin/GPTQ models are in the top 3 selections of each other.
6
-
7
-Note: Marlin internally uses locks to synchronize the threads. This can
8
-result in very slight nondeterminism for Marlin. As a result, we re-run the test
9
-up to 3 times to see if we pass.
10
-
11
-Run `pytest tests/models/test_marlin.py`.
12
-"""
13
-from dataclasses import dataclass
14
-
15
-import pytest
16
-
17
-from tests.quantization.utils import is_quant_method_supported
18
-
19
-from ...utils import check_logprobs_close
20
-
21
-
22
-@dataclass
23
-class ModelPair:
24
- model_marlin: str
25
- model_gptq: str
26
-
27
-
28
-model_pairs = [
29
- ModelPair(model_marlin="nm-testing/zephyr-beta-7b-marlin-g128",
30
- model_gptq="nm-testing/zephyr-beta-7b-gptq-g128"),
31
- ModelPair(model_marlin="robertgshaw2/zephyr-7b-beta-channelwise-marlin",
32
- model_gptq="robertgshaw2/zephyr-7b-beta-channelwise-gptq"),
33
- ModelPair(model_marlin="robertgshaw2/TinyLlama-1.1B-Chat-v1.0-g128-marlin",
34
- model_gptq="robertgshaw2/TinyLlama-1.1B-Chat-v1.0-g128-gptq")
35
-]
36
-
37
-
38
-@pytest.mark.flaky(reruns=2)
39
-@pytest.mark.skipif(not is_quant_method_supported("marlin"),
40
- reason="Marlin is not supported on this GPU type.")
41
-@pytest.mark.parametrize("model_pair", model_pairs)
42
-@pytest.mark.parametrize("dtype", ["half"])
43
-@pytest.mark.parametrize("max_tokens", [32])
44
-@pytest.mark.parametrize("num_logprobs", [5])
45
-def test_models(
46
- vllm_runner,
47
- example_prompts,
48
- model_pair: ModelPair,
49
- dtype: str,
50
- max_tokens: int,
51
- num_logprobs: int,
52
-) -> None:
53
- with vllm_runner(model_pair.model_marlin,
54
- dtype=dtype,
55
- quantization="marlin") as marlin_model:
56
- marlin_outputs = marlin_model.generate_greedy_logprobs(
57
- example_prompts, max_tokens, num_logprobs)
58
-
59
- with vllm_runner(model_pair.model_gptq, dtype=dtype,
60
- quantization="gptq") as gptq_model:
61
- gptq_outputs = gptq_model.generate_greedy_logprobs(
62
- example_prompts, max_tokens, num_logprobs)
63
-
64
- check_logprobs_close(
65
- outputs_0_lst=gptq_outputs,
66
- outputs_1_lst=marlin_outputs,
67
- name_0="gptq",
68
- name_1="marlin",
69
- )
tests/models/decoder_only/language/test_mistral.py
+21
-16
@@ -4,7 +4,7 @@ Run `pytest tests/models/test_mistral.py`.
4
"""
5
import pytest
6
7
-from vllm import LLM, SamplingParams
7
+from vllm import SamplingParams
8
9
from ...utils import check_logprobs_close
10
@@ -15,6 +15,10 @@ MODELS = [
15
# "mistralai/Mistral-Nemo-Instruct-2407"
16
]
17
18
+MISTRAL_FORMAT_MODELS = [
19
+ "mistralai/Mistral-7B-Instruct-v0.3",
20
+]
21
+
22
SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
23
SYMBOLIC_LANG_PROMPTS = [
24
"勇敢な船乗りについての詩を書く", # japanese
@@ -95,7 +99,7 @@ def test_models(
99
)
100
101
98
-@pytest.mark.parametrize("model", MODELS[1:])
102
+@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
103
@pytest.mark.parametrize("dtype", ["bfloat16"])
104
@pytest.mark.parametrize("max_tokens", [64])
105
@pytest.mark.parametrize("num_logprobs", [5])
@@ -135,28 +139,29 @@ def test_mistral_format(
139
)
140
141
138
-@pytest.mark.parametrize("model", MODELS[1:])
142
+@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
143
@pytest.mark.parametrize("dtype", ["bfloat16"])
140
-@pytest.mark.parametrize("prompt", SYMBOLIC_LANG_PROMPTS)
144
def test_mistral_symbolic_languages(
145
+ vllm_runner,
146
model: str,
147
dtype: str,
144
- prompt: str,
148
) -> None:
146
- prompt = "hi"
147
- msg = {"role": "user", "content": prompt}
148
- llm = LLM(model=model,
149
- dtype=dtype,
150
- max_model_len=8192,
151
- tokenizer_mode="mistral",
152
- config_format="mistral",
153
- load_format="mistral")
154
- outputs = llm.chat([msg], sampling_params=SAMPLING_PARAMS)
155
- assert "�" not in outputs[0].outputs[0].text.strip()
149
+ with vllm_runner(model,
150
+ dtype=dtype,
151
+ max_model_len=8192,
152
+ tokenizer_mode="mistral",
153
+ config_format="mistral",
154
+ load_format="mistral") as vllm_model:
155
+ for prompt in SYMBOLIC_LANG_PROMPTS:
156
+ msg = {"role": "user", "content": prompt}
157
+ outputs = vllm_model.model.chat([msg],
158
+ sampling_params=SAMPLING_PARAMS)
159
+ assert "�" not in outputs[0].outputs[0].text.strip()
160
161
162
@pytest.mark.parametrize("dtype", ["bfloat16"])
159
-@pytest.mark.parametrize("model", MODELS[1:]) # v1 can't do func calling
163
+@pytest.mark.parametrize("model",
164
+ MISTRAL_FORMAT_MODELS) # v1 can't do func calling
165
def test_mistral_function_calling(
166
vllm_runner,
167
model: str,
tests/models/decoder_only/language/test_models.py
+37
-32
@@ -7,25 +7,39 @@ Run `pytest tests/models/test_models.py`.
7
"""
8
import pytest
9
10
-from ...utils import check_outputs_equal
10
+from vllm.platforms import current_platform
11
+
12
+from ...utils import check_logprobs_close
13
14
MODELS = [
13
- "facebook/opt-125m",
14
- "gpt2",
15
- "bigcode/tiny_starcoder_py",
16
- "EleutherAI/pythia-70m",
17
- "bigscience/bloom-560m", # Testing alibi slopes.
18
- "microsoft/phi-2",
19
- "stabilityai/stablelm-3b-4e1t",
20
- # "allenai/OLMo-1B", # Broken
21
- "bigcode/starcoder2-3b",
22
- "google/gemma-1.1-2b-it",
15
+ "facebook/opt-125m", # opt
16
+ "openai-community/gpt2", # gpt2
17
+ # "Milos/slovak-gpt-j-405M", # gptj
18
+ # "bigcode/tiny_starcoder_py", # gpt_bigcode
19
+ # "EleutherAI/pythia-70m", # gpt_neox
20
+ "bigscience/bloom-560m", # bloom - testing alibi slopes
21
+ "microsoft/phi-2", # phi
22
+ # "stabilityai/stablelm-3b-4e1t", # stablelm
23
+ # "bigcode/starcoder2-3b", # starcoder2
24
+ "google/gemma-1.1-2b-it", # gemma
25
+ "Qwen/Qwen2.5-0.5B-Instruct", # qwen2
26
+ "meta-llama/Llama-3.2-1B-Instruct", # llama
27
]
28
29
+if not current_platform.is_cpu():
30
+ MODELS += [
31
+ # fused_moe which not supported on CPU
32
+ "openbmb/MiniCPM3-4B",
33
+ ]
34
+
35
+# TODO: remove this after CPU float16 support ready
36
+target_dtype = "float" if current_platform.is_cpu() else "half"
37
+
38
39
@pytest.mark.parametrize("model", MODELS)
27
-@pytest.mark.parametrize("dtype", ["float"])
28
-@pytest.mark.parametrize("max_tokens", [96])
40
+@pytest.mark.parametrize("dtype", [target_dtype])
41
+@pytest.mark.parametrize("max_tokens", [32])
42
+@pytest.mark.parametrize("num_logprobs", [5])
43
def test_models(
44
hf_runner,
45
vllm_runner,
@@ -33,33 +47,24 @@ def test_models(
47
model: str,
48
dtype: str,
49
max_tokens: int,
50
+ num_logprobs: int,
51
) -> None:
37
- # To pass the small model tests, we need full precision.
38
- assert dtype == "float"
52
53
with hf_runner(model, dtype=dtype) as hf_model:
41
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
54
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
55
+ example_prompts, max_tokens, num_logprobs)
56
57
with vllm_runner(model, dtype=dtype) as vllm_model:
44
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
58
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
59
+ example_prompts, max_tokens, num_logprobs)
60
+ # This test is for verifying whether the model's extra_repr
61
+ # can be printed correctly.
62
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
63
+ model_runner.model)
64
46
- check_outputs_equal(
65
+ check_logprobs_close(
66
outputs_0_lst=hf_outputs,
67
outputs_1_lst=vllm_outputs,
68
name_0="hf",
69
name_1="vllm",
70
)
52
-
53
-
54
-@pytest.mark.parametrize("model", MODELS)
55
-@pytest.mark.parametrize("dtype", ["float"])
56
-def test_model_print(
57
- vllm_runner,
58
- model: str,
59
- dtype: str,
60
-) -> None:
61
- with vllm_runner(model, dtype=dtype) as vllm_model:
62
- # This test is for verifying whether the model's extra_repr
63
- # can be printed correctly.
64
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
65
- model_runner.model)
tests/models/decoder_only/language/test_qwen.py
deleted
-34
@@ -1,34 +0,0 @@
1
-"""Ensure that a text-only Qwen model can be run without throwing an error.
2
-We explicitly test this because Qwen is implemented as a multimodal and
3
-supports a visual encoder for models like Qwen-VL.
4
-"""
5
-from typing import List, Type
6
-
7
-import pytest
8
-
9
-from ....conftest import VllmRunner
10
-
11
-models = [
12
- "Qwen/Qwen-7B-Chat" # Has no visual encoder
13
-]
14
-
15
-
16
-@pytest.mark.parametrize("model", models)
17
-@pytest.mark.parametrize("dtype", ["bfloat16"])
18
-@pytest.mark.parametrize("max_tokens", [32])
19
-@pytest.mark.parametrize("num_logprobs", [5])
20
-def test_text_only_qwen_model_can_be_loaded_and_run(
21
- vllm_runner: Type[VllmRunner],
22
- example_prompts: List[str],
23
- model: str,
24
- *,
25
- dtype: str,
26
- max_tokens: int,
27
- num_logprobs: int,
28
-):
29
- with vllm_runner(model, dtype=dtype) as vllm_model:
30
- vllm_model.generate_greedy_logprobs(
31
- example_prompts,
32
- max_tokens,
33
- num_logprobs=num_logprobs,
34
- )