@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 02462465ea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[CI] Prune tests/models/decoder_only/language/* tests (#9940)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 5, 2024 at 16:02 UTC 02462465ea1c45163fde632fb94e0e4939ee8a59
9 files changed +70 -270
.buildkite/test-pipeline.yaml
+1 -2
@@ -321,7 +321,6 @@ steps:
321 - tests/models/decoder_only/language
322 commands:
323 - pytest -v -s models/decoder_only/language/test_models.py
324 - - pytest -v -s models/decoder_only/language/test_big_models.py
324
325 - label: Decoder-only Language Models Test (Extended) # 1h20min
326 nightly: true
@@ -329,7 +328,7 @@ steps:
328 - vllm/
329 - tests/models/decoder_only/language
330 commands:
332 - - pytest -v -s models/decoder_only/language --ignore=models/decoder_only/language/test_models.py --ignore=models/decoder_only/language/test_big_models.py
331 + - pytest -v -s models/decoder_only/language --ignore=models/decoder_only/language/test_models.py
332
333 - label: Decoder-only Multi-Modal Models Test (Standard)
334 #mirror_hardwares: [amd]
tests/models/decoder_only/language/test_big_models.py deleted
-93
@@ -1,93 +0,0 @@
1 -"""Compare the outputs of HF and vLLM when using greedy sampling.
2 -
3 -This tests bigger models and use half precision.
4 -
5 -Run `pytest tests/models/test_big_models.py`.
6 -"""
7 -import pytest
8 -
9 -from vllm.platforms import current_platform
10 -
11 -from ...utils import check_logprobs_close, check_outputs_equal
12 -
13 -MODELS = [
14 - "meta-llama/Llama-2-7b-hf",
15 - # "mistralai/Mistral-7B-v0.1", # Tested by test_mistral.py
16 - # "Deci/DeciLM-7b", # Broken
17 - # "tiiuae/falcon-7b", # Broken
18 - "EleutherAI/gpt-j-6b",
19 - # "mosaicml/mpt-7b", # Broken
20 - # "Qwen/Qwen1.5-0.5B" # Broken,
21 -]
22 -
23 -if not current_platform.is_cpu():
24 - MODELS += [
25 - # fused_moe which not supported on CPU
26 - "openbmb/MiniCPM3-4B",
27 - # Head size isn't supported on CPU
28 - "h2oai/h2o-danube3-4b-base",
29 - ]
30 -
31 -# TODO: remove this after CPU float16 support ready
32 -target_dtype = "float" if current_platform.is_cpu() else "half"
33 -
34 -
35 -@pytest.mark.parametrize("model", MODELS)
36 -@pytest.mark.parametrize("dtype", [target_dtype])
37 -@pytest.mark.parametrize("max_tokens", [32])
38 -def test_models(
39 - hf_runner,
40 - vllm_runner,
41 - example_prompts,
42 - model: str,
43 - dtype: str,
44 - max_tokens: int,
45 -) -> None:
46 -
47 - if model == "openbmb/MiniCPM3-4B":
48 - # the output becomes slightly different when upgrading to
49 - # pytorch 2.5 . Changing to logprobs checks instead of exact
50 - # output checks.
51 - NUM_LOG_PROBS = 8
52 - with hf_runner(model, dtype=dtype) as hf_model:
53 - hf_outputs = hf_model.generate_greedy_logprobs_limit(
54 - example_prompts, max_tokens, NUM_LOG_PROBS)
55 -
56 - with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
57 - vllm_outputs = vllm_model.generate_greedy_logprobs(
58 - example_prompts, max_tokens, NUM_LOG_PROBS)
59 -
60 - check_logprobs_close(
61 - outputs_0_lst=hf_outputs,
62 - outputs_1_lst=vllm_outputs,
63 - name_0="hf",
64 - name_1="vllm",
65 - )
66 - else:
67 - with hf_runner(model, dtype=dtype) as hf_model:
68 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
69 -
70 - with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
71 - vllm_outputs = vllm_model.generate_greedy(example_prompts,
72 - max_tokens)
73 -
74 - check_outputs_equal(
75 - outputs_0_lst=hf_outputs,
76 - outputs_1_lst=vllm_outputs,
77 - name_0="hf",
78 - name_1="vllm",
79 - )
80 -
81 -
82 -@pytest.mark.parametrize("model", MODELS)
83 -@pytest.mark.parametrize("dtype", [target_dtype])
84 -def test_model_print(
85 - vllm_runner,
86 - model: str,
87 - dtype: str,
88 -) -> None:
89 - with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
90 - # This test is for verifying whether the model's extra_repr
91 - # can be printed correctly.
92 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
93 - model_runner.model)
tests/models/decoder_only/language/test_fp8.py
+5 -5
@@ -21,11 +21,11 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
21 "kv_cache_dtype,base_model,test_model,scale_path",
22 [
23 # Test FP8 checkpoint w. fp8_e4m3 kv-cache scaling factors.
24 - ("fp8_e4m3", "meta-llama/Meta-Llama-3-8B-Instruct",
25 - "nm-testing/Meta-Llama-3-8B-Instruct-FP8-KV", None),
24 + ("fp8_e4m3", "meta-llama/Llama-3.2-1B-Instruct",
25 + "nm-testing/Llama-3.2-1B-Instruct-FP8-KV", None),
26 # Test FP16 checkpoint w. fp8_e5m2 kv-cache.
27 - ("fp8_e5m2", "meta-llama/Meta-Llama-3-8B-Instruct",
28 - "meta-llama/Meta-Llama-3-8B-Instruct", None),
27 + ("fp8_e5m2", "meta-llama/Llama-3.2-1B-Instruct",
28 + "meta-llama/Llama-3.2-1B-Instruct", None),
29 # Test FP16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
30 ("fp8_e4m3", "meta-llama/Llama-2-7b-chat-hf",
31 "meta-llama/Llama-2-7b-chat-hf",
@@ -33,7 +33,7 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
33 ])
34 # Due to low-precision numerical divergence, we only test logprob of 4 tokens
35 @pytest.mark.parametrize("max_tokens", [4])
36 -@pytest.mark.parametrize("enforce_eager", [False, True])
36 +@pytest.mark.parametrize("enforce_eager", [True])
37 @pytest.mark.parametrize("backend", ["FLASH_ATTN", "XFORMERS", "FLASHINFER"])
38 # NOTE: Increasing this in this suite will fail CI because we currently cannot
39 # reset distributed env properly. Use a value > 1 just when you test.
tests/models/decoder_only/language/test_gptq_marlin.py
-13
@@ -22,24 +22,11 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
22 MAX_MODEL_LEN = 1024
23
24 MODELS = [
25 - # act_order==False, group_size=channelwise
26 - ("robertgshaw2/zephyr-7b-beta-channelwise-gptq", "main"),
27 - # act_order==False, group_size=128
28 - ("TheBloke/Llama-2-7B-GPTQ", "main"),
29 -
25 # act_order==True, group_size=128
26 ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "main"),
32 - # act_order==True, group_size=64
33 - ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-4bit-64g-actorder_True"),
34 - # act_order==True, group_size=32
35 - ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-4bit-32g-actorder_True"),
27
28 # 8-bit, act_order==True, group_size=channelwise
29 ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit--1g-actorder_True"),
39 - # 8-bit, act_order==True, group_size=128
40 - ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-128g-actorder_True"),
41 - # 8-bit, act_order==True, group_size=32
42 - ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit-32g-actorder_True"),
30
31 # 4-bit, act_order==True, group_size=128
32 ("TechxGenus/gemma-1.1-2b-it-GPTQ", "main")
tests/models/decoder_only/language/test_gptq_marlin_24.py
+6 -6
@@ -25,16 +25,16 @@ model_pairs = [
25 # 4-bit, group_size == 128
26 ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-4bit-g128",
27 model_gptq="alexm-nm/tinyllama-24-gptq-4bit-g128"),
28 - # 4-bit, group_size == channelwise
29 - ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-4bit-channelwise",
30 - model_gptq="alexm-nm/tinyllama-24-gptq-4bit-channelwise"),
28 + # # 4-bit, group_size == channelwise
29 + # ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-4bit-channelwise",
30 + # model_gptq="alexm-nm/tinyllama-24-gptq-4bit-channelwise"),
31
32 # 8-bit, group_size == 128
33 ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-8bit-g128",
34 model_gptq="alexm-nm/tinyllama-24-gptq-8bit-g128"),
35 - # 8-bit, group_size == channelwise
36 - ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-8bit-channelwise",
37 - model_gptq="alexm-nm/tinyllama-24-gptq-8bit-channelwise"),
35 + # # 8-bit, group_size == channelwise
36 + # ModelPair(model_marlin="alexm-nm/tinyllama-24-marlin24-8bit-channelwise",
37 + # model_gptq="alexm-nm/tinyllama-24-gptq-8bit-channelwise"),
38 ]
39
40
tests/models/decoder_only/language/test_marlin.py deleted
-69
@@ -1,69 +0,0 @@
1 -"""Compare the outputs of a GPTQ model to a Marlin model.
2 -
3 -Note: GPTQ and Marlin do not have bitwise correctness.
4 -As a result, in this test, we just confirm that the top selected tokens of the
5 -Marlin/GPTQ models are in the top 3 selections of each other.
6 -
7 -Note: Marlin internally uses locks to synchronize the threads. This can
8 -result in very slight nondeterminism for Marlin. As a result, we re-run the test
9 -up to 3 times to see if we pass.
10 -
11 -Run `pytest tests/models/test_marlin.py`.
12 -"""
13 -from dataclasses import dataclass
14 -
15 -import pytest
16 -
17 -from tests.quantization.utils import is_quant_method_supported
18 -
19 -from ...utils import check_logprobs_close
20 -
21 -
22 -@dataclass
23 -class ModelPair:
24 - model_marlin: str
25 - model_gptq: str
26 -
27 -
28 -model_pairs = [
29 - ModelPair(model_marlin="nm-testing/zephyr-beta-7b-marlin-g128",
30 - model_gptq="nm-testing/zephyr-beta-7b-gptq-g128"),
31 - ModelPair(model_marlin="robertgshaw2/zephyr-7b-beta-channelwise-marlin",
32 - model_gptq="robertgshaw2/zephyr-7b-beta-channelwise-gptq"),
33 - ModelPair(model_marlin="robertgshaw2/TinyLlama-1.1B-Chat-v1.0-g128-marlin",
34 - model_gptq="robertgshaw2/TinyLlama-1.1B-Chat-v1.0-g128-gptq")
35 -]
36 -
37 -
38 -@pytest.mark.flaky(reruns=2)
39 -@pytest.mark.skipif(not is_quant_method_supported("marlin"),
40 - reason="Marlin is not supported on this GPU type.")
41 -@pytest.mark.parametrize("model_pair", model_pairs)
42 -@pytest.mark.parametrize("dtype", ["half"])
43 -@pytest.mark.parametrize("max_tokens", [32])
44 -@pytest.mark.parametrize("num_logprobs", [5])
45 -def test_models(
46 - vllm_runner,
47 - example_prompts,
48 - model_pair: ModelPair,
49 - dtype: str,
50 - max_tokens: int,
51 - num_logprobs: int,
52 -) -> None:
53 - with vllm_runner(model_pair.model_marlin,
54 - dtype=dtype,
55 - quantization="marlin") as marlin_model:
56 - marlin_outputs = marlin_model.generate_greedy_logprobs(
57 - example_prompts, max_tokens, num_logprobs)
58 -
59 - with vllm_runner(model_pair.model_gptq, dtype=dtype,
60 - quantization="gptq") as gptq_model:
61 - gptq_outputs = gptq_model.generate_greedy_logprobs(
62 - example_prompts, max_tokens, num_logprobs)
63 -
64 - check_logprobs_close(
65 - outputs_0_lst=gptq_outputs,
66 - outputs_1_lst=marlin_outputs,
67 - name_0="gptq",
68 - name_1="marlin",
69 - )
tests/models/decoder_only/language/test_mistral.py
+21 -16
@@ -4,7 +4,7 @@ Run `pytest tests/models/test_mistral.py`.
4 """
5 import pytest
6
7 -from vllm import LLM, SamplingParams
7 +from vllm import SamplingParams
8
9 from ...utils import check_logprobs_close
10
@@ -15,6 +15,10 @@ MODELS = [
15 # "mistralai/Mistral-Nemo-Instruct-2407"
16 ]
17
18 +MISTRAL_FORMAT_MODELS = [
19 + "mistralai/Mistral-7B-Instruct-v0.3",
20 +]
21 +
22 SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
23 SYMBOLIC_LANG_PROMPTS = [
24 "勇敢な船乗りについての詩を書く", # japanese
@@ -95,7 +99,7 @@ def test_models(
99 )
100
101
98 -@pytest.mark.parametrize("model", MODELS[1:])
102 +@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
103 @pytest.mark.parametrize("dtype", ["bfloat16"])
104 @pytest.mark.parametrize("max_tokens", [64])
105 @pytest.mark.parametrize("num_logprobs", [5])
@@ -135,28 +139,29 @@ def test_mistral_format(
139 )
140
141
138 -@pytest.mark.parametrize("model", MODELS[1:])
142 +@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
143 @pytest.mark.parametrize("dtype", ["bfloat16"])
140 -@pytest.mark.parametrize("prompt", SYMBOLIC_LANG_PROMPTS)
144 def test_mistral_symbolic_languages(
145 + vllm_runner,
146 model: str,
147 dtype: str,
144 - prompt: str,
148 ) -> None:
146 - prompt = "hi"
147 - msg = {"role": "user", "content": prompt}
148 - llm = LLM(model=model,
149 - dtype=dtype,
150 - max_model_len=8192,
151 - tokenizer_mode="mistral",
152 - config_format="mistral",
153 - load_format="mistral")
154 - outputs = llm.chat([msg], sampling_params=SAMPLING_PARAMS)
155 - assert "�" not in outputs[0].outputs[0].text.strip()
149 + with vllm_runner(model,
150 + dtype=dtype,
151 + max_model_len=8192,
152 + tokenizer_mode="mistral",
153 + config_format="mistral",
154 + load_format="mistral") as vllm_model:
155 + for prompt in SYMBOLIC_LANG_PROMPTS:
156 + msg = {"role": "user", "content": prompt}
157 + outputs = vllm_model.model.chat([msg],
158 + sampling_params=SAMPLING_PARAMS)
159 + assert "�" not in outputs[0].outputs[0].text.strip()
160
161
162 @pytest.mark.parametrize("dtype", ["bfloat16"])
159 -@pytest.mark.parametrize("model", MODELS[1:]) # v1 can't do func calling
163 +@pytest.mark.parametrize("model",
164 + MISTRAL_FORMAT_MODELS) # v1 can't do func calling
165 def test_mistral_function_calling(
166 vllm_runner,
167 model: str,
tests/models/decoder_only/language/test_models.py
+37 -32
@@ -7,25 +7,39 @@ Run `pytest tests/models/test_models.py`.
7 """
8 import pytest
9
10 -from ...utils import check_outputs_equal
10 +from vllm.platforms import current_platform
11 +
12 +from ...utils import check_logprobs_close
13
14 MODELS = [
13 - "facebook/opt-125m",
14 - "gpt2",
15 - "bigcode/tiny_starcoder_py",
16 - "EleutherAI/pythia-70m",
17 - "bigscience/bloom-560m", # Testing alibi slopes.
18 - "microsoft/phi-2",
19 - "stabilityai/stablelm-3b-4e1t",
20 - # "allenai/OLMo-1B", # Broken
21 - "bigcode/starcoder2-3b",
22 - "google/gemma-1.1-2b-it",
15 + "facebook/opt-125m", # opt
16 + "openai-community/gpt2", # gpt2
17 + # "Milos/slovak-gpt-j-405M", # gptj
18 + # "bigcode/tiny_starcoder_py", # gpt_bigcode
19 + # "EleutherAI/pythia-70m", # gpt_neox
20 + "bigscience/bloom-560m", # bloom - testing alibi slopes
21 + "microsoft/phi-2", # phi
22 + # "stabilityai/stablelm-3b-4e1t", # stablelm
23 + # "bigcode/starcoder2-3b", # starcoder2
24 + "google/gemma-1.1-2b-it", # gemma
25 + "Qwen/Qwen2.5-0.5B-Instruct", # qwen2
26 + "meta-llama/Llama-3.2-1B-Instruct", # llama
27 ]
28
29 +if not current_platform.is_cpu():
30 + MODELS += [
31 + # fused_moe which not supported on CPU
32 + "openbmb/MiniCPM3-4B",
33 + ]
34 +
35 +# TODO: remove this after CPU float16 support ready
36 +target_dtype = "float" if current_platform.is_cpu() else "half"
37 +
38
39 @pytest.mark.parametrize("model", MODELS)
27 -@pytest.mark.parametrize("dtype", ["float"])
28 -@pytest.mark.parametrize("max_tokens", [96])
40 +@pytest.mark.parametrize("dtype", [target_dtype])
41 +@pytest.mark.parametrize("max_tokens", [32])
42 +@pytest.mark.parametrize("num_logprobs", [5])
43 def test_models(
44 hf_runner,
45 vllm_runner,
@@ -33,33 +47,24 @@ def test_models(
47 model: str,
48 dtype: str,
49 max_tokens: int,
50 + num_logprobs: int,
51 ) -> None:
37 - # To pass the small model tests, we need full precision.
38 - assert dtype == "float"
52
53 with hf_runner(model, dtype=dtype) as hf_model:
41 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
54 + hf_outputs = hf_model.generate_greedy_logprobs_limit(
55 + example_prompts, max_tokens, num_logprobs)
56
57 with vllm_runner(model, dtype=dtype) as vllm_model:
44 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
58 + vllm_outputs = vllm_model.generate_greedy_logprobs(
59 + example_prompts, max_tokens, num_logprobs)
60 + # This test is for verifying whether the model's extra_repr
61 + # can be printed correctly.
62 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
63 + model_runner.model)
64
46 - check_outputs_equal(
65 + check_logprobs_close(
66 outputs_0_lst=hf_outputs,
67 outputs_1_lst=vllm_outputs,
68 name_0="hf",
69 name_1="vllm",
70 )
52 -
53 -
54 -@pytest.mark.parametrize("model", MODELS)
55 -@pytest.mark.parametrize("dtype", ["float"])
56 -def test_model_print(
57 - vllm_runner,
58 - model: str,
59 - dtype: str,
60 -) -> None:
61 - with vllm_runner(model, dtype=dtype) as vllm_model:
62 - # This test is for verifying whether the model's extra_repr
63 - # can be printed correctly.
64 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
65 - model_runner.model)
tests/models/decoder_only/language/test_qwen.py deleted
-34
@@ -1,34 +0,0 @@
1 -"""Ensure that a text-only Qwen model can be run without throwing an error.
2 -We explicitly test this because Qwen is implemented as a multimodal and
3 -supports a visual encoder for models like Qwen-VL.
4 -"""
5 -from typing import List, Type
6 -
7 -import pytest
8 -
9 -from ....conftest import VllmRunner
10 -
11 -models = [
12 - "Qwen/Qwen-7B-Chat" # Has no visual encoder
13 -]
14 -
15 -
16 -@pytest.mark.parametrize("model", models)
17 -@pytest.mark.parametrize("dtype", ["bfloat16"])
18 -@pytest.mark.parametrize("max_tokens", [32])
19 -@pytest.mark.parametrize("num_logprobs", [5])
20 -def test_text_only_qwen_model_can_be_loaded_and_run(
21 - vllm_runner: Type[VllmRunner],
22 - example_prompts: List[str],
23 - model: str,
24 - *,
25 - dtype: str,
26 - max_tokens: int,
27 - num_logprobs: int,
28 -):
29 - with vllm_runner(model, dtype=dtype) as vllm_model:
30 - vllm_model.generate_greedy_logprobs(
31 - example_prompts,
32 - max_tokens,
33 - num_logprobs=num_logprobs,
34 - )