A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[MISC] Consolidate FP8 kv-cache tests (#8131)
Cody Yu committed
Sep 4, 2024 at 11:53 UTC
2ad2e5608eeede10683412bbbfaf30b3a68019dc
4 files changed
+94
-233
.buildkite/run-cpu-test.sh
+6
-1
@@ -23,7 +23,12 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23
# Run basic model test
24
docker exec cpu-test bash -c "
25
pip install pytest matplotlib einops transformers_stream_generator
26
- pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_oot_registration.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
26
+ pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py \
27
+ --ignore=tests/models/test_oot_registration.py \
28
+ --ignore=tests/models/test_registry.py \
29
+ --ignore=tests/models/test_fp8.py \
30
+ --ignore=tests/models/test_jamba.py \
31
+ --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
32
33
# online inference
34
docker exec cpu-test bash -c "
tests/basic_correctness/test_chunked_prefill.py
+7
-36
@@ -16,18 +16,6 @@ MODELS = [
16
"facebook/opt-125m",
17
"meta-llama/Llama-2-7b-hf",
18
]
19
-E5M2_KV_MODELS = [
20
- "facebook/opt-125m",
21
- "meta-llama/Llama-2-7b-chat-hf",
22
-]
23
-E4M3_KV_MODELS = [
24
- "meta-llama/Llama-2-7b-chat-hf", "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V",
25
- "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme"
26
-]
27
-KV_CACHE_QUANTIZATION_PATHS = {
28
- "meta-llama/Llama-2-7b-chat-hf":
29
- "./tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json"
30
-}
19
20
21
@pytest.mark.parametrize("model", MODELS)
@@ -78,10 +66,10 @@ def test_models(
66
)
67
68
81
-@pytest.mark.parametrize("kv_cache_dtype,model",
82
- [("fp8_e5m2", m)
83
- for m in E5M2_KV_MODELS] + [("fp8_e4m3", m)
84
- for m in E4M3_KV_MODELS])
69
+@pytest.mark.parametrize(
70
+ "kv_cache_dtype,model",
71
+ [("fp8_e4m3",
72
+ "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme")])
73
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
74
@pytest.mark.parametrize("max_tokens", [4])
75
@pytest.mark.parametrize("chunked_prefill_token_size", [4, 16])
@@ -104,30 +92,15 @@ def test_models_with_fp8_kv_cache(
92
disable_async_output_proc: bool,
93
) -> None:
94
"""
107
- Only checks log probs match between chunked-prefill and
108
- non-chunked-prefill version of vLLM model runner.
109
-
110
- This test is used when there is discrepancy in kernels
111
- / numerics (e.g. when using lower-precision types like FP8).
95
+ Check output logprobs match between no_chunked_prefill and chunked_prefill
96
+ with fp8 kv cache. General fp8 kv-cache tests are covered in test_fp8.py,
97
+ so here we only check chunked prefill.
98
"""
99
NUM_LOG_PROBS = 8
100
115
- if model == "facebook/opt-125m":
116
- pytest.skip(
117
- "#7378: CUDA illegal memory access (undiagnosed) facebook/opt-125m"
118
- )
119
- if ((model, kv_cache_dtype, chunked_prefill_token_size) == (
120
- "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V", "fp8_e4m3", 4)):
121
- pytest.skip("flakey test, see: #7874 #8051")
122
-
101
max_num_seqs = chunked_prefill_token_size
102
max_num_batched_tokens = chunked_prefill_token_size
103
126
- extra_kwargs = {}
127
- if model in KV_CACHE_QUANTIZATION_PATHS:
128
- extra_kwargs["quantization_param_path"] = KV_CACHE_QUANTIZATION_PATHS[
129
- model]
130
-
104
with vllm_runner(
105
model,
106
tensor_parallel_size=tensor_parallel_size,
@@ -135,7 +108,6 @@ def test_models_with_fp8_kv_cache(
108
max_num_seqs=max_num_seqs,
109
kv_cache_dtype=kv_cache_dtype,
110
disable_async_output_proc=disable_async_output_proc,
138
- **extra_kwargs,
111
) as vllm_model:
112
no_chunked_prefill_outputs = vllm_model.generate_greedy_logprobs(
113
example_prompts, max_tokens, NUM_LOG_PROBS)
@@ -149,7 +121,6 @@ def test_models_with_fp8_kv_cache(
121
max_num_seqs=max_num_seqs,
122
kv_cache_dtype=kv_cache_dtype,
123
disable_async_output_proc=disable_async_output_proc,
152
- **extra_kwargs,
124
) as vllm_model:
125
chunked_prefill_outputs = vllm_model.generate_greedy_logprobs(
126
example_prompts, max_tokens, NUM_LOG_PROBS)
tests/models/test_fp8.py
+81
-100
@@ -3,116 +3,97 @@
3
Note: these tests will only pass on L4 GPU.
4
"""
5
import os
6
-from typing import List
6
+from typing import Optional
7
8
import pytest
9
-import torch
10
-from transformers import AutoTokenizer
9
10
+from tests.kernels.utils import override_backend_env_variable
11
from tests.quantization.utils import is_quant_method_supported
13
-from vllm import LLM, SamplingParams
12
15
-os.environ["TOKENIZERS_PARALLELISM"] = "true"
16
-
17
-MAX_MODEL_LEN = 1024
18
-
19
-MODELS = [
20
- "nm-testing/Meta-Llama-3-8B-Instruct-FP8-KV",
21
- "meta-llama/Meta-Llama-3-8B-Instruct",
22
-]
13
+from ..models.utils import check_logprobs_close
14
24
-EXPECTED_STRS_MAP = {
25
- "nm-testing/Meta-Llama-3-8B-Instruct-FP8-KV": {
26
- "auto": [
27
- 'LLaMA is a high-throughput and memory-efficient inference and serving engine for Large Language Models (',
28
- 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
29
- 'Artificial intelligence (AI) and human intelligence (HI) process information in distinct ways, with both',
30
- 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
31
- 'Zeta-5, a highly advanced robot designed for menial labor, whirred and beep',
32
- 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
33
- 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
34
- 'Here are the translations:\n\n**Japanese:** (Haya aki no tori, nemuri no'
35
- ],
36
- "fp8": [
37
- 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
38
- 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
39
- 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
40
- 'A neural network is a complex system made up of several basic components that work together to enable it to',
41
- 'Zeta-5, a highly advanced robot designed for menial labor, had never experienced anything like',
42
- 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. Here',
43
- 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
44
- 'Here are the translations:\n\n**Japanese:** (Haya kotori wa mushi o tsuk'
45
- ]
46
- },
47
- "meta-llama/Meta-Llama-3-8B-Instruct": {
48
- "auto": [
49
- 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
50
- 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
51
- 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
52
- 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
53
- 'In the vast, sterile laboratory, Robot 3456-Alpha, or "Alpha" for short',
54
- 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
55
- 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
56
- 'Here are the translations:\n\n**Japanese:** (Haya aki wa mushi o tsukamu'
57
- ],
58
- "fp8": [
59
- 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
60
- 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
61
- 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
62
- 'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
63
- 'In the year 2154, robotics engineer Dr. Rachel Kim had spent years perfecting her latest',
64
- 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
65
- 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
66
- 'Here are the translations:\n\n**Japanese:** (Haya tori, mushi o tsukamu'
67
- ]
68
- },
69
-}
15
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
16
17
72
-# This test compares against golden strings for exact match since
73
-# there is no baseline implementation to compare against
74
-# and is unstable w.r.t specifics of the fp8 implementation or
75
-# the hardware being run on.
76
-# Disabled to prevent it from breaking the build
77
-@pytest.mark.skip(
78
- reason=
79
- "Prevent unstable test based on golden strings from breaking the build.")
18
@pytest.mark.skipif(not is_quant_method_supported("fp8"),
19
reason="fp8 is not supported on this GPU type.")
82
-@pytest.mark.parametrize("model_name", MODELS)
83
-@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
84
-def test_models(example_prompts, model_name, kv_cache_dtype) -> None:
85
- model = LLM(model=model_name,
86
- max_model_len=MAX_MODEL_LEN,
87
- trust_remote_code=True,
88
- enforce_eager=True,
89
- quantization="fp8",
90
- kv_cache_dtype=kv_cache_dtype)
20
+@pytest.mark.parametrize(
21
+ "kv_cache_dtype,base_model,test_model,scale_path",
22
+ [
23
+ # Test FP8 checkpoint w. fp8_e4m3 kv-cache scaling factors.
24
+ ("fp8_e4m3", "meta-llama/Meta-Llama-3-8B-Instruct",
25
+ "nm-testing/Meta-Llama-3-8B-Instruct-FP8-KV", None),
26
+ # Test FP16 checkpoint w. fp8_e5m2 kv-cache.
27
+ ("fp8_e5m2", "meta-llama/Meta-Llama-3-8B-Instruct",
28
+ "meta-llama/Meta-Llama-3-8B-Instruct", None),
29
+ # Test FP16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
30
+ ("fp8_e4m3", "meta-llama/Llama-2-7b-chat-hf",
31
+ "meta-llama/Llama-2-7b-chat-hf",
32
+ "./tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json")
33
+ ])
34
+# Due to low-precision numerical divergence, we only test logprob of 4 tokens
35
+@pytest.mark.parametrize("max_tokens", [4])
36
+@pytest.mark.parametrize("enforce_eager", [False, True])
37
+@pytest.mark.parametrize("backend", ["FLASH_ATTN", "XFORMERS", "FLASHINFER"])
38
+# NOTE: Increasing this in this suite will fail CI because we currently cannot
39
+# reset distributed env properly. Use a value > 1 just when you test.
40
+@pytest.mark.parametrize("tensor_parallel_size", [1])
41
+# Due to low-precision numerical divergence, this test is too sensitive for
42
+# the async postprocessor
43
+@pytest.mark.parametrize("disable_async_output_proc", [True])
44
+def test_models(
45
+ vllm_runner,
46
+ example_prompts,
47
+ kv_cache_dtype: str,
48
+ base_model: str,
49
+ test_model: str,
50
+ scale_path: Optional[str],
51
+ max_tokens: int,
52
+ enforce_eager: bool,
53
+ backend: str,
54
+ tensor_parallel_size: int,
55
+ disable_async_output_proc: bool,
56
+ monkeypatch,
57
+) -> None:
58
+ """
59
+ Only checks log probs match to cover the discrepancy in
60
+ numerical sensitive kernels.
61
+ """
62
+ override_backend_env_variable(monkeypatch, backend)
63
+
64
+ MAX_MODEL_LEN = 1024
65
+ NUM_LOG_PROBS = 8
66
+
67
+ with vllm_runner(
68
+ base_model,
69
+ max_model_len=MAX_MODEL_LEN,
70
+ tensor_parallel_size=tensor_parallel_size,
71
+ enforce_eager=enforce_eager,
72
+ kv_cache_dtype="auto",
73
+ disable_async_output_proc=disable_async_output_proc,
74
+ ) as vllm_model:
75
+ baseline_outputs = vllm_model.generate_greedy_logprobs(
76
+ example_prompts, max_tokens, NUM_LOG_PROBS)
77
92
- tokenizer = AutoTokenizer.from_pretrained(model_name)
93
- formatted_prompts = [
94
- tokenizer.apply_chat_template([{
95
- "role": "user",
96
- "content": prompt
97
- }],
98
- tokenize=False,
99
- add_generation_prompt=True)
100
- for prompt in example_prompts
101
- ]
78
+ extra_kwargs = {}
79
+ if scale_path is not None:
80
+ extra_kwargs["quantization_param_path"] = scale_path
81
103
- params = SamplingParams(max_tokens=20, temperature=0)
104
- generations: List[str] = []
105
- # Note: these need to be run 1 at a time due to numerical precision,
106
- # since the expected strs were generated this way.
107
- for prompt in formatted_prompts:
108
- outputs = model.generate(prompt, params)
109
- generations.append(outputs[0].outputs[0].text)
110
- del model
82
+ with vllm_runner(
83
+ test_model,
84
+ max_model_len=MAX_MODEL_LEN,
85
+ tensor_parallel_size=tensor_parallel_size,
86
+ enforce_eager=enforce_eager,
87
+ kv_cache_dtype=kv_cache_dtype,
88
+ disable_async_output_proc=disable_async_output_proc,
89
+ **extra_kwargs,
90
+ ) as vllm_model:
91
+ test_outputs = vllm_model.generate_greedy_logprobs(
92
+ example_prompts, max_tokens, NUM_LOG_PROBS)
93
112
- print(model_name, kv_cache_dtype, generations)
113
- expected_strs = EXPECTED_STRS_MAP[model_name][kv_cache_dtype]
114
- for i in range(len(example_prompts)):
115
- generated_str = generations[i]
116
- expected_str = expected_strs[i]
117
- assert expected_str == generated_str, (
118
- f"Test{i}:\nExpected: {expected_str!r}\nvLLM: {generated_str!r}")
94
+ check_logprobs_close(
95
+ outputs_0_lst=baseline_outputs,
96
+ outputs_1_lst=test_outputs,
97
+ name_0="fp16_kv_cache",
98
+ name_1="fp8_kv_cache",
99
+ )
tests/models/test_fp8kv_flashinfer.py
deleted
-96
@@ -1,96 +0,0 @@
1
-# flake8: noqa
2
-"""Tests fp8 models against ground truth generation
3
-This verifies the flashinfer backend with fp8
4
-quantization and fp8 KV Cache without scaling
5
-factors Note: these tests will only pass on H100 GPU.
6
-"""
7
-import os
8
-from typing import List
9
-
10
-import pytest
11
-from transformers import AutoTokenizer
12
-
13
-from tests.quantization.utils import is_quant_method_supported
14
-from vllm import LLM, SamplingParams
15
-
16
-os.environ["TOKENIZERS_PARALLELISM"] = "true"
17
-
18
-MAX_MODEL_LEN = 1024
19
-
20
-MODELS = [
21
- "nm-testing/Meta-Llama-3-8B-Instruct-FP8",
22
-]
23
-
24
-EXPECTED_STRS_MAP = {
25
- "nm-testing/Meta-Llama-3-8B-Instruct-FP8": {
26
- "auto": [
27
- 'LLaMA is a high-throughput and memory-efficient inference and serving engine for Large Language Models (',
28
- 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
29
- 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
30
- 'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
31
- 'In the sterile, metallic halls of the robotics lab, a peculiar phenomenon occurred. Zeta-5',
32
- 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
33
- 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
34
- 'Here are the translations:\n\n**Japanese:** (Haya aki no tori, mushi o',
35
- ],
36
- "fp8": [
37
- 'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
38
- 'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
39
- 'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
40
- 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
41
- 'Zeta-5, a highly advanced robot designed for menial labor, whirred and beep',
42
- 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. Here',
43
- 'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
44
- 'Here are the translations:\n\n**Japanese:** (Haya aki no tori, guri o',
45
- ]
46
- }
47
-}
48
-
49
-
50
-# This test compares against golden strings for exact match since
51
-# there is no baseline implementation to compare against
52
-# and is unstable w.r.t specifics of the fp8 implementation or
53
-# the hardware being run on.
54
-# No assert to prevent it from breaking the build
55
-@pytest.mark.skipif(not is_quant_method_supported("fp8"),
56
- reason="fp8 is not supported on this GPU type.")
57
-@pytest.mark.parametrize("model_name", MODELS)
58
-@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
59
-@pytest.mark.parametrize("backend", ["XFORMERS", "FLASHINFER"])
60
-def test_models(example_prompts, model_name, kv_cache_dtype, backend) -> None:
61
- # Note that the golden strings may not work for FLASHINFER Backend.
62
- # The intention is to test the path
63
- os.environ["VLLM_ATTENTION_BACKEND"] = backend
64
- model = LLM(model=model_name,
65
- max_model_len=MAX_MODEL_LEN,
66
- trust_remote_code=True,
67
- quantization="fp8",
68
- kv_cache_dtype=kv_cache_dtype)
69
-
70
- tokenizer = AutoTokenizer.from_pretrained(model_name)
71
- formatted_prompts = [
72
- tokenizer.apply_chat_template([{
73
- "role": "user",
74
- "content": prompt
75
- }],
76
- tokenize=False,
77
- add_generation_prompt=True)
78
- for prompt in example_prompts
79
- ]
80
-
81
- params = SamplingParams(max_tokens=20, temperature=0)
82
- generations: List[str] = []
83
- # Note: these need to be run 1 at a time due to numerical precision,
84
- # since the expected strs were generated this way.
85
- for prompt in formatted_prompts:
86
- outputs = model.generate(prompt, params)
87
- generations.append(outputs[0].outputs[0].text)
88
- del model
89
-
90
- print(f"Testing: {model_name} with kv_cache_dtype: {kv_cache_dtype}")
91
- expected_strs = EXPECTED_STRS_MAP[model_name][kv_cache_dtype]
92
- for i in range(len(example_prompts)):
93
- generated_str = generations[i]
94
- expected_str = expected_strs[i]
95
- print(f"generated_str\n: {generated_str}")
96
- print(f"expected_str\n: {expected_str}")