A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ci][test] add correctness test for cpu offloading (#6549)
youkaichao committed
Jul 18, 2024 at 16:41 UTC
f53b8f0d051a49402e5c4aef40764a7de8d5d50c
4 files changed
+105
-85
.buildkite/test-pipeline.yaml
+1
@@ -46,6 +46,7 @@ steps:
46
commands:
47
- pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
48
- pytest -v -s basic_correctness/test_basic_correctness.py
49
+ - pytest -v -s basic_correctness/test_cpu_offload.py
50
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
51
- VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_chunked_prefill.py
52
- VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
tests/basic_correctness/test_cpu_offload.py
new
+8
@@ -0,0 +1,8 @@
1
+from ..utils import compare_two_settings
2
+
3
+
4
+def test_cpu_offload():
5
+ compare_two_settings("meta-llama/Llama-2-7b-hf", [],
6
+ ["--cpu-offload-gb", "4"])
7
+ compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
8
+ [], ["--cpu-offload-gb", "1"])
tests/distributed/test_pipeline_parallel.py
+2
-85
@@ -1,7 +1,6 @@
1
import pytest
2
-from transformers import AutoTokenizer
2
4
-from ..utils import RemoteOpenAIServer
3
+from ..utils import compare_two_settings
4
5
6
@pytest.mark.parametrize(
@@ -13,7 +12,6 @@ from ..utils import RemoteOpenAIServer
12
(1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
13
])
14
def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
16
- tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
15
16
pp_args = [
17
# use half precision for speed and memory savings in CI environment
@@ -48,85 +46,4 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
46
pp_args.append("--enforce-eager")
47
tp_args.append("--enforce-eager")
48
51
- prompt = "Hello, my name is"
52
- token_ids = tokenizer(prompt)["input_ids"]
53
- results = []
54
- for args in (pp_args, tp_args):
55
- with RemoteOpenAIServer(MODEL_NAME, args) as server:
56
- client = server.get_client()
57
-
58
- # test models list
59
- models = client.models.list()
60
- models = models.data
61
- served_model = models[0]
62
- results.append({
63
- "test": "models_list",
64
- "id": served_model.id,
65
- "root": served_model.root,
66
- })
67
-
68
- # test with text prompt
69
- completion = client.completions.create(model=MODEL_NAME,
70
- prompt=prompt,
71
- max_tokens=5,
72
- temperature=0.0)
73
-
74
- results.append({
75
- "test": "single_completion",
76
- "text": completion.choices[0].text,
77
- "finish_reason": completion.choices[0].finish_reason,
78
- "usage": completion.usage,
79
- })
80
-
81
- # test using token IDs
82
- completion = client.completions.create(
83
- model=MODEL_NAME,
84
- prompt=token_ids,
85
- max_tokens=5,
86
- temperature=0.0,
87
- )
88
-
89
- results.append({
90
- "test": "token_ids",
91
- "text": completion.choices[0].text,
92
- "finish_reason": completion.choices[0].finish_reason,
93
- "usage": completion.usage,
94
- })
95
-
96
- # test simple list
97
- batch = client.completions.create(
98
- model=MODEL_NAME,
99
- prompt=[prompt, prompt],
100
- max_tokens=5,
101
- temperature=0.0,
102
- )
103
-
104
- results.append({
105
- "test": "simple_list",
106
- "text0": batch.choices[0].text,
107
- "text1": batch.choices[1].text,
108
- })
109
-
110
- # test streaming
111
- batch = client.completions.create(
112
- model=MODEL_NAME,
113
- prompt=[prompt, prompt],
114
- max_tokens=5,
115
- temperature=0.0,
116
- stream=True,
117
- )
118
- texts = [""] * 2
119
- for chunk in batch:
120
- assert len(chunk.choices) == 1
121
- choice = chunk.choices[0]
122
- texts[choice.index] += choice.text
123
- results.append({
124
- "test": "streaming",
125
- "texts": texts,
126
- })
127
-
128
- n = len(results) // 2
129
- pp_results = results[:n]
130
- tp_results = results[n:]
131
- for pp, tp in zip(pp_results, tp_results):
132
- assert pp == tp
49
+ compare_two_settings(MODEL_NAME, pp_args, tp_args)
tests/utils.py
+94
@@ -10,6 +10,7 @@ from typing import Any, Dict, List
10
import openai
11
import ray
12
import requests
13
+from transformers import AutoTokenizer
14
15
from vllm.distributed import (ensure_model_parallel_initialized,
16
init_distributed_environment)
@@ -124,6 +125,99 @@ class RemoteOpenAIServer:
125
)
126
127
128
+def compare_two_settings(model: str, arg1: List[str], arg2: List[str]):
129
+ """
130
+ Launch API server with two different sets of arguments and compare the
131
+ results of the API calls. The arguments are after the model name.
132
+ """
133
+
134
+ tokenizer = AutoTokenizer.from_pretrained(model)
135
+
136
+ prompt = "Hello, my name is"
137
+ token_ids = tokenizer(prompt)["input_ids"]
138
+ results = []
139
+ for args in (arg1, arg2):
140
+ with RemoteOpenAIServer(model, args) as server:
141
+ client = server.get_client()
142
+
143
+ # test models list
144
+ models = client.models.list()
145
+ models = models.data
146
+ served_model = models[0]
147
+ results.append({
148
+ "test": "models_list",
149
+ "id": served_model.id,
150
+ "root": served_model.root,
151
+ })
152
+
153
+ # test with text prompt
154
+ completion = client.completions.create(model=model,
155
+ prompt=prompt,
156
+ max_tokens=5,
157
+ temperature=0.0)
158
+
159
+ results.append({
160
+ "test": "single_completion",
161
+ "text": completion.choices[0].text,
162
+ "finish_reason": completion.choices[0].finish_reason,
163
+ "usage": completion.usage,
164
+ })
165
+
166
+ # test using token IDs
167
+ completion = client.completions.create(
168
+ model=model,
169
+ prompt=token_ids,
170
+ max_tokens=5,
171
+ temperature=0.0,
172
+ )
173
+
174
+ results.append({
175
+ "test": "token_ids",
176
+ "text": completion.choices[0].text,
177
+ "finish_reason": completion.choices[0].finish_reason,
178
+ "usage": completion.usage,
179
+ })
180
+
181
+ # test simple list
182
+ batch = client.completions.create(
183
+ model=model,
184
+ prompt=[prompt, prompt],
185
+ max_tokens=5,
186
+ temperature=0.0,
187
+ )
188
+
189
+ results.append({
190
+ "test": "simple_list",
191
+ "text0": batch.choices[0].text,
192
+ "text1": batch.choices[1].text,
193
+ })
194
+
195
+ # test streaming
196
+ batch = client.completions.create(
197
+ model=model,
198
+ prompt=[prompt, prompt],
199
+ max_tokens=5,
200
+ temperature=0.0,
201
+ stream=True,
202
+ )
203
+ texts = [""] * 2
204
+ for chunk in batch:
205
+ assert len(chunk.choices) == 1
206
+ choice = chunk.choices[0]
207
+ texts[choice.index] += choice.text
208
+ results.append({
209
+ "test": "streaming",
210
+ "texts": texts,
211
+ })
212
+
213
+ n = len(results) // 2
214
+ arg1_results = results[:n]
215
+ arg2_results = results[n:]
216
+ for arg1_result, arg2_result in zip(arg1_results, arg2_results):
217
+ assert arg1_result == arg2_result, \
218
+ f"Results for {model=} are not the same with {arg1=} and {arg2=}"
219
+
220
+
221
def init_test_distributed_environment(
222
tp_size: int,
223
pp_size: int,