A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Support Batch Completion in Server (#2529)
Simon Mo committed
Jan 24, 2024 at 17:11 UTC
3a7dd7e367277c47472912e84375fa912df07328
2 files changed
+215
-105
tests/entrypoints/test_openai_server.py
+53
-2
@@ -1,5 +1,6 @@
1
-import time
1
+import os
2
import subprocess
3
+import time
4
5
import sys
6
import pytest
@@ -17,8 +18,11 @@ pytestmark = pytest.mark.asyncio
18
class ServerRunner:
19
20
def __init__(self, args):
21
+ env = os.environ.copy()
22
+ env["PYTHONUNBUFFERED"] = "1"
23
self.proc = subprocess.Popen(
24
["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
25
+ env=env,
26
stdout=sys.stdout,
27
stderr=sys.stderr,
28
)
@@ -58,7 +62,8 @@ def server():
62
"--dtype",
63
"bfloat16", # use half precision for speed and memory savings in CI environment
64
"--max-model-len",
61
- "8192"
65
+ "8192",
66
+ "--enforce-eager",
67
])
68
ray.get(server_runner.ready.remote())
69
yield server_runner
@@ -199,5 +204,51 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI):
204
assert "".join(chunks) == output
205
206
207
+async def test_batch_completions(server, client: openai.AsyncOpenAI):
208
+ # test simple list
209
+ batch = await client.completions.create(
210
+ model=MODEL_NAME,
211
+ prompt=["Hello, my name is", "Hello, my name is"],
212
+ max_tokens=5,
213
+ temperature=0.0,
214
+ )
215
+ assert len(batch.choices) == 2
216
+ assert batch.choices[0].text == batch.choices[1].text
217
+
218
+ # test n = 2
219
+ batch = await client.completions.create(
220
+ model=MODEL_NAME,
221
+ prompt=["Hello, my name is", "Hello, my name is"],
222
+ n=2,
223
+ max_tokens=5,
224
+ temperature=0.0,
225
+ extra_body=dict(
226
+ # NOTE: this has to be true for n > 1 in vLLM, but not necessary for official client.
227
+ use_beam_search=True),
228
+ )
229
+ assert len(batch.choices) == 4
230
+ assert batch.choices[0].text != batch.choices[
231
+ 1].text, "beam search should be different"
232
+ assert batch.choices[0].text == batch.choices[
233
+ 2].text, "two copies of the same prompt should be the same"
234
+ assert batch.choices[1].text == batch.choices[
235
+ 3].text, "two copies of the same prompt should be the same"
236
+
237
+ # test streaming
238
+ batch = await client.completions.create(
239
+ model=MODEL_NAME,
240
+ prompt=["Hello, my name is", "Hello, my name is"],
241
+ max_tokens=5,
242
+ temperature=0.0,
243
+ stream=True,
244
+ )
245
+ texts = [""] * 2
246
+ async for chunk in batch:
247
+ assert len(chunk.choices) == 1
248
+ choice = chunk.choices[0]
249
+ texts[choice.index] += choice.text
250
+ assert texts[0] == texts[1]
251
+
252
+
253
if __name__ == "__main__":
254
pytest.main([__file__])
vllm/entrypoints/openai/serving_completion.py
+162
-103
@@ -1,6 +1,7 @@
1
+import asyncio
2
import time
3
from fastapi import Request
3
-from typing import AsyncGenerator, AsyncIterator
4
+from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional
5
from vllm.logger import init_logger
6
from vllm.utils import random_uuid
7
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -18,48 +19,68 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing
19
20
logger = init_logger(__name__)
21
22
+TypeTokenIDs = list[int]
23
+TypeTopLogProbs = List[Optional[dict[int, float]]]
24
+TypeCreateLogProbsFn = Callable[
25
+ [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], LogProbs]
26
+
27
28
async def completion_stream_generator(
23
- request: CompletionRequest,
24
- result_generator: AsyncIterator[RequestOutput],
25
- echo_without_generation, create_logprobs_fn, request_id, created_time,
26
- model_name) -> AsyncGenerator[str, None]:
27
- previous_texts = [""] * request.n
28
- previous_num_tokens = [0] * request.n
29
- has_echoed = [False] * request.n
30
-
31
- async for res in result_generator:
32
- # TODO: handle client disconnect for streaming
29
+ request: CompletionRequest,
30
+ raw_request: Request,
31
+ on_abort,
32
+ result_generator: AsyncIterator[tuple[int, RequestOutput]],
33
+ create_logprobs_fn: TypeCreateLogProbsFn,
34
+ request_id: str,
35
+ created_time: int,
36
+ model_name: str,
37
+ num_prompts: int,
38
+) -> AsyncGenerator[str, None]:
39
+ previous_texts = [""] * request.n * num_prompts
40
+ previous_num_tokens = [0] * request.n * num_prompts
41
+ has_echoed = [False] * request.n * num_prompts
42
+
43
+ async for prompt_idx, res in result_generator:
44
+
45
+ # Abort the request if the client disconnects.
46
+ if await raw_request.is_disconnected():
47
+ await on_abort(f"{request_id}-{prompt_idx}")
48
+ raise StopAsyncIteration()
49
+
50
for output in res.outputs:
34
- i = output.index
35
- delta_text = output.text[len(previous_texts[i]):]
36
- token_ids = output.token_ids[previous_num_tokens[i]:]
37
- if request.logprobs is not None:
38
- top_logprobs = output.logprobs[previous_num_tokens[i]:]
39
- else:
40
- top_logprobs = None
41
- offsets = len(previous_texts[i])
42
- if request.echo and not has_echoed[i]:
43
- if not echo_without_generation:
44
- delta_text = res.prompt + delta_text
45
- token_ids = res.prompt_token_ids + token_ids
46
- if top_logprobs:
47
- top_logprobs = res.prompt_logprobs + top_logprobs
48
- else: # only just return the prompt
49
- delta_text = res.prompt
50
- token_ids = res.prompt_token_ids
51
- if top_logprobs:
52
- top_logprobs = res.prompt_logprobs
51
+ i = output.index + prompt_idx * request.n
52
+ # TODO(simon): optimize the performance by avoiding full text O(n^2) sending.
53
+
54
+ if request.echo and request.max_tokens == 0:
55
+ # only return the prompt
56
+ delta_text = res.prompt
57
+ delta_token_ids = res.prompt_token_ids
58
+ top_logprobs = res.prompt_logprobs
59
+ has_echoed[i] = True
60
+ elif request.echo and request.max_tokens > 0 and not has_echoed[i]:
61
+ # echo the prompt and first token
62
+ delta_text = res.prompt + output.text
63
+ delta_token_ids = res.prompt_token_ids + output.token_ids
64
+ top_logprobs = res.prompt_logprobs + (output.logprobs or [])
65
has_echoed[i] = True
66
+ else:
67
+ # return just the delta
68
+ delta_text = output.text[len(previous_texts[i]):]
69
+ delta_token_ids = output.token_ids[previous_num_tokens[i]:]
70
+ top_logprobs = output.logprobs[
71
+ previous_num_tokens[i]:] if output.logprobs else None
72
+
73
if request.logprobs is not None:
74
+ assert top_logprobs is not None, "top_logprobs must be provided when logprobs is requested"
75
logprobs = create_logprobs_fn(
56
- token_ids=token_ids,
76
+ token_ids=delta_token_ids,
77
top_logprobs=top_logprobs,
78
num_output_top_logprobs=request.logprobs,
59
- initial_text_offset=offsets,
79
+ initial_text_offset=len(previous_texts[i]),
80
)
81
else:
82
logprobs = None
83
+
84
previous_texts[i] = output.text
85
previous_num_tokens[i] = len(output.token_ids)
86
finish_reason = output.finish_reason
@@ -77,7 +98,7 @@ async def completion_stream_generator(
98
]).model_dump_json(exclude_unset=True)
99
yield f"data: {response_json}\n\n"
100
80
- if output.finish_reason is not None:
101
+ if output.finish_reason is not None: # return final usage
102
logprobs = LogProbs() if request.logprobs is not None else None
103
prompt_tokens = len(res.prompt_token_ids)
104
completion_tokens = len(output.token_ids)
@@ -129,51 +150,58 @@ def parse_prompt_format(prompt) -> tuple[bool, list]:
150
return prompt_is_tokens, prompts
151
152
132
-def request_output_to_completion_response(final_res: RequestOutput, request,
133
- echo_without_generation,
134
- create_logprobs_fn, request_id,
135
- created_time,
136
- model_name) -> CompletionResponse:
137
- assert final_res is not None
153
+def request_output_to_completion_response(
154
+ final_res_batch: list[RequestOutput],
155
+ request: CompletionRequest,
156
+ create_logprobs_fn: TypeCreateLogProbsFn,
157
+ request_id: str,
158
+ created_time: int,
159
+ model_name: str,
160
+) -> CompletionResponse:
161
choices = []
139
- prompt_token_ids = final_res.prompt_token_ids
140
- prompt_logprobs = final_res.prompt_logprobs
141
- prompt_text = final_res.prompt
142
- for output in final_res.outputs:
143
- if request.logprobs is not None:
144
- if not echo_without_generation:
162
+ num_prompt_tokens = 0
163
+ num_generated_tokens = 0
164
+ for final_res in final_res_batch:
165
+ assert final_res is not None
166
+ prompt_token_ids = final_res.prompt_token_ids
167
+ prompt_logprobs = final_res.prompt_logprobs
168
+ prompt_text = final_res.prompt
169
+
170
+ for output in final_res.outputs:
171
+ if request.echo and request.max_tokens == 0:
172
+ token_ids = prompt_token_ids
173
+ top_logprobs = prompt_logprobs
174
+ output_text = prompt_text
175
+ elif request.echo and request.max_tokens > 0:
176
+ token_ids = prompt_token_ids + output.token_ids
177
+ top_logprobs = prompt_logprobs + output.logprobs
178
+ output_text = prompt_text + output.text
179
+ else:
180
token_ids = output.token_ids
181
top_logprobs = output.logprobs
147
- if request.echo:
148
- token_ids = prompt_token_ids + token_ids
149
- top_logprobs = prompt_logprobs + top_logprobs
182
+ output_text = output.text
183
+
184
+ if request.logprobs is not None:
185
+ logprobs = create_logprobs_fn(
186
+ token_ids=token_ids,
187
+ top_logprobs=top_logprobs,
188
+ num_output_top_logprobs=request.logprobs,
189
+ )
190
else:
151
- token_ids = prompt_token_ids
152
- top_logprobs = prompt_logprobs
153
- logprobs = create_logprobs_fn(
154
- token_ids=token_ids,
155
- top_logprobs=top_logprobs,
156
- num_output_top_logprobs=request.logprobs,
191
+ logprobs = None
192
+
193
+ choice_data = CompletionResponseChoice(
194
+ index=len(choices),
195
+ text=output_text,
196
+ logprobs=logprobs,
197
+ finish_reason=output.finish_reason,
198
)
158
- else:
159
- logprobs = None
160
- if not echo_without_generation:
161
- output_text = output.text
162
- if request.echo:
163
- output_text = prompt_text + output_text
164
- else:
165
- output_text = prompt_text
166
- choice_data = CompletionResponseChoice(
167
- index=output.index,
168
- text=output_text,
169
- logprobs=logprobs,
170
- finish_reason=output.finish_reason,
171
- )
172
- choices.append(choice_data)
173
-
174
- num_prompt_tokens = len(final_res.prompt_token_ids)
175
- num_generated_tokens = sum(
176
- len(output.token_ids) for output in final_res.outputs)
199
+ choices.append(choice_data)
200
+
201
+ num_prompt_tokens += len(prompt_token_ids)
202
+ num_generated_tokens += sum(
203
+ len(output.token_ids) for output in final_res.outputs)
204
+
205
usage = UsageInfo(
206
prompt_tokens=num_prompt_tokens,
207
completion_tokens=num_generated_tokens,
@@ -189,6 +217,36 @@ def request_output_to_completion_response(final_res: RequestOutput, request,
217
)
218
219
220
+def merge_async_iterators(*iterators):
221
+ """Merge multiple asynchronous iterators into a single iterator.
222
+
223
+ This method handle the case where some iterators finish before others.
224
+ When it yields, it yields a tuple (i, item) where i is the index of the
225
+ iterator that yields the item.
226
+ """
227
+ queue = asyncio.Queue()
228
+
229
+ finished = [False] * len(iterators)
230
+
231
+ async def producer(i, iterator):
232
+ async for item in iterator:
233
+ await queue.put((i, item))
234
+ finished[i] = True
235
+
236
+ _tasks = [
237
+ asyncio.create_task(producer(i, iterator))
238
+ for i, iterator in enumerate(iterators)
239
+ ]
240
+
241
+ async def consumer():
242
+ while not all(finished) or not queue.empty():
243
+ item = await queue.get()
244
+ yield item
245
+ await asyncio.gather(*_tasks)
246
+
247
+ return consumer()
248
+
249
+
250
class OpenAIServingCompletion(OpenAIServing):
251
252
def __init__(self, engine: AsyncLLMEngine, served_model: str):
@@ -210,9 +268,6 @@ class OpenAIServingCompletion(OpenAIServing):
268
if error_check_ret is not None:
269
return error_check_ret
270
213
- # OpenAI API supports echoing the prompt when max_tokens is 0.
214
- echo_without_generation = request.echo and request.max_tokens == 0
215
-
271
# Return error for unsupported features.
272
if request.suffix is not None:
273
return self.create_error_response(
@@ -226,30 +281,30 @@ class OpenAIServingCompletion(OpenAIServing):
281
created_time = int(time.monotonic())
282
283
# Schedule the request and get the result generator.
284
+ generators = []
285
try:
286
sampling_params = request.to_sampling_params()
231
-
287
prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
288
234
- if len(prompts) > 1:
235
- raise ValueError(
236
- "Batching in completion API is not supported.")
237
- prompt = prompts[0]
238
-
239
- if prompt_is_tokens:
240
- input_ids = self._validate_prompt_and_tokenize(
241
- request, prompt_ids=prompt)
242
- else:
243
- input_ids = self._validate_prompt_and_tokenize(request,
244
- prompt=prompt)
245
-
246
- result_generator = self.engine.generate(None,
247
- sampling_params,
248
- request_id,
249
- prompt_token_ids=input_ids)
289
+ for i, prompt in enumerate(prompts):
290
+ if prompt_is_tokens:
291
+ input_ids = self._validate_prompt_and_tokenize(
292
+ request, prompt_ids=prompt)
293
+ else:
294
+ input_ids = self._validate_prompt_and_tokenize(
295
+ request, prompt=prompt)
296
+
297
+ generators.append(
298
+ self.engine.generate(None,
299
+ sampling_params,
300
+ f"{request_id}-{i}",
301
+ prompt_token_ids=input_ids))
302
except ValueError as e:
303
return self.create_error_response(str(e))
304
305
+ result_generator: AsyncIterator[tuple[
306
+ int, RequestOutput]] = merge_async_iterators(*generators)
307
+
308
# Similar to the OpenAI API, when n != best_of, we do not stream the
309
# results. In addition, we do not stream the results when use beam search.
310
stream = (request.stream
@@ -258,23 +313,27 @@ class OpenAIServingCompletion(OpenAIServing):
313
314
# Streaming response
315
if stream:
261
- return completion_stream_generator(request, result_generator,
262
- echo_without_generation,
316
+ return completion_stream_generator(request,
317
+ raw_request,
318
+ self.engine.abort,
319
+ result_generator,
320
self._create_logprobs,
264
- request_id, created_time,
265
- model_name)
321
+ request_id,
322
+ created_time,
323
+ model_name,
324
+ num_prompts=len(prompts))
325
326
# Non-streaming response
268
- final_res: RequestOutput = None
269
- async for res in result_generator:
327
+ final_res_batch: RequestOutput = [None] * len(prompts)
328
+ async for i, res in result_generator:
329
if await raw_request.is_disconnected():
330
# Abort the request if the client disconnects.
272
- await self.engine.abort(request_id)
331
+ await self.engine.abort(f"{request_id}-{i}")
332
return self.create_error_response("Client disconnected")
274
- final_res = res
333
+ final_res_batch[i] = res
334
response = request_output_to_completion_response(
276
- final_res, request, echo_without_generation, self._create_logprobs,
277
- request_id, created_time, model_name)
335
+ final_res_batch, request, self._create_logprobs, request_id,
336
+ created_time, model_name)
337
338
# When user requests streaming but we don't stream, we still need to
339
# return a streaming response with a single event.