@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3a7dd7e367
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Support Batch Completion in Server (#2529)

Simon Mo committed Jan 24, 2024 at 17:11 UTC 3a7dd7e367277c47472912e84375fa912df07328
2 files changed +215 -105
tests/entrypoints/test_openai_server.py
+53 -2
@@ -1,5 +1,6 @@
1 -import time
1 +import os
2 import subprocess
3 +import time
4
5 import sys
6 import pytest
@@ -17,8 +18,11 @@ pytestmark = pytest.mark.asyncio
18 class ServerRunner:
19
20 def __init__(self, args):
21 + env = os.environ.copy()
22 + env["PYTHONUNBUFFERED"] = "1"
23 self.proc = subprocess.Popen(
24 ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
25 + env=env,
26 stdout=sys.stdout,
27 stderr=sys.stderr,
28 )
@@ -58,7 +62,8 @@ def server():
62 "--dtype",
63 "bfloat16", # use half precision for speed and memory savings in CI environment
64 "--max-model-len",
61 - "8192"
65 + "8192",
66 + "--enforce-eager",
67 ])
68 ray.get(server_runner.ready.remote())
69 yield server_runner
@@ -199,5 +204,51 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI):
204 assert "".join(chunks) == output
205
206
207 +async def test_batch_completions(server, client: openai.AsyncOpenAI):
208 + # test simple list
209 + batch = await client.completions.create(
210 + model=MODEL_NAME,
211 + prompt=["Hello, my name is", "Hello, my name is"],
212 + max_tokens=5,
213 + temperature=0.0,
214 + )
215 + assert len(batch.choices) == 2
216 + assert batch.choices[0].text == batch.choices[1].text
217 +
218 + # test n = 2
219 + batch = await client.completions.create(
220 + model=MODEL_NAME,
221 + prompt=["Hello, my name is", "Hello, my name is"],
222 + n=2,
223 + max_tokens=5,
224 + temperature=0.0,
225 + extra_body=dict(
226 + # NOTE: this has to be true for n > 1 in vLLM, but not necessary for official client.
227 + use_beam_search=True),
228 + )
229 + assert len(batch.choices) == 4
230 + assert batch.choices[0].text != batch.choices[
231 + 1].text, "beam search should be different"
232 + assert batch.choices[0].text == batch.choices[
233 + 2].text, "two copies of the same prompt should be the same"
234 + assert batch.choices[1].text == batch.choices[
235 + 3].text, "two copies of the same prompt should be the same"
236 +
237 + # test streaming
238 + batch = await client.completions.create(
239 + model=MODEL_NAME,
240 + prompt=["Hello, my name is", "Hello, my name is"],
241 + max_tokens=5,
242 + temperature=0.0,
243 + stream=True,
244 + )
245 + texts = [""] * 2
246 + async for chunk in batch:
247 + assert len(chunk.choices) == 1
248 + choice = chunk.choices[0]
249 + texts[choice.index] += choice.text
250 + assert texts[0] == texts[1]
251 +
252 +
253 if __name__ == "__main__":
254 pytest.main([__file__])
vllm/entrypoints/openai/serving_completion.py
+162 -103
@@ -1,6 +1,7 @@
1 +import asyncio
2 import time
3 from fastapi import Request
3 -from typing import AsyncGenerator, AsyncIterator
4 +from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional
5 from vllm.logger import init_logger
6 from vllm.utils import random_uuid
7 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -18,48 +19,68 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing
19
20 logger = init_logger(__name__)
21
22 +TypeTokenIDs = list[int]
23 +TypeTopLogProbs = List[Optional[dict[int, float]]]
24 +TypeCreateLogProbsFn = Callable[
25 + [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], LogProbs]
26 +
27
28 async def completion_stream_generator(
23 - request: CompletionRequest,
24 - result_generator: AsyncIterator[RequestOutput],
25 - echo_without_generation, create_logprobs_fn, request_id, created_time,
26 - model_name) -> AsyncGenerator[str, None]:
27 - previous_texts = [""] * request.n
28 - previous_num_tokens = [0] * request.n
29 - has_echoed = [False] * request.n
30 -
31 - async for res in result_generator:
32 - # TODO: handle client disconnect for streaming
29 + request: CompletionRequest,
30 + raw_request: Request,
31 + on_abort,
32 + result_generator: AsyncIterator[tuple[int, RequestOutput]],
33 + create_logprobs_fn: TypeCreateLogProbsFn,
34 + request_id: str,
35 + created_time: int,
36 + model_name: str,
37 + num_prompts: int,
38 +) -> AsyncGenerator[str, None]:
39 + previous_texts = [""] * request.n * num_prompts
40 + previous_num_tokens = [0] * request.n * num_prompts
41 + has_echoed = [False] * request.n * num_prompts
42 +
43 + async for prompt_idx, res in result_generator:
44 +
45 + # Abort the request if the client disconnects.
46 + if await raw_request.is_disconnected():
47 + await on_abort(f"{request_id}-{prompt_idx}")
48 + raise StopAsyncIteration()
49 +
50 for output in res.outputs:
34 - i = output.index
35 - delta_text = output.text[len(previous_texts[i]):]
36 - token_ids = output.token_ids[previous_num_tokens[i]:]
37 - if request.logprobs is not None:
38 - top_logprobs = output.logprobs[previous_num_tokens[i]:]
39 - else:
40 - top_logprobs = None
41 - offsets = len(previous_texts[i])
42 - if request.echo and not has_echoed[i]:
43 - if not echo_without_generation:
44 - delta_text = res.prompt + delta_text
45 - token_ids = res.prompt_token_ids + token_ids
46 - if top_logprobs:
47 - top_logprobs = res.prompt_logprobs + top_logprobs
48 - else: # only just return the prompt
49 - delta_text = res.prompt
50 - token_ids = res.prompt_token_ids
51 - if top_logprobs:
52 - top_logprobs = res.prompt_logprobs
51 + i = output.index + prompt_idx * request.n
52 + # TODO(simon): optimize the performance by avoiding full text O(n^2) sending.
53 +
54 + if request.echo and request.max_tokens == 0:
55 + # only return the prompt
56 + delta_text = res.prompt
57 + delta_token_ids = res.prompt_token_ids
58 + top_logprobs = res.prompt_logprobs
59 + has_echoed[i] = True
60 + elif request.echo and request.max_tokens > 0 and not has_echoed[i]:
61 + # echo the prompt and first token
62 + delta_text = res.prompt + output.text
63 + delta_token_ids = res.prompt_token_ids + output.token_ids
64 + top_logprobs = res.prompt_logprobs + (output.logprobs or [])
65 has_echoed[i] = True
66 + else:
67 + # return just the delta
68 + delta_text = output.text[len(previous_texts[i]):]
69 + delta_token_ids = output.token_ids[previous_num_tokens[i]:]
70 + top_logprobs = output.logprobs[
71 + previous_num_tokens[i]:] if output.logprobs else None
72 +
73 if request.logprobs is not None:
74 + assert top_logprobs is not None, "top_logprobs must be provided when logprobs is requested"
75 logprobs = create_logprobs_fn(
56 - token_ids=token_ids,
76 + token_ids=delta_token_ids,
77 top_logprobs=top_logprobs,
78 num_output_top_logprobs=request.logprobs,
59 - initial_text_offset=offsets,
79 + initial_text_offset=len(previous_texts[i]),
80 )
81 else:
82 logprobs = None
83 +
84 previous_texts[i] = output.text
85 previous_num_tokens[i] = len(output.token_ids)
86 finish_reason = output.finish_reason
@@ -77,7 +98,7 @@ async def completion_stream_generator(
98 ]).model_dump_json(exclude_unset=True)
99 yield f"data: {response_json}\n\n"
100
80 - if output.finish_reason is not None:
101 + if output.finish_reason is not None: # return final usage
102 logprobs = LogProbs() if request.logprobs is not None else None
103 prompt_tokens = len(res.prompt_token_ids)
104 completion_tokens = len(output.token_ids)
@@ -129,51 +150,58 @@ def parse_prompt_format(prompt) -> tuple[bool, list]:
150 return prompt_is_tokens, prompts
151
152
132 -def request_output_to_completion_response(final_res: RequestOutput, request,
133 - echo_without_generation,
134 - create_logprobs_fn, request_id,
135 - created_time,
136 - model_name) -> CompletionResponse:
137 - assert final_res is not None
153 +def request_output_to_completion_response(
154 + final_res_batch: list[RequestOutput],
155 + request: CompletionRequest,
156 + create_logprobs_fn: TypeCreateLogProbsFn,
157 + request_id: str,
158 + created_time: int,
159 + model_name: str,
160 +) -> CompletionResponse:
161 choices = []
139 - prompt_token_ids = final_res.prompt_token_ids
140 - prompt_logprobs = final_res.prompt_logprobs
141 - prompt_text = final_res.prompt
142 - for output in final_res.outputs:
143 - if request.logprobs is not None:
144 - if not echo_without_generation:
162 + num_prompt_tokens = 0
163 + num_generated_tokens = 0
164 + for final_res in final_res_batch:
165 + assert final_res is not None
166 + prompt_token_ids = final_res.prompt_token_ids
167 + prompt_logprobs = final_res.prompt_logprobs
168 + prompt_text = final_res.prompt
169 +
170 + for output in final_res.outputs:
171 + if request.echo and request.max_tokens == 0:
172 + token_ids = prompt_token_ids
173 + top_logprobs = prompt_logprobs
174 + output_text = prompt_text
175 + elif request.echo and request.max_tokens > 0:
176 + token_ids = prompt_token_ids + output.token_ids
177 + top_logprobs = prompt_logprobs + output.logprobs
178 + output_text = prompt_text + output.text
179 + else:
180 token_ids = output.token_ids
181 top_logprobs = output.logprobs
147 - if request.echo:
148 - token_ids = prompt_token_ids + token_ids
149 - top_logprobs = prompt_logprobs + top_logprobs
182 + output_text = output.text
183 +
184 + if request.logprobs is not None:
185 + logprobs = create_logprobs_fn(
186 + token_ids=token_ids,
187 + top_logprobs=top_logprobs,
188 + num_output_top_logprobs=request.logprobs,
189 + )
190 else:
151 - token_ids = prompt_token_ids
152 - top_logprobs = prompt_logprobs
153 - logprobs = create_logprobs_fn(
154 - token_ids=token_ids,
155 - top_logprobs=top_logprobs,
156 - num_output_top_logprobs=request.logprobs,
191 + logprobs = None
192 +
193 + choice_data = CompletionResponseChoice(
194 + index=len(choices),
195 + text=output_text,
196 + logprobs=logprobs,
197 + finish_reason=output.finish_reason,
198 )
158 - else:
159 - logprobs = None
160 - if not echo_without_generation:
161 - output_text = output.text
162 - if request.echo:
163 - output_text = prompt_text + output_text
164 - else:
165 - output_text = prompt_text
166 - choice_data = CompletionResponseChoice(
167 - index=output.index,
168 - text=output_text,
169 - logprobs=logprobs,
170 - finish_reason=output.finish_reason,
171 - )
172 - choices.append(choice_data)
173 -
174 - num_prompt_tokens = len(final_res.prompt_token_ids)
175 - num_generated_tokens = sum(
176 - len(output.token_ids) for output in final_res.outputs)
199 + choices.append(choice_data)
200 +
201 + num_prompt_tokens += len(prompt_token_ids)
202 + num_generated_tokens += sum(
203 + len(output.token_ids) for output in final_res.outputs)
204 +
205 usage = UsageInfo(
206 prompt_tokens=num_prompt_tokens,
207 completion_tokens=num_generated_tokens,
@@ -189,6 +217,36 @@ def request_output_to_completion_response(final_res: RequestOutput, request,
217 )
218
219
220 +def merge_async_iterators(*iterators):
221 + """Merge multiple asynchronous iterators into a single iterator.
222 +
223 + This method handle the case where some iterators finish before others.
224 + When it yields, it yields a tuple (i, item) where i is the index of the
225 + iterator that yields the item.
226 + """
227 + queue = asyncio.Queue()
228 +
229 + finished = [False] * len(iterators)
230 +
231 + async def producer(i, iterator):
232 + async for item in iterator:
233 + await queue.put((i, item))
234 + finished[i] = True
235 +
236 + _tasks = [
237 + asyncio.create_task(producer(i, iterator))
238 + for i, iterator in enumerate(iterators)
239 + ]
240 +
241 + async def consumer():
242 + while not all(finished) or not queue.empty():
243 + item = await queue.get()
244 + yield item
245 + await asyncio.gather(*_tasks)
246 +
247 + return consumer()
248 +
249 +
250 class OpenAIServingCompletion(OpenAIServing):
251
252 def __init__(self, engine: AsyncLLMEngine, served_model: str):
@@ -210,9 +268,6 @@ class OpenAIServingCompletion(OpenAIServing):
268 if error_check_ret is not None:
269 return error_check_ret
270
213 - # OpenAI API supports echoing the prompt when max_tokens is 0.
214 - echo_without_generation = request.echo and request.max_tokens == 0
215 -
271 # Return error for unsupported features.
272 if request.suffix is not None:
273 return self.create_error_response(
@@ -226,30 +281,30 @@ class OpenAIServingCompletion(OpenAIServing):
281 created_time = int(time.monotonic())
282
283 # Schedule the request and get the result generator.
284 + generators = []
285 try:
286 sampling_params = request.to_sampling_params()
231 -
287 prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
288
234 - if len(prompts) > 1:
235 - raise ValueError(
236 - "Batching in completion API is not supported.")
237 - prompt = prompts[0]
238 -
239 - if prompt_is_tokens:
240 - input_ids = self._validate_prompt_and_tokenize(
241 - request, prompt_ids=prompt)
242 - else:
243 - input_ids = self._validate_prompt_and_tokenize(request,
244 - prompt=prompt)
245 -
246 - result_generator = self.engine.generate(None,
247 - sampling_params,
248 - request_id,
249 - prompt_token_ids=input_ids)
289 + for i, prompt in enumerate(prompts):
290 + if prompt_is_tokens:
291 + input_ids = self._validate_prompt_and_tokenize(
292 + request, prompt_ids=prompt)
293 + else:
294 + input_ids = self._validate_prompt_and_tokenize(
295 + request, prompt=prompt)
296 +
297 + generators.append(
298 + self.engine.generate(None,
299 + sampling_params,
300 + f"{request_id}-{i}",
301 + prompt_token_ids=input_ids))
302 except ValueError as e:
303 return self.create_error_response(str(e))
304
305 + result_generator: AsyncIterator[tuple[
306 + int, RequestOutput]] = merge_async_iterators(*generators)
307 +
308 # Similar to the OpenAI API, when n != best_of, we do not stream the
309 # results. In addition, we do not stream the results when use beam search.
310 stream = (request.stream
@@ -258,23 +313,27 @@ class OpenAIServingCompletion(OpenAIServing):
313
314 # Streaming response
315 if stream:
261 - return completion_stream_generator(request, result_generator,
262 - echo_without_generation,
316 + return completion_stream_generator(request,
317 + raw_request,
318 + self.engine.abort,
319 + result_generator,
320 self._create_logprobs,
264 - request_id, created_time,
265 - model_name)
321 + request_id,
322 + created_time,
323 + model_name,
324 + num_prompts=len(prompts))
325
326 # Non-streaming response
268 - final_res: RequestOutput = None
269 - async for res in result_generator:
327 + final_res_batch: RequestOutput = [None] * len(prompts)
328 + async for i, res in result_generator:
329 if await raw_request.is_disconnected():
330 # Abort the request if the client disconnects.
272 - await self.engine.abort(request_id)
331 + await self.engine.abort(f"{request_id}-{i}")
332 return self.create_error_response("Client disconnected")
274 - final_res = res
333 + final_res_batch[i] = res
334 response = request_output_to_completion_response(
276 - final_res, request, echo_without_generation, self._create_logprobs,
277 - request_id, created_time, model_name)
335 + final_res_batch, request, self._create_logprobs, request_id,
336 + created_time, model_name)
337
338 # When user requests streaming but we don't stream, we still need to
339 # return a streaming response with a single event.