A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
feat: add ChatCompletion endpoint in OpenAI demo server. (#330)
Ricardo Lu committed
Jul 3, 2023 at 13:54 UTC
49b26e2cec8c56594668905e853fe4af34336b05
2 files changed
+284
-6
vllm/entrypoints/openai/api_server.py
+236
-3
@@ -4,7 +4,7 @@ import argparse
4
from http import HTTPStatus
5
import json
6
import time
7
-from typing import AsyncGenerator, Dict, List, Optional
7
+from typing import AsyncGenerator, Dict, List, Optional, Union, Any
8
9
import fastapi
10
from fastapi import BackgroundTasks, Request
@@ -17,8 +17,12 @@ from vllm.engine.arg_utils import AsyncEngineArgs
17
from vllm.engine.async_llm_engine import AsyncLLMEngine
18
from vllm.entrypoints.openai.protocol import (
19
CompletionRequest, CompletionResponse, CompletionResponseChoice,
20
- CompletionResponseStreamChoice, CompletionStreamResponse, ErrorResponse,
21
- LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo)
20
+ CompletionResponseStreamChoice, CompletionStreamResponse,
21
+ ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice,
22
+ ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse,
23
+ ChatMessage, DeltaMessage, ErrorResponse, LogProbs,
24
+ ModelCard, ModelList, ModelPermission, UsageInfo)
25
+from fastchat.conversation import Conversation, SeparatorStyle, get_conv_template
26
from vllm.logger import init_logger
27
from vllm.outputs import RequestOutput
28
from vllm.sampling_params import SamplingParams
@@ -55,6 +59,70 @@ async def check_model(request) -> Optional[JSONResponse]:
59
return ret
60
61
62
+async def get_gen_prompt(request) -> str:
63
+ conv = get_conv_template(request.model)
64
+ conv = Conversation(
65
+ name=conv.name,
66
+ system=conv.system,
67
+ roles=conv.roles,
68
+ messages=list(conv.messages), # prevent in-place modification
69
+ offset=conv.offset,
70
+ sep_style=SeparatorStyle(conv.sep_style),
71
+ sep=conv.sep,
72
+ sep2=conv.sep2,
73
+ stop_str=conv.stop_str,
74
+ stop_token_ids=conv.stop_token_ids,
75
+ )
76
+
77
+ if isinstance(request.messages, str):
78
+ prompt = request.messages
79
+ else:
80
+ for message in request.messages:
81
+ msg_role = message["role"]
82
+ if msg_role == "system":
83
+ conv.system = message["content"]
84
+ elif msg_role == "user":
85
+ conv.append_message(conv.roles[0], message["content"])
86
+ elif msg_role == "assistant":
87
+ conv.append_message(conv.roles[1], message["content"])
88
+ else:
89
+ raise ValueError(f"Unknown role: {msg_role}")
90
+
91
+ # Add a blank message for the assistant.
92
+ conv.append_message(conv.roles[1], None)
93
+ prompt = conv.get_prompt()
94
+
95
+ return prompt
96
+
97
+
98
+async def check_length(request, prompt, engine):
99
+ if hasattr(engine.engine.model_config.hf_config, "max_sequence_length"):
100
+ context_len = engine.engine.model_config.hf_config.max_sequence_length
101
+ elif hasattr(engine.engine.model_config.hf_config, "seq_length"):
102
+ context_len = engine.engine.model_config.hf_config.seq_length
103
+ elif hasattr(engine.engine.model_config.hf_config, "max_position_embeddings"):
104
+ context_len = engine.engine.model_config.hf_config.max_position_embeddings
105
+ elif hasattr(engine.engine.model_config.hf_config, "seq_length"):
106
+ context_len = engine.engine.model_config.hf_config.seq_length
107
+ else:
108
+ context_len = 2048
109
+
110
+ input_ids = tokenizer(prompt).input_ids
111
+ token_num = len(input_ids)
112
+
113
+ if token_num + request.max_tokens > context_len:
114
+ return create_error_response(
115
+ HTTPStatus.BAD_REQUEST,
116
+ f"This model's maximum context length is {context_len} tokens. "
117
+ f"However, you requested {request.max_tokens + token_num} tokens "
118
+ f"({token_num} in the messages, "
119
+ f"{request.max_tokens} in the completion). "
120
+ f"Please reduce the length of the messages or completion.",
121
+ )
122
+ else:
123
+ return None
124
+
125
+
126
@app.get("/v1/models")
127
async def show_available_models():
128
"""Show available models. Right now we only have one model."""
@@ -85,6 +153,171 @@ def create_logprobs(token_ids: List[int],
153
return logprobs
154
155
156
+@app.post("/v1/chat/completions")
157
+async def create_chat_completion(raw_request: Request):
158
+ """Completion API similar to OpenAI's API.
159
+
160
+ See https://platform.openai.com/docs/api-reference/chat/create
161
+ for the API specification. This API mimics the OpenAI ChatCompletion API.
162
+
163
+ NOTE: Currently we do not support the following features:
164
+ - function_call (Users should implement this by themselves)
165
+ - logit_bias (to be supported by vLLM engine)
166
+ """
167
+ request = ChatCompletionRequest(**await raw_request.json())
168
+ logger.info(f"Received chat completion request: {request}")
169
+
170
+ error_check_ret = await check_model(request)
171
+ if error_check_ret is not None:
172
+ return error_check_ret
173
+
174
+ if request.logit_bias is not None:
175
+ # TODO: support logit_bias in vLLM engine.
176
+ return create_error_response(HTTPStatus.BAD_REQUEST,
177
+ "logit_bias is not currently supported")
178
+
179
+ prompt = await get_gen_prompt(request)
180
+ error_check_ret = await check_length(request, prompt, engine)
181
+ if error_check_ret is not None:
182
+ return error_check_ret
183
+
184
+ model_name = request.model
185
+ request_id = f"cmpl-{random_uuid()}"
186
+ created_time = int(time.time())
187
+ try:
188
+ sampling_params = SamplingParams(
189
+ n=request.n,
190
+ presence_penalty=request.presence_penalty,
191
+ frequency_penalty=request.frequency_penalty,
192
+ temperature=request.temperature,
193
+ top_p=request.top_p,
194
+ stop=request.stop,
195
+ max_tokens=request.max_tokens,
196
+ best_of=request.best_of,
197
+ top_k=request.top_k,
198
+ ignore_eos=request.ignore_eos,
199
+ use_beam_search=request.use_beam_search,
200
+ )
201
+ except ValueError as e:
202
+ return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
203
+
204
+ result_generator = engine.generate(prompt, sampling_params,
205
+ request_id)
206
+
207
+ async def abort_request() -> None:
208
+ await engine.abort(request_id)
209
+
210
+ def create_stream_response_json(index: int,
211
+ text: str,
212
+ finish_reason: Optional[str] = None) -> str:
213
+ choice_data = ChatCompletionResponseStreamChoice(
214
+ index=index,
215
+ delta=DeltaMessage(content=text),
216
+ finish_reason=finish_reason,
217
+ )
218
+ response = ChatCompletionStreamResponse(
219
+ id=request_id,
220
+ created=created_time,
221
+ model=model_name,
222
+ choices=[choice_data],
223
+ )
224
+ response_json = response.json(ensure_ascii=False)
225
+
226
+ return response_json
227
+
228
+ async def completion_stream_generator() -> AsyncGenerator[str, None]:
229
+ # First chunk with role
230
+ for i in range(request.n):
231
+ choice_data = ChatCompletionResponseStreamChoice(
232
+ index=i,
233
+ delta=DeltaMessage(role="assistant"),
234
+ finish_reason=None,
235
+ )
236
+ chunk = ChatCompletionStreamResponse(
237
+ id=request_id, choices=[choice_data], model=model_name
238
+ )
239
+ yield f"data: {chunk.json(exclude_unset=True, ensure_ascii=False)}\n\n"
240
+
241
+ previous_texts = [""] * request.n
242
+ previous_num_tokens = [0] * request.n
243
+ async for res in result_generator:
244
+ res: RequestOutput
245
+ for output in res.outputs:
246
+ i = output.index
247
+ delta_text = output.text[len(previous_texts[i]):]
248
+ previous_texts[i] = output.text
249
+ previous_num_tokens[i] = len(output.token_ids)
250
+ response_json = create_stream_response_json(
251
+ index=i,
252
+ text=delta_text,
253
+ )
254
+ yield f"data: {response_json}\n\n"
255
+ if output.finish_reason is not None:
256
+ response_json = create_stream_response_json(
257
+ index=i,
258
+ text="",
259
+ finish_reason=output.finish_reason,
260
+ )
261
+ yield f"data: {response_json}\n\n"
262
+ yield "data: [DONE]\n\n"
263
+
264
+ # Streaming response
265
+ if request.stream:
266
+ background_tasks = BackgroundTasks()
267
+ # Abort the request if the client disconnects.
268
+ background_tasks.add_task(abort_request)
269
+ return StreamingResponse(completion_stream_generator(),
270
+ media_type="text/event-stream",
271
+ background=background_tasks)
272
+
273
+ # Non-streaming response
274
+ final_res: RequestOutput = None
275
+ async for res in result_generator:
276
+ if await raw_request.is_disconnected():
277
+ # Abort the request if the client disconnects.
278
+ await abort_request()
279
+ return create_error_response(HTTPStatus.BAD_REQUEST,
280
+ "Client disconnected")
281
+ final_res = res
282
+ assert final_res is not None
283
+ choices = []
284
+ for output in final_res.outputs:
285
+ choice_data = ChatCompletionResponseChoice(
286
+ index=output.index,
287
+ message=ChatMessage(role="assistant", content=output.text),
288
+ finish_reason=output.finish_reason,
289
+ )
290
+ choices.append(choice_data)
291
+
292
+ num_prompt_tokens = len(final_res.prompt_token_ids)
293
+ num_generated_tokens = sum(len(output.token_ids)
294
+ for output in final_res.outputs)
295
+ usage = UsageInfo(
296
+ prompt_tokens=num_prompt_tokens,
297
+ completion_tokens=num_generated_tokens,
298
+ total_tokens=num_prompt_tokens + num_generated_tokens,
299
+ )
300
+ response = ChatCompletionResponse(
301
+ id=request_id,
302
+ created=created_time,
303
+ model=model_name,
304
+ choices=choices,
305
+ usage=usage,
306
+ )
307
+
308
+ if request.stream:
309
+ # When user requests streaming but we don't stream, we still need to
310
+ # return a streaming response with a single event.
311
+ response_json = response.json(ensure_ascii=False)
312
+ async def fake_stream_generator() -> AsyncGenerator[str, None]:
313
+ yield f"data: {response_json}\n\n"
314
+ yield "data: [DONE]\n\n"
315
+ return StreamingResponse(fake_stream_generator(),
316
+ media_type="text/event-stream")
317
+
318
+ return response
319
+
320
+
321
@app.post("/v1/completions")
322
async def create_completion(raw_request: Request):
323
"""Completion API similar to OpenAI's API.
vllm/entrypoints/openai/protocol.py
+48
-3
@@ -53,16 +53,22 @@ class UsageInfo(BaseModel):
53
54
class ChatCompletionRequest(BaseModel):
55
model: str
56
- messages: List[Dict[str, str]]
56
+ messages: Union[str, List[Dict[str, str]]]
57
temperature: Optional[float] = 0.7
58
top_p: Optional[float] = 1.0
59
n: Optional[int] = 1
60
- max_tokens: Optional[int] = None
61
- stop: Optional[Union[str, List[str]]] = None
60
+ max_tokens: Optional[int] = 16
61
+ stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
62
stream: Optional[bool] = False
63
presence_penalty: Optional[float] = 0.0
64
frequency_penalty: Optional[float] = 0.0
65
+ logit_bias: Optional[Dict[str, float]] = None
66
user: Optional[str] = None
67
+ # Additional parameters supported by vLLM
68
+ best_of: Optional[int] = None
69
+ top_k: Optional[int] = -1
70
+ ignore_eos: Optional[bool] = False
71
+ use_beam_search: Optional[bool] = False
72
73
74
class CompletionRequest(BaseModel):
@@ -124,3 +130,42 @@ class CompletionStreamResponse(BaseModel):
130
created: int = Field(default_factory=lambda: int(time.time()))
131
model: str
132
choices: List[CompletionResponseStreamChoice]
133
+
134
+
135
+class ChatMessage(BaseModel):
136
+ role: str
137
+ content: str
138
+
139
+
140
+class ChatCompletionResponseChoice(BaseModel):
141
+ index: int
142
+ message: ChatMessage
143
+ finish_reason: Optional[Literal["stop", "length"]] = None
144
+
145
+
146
+class ChatCompletionResponse(BaseModel):
147
+ id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
148
+ object: str = "chat.completion"
149
+ created: int = Field(default_factory=lambda: int(time.time()))
150
+ model: str
151
+ choices: List[ChatCompletionResponseChoice]
152
+ usage: UsageInfo
153
+
154
+
155
+class DeltaMessage(BaseModel):
156
+ role: Optional[str] = None
157
+ content: Optional[str] = None
158
+
159
+
160
+class ChatCompletionResponseStreamChoice(BaseModel):
161
+ index: int
162
+ delta: DeltaMessage
163
+ finish_reason: Optional[Literal["stop", "length"]] = None
164
+
165
+
166
+class ChatCompletionStreamResponse(BaseModel):
167
+ id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
168
+ object: str = "chat.completion.chunk"
169
+ created: int = Field(default_factory=lambda: int(time.time()))
170
+ model: str
171
+ choices: List[ChatCompletionResponseStreamChoice]