A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
OpenAI Server refactoring (#2360)
FlorianJoncour committed
Jan 17, 2024 at 05:33 UTC
14cc317ba48229d93ee2417822d96ccb8db56abe
8 files changed
+954
-643
.buildkite/test-pipeline.yaml
+3
@@ -19,6 +19,9 @@ steps:
19
- label: Engine Test
20
command: pytest -v -s engine
21
22
+- label: Entrypoints Test
23
+ command: pytest -v -s entrypoints
24
+
25
- label: Kernels Test
26
command: pytest -v -s kernels
27
soft_fail: true
requirements-dev.txt
+3
@@ -16,3 +16,6 @@ pytest-asyncio
16
httpx
17
einops # required for MPT
18
flash_attn # required for HuggingFace's llama implementation
19
+openai
20
+requests
21
+ray
\ No newline at end of file
tests/async_engine/test_chat_template.py
renamed
+17
-19
@@ -1,12 +1,12 @@
1
-from argparse import Namespace
1
from dataclasses import dataclass
2
import os
3
import pathlib
4
5
import pytest
7
-from fastapi.testclient import TestClient
6
9
-from vllm.entrypoints.openai.api_server import *
7
+from vllm.transformers_utils.tokenizer import get_tokenizer
8
+from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
9
+from vllm.entrypoints.openai.protocol import ChatCompletionRequest
10
11
chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
12
__file__))).parent.parent / "examples/template_chatml.jinja"
@@ -48,7 +48,6 @@ TEST_MESSAGES = [
48
'content': 'What is the capital of'
49
},
50
]
51
-client = TestClient(app)
51
52
53
@dataclass
@@ -56,13 +55,17 @@ class MockTokenizer:
55
chat_template = None
56
57
58
+@dataclass
59
+class MockServingChat:
60
+ tokenizer: MockTokenizer
61
+
62
+
63
def test_load_chat_template():
64
# Testing chatml template
61
- mock_args = Namespace(chat_template=chatml_jinja_path)
65
tokenizer = MockTokenizer()
63
-
64
- # Call the function with the mocked args
65
- load_chat_template(mock_args, tokenizer)
66
+ mock_serving_chat = MockServingChat(tokenizer)
67
+ OpenAIServingChat._load_chat_template(mock_serving_chat,
68
+ chat_template=chatml_jinja_path)
69
70
template_content = tokenizer.chat_template
71
@@ -76,11 +79,11 @@ def test_load_chat_template():
79
def test_no_load_chat_template():
80
# Testing chatml template
81
template = "../../examples/does_not_exist"
79
- mock_args = Namespace(chat_template=template)
82
tokenizer = MockTokenizer()
83
82
- # Call the function with the mocked args
83
- load_chat_template(mock_args, tokenizer=tokenizer)
84
+ mock_serving_chat = MockServingChat(tokenizer)
85
+ OpenAIServingChat._load_chat_template(mock_serving_chat,
86
+ chat_template=template)
87
template_content = tokenizer.chat_template
88
89
# Test assertions
@@ -97,9 +100,9 @@ async def test_get_gen_prompt(model, template, add_generation_prompt,
100
expected_output):
101
# Initialize the tokenizer
102
tokenizer = get_tokenizer(tokenizer_name=model)
100
-
101
- mock_args = Namespace(chat_template=template)
102
- load_chat_template(mock_args, tokenizer)
103
+ mock_serving_chat = MockServingChat(tokenizer)
104
+ OpenAIServingChat._load_chat_template(mock_serving_chat,
105
+ chat_template=template)
106
107
# Create a mock request object using keyword arguments
108
mock_request = ChatCompletionRequest(
@@ -115,8 +118,3 @@ async def test_get_gen_prompt(model, template, add_generation_prompt,
118
119
# Test assertion
120
assert result == expected_output, f"The generated prompt does not match the expected output for model {model} and template {template}"
118
-
119
-
120
-def test_health_endpoint():
121
- response = client.get("/health")
122
- assert response.status_code == 200
tests/entrypoints/test_openai_server.py
new
+193
@@ -0,0 +1,193 @@
1
+import time
2
+import subprocess
3
+
4
+import sys
5
+import pytest
6
+import requests
7
+import ray # using Ray for overall ease of process management, parallel requests, and debugging.
8
+import openai # use the official client for correctness check
9
+
10
+MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
11
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta" # any model with a chat template should work here
12
+
13
+pytestmark = pytest.mark.asyncio
14
+
15
+
16
+@ray.remote(num_gpus=1)
17
+class ServerRunner:
18
+
19
+ def __init__(self, args):
20
+ self.proc = subprocess.Popen(
21
+ ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
22
+ stdout=sys.stdout,
23
+ stderr=sys.stderr,
24
+ )
25
+ self._wait_for_server()
26
+
27
+ def ready(self):
28
+ return True
29
+
30
+ def _wait_for_server(self):
31
+ # run health check
32
+ start = time.time()
33
+ while True:
34
+ try:
35
+ if requests.get(
36
+ "http://localhost:8000/health").status_code == 200:
37
+ break
38
+ except Exception as err:
39
+ if self.proc.poll() is not None:
40
+ raise RuntimeError("Server exited unexpectedly.") from err
41
+
42
+ time.sleep(0.5)
43
+ if time.time() - start > MAX_SERVER_START_WAIT_S:
44
+ raise RuntimeError(
45
+ "Server failed to start in time.") from err
46
+
47
+ def __del__(self):
48
+ if hasattr(self, "proc"):
49
+ self.proc.terminate()
50
+
51
+
52
+@pytest.fixture(scope="session")
53
+def server():
54
+ ray.init()
55
+ server_runner = ServerRunner.remote([
56
+ "--model",
57
+ MODEL_NAME,
58
+ "--dtype",
59
+ "bfloat16", # use half precision for speed and memory savings in CI environment
60
+ "--max-model-len",
61
+ "8192"
62
+ ])
63
+ ray.get(server_runner.ready.remote())
64
+ yield server_runner
65
+ ray.shutdown()
66
+
67
+
68
+@pytest.fixture(scope="session")
69
+def client():
70
+ client = openai.AsyncOpenAI(
71
+ base_url="http://localhost:8000/v1",
72
+ api_key="token-abc123",
73
+ )
74
+ yield client
75
+
76
+
77
+async def test_single_completion(server, client: openai.AsyncOpenAI):
78
+ completion = await client.completions.create(model=MODEL_NAME,
79
+ prompt="Hello, my name is",
80
+ max_tokens=5,
81
+ temperature=0.0)
82
+
83
+ assert completion.id is not None
84
+ assert completion.choices is not None and len(completion.choices) == 1
85
+ assert completion.choices[0].text is not None and len(
86
+ completion.choices[0].text) >= 5
87
+ assert completion.choices[0].finish_reason == "length"
88
+ assert completion.usage == openai.types.CompletionUsage(
89
+ completion_tokens=5, prompt_tokens=6, total_tokens=11)
90
+
91
+
92
+async def test_single_chat_session(server, client: openai.AsyncOpenAI):
93
+ messages = [{
94
+ "role": "system",
95
+ "content": "you are a helpful assistant"
96
+ }, {
97
+ "role": "user",
98
+ "content": "what is 1+1?"
99
+ }]
100
+
101
+ # test single completion
102
+ chat_completion = await client.chat.completions.create(
103
+ model=MODEL_NAME,
104
+ messages=messages,
105
+ max_tokens=10,
106
+ )
107
+ assert chat_completion.id is not None
108
+ assert chat_completion.choices is not None and len(
109
+ chat_completion.choices) == 1
110
+ assert chat_completion.choices[0].message is not None
111
+ message = chat_completion.choices[0].message
112
+ assert message.content is not None and len(message.content) >= 10
113
+ assert message.role == "assistant"
114
+ messages.append({"role": "assistant", "content": message.content})
115
+
116
+ # test multi-turn dialogue
117
+ messages.append({"role": "user", "content": "express your result in json"})
118
+ chat_completion = await client.chat.completions.create(
119
+ model=MODEL_NAME,
120
+ messages=messages,
121
+ max_tokens=10,
122
+ )
123
+ message = chat_completion.choices[0].message
124
+ assert message.content is not None and len(message.content) >= 0
125
+
126
+
127
+async def test_completion_streaming(server, client: openai.AsyncOpenAI):
128
+ prompt = "What is an LLM?"
129
+
130
+ single_completion = await client.completions.create(
131
+ model=MODEL_NAME,
132
+ prompt=prompt,
133
+ max_tokens=5,
134
+ temperature=0.0,
135
+ )
136
+ single_output = single_completion.choices[0].text
137
+ single_usage = single_completion.usage
138
+
139
+ stream = await client.completions.create(
140
+ model=MODEL_NAME,
141
+ prompt=prompt,
142
+ max_tokens=5,
143
+ temperature=0.0,
144
+ stream=True,
145
+ )
146
+ chunks = []
147
+ async for chunk in stream:
148
+ chunks.append(chunk.choices[0].text)
149
+ assert chunk.choices[0].finish_reason == "length"
150
+ assert chunk.usage == single_usage
151
+ assert "".join(chunks) == single_output
152
+
153
+
154
+async def test_chat_streaming(server, client: openai.AsyncOpenAI):
155
+ messages = [{
156
+ "role": "system",
157
+ "content": "you are a helpful assistant"
158
+ }, {
159
+ "role": "user",
160
+ "content": "what is 1+1?"
161
+ }]
162
+
163
+ # test single completion
164
+ chat_completion = await client.chat.completions.create(
165
+ model=MODEL_NAME,
166
+ messages=messages,
167
+ max_tokens=10,
168
+ temperature=0.0,
169
+ )
170
+ output = chat_completion.choices[0].message.content
171
+ stop_reason = chat_completion.choices[0].finish_reason
172
+
173
+ # test streaming
174
+ stream = await client.chat.completions.create(
175
+ model=MODEL_NAME,
176
+ messages=messages,
177
+ max_tokens=10,
178
+ temperature=0.0,
179
+ stream=True,
180
+ )
181
+ chunks = []
182
+ async for chunk in stream:
183
+ delta = chunk.choices[0].delta
184
+ if delta.role:
185
+ assert delta.role == "assistant"
186
+ if delta.content:
187
+ chunks.append(delta.content)
188
+ assert chunk.choices[0].finish_reason == stop_reason
189
+ assert "".join(chunks) == output
190
+
191
+
192
+if __name__ == "__main__":
193
+ pytest.main([__file__])
vllm/entrypoints/openai/api_server.py
+25
-624
@@ -1,19 +1,12 @@
1
-# Adapted from
2
-# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py
3
-
1
import argparse
2
import asyncio
6
-import codecs
3
import json
8
-import time
4
from contextlib import asynccontextmanager
10
-from http import HTTPStatus
11
-from typing import AsyncGenerator, Dict, List, Optional, Tuple, Union
12
-
5
from aioprometheus import MetricsMiddleware
6
from aioprometheus.asgi.starlette import metrics
7
import fastapi
8
import uvicorn
9
+from http import HTTPStatus
10
from fastapi import Request
11
from fastapi.exceptions import RequestValidationError
12
from fastapi.middleware.cors import CORSMiddleware
@@ -22,26 +15,16 @@ from fastapi.responses import JSONResponse, StreamingResponse, Response
15
from vllm.engine.arg_utils import AsyncEngineArgs
16
from vllm.engine.async_llm_engine import AsyncLLMEngine
17
from vllm.engine.metrics import add_global_metrics_labels
25
-from vllm.entrypoints.openai.protocol import (
26
- CompletionRequest, CompletionResponse, CompletionResponseChoice,
27
- CompletionResponseStreamChoice, CompletionStreamResponse,
28
- ChatCompletionRequest, ChatCompletionResponse,
29
- ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
30
- ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
31
- LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo)
18
+from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest, ErrorResponse
19
from vllm.logger import init_logger
33
-from vllm.outputs import RequestOutput
34
-from vllm.sampling_params import SamplingParams
35
-from vllm.transformers_utils.tokenizer import get_tokenizer
36
-from vllm.utils import random_uuid
20
+from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
21
+from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
22
23
TIMEOUT_KEEP_ALIVE = 5 # seconds
24
25
+openai_serving_chat: OpenAIServingChat = None
26
+openai_serving_completion: OpenAIServingCompletion = None
27
logger = init_logger(__name__)
41
-served_model = None
42
-engine_args = None
43
-engine = None
44
-response_role = None
28
29
30
@asynccontextmanager
@@ -120,72 +103,10 @@ app.add_middleware(MetricsMiddleware) # Trace HTTP server metrics
103
app.add_route("/metrics", metrics) # Exposes HTTP metrics
104
105
123
-def create_error_response(status_code: HTTPStatus,
124
- message: str) -> JSONResponse:
125
- return JSONResponse(ErrorResponse(message=message,
126
- type="invalid_request_error").dict(),
127
- status_code=status_code.value)
128
-
129
-
130
-def load_chat_template(args, tokenizer):
131
- if args.chat_template is not None:
132
- try:
133
- with open(args.chat_template, "r") as f:
134
- chat_template = f.read()
135
- except OSError:
136
- # If opening a file fails, set chat template to be args to
137
- # ensure we decode so our escape are interpreted correctly
138
- chat_template = codecs.decode(args.chat_template, "unicode_escape")
139
-
140
- tokenizer.chat_template = chat_template
141
- logger.info(
142
- f"Using supplied chat template:\n{tokenizer.chat_template}")
143
- elif tokenizer.chat_template is not None:
144
- logger.info(f"Using default chat template:\n{tokenizer.chat_template}")
145
- else:
146
- logger.warning("No chat template provided. Chat API will not work.")
147
-
148
-
106
@app.exception_handler(RequestValidationError)
107
async def validation_exception_handler(_, exc):
151
- return create_error_response(HTTPStatus.BAD_REQUEST, str(exc))
152
-
153
-
154
-async def check_model(request) -> Optional[JSONResponse]:
155
- if request.model == served_model:
156
- return
157
- ret = create_error_response(
158
- HTTPStatus.NOT_FOUND,
159
- f"The model `{request.model}` does not exist.",
160
- )
161
- return ret
162
-
163
-
164
-async def check_length(
165
- request: Union[ChatCompletionRequest, CompletionRequest],
166
- prompt: Optional[str] = None,
167
- prompt_ids: Optional[List[int]] = None
168
-) -> Tuple[List[int], Optional[JSONResponse]]:
169
- assert (not (prompt is None and prompt_ids is None)
170
- and not (prompt is not None and prompt_ids is not None)
171
- ), "Either prompt or prompt_ids should be provided."
172
- input_ids = prompt_ids if prompt_ids is not None else tokenizer(
173
- prompt).input_ids
174
- token_num = len(input_ids)
175
-
176
- if request.max_tokens is None:
177
- request.max_tokens = max_model_len - token_num
178
- if token_num + request.max_tokens > max_model_len:
179
- return input_ids, create_error_response(
180
- HTTPStatus.BAD_REQUEST,
181
- f"This model's maximum context length is {max_model_len} tokens. "
182
- f"However, you requested {request.max_tokens + token_num} tokens "
183
- f"({token_num} in the messages, "
184
- f"{request.max_tokens} in the completion). "
185
- f"Please reduce the length of the messages or completion.",
186
- )
187
- else:
188
- return input_ids, None
108
+ err = openai_serving_chat.create_error_response(message=str(exc))
109
+ return JSONResponse(err.dict(), status_code=HTTPStatus.BAD_REQUEST)
110
111
112
@app.get("/health")
@@ -196,544 +117,31 @@ async def health() -> Response:
117
118
@app.get("/v1/models")
119
async def show_available_models():
199
- """Show available models. Right now we only have one model."""
200
- model_cards = [
201
- ModelCard(id=served_model,
202
- root=served_model,
203
- permission=[ModelPermission()])
204
- ]
205
- return ModelList(data=model_cards)
206
-
207
-
208
-def create_logprobs(
209
- token_ids: List[int],
210
- top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None,
211
- num_output_top_logprobs: Optional[int] = None,
212
- initial_text_offset: int = 0,
213
-) -> LogProbs:
214
- """Create OpenAI-style logprobs."""
215
- logprobs = LogProbs()
216
- last_token_len = 0
217
- if num_output_top_logprobs:
218
- logprobs.top_logprobs = []
219
- for i, token_id in enumerate(token_ids):
220
- step_top_logprobs = top_logprobs[i]
221
- if step_top_logprobs is not None:
222
- token_logprob = step_top_logprobs[token_id]
223
- else:
224
- token_logprob = None
225
- token = tokenizer.convert_ids_to_tokens(token_id)
226
- logprobs.tokens.append(token)
227
- logprobs.token_logprobs.append(token_logprob)
228
- if len(logprobs.text_offset) == 0:
229
- logprobs.text_offset.append(initial_text_offset)
230
- else:
231
- logprobs.text_offset.append(logprobs.text_offset[-1] +
232
- last_token_len)
233
- last_token_len = len(token)
234
-
235
- if num_output_top_logprobs:
236
- logprobs.top_logprobs.append({
237
- tokenizer.convert_ids_to_tokens(i): p
238
- for i, p in step_top_logprobs.items()
239
- } if step_top_logprobs else None)
240
- return logprobs
120
+ models = await openai_serving_chat.show_available_models()
121
+ return JSONResponse(content=models.dict())
122
123
124
@app.post("/v1/chat/completions")
125
async def create_chat_completion(request: ChatCompletionRequest,
126
raw_request: Request):
246
- """Completion API similar to OpenAI's API.
247
-
248
- See https://platform.openai.com/docs/api-reference/chat/create
249
- for the API specification. This API mimics the OpenAI ChatCompletion API.
250
-
251
- NOTE: Currently we do not support the following features:
252
- - function_call (Users should implement this by themselves)
253
- - logit_bias (to be supported by vLLM engine)
254
- """
255
- error_check_ret = await check_model(request)
256
- if error_check_ret is not None:
257
- return error_check_ret
258
-
259
- if request.logit_bias is not None and len(request.logit_bias) > 0:
260
- # TODO: support logit_bias in vLLM engine.
261
- return create_error_response(HTTPStatus.BAD_REQUEST,
262
- "logit_bias is not currently supported")
263
-
264
- try:
265
- prompt = tokenizer.apply_chat_template(
266
- conversation=request.messages,
267
- tokenize=False,
268
- add_generation_prompt=request.add_generation_prompt)
269
- except Exception as e:
270
- logger.error(f"Error in applying chat template from request: {str(e)}")
271
- return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
272
-
273
- token_ids, error_check_ret = await check_length(request, prompt=prompt)
274
- if error_check_ret is not None:
275
- return error_check_ret
276
-
277
- model_name = request.model
278
- request_id = f"cmpl-{random_uuid()}"
279
- created_time = int(time.monotonic())
280
- chunk_object_type = "chat.completion.chunk"
281
- try:
282
- spaces_between_special_tokens = request.spaces_between_special_tokens
283
- sampling_params = SamplingParams(
284
- n=request.n,
285
- presence_penalty=request.presence_penalty,
286
- frequency_penalty=request.frequency_penalty,
287
- repetition_penalty=request.repetition_penalty,
288
- temperature=request.temperature,
289
- top_p=request.top_p,
290
- min_p=request.min_p,
291
- stop=request.stop,
292
- stop_token_ids=request.stop_token_ids,
293
- max_tokens=request.max_tokens,
294
- best_of=request.best_of,
295
- top_k=request.top_k,
296
- ignore_eos=request.ignore_eos,
297
- use_beam_search=request.use_beam_search,
298
- skip_special_tokens=request.skip_special_tokens,
299
- spaces_between_special_tokens=spaces_between_special_tokens,
300
- )
301
- except ValueError as e:
302
- return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
303
-
304
- result_generator = engine.generate(prompt, sampling_params, request_id,
305
- token_ids)
306
-
307
- def get_role() -> str:
308
- if request.add_generation_prompt:
309
- return response_role
310
- else:
311
- return request.messages[-1]["role"]
312
-
313
- async def completion_stream_generator() -> AsyncGenerator[str, None]:
314
- # Send first response for each request.n (index) with the role
315
- role = get_role()
316
- for i in range(request.n):
317
- choice_data = ChatCompletionResponseStreamChoice(
318
- index=i, delta=DeltaMessage(role=role), finish_reason=None)
319
- chunk = ChatCompletionStreamResponse(id=request_id,
320
- object=chunk_object_type,
321
- created=created_time,
322
- choices=[choice_data],
323
- model=model_name)
324
- data = chunk.json(exclude_unset=True, ensure_ascii=False)
325
- yield f"data: {data}\n\n"
326
-
327
- # Send response to echo the input portion of the last message
328
- if request.echo:
329
- last_msg_content = ""
330
- if request.messages and isinstance(
331
- request.messages, list) and request.messages[-1].get(
332
- "content") and request.messages[-1].get(
333
- "role") == role:
334
- last_msg_content = request.messages[-1]["content"]
335
- if last_msg_content:
336
- for i in range(request.n):
337
- choice_data = ChatCompletionResponseStreamChoice(
338
- index=i,
339
- delta=DeltaMessage(content=last_msg_content),
340
- finish_reason=None)
341
- chunk = ChatCompletionStreamResponse(
342
- id=request_id,
343
- object=chunk_object_type,
344
- created=created_time,
345
- choices=[choice_data],
346
- model=model_name)
347
- data = chunk.json(exclude_unset=True, ensure_ascii=False)
348
- yield f"data: {data}\n\n"
349
-
350
- # Send response for each token for each request.n (index)
351
- previous_texts = [""] * request.n
352
- previous_num_tokens = [0] * request.n
353
- finish_reason_sent = [False] * request.n
354
- async for res in result_generator:
355
- res: RequestOutput
356
- for output in res.outputs:
357
- i = output.index
358
-
359
- if finish_reason_sent[i]:
360
- continue
361
-
362
- if output.finish_reason is None:
363
- # Send token-by-token response for each request.n
364
- delta_text = output.text[len(previous_texts[i]):]
365
- previous_texts[i] = output.text
366
- previous_num_tokens[i] = len(output.token_ids)
367
- choice_data = ChatCompletionResponseStreamChoice(
368
- index=i,
369
- delta=DeltaMessage(content=delta_text),
370
- finish_reason=None)
371
- chunk = ChatCompletionStreamResponse(
372
- id=request_id,
373
- object=chunk_object_type,
374
- created=created_time,
375
- choices=[choice_data],
376
- model=model_name)
377
- data = chunk.json(exclude_unset=True, ensure_ascii=False)
378
- yield f"data: {data}\n\n"
379
- else:
380
- # Send the finish response for each request.n only once
381
- prompt_tokens = len(res.prompt_token_ids)
382
- final_usage = UsageInfo(
383
- prompt_tokens=prompt_tokens,
384
- completion_tokens=previous_num_tokens[i],
385
- total_tokens=prompt_tokens + previous_num_tokens[i],
386
- )
387
- choice_data = ChatCompletionResponseStreamChoice(
388
- index=i, delta=[], finish_reason=output.finish_reason)
389
- chunk = ChatCompletionStreamResponse(
390
- id=request_id,
391
- object=chunk_object_type,
392
- created=created_time,
393
- choices=[choice_data],
394
- model=model_name)
395
- if final_usage is not None:
396
- chunk.usage = final_usage
397
- data = chunk.json(exclude_unset=True,
398
- exclude_none=True,
399
- ensure_ascii=False)
400
- yield f"data: {data}\n\n"
401
- finish_reason_sent[i] = True
402
- # Send the final done message after all response.n are finished
403
- yield "data: [DONE]\n\n"
404
-
405
- async def completion_full_generator():
406
- final_res: RequestOutput = None
407
- async for res in result_generator:
408
- if await raw_request.is_disconnected():
409
- # Abort the request if the client disconnects.
410
- await engine.abort(request_id)
411
- return create_error_response(HTTPStatus.BAD_REQUEST,
412
- "Client disconnected")
413
- final_res = res
414
- assert final_res is not None
415
-
416
- choices = []
417
- role = get_role()
418
- for output in final_res.outputs:
419
- choice_data = ChatCompletionResponseChoice(
420
- index=output.index,
421
- message=ChatMessage(role=role, content=output.text),
422
- finish_reason=output.finish_reason,
423
- )
424
- choices.append(choice_data)
425
-
426
- if request.echo:
427
- last_msg_content = ""
428
- if request.messages and isinstance(
429
- request.messages, list) and request.messages[-1].get(
430
- "content") and request.messages[-1].get(
431
- "role") == role:
432
- last_msg_content = request.messages[-1]["content"]
433
-
434
- for choice in choices:
435
- full_message = last_msg_content + choice.message.content
436
- choice.message.content = full_message
437
-
438
- num_prompt_tokens = len(final_res.prompt_token_ids)
439
- num_generated_tokens = sum(
440
- len(output.token_ids) for output in final_res.outputs)
441
- usage = UsageInfo(
442
- prompt_tokens=num_prompt_tokens,
443
- completion_tokens=num_generated_tokens,
444
- total_tokens=num_prompt_tokens + num_generated_tokens,
445
- )
446
- response = ChatCompletionResponse(
447
- id=request_id,
448
- created=created_time,
449
- model=model_name,
450
- choices=choices,
451
- usage=usage,
452
- )
453
-
454
- return response
455
-
456
- # Streaming response
457
- if request.stream:
458
- return StreamingResponse(completion_stream_generator(),
127
+ generator = await openai_serving_chat.create_chat_completion(
128
+ request, raw_request)
129
+ if request.stream and not isinstance(generator, ErrorResponse):
130
+ return StreamingResponse(content=generator,
131
media_type="text/event-stream")
132
else:
461
- return await completion_full_generator()
133
+ return JSONResponse(content=generator.dict())
134
135
136
@app.post("/v1/completions")
137
async def create_completion(request: CompletionRequest, raw_request: Request):
466
- """Completion API similar to OpenAI's API.
467
-
468
- See https://platform.openai.com/docs/api-reference/completions/create
469
- for the API specification. This API mimics the OpenAI Completion API.
470
-
471
- NOTE: Currently we do not support the following features:
472
- - suffix (the language models we currently support do not support
473
- suffix)
474
- - logit_bias (to be supported by vLLM engine)
475
- """
476
-
477
- error_check_ret = await check_model(request)
478
- if error_check_ret is not None:
479
- return error_check_ret
480
-
481
- # OpenAI API supports echoing the prompt when max_tokens is 0.
482
- echo_without_generation = request.echo and request.max_tokens == 0
483
-
484
- if request.suffix is not None:
485
- # The language models we currently support do not support suffix.
486
- return create_error_response(HTTPStatus.BAD_REQUEST,
487
- "suffix is not currently supported")
488
-
489
- if request.logit_bias is not None and len(request.logit_bias) > 0:
490
- # TODO: support logit_bias in vLLM engine.
491
- return create_error_response(HTTPStatus.BAD_REQUEST,
492
- "logit_bias is not currently supported")
493
-
494
- model_name = request.model
495
- request_id = f"cmpl-{random_uuid()}"
496
-
497
- use_token_ids = False
498
- if isinstance(request.prompt, list):
499
- if len(request.prompt) == 0:
500
- return create_error_response(HTTPStatus.BAD_REQUEST,
501
- "please provide at least one prompt")
502
- first_element = request.prompt[0]
503
- if isinstance(first_element, int):
504
- use_token_ids = True
505
- prompt = request.prompt
506
- elif isinstance(first_element, (str, list)):
507
- # TODO: handles multiple prompt case in list[list[int]]
508
- if len(request.prompt) > 1:
509
- return create_error_response(
510
- HTTPStatus.BAD_REQUEST,
511
- "multiple prompts in a batch is not currently supported")
512
- use_token_ids = not isinstance(first_element, str)
513
- prompt = request.prompt[0]
514
- else:
515
- prompt = request.prompt
516
-
517
- if use_token_ids:
518
- _, error_check_ret = await check_length(request, prompt_ids=prompt)
519
- else:
520
- token_ids, error_check_ret = await check_length(request, prompt=prompt)
521
- if error_check_ret is not None:
522
- return error_check_ret
523
-
524
- created_time = int(time.monotonic())
525
- try:
526
- spaces_between_special_tokens = request.spaces_between_special_tokens
527
- sampling_params = SamplingParams(
528
- n=request.n,
529
- best_of=request.best_of,
530
- presence_penalty=request.presence_penalty,
531
- frequency_penalty=request.frequency_penalty,
532
- repetition_penalty=request.repetition_penalty,
533
- temperature=request.temperature,
534
- top_p=request.top_p,
535
- top_k=request.top_k,
536
- min_p=request.min_p,
537
- stop=request.stop,
538
- stop_token_ids=request.stop_token_ids,
539
- ignore_eos=request.ignore_eos,
540
- max_tokens=request.max_tokens
541
- if not echo_without_generation else 1,
542
- logprobs=request.logprobs,
543
- use_beam_search=request.use_beam_search,
544
- prompt_logprobs=request.logprobs if request.echo else None,
545
- skip_special_tokens=request.skip_special_tokens,
546
- spaces_between_special_tokens=spaces_between_special_tokens,
547
- )
548
- except ValueError as e:
549
- return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
550
-
551
- if use_token_ids:
552
- result_generator = engine.generate(None,
553
- sampling_params,
554
- request_id,
555
- prompt_token_ids=prompt)
556
- else:
557
- result_generator = engine.generate(prompt, sampling_params, request_id,
558
- token_ids)
559
-
560
- # Similar to the OpenAI API, when n != best_of, we do not stream the
561
- # results. In addition, we do not stream the results when use beam search.
562
- stream = (request.stream
563
- and (request.best_of is None or request.n == request.best_of)
564
- and not request.use_beam_search)
565
-
566
- def create_stream_response_json(
567
- index: int,
568
- text: str,
569
- logprobs: Optional[LogProbs] = None,
570
- finish_reason: Optional[str] = None,
571
- usage: Optional[UsageInfo] = None,
572
- ) -> str:
573
- choice_data = CompletionResponseStreamChoice(
574
- index=index,
575
- text=text,
576
- logprobs=logprobs,
577
- finish_reason=finish_reason,
578
- )
579
- response = CompletionStreamResponse(
580
- id=request_id,
581
- created=created_time,
582
- model=model_name,
583
- choices=[choice_data],
584
- )
585
- if usage is not None:
586
- response.usage = usage
587
- response_json = response.json(exclude_unset=True, ensure_ascii=False)
588
-
589
- return response_json
590
-
591
- async def completion_stream_generator() -> AsyncGenerator[str, None]:
592
- previous_texts = [""] * request.n
593
- previous_num_tokens = [0] * request.n
594
- has_echoed = [False] * request.n
595
- async for res in result_generator:
596
- res: RequestOutput
597
- for output in res.outputs:
598
- i = output.index
599
- delta_text = output.text[len(previous_texts[i]):]
600
- token_ids = output.token_ids[previous_num_tokens[i]:]
601
- if request.logprobs is not None:
602
- top_logprobs = output.logprobs[previous_num_tokens[i]:]
603
- else:
604
- top_logprobs = None
605
- offsets = len(previous_texts[i])
606
- if request.echo and not has_echoed[i]:
607
- if not echo_without_generation:
608
- delta_text = res.prompt + delta_text
609
- token_ids = res.prompt_token_ids + token_ids
610
- if top_logprobs:
611
- top_logprobs = res.prompt_logprobs + top_logprobs
612
- else: # only just return the prompt
613
- delta_text = res.prompt
614
- token_ids = res.prompt_token_ids
615
- if top_logprobs:
616
- top_logprobs = res.prompt_logprobs
617
- has_echoed[i] = True
618
- if request.logprobs is not None:
619
- logprobs = create_logprobs(
620
- token_ids=token_ids,
621
- top_logprobs=top_logprobs,
622
- num_output_top_logprobs=request.logprobs,
623
- initial_text_offset=offsets,
624
- )
625
- else:
626
- logprobs = None
627
- previous_texts[i] = output.text
628
- previous_num_tokens[i] = len(output.token_ids)
629
- finish_reason = output.finish_reason
630
- response_json = create_stream_response_json(
631
- index=i,
632
- text=delta_text,
633
- logprobs=logprobs,
634
- finish_reason=finish_reason,
635
- )
636
- yield f"data: {response_json}\n\n"
637
- if output.finish_reason is not None:
638
- logprobs = (LogProbs()
639
- if request.logprobs is not None else None)
640
- prompt_tokens = len(res.prompt_token_ids)
641
- completion_tokens = len(output.token_ids)
642
- final_usage = UsageInfo(
643
- prompt_tokens=prompt_tokens,
644
- completion_tokens=completion_tokens,
645
- total_tokens=prompt_tokens + completion_tokens,
646
- )
647
- response_json = create_stream_response_json(
648
- index=i,
649
- text="",
650
- logprobs=logprobs,
651
- finish_reason=output.finish_reason,
652
- usage=final_usage,
653
- )
654
- yield f"data: {response_json}\n\n"
655
- yield "data: [DONE]\n\n"
656
-
657
- # Streaming response
658
- if stream:
659
- return StreamingResponse(completion_stream_generator(),
660
- media_type="text/event-stream")
661
-
662
- # Non-streaming response
663
- final_res: RequestOutput = None
664
- async for res in result_generator:
665
- if await raw_request.is_disconnected():
666
- # Abort the request if the client disconnects.
667
- await engine.abort(request_id)
668
- return create_error_response(HTTPStatus.BAD_REQUEST,
669
- "Client disconnected")
670
- final_res = res
671
- assert final_res is not None
672
- choices = []
673
- prompt_token_ids = final_res.prompt_token_ids
674
- prompt_logprobs = final_res.prompt_logprobs
675
- prompt_text = final_res.prompt
676
- for output in final_res.outputs:
677
- if request.logprobs is not None:
678
- if not echo_without_generation:
679
- token_ids = output.token_ids
680
- top_logprobs = output.logprobs
681
- if request.echo:
682
- token_ids = prompt_token_ids + token_ids
683
- top_logprobs = prompt_logprobs + top_logprobs
684
- else:
685
- token_ids = prompt_token_ids
686
- top_logprobs = prompt_logprobs
687
- logprobs = create_logprobs(
688
- token_ids=token_ids,
689
- top_logprobs=top_logprobs,
690
- num_output_top_logprobs=request.logprobs,
691
- )
692
- else:
693
- logprobs = None
694
- if not echo_without_generation:
695
- output_text = output.text
696
- if request.echo:
697
- output_text = prompt_text + output_text
698
- else:
699
- output_text = prompt_text
700
- choice_data = CompletionResponseChoice(
701
- index=output.index,
702
- text=output_text,
703
- logprobs=logprobs,
704
- finish_reason=output.finish_reason,
705
- )
706
- choices.append(choice_data)
707
-
708
- num_prompt_tokens = len(final_res.prompt_token_ids)
709
- num_generated_tokens = sum(
710
- len(output.token_ids) for output in final_res.outputs)
711
- usage = UsageInfo(
712
- prompt_tokens=num_prompt_tokens,
713
- completion_tokens=num_generated_tokens,
714
- total_tokens=num_prompt_tokens + num_generated_tokens,
715
- )
716
- response = CompletionResponse(
717
- id=request_id,
718
- created=created_time,
719
- model=model_name,
720
- choices=choices,
721
- usage=usage,
722
- )
723
-
724
- if request.stream:
725
- # When user requests streaming but we don't stream, we still need to
726
- # return a streaming response with a single event.
727
- response_json = response.json(ensure_ascii=False)
728
-
729
- async def fake_stream_generator() -> AsyncGenerator[str, None]:
730
- yield f"data: {response_json}\n\n"
731
- yield "data: [DONE]\n\n"
732
-
733
- return StreamingResponse(fake_stream_generator(),
138
+ generator = await openai_serving_completion.create_completion(
139
+ request, raw_request)
140
+ if request.stream and not isinstance(generator, ErrorResponse):
141
+ return StreamingResponse(content=generator,
142
media_type="text/event-stream")
735
-
736
- return response
143
+ else:
144
+ return JSONResponse(content=generator.dict())
145
146
147
if __name__ == "__main__":
@@ -754,19 +162,12 @@ if __name__ == "__main__":
162
else:
163
served_model = args.model
164
757
- response_role = args.response_role
758
-
165
engine_args = AsyncEngineArgs.from_cli_args(args)
166
engine = AsyncLLMEngine.from_engine_args(engine_args)
761
- engine_model_config = asyncio.run(engine.get_model_config())
762
- max_model_len = engine_model_config.max_model_len
763
-
764
- # A separate tokenizer to map token IDs to strings.
765
- tokenizer = get_tokenizer(
766
- engine_model_config.tokenizer,
767
- tokenizer_mode=engine_model_config.tokenizer_mode,
768
- trust_remote_code=engine_model_config.trust_remote_code)
769
- load_chat_template(args, tokenizer)
167
+ openai_serving_chat = OpenAIServingChat(engine, served_model,
168
+ args.response_role,
169
+ args.chat_template)
170
+ openai_serving_completion = OpenAIServingCompletion(engine, served_model)
171
172
# Register labels for metrics
173
add_global_metrics_labels(model_name=engine_args.model)
vllm/entrypoints/openai/serving_chat.py
new
+288
@@ -0,0 +1,288 @@
1
+import time
2
+import codecs
3
+from fastapi import Request
4
+from typing import AsyncGenerator, AsyncIterator, Union
5
+from vllm.logger import init_logger
6
+from vllm.utils import random_uuid
7
+from vllm.engine.async_llm_engine import AsyncLLMEngine
8
+from vllm.entrypoints.openai.protocol import (
9
+ ChatCompletionRequest, ChatCompletionResponse,
10
+ ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
11
+ ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
12
+ UsageInfo)
13
+from vllm.outputs import RequestOutput
14
+from vllm.sampling_params import SamplingParams
15
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
16
+
17
+logger = init_logger(__name__)
18
+
19
+
20
+class OpenAIServingChat(OpenAIServing):
21
+
22
+ def __init__(self,
23
+ engine: AsyncLLMEngine,
24
+ served_model: str,
25
+ response_role: str,
26
+ chat_template=None):
27
+ super().__init__(engine=engine, served_model=served_model)
28
+ self.response_role = response_role
29
+ self._load_chat_template(chat_template)
30
+
31
+ async def create_chat_completion(
32
+ self, request: ChatCompletionRequest, raw_request: Request
33
+ ) -> Union[ErrorResponse, AsyncGenerator[str, None],
34
+ ChatCompletionResponse]:
35
+ """Completion API similar to OpenAI's API.
36
+
37
+ See https://platform.openai.com/docs/api-reference/chat/create
38
+ for the API specification. This API mimics the OpenAI ChatCompletion API.
39
+
40
+ NOTE: Currently we do not support the following features:
41
+ - function_call (Users should implement this by themselves)
42
+ - logit_bias (to be supported by vLLM engine)
43
+ """
44
+ error_check_ret = await self._check_model(request)
45
+ if error_check_ret is not None:
46
+ return error_check_ret
47
+
48
+ if request.logit_bias is not None and len(request.logit_bias) > 0:
49
+ # TODO: support logit_bias in vLLM engine.
50
+ return self.create_error_response(
51
+ "logit_bias is not currently supported")
52
+
53
+ try:
54
+ prompt = self.tokenizer.apply_chat_template(
55
+ conversation=request.messages,
56
+ tokenize=False,
57
+ add_generation_prompt=request.add_generation_prompt)
58
+ except Exception as e:
59
+ logger.error(
60
+ f"Error in applying chat template from request: {str(e)}")
61
+ return self.create_error_response(str(e))
62
+
63
+ token_ids, error_check_ret = await self._check_length(request,
64
+ prompt=prompt)
65
+ if error_check_ret is not None:
66
+ return error_check_ret
67
+
68
+ request_id = f"cmpl-{random_uuid()}"
69
+ try:
70
+ spaces_between_special_tokens = request.spaces_between_special_tokens
71
+ sampling_params = SamplingParams(
72
+ n=request.n,
73
+ presence_penalty=request.presence_penalty,
74
+ frequency_penalty=request.frequency_penalty,
75
+ repetition_penalty=request.repetition_penalty,
76
+ temperature=request.temperature,
77
+ top_p=request.top_p,
78
+ min_p=request.min_p,
79
+ stop=request.stop,
80
+ stop_token_ids=request.stop_token_ids,
81
+ max_tokens=request.max_tokens,
82
+ best_of=request.best_of,
83
+ top_k=request.top_k,
84
+ ignore_eos=request.ignore_eos,
85
+ use_beam_search=request.use_beam_search,
86
+ skip_special_tokens=request.skip_special_tokens,
87
+ spaces_between_special_tokens=spaces_between_special_tokens,
88
+ )
89
+ except ValueError as e:
90
+ return self.create_error_response(str(e))
91
+
92
+ result_generator = self.engine.generate(prompt, sampling_params,
93
+ request_id, token_ids)
94
+ # Streaming response
95
+ if request.stream:
96
+ return self.chat_completion_stream_generator(
97
+ request, result_generator, request_id)
98
+ else:
99
+ return await self.chat_completion_full_generator(
100
+ request, raw_request, result_generator, request_id)
101
+
102
+ def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
103
+ if request.add_generation_prompt:
104
+ return self.response_role
105
+ else:
106
+ return request.messages[-1].role
107
+
108
+ async def chat_completion_stream_generator(
109
+ self, request: ChatCompletionRequest,
110
+ result_generator: AsyncIterator[RequestOutput], request_id: str
111
+ ) -> Union[ErrorResponse, AsyncGenerator[str, None]]:
112
+
113
+ model_name = request.model
114
+ created_time = int(time.monotonic())
115
+ chunk_object_type = "chat.completion.chunk"
116
+
117
+ # Send first response for each request.n (index) with the role
118
+ role = self.get_chat_request_role(request)
119
+ for i in range(request.n):
120
+ choice_data = ChatCompletionResponseStreamChoice(
121
+ index=i, delta=DeltaMessage(role=role), finish_reason=None)
122
+ chunk = ChatCompletionStreamResponse(id=request_id,
123
+ object=chunk_object_type,
124
+ created=created_time,
125
+ choices=[choice_data],
126
+ model=model_name)
127
+ data = chunk.json(exclude_unset=True, ensure_ascii=False)
128
+ yield f"data: {data}\n\n"
129
+
130
+ # Send response to echo the input portion of the last message
131
+ if request.echo:
132
+ last_msg_content = ""
133
+ if request.messages and isinstance(
134
+ request.messages, list) and request.messages[-1].get(
135
+ "content") and request.messages[-1].get(
136
+ "role") == role:
137
+ last_msg_content = request.messages[-1]["content"]
138
+ if last_msg_content:
139
+ for i in range(request.n):
140
+ choice_data = ChatCompletionResponseStreamChoice(
141
+ index=i,
142
+ delta=DeltaMessage(content=last_msg_content),
143
+ finish_reason=None)
144
+ chunk = ChatCompletionStreamResponse(
145
+ id=request_id,
146
+ object=chunk_object_type,
147
+ created=created_time,
148
+ choices=[choice_data],
149
+ model=model_name)
150
+ data = chunk.json(exclude_unset=True, ensure_ascii=False)
151
+ yield f"data: {data}\n\n"
152
+
153
+ # Send response for each token for each request.n (index)
154
+ previous_texts = [""] * request.n
155
+ previous_num_tokens = [0] * request.n
156
+ finish_reason_sent = [False] * request.n
157
+ async for res in result_generator:
158
+ res: RequestOutput
159
+ for output in res.outputs:
160
+ i = output.index
161
+
162
+ if finish_reason_sent[i]:
163
+ continue
164
+
165
+ delta_text = output.text[len(previous_texts[i]):]
166
+ previous_texts[i] = output.text
167
+ previous_num_tokens[i] = len(output.token_ids)
168
+
169
+ if output.finish_reason is None:
170
+ # Send token-by-token response for each request.n
171
+ choice_data = ChatCompletionResponseStreamChoice(
172
+ index=i,
173
+ delta=DeltaMessage(content=delta_text),
174
+ finish_reason=None)
175
+ chunk = ChatCompletionStreamResponse(
176
+ id=request_id,
177
+ object=chunk_object_type,
178
+ created=created_time,
179
+ choices=[choice_data],
180
+ model=model_name)
181
+ data = chunk.json(exclude_unset=True, ensure_ascii=False)
182
+ yield f"data: {data}\n\n"
183
+ else:
184
+ # Send the finish response for each request.n only once
185
+ prompt_tokens = len(res.prompt_token_ids)
186
+ final_usage = UsageInfo(
187
+ prompt_tokens=prompt_tokens,
188
+ completion_tokens=previous_num_tokens[i],
189
+ total_tokens=prompt_tokens + previous_num_tokens[i],
190
+ )
191
+ choice_data = ChatCompletionResponseStreamChoice(
192
+ index=i,
193
+ delta=DeltaMessage(content=delta_text),
194
+ finish_reason=output.finish_reason)
195
+ chunk = ChatCompletionStreamResponse(
196
+ id=request_id,
197
+ object=chunk_object_type,
198
+ created=created_time,
199
+ choices=[choice_data],
200
+ model=model_name)
201
+ if final_usage is not None:
202
+ chunk.usage = final_usage
203
+ data = chunk.json(exclude_unset=True,
204
+ exclude_none=True,
205
+ ensure_ascii=False)
206
+ yield f"data: {data}\n\n"
207
+ finish_reason_sent[i] = True
208
+ # Send the final done message after all response.n are finished
209
+ yield "data: [DONE]\n\n"
210
+
211
+ async def chat_completion_full_generator(
212
+ self, request: ChatCompletionRequest, raw_request: Request,
213
+ result_generator: AsyncIterator[RequestOutput],
214
+ request_id: str) -> Union[ErrorResponse, ChatCompletionResponse]:
215
+
216
+ model_name = request.model
217
+ created_time = int(time.monotonic())
218
+ final_res: RequestOutput = None
219
+
220
+ async for res in result_generator:
221
+ if await raw_request.is_disconnected():
222
+ # Abort the request if the client disconnects.
223
+ await self.engine.abort(request_id)
224
+ return self.create_error_response("Client disconnected")
225
+ final_res = res
226
+ assert final_res is not None
227
+
228
+ choices = []
229
+ role = self.get_chat_request_role(request)
230
+ for output in final_res.outputs:
231
+ choice_data = ChatCompletionResponseChoice(
232
+ index=output.index,
233
+ message=ChatMessage(role=role, content=output.text),
234
+ finish_reason=output.finish_reason,
235
+ )
236
+ choices.append(choice_data)
237
+
238
+ if request.echo:
239
+ last_msg_content = ""
240
+ if request.messages and isinstance(
241
+ request.messages, list) and request.messages[-1].get(
242
+ "content") and request.messages[-1].get(
243
+ "role") == role:
244
+ last_msg_content = request.messages[-1]["content"]
245
+
246
+ for choice in choices:
247
+ full_message = last_msg_content + choice.message.content
248
+ choice.message.content = full_message
249
+
250
+ num_prompt_tokens = len(final_res.prompt_token_ids)
251
+ num_generated_tokens = sum(
252
+ len(output.token_ids) for output in final_res.outputs)
253
+ usage = UsageInfo(
254
+ prompt_tokens=num_prompt_tokens,
255
+ completion_tokens=num_generated_tokens,
256
+ total_tokens=num_prompt_tokens + num_generated_tokens,
257
+ )
258
+ response = ChatCompletionResponse(
259
+ id=request_id,
260
+ created=created_time,
261
+ model=model_name,
262
+ choices=choices,
263
+ usage=usage,
264
+ )
265
+
266
+ return response
267
+
268
+ def _load_chat_template(self, chat_template):
269
+ if chat_template is not None:
270
+ try:
271
+ with open(chat_template, "r") as f:
272
+ self.tokenizer.chat_template = f.read()
273
+ except OSError:
274
+ # If opening a file fails, set chat template to be args to
275
+ # ensure we decode so our escape are interpreted correctly
276
+ self.tokenizer.chat_template = codecs.decode(
277
+ chat_template, "unicode_escape")
278
+
279
+ logger.info(
280
+ f"Using supplied chat template:\n{self.tokenizer.chat_template}"
281
+ )
282
+ elif self.tokenizer.chat_template is not None:
283
+ logger.info(
284
+ f"Using default chat template:\n{self.tokenizer.chat_template}"
285
+ )
286
+ else:
287
+ logger.warning(
288
+ "No chat template provided. Chat API will not work.")
vllm/entrypoints/openai/serving_completion.py
new
+295
@@ -0,0 +1,295 @@
1
+import time
2
+from fastapi import Request
3
+from typing import AsyncGenerator, Optional
4
+from vllm.logger import init_logger
5
+from vllm.utils import random_uuid
6
+from vllm.engine.async_llm_engine import AsyncLLMEngine
7
+from .protocol import (CompletionRequest, CompletionResponse,
8
+ CompletionResponseChoice,
9
+ CompletionResponseStreamChoice,
10
+ CompletionStreamResponse, LogProbs, UsageInfo)
11
+from vllm.outputs import RequestOutput
12
+from vllm.sampling_params import SamplingParams
13
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
14
+
15
+logger = init_logger(__name__)
16
+
17
+
18
+class OpenAIServingCompletion(OpenAIServing):
19
+
20
+ def __init__(self, engine: AsyncLLMEngine, served_model: str):
21
+ super().__init__(engine=engine, served_model=served_model)
22
+
23
+ async def create_completion(self, request: CompletionRequest,
24
+ raw_request: Request):
25
+ """Completion API similar to OpenAI's API.
26
+
27
+ See https://platform.openai.com/docs/api-reference/completions/create
28
+ for the API specification. This API mimics the OpenAI Completion API.
29
+
30
+ NOTE: Currently we do not support the following features:
31
+ - suffix (the language models we currently support do not support
32
+ suffix)
33
+ - logit_bias (to be supported by vLLM engine)
34
+ """
35
+
36
+ error_check_ret = await self._check_model(request)
37
+ if error_check_ret is not None:
38
+ return error_check_ret
39
+
40
+ # OpenAI API supports echoing the prompt when max_tokens is 0.
41
+ echo_without_generation = request.echo and request.max_tokens == 0
42
+
43
+ if request.suffix is not None:
44
+ # The language models we currently support do not support suffix.
45
+ return self.create_error_response(
46
+ "suffix is not currently supported")
47
+
48
+ if request.logit_bias is not None and len(request.logit_bias) > 0:
49
+ # TODO: support logit_bias in vLLM engine.
50
+ return self.create_error_response(
51
+ "logit_bias is not currently supported")
52
+
53
+ model_name = request.model
54
+ request_id = f"cmpl-{random_uuid()}"
55
+
56
+ use_token_ids = False
57
+ if isinstance(request.prompt, list):
58
+ if len(request.prompt) == 0:
59
+ return self.create_error_response(
60
+ "please provide at least one prompt")
61
+ first_element = request.prompt[0]
62
+ if isinstance(first_element, int):
63
+ use_token_ids = True
64
+ prompt = request.prompt
65
+ elif isinstance(first_element, (str, list)):
66
+ # TODO: handles multiple prompt case in list[list[int]]
67
+ if len(request.prompt) > 1:
68
+ return self.create_error_response(
69
+ "multiple prompts in a batch is not currently supported"
70
+ )
71
+ use_token_ids = not isinstance(first_element, str)
72
+ prompt = request.prompt[0]
73
+ else:
74
+ prompt = request.prompt
75
+
76
+ if use_token_ids:
77
+ _, error_check_ret = await self._check_length(request,
78
+ prompt_ids=prompt)
79
+ else:
80
+ token_ids, error_check_ret = await self._check_length(
81
+ request, prompt=prompt)
82
+ if error_check_ret is not None:
83
+ return error_check_ret
84
+
85
+ created_time = int(time.monotonic())
86
+ try:
87
+ spaces_between_special_tokens = request.spaces_between_special_tokens
88
+ sampling_params = SamplingParams(
89
+ n=request.n,
90
+ best_of=request.best_of,
91
+ presence_penalty=request.presence_penalty,
92
+ frequency_penalty=request.frequency_penalty,
93
+ repetition_penalty=request.repetition_penalty,
94
+ temperature=request.temperature,
95
+ top_p=request.top_p,
96
+ top_k=request.top_k,
97
+ min_p=request.min_p,
98
+ stop=request.stop,
99
+ stop_token_ids=request.stop_token_ids,
100
+ ignore_eos=request.ignore_eos,
101
+ max_tokens=request.max_tokens
102
+ if not echo_without_generation else 1,
103
+ logprobs=request.logprobs,
104
+ use_beam_search=request.use_beam_search,
105
+ prompt_logprobs=request.logprobs if request.echo else None,
106
+ skip_special_tokens=request.skip_special_tokens,
107
+ spaces_between_special_tokens=spaces_between_special_tokens,
108
+ )
109
+ except ValueError as e:
110
+ return self.create_error_response(str(e))
111
+
112
+ if use_token_ids:
113
+ result_generator = self.engine.generate(None,
114
+ sampling_params,
115
+ request_id,
116
+ prompt_token_ids=prompt)
117
+ else:
118
+ result_generator = self.engine.generate(prompt, sampling_params,
119
+ request_id, token_ids)
120
+
121
+ # Similar to the OpenAI API, when n != best_of, we do not stream the
122
+ # results. In addition, we do not stream the results when use beam search.
123
+ stream = (request.stream
124
+ and (request.best_of is None or request.n == request.best_of)
125
+ and not request.use_beam_search)
126
+
127
+ def create_stream_response_json(
128
+ index: int,
129
+ text: str,
130
+ logprobs: Optional[LogProbs] = None,
131
+ finish_reason: Optional[str] = None,
132
+ usage: Optional[UsageInfo] = None,
133
+ ) -> str:
134
+ choice_data = CompletionResponseStreamChoice(
135
+ index=index,
136
+ text=text,
137
+ logprobs=logprobs,
138
+ finish_reason=finish_reason,
139
+ )
140
+ response = CompletionStreamResponse(
141
+ id=request_id,
142
+ created=created_time,
143
+ model=model_name,
144
+ choices=[choice_data],
145
+ )
146
+ if usage is not None:
147
+ response.usage = usage
148
+ response_json = response.json(exclude_unset=True,
149
+ ensure_ascii=False)
150
+
151
+ return response_json
152
+
153
+ async def completion_stream_generator() -> AsyncGenerator[str, None]:
154
+ previous_texts = [""] * request.n
155
+ previous_num_tokens = [0] * request.n
156
+ has_echoed = [False] * request.n
157
+ async for res in result_generator:
158
+ res: RequestOutput
159
+ for output in res.outputs:
160
+ i = output.index
161
+ delta_text = output.text[len(previous_texts[i]):]
162
+ token_ids = output.token_ids[previous_num_tokens[i]:]
163
+ if request.logprobs is not None:
164
+ top_logprobs = output.logprobs[previous_num_tokens[i]:]
165
+ else:
166
+ top_logprobs = None
167
+ offsets = len(previous_texts[i])
168
+ if request.echo and not has_echoed[i]:
169
+ if not echo_without_generation:
170
+ delta_text = res.prompt + delta_text
171
+ token_ids = res.prompt_token_ids + token_ids
172
+ if top_logprobs:
173
+ top_logprobs = res.prompt_logprobs + top_logprobs
174
+ else: # only just return the prompt
175
+ delta_text = res.prompt
176
+ token_ids = res.prompt_token_ids
177
+ if top_logprobs:
178
+ top_logprobs = res.prompt_logprobs
179
+ has_echoed[i] = True
180
+ if request.logprobs is not None:
181
+ logprobs = self._create_logprobs(
182
+ token_ids=token_ids,
183
+ top_logprobs=top_logprobs,
184
+ num_output_top_logprobs=request.logprobs,
185
+ initial_text_offset=offsets,
186
+ )
187
+ else:
188
+ logprobs = None
189
+ previous_texts[i] = output.text
190
+ previous_num_tokens[i] = len(output.token_ids)
191
+ finish_reason = output.finish_reason
192
+ response_json = create_stream_response_json(
193
+ index=i,
194
+ text=delta_text,
195
+ logprobs=logprobs,
196
+ finish_reason=finish_reason,
197
+ )
198
+ yield f"data: {response_json}\n\n"
199
+ if output.finish_reason is not None:
200
+ logprobs = (LogProbs()
201
+ if request.logprobs is not None else None)
202
+ prompt_tokens = len(res.prompt_token_ids)
203
+ completion_tokens = len(output.token_ids)
204
+ final_usage = UsageInfo(
205
+ prompt_tokens=prompt_tokens,
206
+ completion_tokens=completion_tokens,
207
+ total_tokens=prompt_tokens + completion_tokens,
208
+ )
209
+ response_json = create_stream_response_json(
210
+ index=i,
211
+ text="",
212
+ logprobs=logprobs,
213
+ finish_reason=output.finish_reason,
214
+ usage=final_usage,
215
+ )
216
+ yield f"data: {response_json}\n\n"
217
+ yield "data: [DONE]\n\n"
218
+
219
+ # Streaming response
220
+ if stream:
221
+ return completion_stream_generator()
222
+
223
+ # Non-streaming response
224
+ final_res: RequestOutput = None
225
+ async for res in result_generator:
226
+ if await raw_request.is_disconnected():
227
+ # Abort the request if the client disconnects.
228
+ await self.engine.abort(request_id)
229
+ return self.create_error_response("Client disconnected")
230
+ final_res = res
231
+ assert final_res is not None
232
+ choices = []
233
+ prompt_token_ids = final_res.prompt_token_ids
234
+ prompt_logprobs = final_res.prompt_logprobs
235
+ prompt_text = final_res.prompt
236
+ for output in final_res.outputs:
237
+ if request.logprobs is not None:
238
+ if not echo_without_generation:
239
+ token_ids = output.token_ids
240
+ top_logprobs = output.logprobs
241
+ if request.echo:
242
+ token_ids = prompt_token_ids + token_ids
243
+ top_logprobs = prompt_logprobs + top_logprobs
244
+ else:
245
+ token_ids = prompt_token_ids
246
+ top_logprobs = prompt_logprobs
247
+ logprobs = self._create_logprobs(
248
+ token_ids=token_ids,
249
+ top_logprobs=top_logprobs,
250
+ num_output_top_logprobs=request.logprobs,
251
+ )
252
+ else:
253
+ logprobs = None
254
+ if not echo_without_generation:
255
+ output_text = output.text
256
+ if request.echo:
257
+ output_text = prompt_text + output_text
258
+ else:
259
+ output_text = prompt_text
260
+ choice_data = CompletionResponseChoice(
261
+ index=output.index,
262
+ text=output_text,
263
+ logprobs=logprobs,
264
+ finish_reason=output.finish_reason,
265
+ )
266
+ choices.append(choice_data)
267
+
268
+ num_prompt_tokens = len(final_res.prompt_token_ids)
269
+ num_generated_tokens = sum(
270
+ len(output.token_ids) for output in final_res.outputs)
271
+ usage = UsageInfo(
272
+ prompt_tokens=num_prompt_tokens,
273
+ completion_tokens=num_generated_tokens,
274
+ total_tokens=num_prompt_tokens + num_generated_tokens,
275
+ )
276
+ response = CompletionResponse(
277
+ id=request_id,
278
+ created=created_time,
279
+ model=model_name,
280
+ choices=choices,
281
+ usage=usage,
282
+ )
283
+
284
+ if request.stream:
285
+ # When user requests streaming but we don't stream, we still need to
286
+ # return a streaming response with a single event.
287
+ response_json = response.json(ensure_ascii=False)
288
+
289
+ async def fake_stream_generator() -> AsyncGenerator[str, None]:
290
+ yield f"data: {response_json}\n\n"
291
+ yield "data: [DONE]\n\n"
292
+
293
+ return fake_stream_generator()
294
+
295
+ return response
vllm/entrypoints/openai/serving_engine.py
new
+130
@@ -0,0 +1,130 @@
1
+import asyncio
2
+from http import HTTPStatus
3
+from typing import Dict, List, Optional, Tuple, Union
4
+from vllm.logger import init_logger
5
+from vllm.transformers_utils.tokenizer import get_tokenizer
6
+from vllm.engine.async_llm_engine import AsyncLLMEngine
7
+from vllm.entrypoints.openai.protocol import (CompletionRequest,
8
+ ChatCompletionRequest,
9
+ ErrorResponse, LogProbs,
10
+ ModelCard, ModelList,
11
+ ModelPermission)
12
+
13
+logger = init_logger(__name__)
14
+
15
+
16
+class OpenAIServing:
17
+
18
+ def __init__(self, engine: AsyncLLMEngine, served_model: str):
19
+ self.engine = engine
20
+ self.served_model = served_model
21
+
22
+ self.max_model_len = 0
23
+ self.tokenizer = None
24
+
25
+ try:
26
+ event_loop = asyncio.get_running_loop()
27
+ except RuntimeError:
28
+ event_loop = None
29
+
30
+ if event_loop is not None and event_loop.is_running(
31
+ ): # If the current is instanced by Ray Serve, there is already a running event loop
32
+ event_loop.create_task(self._post_init())
33
+ else: # When using single vLLM without engine_use_ray
34
+ asyncio.run(self._post_init())
35
+
36
+ async def _post_init(self):
37
+ engine_model_config = await self.engine.get_model_config()
38
+ self.max_model_len = engine_model_config.max_model_len
39
+
40
+ # A separate tokenizer to map token IDs to strings.
41
+ self.tokenizer = get_tokenizer(
42
+ engine_model_config.tokenizer,
43
+ tokenizer_mode=engine_model_config.tokenizer_mode,
44
+ trust_remote_code=engine_model_config.trust_remote_code)
45
+
46
+ async def show_available_models(self) -> ModelList:
47
+ """Show available models. Right now we only have one model."""
48
+ model_cards = [
49
+ ModelCard(id=self.served_model,
50
+ root=self.served_model,
51
+ permission=[ModelPermission()])
52
+ ]
53
+ return ModelList(data=model_cards)
54
+
55
+ def _create_logprobs(
56
+ self,
57
+ token_ids: List[int],
58
+ top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None,
59
+ num_output_top_logprobs: Optional[int] = None,
60
+ initial_text_offset: int = 0,
61
+ ) -> LogProbs:
62
+ """Create OpenAI-style logprobs."""
63
+ logprobs = LogProbs()
64
+ last_token_len = 0
65
+ if num_output_top_logprobs:
66
+ logprobs.top_logprobs = []
67
+ for i, token_id in enumerate(token_ids):
68
+ step_top_logprobs = top_logprobs[i]
69
+ if step_top_logprobs is not None:
70
+ token_logprob = step_top_logprobs[token_id]
71
+ else:
72
+ token_logprob = None
73
+ token = self.tokenizer.convert_ids_to_tokens(token_id)
74
+ logprobs.tokens.append(token)
75
+ logprobs.token_logprobs.append(token_logprob)
76
+ if len(logprobs.text_offset) == 0:
77
+ logprobs.text_offset.append(initial_text_offset)
78
+ else:
79
+ logprobs.text_offset.append(logprobs.text_offset[-1] +
80
+ last_token_len)
81
+ last_token_len = len(token)
82
+
83
+ if num_output_top_logprobs:
84
+ logprobs.top_logprobs.append({
85
+ self.tokenizer.convert_ids_to_tokens(i): p
86
+ for i, p in step_top_logprobs.items()
87
+ } if step_top_logprobs else None)
88
+ return logprobs
89
+
90
+ def create_error_response(
91
+ self,
92
+ message: str,
93
+ err_type: str = "BadRequestError",
94
+ status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse:
95
+ return ErrorResponse(message=message,
96
+ type=err_type,
97
+ code=status_code.value)
98
+
99
+ async def _check_model(self, request) -> Optional[ErrorResponse]:
100
+ if request.model == self.served_model:
101
+ return
102
+ return self.create_error_response(
103
+ message=f"The model `{request.model}` does not exist.",
104
+ err_type="NotFoundError",
105
+ status_code=HTTPStatus.NOT_FOUND)
106
+
107
+ async def _check_length(
108
+ self,
109
+ request: Union[ChatCompletionRequest, CompletionRequest],
110
+ prompt: Optional[str] = None,
111
+ prompt_ids: Optional[List[int]] = None
112
+ ) -> Tuple[List[int], Optional[ErrorResponse]]:
113
+ assert (not (prompt is None and prompt_ids is None)
114
+ and not (prompt is not None and prompt_ids is not None)
115
+ ), "Either prompt or prompt_ids should be provided."
116
+ input_ids = prompt_ids if prompt_ids is not None else self.tokenizer(
117
+ prompt).input_ids
118
+ token_num = len(input_ids)
119
+
120
+ if request.max_tokens is None:
121
+ request.max_tokens = self.max_model_len - token_num
122
+ if token_num + request.max_tokens > self.max_model_len:
123
+ return input_ids, self.create_error_response(
124
+ f"This model's maximum context length is {self.max_model_len} tokens. "
125
+ f"However, you requested {request.max_tokens + token_num} tokens "
126
+ f"({token_num} in the messages, "
127
+ f"{request.max_tokens} in the completion). "
128
+ f"Please reduce the length of the messages or completion.", )
129
+ else:
130
+ return input_ids, None