@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 14cc317ba4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

OpenAI Server refactoring (#2360)

FlorianJoncour committed Jan 17, 2024 at 05:33 UTC 14cc317ba48229d93ee2417822d96ccb8db56abe
8 files changed +954 -643
.buildkite/test-pipeline.yaml
+3
@@ -19,6 +19,9 @@ steps:
19 - label: Engine Test
20 command: pytest -v -s engine
21
22 +- label: Entrypoints Test
23 + command: pytest -v -s entrypoints
24 +
25 - label: Kernels Test
26 command: pytest -v -s kernels
27 soft_fail: true
requirements-dev.txt
+3
@@ -16,3 +16,6 @@ pytest-asyncio
16 httpx
17 einops # required for MPT
18 flash_attn # required for HuggingFace's llama implementation
19 +openai
20 +requests
21 +ray
\ No newline at end of file
tests/async_engine/test_chat_template.py renamed
+17 -19
@@ -1,12 +1,12 @@
1 -from argparse import Namespace
1 from dataclasses import dataclass
2 import os
3 import pathlib
4
5 import pytest
7 -from fastapi.testclient import TestClient
6
9 -from vllm.entrypoints.openai.api_server import *
7 +from vllm.transformers_utils.tokenizer import get_tokenizer
8 +from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
9 +from vllm.entrypoints.openai.protocol import ChatCompletionRequest
10
11 chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
12 __file__))).parent.parent / "examples/template_chatml.jinja"
@@ -48,7 +48,6 @@ TEST_MESSAGES = [
48 'content': 'What is the capital of'
49 },
50 ]
51 -client = TestClient(app)
51
52
53 @dataclass
@@ -56,13 +55,17 @@ class MockTokenizer:
55 chat_template = None
56
57
58 +@dataclass
59 +class MockServingChat:
60 + tokenizer: MockTokenizer
61 +
62 +
63 def test_load_chat_template():
64 # Testing chatml template
61 - mock_args = Namespace(chat_template=chatml_jinja_path)
65 tokenizer = MockTokenizer()
63 -
64 - # Call the function with the mocked args
65 - load_chat_template(mock_args, tokenizer)
66 + mock_serving_chat = MockServingChat(tokenizer)
67 + OpenAIServingChat._load_chat_template(mock_serving_chat,
68 + chat_template=chatml_jinja_path)
69
70 template_content = tokenizer.chat_template
71
@@ -76,11 +79,11 @@ def test_load_chat_template():
79 def test_no_load_chat_template():
80 # Testing chatml template
81 template = "../../examples/does_not_exist"
79 - mock_args = Namespace(chat_template=template)
82 tokenizer = MockTokenizer()
83
82 - # Call the function with the mocked args
83 - load_chat_template(mock_args, tokenizer=tokenizer)
84 + mock_serving_chat = MockServingChat(tokenizer)
85 + OpenAIServingChat._load_chat_template(mock_serving_chat,
86 + chat_template=template)
87 template_content = tokenizer.chat_template
88
89 # Test assertions
@@ -97,9 +100,9 @@ async def test_get_gen_prompt(model, template, add_generation_prompt,
100 expected_output):
101 # Initialize the tokenizer
102 tokenizer = get_tokenizer(tokenizer_name=model)
100 -
101 - mock_args = Namespace(chat_template=template)
102 - load_chat_template(mock_args, tokenizer)
103 + mock_serving_chat = MockServingChat(tokenizer)
104 + OpenAIServingChat._load_chat_template(mock_serving_chat,
105 + chat_template=template)
106
107 # Create a mock request object using keyword arguments
108 mock_request = ChatCompletionRequest(
@@ -115,8 +118,3 @@ async def test_get_gen_prompt(model, template, add_generation_prompt,
118
119 # Test assertion
120 assert result == expected_output, f"The generated prompt does not match the expected output for model {model} and template {template}"
118 -
119 -
120 -def test_health_endpoint():
121 - response = client.get("/health")
122 - assert response.status_code == 200
tests/entrypoints/test_openai_server.py new
+193
@@ -0,0 +1,193 @@
1 +import time
2 +import subprocess
3 +
4 +import sys
5 +import pytest
6 +import requests
7 +import ray # using Ray for overall ease of process management, parallel requests, and debugging.
8 +import openai # use the official client for correctness check
9 +
10 +MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
11 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta" # any model with a chat template should work here
12 +
13 +pytestmark = pytest.mark.asyncio
14 +
15 +
16 +@ray.remote(num_gpus=1)
17 +class ServerRunner:
18 +
19 + def __init__(self, args):
20 + self.proc = subprocess.Popen(
21 + ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
22 + stdout=sys.stdout,
23 + stderr=sys.stderr,
24 + )
25 + self._wait_for_server()
26 +
27 + def ready(self):
28 + return True
29 +
30 + def _wait_for_server(self):
31 + # run health check
32 + start = time.time()
33 + while True:
34 + try:
35 + if requests.get(
36 + "http://localhost:8000/health").status_code == 200:
37 + break
38 + except Exception as err:
39 + if self.proc.poll() is not None:
40 + raise RuntimeError("Server exited unexpectedly.") from err
41 +
42 + time.sleep(0.5)
43 + if time.time() - start > MAX_SERVER_START_WAIT_S:
44 + raise RuntimeError(
45 + "Server failed to start in time.") from err
46 +
47 + def __del__(self):
48 + if hasattr(self, "proc"):
49 + self.proc.terminate()
50 +
51 +
52 +@pytest.fixture(scope="session")
53 +def server():
54 + ray.init()
55 + server_runner = ServerRunner.remote([
56 + "--model",
57 + MODEL_NAME,
58 + "--dtype",
59 + "bfloat16", # use half precision for speed and memory savings in CI environment
60 + "--max-model-len",
61 + "8192"
62 + ])
63 + ray.get(server_runner.ready.remote())
64 + yield server_runner
65 + ray.shutdown()
66 +
67 +
68 +@pytest.fixture(scope="session")
69 +def client():
70 + client = openai.AsyncOpenAI(
71 + base_url="http://localhost:8000/v1",
72 + api_key="token-abc123",
73 + )
74 + yield client
75 +
76 +
77 +async def test_single_completion(server, client: openai.AsyncOpenAI):
78 + completion = await client.completions.create(model=MODEL_NAME,
79 + prompt="Hello, my name is",
80 + max_tokens=5,
81 + temperature=0.0)
82 +
83 + assert completion.id is not None
84 + assert completion.choices is not None and len(completion.choices) == 1
85 + assert completion.choices[0].text is not None and len(
86 + completion.choices[0].text) >= 5
87 + assert completion.choices[0].finish_reason == "length"
88 + assert completion.usage == openai.types.CompletionUsage(
89 + completion_tokens=5, prompt_tokens=6, total_tokens=11)
90 +
91 +
92 +async def test_single_chat_session(server, client: openai.AsyncOpenAI):
93 + messages = [{
94 + "role": "system",
95 + "content": "you are a helpful assistant"
96 + }, {
97 + "role": "user",
98 + "content": "what is 1+1?"
99 + }]
100 +
101 + # test single completion
102 + chat_completion = await client.chat.completions.create(
103 + model=MODEL_NAME,
104 + messages=messages,
105 + max_tokens=10,
106 + )
107 + assert chat_completion.id is not None
108 + assert chat_completion.choices is not None and len(
109 + chat_completion.choices) == 1
110 + assert chat_completion.choices[0].message is not None
111 + message = chat_completion.choices[0].message
112 + assert message.content is not None and len(message.content) >= 10
113 + assert message.role == "assistant"
114 + messages.append({"role": "assistant", "content": message.content})
115 +
116 + # test multi-turn dialogue
117 + messages.append({"role": "user", "content": "express your result in json"})
118 + chat_completion = await client.chat.completions.create(
119 + model=MODEL_NAME,
120 + messages=messages,
121 + max_tokens=10,
122 + )
123 + message = chat_completion.choices[0].message
124 + assert message.content is not None and len(message.content) >= 0
125 +
126 +
127 +async def test_completion_streaming(server, client: openai.AsyncOpenAI):
128 + prompt = "What is an LLM?"
129 +
130 + single_completion = await client.completions.create(
131 + model=MODEL_NAME,
132 + prompt=prompt,
133 + max_tokens=5,
134 + temperature=0.0,
135 + )
136 + single_output = single_completion.choices[0].text
137 + single_usage = single_completion.usage
138 +
139 + stream = await client.completions.create(
140 + model=MODEL_NAME,
141 + prompt=prompt,
142 + max_tokens=5,
143 + temperature=0.0,
144 + stream=True,
145 + )
146 + chunks = []
147 + async for chunk in stream:
148 + chunks.append(chunk.choices[0].text)
149 + assert chunk.choices[0].finish_reason == "length"
150 + assert chunk.usage == single_usage
151 + assert "".join(chunks) == single_output
152 +
153 +
154 +async def test_chat_streaming(server, client: openai.AsyncOpenAI):
155 + messages = [{
156 + "role": "system",
157 + "content": "you are a helpful assistant"
158 + }, {
159 + "role": "user",
160 + "content": "what is 1+1?"
161 + }]
162 +
163 + # test single completion
164 + chat_completion = await client.chat.completions.create(
165 + model=MODEL_NAME,
166 + messages=messages,
167 + max_tokens=10,
168 + temperature=0.0,
169 + )
170 + output = chat_completion.choices[0].message.content
171 + stop_reason = chat_completion.choices[0].finish_reason
172 +
173 + # test streaming
174 + stream = await client.chat.completions.create(
175 + model=MODEL_NAME,
176 + messages=messages,
177 + max_tokens=10,
178 + temperature=0.0,
179 + stream=True,
180 + )
181 + chunks = []
182 + async for chunk in stream:
183 + delta = chunk.choices[0].delta
184 + if delta.role:
185 + assert delta.role == "assistant"
186 + if delta.content:
187 + chunks.append(delta.content)
188 + assert chunk.choices[0].finish_reason == stop_reason
189 + assert "".join(chunks) == output
190 +
191 +
192 +if __name__ == "__main__":
193 + pytest.main([__file__])
vllm/entrypoints/openai/api_server.py
+25 -624
@@ -1,19 +1,12 @@
1 -# Adapted from
2 -# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py
3 -
1 import argparse
2 import asyncio
6 -import codecs
3 import json
8 -import time
4 from contextlib import asynccontextmanager
10 -from http import HTTPStatus
11 -from typing import AsyncGenerator, Dict, List, Optional, Tuple, Union
12 -
5 from aioprometheus import MetricsMiddleware
6 from aioprometheus.asgi.starlette import metrics
7 import fastapi
8 import uvicorn
9 +from http import HTTPStatus
10 from fastapi import Request
11 from fastapi.exceptions import RequestValidationError
12 from fastapi.middleware.cors import CORSMiddleware
@@ -22,26 +15,16 @@ from fastapi.responses import JSONResponse, StreamingResponse, Response
15 from vllm.engine.arg_utils import AsyncEngineArgs
16 from vllm.engine.async_llm_engine import AsyncLLMEngine
17 from vllm.engine.metrics import add_global_metrics_labels
25 -from vllm.entrypoints.openai.protocol import (
26 - CompletionRequest, CompletionResponse, CompletionResponseChoice,
27 - CompletionResponseStreamChoice, CompletionStreamResponse,
28 - ChatCompletionRequest, ChatCompletionResponse,
29 - ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
30 - ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
31 - LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo)
18 +from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest, ErrorResponse
19 from vllm.logger import init_logger
33 -from vllm.outputs import RequestOutput
34 -from vllm.sampling_params import SamplingParams
35 -from vllm.transformers_utils.tokenizer import get_tokenizer
36 -from vllm.utils import random_uuid
20 +from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
21 +from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
22
23 TIMEOUT_KEEP_ALIVE = 5 # seconds
24
25 +openai_serving_chat: OpenAIServingChat = None
26 +openai_serving_completion: OpenAIServingCompletion = None
27 logger = init_logger(__name__)
41 -served_model = None
42 -engine_args = None
43 -engine = None
44 -response_role = None
28
29
30 @asynccontextmanager
@@ -120,72 +103,10 @@ app.add_middleware(MetricsMiddleware) # Trace HTTP server metrics
103 app.add_route("/metrics", metrics) # Exposes HTTP metrics
104
105
123 -def create_error_response(status_code: HTTPStatus,
124 - message: str) -> JSONResponse:
125 - return JSONResponse(ErrorResponse(message=message,
126 - type="invalid_request_error").dict(),
127 - status_code=status_code.value)
128 -
129 -
130 -def load_chat_template(args, tokenizer):
131 - if args.chat_template is not None:
132 - try:
133 - with open(args.chat_template, "r") as f:
134 - chat_template = f.read()
135 - except OSError:
136 - # If opening a file fails, set chat template to be args to
137 - # ensure we decode so our escape are interpreted correctly
138 - chat_template = codecs.decode(args.chat_template, "unicode_escape")
139 -
140 - tokenizer.chat_template = chat_template
141 - logger.info(
142 - f"Using supplied chat template:\n{tokenizer.chat_template}")
143 - elif tokenizer.chat_template is not None:
144 - logger.info(f"Using default chat template:\n{tokenizer.chat_template}")
145 - else:
146 - logger.warning("No chat template provided. Chat API will not work.")
147 -
148 -
106 @app.exception_handler(RequestValidationError)
107 async def validation_exception_handler(_, exc):
151 - return create_error_response(HTTPStatus.BAD_REQUEST, str(exc))
152 -
153 -
154 -async def check_model(request) -> Optional[JSONResponse]:
155 - if request.model == served_model:
156 - return
157 - ret = create_error_response(
158 - HTTPStatus.NOT_FOUND,
159 - f"The model `{request.model}` does not exist.",
160 - )
161 - return ret
162 -
163 -
164 -async def check_length(
165 - request: Union[ChatCompletionRequest, CompletionRequest],
166 - prompt: Optional[str] = None,
167 - prompt_ids: Optional[List[int]] = None
168 -) -> Tuple[List[int], Optional[JSONResponse]]:
169 - assert (not (prompt is None and prompt_ids is None)
170 - and not (prompt is not None and prompt_ids is not None)
171 - ), "Either prompt or prompt_ids should be provided."
172 - input_ids = prompt_ids if prompt_ids is not None else tokenizer(
173 - prompt).input_ids
174 - token_num = len(input_ids)
175 -
176 - if request.max_tokens is None:
177 - request.max_tokens = max_model_len - token_num
178 - if token_num + request.max_tokens > max_model_len:
179 - return input_ids, create_error_response(
180 - HTTPStatus.BAD_REQUEST,
181 - f"This model's maximum context length is {max_model_len} tokens. "
182 - f"However, you requested {request.max_tokens + token_num} tokens "
183 - f"({token_num} in the messages, "
184 - f"{request.max_tokens} in the completion). "
185 - f"Please reduce the length of the messages or completion.",
186 - )
187 - else:
188 - return input_ids, None
108 + err = openai_serving_chat.create_error_response(message=str(exc))
109 + return JSONResponse(err.dict(), status_code=HTTPStatus.BAD_REQUEST)
110
111
112 @app.get("/health")
@@ -196,544 +117,31 @@ async def health() -> Response:
117
118 @app.get("/v1/models")
119 async def show_available_models():
199 - """Show available models. Right now we only have one model."""
200 - model_cards = [
201 - ModelCard(id=served_model,
202 - root=served_model,
203 - permission=[ModelPermission()])
204 - ]
205 - return ModelList(data=model_cards)
206 -
207 -
208 -def create_logprobs(
209 - token_ids: List[int],
210 - top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None,
211 - num_output_top_logprobs: Optional[int] = None,
212 - initial_text_offset: int = 0,
213 -) -> LogProbs:
214 - """Create OpenAI-style logprobs."""
215 - logprobs = LogProbs()
216 - last_token_len = 0
217 - if num_output_top_logprobs:
218 - logprobs.top_logprobs = []
219 - for i, token_id in enumerate(token_ids):
220 - step_top_logprobs = top_logprobs[i]
221 - if step_top_logprobs is not None:
222 - token_logprob = step_top_logprobs[token_id]
223 - else:
224 - token_logprob = None
225 - token = tokenizer.convert_ids_to_tokens(token_id)
226 - logprobs.tokens.append(token)
227 - logprobs.token_logprobs.append(token_logprob)
228 - if len(logprobs.text_offset) == 0:
229 - logprobs.text_offset.append(initial_text_offset)
230 - else:
231 - logprobs.text_offset.append(logprobs.text_offset[-1] +
232 - last_token_len)
233 - last_token_len = len(token)
234 -
235 - if num_output_top_logprobs:
236 - logprobs.top_logprobs.append({
237 - tokenizer.convert_ids_to_tokens(i): p
238 - for i, p in step_top_logprobs.items()
239 - } if step_top_logprobs else None)
240 - return logprobs
120 + models = await openai_serving_chat.show_available_models()
121 + return JSONResponse(content=models.dict())
122
123
124 @app.post("/v1/chat/completions")
125 async def create_chat_completion(request: ChatCompletionRequest,
126 raw_request: Request):
246 - """Completion API similar to OpenAI's API.
247 -
248 - See https://platform.openai.com/docs/api-reference/chat/create
249 - for the API specification. This API mimics the OpenAI ChatCompletion API.
250 -
251 - NOTE: Currently we do not support the following features:
252 - - function_call (Users should implement this by themselves)
253 - - logit_bias (to be supported by vLLM engine)
254 - """
255 - error_check_ret = await check_model(request)
256 - if error_check_ret is not None:
257 - return error_check_ret
258 -
259 - if request.logit_bias is not None and len(request.logit_bias) > 0:
260 - # TODO: support logit_bias in vLLM engine.
261 - return create_error_response(HTTPStatus.BAD_REQUEST,
262 - "logit_bias is not currently supported")
263 -
264 - try:
265 - prompt = tokenizer.apply_chat_template(
266 - conversation=request.messages,
267 - tokenize=False,
268 - add_generation_prompt=request.add_generation_prompt)
269 - except Exception as e:
270 - logger.error(f"Error in applying chat template from request: {str(e)}")
271 - return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
272 -
273 - token_ids, error_check_ret = await check_length(request, prompt=prompt)
274 - if error_check_ret is not None:
275 - return error_check_ret
276 -
277 - model_name = request.model
278 - request_id = f"cmpl-{random_uuid()}"
279 - created_time = int(time.monotonic())
280 - chunk_object_type = "chat.completion.chunk"
281 - try:
282 - spaces_between_special_tokens = request.spaces_between_special_tokens
283 - sampling_params = SamplingParams(
284 - n=request.n,
285 - presence_penalty=request.presence_penalty,
286 - frequency_penalty=request.frequency_penalty,
287 - repetition_penalty=request.repetition_penalty,
288 - temperature=request.temperature,
289 - top_p=request.top_p,
290 - min_p=request.min_p,
291 - stop=request.stop,
292 - stop_token_ids=request.stop_token_ids,
293 - max_tokens=request.max_tokens,
294 - best_of=request.best_of,
295 - top_k=request.top_k,
296 - ignore_eos=request.ignore_eos,
297 - use_beam_search=request.use_beam_search,
298 - skip_special_tokens=request.skip_special_tokens,
299 - spaces_between_special_tokens=spaces_between_special_tokens,
300 - )
301 - except ValueError as e:
302 - return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
303 -
304 - result_generator = engine.generate(prompt, sampling_params, request_id,
305 - token_ids)
306 -
307 - def get_role() -> str:
308 - if request.add_generation_prompt:
309 - return response_role
310 - else:
311 - return request.messages[-1]["role"]
312 -
313 - async def completion_stream_generator() -> AsyncGenerator[str, None]:
314 - # Send first response for each request.n (index) with the role
315 - role = get_role()
316 - for i in range(request.n):
317 - choice_data = ChatCompletionResponseStreamChoice(
318 - index=i, delta=DeltaMessage(role=role), finish_reason=None)
319 - chunk = ChatCompletionStreamResponse(id=request_id,
320 - object=chunk_object_type,
321 - created=created_time,
322 - choices=[choice_data],
323 - model=model_name)
324 - data = chunk.json(exclude_unset=True, ensure_ascii=False)
325 - yield f"data: {data}\n\n"
326 -
327 - # Send response to echo the input portion of the last message
328 - if request.echo:
329 - last_msg_content = ""
330 - if request.messages and isinstance(
331 - request.messages, list) and request.messages[-1].get(
332 - "content") and request.messages[-1].get(
333 - "role") == role:
334 - last_msg_content = request.messages[-1]["content"]
335 - if last_msg_content:
336 - for i in range(request.n):
337 - choice_data = ChatCompletionResponseStreamChoice(
338 - index=i,
339 - delta=DeltaMessage(content=last_msg_content),
340 - finish_reason=None)
341 - chunk = ChatCompletionStreamResponse(
342 - id=request_id,
343 - object=chunk_object_type,
344 - created=created_time,
345 - choices=[choice_data],
346 - model=model_name)
347 - data = chunk.json(exclude_unset=True, ensure_ascii=False)
348 - yield f"data: {data}\n\n"
349 -
350 - # Send response for each token for each request.n (index)
351 - previous_texts = [""] * request.n
352 - previous_num_tokens = [0] * request.n
353 - finish_reason_sent = [False] * request.n
354 - async for res in result_generator:
355 - res: RequestOutput
356 - for output in res.outputs:
357 - i = output.index
358 -
359 - if finish_reason_sent[i]:
360 - continue
361 -
362 - if output.finish_reason is None:
363 - # Send token-by-token response for each request.n
364 - delta_text = output.text[len(previous_texts[i]):]
365 - previous_texts[i] = output.text
366 - previous_num_tokens[i] = len(output.token_ids)
367 - choice_data = ChatCompletionResponseStreamChoice(
368 - index=i,
369 - delta=DeltaMessage(content=delta_text),
370 - finish_reason=None)
371 - chunk = ChatCompletionStreamResponse(
372 - id=request_id,
373 - object=chunk_object_type,
374 - created=created_time,
375 - choices=[choice_data],
376 - model=model_name)
377 - data = chunk.json(exclude_unset=True, ensure_ascii=False)
378 - yield f"data: {data}\n\n"
379 - else:
380 - # Send the finish response for each request.n only once
381 - prompt_tokens = len(res.prompt_token_ids)
382 - final_usage = UsageInfo(
383 - prompt_tokens=prompt_tokens,
384 - completion_tokens=previous_num_tokens[i],
385 - total_tokens=prompt_tokens + previous_num_tokens[i],
386 - )
387 - choice_data = ChatCompletionResponseStreamChoice(
388 - index=i, delta=[], finish_reason=output.finish_reason)
389 - chunk = ChatCompletionStreamResponse(
390 - id=request_id,
391 - object=chunk_object_type,
392 - created=created_time,
393 - choices=[choice_data],
394 - model=model_name)
395 - if final_usage is not None:
396 - chunk.usage = final_usage
397 - data = chunk.json(exclude_unset=True,
398 - exclude_none=True,
399 - ensure_ascii=False)
400 - yield f"data: {data}\n\n"
401 - finish_reason_sent[i] = True
402 - # Send the final done message after all response.n are finished
403 - yield "data: [DONE]\n\n"
404 -
405 - async def completion_full_generator():
406 - final_res: RequestOutput = None
407 - async for res in result_generator:
408 - if await raw_request.is_disconnected():
409 - # Abort the request if the client disconnects.
410 - await engine.abort(request_id)
411 - return create_error_response(HTTPStatus.BAD_REQUEST,
412 - "Client disconnected")
413 - final_res = res
414 - assert final_res is not None
415 -
416 - choices = []
417 - role = get_role()
418 - for output in final_res.outputs:
419 - choice_data = ChatCompletionResponseChoice(
420 - index=output.index,
421 - message=ChatMessage(role=role, content=output.text),
422 - finish_reason=output.finish_reason,
423 - )
424 - choices.append(choice_data)
425 -
426 - if request.echo:
427 - last_msg_content = ""
428 - if request.messages and isinstance(
429 - request.messages, list) and request.messages[-1].get(
430 - "content") and request.messages[-1].get(
431 - "role") == role:
432 - last_msg_content = request.messages[-1]["content"]
433 -
434 - for choice in choices:
435 - full_message = last_msg_content + choice.message.content
436 - choice.message.content = full_message
437 -
438 - num_prompt_tokens = len(final_res.prompt_token_ids)
439 - num_generated_tokens = sum(
440 - len(output.token_ids) for output in final_res.outputs)
441 - usage = UsageInfo(
442 - prompt_tokens=num_prompt_tokens,
443 - completion_tokens=num_generated_tokens,
444 - total_tokens=num_prompt_tokens + num_generated_tokens,
445 - )
446 - response = ChatCompletionResponse(
447 - id=request_id,
448 - created=created_time,
449 - model=model_name,
450 - choices=choices,
451 - usage=usage,
452 - )
453 -
454 - return response
455 -
456 - # Streaming response
457 - if request.stream:
458 - return StreamingResponse(completion_stream_generator(),
127 + generator = await openai_serving_chat.create_chat_completion(
128 + request, raw_request)
129 + if request.stream and not isinstance(generator, ErrorResponse):
130 + return StreamingResponse(content=generator,
131 media_type="text/event-stream")
132 else:
461 - return await completion_full_generator()
133 + return JSONResponse(content=generator.dict())
134
135
136 @app.post("/v1/completions")
137 async def create_completion(request: CompletionRequest, raw_request: Request):
466 - """Completion API similar to OpenAI's API.
467 -
468 - See https://platform.openai.com/docs/api-reference/completions/create
469 - for the API specification. This API mimics the OpenAI Completion API.
470 -
471 - NOTE: Currently we do not support the following features:
472 - - suffix (the language models we currently support do not support
473 - suffix)
474 - - logit_bias (to be supported by vLLM engine)
475 - """
476 -
477 - error_check_ret = await check_model(request)
478 - if error_check_ret is not None:
479 - return error_check_ret
480 -
481 - # OpenAI API supports echoing the prompt when max_tokens is 0.
482 - echo_without_generation = request.echo and request.max_tokens == 0
483 -
484 - if request.suffix is not None:
485 - # The language models we currently support do not support suffix.
486 - return create_error_response(HTTPStatus.BAD_REQUEST,
487 - "suffix is not currently supported")
488 -
489 - if request.logit_bias is not None and len(request.logit_bias) > 0:
490 - # TODO: support logit_bias in vLLM engine.
491 - return create_error_response(HTTPStatus.BAD_REQUEST,
492 - "logit_bias is not currently supported")
493 -
494 - model_name = request.model
495 - request_id = f"cmpl-{random_uuid()}"
496 -
497 - use_token_ids = False
498 - if isinstance(request.prompt, list):
499 - if len(request.prompt) == 0:
500 - return create_error_response(HTTPStatus.BAD_REQUEST,
501 - "please provide at least one prompt")
502 - first_element = request.prompt[0]
503 - if isinstance(first_element, int):
504 - use_token_ids = True
505 - prompt = request.prompt
506 - elif isinstance(first_element, (str, list)):
507 - # TODO: handles multiple prompt case in list[list[int]]
508 - if len(request.prompt) > 1:
509 - return create_error_response(
510 - HTTPStatus.BAD_REQUEST,
511 - "multiple prompts in a batch is not currently supported")
512 - use_token_ids = not isinstance(first_element, str)
513 - prompt = request.prompt[0]
514 - else:
515 - prompt = request.prompt
516 -
517 - if use_token_ids:
518 - _, error_check_ret = await check_length(request, prompt_ids=prompt)
519 - else:
520 - token_ids, error_check_ret = await check_length(request, prompt=prompt)
521 - if error_check_ret is not None:
522 - return error_check_ret
523 -
524 - created_time = int(time.monotonic())
525 - try:
526 - spaces_between_special_tokens = request.spaces_between_special_tokens
527 - sampling_params = SamplingParams(
528 - n=request.n,
529 - best_of=request.best_of,
530 - presence_penalty=request.presence_penalty,
531 - frequency_penalty=request.frequency_penalty,
532 - repetition_penalty=request.repetition_penalty,
533 - temperature=request.temperature,
534 - top_p=request.top_p,
535 - top_k=request.top_k,
536 - min_p=request.min_p,
537 - stop=request.stop,
538 - stop_token_ids=request.stop_token_ids,
539 - ignore_eos=request.ignore_eos,
540 - max_tokens=request.max_tokens
541 - if not echo_without_generation else 1,
542 - logprobs=request.logprobs,
543 - use_beam_search=request.use_beam_search,
544 - prompt_logprobs=request.logprobs if request.echo else None,
545 - skip_special_tokens=request.skip_special_tokens,
546 - spaces_between_special_tokens=spaces_between_special_tokens,
547 - )
548 - except ValueError as e:
549 - return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
550 -
551 - if use_token_ids:
552 - result_generator = engine.generate(None,
553 - sampling_params,
554 - request_id,
555 - prompt_token_ids=prompt)
556 - else:
557 - result_generator = engine.generate(prompt, sampling_params, request_id,
558 - token_ids)
559 -
560 - # Similar to the OpenAI API, when n != best_of, we do not stream the
561 - # results. In addition, we do not stream the results when use beam search.
562 - stream = (request.stream
563 - and (request.best_of is None or request.n == request.best_of)
564 - and not request.use_beam_search)
565 -
566 - def create_stream_response_json(
567 - index: int,
568 - text: str,
569 - logprobs: Optional[LogProbs] = None,
570 - finish_reason: Optional[str] = None,
571 - usage: Optional[UsageInfo] = None,
572 - ) -> str:
573 - choice_data = CompletionResponseStreamChoice(
574 - index=index,
575 - text=text,
576 - logprobs=logprobs,
577 - finish_reason=finish_reason,
578 - )
579 - response = CompletionStreamResponse(
580 - id=request_id,
581 - created=created_time,
582 - model=model_name,
583 - choices=[choice_data],
584 - )
585 - if usage is not None:
586 - response.usage = usage
587 - response_json = response.json(exclude_unset=True, ensure_ascii=False)
588 -
589 - return response_json
590 -
591 - async def completion_stream_generator() -> AsyncGenerator[str, None]:
592 - previous_texts = [""] * request.n
593 - previous_num_tokens = [0] * request.n
594 - has_echoed = [False] * request.n
595 - async for res in result_generator:
596 - res: RequestOutput
597 - for output in res.outputs:
598 - i = output.index
599 - delta_text = output.text[len(previous_texts[i]):]
600 - token_ids = output.token_ids[previous_num_tokens[i]:]
601 - if request.logprobs is not None:
602 - top_logprobs = output.logprobs[previous_num_tokens[i]:]
603 - else:
604 - top_logprobs = None
605 - offsets = len(previous_texts[i])
606 - if request.echo and not has_echoed[i]:
607 - if not echo_without_generation:
608 - delta_text = res.prompt + delta_text
609 - token_ids = res.prompt_token_ids + token_ids
610 - if top_logprobs:
611 - top_logprobs = res.prompt_logprobs + top_logprobs
612 - else: # only just return the prompt
613 - delta_text = res.prompt
614 - token_ids = res.prompt_token_ids
615 - if top_logprobs:
616 - top_logprobs = res.prompt_logprobs
617 - has_echoed[i] = True
618 - if request.logprobs is not None:
619 - logprobs = create_logprobs(
620 - token_ids=token_ids,
621 - top_logprobs=top_logprobs,
622 - num_output_top_logprobs=request.logprobs,
623 - initial_text_offset=offsets,
624 - )
625 - else:
626 - logprobs = None
627 - previous_texts[i] = output.text
628 - previous_num_tokens[i] = len(output.token_ids)
629 - finish_reason = output.finish_reason
630 - response_json = create_stream_response_json(
631 - index=i,
632 - text=delta_text,
633 - logprobs=logprobs,
634 - finish_reason=finish_reason,
635 - )
636 - yield f"data: {response_json}\n\n"
637 - if output.finish_reason is not None:
638 - logprobs = (LogProbs()
639 - if request.logprobs is not None else None)
640 - prompt_tokens = len(res.prompt_token_ids)
641 - completion_tokens = len(output.token_ids)
642 - final_usage = UsageInfo(
643 - prompt_tokens=prompt_tokens,
644 - completion_tokens=completion_tokens,
645 - total_tokens=prompt_tokens + completion_tokens,
646 - )
647 - response_json = create_stream_response_json(
648 - index=i,
649 - text="",
650 - logprobs=logprobs,
651 - finish_reason=output.finish_reason,
652 - usage=final_usage,
653 - )
654 - yield f"data: {response_json}\n\n"
655 - yield "data: [DONE]\n\n"
656 -
657 - # Streaming response
658 - if stream:
659 - return StreamingResponse(completion_stream_generator(),
660 - media_type="text/event-stream")
661 -
662 - # Non-streaming response
663 - final_res: RequestOutput = None
664 - async for res in result_generator:
665 - if await raw_request.is_disconnected():
666 - # Abort the request if the client disconnects.
667 - await engine.abort(request_id)
668 - return create_error_response(HTTPStatus.BAD_REQUEST,
669 - "Client disconnected")
670 - final_res = res
671 - assert final_res is not None
672 - choices = []
673 - prompt_token_ids = final_res.prompt_token_ids
674 - prompt_logprobs = final_res.prompt_logprobs
675 - prompt_text = final_res.prompt
676 - for output in final_res.outputs:
677 - if request.logprobs is not None:
678 - if not echo_without_generation:
679 - token_ids = output.token_ids
680 - top_logprobs = output.logprobs
681 - if request.echo:
682 - token_ids = prompt_token_ids + token_ids
683 - top_logprobs = prompt_logprobs + top_logprobs
684 - else:
685 - token_ids = prompt_token_ids
686 - top_logprobs = prompt_logprobs
687 - logprobs = create_logprobs(
688 - token_ids=token_ids,
689 - top_logprobs=top_logprobs,
690 - num_output_top_logprobs=request.logprobs,
691 - )
692 - else:
693 - logprobs = None
694 - if not echo_without_generation:
695 - output_text = output.text
696 - if request.echo:
697 - output_text = prompt_text + output_text
698 - else:
699 - output_text = prompt_text
700 - choice_data = CompletionResponseChoice(
701 - index=output.index,
702 - text=output_text,
703 - logprobs=logprobs,
704 - finish_reason=output.finish_reason,
705 - )
706 - choices.append(choice_data)
707 -
708 - num_prompt_tokens = len(final_res.prompt_token_ids)
709 - num_generated_tokens = sum(
710 - len(output.token_ids) for output in final_res.outputs)
711 - usage = UsageInfo(
712 - prompt_tokens=num_prompt_tokens,
713 - completion_tokens=num_generated_tokens,
714 - total_tokens=num_prompt_tokens + num_generated_tokens,
715 - )
716 - response = CompletionResponse(
717 - id=request_id,
718 - created=created_time,
719 - model=model_name,
720 - choices=choices,
721 - usage=usage,
722 - )
723 -
724 - if request.stream:
725 - # When user requests streaming but we don't stream, we still need to
726 - # return a streaming response with a single event.
727 - response_json = response.json(ensure_ascii=False)
728 -
729 - async def fake_stream_generator() -> AsyncGenerator[str, None]:
730 - yield f"data: {response_json}\n\n"
731 - yield "data: [DONE]\n\n"
732 -
733 - return StreamingResponse(fake_stream_generator(),
138 + generator = await openai_serving_completion.create_completion(
139 + request, raw_request)
140 + if request.stream and not isinstance(generator, ErrorResponse):
141 + return StreamingResponse(content=generator,
142 media_type="text/event-stream")
735 -
736 - return response
143 + else:
144 + return JSONResponse(content=generator.dict())
145
146
147 if __name__ == "__main__":
@@ -754,19 +162,12 @@ if __name__ == "__main__":
162 else:
163 served_model = args.model
164
757 - response_role = args.response_role
758 -
165 engine_args = AsyncEngineArgs.from_cli_args(args)
166 engine = AsyncLLMEngine.from_engine_args(engine_args)
761 - engine_model_config = asyncio.run(engine.get_model_config())
762 - max_model_len = engine_model_config.max_model_len
763 -
764 - # A separate tokenizer to map token IDs to strings.
765 - tokenizer = get_tokenizer(
766 - engine_model_config.tokenizer,
767 - tokenizer_mode=engine_model_config.tokenizer_mode,
768 - trust_remote_code=engine_model_config.trust_remote_code)
769 - load_chat_template(args, tokenizer)
167 + openai_serving_chat = OpenAIServingChat(engine, served_model,
168 + args.response_role,
169 + args.chat_template)
170 + openai_serving_completion = OpenAIServingCompletion(engine, served_model)
171
172 # Register labels for metrics
173 add_global_metrics_labels(model_name=engine_args.model)
vllm/entrypoints/openai/serving_chat.py new
+288
@@ -0,0 +1,288 @@
1 +import time
2 +import codecs
3 +from fastapi import Request
4 +from typing import AsyncGenerator, AsyncIterator, Union
5 +from vllm.logger import init_logger
6 +from vllm.utils import random_uuid
7 +from vllm.engine.async_llm_engine import AsyncLLMEngine
8 +from vllm.entrypoints.openai.protocol import (
9 + ChatCompletionRequest, ChatCompletionResponse,
10 + ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
11 + ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
12 + UsageInfo)
13 +from vllm.outputs import RequestOutput
14 +from vllm.sampling_params import SamplingParams
15 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
16 +
17 +logger = init_logger(__name__)
18 +
19 +
20 +class OpenAIServingChat(OpenAIServing):
21 +
22 + def __init__(self,
23 + engine: AsyncLLMEngine,
24 + served_model: str,
25 + response_role: str,
26 + chat_template=None):
27 + super().__init__(engine=engine, served_model=served_model)
28 + self.response_role = response_role
29 + self._load_chat_template(chat_template)
30 +
31 + async def create_chat_completion(
32 + self, request: ChatCompletionRequest, raw_request: Request
33 + ) -> Union[ErrorResponse, AsyncGenerator[str, None],
34 + ChatCompletionResponse]:
35 + """Completion API similar to OpenAI's API.
36 +
37 + See https://platform.openai.com/docs/api-reference/chat/create
38 + for the API specification. This API mimics the OpenAI ChatCompletion API.
39 +
40 + NOTE: Currently we do not support the following features:
41 + - function_call (Users should implement this by themselves)
42 + - logit_bias (to be supported by vLLM engine)
43 + """
44 + error_check_ret = await self._check_model(request)
45 + if error_check_ret is not None:
46 + return error_check_ret
47 +
48 + if request.logit_bias is not None and len(request.logit_bias) > 0:
49 + # TODO: support logit_bias in vLLM engine.
50 + return self.create_error_response(
51 + "logit_bias is not currently supported")
52 +
53 + try:
54 + prompt = self.tokenizer.apply_chat_template(
55 + conversation=request.messages,
56 + tokenize=False,
57 + add_generation_prompt=request.add_generation_prompt)
58 + except Exception as e:
59 + logger.error(
60 + f"Error in applying chat template from request: {str(e)}")
61 + return self.create_error_response(str(e))
62 +
63 + token_ids, error_check_ret = await self._check_length(request,
64 + prompt=prompt)
65 + if error_check_ret is not None:
66 + return error_check_ret
67 +
68 + request_id = f"cmpl-{random_uuid()}"
69 + try:
70 + spaces_between_special_tokens = request.spaces_between_special_tokens
71 + sampling_params = SamplingParams(
72 + n=request.n,
73 + presence_penalty=request.presence_penalty,
74 + frequency_penalty=request.frequency_penalty,
75 + repetition_penalty=request.repetition_penalty,
76 + temperature=request.temperature,
77 + top_p=request.top_p,
78 + min_p=request.min_p,
79 + stop=request.stop,
80 + stop_token_ids=request.stop_token_ids,
81 + max_tokens=request.max_tokens,
82 + best_of=request.best_of,
83 + top_k=request.top_k,
84 + ignore_eos=request.ignore_eos,
85 + use_beam_search=request.use_beam_search,
86 + skip_special_tokens=request.skip_special_tokens,
87 + spaces_between_special_tokens=spaces_between_special_tokens,
88 + )
89 + except ValueError as e:
90 + return self.create_error_response(str(e))
91 +
92 + result_generator = self.engine.generate(prompt, sampling_params,
93 + request_id, token_ids)
94 + # Streaming response
95 + if request.stream:
96 + return self.chat_completion_stream_generator(
97 + request, result_generator, request_id)
98 + else:
99 + return await self.chat_completion_full_generator(
100 + request, raw_request, result_generator, request_id)
101 +
102 + def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
103 + if request.add_generation_prompt:
104 + return self.response_role
105 + else:
106 + return request.messages[-1].role
107 +
108 + async def chat_completion_stream_generator(
109 + self, request: ChatCompletionRequest,
110 + result_generator: AsyncIterator[RequestOutput], request_id: str
111 + ) -> Union[ErrorResponse, AsyncGenerator[str, None]]:
112 +
113 + model_name = request.model
114 + created_time = int(time.monotonic())
115 + chunk_object_type = "chat.completion.chunk"
116 +
117 + # Send first response for each request.n (index) with the role
118 + role = self.get_chat_request_role(request)
119 + for i in range(request.n):
120 + choice_data = ChatCompletionResponseStreamChoice(
121 + index=i, delta=DeltaMessage(role=role), finish_reason=None)
122 + chunk = ChatCompletionStreamResponse(id=request_id,
123 + object=chunk_object_type,
124 + created=created_time,
125 + choices=[choice_data],
126 + model=model_name)
127 + data = chunk.json(exclude_unset=True, ensure_ascii=False)
128 + yield f"data: {data}\n\n"
129 +
130 + # Send response to echo the input portion of the last message
131 + if request.echo:
132 + last_msg_content = ""
133 + if request.messages and isinstance(
134 + request.messages, list) and request.messages[-1].get(
135 + "content") and request.messages[-1].get(
136 + "role") == role:
137 + last_msg_content = request.messages[-1]["content"]
138 + if last_msg_content:
139 + for i in range(request.n):
140 + choice_data = ChatCompletionResponseStreamChoice(
141 + index=i,
142 + delta=DeltaMessage(content=last_msg_content),
143 + finish_reason=None)
144 + chunk = ChatCompletionStreamResponse(
145 + id=request_id,
146 + object=chunk_object_type,
147 + created=created_time,
148 + choices=[choice_data],
149 + model=model_name)
150 + data = chunk.json(exclude_unset=True, ensure_ascii=False)
151 + yield f"data: {data}\n\n"
152 +
153 + # Send response for each token for each request.n (index)
154 + previous_texts = [""] * request.n
155 + previous_num_tokens = [0] * request.n
156 + finish_reason_sent = [False] * request.n
157 + async for res in result_generator:
158 + res: RequestOutput
159 + for output in res.outputs:
160 + i = output.index
161 +
162 + if finish_reason_sent[i]:
163 + continue
164 +
165 + delta_text = output.text[len(previous_texts[i]):]
166 + previous_texts[i] = output.text
167 + previous_num_tokens[i] = len(output.token_ids)
168 +
169 + if output.finish_reason is None:
170 + # Send token-by-token response for each request.n
171 + choice_data = ChatCompletionResponseStreamChoice(
172 + index=i,
173 + delta=DeltaMessage(content=delta_text),
174 + finish_reason=None)
175 + chunk = ChatCompletionStreamResponse(
176 + id=request_id,
177 + object=chunk_object_type,
178 + created=created_time,
179 + choices=[choice_data],
180 + model=model_name)
181 + data = chunk.json(exclude_unset=True, ensure_ascii=False)
182 + yield f"data: {data}\n\n"
183 + else:
184 + # Send the finish response for each request.n only once
185 + prompt_tokens = len(res.prompt_token_ids)
186 + final_usage = UsageInfo(
187 + prompt_tokens=prompt_tokens,
188 + completion_tokens=previous_num_tokens[i],
189 + total_tokens=prompt_tokens + previous_num_tokens[i],
190 + )
191 + choice_data = ChatCompletionResponseStreamChoice(
192 + index=i,
193 + delta=DeltaMessage(content=delta_text),
194 + finish_reason=output.finish_reason)
195 + chunk = ChatCompletionStreamResponse(
196 + id=request_id,
197 + object=chunk_object_type,
198 + created=created_time,
199 + choices=[choice_data],
200 + model=model_name)
201 + if final_usage is not None:
202 + chunk.usage = final_usage
203 + data = chunk.json(exclude_unset=True,
204 + exclude_none=True,
205 + ensure_ascii=False)
206 + yield f"data: {data}\n\n"
207 + finish_reason_sent[i] = True
208 + # Send the final done message after all response.n are finished
209 + yield "data: [DONE]\n\n"
210 +
211 + async def chat_completion_full_generator(
212 + self, request: ChatCompletionRequest, raw_request: Request,
213 + result_generator: AsyncIterator[RequestOutput],
214 + request_id: str) -> Union[ErrorResponse, ChatCompletionResponse]:
215 +
216 + model_name = request.model
217 + created_time = int(time.monotonic())
218 + final_res: RequestOutput = None
219 +
220 + async for res in result_generator:
221 + if await raw_request.is_disconnected():
222 + # Abort the request if the client disconnects.
223 + await self.engine.abort(request_id)
224 + return self.create_error_response("Client disconnected")
225 + final_res = res
226 + assert final_res is not None
227 +
228 + choices = []
229 + role = self.get_chat_request_role(request)
230 + for output in final_res.outputs:
231 + choice_data = ChatCompletionResponseChoice(
232 + index=output.index,
233 + message=ChatMessage(role=role, content=output.text),
234 + finish_reason=output.finish_reason,
235 + )
236 + choices.append(choice_data)
237 +
238 + if request.echo:
239 + last_msg_content = ""
240 + if request.messages and isinstance(
241 + request.messages, list) and request.messages[-1].get(
242 + "content") and request.messages[-1].get(
243 + "role") == role:
244 + last_msg_content = request.messages[-1]["content"]
245 +
246 + for choice in choices:
247 + full_message = last_msg_content + choice.message.content
248 + choice.message.content = full_message
249 +
250 + num_prompt_tokens = len(final_res.prompt_token_ids)
251 + num_generated_tokens = sum(
252 + len(output.token_ids) for output in final_res.outputs)
253 + usage = UsageInfo(
254 + prompt_tokens=num_prompt_tokens,
255 + completion_tokens=num_generated_tokens,
256 + total_tokens=num_prompt_tokens + num_generated_tokens,
257 + )
258 + response = ChatCompletionResponse(
259 + id=request_id,
260 + created=created_time,
261 + model=model_name,
262 + choices=choices,
263 + usage=usage,
264 + )
265 +
266 + return response
267 +
268 + def _load_chat_template(self, chat_template):
269 + if chat_template is not None:
270 + try:
271 + with open(chat_template, "r") as f:
272 + self.tokenizer.chat_template = f.read()
273 + except OSError:
274 + # If opening a file fails, set chat template to be args to
275 + # ensure we decode so our escape are interpreted correctly
276 + self.tokenizer.chat_template = codecs.decode(
277 + chat_template, "unicode_escape")
278 +
279 + logger.info(
280 + f"Using supplied chat template:\n{self.tokenizer.chat_template}"
281 + )
282 + elif self.tokenizer.chat_template is not None:
283 + logger.info(
284 + f"Using default chat template:\n{self.tokenizer.chat_template}"
285 + )
286 + else:
287 + logger.warning(
288 + "No chat template provided. Chat API will not work.")
vllm/entrypoints/openai/serving_completion.py new
+295
@@ -0,0 +1,295 @@
1 +import time
2 +from fastapi import Request
3 +from typing import AsyncGenerator, Optional
4 +from vllm.logger import init_logger
5 +from vllm.utils import random_uuid
6 +from vllm.engine.async_llm_engine import AsyncLLMEngine
7 +from .protocol import (CompletionRequest, CompletionResponse,
8 + CompletionResponseChoice,
9 + CompletionResponseStreamChoice,
10 + CompletionStreamResponse, LogProbs, UsageInfo)
11 +from vllm.outputs import RequestOutput
12 +from vllm.sampling_params import SamplingParams
13 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
14 +
15 +logger = init_logger(__name__)
16 +
17 +
18 +class OpenAIServingCompletion(OpenAIServing):
19 +
20 + def __init__(self, engine: AsyncLLMEngine, served_model: str):
21 + super().__init__(engine=engine, served_model=served_model)
22 +
23 + async def create_completion(self, request: CompletionRequest,
24 + raw_request: Request):
25 + """Completion API similar to OpenAI's API.
26 +
27 + See https://platform.openai.com/docs/api-reference/completions/create
28 + for the API specification. This API mimics the OpenAI Completion API.
29 +
30 + NOTE: Currently we do not support the following features:
31 + - suffix (the language models we currently support do not support
32 + suffix)
33 + - logit_bias (to be supported by vLLM engine)
34 + """
35 +
36 + error_check_ret = await self._check_model(request)
37 + if error_check_ret is not None:
38 + return error_check_ret
39 +
40 + # OpenAI API supports echoing the prompt when max_tokens is 0.
41 + echo_without_generation = request.echo and request.max_tokens == 0
42 +
43 + if request.suffix is not None:
44 + # The language models we currently support do not support suffix.
45 + return self.create_error_response(
46 + "suffix is not currently supported")
47 +
48 + if request.logit_bias is not None and len(request.logit_bias) > 0:
49 + # TODO: support logit_bias in vLLM engine.
50 + return self.create_error_response(
51 + "logit_bias is not currently supported")
52 +
53 + model_name = request.model
54 + request_id = f"cmpl-{random_uuid()}"
55 +
56 + use_token_ids = False
57 + if isinstance(request.prompt, list):
58 + if len(request.prompt) == 0:
59 + return self.create_error_response(
60 + "please provide at least one prompt")
61 + first_element = request.prompt[0]
62 + if isinstance(first_element, int):
63 + use_token_ids = True
64 + prompt = request.prompt
65 + elif isinstance(first_element, (str, list)):
66 + # TODO: handles multiple prompt case in list[list[int]]
67 + if len(request.prompt) > 1:
68 + return self.create_error_response(
69 + "multiple prompts in a batch is not currently supported"
70 + )
71 + use_token_ids = not isinstance(first_element, str)
72 + prompt = request.prompt[0]
73 + else:
74 + prompt = request.prompt
75 +
76 + if use_token_ids:
77 + _, error_check_ret = await self._check_length(request,
78 + prompt_ids=prompt)
79 + else:
80 + token_ids, error_check_ret = await self._check_length(
81 + request, prompt=prompt)
82 + if error_check_ret is not None:
83 + return error_check_ret
84 +
85 + created_time = int(time.monotonic())
86 + try:
87 + spaces_between_special_tokens = request.spaces_between_special_tokens
88 + sampling_params = SamplingParams(
89 + n=request.n,
90 + best_of=request.best_of,
91 + presence_penalty=request.presence_penalty,
92 + frequency_penalty=request.frequency_penalty,
93 + repetition_penalty=request.repetition_penalty,
94 + temperature=request.temperature,
95 + top_p=request.top_p,
96 + top_k=request.top_k,
97 + min_p=request.min_p,
98 + stop=request.stop,
99 + stop_token_ids=request.stop_token_ids,
100 + ignore_eos=request.ignore_eos,
101 + max_tokens=request.max_tokens
102 + if not echo_without_generation else 1,
103 + logprobs=request.logprobs,
104 + use_beam_search=request.use_beam_search,
105 + prompt_logprobs=request.logprobs if request.echo else None,
106 + skip_special_tokens=request.skip_special_tokens,
107 + spaces_between_special_tokens=spaces_between_special_tokens,
108 + )
109 + except ValueError as e:
110 + return self.create_error_response(str(e))
111 +
112 + if use_token_ids:
113 + result_generator = self.engine.generate(None,
114 + sampling_params,
115 + request_id,
116 + prompt_token_ids=prompt)
117 + else:
118 + result_generator = self.engine.generate(prompt, sampling_params,
119 + request_id, token_ids)
120 +
121 + # Similar to the OpenAI API, when n != best_of, we do not stream the
122 + # results. In addition, we do not stream the results when use beam search.
123 + stream = (request.stream
124 + and (request.best_of is None or request.n == request.best_of)
125 + and not request.use_beam_search)
126 +
127 + def create_stream_response_json(
128 + index: int,
129 + text: str,
130 + logprobs: Optional[LogProbs] = None,
131 + finish_reason: Optional[str] = None,
132 + usage: Optional[UsageInfo] = None,
133 + ) -> str:
134 + choice_data = CompletionResponseStreamChoice(
135 + index=index,
136 + text=text,
137 + logprobs=logprobs,
138 + finish_reason=finish_reason,
139 + )
140 + response = CompletionStreamResponse(
141 + id=request_id,
142 + created=created_time,
143 + model=model_name,
144 + choices=[choice_data],
145 + )
146 + if usage is not None:
147 + response.usage = usage
148 + response_json = response.json(exclude_unset=True,
149 + ensure_ascii=False)
150 +
151 + return response_json
152 +
153 + async def completion_stream_generator() -> AsyncGenerator[str, None]:
154 + previous_texts = [""] * request.n
155 + previous_num_tokens = [0] * request.n
156 + has_echoed = [False] * request.n
157 + async for res in result_generator:
158 + res: RequestOutput
159 + for output in res.outputs:
160 + i = output.index
161 + delta_text = output.text[len(previous_texts[i]):]
162 + token_ids = output.token_ids[previous_num_tokens[i]:]
163 + if request.logprobs is not None:
164 + top_logprobs = output.logprobs[previous_num_tokens[i]:]
165 + else:
166 + top_logprobs = None
167 + offsets = len(previous_texts[i])
168 + if request.echo and not has_echoed[i]:
169 + if not echo_without_generation:
170 + delta_text = res.prompt + delta_text
171 + token_ids = res.prompt_token_ids + token_ids
172 + if top_logprobs:
173 + top_logprobs = res.prompt_logprobs + top_logprobs
174 + else: # only just return the prompt
175 + delta_text = res.prompt
176 + token_ids = res.prompt_token_ids
177 + if top_logprobs:
178 + top_logprobs = res.prompt_logprobs
179 + has_echoed[i] = True
180 + if request.logprobs is not None:
181 + logprobs = self._create_logprobs(
182 + token_ids=token_ids,
183 + top_logprobs=top_logprobs,
184 + num_output_top_logprobs=request.logprobs,
185 + initial_text_offset=offsets,
186 + )
187 + else:
188 + logprobs = None
189 + previous_texts[i] = output.text
190 + previous_num_tokens[i] = len(output.token_ids)
191 + finish_reason = output.finish_reason
192 + response_json = create_stream_response_json(
193 + index=i,
194 + text=delta_text,
195 + logprobs=logprobs,
196 + finish_reason=finish_reason,
197 + )
198 + yield f"data: {response_json}\n\n"
199 + if output.finish_reason is not None:
200 + logprobs = (LogProbs()
201 + if request.logprobs is not None else None)
202 + prompt_tokens = len(res.prompt_token_ids)
203 + completion_tokens = len(output.token_ids)
204 + final_usage = UsageInfo(
205 + prompt_tokens=prompt_tokens,
206 + completion_tokens=completion_tokens,
207 + total_tokens=prompt_tokens + completion_tokens,
208 + )
209 + response_json = create_stream_response_json(
210 + index=i,
211 + text="",
212 + logprobs=logprobs,
213 + finish_reason=output.finish_reason,
214 + usage=final_usage,
215 + )
216 + yield f"data: {response_json}\n\n"
217 + yield "data: [DONE]\n\n"
218 +
219 + # Streaming response
220 + if stream:
221 + return completion_stream_generator()
222 +
223 + # Non-streaming response
224 + final_res: RequestOutput = None
225 + async for res in result_generator:
226 + if await raw_request.is_disconnected():
227 + # Abort the request if the client disconnects.
228 + await self.engine.abort(request_id)
229 + return self.create_error_response("Client disconnected")
230 + final_res = res
231 + assert final_res is not None
232 + choices = []
233 + prompt_token_ids = final_res.prompt_token_ids
234 + prompt_logprobs = final_res.prompt_logprobs
235 + prompt_text = final_res.prompt
236 + for output in final_res.outputs:
237 + if request.logprobs is not None:
238 + if not echo_without_generation:
239 + token_ids = output.token_ids
240 + top_logprobs = output.logprobs
241 + if request.echo:
242 + token_ids = prompt_token_ids + token_ids
243 + top_logprobs = prompt_logprobs + top_logprobs
244 + else:
245 + token_ids = prompt_token_ids
246 + top_logprobs = prompt_logprobs
247 + logprobs = self._create_logprobs(
248 + token_ids=token_ids,
249 + top_logprobs=top_logprobs,
250 + num_output_top_logprobs=request.logprobs,
251 + )
252 + else:
253 + logprobs = None
254 + if not echo_without_generation:
255 + output_text = output.text
256 + if request.echo:
257 + output_text = prompt_text + output_text
258 + else:
259 + output_text = prompt_text
260 + choice_data = CompletionResponseChoice(
261 + index=output.index,
262 + text=output_text,
263 + logprobs=logprobs,
264 + finish_reason=output.finish_reason,
265 + )
266 + choices.append(choice_data)
267 +
268 + num_prompt_tokens = len(final_res.prompt_token_ids)
269 + num_generated_tokens = sum(
270 + len(output.token_ids) for output in final_res.outputs)
271 + usage = UsageInfo(
272 + prompt_tokens=num_prompt_tokens,
273 + completion_tokens=num_generated_tokens,
274 + total_tokens=num_prompt_tokens + num_generated_tokens,
275 + )
276 + response = CompletionResponse(
277 + id=request_id,
278 + created=created_time,
279 + model=model_name,
280 + choices=choices,
281 + usage=usage,
282 + )
283 +
284 + if request.stream:
285 + # When user requests streaming but we don't stream, we still need to
286 + # return a streaming response with a single event.
287 + response_json = response.json(ensure_ascii=False)
288 +
289 + async def fake_stream_generator() -> AsyncGenerator[str, None]:
290 + yield f"data: {response_json}\n\n"
291 + yield "data: [DONE]\n\n"
292 +
293 + return fake_stream_generator()
294 +
295 + return response
vllm/entrypoints/openai/serving_engine.py new
+130
@@ -0,0 +1,130 @@
1 +import asyncio
2 +from http import HTTPStatus
3 +from typing import Dict, List, Optional, Tuple, Union
4 +from vllm.logger import init_logger
5 +from vllm.transformers_utils.tokenizer import get_tokenizer
6 +from vllm.engine.async_llm_engine import AsyncLLMEngine
7 +from vllm.entrypoints.openai.protocol import (CompletionRequest,
8 + ChatCompletionRequest,
9 + ErrorResponse, LogProbs,
10 + ModelCard, ModelList,
11 + ModelPermission)
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +class OpenAIServing:
17 +
18 + def __init__(self, engine: AsyncLLMEngine, served_model: str):
19 + self.engine = engine
20 + self.served_model = served_model
21 +
22 + self.max_model_len = 0
23 + self.tokenizer = None
24 +
25 + try:
26 + event_loop = asyncio.get_running_loop()
27 + except RuntimeError:
28 + event_loop = None
29 +
30 + if event_loop is not None and event_loop.is_running(
31 + ): # If the current is instanced by Ray Serve, there is already a running event loop
32 + event_loop.create_task(self._post_init())
33 + else: # When using single vLLM without engine_use_ray
34 + asyncio.run(self._post_init())
35 +
36 + async def _post_init(self):
37 + engine_model_config = await self.engine.get_model_config()
38 + self.max_model_len = engine_model_config.max_model_len
39 +
40 + # A separate tokenizer to map token IDs to strings.
41 + self.tokenizer = get_tokenizer(
42 + engine_model_config.tokenizer,
43 + tokenizer_mode=engine_model_config.tokenizer_mode,
44 + trust_remote_code=engine_model_config.trust_remote_code)
45 +
46 + async def show_available_models(self) -> ModelList:
47 + """Show available models. Right now we only have one model."""
48 + model_cards = [
49 + ModelCard(id=self.served_model,
50 + root=self.served_model,
51 + permission=[ModelPermission()])
52 + ]
53 + return ModelList(data=model_cards)
54 +
55 + def _create_logprobs(
56 + self,
57 + token_ids: List[int],
58 + top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None,
59 + num_output_top_logprobs: Optional[int] = None,
60 + initial_text_offset: int = 0,
61 + ) -> LogProbs:
62 + """Create OpenAI-style logprobs."""
63 + logprobs = LogProbs()
64 + last_token_len = 0
65 + if num_output_top_logprobs:
66 + logprobs.top_logprobs = []
67 + for i, token_id in enumerate(token_ids):
68 + step_top_logprobs = top_logprobs[i]
69 + if step_top_logprobs is not None:
70 + token_logprob = step_top_logprobs[token_id]
71 + else:
72 + token_logprob = None
73 + token = self.tokenizer.convert_ids_to_tokens(token_id)
74 + logprobs.tokens.append(token)
75 + logprobs.token_logprobs.append(token_logprob)
76 + if len(logprobs.text_offset) == 0:
77 + logprobs.text_offset.append(initial_text_offset)
78 + else:
79 + logprobs.text_offset.append(logprobs.text_offset[-1] +
80 + last_token_len)
81 + last_token_len = len(token)
82 +
83 + if num_output_top_logprobs:
84 + logprobs.top_logprobs.append({
85 + self.tokenizer.convert_ids_to_tokens(i): p
86 + for i, p in step_top_logprobs.items()
87 + } if step_top_logprobs else None)
88 + return logprobs
89 +
90 + def create_error_response(
91 + self,
92 + message: str,
93 + err_type: str = "BadRequestError",
94 + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse:
95 + return ErrorResponse(message=message,
96 + type=err_type,
97 + code=status_code.value)
98 +
99 + async def _check_model(self, request) -> Optional[ErrorResponse]:
100 + if request.model == self.served_model:
101 + return
102 + return self.create_error_response(
103 + message=f"The model `{request.model}` does not exist.",
104 + err_type="NotFoundError",
105 + status_code=HTTPStatus.NOT_FOUND)
106 +
107 + async def _check_length(
108 + self,
109 + request: Union[ChatCompletionRequest, CompletionRequest],
110 + prompt: Optional[str] = None,
111 + prompt_ids: Optional[List[int]] = None
112 + ) -> Tuple[List[int], Optional[ErrorResponse]]:
113 + assert (not (prompt is None and prompt_ids is None)
114 + and not (prompt is not None and prompt_ids is not None)
115 + ), "Either prompt or prompt_ids should be provided."
116 + input_ids = prompt_ids if prompt_ids is not None else self.tokenizer(
117 + prompt).input_ids
118 + token_num = len(input_ids)
119 +
120 + if request.max_tokens is None:
121 + request.max_tokens = self.max_model_len - token_num
122 + if token_num + request.max_tokens > self.max_model_len:
123 + return input_ids, self.create_error_response(
124 + f"This model's maximum context length is {self.max_model_len} tokens. "
125 + f"However, you requested {request.max_tokens + token_num} tokens "
126 + f"({token_num} in the messages, "
127 + f"{request.max_tokens} in the completion). "
128 + f"Please reduce the length of the messages or completion.", )
129 + else:
130 + return input_ids, None