A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Frontend] Move async logic outside of constructor (#4674)
Cyrus Leung committed
May 9, 2024 at 13:48 UTC
f12b20deccbc6c8bb5cdeac053d75178341c66c1
7 files changed
+96
-102
tests/async_engine/test_chat_template.py
+13
-17
@@ -60,13 +60,12 @@ class MockServingChat:
60
tokenizer: MockTokenizer
61
62
63
-@pytest.mark.asyncio
64
-async def test_load_chat_template():
63
+def test_load_chat_template():
64
# Testing chatml template
65
tokenizer = MockTokenizer()
66
mock_serving_chat = MockServingChat(tokenizer)
68
- await OpenAIServingChat._load_chat_template(
69
- mock_serving_chat, chat_template=chatml_jinja_path)
67
+ OpenAIServingChat._load_chat_template(mock_serving_chat,
68
+ chat_template=chatml_jinja_path)
69
70
template_content = tokenizer.chat_template
71
@@ -77,8 +76,7 @@ async def test_load_chat_template():
76
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
77
78
80
-@pytest.mark.asyncio
81
-async def test_no_load_chat_template_filelike():
79
+def test_no_load_chat_template_filelike():
80
# Testing chatml template
81
template = "../../examples/does_not_exist"
82
tokenizer = MockTokenizer()
@@ -86,35 +84,33 @@ async def test_no_load_chat_template_filelike():
84
mock_serving_chat = MockServingChat(tokenizer)
85
86
with pytest.raises(ValueError, match="looks like a file path"):
89
- await OpenAIServingChat._load_chat_template(mock_serving_chat,
90
- chat_template=template)
87
+ OpenAIServingChat._load_chat_template(mock_serving_chat,
88
+ chat_template=template)
89
90
93
-@pytest.mark.asyncio
94
-async def test_no_load_chat_template_literallike():
91
+def test_no_load_chat_template_literallike():
92
# Testing chatml template
93
template = "{{ messages }}"
94
tokenizer = MockTokenizer()
95
96
mock_serving_chat = MockServingChat(tokenizer)
100
- await OpenAIServingChat._load_chat_template(mock_serving_chat,
101
- chat_template=template)
97
+ OpenAIServingChat._load_chat_template(mock_serving_chat,
98
+ chat_template=template)
99
template_content = tokenizer.chat_template
100
101
assert template_content == template
102
103
107
-@pytest.mark.asyncio
104
@pytest.mark.parametrize(
105
"model,template,add_generation_prompt,expected_output",
106
MODEL_TEMPLATE_GENERATON_OUTPUT)
111
-async def test_get_gen_prompt(model, template, add_generation_prompt,
112
- expected_output):
107
+def test_get_gen_prompt(model, template, add_generation_prompt,
108
+ expected_output):
109
# Initialize the tokenizer
110
tokenizer = get_tokenizer(tokenizer_name=model)
111
mock_serving_chat = MockServingChat(tokenizer)
116
- await OpenAIServingChat._load_chat_template(mock_serving_chat,
117
- chat_template=template)
112
+ OpenAIServingChat._load_chat_template(mock_serving_chat,
113
+ chat_template=template)
114
115
# Create a mock request object using keyword arguments
116
mock_request = ChatCompletionRequest(
tests/entrypoints/openai/test_serving_chat.py
+6
-2
@@ -20,11 +20,15 @@ class MockModelConfig:
20
class MockEngine:
21
22
async def get_model_config(self):
23
- return MockModelConfig
23
+ return MockModelConfig()
24
25
26
async def _async_serving_chat_init():
27
- serving_completion = OpenAIServingChat(MockEngine(),
27
+ engine = MockEngine()
28
+ model_config = await engine.get_model_config()
29
+
30
+ serving_completion = OpenAIServingChat(engine,
31
+ model_config,
32
served_model_names=[MODEL_NAME],
33
response_role="assistant",
34
chat_template=CHAT_TEMPLATE)
vllm/engine/arg_utils.py
+1
-1
@@ -516,7 +516,7 @@ class EngineArgs:
516
return parser
517
518
@classmethod
519
- def from_cli_args(cls, args: argparse.Namespace) -> 'EngineArgs':
519
+ def from_cli_args(cls, args: argparse.Namespace):
520
# Get the list of attributes of this dataclass.
521
attrs = [attr.name for attr in dataclasses.fields(cls)]
522
# Set the attributes from the parsed arguments.
vllm/entrypoints/openai/api_server.py
+20
-3
@@ -4,7 +4,7 @@ import inspect
4
import re
5
from contextlib import asynccontextmanager
6
from http import HTTPStatus
7
-from typing import Set
7
+from typing import Optional, Set
8
9
import fastapi
10
import uvicorn
@@ -164,15 +164,32 @@ if __name__ == "__main__":
164
served_model_names = args.served_model_name
165
else:
166
served_model_names = [args.model]
167
+
168
engine_args = AsyncEngineArgs.from_cli_args(args)
169
engine = AsyncLLMEngine.from_engine_args(
170
engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
170
- openai_serving_chat = OpenAIServingChat(engine, served_model_names,
171
+
172
+ event_loop: Optional[asyncio.AbstractEventLoop]
173
+ try:
174
+ event_loop = asyncio.get_running_loop()
175
+ except RuntimeError:
176
+ event_loop = None
177
+
178
+ if event_loop is not None and event_loop.is_running():
179
+ # If the current is instanced by Ray Serve,
180
+ # there is already a running event loop
181
+ model_config = event_loop.run_until_complete(engine.get_model_config())
182
+ else:
183
+ # When using single vLLM without engine_use_ray
184
+ model_config = asyncio.run(engine.get_model_config())
185
+
186
+ openai_serving_chat = OpenAIServingChat(engine, model_config,
187
+ served_model_names,
188
args.response_role,
189
args.lora_modules,
190
args.chat_template)
191
openai_serving_completion = OpenAIServingCompletion(
175
- engine, served_model_names, args.lora_modules)
192
+ engine, model_config, served_model_names, args.lora_modules)
193
194
app.root_path = args.root_path
195
uvicorn.run(app,
vllm/entrypoints/openai/serving_chat.py
+35
-37
@@ -1,4 +1,3 @@
1
-import asyncio
1
import codecs
2
import time
3
from typing import (AsyncGenerator, AsyncIterator, Awaitable, Iterable, List,
@@ -8,6 +7,7 @@ from fastapi import Request
7
from openai.types.chat import (ChatCompletionContentPartParam,
8
ChatCompletionRole)
9
10
+from vllm.config import ModelConfig
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
12
from vllm.entrypoints.openai.protocol import (
13
ChatCompletionRequest, ChatCompletionResponse,
@@ -35,17 +35,47 @@ class OpenAIServingChat(OpenAIServing):
35
36
def __init__(self,
37
engine: AsyncLLMEngine,
38
+ model_config: ModelConfig,
39
served_model_names: List[str],
40
response_role: str,
41
lora_modules: Optional[List[LoRAModulePath]] = None,
42
chat_template: Optional[str] = None):
43
super().__init__(engine=engine,
44
+ model_config=model_config,
45
served_model_names=served_model_names,
44
- lora_modules=lora_modules,
45
- await_post_init=self._load_chat_template(
46
- chat_template=chat_template))
46
+ lora_modules=lora_modules)
47
48
self.response_role = response_role
49
+ self._load_chat_template(chat_template)
50
+
51
+ def _load_chat_template(self, chat_template: Optional[str]):
52
+ tokenizer = self.tokenizer
53
+
54
+ if chat_template is not None:
55
+ try:
56
+ with open(chat_template, "r") as f:
57
+ tokenizer.chat_template = f.read()
58
+ except OSError as e:
59
+ JINJA_CHARS = "{}\n"
60
+ if not any(c in chat_template for c in JINJA_CHARS):
61
+ msg = (f"The supplied chat template ({chat_template}) "
62
+ f"looks like a file path, but it failed to be "
63
+ f"opened. Reason: {e}")
64
+ raise ValueError(msg) from e
65
+
66
+ # If opening a file fails, set chat template to be args to
67
+ # ensure we decode so our escape are interpreted correctly
68
+ tokenizer.chat_template = codecs.decode(
69
+ chat_template, "unicode_escape")
70
+
71
+ logger.info("Using supplied chat template:\n%s",
72
+ tokenizer.chat_template)
73
+ elif tokenizer.chat_template is not None:
74
+ logger.info("Using default chat template:\n%s",
75
+ tokenizer.chat_template)
76
+ else:
77
+ logger.warning(
78
+ "No chat template provided. Chat API will not work.")
79
80
def _parse_chat_message_content(
81
self,
@@ -357,36 +387,4 @@ class OpenAIServingChat(OpenAIServing):
387
usage=usage,
388
)
389
360
- return response
361
-
362
- async def _load_chat_template(self, chat_template: Optional[str]):
363
- while self.tokenizer is None:
364
- # Give the parent class time to load the tokenizer
365
- await asyncio.sleep(0.1)
366
- tokenizer = self.tokenizer
367
-
368
- if chat_template is not None:
369
- try:
370
- with open(chat_template, "r") as f:
371
- tokenizer.chat_template = f.read()
372
- except OSError as e:
373
- JINJA_CHARS = "{}\n"
374
- if not any(c in chat_template for c in JINJA_CHARS):
375
- msg = (f"The supplied chat template ({chat_template}) "
376
- f"looks like a file path, but it failed to be "
377
- f"opened. Reason: {e}")
378
- raise ValueError(msg) from e
379
-
380
- # If opening a file fails, set chat template to be args to
381
- # ensure we decode so our escape are interpreted correctly
382
- tokenizer.chat_template = codecs.decode(
383
- chat_template, "unicode_escape")
384
-
385
- logger.info("Using supplied chat template:\n%s",
386
- tokenizer.chat_template)
387
- elif tokenizer.chat_template is not None:
388
- logger.info("Using default chat template:\n%s",
389
- tokenizer.chat_template)
390
- else:
391
- logger.warning(
392
- "No chat template provided. Chat API will not work.")
390
+ return response
\ No newline at end of file
vllm/entrypoints/openai/serving_completion.py
+4
-3
@@ -4,6 +4,7 @@ from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
4
5
from fastapi import Request
6
7
+from vllm.config import ModelConfig
8
from vllm.engine.async_llm_engine import AsyncLLMEngine
9
from vllm.entrypoints.openai.protocol import (CompletionRequest,
10
CompletionResponse,
@@ -52,11 +53,11 @@ def parse_prompt_format(prompt) -> Tuple[bool, list]:
53
54
class OpenAIServingCompletion(OpenAIServing):
55
55
- def __init__(self,
56
- engine: AsyncLLMEngine,
56
+ def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
57
served_model_names: List[str],
58
- lora_modules: Optional[List[LoRAModulePath]] = None):
58
+ lora_modules: Optional[List[LoRAModulePath]]):
59
super().__init__(engine=engine,
60
+ model_config=model_config,
61
served_model_names=served_model_names,
62
lora_modules=lora_modules)
63
vllm/entrypoints/openai/serving_engine.py
+17
-39
@@ -1,13 +1,12 @@
1
-import asyncio
1
import json
2
from dataclasses import dataclass
3
from http import HTTPStatus
5
-from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union
4
+from typing import Dict, List, Optional, Tuple, Union
5
6
from pydantic import Field
8
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
7
from typing_extensions import Annotated
8
9
+from vllm.config import ModelConfig
10
from vllm.engine.async_llm_engine import AsyncLLMEngine
11
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
CompletionRequest, ErrorResponse,
@@ -29,13 +28,24 @@ class LoRAModulePath:
28
29
class OpenAIServing:
30
32
- def __init__(self,
33
- engine: AsyncLLMEngine,
31
+ def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
32
served_model_names: List[str],
35
- lora_modules: Optional[List[LoRAModulePath]],
36
- await_post_init: Optional[Awaitable[Any]] = None):
33
+ lora_modules: Optional[List[LoRAModulePath]]):
34
+ super().__init__()
35
+
36
self.engine = engine
37
+ self.max_model_len = model_config.max_model_len
38
+
39
+ # A separate tokenizer to map token IDs to strings.
40
+ self.tokenizer = get_tokenizer(
41
+ model_config.tokenizer,
42
+ tokenizer_mode=model_config.tokenizer_mode,
43
+ tokenizer_revision=model_config.tokenizer_revision,
44
+ trust_remote_code=model_config.trust_remote_code,
45
+ truncation_side="left")
46
+
47
self.served_model_names = served_model_names
48
+
49
if lora_modules is None:
50
self.lora_requests = []
51
else:
@@ -47,38 +57,6 @@ class OpenAIServing:
57
) for i, lora in enumerate(lora_modules, start=1)
58
]
59
50
- self.max_model_len = 0
51
- # Lazy initialized
52
- self.tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
53
-
54
- try:
55
- event_loop = asyncio.get_running_loop()
56
- except RuntimeError:
57
- event_loop = None
58
-
59
- if event_loop is not None and event_loop.is_running():
60
- # If the current is instanced by Ray Serve,
61
- # there is already a running event loop
62
- event_loop.create_task(self._post_init(await_post_init))
63
- else:
64
- # When using single vLLM without engine_use_ray
65
- asyncio.run(self._post_init(await_post_init))
66
-
67
- async def _post_init(self, await_post_init):
68
- engine_model_config = await self.engine.get_model_config()
69
- self.max_model_len = engine_model_config.max_model_len
70
-
71
- # A separate tokenizer to map token IDs to strings.
72
- self.tokenizer = get_tokenizer(
73
- engine_model_config.tokenizer,
74
- tokenizer_mode=engine_model_config.tokenizer_mode,
75
- tokenizer_revision=engine_model_config.tokenizer_revision,
76
- trust_remote_code=engine_model_config.trust_remote_code,
77
- truncation_side="left")
78
-
79
- if await_post_init is not None:
80
- await await_post_init
81
-
60
async def show_available_models(self) -> ModelList:
61
"""Show available models. Right now we only have one model."""
62
model_cards = [