A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
Fix/async chat serving (#2727)
Sebastian Schoennenbeck committed
May 3, 2024 at 20:04 UTC
f8e7adda21810104382bdf3febe3ea02c72f7348
5 files changed
+73
-21
tests/async_engine/test_chat_template.py
+14
-11
@@ -60,12 +60,13 @@ class MockServingChat:
60
tokenizer: MockTokenizer
61
62
63
-def test_load_chat_template():
63
+@pytest.mark.asyncio
64
+async def test_load_chat_template():
65
# Testing chatml template
66
tokenizer = MockTokenizer()
67
mock_serving_chat = MockServingChat(tokenizer)
67
- OpenAIServingChat._load_chat_template(mock_serving_chat,
68
- chat_template=chatml_jinja_path)
68
+ await OpenAIServingChat._load_chat_template(
69
+ mock_serving_chat, chat_template=chatml_jinja_path)
70
71
template_content = tokenizer.chat_template
72
@@ -76,7 +77,8 @@ def test_load_chat_template():
77
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
78
79
79
-def test_no_load_chat_template_filelike():
80
+@pytest.mark.asyncio
81
+async def test_no_load_chat_template_filelike():
82
# Testing chatml template
83
template = "../../examples/does_not_exist"
84
tokenizer = MockTokenizer()
@@ -84,18 +86,19 @@ def test_no_load_chat_template_filelike():
86
mock_serving_chat = MockServingChat(tokenizer)
87
88
with pytest.raises(ValueError, match="looks like a file path"):
87
- OpenAIServingChat._load_chat_template(mock_serving_chat,
88
- chat_template=template)
89
+ await OpenAIServingChat._load_chat_template(mock_serving_chat,
90
+ chat_template=template)
91
92
91
-def test_no_load_chat_template_literallike():
93
+@pytest.mark.asyncio
94
+async def test_no_load_chat_template_literallike():
95
# Testing chatml template
96
template = "{{ messages }}"
97
tokenizer = MockTokenizer()
98
99
mock_serving_chat = MockServingChat(tokenizer)
97
- OpenAIServingChat._load_chat_template(mock_serving_chat,
98
- chat_template=template)
100
+ await OpenAIServingChat._load_chat_template(mock_serving_chat,
101
+ chat_template=template)
102
template_content = tokenizer.chat_template
103
104
assert template_content == template
@@ -110,8 +113,8 @@ async def test_get_gen_prompt(model, template, add_generation_prompt,
113
# Initialize the tokenizer
114
tokenizer = get_tokenizer(tokenizer_name=model)
115
mock_serving_chat = MockServingChat(tokenizer)
113
- OpenAIServingChat._load_chat_template(mock_serving_chat,
114
- chat_template=template)
116
+ await OpenAIServingChat._load_chat_template(mock_serving_chat,
117
+ chat_template=template)
118
119
# Create a mock request object using keyword arguments
120
mock_request = ChatCompletionRequest(
tests/entrypoints/openai/test_serving_chat.py
new
+37
@@ -0,0 +1,37 @@
1
+import asyncio
2
+from dataclasses import dataclass
3
+
4
+from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
5
+
6
+MODEL_NAME = "openai-community/gpt2"
7
+CHAT_TEMPLATE = "Dummy chat template for testing {}"
8
+
9
+
10
+@dataclass
11
+class MockModelConfig:
12
+ tokenizer = MODEL_NAME
13
+ trust_remote_code = False
14
+ tokenizer_mode = "auto"
15
+ max_model_len = 100
16
+ tokenizer_revision = None
17
+
18
+
19
+@dataclass
20
+class MockEngine:
21
+
22
+ async def get_model_config(self):
23
+ return MockModelConfig
24
+
25
+
26
+async def _async_serving_chat_init():
27
+ serving_completion = OpenAIServingChat(MockEngine(),
28
+ served_model_names=[MODEL_NAME],
29
+ response_role="assistant",
30
+ chat_template=CHAT_TEMPLATE)
31
+ return serving_completion
32
+
33
+
34
+def test_async_serving_chat_init():
35
+ serving_completion = asyncio.run(_async_serving_chat_init())
36
+ assert serving_completion.tokenizer is not None
37
+ assert serving_completion.tokenizer.chat_template == CHAT_TEMPLATE
tests/entrypoints/test_openai_server.py
+1
-1
@@ -150,7 +150,7 @@ def server(zephyr_lora_files):
150
ray.shutdown()
151
152
153
-@pytest.fixture(scope="session")
153
+@pytest.fixture(scope="module")
154
def client():
155
client = openai.AsyncOpenAI(
156
base_url="http://localhost:8000/v1",
vllm/entrypoints/openai/serving_chat.py
+9
-3
@@ -1,3 +1,4 @@
1
+import asyncio
2
import codecs
3
import time
4
from typing import (AsyncGenerator, AsyncIterator, Awaitable, Iterable, List,
@@ -40,9 +41,11 @@ class OpenAIServingChat(OpenAIServing):
41
chat_template: Optional[str] = None):
42
super().__init__(engine=engine,
43
served_model_names=served_model_names,
43
- lora_modules=lora_modules)
44
+ lora_modules=lora_modules,
45
+ await_post_init=self._load_chat_template(
46
+ chat_template=chat_template))
47
+
48
self.response_role = response_role
45
- self._load_chat_template(chat_template)
49
50
def _parse_chat_message_content(
51
self,
@@ -356,7 +359,10 @@ class OpenAIServingChat(OpenAIServing):
359
360
return response
361
359
- def _load_chat_template(self, chat_template: Optional[str]):
362
+ async def _load_chat_template(self, chat_template: Optional[str]):
363
+ while self.tokenizer is None:
364
+ # Give the parent class time to load the tokenizer
365
+ await asyncio.sleep(0.1)
366
tokenizer = self.tokenizer
367
368
if chat_template is not None:
vllm/entrypoints/openai/serving_engine.py
+12
-6
@@ -2,7 +2,7 @@ import asyncio
2
import json
3
from dataclasses import dataclass
4
from http import HTTPStatus
5
-from typing import Dict, List, Optional, Tuple, Union
5
+from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union
6
7
from pydantic import Field
8
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
@@ -29,8 +29,11 @@ class LoRAModulePath:
29
30
class OpenAIServing:
31
32
- def __init__(self, engine: AsyncLLMEngine, served_model_names: List[str],
33
- lora_modules: Optional[List[LoRAModulePath]]):
32
+ def __init__(self,
33
+ engine: AsyncLLMEngine,
34
+ served_model_names: List[str],
35
+ lora_modules: Optional[List[LoRAModulePath]],
36
+ await_post_init: Optional[Awaitable[Any]] = None):
37
self.engine = engine
38
self.served_model_names = served_model_names
39
if lora_modules is None:
@@ -56,12 +59,12 @@ class OpenAIServing:
59
if event_loop is not None and event_loop.is_running():
60
# If the current is instanced by Ray Serve,
61
# there is already a running event loop
59
- event_loop.create_task(self._post_init())
62
+ event_loop.create_task(self._post_init(await_post_init))
63
else:
64
# When using single vLLM without engine_use_ray
62
- asyncio.run(self._post_init())
65
+ asyncio.run(self._post_init(await_post_init))
66
64
- async def _post_init(self):
67
+ async def _post_init(self, await_post_init):
68
engine_model_config = await self.engine.get_model_config()
69
self.max_model_len = engine_model_config.max_model_len
70
@@ -73,6 +76,9 @@ class OpenAIServing:
76
trust_remote_code=engine_model_config.trust_remote_code,
77
truncation_side="left")
78
79
+ if await_post_init is not None:
80
+ await await_post_init
81
+
82
async def show_available_models(self) -> ModelList:
83
"""Show available models. Right now we only have one model."""
84
model_cards = [