@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f8e7adda21
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

Fix/async chat serving (#2727)

Sebastian Schoennenbeck committed May 3, 2024 at 20:04 UTC f8e7adda21810104382bdf3febe3ea02c72f7348
5 files changed +73 -21
tests/async_engine/test_chat_template.py
+14 -11
@@ -60,12 +60,13 @@ class MockServingChat:
60 tokenizer: MockTokenizer
61
62
63 -def test_load_chat_template():
63 +@pytest.mark.asyncio
64 +async def test_load_chat_template():
65 # Testing chatml template
66 tokenizer = MockTokenizer()
67 mock_serving_chat = MockServingChat(tokenizer)
67 - OpenAIServingChat._load_chat_template(mock_serving_chat,
68 - chat_template=chatml_jinja_path)
68 + await OpenAIServingChat._load_chat_template(
69 + mock_serving_chat, chat_template=chatml_jinja_path)
70
71 template_content = tokenizer.chat_template
72
@@ -76,7 +77,8 @@ def test_load_chat_template():
77 {% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
78
79
79 -def test_no_load_chat_template_filelike():
80 +@pytest.mark.asyncio
81 +async def test_no_load_chat_template_filelike():
82 # Testing chatml template
83 template = "../../examples/does_not_exist"
84 tokenizer = MockTokenizer()
@@ -84,18 +86,19 @@ def test_no_load_chat_template_filelike():
86 mock_serving_chat = MockServingChat(tokenizer)
87
88 with pytest.raises(ValueError, match="looks like a file path"):
87 - OpenAIServingChat._load_chat_template(mock_serving_chat,
88 - chat_template=template)
89 + await OpenAIServingChat._load_chat_template(mock_serving_chat,
90 + chat_template=template)
91
92
91 -def test_no_load_chat_template_literallike():
93 +@pytest.mark.asyncio
94 +async def test_no_load_chat_template_literallike():
95 # Testing chatml template
96 template = "{{ messages }}"
97 tokenizer = MockTokenizer()
98
99 mock_serving_chat = MockServingChat(tokenizer)
97 - OpenAIServingChat._load_chat_template(mock_serving_chat,
98 - chat_template=template)
100 + await OpenAIServingChat._load_chat_template(mock_serving_chat,
101 + chat_template=template)
102 template_content = tokenizer.chat_template
103
104 assert template_content == template
@@ -110,8 +113,8 @@ async def test_get_gen_prompt(model, template, add_generation_prompt,
113 # Initialize the tokenizer
114 tokenizer = get_tokenizer(tokenizer_name=model)
115 mock_serving_chat = MockServingChat(tokenizer)
113 - OpenAIServingChat._load_chat_template(mock_serving_chat,
114 - chat_template=template)
116 + await OpenAIServingChat._load_chat_template(mock_serving_chat,
117 + chat_template=template)
118
119 # Create a mock request object using keyword arguments
120 mock_request = ChatCompletionRequest(
tests/entrypoints/openai/test_serving_chat.py new
+37
@@ -0,0 +1,37 @@
1 +import asyncio
2 +from dataclasses import dataclass
3 +
4 +from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
5 +
6 +MODEL_NAME = "openai-community/gpt2"
7 +CHAT_TEMPLATE = "Dummy chat template for testing {}"
8 +
9 +
10 +@dataclass
11 +class MockModelConfig:
12 + tokenizer = MODEL_NAME
13 + trust_remote_code = False
14 + tokenizer_mode = "auto"
15 + max_model_len = 100
16 + tokenizer_revision = None
17 +
18 +
19 +@dataclass
20 +class MockEngine:
21 +
22 + async def get_model_config(self):
23 + return MockModelConfig
24 +
25 +
26 +async def _async_serving_chat_init():
27 + serving_completion = OpenAIServingChat(MockEngine(),
28 + served_model_names=[MODEL_NAME],
29 + response_role="assistant",
30 + chat_template=CHAT_TEMPLATE)
31 + return serving_completion
32 +
33 +
34 +def test_async_serving_chat_init():
35 + serving_completion = asyncio.run(_async_serving_chat_init())
36 + assert serving_completion.tokenizer is not None
37 + assert serving_completion.tokenizer.chat_template == CHAT_TEMPLATE
tests/entrypoints/test_openai_server.py
+1 -1
@@ -150,7 +150,7 @@ def server(zephyr_lora_files):
150 ray.shutdown()
151
152
153 -@pytest.fixture(scope="session")
153 +@pytest.fixture(scope="module")
154 def client():
155 client = openai.AsyncOpenAI(
156 base_url="http://localhost:8000/v1",
vllm/entrypoints/openai/serving_chat.py
+9 -3
@@ -1,3 +1,4 @@
1 +import asyncio
2 import codecs
3 import time
4 from typing import (AsyncGenerator, AsyncIterator, Awaitable, Iterable, List,
@@ -40,9 +41,11 @@ class OpenAIServingChat(OpenAIServing):
41 chat_template: Optional[str] = None):
42 super().__init__(engine=engine,
43 served_model_names=served_model_names,
43 - lora_modules=lora_modules)
44 + lora_modules=lora_modules,
45 + await_post_init=self._load_chat_template(
46 + chat_template=chat_template))
47 +
48 self.response_role = response_role
45 - self._load_chat_template(chat_template)
49
50 def _parse_chat_message_content(
51 self,
@@ -356,7 +359,10 @@ class OpenAIServingChat(OpenAIServing):
359
360 return response
361
359 - def _load_chat_template(self, chat_template: Optional[str]):
362 + async def _load_chat_template(self, chat_template: Optional[str]):
363 + while self.tokenizer is None:
364 + # Give the parent class time to load the tokenizer
365 + await asyncio.sleep(0.1)
366 tokenizer = self.tokenizer
367
368 if chat_template is not None:
vllm/entrypoints/openai/serving_engine.py
+12 -6
@@ -2,7 +2,7 @@ import asyncio
2 import json
3 from dataclasses import dataclass
4 from http import HTTPStatus
5 -from typing import Dict, List, Optional, Tuple, Union
5 +from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union
6
7 from pydantic import Field
8 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
@@ -29,8 +29,11 @@ class LoRAModulePath:
29
30 class OpenAIServing:
31
32 - def __init__(self, engine: AsyncLLMEngine, served_model_names: List[str],
33 - lora_modules: Optional[List[LoRAModulePath]]):
32 + def __init__(self,
33 + engine: AsyncLLMEngine,
34 + served_model_names: List[str],
35 + lora_modules: Optional[List[LoRAModulePath]],
36 + await_post_init: Optional[Awaitable[Any]] = None):
37 self.engine = engine
38 self.served_model_names = served_model_names
39 if lora_modules is None:
@@ -56,12 +59,12 @@ class OpenAIServing:
59 if event_loop is not None and event_loop.is_running():
60 # If the current is instanced by Ray Serve,
61 # there is already a running event loop
59 - event_loop.create_task(self._post_init())
62 + event_loop.create_task(self._post_init(await_post_init))
63 else:
64 # When using single vLLM without engine_use_ray
62 - asyncio.run(self._post_init())
65 + asyncio.run(self._post_init(await_post_init))
66
64 - async def _post_init(self):
67 + async def _post_init(self, await_post_init):
68 engine_model_config = await self.engine.get_model_config()
69 self.max_model_len = engine_model_config.max_model_len
70
@@ -73,6 +76,9 @@ class OpenAIServing:
76 trust_remote_code=engine_model_config.trust_remote_code,
77 truncation_side="left")
78
79 + if await_post_init is not None:
80 + await await_post_init
81 +
82 async def show_available_models(self) -> ModelList:
83 """Show available models. Right now we only have one model."""
84 model_cards = [