@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f12b20decc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Frontend] Move async logic outside of constructor (#4674)

Cyrus Leung committed May 9, 2024 at 13:48 UTC f12b20deccbc6c8bb5cdeac053d75178341c66c1
7 files changed +96 -102
tests/async_engine/test_chat_template.py
+13 -17
@@ -60,13 +60,12 @@ class MockServingChat:
60 tokenizer: MockTokenizer
61
62
63 -@pytest.mark.asyncio
64 -async def test_load_chat_template():
63 +def test_load_chat_template():
64 # Testing chatml template
65 tokenizer = MockTokenizer()
66 mock_serving_chat = MockServingChat(tokenizer)
68 - await OpenAIServingChat._load_chat_template(
69 - mock_serving_chat, chat_template=chatml_jinja_path)
67 + OpenAIServingChat._load_chat_template(mock_serving_chat,
68 + chat_template=chatml_jinja_path)
69
70 template_content = tokenizer.chat_template
71
@@ -77,8 +76,7 @@ async def test_load_chat_template():
76 {% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
77
78
80 -@pytest.mark.asyncio
81 -async def test_no_load_chat_template_filelike():
79 +def test_no_load_chat_template_filelike():
80 # Testing chatml template
81 template = "../../examples/does_not_exist"
82 tokenizer = MockTokenizer()
@@ -86,35 +84,33 @@ async def test_no_load_chat_template_filelike():
84 mock_serving_chat = MockServingChat(tokenizer)
85
86 with pytest.raises(ValueError, match="looks like a file path"):
89 - await OpenAIServingChat._load_chat_template(mock_serving_chat,
90 - chat_template=template)
87 + OpenAIServingChat._load_chat_template(mock_serving_chat,
88 + chat_template=template)
89
90
93 -@pytest.mark.asyncio
94 -async def test_no_load_chat_template_literallike():
91 +def test_no_load_chat_template_literallike():
92 # Testing chatml template
93 template = "{{ messages }}"
94 tokenizer = MockTokenizer()
95
96 mock_serving_chat = MockServingChat(tokenizer)
100 - await OpenAIServingChat._load_chat_template(mock_serving_chat,
101 - chat_template=template)
97 + OpenAIServingChat._load_chat_template(mock_serving_chat,
98 + chat_template=template)
99 template_content = tokenizer.chat_template
100
101 assert template_content == template
102
103
107 -@pytest.mark.asyncio
104 @pytest.mark.parametrize(
105 "model,template,add_generation_prompt,expected_output",
106 MODEL_TEMPLATE_GENERATON_OUTPUT)
111 -async def test_get_gen_prompt(model, template, add_generation_prompt,
112 - expected_output):
107 +def test_get_gen_prompt(model, template, add_generation_prompt,
108 + expected_output):
109 # Initialize the tokenizer
110 tokenizer = get_tokenizer(tokenizer_name=model)
111 mock_serving_chat = MockServingChat(tokenizer)
116 - await OpenAIServingChat._load_chat_template(mock_serving_chat,
117 - chat_template=template)
112 + OpenAIServingChat._load_chat_template(mock_serving_chat,
113 + chat_template=template)
114
115 # Create a mock request object using keyword arguments
116 mock_request = ChatCompletionRequest(
tests/entrypoints/openai/test_serving_chat.py
+6 -2
@@ -20,11 +20,15 @@ class MockModelConfig:
20 class MockEngine:
21
22 async def get_model_config(self):
23 - return MockModelConfig
23 + return MockModelConfig()
24
25
26 async def _async_serving_chat_init():
27 - serving_completion = OpenAIServingChat(MockEngine(),
27 + engine = MockEngine()
28 + model_config = await engine.get_model_config()
29 +
30 + serving_completion = OpenAIServingChat(engine,
31 + model_config,
32 served_model_names=[MODEL_NAME],
33 response_role="assistant",
34 chat_template=CHAT_TEMPLATE)
vllm/engine/arg_utils.py
+1 -1
@@ -516,7 +516,7 @@ class EngineArgs:
516 return parser
517
518 @classmethod
519 - def from_cli_args(cls, args: argparse.Namespace) -> 'EngineArgs':
519 + def from_cli_args(cls, args: argparse.Namespace):
520 # Get the list of attributes of this dataclass.
521 attrs = [attr.name for attr in dataclasses.fields(cls)]
522 # Set the attributes from the parsed arguments.
vllm/entrypoints/openai/api_server.py
+20 -3
@@ -4,7 +4,7 @@ import inspect
4 import re
5 from contextlib import asynccontextmanager
6 from http import HTTPStatus
7 -from typing import Set
7 +from typing import Optional, Set
8
9 import fastapi
10 import uvicorn
@@ -164,15 +164,32 @@ if __name__ == "__main__":
164 served_model_names = args.served_model_name
165 else:
166 served_model_names = [args.model]
167 +
168 engine_args = AsyncEngineArgs.from_cli_args(args)
169 engine = AsyncLLMEngine.from_engine_args(
170 engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
170 - openai_serving_chat = OpenAIServingChat(engine, served_model_names,
171 +
172 + event_loop: Optional[asyncio.AbstractEventLoop]
173 + try:
174 + event_loop = asyncio.get_running_loop()
175 + except RuntimeError:
176 + event_loop = None
177 +
178 + if event_loop is not None and event_loop.is_running():
179 + # If the current is instanced by Ray Serve,
180 + # there is already a running event loop
181 + model_config = event_loop.run_until_complete(engine.get_model_config())
182 + else:
183 + # When using single vLLM without engine_use_ray
184 + model_config = asyncio.run(engine.get_model_config())
185 +
186 + openai_serving_chat = OpenAIServingChat(engine, model_config,
187 + served_model_names,
188 args.response_role,
189 args.lora_modules,
190 args.chat_template)
191 openai_serving_completion = OpenAIServingCompletion(
175 - engine, served_model_names, args.lora_modules)
192 + engine, model_config, served_model_names, args.lora_modules)
193
194 app.root_path = args.root_path
195 uvicorn.run(app,
vllm/entrypoints/openai/serving_chat.py
+35 -37
@@ -1,4 +1,3 @@
1 -import asyncio
1 import codecs
2 import time
3 from typing import (AsyncGenerator, AsyncIterator, Awaitable, Iterable, List,
@@ -8,6 +7,7 @@ from fastapi import Request
7 from openai.types.chat import (ChatCompletionContentPartParam,
8 ChatCompletionRole)
9
10 +from vllm.config import ModelConfig
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
12 from vllm.entrypoints.openai.protocol import (
13 ChatCompletionRequest, ChatCompletionResponse,
@@ -35,17 +35,47 @@ class OpenAIServingChat(OpenAIServing):
35
36 def __init__(self,
37 engine: AsyncLLMEngine,
38 + model_config: ModelConfig,
39 served_model_names: List[str],
40 response_role: str,
41 lora_modules: Optional[List[LoRAModulePath]] = None,
42 chat_template: Optional[str] = None):
43 super().__init__(engine=engine,
44 + model_config=model_config,
45 served_model_names=served_model_names,
44 - lora_modules=lora_modules,
45 - await_post_init=self._load_chat_template(
46 - chat_template=chat_template))
46 + lora_modules=lora_modules)
47
48 self.response_role = response_role
49 + self._load_chat_template(chat_template)
50 +
51 + def _load_chat_template(self, chat_template: Optional[str]):
52 + tokenizer = self.tokenizer
53 +
54 + if chat_template is not None:
55 + try:
56 + with open(chat_template, "r") as f:
57 + tokenizer.chat_template = f.read()
58 + except OSError as e:
59 + JINJA_CHARS = "{}\n"
60 + if not any(c in chat_template for c in JINJA_CHARS):
61 + msg = (f"The supplied chat template ({chat_template}) "
62 + f"looks like a file path, but it failed to be "
63 + f"opened. Reason: {e}")
64 + raise ValueError(msg) from e
65 +
66 + # If opening a file fails, set chat template to be args to
67 + # ensure we decode so our escape are interpreted correctly
68 + tokenizer.chat_template = codecs.decode(
69 + chat_template, "unicode_escape")
70 +
71 + logger.info("Using supplied chat template:\n%s",
72 + tokenizer.chat_template)
73 + elif tokenizer.chat_template is not None:
74 + logger.info("Using default chat template:\n%s",
75 + tokenizer.chat_template)
76 + else:
77 + logger.warning(
78 + "No chat template provided. Chat API will not work.")
79
80 def _parse_chat_message_content(
81 self,
@@ -357,36 +387,4 @@ class OpenAIServingChat(OpenAIServing):
387 usage=usage,
388 )
389
360 - return response
361 -
362 - async def _load_chat_template(self, chat_template: Optional[str]):
363 - while self.tokenizer is None:
364 - # Give the parent class time to load the tokenizer
365 - await asyncio.sleep(0.1)
366 - tokenizer = self.tokenizer
367 -
368 - if chat_template is not None:
369 - try:
370 - with open(chat_template, "r") as f:
371 - tokenizer.chat_template = f.read()
372 - except OSError as e:
373 - JINJA_CHARS = "{}\n"
374 - if not any(c in chat_template for c in JINJA_CHARS):
375 - msg = (f"The supplied chat template ({chat_template}) "
376 - f"looks like a file path, but it failed to be "
377 - f"opened. Reason: {e}")
378 - raise ValueError(msg) from e
379 -
380 - # If opening a file fails, set chat template to be args to
381 - # ensure we decode so our escape are interpreted correctly
382 - tokenizer.chat_template = codecs.decode(
383 - chat_template, "unicode_escape")
384 -
385 - logger.info("Using supplied chat template:\n%s",
386 - tokenizer.chat_template)
387 - elif tokenizer.chat_template is not None:
388 - logger.info("Using default chat template:\n%s",
389 - tokenizer.chat_template)
390 - else:
391 - logger.warning(
392 - "No chat template provided. Chat API will not work.")
390 + return response
\ No newline at end of file
vllm/entrypoints/openai/serving_completion.py
+4 -3
@@ -4,6 +4,7 @@ from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
4
5 from fastapi import Request
6
7 +from vllm.config import ModelConfig
8 from vllm.engine.async_llm_engine import AsyncLLMEngine
9 from vllm.entrypoints.openai.protocol import (CompletionRequest,
10 CompletionResponse,
@@ -52,11 +53,11 @@ def parse_prompt_format(prompt) -> Tuple[bool, list]:
53
54 class OpenAIServingCompletion(OpenAIServing):
55
55 - def __init__(self,
56 - engine: AsyncLLMEngine,
56 + def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
57 served_model_names: List[str],
58 - lora_modules: Optional[List[LoRAModulePath]] = None):
58 + lora_modules: Optional[List[LoRAModulePath]]):
59 super().__init__(engine=engine,
60 + model_config=model_config,
61 served_model_names=served_model_names,
62 lora_modules=lora_modules)
63
vllm/entrypoints/openai/serving_engine.py
+17 -39
@@ -1,13 +1,12 @@
1 -import asyncio
1 import json
2 from dataclasses import dataclass
3 from http import HTTPStatus
5 -from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union
4 +from typing import Dict, List, Optional, Tuple, Union
5
6 from pydantic import Field
8 -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
7 from typing_extensions import Annotated
8
9 +from vllm.config import ModelConfig
10 from vllm.engine.async_llm_engine import AsyncLLMEngine
11 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12 CompletionRequest, ErrorResponse,
@@ -29,13 +28,24 @@ class LoRAModulePath:
28
29 class OpenAIServing:
30
32 - def __init__(self,
33 - engine: AsyncLLMEngine,
31 + def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
32 served_model_names: List[str],
35 - lora_modules: Optional[List[LoRAModulePath]],
36 - await_post_init: Optional[Awaitable[Any]] = None):
33 + lora_modules: Optional[List[LoRAModulePath]]):
34 + super().__init__()
35 +
36 self.engine = engine
37 + self.max_model_len = model_config.max_model_len
38 +
39 + # A separate tokenizer to map token IDs to strings.
40 + self.tokenizer = get_tokenizer(
41 + model_config.tokenizer,
42 + tokenizer_mode=model_config.tokenizer_mode,
43 + tokenizer_revision=model_config.tokenizer_revision,
44 + trust_remote_code=model_config.trust_remote_code,
45 + truncation_side="left")
46 +
47 self.served_model_names = served_model_names
48 +
49 if lora_modules is None:
50 self.lora_requests = []
51 else:
@@ -47,38 +57,6 @@ class OpenAIServing:
57 ) for i, lora in enumerate(lora_modules, start=1)
58 ]
59
50 - self.max_model_len = 0
51 - # Lazy initialized
52 - self.tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
53 -
54 - try:
55 - event_loop = asyncio.get_running_loop()
56 - except RuntimeError:
57 - event_loop = None
58 -
59 - if event_loop is not None and event_loop.is_running():
60 - # If the current is instanced by Ray Serve,
61 - # there is already a running event loop
62 - event_loop.create_task(self._post_init(await_post_init))
63 - else:
64 - # When using single vLLM without engine_use_ray
65 - asyncio.run(self._post_init(await_post_init))
66 -
67 - async def _post_init(self, await_post_init):
68 - engine_model_config = await self.engine.get_model_config()
69 - self.max_model_len = engine_model_config.max_model_len
70 -
71 - # A separate tokenizer to map token IDs to strings.
72 - self.tokenizer = get_tokenizer(
73 - engine_model_config.tokenizer,
74 - tokenizer_mode=engine_model_config.tokenizer_mode,
75 - tokenizer_revision=engine_model_config.tokenizer_revision,
76 - trust_remote_code=engine_model_config.trust_remote_code,
77 - truncation_side="left")
78 -
79 - if await_post_init is not None:
80 - await await_post_init
81 -
60 async def show_available_models(self) -> ModelList:
61 """Show available models. Right now we only have one model."""
62 model_cards = [