A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Frontend] don't block event loop in tokenization (preprocess) in OpenAI compatible server (#10635)
Signed-off-by: Tomer Asida <tomera@ai21.com>
tomeras91 committed
Nov 27, 2024 at 23:21 UTC
395b1c74543053ebf25d4ab3af828cd145506caa
7 files changed
+206
-56
tests/entrypoints/openai/test_async_tokenization.py
new
+137
@@ -0,0 +1,137 @@
1
+import asyncio
2
+import contextlib
3
+import random
4
+import time
5
+from typing import Callable
6
+
7
+import openai
8
+import pytest
9
+import pytest_asyncio
10
+import requests
11
+
12
+from tests.utils import RemoteOpenAIServer
13
+
14
+MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
15
+
16
+
17
+@pytest.fixture(scope="module")
18
+def server(): # noqa: F811
19
+ args = [
20
+ # use half precision for speed and memory savings in CI environment
21
+ "--dtype",
22
+ "bfloat16",
23
+ "--max-model-len",
24
+ "8192",
25
+ "--enforce-eager",
26
+ "--max-num-seqs",
27
+ "128",
28
+ "--load-format",
29
+ "dummy",
30
+ ]
31
+
32
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
33
+ yield remote_server
34
+
35
+
36
+@pytest_asyncio.fixture
37
+async def client(server):
38
+ async with server.get_async_client() as async_client:
39
+ yield async_client
40
+
41
+
42
+@pytest.mark.asyncio
43
+@pytest.mark.parametrize(
44
+ ids=["completion", "chat"],
45
+ argnames=["create_func_gen", "content_body"],
46
+ argvalues=[
47
+ (lambda x: x.completions.create, {
48
+ "prompt": " ".join(['A'] * 10_000)
49
+ }),
50
+ (lambda x: x.chat.completions.create, {
51
+ "messages": [{
52
+ "role": "user",
53
+ "content": " ".join(['A'] * 10_000)
54
+ }]
55
+ }),
56
+ ],
57
+)
58
+async def test_with_and_without_truncate(
59
+ server: RemoteOpenAIServer,
60
+ client: openai.AsyncOpenAI,
61
+ create_func_gen: Callable,
62
+ content_body: dict,
63
+):
64
+ create_func = create_func_gen(client)
65
+ body = {"model": MODEL_NAME, **content_body, "max_tokens": 10}
66
+
67
+ num_requests = 10
68
+ truncate_prompt_tokens = ([1000] * (num_requests // 2) + [None] *
69
+ (num_requests - num_requests // 2))
70
+ random.shuffle(truncate_prompt_tokens)
71
+
72
+ bodies = [{
73
+ **body, "extra_body": {
74
+ 'truncate_prompt_tokens': t
75
+ }
76
+ } for t in truncate_prompt_tokens]
77
+
78
+ async def get_status_code(**kwargs):
79
+ try:
80
+ await create_func(**kwargs)
81
+ return 200
82
+ except openai.APIStatusError as e:
83
+ return e.status_code
84
+
85
+ responses = await asyncio.gather(*[get_status_code(**b) for b in bodies])
86
+ assert 500 not in responses
87
+
88
+
89
+@pytest.mark.asyncio
90
+@pytest.mark.parametrize(
91
+ ids=["single completion", "multiple completions", "chat"],
92
+ argnames=["create_func_gen", "content_body"],
93
+ argvalues=[
94
+ (lambda x: x.completions.create, {
95
+ "prompt": " ".join(['A'] * 300_000)
96
+ }),
97
+ (lambda x: x.completions.create, {
98
+ "prompt": [" ".join(['A'] * 300_000)] * 2
99
+ }),
100
+ (lambda x: x.chat.completions.create, {
101
+ "messages": [{
102
+ "role": "user",
103
+ "content": " ".join(['A'] * 300_000)
104
+ }]
105
+ }),
106
+ ],
107
+)
108
+async def test_healthcheck_response_time(
109
+ server: RemoteOpenAIServer,
110
+ client: openai.AsyncOpenAI,
111
+ create_func_gen: Callable,
112
+ content_body: dict,
113
+):
114
+ num_requests = 50
115
+
116
+ create_func = create_func_gen(client)
117
+ body = {"model": MODEL_NAME, **content_body, "max_tokens": 10}
118
+
119
+ def get_response_time(url):
120
+ start_time = time.monotonic()
121
+ res = requests.get(url)
122
+ end_time = time.monotonic()
123
+ assert res.status_code == 200
124
+ return end_time - start_time
125
+
126
+ no_load_response_time = get_response_time(server.url_for("health"))
127
+ tasks = [
128
+ asyncio.create_task(create_func(**body)) for _ in range(num_requests)
129
+ ]
130
+ await asyncio.sleep(1) # give the tasks a chance to start running
131
+ load_response_time = get_response_time(server.url_for("health"))
132
+
133
+ with contextlib.suppress(openai.APIStatusError):
134
+ await asyncio.gather(*tasks)
135
+
136
+ assert load_response_time < 100 * no_load_response_time
137
+ assert load_response_time < 0.1
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -101,7 +101,7 @@ class OpenAIServingCompletion(OpenAIServing):
101
102
tokenizer = await self.engine_client.get_tokenizer(lora_request)
103
104
- request_prompts, engine_prompts = self._preprocess_completion(
104
+ request_prompts, engine_prompts = await self._preprocess_completion(
105
request,
106
tokenizer,
107
request.prompt,
vllm/entrypoints/openai/serving_embedding.py
+8
-7
@@ -156,13 +156,14 @@ class OpenAIServingEmbedding(OpenAIServing):
156
add_special_tokens=request.add_special_tokens,
157
)
158
else:
159
- request_prompts, engine_prompts = self._preprocess_completion(
160
- request,
161
- tokenizer,
162
- request.input,
163
- truncate_prompt_tokens=truncate_prompt_tokens,
164
- add_special_tokens=request.add_special_tokens,
165
- )
159
+ (request_prompts,
160
+ engine_prompts) = await self._preprocess_completion(
161
+ request,
162
+ tokenizer,
163
+ request.input,
164
+ truncate_prompt_tokens=truncate_prompt_tokens,
165
+ add_special_tokens=request.add_special_tokens,
166
+ )
167
except ValueError as e:
168
logger.exception("Error in preprocessing prompt inputs")
169
return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_engine.py
+40
-35
@@ -1,5 +1,6 @@
1
import json
2
import pathlib
3
+from concurrent.futures.thread import ThreadPoolExecutor
4
from dataclasses import dataclass
5
from http import HTTPStatus
6
from typing import (Any, Callable, Dict, Iterable, Iterator, List, Mapping,
@@ -46,7 +47,7 @@ from vllm.sequence import Logprob
47
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
48
log_tracing_disabled_warning)
49
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
49
-from vllm.utils import AtomicCounter, is_list_of
50
+from vllm.utils import AtomicCounter, is_list_of, make_async
51
52
logger = init_logger(__name__)
53
@@ -140,6 +141,14 @@ class OpenAIServing:
141
self.request_logger = request_logger
142
self.return_tokens_as_token_ids = return_tokens_as_token_ids
143
144
+ self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)
145
+
146
+ self._tokenize_prompt_input_async = make_async(
147
+ self._tokenize_prompt_input, executor=self._tokenizer_executor)
148
+ self._tokenize_prompt_input_or_inputs_async = make_async(
149
+ self._tokenize_prompt_input_or_inputs,
150
+ executor=self._tokenizer_executor)
151
+
152
async def show_available_models(self) -> ModelList:
153
"""Show available models. Right now we only have one model."""
154
model_cards = [
@@ -368,7 +377,7 @@ class OpenAIServing:
377
input_or_inputs: Union[str, List[str], List[int], List[List[int]]],
378
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
379
add_special_tokens: bool = True,
371
- ) -> Iterator[TextTokensPrompt]:
380
+ ) -> List[TextTokensPrompt]:
381
"""
382
Tokenize/detokenize depending on the input format.
383
@@ -376,45 +385,41 @@ class OpenAIServing:
385
, each input can be a string or array of tokens. Note that each request
386
can pass one or more inputs.
387
"""
379
- for prompt_input in parse_and_batch_prompt(input_or_inputs):
380
- # Although our type checking is based on mypy,
381
- # VSCode Pyright extension should still work properly
382
- # "is True" is required for Pyright to perform type narrowing
383
- # See: https://github.com/microsoft/pyright/issues/7672
384
- if prompt_input["is_tokens"] is False:
385
- yield self._normalize_prompt_text_to_input(
386
- request,
387
- tokenizer,
388
- prompt=prompt_input["content"],
389
- truncate_prompt_tokens=truncate_prompt_tokens,
390
- add_special_tokens=add_special_tokens,
391
- )
392
- else:
393
- yield self._normalize_prompt_tokens_to_input(
394
- request,
395
- tokenizer,
396
- prompt_ids=prompt_input["content"],
397
- truncate_prompt_tokens=truncate_prompt_tokens,
398
- )
388
+ # Although our type checking is based on mypy,
389
+ # VSCode Pyright extension should still work properly
390
+ # "is True" is required for Pyright to perform type narrowing
391
+ # See: https://github.com/microsoft/pyright/issues/7672
392
+ return [
393
+ self._normalize_prompt_text_to_input(
394
+ request,
395
+ tokenizer,
396
+ prompt=prompt_input["content"],
397
+ truncate_prompt_tokens=truncate_prompt_tokens,
398
+ add_special_tokens=add_special_tokens)
399
+ if prompt_input["is_tokens"] is False else
400
+ self._normalize_prompt_tokens_to_input(
401
+ request,
402
+ tokenizer,
403
+ prompt_ids=prompt_input["content"],
404
+ truncate_prompt_tokens=truncate_prompt_tokens)
405
+ for prompt_input in parse_and_batch_prompt(input_or_inputs)
406
+ ]
407
400
- def _preprocess_completion(
408
+ async def _preprocess_completion(
409
self,
410
request: CompletionLikeRequest,
411
tokenizer: AnyTokenizer,
412
input_or_inputs: Union[str, List[str], List[int], List[List[int]]],
413
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
414
add_special_tokens: bool = True,
407
- ) -> Tuple[Sequence[TextTokensPrompt], List[TokensPrompt]]:
408
- request_prompts = [
409
- request_prompt
410
- for request_prompt in self._tokenize_prompt_input_or_inputs(
411
- request,
412
- tokenizer,
413
- input_or_inputs,
414
- truncate_prompt_tokens=truncate_prompt_tokens,
415
- add_special_tokens=add_special_tokens,
416
- )
417
- ]
415
+ ) -> Tuple[List[TextTokensPrompt], List[TokensPrompt]]:
416
+ request_prompts = await self._tokenize_prompt_input_or_inputs_async(
417
+ request,
418
+ tokenizer,
419
+ input_or_inputs,
420
+ truncate_prompt_tokens=truncate_prompt_tokens,
421
+ add_special_tokens=add_special_tokens,
422
+ )
423
424
engine_prompts = [
425
TokensPrompt(prompt_token_ids=request_prompt["prompt_token_ids"])
@@ -493,7 +498,7 @@ class OpenAIServing:
498
request=request)
499
500
if isinstance(request_prompt, str):
496
- prompt_inputs = self._tokenize_prompt_input(
501
+ prompt_inputs = await self._tokenize_prompt_input_async(
502
request,
503
tokenizer,
504
request_prompt,
vllm/entrypoints/openai/serving_score.py
+6
-4
@@ -15,7 +15,7 @@ from vllm.inputs.data import TokensPrompt
15
from vllm.logger import init_logger
16
from vllm.outputs import EmbeddingRequestOutput
17
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
18
-from vllm.utils import merge_async_iterators, random_uuid
18
+from vllm.utils import make_async, merge_async_iterators, random_uuid
19
20
logger = init_logger(__name__)
21
@@ -145,9 +145,11 @@ class OpenAIServingScores(OpenAIServing):
145
tokenization_kwargs["truncation"] = True
146
tokenization_kwargs["max_length"] = truncate_prompt_tokens
147
148
- prompt_inputs = tokenizer(text=q,
149
- text_pair=t,
150
- **tokenization_kwargs)
148
+ tokenize_async = make_async(tokenizer.__call__,
149
+ executor=self._tokenizer_executor)
150
+ prompt_inputs = await tokenize_async(text=q,
151
+ text_pair=t,
152
+ **tokenization_kwargs)
153
engine_prompt = TokensPrompt(
154
prompt_token_ids=prompt_inputs["input_ids"],
155
token_type_ids=prompt_inputs.get("token_type_ids"))
vllm/entrypoints/openai/serving_tokenization.py
+8
-7
@@ -81,12 +81,13 @@ class OpenAIServingTokenization(OpenAIServing):
81
add_special_tokens=request.add_special_tokens,
82
)
83
else:
84
- request_prompts, engine_prompts = self._preprocess_completion(
85
- request,
86
- tokenizer,
87
- request.prompt,
88
- add_special_tokens=request.add_special_tokens,
89
- )
84
+ (request_prompts,
85
+ engine_prompts) = await self._preprocess_completion(
86
+ request,
87
+ tokenizer,
88
+ request.prompt,
89
+ add_special_tokens=request.add_special_tokens,
90
+ )
91
except ValueError as e:
92
logger.exception("Error in preprocessing prompt inputs")
93
return self.create_error_response(str(e))
@@ -134,7 +135,7 @@ class OpenAIServingTokenization(OpenAIServing):
135
# Silently ignore prompt adapter since it does not affect tokenization
136
# (Unlike in Embeddings API where an error is raised)
137
137
- prompt_input = self._tokenize_prompt_input(
138
+ prompt_input = await self._tokenize_prompt_input_async(
139
request,
140
tokenizer,
141
request.tokens,
vllm/utils.py
+6
-2
@@ -1,5 +1,6 @@
1
import argparse
2
import asyncio
3
+import concurrent
4
import contextlib
5
import datetime
6
import enum
@@ -351,7 +352,10 @@ def in_wsl() -> bool:
352
return "microsoft" in " ".join(uname()).lower()
353
354
354
-def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
355
+def make_async(
356
+ func: Callable[P, T],
357
+ executor: Optional[concurrent.futures.Executor] = None
358
+) -> Callable[P, Awaitable[T]]:
359
"""Take a blocking function, and run it on in an executor thread.
360
361
This function prevents the blocking function from blocking the
@@ -362,7 +366,7 @@ def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
366
def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> asyncio.Future:
367
loop = asyncio.get_event_loop()
368
p_func = partial(func, *args, **kwargs)
365
- return loop.run_in_executor(executor=None, func=p_func)
369
+ return loop.run_in_executor(executor=executor, func=p_func)
370
371
return _async_wrapper
372