@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 395b1c7454
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Frontend] don't block event loop in tokenization (preprocess) in OpenAI compatible server (#10635)

Signed-off-by: Tomer Asida <tomera@ai21.com>

tomeras91 committed Nov 27, 2024 at 23:21 UTC 395b1c74543053ebf25d4ab3af828cd145506caa
7 files changed +206 -56
tests/entrypoints/openai/test_async_tokenization.py new
+137
@@ -0,0 +1,137 @@
1 +import asyncio
2 +import contextlib
3 +import random
4 +import time
5 +from typing import Callable
6 +
7 +import openai
8 +import pytest
9 +import pytest_asyncio
10 +import requests
11 +
12 +from tests.utils import RemoteOpenAIServer
13 +
14 +MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
15 +
16 +
17 +@pytest.fixture(scope="module")
18 +def server(): # noqa: F811
19 + args = [
20 + # use half precision for speed and memory savings in CI environment
21 + "--dtype",
22 + "bfloat16",
23 + "--max-model-len",
24 + "8192",
25 + "--enforce-eager",
26 + "--max-num-seqs",
27 + "128",
28 + "--load-format",
29 + "dummy",
30 + ]
31 +
32 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
33 + yield remote_server
34 +
35 +
36 +@pytest_asyncio.fixture
37 +async def client(server):
38 + async with server.get_async_client() as async_client:
39 + yield async_client
40 +
41 +
42 +@pytest.mark.asyncio
43 +@pytest.mark.parametrize(
44 + ids=["completion", "chat"],
45 + argnames=["create_func_gen", "content_body"],
46 + argvalues=[
47 + (lambda x: x.completions.create, {
48 + "prompt": " ".join(['A'] * 10_000)
49 + }),
50 + (lambda x: x.chat.completions.create, {
51 + "messages": [{
52 + "role": "user",
53 + "content": " ".join(['A'] * 10_000)
54 + }]
55 + }),
56 + ],
57 +)
58 +async def test_with_and_without_truncate(
59 + server: RemoteOpenAIServer,
60 + client: openai.AsyncOpenAI,
61 + create_func_gen: Callable,
62 + content_body: dict,
63 +):
64 + create_func = create_func_gen(client)
65 + body = {"model": MODEL_NAME, **content_body, "max_tokens": 10}
66 +
67 + num_requests = 10
68 + truncate_prompt_tokens = ([1000] * (num_requests // 2) + [None] *
69 + (num_requests - num_requests // 2))
70 + random.shuffle(truncate_prompt_tokens)
71 +
72 + bodies = [{
73 + **body, "extra_body": {
74 + 'truncate_prompt_tokens': t
75 + }
76 + } for t in truncate_prompt_tokens]
77 +
78 + async def get_status_code(**kwargs):
79 + try:
80 + await create_func(**kwargs)
81 + return 200
82 + except openai.APIStatusError as e:
83 + return e.status_code
84 +
85 + responses = await asyncio.gather(*[get_status_code(**b) for b in bodies])
86 + assert 500 not in responses
87 +
88 +
89 +@pytest.mark.asyncio
90 +@pytest.mark.parametrize(
91 + ids=["single completion", "multiple completions", "chat"],
92 + argnames=["create_func_gen", "content_body"],
93 + argvalues=[
94 + (lambda x: x.completions.create, {
95 + "prompt": " ".join(['A'] * 300_000)
96 + }),
97 + (lambda x: x.completions.create, {
98 + "prompt": [" ".join(['A'] * 300_000)] * 2
99 + }),
100 + (lambda x: x.chat.completions.create, {
101 + "messages": [{
102 + "role": "user",
103 + "content": " ".join(['A'] * 300_000)
104 + }]
105 + }),
106 + ],
107 +)
108 +async def test_healthcheck_response_time(
109 + server: RemoteOpenAIServer,
110 + client: openai.AsyncOpenAI,
111 + create_func_gen: Callable,
112 + content_body: dict,
113 +):
114 + num_requests = 50
115 +
116 + create_func = create_func_gen(client)
117 + body = {"model": MODEL_NAME, **content_body, "max_tokens": 10}
118 +
119 + def get_response_time(url):
120 + start_time = time.monotonic()
121 + res = requests.get(url)
122 + end_time = time.monotonic()
123 + assert res.status_code == 200
124 + return end_time - start_time
125 +
126 + no_load_response_time = get_response_time(server.url_for("health"))
127 + tasks = [
128 + asyncio.create_task(create_func(**body)) for _ in range(num_requests)
129 + ]
130 + await asyncio.sleep(1) # give the tasks a chance to start running
131 + load_response_time = get_response_time(server.url_for("health"))
132 +
133 + with contextlib.suppress(openai.APIStatusError):
134 + await asyncio.gather(*tasks)
135 +
136 + assert load_response_time < 100 * no_load_response_time
137 + assert load_response_time < 0.1
vllm/entrypoints/openai/serving_completion.py
+1 -1
@@ -101,7 +101,7 @@ class OpenAIServingCompletion(OpenAIServing):
101
102 tokenizer = await self.engine_client.get_tokenizer(lora_request)
103
104 - request_prompts, engine_prompts = self._preprocess_completion(
104 + request_prompts, engine_prompts = await self._preprocess_completion(
105 request,
106 tokenizer,
107 request.prompt,
vllm/entrypoints/openai/serving_embedding.py
+8 -7
@@ -156,13 +156,14 @@ class OpenAIServingEmbedding(OpenAIServing):
156 add_special_tokens=request.add_special_tokens,
157 )
158 else:
159 - request_prompts, engine_prompts = self._preprocess_completion(
160 - request,
161 - tokenizer,
162 - request.input,
163 - truncate_prompt_tokens=truncate_prompt_tokens,
164 - add_special_tokens=request.add_special_tokens,
165 - )
159 + (request_prompts,
160 + engine_prompts) = await self._preprocess_completion(
161 + request,
162 + tokenizer,
163 + request.input,
164 + truncate_prompt_tokens=truncate_prompt_tokens,
165 + add_special_tokens=request.add_special_tokens,
166 + )
167 except ValueError as e:
168 logger.exception("Error in preprocessing prompt inputs")
169 return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_engine.py
+40 -35
@@ -1,5 +1,6 @@
1 import json
2 import pathlib
3 +from concurrent.futures.thread import ThreadPoolExecutor
4 from dataclasses import dataclass
5 from http import HTTPStatus
6 from typing import (Any, Callable, Dict, Iterable, Iterator, List, Mapping,
@@ -46,7 +47,7 @@ from vllm.sequence import Logprob
47 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
48 log_tracing_disabled_warning)
49 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
49 -from vllm.utils import AtomicCounter, is_list_of
50 +from vllm.utils import AtomicCounter, is_list_of, make_async
51
52 logger = init_logger(__name__)
53
@@ -140,6 +141,14 @@ class OpenAIServing:
141 self.request_logger = request_logger
142 self.return_tokens_as_token_ids = return_tokens_as_token_ids
143
144 + self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)
145 +
146 + self._tokenize_prompt_input_async = make_async(
147 + self._tokenize_prompt_input, executor=self._tokenizer_executor)
148 + self._tokenize_prompt_input_or_inputs_async = make_async(
149 + self._tokenize_prompt_input_or_inputs,
150 + executor=self._tokenizer_executor)
151 +
152 async def show_available_models(self) -> ModelList:
153 """Show available models. Right now we only have one model."""
154 model_cards = [
@@ -368,7 +377,7 @@ class OpenAIServing:
377 input_or_inputs: Union[str, List[str], List[int], List[List[int]]],
378 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
379 add_special_tokens: bool = True,
371 - ) -> Iterator[TextTokensPrompt]:
380 + ) -> List[TextTokensPrompt]:
381 """
382 Tokenize/detokenize depending on the input format.
383
@@ -376,45 +385,41 @@ class OpenAIServing:
385 , each input can be a string or array of tokens. Note that each request
386 can pass one or more inputs.
387 """
379 - for prompt_input in parse_and_batch_prompt(input_or_inputs):
380 - # Although our type checking is based on mypy,
381 - # VSCode Pyright extension should still work properly
382 - # "is True" is required for Pyright to perform type narrowing
383 - # See: https://github.com/microsoft/pyright/issues/7672
384 - if prompt_input["is_tokens"] is False:
385 - yield self._normalize_prompt_text_to_input(
386 - request,
387 - tokenizer,
388 - prompt=prompt_input["content"],
389 - truncate_prompt_tokens=truncate_prompt_tokens,
390 - add_special_tokens=add_special_tokens,
391 - )
392 - else:
393 - yield self._normalize_prompt_tokens_to_input(
394 - request,
395 - tokenizer,
396 - prompt_ids=prompt_input["content"],
397 - truncate_prompt_tokens=truncate_prompt_tokens,
398 - )
388 + # Although our type checking is based on mypy,
389 + # VSCode Pyright extension should still work properly
390 + # "is True" is required for Pyright to perform type narrowing
391 + # See: https://github.com/microsoft/pyright/issues/7672
392 + return [
393 + self._normalize_prompt_text_to_input(
394 + request,
395 + tokenizer,
396 + prompt=prompt_input["content"],
397 + truncate_prompt_tokens=truncate_prompt_tokens,
398 + add_special_tokens=add_special_tokens)
399 + if prompt_input["is_tokens"] is False else
400 + self._normalize_prompt_tokens_to_input(
401 + request,
402 + tokenizer,
403 + prompt_ids=prompt_input["content"],
404 + truncate_prompt_tokens=truncate_prompt_tokens)
405 + for prompt_input in parse_and_batch_prompt(input_or_inputs)
406 + ]
407
400 - def _preprocess_completion(
408 + async def _preprocess_completion(
409 self,
410 request: CompletionLikeRequest,
411 tokenizer: AnyTokenizer,
412 input_or_inputs: Union[str, List[str], List[int], List[List[int]]],
413 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
414 add_special_tokens: bool = True,
407 - ) -> Tuple[Sequence[TextTokensPrompt], List[TokensPrompt]]:
408 - request_prompts = [
409 - request_prompt
410 - for request_prompt in self._tokenize_prompt_input_or_inputs(
411 - request,
412 - tokenizer,
413 - input_or_inputs,
414 - truncate_prompt_tokens=truncate_prompt_tokens,
415 - add_special_tokens=add_special_tokens,
416 - )
417 - ]
415 + ) -> Tuple[List[TextTokensPrompt], List[TokensPrompt]]:
416 + request_prompts = await self._tokenize_prompt_input_or_inputs_async(
417 + request,
418 + tokenizer,
419 + input_or_inputs,
420 + truncate_prompt_tokens=truncate_prompt_tokens,
421 + add_special_tokens=add_special_tokens,
422 + )
423
424 engine_prompts = [
425 TokensPrompt(prompt_token_ids=request_prompt["prompt_token_ids"])
@@ -493,7 +498,7 @@ class OpenAIServing:
498 request=request)
499
500 if isinstance(request_prompt, str):
496 - prompt_inputs = self._tokenize_prompt_input(
501 + prompt_inputs = await self._tokenize_prompt_input_async(
502 request,
503 tokenizer,
504 request_prompt,
vllm/entrypoints/openai/serving_score.py
+6 -4
@@ -15,7 +15,7 @@ from vllm.inputs.data import TokensPrompt
15 from vllm.logger import init_logger
16 from vllm.outputs import EmbeddingRequestOutput
17 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
18 -from vllm.utils import merge_async_iterators, random_uuid
18 +from vllm.utils import make_async, merge_async_iterators, random_uuid
19
20 logger = init_logger(__name__)
21
@@ -145,9 +145,11 @@ class OpenAIServingScores(OpenAIServing):
145 tokenization_kwargs["truncation"] = True
146 tokenization_kwargs["max_length"] = truncate_prompt_tokens
147
148 - prompt_inputs = tokenizer(text=q,
149 - text_pair=t,
150 - **tokenization_kwargs)
148 + tokenize_async = make_async(tokenizer.__call__,
149 + executor=self._tokenizer_executor)
150 + prompt_inputs = await tokenize_async(text=q,
151 + text_pair=t,
152 + **tokenization_kwargs)
153 engine_prompt = TokensPrompt(
154 prompt_token_ids=prompt_inputs["input_ids"],
155 token_type_ids=prompt_inputs.get("token_type_ids"))
vllm/entrypoints/openai/serving_tokenization.py
+8 -7
@@ -81,12 +81,13 @@ class OpenAIServingTokenization(OpenAIServing):
81 add_special_tokens=request.add_special_tokens,
82 )
83 else:
84 - request_prompts, engine_prompts = self._preprocess_completion(
85 - request,
86 - tokenizer,
87 - request.prompt,
88 - add_special_tokens=request.add_special_tokens,
89 - )
84 + (request_prompts,
85 + engine_prompts) = await self._preprocess_completion(
86 + request,
87 + tokenizer,
88 + request.prompt,
89 + add_special_tokens=request.add_special_tokens,
90 + )
91 except ValueError as e:
92 logger.exception("Error in preprocessing prompt inputs")
93 return self.create_error_response(str(e))
@@ -134,7 +135,7 @@ class OpenAIServingTokenization(OpenAIServing):
135 # Silently ignore prompt adapter since it does not affect tokenization
136 # (Unlike in Embeddings API where an error is raised)
137
137 - prompt_input = self._tokenize_prompt_input(
138 + prompt_input = await self._tokenize_prompt_input_async(
139 request,
140 tokenizer,
141 request.tokens,
vllm/utils.py
+6 -2
@@ -1,5 +1,6 @@
1 import argparse
2 import asyncio
3 +import concurrent
4 import contextlib
5 import datetime
6 import enum
@@ -351,7 +352,10 @@ def in_wsl() -> bool:
352 return "microsoft" in " ".join(uname()).lower()
353
354
354 -def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
355 +def make_async(
356 + func: Callable[P, T],
357 + executor: Optional[concurrent.futures.Executor] = None
358 +) -> Callable[P, Awaitable[T]]:
359 """Take a blocking function, and run it on in an executor thread.
360
361 This function prevents the blocking function from blocking the
@@ -362,7 +366,7 @@ def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
366 def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> asyncio.Future:
367 loop = asyncio.get_event_loop()
368 p_func = partial(func, *args, **kwargs)
365 - return loop.run_in_executor(executor=None, func=p_func)
369 + return loop.run_in_executor(executor=executor, func=p_func)
370
371 return _async_wrapper
372