A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Frontend] Use request id from header (#10968)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Dec 9, 2024 at 22:46 UTC
980ad394a83a6f12c576a035922db3c2e743beff
8 files changed
+27
-13
docs/requirements-docs.txt
+1
@@ -16,5 +16,6 @@ mistral_common >= 1.5.0
16
aiohttp
17
starlette
18
openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
19
+fastapi # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
20
partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
21
requests
vllm/entrypoints/openai/api_server.py
+2
-2
@@ -305,7 +305,7 @@ async def health(raw_request: Request) -> Response:
305
async def tokenize(request: TokenizeRequest, raw_request: Request):
306
handler = tokenization(raw_request)
307
308
- generator = await handler.create_tokenize(request)
308
+ generator = await handler.create_tokenize(request, raw_request)
309
if isinstance(generator, ErrorResponse):
310
return JSONResponse(content=generator.model_dump(),
311
status_code=generator.code)
@@ -319,7 +319,7 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
319
async def detokenize(request: DetokenizeRequest, raw_request: Request):
320
handler = tokenization(raw_request)
321
322
- generator = await handler.create_detokenize(request)
322
+ generator = await handler.create_detokenize(request, raw_request)
323
if isinstance(generator, ErrorResponse):
324
return JSONResponse(content=generator.model_dump(),
325
status_code=generator.code)
vllm/entrypoints/openai/serving_chat.py
+2
-1
@@ -176,7 +176,8 @@ class OpenAIServingChat(OpenAIServing):
176
logger.exception("Error in preprocessing prompt inputs")
177
return self.create_error_response(str(e))
178
179
- request_id = f"chatcmpl-{request.request_id}"
179
+ request_id = "chatcmpl-" \
180
+ f"{self._base_request_id(raw_request, request.request_id)}"
181
182
request_metadata = RequestResponseMetadata(request_id=request_id)
183
if raw_request:
vllm/entrypoints/openai/serving_completion.py
+2
-2
@@ -30,7 +30,7 @@ from vllm.outputs import RequestOutput
30
from vllm.sampling_params import BeamSearchParams, SamplingParams
31
from vllm.sequence import Logprob
32
from vllm.transformers_utils.tokenizer import AnyTokenizer
33
-from vllm.utils import merge_async_iterators, random_uuid
33
+from vllm.utils import merge_async_iterators
34
35
logger = init_logger(__name__)
36
@@ -86,7 +86,7 @@ class OpenAIServingCompletion(OpenAIServing):
86
"suffix is not currently supported")
87
88
model_name = self.base_model_paths[0].name
89
- request_id = f"cmpl-{random_uuid()}"
89
+ request_id = f"cmpl-{self._base_request_id(raw_request)}"
90
created_time = int(time.time())
91
92
request_metadata = RequestResponseMetadata(request_id=request_id)
vllm/entrypoints/openai/serving_embedding.py
+2
-2
@@ -19,7 +19,7 @@ from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
19
from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
20
from vllm.logger import init_logger
21
from vllm.outputs import PoolingOutput, PoolingRequestOutput
22
-from vllm.utils import merge_async_iterators, random_uuid
22
+from vllm.utils import merge_async_iterators
23
24
logger = init_logger(__name__)
25
@@ -110,7 +110,7 @@ class OpenAIServingEmbedding(OpenAIServing):
110
"dimensions is currently not supported")
111
112
model_name = request.model
113
- request_id = f"embd-{random_uuid()}"
113
+ request_id = f"embd-{self._base_request_id(raw_request)}"
114
created_time = int(time.monotonic())
115
116
truncate_prompt_tokens = None
vllm/entrypoints/openai/serving_engine.py
+10
-1
@@ -6,6 +6,7 @@ from http import HTTPStatus
6
from typing import (Any, Callable, Dict, Iterable, Iterator, List, Mapping,
7
Optional, Sequence, Tuple, TypedDict, Union)
8
9
+from fastapi import Request
10
from pydantic import Field
11
from starlette.datastructures import Headers
12
from typing_extensions import Annotated
@@ -47,7 +48,7 @@ from vllm.sequence import Logprob
48
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
49
log_tracing_disabled_warning)
50
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
50
-from vllm.utils import AtomicCounter, is_list_of, make_async
51
+from vllm.utils import AtomicCounter, is_list_of, make_async, random_uuid
52
53
logger = init_logger(__name__)
54
@@ -565,6 +566,14 @@ class OpenAIServing:
566
567
return None
568
569
+ @staticmethod
570
+ def _base_request_id(raw_request: Request,
571
+ default: Optional[str] = None) -> Optional[str]:
572
+ """Pulls the request id to use from a header, if provided"""
573
+ default = default or random_uuid()
574
+ return raw_request.headers.get(
575
+ "X-Request-Id", default) if raw_request is not None else default
576
+
577
@staticmethod
578
def _get_decoded_token(logprob: Logprob,
579
token_id: int,
vllm/entrypoints/openai/serving_score.py
+2
-2
@@ -15,7 +15,7 @@ from vllm.inputs.data import TokensPrompt
15
from vllm.logger import init_logger
16
from vllm.outputs import PoolingRequestOutput
17
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
18
-from vllm.utils import make_async, merge_async_iterators, random_uuid
18
+from vllm.utils import make_async, merge_async_iterators
19
20
logger = init_logger(__name__)
21
@@ -102,7 +102,7 @@ class OpenAIServingScores(OpenAIServing):
102
return error_check_ret
103
104
model_name = request.model
105
- request_id = f"score-{random_uuid()}"
105
+ request_id = f"score-{self._base_request_id(raw_request)}"
106
created_time = int(time.monotonic())
107
truncate_prompt_tokens = request.truncate_prompt_tokens
108
vllm/entrypoints/openai/serving_tokenization.py
+6
-3
@@ -1,5 +1,7 @@
1
from typing import Final, List, Optional, Union
2
3
+from fastapi import Request
4
+
5
from vllm.config import ModelConfig
6
from vllm.engine.protocol import EngineClient
7
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
@@ -17,7 +19,6 @@ from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
19
LoRAModulePath,
20
OpenAIServing)
21
from vllm.logger import init_logger
20
-from vllm.utils import random_uuid
22
23
logger = init_logger(__name__)
24
@@ -48,12 +49,13 @@ class OpenAIServingTokenization(OpenAIServing):
49
async def create_tokenize(
50
self,
51
request: TokenizeRequest,
52
+ raw_request: Request,
53
) -> Union[TokenizeResponse, ErrorResponse]:
54
error_check_ret = await self._check_model(request)
55
if error_check_ret is not None:
56
return error_check_ret
57
56
- request_id = f"tokn-{random_uuid()}"
58
+ request_id = f"tokn-{self._base_request_id(raw_request)}"
59
60
try:
61
(
@@ -112,12 +114,13 @@ class OpenAIServingTokenization(OpenAIServing):
114
async def create_detokenize(
115
self,
116
request: DetokenizeRequest,
117
+ raw_request: Request,
118
) -> Union[DetokenizeResponse, ErrorResponse]:
119
error_check_ret = await self._check_model(request)
120
if error_check_ret is not None:
121
return error_check_ret
122
120
- request_id = f"tokn-{random_uuid()}"
123
+ request_id = f"tokn-{self._base_request_id(raw_request)}"
124
125
(
126
lora_request,