@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 980ad394a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Frontend] Use request id from header (#10968)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Dec 9, 2024 at 22:46 UTC 980ad394a83a6f12c576a035922db3c2e743beff
8 files changed +27 -13
docs/requirements-docs.txt
+1
@@ -16,5 +16,6 @@ mistral_common >= 1.5.0
16 aiohttp
17 starlette
18 openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
19 +fastapi # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
20 partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
21 requests
vllm/entrypoints/openai/api_server.py
+2 -2
@@ -305,7 +305,7 @@ async def health(raw_request: Request) -> Response:
305 async def tokenize(request: TokenizeRequest, raw_request: Request):
306 handler = tokenization(raw_request)
307
308 - generator = await handler.create_tokenize(request)
308 + generator = await handler.create_tokenize(request, raw_request)
309 if isinstance(generator, ErrorResponse):
310 return JSONResponse(content=generator.model_dump(),
311 status_code=generator.code)
@@ -319,7 +319,7 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
319 async def detokenize(request: DetokenizeRequest, raw_request: Request):
320 handler = tokenization(raw_request)
321
322 - generator = await handler.create_detokenize(request)
322 + generator = await handler.create_detokenize(request, raw_request)
323 if isinstance(generator, ErrorResponse):
324 return JSONResponse(content=generator.model_dump(),
325 status_code=generator.code)
vllm/entrypoints/openai/serving_chat.py
+2 -1
@@ -176,7 +176,8 @@ class OpenAIServingChat(OpenAIServing):
176 logger.exception("Error in preprocessing prompt inputs")
177 return self.create_error_response(str(e))
178
179 - request_id = f"chatcmpl-{request.request_id}"
179 + request_id = "chatcmpl-" \
180 + f"{self._base_request_id(raw_request, request.request_id)}"
181
182 request_metadata = RequestResponseMetadata(request_id=request_id)
183 if raw_request:
vllm/entrypoints/openai/serving_completion.py
+2 -2
@@ -30,7 +30,7 @@ from vllm.outputs import RequestOutput
30 from vllm.sampling_params import BeamSearchParams, SamplingParams
31 from vllm.sequence import Logprob
32 from vllm.transformers_utils.tokenizer import AnyTokenizer
33 -from vllm.utils import merge_async_iterators, random_uuid
33 +from vllm.utils import merge_async_iterators
34
35 logger = init_logger(__name__)
36
@@ -86,7 +86,7 @@ class OpenAIServingCompletion(OpenAIServing):
86 "suffix is not currently supported")
87
88 model_name = self.base_model_paths[0].name
89 - request_id = f"cmpl-{random_uuid()}"
89 + request_id = f"cmpl-{self._base_request_id(raw_request)}"
90 created_time = int(time.time())
91
92 request_metadata = RequestResponseMetadata(request_id=request_id)
vllm/entrypoints/openai/serving_embedding.py
+2 -2
@@ -19,7 +19,7 @@ from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
19 from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
20 from vllm.logger import init_logger
21 from vllm.outputs import PoolingOutput, PoolingRequestOutput
22 -from vllm.utils import merge_async_iterators, random_uuid
22 +from vllm.utils import merge_async_iterators
23
24 logger = init_logger(__name__)
25
@@ -110,7 +110,7 @@ class OpenAIServingEmbedding(OpenAIServing):
110 "dimensions is currently not supported")
111
112 model_name = request.model
113 - request_id = f"embd-{random_uuid()}"
113 + request_id = f"embd-{self._base_request_id(raw_request)}"
114 created_time = int(time.monotonic())
115
116 truncate_prompt_tokens = None
vllm/entrypoints/openai/serving_engine.py
+10 -1
@@ -6,6 +6,7 @@ from http import HTTPStatus
6 from typing import (Any, Callable, Dict, Iterable, Iterator, List, Mapping,
7 Optional, Sequence, Tuple, TypedDict, Union)
8
9 +from fastapi import Request
10 from pydantic import Field
11 from starlette.datastructures import Headers
12 from typing_extensions import Annotated
@@ -47,7 +48,7 @@ from vllm.sequence import Logprob
48 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
49 log_tracing_disabled_warning)
50 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
50 -from vllm.utils import AtomicCounter, is_list_of, make_async
51 +from vllm.utils import AtomicCounter, is_list_of, make_async, random_uuid
52
53 logger = init_logger(__name__)
54
@@ -565,6 +566,14 @@ class OpenAIServing:
566
567 return None
568
569 + @staticmethod
570 + def _base_request_id(raw_request: Request,
571 + default: Optional[str] = None) -> Optional[str]:
572 + """Pulls the request id to use from a header, if provided"""
573 + default = default or random_uuid()
574 + return raw_request.headers.get(
575 + "X-Request-Id", default) if raw_request is not None else default
576 +
577 @staticmethod
578 def _get_decoded_token(logprob: Logprob,
579 token_id: int,
vllm/entrypoints/openai/serving_score.py
+2 -2
@@ -15,7 +15,7 @@ from vllm.inputs.data import TokensPrompt
15 from vllm.logger import init_logger
16 from vllm.outputs import PoolingRequestOutput
17 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
18 -from vllm.utils import make_async, merge_async_iterators, random_uuid
18 +from vllm.utils import make_async, merge_async_iterators
19
20 logger = init_logger(__name__)
21
@@ -102,7 +102,7 @@ class OpenAIServingScores(OpenAIServing):
102 return error_check_ret
103
104 model_name = request.model
105 - request_id = f"score-{random_uuid()}"
105 + request_id = f"score-{self._base_request_id(raw_request)}"
106 created_time = int(time.monotonic())
107 truncate_prompt_tokens = request.truncate_prompt_tokens
108
vllm/entrypoints/openai/serving_tokenization.py
+6 -3
@@ -1,5 +1,7 @@
1 from typing import Final, List, Optional, Union
2
3 +from fastapi import Request
4 +
5 from vllm.config import ModelConfig
6 from vllm.engine.protocol import EngineClient
7 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
@@ -17,7 +19,6 @@ from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
19 LoRAModulePath,
20 OpenAIServing)
21 from vllm.logger import init_logger
20 -from vllm.utils import random_uuid
22
23 logger = init_logger(__name__)
24
@@ -48,12 +49,13 @@ class OpenAIServingTokenization(OpenAIServing):
49 async def create_tokenize(
50 self,
51 request: TokenizeRequest,
52 + raw_request: Request,
53 ) -> Union[TokenizeResponse, ErrorResponse]:
54 error_check_ret = await self._check_model(request)
55 if error_check_ret is not None:
56 return error_check_ret
57
56 - request_id = f"tokn-{random_uuid()}"
58 + request_id = f"tokn-{self._base_request_id(raw_request)}"
59
60 try:
61 (
@@ -112,12 +114,13 @@ class OpenAIServingTokenization(OpenAIServing):
114 async def create_detokenize(
115 self,
116 request: DetokenizeRequest,
117 + raw_request: Request,
118 ) -> Union[DetokenizeResponse, ErrorResponse]:
119 error_check_ret = await self._check_model(request)
120 if error_check_ret is not None:
121 return error_check_ret
122
120 - request_id = f"tokn-{random_uuid()}"
123 + request_id = f"tokn-{self._base_request_id(raw_request)}"
124
125 (
126 lora_request,