A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Add --log-error-stack to print stack trace for error response (#22960)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Aug 26, 2025 at 21:58 UTC
32102644213a6367d10ec3a92ae76fb0004f3a52
13 files changed
+51
-8
vllm/entrypoints/openai/api_server.py
+10
@@ -1749,6 +1749,7 @@ async def init_app_state(
1749
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
1750
enable_force_include_usage=args.enable_force_include_usage,
1751
enable_log_outputs=args.enable_log_outputs,
1752
+ log_error_stack=args.log_error_stack,
1753
) if "generate" in supported_tasks else None
1754
state.openai_serving_chat = OpenAIServingChat(
1755
engine_client,
@@ -1767,6 +1768,7 @@ async def init_app_state(
1768
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
1769
enable_force_include_usage=args.enable_force_include_usage,
1770
enable_log_outputs=args.enable_log_outputs,
1771
+ log_error_stack=args.log_error_stack,
1772
) if "generate" in supported_tasks else None
1773
state.openai_serving_completion = OpenAIServingCompletion(
1774
engine_client,
@@ -1776,6 +1778,7 @@ async def init_app_state(
1778
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
1779
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
1780
enable_force_include_usage=args.enable_force_include_usage,
1781
+ log_error_stack=args.log_error_stack,
1782
) if "generate" in supported_tasks else None
1783
state.openai_serving_pooling = OpenAIServingPooling(
1784
engine_client,
@@ -1784,6 +1787,7 @@ async def init_app_state(
1787
request_logger=request_logger,
1788
chat_template=resolved_chat_template,
1789
chat_template_content_format=args.chat_template_content_format,
1790
+ log_error_stack=args.log_error_stack,
1791
) if "encode" in supported_tasks else None
1792
state.openai_serving_embedding = OpenAIServingEmbedding(
1793
engine_client,
@@ -1792,12 +1796,14 @@ async def init_app_state(
1796
request_logger=request_logger,
1797
chat_template=resolved_chat_template,
1798
chat_template_content_format=args.chat_template_content_format,
1799
+ log_error_stack=args.log_error_stack,
1800
) if "embed" in supported_tasks else None
1801
state.openai_serving_classification = ServingClassification(
1802
engine_client,
1803
model_config,
1804
state.openai_serving_models,
1805
request_logger=request_logger,
1806
+ log_error_stack=args.log_error_stack,
1807
) if "classify" in supported_tasks else None
1808
1809
enable_serving_reranking = ("classify" in supported_tasks and getattr(
@@ -1807,6 +1813,7 @@ async def init_app_state(
1813
model_config,
1814
state.openai_serving_models,
1815
request_logger=request_logger,
1816
+ log_error_stack=args.log_error_stack,
1817
) if ("embed" in supported_tasks or enable_serving_reranking) else None
1818
1819
state.openai_serving_tokenization = OpenAIServingTokenization(
@@ -1816,18 +1823,21 @@ async def init_app_state(
1823
request_logger=request_logger,
1824
chat_template=resolved_chat_template,
1825
chat_template_content_format=args.chat_template_content_format,
1826
+ log_error_stack=args.log_error_stack,
1827
)
1828
state.openai_serving_transcription = OpenAIServingTranscription(
1829
engine_client,
1830
model_config,
1831
state.openai_serving_models,
1832
request_logger=request_logger,
1833
+ log_error_stack=args.log_error_stack,
1834
) if "transcription" in supported_tasks else None
1835
state.openai_serving_translation = OpenAIServingTranslation(
1836
engine_client,
1837
model_config,
1838
state.openai_serving_models,
1839
request_logger=request_logger,
1840
+ log_error_stack=args.log_error_stack,
1841
) if "transcription" in supported_tasks else None
1842
1843
state.enable_server_load_tracking = args.enable_server_load_tracking
vllm/entrypoints/openai/cli_args.py
+2
@@ -180,6 +180,8 @@ schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
180
h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
181
"""Maximum number of HTTP headers allowed in a request for h11 parser.
182
Helps mitigate header abuse. Default: 256."""
183
+ log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
184
+ """If set to True, log the stack trace of error responses"""
185
186
@staticmethod
187
def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
vllm/entrypoints/openai/serving_chat.py
+3
-1
@@ -76,13 +76,15 @@ class OpenAIServingChat(OpenAIServing):
76
enable_prompt_tokens_details: bool = False,
77
enable_force_include_usage: bool = False,
78
enable_log_outputs: bool = False,
79
+ log_error_stack: bool = False,
80
) -> None:
81
super().__init__(engine_client=engine_client,
82
model_config=model_config,
83
models=models,
84
request_logger=request_logger,
85
return_tokens_as_token_ids=return_tokens_as_token_ids,
85
- enable_force_include_usage=enable_force_include_usage)
86
+ enable_force_include_usage=enable_force_include_usage,
87
+ log_error_stack=log_error_stack)
88
89
self.response_role = response_role
90
self.chat_template = chat_template
vllm/entrypoints/openai/serving_classification.py
+2
@@ -129,12 +129,14 @@ class ServingClassification(ClassificationMixin):
129
models: OpenAIServingModels,
130
*,
131
request_logger: Optional[RequestLogger],
132
+ log_error_stack: bool = False,
133
) -> None:
134
super().__init__(
135
engine_client=engine_client,
136
model_config=model_config,
137
models=models,
138
request_logger=request_logger,
139
+ log_error_stack=log_error_stack,
140
)
141
142
async def create_classify(
vllm/entrypoints/openai/serving_completion.py
+2
@@ -59,6 +59,7 @@ class OpenAIServingCompletion(OpenAIServing):
59
return_tokens_as_token_ids: bool = False,
60
enable_prompt_tokens_details: bool = False,
61
enable_force_include_usage: bool = False,
62
+ log_error_stack: bool = False,
63
):
64
super().__init__(
65
engine_client=engine_client,
@@ -67,6 +68,7 @@ class OpenAIServingCompletion(OpenAIServing):
68
request_logger=request_logger,
69
return_tokens_as_token_ids=return_tokens_as_token_ids,
70
enable_force_include_usage=enable_force_include_usage,
71
+ log_error_stack=log_error_stack,
72
)
73
self.enable_prompt_tokens_details = enable_prompt_tokens_details
74
self.default_sampling_params = (
vllm/entrypoints/openai/serving_embedding.py
+3
-1
@@ -593,11 +593,13 @@ class OpenAIServingEmbedding(EmbeddingMixin):
593
request_logger: Optional[RequestLogger],
594
chat_template: Optional[str],
595
chat_template_content_format: ChatTemplateContentFormatOption,
596
+ log_error_stack: bool = False,
597
) -> None:
598
super().__init__(engine_client=engine_client,
599
model_config=model_config,
600
models=models,
600
- request_logger=request_logger)
601
+ request_logger=request_logger,
602
+ log_error_stack=log_error_stack)
603
604
self.chat_template = chat_template
605
self.chat_template_content_format: Final = chat_template_content_format
vllm/entrypoints/openai/serving_engine.py
+9
@@ -5,6 +5,7 @@ import io
5
import json
6
import sys
7
import time
8
+import traceback
9
from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence
10
from concurrent.futures import ThreadPoolExecutor
11
from http import HTTPStatus
@@ -205,6 +206,7 @@ class OpenAIServing:
206
request_logger: Optional[RequestLogger],
207
return_tokens_as_token_ids: bool = False,
208
enable_force_include_usage: bool = False,
209
+ log_error_stack: bool = False,
210
):
211
super().__init__()
212
@@ -222,6 +224,7 @@ class OpenAIServing:
224
225
self._async_tokenizer_pool: dict[AnyTokenizer,
226
AsyncMicrobatchTokenizer] = {}
227
+ self.log_error_stack = log_error_stack
228
229
def _get_async_tokenizer(self, tokenizer) -> AsyncMicrobatchTokenizer:
230
"""
@@ -412,6 +415,12 @@ class OpenAIServing:
415
message: str,
416
err_type: str = "BadRequestError",
417
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse:
418
+ if self.log_error_stack:
419
+ exc_type, _, _ = sys.exc_info()
420
+ if exc_type is not None:
421
+ traceback.print_exc()
422
+ else:
423
+ traceback.print_stack()
424
return ErrorResponse(error=ErrorInfo(
425
message=message, type=err_type, code=status_code.value))
426
vllm/entrypoints/openai/serving_pooling.py
+3
-1
@@ -58,11 +58,13 @@ class OpenAIServingPooling(OpenAIServing):
58
request_logger: Optional[RequestLogger],
59
chat_template: Optional[str],
60
chat_template_content_format: ChatTemplateContentFormatOption,
61
+ log_error_stack: bool = False,
62
) -> None:
63
super().__init__(engine_client=engine_client,
64
model_config=model_config,
65
models=models,
65
- request_logger=request_logger)
66
+ request_logger=request_logger,
67
+ log_error_stack=log_error_stack)
68
69
self.chat_template = chat_template
70
self.chat_template_content_format: Final = chat_template_content_format
vllm/entrypoints/openai/serving_responses.py
+2
@@ -88,6 +88,7 @@ class OpenAIServingResponses(OpenAIServing):
88
enable_prompt_tokens_details: bool = False,
89
enable_force_include_usage: bool = False,
90
enable_log_outputs: bool = False,
91
+ log_error_stack: bool = False,
92
) -> None:
93
super().__init__(
94
engine_client=engine_client,
@@ -96,6 +97,7 @@ class OpenAIServingResponses(OpenAIServing):
97
request_logger=request_logger,
98
return_tokens_as_token_ids=return_tokens_as_token_ids,
99
enable_force_include_usage=enable_force_include_usage,
100
+ log_error_stack=log_error_stack,
101
)
102
103
self.chat_template = chat_template
vllm/entrypoints/openai/serving_score.py
+3
-1
@@ -47,11 +47,13 @@ class ServingScores(OpenAIServing):
47
models: OpenAIServingModels,
48
*,
49
request_logger: Optional[RequestLogger],
50
+ log_error_stack: bool = False,
51
) -> None:
52
super().__init__(engine_client=engine_client,
53
model_config=model_config,
54
models=models,
54
- request_logger=request_logger)
55
+ request_logger=request_logger,
56
+ log_error_stack=log_error_stack)
57
58
async def _embedding_score(
59
self,
vllm/entrypoints/openai/serving_tokenization.py
+3
-1
@@ -39,11 +39,13 @@ class OpenAIServingTokenization(OpenAIServing):
39
request_logger: Optional[RequestLogger],
40
chat_template: Optional[str],
41
chat_template_content_format: ChatTemplateContentFormatOption,
42
+ log_error_stack: bool = False,
43
) -> None:
44
super().__init__(engine_client=engine_client,
45
model_config=model_config,
46
models=models,
46
- request_logger=request_logger)
47
+ request_logger=request_logger,
48
+ log_error_stack=log_error_stack)
49
50
self.chat_template = chat_template
51
self.chat_template_content_format: Final = chat_template_content_format
vllm/entrypoints/openai/serving_transcription.py
+6
-2
@@ -32,13 +32,15 @@ class OpenAIServingTranscription(OpenAISpeechToText):
32
*,
33
request_logger: Optional[RequestLogger],
34
return_tokens_as_token_ids: bool = False,
35
+ log_error_stack: bool = False,
36
):
37
super().__init__(engine_client=engine_client,
38
model_config=model_config,
39
models=models,
40
request_logger=request_logger,
41
return_tokens_as_token_ids=return_tokens_as_token_ids,
41
- task_type="transcribe")
42
+ task_type="transcribe",
43
+ log_error_stack=log_error_stack)
44
45
async def create_transcription(
46
self, audio_data: bytes, request: TranscriptionRequest,
@@ -88,13 +90,15 @@ class OpenAIServingTranslation(OpenAISpeechToText):
90
*,
91
request_logger: Optional[RequestLogger],
92
return_tokens_as_token_ids: bool = False,
93
+ log_error_stack: bool = False,
94
):
95
super().__init__(engine_client=engine_client,
96
model_config=model_config,
97
models=models,
98
request_logger=request_logger,
99
return_tokens_as_token_ids=return_tokens_as_token_ids,
97
- task_type="translate")
100
+ task_type="translate",
101
+ log_error_stack=log_error_stack)
102
103
async def create_translation(
104
self, audio_data: bytes, request: TranslationRequest,
vllm/entrypoints/openai/speech_to_text.py
+3
-1
@@ -53,12 +53,14 @@ class OpenAISpeechToText(OpenAIServing):
53
request_logger: Optional[RequestLogger],
54
return_tokens_as_token_ids: bool = False,
55
task_type: Literal["transcribe", "translate"] = "transcribe",
56
+ log_error_stack: bool = False,
57
):
58
super().__init__(engine_client=engine_client,
59
model_config=model_config,
60
models=models,
61
request_logger=request_logger,
61
- return_tokens_as_token_ids=return_tokens_as_token_ids)
62
+ return_tokens_as_token_ids=return_tokens_as_token_ids,
63
+ log_error_stack=log_error_stack)
64
65
self.default_sampling_params = (
66
self.model_config.get_diff_sampling_param())