A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Frontend] Improve `StreamingResponse` Exception Handling (#11752)
Robert Shaw committed
Jan 5, 2025 at 16:35 UTC
33fc1e2e86ce5d60940463f8f71daaa61728d3b7
2 files changed
+3
-3
vllm/entrypoints/openai/serving_chat.py
+2
-2
@@ -301,7 +301,7 @@ class OpenAIServingChat(OpenAIServing):
301
] * num_choices
302
else:
303
tool_parsers = [None] * num_choices
304
- except RuntimeError as e:
304
+ except Exception as e:
305
logger.exception("Error in tool parser creation.")
306
data = self.create_streaming_error_response(str(e))
307
yield f"data: {data}\n\n"
@@ -591,7 +591,7 @@ class OpenAIServingChat(OpenAIServing):
591
completion_tokens=num_completion_tokens,
592
total_tokens=num_prompt_tokens + num_completion_tokens)
593
594
- except ValueError as e:
594
+ except Exception as e:
595
# TODO: Use a vllm-specific Validation Error
596
logger.exception("Error in chat completion stream generator.")
597
data = self.create_streaming_error_response(str(e))
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -371,7 +371,7 @@ class OpenAIServingCompletion(OpenAIServing):
371
# report to FastAPI middleware aggregate usage across all choices
372
request_metadata.final_usage_info = final_usage_info
373
374
- except ValueError as e:
374
+ except Exception as e:
375
# TODO: Use a vllm-specific Validation Error
376
data = self.create_streaming_error_response(str(e))
377
yield f"data: {data}\n\n"