@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 33fc1e2e86
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Frontend] Improve `StreamingResponse` Exception Handling (#11752)

Robert Shaw committed Jan 5, 2025 at 16:35 UTC 33fc1e2e86ce5d60940463f8f71daaa61728d3b7
2 files changed +3 -3
vllm/entrypoints/openai/serving_chat.py
+2 -2
@@ -301,7 +301,7 @@ class OpenAIServingChat(OpenAIServing):
301 ] * num_choices
302 else:
303 tool_parsers = [None] * num_choices
304 - except RuntimeError as e:
304 + except Exception as e:
305 logger.exception("Error in tool parser creation.")
306 data = self.create_streaming_error_response(str(e))
307 yield f"data: {data}\n\n"
@@ -591,7 +591,7 @@ class OpenAIServingChat(OpenAIServing):
591 completion_tokens=num_completion_tokens,
592 total_tokens=num_prompt_tokens + num_completion_tokens)
593
594 - except ValueError as e:
594 + except Exception as e:
595 # TODO: Use a vllm-specific Validation Error
596 logger.exception("Error in chat completion stream generator.")
597 data = self.create_streaming_error_response(str(e))
vllm/entrypoints/openai/serving_completion.py
+1 -1
@@ -371,7 +371,7 @@ class OpenAIServingCompletion(OpenAIServing):
371 # report to FastAPI middleware aggregate usage across all choices
372 request_metadata.final_usage_info = final_usage_info
373
374 - except ValueError as e:
374 + except Exception as e:
375 # TODO: Use a vllm-specific Validation Error
376 data = self.create_streaming_error_response(str(e))
377 yield f"data: {data}\n\n"