@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 29862b884b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Frontend] Adjust try/except blocks in API impl (#10056)

Signed-off-by: Nick Hill <nhill@redhat.com>

Nick Hill committed Nov 7, 2024 at 04:07 UTC 29862b884bb5c59a35a9bcf62913c233d8b82471
2 files changed +5 -11
vllm/entrypoints/openai/serving_completion.py
+2 -6
@@ -189,13 +189,7 @@ class OpenAIServingCompletion(OpenAIServing):
189 try:
190 async for i, res in result_generator:
191 final_res_batch[i] = res
192 - except asyncio.CancelledError:
193 - return self.create_error_response("Client disconnected")
194 - except ValueError as e:
195 - # TODO: Use a vllm-specific Validation Error
196 - return self.create_error_response(str(e))
192
198 - try:
193 for i, final_res in enumerate(final_res_batch):
194 assert final_res is not None
195
@@ -217,6 +211,8 @@ class OpenAIServingCompletion(OpenAIServing):
211 tokenizer,
212 request_metadata,
213 )
214 + except asyncio.CancelledError:
215 + return self.create_error_response("Client disconnected")
216 except ValueError as e:
217 # TODO: Use a vllm-specific Validation Error
218 return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_embedding.py
+3 -5
@@ -205,12 +205,8 @@ class OpenAIServingEmbedding(OpenAIServing):
205 try:
206 async for i, res in result_generator:
207 final_res_batch[i] = res
208 - except asyncio.CancelledError:
209 - return self.create_error_response("Client disconnected")
208
211 - try:
212 - for final_res in final_res_batch:
213 - assert final_res is not None
209 + assert all(final_res is not None for final_res in final_res_batch)
210
211 final_res_batch_checked = cast(List[EmbeddingRequestOutput],
212 final_res_batch)
@@ -218,6 +214,8 @@ class OpenAIServingEmbedding(OpenAIServing):
214 response = request_output_to_embedding_response(
215 final_res_batch_checked, request_id, created_time, model_name,
216 encoding_format)
217 + except asyncio.CancelledError:
218 + return self.create_error_response("Client disconnected")
219 except ValueError as e:
220 # TODO: Use a vllm-specific Validation Error
221 return self.create_error_response(str(e))