A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Frontend] Adjust try/except blocks in API impl (#10056)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Nov 7, 2024 at 04:07 UTC
29862b884bb5c59a35a9bcf62913c233d8b82471
2 files changed
+5
-11
vllm/entrypoints/openai/serving_completion.py
+2
-6
@@ -189,13 +189,7 @@ class OpenAIServingCompletion(OpenAIServing):
189
try:
190
async for i, res in result_generator:
191
final_res_batch[i] = res
192
- except asyncio.CancelledError:
193
- return self.create_error_response("Client disconnected")
194
- except ValueError as e:
195
- # TODO: Use a vllm-specific Validation Error
196
- return self.create_error_response(str(e))
192
198
- try:
193
for i, final_res in enumerate(final_res_batch):
194
assert final_res is not None
195
@@ -217,6 +211,8 @@ class OpenAIServingCompletion(OpenAIServing):
211
tokenizer,
212
request_metadata,
213
)
214
+ except asyncio.CancelledError:
215
+ return self.create_error_response("Client disconnected")
216
except ValueError as e:
217
# TODO: Use a vllm-specific Validation Error
218
return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_embedding.py
+3
-5
@@ -205,12 +205,8 @@ class OpenAIServingEmbedding(OpenAIServing):
205
try:
206
async for i, res in result_generator:
207
final_res_batch[i] = res
208
- except asyncio.CancelledError:
209
- return self.create_error_response("Client disconnected")
208
211
- try:
212
- for final_res in final_res_batch:
213
- assert final_res is not None
209
+ assert all(final_res is not None for final_res in final_res_batch)
210
211
final_res_batch_checked = cast(List[EmbeddingRequestOutput],
212
final_res_batch)
@@ -218,6 +214,8 @@ class OpenAIServingEmbedding(OpenAIServing):
214
response = request_output_to_embedding_response(
215
final_res_batch_checked, request_id, created_time, model_name,
216
encoding_format)
217
+ except asyncio.CancelledError:
218
+ return self.create_error_response("Client disconnected")
219
except ValueError as e:
220
# TODO: Use a vllm-specific Validation Error
221
return self.create_error_response(str(e))