A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Bugfix][Benchmarks] Ensure `async_request_deepspeed_mii` uses the OpenAI choices key (#15926)
Signed-off-by: Brayden Zhong <b8zhong@uwaterloo.ca>
Brayden Zhong committed
Apr 2, 2025 at 05:19 UTC
252937806c4065b6b7226f80bf7a2045465f739a
1 file changed
+9
-1
benchmarks/backend_request_func.py
+9
-1
@@ -219,7 +219,15 @@ async def async_request_deepspeed_mii(
219
if response.status == 200:
220
parsed_resp = await response.json()
221
output.latency = time.perf_counter() - st
222
- output.generated_text = parsed_resp["text"][0]
222
+ if "choices" in parsed_resp:
223
+ output.generated_text = parsed_resp["choices"][0][
224
+ "text"]
225
+ elif "text" in parsed_resp:
226
+ output.generated_text = parsed_resp["text"][0]
227
+ else:
228
+ output.error = ("Unexpected response format: "
229
+ "neither 'choices' nor 'text' found")
230
+ output.success = False
231
output.success = True
232
else:
233
output.error = response.reason or ""