A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Misc] Fix OpenAI API Compatibility Issues in Benchmark Script (#12357)
Signed-off-by: Junichi Sato <junichi.sato@sbintuitions.co.jp>
Junichi Sato committed
Jan 24, 2025 at 07:02 UTC
9726ad676d04d4b424d266212ac85000efdcd64d
1 file changed
+6
-4
benchmarks/backend_request_func.py
+6
-4
@@ -245,11 +245,12 @@ async def async_request_openai_completions(
245
"max_tokens": request_func_input.output_len,
246
"logprobs": request_func_input.logprobs,
247
"stream": True,
248
- "ignore_eos": request_func_input.ignore_eos,
248
"stream_options": {
249
"include_usage": True,
250
},
251
}
252
+ if request_func_input.ignore_eos:
253
+ payload["ignore_eos"] = request_func_input.ignore_eos
254
if request_func_input.extra_body:
255
payload.update(request_func_input.extra_body)
256
headers = {
@@ -297,7 +298,7 @@ async def async_request_openai_completions(
298
most_recent_timestamp)
299
300
most_recent_timestamp = timestamp
300
- generated_text += text
301
+ generated_text += text or ""
302
elif usage := data.get("usage"):
303
output.output_tokens = usage.get(
304
"completion_tokens")
@@ -348,11 +349,12 @@ async def async_request_openai_chat_completions(
349
"temperature": 0.0,
350
"max_completion_tokens": request_func_input.output_len,
351
"stream": True,
351
- "ignore_eos": request_func_input.ignore_eos,
352
"stream_options": {
353
"include_usage": True,
354
},
355
}
356
+ if request_func_input.ignore_eos:
357
+ payload["ignore_eos"] = request_func_input.ignore_eos
358
if request_func_input.extra_body:
359
payload.update(request_func_input.extra_body)
360
headers = {
@@ -394,7 +396,7 @@ async def async_request_openai_chat_completions(
396
output.itl.append(timestamp -
397
most_recent_timestamp)
398
397
- generated_text += content
399
+ generated_text += content or ""
400
elif usage := data.get("usage"):
401
output.output_tokens = usage.get(
402
"completion_tokens")