@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9726ad676d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Misc] Fix OpenAI API Compatibility Issues in Benchmark Script (#12357)

Signed-off-by: Junichi Sato <junichi.sato@sbintuitions.co.jp>

Junichi Sato committed Jan 24, 2025 at 07:02 UTC 9726ad676d04d4b424d266212ac85000efdcd64d
1 file changed +6 -4
benchmarks/backend_request_func.py
+6 -4
@@ -245,11 +245,12 @@ async def async_request_openai_completions(
245 "max_tokens": request_func_input.output_len,
246 "logprobs": request_func_input.logprobs,
247 "stream": True,
248 - "ignore_eos": request_func_input.ignore_eos,
248 "stream_options": {
249 "include_usage": True,
250 },
251 }
252 + if request_func_input.ignore_eos:
253 + payload["ignore_eos"] = request_func_input.ignore_eos
254 if request_func_input.extra_body:
255 payload.update(request_func_input.extra_body)
256 headers = {
@@ -297,7 +298,7 @@ async def async_request_openai_completions(
298 most_recent_timestamp)
299
300 most_recent_timestamp = timestamp
300 - generated_text += text
301 + generated_text += text or ""
302 elif usage := data.get("usage"):
303 output.output_tokens = usage.get(
304 "completion_tokens")
@@ -348,11 +349,12 @@ async def async_request_openai_chat_completions(
349 "temperature": 0.0,
350 "max_completion_tokens": request_func_input.output_len,
351 "stream": True,
351 - "ignore_eos": request_func_input.ignore_eos,
352 "stream_options": {
353 "include_usage": True,
354 },
355 }
356 + if request_func_input.ignore_eos:
357 + payload["ignore_eos"] = request_func_input.ignore_eos
358 if request_func_input.extra_body:
359 payload.update(request_func_input.extra_body)
360 headers = {
@@ -394,7 +396,7 @@ async def async_request_openai_chat_completions(
396 output.itl.append(timestamp -
397 most_recent_timestamp)
398
397 - generated_text += content
399 + generated_text += content or ""
400 elif usage := data.get("usage"):
401 output.output_tokens = usage.get(
402 "completion_tokens")