A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Benchmark] fix request loss if "ping" is returned (#19535)
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Wang, Yi committed
Jun 22, 2025 at 15:21 UTC
202c5df9357e7c52b51e19abc70e8444f3f85ada
2 files changed
+23
-5
benchmarks/backend_request_func.py
+7
-1
@@ -404,8 +404,14 @@ async def async_request_openai_chat_completions(
404
chunk_bytes = chunk_bytes.strip()
405
if not chunk_bytes:
406
continue
407
+ chunk_bytes = chunk_bytes.decode("utf-8")
408
+ # NOTE: SSE comments (often used as pings) start with a colon.
409
+ # These are not JSON data payload and should be skipped.
410
+ if chunk_bytes.startswith(":"):
411
+ continue
412
+
413
+ chunk = chunk_bytes.removeprefix("data: ")
414
408
- chunk = chunk_bytes.decode("utf-8").removeprefix("data: ")
415
if chunk != "[DONE]":
416
timestamp = time.perf_counter()
417
data = json.loads(chunk)
vllm/benchmarks/endpoint_request_func.py
+16
-4
@@ -104,9 +104,15 @@ async def async_request_openai_completions(
104
chunk_bytes = chunk_bytes.strip()
105
if not chunk_bytes:
106
continue
107
+ chunk_bytes = chunk_bytes.decode("utf-8")
108
+ # NOTE: SSE comments (often used as pings) start with
109
+ # a colon. These are not JSON data payload and should
110
+ # be skipped.
111
+ if chunk_bytes.startswith(":"):
112
+ continue
113
+
114
+ chunk = chunk_bytes.removeprefix("data: ")
115
108
- chunk = chunk_bytes.decode("utf-8").removeprefix(
109
- "data: ")
116
if chunk != "[DONE]":
117
data = json.loads(chunk)
118
@@ -213,9 +219,15 @@ async def async_request_openai_chat_completions(
219
chunk_bytes = chunk_bytes.strip()
220
if not chunk_bytes:
221
continue
222
+ chunk_bytes = chunk_bytes.decode("utf-8")
223
+ # NOTE: SSE comments (often used as pings) start with
224
+ # a colon. These are not JSON data payload and should
225
+ # be skipped.
226
+ if chunk_bytes.startswith(":"):
227
+ continue
228
+
229
+ chunk = chunk_bytes.removeprefix("data: ")
230
217
- chunk = chunk_bytes.decode("utf-8").removeprefix(
218
- "data: ")
231
if chunk != "[DONE]":
232
timestamp = time.perf_counter()
233
data = json.loads(chunk)