@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 88407532e7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix]if the content is started with ":"(response of ping), client should i… (#5303)

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Wang, Yi committed Jun 13, 2024 at 11:16 UTC 88407532e7ec2dd3313f6cb3a31d8dd1fa868178
1 file changed +6 -2
benchmarks/backend_request_func.py
+6 -2
@@ -68,9 +68,13 @@ async def async_request_tgi(
68 chunk_bytes = chunk_bytes.strip()
69 if not chunk_bytes:
70 continue
71 + chunk_bytes = chunk_bytes.decode("utf-8")
72
72 - chunk = remove_prefix(chunk_bytes.decode("utf-8"),
73 - "data:")
73 + #NOTE: Sometimes TGI returns a ping response without
74 + # any data, we should skip it.
75 + if chunk_bytes.startswith(":"):
76 + continue
77 + chunk = remove_prefix(chunk_bytes, "data:")
78
79 data = json.loads(chunk)
80 timestamp = time.perf_counter()