A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix]if the content is started with ":"(response of ping), client should i… (#5303)
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com> Co-authored-by: Roger Wang <ywang@roblox.com>
Wang, Yi committed
Jun 13, 2024 at 11:16 UTC
88407532e7ec2dd3313f6cb3a31d8dd1fa868178
1 file changed
+6
-2
benchmarks/backend_request_func.py
+6
-2
@@ -68,9 +68,13 @@ async def async_request_tgi(
68
chunk_bytes = chunk_bytes.strip()
69
if not chunk_bytes:
70
continue
71
+ chunk_bytes = chunk_bytes.decode("utf-8")
72
72
- chunk = remove_prefix(chunk_bytes.decode("utf-8"),
73
- "data:")
73
+ #NOTE: Sometimes TGI returns a ping response without
74
+ # any data, we should skip it.
75
+ if chunk_bytes.startswith(":"):
76
+ continue
77
+ chunk = remove_prefix(chunk_bytes, "data:")
78
79
data = json.loads(chunk)
80
timestamp = time.perf_counter()