A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] fix output parsing error for trtllm backend (#4137)
Co-authored-by: Roger Wang <ywang@roblox.com>
Elinx committed
Apr 17, 2024 at 19:07 UTC
fe3b5bbc23a99533bc7d4a94ae073828ed025974
1 file changed
+1
-1
benchmarks/backend_request_func.py
+1
-1
@@ -135,6 +135,7 @@ async def async_request_trt_llm(
135
"data:")
136
137
data = json.loads(chunk)
138
+ output.generated_text += data["text_output"]
139
timestamp = time.perf_counter()
140
# First token
141
if ttft == 0.0:
@@ -149,7 +150,6 @@ async def async_request_trt_llm(
150
most_recent_timestamp = timestamp
151
152
output.latency = most_recent_timestamp - st
152
- output.generated_text = json.loads(data)["text_output"]
153
output.success = True
154
155
else: