@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fe3b5bbc23
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] fix output parsing error for trtllm backend (#4137)

Co-authored-by: Roger Wang <ywang@roblox.com>

Elinx committed Apr 17, 2024 at 19:07 UTC fe3b5bbc23a99533bc7d4a94ae073828ed025974
1 file changed +1 -1
benchmarks/backend_request_func.py
+1 -1
@@ -135,6 +135,7 @@ async def async_request_trt_llm(
135 "data:")
136
137 data = json.loads(chunk)
138 + output.generated_text += data["text_output"]
139 timestamp = time.perf_counter()
140 # First token
141 if ttft == 0.0:
@@ -149,7 +150,6 @@ async def async_request_trt_llm(
150 most_recent_timestamp = timestamp
151
152 output.latency = most_recent_timestamp - st
152 - output.generated_text = json.loads(data)["text_output"]
153 output.success = True
154
155 else: