A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc] Fix typo in #5895 (#10145)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 8, 2024 at 17:07 UTC
f4c2187e2967ef4052d173b422b0249ab9532753
1 file changed
+2
-2
benchmarks/backend_request_func.py
+2
-2
@@ -256,7 +256,7 @@ async def async_request_openai_completions(
256
async with session.post(url=api_url, json=payload,
257
headers=headers) as response:
258
if response.status == 200:
259
- first_valid_chunk_received = False
259
+ first_chunk_received = False
260
async for chunk_bytes in response.content:
261
chunk_bytes = chunk_bytes.strip()
262
if not chunk_bytes:
@@ -275,7 +275,7 @@ async def async_request_openai_completions(
275
if data["choices"][0]["text"]:
276
timestamp = time.perf_counter()
277
# First token
278
- if not first_valid_chunk_received:
278
+ if not first_chunk_received:
279
first_chunk_received = True
280
ttft = time.perf_counter() - st
281
output.ttft = ttft