@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f4c2187e29
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc] Fix typo in #5895 (#10145)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 8, 2024 at 17:07 UTC f4c2187e2967ef4052d173b422b0249ab9532753
1 file changed +2 -2
benchmarks/backend_request_func.py
+2 -2
@@ -256,7 +256,7 @@ async def async_request_openai_completions(
256 async with session.post(url=api_url, json=payload,
257 headers=headers) as response:
258 if response.status == 200:
259 - first_valid_chunk_received = False
259 + first_chunk_received = False
260 async for chunk_bytes in response.content:
261 chunk_bytes = chunk_bytes.strip()
262 if not chunk_bytes:
@@ -275,7 +275,7 @@ async def async_request_openai_completions(
275 if data["choices"][0]["text"]:
276 timestamp = time.perf_counter()
277 # First token
278 - if not first_valid_chunk_received:
278 + if not first_chunk_received:
279 first_chunk_received = True
280 ttft = time.perf_counter() - st
281 output.ttft = ttft