A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend] Reduce frequency of client cancellation checking (#7959)
Nick Hill committed
Oct 21, 2024 at 21:28 UTC
9d9186be971f0553cea771177db43edafb005b72
1 file changed
+38
-19
vllm/utils.py
+38
-19
@@ -13,10 +13,11 @@ import subprocess
13
import sys
14
import tempfile
15
import threading
16
+import time
17
import uuid
18
import warnings
19
import weakref
19
-from asyncio import FIRST_COMPLETED, ensure_future
20
+from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
21
from collections.abc import Mapping
22
from functools import lru_cache, partial, wraps
23
from platform import uname
@@ -437,6 +438,12 @@ def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
438
return _async_wrapper
439
440
441
+def _next_task(iterator: AsyncGenerator[T, None],
442
+ loop: AbstractEventLoop) -> Task:
443
+ # Can use anext() in python >= 3.10
444
+ return loop.create_task(iterator.__anext__()) # type: ignore[arg-type]
445
+
446
+
447
async def iterate_with_cancellation(
448
iterator: AsyncGenerator[T, None],
449
is_cancelled: Callable[[], Awaitable[bool]],
@@ -445,19 +452,27 @@ async def iterate_with_cancellation(
452
at least once per second to check for client cancellation.
453
"""
454
448
- # Can use anext() in python >= 3.10
449
- awaits = [ensure_future(iterator.__anext__())]
455
+ loop = asyncio.get_running_loop()
456
+
457
+ awaits: List[Future[T]] = [_next_task(iterator, loop)]
458
+ next_cancel_check: float = 0
459
while True:
451
- done, pending = await asyncio.wait(awaits, timeout=1)
452
- if await is_cancelled():
453
- with contextlib.suppress(BaseException):
454
- awaits[0].cancel()
455
- await iterator.aclose()
456
- raise asyncio.CancelledError("client cancelled")
460
+ done, pending = await asyncio.wait(awaits, timeout=1.5)
461
+
462
+ # Check for cancellation at most once per second
463
+ time_now = time.time()
464
+ if time_now >= next_cancel_check:
465
+ if await is_cancelled():
466
+ with contextlib.suppress(BaseException):
467
+ awaits[0].cancel()
468
+ await iterator.aclose()
469
+ raise asyncio.CancelledError("client cancelled")
470
+ next_cancel_check = time_now + 1
471
+
472
if done:
473
try:
474
item = await awaits[0]
460
- awaits[0] = ensure_future(iterator.__anext__())
475
+ awaits[0] = _next_task(iterator, loop)
476
yield item
477
except StopAsyncIteration:
478
# we are done
@@ -478,25 +493,29 @@ async def merge_async_iterators(
493
to check for client cancellation.
494
"""
495
481
- # Can use anext() in python >= 3.10
482
- awaits = {
483
- ensure_future(pair[1].__anext__()): pair
484
- for pair in enumerate(iterators)
485
- }
486
- timeout = None if is_cancelled is None else 1
496
+ loop = asyncio.get_running_loop()
497
+
498
+ awaits = {_next_task(pair[1], loop): pair for pair in enumerate(iterators)}
499
+ timeout = None if is_cancelled is None else 1.5
500
+ next_cancel_check: float = 0
501
try:
502
while awaits:
503
done, pending = await asyncio.wait(awaits.keys(),
504
return_when=FIRST_COMPLETED,
505
timeout=timeout)
492
- if is_cancelled is not None and await is_cancelled():
493
- raise asyncio.CancelledError("client cancelled")
506
+ if is_cancelled is not None:
507
+ # Check for cancellation at most once per second
508
+ time_now = time.time()
509
+ if time_now >= next_cancel_check:
510
+ if await is_cancelled():
511
+ raise asyncio.CancelledError("client cancelled")
512
+ next_cancel_check = time_now + 1
513
for d in done:
514
pair = awaits.pop(d)
515
try:
516
item = await d
517
i, it = pair
499
- awaits[ensure_future(it.__anext__())] = pair
518
+ awaits[_next_task(it, loop)] = pair
519
yield i, item
520
except StopAsyncIteration:
521
pass