@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9d9186be97
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Frontend] Reduce frequency of client cancellation checking (#7959)

Nick Hill committed Oct 21, 2024 at 21:28 UTC 9d9186be971f0553cea771177db43edafb005b72
1 file changed +38 -19
vllm/utils.py
+38 -19
@@ -13,10 +13,11 @@ import subprocess
13 import sys
14 import tempfile
15 import threading
16 +import time
17 import uuid
18 import warnings
19 import weakref
19 -from asyncio import FIRST_COMPLETED, ensure_future
20 +from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
21 from collections.abc import Mapping
22 from functools import lru_cache, partial, wraps
23 from platform import uname
@@ -437,6 +438,12 @@ def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
438 return _async_wrapper
439
440
441 +def _next_task(iterator: AsyncGenerator[T, None],
442 + loop: AbstractEventLoop) -> Task:
443 + # Can use anext() in python >= 3.10
444 + return loop.create_task(iterator.__anext__()) # type: ignore[arg-type]
445 +
446 +
447 async def iterate_with_cancellation(
448 iterator: AsyncGenerator[T, None],
449 is_cancelled: Callable[[], Awaitable[bool]],
@@ -445,19 +452,27 @@ async def iterate_with_cancellation(
452 at least once per second to check for client cancellation.
453 """
454
448 - # Can use anext() in python >= 3.10
449 - awaits = [ensure_future(iterator.__anext__())]
455 + loop = asyncio.get_running_loop()
456 +
457 + awaits: List[Future[T]] = [_next_task(iterator, loop)]
458 + next_cancel_check: float = 0
459 while True:
451 - done, pending = await asyncio.wait(awaits, timeout=1)
452 - if await is_cancelled():
453 - with contextlib.suppress(BaseException):
454 - awaits[0].cancel()
455 - await iterator.aclose()
456 - raise asyncio.CancelledError("client cancelled")
460 + done, pending = await asyncio.wait(awaits, timeout=1.5)
461 +
462 + # Check for cancellation at most once per second
463 + time_now = time.time()
464 + if time_now >= next_cancel_check:
465 + if await is_cancelled():
466 + with contextlib.suppress(BaseException):
467 + awaits[0].cancel()
468 + await iterator.aclose()
469 + raise asyncio.CancelledError("client cancelled")
470 + next_cancel_check = time_now + 1
471 +
472 if done:
473 try:
474 item = await awaits[0]
460 - awaits[0] = ensure_future(iterator.__anext__())
475 + awaits[0] = _next_task(iterator, loop)
476 yield item
477 except StopAsyncIteration:
478 # we are done
@@ -478,25 +493,29 @@ async def merge_async_iterators(
493 to check for client cancellation.
494 """
495
481 - # Can use anext() in python >= 3.10
482 - awaits = {
483 - ensure_future(pair[1].__anext__()): pair
484 - for pair in enumerate(iterators)
485 - }
486 - timeout = None if is_cancelled is None else 1
496 + loop = asyncio.get_running_loop()
497 +
498 + awaits = {_next_task(pair[1], loop): pair for pair in enumerate(iterators)}
499 + timeout = None if is_cancelled is None else 1.5
500 + next_cancel_check: float = 0
501 try:
502 while awaits:
503 done, pending = await asyncio.wait(awaits.keys(),
504 return_when=FIRST_COMPLETED,
505 timeout=timeout)
492 - if is_cancelled is not None and await is_cancelled():
493 - raise asyncio.CancelledError("client cancelled")
506 + if is_cancelled is not None:
507 + # Check for cancellation at most once per second
508 + time_now = time.time()
509 + if time_now >= next_cancel_check:
510 + if await is_cancelled():
511 + raise asyncio.CancelledError("client cancelled")
512 + next_cancel_check = time_now + 1
513 for d in done:
514 pair = awaits.pop(d)
515 try:
516 item = await d
517 i, it = pair
499 - awaits[ensure_future(it.__anext__())] = pair
518 + awaits[_next_task(it, loop)] = pair
519 yield i, item
520 except StopAsyncIteration:
521 pass