@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fc1493a01e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[FrontEnd] Make `merge_async_iterators` `is_cancelled` arg optional (#7282)

Nick Hill committed Aug 7, 2024 at 13:35 UTC fc1493a01edf0ae6f50299daae15679e4c08d74c
1 file changed +6 -5
vllm/utils.py
+6 -5
@@ -405,7 +405,7 @@ async def iterate_with_cancellation(
405
406 async def merge_async_iterators(
407 *iterators: AsyncGenerator[T, None],
408 - is_cancelled: Callable[[], Awaitable[bool]],
408 + is_cancelled: Optional[Callable[[], Awaitable[bool]]] = None,
409 ) -> AsyncGenerator[Tuple[int, T], None]:
410 """Merge multiple asynchronous iterators into a single iterator.
411
@@ -413,8 +413,8 @@ async def merge_async_iterators(
413 When it yields, it yields a tuple (i, item) where i is the index of the
414 iterator that yields the item.
415
416 - It also polls the provided function at least once per second to check
417 - for client cancellation.
416 + It also optionally polls a provided function at least once per second
417 + to check for client cancellation.
418 """
419
420 # Can use anext() in python >= 3.10
@@ -422,12 +422,13 @@ async def merge_async_iterators(
422 ensure_future(pair[1].__anext__()): pair
423 for pair in enumerate(iterators)
424 }
425 + timeout = None if is_cancelled is None else 1
426 try:
427 while awaits:
428 done, pending = await asyncio.wait(awaits.keys(),
429 return_when=FIRST_COMPLETED,
429 - timeout=1)
430 - if await is_cancelled():
430 + timeout=timeout)
431 + if is_cancelled is not None and await is_cancelled():
432 raise asyncio.CancelledError("client cancelled")
433 for d in done:
434 pair = awaits.pop(d)