A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[FrontEnd] Make `merge_async_iterators` `is_cancelled` arg optional (#7282)
Nick Hill committed
Aug 7, 2024 at 13:35 UTC
fc1493a01edf0ae6f50299daae15679e4c08d74c
1 file changed
+6
-5
vllm/utils.py
+6
-5
@@ -405,7 +405,7 @@ async def iterate_with_cancellation(
405
406
async def merge_async_iterators(
407
*iterators: AsyncGenerator[T, None],
408
- is_cancelled: Callable[[], Awaitable[bool]],
408
+ is_cancelled: Optional[Callable[[], Awaitable[bool]]] = None,
409
) -> AsyncGenerator[Tuple[int, T], None]:
410
"""Merge multiple asynchronous iterators into a single iterator.
411
@@ -413,8 +413,8 @@ async def merge_async_iterators(
413
When it yields, it yields a tuple (i, item) where i is the index of the
414
iterator that yields the item.
415
416
- It also polls the provided function at least once per second to check
417
- for client cancellation.
416
+ It also optionally polls a provided function at least once per second
417
+ to check for client cancellation.
418
"""
419
420
# Can use anext() in python >= 3.10
@@ -422,12 +422,13 @@ async def merge_async_iterators(
422
ensure_future(pair[1].__anext__()): pair
423
for pair in enumerate(iterators)
424
}
425
+ timeout = None if is_cancelled is None else 1
426
try:
427
while awaits:
428
done, pending = await asyncio.wait(awaits.keys(),
429
return_when=FIRST_COMPLETED,
429
- timeout=1)
430
- if await is_cancelled():
430
+ timeout=timeout)
431
+ if is_cancelled is not None and await is_cancelled():
432
raise asyncio.CancelledError("client cancelled")
433
for d in done:
434
pair = awaits.pop(d)