@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eecd864388
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix][CI/Build] Fix test and improve code for `merge_async_iterators` (#5096)

Cyrus Leung committed May 30, 2024 at 07:02 UTC eecd864388cba75421215411d42bde1c328fa518
3 files changed +62 -45
tests/async_engine/test_merge_async_iterators.py deleted
-41
@@ -1,41 +0,0 @@
1 -import asyncio
2 -from typing import AsyncIterator, Tuple
3 -
4 -import pytest
5 -
6 -from vllm.utils import merge_async_iterators
7 -
8 -
9 -@pytest.mark.asyncio
10 -async def test_merge_async_iterators():
11 -
12 - async def mock_async_iterator(idx: int) -> AsyncIterator[str]:
13 - try:
14 - while True:
15 - yield f"item from iterator {idx}"
16 - await asyncio.sleep(0.1)
17 - except asyncio.CancelledError:
18 - pass
19 -
20 - iterators = [mock_async_iterator(i) for i in range(3)]
21 - merged_iterator: AsyncIterator[Tuple[int, str]] = merge_async_iterators(
22 - *iterators)
23 -
24 - async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
25 - async for idx, output in generator:
26 - print(f"idx: {idx}, output: {output}")
27 -
28 - task = asyncio.create_task(stream_output(merged_iterator))
29 - await asyncio.sleep(0.5)
30 - task.cancel()
31 - with pytest.raises(asyncio.CancelledError):
32 - await task
33 -
34 - for iterator in iterators:
35 - try:
36 - await asyncio.wait_for(anext(iterator), 1)
37 - except StopAsyncIteration:
38 - # All iterators should be cancelled and print this message.
39 - print("Iterator was cancelled normally")
40 - except (Exception, asyncio.CancelledError) as e:
41 - raise AssertionError() from e
tests/test_utils.py
+56 -1
@@ -1,9 +1,64 @@
1 +import asyncio
2 +import sys
3 +from typing import (TYPE_CHECKING, Any, AsyncIterator, Awaitable, Protocol,
4 + Tuple, TypeVar)
5 +
6 import pytest
7
3 -from vllm.utils import deprecate_kwargs
8 +from vllm.utils import deprecate_kwargs, merge_async_iterators
9
10 from .utils import error_on_warning
11
12 +if sys.version_info < (3, 10):
13 + if TYPE_CHECKING:
14 + _AwaitableT = TypeVar("_AwaitableT", bound=Awaitable[Any])
15 + _AwaitableT_co = TypeVar("_AwaitableT_co",
16 + bound=Awaitable[Any],
17 + covariant=True)
18 +
19 + class _SupportsSynchronousAnext(Protocol[_AwaitableT_co]):
20 +
21 + def __anext__(self) -> _AwaitableT_co:
22 + ...
23 +
24 + def anext(i: "_SupportsSynchronousAnext[_AwaitableT]", /) -> "_AwaitableT":
25 + return i.__anext__()
26 +
27 +
28 +@pytest.mark.asyncio
29 +async def test_merge_async_iterators():
30 +
31 + async def mock_async_iterator(idx: int) -> AsyncIterator[str]:
32 + try:
33 + while True:
34 + yield f"item from iterator {idx}"
35 + await asyncio.sleep(0.1)
36 + except asyncio.CancelledError:
37 + pass
38 +
39 + iterators = [mock_async_iterator(i) for i in range(3)]
40 + merged_iterator: AsyncIterator[Tuple[int, str]] = merge_async_iterators(
41 + *iterators)
42 +
43 + async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
44 + async for idx, output in generator:
45 + print(f"idx: {idx}, output: {output}")
46 +
47 + task = asyncio.create_task(stream_output(merged_iterator))
48 + await asyncio.sleep(0.5)
49 + task.cancel()
50 + with pytest.raises(asyncio.CancelledError):
51 + await task
52 +
53 + for iterator in iterators:
54 + try:
55 + await asyncio.wait_for(anext(iterator), 1)
56 + except StopAsyncIteration:
57 + # All iterators should be cancelled and print this message.
58 + print("Iterator was cancelled normally")
59 + except (Exception, asyncio.CancelledError) as e:
60 + raise AssertionError() from e
61 +
62
63 def test_deprecate_kwargs_always():
64
vllm/utils.py
+6 -3
@@ -5,6 +5,7 @@ import gc
5 import os
6 import socket
7 import subprocess
8 +import sys
9 import tempfile
10 import threading
11 import uuid
@@ -234,9 +235,11 @@ def merge_async_iterators(
235 yield item
236 except (Exception, asyncio.CancelledError) as e:
237 for task in _tasks:
237 - # NOTE: Pass the error msg in cancel()
238 - # when only Python 3.9+ is supported.
239 - task.cancel()
238 + if sys.version_info >= (3, 9):
239 + # msg parameter only supported in Python 3.9+
240 + task.cancel(e)
241 + else:
242 + task.cancel()
243 raise e
244 await asyncio.gather(*_tasks)
245