A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix][CI/Build] Fix test and improve code for `merge_async_iterators` (#5096)
Cyrus Leung committed
May 30, 2024 at 07:02 UTC
eecd864388cba75421215411d42bde1c328fa518
3 files changed
+62
-45
tests/async_engine/test_merge_async_iterators.py
deleted
-41
@@ -1,41 +0,0 @@
1
-import asyncio
2
-from typing import AsyncIterator, Tuple
3
-
4
-import pytest
5
-
6
-from vllm.utils import merge_async_iterators
7
-
8
-
9
-@pytest.mark.asyncio
10
-async def test_merge_async_iterators():
11
-
12
- async def mock_async_iterator(idx: int) -> AsyncIterator[str]:
13
- try:
14
- while True:
15
- yield f"item from iterator {idx}"
16
- await asyncio.sleep(0.1)
17
- except asyncio.CancelledError:
18
- pass
19
-
20
- iterators = [mock_async_iterator(i) for i in range(3)]
21
- merged_iterator: AsyncIterator[Tuple[int, str]] = merge_async_iterators(
22
- *iterators)
23
-
24
- async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
25
- async for idx, output in generator:
26
- print(f"idx: {idx}, output: {output}")
27
-
28
- task = asyncio.create_task(stream_output(merged_iterator))
29
- await asyncio.sleep(0.5)
30
- task.cancel()
31
- with pytest.raises(asyncio.CancelledError):
32
- await task
33
-
34
- for iterator in iterators:
35
- try:
36
- await asyncio.wait_for(anext(iterator), 1)
37
- except StopAsyncIteration:
38
- # All iterators should be cancelled and print this message.
39
- print("Iterator was cancelled normally")
40
- except (Exception, asyncio.CancelledError) as e:
41
- raise AssertionError() from e
tests/test_utils.py
+56
-1
@@ -1,9 +1,64 @@
1
+import asyncio
2
+import sys
3
+from typing import (TYPE_CHECKING, Any, AsyncIterator, Awaitable, Protocol,
4
+ Tuple, TypeVar)
5
+
6
import pytest
7
3
-from vllm.utils import deprecate_kwargs
8
+from vllm.utils import deprecate_kwargs, merge_async_iterators
9
10
from .utils import error_on_warning
11
12
+if sys.version_info < (3, 10):
13
+ if TYPE_CHECKING:
14
+ _AwaitableT = TypeVar("_AwaitableT", bound=Awaitable[Any])
15
+ _AwaitableT_co = TypeVar("_AwaitableT_co",
16
+ bound=Awaitable[Any],
17
+ covariant=True)
18
+
19
+ class _SupportsSynchronousAnext(Protocol[_AwaitableT_co]):
20
+
21
+ def __anext__(self) -> _AwaitableT_co:
22
+ ...
23
+
24
+ def anext(i: "_SupportsSynchronousAnext[_AwaitableT]", /) -> "_AwaitableT":
25
+ return i.__anext__()
26
+
27
+
28
+@pytest.mark.asyncio
29
+async def test_merge_async_iterators():
30
+
31
+ async def mock_async_iterator(idx: int) -> AsyncIterator[str]:
32
+ try:
33
+ while True:
34
+ yield f"item from iterator {idx}"
35
+ await asyncio.sleep(0.1)
36
+ except asyncio.CancelledError:
37
+ pass
38
+
39
+ iterators = [mock_async_iterator(i) for i in range(3)]
40
+ merged_iterator: AsyncIterator[Tuple[int, str]] = merge_async_iterators(
41
+ *iterators)
42
+
43
+ async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
44
+ async for idx, output in generator:
45
+ print(f"idx: {idx}, output: {output}")
46
+
47
+ task = asyncio.create_task(stream_output(merged_iterator))
48
+ await asyncio.sleep(0.5)
49
+ task.cancel()
50
+ with pytest.raises(asyncio.CancelledError):
51
+ await task
52
+
53
+ for iterator in iterators:
54
+ try:
55
+ await asyncio.wait_for(anext(iterator), 1)
56
+ except StopAsyncIteration:
57
+ # All iterators should be cancelled and print this message.
58
+ print("Iterator was cancelled normally")
59
+ except (Exception, asyncio.CancelledError) as e:
60
+ raise AssertionError() from e
61
+
62
63
def test_deprecate_kwargs_always():
64
vllm/utils.py
+6
-3
@@ -5,6 +5,7 @@ import gc
5
import os
6
import socket
7
import subprocess
8
+import sys
9
import tempfile
10
import threading
11
import uuid
@@ -234,9 +235,11 @@ def merge_async_iterators(
235
yield item
236
except (Exception, asyncio.CancelledError) as e:
237
for task in _tasks:
237
- # NOTE: Pass the error msg in cancel()
238
- # when only Python 3.9+ is supported.
239
- task.cancel()
238
+ if sys.version_info >= (3, 9):
239
+ # msg parameter only supported in Python 3.9+
240
+ task.cancel(e)
241
+ else:
242
+ task.cancel()
243
raise e
244
await asyncio.gather(*_tasks)
245