A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Abort requests when the connection to /v1/completions is interrupted (#4363)
Ruoyu Qin committed
Apr 28, 2024 at 00:48 UTC
dfea17314827845d55dabb03ebe905f58e6682e4
2 files changed
+53
-5
tests/async_engine/test_merge_async_iterators.py
new
+41
@@ -0,0 +1,41 @@
1
+import asyncio
2
+from typing import AsyncIterator, Tuple
3
+
4
+import pytest
5
+
6
+from vllm.utils import merge_async_iterators
7
+
8
+
9
+@pytest.mark.asyncio
10
+async def test_merge_async_iterators():
11
+
12
+ async def mock_async_iterator(idx: int) -> AsyncIterator[str]:
13
+ try:
14
+ while True:
15
+ yield f"item from iterator {idx}"
16
+ await asyncio.sleep(0.1)
17
+ except asyncio.CancelledError:
18
+ pass
19
+
20
+ iterators = [mock_async_iterator(i) for i in range(3)]
21
+ merged_iterator: AsyncIterator[Tuple[int, str]] = merge_async_iterators(
22
+ *iterators)
23
+
24
+ async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
25
+ async for idx, output in generator:
26
+ print(f"idx: {idx}, output: {output}")
27
+
28
+ task = asyncio.create_task(stream_output(merged_iterator))
29
+ await asyncio.sleep(0.5)
30
+ task.cancel()
31
+ with pytest.raises(asyncio.CancelledError):
32
+ await task
33
+
34
+ for iterator in iterators:
35
+ try:
36
+ await asyncio.wait_for(anext(iterator), 1)
37
+ except StopAsyncIteration:
38
+ # All iterators should be cancelled and print this message.
39
+ print("Iterator was cancelled normally")
40
+ except (Exception, asyncio.CancelledError) as e:
41
+ raise AssertionError() from e
vllm/utils.py
+12
-5
@@ -225,11 +225,18 @@ def merge_async_iterators(
225
]
226
227
async def consumer():
228
- while not all(finished) or not queue.empty():
229
- item = await queue.get()
230
- if isinstance(item, Exception):
231
- raise item
232
- yield item
228
+ try:
229
+ while not all(finished) or not queue.empty():
230
+ item = await queue.get()
231
+ if isinstance(item, Exception):
232
+ raise item
233
+ yield item
234
+ except (Exception, asyncio.CancelledError) as e:
235
+ for task in _tasks:
236
+ # NOTE: Pass the error msg in cancel()
237
+ # when only Python 3.9+ is supported.
238
+ task.cancel()
239
+ raise e
240
await asyncio.gather(*_tasks)
241
242
return consumer()