@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dfea173148
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Abort requests when the connection to /v1/completions is interrupted (#4363)

Ruoyu Qin committed Apr 28, 2024 at 00:48 UTC dfea17314827845d55dabb03ebe905f58e6682e4
2 files changed +53 -5
tests/async_engine/test_merge_async_iterators.py new
+41
@@ -0,0 +1,41 @@
1 +import asyncio
2 +from typing import AsyncIterator, Tuple
3 +
4 +import pytest
5 +
6 +from vllm.utils import merge_async_iterators
7 +
8 +
9 +@pytest.mark.asyncio
10 +async def test_merge_async_iterators():
11 +
12 + async def mock_async_iterator(idx: int) -> AsyncIterator[str]:
13 + try:
14 + while True:
15 + yield f"item from iterator {idx}"
16 + await asyncio.sleep(0.1)
17 + except asyncio.CancelledError:
18 + pass
19 +
20 + iterators = [mock_async_iterator(i) for i in range(3)]
21 + merged_iterator: AsyncIterator[Tuple[int, str]] = merge_async_iterators(
22 + *iterators)
23 +
24 + async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
25 + async for idx, output in generator:
26 + print(f"idx: {idx}, output: {output}")
27 +
28 + task = asyncio.create_task(stream_output(merged_iterator))
29 + await asyncio.sleep(0.5)
30 + task.cancel()
31 + with pytest.raises(asyncio.CancelledError):
32 + await task
33 +
34 + for iterator in iterators:
35 + try:
36 + await asyncio.wait_for(anext(iterator), 1)
37 + except StopAsyncIteration:
38 + # All iterators should be cancelled and print this message.
39 + print("Iterator was cancelled normally")
40 + except (Exception, asyncio.CancelledError) as e:
41 + raise AssertionError() from e
vllm/utils.py
+12 -5
@@ -225,11 +225,18 @@ def merge_async_iterators(
225 ]
226
227 async def consumer():
228 - while not all(finished) or not queue.empty():
229 - item = await queue.get()
230 - if isinstance(item, Exception):
231 - raise item
232 - yield item
228 + try:
229 + while not all(finished) or not queue.empty():
230 + item = await queue.get()
231 + if isinstance(item, Exception):
232 + raise item
233 + yield item
234 + except (Exception, asyncio.CancelledError) as e:
235 + for task in _tasks:
236 + # NOTE: Pass the error msg in cancel()
237 + # when only Python 3.9+ is supported.
238 + task.cancel()
239 + raise e
240 await asyncio.gather(*_tasks)
241
242 return consumer()