A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[BugFix] Avoid premature async generator exit and raise all exception variations (#7698)
Nick Hill committed
Aug 21, 2024 at 11:45 UTC
c75363fbc0ffd9d56bab452dd12adbf3628d73b4
2 files changed
+101
-21
tests/async_engine/test_async_llm_engine.py
+88
-13
@@ -1,14 +1,19 @@
1
import asyncio
2
import os
3
+from asyncio import CancelledError
4
from dataclasses import dataclass
5
+from typing import Optional
6
7
import pytest
8
+import pytest_asyncio
9
import torch
10
11
from vllm import SamplingParams
12
from vllm.config import ParallelConfig
13
from vllm.engine.async_llm_engine import AsyncEngineArgs, AsyncLLMEngine
14
+from vllm.outputs import RequestOutput as RealRequestOutput
15
16
+from ..conftest import cleanup
17
from ..utils import wait_for_gpu_memory_to_clear
18
19
@@ -118,15 +123,38 @@ async def test_new_requests_event():
123
os.environ.pop("VLLM_ALLOW_ENGINE_USE_RAY")
124
125
121
-def test_asyncio_run():
126
+def start_engine():
127
wait_for_gpu_memory_to_clear(
128
devices=list(range(torch.cuda.device_count())),
129
threshold_bytes=2 * 2**30,
130
timeout_s=60,
131
)
132
128
- engine = AsyncLLMEngine.from_engine_args(
129
- AsyncEngineArgs(model="facebook/opt-125m"))
133
+ return AsyncLLMEngine.from_engine_args(
134
+ AsyncEngineArgs(model="facebook/opt-125m", enforce_eager=True))
135
+
136
+
137
+@pytest_asyncio.fixture(scope="module")
138
+async def async_engine():
139
+ engine = await asyncio.get_event_loop().run_in_executor(executor=None,
140
+ func=start_engine)
141
+ try:
142
+ yield engine
143
+ finally:
144
+ engine.shutdown_background_loop()
145
+ del engine
146
+ await asyncio.sleep(0.1)
147
+ cleanup()
148
+
149
+
150
+@pytest.fixture()
151
+def should_do_global_cleanup_after_test(request) -> bool:
152
+ # So we can share the async engine fixture between these tests
153
+ return False
154
+
155
+
156
+@pytest.mark.asyncio(scope="module")
157
+async def test_asyncio_run(async_engine):
158
159
async def run(prompt: str):
160
sampling_params = SamplingParams(
@@ -134,17 +162,64 @@ def test_asyncio_run():
162
max_tokens=32,
163
)
164
137
- async for output in engine.generate(prompt,
138
- sampling_params,
139
- request_id=prompt):
165
+ async for output in async_engine.generate(prompt,
166
+ sampling_params,
167
+ request_id=prompt):
168
final_output = output
169
return final_output
170
143
- async def generate():
144
- return await asyncio.gather(
145
- run("test0"),
146
- run("test1"),
147
- )
148
-
149
- results = asyncio.run(generate())
171
+ results = await asyncio.gather(
172
+ run("test0"),
173
+ run("test1"),
174
+ )
175
assert len(results) == 2
176
+
177
+
178
+@pytest.mark.asyncio(scope="module")
179
+async def test_cancellation(async_engine):
180
+ sampling_params = SamplingParams(
181
+ temperature=0,
182
+ min_tokens=10,
183
+ max_tokens=10,
184
+ )
185
+
186
+ i = 0
187
+ with pytest.raises(CancelledError):
188
+ async for output in async_engine.generate("test2",
189
+ sampling_params,
190
+ request_id="test2"):
191
+ assert not output.finished
192
+ i += 1
193
+ if i == 5:
194
+ await async_engine.abort("test2")
195
+
196
+ assert i == 5
197
+
198
+
199
+@pytest.mark.asyncio(scope="module")
200
+async def test_delayed_generator(async_engine):
201
+ sampling_params = SamplingParams(
202
+ temperature=0,
203
+ min_tokens=10,
204
+ max_tokens=10,
205
+ )
206
+
207
+ stream = async_engine.generate("test3",
208
+ sampling_params,
209
+ request_id="test3")
210
+ i = 0
211
+ final_output: Optional[RealRequestOutput] = None
212
+ async for output in stream:
213
+ final_output = output
214
+ if i == 0:
215
+ # wait for generation to complete before consuming
216
+ # the remaining messages
217
+ await asyncio.sleep(1)
218
+ if i < 9:
219
+ assert not output.finished
220
+ i += 1
221
+
222
+ assert i == 10
223
+ assert final_output is not None
224
+ assert len(final_output.outputs[0].token_ids) == 10
225
+ assert final_output.finished
vllm/engine/async_llm_engine.py
+13
-8
@@ -2,8 +2,8 @@ import asyncio
2
import time
3
from dataclasses import dataclass
4
from functools import partial
5
-from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
6
- Optional, Set, Tuple, Type, Union)
5
+from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
6
+ Mapping, Optional, Set, Tuple, Type, Union)
7
8
import torch
9
from typing_extensions import assert_never
@@ -85,9 +85,8 @@ class AsyncStream:
85
86
def put(self, item: Union[RequestOutput, EmbeddingRequestOutput,
87
Exception]) -> None:
88
- if self._finished:
89
- return
90
- self._queue.put_nowait(item)
88
+ if not self._finished:
89
+ self._queue.put_nowait(item)
90
91
def finish(
92
self,
@@ -96,7 +95,7 @@ class AsyncStream:
95
if not self._finished:
96
self._finished = True
97
self._queue.put_nowait(
99
- exception if exception is not None else STOP_ITERATION)
98
+ exception if self._is_raisable(exception) else STOP_ITERATION)
99
100
@property
101
def finished(self) -> bool:
@@ -106,9 +105,9 @@ class AsyncStream:
105
self
106
) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
107
try:
109
- while not self._finished:
108
+ while True:
109
result = await self._queue.get()
111
- if isinstance(result, Exception):
110
+ if self._is_raisable(result):
111
if result == STOP_ITERATION:
112
return
113
raise result
@@ -117,6 +116,12 @@ class AsyncStream:
116
self._cancel(self.request_id)
117
raise asyncio.CancelledError from None
118
119
+ @staticmethod
120
+ def _is_raisable(value: Any):
121
+ return isinstance(value, BaseException) or \
122
+ (isinstance(value, type) and \
123
+ issubclass(value, BaseException))
124
+
125
126
class RequestTracker:
127
"""Synchronous abstraction for tracking requests."""