@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c75363fbc0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[BugFix] Avoid premature async generator exit and raise all exception variations (#7698)

Nick Hill committed Aug 21, 2024 at 11:45 UTC c75363fbc0ffd9d56bab452dd12adbf3628d73b4
2 files changed +101 -21
tests/async_engine/test_async_llm_engine.py
+88 -13
@@ -1,14 +1,19 @@
1 import asyncio
2 import os
3 +from asyncio import CancelledError
4 from dataclasses import dataclass
5 +from typing import Optional
6
7 import pytest
8 +import pytest_asyncio
9 import torch
10
11 from vllm import SamplingParams
12 from vllm.config import ParallelConfig
13 from vllm.engine.async_llm_engine import AsyncEngineArgs, AsyncLLMEngine
14 +from vllm.outputs import RequestOutput as RealRequestOutput
15
16 +from ..conftest import cleanup
17 from ..utils import wait_for_gpu_memory_to_clear
18
19
@@ -118,15 +123,38 @@ async def test_new_requests_event():
123 os.environ.pop("VLLM_ALLOW_ENGINE_USE_RAY")
124
125
121 -def test_asyncio_run():
126 +def start_engine():
127 wait_for_gpu_memory_to_clear(
128 devices=list(range(torch.cuda.device_count())),
129 threshold_bytes=2 * 2**30,
130 timeout_s=60,
131 )
132
128 - engine = AsyncLLMEngine.from_engine_args(
129 - AsyncEngineArgs(model="facebook/opt-125m"))
133 + return AsyncLLMEngine.from_engine_args(
134 + AsyncEngineArgs(model="facebook/opt-125m", enforce_eager=True))
135 +
136 +
137 +@pytest_asyncio.fixture(scope="module")
138 +async def async_engine():
139 + engine = await asyncio.get_event_loop().run_in_executor(executor=None,
140 + func=start_engine)
141 + try:
142 + yield engine
143 + finally:
144 + engine.shutdown_background_loop()
145 + del engine
146 + await asyncio.sleep(0.1)
147 + cleanup()
148 +
149 +
150 +@pytest.fixture()
151 +def should_do_global_cleanup_after_test(request) -> bool:
152 + # So we can share the async engine fixture between these tests
153 + return False
154 +
155 +
156 +@pytest.mark.asyncio(scope="module")
157 +async def test_asyncio_run(async_engine):
158
159 async def run(prompt: str):
160 sampling_params = SamplingParams(
@@ -134,17 +162,64 @@ def test_asyncio_run():
162 max_tokens=32,
163 )
164
137 - async for output in engine.generate(prompt,
138 - sampling_params,
139 - request_id=prompt):
165 + async for output in async_engine.generate(prompt,
166 + sampling_params,
167 + request_id=prompt):
168 final_output = output
169 return final_output
170
143 - async def generate():
144 - return await asyncio.gather(
145 - run("test0"),
146 - run("test1"),
147 - )
148 -
149 - results = asyncio.run(generate())
171 + results = await asyncio.gather(
172 + run("test0"),
173 + run("test1"),
174 + )
175 assert len(results) == 2
176 +
177 +
178 +@pytest.mark.asyncio(scope="module")
179 +async def test_cancellation(async_engine):
180 + sampling_params = SamplingParams(
181 + temperature=0,
182 + min_tokens=10,
183 + max_tokens=10,
184 + )
185 +
186 + i = 0
187 + with pytest.raises(CancelledError):
188 + async for output in async_engine.generate("test2",
189 + sampling_params,
190 + request_id="test2"):
191 + assert not output.finished
192 + i += 1
193 + if i == 5:
194 + await async_engine.abort("test2")
195 +
196 + assert i == 5
197 +
198 +
199 +@pytest.mark.asyncio(scope="module")
200 +async def test_delayed_generator(async_engine):
201 + sampling_params = SamplingParams(
202 + temperature=0,
203 + min_tokens=10,
204 + max_tokens=10,
205 + )
206 +
207 + stream = async_engine.generate("test3",
208 + sampling_params,
209 + request_id="test3")
210 + i = 0
211 + final_output: Optional[RealRequestOutput] = None
212 + async for output in stream:
213 + final_output = output
214 + if i == 0:
215 + # wait for generation to complete before consuming
216 + # the remaining messages
217 + await asyncio.sleep(1)
218 + if i < 9:
219 + assert not output.finished
220 + i += 1
221 +
222 + assert i == 10
223 + assert final_output is not None
224 + assert len(final_output.outputs[0].token_ids) == 10
225 + assert final_output.finished
vllm/engine/async_llm_engine.py
+13 -8
@@ -2,8 +2,8 @@ import asyncio
2 import time
3 from dataclasses import dataclass
4 from functools import partial
5 -from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
6 - Optional, Set, Tuple, Type, Union)
5 +from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
6 + Mapping, Optional, Set, Tuple, Type, Union)
7
8 import torch
9 from typing_extensions import assert_never
@@ -85,9 +85,8 @@ class AsyncStream:
85
86 def put(self, item: Union[RequestOutput, EmbeddingRequestOutput,
87 Exception]) -> None:
88 - if self._finished:
89 - return
90 - self._queue.put_nowait(item)
88 + if not self._finished:
89 + self._queue.put_nowait(item)
90
91 def finish(
92 self,
@@ -96,7 +95,7 @@ class AsyncStream:
95 if not self._finished:
96 self._finished = True
97 self._queue.put_nowait(
99 - exception if exception is not None else STOP_ITERATION)
98 + exception if self._is_raisable(exception) else STOP_ITERATION)
99
100 @property
101 def finished(self) -> bool:
@@ -106,9 +105,9 @@ class AsyncStream:
105 self
106 ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
107 try:
109 - while not self._finished:
108 + while True:
109 result = await self._queue.get()
111 - if isinstance(result, Exception):
110 + if self._is_raisable(result):
111 if result == STOP_ITERATION:
112 return
113 raise result
@@ -117,6 +116,12 @@ class AsyncStream:
116 self._cancel(self.request_id)
117 raise asyncio.CancelledError from None
118
119 + @staticmethod
120 + def _is_raisable(value: Any):
121 + return isinstance(value, BaseException) or \
122 + (isinstance(value, type) and \
123 + issubclass(value, BaseException))
124 +
125
126 class RequestTracker:
127 """Synchronous abstraction for tracking requests."""