A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] AsyncLLMEngine hangs with asyncio.run (#5654)
zifeitong committed
Jun 19, 2024 at 13:57 UTC
78687504f7eb6d7523bff15b1bca8c9cbb74656a
5 files changed
+271
-47
tests/async_engine/test_async_llm_engine.py
+37
-1
@@ -2,8 +2,12 @@ import asyncio
2
from dataclasses import dataclass
3
4
import pytest
5
+import torch
6
6
-from vllm.engine.async_llm_engine import AsyncLLMEngine
7
+from vllm import SamplingParams
8
+from vllm.engine.async_llm_engine import AsyncEngineArgs, AsyncLLMEngine
9
+
10
+from ..utils import wait_for_gpu_memory_to_clear
11
12
13
@dataclass
@@ -94,3 +98,35 @@ async def test_new_requests_event():
98
assert engine.get_model_config() is not None
99
assert engine.get_tokenizer() is not None
100
assert engine.get_decoding_config() is not None
101
+
102
+
103
+def test_asyncio_run():
104
+ wait_for_gpu_memory_to_clear(
105
+ devices=list(range(torch.cuda.device_count())),
106
+ threshold_bytes=2 * 2**30,
107
+ timeout_s=60,
108
+ )
109
+
110
+ engine = AsyncLLMEngine.from_engine_args(
111
+ AsyncEngineArgs(model="facebook/opt-125m"))
112
+
113
+ async def run(prompt: str):
114
+ sampling_params = SamplingParams(
115
+ temperature=0,
116
+ max_tokens=32,
117
+ )
118
+
119
+ async for output in engine.generate(prompt,
120
+ sampling_params,
121
+ request_id=prompt):
122
+ final_output = output
123
+ return final_output
124
+
125
+ async def generate():
126
+ return await asyncio.gather(
127
+ run("test0"),
128
+ run("test1"),
129
+ )
130
+
131
+ results = asyncio.run(generate())
132
+ assert len(results) == 2
tests/spec_decode/e2e/conftest.py
+1
-42
@@ -1,5 +1,4 @@
1
import asyncio
2
-import time
2
from itertools import cycle
3
from typing import Dict, List, Optional, Tuple, Union
4
@@ -7,12 +6,6 @@ import pytest
6
import ray
7
import torch
8
10
-from vllm.utils import is_hip
11
-
12
-if (not is_hip()):
13
- from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
14
- nvmlInit)
15
-
9
from vllm import LLM
10
from vllm.engine.arg_utils import AsyncEngineArgs
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -26,6 +19,7 @@ from vllm.usage.usage_lib import UsageContext
19
from vllm.utils import Counter, random_uuid
20
21
from ...conftest import cleanup
22
+from ...utils import wait_for_gpu_memory_to_clear
23
24
25
class AsyncLLM:
@@ -291,38 +285,3 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
285
print(f'{i=} {baseline_token_ids=}')
286
print(f'{i=} {spec_token_ids=}')
287
assert baseline_token_ids == spec_token_ids
294
-
295
-
296
-def wait_for_gpu_memory_to_clear(devices: List[int],
297
- threshold_bytes: int,
298
- timeout_s: float = 120) -> None:
299
- # Use nvml instead of pytorch to reduce measurement error from torch cuda
300
- # context.
301
- nvmlInit()
302
- start_time = time.time()
303
- while True:
304
- output: Dict[int, str] = {}
305
- output_raw: Dict[int, float] = {}
306
- for device in devices:
307
- dev_handle = nvmlDeviceGetHandleByIndex(device)
308
- mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
309
- gb_used = mem_info.used / 2**30
310
- output_raw[device] = gb_used
311
- output[device] = f'{gb_used:.02f}'
312
-
313
- print('gpu memory used (GB): ', end='')
314
- for k, v in output.items():
315
- print(f'{k}={v}; ', end='')
316
- print('')
317
-
318
- dur_s = time.time() - start_time
319
- if all(v <= (threshold_bytes / 2**30) for v in output_raw.values()):
320
- print(f'Done waiting for free GPU memory on devices {devices=} '
321
- f'({threshold_bytes/2**30=}) {dur_s=:.02f}')
322
- break
323
-
324
- if dur_s >= timeout_s:
325
- raise ValueError(f'Memory of devices {devices=} not free after '
326
- f'{dur_s=:.02f} ({threshold_bytes/2**30=})')
327
-
328
- time.sleep(5)
tests/utils.py
+41
-2
@@ -4,7 +4,7 @@ import sys
4
import time
5
import warnings
6
from contextlib import contextmanager
7
-from typing import List
7
+from typing import Dict, List
8
9
import openai
10
import ray
@@ -13,7 +13,11 @@ import requests
13
from vllm.distributed import (ensure_model_parallel_initialized,
14
init_distributed_environment)
15
from vllm.entrypoints.openai.cli_args import make_arg_parser
16
-from vllm.utils import get_open_port
16
+from vllm.utils import get_open_port, is_hip
17
+
18
+if (not is_hip()):
19
+ from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
20
+ nvmlInit)
21
22
# Path to root of repository so that utilities can be imported by ray workers
23
VLLM_PATH = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))
@@ -154,3 +158,38 @@ def error_on_warning():
158
warnings.simplefilter("error")
159
160
yield
161
+
162
+
163
+def wait_for_gpu_memory_to_clear(devices: List[int],
164
+ threshold_bytes: int,
165
+ timeout_s: float = 120) -> None:
166
+ # Use nvml instead of pytorch to reduce measurement error from torch cuda
167
+ # context.
168
+ nvmlInit()
169
+ start_time = time.time()
170
+ while True:
171
+ output: Dict[int, str] = {}
172
+ output_raw: Dict[int, float] = {}
173
+ for device in devices:
174
+ dev_handle = nvmlDeviceGetHandleByIndex(device)
175
+ mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
176
+ gb_used = mem_info.used / 2**30
177
+ output_raw[device] = gb_used
178
+ output[device] = f'{gb_used:.02f}'
179
+
180
+ print('gpu memory used (GB): ', end='')
181
+ for k, v in output.items():
182
+ print(f'{k}={v}; ', end='')
183
+ print('')
184
+
185
+ dur_s = time.time() - start_time
186
+ if all(v <= (threshold_bytes / 2**30) for v in output_raw.values()):
187
+ print(f'Done waiting for free GPU memory on devices {devices=} '
188
+ f'({threshold_bytes/2**30=}) {dur_s=:.02f}')
189
+ break
190
+
191
+ if dur_s >= timeout_s:
192
+ raise ValueError(f'Memory of devices {devices=} not free after '
193
+ f'{dur_s=:.02f} ({threshold_bytes/2**30=})')
194
+
195
+ time.sleep(5)
vllm/engine/async_llm_engine.py
+3
-2
@@ -10,6 +10,7 @@ import vllm.envs as envs
10
from vllm.config import DecodingConfig, ModelConfig
11
from vllm.core.scheduler import SchedulerOutputs
12
from vllm.engine.arg_utils import AsyncEngineArgs
13
+from vllm.engine.async_timeout import asyncio_timeout
14
from vllm.engine.llm_engine import LLMEngine
15
from vllm.executor.ray_utils import initialize_ray_cluster, ray
16
from vllm.inputs import LLMInputs, PromptInputs
@@ -545,8 +546,8 @@ class AsyncLLMEngine:
546
# Abort if iteration takes too long due to unrecoverable errors
547
# (eg. NCCL timeouts).
548
try:
548
- has_requests_in_progress = await asyncio.wait_for(
549
- self.engine_step(), ENGINE_ITERATION_TIMEOUT_S)
549
+ async with asyncio_timeout(ENGINE_ITERATION_TIMEOUT_S):
550
+ has_requests_in_progress = await self.engine_step()
551
except asyncio.TimeoutError as exc:
552
logger.error(
553
"Engine iteration timed out. This should never happen!")
vllm/engine/async_timeout.py
new
+189
@@ -0,0 +1,189 @@
1
+# Workaround for https://github.com/python/cpython/issues/86296
2
+#
3
+# From https://github.com/aio-libs/async-timeout/blob/master/async_timeout/__init__.py
4
+# Licensed under the Apache License (Apache-2.0)
5
+
6
+import asyncio
7
+import enum
8
+import sys
9
+import warnings
10
+from types import TracebackType
11
+from typing import Any, Optional, Type
12
+
13
+if sys.version_info[:2] >= (3, 11):
14
+ from asyncio import timeout as asyncio_timeout
15
+else:
16
+
17
+ def asyncio_timeout(delay: Optional[float]) -> "Timeout":
18
+ """timeout context manager.
19
+ Useful in cases when you want to apply timeout logic around block
20
+ of code or in cases when asyncio.wait_for is not suitable. For example:
21
+ >>> async with timeout(0.001):
22
+ ... async with aiohttp.get('https://github.com') as r:
23
+ ... await r.text()
24
+ delay - value in seconds or None to disable timeout logic
25
+ """
26
+ loop = asyncio.get_running_loop()
27
+ deadline = loop.time() + delay if delay is not None else None
28
+ return Timeout(deadline, loop)
29
+
30
+ class _State(enum.Enum):
31
+ INIT = "INIT"
32
+ ENTER = "ENTER"
33
+ TIMEOUT = "TIMEOUT"
34
+ EXIT = "EXIT"
35
+
36
+ class Timeout:
37
+ # Internal class, please don't instantiate it directly
38
+ # Use timeout() and timeout_at() public factories instead.
39
+ #
40
+ # Implementation note: `async with timeout()` is preferred
41
+ # over `with timeout()`.
42
+ # While technically the Timeout class implementation
43
+ # doesn't need to be async at all,
44
+ # the `async with` statement explicitly points that
45
+ # the context manager should be used from async function context.
46
+ #
47
+ # This design allows to avoid many silly misusages.
48
+ #
49
+ # TimeoutError is raised immediately when scheduled
50
+ # if the deadline is passed.
51
+ # The purpose is to time out as soon as possible
52
+ # without waiting for the next await expression.
53
+
54
+ __slots__ = ("_deadline", "_loop", "_state", "_timeout_handler")
55
+
56
+ def __init__(self, deadline: Optional[float],
57
+ loop: asyncio.AbstractEventLoop) -> None:
58
+ self._loop = loop
59
+ self._state = _State.INIT
60
+
61
+ self._timeout_handler = None # type: Optional[asyncio.Handle]
62
+ if deadline is None:
63
+ self._deadline = None # type: Optional[float]
64
+ else:
65
+ self.update(deadline)
66
+
67
+ def __enter__(self) -> "Timeout":
68
+ warnings.warn(
69
+ "with timeout() is deprecated, use async with timeout()",
70
+ DeprecationWarning,
71
+ stacklevel=2,
72
+ )
73
+ self._do_enter()
74
+ return self
75
+
76
+ def __exit__(
77
+ self,
78
+ exc_type: Optional[Type[BaseException]],
79
+ exc_val: Optional[BaseException],
80
+ exc_tb: Optional[TracebackType],
81
+ ) -> Optional[bool]:
82
+ self._do_exit(exc_type)
83
+ return None
84
+
85
+ async def __aenter__(self) -> "Timeout":
86
+ self._do_enter()
87
+ return self
88
+
89
+ async def __aexit__(
90
+ self,
91
+ exc_type: Optional[Type[BaseException]],
92
+ exc_val: Optional[BaseException],
93
+ exc_tb: Optional[TracebackType],
94
+ ) -> Optional[bool]:
95
+ self._do_exit(exc_type)
96
+ return None
97
+
98
+ @property
99
+ def expired(self) -> bool:
100
+ """Is timeout expired during execution?"""
101
+ return self._state == _State.TIMEOUT
102
+
103
+ @property
104
+ def deadline(self) -> Optional[float]:
105
+ return self._deadline
106
+
107
+ def reject(self) -> None:
108
+ """Reject scheduled timeout if any."""
109
+ # cancel is maybe better name but
110
+ # task.cancel() raises CancelledError in asyncio world.
111
+ if self._state not in (_State.INIT, _State.ENTER):
112
+ raise RuntimeError(f"invalid state {self._state.value}")
113
+ self._reject()
114
+
115
+ def _reject(self) -> None:
116
+ if self._timeout_handler is not None:
117
+ self._timeout_handler.cancel()
118
+ self._timeout_handler = None
119
+
120
+ def shift(self, delay: float) -> None:
121
+ """Advance timeout on delay seconds.
122
+ The delay can be negative.
123
+ Raise RuntimeError if shift is called when deadline is not scheduled
124
+ """
125
+ deadline = self._deadline
126
+ if deadline is None:
127
+ raise RuntimeError(
128
+ "cannot shift timeout if deadline is not scheduled")
129
+ self.update(deadline + delay)
130
+
131
+ def update(self, deadline: float) -> None:
132
+ """Set deadline to absolute value.
133
+ deadline argument points on the time in the same clock system
134
+ as loop.time().
135
+ If new deadline is in the past the timeout is raised immediately.
136
+ Please note: it is not POSIX time but a time with
137
+ undefined starting base, e.g. the time of the system power on.
138
+ """
139
+ if self._state == _State.EXIT:
140
+ raise RuntimeError(
141
+ "cannot reschedule after exit from context manager")
142
+ if self._state == _State.TIMEOUT:
143
+ raise RuntimeError("cannot reschedule expired timeout")
144
+ if self._timeout_handler is not None:
145
+ self._timeout_handler.cancel()
146
+ self._deadline = deadline
147
+ if self._state != _State.INIT:
148
+ self._reschedule()
149
+
150
+ def _reschedule(self) -> None:
151
+ assert self._state == _State.ENTER
152
+ deadline = self._deadline
153
+ if deadline is None:
154
+ return
155
+
156
+ now = self._loop.time()
157
+ if self._timeout_handler is not None:
158
+ self._timeout_handler.cancel()
159
+
160
+ task = asyncio.current_task()
161
+ if deadline <= now:
162
+ self._timeout_handler = self._loop.call_soon(
163
+ self._on_timeout, task)
164
+ else:
165
+ self._timeout_handler = self._loop.call_at(
166
+ deadline, self._on_timeout, task)
167
+
168
+ def _do_enter(self) -> None:
169
+ if self._state != _State.INIT:
170
+ raise RuntimeError(f"invalid state {self._state.value}")
171
+ self._state = _State.ENTER
172
+ self._reschedule()
173
+
174
+ def _do_exit(self, exc_type: Optional[Type[BaseException]]) -> None:
175
+ if exc_type is asyncio.CancelledError and \
176
+ self._state == _State.TIMEOUT:
177
+ self._timeout_handler = None
178
+ raise asyncio.TimeoutError
179
+ # timeout has not expired
180
+ self._state = _State.EXIT
181
+ self._reject()
182
+ return None
183
+
184
+ def _on_timeout(self, task: "Optional[asyncio.Task[Any]]") -> None:
185
+ if task:
186
+ task.cancel()
187
+ self._state = _State.TIMEOUT
188
+ # drop the reference early
189
+ self._timeout_handler = None