@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 78687504f7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] AsyncLLMEngine hangs with asyncio.run (#5654)

zifeitong committed Jun 19, 2024 at 13:57 UTC 78687504f7eb6d7523bff15b1bca8c9cbb74656a
5 files changed +271 -47
tests/async_engine/test_async_llm_engine.py
+37 -1
@@ -2,8 +2,12 @@ import asyncio
2 from dataclasses import dataclass
3
4 import pytest
5 +import torch
6
6 -from vllm.engine.async_llm_engine import AsyncLLMEngine
7 +from vllm import SamplingParams
8 +from vllm.engine.async_llm_engine import AsyncEngineArgs, AsyncLLMEngine
9 +
10 +from ..utils import wait_for_gpu_memory_to_clear
11
12
13 @dataclass
@@ -94,3 +98,35 @@ async def test_new_requests_event():
98 assert engine.get_model_config() is not None
99 assert engine.get_tokenizer() is not None
100 assert engine.get_decoding_config() is not None
101 +
102 +
103 +def test_asyncio_run():
104 + wait_for_gpu_memory_to_clear(
105 + devices=list(range(torch.cuda.device_count())),
106 + threshold_bytes=2 * 2**30,
107 + timeout_s=60,
108 + )
109 +
110 + engine = AsyncLLMEngine.from_engine_args(
111 + AsyncEngineArgs(model="facebook/opt-125m"))
112 +
113 + async def run(prompt: str):
114 + sampling_params = SamplingParams(
115 + temperature=0,
116 + max_tokens=32,
117 + )
118 +
119 + async for output in engine.generate(prompt,
120 + sampling_params,
121 + request_id=prompt):
122 + final_output = output
123 + return final_output
124 +
125 + async def generate():
126 + return await asyncio.gather(
127 + run("test0"),
128 + run("test1"),
129 + )
130 +
131 + results = asyncio.run(generate())
132 + assert len(results) == 2
tests/spec_decode/e2e/conftest.py
+1 -42
@@ -1,5 +1,4 @@
1 import asyncio
2 -import time
2 from itertools import cycle
3 from typing import Dict, List, Optional, Tuple, Union
4
@@ -7,12 +6,6 @@ import pytest
6 import ray
7 import torch
8
10 -from vllm.utils import is_hip
11 -
12 -if (not is_hip()):
13 - from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
14 - nvmlInit)
15 -
9 from vllm import LLM
10 from vllm.engine.arg_utils import AsyncEngineArgs
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -26,6 +19,7 @@ from vllm.usage.usage_lib import UsageContext
19 from vllm.utils import Counter, random_uuid
20
21 from ...conftest import cleanup
22 +from ...utils import wait_for_gpu_memory_to_clear
23
24
25 class AsyncLLM:
@@ -291,38 +285,3 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
285 print(f'{i=} {baseline_token_ids=}')
286 print(f'{i=} {spec_token_ids=}')
287 assert baseline_token_ids == spec_token_ids
294 -
295 -
296 -def wait_for_gpu_memory_to_clear(devices: List[int],
297 - threshold_bytes: int,
298 - timeout_s: float = 120) -> None:
299 - # Use nvml instead of pytorch to reduce measurement error from torch cuda
300 - # context.
301 - nvmlInit()
302 - start_time = time.time()
303 - while True:
304 - output: Dict[int, str] = {}
305 - output_raw: Dict[int, float] = {}
306 - for device in devices:
307 - dev_handle = nvmlDeviceGetHandleByIndex(device)
308 - mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
309 - gb_used = mem_info.used / 2**30
310 - output_raw[device] = gb_used
311 - output[device] = f'{gb_used:.02f}'
312 -
313 - print('gpu memory used (GB): ', end='')
314 - for k, v in output.items():
315 - print(f'{k}={v}; ', end='')
316 - print('')
317 -
318 - dur_s = time.time() - start_time
319 - if all(v <= (threshold_bytes / 2**30) for v in output_raw.values()):
320 - print(f'Done waiting for free GPU memory on devices {devices=} '
321 - f'({threshold_bytes/2**30=}) {dur_s=:.02f}')
322 - break
323 -
324 - if dur_s >= timeout_s:
325 - raise ValueError(f'Memory of devices {devices=} not free after '
326 - f'{dur_s=:.02f} ({threshold_bytes/2**30=})')
327 -
328 - time.sleep(5)
tests/utils.py
+41 -2
@@ -4,7 +4,7 @@ import sys
4 import time
5 import warnings
6 from contextlib import contextmanager
7 -from typing import List
7 +from typing import Dict, List
8
9 import openai
10 import ray
@@ -13,7 +13,11 @@ import requests
13 from vllm.distributed import (ensure_model_parallel_initialized,
14 init_distributed_environment)
15 from vllm.entrypoints.openai.cli_args import make_arg_parser
16 -from vllm.utils import get_open_port
16 +from vllm.utils import get_open_port, is_hip
17 +
18 +if (not is_hip()):
19 + from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
20 + nvmlInit)
21
22 # Path to root of repository so that utilities can be imported by ray workers
23 VLLM_PATH = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))
@@ -154,3 +158,38 @@ def error_on_warning():
158 warnings.simplefilter("error")
159
160 yield
161 +
162 +
163 +def wait_for_gpu_memory_to_clear(devices: List[int],
164 + threshold_bytes: int,
165 + timeout_s: float = 120) -> None:
166 + # Use nvml instead of pytorch to reduce measurement error from torch cuda
167 + # context.
168 + nvmlInit()
169 + start_time = time.time()
170 + while True:
171 + output: Dict[int, str] = {}
172 + output_raw: Dict[int, float] = {}
173 + for device in devices:
174 + dev_handle = nvmlDeviceGetHandleByIndex(device)
175 + mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
176 + gb_used = mem_info.used / 2**30
177 + output_raw[device] = gb_used
178 + output[device] = f'{gb_used:.02f}'
179 +
180 + print('gpu memory used (GB): ', end='')
181 + for k, v in output.items():
182 + print(f'{k}={v}; ', end='')
183 + print('')
184 +
185 + dur_s = time.time() - start_time
186 + if all(v <= (threshold_bytes / 2**30) for v in output_raw.values()):
187 + print(f'Done waiting for free GPU memory on devices {devices=} '
188 + f'({threshold_bytes/2**30=}) {dur_s=:.02f}')
189 + break
190 +
191 + if dur_s >= timeout_s:
192 + raise ValueError(f'Memory of devices {devices=} not free after '
193 + f'{dur_s=:.02f} ({threshold_bytes/2**30=})')
194 +
195 + time.sleep(5)
vllm/engine/async_llm_engine.py
+3 -2
@@ -10,6 +10,7 @@ import vllm.envs as envs
10 from vllm.config import DecodingConfig, ModelConfig
11 from vllm.core.scheduler import SchedulerOutputs
12 from vllm.engine.arg_utils import AsyncEngineArgs
13 +from vllm.engine.async_timeout import asyncio_timeout
14 from vllm.engine.llm_engine import LLMEngine
15 from vllm.executor.ray_utils import initialize_ray_cluster, ray
16 from vllm.inputs import LLMInputs, PromptInputs
@@ -545,8 +546,8 @@ class AsyncLLMEngine:
546 # Abort if iteration takes too long due to unrecoverable errors
547 # (eg. NCCL timeouts).
548 try:
548 - has_requests_in_progress = await asyncio.wait_for(
549 - self.engine_step(), ENGINE_ITERATION_TIMEOUT_S)
549 + async with asyncio_timeout(ENGINE_ITERATION_TIMEOUT_S):
550 + has_requests_in_progress = await self.engine_step()
551 except asyncio.TimeoutError as exc:
552 logger.error(
553 "Engine iteration timed out. This should never happen!")
vllm/engine/async_timeout.py new
+189
@@ -0,0 +1,189 @@
1 +# Workaround for https://github.com/python/cpython/issues/86296
2 +#
3 +# From https://github.com/aio-libs/async-timeout/blob/master/async_timeout/__init__.py
4 +# Licensed under the Apache License (Apache-2.0)
5 +
6 +import asyncio
7 +import enum
8 +import sys
9 +import warnings
10 +from types import TracebackType
11 +from typing import Any, Optional, Type
12 +
13 +if sys.version_info[:2] >= (3, 11):
14 + from asyncio import timeout as asyncio_timeout
15 +else:
16 +
17 + def asyncio_timeout(delay: Optional[float]) -> "Timeout":
18 + """timeout context manager.
19 + Useful in cases when you want to apply timeout logic around block
20 + of code or in cases when asyncio.wait_for is not suitable. For example:
21 + >>> async with timeout(0.001):
22 + ... async with aiohttp.get('https://github.com') as r:
23 + ... await r.text()
24 + delay - value in seconds or None to disable timeout logic
25 + """
26 + loop = asyncio.get_running_loop()
27 + deadline = loop.time() + delay if delay is not None else None
28 + return Timeout(deadline, loop)
29 +
30 + class _State(enum.Enum):
31 + INIT = "INIT"
32 + ENTER = "ENTER"
33 + TIMEOUT = "TIMEOUT"
34 + EXIT = "EXIT"
35 +
36 + class Timeout:
37 + # Internal class, please don't instantiate it directly
38 + # Use timeout() and timeout_at() public factories instead.
39 + #
40 + # Implementation note: `async with timeout()` is preferred
41 + # over `with timeout()`.
42 + # While technically the Timeout class implementation
43 + # doesn't need to be async at all,
44 + # the `async with` statement explicitly points that
45 + # the context manager should be used from async function context.
46 + #
47 + # This design allows to avoid many silly misusages.
48 + #
49 + # TimeoutError is raised immediately when scheduled
50 + # if the deadline is passed.
51 + # The purpose is to time out as soon as possible
52 + # without waiting for the next await expression.
53 +
54 + __slots__ = ("_deadline", "_loop", "_state", "_timeout_handler")
55 +
56 + def __init__(self, deadline: Optional[float],
57 + loop: asyncio.AbstractEventLoop) -> None:
58 + self._loop = loop
59 + self._state = _State.INIT
60 +
61 + self._timeout_handler = None # type: Optional[asyncio.Handle]
62 + if deadline is None:
63 + self._deadline = None # type: Optional[float]
64 + else:
65 + self.update(deadline)
66 +
67 + def __enter__(self) -> "Timeout":
68 + warnings.warn(
69 + "with timeout() is deprecated, use async with timeout()",
70 + DeprecationWarning,
71 + stacklevel=2,
72 + )
73 + self._do_enter()
74 + return self
75 +
76 + def __exit__(
77 + self,
78 + exc_type: Optional[Type[BaseException]],
79 + exc_val: Optional[BaseException],
80 + exc_tb: Optional[TracebackType],
81 + ) -> Optional[bool]:
82 + self._do_exit(exc_type)
83 + return None
84 +
85 + async def __aenter__(self) -> "Timeout":
86 + self._do_enter()
87 + return self
88 +
89 + async def __aexit__(
90 + self,
91 + exc_type: Optional[Type[BaseException]],
92 + exc_val: Optional[BaseException],
93 + exc_tb: Optional[TracebackType],
94 + ) -> Optional[bool]:
95 + self._do_exit(exc_type)
96 + return None
97 +
98 + @property
99 + def expired(self) -> bool:
100 + """Is timeout expired during execution?"""
101 + return self._state == _State.TIMEOUT
102 +
103 + @property
104 + def deadline(self) -> Optional[float]:
105 + return self._deadline
106 +
107 + def reject(self) -> None:
108 + """Reject scheduled timeout if any."""
109 + # cancel is maybe better name but
110 + # task.cancel() raises CancelledError in asyncio world.
111 + if self._state not in (_State.INIT, _State.ENTER):
112 + raise RuntimeError(f"invalid state {self._state.value}")
113 + self._reject()
114 +
115 + def _reject(self) -> None:
116 + if self._timeout_handler is not None:
117 + self._timeout_handler.cancel()
118 + self._timeout_handler = None
119 +
120 + def shift(self, delay: float) -> None:
121 + """Advance timeout on delay seconds.
122 + The delay can be negative.
123 + Raise RuntimeError if shift is called when deadline is not scheduled
124 + """
125 + deadline = self._deadline
126 + if deadline is None:
127 + raise RuntimeError(
128 + "cannot shift timeout if deadline is not scheduled")
129 + self.update(deadline + delay)
130 +
131 + def update(self, deadline: float) -> None:
132 + """Set deadline to absolute value.
133 + deadline argument points on the time in the same clock system
134 + as loop.time().
135 + If new deadline is in the past the timeout is raised immediately.
136 + Please note: it is not POSIX time but a time with
137 + undefined starting base, e.g. the time of the system power on.
138 + """
139 + if self._state == _State.EXIT:
140 + raise RuntimeError(
141 + "cannot reschedule after exit from context manager")
142 + if self._state == _State.TIMEOUT:
143 + raise RuntimeError("cannot reschedule expired timeout")
144 + if self._timeout_handler is not None:
145 + self._timeout_handler.cancel()
146 + self._deadline = deadline
147 + if self._state != _State.INIT:
148 + self._reschedule()
149 +
150 + def _reschedule(self) -> None:
151 + assert self._state == _State.ENTER
152 + deadline = self._deadline
153 + if deadline is None:
154 + return
155 +
156 + now = self._loop.time()
157 + if self._timeout_handler is not None:
158 + self._timeout_handler.cancel()
159 +
160 + task = asyncio.current_task()
161 + if deadline <= now:
162 + self._timeout_handler = self._loop.call_soon(
163 + self._on_timeout, task)
164 + else:
165 + self._timeout_handler = self._loop.call_at(
166 + deadline, self._on_timeout, task)
167 +
168 + def _do_enter(self) -> None:
169 + if self._state != _State.INIT:
170 + raise RuntimeError(f"invalid state {self._state.value}")
171 + self._state = _State.ENTER
172 + self._reschedule()
173 +
174 + def _do_exit(self, exc_type: Optional[Type[BaseException]]) -> None:
175 + if exc_type is asyncio.CancelledError and \
176 + self._state == _State.TIMEOUT:
177 + self._timeout_handler = None
178 + raise asyncio.TimeoutError
179 + # timeout has not expired
180 + self._state = _State.EXIT
181 + self._reject()
182 + return None
183 +
184 + def _on_timeout(self, task: "Optional[asyncio.Task[Any]]") -> None:
185 + if task:
186 + task.cancel()
187 + self._state = _State.TIMEOUT
188 + # drop the reference early
189 + self._timeout_handler = None