@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c07ece5ca4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Make `AsyncLLMEngine` more robust & fix batched abort (#969)

Signed-off-by: Antoni Baum <antoni.baum@protonmail.com> Co-authored-by: Avnish Narayan <38871737+avnishn@users.noreply.github.com>

Antoni Baum committed Sep 7, 2023 at 13:43 UTC c07ece5ca490a90b2b19c33ab7da2d21e015d7bd
7 files changed +345 -55
tests/async_engine/api_server_async_engine.py new
+51
@@ -0,0 +1,51 @@
1 +"""vllm.entrypoints.api_server with some extra logging for testing."""
2 +import argparse
3 +from typing import Any, Dict
4 +
5 +import uvicorn
6 +from fastapi.responses import JSONResponse, Response
7 +
8 +import vllm.entrypoints.api_server
9 +from vllm.engine.arg_utils import AsyncEngineArgs
10 +from vllm.engine.async_llm_engine import AsyncLLMEngine
11 +
12 +app = vllm.entrypoints.api_server.app
13 +
14 +
15 +class AsyncLLMEngineWithStats(AsyncLLMEngine):
16 +
17 + def __init__(self, *args, **kwargs):
18 + super().__init__(*args, **kwargs)
19 + self._num_aborts = 0
20 +
21 + async def abort(self, request_id: str) -> None:
22 + await super().abort(request_id)
23 + self._num_aborts += 1
24 +
25 + def testing_stats(self) -> Dict[str, Any]:
26 + return {"num_aborted_requests": self._num_aborts}
27 +
28 +
29 +@app.get("/stats")
30 +def stats() -> Response:
31 + """Get the statistics of the engine."""
32 + return JSONResponse(engine.testing_stats())
33 +
34 +
35 +if __name__ == "__main__":
36 + parser = argparse.ArgumentParser()
37 + parser.add_argument("--host", type=str, default="localhost")
38 + parser.add_argument("--port", type=int, default=8000)
39 + parser = AsyncEngineArgs.add_cli_args(parser)
40 + args = parser.parse_args()
41 +
42 + engine_args = AsyncEngineArgs.from_cli_args(args)
43 + engine = AsyncLLMEngineWithStats.from_engine_args(engine_args,
44 + start_engine_loop=False)
45 + vllm.entrypoints.api_server.engine = engine
46 + uvicorn.run(
47 + app,
48 + host=args.host,
49 + port=args.port,
50 + log_level="debug",
51 + timeout_keep_alive=vllm.entrypoints.api_server.TIMEOUT_KEEP_ALIVE)
tests/async_engine/test_api_server.py new
+86
@@ -0,0 +1,86 @@
1 +import subprocess
2 +import sys
3 +import time
4 +from multiprocessing import Pool
5 +from pathlib import Path
6 +
7 +import pytest
8 +import requests
9 +
10 +
11 +def _query_server(prompt: str) -> dict:
12 + response = requests.post("http://localhost:8000/generate",
13 + json={
14 + "prompt": prompt,
15 + "max_tokens": 100,
16 + "temperature": 0,
17 + "ignore_eos": True
18 + })
19 + response.raise_for_status()
20 + return response.json()
21 +
22 +
23 +@pytest.fixture
24 +def api_server():
25 + script_path = Path(__file__).parent.joinpath(
26 + "api_server_async_engine.py").absolute()
27 + uvicorn_process = subprocess.Popen([
28 + sys.executable, "-u",
29 + str(script_path), "--model", "facebook/opt-125m"
30 + ])
31 + yield
32 + uvicorn_process.terminate()
33 +
34 +
35 +def test_api_server(api_server):
36 + """
37 + Run the API server and test it.
38 +
39 + We run both the server and requests in separate processes.
40 +
41 + We test that the server can handle incoming requests, including
42 + multiple requests at the same time, and that it can handle requests
43 + being cancelled without crashing.
44 + """
45 + with Pool(32) as pool:
46 + # Wait until the server is ready
47 + prompts = ["Hello world"] * 1
48 + result = None
49 + while not result:
50 + try:
51 + for result in pool.map(_query_server, prompts):
52 + break
53 + except:
54 + time.sleep(1)
55 +
56 + # Actual tests start here
57 + # Try with 1 prompt
58 + for result in pool.map(_query_server, prompts):
59 + assert result
60 +
61 + num_aborted_requests = requests.get(
62 + "http://localhost:8000/stats").json()["num_aborted_requests"]
63 + assert num_aborted_requests == 0
64 +
65 + # Try with 100 prompts
66 + prompts = ["Hello world"] * 100
67 + for result in pool.map(_query_server, prompts):
68 + assert result
69 +
70 + # Cancel requests
71 + pool.map_async(_query_server, prompts)
72 + time.sleep(0.01)
73 + pool.terminate()
74 + pool.join()
75 +
76 + # check cancellation stats
77 + num_aborted_requests = requests.get(
78 + "http://localhost:8000/stats").json()["num_aborted_requests"]
79 + assert num_aborted_requests > 0
80 +
81 + # check that server still runs after cancellations
82 + with Pool(32) as pool:
83 + # Try with 100 prompts
84 + prompts = ["Hello world"] * 100
85 + for result in pool.map(_query_server, prompts):
86 + assert result
tests/async_engine/test_request_tracker.py new
+54
@@ -0,0 +1,54 @@
1 +import pytest
2 +
3 +from vllm.engine.async_llm_engine import RequestTracker
4 +from vllm.outputs import RequestOutput
5 +
6 +
7 +def test_request_tracker():
8 + tracker = RequestTracker()
9 + stream_1 = tracker.add_request("1")
10 + new, finished = tracker.get_new_and_finished_requests()
11 + assert len(new) == 1
12 + assert new[0]["request_id"] == "1"
13 + assert not finished
14 + assert not stream_1.finished
15 +
16 + stream_2 = tracker.add_request("2")
17 + stream_3 = tracker.add_request("3")
18 + new, finished = tracker.get_new_and_finished_requests()
19 + assert len(new) == 2
20 + assert new[0]["request_id"] == "2"
21 + assert new[1]["request_id"] == "3"
22 + assert not finished
23 + assert not stream_2.finished
24 + assert not stream_3.finished
25 +
26 + # request_ids must be unique
27 + with pytest.raises(KeyError):
28 + tracker.add_request("1")
29 +
30 + tracker.abort_request("1")
31 + new, finished = tracker.get_new_and_finished_requests()
32 + assert len(finished) == 1
33 + assert "1" in finished
34 + assert not new
35 + assert stream_1.finished
36 +
37 + stream_4 = tracker.add_request("4")
38 + tracker.abort_request("4")
39 + new, finished = tracker.get_new_and_finished_requests()
40 + assert len(finished) == 1
41 + assert "4" in finished
42 + assert not new
43 + assert stream_4.finished
44 +
45 + stream_5 = tracker.add_request("5")
46 + tracker.process_request_output(
47 + RequestOutput("2", "output", [], [], finished=True))
48 + new, finished = tracker.get_new_and_finished_requests()
49 + assert len(finished) == 1
50 + assert "2" in finished
51 + assert len(new) == 1
52 + assert new[0]["request_id"] == "5"
53 + assert stream_2.finished
54 + assert not stream_5.finished
vllm/core/scheduler.py
+4 -1
@@ -92,7 +92,10 @@ class Scheduler:
92 request_id = (request_id, )
93 request_ids = set(request_id)
94 for state_queue in [self.waiting, self.running, self.swapped]:
95 - for seq_group in state_queue:
95 + # We need to reverse the list as we are removing elements
96 + # from it as we iterate over it. If we don't do it,
97 + # indices will get messed up and we will skip over elements.
98 + for seq_group in reversed(state_queue):
99 if seq_group.request_id in request_ids:
100 # Remove the sequence group from the state queue.
101 state_queue.remove(seq_group)
vllm/engine/async_llm_engine.py
+148 -54
@@ -1,7 +1,7 @@
1 import asyncio
2 import time
3 from functools import partial
4 -from typing import Any, Dict, Iterable, List, Optional, Type, Union
4 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union
5
6 from vllm.config import ModelConfig
7 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -14,6 +14,28 @@ from vllm.sampling_params import SamplingParams
14 logger = init_logger(__name__)
15
16
17 +class AsyncEngineDeadError(RuntimeError):
18 + pass
19 +
20 +
21 +def _raise_exception_on_finish(task: asyncio.Task,
22 + request_tracker: "RequestTracker") -> None:
23 + msg = ("Task finished unexpectedly. This should never happen! "
24 + "Please open an issue on Github.")
25 + try:
26 + try:
27 + task.result()
28 + except asyncio.CancelledError:
29 + return
30 + except Exception as exc:
31 + raise AsyncEngineDeadError(
32 + msg + " See stack trace above for the actual cause.") from exc
33 + raise AsyncEngineDeadError(msg)
34 + except Exception as exc:
35 + request_tracker.propagate_exception(exc)
36 + raise exc
37 +
38 +
39 class AsyncStream:
40 """A stream of RequestOutputs for a request that can be
41 iterated over asynchronously."""
@@ -43,15 +65,90 @@ class AsyncStream:
65 result = await self._queue.get()
66 if result is StopIteration:
67 raise StopAsyncIteration
68 + elif isinstance(result, Exception):
69 + raise result
70 return result
71
72
49 -def _raise_exception_on_finish(task: asyncio.Task) -> None:
50 - try:
51 - task.result()
52 - except Exception as e:
53 - raise RuntimeError("Task finished unexpectedly.") from e
54 - raise RuntimeError("Task finished unexpectedly.")
73 +class RequestTracker:
74 + """Synchronous abstraction for tracking requests."""
75 +
76 + def __init__(self) -> None:
77 + self._request_streams: Dict[str, AsyncStream] = {}
78 + self._finished_requests: asyncio.Queue[str] = asyncio.Queue()
79 + self._new_requests: asyncio.Queue[Tuple[AsyncStream,
80 + dict]] = asyncio.Queue()
81 +
82 + def __contains__(self, item):
83 + return item in self._request_streams
84 +
85 + def propagate_exception(self, exc: Exception) -> None:
86 + """Propagate an exception to all request streams."""
87 + for stream in self._request_streams.values():
88 + stream.put(exc)
89 +
90 + def process_request_output(self,
91 + request_output: RequestOutput,
92 + *,
93 + verbose: bool = False) -> None:
94 + """Process a request output from the engine."""
95 + request_id = request_output.request_id
96 +
97 + self._request_streams[request_id].put(request_output)
98 + if request_output.finished:
99 + if verbose:
100 + logger.info(f"Finished request {request_id}.")
101 + self.abort_request(request_id)
102 +
103 + def add_request(self, request_id: str,
104 + **engine_add_request_kwargs) -> AsyncStream:
105 + """Add a request to be sent to the engine on the next background
106 + loop iteration."""
107 + if request_id in self._request_streams:
108 + raise KeyError(f"Request {request_id} already exists.")
109 +
110 + stream = AsyncStream(request_id)
111 + self._new_requests.put_nowait((stream, {
112 + "request_id": request_id,
113 + **engine_add_request_kwargs
114 + }))
115 + return stream
116 +
117 + def abort_request(self, request_id: str, *, verbose: bool = False) -> None:
118 + """Abort a request during next background loop iteration."""
119 + if verbose:
120 + logger.info(f"Aborted request {request_id}.")
121 +
122 + self._finished_requests.put_nowait(request_id)
123 +
124 + if request_id not in self._request_streams or self._request_streams[
125 + request_id].finished:
126 + # The request has already finished or been aborted.
127 + return
128 +
129 + self._request_streams[request_id].finish()
130 +
131 + def get_new_and_finished_requests(self) -> Tuple[List[dict], Set[str]]:
132 + """Get the new requests and finished requests to be
133 + sent to the engine."""
134 + new_requests: List[dict] = []
135 + finished_requests: Set[str] = set()
136 +
137 + while not self._finished_requests.empty():
138 + request_id = self._finished_requests.get_nowait()
139 + finished_requests.add(request_id)
140 + self._request_streams.pop(request_id, None)
141 +
142 + while not self._new_requests.empty():
143 + stream, new_request = self._new_requests.get_nowait()
144 + if stream.request_id in finished_requests:
145 + # The request has already been aborted.
146 + stream.finish()
147 + continue
148 + self._request_streams[stream.request_id] = stream
149 + new_requests.append(new_request)
150 +
151 + return new_requests, finished_requests
152
153
154 class _AsyncLLMEngine(LLMEngine):
@@ -150,16 +247,15 @@ class AsyncLLMEngine:
247 self.log_requests = log_requests
248 self.engine = self._init_engine(*args, **kwargs)
249
153 - # Request id -> stream.
154 - self.request_streams: Dict[str, AsyncStream] = {}
155 - self.finished_requests: asyncio.Queue[str] = asyncio.Queue()
250 + self.request_tracker: RequestTracker = RequestTracker()
251 self.background_loop = None
252 if start_engine_loop:
253 self.start_background_loop()
254
255 @property
256 def is_running(self) -> bool:
162 - return self.background_loop is not None
257 + return (self.background_loop is not None
258 + and not self.background_loop.done())
259
260 def start_background_loop(self) -> None:
261 """Start the background loop."""
@@ -167,7 +263,9 @@ class AsyncLLMEngine:
263 raise RuntimeError("Background loop is already running.")
264 self.background_loop = asyncio.get_event_loop().create_task(
265 self.run_engine_loop())
170 - self.background_loop.add_done_callback(_raise_exception_on_finish)
266 + self.background_loop.add_done_callback(
267 + partial(_raise_exception_on_finish,
268 + request_tracker=self.request_tracker))
269
270 def _init_engine(self, *args,
271 **kwargs) -> Union[_AsyncLLMEngine, "ray.ObjectRef"]:
@@ -181,6 +279,21 @@ class AsyncLLMEngine:
279
280 async def engine_step(self):
281 """Kick the engine to process the waiting requests."""
282 +
283 + new_requests, finished_requests = (
284 + self.request_tracker.get_new_and_finished_requests())
285 +
286 + for new_request in new_requests:
287 + # Add the request into the vLLM engine's waiting queue.
288 + # TODO: Maybe add add_request_batch to reduce Ray overhead
289 + if self.engine_use_ray:
290 + await self.engine.add_request.remote(**new_request)
291 + else:
292 + self.engine.add_request(**new_request)
293 +
294 + if finished_requests:
295 + await self._engine_abort(finished_requests)
296 +
297 if self.engine_use_ray:
298 request_outputs = await self.engine.step.remote()
299 else:
@@ -188,20 +301,8 @@ class AsyncLLMEngine:
301
302 # Put the outputs into the corresponding streams.
303 for request_output in request_outputs:
191 - request_id = request_output.request_id
192 - self.request_streams[request_id].put(request_output)
193 - if request_output.finished:
194 - if self.log_requests:
195 - logger.info(f"Finished request {request_id}.")
196 - self.request_streams[request_id].finish()
197 - self.finished_requests.put_nowait(request_id)
198 -
199 - finished_request = set()
200 - while not self.finished_requests.empty():
201 - finished_request.add(self.finished_requests.get_nowait())
202 - await self._engine_abort(finished_request)
203 - for request_id in finished_request:
204 - del self.request_streams[request_id]
304 + self.request_tracker.process_request_output(
305 + request_output, verbose=self.log_requests)
306
307 async def _engine_abort(self, request_ids: Iterable[str]):
308 if self.engine_use_ray:
@@ -228,25 +329,19 @@ class AsyncLLMEngine:
329 f"sampling params: {sampling_params}, "
330 f"prompt token ids: {prompt_token_ids}.")
331
231 - if request_id in self.request_streams:
232 - raise KeyError(f"Request {request_id} already exists.")
233 - stream = AsyncStream(request_id)
234 - self.request_streams[request_id] = stream
332 + if not self.is_running:
333 + raise AsyncEngineDeadError(
334 + "Background loop is not running. If it was running, "
335 + "inspect the output to find the stacktrace of the "
336 + "error that caused the background loop to stop "
337 + "(AsyncEngineDeadError).")
338
236 - # Add the request into the vLLM engine's waiting queue.
237 - if self.engine_use_ray:
238 - await self.engine.add_request.remote(
239 - request_id,
240 - prompt,
241 - sampling_params,
242 - prompt_token_ids=prompt_token_ids,
243 - arrival_time=arrival_time)
244 - else:
245 - self.engine.add_request(request_id,
246 - prompt,
247 - sampling_params,
248 - prompt_token_ids=prompt_token_ids,
249 - arrival_time=arrival_time)
339 + stream = self.request_tracker.add_request(
340 + request_id,
341 + prompt=prompt,
342 + sampling_params=sampling_params,
343 + prompt_token_ids=prompt_token_ids,
344 + arrival_time=arrival_time)
345
346 return stream
347
@@ -300,6 +395,13 @@ class AsyncLLMEngine:
395 Args:
396 request_id: The unique id of the request.
397 """
398 + if not self.is_running:
399 + raise AsyncEngineDeadError(
400 + "Background loop is not running. If it was running, "
401 + "inspect the output to find the stacktrace of the "
402 + "error that caused the background loop to stop "
403 + "(AsyncEngineDeadError).")
404 +
405 return self._abort(request_id)
406
407 def _abort(self, request_id: str) -> None:
@@ -311,16 +413,8 @@ class AsyncLLMEngine:
413 Args:
414 request_id: The unique id of the request.
415 """
314 - if request_id not in self.request_streams or self.request_streams[
315 - request_id].finished:
316 - # The request has already finished or been aborted.
317 - return
318 -
319 - if self.log_requests:
320 - logger.info(f"Aborted request {request_id}.")
321 -
322 - self.request_streams[request_id].finish()
323 - self.finished_requests.put_nowait(request_id)
416 + self.request_tracker.abort_request(request_id,
417 + verbose=self.log_requests)
418
419 async def get_model_config(self) -> ModelConfig:
420 """Get the model configuration of the vLLM engine."""
vllm/entrypoints/api_server.py
+1
@@ -14,6 +14,7 @@ from vllm.utils import random_uuid
14 TIMEOUT_KEEP_ALIVE = 5 # seconds.
15 TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds.
16 app = FastAPI()
17 +engine = None
18
19
20 @app.post("/generate")
vllm/entrypoints/openai/api_server.py
+1
@@ -44,6 +44,7 @@ TIMEOUT_KEEP_ALIVE = 5 # seconds
44 logger = init_logger(__name__)
45 served_model = None
46 app = fastapi.FastAPI()
47 +engine = None
48
49
50 def create_error_response(status_code: HTTPStatus,