A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Refactor AsyncLLMEngine (#880)
Antoni Baum committed
Sep 3, 2023 at 21:43 UTC
ce741ba3e4fea00bacd2e1c609ca587ec35eb161
3 files changed
+267
-145
vllm/core/scheduler.py
+9
-4
@@ -1,6 +1,6 @@
1
import enum
2
import time
3
-from typing import Dict, List, Optional, Tuple
3
+from typing import Dict, Iterable, List, Optional, Tuple, Union
4
5
from vllm.config import CacheConfig, SchedulerConfig
6
from vllm.core.block_manager import BlockSpaceManager
@@ -87,17 +87,22 @@ class Scheduler:
87
# Add sequence groups to the waiting queue.
88
self.waiting.append(seq_group)
89
90
- def abort_seq_group(self, request_id: str) -> None:
90
+ def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
91
+ if isinstance(request_id, str):
92
+ request_id = (request_id, )
93
+ request_ids = set(request_id)
94
for state_queue in [self.waiting, self.running, self.swapped]:
95
for seq_group in state_queue:
93
- if seq_group.request_id == request_id:
96
+ if seq_group.request_id in request_ids:
97
# Remove the sequence group from the state queue.
98
state_queue.remove(seq_group)
99
for seq in seq_group.seqs:
100
if seq.is_finished():
101
continue
102
self.free_seq(seq, SequenceStatus.FINISHED_ABORTED)
100
- return
103
+ request_ids.remove(seq_group.request_id)
104
+ if not request_ids:
105
+ return
106
107
def has_unfinished_seqs(self) -> bool:
108
return self.waiting or self.running or self.swapped
vllm/engine/async_llm_engine.py
+212
-109
@@ -1,6 +1,7 @@
1
import asyncio
2
import time
3
-from typing import Dict, List, Optional
3
+from functools import partial
4
+from typing import Any, Dict, Iterable, List, Optional, Set, Type, Union
5
6
from vllm.config import ModelConfig
7
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -12,7 +13,105 @@ from vllm.sampling_params import SamplingParams
13
14
logger = init_logger(__name__)
15
15
-TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
16
+
17
+class AsyncStream:
18
+ """A stream of RequestOutputs for a request that can be
19
+ iterated over asynchronously."""
20
+
21
+ def __init__(self, request_id: str) -> None:
22
+ self.request_id = request_id
23
+ self._queue = asyncio.Queue()
24
+ self._finished = False
25
+
26
+ def put(self, item: RequestOutput) -> None:
27
+ if self._finished:
28
+ return
29
+ self._queue.put_nowait(item)
30
+
31
+ def finish(self) -> None:
32
+ self._queue.put_nowait(StopIteration)
33
+ self._finished = True
34
+
35
+ @property
36
+ def finished(self) -> bool:
37
+ return self._finished
38
+
39
+ def __aiter__(self):
40
+ return self
41
+
42
+ async def __anext__(self) -> RequestOutput:
43
+ result = await self._queue.get()
44
+ if result is StopIteration:
45
+ raise StopAsyncIteration
46
+ return result
47
+
48
+
49
+def _raise_exception_on_finish(task: asyncio.Task) -> None:
50
+ try:
51
+ task.result()
52
+ except Exception as e:
53
+ raise RuntimeError("Task finished unexpectedly.") from e
54
+ raise RuntimeError("Task finished unexpectedly.")
55
+
56
+
57
+class _AsyncLLMEngine(LLMEngine):
58
+ """Extension of LLMEngine to add async methods."""
59
+
60
+ async def step_async(self) -> List[RequestOutput]:
61
+ """Performs one decoding iteration and returns newly generated results.
62
+ The workers are ran asynchronously if possible.
63
+
64
+ This function performs one decoding iteration of the engine. It first
65
+ schedules the sequences to be executed in the next iteration and the
66
+ token blocks to be swapped in/out/copy. Then, it executes the model
67
+ and updates the scheduler with the model outputs. Finally, it decodes
68
+ the sequences and returns the newly generated results.
69
+ """
70
+ (seq_group_metadata_list, scheduler_outputs,
71
+ early_return) = self._schedule()
72
+ if early_return is not None:
73
+ return early_return
74
+
75
+ # Execute the model.
76
+ output = await self._run_workers_async(
77
+ "execute_model",
78
+ seq_group_metadata_list=seq_group_metadata_list,
79
+ blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
80
+ blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
81
+ blocks_to_copy=scheduler_outputs.blocks_to_copy,
82
+ )
83
+
84
+ return self._process_worker_outputs(output, scheduler_outputs)
85
+
86
+ async def _run_workers_async(
87
+ self,
88
+ method: str,
89
+ *args,
90
+ get_all_outputs: bool = False,
91
+ **kwargs,
92
+ ) -> Any:
93
+ """Runs the given method on all workers."""
94
+ all_outputs = []
95
+ for worker in self.workers:
96
+ if self.parallel_config.worker_use_ray:
97
+ executor = partial(worker.execute_method.remote, method)
98
+ else:
99
+ executor = getattr(worker, method)
100
+
101
+ output = executor(*args, **kwargs)
102
+ all_outputs.append(output)
103
+
104
+ if self.parallel_config.worker_use_ray:
105
+ all_outputs = await asyncio.gather(*all_outputs)
106
+
107
+ if get_all_outputs:
108
+ return all_outputs
109
+
110
+ # Make sure all workers have the same results.
111
+ output = all_outputs[0]
112
+ for other_output in all_outputs[1:]:
113
+ assert output == other_output
114
+ return output
115
116
117
class AsyncLLMEngine:
@@ -37,49 +136,111 @@ class AsyncLLMEngine:
136
*args, *kwargs: Arguments for LLMEngine.
137
"""
138
139
+ _engine_class: Type[_AsyncLLMEngine] = _AsyncLLMEngine
140
+
141
def __init__(self,
142
worker_use_ray: bool,
143
engine_use_ray: bool,
144
*args,
145
log_requests: bool = True,
146
+ start_engine_loop: bool = False,
147
**kwargs) -> None:
148
self.worker_use_ray = worker_use_ray
149
self.engine_use_ray = engine_use_ray
150
self.log_requests = log_requests
151
+ self.engine = self._init_engine(*args, **kwargs)
152
+
153
+ # Request id -> stream.
154
+ self.request_streams: Dict[str, AsyncStream] = {}
155
+ self.finished_requests: Set[str] = set()
156
+ self.background_loop = None
157
+ if start_engine_loop:
158
+ self._start_background_loop()
159
+
160
+ def _start_background_loop(self) -> None:
161
+ """Start the background loop."""
162
+ if self.background_loop is not None:
163
+ raise RuntimeError("Background loop is already running.")
164
+ self.background_loop = asyncio.get_event_loop().create_task(
165
+ self.run_engine_loop())
166
+ self.background_loop.add_done_callback(_raise_exception_on_finish)
167
+
168
+ def _init_engine(self, *args,
169
+ **kwargs) -> Union[_AsyncLLMEngine, "ray.ObjectRef"]:
170
if not self.engine_use_ray:
50
- engine_class = LLMEngine
171
+ engine_class = self._engine_class
172
elif self.worker_use_ray:
52
- engine_class = ray.remote(num_cpus=0)(LLMEngine).remote
173
+ engine_class = ray.remote(num_cpus=0)(self._engine_class).remote
174
else:
54
- engine_class = ray.remote(num_gpus=1)(LLMEngine).remote
55
- self.engine = engine_class(*args, **kwargs)
56
- # Request id -> request output.
57
- self.request_outputs: Dict[str, RequestOutput] = {}
58
- # Request id -> event to notify that there is new output.
59
- self.request_events: Dict[str, asyncio.Event] = {}
60
- self.is_engine_running = False
61
- self.kicking_request_id: Optional[str] = None
62
-
63
- async def engine_step(self, kicking_request_id: Optional[str] = None):
175
+ engine_class = ray.remote(num_gpus=1)(self._engine_class).remote
176
+ return engine_class(*args, **kwargs)
177
+
178
+ async def engine_step(self):
179
"""Kick the engine to process the waiting requests."""
65
- self.is_engine_running = True
66
- self.kicking_request_id = kicking_request_id
180
if self.engine_use_ray:
181
request_outputs = await self.engine.step.remote()
182
else:
70
- # Yield to the event loop to allow other coroutines to run
71
- # while is_engine_running is True. This let the engine to add new
72
- # requests into the queue.
73
- await asyncio.sleep(0)
74
- request_outputs = self.engine.step()
75
- self.is_engine_running = False
76
- self.kicking_request_id = None
183
+ request_outputs = await self.engine.step_async()
184
78
- # Notify the waiting coroutines that there are new outputs ready.
185
+ # Put the outputs into the corresponding streams.
186
for request_output in request_outputs:
187
request_id = request_output.request_id
81
- self.request_outputs[request_id] = request_output
82
- self.request_events[request_id].set()
188
+ self.request_streams[request_id].put(request_output)
189
+ if request_output.finished:
190
+ if self.log_requests:
191
+ logger.info(f"Finished request {request_id}.")
192
+ self.request_streams[request_id].finish()
193
+ self.finished_requests.add(request_id)
194
+
195
+ await self._engine_abort(self.finished_requests)
196
+ for request_id in self.finished_requests:
197
+ del self.request_streams[request_id]
198
+ self.finished_requests.clear()
199
+
200
+ async def _engine_abort(self, request_ids: Iterable[str]):
201
+ if self.engine_use_ray:
202
+ await self.engine.abort_request.remote(request_ids)
203
+ else:
204
+ self.engine.abort_request(request_ids)
205
+
206
+ async def run_engine_loop(self):
207
+ while True:
208
+ await self.engine_step()
209
+ await asyncio.sleep(0)
210
+
211
+ async def add_request(
212
+ self,
213
+ request_id: str,
214
+ prompt: Optional[str],
215
+ sampling_params: SamplingParams,
216
+ prompt_token_ids: Optional[List[int]] = None,
217
+ arrival_time: Optional[float] = None,
218
+ ) -> AsyncStream:
219
+ if self.log_requests:
220
+ logger.info(f"Received request {request_id}: "
221
+ f"prompt: {prompt!r}, "
222
+ f"sampling params: {sampling_params}, "
223
+ f"prompt token ids: {prompt_token_ids}.")
224
+
225
+ stream = AsyncStream(request_id)
226
+ self.request_streams[request_id] = stream
227
+
228
+ # Add the request into the vLLM engine's waiting queue.
229
+ if self.engine_use_ray:
230
+ await self.engine.add_request.remote(
231
+ request_id,
232
+ prompt,
233
+ sampling_params,
234
+ prompt_token_ids=prompt_token_ids,
235
+ arrival_time=arrival_time)
236
+ else:
237
+ self.engine.add_request(request_id,
238
+ prompt,
239
+ sampling_params,
240
+ prompt_token_ids=prompt_token_ids,
241
+ arrival_time=arrival_time)
242
+
243
+ return stream
244
245
async def generate(
246
self,
@@ -108,78 +269,32 @@ class AsyncLLMEngine:
269
# Preprocess the request.
270
arrival_time = time.time()
271
111
- # Create an event to notify us that there is new output from the
112
- # vLLM engine.
113
- request_event = asyncio.Event()
114
- self.request_events[request_id] = request_event
272
+ try:
273
+ stream = await self.add_request(request_id,
274
+ prompt,
275
+ sampling_params,
276
+ prompt_token_ids=prompt_token_ids,
277
+ arrival_time=arrival_time)
278
116
- if self.log_requests:
117
- logger.info(f"Received request {request_id}: "
118
- f"prompt: {prompt!r}, "
119
- f"sampling params: {sampling_params}, "
120
- f"prompt token ids: {prompt_token_ids}.")
279
+ async for request_output in stream:
280
+ yield request_output
281
+ except Exception as e:
282
+ # If there is an exception, abort the request.
283
+ self._abort(request_id)
284
+ raise e
285
122
- # Add the request into the vLLM engine's waiting queue.
123
- if self.engine_use_ray:
124
- await self.engine.add_request.remote(
125
- request_id,
126
- prompt,
127
- sampling_params,
128
- prompt_token_ids=prompt_token_ids,
129
- arrival_time=arrival_time)
130
- else:
131
- self.engine.add_request(request_id,
132
- prompt,
133
- sampling_params,
134
- prompt_token_ids=prompt_token_ids,
135
- arrival_time=arrival_time)
286
+ async def abort(self, request_id: str) -> None:
287
+ """Abort a request.
288
137
- # The vLLM engine does not have a background loop that keeps
138
- # processing incoming requests. Therefore, we need to keep kicking
139
- # the engine to process the requests.
140
- while True:
141
- if request_id not in self.request_events:
142
- # The request has been aborted.
143
- return
144
-
145
- # Kick the engine if the engine is not running.
146
- if not self.is_engine_running:
147
- try:
148
- await self.engine_step(request_id)
149
- except RuntimeError as e:
150
- await self.abort(request_id)
151
- raise e
152
-
153
- # Wait for new output. The group_event will be set in engine_step
154
- # when there is new output available for the sequence group.
155
- # Added a timeout to prevent deadlock.
156
- try:
157
- await asyncio.wait_for(request_event.wait(),
158
- timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
159
- except asyncio.TimeoutError:
160
- continue
161
- # Reset the event to wait for the next output.
162
- request_event.clear()
163
-
164
- # Decode and return new outputs.
165
- request_output = self.request_outputs[request_id]
166
- yield request_output
167
-
168
- # Once finished, release the resources of the sequence group.
169
- if request_output.finished:
170
- if self.log_requests:
171
- logger.info(f"Finished request {request_id}.")
289
+ Abort a submitted request. If the request is finished or not found,
290
+ this method will be a no-op.
291
173
- del self.request_outputs[request_id]
174
- del self.request_events[request_id]
175
- # Kick the engine if the engine is not running. This is to
176
- # prevent that there are still requests in engine's waiting
177
- # queue to be executed.
178
- if not self.is_engine_running:
179
- await self.engine_step()
180
- break
292
+ Args:
293
+ request_id: The unique id of the request.
294
+ """
295
+ return self._abort(request_id)
296
182
- async def abort(self, request_id: str) -> None:
297
+ def _abort(self, request_id: str) -> None:
298
"""Abort a request.
299
300
Abort a submitted request. If the request is finished or not found,
@@ -188,28 +303,16 @@ class AsyncLLMEngine:
303
Args:
304
request_id: The unique id of the request.
305
"""
191
- if request_id not in self.request_events:
306
+ if request_id not in self.request_streams or self.request_streams[
307
+ request_id].finished:
308
# The request has already finished or been aborted.
309
return
310
311
if self.log_requests:
312
logger.info(f"Aborted request {request_id}.")
313
198
- if self.engine_use_ray:
199
- await self.engine.abort_request.remote(request_id)
200
- else:
201
- self.engine.abort_request(request_id)
202
-
203
- if request_id in self.request_events:
204
- del self.request_events[request_id]
205
- if request_id in self.request_outputs:
206
- del self.request_outputs[request_id]
207
-
208
- # To prevent deadlock when a request is aborted while the engine is
209
- # running.
210
- if self.kicking_request_id == request_id:
211
- self.is_engine_running = False
212
- self.kicking_request_id = None
314
+ self.request_streams[request_id].finish()
315
+ self.finished_requests.add(request_id)
316
317
async def get_model_config(self) -> ModelConfig:
318
"""Get the model configuration of the vLLM engine."""
vllm/engine/llm_engine.py
+46
-32
@@ -1,17 +1,18 @@
1
-import time
1
import copy
2
+import time
3
from functools import partial
4
-from typing import Any, List, Optional, Tuple, TYPE_CHECKING
4
+from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union
5
6
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7
SchedulerConfig)
8
-from vllm.core.scheduler import Scheduler
8
+from vllm.core.scheduler import Scheduler, SchedulerOutputs
9
from vllm.engine.arg_utils import EngineArgs
10
-from vllm.engine.ray_utils import initialize_cluster, ray, RayWorker
10
+from vllm.engine.ray_utils import RayWorker, initialize_cluster, ray
11
from vllm.logger import init_logger
12
from vllm.outputs import RequestOutput
13
from vllm.sampling_params import SamplingParams
14
-from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
14
+from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupMetadata,
15
+ SequenceStatus)
16
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
17
get_tokenizer)
18
from vllm.utils import Counter
@@ -135,7 +136,8 @@ class LLMEngine:
136
get_all_outputs=True,
137
)
138
138
- def _init_workers_ray(self, placement_group: "PlacementGroup"):
139
+ def _init_workers_ray(self, placement_group: "PlacementGroup",
140
+ **ray_remote_kwargs):
141
# Lazy import the Worker to avoid importing torch.cuda/xformers
142
# before CUDA_VISIBLE_DEVICES is set in the Worker
143
from vllm.worker.worker import Worker # pylint: disable=import-outside-toplevel
@@ -150,6 +152,7 @@ class LLMEngine:
152
scheduling_strategy=PlacementGroupSchedulingStrategy(
153
placement_group=placement_group,
154
placement_group_capture_child_tasks=True),
155
+ **ray_remote_kwargs,
156
)(RayWorker).remote()
157
self.workers.append(worker)
158
@@ -268,11 +271,11 @@ class LLMEngine:
271
# Add the sequence group to the scheduler.
272
self.scheduler.add_seq_group(seq_group)
273
271
- def abort_request(self, request_id: str) -> None:
272
- """Aborts a request with the given ID.
274
+ def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
275
+ """Aborts a request(s) with the given ID.
276
277
Args:
275
- request_id: The ID of the request to abort.
278
+ request_id: The ID(s) of the request to abort.
279
"""
280
self.scheduler.abort_seq_group(request_id)
281
@@ -288,35 +291,21 @@ class LLMEngine:
291
"""Returns True if there are unfinished requests."""
292
return self.scheduler.has_unfinished_seqs()
293
291
- def step(self) -> List[RequestOutput]:
292
- """Performs one decoding iteration and returns newly generated results.
293
-
294
- This function performs one decoding iteration of the engine. It first
295
- schedules the sequences to be executed in the next iteration and the
296
- token blocks to be swapped in/out/copy. Then, it executes the model
297
- and updates the scheduler with the model outputs. Finally, it decodes
298
- the sequences and returns the newly generated results.
299
- """
294
+ def _schedule(
295
+ self
296
+ ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
297
+ Optional[List[RequestOutput]]]:
298
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
299
if scheduler_outputs.is_empty():
302
- if not scheduler_outputs.ignored_seq_groups:
303
- # Nothing to do.
304
- return []
305
- # If there are ignored seq groups, we need to return them as the
306
- # request outputs.
307
- return [
300
+ return seq_group_metadata_list, scheduler_outputs, [
301
RequestOutput.from_seq_group(seq_group)
302
for seq_group in scheduler_outputs.ignored_seq_groups
303
]
304
+ return seq_group_metadata_list, scheduler_outputs, None
305
312
- # Execute the model.
313
- output = self._run_workers(
314
- "execute_model",
315
- seq_group_metadata_list=seq_group_metadata_list,
316
- blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
317
- blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
318
- blocks_to_copy=scheduler_outputs.blocks_to_copy,
319
- )
306
+ def _process_worker_outputs(
307
+ self, output,
308
+ scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]:
309
# Update the scheduler with the model outputs.
310
seq_groups = self.scheduler.update(output)
311
@@ -339,6 +328,31 @@ class LLMEngine:
328
scheduler_outputs.num_batched_tokens)
329
return request_outputs
330
331
+ def step(self) -> List[RequestOutput]:
332
+ """Performs one decoding iteration and returns newly generated results.
333
+
334
+ This function performs one decoding iteration of the engine. It first
335
+ schedules the sequences to be executed in the next iteration and the
336
+ token blocks to be swapped in/out/copy. Then, it executes the model
337
+ and updates the scheduler with the model outputs. Finally, it decodes
338
+ the sequences and returns the newly generated results.
339
+ """
340
+ (seq_group_metadata_list, scheduler_outputs,
341
+ early_return) = self._schedule()
342
+ if early_return is not None:
343
+ return early_return
344
+
345
+ # Execute the model.
346
+ output = self._run_workers(
347
+ "execute_model",
348
+ seq_group_metadata_list=seq_group_metadata_list,
349
+ blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
350
+ blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
351
+ blocks_to_copy=scheduler_outputs.blocks_to_copy,
352
+ )
353
+
354
+ return self._process_worker_outputs(output, scheduler_outputs)
355
+
356
def _log_system_stats(
357
self,
358
prompt_run: bool,