@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ce741ba3e4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Refactor AsyncLLMEngine (#880)

Antoni Baum committed Sep 3, 2023 at 21:43 UTC ce741ba3e4fea00bacd2e1c609ca587ec35eb161
3 files changed +267 -145
vllm/core/scheduler.py
+9 -4
@@ -1,6 +1,6 @@
1 import enum
2 import time
3 -from typing import Dict, List, Optional, Tuple
3 +from typing import Dict, Iterable, List, Optional, Tuple, Union
4
5 from vllm.config import CacheConfig, SchedulerConfig
6 from vllm.core.block_manager import BlockSpaceManager
@@ -87,17 +87,22 @@ class Scheduler:
87 # Add sequence groups to the waiting queue.
88 self.waiting.append(seq_group)
89
90 - def abort_seq_group(self, request_id: str) -> None:
90 + def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
91 + if isinstance(request_id, str):
92 + request_id = (request_id, )
93 + request_ids = set(request_id)
94 for state_queue in [self.waiting, self.running, self.swapped]:
95 for seq_group in state_queue:
93 - if seq_group.request_id == request_id:
96 + if seq_group.request_id in request_ids:
97 # Remove the sequence group from the state queue.
98 state_queue.remove(seq_group)
99 for seq in seq_group.seqs:
100 if seq.is_finished():
101 continue
102 self.free_seq(seq, SequenceStatus.FINISHED_ABORTED)
100 - return
103 + request_ids.remove(seq_group.request_id)
104 + if not request_ids:
105 + return
106
107 def has_unfinished_seqs(self) -> bool:
108 return self.waiting or self.running or self.swapped
vllm/engine/async_llm_engine.py
+212 -109
@@ -1,6 +1,7 @@
1 import asyncio
2 import time
3 -from typing import Dict, List, Optional
3 +from functools import partial
4 +from typing import Any, Dict, Iterable, List, Optional, Set, Type, Union
5
6 from vllm.config import ModelConfig
7 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -12,7 +13,105 @@ from vllm.sampling_params import SamplingParams
13
14 logger = init_logger(__name__)
15
15 -TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
16 +
17 +class AsyncStream:
18 + """A stream of RequestOutputs for a request that can be
19 + iterated over asynchronously."""
20 +
21 + def __init__(self, request_id: str) -> None:
22 + self.request_id = request_id
23 + self._queue = asyncio.Queue()
24 + self._finished = False
25 +
26 + def put(self, item: RequestOutput) -> None:
27 + if self._finished:
28 + return
29 + self._queue.put_nowait(item)
30 +
31 + def finish(self) -> None:
32 + self._queue.put_nowait(StopIteration)
33 + self._finished = True
34 +
35 + @property
36 + def finished(self) -> bool:
37 + return self._finished
38 +
39 + def __aiter__(self):
40 + return self
41 +
42 + async def __anext__(self) -> RequestOutput:
43 + result = await self._queue.get()
44 + if result is StopIteration:
45 + raise StopAsyncIteration
46 + return result
47 +
48 +
49 +def _raise_exception_on_finish(task: asyncio.Task) -> None:
50 + try:
51 + task.result()
52 + except Exception as e:
53 + raise RuntimeError("Task finished unexpectedly.") from e
54 + raise RuntimeError("Task finished unexpectedly.")
55 +
56 +
57 +class _AsyncLLMEngine(LLMEngine):
58 + """Extension of LLMEngine to add async methods."""
59 +
60 + async def step_async(self) -> List[RequestOutput]:
61 + """Performs one decoding iteration and returns newly generated results.
62 + The workers are ran asynchronously if possible.
63 +
64 + This function performs one decoding iteration of the engine. It first
65 + schedules the sequences to be executed in the next iteration and the
66 + token blocks to be swapped in/out/copy. Then, it executes the model
67 + and updates the scheduler with the model outputs. Finally, it decodes
68 + the sequences and returns the newly generated results.
69 + """
70 + (seq_group_metadata_list, scheduler_outputs,
71 + early_return) = self._schedule()
72 + if early_return is not None:
73 + return early_return
74 +
75 + # Execute the model.
76 + output = await self._run_workers_async(
77 + "execute_model",
78 + seq_group_metadata_list=seq_group_metadata_list,
79 + blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
80 + blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
81 + blocks_to_copy=scheduler_outputs.blocks_to_copy,
82 + )
83 +
84 + return self._process_worker_outputs(output, scheduler_outputs)
85 +
86 + async def _run_workers_async(
87 + self,
88 + method: str,
89 + *args,
90 + get_all_outputs: bool = False,
91 + **kwargs,
92 + ) -> Any:
93 + """Runs the given method on all workers."""
94 + all_outputs = []
95 + for worker in self.workers:
96 + if self.parallel_config.worker_use_ray:
97 + executor = partial(worker.execute_method.remote, method)
98 + else:
99 + executor = getattr(worker, method)
100 +
101 + output = executor(*args, **kwargs)
102 + all_outputs.append(output)
103 +
104 + if self.parallel_config.worker_use_ray:
105 + all_outputs = await asyncio.gather(*all_outputs)
106 +
107 + if get_all_outputs:
108 + return all_outputs
109 +
110 + # Make sure all workers have the same results.
111 + output = all_outputs[0]
112 + for other_output in all_outputs[1:]:
113 + assert output == other_output
114 + return output
115
116
117 class AsyncLLMEngine:
@@ -37,49 +136,111 @@ class AsyncLLMEngine:
136 *args, *kwargs: Arguments for LLMEngine.
137 """
138
139 + _engine_class: Type[_AsyncLLMEngine] = _AsyncLLMEngine
140 +
141 def __init__(self,
142 worker_use_ray: bool,
143 engine_use_ray: bool,
144 *args,
145 log_requests: bool = True,
146 + start_engine_loop: bool = False,
147 **kwargs) -> None:
148 self.worker_use_ray = worker_use_ray
149 self.engine_use_ray = engine_use_ray
150 self.log_requests = log_requests
151 + self.engine = self._init_engine(*args, **kwargs)
152 +
153 + # Request id -> stream.
154 + self.request_streams: Dict[str, AsyncStream] = {}
155 + self.finished_requests: Set[str] = set()
156 + self.background_loop = None
157 + if start_engine_loop:
158 + self._start_background_loop()
159 +
160 + def _start_background_loop(self) -> None:
161 + """Start the background loop."""
162 + if self.background_loop is not None:
163 + raise RuntimeError("Background loop is already running.")
164 + self.background_loop = asyncio.get_event_loop().create_task(
165 + self.run_engine_loop())
166 + self.background_loop.add_done_callback(_raise_exception_on_finish)
167 +
168 + def _init_engine(self, *args,
169 + **kwargs) -> Union[_AsyncLLMEngine, "ray.ObjectRef"]:
170 if not self.engine_use_ray:
50 - engine_class = LLMEngine
171 + engine_class = self._engine_class
172 elif self.worker_use_ray:
52 - engine_class = ray.remote(num_cpus=0)(LLMEngine).remote
173 + engine_class = ray.remote(num_cpus=0)(self._engine_class).remote
174 else:
54 - engine_class = ray.remote(num_gpus=1)(LLMEngine).remote
55 - self.engine = engine_class(*args, **kwargs)
56 - # Request id -> request output.
57 - self.request_outputs: Dict[str, RequestOutput] = {}
58 - # Request id -> event to notify that there is new output.
59 - self.request_events: Dict[str, asyncio.Event] = {}
60 - self.is_engine_running = False
61 - self.kicking_request_id: Optional[str] = None
62 -
63 - async def engine_step(self, kicking_request_id: Optional[str] = None):
175 + engine_class = ray.remote(num_gpus=1)(self._engine_class).remote
176 + return engine_class(*args, **kwargs)
177 +
178 + async def engine_step(self):
179 """Kick the engine to process the waiting requests."""
65 - self.is_engine_running = True
66 - self.kicking_request_id = kicking_request_id
180 if self.engine_use_ray:
181 request_outputs = await self.engine.step.remote()
182 else:
70 - # Yield to the event loop to allow other coroutines to run
71 - # while is_engine_running is True. This let the engine to add new
72 - # requests into the queue.
73 - await asyncio.sleep(0)
74 - request_outputs = self.engine.step()
75 - self.is_engine_running = False
76 - self.kicking_request_id = None
183 + request_outputs = await self.engine.step_async()
184
78 - # Notify the waiting coroutines that there are new outputs ready.
185 + # Put the outputs into the corresponding streams.
186 for request_output in request_outputs:
187 request_id = request_output.request_id
81 - self.request_outputs[request_id] = request_output
82 - self.request_events[request_id].set()
188 + self.request_streams[request_id].put(request_output)
189 + if request_output.finished:
190 + if self.log_requests:
191 + logger.info(f"Finished request {request_id}.")
192 + self.request_streams[request_id].finish()
193 + self.finished_requests.add(request_id)
194 +
195 + await self._engine_abort(self.finished_requests)
196 + for request_id in self.finished_requests:
197 + del self.request_streams[request_id]
198 + self.finished_requests.clear()
199 +
200 + async def _engine_abort(self, request_ids: Iterable[str]):
201 + if self.engine_use_ray:
202 + await self.engine.abort_request.remote(request_ids)
203 + else:
204 + self.engine.abort_request(request_ids)
205 +
206 + async def run_engine_loop(self):
207 + while True:
208 + await self.engine_step()
209 + await asyncio.sleep(0)
210 +
211 + async def add_request(
212 + self,
213 + request_id: str,
214 + prompt: Optional[str],
215 + sampling_params: SamplingParams,
216 + prompt_token_ids: Optional[List[int]] = None,
217 + arrival_time: Optional[float] = None,
218 + ) -> AsyncStream:
219 + if self.log_requests:
220 + logger.info(f"Received request {request_id}: "
221 + f"prompt: {prompt!r}, "
222 + f"sampling params: {sampling_params}, "
223 + f"prompt token ids: {prompt_token_ids}.")
224 +
225 + stream = AsyncStream(request_id)
226 + self.request_streams[request_id] = stream
227 +
228 + # Add the request into the vLLM engine's waiting queue.
229 + if self.engine_use_ray:
230 + await self.engine.add_request.remote(
231 + request_id,
232 + prompt,
233 + sampling_params,
234 + prompt_token_ids=prompt_token_ids,
235 + arrival_time=arrival_time)
236 + else:
237 + self.engine.add_request(request_id,
238 + prompt,
239 + sampling_params,
240 + prompt_token_ids=prompt_token_ids,
241 + arrival_time=arrival_time)
242 +
243 + return stream
244
245 async def generate(
246 self,
@@ -108,78 +269,32 @@ class AsyncLLMEngine:
269 # Preprocess the request.
270 arrival_time = time.time()
271
111 - # Create an event to notify us that there is new output from the
112 - # vLLM engine.
113 - request_event = asyncio.Event()
114 - self.request_events[request_id] = request_event
272 + try:
273 + stream = await self.add_request(request_id,
274 + prompt,
275 + sampling_params,
276 + prompt_token_ids=prompt_token_ids,
277 + arrival_time=arrival_time)
278
116 - if self.log_requests:
117 - logger.info(f"Received request {request_id}: "
118 - f"prompt: {prompt!r}, "
119 - f"sampling params: {sampling_params}, "
120 - f"prompt token ids: {prompt_token_ids}.")
279 + async for request_output in stream:
280 + yield request_output
281 + except Exception as e:
282 + # If there is an exception, abort the request.
283 + self._abort(request_id)
284 + raise e
285
122 - # Add the request into the vLLM engine's waiting queue.
123 - if self.engine_use_ray:
124 - await self.engine.add_request.remote(
125 - request_id,
126 - prompt,
127 - sampling_params,
128 - prompt_token_ids=prompt_token_ids,
129 - arrival_time=arrival_time)
130 - else:
131 - self.engine.add_request(request_id,
132 - prompt,
133 - sampling_params,
134 - prompt_token_ids=prompt_token_ids,
135 - arrival_time=arrival_time)
286 + async def abort(self, request_id: str) -> None:
287 + """Abort a request.
288
137 - # The vLLM engine does not have a background loop that keeps
138 - # processing incoming requests. Therefore, we need to keep kicking
139 - # the engine to process the requests.
140 - while True:
141 - if request_id not in self.request_events:
142 - # The request has been aborted.
143 - return
144 -
145 - # Kick the engine if the engine is not running.
146 - if not self.is_engine_running:
147 - try:
148 - await self.engine_step(request_id)
149 - except RuntimeError as e:
150 - await self.abort(request_id)
151 - raise e
152 -
153 - # Wait for new output. The group_event will be set in engine_step
154 - # when there is new output available for the sequence group.
155 - # Added a timeout to prevent deadlock.
156 - try:
157 - await asyncio.wait_for(request_event.wait(),
158 - timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
159 - except asyncio.TimeoutError:
160 - continue
161 - # Reset the event to wait for the next output.
162 - request_event.clear()
163 -
164 - # Decode and return new outputs.
165 - request_output = self.request_outputs[request_id]
166 - yield request_output
167 -
168 - # Once finished, release the resources of the sequence group.
169 - if request_output.finished:
170 - if self.log_requests:
171 - logger.info(f"Finished request {request_id}.")
289 + Abort a submitted request. If the request is finished or not found,
290 + this method will be a no-op.
291
173 - del self.request_outputs[request_id]
174 - del self.request_events[request_id]
175 - # Kick the engine if the engine is not running. This is to
176 - # prevent that there are still requests in engine's waiting
177 - # queue to be executed.
178 - if not self.is_engine_running:
179 - await self.engine_step()
180 - break
292 + Args:
293 + request_id: The unique id of the request.
294 + """
295 + return self._abort(request_id)
296
182 - async def abort(self, request_id: str) -> None:
297 + def _abort(self, request_id: str) -> None:
298 """Abort a request.
299
300 Abort a submitted request. If the request is finished or not found,
@@ -188,28 +303,16 @@ class AsyncLLMEngine:
303 Args:
304 request_id: The unique id of the request.
305 """
191 - if request_id not in self.request_events:
306 + if request_id not in self.request_streams or self.request_streams[
307 + request_id].finished:
308 # The request has already finished or been aborted.
309 return
310
311 if self.log_requests:
312 logger.info(f"Aborted request {request_id}.")
313
198 - if self.engine_use_ray:
199 - await self.engine.abort_request.remote(request_id)
200 - else:
201 - self.engine.abort_request(request_id)
202 -
203 - if request_id in self.request_events:
204 - del self.request_events[request_id]
205 - if request_id in self.request_outputs:
206 - del self.request_outputs[request_id]
207 -
208 - # To prevent deadlock when a request is aborted while the engine is
209 - # running.
210 - if self.kicking_request_id == request_id:
211 - self.is_engine_running = False
212 - self.kicking_request_id = None
314 + self.request_streams[request_id].finish()
315 + self.finished_requests.add(request_id)
316
317 async def get_model_config(self) -> ModelConfig:
318 """Get the model configuration of the vLLM engine."""
vllm/engine/llm_engine.py
+46 -32
@@ -1,17 +1,18 @@
1 -import time
1 import copy
2 +import time
3 from functools import partial
4 -from typing import Any, List, Optional, Tuple, TYPE_CHECKING
4 +from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union
5
6 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7 SchedulerConfig)
8 -from vllm.core.scheduler import Scheduler
8 +from vllm.core.scheduler import Scheduler, SchedulerOutputs
9 from vllm.engine.arg_utils import EngineArgs
10 -from vllm.engine.ray_utils import initialize_cluster, ray, RayWorker
10 +from vllm.engine.ray_utils import RayWorker, initialize_cluster, ray
11 from vllm.logger import init_logger
12 from vllm.outputs import RequestOutput
13 from vllm.sampling_params import SamplingParams
14 -from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
14 +from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupMetadata,
15 + SequenceStatus)
16 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
17 get_tokenizer)
18 from vllm.utils import Counter
@@ -135,7 +136,8 @@ class LLMEngine:
136 get_all_outputs=True,
137 )
138
138 - def _init_workers_ray(self, placement_group: "PlacementGroup"):
139 + def _init_workers_ray(self, placement_group: "PlacementGroup",
140 + **ray_remote_kwargs):
141 # Lazy import the Worker to avoid importing torch.cuda/xformers
142 # before CUDA_VISIBLE_DEVICES is set in the Worker
143 from vllm.worker.worker import Worker # pylint: disable=import-outside-toplevel
@@ -150,6 +152,7 @@ class LLMEngine:
152 scheduling_strategy=PlacementGroupSchedulingStrategy(
153 placement_group=placement_group,
154 placement_group_capture_child_tasks=True),
155 + **ray_remote_kwargs,
156 )(RayWorker).remote()
157 self.workers.append(worker)
158
@@ -268,11 +271,11 @@ class LLMEngine:
271 # Add the sequence group to the scheduler.
272 self.scheduler.add_seq_group(seq_group)
273
271 - def abort_request(self, request_id: str) -> None:
272 - """Aborts a request with the given ID.
274 + def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
275 + """Aborts a request(s) with the given ID.
276
277 Args:
275 - request_id: The ID of the request to abort.
278 + request_id: The ID(s) of the request to abort.
279 """
280 self.scheduler.abort_seq_group(request_id)
281
@@ -288,35 +291,21 @@ class LLMEngine:
291 """Returns True if there are unfinished requests."""
292 return self.scheduler.has_unfinished_seqs()
293
291 - def step(self) -> List[RequestOutput]:
292 - """Performs one decoding iteration and returns newly generated results.
293 -
294 - This function performs one decoding iteration of the engine. It first
295 - schedules the sequences to be executed in the next iteration and the
296 - token blocks to be swapped in/out/copy. Then, it executes the model
297 - and updates the scheduler with the model outputs. Finally, it decodes
298 - the sequences and returns the newly generated results.
299 - """
294 + def _schedule(
295 + self
296 + ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
297 + Optional[List[RequestOutput]]]:
298 seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
299 if scheduler_outputs.is_empty():
302 - if not scheduler_outputs.ignored_seq_groups:
303 - # Nothing to do.
304 - return []
305 - # If there are ignored seq groups, we need to return them as the
306 - # request outputs.
307 - return [
300 + return seq_group_metadata_list, scheduler_outputs, [
301 RequestOutput.from_seq_group(seq_group)
302 for seq_group in scheduler_outputs.ignored_seq_groups
303 ]
304 + return seq_group_metadata_list, scheduler_outputs, None
305
312 - # Execute the model.
313 - output = self._run_workers(
314 - "execute_model",
315 - seq_group_metadata_list=seq_group_metadata_list,
316 - blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
317 - blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
318 - blocks_to_copy=scheduler_outputs.blocks_to_copy,
319 - )
306 + def _process_worker_outputs(
307 + self, output,
308 + scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]:
309 # Update the scheduler with the model outputs.
310 seq_groups = self.scheduler.update(output)
311
@@ -339,6 +328,31 @@ class LLMEngine:
328 scheduler_outputs.num_batched_tokens)
329 return request_outputs
330
331 + def step(self) -> List[RequestOutput]:
332 + """Performs one decoding iteration and returns newly generated results.
333 +
334 + This function performs one decoding iteration of the engine. It first
335 + schedules the sequences to be executed in the next iteration and the
336 + token blocks to be swapped in/out/copy. Then, it executes the model
337 + and updates the scheduler with the model outputs. Finally, it decodes
338 + the sequences and returns the newly generated results.
339 + """
340 + (seq_group_metadata_list, scheduler_outputs,
341 + early_return) = self._schedule()
342 + if early_return is not None:
343 + return early_return
344 +
345 + # Execute the model.
346 + output = self._run_workers(
347 + "execute_model",
348 + seq_group_metadata_list=seq_group_metadata_list,
349 + blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
350 + blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
351 + blocks_to_copy=scheduler_outputs.blocks_to_copy,
352 + )
353 +
354 + return self._process_worker_outputs(output, scheduler_outputs)
355 +
356 def _log_system_stats(
357 self,
358 prompt_run: bool,