A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Refactor scheduler (#658)
Woosuk Kwon committed
Aug 2, 2023 at 16:42 UTC
55fe8a81ec5deab319c7c6b02913c21273b764ca
4 files changed
+205
-144
examples/llm_engine_example.py
+1
-1
@@ -28,7 +28,7 @@ def main(args: argparse.Namespace):
28
# Run the engine by calling `engine.step()` manually.
29
request_id = 0
30
while True:
31
- # To test iteration-level scheduling, we add one request at each step.
31
+ # To test continuous batching, we add one request at each step.
32
if test_prompts:
33
prompt, sampling_params = test_prompts.pop(0)
34
engine.add_request(str(request_id), prompt, sampling_params)
vllm/core/scheduler.py
+96
-129
@@ -12,8 +12,6 @@ from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
12
13
logger = init_logger(__name__)
14
15
-_LOGGING_INTERVAL_SEC = 5
16
-
15
16
class PreemptionMode(enum.Enum):
17
"""Preemption modes.
@@ -32,19 +30,28 @@ class SchedulerOutputs:
30
31
def __init__(
32
self,
33
+ scheduled_seq_groups: List[SequenceGroup],
34
+ prompt_run: bool,
35
+ num_batched_tokens: int,
36
blocks_to_swap_in: Dict[int, int],
37
blocks_to_swap_out: Dict[int, int],
38
blocks_to_copy: Dict[int, List[int]],
39
+ ignored_seq_groups: List[SequenceGroup],
40
) -> None:
41
+ self.scheduled_seq_groups = scheduled_seq_groups
42
+ self.prompt_run = prompt_run
43
+ self.num_batched_tokens = num_batched_tokens
44
self.blocks_to_swap_in = blocks_to_swap_in
45
self.blocks_to_swap_out = blocks_to_swap_out
46
self.blocks_to_copy = blocks_to_copy
47
# Swap in and swap out should never happen at the same time.
48
assert not (blocks_to_swap_in and blocks_to_swap_out)
49
+ self.ignored_seq_groups = ignored_seq_groups
50
51
def is_empty(self) -> bool:
46
- return (not self.blocks_to_swap_in and not self.blocks_to_swap_out
47
- and not self.blocks_to_copy)
52
+ # NOTE: We do not consider the ignored sequence groups.
53
+ return (not self.scheduled_seq_groups and not self.blocks_to_swap_in
54
+ and not self.blocks_to_swap_out and not self.blocks_to_copy)
55
56
57
class Scheduler:
@@ -53,11 +60,9 @@ class Scheduler:
60
self,
61
scheduler_config: SchedulerConfig,
62
cache_config: CacheConfig,
56
- log_stats: bool,
63
) -> None:
64
self.scheduler_config = scheduler_config
65
self.cache_config = cache_config
60
- self.log_stats = log_stats
66
67
# Instantiate the scheduling policy.
68
self.policy = PolicyFactory.get_policy(policy_name="fcfs")
@@ -75,10 +80,6 @@ class Scheduler:
80
# Sequence groups in the SWAPPED state.
81
self.swapped: List[SequenceGroup] = []
82
78
- self.last_logging_time: float = 0.0
79
- # List[timestamp, num_tokens]
80
- self.num_input_tokens: List[Tuple[float, int]] = []
81
-
83
def add_seq_group(self, seq_group: SequenceGroup) -> None:
84
# Add sequence groups to the waiting queue.
85
self.waiting.append(seq_group)
@@ -101,21 +102,80 @@ class Scheduler:
102
def get_num_unfinished_seq_groups(self) -> int:
103
return len(self.waiting) + len(self.running) + len(self.swapped)
104
104
- def _schedule(
105
- self) -> Tuple[SchedulerOutputs, List[str], List[SequenceGroup]]:
105
+ def _schedule(self) -> SchedulerOutputs:
106
# Blocks that need to be swaped or copied before model execution.
107
blocks_to_swap_in: Dict[int, int] = {}
108
blocks_to_swap_out: Dict[int, int] = {}
109
blocks_to_copy: Dict[int, List[int]] = {}
110
- ignored_seq_groups: List[SequenceGroup] = []
110
111
# Fix the current time.
112
now = time.time()
113
115
- # NOTE(woosuk): We prioritize the sequence groups in the RUNNING state
116
- # in order to minimize the preemption overheads.
117
- # Preemption happens only when there is no available slot to keep all
118
- # the sequence groups in the RUNNING state.
114
+ # Join waiting sequences if possible.
115
+ if not self.swapped:
116
+ ignored_seq_groups: List[SequenceGroup] = []
117
+ scheduled: List[SequenceGroup] = []
118
+ num_batched_tokens = 0
119
+ # Optimization: We do not sort the waiting queue since the preempted
120
+ # sequence groups are added to the front and the new sequence groups
121
+ # are added to the back.
122
+ while self.waiting:
123
+ seq_group = self.waiting[0]
124
+
125
+ num_prompt_tokens = seq_group.get_seqs()[0].get_len()
126
+ prompt_limit = min(
127
+ self.scheduler_config.max_model_len,
128
+ self.scheduler_config.max_num_batched_tokens)
129
+ if num_prompt_tokens > prompt_limit:
130
+ logger.warning(
131
+ f"Input prompt ({num_prompt_tokens} tokens) is too long"
132
+ f" and exceeds limit of {prompt_limit}")
133
+ for seq in seq_group.get_seqs():
134
+ seq.status = SequenceStatus.FINISHED_IGNORED
135
+ ignored_seq_groups.append(seq_group)
136
+ self.waiting.pop(0)
137
+ break
138
+
139
+ # If the sequence group cannot be allocated, stop.
140
+ if not self.block_manager.can_allocate(seq_group):
141
+ break
142
+
143
+ # If the number of batched tokens exceeds the limit, stop.
144
+ if (num_batched_tokens + num_prompt_tokens >
145
+ self.scheduler_config.max_num_batched_tokens):
146
+ break
147
+
148
+ # The total number of sequences in the RUNNING state should not
149
+ # exceed the maximum number of sequences.
150
+ num_new_seqs = seq_group.num_seqs(
151
+ status=SequenceStatus.WAITING)
152
+ num_curr_seqs = sum(
153
+ seq_group.num_seqs(status=SequenceStatus.RUNNING)
154
+ for seq_group in self.running)
155
+ if (num_curr_seqs + num_new_seqs >
156
+ self.scheduler_config.max_num_seqs):
157
+ break
158
+
159
+ seq_group = self.waiting.pop(0)
160
+ self._allocate(seq_group)
161
+ self.running.append(seq_group)
162
+ num_batched_tokens += num_prompt_tokens
163
+ scheduled.append(seq_group)
164
+
165
+ if scheduled:
166
+ scheduler_outputs = SchedulerOutputs(
167
+ scheduled_seq_groups=scheduled,
168
+ prompt_run=True,
169
+ num_batched_tokens=num_batched_tokens,
170
+ blocks_to_swap_in=blocks_to_swap_in,
171
+ blocks_to_swap_out=blocks_to_swap_out,
172
+ blocks_to_copy=blocks_to_copy,
173
+ ignored_seq_groups=ignored_seq_groups,
174
+ )
175
+ return scheduler_outputs
176
+
177
+ # NOTE(woosuk): Preemption happens only when there is no available slot
178
+ # to keep all the sequence groups in the RUNNING state.
179
# In this case, the policy is responsible for deciding which sequence
180
# groups to preempt.
181
self.running = self.policy.sort_by_priority(now, self.running)
@@ -173,124 +233,26 @@ class Scheduler:
233
seq_group.num_seqs(status=SequenceStatus.RUNNING)
234
for seq_group in self.running)
235
176
- # Join waiting sequences if possible.
177
- prompt_group_ids: List[str] = []
178
- # NOTE(woosuk): The sequence groups in the SWAPPED state are strictly
179
- # prioritized over the sequence groups in the WAITING state.
180
- # This is because we want to bound the amount of CPU memory taken by
181
- # the swapped sequence groups.
182
- if not self.swapped:
183
- # Optimization: We do not sort the waiting queue since the preempted
184
- # sequence groups are added to the front and the new sequence groups
185
- # are added to the back.
186
- while self.waiting:
187
- seq_group = self.waiting[0]
188
- # If the sequence group has been preempted in this step, stop.
189
- if seq_group in preempted:
190
- break
191
-
192
- num_prompt_tokens = seq_group.get_seqs()[0].get_len()
193
- prompt_limit = min(
194
- self.scheduler_config.max_model_len,
195
- self.scheduler_config.max_num_batched_tokens)
196
- if num_prompt_tokens > prompt_limit:
197
- logger.warning(
198
- f"Input prompt ({num_prompt_tokens} tokens) is too long"
199
- f" and exceeds limit of {prompt_limit}")
200
- for seq in seq_group.get_seqs():
201
- seq.status = SequenceStatus.FINISHED_IGNORED
202
- ignored_seq_groups.append(seq_group)
203
- self.waiting.pop(0)
204
- break
205
-
206
- # If the sequence group cannot be allocated, stop.
207
- if not self.block_manager.can_allocate(seq_group):
208
- break
209
-
210
- # If the number of batched tokens exceeds the limit, stop.
211
- if (num_batched_tokens + num_prompt_tokens >
212
- self.scheduler_config.max_num_batched_tokens):
213
- break
214
-
215
- # The total number of sequences in the RUNNING state should not
216
- # exceed the maximum number of sequences.
217
- num_new_seqs = seq_group.num_seqs(
218
- status=SequenceStatus.WAITING)
219
- num_curr_seqs = sum(
220
- seq_group.num_seqs(status=SequenceStatus.RUNNING)
221
- for seq_group in self.running)
222
- if (num_curr_seqs + num_new_seqs >
223
- self.scheduler_config.max_num_seqs):
224
- break
225
-
226
- seq_group = self.waiting.pop(0)
227
- self._allocate(seq_group)
228
- self.running.append(seq_group)
229
- num_batched_tokens += num_prompt_tokens
230
- prompt_group_ids.append(seq_group.request_id)
231
-
236
scheduler_outputs = SchedulerOutputs(
237
+ scheduled_seq_groups=self.running,
238
+ prompt_run=False,
239
+ num_batched_tokens=num_batched_tokens,
240
blocks_to_swap_in=blocks_to_swap_in,
241
blocks_to_swap_out=blocks_to_swap_out,
242
blocks_to_copy=blocks_to_copy,
243
+ ignored_seq_groups=[],
244
)
237
- if not self.log_stats:
238
- return scheduler_outputs, prompt_group_ids, ignored_seq_groups
245
+ return scheduler_outputs
246
240
- # TODO(woosuk): Move the below code to the engine.
241
- now = time.time()
242
- if num_batched_tokens > 0:
243
- self.num_input_tokens.append((now, num_batched_tokens))
244
- elapsed_time = now - self.last_logging_time
245
- if elapsed_time > _LOGGING_INTERVAL_SEC:
246
- self.last_logging_time = now
247
- self.num_input_tokens = [(t, n) for t, n in self.num_input_tokens
248
- if now - t < _LOGGING_INTERVAL_SEC]
249
- if len(self.num_input_tokens) > 1:
250
- total_num_tokens = sum(n
251
- for _, n in self.num_input_tokens[:-1])
252
- window = now - self.num_input_tokens[0][0]
253
- avg_throughput = total_num_tokens / window
254
- else:
255
- avg_throughput = 0.0
256
-
257
- total_num_gpu_blocks = self.cache_config.num_gpu_blocks
258
- num_free_gpu_blocks = self.block_manager.get_num_free_gpu_blocks()
259
- num_used_gpu_blocks = total_num_gpu_blocks - num_free_gpu_blocks
260
- gpu_cache_usage = num_used_gpu_blocks / total_num_gpu_blocks
261
-
262
- total_num_cpu_blocks = self.cache_config.num_cpu_blocks
263
- if total_num_cpu_blocks > 0:
264
- num_free_cpu_blocks = (
265
- self.block_manager.get_num_free_cpu_blocks())
266
- num_used_cpu_blocks = total_num_cpu_blocks - num_free_cpu_blocks
267
- cpu_cache_usage = num_used_cpu_blocks / total_num_cpu_blocks
268
- else:
269
- cpu_cache_usage = 0.0
270
-
271
- logger.info(f"Throughput: {avg_throughput:.1f} tokens/s, "
272
- f"Running: {len(self.running)} reqs, "
273
- f"Swapped: {len(self.swapped)} reqs, "
274
- f"Pending: {len(self.waiting)} reqs, "
275
- f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
276
- f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
277
- return scheduler_outputs, prompt_group_ids, ignored_seq_groups
278
-
279
- def schedule(
280
- self
281
- ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
282
- List[SequenceGroup]]:
247
+ def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
248
# Schedule sequence groups.
249
# This function call changes the internal states of the scheduler
250
# such as self.running, self.swapped, and self.waiting.
286
- (scheduler_outputs, prompt_group_ids,
287
- ignored_seq_groups) = self._schedule()
251
+ scheduler_outputs = self._schedule()
252
253
# Create input data structures.
254
seq_group_metadata_list: List[SequenceGroupMetadata] = []
291
- for seq_group in self.running:
292
- is_prompt = seq_group.request_id in prompt_group_ids
293
-
255
+ for seq_group in scheduler_outputs.scheduled_seq_groups:
256
seq_data: Dict[int, List[SequenceData]] = {}
257
block_tables: Dict[int, List[int]] = {}
258
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
@@ -300,20 +262,27 @@ class Scheduler:
262
263
seq_group_metadata = SequenceGroupMetadata(
264
request_id=seq_group.request_id,
303
- is_prompt=is_prompt,
265
+ is_prompt=scheduler_outputs.prompt_run,
266
seq_data=seq_data,
267
sampling_params=seq_group.sampling_params,
268
block_tables=block_tables,
269
)
270
seq_group_metadata_list.append(seq_group_metadata)
309
- return seq_group_metadata_list, scheduler_outputs, ignored_seq_groups
271
+ return seq_group_metadata_list, scheduler_outputs
272
273
def update(
274
self,
275
seq_outputs: Dict[int, SequenceOutputs],
276
) -> List[SequenceGroup]:
315
- # Update the running sequences and free blocks.
277
+ scheduled: List[SequenceGroup] = []
278
for seq_group in self.running:
279
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
280
+ if seq.seq_id in seq_outputs:
281
+ scheduled.append(seq_group)
282
+ break
283
+
284
+ # Update the scheduled sequences and free blocks.
285
+ for seq_group in scheduled:
286
# Process beam search results before processing the new tokens.
287
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
288
output = seq_outputs[seq.seq_id]
@@ -331,9 +300,7 @@ class Scheduler:
300
# Append a new token to the sequence.
301
output = seq_outputs[seq.seq_id]
302
seq.append_token_id(output.output_token, output.logprobs)
334
- # Return a shallow copy of the running queue to prevent the queue
335
- # from being modified by the caller.
336
- return self.running.copy()
303
+ return scheduled
304
305
def free_seq(self, seq: Sequence, finish_status: SequenceStatus) -> None:
306
seq.status = finish_status
vllm/engine/llm_engine.py
+91
-9
@@ -1,7 +1,7 @@
1
import time
2
import copy
3
from functools import partial
4
-from typing import Any, List, Optional, TYPE_CHECKING
4
+from typing import Any, List, Optional, Tuple, TYPE_CHECKING
5
6
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7
SchedulerConfig)
@@ -25,6 +25,8 @@ if TYPE_CHECKING:
25
26
logger = init_logger(__name__)
27
28
+_LOGGING_INTERVAL_SEC = 5
29
+
30
31
class LLMEngine:
32
"""An LLM engine that receives requests and generates texts.
@@ -102,7 +104,14 @@ class LLMEngine:
104
self._init_cache()
105
106
# Create the scheduler.
105
- self.scheduler = Scheduler(scheduler_config, cache_config, log_stats)
107
+ self.scheduler = Scheduler(scheduler_config, cache_config)
108
+
109
+ # Logging.
110
+ self.last_logging_time = 0.0
111
+ # List of (timestamp, num_tokens)
112
+ self.num_prompt_tokens: List[Tuple[float, int]] = []
113
+ # List of (timestamp, num_tokens)
114
+ self.num_generation_tokens: List[Tuple[float, int]] = []
115
116
def _init_workers(self, distributed_init_method: str):
117
# Lazy import the Worker to avoid importing torch.cuda/xformers
@@ -288,12 +297,17 @@ class LLMEngine:
297
and updates the scheduler with the model outputs. Finally, it decodes
298
the sequences and returns the newly generated results.
299
"""
291
- (seq_group_metadata_list, scheduler_outputs,
292
- ignored_seq_groups) = self.scheduler.schedule()
293
- if ((not seq_group_metadata_list) and scheduler_outputs.is_empty()
294
- and (not ignored_seq_groups)):
295
- # Nothing to do.
296
- return []
300
+ seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
301
+ if scheduler_outputs.is_empty():
302
+ if not scheduler_outputs.ignored_seq_groups:
303
+ # Nothing to do.
304
+ return []
305
+ # If there are ignored seq groups, we need to return them as the
306
+ # request outputs.
307
+ return [
308
+ RequestOutput.from_seq_group(seq_group)
309
+ for seq_group in scheduler_outputs.ignored_seq_groups
310
+ ]
311
312
# Execute the model.
313
output = self._run_workers(
@@ -315,11 +329,79 @@ class LLMEngine:
329
330
# Create the outputs.
331
request_outputs: List[RequestOutput] = []
318
- for seq_group in seq_groups + ignored_seq_groups:
332
+ for seq_group in seq_groups + scheduler_outputs.ignored_seq_groups:
333
request_output = RequestOutput.from_seq_group(seq_group)
334
request_outputs.append(request_output)
335
+
336
+ if self.log_stats:
337
+ # Log the system stats.
338
+ self._log_system_stats(scheduler_outputs.prompt_run,
339
+ scheduler_outputs.num_batched_tokens)
340
return request_outputs
341
342
+ def _log_system_stats(
343
+ self,
344
+ prompt_run: bool,
345
+ num_batched_tokens: int,
346
+ ) -> None:
347
+ now = time.time()
348
+ # Log the number of batched input tokens.
349
+ if prompt_run:
350
+ self.num_prompt_tokens.append((now, num_batched_tokens))
351
+ else:
352
+ self.num_generation_tokens.append((now, num_batched_tokens))
353
+
354
+ elapsed_time = now - self.last_logging_time
355
+ if elapsed_time < _LOGGING_INTERVAL_SEC:
356
+ return
357
+
358
+ # Discard the old stats.
359
+ self.num_prompt_tokens = [(t, n) for t, n in self.num_prompt_tokens
360
+ if now - t < _LOGGING_INTERVAL_SEC]
361
+ self.num_generation_tokens = [(t, n)
362
+ for t, n in self.num_generation_tokens
363
+ if now - t < _LOGGING_INTERVAL_SEC]
364
+
365
+ if len(self.num_prompt_tokens) > 1:
366
+ total_num_tokens = sum(n for _, n in self.num_prompt_tokens[:-1])
367
+ window = now - self.num_prompt_tokens[0][0]
368
+ avg_prompt_throughput = total_num_tokens / window
369
+ else:
370
+ avg_prompt_throughput = 0.0
371
+ if len(self.num_generation_tokens) > 1:
372
+ total_num_tokens = sum(n
373
+ for _, n in self.num_generation_tokens[:-1])
374
+ window = now - self.num_generation_tokens[0][0]
375
+ avg_generation_throughput = total_num_tokens / window
376
+ else:
377
+ avg_generation_throughput = 0.0
378
+
379
+ total_num_gpu_blocks = self.cache_config.num_gpu_blocks
380
+ num_free_gpu_blocks = (
381
+ self.scheduler.block_manager.get_num_free_gpu_blocks())
382
+ num_used_gpu_blocks = total_num_gpu_blocks - num_free_gpu_blocks
383
+ gpu_cache_usage = num_used_gpu_blocks / total_num_gpu_blocks
384
+
385
+ total_num_cpu_blocks = self.cache_config.num_cpu_blocks
386
+ if total_num_cpu_blocks > 0:
387
+ num_free_cpu_blocks = (
388
+ self.scheduler.block_manager.get_num_free_cpu_blocks())
389
+ num_used_cpu_blocks = total_num_cpu_blocks - num_free_cpu_blocks
390
+ cpu_cache_usage = num_used_cpu_blocks / total_num_cpu_blocks
391
+ else:
392
+ cpu_cache_usage = 0.0
393
+
394
+ logger.info("Avg prompt throughput: "
395
+ f"{avg_prompt_throughput:.1f} tokens/s, "
396
+ "Avg generation throughput: "
397
+ f"{avg_generation_throughput:.1f} tokens/s, "
398
+ f"Running: {len(self.scheduler.running)} reqs, "
399
+ f"Swapped: {len(self.scheduler.swapped)} reqs, "
400
+ f"Pending: {len(self.scheduler.waiting)} reqs, "
401
+ f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
402
+ f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
403
+ self.last_logging_time = now
404
+
405
def _decode_sequences(self, seq_groups: List[SequenceGroup]) -> None:
406
"""Decodes the sequence outputs."""
407
for seq_group in seq_groups:
vllm/model_executor/layers/attention.py
+17
-5
@@ -20,12 +20,20 @@ class PagedAttention(nn.Module):
20
"""GPT-style multi-head PagedAttention.
21
22
This class takes flattened 1D query, key, and value tensors as input. The
23
- input 1D tensors can be split into three parts: the prompt tokens, the
24
- generation tokens, and the paddings.
23
+ input 1D tensors can either contain prompt tokens or generation tokens, in
24
+ addition to paddings.
25
26
- |<------------------------------------- num_valid_tokens ------------------------------------->|
27
- |<--------------- num_prompt_tokens -------------->|<------- num_generation_tokens (M) ------->|
28
- |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|<--generation_0-->|...|<--generation_M-1-->|<--padding-->|
26
+ If the input tensors contain prompt tokens, the layout is as follows:
27
+
28
+ |<---------------------- num_valid_tokens ---------------------->|
29
+ |<--------------- num_prompt_tokens -------------->|
30
+ |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|<--padding-->|
31
+
32
+ Otherwise, the layout is as follows:
33
+
34
+ |<------------------ num_valid_tokens ------------------->|
35
+ |<------- num_generation_tokens (M) ------->|
36
+ |<--generation_0-->|...|<--generation_M-1-->|<--padding-->|
37
38
The prompts might have different lengths, while the generation tokens always
39
have length 1. The paddings are appended to make the input length a multiple
@@ -188,6 +196,8 @@ class PagedAttention(nn.Module):
196
# Compute the attention op for prompts.
197
num_prompt_tokens = input_metadata.num_prompt_tokens
198
if num_prompt_tokens > 0:
199
+ # Prompt run.
200
+ assert input_metadata.num_generation_tokens == 0
201
self.set_attn_bias(input_metadata)
202
self.multi_query_kv_attention(
203
output[:num_prompt_tokens],
@@ -217,6 +227,8 @@ class PagedAttention(nn.Module):
227
)
228
229
if input_metadata.num_generation_tokens > 0:
230
+ # Decoding run.
231
+ assert input_metadata.num_prompt_tokens == 0
232
assert key_cache is not None and value_cache is not None, (
233
"key_cache and value_cache must be provided when "
234
"generating tokens.")