A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[2/N] Chunked prefill data update (#3538)
SangBin Cho committed
Mar 29, 2024 at 02:06 UTC
b51c1cc9d2f223cfa3aef1426bced10dfde28dbb
11 files changed
+272
-76
benchmarks/benchmark_latency.py
+13
-1
@@ -26,7 +26,9 @@ def main(args: argparse.Namespace):
26
kv_cache_dtype=args.kv_cache_dtype,
27
device=args.device,
28
ray_workers_use_nsight=args.ray_workers_use_nsight,
29
- download_dir=args.download_dir)
29
+ enable_chunked_prefill=args.enable_chunked_prefill,
30
+ download_dir=args.download_dir,
31
+ block_size=args.block_size)
32
33
sampling_params = SamplingParams(
34
n=args.n,
@@ -145,6 +147,16 @@ if __name__ == '__main__':
147
default="cuda",
148
choices=["cuda"],
149
help='device type for vLLM execution, supporting CUDA only currently.')
150
+ parser.add_argument('--block-size',
151
+ type=int,
152
+ default=16,
153
+ help='block size of key/value cache')
154
+ parser.add_argument(
155
+ '--enable-chunked-prefill',
156
+ type=bool,
157
+ default=False,
158
+ help='If True, the prefill requests can be chunked based on the '
159
+ 'max_num_batched_tokens')
160
parser.add_argument(
161
"--ray-workers-use-nsight",
162
action='store_true',
tests/conftest.py
+4
@@ -256,6 +256,8 @@ class VllmRunner:
256
dtype: str = "half",
257
disable_log_stats: bool = True,
258
tensor_parallel_size: int = 1,
259
+ block_size: int = 16,
260
+ enable_chunked_prefill: bool = False,
261
**kwargs,
262
) -> None:
263
self.model = LLM(
@@ -266,6 +268,8 @@ class VllmRunner:
268
swap_space=0,
269
disable_log_stats=disable_log_stats,
270
tensor_parallel_size=tensor_parallel_size,
271
+ block_size=block_size,
272
+ enable_chunked_prefill=enable_chunked_prefill,
273
**kwargs,
274
)
275
tests/core/test_scheduler.py
+13
-9
@@ -10,6 +10,10 @@ from vllm.sequence import Logprob, SequenceGroup
10
from .utils import create_dummy_prompt
11
12
13
+def get_sequence_groups(scheduler_output):
14
+ return [s.seq_group for s in scheduler_output.scheduled_seq_groups]
15
+
16
+
17
def test_scheduler_add_seq_group():
18
block_size = 4
19
scheduler_config = SchedulerConfig(100, 64, 1)
@@ -57,9 +61,9 @@ def test_scheduler_schedule_simple():
61
cache_config.num_cpu_blocks = 8
62
cache_config.num_gpu_blocks = 8
63
scheduler = Scheduler(scheduler_config, cache_config, None)
64
+ running: List[SequenceGroup] = []
65
66
# Add seq groups to scheduler.
62
- running: List[SequenceGroup] = []
67
for i in range(num_seq_group):
68
_, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
69
scheduler.add_seq_group(seq_group)
@@ -68,7 +72,7 @@ def test_scheduler_schedule_simple():
72
# Schedule seq groups prompts.
73
num_tokens = block_size * num_seq_group
74
seq_group_meta, out = scheduler.schedule()
71
- assert set(out.scheduled_seq_groups) == set(running)
75
+ assert set(get_sequence_groups(out)) == set(running)
76
assert out.num_batched_tokens == num_tokens
77
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
78
and not out.blocks_to_swap_out)
@@ -76,7 +80,7 @@ def test_scheduler_schedule_simple():
80
81
# Schedule seq groups generation.
82
seq_group_meta, out = scheduler.schedule()
79
- assert set(out.scheduled_seq_groups) == set(running)
83
+ assert set(get_sequence_groups(out)) == set(running)
84
assert out.num_batched_tokens == num_seq_group
85
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
86
and not out.blocks_to_swap_out)
@@ -100,7 +104,7 @@ def test_scheduler_schedule_preempt_abort():
104
105
# Schedule seq groups prompts.
106
seq_group_meta, out = scheduler.schedule()
103
- assert out.scheduled_seq_groups == [seq_group_a, seq_group_b]
107
+ assert get_sequence_groups(out) == [seq_group_a, seq_group_b]
108
assert out.num_batched_tokens == block_size * 2 # seq_a and seq_b
109
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
110
and not out.blocks_to_swap_out)
@@ -115,7 +119,7 @@ def test_scheduler_schedule_preempt_abort():
119
120
# Schedule seq groups generation and preempt seq group b.
121
seq_group_meta, out = scheduler.schedule()
118
- assert out.scheduled_seq_groups == [seq_group_a]
122
+ assert get_sequence_groups(out) == [seq_group_a]
123
assert out.num_batched_tokens == 1
124
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
125
and not out.blocks_to_swap_out)
@@ -125,7 +129,7 @@ def test_scheduler_schedule_preempt_abort():
129
# Abort seq group a. Re-schedule seq group b prompt with recomputation.
130
scheduler.abort_seq_group("1")
131
seq_group_meta, out = scheduler.schedule()
128
- assert out.scheduled_seq_groups == [seq_group_b]
132
+ assert get_sequence_groups(out) == [seq_group_b]
133
assert out.num_batched_tokens == 5 # 4 prompt + 1 generation.
134
assert (not out.blocks_to_copy and not out.blocks_to_swap_in
135
and not out.blocks_to_swap_out)
@@ -155,11 +159,11 @@ def test_scheduler_max_seqs():
159
160
# Schedule seq groups prompts.
161
_, out = scheduler.schedule()
158
- assert set(out.scheduled_seq_groups) == set([all_seq_groups[0]])
162
+ assert set(get_sequence_groups(out)) == set([all_seq_groups[0]])
163
164
# Schedule seq groups generation.
165
_, out = scheduler.schedule()
162
- assert set(out.scheduled_seq_groups) == set([all_seq_groups[0]])
166
+ assert set(get_sequence_groups(out)) == set([all_seq_groups[0]])
167
168
# Append 2 more seq group
169
scheduler.add_seq_group(all_seq_groups[1])
@@ -169,7 +173,7 @@ def test_scheduler_max_seqs():
173
# Only 1 seq group should be scheduled since max_seq_group is 2
174
# and one is prompting.
175
_, out = scheduler.schedule()
172
- assert set(out.scheduled_seq_groups) == set([all_seq_groups[1]])
176
+ assert set(get_sequence_groups(out)) == set([all_seq_groups[1]])
177
178
179
def test_scheduler_delay_factor():
tests/test_sequence.py
+23
-1
@@ -1,6 +1,7 @@
1
import pytest
2
3
-from vllm.sequence import SamplerOutput, SequenceGroupOutput, SequenceOutput
3
+from vllm.sequence import (SamplerOutput, SequenceData, SequenceGroupOutput,
4
+ SequenceOutput)
5
6
7
@pytest.fixture
@@ -48,3 +49,24 @@ def test_sampler_output_eq(sample_outputs):
49
sampler_output3 = SamplerOutput(outputs=sample_outputs[:-1])
50
assert sampler_output1 == sampler_output2
51
assert sampler_output1 != sampler_output3
52
+
53
+
54
+def test_sequence_data_prefill():
55
+ seq_data = SequenceData(prompt_token_ids=[1, 2, 3, 4])
56
+ assert seq_data.get_num_uncomputed_tokens() == 4
57
+ assert seq_data.get_num_computed_tokens() == 0
58
+ # advance by 2
59
+ seq_data.update_num_computed_tokens(2)
60
+ assert seq_data.get_num_uncomputed_tokens() == 2
61
+ assert seq_data.get_num_computed_tokens() == 2
62
+
63
+ # advance by 1
64
+ seq_data.update_num_computed_tokens(1)
65
+ assert seq_data.get_num_uncomputed_tokens() == 1
66
+ assert seq_data.get_num_computed_tokens() == 3
67
+
68
+ # append tokens and reset, simulating recompute
69
+ seq_data.append_token_id(1, logprob=0.0)
70
+ seq_data.reset_num_computed_tokens()
71
+ assert seq_data.get_num_uncomputed_tokens() == 5
72
+ assert seq_data.get_num_computed_tokens() == 0
tests/worker/test_model_runner.py
+20
-17
@@ -18,15 +18,16 @@ def test_prepare_prompt(batch_size):
18
# make sure all tokens fit into one block
19
prompt_len = i % (model_runner.block_size - 1) + 1
20
prompt_lens.append(prompt_len)
21
- seq_data = list(range(prompt_len))
22
- seq_group_metadata_list.append(
23
- SequenceGroupMetadata(
24
- request_id=f"test_{i}",
25
- is_prompt=True,
26
- seq_data={0: SequenceData(seq_data)},
27
- sampling_params=SamplingParams(temperature=0),
28
- block_tables=block_tables,
29
- ))
21
+ seq_data = SequenceData(list(range(prompt_len)))
22
+ seq_group_metadata = SequenceGroupMetadata(
23
+ request_id=f"test_{i}",
24
+ is_prompt=True,
25
+ seq_data={0: seq_data},
26
+ sampling_params=SamplingParams(temperature=0),
27
+ block_tables=block_tables,
28
+ )
29
+ assert seq_group_metadata.token_chunk_size == seq_data.get_len()
30
+ seq_group_metadata_list.append(seq_group_metadata)
31
32
expected_selected_token_indices = []
33
selected_token_start_idx = 0
@@ -131,14 +132,16 @@ def test_prepare_decode_cuda_graph(batch_size):
132
prompt_len = i % (model_runner.block_size - 1) + 1
133
prompt_lens.append(prompt_len)
134
seq_data = list(range(prompt_len))
134
- seq_group_metadata_list.append(
135
- SequenceGroupMetadata(
136
- request_id=f"test_{i}",
137
- is_prompt=False,
138
- seq_data={0: SequenceData(seq_data)},
139
- sampling_params=SamplingParams(temperature=0),
140
- block_tables={0: [1]},
141
- ))
135
+ seq_data = SequenceData(seq_data)
136
+ seq_group_metadata = SequenceGroupMetadata(
137
+ request_id=f"test_{i}",
138
+ is_prompt=False,
139
+ seq_data={0: seq_data},
140
+ sampling_params=SamplingParams(temperature=0),
141
+ block_tables={0: [1]},
142
+ )
143
+ assert seq_group_metadata.token_chunk_size == 1
144
+ seq_group_metadata_list.append(seq_group_metadata)
145
146
input_tokens, input_positions, attn_metadata, _, _, _ = (
147
model_runner._prepare_decode(seq_group_metadata_list))
vllm/config.py
+4
@@ -533,6 +533,8 @@ class SchedulerConfig:
533
delay_factor: Apply a delay (of delay factor multiplied by previous
534
prompt latency) before scheduling next prompt.
535
use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
536
+ enable_chunked_prefill: If True, prefill requests can be chunked based
537
+ on the remaining max_num_batched_tokens.
538
"""
539
540
def __init__(
@@ -542,6 +544,7 @@ class SchedulerConfig:
544
max_model_len: int,
545
use_v2_block_manager: bool = False,
546
delay_factor: float = 0.0,
547
+ enable_chunked_prefill: bool = False,
548
) -> None:
549
if max_num_batched_tokens is not None:
550
self.max_num_batched_tokens = max_num_batched_tokens
@@ -553,6 +556,7 @@ class SchedulerConfig:
556
self.max_model_len = max_model_len
557
self.delay_factor = delay_factor
558
self.use_v2_block_manager = use_v2_block_manager
559
+ self.chunked_prefill_enabled = enable_chunked_prefill
560
self._verify_args()
561
562
def _verify_args(self) -> None:
vllm/core/scheduler.py
+81
-27
@@ -1,6 +1,7 @@
1
import enum
2
import time
3
from collections import deque
4
+from dataclasses import dataclass
5
from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
6
7
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
@@ -27,11 +28,24 @@ class PreemptionMode(enum.Enum):
28
RECOMPUTE = enum.auto()
29
30
31
+# seq_group: SequenceGroup to schedule.
32
+# token_chunk_size: The number of prefill tokens to be processed in the next
33
+# step.
34
+@dataclass
35
+class ScheduledSequenceGroup:
36
+ # A sequence group that's scheduled.
37
+ seq_group: SequenceGroup
38
+ # The total chunk size (number of tokens) to process for next iteration.
39
+ # 1 for decoding. Same as prompt tokens for prefill, but if prefill is
40
+ # chunked, it can be smaller than that.
41
+ token_chunk_size: int
42
+
43
+
44
class SchedulerOutputs:
45
46
def __init__(
47
self,
34
- scheduled_seq_groups: Iterable[SequenceGroup],
48
+ scheduled_seq_groups: Iterable[ScheduledSequenceGroup],
49
prompt_run: bool,
50
num_batched_tokens: int,
51
blocks_to_swap_in: Dict[int, int],
@@ -39,17 +53,41 @@ class SchedulerOutputs:
53
blocks_to_copy: Dict[int, List[int]],
54
ignored_seq_groups: List[SequenceGroup],
55
) -> None:
42
- self.scheduled_seq_groups = scheduled_seq_groups
43
- self.prompt_run = prompt_run
44
- self.num_batched_tokens = num_batched_tokens
45
- self.blocks_to_swap_in = blocks_to_swap_in
46
- self.blocks_to_swap_out = blocks_to_swap_out
47
- self.blocks_to_copy = blocks_to_copy
56
+ """A list of sequence groups to be scheduled as a single batch.
57
+
58
+ Args:
59
+ scheduled_seq_groups: A tuple of scheduled sequence group and its
60
+ token chunk size.
61
+ prompt_run: True if all sequence groups are in prefill phase.
62
+ If False, all sequence groups are in decoding phase.
63
+ num_batched_tokens: Total number of batched tokens.
64
+ blocks_to_swap_in: Blocks to swap in. Dict of CPU -> GPU block
65
+ number.
66
+ blocks_to_swap_out: Blocks to swap out. Dict of GPU -> CPU block
67
+ number.
68
+ blocks_to_copy: Blocks to copy. Source to a list of dest blocks.
69
+ ignored_seq_groups: Sequence groups that are going to be ignored.
70
+ """
71
+ # A tuple of scheduled sequence group and its chunk size.
72
+ self.scheduled_seq_groups: ScheduledSequenceGroup = scheduled_seq_groups
73
+ # True if all sequence groups are in prefill phase. If False, all
74
+ # sequence groups are in decoding phase.
75
+ self.prompt_run: bool = prompt_run
76
+ # Total number of batched tokens.
77
+ self.num_batched_tokens: int = num_batched_tokens
78
+ # Blocks to swap in. Dict of CPU -> GPU block number.
79
+ self.blocks_to_swap_in: Dict[int, int] = blocks_to_swap_in
80
+ # Blocks to swap out. Dict of GPU -> CPU block number.
81
+ self.blocks_to_swap_out: Dict[int, int] = blocks_to_swap_out
82
+ # Blocks to copy. Source to a list of dest blocks.
83
+ self.blocks_to_copy: Dict[int, List[int]] = blocks_to_copy
84
+ # Sequence groups that are going to be ignored.
85
+ self.ignored_seq_groups: List[SequenceGroup] = ignored_seq_groups
86
+
87
# Swap in and swap out should never happen at the same time.
88
assert not (blocks_to_swap_in and blocks_to_swap_out)
50
- self.ignored_seq_groups = ignored_seq_groups
89
52
- self.num_loras = len(self.lora_requests)
90
+ self.num_loras: int = len(self.lora_requests)
91
if self.num_loras > 0:
92
self._sort_by_lora_ids()
93
@@ -59,13 +97,13 @@ class SchedulerOutputs:
97
and not self.blocks_to_swap_out and not self.blocks_to_copy)
98
99
def _sort_by_lora_ids(self) -> bool:
62
- self.scheduled_seq_groups = sorted(self.scheduled_seq_groups,
63
- key=lambda g:
64
- (g.lora_int_id, g.request_id))
100
+ self.scheduled_seq_groups = sorted(
101
+ self.scheduled_seq_groups,
102
+ key=lambda g: (g.seq_group.lora_int_id, g.seq_group.request_id))
103
104
@property
105
def lora_requests(self) -> Set[LoRARequest]:
68
- return {g.lora_request for g in self.scheduled_seq_groups}
106
+ return {g.seq_group.lora_request for g in self.scheduled_seq_groups}
107
108
109
class Scheduler:
@@ -198,11 +236,13 @@ class Scheduler:
236
assert len(waiting_seqs) == 1, (
237
"Waiting sequence group should have only one prompt "
238
"sequence.")
201
- num_prompt_tokens = waiting_seqs[0].get_len()
202
- if num_prompt_tokens > self.prompt_limit:
239
+ # get_len includes output tokens if the request has been
240
+ # preempted.
241
+ num_prefill_tokens = waiting_seqs[0].get_len()
242
+ if num_prefill_tokens > self.prompt_limit:
243
logger.warning(
204
- f"Input prompt ({num_prompt_tokens} tokens) is too long"
205
- f" and exceeds limit of {self.prompt_limit}")
244
+ f"Input prompt ({num_prefill_tokens} tokens) is too "
245
+ f"long and exceeds limit of {self.prompt_limit}")
246
for seq in waiting_seqs:
247
seq.status = SequenceStatus.FINISHED_IGNORED
248
ignored_seq_groups.append(seq_group)
@@ -215,8 +255,8 @@ class Scheduler:
255
break
256
elif can_allocate == AllocStatus.NEVER:
257
logger.warning(
218
- f"Input prompt ({num_prompt_tokens} tokens) is too long"
219
- f" and exceeds the capacity of block_manager")
258
+ f"Input prompt ({num_prefill_tokens} tokens) is too "
259
+ f"long and exceeds the capacity of block_manager")
260
for seq in waiting_seqs:
261
seq.status = SequenceStatus.FINISHED_IGNORED
262
ignored_seq_groups.append(seq_group)
@@ -235,7 +275,7 @@ class Scheduler:
275
continue
276
277
# If the number of batched tokens exceeds the limit, stop.
238
- num_batched_tokens += num_prompt_tokens
278
+ num_batched_tokens += num_prefill_tokens
279
if (num_batched_tokens >
280
self.scheduler_config.max_num_batched_tokens):
281
break
@@ -253,8 +293,10 @@ class Scheduler:
293
self._allocate(seq_group)
294
self.running.append(seq_group)
295
num_curr_seqs += num_new_seqs
256
- scheduled.append(seq_group)
257
-
296
+ scheduled.append(
297
+ ScheduledSequenceGroup(
298
+ seq_group=seq_group,
299
+ token_chunk_size=num_prefill_tokens))
300
self.waiting.extendleft(leftover_waiting_sequences)
301
302
if scheduled or ignored_seq_groups:
@@ -352,7 +394,11 @@ class Scheduler:
394
for seq_group in self.running)
395
396
scheduler_outputs = SchedulerOutputs(
355
- scheduled_seq_groups=self.running,
397
+ scheduled_seq_groups=[
398
+ ScheduledSequenceGroup(seq_group=running_group,
399
+ token_chunk_size=1)
400
+ for running_group in self.running
401
+ ],
402
prompt_run=False,
403
num_batched_tokens=num_batched_tokens,
404
blocks_to_swap_in=blocks_to_swap_in,
@@ -371,10 +417,14 @@ class Scheduler:
417
418
# Create input data structures.
419
seq_group_metadata_list: List[SequenceGroupMetadata] = []
374
- for seq_group in scheduler_outputs.scheduled_seq_groups:
420
+ for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
421
+ seq_group = scheduled_seq_group.seq_group
422
+ token_chunk_size = scheduled_seq_group.token_chunk_size
423
seq_group.maybe_set_first_scheduled_time(now)
424
425
+ # seq_id -> SequenceData
426
seq_data: Dict[int, SequenceData] = {}
427
+ # seq_id -> physical block numbers
428
block_tables: Dict[int, List[int]] = {}
429
430
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
@@ -393,6 +443,7 @@ class Scheduler:
443
seq_data=seq_data,
444
sampling_params=seq_group.sampling_params,
445
block_tables=block_tables,
446
+ token_chunk_size=token_chunk_size,
447
lora_request=seq_group.lora_request,
448
computed_block_nums=common_computed_block_nums,
449
state=seq_group.state,
@@ -409,8 +460,9 @@ class Scheduler:
460
# batch will have been computed before the next scheduling invocation.
461
# This is because the engine assumes that a failure in model execution
462
# will crash the vLLM instance / will not retry.
412
- for seq_group in scheduler_outputs.scheduled_seq_groups:
413
- self.block_manager.mark_blocks_as_computed(seq_group)
463
+ for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
464
+ self.block_manager.mark_blocks_as_computed(
465
+ scheduled_seq_group.seq_group)
466
467
return seq_group_metadata_list, scheduler_outputs
468
@@ -418,6 +470,7 @@ class Scheduler:
470
self.block_manager.fork(parent_seq, child_seq)
471
472
def free_seq(self, seq: Sequence) -> None:
473
+ """Free a sequence from a block table."""
474
self.block_manager.free(seq)
475
476
def free_finished_seq_groups(self) -> None:
@@ -480,7 +533,8 @@ class Scheduler:
533
assert len(seqs) == 1
534
for seq in seqs:
535
seq.status = SequenceStatus.WAITING
483
- self.block_manager.free(seq)
536
+ self.free_seq(seq)
537
+ seq.reset_state_for_recompute()
538
# NOTE: For FCFS, we insert the preempted sequence group to the front
539
# of the waiting queue.
540
self.waiting.appendleft(seq_group)
vllm/engine/arg_utils.py
+15
-5
@@ -62,6 +62,7 @@ class EngineArgs:
62
image_input_shape: Optional[str] = None
63
image_feature_size: Optional[int] = None
64
scheduler_delay_factor: float = 0.0
65
+ enable_chunked_prefill: bool = False
66
67
def __post_init__(self):
68
if self.tokenizer is None:
@@ -356,6 +357,12 @@ class EngineArgs:
357
default=EngineArgs.scheduler_delay_factor,
358
help='Apply a delay (of delay factor multiplied by previous'
359
'prompt latency) before scheduling next prompt.')
360
+ parser.add_argument(
361
+ '--enable-chunked-prefill',
362
+ type=bool,
363
+ default=False,
364
+ help='If True, the prefill requests can be chunked based on the '
365
+ 'max_num_batched_tokens')
366
return parser
367
368
@classmethod
@@ -394,11 +401,14 @@ class EngineArgs:
401
self.tokenizer_pool_type,
402
self.tokenizer_pool_extra_config,
403
), self.ray_workers_use_nsight)
397
- scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
398
- self.max_num_seqs,
399
- model_config.max_model_len,
400
- self.use_v2_block_manager,
401
- self.scheduler_delay_factor)
404
+ scheduler_config = SchedulerConfig(
405
+ self.max_num_batched_tokens,
406
+ self.max_num_seqs,
407
+ model_config.max_model_len,
408
+ self.use_v2_block_manager,
409
+ delay_factor=self.scheduler_delay_factor,
410
+ enable_chunked_prefill=self.enable_chunked_prefill,
411
+ )
412
lora_config = LoRAConfig(
413
max_lora_rank=self.max_lora_rank,
414
max_loras=self.max_loras,
vllm/engine/llm_engine.py
+14
-7
@@ -553,7 +553,10 @@ class LLMEngine:
553
# Update the scheduled sequence groups with the model outputs.
554
scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
555
556
- for seq_group, outputs in zip(scheduled_seq_groups, output):
556
+ for scheduled_seq_group, outputs in zip(scheduled_seq_groups, output):
557
+ seq_group = scheduled_seq_group.seq_group
558
+ token_chunk_size = scheduled_seq_group.token_chunk_size
559
+ seq_group.update_num_computed_tokens(token_chunk_size)
560
self._process_sequence_group_outputs(seq_group, outputs)
561
562
# Free the finished sequence groups.
@@ -561,7 +564,8 @@ class LLMEngine:
564
565
# Create the outputs.
566
request_outputs: List[RequestOutput] = []
564
- for seq_group in scheduled_seq_groups:
567
+ for scheduled_seq_group in scheduled_seq_groups:
568
+ seq_group = scheduled_seq_group.seq_group
569
seq_group.maybe_set_first_token_time(now)
570
request_output = RequestOutput.from_seq_group(seq_group)
571
request_outputs.append(request_output)
@@ -676,17 +680,20 @@ class LLMEngine:
680
# Number of Tokens.
681
if prompt_run:
682
num_prompt_tokens = sum(
679
- len(seq_group.prompt_token_ids)
680
- for seq_group in scheduler_outputs.scheduled_seq_groups)
683
+ len(scheduled_seq_group.seq_group.prompt_token_ids)
684
+ for scheduled_seq_group in
685
+ scheduler_outputs.scheduled_seq_groups)
686
num_generation_tokens = sum(
682
- seq_group.num_seqs()
683
- for seq_group in scheduler_outputs.scheduled_seq_groups)
687
+ scheduled_seq_group.seq_group.num_seqs()
688
+ for scheduled_seq_group in
689
+ scheduler_outputs.scheduled_seq_groups)
690
else:
691
num_generation_tokens = scheduler_outputs.num_batched_tokens
692
693
# Latency Timings.
694
time_last_iters = []
689
- for seq_group in scheduler_outputs.scheduled_seq_groups:
695
+ for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
696
+ seq_group = scheduled_seq_group.seq_group
697
# Time since last token.
698
# (n.b. updates seq_group.metrics.last_token_time)
699
time_last_iters.append(seq_group.get_last_latency(now))
vllm/sequence.py
+55
@@ -113,6 +113,8 @@ class SequenceData:
113
self.prompt_token_ids = prompt_token_ids
114
self.output_token_ids = output_token_ids
115
self.cumulative_logprob = 0.0
116
+ # The number of tokens that are computed (that run against the model).
117
+ self._num_computed_tokens = 0
118
119
def append_token_id(self, token_id: int, logprob: float) -> None:
120
self.output_token_ids.append(token_id)
@@ -130,6 +132,28 @@ class SequenceData:
132
def get_token_ids(self) -> List[int]:
133
return self.prompt_token_ids + self.output_token_ids
134
135
+ def get_num_computed_tokens(self) -> int:
136
+ """Return the number of prefill tokens that are already computed."""
137
+ return self._num_computed_tokens
138
+
139
+ def update_num_computed_tokens(self, num_new_computed_tokens: int) -> int:
140
+ """Update number of tokens computed so far."""
141
+ self._num_computed_tokens += num_new_computed_tokens
142
+
143
+ def reset_num_computed_tokens(self) -> None:
144
+ """Reset the number of computed tokens from this sequence. It is
145
+ supposed to be called when a sequence needs to be started from
146
+ the beginning again (e.g., sequence is preempted).
147
+ """
148
+ self._num_computed_tokens = 0
149
+
150
+ def get_num_uncomputed_tokens(self) -> int:
151
+ """Return the number of prefil tokens that are not computed."""
152
+ # we use `get_len()` which includes prompt_len + output_len instead
153
+ # of prompt_len here. This is because during recompute we need to
154
+ # prefill for both prompt and output.
155
+ return self.get_len() - self.get_num_computed_tokens()
156
+
157
def get_last_token_id(self) -> int:
158
if not self.output_token_ids:
159
return self.prompt_token_ids[-1]
@@ -208,6 +232,10 @@ class Sequence:
232
def num_hashed_tokens_of_block(self, logical_idx: int):
233
return logical_idx * self.block_size + self.block_size
234
235
+ def reset_state_for_recompute(self):
236
+ """Reset the sequence states for recomputation."""
237
+ self.data.reset_num_computed_tokens()
238
+
239
def _append_logical_block(self) -> None:
240
block = LogicalTokenBlock(
241
block_number=len(self.logical_token_blocks),
@@ -430,6 +458,18 @@ class SequenceGroup:
458
def get_finished_seqs(self) -> List[Sequence]:
459
return [seq for seq in self.seqs_dict.values() if seq.is_finished()]
460
461
+ def update_num_computed_tokens(self, num_new_computed_tokens: int):
462
+ """Update number of tokens computed so far."""
463
+ for seq in self.seqs_dict.values():
464
+ seq.data.update_num_computed_tokens(num_new_computed_tokens)
465
+
466
+ def get_num_uncomputed_tokens(self) -> int:
467
+ # All sequences in the group should have the same prompt, so the
468
+ # number of unfinished prefill tokens are the same across all
469
+ # sequences.
470
+ return list(
471
+ self.seqs_dict.values())[0].data.get_num_uncomputed_tokens()
472
+
473
def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
474
return len(self.get_seqs(status))
475
@@ -473,6 +513,8 @@ class SequenceGroupMetadata:
513
sampling_params: The sampling parameters used to generate the outputs.
514
block_tables: The block tables. (Seq id -> list of physical block
515
numbers)
516
+ token_chunk_size: The number of tokens to be processed. None if
517
+ chunking is not required.
518
state: Internal state tied to this sequence group.
519
lora_request: LoRA request.
520
multi_modal_data: Multi modal data.
@@ -485,6 +527,7 @@ class SequenceGroupMetadata:
527
seq_data: Dict[int, SequenceData],
528
sampling_params: SamplingParams,
529
block_tables: Dict[int, List[int]],
530
+ token_chunk_size: Optional[int] = None,
531
lora_request: Optional[LoRARequest] = None,
532
computed_block_nums: Optional[List[int]] = None,
533
state: Optional[SequenceGroupState] = None,
@@ -499,11 +542,23 @@ class SequenceGroupMetadata:
542
self.computed_block_nums = computed_block_nums
543
self.multi_modal_data = multi_modal_data
544
self.state = SequenceGroupState() if state is None else state
545
+ self._token_chunk_size = token_chunk_size
546
+
547
+ if self._token_chunk_size is None:
548
+ if is_prompt:
549
+ self._token_chunk_size = list(seq_data.values())[0].get_len()
550
+ else:
551
+ self._token_chunk_size = 1
552
553
@property
554
def lora_int_id(self) -> int:
555
return self.lora_request.lora_int_id if self.lora_request else 0
556
557
+ @property
558
+ def token_chunk_size(self) -> int:
559
+ """Return the number of tokens to be processed (chunk size)."""
560
+ return self._token_chunk_size
561
+
562
563
class SequenceOutput:
564
"""The model output associated with a sequence.
vllm/worker/model_runner.py
+30
-9
@@ -150,39 +150,58 @@ class ModelRunner:
150
subquery_lens: List[int] = []
151
prefix_block_tables: List[List[int]] = []
152
multi_modal_input_list: List[torch.Tensor] = []
153
+
154
for seq_group_metadata in seq_group_metadata_list:
155
assert seq_group_metadata.is_prompt
156
seq_ids = list(seq_group_metadata.seq_data.keys())
157
assert len(seq_ids) == 1
158
seq_id = seq_ids[0]
159
160
+ computed_block_nums = seq_group_metadata.computed_block_nums
161
+ if (self.scheduler_config is not None
162
+ and self.scheduler_config.chunked_prefill_enabled
163
+ and computed_block_nums is not None):
164
+ raise RuntimeError(
165
+ "chunked prefill cannot be used with prefix caching "
166
+ "now.")
167
+
168
+ token_chunk_size = seq_group_metadata.token_chunk_size
169
seq_data = seq_group_metadata.seq_data[seq_id]
160
- prompt_tokens = seq_data.get_token_ids()
170
+ computed_len = seq_data.get_num_computed_tokens()
171
+ # We should use get_len here because in case of preemption
172
+ # it contains output tokens.
173
+ prefill_end = min(seq_data.get_len(),
174
+ computed_len + token_chunk_size)
175
+ # TODO(sang): Rename it after chunked prefill is introduced.
176
+ prompt_tokens = seq_data.get_token_ids()[computed_len:prefill_end]
177
prompt_len = len(prompt_tokens)
178
+ # Right now, the prefill_end is always same as the length of
179
+ # sequence. However, once chunked prefill is introduced, this
180
+ # assumption can be changed.
181
+ assert prefill_end == seq_data.get_len()
182
prompt_lens.append(prompt_len)
163
- computed_len = 0
183
184
# NOTE: This only works for oooooooxxx style attention.
166
- computed_block_nums = seq_group_metadata.computed_block_nums
185
if computed_block_nums is not None and len(
186
computed_block_nums) > 0 and self.sliding_window is None:
187
# Prefix is not supported with sliding_window
188
computed_len = len(computed_block_nums) * self.block_size
189
prompt_tokens = prompt_tokens[computed_len:]
190
prefix_block_tables.append(computed_block_nums)
173
- context_len = computed_len
191
else:
192
prefix_block_tables.append([])
176
- context_len = 0
193
+ # Right now, prefill start is always 0. However, this
194
+ # assumption can be changed once chunked prefill is introduced.
195
+ assert computed_len == 0
196
+
197
# actual prompt lens
178
- context_lens.append(context_len)
198
+ context_lens.append(computed_len)
199
subquery_lens.append(prompt_len - computed_len)
200
201
input_tokens.extend(prompt_tokens)
202
# NOTE(woosuk): Here we assume that the first token in the prompt
203
# is always the first token in the sequence.
184
- input_positions.extend(
185
- list(range(computed_len, computed_len + len(prompt_tokens))))
204
+ input_positions.extend(list(range(computed_len, prefill_end)))
205
206
lora_id = seq_group_metadata.lora_int_id
207
@@ -218,7 +237,8 @@ class ModelRunner:
237
"Prefix caching is currently not supported with "
238
"sliding window attention")
239
start_idx = max(0, prompt_len - self.sliding_window)
221
- for i in range(computed_len, prompt_len):
240
+
241
+ for i in range(computed_len, prefill_end):
242
if i < start_idx:
243
slot_mapping.append(_PAD_SLOT_ID)
244
continue
@@ -331,6 +351,7 @@ class ModelRunner:
351
352
for seq_group_metadata in seq_group_metadata_list:
353
assert not seq_group_metadata.is_prompt
354
+ assert seq_group_metadata.token_chunk_size == 1
355
356
seq_ids = list(seq_group_metadata.seq_data.keys())
357
lora_id = seq_group_metadata.lora_int_id