@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b51c1cc9d2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[2/N] Chunked prefill data update (#3538)

SangBin Cho committed Mar 29, 2024 at 02:06 UTC b51c1cc9d2f223cfa3aef1426bced10dfde28dbb
11 files changed +272 -76
benchmarks/benchmark_latency.py
+13 -1
@@ -26,7 +26,9 @@ def main(args: argparse.Namespace):
26 kv_cache_dtype=args.kv_cache_dtype,
27 device=args.device,
28 ray_workers_use_nsight=args.ray_workers_use_nsight,
29 - download_dir=args.download_dir)
29 + enable_chunked_prefill=args.enable_chunked_prefill,
30 + download_dir=args.download_dir,
31 + block_size=args.block_size)
32
33 sampling_params = SamplingParams(
34 n=args.n,
@@ -145,6 +147,16 @@ if __name__ == '__main__':
147 default="cuda",
148 choices=["cuda"],
149 help='device type for vLLM execution, supporting CUDA only currently.')
150 + parser.add_argument('--block-size',
151 + type=int,
152 + default=16,
153 + help='block size of key/value cache')
154 + parser.add_argument(
155 + '--enable-chunked-prefill',
156 + type=bool,
157 + default=False,
158 + help='If True, the prefill requests can be chunked based on the '
159 + 'max_num_batched_tokens')
160 parser.add_argument(
161 "--ray-workers-use-nsight",
162 action='store_true',
tests/conftest.py
+4
@@ -256,6 +256,8 @@ class VllmRunner:
256 dtype: str = "half",
257 disable_log_stats: bool = True,
258 tensor_parallel_size: int = 1,
259 + block_size: int = 16,
260 + enable_chunked_prefill: bool = False,
261 **kwargs,
262 ) -> None:
263 self.model = LLM(
@@ -266,6 +268,8 @@ class VllmRunner:
268 swap_space=0,
269 disable_log_stats=disable_log_stats,
270 tensor_parallel_size=tensor_parallel_size,
271 + block_size=block_size,
272 + enable_chunked_prefill=enable_chunked_prefill,
273 **kwargs,
274 )
275
tests/core/test_scheduler.py
+13 -9
@@ -10,6 +10,10 @@ from vllm.sequence import Logprob, SequenceGroup
10 from .utils import create_dummy_prompt
11
12
13 +def get_sequence_groups(scheduler_output):
14 + return [s.seq_group for s in scheduler_output.scheduled_seq_groups]
15 +
16 +
17 def test_scheduler_add_seq_group():
18 block_size = 4
19 scheduler_config = SchedulerConfig(100, 64, 1)
@@ -57,9 +61,9 @@ def test_scheduler_schedule_simple():
61 cache_config.num_cpu_blocks = 8
62 cache_config.num_gpu_blocks = 8
63 scheduler = Scheduler(scheduler_config, cache_config, None)
64 + running: List[SequenceGroup] = []
65
66 # Add seq groups to scheduler.
62 - running: List[SequenceGroup] = []
67 for i in range(num_seq_group):
68 _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
69 scheduler.add_seq_group(seq_group)
@@ -68,7 +72,7 @@ def test_scheduler_schedule_simple():
72 # Schedule seq groups prompts.
73 num_tokens = block_size * num_seq_group
74 seq_group_meta, out = scheduler.schedule()
71 - assert set(out.scheduled_seq_groups) == set(running)
75 + assert set(get_sequence_groups(out)) == set(running)
76 assert out.num_batched_tokens == num_tokens
77 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
78 and not out.blocks_to_swap_out)
@@ -76,7 +80,7 @@ def test_scheduler_schedule_simple():
80
81 # Schedule seq groups generation.
82 seq_group_meta, out = scheduler.schedule()
79 - assert set(out.scheduled_seq_groups) == set(running)
83 + assert set(get_sequence_groups(out)) == set(running)
84 assert out.num_batched_tokens == num_seq_group
85 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
86 and not out.blocks_to_swap_out)
@@ -100,7 +104,7 @@ def test_scheduler_schedule_preempt_abort():
104
105 # Schedule seq groups prompts.
106 seq_group_meta, out = scheduler.schedule()
103 - assert out.scheduled_seq_groups == [seq_group_a, seq_group_b]
107 + assert get_sequence_groups(out) == [seq_group_a, seq_group_b]
108 assert out.num_batched_tokens == block_size * 2 # seq_a and seq_b
109 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
110 and not out.blocks_to_swap_out)
@@ -115,7 +119,7 @@ def test_scheduler_schedule_preempt_abort():
119
120 # Schedule seq groups generation and preempt seq group b.
121 seq_group_meta, out = scheduler.schedule()
118 - assert out.scheduled_seq_groups == [seq_group_a]
122 + assert get_sequence_groups(out) == [seq_group_a]
123 assert out.num_batched_tokens == 1
124 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
125 and not out.blocks_to_swap_out)
@@ -125,7 +129,7 @@ def test_scheduler_schedule_preempt_abort():
129 # Abort seq group a. Re-schedule seq group b prompt with recomputation.
130 scheduler.abort_seq_group("1")
131 seq_group_meta, out = scheduler.schedule()
128 - assert out.scheduled_seq_groups == [seq_group_b]
132 + assert get_sequence_groups(out) == [seq_group_b]
133 assert out.num_batched_tokens == 5 # 4 prompt + 1 generation.
134 assert (not out.blocks_to_copy and not out.blocks_to_swap_in
135 and not out.blocks_to_swap_out)
@@ -155,11 +159,11 @@ def test_scheduler_max_seqs():
159
160 # Schedule seq groups prompts.
161 _, out = scheduler.schedule()
158 - assert set(out.scheduled_seq_groups) == set([all_seq_groups[0]])
162 + assert set(get_sequence_groups(out)) == set([all_seq_groups[0]])
163
164 # Schedule seq groups generation.
165 _, out = scheduler.schedule()
162 - assert set(out.scheduled_seq_groups) == set([all_seq_groups[0]])
166 + assert set(get_sequence_groups(out)) == set([all_seq_groups[0]])
167
168 # Append 2 more seq group
169 scheduler.add_seq_group(all_seq_groups[1])
@@ -169,7 +173,7 @@ def test_scheduler_max_seqs():
173 # Only 1 seq group should be scheduled since max_seq_group is 2
174 # and one is prompting.
175 _, out = scheduler.schedule()
172 - assert set(out.scheduled_seq_groups) == set([all_seq_groups[1]])
176 + assert set(get_sequence_groups(out)) == set([all_seq_groups[1]])
177
178
179 def test_scheduler_delay_factor():
tests/test_sequence.py
+23 -1
@@ -1,6 +1,7 @@
1 import pytest
2
3 -from vllm.sequence import SamplerOutput, SequenceGroupOutput, SequenceOutput
3 +from vllm.sequence import (SamplerOutput, SequenceData, SequenceGroupOutput,
4 + SequenceOutput)
5
6
7 @pytest.fixture
@@ -48,3 +49,24 @@ def test_sampler_output_eq(sample_outputs):
49 sampler_output3 = SamplerOutput(outputs=sample_outputs[:-1])
50 assert sampler_output1 == sampler_output2
51 assert sampler_output1 != sampler_output3
52 +
53 +
54 +def test_sequence_data_prefill():
55 + seq_data = SequenceData(prompt_token_ids=[1, 2, 3, 4])
56 + assert seq_data.get_num_uncomputed_tokens() == 4
57 + assert seq_data.get_num_computed_tokens() == 0
58 + # advance by 2
59 + seq_data.update_num_computed_tokens(2)
60 + assert seq_data.get_num_uncomputed_tokens() == 2
61 + assert seq_data.get_num_computed_tokens() == 2
62 +
63 + # advance by 1
64 + seq_data.update_num_computed_tokens(1)
65 + assert seq_data.get_num_uncomputed_tokens() == 1
66 + assert seq_data.get_num_computed_tokens() == 3
67 +
68 + # append tokens and reset, simulating recompute
69 + seq_data.append_token_id(1, logprob=0.0)
70 + seq_data.reset_num_computed_tokens()
71 + assert seq_data.get_num_uncomputed_tokens() == 5
72 + assert seq_data.get_num_computed_tokens() == 0
tests/worker/test_model_runner.py
+20 -17
@@ -18,15 +18,16 @@ def test_prepare_prompt(batch_size):
18 # make sure all tokens fit into one block
19 prompt_len = i % (model_runner.block_size - 1) + 1
20 prompt_lens.append(prompt_len)
21 - seq_data = list(range(prompt_len))
22 - seq_group_metadata_list.append(
23 - SequenceGroupMetadata(
24 - request_id=f"test_{i}",
25 - is_prompt=True,
26 - seq_data={0: SequenceData(seq_data)},
27 - sampling_params=SamplingParams(temperature=0),
28 - block_tables=block_tables,
29 - ))
21 + seq_data = SequenceData(list(range(prompt_len)))
22 + seq_group_metadata = SequenceGroupMetadata(
23 + request_id=f"test_{i}",
24 + is_prompt=True,
25 + seq_data={0: seq_data},
26 + sampling_params=SamplingParams(temperature=0),
27 + block_tables=block_tables,
28 + )
29 + assert seq_group_metadata.token_chunk_size == seq_data.get_len()
30 + seq_group_metadata_list.append(seq_group_metadata)
31
32 expected_selected_token_indices = []
33 selected_token_start_idx = 0
@@ -131,14 +132,16 @@ def test_prepare_decode_cuda_graph(batch_size):
132 prompt_len = i % (model_runner.block_size - 1) + 1
133 prompt_lens.append(prompt_len)
134 seq_data = list(range(prompt_len))
134 - seq_group_metadata_list.append(
135 - SequenceGroupMetadata(
136 - request_id=f"test_{i}",
137 - is_prompt=False,
138 - seq_data={0: SequenceData(seq_data)},
139 - sampling_params=SamplingParams(temperature=0),
140 - block_tables={0: [1]},
141 - ))
135 + seq_data = SequenceData(seq_data)
136 + seq_group_metadata = SequenceGroupMetadata(
137 + request_id=f"test_{i}",
138 + is_prompt=False,
139 + seq_data={0: seq_data},
140 + sampling_params=SamplingParams(temperature=0),
141 + block_tables={0: [1]},
142 + )
143 + assert seq_group_metadata.token_chunk_size == 1
144 + seq_group_metadata_list.append(seq_group_metadata)
145
146 input_tokens, input_positions, attn_metadata, _, _, _ = (
147 model_runner._prepare_decode(seq_group_metadata_list))
vllm/config.py
+4
@@ -533,6 +533,8 @@ class SchedulerConfig:
533 delay_factor: Apply a delay (of delay factor multiplied by previous
534 prompt latency) before scheduling next prompt.
535 use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
536 + enable_chunked_prefill: If True, prefill requests can be chunked based
537 + on the remaining max_num_batched_tokens.
538 """
539
540 def __init__(
@@ -542,6 +544,7 @@ class SchedulerConfig:
544 max_model_len: int,
545 use_v2_block_manager: bool = False,
546 delay_factor: float = 0.0,
547 + enable_chunked_prefill: bool = False,
548 ) -> None:
549 if max_num_batched_tokens is not None:
550 self.max_num_batched_tokens = max_num_batched_tokens
@@ -553,6 +556,7 @@ class SchedulerConfig:
556 self.max_model_len = max_model_len
557 self.delay_factor = delay_factor
558 self.use_v2_block_manager = use_v2_block_manager
559 + self.chunked_prefill_enabled = enable_chunked_prefill
560 self._verify_args()
561
562 def _verify_args(self) -> None:
vllm/core/scheduler.py
+81 -27
@@ -1,6 +1,7 @@
1 import enum
2 import time
3 from collections import deque
4 +from dataclasses import dataclass
5 from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
6
7 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
@@ -27,11 +28,24 @@ class PreemptionMode(enum.Enum):
28 RECOMPUTE = enum.auto()
29
30
31 +# seq_group: SequenceGroup to schedule.
32 +# token_chunk_size: The number of prefill tokens to be processed in the next
33 +# step.
34 +@dataclass
35 +class ScheduledSequenceGroup:
36 + # A sequence group that's scheduled.
37 + seq_group: SequenceGroup
38 + # The total chunk size (number of tokens) to process for next iteration.
39 + # 1 for decoding. Same as prompt tokens for prefill, but if prefill is
40 + # chunked, it can be smaller than that.
41 + token_chunk_size: int
42 +
43 +
44 class SchedulerOutputs:
45
46 def __init__(
47 self,
34 - scheduled_seq_groups: Iterable[SequenceGroup],
48 + scheduled_seq_groups: Iterable[ScheduledSequenceGroup],
49 prompt_run: bool,
50 num_batched_tokens: int,
51 blocks_to_swap_in: Dict[int, int],
@@ -39,17 +53,41 @@ class SchedulerOutputs:
53 blocks_to_copy: Dict[int, List[int]],
54 ignored_seq_groups: List[SequenceGroup],
55 ) -> None:
42 - self.scheduled_seq_groups = scheduled_seq_groups
43 - self.prompt_run = prompt_run
44 - self.num_batched_tokens = num_batched_tokens
45 - self.blocks_to_swap_in = blocks_to_swap_in
46 - self.blocks_to_swap_out = blocks_to_swap_out
47 - self.blocks_to_copy = blocks_to_copy
56 + """A list of sequence groups to be scheduled as a single batch.
57 +
58 + Args:
59 + scheduled_seq_groups: A tuple of scheduled sequence group and its
60 + token chunk size.
61 + prompt_run: True if all sequence groups are in prefill phase.
62 + If False, all sequence groups are in decoding phase.
63 + num_batched_tokens: Total number of batched tokens.
64 + blocks_to_swap_in: Blocks to swap in. Dict of CPU -> GPU block
65 + number.
66 + blocks_to_swap_out: Blocks to swap out. Dict of GPU -> CPU block
67 + number.
68 + blocks_to_copy: Blocks to copy. Source to a list of dest blocks.
69 + ignored_seq_groups: Sequence groups that are going to be ignored.
70 + """
71 + # A tuple of scheduled sequence group and its chunk size.
72 + self.scheduled_seq_groups: ScheduledSequenceGroup = scheduled_seq_groups
73 + # True if all sequence groups are in prefill phase. If False, all
74 + # sequence groups are in decoding phase.
75 + self.prompt_run: bool = prompt_run
76 + # Total number of batched tokens.
77 + self.num_batched_tokens: int = num_batched_tokens
78 + # Blocks to swap in. Dict of CPU -> GPU block number.
79 + self.blocks_to_swap_in: Dict[int, int] = blocks_to_swap_in
80 + # Blocks to swap out. Dict of GPU -> CPU block number.
81 + self.blocks_to_swap_out: Dict[int, int] = blocks_to_swap_out
82 + # Blocks to copy. Source to a list of dest blocks.
83 + self.blocks_to_copy: Dict[int, List[int]] = blocks_to_copy
84 + # Sequence groups that are going to be ignored.
85 + self.ignored_seq_groups: List[SequenceGroup] = ignored_seq_groups
86 +
87 # Swap in and swap out should never happen at the same time.
88 assert not (blocks_to_swap_in and blocks_to_swap_out)
50 - self.ignored_seq_groups = ignored_seq_groups
89
52 - self.num_loras = len(self.lora_requests)
90 + self.num_loras: int = len(self.lora_requests)
91 if self.num_loras > 0:
92 self._sort_by_lora_ids()
93
@@ -59,13 +97,13 @@ class SchedulerOutputs:
97 and not self.blocks_to_swap_out and not self.blocks_to_copy)
98
99 def _sort_by_lora_ids(self) -> bool:
62 - self.scheduled_seq_groups = sorted(self.scheduled_seq_groups,
63 - key=lambda g:
64 - (g.lora_int_id, g.request_id))
100 + self.scheduled_seq_groups = sorted(
101 + self.scheduled_seq_groups,
102 + key=lambda g: (g.seq_group.lora_int_id, g.seq_group.request_id))
103
104 @property
105 def lora_requests(self) -> Set[LoRARequest]:
68 - return {g.lora_request for g in self.scheduled_seq_groups}
106 + return {g.seq_group.lora_request for g in self.scheduled_seq_groups}
107
108
109 class Scheduler:
@@ -198,11 +236,13 @@ class Scheduler:
236 assert len(waiting_seqs) == 1, (
237 "Waiting sequence group should have only one prompt "
238 "sequence.")
201 - num_prompt_tokens = waiting_seqs[0].get_len()
202 - if num_prompt_tokens > self.prompt_limit:
239 + # get_len includes output tokens if the request has been
240 + # preempted.
241 + num_prefill_tokens = waiting_seqs[0].get_len()
242 + if num_prefill_tokens > self.prompt_limit:
243 logger.warning(
204 - f"Input prompt ({num_prompt_tokens} tokens) is too long"
205 - f" and exceeds limit of {self.prompt_limit}")
244 + f"Input prompt ({num_prefill_tokens} tokens) is too "
245 + f"long and exceeds limit of {self.prompt_limit}")
246 for seq in waiting_seqs:
247 seq.status = SequenceStatus.FINISHED_IGNORED
248 ignored_seq_groups.append(seq_group)
@@ -215,8 +255,8 @@ class Scheduler:
255 break
256 elif can_allocate == AllocStatus.NEVER:
257 logger.warning(
218 - f"Input prompt ({num_prompt_tokens} tokens) is too long"
219 - f" and exceeds the capacity of block_manager")
258 + f"Input prompt ({num_prefill_tokens} tokens) is too "
259 + f"long and exceeds the capacity of block_manager")
260 for seq in waiting_seqs:
261 seq.status = SequenceStatus.FINISHED_IGNORED
262 ignored_seq_groups.append(seq_group)
@@ -235,7 +275,7 @@ class Scheduler:
275 continue
276
277 # If the number of batched tokens exceeds the limit, stop.
238 - num_batched_tokens += num_prompt_tokens
278 + num_batched_tokens += num_prefill_tokens
279 if (num_batched_tokens >
280 self.scheduler_config.max_num_batched_tokens):
281 break
@@ -253,8 +293,10 @@ class Scheduler:
293 self._allocate(seq_group)
294 self.running.append(seq_group)
295 num_curr_seqs += num_new_seqs
256 - scheduled.append(seq_group)
257 -
296 + scheduled.append(
297 + ScheduledSequenceGroup(
298 + seq_group=seq_group,
299 + token_chunk_size=num_prefill_tokens))
300 self.waiting.extendleft(leftover_waiting_sequences)
301
302 if scheduled or ignored_seq_groups:
@@ -352,7 +394,11 @@ class Scheduler:
394 for seq_group in self.running)
395
396 scheduler_outputs = SchedulerOutputs(
355 - scheduled_seq_groups=self.running,
397 + scheduled_seq_groups=[
398 + ScheduledSequenceGroup(seq_group=running_group,
399 + token_chunk_size=1)
400 + for running_group in self.running
401 + ],
402 prompt_run=False,
403 num_batched_tokens=num_batched_tokens,
404 blocks_to_swap_in=blocks_to_swap_in,
@@ -371,10 +417,14 @@ class Scheduler:
417
418 # Create input data structures.
419 seq_group_metadata_list: List[SequenceGroupMetadata] = []
374 - for seq_group in scheduler_outputs.scheduled_seq_groups:
420 + for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
421 + seq_group = scheduled_seq_group.seq_group
422 + token_chunk_size = scheduled_seq_group.token_chunk_size
423 seq_group.maybe_set_first_scheduled_time(now)
424
425 + # seq_id -> SequenceData
426 seq_data: Dict[int, SequenceData] = {}
427 + # seq_id -> physical block numbers
428 block_tables: Dict[int, List[int]] = {}
429
430 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
@@ -393,6 +443,7 @@ class Scheduler:
443 seq_data=seq_data,
444 sampling_params=seq_group.sampling_params,
445 block_tables=block_tables,
446 + token_chunk_size=token_chunk_size,
447 lora_request=seq_group.lora_request,
448 computed_block_nums=common_computed_block_nums,
449 state=seq_group.state,
@@ -409,8 +460,9 @@ class Scheduler:
460 # batch will have been computed before the next scheduling invocation.
461 # This is because the engine assumes that a failure in model execution
462 # will crash the vLLM instance / will not retry.
412 - for seq_group in scheduler_outputs.scheduled_seq_groups:
413 - self.block_manager.mark_blocks_as_computed(seq_group)
463 + for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
464 + self.block_manager.mark_blocks_as_computed(
465 + scheduled_seq_group.seq_group)
466
467 return seq_group_metadata_list, scheduler_outputs
468
@@ -418,6 +470,7 @@ class Scheduler:
470 self.block_manager.fork(parent_seq, child_seq)
471
472 def free_seq(self, seq: Sequence) -> None:
473 + """Free a sequence from a block table."""
474 self.block_manager.free(seq)
475
476 def free_finished_seq_groups(self) -> None:
@@ -480,7 +533,8 @@ class Scheduler:
533 assert len(seqs) == 1
534 for seq in seqs:
535 seq.status = SequenceStatus.WAITING
483 - self.block_manager.free(seq)
536 + self.free_seq(seq)
537 + seq.reset_state_for_recompute()
538 # NOTE: For FCFS, we insert the preempted sequence group to the front
539 # of the waiting queue.
540 self.waiting.appendleft(seq_group)
vllm/engine/arg_utils.py
+15 -5
@@ -62,6 +62,7 @@ class EngineArgs:
62 image_input_shape: Optional[str] = None
63 image_feature_size: Optional[int] = None
64 scheduler_delay_factor: float = 0.0
65 + enable_chunked_prefill: bool = False
66
67 def __post_init__(self):
68 if self.tokenizer is None:
@@ -356,6 +357,12 @@ class EngineArgs:
357 default=EngineArgs.scheduler_delay_factor,
358 help='Apply a delay (of delay factor multiplied by previous'
359 'prompt latency) before scheduling next prompt.')
360 + parser.add_argument(
361 + '--enable-chunked-prefill',
362 + type=bool,
363 + default=False,
364 + help='If True, the prefill requests can be chunked based on the '
365 + 'max_num_batched_tokens')
366 return parser
367
368 @classmethod
@@ -394,11 +401,14 @@ class EngineArgs:
401 self.tokenizer_pool_type,
402 self.tokenizer_pool_extra_config,
403 ), self.ray_workers_use_nsight)
397 - scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
398 - self.max_num_seqs,
399 - model_config.max_model_len,
400 - self.use_v2_block_manager,
401 - self.scheduler_delay_factor)
404 + scheduler_config = SchedulerConfig(
405 + self.max_num_batched_tokens,
406 + self.max_num_seqs,
407 + model_config.max_model_len,
408 + self.use_v2_block_manager,
409 + delay_factor=self.scheduler_delay_factor,
410 + enable_chunked_prefill=self.enable_chunked_prefill,
411 + )
412 lora_config = LoRAConfig(
413 max_lora_rank=self.max_lora_rank,
414 max_loras=self.max_loras,
vllm/engine/llm_engine.py
+14 -7
@@ -553,7 +553,10 @@ class LLMEngine:
553 # Update the scheduled sequence groups with the model outputs.
554 scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
555
556 - for seq_group, outputs in zip(scheduled_seq_groups, output):
556 + for scheduled_seq_group, outputs in zip(scheduled_seq_groups, output):
557 + seq_group = scheduled_seq_group.seq_group
558 + token_chunk_size = scheduled_seq_group.token_chunk_size
559 + seq_group.update_num_computed_tokens(token_chunk_size)
560 self._process_sequence_group_outputs(seq_group, outputs)
561
562 # Free the finished sequence groups.
@@ -561,7 +564,8 @@ class LLMEngine:
564
565 # Create the outputs.
566 request_outputs: List[RequestOutput] = []
564 - for seq_group in scheduled_seq_groups:
567 + for scheduled_seq_group in scheduled_seq_groups:
568 + seq_group = scheduled_seq_group.seq_group
569 seq_group.maybe_set_first_token_time(now)
570 request_output = RequestOutput.from_seq_group(seq_group)
571 request_outputs.append(request_output)
@@ -676,17 +680,20 @@ class LLMEngine:
680 # Number of Tokens.
681 if prompt_run:
682 num_prompt_tokens = sum(
679 - len(seq_group.prompt_token_ids)
680 - for seq_group in scheduler_outputs.scheduled_seq_groups)
683 + len(scheduled_seq_group.seq_group.prompt_token_ids)
684 + for scheduled_seq_group in
685 + scheduler_outputs.scheduled_seq_groups)
686 num_generation_tokens = sum(
682 - seq_group.num_seqs()
683 - for seq_group in scheduler_outputs.scheduled_seq_groups)
687 + scheduled_seq_group.seq_group.num_seqs()
688 + for scheduled_seq_group in
689 + scheduler_outputs.scheduled_seq_groups)
690 else:
691 num_generation_tokens = scheduler_outputs.num_batched_tokens
692
693 # Latency Timings.
694 time_last_iters = []
689 - for seq_group in scheduler_outputs.scheduled_seq_groups:
695 + for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
696 + seq_group = scheduled_seq_group.seq_group
697 # Time since last token.
698 # (n.b. updates seq_group.metrics.last_token_time)
699 time_last_iters.append(seq_group.get_last_latency(now))
vllm/sequence.py
+55
@@ -113,6 +113,8 @@ class SequenceData:
113 self.prompt_token_ids = prompt_token_ids
114 self.output_token_ids = output_token_ids
115 self.cumulative_logprob = 0.0
116 + # The number of tokens that are computed (that run against the model).
117 + self._num_computed_tokens = 0
118
119 def append_token_id(self, token_id: int, logprob: float) -> None:
120 self.output_token_ids.append(token_id)
@@ -130,6 +132,28 @@ class SequenceData:
132 def get_token_ids(self) -> List[int]:
133 return self.prompt_token_ids + self.output_token_ids
134
135 + def get_num_computed_tokens(self) -> int:
136 + """Return the number of prefill tokens that are already computed."""
137 + return self._num_computed_tokens
138 +
139 + def update_num_computed_tokens(self, num_new_computed_tokens: int) -> int:
140 + """Update number of tokens computed so far."""
141 + self._num_computed_tokens += num_new_computed_tokens
142 +
143 + def reset_num_computed_tokens(self) -> None:
144 + """Reset the number of computed tokens from this sequence. It is
145 + supposed to be called when a sequence needs to be started from
146 + the beginning again (e.g., sequence is preempted).
147 + """
148 + self._num_computed_tokens = 0
149 +
150 + def get_num_uncomputed_tokens(self) -> int:
151 + """Return the number of prefil tokens that are not computed."""
152 + # we use `get_len()` which includes prompt_len + output_len instead
153 + # of prompt_len here. This is because during recompute we need to
154 + # prefill for both prompt and output.
155 + return self.get_len() - self.get_num_computed_tokens()
156 +
157 def get_last_token_id(self) -> int:
158 if not self.output_token_ids:
159 return self.prompt_token_ids[-1]
@@ -208,6 +232,10 @@ class Sequence:
232 def num_hashed_tokens_of_block(self, logical_idx: int):
233 return logical_idx * self.block_size + self.block_size
234
235 + def reset_state_for_recompute(self):
236 + """Reset the sequence states for recomputation."""
237 + self.data.reset_num_computed_tokens()
238 +
239 def _append_logical_block(self) -> None:
240 block = LogicalTokenBlock(
241 block_number=len(self.logical_token_blocks),
@@ -430,6 +458,18 @@ class SequenceGroup:
458 def get_finished_seqs(self) -> List[Sequence]:
459 return [seq for seq in self.seqs_dict.values() if seq.is_finished()]
460
461 + def update_num_computed_tokens(self, num_new_computed_tokens: int):
462 + """Update number of tokens computed so far."""
463 + for seq in self.seqs_dict.values():
464 + seq.data.update_num_computed_tokens(num_new_computed_tokens)
465 +
466 + def get_num_uncomputed_tokens(self) -> int:
467 + # All sequences in the group should have the same prompt, so the
468 + # number of unfinished prefill tokens are the same across all
469 + # sequences.
470 + return list(
471 + self.seqs_dict.values())[0].data.get_num_uncomputed_tokens()
472 +
473 def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
474 return len(self.get_seqs(status))
475
@@ -473,6 +513,8 @@ class SequenceGroupMetadata:
513 sampling_params: The sampling parameters used to generate the outputs.
514 block_tables: The block tables. (Seq id -> list of physical block
515 numbers)
516 + token_chunk_size: The number of tokens to be processed. None if
517 + chunking is not required.
518 state: Internal state tied to this sequence group.
519 lora_request: LoRA request.
520 multi_modal_data: Multi modal data.
@@ -485,6 +527,7 @@ class SequenceGroupMetadata:
527 seq_data: Dict[int, SequenceData],
528 sampling_params: SamplingParams,
529 block_tables: Dict[int, List[int]],
530 + token_chunk_size: Optional[int] = None,
531 lora_request: Optional[LoRARequest] = None,
532 computed_block_nums: Optional[List[int]] = None,
533 state: Optional[SequenceGroupState] = None,
@@ -499,11 +542,23 @@ class SequenceGroupMetadata:
542 self.computed_block_nums = computed_block_nums
543 self.multi_modal_data = multi_modal_data
544 self.state = SequenceGroupState() if state is None else state
545 + self._token_chunk_size = token_chunk_size
546 +
547 + if self._token_chunk_size is None:
548 + if is_prompt:
549 + self._token_chunk_size = list(seq_data.values())[0].get_len()
550 + else:
551 + self._token_chunk_size = 1
552
553 @property
554 def lora_int_id(self) -> int:
555 return self.lora_request.lora_int_id if self.lora_request else 0
556
557 + @property
558 + def token_chunk_size(self) -> int:
559 + """Return the number of tokens to be processed (chunk size)."""
560 + return self._token_chunk_size
561 +
562
563 class SequenceOutput:
564 """The model output associated with a sequence.
vllm/worker/model_runner.py
+30 -9
@@ -150,39 +150,58 @@ class ModelRunner:
150 subquery_lens: List[int] = []
151 prefix_block_tables: List[List[int]] = []
152 multi_modal_input_list: List[torch.Tensor] = []
153 +
154 for seq_group_metadata in seq_group_metadata_list:
155 assert seq_group_metadata.is_prompt
156 seq_ids = list(seq_group_metadata.seq_data.keys())
157 assert len(seq_ids) == 1
158 seq_id = seq_ids[0]
159
160 + computed_block_nums = seq_group_metadata.computed_block_nums
161 + if (self.scheduler_config is not None
162 + and self.scheduler_config.chunked_prefill_enabled
163 + and computed_block_nums is not None):
164 + raise RuntimeError(
165 + "chunked prefill cannot be used with prefix caching "
166 + "now.")
167 +
168 + token_chunk_size = seq_group_metadata.token_chunk_size
169 seq_data = seq_group_metadata.seq_data[seq_id]
160 - prompt_tokens = seq_data.get_token_ids()
170 + computed_len = seq_data.get_num_computed_tokens()
171 + # We should use get_len here because in case of preemption
172 + # it contains output tokens.
173 + prefill_end = min(seq_data.get_len(),
174 + computed_len + token_chunk_size)
175 + # TODO(sang): Rename it after chunked prefill is introduced.
176 + prompt_tokens = seq_data.get_token_ids()[computed_len:prefill_end]
177 prompt_len = len(prompt_tokens)
178 + # Right now, the prefill_end is always same as the length of
179 + # sequence. However, once chunked prefill is introduced, this
180 + # assumption can be changed.
181 + assert prefill_end == seq_data.get_len()
182 prompt_lens.append(prompt_len)
163 - computed_len = 0
183
184 # NOTE: This only works for oooooooxxx style attention.
166 - computed_block_nums = seq_group_metadata.computed_block_nums
185 if computed_block_nums is not None and len(
186 computed_block_nums) > 0 and self.sliding_window is None:
187 # Prefix is not supported with sliding_window
188 computed_len = len(computed_block_nums) * self.block_size
189 prompt_tokens = prompt_tokens[computed_len:]
190 prefix_block_tables.append(computed_block_nums)
173 - context_len = computed_len
191 else:
192 prefix_block_tables.append([])
176 - context_len = 0
193 + # Right now, prefill start is always 0. However, this
194 + # assumption can be changed once chunked prefill is introduced.
195 + assert computed_len == 0
196 +
197 # actual prompt lens
178 - context_lens.append(context_len)
198 + context_lens.append(computed_len)
199 subquery_lens.append(prompt_len - computed_len)
200
201 input_tokens.extend(prompt_tokens)
202 # NOTE(woosuk): Here we assume that the first token in the prompt
203 # is always the first token in the sequence.
184 - input_positions.extend(
185 - list(range(computed_len, computed_len + len(prompt_tokens))))
204 + input_positions.extend(list(range(computed_len, prefill_end)))
205
206 lora_id = seq_group_metadata.lora_int_id
207
@@ -218,7 +237,8 @@ class ModelRunner:
237 "Prefix caching is currently not supported with "
238 "sliding window attention")
239 start_idx = max(0, prompt_len - self.sliding_window)
221 - for i in range(computed_len, prompt_len):
240 +
241 + for i in range(computed_len, prefill_end):
242 if i < start_idx:
243 slot_mapping.append(_PAD_SLOT_ID)
244 continue
@@ -331,6 +351,7 @@ class ModelRunner:
351
352 for seq_group_metadata in seq_group_metadata_list:
353 assert not seq_group_metadata.is_prompt
354 + assert seq_group_metadata.token_chunk_size == 1
355
356 seq_ids = list(seq_group_metadata.seq_data.keys())
357 lora_id = seq_group_metadata.lora_int_id