@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c2ec430ab5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core] Multi-Step + Single Step Prefills via Chunked Prefill code path (#8378)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Sep 27, 2024 at 16:32 UTC c2ec430ab5713d0626c1a7809718ef6c4eebf389
19 files changed +514 -109
csrc/prepare_inputs/advance_step.cu
+1 -1
@@ -52,7 +52,7 @@ __global__ void advance_step_flashattn_kernel(
52 slot_mapping_ptr[cur_query_id] = slot_num;
53 }
54
55 -inline void verify_tensor(std::string const& name, torch::Tensor& t,
55 +inline void verify_tensor(std::string const& name, torch::Tensor const& t,
56 int64_t const size_0, int64_t const size_1,
57 c10::ScalarType const type) {
58 bool size_0_cond = true;
tests/multi_step/test_correctness_async_llm.py
+9
@@ -37,6 +37,7 @@ DEFAULT_SERVER_ARGS: List[str] = [
37 @pytest.mark.parametrize("num_logprobs", [5])
38 @pytest.mark.parametrize("is_async", [True])
39 @pytest.mark.parametrize("attention_backend", ["FLASHINFER", "FLASH_ATTN"])
40 +@pytest.mark.parametrize("enable_chunked_prefill", [True, False])
41 @pytest.mark.asyncio
42 async def test_multi_step(
43 example_prompts,
@@ -49,6 +50,7 @@ async def test_multi_step(
50 is_async: bool,
51 num_logprobs: Optional[int],
52 attention_backend: str,
53 + enable_chunked_prefill: bool,
54 monkeypatch,
55 ) -> None:
56 """Test vLLM engine with multi-step scheduling in an OpenAI-protocol
@@ -74,6 +76,10 @@ async def test_multi_step(
76 num_logprobs: corresponds to the `logprobs` argument to the OpenAI
77 completions endpoint; `None` -> no logprobs
78 """
79 + if enable_chunked_prefill and \
80 + (pp_size > 1 or attention_backend != "FLASH_ATTN"):
81 + pytest.skip("Multi-step with Chunked-Prefill only supports"
82 + "PP=1 and FLASH_ATTN backend")
83
84 override_backend_env_variable(monkeypatch, attention_backend)
85
@@ -93,6 +99,9 @@ async def test_multi_step(
99 if eager_mode:
100 ms_server_args.append("--enforce-eager")
101
102 + if enable_chunked_prefill:
103 + ms_server_args.append("--enable-chunked-prefill")
104 +
105 distributed_args = [
106 "--tensor-parallel-size",
107 str(tp_size),
tests/multi_step/test_correctness_llm.py
+4
@@ -16,6 +16,7 @@ NUM_PROMPTS = [10]
16 @pytest.mark.parametrize("model", MODELS)
17 @pytest.mark.parametrize("dtype", ["half"])
18 @pytest.mark.parametrize("tp_size", [1])
19 +@pytest.mark.parametrize("enable_chunked_prefill", [False, True])
20 @pytest.mark.parametrize("max_tokens", [5])
21 @pytest.mark.parametrize("enforce_eager", [True])
22 @pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
@@ -28,6 +29,7 @@ def test_multi_step_llm(
29 model: str,
30 dtype: str,
31 tp_size: int,
32 + enable_chunked_prefill: bool,
33 max_tokens: int,
34 enforce_eager: int,
35 num_scheduler_steps: int,
@@ -51,6 +53,7 @@ def test_multi_step_llm(
53 model: model under test (same for single- and multi-step engines)
54 dtype: tensor datatype for engine to utilize
55 tp_size: degree of tensor-parallelism
56 + enable_chunked_prefill: chunked-prefill on/off
57 max_tokens: the maximum number of tokens to generate
58 enforce_eager
59 num_scheduler_steps: for multi-step scheduling, GPU-side steps per
@@ -73,6 +76,7 @@ def test_multi_step_llm(
76 gpu_memory_utilization=0.7,
77 tensor_parallel_size=tp_size,
78 use_v2_block_manager=True,
79 + enable_chunked_prefill=enable_chunked_prefill,
80 num_scheduler_steps=num_scheduler_steps,
81 ) as vllm_model:
82 vllm_outputs = (vllm_model.generate_greedy(prompts, max_tokens)
vllm/attention/backends/flash_attn.py
+27 -5
@@ -342,9 +342,13 @@ class FlashAttentionMetadata(AttentionMetadata):
342 )
343 return self._cached_decode_metadata
344
345 - def advance_step(self, model_input: "ModelInputForGPUWithSamplingMetadata",
345 + def advance_step(self,
346 + model_input: "ModelInputForGPUWithSamplingMetadata",
347 sampled_token_ids: Optional[torch.Tensor],
347 - block_size: int, num_seqs: int, num_queries: int):
348 + block_size: int,
349 + num_seqs: int,
350 + num_queries: int,
351 + turn_prefills_into_decodes: bool = False):
352 """
353 Update metadata in-place to advance one decode step.
354 """
@@ -355,6 +359,23 @@ class FlashAttentionMetadata(AttentionMetadata):
359 assert num_seqs > num_queries
360 assert self.use_cuda_graph
361
362 + if turn_prefills_into_decodes:
363 + # When Mutli-Step is enabled with Chunked-Prefill, prefills and
364 + # decodes are scheduled together. In the first step, all the
365 + # prefills turn into decodes. This update reflects that
366 + # conversion.
367 + assert self.num_decode_tokens + self.num_prefills == num_seqs
368 + self.num_decode_tokens += self.num_prefills
369 + self.num_prefills = 0
370 + self.num_prefill_tokens = 0
371 + self.max_prefill_seq_len = 0
372 + self.max_query_len = 1
373 +
374 + self.slot_mapping = self.slot_mapping[:num_seqs]
375 + else:
376 + assert self.seq_lens is not None
377 + assert self.max_decode_seq_len == max(self.seq_lens)
378 +
379 assert self.num_prefills == 0
380 assert self.num_prefill_tokens == 0
381 assert self.num_decode_tokens == num_seqs
@@ -366,7 +387,6 @@ class FlashAttentionMetadata(AttentionMetadata):
387 assert self.seq_lens_tensor.shape == (num_seqs, )
388 assert self.max_query_len == 1
389 assert self.max_prefill_seq_len == 0
369 - assert self.max_decode_seq_len == max(self.seq_lens)
390
391 assert self.query_start_loc is not None
392 assert self.query_start_loc.shape == (num_queries + 1, )
@@ -706,8 +726,10 @@ class FlashAttentionImpl(AttentionImpl):
726
727 num_prefill_tokens = attn_metadata.num_prefill_tokens
728 num_decode_tokens = attn_metadata.num_decode_tokens
709 - assert key.shape[0] == num_prefill_tokens + num_decode_tokens
710 - assert value.shape[0] == num_prefill_tokens + num_decode_tokens
729 + assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
730 + f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
731 + assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
732 + f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
733
734 # Query for decode. KV is not needed because it is already cached.
735 decode_query = query[num_prefill_tokens:]
vllm/attention/backends/flashinfer.py
+12 -8
@@ -410,18 +410,22 @@ class FlashInferMetadata(AttentionMetadata):
410
411 return self
412
413 - def advance_step(
414 - self,
415 - model_input: "ModelInputForGPUWithSamplingMetadata",
416 - sampled_token_ids: Optional[torch.Tensor],
417 - block_size: int,
418 - num_seqs: int,
419 - num_queries: int,
420 - ):
413 + def advance_step(self,
414 + model_input: "ModelInputForGPUWithSamplingMetadata",
415 + sampled_token_ids: Optional[torch.Tensor],
416 + block_size: int,
417 + num_seqs: int,
418 + num_queries: int,
419 + turn_prefills_into_decodes: bool = False):
420 """
421 Update metadata in-place to advance one decode step.
422 """
423
424 + assert not turn_prefills_into_decodes, \
425 + ("Chunked prefill is not supported with flashinfer yet."
426 + "turn_prefills_into_decodes is a Multi-Step + Chunked-Prefill "
427 + "specific parameter.")
428 +
429 assert num_seqs > 0
430 assert num_queries > 0
431 assert model_input.attn_metadata is not None
vllm/config.py
+10 -3
@@ -983,9 +983,16 @@ class SchedulerConfig:
983 policy: str = "fcfs") -> None:
984 if max_num_batched_tokens is None:
985 if enable_chunked_prefill:
986 - # It is the values that have the best balance between ITL
987 - # and TTFT on A100. Note it is not optimized for throughput.
988 - max_num_batched_tokens = 512
986 + if num_scheduler_steps > 1:
987 + # Multi-step Chunked-Prefill doesn't allow prompt-chunking
988 + # for now. Have max_num_batched_tokens set to max_model_len
989 + # so we don't reject sequences on account of a short
990 + # max_num_batched_tokens.
991 + max_num_batched_tokens = max(max_model_len, 2048)
992 + else:
993 + # It is the values that have the best balance between ITL
994 + # and TTFT on A100. Note it is not optimized for throughput.
995 + max_num_batched_tokens = 512
996 else:
997 # If max_model_len is too short, use 2048 as the default value
998 # for higher throughput.
vllm/core/block/block_table.py
+9 -4
@@ -55,9 +55,12 @@ class BlockTable:
55 self._num_full_slots = self._get_num_token_ids()
56
57 @staticmethod
58 - def get_num_required_blocks(token_ids: List[int], block_size: int) -> int:
58 + def get_num_required_blocks(token_ids: List[int],
59 + block_size: int,
60 + num_lookahead_slots: int = 0) -> int:
61 """Calculates the minimum number of blocks required to store a given
60 - sequence of token IDs.
62 + sequence of token IDs along with any look-ahead slots that may be
63 + required (like in multi-step + chunked-prefill).
64
65 This assumes worst-case scenario, where every block requires a new
66 allocation (e.g. ignoring prefix caching).
@@ -66,12 +69,14 @@ class BlockTable:
69 token_ids (List[int]): The sequence of token IDs to be stored.
70 block_size (int): The maximum number of tokens that can be stored in
71 a single block.
72 + num_lookahead_slots (int): look-ahead slots that the sequence may
73 + require.
74
75 Returns:
76 int: The minimum number of blocks required to store the given
72 - sequence of token IDs.
77 + sequence of token IDs along with any required look-ahead slots.
78 """
74 - return cdiv(len(token_ids), block_size)
79 + return cdiv(len(token_ids) + num_lookahead_slots, block_size)
80
81 def allocate(self,
82 token_ids: List[int],
vllm/core/block_manager_v1.py
+6 -1
@@ -281,10 +281,15 @@ class BlockSpaceManagerV1(BlockSpaceManager):
281 def _get_seq_num_required_blocks(self, seq: Optional[Sequence]) -> int:
282 return 0 if seq is None else seq.n_blocks
283
284 - def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
284 + def can_allocate(self,
285 + seq_group: SequenceGroup,
286 + num_lookahead_slots: int = 0) -> AllocStatus:
287 # FIXME(woosuk): Here we assume that all sequences in the group share
288 # the same prompt. This may not be true for preempted sequences.
289
290 + assert (num_lookahead_slots == 0
291 + ), "lookahead allocation not supported in BlockSpaceManagerV1"
292 +
293 check_no_caching_or_swa_for_blockmgr_encdec(self, seq_group)
294
295 self_num_required_blocks = self._get_seq_num_required_blocks(
vllm/core/block_manager_v2.py
+4 -1
@@ -107,7 +107,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
107 self._last_access_blocks_tracker = LastAccessBlocksTracker(
108 self.block_allocator)
109
110 - def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
110 + def can_allocate(self,
111 + seq_group: SequenceGroup,
112 + num_lookahead_slots: int = 0) -> AllocStatus:
113 # FIXME(woosuk): Here we assume that all sequences in the group share
114 # the same prompt. This may not be true for preempted sequences.
115
@@ -117,6 +119,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
119 num_required_blocks = BlockTable.get_num_required_blocks(
120 seq.get_token_ids(),
121 block_size=self.block_size,
122 + num_lookahead_slots=num_lookahead_slots,
123 )
124
125 if seq_group.is_encoder_decoder():
vllm/core/embedding_model_block_manager.py
+3 -1
@@ -21,7 +21,9 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
21 ) -> None:
22 pass
23
24 - def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
24 + def can_allocate(self,
25 + seq_group: SequenceGroup,
26 + num_lookahead_slots: int = 0) -> AllocStatus:
27 # Always return OK for dummy purposes
28 return AllocStatus.OK
29
vllm/core/interfaces.py
+3 -1
@@ -44,7 +44,9 @@ class BlockSpaceManager(ABC):
44 raise ValueError(f"Unknown version {version=}")
45
46 @abstractmethod
47 - def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
47 + def can_allocate(self,
48 + seq_group: SequenceGroup,
49 + num_lookahead_slots: int = 0) -> AllocStatus:
50 pass
51
52 @abstractmethod
vllm/core/scheduler.py
+101 -33
@@ -522,7 +522,7 @@ class Scheduler:
522 ret.swapped_out.clear()
523
524 ret.num_lookahead_slots = self._get_num_lookahead_slots(
525 - is_prefill=False)
525 + is_prefill=False, enable_chunking=enable_chunking)
526
527 ret.decode_seq_groups_list.clear()
528 ret.prefill_seq_groups_list.clear()
@@ -561,7 +561,7 @@ class Scheduler:
561
562 # NOTE(woosuk): Preemption happens only when there is no available
563 # slot to keep all the sequence groups in the RUNNING state.
564 - while not self._can_append_slots(seq_group):
564 + while not self._can_append_slots(seq_group, enable_chunking):
565 budget.subtract_num_batched_tokens(seq_group.request_id,
566 num_running_tokens)
567 num_running_seqs = seq_group.get_max_num_running_seqs()
@@ -611,7 +611,7 @@ class Scheduler:
611 if not cont_loop:
612 break
613 else:
614 - self._append_slots(seq_group, blocks_to_copy)
614 + self._append_slots(seq_group, blocks_to_copy, enable_chunking)
615 is_prefill = seq_group.is_prefill()
616
617 scheduled_seq_group: ScheduledSequenceGroup = \
@@ -684,7 +684,8 @@ class Scheduler:
684 # If the sequence group cannot be swapped in, stop.
685 is_prefill = seq_group.is_prefill()
686 alloc_status = self.block_manager.can_swap_in(
687 - seq_group, self._get_num_lookahead_slots(is_prefill))
687 + seq_group,
688 + self._get_num_lookahead_slots(is_prefill, enable_chunking))
689 if alloc_status == AllocStatus.LATER:
690 break
691 elif alloc_status == AllocStatus.NEVER:
@@ -727,7 +728,7 @@ class Scheduler:
728 curr_loras.add(lora_int_id)
729 swapped_queue.popleft()
730 self._swap_in(seq_group, blocks_to_swap_in)
730 - self._append_slots(seq_group, blocks_to_copy)
731 + self._append_slots(seq_group, blocks_to_copy, enable_chunking)
732 is_prefill = seq_group.is_prefill()
733 if is_prefill:
734 prefill_seq_groups.append(
@@ -747,12 +748,13 @@ class Scheduler:
748 blocks_to_swap_in=blocks_to_swap_in,
749 blocks_to_copy=blocks_to_copy,
750 num_lookahead_slots=self._get_num_lookahead_slots(
750 - is_prefill=False),
751 + is_prefill=False, enable_chunking=enable_chunking),
752 infeasible_seq_groups=infeasible_seq_groups,
753 )
754
755 def _get_prompt_limit(self, seq_group: SequenceGroup) -> int:
755 - if self.scheduler_config.chunked_prefill_enabled:
756 + if self.scheduler_config.chunked_prefill_enabled and \
757 + not self.scheduler_config.is_multi_step:
758 prompt_limit = self.scheduler_config.max_model_len
759 else:
760 prompt_limit = min(self.scheduler_config.max_model_len,
@@ -899,15 +901,21 @@ class Scheduler:
901 waiting_queue.popleft()
902 continue
903
904 + num_lookahead_slots: int = 0
905 + if self.scheduler_config.is_multi_step and enable_chunking:
906 + num_lookahead_slots = self._get_num_lookahead_slots(
907 + True, enable_chunking)
908 +
909 # If the sequence group cannot be allocated, stop.
903 - can_allocate = self.block_manager.can_allocate(seq_group)
910 + can_allocate = self.block_manager.can_allocate(
911 + seq_group, num_lookahead_slots=num_lookahead_slots)
912 if can_allocate == AllocStatus.LATER:
913 break
914 elif can_allocate == AllocStatus.NEVER:
915 logger.warning(
908 - "Input prompt (%d tokens) is too long"
909 - " and exceeds the capacity of block_manager",
910 - num_new_tokens)
916 + "Input prompt (%d tokens) + lookahead slots (%d) is "
917 + "too long and exceeds the capacity of block_manager",
918 + num_new_tokens, num_lookahead_slots)
919 for seq in waiting_seqs:
920 seq.status = SequenceStatus.FINISHED_IGNORED
921 ignored_seq_groups.append(seq_group)
@@ -939,9 +947,24 @@ class Scheduler:
947 curr_loras.add(lora_int_id)
948 waiting_queue.popleft()
949 self._allocate_and_set_running(seq_group)
942 - seq_group.init_multi_step(
943 - num_scheduler_steps=self._get_num_lookahead_slots(
944 - is_prefill=True) + 1)
950 +
951 + if enable_chunking and self.scheduler_config.is_multi_step:
952 + blocks_to_copy: List[Tuple[int, int]] = []
953 + # init_multi_step_from_lookahead_slots happens in append_slots
954 + self._append_slots(seq_group, blocks_to_copy, enable_chunking)
955 + # This assert will trip when a copy-on-write happens. This is
956 + # not a concern as the very first sequence-group block
957 + # allocation happens above. Still, we have the assert to
958 + # catch any edge-cases.
959 + assert not blocks_to_copy
960 + else:
961 + seq_group.init_multi_step_from_lookahead_slots(
962 + num_lookahead_slots,
963 + num_scheduler_steps=self.scheduler_config.
964 + num_scheduler_steps,
965 + is_multi_step=self.scheduler_config.is_multi_step,
966 + enable_chunking=enable_chunking)
967 +
968 seq_groups.append(
969 ScheduledSequenceGroup(seq_group=seq_group,
970 token_chunk_size=num_new_tokens))
@@ -956,7 +979,8 @@ class Scheduler:
979 return SchedulerPrefillOutputs(
980 seq_groups=seq_groups,
981 ignored_seq_groups=ignored_seq_groups,
959 - num_lookahead_slots=self._get_num_lookahead_slots(is_prefill=True))
982 + num_lookahead_slots=self._get_num_lookahead_slots(
983 + is_prefill=True, enable_chunking=enable_chunking))
984
985 def _schedule_default(self) -> SchedulerOutputs:
986 """Schedule queued requests.
@@ -1153,7 +1177,8 @@ class Scheduler:
1177 else:
1178 return self._schedule_default()
1179
1156 - def _can_append_slots(self, seq_group: SequenceGroup) -> bool:
1180 + def _can_append_slots(self, seq_group: SequenceGroup,
1181 + enable_chunking: bool) -> bool:
1182 """Determine whether or not we have enough space in the KV cache to
1183 continue generation of the sequence group.
1184 """
@@ -1164,13 +1189,17 @@ class Scheduler:
1189 self.artificial_preempt_cnt -= 1
1190 return False
1191
1167 - # Appending slots only occurs in decoding.
1168 - is_prefill = False
1192 + is_prefill = seq_group.is_prefill()
1193 + num_lookahead_slots = self._get_num_lookahead_slots(
1194 + is_prefill, enable_chunking)
1195 +
1196 + if is_prefill and num_lookahead_slots > 0:
1197 + # Appending prefill slots only happens multi-step and
1198 + # chunked-prefill are enabled together.
1199 + assert self.scheduler_config.is_multi_step and enable_chunking
1200
1201 return self.block_manager.can_append_slots(
1171 - seq_group=seq_group,
1172 - num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
1173 - )
1202 + seq_group=seq_group, num_lookahead_slots=num_lookahead_slots)
1203
1204 def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool:
1205 no_beam_search = seq_group.sampling_params is None or (
@@ -1186,7 +1215,7 @@ class Scheduler:
1215 # such as self.running, self.swapped, and self.waiting.
1216 scheduler_start_time = time.perf_counter()
1217
1189 - scheduler_outputs = self._schedule()
1218 + scheduler_outputs: SchedulerOutputs = self._schedule()
1219 now = time.time()
1220
1221 if not self.cache_config.enable_prefix_caching:
@@ -1383,11 +1412,10 @@ class Scheduler:
1412 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
1413 seq.status = SequenceStatus.RUNNING
1414
1386 - def _append_slots(
1387 - self,
1388 - seq_group: SequenceGroup,
1389 - blocks_to_copy: List[Tuple[int, int]],
1390 - ) -> None:
1415 + def _append_slots(self,
1416 + seq_group: SequenceGroup,
1417 + blocks_to_copy: List[Tuple[int, int]],
1418 + enable_chunking: bool = False) -> None:
1419 """Appends new slots to the sequences in the given sequence group.
1420
1421 Args:
@@ -1398,11 +1426,25 @@ class Scheduler:
1426 int is the destination block index. This list is updated with
1427 the new source and destination block indices for the appended
1428 slots.
1429 + enable_chunking (bool): True if chunked prefill is enabled.
1430 """
1402 - num_lookahead_slots = self._get_num_lookahead_slots(is_prefill=False)
1403 - seq_group.init_multi_step(num_scheduler_steps=num_lookahead_slots + 1)
1404 -
1405 - for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
1431 + is_prefill: bool = seq_group.is_prefill()
1432 + num_lookahead_slots: int = self._get_num_lookahead_slots(
1433 + is_prefill, enable_chunking)
1434 +
1435 + seq_group.init_multi_step_from_lookahead_slots(
1436 + num_lookahead_slots,
1437 + num_scheduler_steps=self.scheduler_config.num_scheduler_steps,
1438 + is_multi_step=self.scheduler_config.is_multi_step,
1439 + enable_chunking=enable_chunking)
1440 +
1441 + seq_status: Optional[SequenceStatus] = SequenceStatus.RUNNING
1442 + if self.scheduler_config.is_multi_step and enable_chunking:
1443 + # In multi-step chunked-prefill any sequence type can have
1444 + # slots appended.
1445 + seq_status = None
1446 +
1447 + for seq in seq_group.get_seqs(status=seq_status):
1448 cows = self.block_manager.append_slots(seq, num_lookahead_slots)
1449 if len(cows) > 0:
1450 blocks_to_copy.extend(cows)
@@ -1513,16 +1555,32 @@ class Scheduler:
1555 passed_delay = True
1556 return passed_delay
1557
1516 - def _get_num_lookahead_slots(self, is_prefill: bool) -> int:
1558 + def _get_num_lookahead_slots(self, is_prefill: bool,
1559 + enable_chunking: bool) -> int:
1560 """The number of slots to allocate per sequence per step, beyond known
1561 token ids. Speculative decoding uses these slots to store KV activations
1562 of tokens which may or may not be accepted.
1563
1564 Speculative decoding does not yet support prefill, so we do not perform
1565 lookahead allocation for prefill.
1566 +
1567 + When chunking is enabled with multi-step, we allocate lookahead slots
1568 + for the prefills for when the prefills turn into decodes in the first
1569 + step.
1570 """
1571 if is_prefill:
1525 - return 0
1572 + if self.scheduler_config.is_multi_step and enable_chunking:
1573 + # num_lookahead_slots was introduced in the context of decodes,
1574 + # in Speculative Decoding.
1575 + # When the num_scheduler_steps is 8, say, then the
1576 + # num_lookahead_slots is 7. Meaning, we are doing a 1-step of
1577 + # decode anyways and we wish to do 7 more.
1578 + #
1579 + # "lookaheads" for prefills, is introduced in support for
1580 + # Chunked-Prefill in Multi-Step.
1581 + return self.scheduler_config.num_lookahead_slots + 1
1582 + else:
1583 + return 0
1584
1585 return self.scheduler_config.num_lookahead_slots
1586
@@ -1565,6 +1623,16 @@ class Scheduler:
1623 if remaining_token_budget < num_new_tokens:
1624 num_new_tokens = (remaining_token_budget //
1625 block_size) * block_size
1626 + elif self.scheduler_config.is_multi_step:
1627 + if num_new_tokens > self._get_prompt_limit(seq_group):
1628 + # If the seq_group is in prompt-stage, pass the
1629 + # num_new_tokens as-is so the caller can ignore
1630 + # the sequence.
1631 + pass
1632 + else:
1633 + num_new_tokens = 0 \
1634 + if num_new_tokens > remaining_token_budget \
1635 + else num_new_tokens
1636 else:
1637 num_new_tokens = min(num_new_tokens, remaining_token_budget)
1638 return num_new_tokens
vllm/engine/arg_utils.py
+7 -3
@@ -980,9 +980,13 @@ class EngineArgs:
980 if speculative_config is not None:
981 raise ValueError("Speculative decoding is not supported with "
982 "multi-step (--num-scheduler-steps > 1)")
983 - if self.enable_chunked_prefill:
984 - raise ValueError("Chunked prefill is not supported with "
985 - "multi-step (--num-scheduler-steps > 1)")
983 + if self.enable_chunked_prefill and self.enable_prefix_caching:
984 + raise ValueError("Multi-Step is not supported with "
985 + "both Chunked-Prefill and Prefix-Caching "
986 + "enabled together.")
987 + if self.enable_chunked_prefill and self.pipeline_parallel_size > 1:
988 + raise ValueError("Multi-Step Chunked-Prefill is not supported "
989 + "for pipeline-parallel-size > 1")
990
991 # make sure num_lookahead_slots is set the higher value depending on
992 # if we are using speculative decoding or multi-step
vllm/engine/async_llm_engine.py
+8 -1
@@ -363,11 +363,18 @@ class _AsyncLLMEngine(LLMEngine):
363 self.cached_scheduler_outputs[
364 virtual_engine] = SchedulerOutputState()
365
366 + # is_first_step_output is True only when the num_steps of all
367 + # the sequences are 1. When the num_steps > 1,
368 + # multi_step_model_runner does the first-step output append.
369 + is_first_step_output: bool = False if not seq_group_metadata_list \
370 + else seq_group_metadata_list[0].state.num_steps == 1
371 +
372 ctx.append_output(outputs=outputs,
373 seq_group_metadata_list=seq_group_metadata_list,
374 scheduler_outputs=scheduler_outputs,
375 is_async=allow_async_output_proc,
370 - is_last_step=True)
376 + is_last_step=True,
377 + is_first_step_output=is_first_step_output)
378
379 if outputs and allow_async_output_proc:
380 assert len(
vllm/engine/llm_engine.py
+113 -17
@@ -90,6 +90,12 @@ class OutputData(NamedTuple):
90 scheduler_outputs: SchedulerOutputs
91 is_async: bool
92 is_last_step: bool
93 + # Indicates if this output is from the first step of the
94 + # multi-step. When multi-step is disabled, this is always
95 + # set to True.
96 + # is_first_step_output is invalid when `outputs` has
97 + # outputs from multiple steps.
98 + is_first_step_output: Optional[bool]
99 skip: List[int]
100
101
@@ -108,13 +114,15 @@ class SchedulerContext:
114 def append_output(self, outputs: List[SamplerOutput],
115 seq_group_metadata_list: List[SequenceGroupMetadata],
116 scheduler_outputs: SchedulerOutputs, is_async: bool,
111 - is_last_step: bool):
117 + is_last_step: bool,
118 + is_first_step_output: Optional[bool]):
119 self.output_queue.append(
120 OutputData(outputs=outputs,
121 seq_group_metadata_list=seq_group_metadata_list,
122 scheduler_outputs=scheduler_outputs,
123 is_async=is_async,
124 is_last_step=is_last_step,
125 + is_first_step_output=is_first_step_output,
126 skip=[]))
127
128
@@ -237,9 +245,10 @@ class LLMEngine:
245 "quantization_param_path=%s, device_config=%s, "
246 "decoding_config=%r, observability_config=%r, "
247 "seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
240 - "num_scheduler_steps=%d, multi_step_stream_outputs=%s, "
241 - "enable_prefix_caching=%s, use_async_output_proc=%s, "
242 - "use_cached_outputs=%s, mm_processor_kwargs=%s)",
248 + "num_scheduler_steps=%d, chunked_prefill_enabled=%s "
249 + "multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
250 + "use_async_output_proc=%s, use_cached_outputs=%s, "
251 + "mm_processor_kwargs=%s)",
252 VLLM_VERSION,
253 model_config.model,
254 speculative_config,
@@ -270,6 +279,7 @@ class LLMEngine:
279 model_config.served_model_name,
280 scheduler_config.use_v2_block_manager,
281 scheduler_config.num_scheduler_steps,
282 + scheduler_config.chunked_prefill_enabled,
283 scheduler_config.multi_step_stream_outputs,
284 cache_config.enable_prefix_caching,
285 model_config.use_async_output_proc,
@@ -957,8 +967,66 @@ class LLMEngine:
967
968 ctx: The virtual engine context to work on
969 request_id: If provided, then only this request is going to be processed
960 -
970 """
971 +
972 + def update_prefill_num_computed_tokens(
973 + seq_group: SequenceGroup,
974 + seq_group_meta: SequenceGroupMetadata, num_outputs: int,
975 + is_first_step_output: Optional[bool]) -> None:
976 + """
977 + When multi-step and chunked-prefill are enabled together, the
978 + prefill sequence scheduled for multi-step execution turn into
979 + decodes in the first step itself. This function accounts
980 + for that conversion.
981 +
982 + seq_group: SequenceGroup - A prefill seq_group
983 + seq_group_meta: SequenceGroupMetadata - Metadata of the given
984 + prefill seq_group
985 + num_outputs: int - number of output tokens being processed for the
986 + given seq_group
987 + is_first_step_output: Optional[bool] -
988 + If multi-step is enabled and num_outputs is 1, this value
989 + indicates if this outputs belongs to the first step in the
990 + multi-step.
991 + If multi-step is enabled and num_outputs > 1, this value
992 + must be None, as num_outputs > 1 indicates that outputs from
993 + all the steps in multi-step are submitted in a single burst.
994 + When multi-step is disabled, this value is always True.
995 + """
996 +
997 + assert seq_group_meta.is_prompt
998 +
999 + token_chunk_size = seq_group_meta.token_chunk_size
1000 +
1001 + if num_outputs == 1:
1002 + assert is_first_step_output is not None
1003 +
1004 + if seq_group_meta.state.num_steps == 1:
1005 + assert is_first_step_output is True
1006 + seq_group.update_num_computed_tokens(token_chunk_size)
1007 + return
1008 +
1009 + # multi-step prefill is only supported when multi-step is
1010 + # enabled with chunked prefill
1011 + assert self.scheduler_config.is_multi_step and \
1012 + self.scheduler_config.chunked_prefill_enabled
1013 + if is_first_step_output is True:
1014 + # This sequence is a prompt during the first step only.
1015 + seq_group.update_num_computed_tokens(token_chunk_size)
1016 + return
1017 +
1018 + assert is_first_step_output is None
1019 +
1020 + # multi-step prefill is only supported when multi-step is
1021 + # enabled with chunked prefill. Outputs from all the steps are
1022 + # submitted in a single burst.
1023 + assert self.scheduler_config.is_multi_step and \
1024 + self.scheduler_config.chunked_prefill_enabled
1025 + assert num_outputs == seq_group_meta.state.num_steps, \
1026 + f"#outputs {len(outputs)} - num steps {seq_group_meta.state.num_steps}" #noqa
1027 + # This sequence is a prompt during the first step only.
1028 + seq_group.update_num_computed_tokens(token_chunk_size)
1029 +
1030 now = time.time()
1031
1032 if len(ctx.output_queue) == 0:
@@ -969,20 +1037,27 @@ class LLMEngine:
1037 # When we process only one request, no pop is required
1038 # (since later we will process all of the rest)
1039 (outputs, seq_group_metadata_list, scheduler_outputs, is_async,
972 - is_last_step, skip) = ctx.output_queue[0]
1040 + is_last_step, is_first_step_output, skip) = ctx.output_queue[0]
1041 else:
1042 (outputs, seq_group_metadata_list, scheduler_outputs, is_async,
975 - is_last_step, skip) = ctx.output_queue.popleft()
1043 + is_last_step, is_first_step_output,
1044 + skip) = ctx.output_queue.popleft()
1045
1046 # Sanity check
1047 assert len(seq_group_metadata_list) == len(
1048 scheduler_outputs.scheduled_seq_groups)
1049
981 - # Organize outputs by [step][sequence group] instead of
982 - # [sequence group][step].
983 - if len(outputs) > 1:
1050 + has_multiple_outputs: bool = len(outputs) > 1
1051 + if has_multiple_outputs:
1052 + assert self.scheduler_config.is_multi_step or \
1053 + self.speculative_config
1054 + # Organize outputs by [step][sequence group] instead of
1055 + # [sequence group][step].
1056 outputs_by_sequence_group = create_output_by_sequence_group(
1057 outputs, num_seq_groups=len(seq_group_metadata_list))
1058 + # We have outputs for multiple steps submitted in a single burst,
1059 + # so invalidate is_first_step_output.
1060 + is_first_step_output = None
1061 else:
1062 outputs_by_sequence_group = outputs
1063
@@ -1018,14 +1093,17 @@ class LLMEngine:
1093 finished_before.append(i)
1094 continue
1095
1021 - if len(outputs) > 1:
1096 + if has_multiple_outputs:
1097 output = outputs_by_sequence_group[i]
1098 else:
1099 output = [outputs_by_sequence_group[0][i]]
1100
1026 - if not is_async:
1027 - seq_group.update_num_computed_tokens(
1028 - scheduled_seq_group.token_chunk_size)
1101 + if not is_async and seq_group_meta.is_prompt:
1102 + # Updates for all decodes happen when we actually append the
1103 + # token ids to the seq in process_outputs.
1104 + update_prefill_num_computed_tokens(seq_group, seq_group_meta,
1105 + len(output),
1106 + is_first_step_output)
1107
1108 if outputs:
1109 for o in outputs:
@@ -1159,8 +1237,18 @@ class LLMEngine:
1237 if seq_group.is_finished():
1238 continue
1239
1162 - seq_group.update_num_computed_tokens(
1163 - seq_group_metadata.token_chunk_size)
1240 + if seq_group_metadata.is_prompt:
1241 + if self.scheduler_config.is_multi_step and \
1242 + self.scheduler_config.chunked_prefill_enabled:
1243 + # Prompts are scheduled in multi-step only when
1244 + # chunking is enabled. These prompts turn into
1245 + # decodes after the very first step. Therefore,
1246 + # we skip the update to the num_computed_tokens
1247 + # here.
1248 + pass
1249 + else:
1250 + seq_group.update_num_computed_tokens(
1251 + seq_group_metadata.token_chunk_size)
1252
1253 if seq_group_metadata.do_sample:
1254 assert len(sequence_group_outputs.samples) == 1, (
@@ -1172,6 +1260,7 @@ class LLMEngine:
1260 assert len(seq_group.seqs) == 1
1261 seq = seq_group.seqs[0]
1262 seq.append_token_id(sample.output_token, sample.logprobs)
1263 + seq_group.update_num_computed_tokens(1)
1264
1265 def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1266 """Performs one decoding iteration and returns newly generated results.
@@ -1324,12 +1413,19 @@ class LLMEngine:
1413 if self.scheduler_config.is_multi_step:
1414 self.cached_scheduler_outputs[0] = SchedulerOutputState()
1415
1416 + # is_first_step_output is True only when the num_steps of all
1417 + # the sequences are 1. When the num_steps > 1,
1418 + # multi_step_model_runner does the first-step output append.
1419 + is_first_step_output: bool = False if not seq_group_metadata_list \
1420 + else seq_group_metadata_list[0].state.num_steps == 1
1421 +
1422 # Add results to the output_queue
1423 ctx.append_output(outputs=outputs,
1424 seq_group_metadata_list=seq_group_metadata_list,
1425 scheduler_outputs=scheduler_outputs,
1426 is_async=allow_async_output_proc,
1332 - is_last_step=True)
1427 + is_last_step=True,
1428 + is_first_step_output=is_first_step_output)
1429
1430 if outputs and allow_async_output_proc:
1431 assert len(outputs) == 1, (
vllm/engine/output_processor/multi_step.py
+1
@@ -170,6 +170,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
170 token_id=output_token_id,
171 logprobs=output_logprob,
172 )
173 + seq.data.update_num_computed_tokens(1)
174
175 self._process_decode_and_stop(seq, sampling_params)
176
vllm/sequence.py
+43 -3
@@ -743,10 +743,35 @@ class SequenceGroup:
743 return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens\
744 if self.prompt_adapter_request else 0
745
746 - def init_multi_step(self, num_scheduler_steps: int) -> None:
747 - self.state.num_steps = num_scheduler_steps
746 + def init_multi_step(self, num_steps: int) -> None:
747 + self.state.num_steps = num_steps
748 self.state.current_step = 0
749
750 + def init_multi_step_from_lookahead_slots(self, num_lookahead_slots: int,
751 + num_scheduler_steps: int,
752 + is_multi_step: bool,
753 + enable_chunking: bool) -> None:
754 +
755 + if not is_multi_step:
756 + self.init_multi_step(num_steps=num_scheduler_steps)
757 + return
758 +
759 + # Multi-Step case
760 + is_prefill = self.is_prefill()
761 +
762 + # The asserts below reflect the expectations of the current system.
763 + if is_prefill and enable_chunking:
764 + assert num_lookahead_slots == num_scheduler_steps
765 + self.init_multi_step(num_steps=num_lookahead_slots)
766 + else:
767 + is_decode: bool = not is_prefill
768 + # If it is a prefill, num_lookahead_slots must be 0
769 + assert num_lookahead_slots == 0 or is_decode
770 + # If it is a decode, num_lookahead_slots + 1 must match
771 + # the scheduler steps.
772 + assert num_lookahead_slots + 1 == num_scheduler_steps or is_prefill
773 + self.init_multi_step(num_steps=num_lookahead_slots + 1)
774 +
775 def get_last_latency(self, now: float) -> Optional[float]:
776 """Sets the last token time for Request level timings."""
777 # If still in prefill phase, raise Error.
@@ -1010,6 +1035,20 @@ class SequenceGroupMetadata(
1035 return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens \
1036 if self.prompt_adapter_request else 0
1037
1038 + # Multi-Step Chunked-Prefill property
1039 + @property
1040 + def is_single_step_prompt(self) -> bool:
1041 + # do_sample is true, only when the token_chunk_size matches the
1042 + # num_uncomputed_tokens of the sequence. This indicates that
1043 + # the prompt will finish processing in a single `execute_model`
1044 + # step.
1045 + return self.is_prompt and self.do_sample
1046 +
1047 + def get_first_seq_id(self) -> int:
1048 + # This is an efficient way of fetching the seq_id when
1049 + # we know this SequenceGroup has only one sequence.
1050 + return next(iter(self.seq_data))
1051 +
1052 def apply_delta(self,
1053 sequence_group_metadata_delta: SequenceGroupMetadataDelta):
1054 for id, delta in sequence_group_metadata_delta.seq_data_delta.items():
@@ -1022,7 +1061,8 @@ class SequenceGroupMetadata(
1061
1062 def finish_step(self) -> None:
1063 assert self.state is not None
1025 - assert self.state.current_step < self.state.num_steps
1064 + assert self.state.current_step < self.state.num_steps, \
1065 + f"current step {self.state.current_step}, num_steps {self.state.num_steps}" # noqa
1066 self.state.current_step += 1
1067
1068
vllm/worker/multi_step_model_runner.py
+150 -25
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
14 get_pythonized_sample_results)
15 from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
16 Logprob, SequenceGroupMetadata, SequenceOutput)
17 -from vllm.utils import PyObjectCache
17 +from vllm.utils import PyObjectCache, async_tensor_h2d
18 from vllm.worker.model_runner import (GPUModelRunnerBase,
19 ModelInputForGPUWithSamplingMetadata)
20 from vllm.worker.model_runner_base import (
@@ -30,6 +30,14 @@ if TYPE_CHECKING:
30 logger = init_logger(__name__)
31
32 MULTI_STEP_ATTENTION_BACKENDS = ["flash-attn", "rocm-flash-attn", "flashinfer"]
33 +MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS = ["flash-attn"]
34 +
35 +def _get_supported_attention_backends(chunked_prefill_enabled: bool) \
36 + -> List[str]:
37 + if chunked_prefill_enabled:
38 + return MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS
39 + else:
40 + return MULTI_STEP_ATTENTION_BACKENDS
41
42
43 def seq_output_builder():
@@ -144,11 +152,13 @@ class StatefulModelInput(BroadcastableModelInput):
152 is_multi_step: bool = True
153 is_last_step: bool = False
154 is_first_multi_step: bool = False
155 + base_output_proc_callback: Optional[Callable] = None
156 # ping-pong data structures for multi-step to wait on the previous step
157 step_cuda_events: List[torch.cuda.Event] = field(
158 default_factory=lambda: [torch.cuda.Event(blocking=True)] * 2)
159 num_seqs: int = -1
160 num_queries: int = -1
161 + num_single_step_prefills: int = 0
162
163 def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
164 assert self.frozen_model_input is not None
@@ -161,6 +171,7 @@ class StatefulModelInput(BroadcastableModelInput):
171 'is_first_multi_step': self.is_first_multi_step,
172 'num_seqs': self.num_seqs,
173 'num_queries': self.num_queries,
174 + 'num_single_step_prefills': self.num_single_step_prefills,
175 }
176 tensor_dict.update(new_tensor_dict)
177 return tensor_dict
@@ -209,6 +220,81 @@ class StatefulModelInput(BroadcastableModelInput):
220 sampled_token_ids=sampled_token_ids,
221 pythonized=False))
222
223 + def maybe_advance_sampling_metadata(self, device: str, pin_memory: bool):
224 + """
225 + sampling_metadata.selected_token_indices is constructed for the
226 + first-step in Multi-Step. However, when chunked-prefill is enabled with
227 + multi-step, the scheduled prompts are fully processed in the
228 + first-step and are processed as decodes in the rest of the steps.
229 + This function updates the sampling_metadata.selected_token_indices
230 + to account for this conversion.
231 +
232 + Example:
233 + Let 2 prompts and 2 decodes be scheduled together. Let the
234 + num-tokens to process for the 2 prompts be 5 and 8 respectively.
235 +
236 + In that case, sampling_metadata.sampled_token_indices will be,
237 + [4, 12, 13, 14] as it is constructed for the first-step in
238 + multi-step.
239 + However, the prompts turns to decodes after the first-step
240 + and the num-tokens for the previously-prompt sequences will
241 + be 1 and 1 as they are decodes now. The self.sampled_token_indices
242 + must be updated to [0,1,2,3].
243 + """
244 + assert self.current_step == 1 and self.num_single_step_prefills > 0
245 + if not get_pp_group().is_last_rank:
246 + return
247 +
248 + assert self.frozen_model_input is not None
249 + assert self.frozen_model_input.sampling_metadata is not None
250 + self.frozen_model_input.sampling_metadata.selected_token_indices = \
251 + async_tensor_h2d(list(range(self.num_queries)),
252 + dtype=torch.long,
253 + target_device=device,
254 + pin_memory=pin_memory)
255 +
256 + def maybe_advance_frozen_model_input(self, device: str, pin_memory: bool):
257 + """
258 + Advancing the datastructures of StatefulModelInput::frozen_model_input
259 + is only required when prefills are scheduled with decodes to run in
260 + multi-step. This advancement/correction is required to account for
261 + the conversion of Prefills to Decodes after the first multi-step.
262 + """
263 + if self.current_step != 1 or self.num_single_step_prefills == 0:
264 + return
265 +
266 + assert self.frozen_model_input is not None
267 + fmi = self.frozen_model_input
268 +
269 + # Truncate input_tokens
270 + assert fmi.input_tokens is not None
271 + assert fmi.input_tokens.shape[0] >= self.num_seqs
272 + fmi_new_input_tokens: torch.Tensor = fmi.input_tokens[:self.num_seqs]
273 +
274 + # Update frozen_model_input::input_positons.
275 + assert fmi.input_positions is not None
276 + assert fmi.input_positions.shape[0] >= self.num_seqs
277 + fmi_new_input_positions: torch.Tensor = fmi.input_positions[:self.
278 + num_seqs]
279 +
280 + # Assert unsupported
281 + assert fmi.lora_mapping is None
282 + assert fmi.lora_requests is not None
283 + assert len(fmi.lora_requests) == 0
284 + assert fmi.attn_metadata is not None
285 + assert fmi.prompt_adapter_mapping is None
286 + assert fmi.prompt_adapter_requests is not None
287 + assert len(fmi.prompt_adapter_requests) == 0
288 + assert fmi.multi_modal_kwargs is not None
289 + assert len(fmi.multi_modal_kwargs) == 0
290 +
291 + self.frozen_model_input = dataclasses.replace(
292 + self.frozen_model_input,
293 + input_tokens=fmi_new_input_tokens,
294 + input_positions=fmi_new_input_positions)
295 +
296 + self.maybe_advance_sampling_metadata(device, pin_memory)
297 +
298
299 # MutableModelInputForGPUWithMultiStepMetadata is not subclass of
300 # ModelInputForGPU but it wraps the actual input dataclass and adds multi-step
@@ -220,6 +306,19 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
306 def __init__(self, base_model_runner: GPUModelRunnerBase, *args, **kwargs):
307 super().__init__(*args, **kwargs)
308
309 + # Check attention backend support.
310 + supported_attention_backends: List[str] = \
311 + _get_supported_attention_backends(
312 + self.scheduler_config.chunked_prefill_enabled)
313 + if self.attn_backend.get_name() not in supported_attention_backends:
314 + ms_config_str: str = "Multi-Step + Chunked-Prefill" \
315 + if self.scheduler_config.chunked_prefill_enabled \
316 + else "Multi-Step"
317 + raise ValueError(
318 + f"{ms_config_str} not supported for attention backend: "
319 + f"{self.attn_backend.get_name()}. Set VLLM_ATTENTION_BACKEND "
320 + f"to a value from {supported_attention_backends}.")
321 +
322 # uses the base model runner to execute the model and wraps it with
323 # multi-step logic
324 self._base_model_runner: GPUModelRunnerBase = base_model_runner
@@ -248,14 +347,25 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
347 virtual_engine: int = 0,
348 finished_requests_ids: Optional[List[str]] = None
349 ) -> StatefulModelInput:
251 - frozen_model_input = self._base_model_runner.prepare_model_input(
252 - seq_group_metadata_list, virtual_engine, finished_requests_ids)
350 + frozen_model_input: ModelInputForGPUWithSamplingMetadata = \
351 + self._base_model_runner.prepare_model_input(
352 + seq_group_metadata_list,
353 + virtual_engine,
354 + finished_requests_ids)
355 +
356 + assert frozen_model_input.query_lens is not None
357 + assert frozen_model_input.seq_lens is not None
358 + assert frozen_model_input.attn_metadata is not None
359 + num_queries = len(frozen_model_input.query_lens)
360 + num_seqs = len(frozen_model_input.seq_lens)
361 + num_single_step_prefills = frozen_model_input.attn_metadata.num_prefills
362
363 model_input = StatefulModelInput(
364 frozen_model_input=frozen_model_input,
256 - num_seqs=len(frozen_model_input.seq_lens),
257 - num_queries=len(frozen_model_input.query_lens),
258 - )
365 + num_seqs=num_seqs,
366 + num_queries=num_queries,
367 + num_single_step_prefills=num_single_step_prefills)
368 +
369 return model_input
370
371 def _async_process_outputs(self, model_input: StatefulModelInput,
@@ -265,7 +375,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
375 output_proc_callback()
376
377 cont = True
268 - for model_output in model_input.cached_outputs:
378 + for step_num, model_output in enumerate(model_input.cached_outputs):
379 if not model_output.pythonized:
380 model_output.maybe_pythonize(model_input, self._copy_stream,
381 self.pinned_sampled_token_ids)
@@ -276,7 +386,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
386 seq_group_metadata_list=ctx.seq_group_metadata_list,
387 scheduler_outputs=ctx.scheduler_outputs,
388 is_async=False,
279 - is_last_step=False)
389 + is_last_step=False,
390 + is_first_step_output=step_num == 0)
391
392 output_proc_callback()
393 else:
@@ -292,9 +403,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
403 has_async_callback = output_proc_callback is not None
404
405 outputs = []
295 - for output_id in range(len(model_input.cached_outputs)):
296 - output = model_input.cached_outputs[output_id]
297 - is_last_step = output_id == len(model_input.cached_outputs) - 1
406 + for step_num, output in enumerate(model_input.cached_outputs):
407 + is_last_step = step_num == len(model_input.cached_outputs) - 1
408
409 # For non-async case:
410 # -- We simply add the outputs
@@ -323,7 +433,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
433 seq_group_metadata_list,
434 scheduler_outputs=ctx.scheduler_outputs,
435 is_async=False,
326 - is_last_step=False)
436 + is_last_step=False,
437 + is_first_step_output=step_num == 0)
438 else:
439 outputs.append(output.sampler_output)
440 else:
@@ -389,18 +500,27 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
500 model_input = self._advance_step(
501 model_input, model_input.cached_outputs[-1].sampler_output)
502
392 - output_proc_callback = None
503 + # frozen_model_input may have been updated
504 + frozen_model_input = model_input.frozen_model_input
505 + assert frozen_model_input is not None
506 +
507 + if model_input.base_output_proc_callback is None:
508 + assert frozen_model_input is not None
509 + model_input.base_output_proc_callback = \
510 + frozen_model_input.async_callback
511 +
512 if frozen_model_input.async_callback is not None:
394 - output_proc_callback = frozen_model_input.async_callback
395 - assert output_proc_callback is not None
513 + assert model_input.base_output_proc_callback is not None
514 async_callback = functools.partial(
515 self._async_process_outputs,
516 model_input=model_input,
399 - output_proc_callback=output_proc_callback)
517 + output_proc_callback=model_input.base_output_proc_callback)
518
401 - frozen_model_input = dataclasses.replace( # type: ignore
519 + model_input.frozen_model_input = dataclasses.replace( # type: ignore
520 model_input.frozen_model_input,
521 async_callback=async_callback)
522 + # Update the local instance
523 + frozen_model_input = model_input.frozen_model_input
524 assert frozen_model_input is not None
525
526 # Execute the model
@@ -455,8 +575,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
575
576 # Pythonize the output and block if needed since it is the last step
577 if model_input.is_last_step:
458 - outputs = self._final_process_outputs(model_input,
459 - output_proc_callback)
578 + outputs = self._final_process_outputs(
579 + model_input, model_input.base_output_proc_callback)
580 self.pythonization_cache.reset()
581 return outputs
582
@@ -484,11 +604,14 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
604
605 def _advance_step(self, model_input: StatefulModelInput,
606 out: SamplerOutput) -> StatefulModelInput:
487 - if self.attn_backend.get_name() not in MULTI_STEP_ATTENTION_BACKENDS:
488 - raise ValueError(
489 - f"Multi-step not supported for attention backend: "
490 - f"{self.attn_backend.get_name()}. Set VLLM_ATTENTION_BACKEND "
491 - f"to a value from {MULTI_STEP_ATTENTION_BACKENDS}.")
607 +
608 + model_input.maybe_advance_frozen_model_input(self.device,
609 + self.pin_memory)
610 + frozen_model_input = model_input.frozen_model_input
611 + assert frozen_model_input is not None
612 + assert frozen_model_input.input_tokens is not None
613 + assert frozen_model_input.input_tokens.shape[0] == model_input.num_seqs
614 + assert frozen_model_input.attn_metadata is not None
615
616 sampled_token_ids = model_input.cached_outputs[-1].sampled_token_ids
617 num_seqs = model_input.num_seqs
@@ -498,13 +621,15 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
621 attn_metadata = frozen_model_input.attn_metadata
622 assert attn_metadata is not None
623
624 + turn_prefills_into_decodes: bool = model_input.current_step == 1 and \
625 + model_input.num_single_step_prefills != 0
626 attn_metadata.advance_step(
627 frozen_model_input,
628 sampled_token_ids,
629 self.block_size,
630 num_seqs,
631 num_queries,
507 - )
632 + turn_prefills_into_decodes=turn_prefills_into_decodes)
633
634 return model_input
635
vllm/worker/multi_step_worker.py
+3 -2
@@ -76,8 +76,9 @@ class MultiStepWorker(Worker):
76 frozen_model_input = model_input.frozen_model_input
77 assert frozen_model_input is not None
78 assert frozen_model_input.attn_metadata is not None
79 - # clear the cached decode metadata so that it can be recomputed on
80 - # the workers
79 + # clear the cached metadata so that it can be recomputed on
80 + # the workers.
81 + frozen_model_input.attn_metadata._cached_prefill_metadata = None
82 frozen_model_input.attn_metadata._cached_decode_metadata = None
83
84 model_input.is_first_multi_step = is_first_multi_step