@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ecf7b1757
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[core] [3/N] multi-step args and sequence.py (#7452)

William Lin committed Aug 14, 2024 at 12:32 UTC 2ecf7b175703de020943b33532baaf6a31f69d3a
4 files changed +100 -5
vllm/config.py
+13 -1
@@ -847,7 +847,8 @@ class SchedulerConfig:
847 delay_factor: float = 0.0,
848 enable_chunked_prefill: bool = False,
849 embedding_mode: Optional[bool] = False,
850 - preemption_mode: Optional[str] = None) -> None:
850 + preemption_mode: Optional[str] = None,
851 + num_scheduler_steps: int = 1) -> None:
852 if max_num_batched_tokens is not None:
853 self.max_num_batched_tokens = max_num_batched_tokens
854 else:
@@ -876,6 +877,7 @@ class SchedulerConfig:
877 self.chunked_prefill_enabled = enable_chunked_prefill
878 self.embedding_mode = embedding_mode
879 self.preemption_mode = preemption_mode
880 + self.num_scheduler_steps = num_scheduler_steps
881 self._verify_args()
882
883 def _verify_args(self) -> None:
@@ -901,6 +903,16 @@ class SchedulerConfig:
903 f"({self.num_lookahead_slots}) must be greater than or "
904 "equal to 0.")
905
906 + if self.num_scheduler_steps < 1:
907 + raise ValueError(
908 + "num_scheduler_steps "
909 + f"({self.num_scheduler_steps}) must be greater than or "
910 + "equal to 1.")
911 +
912 + @property
913 + def is_multi_step(self) -> bool:
914 + return self.num_scheduler_steps > 1
915 +
916
917 class DeviceConfig:
918 device: Optional[torch.device]
vllm/core/scheduler.py
+5
@@ -805,6 +805,9 @@ class Scheduler:
805 curr_loras.add(lora_int_id)
806 waiting_queue.popleft()
807 self._allocate_and_set_running(seq_group)
808 + seq_group.init_multi_step(
809 + num_scheduler_steps=self._get_num_lookahead_slots(
810 + is_prefill=True) + 1)
811 seq_groups.append(
812 ScheduledSequenceGroup(seq_group=seq_group,
813 token_chunk_size=num_new_tokens))
@@ -1108,6 +1111,7 @@ class Scheduler:
1111 computed_block_nums=common_computed_block_nums,
1112 encoder_seq_data=encoder_seq_data,
1113 cross_block_table=cross_block_table,
1114 + state=seq_group.state,
1115 # `multi_modal_data` will only be present for the 1st comm
1116 # between engine and worker.
1117 # the subsequent comms can still use delta, but
@@ -1184,6 +1188,7 @@ class Scheduler:
1188 slots.
1189 """
1190 num_lookahead_slots = self._get_num_lookahead_slots(is_prefill=False)
1191 + seq_group.init_multi_step(num_scheduler_steps=num_lookahead_slots + 1)
1192
1193 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
1194 cows = self.block_manager.append_slots(seq, num_lookahead_slots)
vllm/engine/arg_utils.py
+25 -3
@@ -115,6 +115,7 @@ class EngineArgs:
115 lora_dtype: str = 'auto'
116 max_cpu_loras: Optional[int] = None
117 device: str = 'auto'
118 + num_scheduler_steps: int = 1
119 ray_workers_use_nsight: bool = False
120 num_gpu_blocks_override: Optional[int] = None
121 num_lookahead_slots: int = 0
@@ -543,6 +544,11 @@ class EngineArgs:
544 "tpu", "xpu"
545 ],
546 help='Device type for vLLM execution.')
547 + parser.add_argument('--num-scheduler-steps',
548 + type=int,
549 + default=1,
550 + help=('Maximum number of forward steps per '
551 + 'scheduler call.'))
552
553 parser.add_argument(
554 '--scheduler-delay-factor',
@@ -858,18 +864,34 @@ class EngineArgs:
864 disable_logprobs=self.disable_logprobs_during_spec_decoding,
865 )
866
867 + if self.num_scheduler_steps > 1:
868 + raise NotImplementedError("Multi-step is not yet supported.")
869 + if speculative_config is not None:
870 + raise ValueError("Speculative decoding is not supported with "
871 + "multi-step (--num-scheduler-steps > 1)")
872 + if self.enable_chunked_prefill:
873 + raise ValueError("Chunked prefill is not supported with "
874 + "multi-step (--num-scheduler-steps > 1)")
875 +
876 + # make sure num_lookahead_slots is set the higher value depending on
877 + # if we are using speculative decoding or multi-step
878 + num_lookahead_slots = max(self.num_lookahead_slots,
879 + self.num_scheduler_steps - 1)
880 + num_lookahead_slots = num_lookahead_slots \
881 + if speculative_config is None \
882 + else speculative_config.num_lookahead_slots
883 +
884 scheduler_config = SchedulerConfig(
885 max_num_batched_tokens=self.max_num_batched_tokens,
886 max_num_seqs=self.max_num_seqs,
887 max_model_len=model_config.max_model_len,
888 use_v2_block_manager=self.use_v2_block_manager,
866 - num_lookahead_slots=(self.num_lookahead_slots
867 - if speculative_config is None else
868 - speculative_config.num_lookahead_slots),
889 + num_lookahead_slots=num_lookahead_slots,
890 delay_factor=self.scheduler_delay_factor,
891 enable_chunked_prefill=self.enable_chunked_prefill,
892 embedding_mode=model_config.embedding_mode,
893 preemption_mode=self.preemption_mode,
894 + num_scheduler_steps=self.num_scheduler_steps,
895 )
896 lora_config = LoRAConfig(
897 max_lora_rank=self.max_lora_rank,
vllm/sequence.py
+57 -1
@@ -8,6 +8,7 @@ from dataclasses import dataclass, field
8 from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
9 Union, cast)
10
11 +import numpy
12 import torch
13
14 from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs
@@ -489,6 +490,19 @@ class Sequence:
490 f"num_blocks={self.n_blocks}, ")
491
492
493 +@dataclass
494 +class SequenceGroupState:
495 + """Mutable state tied to a specific sequence group"""
496 +
497 + # for multi-step decoding
498 + num_steps: int = 1
499 + current_step: int = 0
500 +
501 + @property
502 + def remaining_steps(self) -> int:
503 + return self.num_steps - self.current_step
504 +
505 +
506 class SequenceGroup:
507 """A group of sequences that are generated from the same prompt.
508
@@ -534,6 +548,7 @@ class SequenceGroup:
548 time_in_queue=None)
549 self.lora_request = lora_request
550 self.prompt_logprobs: Optional[PromptLogprobs] = None
551 + self.state = SequenceGroupState()
552 self.embeddings = embeddings
553 self.pooling_params = pooling_params
554 self.prompt_adapter_request = prompt_adapter_request
@@ -588,6 +603,10 @@ class SequenceGroup:
603 return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens\
604 if self.prompt_adapter_request else 0
605
606 + def init_multi_step(self, num_scheduler_steps: int) -> None:
607 + self.state.num_steps = num_scheduler_steps
608 + self.state.current_step = 0
609 +
610 def get_last_latency(self, now: float) -> Optional[float]:
611 """Sets the last token time for Request level timings."""
612 # If still in prefill phase, raise Error.
@@ -756,6 +775,7 @@ class SequenceGroupMetadata:
775 lora_request: LoRA request.
776 computed_block_nums: The block numbers that are already computed,
777 used in prefix caching.
778 + state: Internal state tied to this sequence group.
779 multi_modal_data: Multi modal data.
780 encoder_seq_data: Optional sequence data for encoder prompt
781 (SequenceGroup.encoder_seq). Should be None
@@ -781,6 +801,7 @@ class SequenceGroupMetadata:
801 token_chunk_size: Optional[int] = None,
802 lora_request: Optional[LoRARequest] = None,
803 computed_block_nums: Optional[List[int]] = None,
804 + state: Optional[SequenceGroupState] = None,
805 multi_modal_data: Optional["MultiModalDataDict"] = None,
806 encoder_seq_data: Optional[SequenceData] = None,
807 cross_block_table: Optional[List[int]] = None,
@@ -796,6 +817,7 @@ class SequenceGroupMetadata:
817 self.prompt_adapter_request = prompt_adapter_request
818 self.computed_block_nums = computed_block_nums
819 self.multi_modal_data = multi_modal_data
820 + self.state = SequenceGroupState() if state is None else state
821 self.encoder_seq_data = encoder_seq_data
822 self.cross_block_table = cross_block_table
823 self._token_chunk_size = token_chunk_size
@@ -834,6 +856,10 @@ class SequenceGroupMetadata:
856 assert self._token_chunk_size is not None
857 return self._token_chunk_size
858
859 + def finish_step(self) -> None:
860 + assert self.state.current_step < self.state.num_steps
861 + self.state.current_step += 1
862 +
863
864 class SequenceOutput:
865 """The model output associated with a sequence.
@@ -971,6 +997,7 @@ class SamplerOutput:
997
998 # On-device tensor containing the sampled token ids.
999 sampled_token_ids: Optional[torch.Tensor] = None
1000 + sampled_token_ids_numpy: Optional[numpy.ndarray] = None
1001
1002 # Spec decode metrics populated by workers.
1003 spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
@@ -1112,6 +1139,33 @@ class ExecuteModelRequest:
1139 num_steps: int = 1
1140 # Finished request ids since last step.
1141 finished_requests_ids: List[str] = field(default_factory=list)
1142 + # The last sampled token ids for multi step decoding.
1143 + last_sampled_token_ids: Optional[torch.Tensor] = None
1144 +
1145 + @property
1146 + def is_first_multi_step(self) -> bool:
1147 + # TODO(will) make this be able to handle batches with variable number of
1148 + # steps
1149 + assert len(self.seq_group_metadata_list) > 0
1150 + first_seq_group = self.seq_group_metadata_list[0]
1151 + return first_seq_group.state.current_step == 0
1152 +
1153 + @property
1154 + def is_last_step(self) -> bool:
1155 + # TODO(will) make this be able to handle batches with variable number of
1156 + # steps
1157 + assert len(self.seq_group_metadata_list) > 0
1158 + first_seq_group = self.seq_group_metadata_list[0]
1159 + num_steps = first_seq_group.state.num_steps
1160 + current_step = first_seq_group.state.current_step
1161 + return num_steps - current_step == 1
1162 +
1163 + @property
1164 + def current_step(self) -> int:
1165 + # TODO(will) make this be able to handle batches with variable number of
1166 + # steps
1167 + assert len(self.seq_group_metadata_list) > 0
1168 + return self.seq_group_metadata_list[0].state.current_step
1169
1170 def clone(
1171 self, seq_group_metadata_list: List[SequenceGroupMetadata]
@@ -1127,4 +1181,6 @@ class ExecuteModelRequest:
1181 running_queue_size=self.running_queue_size,
1182 previous_hidden_states=self.previous_hidden_states,
1183 num_steps=self.num_steps,
1130 - finished_requests_ids=self.finished_requests_ids)
1184 + finished_requests_ids=self.finished_requests_ids,
1185 + last_sampled_token_ids=self.last_sampled_token_ids.clone()
1186 + if self.last_sampled_token_ids is not None else None)