A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[core] [3/N] multi-step args and sequence.py (#7452)
William Lin committed
Aug 14, 2024 at 12:32 UTC
2ecf7b175703de020943b33532baaf6a31f69d3a
4 files changed
+100
-5
vllm/config.py
+13
-1
@@ -847,7 +847,8 @@ class SchedulerConfig:
847
delay_factor: float = 0.0,
848
enable_chunked_prefill: bool = False,
849
embedding_mode: Optional[bool] = False,
850
- preemption_mode: Optional[str] = None) -> None:
850
+ preemption_mode: Optional[str] = None,
851
+ num_scheduler_steps: int = 1) -> None:
852
if max_num_batched_tokens is not None:
853
self.max_num_batched_tokens = max_num_batched_tokens
854
else:
@@ -876,6 +877,7 @@ class SchedulerConfig:
877
self.chunked_prefill_enabled = enable_chunked_prefill
878
self.embedding_mode = embedding_mode
879
self.preemption_mode = preemption_mode
880
+ self.num_scheduler_steps = num_scheduler_steps
881
self._verify_args()
882
883
def _verify_args(self) -> None:
@@ -901,6 +903,16 @@ class SchedulerConfig:
903
f"({self.num_lookahead_slots}) must be greater than or "
904
"equal to 0.")
905
906
+ if self.num_scheduler_steps < 1:
907
+ raise ValueError(
908
+ "num_scheduler_steps "
909
+ f"({self.num_scheduler_steps}) must be greater than or "
910
+ "equal to 1.")
911
+
912
+ @property
913
+ def is_multi_step(self) -> bool:
914
+ return self.num_scheduler_steps > 1
915
+
916
917
class DeviceConfig:
918
device: Optional[torch.device]
vllm/core/scheduler.py
+5
@@ -805,6 +805,9 @@ class Scheduler:
805
curr_loras.add(lora_int_id)
806
waiting_queue.popleft()
807
self._allocate_and_set_running(seq_group)
808
+ seq_group.init_multi_step(
809
+ num_scheduler_steps=self._get_num_lookahead_slots(
810
+ is_prefill=True) + 1)
811
seq_groups.append(
812
ScheduledSequenceGroup(seq_group=seq_group,
813
token_chunk_size=num_new_tokens))
@@ -1108,6 +1111,7 @@ class Scheduler:
1111
computed_block_nums=common_computed_block_nums,
1112
encoder_seq_data=encoder_seq_data,
1113
cross_block_table=cross_block_table,
1114
+ state=seq_group.state,
1115
# `multi_modal_data` will only be present for the 1st comm
1116
# between engine and worker.
1117
# the subsequent comms can still use delta, but
@@ -1184,6 +1188,7 @@ class Scheduler:
1188
slots.
1189
"""
1190
num_lookahead_slots = self._get_num_lookahead_slots(is_prefill=False)
1191
+ seq_group.init_multi_step(num_scheduler_steps=num_lookahead_slots + 1)
1192
1193
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
1194
cows = self.block_manager.append_slots(seq, num_lookahead_slots)
vllm/engine/arg_utils.py
+25
-3
@@ -115,6 +115,7 @@ class EngineArgs:
115
lora_dtype: str = 'auto'
116
max_cpu_loras: Optional[int] = None
117
device: str = 'auto'
118
+ num_scheduler_steps: int = 1
119
ray_workers_use_nsight: bool = False
120
num_gpu_blocks_override: Optional[int] = None
121
num_lookahead_slots: int = 0
@@ -543,6 +544,11 @@ class EngineArgs:
544
"tpu", "xpu"
545
],
546
help='Device type for vLLM execution.')
547
+ parser.add_argument('--num-scheduler-steps',
548
+ type=int,
549
+ default=1,
550
+ help=('Maximum number of forward steps per '
551
+ 'scheduler call.'))
552
553
parser.add_argument(
554
'--scheduler-delay-factor',
@@ -858,18 +864,34 @@ class EngineArgs:
864
disable_logprobs=self.disable_logprobs_during_spec_decoding,
865
)
866
867
+ if self.num_scheduler_steps > 1:
868
+ raise NotImplementedError("Multi-step is not yet supported.")
869
+ if speculative_config is not None:
870
+ raise ValueError("Speculative decoding is not supported with "
871
+ "multi-step (--num-scheduler-steps > 1)")
872
+ if self.enable_chunked_prefill:
873
+ raise ValueError("Chunked prefill is not supported with "
874
+ "multi-step (--num-scheduler-steps > 1)")
875
+
876
+ # make sure num_lookahead_slots is set the higher value depending on
877
+ # if we are using speculative decoding or multi-step
878
+ num_lookahead_slots = max(self.num_lookahead_slots,
879
+ self.num_scheduler_steps - 1)
880
+ num_lookahead_slots = num_lookahead_slots \
881
+ if speculative_config is None \
882
+ else speculative_config.num_lookahead_slots
883
+
884
scheduler_config = SchedulerConfig(
885
max_num_batched_tokens=self.max_num_batched_tokens,
886
max_num_seqs=self.max_num_seqs,
887
max_model_len=model_config.max_model_len,
888
use_v2_block_manager=self.use_v2_block_manager,
866
- num_lookahead_slots=(self.num_lookahead_slots
867
- if speculative_config is None else
868
- speculative_config.num_lookahead_slots),
889
+ num_lookahead_slots=num_lookahead_slots,
890
delay_factor=self.scheduler_delay_factor,
891
enable_chunked_prefill=self.enable_chunked_prefill,
892
embedding_mode=model_config.embedding_mode,
893
preemption_mode=self.preemption_mode,
894
+ num_scheduler_steps=self.num_scheduler_steps,
895
)
896
lora_config = LoRAConfig(
897
max_lora_rank=self.max_lora_rank,
vllm/sequence.py
+57
-1
@@ -8,6 +8,7 @@ from dataclasses import dataclass, field
8
from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
9
Union, cast)
10
11
+import numpy
12
import torch
13
14
from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs
@@ -489,6 +490,19 @@ class Sequence:
490
f"num_blocks={self.n_blocks}, ")
491
492
493
+@dataclass
494
+class SequenceGroupState:
495
+ """Mutable state tied to a specific sequence group"""
496
+
497
+ # for multi-step decoding
498
+ num_steps: int = 1
499
+ current_step: int = 0
500
+
501
+ @property
502
+ def remaining_steps(self) -> int:
503
+ return self.num_steps - self.current_step
504
+
505
+
506
class SequenceGroup:
507
"""A group of sequences that are generated from the same prompt.
508
@@ -534,6 +548,7 @@ class SequenceGroup:
548
time_in_queue=None)
549
self.lora_request = lora_request
550
self.prompt_logprobs: Optional[PromptLogprobs] = None
551
+ self.state = SequenceGroupState()
552
self.embeddings = embeddings
553
self.pooling_params = pooling_params
554
self.prompt_adapter_request = prompt_adapter_request
@@ -588,6 +603,10 @@ class SequenceGroup:
603
return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens\
604
if self.prompt_adapter_request else 0
605
606
+ def init_multi_step(self, num_scheduler_steps: int) -> None:
607
+ self.state.num_steps = num_scheduler_steps
608
+ self.state.current_step = 0
609
+
610
def get_last_latency(self, now: float) -> Optional[float]:
611
"""Sets the last token time for Request level timings."""
612
# If still in prefill phase, raise Error.
@@ -756,6 +775,7 @@ class SequenceGroupMetadata:
775
lora_request: LoRA request.
776
computed_block_nums: The block numbers that are already computed,
777
used in prefix caching.
778
+ state: Internal state tied to this sequence group.
779
multi_modal_data: Multi modal data.
780
encoder_seq_data: Optional sequence data for encoder prompt
781
(SequenceGroup.encoder_seq). Should be None
@@ -781,6 +801,7 @@ class SequenceGroupMetadata:
801
token_chunk_size: Optional[int] = None,
802
lora_request: Optional[LoRARequest] = None,
803
computed_block_nums: Optional[List[int]] = None,
804
+ state: Optional[SequenceGroupState] = None,
805
multi_modal_data: Optional["MultiModalDataDict"] = None,
806
encoder_seq_data: Optional[SequenceData] = None,
807
cross_block_table: Optional[List[int]] = None,
@@ -796,6 +817,7 @@ class SequenceGroupMetadata:
817
self.prompt_adapter_request = prompt_adapter_request
818
self.computed_block_nums = computed_block_nums
819
self.multi_modal_data = multi_modal_data
820
+ self.state = SequenceGroupState() if state is None else state
821
self.encoder_seq_data = encoder_seq_data
822
self.cross_block_table = cross_block_table
823
self._token_chunk_size = token_chunk_size
@@ -834,6 +856,10 @@ class SequenceGroupMetadata:
856
assert self._token_chunk_size is not None
857
return self._token_chunk_size
858
859
+ def finish_step(self) -> None:
860
+ assert self.state.current_step < self.state.num_steps
861
+ self.state.current_step += 1
862
+
863
864
class SequenceOutput:
865
"""The model output associated with a sequence.
@@ -971,6 +997,7 @@ class SamplerOutput:
997
998
# On-device tensor containing the sampled token ids.
999
sampled_token_ids: Optional[torch.Tensor] = None
1000
+ sampled_token_ids_numpy: Optional[numpy.ndarray] = None
1001
1002
# Spec decode metrics populated by workers.
1003
spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
@@ -1112,6 +1139,33 @@ class ExecuteModelRequest:
1139
num_steps: int = 1
1140
# Finished request ids since last step.
1141
finished_requests_ids: List[str] = field(default_factory=list)
1142
+ # The last sampled token ids for multi step decoding.
1143
+ last_sampled_token_ids: Optional[torch.Tensor] = None
1144
+
1145
+ @property
1146
+ def is_first_multi_step(self) -> bool:
1147
+ # TODO(will) make this be able to handle batches with variable number of
1148
+ # steps
1149
+ assert len(self.seq_group_metadata_list) > 0
1150
+ first_seq_group = self.seq_group_metadata_list[0]
1151
+ return first_seq_group.state.current_step == 0
1152
+
1153
+ @property
1154
+ def is_last_step(self) -> bool:
1155
+ # TODO(will) make this be able to handle batches with variable number of
1156
+ # steps
1157
+ assert len(self.seq_group_metadata_list) > 0
1158
+ first_seq_group = self.seq_group_metadata_list[0]
1159
+ num_steps = first_seq_group.state.num_steps
1160
+ current_step = first_seq_group.state.current_step
1161
+ return num_steps - current_step == 1
1162
+
1163
+ @property
1164
+ def current_step(self) -> int:
1165
+ # TODO(will) make this be able to handle batches with variable number of
1166
+ # steps
1167
+ assert len(self.seq_group_metadata_list) > 0
1168
+ return self.seq_group_metadata_list[0].state.current_step
1169
1170
def clone(
1171
self, seq_group_metadata_list: List[SequenceGroupMetadata]
@@ -1127,4 +1181,6 @@ class ExecuteModelRequest:
1181
running_queue_size=self.running_queue_size,
1182
previous_hidden_states=self.previous_hidden_states,
1183
num_steps=self.num_steps,
1130
- finished_requests_ids=self.finished_requests_ids)
1184
+ finished_requests_ids=self.finished_requests_ids,
1185
+ last_sampled_token_ids=self.last_sampled_token_ids.clone()
1186
+ if self.last_sampled_token_ids is not None else None)