A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix incorrect updates to num_computed_tokens in multi-step scheduling (#9038)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Oct 6, 2024 at 15:48 UTC
cb3b2b9ba4a95c413a879e30e2b8674187519a93
6 files changed
+179
-110
tests/core/test_num_computed_tokens_update.py
new
+81
@@ -0,0 +1,81 @@
1
+import pytest
2
+
3
+from tests.conftest import VllmRunner
4
+from tests.core.utils import create_dummy_prompt
5
+from vllm.engine.llm_engine import LLMEngine
6
+from vllm.platforms import current_platform
7
+from vllm.sequence import SequenceGroup
8
+
9
+MODEL = "JackFram/llama-160m"
10
+
11
+
12
+def add_seq_group_to_engine(engine: LLMEngine, seq_group: SequenceGroup):
13
+ scheduler = engine.scheduler[0]
14
+ scheduler.add_seq_group(seq_group)
15
+
16
+
17
+@pytest.mark.parametrize("num_scheduler_steps", [1, 8])
18
+@pytest.mark.parametrize("enable_chunked_prefill", [False, True])
19
+@pytest.mark.parametrize("enforce_eager", [False, True])
20
+def test_num_computed_tokens_update(num_scheduler_steps: int,
21
+ enable_chunked_prefill: bool,
22
+ enforce_eager: bool):
23
+
24
+ is_multi_step = num_scheduler_steps > 1
25
+ is_multi_step_chunked_prefill = is_multi_step and enable_chunked_prefill
26
+
27
+ if is_multi_step_chunked_prefill and current_platform.is_rocm():
28
+ pytest.skip("Multi-step with Chunked-Prefill does not support "
29
+ "rocm_flash_attn backend")
30
+
31
+ # Make a vllm engine
32
+ runner = VllmRunner(model_name=MODEL,
33
+ gpu_memory_utilization=0.7,
34
+ use_v2_block_manager=True,
35
+ num_scheduler_steps=num_scheduler_steps,
36
+ enable_chunked_prefill=enable_chunked_prefill,
37
+ enforce_eager=enforce_eager)
38
+ engine: LLMEngine = runner.model.llm_engine
39
+
40
+ # In multi-step + chunked-prefill there is no separate single prompt step.
41
+ # What is scheduled will run for num_scheduler_steps always.
42
+ num_prompt_steps = num_scheduler_steps \
43
+ if is_multi_step_chunked_prefill else 1
44
+
45
+ num_output_tokens_list = [4, 8, 12, 15, 16, 17]
46
+
47
+ # Create sequence and add to engine
48
+ prompt_len = 10
49
+
50
+ for req_idx, num_output_tokens in enumerate(num_output_tokens_list):
51
+ seq, seq_group = create_dummy_prompt(request_id=str(req_idx),
52
+ prompt_length=prompt_len,
53
+ min_tokens=num_output_tokens,
54
+ max_tokens=num_output_tokens)
55
+ add_seq_group_to_engine(engine, seq_group)
56
+
57
+ assert seq.data.get_num_computed_tokens() == 0
58
+
59
+ for _ in range(num_prompt_steps):
60
+ # prompt steps
61
+ engine.step()
62
+
63
+ if not seq.is_finished():
64
+ prompt_num_computed_tokens = seq.data.get_num_computed_tokens()
65
+ # Test correctness of num_computed_tokens after the prompt steps
66
+ assert prompt_num_computed_tokens == \
67
+ prompt_len + num_prompt_steps - 1
68
+
69
+ decode_step_counter = 0
70
+ while not seq.is_finished():
71
+ # Test correctness of num_computed_tokens after the decode steps
72
+ assert seq.data.get_num_computed_tokens(
73
+ ) == prompt_num_computed_tokens + decode_step_counter
74
+ for _ in range(num_scheduler_steps):
75
+ # decode step
76
+ engine.step()
77
+ decode_step_counter += 1
78
+
79
+ # Test correctness of num_computed_tokens after the sequence finish.
80
+ assert seq.data.get_num_computed_tokens(
81
+ ) == prompt_len + num_output_tokens - 1
tests/core/utils.py
+5
-1
@@ -16,6 +16,8 @@ def create_dummy_prompt(
16
use_beam_search: bool = False,
17
best_of: int = 1,
18
prompt_tokens: Optional[List[int]] = None,
19
+ min_tokens: int = 0,
20
+ max_tokens: int = 16,
21
) -> Tuple[Sequence, SequenceGroup]:
22
if not block_size:
23
block_size = prompt_length
@@ -36,7 +38,9 @@ def create_dummy_prompt(
38
arrival_time=time.time(),
39
sampling_params=SamplingParams(
40
use_beam_search=use_beam_search,
39
- best_of=best_of),
41
+ best_of=best_of,
42
+ max_tokens=max_tokens,
43
+ min_tokens=min_tokens),
44
lora_request=lora_request)
45
46
return prompt, seq_group
vllm/attention/backends/rocm_flash_attn.py
+12
-2
@@ -191,12 +191,22 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
191
)
192
return self._cached_decode_metadata
193
194
- def advance_step(self, model_input: "ModelInputForGPUWithSamplingMetadata",
194
+ def advance_step(self,
195
+ model_input: "ModelInputForGPUWithSamplingMetadata",
196
sampled_token_ids: Optional[torch.Tensor],
196
- block_size: int, num_seqs: int, num_queries: int):
197
+ block_size: int,
198
+ num_seqs: int,
199
+ num_queries: int,
200
+ turn_prefills_into_decodes: bool = False):
201
"""
202
Update metadata in-place to advance one decode step.
203
"""
204
+
205
+ assert not turn_prefills_into_decodes, \
206
+ ("Chunked prefill is not supported with rocm_flash_attn yet."
207
+ "turn_prefills_into_decodes is a Multi-Step + Chunked-Prefill "
208
+ "specific parameter.")
209
+
210
# When using cudagraph, the num_seqs is padded to the next captured
211
# batch sized, but num_queries tracks the actual number of requests in
212
# the batch. For --enforce-eager mode, num_seqs == num_queries
vllm/engine/llm_engine.py
+66
-90
@@ -962,6 +962,45 @@ class LLMEngine:
962
963
return
964
965
+ def _update_num_computed_tokens_for_multi_step_prefill(
966
+ self, seq_group: SequenceGroup,
967
+ seq_group_meta: SequenceGroupMetadata,
968
+ is_first_step_output: Optional[bool]):
969
+ """
970
+ This function updates num_computed_tokens for prompt sequences
971
+ when Multi-Step is enabled.
972
+
973
+ seq_group: SequenceGroup to update the num_computed_tokens for.
974
+ seq_group_meta: Metadata of the given SequenceGroup.
975
+ is_first_step_output: Optional[bool] -
976
+ When available, is_first_step_output indicates if the appended
977
+ output token is the output of the first-step in multi-step.
978
+ A value of None indicates that outputs from all steps in
979
+ in multi-step are submitted in a single burst.
980
+ """
981
+
982
+ assert self.scheduler_config.is_multi_step
983
+
984
+ if not seq_group_meta.is_prompt:
985
+ # num_computed_token updates for multi-step decodes happen after
986
+ # the tokens are appended to the sequence.
987
+ return
988
+
989
+ do_update: bool = False
990
+ if self.scheduler_config.chunked_prefill_enabled:
991
+ # In multi-step + chunked-prefill case, the prompt sequences
992
+ # that are scheduled are fully processed in the first step.
993
+ do_update = is_first_step_output is None or is_first_step_output
994
+ else:
995
+ # Normal multi-step decoding case. In this case prompt-sequences
996
+ # are actually single-stepped. Always update in this case.
997
+ assert seq_group.state.num_steps == 1
998
+ do_update = True
999
+
1000
+ if do_update:
1001
+ seq_group.update_num_computed_tokens(
1002
+ seq_group_meta.token_chunk_size)
1003
+
1004
def _process_model_outputs(self,
1005
ctx: SchedulerContext,
1006
request_id: Optional[str] = None) -> None:
@@ -972,64 +1011,6 @@ class LLMEngine:
1011
request_id: If provided, then only this request is going to be processed
1012
"""
1013
975
- def update_prefill_num_computed_tokens(
976
- seq_group: SequenceGroup,
977
- seq_group_meta: SequenceGroupMetadata, num_outputs: int,
978
- is_first_step_output: Optional[bool]) -> None:
979
- """
980
- When multi-step and chunked-prefill are enabled together, the
981
- prefill sequence scheduled for multi-step execution turn into
982
- decodes in the first step itself. This function accounts
983
- for that conversion.
984
-
985
- seq_group: SequenceGroup - A prefill seq_group
986
- seq_group_meta: SequenceGroupMetadata - Metadata of the given
987
- prefill seq_group
988
- num_outputs: int - number of output tokens being processed for the
989
- given seq_group
990
- is_first_step_output: Optional[bool] -
991
- If multi-step is enabled and num_outputs is 1, this value
992
- indicates if this outputs belongs to the first step in the
993
- multi-step.
994
- If multi-step is enabled and num_outputs > 1, this value
995
- must be None, as num_outputs > 1 indicates that outputs from
996
- all the steps in multi-step are submitted in a single burst.
997
- When multi-step is disabled, this value is always True.
998
- """
999
-
1000
- assert seq_group_meta.is_prompt
1001
-
1002
- token_chunk_size = seq_group_meta.token_chunk_size
1003
-
1004
- if num_outputs == 1:
1005
- assert is_first_step_output is not None
1006
-
1007
- if seq_group_meta.state.num_steps == 1:
1008
- assert is_first_step_output is True
1009
- seq_group.update_num_computed_tokens(token_chunk_size)
1010
- return
1011
-
1012
- # multi-step prefill is only supported when multi-step is
1013
- # enabled with chunked prefill
1014
- assert self.scheduler_config.is_multi_step and \
1015
- self.scheduler_config.chunked_prefill_enabled
1016
- if is_first_step_output is True:
1017
- # This sequence is a prompt during the first step only.
1018
- seq_group.update_num_computed_tokens(token_chunk_size)
1019
- return
1020
-
1021
- assert is_first_step_output is None
1022
-
1023
- # multi-step prefill is only supported when multi-step is
1024
- # enabled with chunked prefill. Outputs from all the steps are
1025
- # submitted in a single burst.
1026
- assert self.scheduler_config.is_multi_step and \
1027
- self.scheduler_config.chunked_prefill_enabled
1028
- assert num_outputs == seq_group_meta.state.num_steps, \
1029
- f"#outputs {len(outputs)} - num steps {seq_group_meta.state.num_steps}" #noqa
1030
- # This sequence is a prompt during the first step only.
1031
- seq_group.update_num_computed_tokens(token_chunk_size)
1032
-
1014
now = time.time()
1015
1016
if len(ctx.output_queue) == 0:
@@ -1090,7 +1071,7 @@ class LLMEngine:
1071
seq_group_meta = seq_group_metadata_list[i]
1072
scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1073
1093
- seq_group = scheduled_seq_group.seq_group
1074
+ seq_group: SequenceGroup = scheduled_seq_group.seq_group
1075
1076
if seq_group.is_finished():
1077
finished_before.append(i)
@@ -1101,14 +1082,14 @@ class LLMEngine:
1082
else:
1083
output = [outputs_by_sequence_group[0][i]]
1084
1104
- if not is_async and seq_group_meta.is_prompt:
1105
- # Updates for all decodes happen when we actually append the
1106
- # token ids to the seq in process_outputs.
1107
- update_prefill_num_computed_tokens(seq_group, seq_group_meta,
1108
- len(output),
1109
- is_first_step_output)
1110
- elif not is_async:
1111
- seq_group.update_num_computed_tokens(1)
1085
+ if not is_async:
1086
+ if self.scheduler_config.is_multi_step:
1087
+ # Updates happen only if the sequence is prefill
1088
+ self._update_num_computed_tokens_for_multi_step_prefill(
1089
+ seq_group, seq_group_meta, is_first_step_output)
1090
+ else:
1091
+ seq_group.update_num_computed_tokens(
1092
+ seq_group_meta.token_chunk_size)
1093
1094
if outputs:
1095
for o in outputs:
@@ -1132,16 +1113,8 @@ class LLMEngine:
1113
else:
1114
self.output_processor.process_prompt_logprob(seq_group, output)
1115
if seq_group_meta.do_sample:
1135
- output_token_num = self.output_processor.process_outputs(
1116
+ self.output_processor.process_outputs(
1117
seq_group, output, is_async)
1137
- if self.speculative_config:
1138
- # We -1 here because we always
1139
- # (w/o speculative decoding) add the number of
1140
- # computed tokens by one in the decoding phase.
1141
- # Therefore, we remove that one token that
1142
- # is already added.
1143
- seq_group.update_num_computed_tokens(output_token_num -
1144
- 1)
1118
1119
if seq_group.is_finished():
1120
finished_now.append(i)
@@ -1250,20 +1223,15 @@ class LLMEngine:
1223
if seq_group.is_finished():
1224
continue
1225
1253
- if seq_group_metadata.is_prompt:
1254
- if self.scheduler_config.is_multi_step and \
1255
- self.scheduler_config.chunked_prefill_enabled:
1256
- # Prompts are scheduled in multi-step only when
1257
- # chunking is enabled. These prompts turn into
1258
- # decodes after the very first step. Therefore,
1259
- # we skip the update to the num_computed_tokens
1260
- # here.
1261
- seq_group.update_num_computed_tokens(1)
1262
- else:
1263
- seq_group.update_num_computed_tokens(
1264
- seq_group_metadata.token_chunk_size)
1226
+ if self.scheduler_config.is_multi_step:
1227
+ # Updates happen only if the sequence is prefill
1228
+ self._update_num_computed_tokens_for_multi_step_prefill(
1229
+ seq_group, seq_group_metadata,
1230
+ seq_group.state.num_steps == 1)
1231
else:
1266
- seq_group.update_num_computed_tokens(1)
1232
+ seq_group.update_num_computed_tokens(
1233
+ seq_group_metadata.token_chunk_size)
1234
+
1235
if seq_group_metadata.do_sample:
1236
assert len(sequence_group_outputs.samples) == 1, (
1237
"Async output processor expects a single sample"
@@ -1273,7 +1241,15 @@ class LLMEngine:
1241
1242
assert len(seq_group.seqs) == 1
1243
seq = seq_group.seqs[0]
1276
- seq.append_token_id(sample.output_token, sample.logprobs)
1244
+
1245
+ if self.scheduler_config.is_multi_step:
1246
+ is_prefill_append = seq.data.get_num_uncomputed_tokens(
1247
+ ) == 0
1248
+ seq.append_token_id(sample.output_token, sample.logprobs)
1249
+ if not is_prefill_append:
1250
+ seq_group.update_num_computed_tokens(1)
1251
+ else:
1252
+ seq.append_token_id(sample.output_token, sample.logprobs)
1253
1254
def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1255
"""Performs one decoding iteration and returns newly generated results.
vllm/engine/output_processor/interfaces.py
+2
-6
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Callable, List, Optional
2
+from typing import Callable, List
3
4
from vllm.config import SchedulerConfig
5
from vllm.core.scheduler import Scheduler
@@ -58,14 +58,10 @@ class SequenceGroupOutputProcessor(ABC):
58
@abstractmethod
59
def process_outputs(self, sequence_group: SequenceGroup,
60
outputs: List[SequenceGroupOutput],
61
- is_async: bool) -> Optional[int]:
61
+ is_async: bool) -> None:
62
"""Process new token ids for the sequence group. Handles logic such as
63
detokenization, stop checking, and freeing/forking sequences in the
64
scheduler.
65
-
66
- Return the number of new tokens generated in the sequence group.
67
- The returned value is optional because it is only used for
68
- speculative decoding mqa scorer.
65
"""
66
pass
67
vllm/engine/output_processor/multi_step.py
+13
-11
@@ -1,5 +1,5 @@
1
import functools
2
-from typing import Callable, List, Optional
2
+from typing import Callable, List
3
4
from vllm.core.scheduler import Scheduler
5
from vllm.engine.output_processor.interfaces import (
@@ -69,7 +69,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
69
def process_outputs(self,
70
sequence_group: SequenceGroup,
71
outputs: List[SequenceGroupOutput],
72
- is_async: bool = False) -> Optional[int]:
72
+ is_async: bool = False) -> None:
73
"""Append new tokens in the outputs to sequences in the sequence group.
74
75
This only supports sequence groups of size 1. It supports greater than
@@ -84,10 +84,6 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
84
tokens from the previous step. If this is true, then
85
no tokens need to be appended since it is already done
86
externally (before the next schedule() call)
87
-
88
- Returns:
89
- The number of tokens appended to the sequence. This is optional
90
- because only speculative decode uses this return value.
87
"""
88
# Sequences can be in RUNNING or FINISHED_ABORTED state
89
# once scheduled, as a sequence is moved to FINSIHED_ABORTED
@@ -110,7 +106,6 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
106
# was already appended, so we only need to do the rest of the
107
# postprocessor: Detokenization + stopping logic
108
self._process_decode_and_stop(seq, sequence_group.sampling_params)
113
- return None
109
else:
110
# Standard multi-step case
111
@@ -126,8 +121,8 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
121
]
122
assert valid_samples
123
129
- return self._process_seq_outputs(seq, valid_samples,
130
- sequence_group.sampling_params)
124
+ self._process_seq_outputs(seq, valid_samples,
125
+ sequence_group.sampling_params)
126
127
def _process_decode_and_stop(self, seq: Sequence,
128
sampling_params: SamplingParams) -> None:
@@ -145,7 +140,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
140
141
def _process_seq_outputs(self, seq: Sequence,
142
valid_samples: List[SequenceOutput],
148
- sampling_params: SamplingParams) -> int:
143
+ sampling_params: SamplingParams) -> None:
144
output_token_ids = [sample.output_token for sample in valid_samples]
145
output_logprobs = [sample.logprobs for sample in valid_samples]
146
@@ -168,6 +163,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
163
output_token_ids = output_token_ids[:i + 1]
164
break
165
166
+ is_prefill_sampled_token = seq.data.get_num_uncomputed_tokens() == 0
167
# Incrementally append tokens to the sequence, as if we had only one new
168
# token.
169
for output_token_id, output_logprob in zip(output_token_ids,
@@ -177,8 +173,14 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
173
logprobs=output_logprob,
174
)
175
176
+ if is_prefill_sampled_token:
177
+ is_prefill_sampled_token = False
178
+ else:
179
+ # Update num_computed_tokens iff the sampled token is not from
180
+ # a prefill step.
181
+ seq.data.update_num_computed_tokens(1)
182
+
183
self._process_decode_and_stop(seq, sampling_params)
184
185
if seq.is_finished():
186
break
184
- return len(output_token_ids)