@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 83c644fe7e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[core][misc] simply output processing with shortcut code path (#7117)

youkaichao committed Aug 4, 2024 at 00:22 UTC 83c644fe7ecee05d3ebe5057acb6e008d7e81eb8
1 file changed +31 -8
vllm/engine/output_processor/single_step.py
+31 -8
@@ -81,6 +81,29 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
81
82 def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
83 outputs: SequenceGroupOutput) -> None:
84 + sampling_params = seq_group.sampling_params
85 + if sampling_params.n == 1 and not sampling_params.use_beam_search:
86 + # only have one output sample
87 + sample = outputs.samples[0]
88 + # only have one sequence
89 + seq = seq_group.seqs[0]
90 + seq.append_token_id(sample.output_token, sample.logprobs)
91 + if sampling_params.detokenize and self.detokenizer:
92 + new_char_count = self.detokenizer.decode_sequence_inplace(
93 + seq, sampling_params)
94 + else:
95 + new_char_count = 0
96 + self.stop_checker.maybe_stop_sequence(
97 + seq,
98 + new_char_count,
99 + sampling_params,
100 + lora_req=seq_group.lora_request,
101 + )
102 + if seq.is_finished():
103 + for scheduler in self.scheduler:
104 + scheduler.free_seq(seq)
105 + return
106 +
107 # Process samples
108 samples = outputs.samples
109 parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
@@ -127,20 +150,20 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
150 child_seqs.append((parent, parent))
151
152 for seq, _ in child_seqs:
130 - if seq_group.sampling_params.detokenize and self.detokenizer:
153 + if sampling_params.detokenize and self.detokenizer:
154 new_char_count = self.detokenizer.decode_sequence_inplace(
132 - seq, seq_group.sampling_params)
155 + seq, sampling_params)
156 else:
157 new_char_count = 0
158 self.stop_checker.maybe_stop_sequence(
159 seq,
160 new_char_count,
138 - seq_group.sampling_params,
161 + sampling_params,
162 lora_req=seq_group.lora_request,
163 )
164
165 # Non-beam search case
143 - if not seq_group.sampling_params.use_beam_search:
166 + if not sampling_params.use_beam_search:
167 # For newly created child sequences, add them to the sequence group
168 # and fork them in block manager if they are not finished.
169 for seq, parent in child_seqs:
@@ -164,8 +187,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
187 # Select the child sequences to keep in the sequence group.
188 selected_child_seqs: List[Tuple[Sequence, Optional[Sequence]]] = []
189 unselected_child_seqs: List[Tuple[Sequence, Optional[Sequence]]] = []
167 - beam_width = seq_group.sampling_params.best_of
168 - length_penalty = seq_group.sampling_params.length_penalty
190 + beam_width = sampling_params.best_of
191 + length_penalty = sampling_params.length_penalty
192
193 # Select the newly finished sequences with the highest scores
194 # to replace existing finished sequences.
@@ -219,8 +242,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
242 best_running_seq = running_child_seqs[0][0]
243 current_worst_seq = all_finished_seqs[beam_width - 1][0]
244 stop_beam_search = self._check_beam_search_early_stopping(
222 - seq_group.sampling_params.early_stopping,
223 - seq_group.sampling_params, best_running_seq, current_worst_seq)
245 + sampling_params.early_stopping, sampling_params,
246 + best_running_seq, current_worst_seq)
247
248 if stop_beam_search:
249 # Stop the beam search and remove all the running sequences from