A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[BUGFIX] When skip_tokenize_init and multistep are set, execution crashes (#12277)
Signed-off-by: maleksan85 <maleksan@amd.com> Co-authored-by: maleksan85 <maleksan@amd.com>
Aleksandr Malyshev committed
Jan 21, 2025 at 15:30 UTC
69196a9bc7aefcd132c68a2184f1092ee3377ba9
1 file changed
+1
-1
vllm/engine/output_processor/multi_step.py
+1
-1
@@ -144,7 +144,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
144
def _process_decode_and_stop(self, seq: Sequence,
145
sampling_params: SamplingParams) -> None:
146
new_char_count = 0
147
- if sampling_params.detokenize:
147
+ if sampling_params.detokenize and self.detokenizer:
148
new_char_count = self.detokenizer.decode_sequence_inplace(
149
seq, sampling_params)
150