@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ec8827288
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Qwen-vl output is inconsistent in speculative decoding (#10350)

Sky Lee committed Nov 15, 2024 at 13:40 UTC 2ec88272881a49d40d91ae0cd858b19d22996c70
1 file changed +2
vllm/spec_decode/batch_expansion.py
+2
@@ -353,6 +353,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
353 seq_data = seq_group_metadata.seq_data[seq_id]
354 prompt_token_ids = seq_data.prompt_token_ids_array
355 new_output_token_ids = [*seq_data.get_output_token_ids(), *token_ids]
356 + mrope_position_delta = seq_data.mrope_position_delta
357
358 new_seq_data_dict = {
359 target_seq_id:
@@ -368,6 +369,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
369 # the kv cache is filled by a previous batch in the batch expansion.
370 for data in new_seq_data_dict.values():
371 data.update_num_computed_tokens(data.get_len() - 1)
372 + data.mrope_position_delta = mrope_position_delta
373
374 return SequenceGroupMetadata(
375 request_id=seq_group_metadata.request_id,