@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d0169e1b0f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Model] Future-proof Qwen2-Audio multi-modal processor (#11776)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 7, 2025 at 11:05 UTC d0169e1b0fa44a80ba40baf92dd2cedd3611076b
1 file changed +4 -2
vllm/model_executor/models/qwen2_audio.py
+4 -2
@@ -227,12 +227,14 @@ class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
227 ]
228
229 def _always_apply_prompt_replacements(self) -> bool:
230 - # HF never applies prompt replacements, so we have to do it ourselves.
230 + # Qwen2-Audio processor will start inserting placeholder tokens
231 + # in an upcoming release:
232 + # https://github.com/huggingface/transformers/pull/35534
233 # NOTE: `_find_placeholders_by_modality` may incorrectly think that HF
234 # has already performed processing for multi-audio input when the input
235 # audios are short (the corresponding placeholders may take up fewer
236 # tokens than the number of audio items)
235 - return True
237 + return not hasattr(self._get_hf_processor(), "audio_token")
238
239
240 @MULTIMODAL_REGISTRY.register_processor(Qwen2AudioMultiModalProcessor)