A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Model] Future-proof Qwen2-Audio multi-modal processor (#11776)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 7, 2025 at 11:05 UTC
d0169e1b0fa44a80ba40baf92dd2cedd3611076b
1 file changed
+4
-2
vllm/model_executor/models/qwen2_audio.py
+4
-2
@@ -227,12 +227,14 @@ class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
227
]
228
229
def _always_apply_prompt_replacements(self) -> bool:
230
- # HF never applies prompt replacements, so we have to do it ourselves.
230
+ # Qwen2-Audio processor will start inserting placeholder tokens
231
+ # in an upcoming release:
232
+ # https://github.com/huggingface/transformers/pull/35534
233
# NOTE: `_find_placeholders_by_modality` may incorrectly think that HF
234
# has already performed processing for multi-audio input when the input
235
# audios are short (the corresponding placeholders may take up fewer
236
# tokens than the number of audio items)
235
- return True
237
+ return not hasattr(self._get_hf_processor(), "audio_token")
238
239
240
@MULTIMODAL_REGISTRY.register_processor(Qwen2AudioMultiModalProcessor)