A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Misc] add mm_processor_kwargs to extra_body for Qwen2.5-VL (#13533)
燃 committed
Feb 20, 2025 at 15:04 UTC
041e29471671a3d43afedb6a749421d56ef4d3e7
4 files changed
+18
-2
vllm/entrypoints/openai/protocol.py
+4
@@ -312,6 +312,10 @@ class ChatCompletionRequest(OpenAIBaseModel):
312
description=("Additional kwargs to pass to the template renderer. "
313
"Will be accessible by the chat template."),
314
)
315
+ mm_processor_kwargs: Optional[Dict[str, Any]] = Field(
316
+ default=None,
317
+ description=("Additional kwargs to pass to the HF processor."),
318
+ )
319
guided_json: Optional[Union[str, dict, BaseModel]] = Field(
320
default=None,
321
description=("If specified, the output will follow the JSON schema."),
vllm/entrypoints/openai/serving_engine.py
+2
@@ -451,6 +451,8 @@ class OpenAIServing:
451
prompt_token_ids=prompt_inputs["prompt_token_ids"])
452
if mm_data is not None:
453
engine_prompt["multi_modal_data"] = mm_data
454
+ if request.mm_processor_kwargs is not None:
455
+ engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
456
457
return conversation, [request_prompt], [engine_prompt]
458
vllm/model_executor/models/qwen2_5_vl.py
+1
-1
@@ -689,7 +689,7 @@ class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
689
min_pixels: Optional[int] = None,
690
max_pixels: Optional[int] = None,
691
size: Optional[dict[str, int]] = None,
692
- fps: Optional[float] = None,
692
+ fps: Optional[Union[float, List[float]]] = None,
693
**kwargs: object,
694
) -> Qwen2_5_VLProcessor:
695
if fps is not None:
vllm/transformers_utils/processor.py
+11
-1
@@ -23,6 +23,15 @@ class HashableDict(dict):
23
return hash(frozenset(self.items()))
24
25
26
+class HashableList(list):
27
+ """
28
+ A list that can be hashed by lru_cache.
29
+ """
30
+
31
+ def __hash__(self) -> int: # type: ignore[override]
32
+ return hash(tuple(self))
33
+
34
+
35
def _merge_mm_kwargs(model_config: "ModelConfig", **kwargs):
36
base_kwargs = model_config.mm_processor_kwargs
37
if base_kwargs is None:
@@ -36,7 +45,8 @@ def _merge_mm_kwargs(model_config: "ModelConfig", **kwargs):
45
for key, value in merged_kwargs.items():
46
if isinstance(value, dict):
47
merged_kwargs[key] = HashableDict(value)
39
-
48
+ if isinstance(value, list):
49
+ merged_kwargs[key] = HashableList(value)
50
return merged_kwargs
51
52