A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 03:30 UTC
[Bugfix] Fix incorrect `image_grid_thw` rank for HunyuanOCR from missing `merge_by_field_config=True` (#29950)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Dec 3, 2025 at 18:05 UTC
a21cd9ed239b853bd587ffe3c9140fe68cd41f59
2 files changed
+24
examples/offline_inference/vision_language_multi_image.py
+23
@@ -309,6 +309,28 @@ def load_h2ovl(question: str, image_urls: list[str]) -> ModelRequestData:
309
)
310
311
312
+# HunyuanOCR
313
+def load_hunyuan_vl(question: str, image_urls: list[str]) -> ModelRequestData:
314
+ model_name = "tencent/HunyuanOCR"
315
+
316
+ engine_args = EngineArgs(
317
+ model=model_name,
318
+ max_model_len=8192,
319
+ limit_mm_per_prompt={"image": len(image_urls)},
320
+ )
321
+
322
+ placeholder = (
323
+ "<｜hy_place▁holder▁no▁100｜><｜hy_place▁holder▁no▁102｜><｜hy_place▁holder▁no▁101｜>" # noqa: E501
324
+ ) * len(image_urls)
325
+ prompt = f"<｜hy_begin▁of▁sentence｜>{placeholder}{question}<｜hy_User｜>"
326
+
327
+ return ModelRequestData(
328
+ engine_args=engine_args,
329
+ prompt=prompt,
330
+ image_data=[fetch_image(url) for url in image_urls],
331
+ )
332
+
333
+
334
def load_hyperclovax_seed_vision(
335
question: str, image_urls: list[str]
336
) -> ModelRequestData:
@@ -1322,6 +1344,7 @@ model_example_map = {
1344
"deepseek_ocr": load_deepseek_ocr,
1345
"gemma3": load_gemma3,
1346
"h2ovl_chat": load_h2ovl,
1347
+ "hunyuan_vl": load_hunyuan_vl,
1348
"hyperclovax_seed_vision": load_hyperclovax_seed_vision,
1349
"idefics3": load_idefics3,
1350
"interns1": load_interns1,
vllm/model_executor/models/hunyuan_vision.py
+1
@@ -785,6 +785,7 @@ class HunYuanVLForConditionalGeneration(
785
SupportsQuant,
786
SupportsXDRoPE,
787
):
788
+ merge_by_field_config = True
789
multimodal_cpu_fields = {"image_grid_thw"}
790
791
# To ensure correct weight loading and mapping.