A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1] Update doc and examples for H2O-VL (#13349)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Feb 16, 2025 at 02:35 UTC
b7d309860ed937f6373e6a3926915b67b86ea3b5
3 files changed
+6
-7
docs/source/models/supported_models.md
+2
-2
@@ -726,7 +726,7 @@ See [this page](#generative-models) for more information on how to use generativ
726
* `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc.
727
*
728
* ✅︎
729
- * \*
729
+ * ✅︎\*
730
- * `Idefics3ForConditionalGeneration`
731
* Idefics3
732
* T + I
@@ -869,7 +869,7 @@ See [this page](#generative-models) for more information on how to use generativ
869
<sup>+</sup> Multiple items can be inputted per text prompt for this modality.
870
871
:::{note}
872
-H2O-VL series models will be available in V1 once we support backends other than FlashAttention.
872
+`h2oai/h2ovl-mississippi-2b` will be available in V1 once we support backends other than FlashAttention.
873
:::
874
875
:::{note}
examples/offline_inference/vision_language.py
+2
-2
@@ -119,7 +119,7 @@ def run_glm4v(question: str, modality: str):
119
def run_h2ovl(question: str, modality: str):
120
assert modality == "image"
121
122
- model_name = "h2oai/h2ovl-mississippi-2b"
122
+ model_name = "h2oai/h2ovl-mississippi-800m"
123
124
llm = LLM(
125
model=model_name,
@@ -136,7 +136,7 @@ def run_h2ovl(question: str, modality: str):
136
add_generation_prompt=True)
137
138
# Stop tokens for H2OVL-Mississippi
139
- # https://huggingface.co/h2oai/h2ovl-mississippi-2b
139
+ # https://huggingface.co/h2oai/h2ovl-mississippi-800m
140
stop_token_ids = [tokenizer.eos_token_id]
141
return llm, prompt, stop_token_ids
142
examples/offline_inference/vision_language_multi_image.py
+2
-3
@@ -78,14 +78,13 @@ def load_deepseek_vl2(question: str, image_urls: List[str]):
78
79
80
def load_h2ovl(question: str, image_urls: List[str]) -> ModelRequestData:
81
- model_name = "h2oai/h2ovl-mississippi-2b"
81
+ model_name = "h2oai/h2ovl-mississippi-800m"
82
83
llm = LLM(
84
model=model_name,
85
trust_remote_code=True,
86
max_model_len=8192,
87
limit_mm_per_prompt={"image": len(image_urls)},
88
- mm_processor_kwargs={"max_dynamic_patch": 4},
88
)
89
90
placeholders = "\n".join(f"Image-{i}: <image>\n"
@@ -99,7 +98,7 @@ def load_h2ovl(question: str, image_urls: List[str]) -> ModelRequestData:
98
add_generation_prompt=True)
99
100
# Stop tokens for H2OVL-Mississippi
102
- # https://huggingface.co/h2oai/h2ovl-mississippi-2b
101
+ # https://huggingface.co/h2oai/h2ovl-mississippi-800m
102
stop_token_ids = [tokenizer.eos_token_id]
103
104
return ModelRequestData(