A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[CI/Build] Add Model Tests for PixtralHF (#9813)
Michael Goin committed
Nov 1, 2024 at 09:55 UTC
30a2e8074246e11a1452ab5e84a7be65ecac6119
1 file changed
+9
tests/models/decoder_only/vision_language/test_models.py
+9
@@ -291,6 +291,15 @@ VLM_TEST_SETTINGS = {
291
# vllm_output_post_proc=model_utils.phi3v_vllm_to_hf_output,
292
# num_logprobs=10,
293
# ),
294
+ "pixtral_hf": VLMTestInfo(
295
+ models=["nm-testing/pixtral-12b-FP8-dynamic"],
296
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
297
+ prompt_formatter=lambda img_prompt: f"<s>[INST]{img_prompt}[/INST]",
298
+ img_idx_to_prompt=lambda idx: "[IMG]",
299
+ max_model_len=8192,
300
+ max_num_seqs=2,
301
+ auto_cls=AutoModelForVision2Seq,
302
+ ),
303
"qwen": VLMTestInfo(
304
models=["Qwen/Qwen-VL"],
305
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),