102bf967f0 · tduncan/Nvidia-Optimization-Stack-VLLM

docs/source/models/supported_models.md

+7

@@ -990,6 +990,13 @@ See [this page](#generative-models) for more information on how to use generativ

990 *

991 * ✅︎

992 * ✅︎

993 +- * `SmolVLMForConditionalGeneration`

994 + * SmolVLM2

995 + * T + I

996 + * `SmolVLM2-2.2B-Instruct`

997 + *

998 + * ✅︎

999 + * ✅︎

1000 - * `UltravoxModel`

1001 * Ultravox

1002 * T + A<sup>E+</sup>

examples/offline_inference/vision_language.py

+29

@@ -298,6 +298,34 @@ def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:

298 )

299

300

301 +# SmolVLM2-2.2B-Instruct

302 +def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:

303 + assert modality == "image"

304 + model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"

305 +

306 + engine_args = EngineArgs(

307 + model=model_name,

308 + max_model_len=8192,

309 + max_num_seqs=2,

310 + enforce_eager=True,

311 + mm_processor_kwargs={

312 + "max_image_size": {

313 + "longest_edge": 384

314 + },

315 + },

316 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

317 + )

318 + prompts = [

319 + (f"<|im_start|>User:<image>{question}<end_of_utterance>\nAssistant:")

320 + for question in questions

321 + ]

322 +

323 + return ModelRequestData(

324 + engine_args=engine_args,

325 + prompts=prompts,

326 + )

327 +

328 +

329 # InternVL

330 def run_internvl(questions: list[str], modality: str) -> ModelRequestData:

331 assert modality == "image"

@@ -955,6 +983,7 @@ model_example_map = {

983 "qwen2_vl": run_qwen2_vl,

984 "qwen2_5_vl": run_qwen2_5_vl,

985 "skywork_chat": run_skyworkr1v,

986 + "smolvlm": run_smolvlm,

987 }

988

989

examples/offline_inference/vision_language_multi_image.py

+28

@@ -217,6 +217,33 @@ def load_idefics3(question: str, image_urls: list[str]) -> ModelRequestData:

217 )

218

219

220 +def load_smolvlm(question: str, image_urls: list[str]) -> ModelRequestData:

221 + model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"

222 +

223 + # The configuration below has been confirmed to launch on a single L40 GPU.

224 + engine_args = EngineArgs(

225 + model=model_name,

226 + max_model_len=8192,

227 + max_num_seqs=16,

228 + enforce_eager=True,

229 + limit_mm_per_prompt={"image": len(image_urls)},

230 + mm_processor_kwargs={

231 + "max_image_size": {

232 + "longest_edge": 384

233 + },

234 + },

235 + )

236 +

237 + placeholders = "\n".join(f"Image-{i}: <image>\n"

238 + for i, _ in enumerate(image_urls, start=1))

239 + prompt = f"<|im_start|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:" # noqa: E501

240 + return ModelRequestData(

241 + engine_args=engine_args,

242 + prompt=prompt,

243 + image_data=[fetch_image(url) for url in image_urls],

244 + )

245 +

246 +

247 def load_internvl(question: str, image_urls: list[str]) -> ModelRequestData:

248 model_name = "OpenGVLab/InternVL2-2B"

249

@@ -614,6 +641,7 @@ model_example_map = {

641 "qwen_vl_chat": load_qwen_vl_chat,

642 "qwen2_vl": load_qwen2_vl,

643 "qwen2_5_vl": load_qwen2_5_vl,

644 + "smolvlm": load_smolvlm,

645 }

646

647

requirements/test.in

+1

@@ -28,6 +28,7 @@ torchvision==0.21.0

28 transformers_stream_generator # required for qwen-vl test

29 matplotlib # required for qwen-vl test

30 mistral_common[opencv] >= 1.5.4 # required for pixtral test

31 +num2words # required for smolvlm test

32 opencv-python-headless >= 4.11.0 # required for video test

33 datamodel_code_generator # required for minicpm3 test

34 lm-eval[api]==0.4.8 # required for model evaluation test

requirements/test.txt

+4

@@ -101,6 +101,8 @@ dill==0.3.8

101 # multiprocess

102 dnspython==2.7.0

103 # via email-validator

104 +docopt==0.6.2

105 + # via num2words

106 docutils==0.16

107 # via awscli

108 einops==0.8.0

@@ -263,6 +265,8 @@ networkx==3.2.1

265 # via torch

266 nltk==3.9.1

267 # via rouge-score

268 +num2words==0.5.14

269 + # via -r requirements/test.in

270 numba==0.61.0

271 # via

272 # -r requirements/test.in

tests/models/decoder_only/vision_language/test_models.py

+10

@@ -493,6 +493,16 @@ VLM_TEST_SETTINGS = {

493 patch_hf_runner=model_utils.skyworkr1v_patch_hf_runner,

494 marks=[large_gpu_mark(min_gb=80)],

495 ),

496 + "smolvlm": VLMTestInfo(

497 + models=["HuggingFaceTB/SmolVLM2-2.2B-Instruct"],

498 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),

499 + prompt_formatter=lambda img_prompt:f"<|im_start|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501

500 + img_idx_to_prompt=lambda idx: "<image>",

501 + max_model_len=8192,

502 + max_num_seqs=2,

503 + auto_cls=AutoModelForImageTextToText,

504 + hf_output_post_proc=model_utils.smolvlm_trunc_hf_output,

505 + ),

506 ### Tensor parallel / multi-gpu broadcast tests

507 "chameleon-broadcast": VLMTestInfo(

508 models=["facebook/chameleon-7b"],

tests/models/decoder_only/vision_language/vlm_utils/model_utils.py

+6

@@ -204,6 +204,12 @@ def idefics3_trunc_hf_output(hf_output: RunnerOutput,

204 return output_ids, output_str, out_logprobs

205

206

207 +def smolvlm_trunc_hf_output(hf_output: RunnerOutput,

208 + model: str) -> RunnerOutput:

209 + # Based on Idefics3

210 + return idefics3_trunc_hf_output(hf_output, model)

211 +

212 +

213 def minicpmv_trunc_hf_output(hf_output: RunnerOutput,

214 model: str) -> RunnerOutput:

215 output_ids, output_str, out_logprobs = hf_output

tests/models/multimodal/processing/test_common.py

+1

@@ -257,6 +257,7 @@ def _test_processing_correctness_mistral(

257 "h2oai/h2ovl-mississippi-800m",

258 "OpenGVLab/InternVL2-1B",

259 "HuggingFaceM4/Idefics3-8B-Llama3",

260 + "HuggingFaceTB/SmolVLM2-2.2B-Instruct",

261 "meta-llama/Llama-4-Scout-17B-16E-Instruct",

262 "llava-hf/llava-1.5-7b-hf",

263 "llava-hf/llava-v1.6-mistral-7b-hf",

tests/models/multimodal/processing/test_smolvlm.py new

+65

@@ -0,0 +1,65 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +"""Tests for smolvlm's multimodal preprocessing kwargs."""

3 +import pytest

4 +from transformers import SmolVLMConfig

5 +

6 +from vllm.multimodal import MULTIMODAL_REGISTRY

7 +

8 +from ....conftest import _ImageAssets

9 +from ...utils import build_model_context

10 +

11 +

12 +@pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"])

13 +# yapf: disable

14 +@pytest.mark.parametrize(

15 + ("mm_processor_kwargs", "expected_toks_per_img"),

16 + [

17 + ({"max_image_size": {"longest_edge": 384}}, 1377),

18 + ({"max_image_size": {"longest_edge": 768}}, 405),

19 + ])

20 +# yapf: enable

21 +@pytest.mark.parametrize("num_imgs", [1, 2])

22 +@pytest.mark.parametrize("kwargs_on_init", [True, False])

23 +def test_processor_override(

24 + image_assets: _ImageAssets,

25 + model_id: str,

26 + mm_processor_kwargs: dict[str, object],

27 + expected_toks_per_img: int,

28 + num_imgs: int,

29 + kwargs_on_init: bool,

30 +):

31 + """Ensure Idefics3MultiModalProcessor handles num_crops properly."""

32 + # Same as the previous test - don't initialize mm_processor_kwargs

33 + # in this test and assume that the kwargs will be correctly expanded by

34 + # the partial when calling the custom input processor.

35 + ctx = build_model_context(

36 + model_id,

37 + mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,

38 + limit_mm_per_prompt={"image": num_imgs},

39 + )

40 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

41 + hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs

42 +

43 + # Build the image str / prompt based on the number of images we pass

44 + placeholders = "<image>" if num_imgs == 1 else "\n".join(

45 + f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))

46 + prompt = f"<|im_start|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501

47 +

48 + # Build mm_data

49 + image_size = ctx.get_hf_config(SmolVLMConfig).vision_config.image_size

50 + dummy_image_size = (image_size * 4, image_size * 4)

51 + dummy_image = image_assets[0].pil_image.resize(dummy_image_size)

52 + mm_data = {"image": [dummy_image] * num_imgs}

53 +

54 + processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)

55 +

56 + # Ensure the placeholders format are correct

57 + hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)

58 + hf_processed_inputs = hf_processor(text=prompt, images=mm_data["image"])

59 + assert processed_inputs["prompt_token_ids"] == hf_processed_inputs[

60 + "input_ids"][0]

61 +

62 + # Ensure we have the right number of placeholders per num_crops size

63 + image_token_id = ctx.get_hf_config().image_token_id

64 + img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)

65 + assert img_tok_count == expected_toks_per_img * num_imgs

tests/models/registry.py

+1

@@ -344,6 +344,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {

344 "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501

345 min_transformers_version="4.49"), # noqa: E501

346 "SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B"),

347 + "SmolVLMForConditionalGeneration": _HfExamplesInfo("HuggingFaceTB/SmolVLM2-2.2B-Instruct"), # noqa: E501

348 "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b", # noqa: E501

349 trust_remote_code=True,

350 max_transformers_version="4.50"),

vllm/entrypoints/chat_utils.py

+1 -1

@@ -498,7 +498,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

498 hf_config.image_token_index)

499 if model_type in ("aya_vision", "chameleon", "deepseek_vl_v2",

500 "internvl_chat", "skywork_chat", "NVLM_D",

501 - "h2ovl_chat", "idefics3"):

501 + "h2ovl_chat", "idefics3", "smolvlm"):

502 return "<image>"

503 if model_type in ("mllama", "llama4"):

504 return "<|image|>"

vllm/model_executor/models/idefics3.py

+14 -5

@@ -206,6 +206,16 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):

206

207 return grid_w * grid_h + 1

208

209 + def _get_image_token(

210 + self,

211 + processor: Optional[Idefics3Processor]) -> tuple[str, str, str]:

212 + if processor is None:

213 + processor = self.get_hf_processor()

214 + image_token = processor.image_token.content

215 + fake_image_token = processor.fake_image_token.content

216 + global_image_token = processor.global_image_tag

217 + return image_token, fake_image_token, global_image_token

218 +

219 def get_image_repl(

220 self,

221 *,

@@ -216,9 +226,8 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):

226 if processor is None:

227 processor = self.get_hf_processor()

228

219 - image_token = processor.image_token.content

220 - fake_image_token = processor.fake_image_token.content

221 - global_img_token = processor.global_image_tag

229 + image_token, fake_image_token, global_img_token = self._get_image_token(

230 + processor)

231 image_seq_len = processor.image_seq_len

232 grid_placeholder = "<row_{n_h}_col_{n_w}>"

233

@@ -300,7 +309,7 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]

309 hf_processor = self.info.get_hf_processor()

310 image_processor: Idefics3ImageProcessor = hf_processor.image_processor

311 longest_edge = image_processor.max_image_size['longest_edge']

303 - image_token = hf_processor.image_token.content

312 + image_token, _, _ = self.info._get_image_token(hf_processor)

313

314 mm_data = {

315 "image":

@@ -382,7 +391,7 @@ class Idefics3MultiModalProcessor(

391 out_mm_kwargs: MultiModalKwargs,

392 ) -> Sequence[PromptUpdate]:

393 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

385 - image_token = hf_processor.image_token.content

394 + image_token, _, _ = self.info._get_image_token(hf_processor)

395

396 def get_replacement_idefics3(item_idx: int) -> PromptUpdateDetails:

397 images = mm_items.get_items("image", ImageProcessorItems)

vllm/model_executor/models/registry.py

+1

@@ -175,6 +175,7 @@ _MULTIMODAL_MODELS = {

175 "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),

176 "InternVLChatModel": ("internvl", "InternVLChatModel"),

177 "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),

178 + "SmolVLMForConditionalGeneration": ("smolvlm","SmolVLMForConditionalGeneration"), # noqa: E501

179 "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),

180 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501

181 "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501

vllm/model_executor/models/smolvlm.py new

+51

@@ -0,0 +1,51 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +

3 +from typing import Dict, Optional

4 +

5 +from transformers import SmolVLMProcessor

6 +

7 +from vllm.config import VllmConfig

8 +from vllm.multimodal import MULTIMODAL_REGISTRY

9 +

10 +# yapf: disable

11 +from .idefics3 import Idefics3DummyInputsBuilder as SmolVLMDummyInputsBuilder

12 +from .idefics3 import Idefics3ForConditionalGeneration

13 +from .idefics3 import Idefics3MultiModalProcessor as SmolVLMMultiModalProcessor

14 +from .idefics3 import Idefics3ProcessingInfo

15 +

16 +# yapf: enable

17 +

18 +

19 +class SmolVLMProcessingInfo(Idefics3ProcessingInfo):

20 +

21 + def get_hf_processor(

22 + self,

23 + *,

24 + max_image_size: Optional[Dict[str, int]] = None,

25 + **kwargs: object,

26 + ) -> SmolVLMProcessor:

27 + if max_image_size is not None:

28 + kwargs["max_image_size"] = max_image_size

29 +

30 + return self.ctx.get_hf_processor(SmolVLMProcessor, **kwargs)

31 +

32 + def _get_image_token(

33 + self, processor: Optional[SmolVLMProcessor]) -> tuple[str, str]:

34 + if processor is None:

35 + processor = self.get_hf_processor()

36 + image_token = processor.image_token

37 + fake_image_token = processor.fake_image_token

38 + global_image_token = processor.global_image_token

39 + return image_token, fake_image_token, global_image_token

40 +

41 +

42 +@MULTIMODAL_REGISTRY.register_processor(SmolVLMMultiModalProcessor,

43 + info=SmolVLMProcessingInfo,

44 + dummy_inputs=SmolVLMDummyInputsBuilder)

45 +class SmolVLMForConditionalGeneration(Idefics3ForConditionalGeneration):

46 +

47 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

48 + super().__init__(

49 + vllm_config=vllm_config,

50 + prefix=prefix,

51 + )

[Model] Add smolvlm support (#16017)