@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4e256cadc2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Remove all references to `yapf` as it's no longer used (#26251)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Oct 5, 2025 at 17:18 UTC 4e256cadc2178cdfb5c54a42836d78cc526f6a61
78 files changed +1994 -1719
csrc/quantization/machete/generate.py
-5
@@ -12,9 +12,6 @@ from functools import reduce
12 from typing import Optional, Union
13
14 import jinja2
15 -
16 -# yapf conflicts with isort for this block
17 -# yapf: disable
15 from vllm_cutlass_library_extension import (
16 DataType,
17 EpilogueScheduleTag,
@@ -31,8 +28,6 @@ from vllm_cutlass_library_extension import (
28 VLLMKernelScheduleTag,
29 )
30
34 -# yapf: enable
35 -
31 #
32 # Generator templating
33 #
examples/others/tensorize_vllm_model.py
+95 -73
@@ -21,8 +21,6 @@ from vllm.utils import FlexibleArgumentParser
21 logger = logging.getLogger()
22
23
24 -# yapf conflicts with isort for this docstring
25 -# yapf: disable
24 """
25 tensorize_vllm_model.py is a script that can be used to serialize and
26 deserialize vLLM models. These models can be loaded using tensorizer
@@ -132,7 +130,8 @@ def get_parser():
130 "can be loaded using tensorizer directly to the GPU "
131 "extremely quickly. Tensor encryption and decryption is "
132 "also supported, although libsodium must be installed to "
135 - "use it.")
133 + "use it."
134 + )
135 parser = EngineArgs.add_cli_args(parser)
136
137 parser.add_argument(
@@ -144,13 +143,14 @@ def get_parser():
143 "along with the model by instantiating a TensorizerConfig object, "
144 "creating a dict from it with TensorizerConfig.to_serializable(), "
145 "and passing it to LoRARequest's initializer with the kwarg "
147 - "tensorizer_config_dict."
146 + "tensorizer_config_dict.",
147 )
148
150 - subparsers = parser.add_subparsers(dest='command', required=True)
149 + subparsers = parser.add_subparsers(dest="command", required=True)
150
151 serialize_parser = subparsers.add_parser(
153 - 'serialize', help="Serialize a model to `--serialized-directory`")
152 + "serialize", help="Serialize a model to `--serialized-directory`"
153 + )
154
155 serialize_parser.add_argument(
156 "--suffix",
@@ -163,7 +163,9 @@ def get_parser():
163 "`--suffix` is `v1`, the serialized model tensors will be "
164 "saved to "
165 "`s3://my-bucket/vllm/EleutherAI/gpt-j-6B/v1/model.tensors`. "
166 - "If none is provided, a random UUID will be used."))
166 + "If none is provided, a random UUID will be used."
167 + ),
168 + )
169 serialize_parser.add_argument(
170 "--serialized-directory",
171 type=str,
@@ -175,108 +177,127 @@ def get_parser():
177 "and the model HuggingFace ID is `EleutherAI/gpt-j-6B`, tensors will "
178 "be saved to `dir/vllm/EleutherAI/gpt-j-6B/suffix/model.tensors`, "
179 "where `suffix` is given by `--suffix` or a random UUID if not "
178 - "provided.")
180 + "provided.",
181 + )
182
183 serialize_parser.add_argument(
184 "--serialization-kwargs",
185 type=tensorizer_kwargs_arg,
186 required=False,
184 - help=("A JSON string containing additional keyword arguments to "
185 - "pass to Tensorizer's TensorSerializer during "
186 - "serialization."))
187 + help=(
188 + "A JSON string containing additional keyword arguments to "
189 + "pass to Tensorizer's TensorSerializer during "
190 + "serialization."
191 + ),
192 + )
193
194 serialize_parser.add_argument(
195 "--keyfile",
196 type=str,
197 required=False,
192 - help=("Encrypt the model weights with a randomly-generated binary key,"
193 - " and save the key at this path"))
198 + help=(
199 + "Encrypt the model weights with a randomly-generated binary key,"
200 + " and save the key at this path"
201 + ),
202 + )
203
204 deserialize_parser = subparsers.add_parser(
196 - 'deserialize',
197 - help=("Deserialize a model from `--path-to-tensors`"
198 - " to verify it can be loaded and used."))
205 + "deserialize",
206 + help=(
207 + "Deserialize a model from `--path-to-tensors`"
208 + " to verify it can be loaded and used."
209 + ),
210 + )
211
212 deserialize_parser.add_argument(
213 "--path-to-tensors",
214 type=str,
215 required=False,
204 - help="The local path or S3 URI to the model tensors to deserialize. ")
216 + help="The local path or S3 URI to the model tensors to deserialize. ",
217 + )
218
219 deserialize_parser.add_argument(
220 "--serialized-directory",
221 type=str,
222 required=False,
223 help="Directory with model artifacts for loading. Assumes a "
211 - "model.tensors file exists therein. Can supersede "
212 - "--path-to-tensors.")
224 + "model.tensors file exists therein. Can supersede "
225 + "--path-to-tensors.",
226 + )
227
228 deserialize_parser.add_argument(
229 "--keyfile",
230 type=str,
231 required=False,
218 - help=("Path to a binary key to use to decrypt the model weights,"
219 - " if the model was serialized with encryption"))
232 + help=(
233 + "Path to a binary key to use to decrypt the model weights,"
234 + " if the model was serialized with encryption"
235 + ),
236 + )
237
238 deserialize_parser.add_argument(
239 "--deserialization-kwargs",
240 type=tensorizer_kwargs_arg,
241 required=False,
225 - help=("A JSON string containing additional keyword arguments to "
226 - "pass to Tensorizer's `TensorDeserializer` during "
227 - "deserialization."))
242 + help=(
243 + "A JSON string containing additional keyword arguments to "
244 + "pass to Tensorizer's `TensorDeserializer` during "
245 + "deserialization."
246 + ),
247 + )
248
249 TensorizerArgs.add_cli_args(deserialize_parser)
250
251 return parser
252
233 -def merge_extra_config_with_tensorizer_config(extra_cfg: dict,
234 - cfg: TensorizerConfig):
253 +
254 +def merge_extra_config_with_tensorizer_config(extra_cfg: dict, cfg: TensorizerConfig):
255 for k, v in extra_cfg.items():
256 if hasattr(cfg, k):
257 setattr(cfg, k, v)
258 logger.info(
259 "Updating TensorizerConfig with %s from "
240 - "--model-loader-extra-config provided", k
260 + "--model-loader-extra-config provided",
261 + k,
262 )
263
264 +
265 def deserialize(args, tensorizer_config):
266 if args.lora_path:
267 tensorizer_config.lora_dir = tensorizer_config.tensorizer_dir
246 - llm = LLM(model=args.model,
247 - load_format="tensorizer",
248 - tensor_parallel_size=args.tensor_parallel_size,
249 - model_loader_extra_config=tensorizer_config,
250 - enable_lora=True,
268 + llm = LLM(
269 + model=args.model,
270 + load_format="tensorizer",
271 + tensor_parallel_size=args.tensor_parallel_size,
272 + model_loader_extra_config=tensorizer_config,
273 + enable_lora=True,
274 )
275 sampling_params = SamplingParams(
253 - temperature=0,
254 - max_tokens=256,
255 - stop=["[/assistant]"]
276 + temperature=0, max_tokens=256, stop=["[/assistant]"]
277 )
278
279 # Truncating this as the extra text isn't necessary
259 - prompts = [
260 - "[user] Write a SQL query to answer the question based on ..."
261 - ]
280 + prompts = ["[user] Write a SQL query to answer the question based on ..."]
281
282 # Test LoRA load
283 print(
284 llm.generate(
266 - prompts,
267 - sampling_params,
268 - lora_request=LoRARequest("sql-lora",
269 - 1,
270 - args.lora_path,
271 - tensorizer_config_dict = tensorizer_config
272 - .to_serializable())
285 + prompts,
286 + sampling_params,
287 + lora_request=LoRARequest(
288 + "sql-lora",
289 + 1,
290 + args.lora_path,
291 + tensorizer_config_dict=tensorizer_config.to_serializable(),
292 + ),
293 )
294 )
295 else:
276 - llm = LLM(model=args.model,
277 - load_format="tensorizer",
278 - tensor_parallel_size=args.tensor_parallel_size,
279 - model_loader_extra_config=tensorizer_config
296 + llm = LLM(
297 + model=args.model,
298 + load_format="tensorizer",
299 + tensor_parallel_size=args.tensor_parallel_size,
300 + model_loader_extra_config=tensorizer_config,
301 )
302 return llm
303
@@ -285,17 +306,20 @@ def main():
306 parser = get_parser()
307 args = parser.parse_args()
308
288 - s3_access_key_id = (getattr(args, 's3_access_key_id', None)
289 - or os.environ.get("S3_ACCESS_KEY_ID", None))
290 - s3_secret_access_key = (getattr(args, 's3_secret_access_key', None)
291 - or os.environ.get("S3_SECRET_ACCESS_KEY", None))
292 - s3_endpoint = (getattr(args, 's3_endpoint', None)
293 - or os.environ.get("S3_ENDPOINT_URL", None))
309 + s3_access_key_id = getattr(args, "s3_access_key_id", None) or os.environ.get(
310 + "S3_ACCESS_KEY_ID", None
311 + )
312 + s3_secret_access_key = getattr(
313 + args, "s3_secret_access_key", None
314 + ) or os.environ.get("S3_SECRET_ACCESS_KEY", None)
315 + s3_endpoint = getattr(args, "s3_endpoint", None) or os.environ.get(
316 + "S3_ENDPOINT_URL", None
317 + )
318
319 credentials = {
320 "s3_access_key_id": s3_access_key_id,
321 "s3_secret_access_key": s3_secret_access_key,
298 - "s3_endpoint": s3_endpoint
322 + "s3_endpoint": s3_endpoint,
323 }
324
325 model_ref = args.model
@@ -309,25 +333,25 @@ def main():
333 if args.model_loader_extra_config:
334 extra_config = json.loads(args.model_loader_extra_config)
335
312 -
313 - tensorizer_dir = (args.serialized_directory or
314 - extra_config.get("tensorizer_dir"))
315 - tensorizer_uri = (getattr(args, "path_to_tensors", None)
316 - or extra_config.get("tensorizer_uri"))
336 + tensorizer_dir = args.serialized_directory or extra_config.get("tensorizer_dir")
337 + tensorizer_uri = getattr(args, "path_to_tensors", None) or extra_config.get(
338 + "tensorizer_uri"
339 + )
340
341 if tensorizer_dir and tensorizer_uri:
319 - parser.error("--serialized-directory and --path-to-tensors "
320 - "cannot both be provided")
342 + parser.error(
343 + "--serialized-directory and --path-to-tensors cannot both be provided"
344 + )
345
346 if not tensorizer_dir and not tensorizer_uri:
323 - parser.error("Either --serialized-directory or --path-to-tensors "
324 - "must be provided")
325 -
347 + parser.error(
348 + "Either --serialized-directory or --path-to-tensors must be provided"
349 + )
350
351 if args.command == "serialize":
352 engine_args = EngineArgs.from_cli_args(args)
353
330 - input_dir = tensorizer_dir.rstrip('/')
354 + input_dir = tensorizer_dir.rstrip("/")
355 suffix = args.suffix if args.suffix else uuid.uuid4().hex
356 base_path = f"{input_dir}/vllm/{model_ref}/{suffix}"
357 if engine_args.tensor_parallel_size > 1:
@@ -339,15 +363,14 @@ def main():
363 tensorizer_uri=model_path,
364 encryption_keyfile=keyfile,
365 serialization_kwargs=args.serialization_kwargs or {},
342 - **credentials
366 + **credentials,
367 )
368
369 if args.lora_path:
370 tensorizer_config.lora_dir = tensorizer_config.tensorizer_dir
371 tensorize_lora_adapter(args.lora_path, tensorizer_config)
372
349 - merge_extra_config_with_tensorizer_config(extra_config,
350 - tensorizer_config)
373 + merge_extra_config_with_tensorizer_config(extra_config, tensorizer_config)
374 tensorize_vllm_model(engine_args, tensorizer_config)
375
376 elif args.command == "deserialize":
@@ -356,11 +379,10 @@ def main():
379 tensorizer_dir=args.serialized_directory,
380 encryption_keyfile=keyfile,
381 deserialization_kwargs=args.deserialization_kwargs or {},
359 - **credentials
382 + **credentials,
383 )
384
362 - merge_extra_config_with_tensorizer_config(extra_config,
363 - tensorizer_config)
385 + merge_extra_config_with_tensorizer_config(extra_config, tensorizer_config)
386 deserialize(args, tensorizer_config)
387 else:
388 raise ValueError("Either serialize or deserialize must be specified.")
tests/compile/test_silu_mul_quant_fusion.py
-5
@@ -8,16 +8,11 @@ import torch
8 import vllm.envs as envs
9 from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
10 from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
11 -
12 -# yapf conflicts with isort for this block
13 -# yapf: disable
11 from vllm.compilation.activation_quant_fusion import (
12 FUSED_OPS,
13 SILU_MUL_OP,
14 ActivationQuantFusionPass,
15 )
19 -
20 -# yapf: enable
16 from vllm.compilation.fusion import QUANT_OPS
17 from vllm.compilation.noop_elimination import NoOpEliminationPass
18 from vllm.compilation.post_cleanup import PostCleanupPass
tests/distributed/test_expert_parallel.py
+22 -20
@@ -107,10 +107,8 @@ class EPTestSettings:
107 # NOTE: You can adjust tp_base locally to fit the model in GPU
108 # The values displayed here are only a rough indicator of the size of the model
109
110 -# yapf: disable
110 TEST_MODELS = {
112 - "deepseek-ai/DeepSeek-V2-Lite-Chat": EPTestSettings.fast(
113 - trust_remote_code=True),
111 + "deepseek-ai/DeepSeek-V2-Lite-Chat": EPTestSettings.fast(trust_remote_code=True),
112 "mistralai/Mixtral-8x7B-Instruct-v0.1": EPTestSettings.fast(tp_base=4),
113 }
114
@@ -192,22 +190,24 @@ def _compare_tp(
190 ]
191
192 try:
195 - compare_two_settings(model_name,
196 - ep_args,
197 - tp_args,
198 - ep_env,
199 - tp_env,
200 - method=method,
201 - max_wait_seconds=360)
193 + compare_two_settings(
194 + model_name,
195 + ep_args,
196 + tp_args,
197 + ep_env,
198 + tp_env,
199 + method=method,
200 + max_wait_seconds=360,
201 + )
202 except Exception:
203 raise
204
205
206 @pytest.mark.parametrize(
207 - ("model_name", "parallel_setup", "distributed_backend", "runner",
208 - "test_options"),
207 + ("model_name", "parallel_setup", "distributed_backend", "runner", "test_options"),
208 [
210 - params for model_name, settings in TEST_MODELS.items()
209 + params
210 + for model_name, settings in TEST_MODELS.items()
211 for params in settings.iter_params(model_name)
212 ],
213 )
@@ -220,10 +220,12 @@ def test_ep(
220 test_options: EPTestOptions,
221 num_gpus_available,
222 ):
223 - _compare_tp(model_name,
224 - parallel_setup,
225 - distributed_backend,
226 - runner,
227 - test_options,
228 - num_gpus_available,
229 - method="generate")
223 + _compare_tp(
224 + model_name,
225 + parallel_setup,
226 + distributed_backend,
227 + runner,
228 + test_options,
229 + num_gpus_available,
230 + method="generate",
231 + )
tests/distributed/test_pipeline_parallel.py
+3 -3
@@ -100,7 +100,6 @@ class PPTestSettings:
100 # NOTE: You can adjust tp_base and/or pp_base locally to fit the model in GPU
101 # The values displayed here are only a rough indicator of the size of the model
102
103 -# yapf: disable
103 TEXT_GENERATION_MODELS = {
104 # [Decoder-only]
105 # Uses Llama
@@ -150,7 +149,9 @@ TEXT_GENERATION_MODELS = {
149 "adept/persimmon-8b-chat": PPTestSettings.fast(),
150 "microsoft/phi-2": PPTestSettings.fast(),
151 "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(),
153 - "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(multi_node_only=True, load_format="dummy"), # noqa: E501
152 + "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(
153 + multi_node_only=True, load_format="dummy"
154 + ), # noqa: E501
155 "Qwen/Qwen-7B-Chat": PPTestSettings.fast(),
156 "Qwen/Qwen2.5-0.5B-Instruct": PPTestSettings.fast(),
157 "Qwen/Qwen1.5-MoE-A2.7B-Chat": PPTestSettings.fast(),
@@ -196,7 +197,6 @@ MULTIMODAL_MODELS = {
197 "Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
198 "fixie-ai/ultravox-v0_5-llama-3_2-1b": PPTestSettings.fast(),
199 }
199 -# yapf: enable
200
201 # NOTE: You can update this on your local machine to run specific tests
202 TEST_MODELS = [
tests/engine/test_arg_utils.py
+10 -25
@@ -287,29 +287,15 @@ def test_prefix_cache_default():
287 assert not engine_args.enable_prefix_caching
288
289
290 -# yapf: disable
291 -@pytest.mark.parametrize(("arg", "expected", "option"), [
292 - (None, None, "mm-processor-kwargs"),
293 - ("{}", {}, "mm-processor-kwargs"),
294 - (
295 - '{"num_crops": 4}',
296 - {
297 - "num_crops": 4
298 - },
299 - "mm-processor-kwargs"
300 - ),
301 - (
302 - '{"foo": {"bar": "baz"}}',
303 - {
304 - "foo":
305 - {
306 - "bar": "baz"
307 - }
308 - },
309 - "mm-processor-kwargs"
310 - ),
311 -])
312 -# yapf: enable
290 +@pytest.mark.parametrize(
291 + ("arg", "expected", "option"),
292 + [
293 + (None, None, "mm-processor-kwargs"),
294 + ("{}", {}, "mm-processor-kwargs"),
295 + ('{"num_crops": 4}', {"num_crops": 4}, "mm-processor-kwargs"),
296 + ('{"foo": {"bar": "baz"}}', {"foo": {"bar": "baz"}}, "mm-processor-kwargs"),
297 + ],
298 +)
299 def test_composite_arg_parser(arg, expected, option):
300 parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
301 if arg is None:
@@ -321,8 +307,7 @@ def test_composite_arg_parser(arg, expected, option):
307
308 def test_human_readable_model_len():
309 # `exit_on_error` disabled to test invalid values below
324 - parser = EngineArgs.add_cli_args(
325 - FlexibleArgumentParser(exit_on_error=False))
310 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser(exit_on_error=False))
311
312 args = parser.parse_args([])
313 assert args.max_model_len is None
tests/entrypoints/test_chat_utils.py
+179 -209
@@ -15,6 +15,7 @@ from vllm.assets.video import VideoAsset
15 from vllm.config import ModelConfig
16 from vllm.entrypoints.chat_utils import (
17 _try_extract_ast,
18 + apply_mistral_chat_template,
19 load_chat_template,
20 parse_chat_messages,
21 parse_chat_messages_futures,
@@ -1855,17 +1856,17 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa
1856
1857 # NOTE: Qwen2-Audio default chat template is specially defined inside
1858 # processor class instead of using `tokenizer_config.json`
1858 -# yapf: disable
1859 @pytest.mark.parametrize(
1860 ("model", "expected_format"),
1861 - [(PHI3V_MODEL_ID, "string"),
1862 - (QWEN2VL_MODEL_ID, "openai"),
1863 - (QWEN25VL_MODEL_ID, "openai"),
1864 - (ULTRAVOX_MODEL_ID, "string"),
1865 - (QWEN2AUDIO_MODEL_ID, "openai"),
1866 - (LLAMA_GUARD_MODEL_ID, "openai")],
1861 + [
1862 + (PHI3V_MODEL_ID, "string"),
1863 + (QWEN2VL_MODEL_ID, "openai"),
1864 + (QWEN25VL_MODEL_ID, "openai"),
1865 + (ULTRAVOX_MODEL_ID, "string"),
1866 + (QWEN2AUDIO_MODEL_ID, "openai"),
1867 + (LLAMA_GUARD_MODEL_ID, "openai"),
1868 + ],
1869 )
1868 -# yapf: enable
1870 def test_resolve_content_format_hf_defined(model, expected_format):
1871 model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
1872 model_info.check_available_online(on_fail="skip")
@@ -1879,7 +1880,8 @@ def test_resolve_content_format_hf_defined(model, expected_format):
1880 hf_overrides=model_info.hf_overrides,
1881 skip_tokenizer_init=model_info.skip_tokenizer_init,
1882 enforce_eager=model_info.enforce_eager,
1882 - dtype=model_info.dtype)
1883 + dtype=model_info.dtype,
1884 + )
1885
1886 tokenizer = get_tokenizer(
1887 model,
@@ -1911,18 +1913,18 @@ def test_resolve_content_format_hf_defined(model, expected_format):
1913 assert resolved_format == expected_format
1914
1915
1914 -# yapf: disable
1916 @pytest.mark.parametrize(
1917 ("model", "expected_format"),
1917 - [("Salesforce/blip2-opt-2.7b", "string"),
1918 - ("facebook/chameleon-7b", "string"),
1919 - ("deepseek-ai/deepseek-vl2-tiny", "string"),
1920 - ("adept/fuyu-8b", "string"),
1921 - ("google/paligemma-3b-mix-224", "string"),
1922 - ("Qwen/Qwen-VL", "string"),
1923 - ("Qwen/Qwen-VL-Chat", "string")],
1918 + [
1919 + ("Salesforce/blip2-opt-2.7b", "string"),
1920 + ("facebook/chameleon-7b", "string"),
1921 + ("deepseek-ai/deepseek-vl2-tiny", "string"),
1922 + ("adept/fuyu-8b", "string"),
1923 + ("google/paligemma-3b-mix-224", "string"),
1924 + ("Qwen/Qwen-VL", "string"),
1925 + ("Qwen/Qwen-VL-Chat", "string"),
1926 + ],
1927 )
1925 -# yapf: enable
1928 def test_resolve_content_format_fallbacks(model, expected_format):
1929 model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
1930 model_info.check_available_online(on_fail="skip")
@@ -1936,7 +1938,8 @@ def test_resolve_content_format_fallbacks(model, expected_format):
1938 hf_overrides=model_info.hf_overrides,
1939 skip_tokenizer_init=model_info.skip_tokenizer_init,
1940 enforce_eager=model_info.enforce_eager,
1939 - dtype=model_info.dtype)
1941 + dtype=model_info.dtype,
1942 + )
1943
1944 tokenizer = get_tokenizer(
1945 model_config.tokenizer,
@@ -1968,30 +1971,30 @@ def test_resolve_content_format_fallbacks(model, expected_format):
1971 assert resolved_format == expected_format
1972
1973
1971 -# yapf: disable
1974 @pytest.mark.parametrize(
1975 ("template_path", "expected_format"),
1974 - [("template_alpaca.jinja", "string"),
1975 - ("template_baichuan.jinja", "string"),
1976 - ("template_chatglm.jinja", "string"),
1977 - ("template_chatglm2.jinja", "string"),
1978 - ("template_chatml.jinja", "string"),
1979 - ("template_dse_qwen2_vl.jinja", "openai"),
1980 - ("template_falcon_180b.jinja", "string"),
1981 - ("template_falcon.jinja", "string"),
1982 - ("template_inkbot.jinja", "string"),
1983 - ("template_teleflm.jinja", "string"),
1984 - ("template_vlm2vec_phi3v.jinja", "openai"),
1985 - ("template_vlm2vec_qwen2vl.jinja", "openai"),
1986 - ("tool_chat_template_granite_20b_fc.jinja", "string"),
1987 - ("tool_chat_template_hermes.jinja", "string"),
1988 - ("tool_chat_template_internlm2_tool.jinja", "string"),
1989 - ("tool_chat_template_llama3.1_json.jinja", "openai"),
1990 - ("tool_chat_template_llama3.2_json.jinja", "openai"),
1991 - ("tool_chat_template_mistral_parallel.jinja", "string"),
1992 - ("tool_chat_template_mistral.jinja", "string")],
1976 + [
1977 + ("template_alpaca.jinja", "string"),
1978 + ("template_baichuan.jinja", "string"),
1979 + ("template_chatglm.jinja", "string"),
1980 + ("template_chatglm2.jinja", "string"),
1981 + ("template_chatml.jinja", "string"),
1982 + ("template_dse_qwen2_vl.jinja", "openai"),
1983 + ("template_falcon_180b.jinja", "string"),
1984 + ("template_falcon.jinja", "string"),
1985 + ("template_inkbot.jinja", "string"),
1986 + ("template_teleflm.jinja", "string"),
1987 + ("template_vlm2vec_phi3v.jinja", "openai"),
1988 + ("template_vlm2vec_qwen2vl.jinja", "openai"),
1989 + ("tool_chat_template_granite_20b_fc.jinja", "string"),
1990 + ("tool_chat_template_hermes.jinja", "string"),
1991 + ("tool_chat_template_internlm2_tool.jinja", "string"),
1992 + ("tool_chat_template_llama3.1_json.jinja", "openai"),
1993 + ("tool_chat_template_llama3.2_json.jinja", "openai"),
1994 + ("tool_chat_template_mistral_parallel.jinja", "string"),
1995 + ("tool_chat_template_mistral.jinja", "string"),
1996 + ],
1997 )
1994 -# yapf: enable
1998 def test_resolve_content_format_examples(template_path, expected_format):
1999 model_config = ModelConfig(
2000 PHI3V_MODEL_ID, # Dummy
@@ -2024,40 +2027,34 @@ def test_resolve_content_format_examples(template_path, expected_format):
2027 assert resolved_format == expected_format
2028
2029
2027 -def test_parse_chat_messages_include_thinking_chunk(mistral_model_config,
2028 - mistral_tokenizer):
2029 - messages = [{
2030 - "role":
2031 - "system",
2032 - "content": [{
2033 - "type": "text",
2034 - "text": "You are a helpful assistant."
2035 - }, {
2036 - "type":
2037 - "thinking",
2038 - "closed":
2039 - True,
2040 - "thinking":
2041 - "Only return the answer when you are confident."
2042 - }]
2043 - }, {
2044 - "role": "user",
2045 - "content": "What is 2+2?"
2046 - }, {
2047 - "role":
2048 - "assistant",
2049 - "content": [{
2050 - "type": "text",
2051 - "text": "Let me think about it."
2052 - }, {
2053 - "type": "thinking",
2054 - "closed": True,
2055 - "thinking": "2+2 = 4"
2056 - }, {
2057 - "type": "text",
2058 - "text": "The answer is 4.",
2059 - }],
2060 - }]
2030 +def test_parse_chat_messages_include_thinking_chunk(
2031 + mistral_model_config, mistral_tokenizer
2032 +):
2033 + messages = [
2034 + {
2035 + "role": "system",
2036 + "content": [
2037 + {"type": "text", "text": "You are a helpful assistant."},
2038 + {
2039 + "type": "thinking",
2040 + "closed": True,
2041 + "thinking": "Only return the answer when you are confident.",
2042 + },
2043 + ],
2044 + },
2045 + {"role": "user", "content": "What is 2+2?"},
2046 + {
2047 + "role": "assistant",
2048 + "content": [
2049 + {"type": "text", "text": "Let me think about it."},
2050 + {"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
2051 + {
2052 + "type": "text",
2053 + "text": "The answer is 4.",
2054 + },
2055 + ],
2056 + },
2057 + ]
2058
2059 conversation_with_thinking, _, _ = parse_chat_messages(
2060 messages,
@@ -2066,122 +2063,105 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config,
2063 content_format="openai",
2064 )
2065
2069 - expected_conversation = [{
2070 - "role":
2071 - "system",
2072 - "content": [{
2073 - "type": "text",
2074 - "text": "You are a helpful assistant."
2075 - }, {
2076 - "type": "text",
2077 - "text": "Only return the answer when you are confident."
2078 - }],
2079 - }, {
2080 - "role":
2081 - "user",
2082 - "content": [{
2083 - "type": "text",
2084 - "text": "What is 2+2?"
2085 - }],
2086 - }, {
2087 - "role":
2088 - "assistant",
2089 - "content": [
2090 - {
2091 - "type": "text",
2092 - "text": "Let me think about it."
2093 - },
2094 - {
2095 - "type": "text",
2096 - "text": "2+2 = 4"
2097 - },
2098 - {
2099 - "type": "text",
2100 - "text": "The answer is 4."
2101 - },
2102 - ]
2103 - }]
2066 + expected_conversation = [
2067 + {
2068 + "role": "system",
2069 + "content": [
2070 + {"type": "text", "text": "You are a helpful assistant."},
2071 + {
2072 + "type": "text",
2073 + "text": "Only return the answer when you are confident.",
2074 + },
2075 + ],
2076 + },
2077 + {
2078 + "role": "user",
2079 + "content": [{"type": "text", "text": "What is 2+2?"}],
2080 + },
2081 + {
2082 + "role": "assistant",
2083 + "content": [
2084 + {"type": "text", "text": "Let me think about it."},
2085 + {"type": "text", "text": "2+2 = 4"},
2086 + {"type": "text", "text": "The answer is 4."},
2087 + ],
2088 + },
2089 + ]
2090
2091 assert conversation_with_thinking == expected_conversation
2092
2093
2094 def test_apply_mistral_chat_template_thinking_chunk():
2109 - # Moved import here to avoid yapf and isort conflicts
2110 - from vllm.entrypoints.chat_utils import apply_mistral_chat_template
2111 - messages = [{
2112 - "role":
2113 - "system",
2114 - "content": [{
2115 - "type": "text",
2116 - "text": "You are a helpful assistant."
2117 - }, {
2118 - "type":
2119 - "thinking",
2120 - "closed":
2121 - True,
2122 - "thinking":
2123 - "Only return the answer when you are confident."
2124 - }]
2125 - }, {
2126 - "role": "user",
2127 - "content": "What is 2+2?"
2128 - }, {
2129 - "role":
2130 - "assistant",
2131 - "content": [{
2132 - "type": "text",
2133 - "text": "Let me think about it."
2134 - }, {
2135 - "type": "thinking",
2136 - "closed": True,
2137 - "thinking": "2+2 = 4"
2138 - }, {
2139 - "type": "text",
2140 - "text": "The answer is 4.",
2141 - }],
2142 - }, {
2143 - "role": "user",
2144 - "content": "Thanks, what is 3+3?"
2145 - }]
2095 + messages = [
2096 + {
2097 + "role": "system",
2098 + "content": [
2099 + {"type": "text", "text": "You are a helpful assistant."},
2100 + {
2101 + "type": "thinking",
2102 + "closed": True,
2103 + "thinking": "Only return the answer when you are confident.",
2104 + },
2105 + ],
2106 + },
2107 + {"role": "user", "content": "What is 2+2?"},
2108 + {
2109 + "role": "assistant",
2110 + "content": [
2111 + {"type": "text", "text": "Let me think about it."},
2112 + {"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
2113 + {
2114 + "type": "text",
2115 + "text": "The answer is 4.",
2116 + },
2117 + ],
2118 + },
2119 + {"role": "user", "content": "Thanks, what is 3+3?"},
2120 + ]
2121
2122 # TODO(Julien): upon model release change to a tokenizer already configured.
2123 # =================================================================
2124 mistral_tokenizer = MistralTokenizer.from_pretrained(
2150 - "mistralai/Devstral-Small-2507")
2125 + "mistralai/Devstral-Small-2507"
2126 + )
2127 assert isinstance(mistral_tokenizer.tokenizer, Tekkenizer)
2128 # Add think special tokens to the tokenizer
2129 mistral_tokenizer.tokenizer._all_special_tokens[35] = SpecialTokenInfo(
2154 - rank=35, is_control=True, token_str=SpecialTokens.begin_think.value)
2130 + rank=35, is_control=True, token_str=SpecialTokens.begin_think.value
2131 + )
2132 mistral_tokenizer.tokenizer._all_special_tokens[36] = SpecialTokenInfo(
2156 - rank=36, is_control=True, token_str=SpecialTokens.end_think.value)
2133 + rank=36, is_control=True, token_str=SpecialTokens.end_think.value
2134 + )
2135 mistral_tokenizer.tokenizer._special_tokens_reverse_vocab = {
2136 k: v
2159 - for k, v in
2160 - mistral_tokenizer.tokenizer._special_tokens_reverse_vocab.items()
2137 + for k, v in mistral_tokenizer.tokenizer._special_tokens_reverse_vocab.items()
2138 if v not in {35, 36}
2139 }
2140 mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
2164 - SpecialTokens.begin_think.value] = 35
2141 + SpecialTokens.begin_think.value
2142 + ] = 35
2143 mistral_tokenizer.tokenizer._special_tokens_reverse_vocab[
2166 - SpecialTokens.end_think.value] = 36
2144 + SpecialTokens.end_think.value
2145 + ] = 36
2146 mistral_tokenizer.instruct.BEGIN_THINK = 35
2147 mistral_tokenizer.instruct.END_THINK = 36
2148 # =================================================================
2149
2171 - tokens_ids = apply_mistral_chat_template(mistral_tokenizer,
2172 - messages,
2173 - chat_template=None,
2174 - tools=None)
2150 + tokens_ids = apply_mistral_chat_template(
2151 + mistral_tokenizer, messages, chat_template=None, tools=None
2152 + )
2153
2154 string_tokens = mistral_tokenizer.mistral.decode(
2177 - tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP)
2155 + tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP
2156 + )
2157
2158 expected_tokens = (
2159 r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the"
2160 r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]"
2161 r"[INST]What is 2+2?[/INST]"
2162 r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>"
2184 - r"[INST]Thanks, what is 3+3?[/INST]")
2163 + r"[INST]Thanks, what is 3+3?[/INST]"
2164 + )
2165
2166 assert string_tokens == expected_tokens
2167
@@ -2192,37 +2172,32 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
2172 ):
2173 audio_uuid = "abcd"
2174 conversation, mm_data, mm_uuids = parse_chat_messages(
2195 - [{
2196 - "role":
2197 - "user",
2198 - "content": [
2199 - {
2200 - "type": "input_audio",
2201 - "input_audio": {},
2202 - "uuid": audio_uuid,
2203 - },
2204 - {
2205 - "type": "text",
2206 - "text": "What does the audio say?"
2207 - },
2208 - ],
2209 - }],
2175 + [
2176 + {
2177 + "role": "user",
2178 + "content": [
2179 + {
2180 + "type": "input_audio",
2181 + "input_audio": {},
2182 + "uuid": audio_uuid,
2183 + },
2184 + {"type": "text", "text": "What does the audio say?"},
2185 + ],
2186 + }
2187 + ],
2188 qwen2_audio_model_config,
2189 qwen2_audio_tokenizer,
2190 content_format="string",
2191 )
2192
2215 - assert conversation == [{
2216 - "role":
2217 - "user",
2218 - "content":
2219 - "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?"
2220 - }]
2193 + assert conversation == [
2194 + {
2195 + "role": "user",
2196 + "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?",
2197 + }
2198 + ]
2199 _assert_mm_data_inputs(mm_data, {"audio": 1})
2222 - _assert_mm_uuids(mm_uuids,
2223 - 1,
2224 - modality="audio",
2225 - expected_uuids=[audio_uuid])
2200 + _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
2201
2202
2203 @pytest.mark.asyncio
@@ -2232,34 +2207,29 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
2207 ):
2208 audio_uuid = "abcd"
2209 conversation, mm_future, mm_uuids = parse_chat_messages_futures(
2235 - [{
2236 - "role":
2237 - "user",
2238 - "content": [
2239 - {
2240 - "type": "input_audio",
2241 - "input_audio": {},
2242 - "uuid": audio_uuid,
2243 - },
2244 - {
2245 - "type": "text",
2246 - "text": "What does the audio say?"
2247 - },
2248 - ],
2249 - }],
2210 + [
2211 + {
2212 + "role": "user",
2213 + "content": [
2214 + {
2215 + "type": "input_audio",
2216 + "input_audio": {},
2217 + "uuid": audio_uuid,
2218 + },
2219 + {"type": "text", "text": "What does the audio say?"},
2220 + ],
2221 + }
2222 + ],
2223 qwen2_audio_model_config,
2224 qwen2_audio_tokenizer,
2225 content_format="string",
2226 )
2227
2255 - assert conversation == [{
2256 - "role":
2257 - "user",
2258 - "content":
2259 - "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?"
2260 - }]
2228 + assert conversation == [
2229 + {
2230 + "role": "user",
2231 + "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?",
2232 + }
2233 + ]
2234 _assert_mm_data_inputs(await mm_future, {"audio": 1})
2262 - _assert_mm_uuids(mm_uuids,
2263 - 1,
2264 - modality="audio",
2265 - expected_uuids=[audio_uuid])
2235 + _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
tests/lora/test_layers.py
-5
@@ -12,9 +12,6 @@ import torch
12 import torch.nn.functional as F
13
14 from vllm.config.lora import LoRAConfig
15 -
16 -# yapf conflicts with isort for this block
17 -# yapf: disable
15 from vllm.lora.layers import (
16 BaseLayerWithLoRA,
17 ColumnParallelLinearWithLoRA,
@@ -32,8 +29,6 @@ from vllm.lora.layers import (
29 RowParallelLinearWithShardedLoRA,
30 VocabParallelEmbeddingWithLoRA,
31 )
35 -
36 -# yapf: enable
32 from vllm.lora.models import LoRALayerWeights, PackedLoRALayerWeights
33 from vllm.lora.punica_wrapper import get_punica_wrapper
34 from vllm.model_executor.layers.linear import (
tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py
-4
@@ -17,8 +17,6 @@ import vllm.model_executor.model_loader.tensorizer
17 from tests.utils import VLLM_PATH, RemoteOpenAIServer
18 from vllm import LLM, SamplingParams
19 from vllm.engine.arg_utils import EngineArgs
20 -
21 -# yapf: disable
20 from vllm.model_executor.model_loader.tensorizer import (
21 TensorizerConfig,
22 TensorSerializer,
@@ -29,8 +27,6 @@ from vllm.model_executor.model_loader.tensorizer import (
27 from vllm.model_executor.model_loader.tensorizer_loader import (
28 BLACKLISTED_TENSORIZER_ARGS,
29 )
32 -
33 -# yapf: enable
30 from vllm.utils import PlaceholderModule
31
32 from .conftest import DummyExecutor, assert_from_collective_rpc
tests/models/multimodal/generation/test_common.py
+191 -169
@@ -45,18 +45,17 @@ from .vlm_utils.types import (
45 if current_platform.is_rocm():
46 os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
47
48 -# yapf: disable
48 COMMON_BROADCAST_SETTINGS = {
49 "test_type": VLMTestType.IMAGE,
50 "dtype": "half",
51 "max_tokens": 5,
52 "tensor_parallel_size": 2,
53 "hf_model_kwargs": {"device_map": "auto"},
55 - "image_size_factors": [(.25, 0.5, 1.0)],
54 + "image_size_factors": [(0.25, 0.5, 1.0)],
55 "distributed_executor_backend": (
56 "ray",
57 "mp",
59 - )
58 + ),
59 }
60
61 ### Test configuration for specific models
@@ -96,22 +95,20 @@ VLM_TEST_SETTINGS = {
95 #### Core tests to always run in the CI
96 "llava": VLMTestInfo(
97 models=["llava-hf/llava-1.5-7b-hf"],
99 - test_type=(
100 - VLMTestType.EMBEDDING,
101 - VLMTestType.IMAGE,
102 - VLMTestType.CUSTOM_INPUTS
103 - ),
98 + test_type=(VLMTestType.EMBEDDING, VLMTestType.IMAGE, VLMTestType.CUSTOM_INPUTS),
99 prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
100 convert_assets_to_embeddings=model_utils.get_llava_embeddings,
101 max_model_len=4096,
102 auto_cls=AutoModelForImageTextToText,
103 vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
109 - custom_test_opts=[CustomTestOptions(
110 - inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
111 - formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
112 - ),
113 - limit_mm_per_prompt={"image": 4},
114 - )],
104 + custom_test_opts=[
105 + CustomTestOptions(
106 + inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
107 + formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
108 + ),
109 + limit_mm_per_prompt={"image": 4},
110 + )
111 + ],
112 # TODO: Revert to "auto" when CPU backend can use torch > 2.6
113 dtype="bfloat16" if current_platform.is_cpu() else "auto",
114 marks=[pytest.mark.core_model, pytest.mark.cpu_model],
@@ -120,27 +117,27 @@ VLM_TEST_SETTINGS = {
117 models=["google/paligemma-3b-mix-224"],
118 test_type=VLMTestType.IMAGE,
119 prompt_formatter=identity,
123 - img_idx_to_prompt = lambda idx: "",
120 + img_idx_to_prompt=lambda idx: "",
121 # Paligemma uses its own sample prompts because the default one fails
125 - single_image_prompts=IMAGE_ASSETS.prompts({
126 - "stop_sign": "caption es",
127 - "cherry_blossom": "What is in the picture?",
128 - }),
122 + single_image_prompts=IMAGE_ASSETS.prompts(
123 + {
124 + "stop_sign": "caption es",
125 + "cherry_blossom": "What is in the picture?",
126 + }
127 + ),
128 auto_cls=AutoModelForImageTextToText,
129 vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
130 dtype="bfloat16",
132 - marks=[pytest.mark.skip(reason="vLLM does not support PrefixLM attention mask")], # noqa: E501
131 + marks=[
132 + pytest.mark.skip(reason="vLLM does not support PrefixLM attention mask")
133 + ], # noqa: E501
134 ),
135 "qwen2_5_vl": VLMTestInfo(
136 models=["Qwen/Qwen2.5-VL-3B-Instruct"],
136 - test_type=(
137 - VLMTestType.IMAGE,
138 - VLMTestType.MULTI_IMAGE,
139 - VLMTestType.VIDEO
140 - ),
141 - prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
142 - img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
143 - video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
137 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
138 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
139 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
140 + video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
141 max_model_len=4096,
142 max_num_seqs=2,
143 auto_cls=AutoModelForImageTextToText,
@@ -150,17 +147,13 @@ VLM_TEST_SETTINGS = {
147 ),
148 "qwen2_5_omni": VLMTestInfo(
149 models=["Qwen/Qwen2.5-Omni-3B"],
153 - test_type=(
154 - VLMTestType.IMAGE,
155 - VLMTestType.MULTI_IMAGE,
156 - VLMTestType.VIDEO
157 - ),
158 - prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
159 - img_idx_to_prompt=lambda idx: "<|vision_bos|><|IMAGE|><|vision_eos|>", # noqa: E501
160 - video_idx_to_prompt=lambda idx: "<|vision_bos|><|VIDEO|><|vision_eos|>", # noqa: E501
150 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
151 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
152 + img_idx_to_prompt=lambda idx: "<|vision_bos|><|IMAGE|><|vision_eos|>", # noqa: E501
153 + video_idx_to_prompt=lambda idx: "<|vision_bos|><|VIDEO|><|vision_eos|>", # noqa: E501
154 max_model_len=4096,
155 max_num_seqs=2,
163 - num_logprobs= 6 if current_platform.is_cpu() else 5,
156 + num_logprobs=6 if current_platform.is_cpu() else 5,
157 auto_cls=AutoModelForTextToWaveform,
158 vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
159 patch_hf_runner=model_utils.qwen2_5_omni_patch_hf_runner,
@@ -168,9 +161,9 @@ VLM_TEST_SETTINGS = {
161 marks=[pytest.mark.core_model, pytest.mark.cpu_model],
162 ),
163 "ultravox": VLMTestInfo(
171 - models = ["fixie-ai/ultravox-v0_5-llama-3_2-1b"],
164 + models=["fixie-ai/ultravox-v0_5-llama-3_2-1b"],
165 test_type=VLMTestType.AUDIO,
173 - prompt_formatter=lambda audio_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{audio_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
166 + prompt_formatter=lambda audio_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{audio_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
167 audio_idx_to_prompt=lambda idx: "<|audio|>",
168 max_model_len=4096,
169 max_num_seqs=2,
@@ -184,9 +177,11 @@ VLM_TEST_SETTINGS = {
177 "llava-onevision-transformers": VLMTestInfo(
178 models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
179 test_type=VLMTestType.IMAGE,
187 - prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
180 + prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
181 max_model_len=16384,
189 - hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
182 + hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
183 + "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
184 + ), # noqa: E501
185 auto_cls=AutoModelForImageTextToText,
186 vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
187 image_size_factors=[(0.25, 0.5, 1.0)],
@@ -201,7 +196,7 @@ VLM_TEST_SETTINGS = {
196 "idefics3-transformers": VLMTestInfo(
197 models=["HuggingFaceTB/SmolVLM-256M-Instruct"],
198 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
204 - prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
199 + prompt_formatter=lambda img_prompt: f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
200 img_idx_to_prompt=lambda idx: "<image>",
201 max_model_len=8192,
202 max_num_seqs=2,
@@ -217,8 +212,8 @@ VLM_TEST_SETTINGS = {
212 "qwen2_5_vl-transformers": VLMTestInfo(
213 models=["Qwen/Qwen2.5-VL-3B-Instruct"],
214 test_type=VLMTestType.IMAGE,
220 - prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
221 - img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
215 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
216 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
217 max_model_len=4096,
218 max_num_seqs=2,
219 auto_cls=AutoModelForImageTextToText,
@@ -228,23 +223,24 @@ VLM_TEST_SETTINGS = {
223 "model_impl": "transformers",
224 },
225 # FIXME: Investigate mrope issue
231 - marks=[large_gpu_mark(min_gb=32),
232 - pytest.mark.skip(reason="Mrope issue")],
226 + marks=[large_gpu_mark(min_gb=32), pytest.mark.skip(reason="Mrope issue")],
227 ),
228 #### Extended model tests
229 "aria": VLMTestInfo(
230 models=["rhymes-ai/Aria"],
231 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
238 - prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
232 + prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
233 img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
234 max_model_len=4096,
235 max_num_seqs=2,
236 auto_cls=AutoModelForImageTextToText,
243 - single_image_prompts=IMAGE_ASSETS.prompts({
244 - "stop_sign": "<vlm_image>Please describe the image shortly.",
245 - "cherry_blossom": "<vlm_image>Please infer the season with reason.", # noqa: E501
246 - }),
247 - multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
237 + single_image_prompts=IMAGE_ASSETS.prompts(
238 + {
239 + "stop_sign": "<vlm_image>Please describe the image shortly.",
240 + "cherry_blossom": "<vlm_image>Please infer the season with reason.", # noqa: E501
241 + }
242 + ),
243 + multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
244 stop_str=["<|im_end|>"],
245 image_size_factors=[(0.10, 0.15)],
246 max_tokens=64,
@@ -253,11 +249,13 @@ VLM_TEST_SETTINGS = {
249 "aya_vision": VLMTestInfo(
250 models=["CohereForAI/aya-vision-8b"],
251 test_type=(VLMTestType.IMAGE),
256 - prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
257 - single_image_prompts=IMAGE_ASSETS.prompts({
258 - "stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
259 - "cherry_blossom": "<image>What is the season?", # noqa: E501
260 - }),
252 + prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
253 + single_image_prompts=IMAGE_ASSETS.prompts(
254 + {
255 + "stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
256 + "cherry_blossom": "<image>What is the season?", # noqa: E501
257 + }
258 + ),
259 multi_image_prompt="<image><image>Describe the two images in detail.", # noqa: E501
260 max_model_len=4096,
261 max_num_seqs=2,
@@ -267,11 +265,13 @@ VLM_TEST_SETTINGS = {
265 "aya_vision-multi_image": VLMTestInfo(
266 models=["CohereForAI/aya-vision-8b"],
267 test_type=(VLMTestType.MULTI_IMAGE),
270 - prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
271 - single_image_prompts=IMAGE_ASSETS.prompts({
272 - "stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
273 - "cherry_blossom": "<image>What is the season?", # noqa: E501
274 - }),
268 + prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
269 + single_image_prompts=IMAGE_ASSETS.prompts(
270 + {
271 + "stop_sign": "<image>What's the content in the center of the image?", # noqa: E501
272 + "cherry_blossom": "<image>What is the season?", # noqa: E501
273 + }
274 + ),
275 multi_image_prompt="<image><image>Describe the two images in detail.", # noqa: E501
276 max_model_len=4096,
277 max_num_seqs=2,
@@ -297,27 +297,29 @@ VLM_TEST_SETTINGS = {
297 max_num_seqs=2,
298 auto_cls=AutoModelForImageTextToText,
299 # For chameleon, we only compare the sequences
300 - vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
301 - hf_output_post_proc = lambda hf_output, model: hf_output[:2],
300 + vllm_output_post_proc=lambda vllm_output, model: vllm_output[:2],
301 + hf_output_post_proc=lambda hf_output, model: hf_output[:2],
302 comparator=check_outputs_equal,
303 max_tokens=8,
304 dtype="bfloat16",
305 ),
306 "deepseek_vl_v2": VLMTestInfo(
307 - models=["Isotr0py/deepseek-vl2-tiny"], # model repo using dynamic module
307 + models=["Isotr0py/deepseek-vl2-tiny"], # model repo using dynamic module
308 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
309 - prompt_formatter=lambda img_prompt: f"<|User|>: {img_prompt}\n\n<|Assistant|>: ", # noqa: E501
309 + prompt_formatter=lambda img_prompt: f"<|User|>: {img_prompt}\n\n<|Assistant|>: ", # noqa: E501
310 max_model_len=4096,
311 max_num_seqs=2,
312 - single_image_prompts=IMAGE_ASSETS.prompts({
313 - "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
314 - "cherry_blossom": "<image>\nPlease infer the season with reason in details.", # noqa: E501
315 - }),
316 - multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?", # noqa: E501
312 + single_image_prompts=IMAGE_ASSETS.prompts(
313 + {
314 + "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
315 + "cherry_blossom": "<image>\nPlease infer the season with reason in details.", # noqa: E501
316 + }
317 + ),
318 + multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?", # noqa: E501
319 patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
320 hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
321 stop_str=["<｜end▁of▁sentence｜>", "<｜begin▁of▁sentence｜>"], # noqa: E501
320 - image_size_factors=[(), (1.0, ), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
322 + image_size_factors=[(), (1.0,), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
323 ),
324 "fuyu": VLMTestInfo(
325 models=["adept/fuyu-8b"],
@@ -336,11 +338,13 @@ VLM_TEST_SETTINGS = {
338 "gemma3": VLMTestInfo(
339 models=["google/gemma-3-4b-it"],
340 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
339 - prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
340 - single_image_prompts=IMAGE_ASSETS.prompts({
341 - "stop_sign": "<start_of_image>What's the content in the center of the image?", # noqa: E501
342 - "cherry_blossom": "<start_of_image>What is the season?", # noqa: E501
343 - }),
341 + prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
342 + single_image_prompts=IMAGE_ASSETS.prompts(
343 + {
344 + "stop_sign": "<start_of_image>What's the content in the center of the image?", # noqa: E501
345 + "cherry_blossom": "<start_of_image>What is the season?", # noqa: E501
346 + }
347 + ),
348 multi_image_prompt="<start_of_image><start_of_image>Describe the two images in detail.", # noqa: E501
349 max_model_len=4096,
350 max_num_seqs=2,
@@ -353,10 +357,12 @@ VLM_TEST_SETTINGS = {
357 models=["zai-org/glm-4v-9b"],
358 test_type=VLMTestType.IMAGE,
359 prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>", # noqa: E501
356 - single_image_prompts=IMAGE_ASSETS.prompts({
357 - "stop_sign": "<|begin_of_image|><|endoftext|><|end_of_image|>What's the content in the center of the image?", # noqa: E501
358 - "cherry_blossom": "<|begin_of_image|><|endoftext|><|end_of_image|>What is the season?", # noqa: E501
359 - }),
360 + single_image_prompts=IMAGE_ASSETS.prompts(
361 + {
362 + "stop_sign": "<|begin_of_image|><|endoftext|><|end_of_image|>What's the content in the center of the image?", # noqa: E501
363 + "cherry_blossom": "<|begin_of_image|><|endoftext|><|end_of_image|>What is the season?", # noqa: E501
364 + }
365 + ),
366 max_model_len=2048,
367 max_num_seqs=2,
368 get_stop_token_ids=lambda tok: [151329, 151336, 151338],
@@ -372,8 +378,8 @@ VLM_TEST_SETTINGS = {
378 models=["zai-org/GLM-4.1V-9B-Thinking"],
379 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
380 prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>", # noqa: E501
375 - img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>", # noqa: E501
376 - video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>", # noqa: E501
381 + img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>", # noqa: E501
382 + video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>", # noqa: E501
383 max_model_len=2048,
384 max_num_seqs=2,
385 get_stop_token_ids=lambda tok: [151329, 151336, 151338],
@@ -390,23 +396,27 @@ VLM_TEST_SETTINGS = {
396 max_num_seqs=2,
397 auto_cls=AutoModelForImageTextToText,
398 patch_hf_runner=model_utils.glm4_1v_patch_hf_runner,
393 - custom_test_opts=[CustomTestOptions(
394 - inputs=custom_inputs.video_with_metadata_glm4_1v(),
395 - limit_mm_per_prompt={"video": 1},
396 - )],
399 + custom_test_opts=[
400 + CustomTestOptions(
401 + inputs=custom_inputs.video_with_metadata_glm4_1v(),
402 + limit_mm_per_prompt={"video": 1},
403 + )
404 + ],
405 marks=[large_gpu_mark(min_gb=32)],
406 ),
407 "h2ovl": VLMTestInfo(
400 - models = [
408 + models=[
409 "h2oai/h2ovl-mississippi-800m",
410 "h2oai/h2ovl-mississippi-2b",
411 ],
412 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
405 - prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>", # noqa: E501
406 - single_image_prompts=IMAGE_ASSETS.prompts({
407 - "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
408 - "cherry_blossom": "<image>\nWhat is the season?",
409 - }),
413 + prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>", # noqa: E501
414 + single_image_prompts=IMAGE_ASSETS.prompts(
415 + {
416 + "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
417 + "cherry_blossom": "<image>\nWhat is the season?",
418 + }
419 + ),
420 multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.", # noqa: E501
421 max_model_len=8192,
422 use_tokenizer_eos=True,
@@ -416,7 +426,7 @@ VLM_TEST_SETTINGS = {
426 "idefics3": VLMTestInfo(
427 models=["HuggingFaceTB/SmolVLM-256M-Instruct"],
428 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
419 - prompt_formatter=lambda img_prompt:f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
429 + prompt_formatter=lambda img_prompt: f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:", # noqa: E501
430 img_idx_to_prompt=lambda idx: "<image>",
431 max_model_len=8192,
432 max_num_seqs=2,
@@ -431,11 +441,13 @@ VLM_TEST_SETTINGS = {
441 # "OpenGVLab/Mono-InternVL-2B",
442 ],
443 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
434 - prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
435 - single_image_prompts=IMAGE_ASSETS.prompts({
436 - "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
437 - "cherry_blossom": "<image>\nWhat is the season?",
438 - }),
444 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
445 + single_image_prompts=IMAGE_ASSETS.prompts(
446 + {
447 + "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
448 + "cherry_blossom": "<image>\nWhat is the season?",
449 + }
450 + ),
451 multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.", # noqa: E501
452 max_model_len=4096,
453 use_tokenizer_eos=True,
@@ -446,7 +458,7 @@ VLM_TEST_SETTINGS = {
458 "OpenGVLab/InternVL3-1B",
459 ],
460 test_type=VLMTestType.VIDEO,
449 - prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
461 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
462 video_idx_to_prompt=lambda idx: "<video>",
463 max_model_len=8192,
464 use_tokenizer_eos=True,
@@ -459,7 +471,7 @@ VLM_TEST_SETTINGS = {
471 VLMTestType.MULTI_IMAGE,
472 VLMTestType.VIDEO,
473 ),
462 - prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
474 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
475 img_idx_to_prompt=lambda idx: "<IMG_CONTEXT>",
476 video_idx_to_prompt=lambda idx: "<video>",
477 max_model_len=8192,
@@ -469,7 +481,7 @@ VLM_TEST_SETTINGS = {
481 "kimi_vl": VLMTestInfo(
482 models=["moonshotai/Kimi-VL-A3B-Instruct"],
483 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
472 - prompt_formatter=lambda img_prompt: f"<|im_user|>user<|im_middle|>{img_prompt}<|im_end|><|im_assistant|>assistant<|im_middle|>", # noqa: E501
484 + prompt_formatter=lambda img_prompt: f"<|im_user|>user<|im_middle|>{img_prompt}<|im_end|><|im_assistant|>assistant<|im_middle|>", # noqa: E501
485 img_idx_to_prompt=lambda _: "<|media_start|>image<|media_content|><|media_pad|><|media_end|>", # noqa: E501
486 max_model_len=8192,
487 max_num_seqs=2,
@@ -480,11 +492,11 @@ VLM_TEST_SETTINGS = {
492 ),
493 "llama4": VLMTestInfo(
494 models=["meta-llama/Llama-4-Scout-17B-16E-Instruct"],
483 - prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|header_start|>user<|header_end|>\n\n{img_prompt}<|eot|><|header_start|>assistant<|header_end|>\n\n", # noqa: E501
495 + prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|header_start|>user<|header_end|>\n\n{img_prompt}<|eot|><|header_start|>assistant<|header_end|>\n\n", # noqa: E501
496 img_idx_to_prompt=lambda _: "<|image|>",
497 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
498 distributed_executor_backend="mp",
487 - image_size_factors=[(.25, 0.5, 1.0)],
499 + image_size_factors=[(0.25, 0.5, 1.0)],
500 hf_model_kwargs={"device_map": "auto"},
501 max_model_len=8192,
502 max_num_seqs=4,
@@ -500,28 +512,34 @@ VLM_TEST_SETTINGS = {
512 max_model_len=10240,
513 auto_cls=AutoModelForImageTextToText,
514 vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
503 - custom_test_opts=[CustomTestOptions(
504 - inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
505 - formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]"
506 - ),
507 - limit_mm_per_prompt={"image": 4},
508 - )],
515 + custom_test_opts=[
516 + CustomTestOptions(
517 + inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
518 + formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]"
519 + ),
520 + limit_mm_per_prompt={"image": 4},
521 + )
522 + ],
523 ),
524 "llava_onevision": VLMTestInfo(
525 models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
526 test_type=VLMTestType.CUSTOM_INPUTS,
513 - prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
527 + prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
528 num_video_frames=16,
529 max_model_len=16384,
516 - hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
530 + hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
531 + "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
532 + ), # noqa: E501
533 auto_cls=AutoModelForImageTextToText,
534 vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
519 - custom_test_opts=[CustomTestOptions(
520 - inputs=custom_inputs.multi_video_multi_aspect_ratio_inputs(
521 - formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
522 - ),
523 - limit_mm_per_prompt={"video": 4},
524 - )],
535 + custom_test_opts=[
536 + CustomTestOptions(
537 + inputs=custom_inputs.multi_video_multi_aspect_ratio_inputs(
538 + formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
539 + ),
540 + limit_mm_per_prompt={"video": 4},
541 + )
542 + ],
543 ),
544 "llava_next_video": VLMTestInfo(
545 models=["llava-hf/LLaVA-NeXT-Video-7B-hf"],
@@ -563,7 +581,9 @@ VLM_TEST_SETTINGS = {
581 img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
582 max_model_len=4096,
583 max_num_seqs=2,
566 - get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
584 + get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(
585 + ["<|im_end|>", "<|endoftext|>"]
586 + ), # noqa: E501
587 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
588 patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,
589 # FIXME: https://huggingface.co/openbmb/MiniCPM-o-2_6/discussions/49
@@ -576,13 +596,15 @@ VLM_TEST_SETTINGS = {
596 img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
597 max_model_len=4096,
598 max_num_seqs=2,
579 - get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
599 + get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(
600 + ["<|im_end|>", "<|endoftext|>"]
601 + ), # noqa: E501
602 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
603 patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,
604 ),
605 "minimax_vl_01": VLMTestInfo(
606 models=["MiniMaxAI/MiniMax-VL-01"],
585 - prompt_formatter=lambda img_prompt: f"<beginning_of_sentence>user: {img_prompt} assistant:<end_of_sentence>", # noqa: E501
607 + prompt_formatter=lambda img_prompt: f"<beginning_of_sentence>user: {img_prompt} assistant:<end_of_sentence>", # noqa: E501
608 img_idx_to_prompt=lambda _: "<image>",
609 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
610 max_model_len=8192,
@@ -604,8 +626,8 @@ VLM_TEST_SETTINGS = {
626 "ovis1_6-gemma2": VLMTestInfo(
627 models=["AIDC-AI/Ovis1.6-Gemma2-9B"],
628 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
607 - prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
608 - img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
629 + prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
630 + img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
631 max_model_len=4096,
632 max_num_seqs=2,
633 dtype="half",
@@ -617,8 +639,8 @@ VLM_TEST_SETTINGS = {
639 "ovis2": VLMTestInfo(
640 models=["AIDC-AI/Ovis2-1B"],
641 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
620 - prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
621 - img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
642 + prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
643 + img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
644 max_model_len=4096,
645 max_num_seqs=2,
646 dtype="half",
@@ -628,13 +650,9 @@ VLM_TEST_SETTINGS = {
650 ),
651 "ovis2_5": VLMTestInfo(
652 models=["AIDC-AI/Ovis2.5-2B"],
631 - test_type=(
632 - VLMTestType.IMAGE,
633 - VLMTestType.MULTI_IMAGE,
634 - VLMTestType.VIDEO
635 - ),
636 - prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
637 - img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
653 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
654 + prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
655 + img_idx_to_prompt=lambda idx: "<image>\n", # noqa: E501
656 video_idx_to_prompt=lambda idx: "<video>\n",
657 max_model_len=4096,
658 max_num_seqs=2,
@@ -646,7 +664,7 @@ VLM_TEST_SETTINGS = {
664 "phi3v": VLMTestInfo(
665 models=["microsoft/Phi-3.5-vision-instruct"],
666 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
649 - prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|end|>\n<|assistant|>\n", # noqa: E501
667 + prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|end|>\n<|assistant|>\n", # noqa: E501
668 img_idx_to_prompt=lambda idx: f"<|image_{idx}|>\n",
669 max_model_len=4096,
670 max_num_seqs=2,
@@ -681,15 +699,11 @@ VLM_TEST_SETTINGS = {
699 ),
700 "qwen2_vl": VLMTestInfo(
701 models=["Qwen/Qwen2-VL-2B-Instruct"],
684 - test_type=(
685 - VLMTestType.IMAGE,
686 - VLMTestType.MULTI_IMAGE,
687 - VLMTestType.VIDEO
688 - ),
689 - prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
690 - img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
691 - video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
692 - multi_image_prompt="Picture 1: <vlm_image>\nPicture 2: <vlm_image>\nDescribe these two images with one paragraph respectively.", # noqa: E501
702 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
703 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
704 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
705 + video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
706 + multi_image_prompt="Picture 1: <vlm_image>\nPicture 2: <vlm_image>\nDescribe these two images with one paragraph respectively.", # noqa: E501
707 max_model_len=4096,
708 max_num_seqs=2,
709 auto_cls=AutoModelForImageTextToText,
@@ -700,11 +714,13 @@ VLM_TEST_SETTINGS = {
714 "skywork_r1v": VLMTestInfo(
715 models=["Skywork/Skywork-R1V-38B"],
716 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
703 - prompt_formatter=lambda img_prompt: f"<｜begin▁of▁sentence｜><｜User｜>\n{img_prompt}<｜Assistant｜><think>\n", # noqa: E501
704 - single_image_prompts=IMAGE_ASSETS.prompts({
705 - "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
706 - "cherry_blossom": "<image>\nWhat is the season?",
707 - }),
717 + prompt_formatter=lambda img_prompt: f"<｜begin▁of▁sentence｜><｜User｜>\n{img_prompt}<｜Assistant｜><think>\n", # noqa: E501
718 + single_image_prompts=IMAGE_ASSETS.prompts(
719 + {
720 + "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
721 + "cherry_blossom": "<image>\nWhat is the season?",
722 + }
723 + ),
724 multi_image_prompt="<image>\n<image>\nDescribe the two images in short.", # noqa: E501
725 max_model_len=4096,
726 use_tokenizer_eos=True,
@@ -737,9 +753,9 @@ VLM_TEST_SETTINGS = {
753 VLMTestType.MULTI_IMAGE,
754 VLMTestType.VIDEO,
755 ),
740 - prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
741 - img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
742 - video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
756 + prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
757 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
758 + video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
759 max_model_len=4096,
760 max_num_seqs=2,
761 auto_cls=AutoModelForImageTextToText,
@@ -752,11 +768,11 @@ VLM_TEST_SETTINGS = {
768 prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
769 max_model_len=4096,
770 auto_cls=AutoModelForImageTextToText,
755 - vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
756 - hf_output_post_proc = lambda hf_output, model: hf_output[:2],
771 + vllm_output_post_proc=lambda vllm_output, model: vllm_output[:2],
772 + hf_output_post_proc=lambda hf_output, model: hf_output[:2],
773 comparator=check_outputs_equal,
774 marks=multi_gpu_marks(num_gpus=2),
759 - **COMMON_BROADCAST_SETTINGS # type: ignore
775 + **COMMON_BROADCAST_SETTINGS, # type: ignore
776 ),
777 "llava-broadcast": VLMTestInfo(
778 models=["llava-hf/llava-1.5-7b-hf"],
@@ -765,7 +781,7 @@ VLM_TEST_SETTINGS = {
781 auto_cls=AutoModelForImageTextToText,
782 vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
783 marks=multi_gpu_marks(num_gpus=2),
768 - **COMMON_BROADCAST_SETTINGS # type: ignore
784 + **COMMON_BROADCAST_SETTINGS, # type: ignore
785 ),
786 "llava_next-broadcast": VLMTestInfo(
787 models=["llava-hf/llava-v1.6-mistral-7b-hf"],
@@ -774,12 +790,12 @@ VLM_TEST_SETTINGS = {
790 auto_cls=AutoModelForImageTextToText,
791 vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
792 marks=multi_gpu_marks(num_gpus=2),
777 - **COMMON_BROADCAST_SETTINGS # type: ignore
793 + **COMMON_BROADCAST_SETTINGS, # type: ignore
794 ),
795 ### Custom input edge-cases for specific models
796 "intern_vl-diff-patches": VLMTestInfo(
797 models=["OpenGVLab/InternVL2-2B"],
782 - prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
798 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
799 test_type=VLMTestType.CUSTOM_INPUTS,
800 max_model_len=4096,
801 use_tokenizer_eos=True,
@@ -788,7 +804,8 @@ VLM_TEST_SETTINGS = {
804 CustomTestOptions(
805 inputs=inp,
806 limit_mm_per_prompt={"image": 2},
791 - ) for inp in custom_inputs.different_patch_input_cases_internvl()
807 + )
808 + for inp in custom_inputs.different_patch_input_cases_internvl()
809 ],
810 ),
811 "llava_onevision-multiple-images": VLMTestInfo(
@@ -797,14 +814,18 @@ VLM_TEST_SETTINGS = {
814 max_model_len=16384,
815 max_num_seqs=2,
816 auto_cls=AutoModelForImageTextToText,
800 - hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
817 + hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
818 + "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
819 + ), # noqa: E501
820 vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
802 - custom_test_opts=[CustomTestOptions(
803 - inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
804 - formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
805 - ),
806 - limit_mm_per_prompt={"image": 4},
807 - )],
821 + custom_test_opts=[
822 + CustomTestOptions(
823 + inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
824 + formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
825 + ),
826 + limit_mm_per_prompt={"image": 4},
827 + )
828 + ],
829 ),
830 # regression test for https://github.com/vllm-project/vllm/issues/15122
831 "qwen2_5_vl-windows-attention": VLMTestInfo(
@@ -814,13 +835,14 @@ VLM_TEST_SETTINGS = {
835 max_num_seqs=2,
836 auto_cls=AutoModelForImageTextToText,
837 vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
817 - custom_test_opts=[CustomTestOptions(
818 - inputs=custom_inputs.windows_attention_image_qwen2_5_vl(),
819 - limit_mm_per_prompt={"image": 1},
820 - )],
838 + custom_test_opts=[
839 + CustomTestOptions(
840 + inputs=custom_inputs.windows_attention_image_qwen2_5_vl(),
841 + limit_mm_per_prompt={"image": 1},
842 + )
843 + ],
844 ),
845 }
823 -# yapf: enable
846
847
848 def _mark_splits(
tests/models/multimodal/generation/vlm_utils/case_filtering.py
+3 -4
@@ -114,7 +114,6 @@ def get_parametrized_options(
114 raise ValueError("Test has type CUSTOM_INPUTS, but none given")
115 iter_kwargs["custom_test_opts"] = test_info.custom_test_opts
116
117 - # yapf: disable
117 # Wrap all model cases in a pytest parameter & pass marks through
118 return [
119 pytest.param(
@@ -122,10 +121,10 @@ def get_parametrized_options(
121 ExpandableVLMTestArgs(
122 **{k: v for k, v in zip(iter_kwargs.keys(), case)}
123 ),
125 - marks=test_info.marks if test_info.marks is not None else []
126 - ) for case in list(itertools.product(*iter_kwargs.values()))
124 + marks=test_info.marks if test_info.marks is not None else [],
125 + )
126 + for case in list(itertools.product(*iter_kwargs.values()))
127 ]
128 - # yapf: enable
128
129 # Get a list per model type, where each entry contains a tuple of all of
130 # that model type's cases, then flatten them into the top level so that
tests/models/multimodal/generation/vlm_utils/model_utils.py
-2
@@ -418,7 +418,6 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
418 self.image_size = self.vision_config.image_size
419
420 def __call__(self, text: str, images: Union[Image, list[Image]], **kwargs):
421 - # yapf: disable
421 from vllm.model_executor.models.h2ovl import (
422 IMG_CONTEXT,
423 IMG_END,
@@ -426,7 +425,6 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
425 image_to_pixel_values_h2ovl,
426 )
427
429 - # yapf: enable
428 images = [images] if isinstance(images, Image) else images
429 pixel_values = [
430 image_to_pixel_values_h2ovl(
tests/models/multimodal/generation/vlm_utils/types.py
+14 -12
@@ -33,24 +33,26 @@ TEST_IMG_PLACEHOLDER = "<vlm_image>"
33 TEST_VIDEO_PLACEHOLDER = "<vlm_video>"
34 TEST_AUDIO_PLACEHOLDER = "<lmm_audio>"
35
36 -# yapf: disable
37 -SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts({
38 - "stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
39 - "cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
40 -})
41 -SINGLE_AUDIO_BASE_PROMPT = AUDIO_ASSETS.prompts({
42 - "mary_had_lamb": f"{TEST_AUDIO_PLACEHOLDER}Transcribe this audio into English.", # noqa: E501
43 - "winning_call": f"{TEST_AUDIO_PLACEHOLDER}What is happening in this audio clip?", # noqa: E501
44 -})
36 +SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts(
37 + {
38 + "stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
39 + "cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
40 + }
41 +)
42 +SINGLE_AUDIO_BASE_PROMPT = AUDIO_ASSETS.prompts(
43 + {
44 + "mary_had_lamb": f"{TEST_AUDIO_PLACEHOLDER}Transcribe this audio into English.", # noqa: E501
45 + "winning_call": f"{TEST_AUDIO_PLACEHOLDER}What is happening in this audio clip?", # noqa: E501
46 + }
47 +)
48
49 MULTI_IMAGE_BASE_PROMPT = f"Image-1: {TEST_IMG_PLACEHOLDER}Image-2: {TEST_IMG_PLACEHOLDER}Describe the two images in detail.\n" # noqa: E501
50 VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"
51
52
50 -IMAGE_SIZE_FACTORS = [(), (1.0, ), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
51 -EMBEDDING_SIZE_FACTORS = [(), (1.0, ), (1.0, 1.0, 1.0)]
53 +IMAGE_SIZE_FACTORS = [(), (1.0,), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
54 +EMBEDDING_SIZE_FACTORS = [(), (1.0,), (1.0, 1.0, 1.0)]
55 RunnerOutput = tuple[list[int], str, Optional[SampleLogprobs]]
53 -# yapf: enable
56
57
58 class PromptWithMultiModalInput(NamedTuple):
tests/models/multimodal/processing/test_common.py
+72 -71
@@ -322,80 +322,81 @@ def _test_processing_correctness_one(
322 )
323
324
325 -# yapf: disable
326 -@pytest.mark.parametrize("model_id", [
327 - "rhymes-ai/Aria",
328 - "CohereForAI/aya-vision-8b",
329 - "Salesforce/blip2-opt-2.7b",
330 - "facebook/chameleon-7b",
331 - "CohereLabs/command-a-vision-07-2025",
332 - "deepseek-ai/deepseek-vl2-tiny",
333 - "baidu/ERNIE-4.5-VL-28B-A3B-PT",
334 - "adept/fuyu-8b",
335 - "google/gemma-3-4b-it",
336 - "google/gemma-3n-E2B-it",
337 - "zai-org/glm-4v-9b",
338 - "zai-org/GLM-4.1V-9B-Thinking",
339 - "zai-org/GLM-4.5V",
340 - "ibm-granite/granite-speech-3.3-2b",
341 - "h2oai/h2ovl-mississippi-800m",
342 - "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
343 - "HuggingFaceM4/Idefics3-8B-Llama3",
344 - "internlm/Intern-S1",
345 - "OpenGVLab/InternVL2-1B",
346 - "OpenGVLab/InternVL3-1B",
347 - "OpenGVLab/InternVL3_5-1B",
348 - "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview",
349 - "OpenGVLab/InternVL3_5-30B-A3B",
350 - "Kwai-Keye/Keye-VL-8B-Preview",
351 - "Kwai-Keye/Keye-VL-1_5-8B",
352 - "moonshotai/Kimi-VL-A3B-Instruct",
353 - "meta-llama/Llama-4-Scout-17B-16E-Instruct",
354 - "llava-hf/llava-1.5-7b-hf",
355 - "llava-hf/llava-v1.6-mistral-7b-hf",
356 - "llava-hf/LLaVA-NeXT-Video-7B-hf",
357 - "llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
358 - "TIGER-Lab/Mantis-8B-siglip-llama3",
359 - "mispeech/midashenglm-7b",
360 - "openbmb/MiniCPM-Llama3-V-2_5",
361 - "openbmb/MiniCPM-o-2_6",
362 - "openbmb/MiniCPM-V-2_6",
363 - "MiniMaxAI/MiniMax-VL-01",
364 - "allenai/Molmo-7B-D-0924",
365 - "allenai/Molmo-7B-O-0924",
366 - "nvidia/NVLM-D-72B",
367 - "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
368 - "AIDC-AI/Ovis1.6-Gemma2-9B",
369 - "AIDC-AI/Ovis1.6-Llama3.2-3B",
370 - "AIDC-AI/Ovis2-1B",
371 - "AIDC-AI/Ovis2.5-2B",
372 - "google/paligemma-3b-mix-224",
373 - "google/paligemma2-3b-ft-docci-448",
374 - "microsoft/Phi-3.5-vision-instruct",
375 - "microsoft/Phi-4-multimodal-instruct",
376 - "mistralai/Pixtral-12B-2409",
377 - "mistral-community/pixtral-12b",
378 - "Qwen/Qwen-VL-Chat",
379 - "Qwen/Qwen2-VL-2B-Instruct",
380 - "Qwen/Qwen2.5-VL-3B-Instruct",
381 - "Qwen/Qwen2-Audio-7B-Instruct",
382 - "Qwen/Qwen2.5-Omni-3B",
383 - "Qwen/Qwen3-VL-4B-Instruct",
384 - "Qwen/Qwen3-VL-30B-A3B-Instruct",
385 - "YannQi/R-4B",
386 - "Skywork/Skywork-R1V-38B",
387 - "HuggingFaceTB/SmolVLM2-2.2B-Instruct",
388 - "stepfun-ai/step3",
389 - "fixie-ai/ultravox-v0_5-llama-3_2-1b",
390 - "openai/whisper-large-v3",
391 - "omni-research/Tarsier-7b",
392 - "omni-research/Tarsier2-Recap-7b",
393 - "mistralai/Voxtral-Mini-3B-2507",
394 -])
325 +@pytest.mark.parametrize(
326 + "model_id",
327 + [
328 + "rhymes-ai/Aria",
329 + "CohereForAI/aya-vision-8b",
330 + "Salesforce/blip2-opt-2.7b",
331 + "facebook/chameleon-7b",
332 + "CohereLabs/command-a-vision-07-2025",
333 + "deepseek-ai/deepseek-vl2-tiny",
334 + "baidu/ERNIE-4.5-VL-28B-A3B-PT",
335 + "adept/fuyu-8b",
336 + "google/gemma-3-4b-it",
337 + "google/gemma-3n-E2B-it",
338 + "zai-org/glm-4v-9b",
339 + "zai-org/GLM-4.1V-9B-Thinking",
340 + "zai-org/GLM-4.5V",
341 + "ibm-granite/granite-speech-3.3-2b",
342 + "h2oai/h2ovl-mississippi-800m",
343 + "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
344 + "HuggingFaceM4/Idefics3-8B-Llama3",
345 + "internlm/Intern-S1",
346 + "OpenGVLab/InternVL2-1B",
347 + "OpenGVLab/InternVL3-1B",
348 + "OpenGVLab/InternVL3_5-1B",
349 + "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview",
350 + "OpenGVLab/InternVL3_5-30B-A3B",
351 + "Kwai-Keye/Keye-VL-8B-Preview",
352 + "Kwai-Keye/Keye-VL-1_5-8B",
353 + "moonshotai/Kimi-VL-A3B-Instruct",
354 + "meta-llama/Llama-4-Scout-17B-16E-Instruct",
355 + "llava-hf/llava-1.5-7b-hf",
356 + "llava-hf/llava-v1.6-mistral-7b-hf",
357 + "llava-hf/LLaVA-NeXT-Video-7B-hf",
358 + "llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
359 + "TIGER-Lab/Mantis-8B-siglip-llama3",
360 + "mispeech/midashenglm-7b",
361 + "openbmb/MiniCPM-Llama3-V-2_5",
362 + "openbmb/MiniCPM-o-2_6",
363 + "openbmb/MiniCPM-V-2_6",
364 + "MiniMaxAI/MiniMax-VL-01",
365 + "allenai/Molmo-7B-D-0924",
366 + "allenai/Molmo-7B-O-0924",
367 + "nvidia/NVLM-D-72B",
368 + "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
369 + "AIDC-AI/Ovis1.6-Gemma2-9B",
370 + "AIDC-AI/Ovis1.6-Llama3.2-3B",
371 + "AIDC-AI/Ovis2-1B",
372 + "AIDC-AI/Ovis2.5-2B",
373 + "google/paligemma-3b-mix-224",
374 + "google/paligemma2-3b-ft-docci-448",
375 + "microsoft/Phi-3.5-vision-instruct",
376 + "microsoft/Phi-4-multimodal-instruct",
377 + "mistralai/Pixtral-12B-2409",
378 + "mistral-community/pixtral-12b",
379 + "Qwen/Qwen-VL-Chat",
380 + "Qwen/Qwen2-VL-2B-Instruct",
381 + "Qwen/Qwen2.5-VL-3B-Instruct",
382 + "Qwen/Qwen2-Audio-7B-Instruct",
383 + "Qwen/Qwen2.5-Omni-3B",
384 + "Qwen/Qwen3-VL-4B-Instruct",
385 + "Qwen/Qwen3-VL-30B-A3B-Instruct",
386 + "YannQi/R-4B",
387 + "Skywork/Skywork-R1V-38B",
388 + "HuggingFaceTB/SmolVLM2-2.2B-Instruct",
389 + "stepfun-ai/step3",
390 + "fixie-ai/ultravox-v0_5-llama-3_2-1b",
391 + "openai/whisper-large-v3",
392 + "omni-research/Tarsier-7b",
393 + "omni-research/Tarsier2-Recap-7b",
394 + "mistralai/Voxtral-Mini-3B-2507",
395 + ],
396 +)
397 @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
398 @pytest.mark.parametrize("num_batches", [32])
399 @pytest.mark.parametrize("simplify_rate", [1.0])
398 -# yapf: enable
400 def test_processing_correctness(
401 model_id: str,
402 hit_rate: float,
tests/models/multimodal/processing/test_idefics3.py
-2
@@ -12,7 +12,6 @@ from ...utils import build_model_context
12
13
14 @pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"])
15 -# yapf: disable
15 @pytest.mark.parametrize(
16 ("mm_processor_kwargs", "expected_toks_per_img"),
17 [
@@ -20,7 +19,6 @@ from ...utils import build_model_context
19 ({"size": {"longest_edge": 728}}, 169 * (2**2 + 1)),
20 ],
21 )
23 -# yapf: enable
22 @pytest.mark.parametrize("num_imgs", [1, 2])
23 @pytest.mark.parametrize("kwargs_on_init", [True, False])
24 def test_processor_override(
tests/models/multimodal/processing/test_phi3v.py
-2
@@ -11,7 +11,6 @@ from ...utils import build_model_context
11
12
13 @pytest.mark.parametrize("model_id", ["microsoft/Phi-3.5-vision-instruct"])
14 -# yapf: disable
14 @pytest.mark.parametrize(
15 ("mm_processor_kwargs", "expected_toks_per_img"),
16 [
@@ -21,7 +20,6 @@ from ...utils import build_model_context
20 ({}, 757),
21 ],
22 )
24 -# yapf: enable
23 @pytest.mark.parametrize("num_imgs", [1, 2])
24 @pytest.mark.parametrize("kwargs_on_init", [True, False])
25 def test_processor_override(
tests/models/multimodal/processing/test_phi4mm.py
-2
@@ -11,7 +11,6 @@ from ...utils import build_model_context
11
12
13 @pytest.mark.parametrize("model_id", ["microsoft/Phi-4-multimodal-instruct"])
14 -# yapf: disable
14 @pytest.mark.parametrize(
15 ("mm_processor_kwargs", "expected_toks_per_img"),
16 [
@@ -21,7 +20,6 @@ from ...utils import build_model_context
20 ({}, 9585),
21 ],
22 )
24 -# yapf: enable
23 @pytest.mark.parametrize("num_imgs", [1, 2])
24 @pytest.mark.parametrize("kwargs_on_init", [True, False])
25 def test_processor_override(
tests/models/multimodal/processing/test_qwen2_vl.py
-2
@@ -10,7 +10,6 @@ from ...utils import build_model_context
10
11
12 @pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
13 -# yapf: disable
13 @pytest.mark.parametrize(
14 ("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"),
15 [
@@ -18,7 +17,6 @@ from ...utils import build_model_context
17 ({"min_pixels": 64**2, "max_pixels": 512**2}, 330, (1320, 1176)),
18 ],
19 )
21 -# yapf: enable
20 @pytest.mark.parametrize("num_imgs", [1, 2])
21 @pytest.mark.parametrize("kwargs_on_init", [True, False])
22 def test_processor_override(
tests/models/multimodal/processing/test_smolvlm.py
-2
@@ -12,7 +12,6 @@ from ...utils import build_model_context
12
13
14 @pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"])
15 -# yapf: disable
15 @pytest.mark.parametrize(
16 ("mm_processor_kwargs", "expected_toks_per_img"),
17 [
@@ -20,7 +19,6 @@ from ...utils import build_model_context
19 ({"max_image_size": {"longest_edge": 768}}, 405),
20 ],
21 )
23 -# yapf: enable
22 @pytest.mark.parametrize("num_imgs", [1, 2])
23 @pytest.mark.parametrize("kwargs_on_init", [True, False])
24 def test_processor_override(
tests/models/multimodal/processing/test_transformers.py
+24 -9
@@ -7,9 +7,7 @@ from vllm.config import ModelConfig
7 from vllm.multimodal import MULTIMODAL_REGISTRY
8
9
10 -# yapf: disable
11 -@pytest.mark.parametrize("model_id",
12 - ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
10 +@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
11 def test_multimodal_processor(model_id):
12 model_config = ModelConfig(
13 model=model_id,
@@ -18,9 +16,9 @@ def test_multimodal_processor(model_id):
16
17 mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
18
21 - image_pil = ImageAsset('cherry_blossom').pil_image
19 + image_pil = ImageAsset("cherry_blossom").pil_image
20 mm_data = {"image": image_pil}
23 - str_prompt = "<|im_start|>user <image>\nWhat is the content of this image?<|im_end|><|im_start|>assistant\n" # noqa: E501
21 + str_prompt = "<|im_start|>user <image>\nWhat is the content of this image?<|im_end|><|im_start|>assistant\n" # noqa: E501
22 str_processed_inputs = mm_processor.apply(
23 prompt=str_prompt,
24 mm_data=mm_data,
@@ -28,8 +26,23 @@ def test_multimodal_processor(model_id):
26 )
27
28 ids_prompt = [
31 - 151644, 872, 220, 151646, 198, 3838, 374, 279, 2213, 315, 419, 2168,
32 - 30, 151645, 151644, 77091, 198
29 + 151644,
30 + 872,
31 + 220,
32 + 151646,
33 + 198,
34 + 3838,
35 + 374,
36 + 279,
37 + 2213,
38 + 315,
39 + 419,
40 + 2168,
41 + 30,
42 + 151645,
43 + 151644,
44 + 77091,
45 + 198,
46 ]
47 ids_processed_inputs = mm_processor.apply(
48 prompt=ids_prompt,
@@ -37,5 +50,7 @@ def test_multimodal_processor(model_id):
50 hf_processor_mm_kwargs={},
51 )
52
40 - assert (str_processed_inputs["prompt_token_ids"]
41 - == ids_processed_inputs["prompt_token_ids"])
53 + assert (
54 + str_processed_inputs["prompt_token_ids"]
55 + == ids_processed_inputs["prompt_token_ids"]
56 + )
tests/models/registry.py
+648 -411
@@ -169,430 +169,625 @@ class _HfExamplesInfo:
169 pytest.skip(msg)
170
171
172 -# yapf: disable
172 _TEXT_GENERATION_EXAMPLE_MODELS = {
173 # [Decoder-only]
175 - "ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-2509",
176 - min_transformers_version="4.56.0",
177 - trust_remote_code=True),
178 - "AquilaModel": _HfExamplesInfo("BAAI/AquilaChat-7B",
179 - trust_remote_code=True),
180 - "AquilaForCausalLM": _HfExamplesInfo("BAAI/AquilaChat2-7B",
181 - trust_remote_code=True),
174 + "ApertusForCausalLM": _HfExamplesInfo(
175 + "swiss-ai/Apertus-8B-2509",
176 + min_transformers_version="4.56.0",
177 + trust_remote_code=True,
178 + ),
179 + "AquilaModel": _HfExamplesInfo("BAAI/AquilaChat-7B", trust_remote_code=True),
180 + "AquilaForCausalLM": _HfExamplesInfo("BAAI/AquilaChat2-7B", trust_remote_code=True),
181 "ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
183 - "ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct",
184 - trust_remote_code=True),
185 - "BaiChuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan-7B",
186 - trust_remote_code=True),
187 - "BaichuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan2-7B-chat",
188 - trust_remote_code=True),
189 - "BailingMoeForCausalLM": _HfExamplesInfo("inclusionAI/Ling-lite-1.5",
190 - trust_remote_code=True),
191 - "BailingMoeV2ForCausalLM": _HfExamplesInfo("inclusionAI/Ling-mini-2.0",
192 - trust_remote_code=True),
193 - "BambaForCausalLM": _HfExamplesInfo("ibm-ai-platform/Bamba-9B-v1",
194 - min_transformers_version="4.55.3",
195 - extras={"tiny": "hmellor/tiny-random-BambaForCausalLM"}), # noqa: E501
196 - "BloomForCausalLM": _HfExamplesInfo("bigscience/bloom-560m",
197 - {"1b": "bigscience/bloomz-1b1"}),
198 - "ChatGLMModel": _HfExamplesInfo("zai-org/chatglm3-6b",
199 - trust_remote_code=True,
200 - max_transformers_version="4.48"),
201 - "ChatGLMForConditionalGeneration": _HfExamplesInfo("thu-coai/ShieldLM-6B-chatglm3", # noqa: E501
202 - trust_remote_code=True),
203 - "CohereForCausalLM": _HfExamplesInfo("CohereForAI/c4ai-command-r-v01",
204 - trust_remote_code=True),
205 - "Cohere2ForCausalLM": _HfExamplesInfo("CohereForAI/c4ai-command-r7b-12-2024", # noqa: E501
206 - trust_remote_code=True),
207 - "CwmForCausalLM": _HfExamplesInfo("facebook/cwm", # noqa: E501
208 - trust_remote_code=True,
209 - is_available_online=False),
182 + "ArcticForCausalLM": _HfExamplesInfo(
183 + "Snowflake/snowflake-arctic-instruct", trust_remote_code=True
184 + ),
185 + "BaiChuanForCausalLM": _HfExamplesInfo(
186 + "baichuan-inc/Baichuan-7B", trust_remote_code=True
187 + ),
188 + "BaichuanForCausalLM": _HfExamplesInfo(
189 + "baichuan-inc/Baichuan2-7B-chat", trust_remote_code=True
190 + ),
191 + "BailingMoeForCausalLM": _HfExamplesInfo(
192 + "inclusionAI/Ling-lite-1.5", trust_remote_code=True
193 + ),
194 + "BailingMoeV2ForCausalLM": _HfExamplesInfo(
195 + "inclusionAI/Ling-mini-2.0", trust_remote_code=True
196 + ),
197 + "BambaForCausalLM": _HfExamplesInfo(
198 + "ibm-ai-platform/Bamba-9B-v1",
199 + min_transformers_version="4.55.3",
200 + extras={"tiny": "hmellor/tiny-random-BambaForCausalLM"},
201 + ),
202 + "BloomForCausalLM": _HfExamplesInfo(
203 + "bigscience/bloom-560m", {"1b": "bigscience/bloomz-1b1"}
204 + ),
205 + "ChatGLMModel": _HfExamplesInfo(
206 + "zai-org/chatglm3-6b", trust_remote_code=True, max_transformers_version="4.48"
207 + ),
208 + "ChatGLMForConditionalGeneration": _HfExamplesInfo(
209 + "thu-coai/ShieldLM-6B-chatglm3",
210 + trust_remote_code=True,
211 + ),
212 + "CohereForCausalLM": _HfExamplesInfo(
213 + "CohereForAI/c4ai-command-r-v01", trust_remote_code=True
214 + ),
215 + "Cohere2ForCausalLM": _HfExamplesInfo(
216 + "CohereForAI/c4ai-command-r7b-12-2024",
217 + trust_remote_code=True,
218 + ),
219 + "CwmForCausalLM": _HfExamplesInfo(
220 + "facebook/cwm",
221 + trust_remote_code=True,
222 + is_available_online=False,
223 + ),
224 "DbrxForCausalLM": _HfExamplesInfo("databricks/dbrx-instruct"),
211 - "DeciLMForCausalLM": _HfExamplesInfo("nvidia/Llama-3_3-Nemotron-Super-49B-v1", # noqa: E501
212 - trust_remote_code=True),
225 + "DeciLMForCausalLM": _HfExamplesInfo(
226 + "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
227 + trust_remote_code=True,
228 + ),
229 "DeepseekForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-llm-7b-chat"),
214 - "DeepseekV2ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V2-Lite-Chat", # noqa: E501
215 - trust_remote_code=True),
216 - "DeepseekV3ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3", # noqa: E501
217 - trust_remote_code=True),
230 + "DeepseekV2ForCausalLM": _HfExamplesInfo(
231 + "deepseek-ai/DeepSeek-V2-Lite-Chat",
232 + trust_remote_code=True,
233 + ),
234 + "DeepseekV3ForCausalLM": _HfExamplesInfo(
235 + "deepseek-ai/DeepSeek-V3",
236 + trust_remote_code=True,
237 + ),
238 "DeepseekV32ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3.2-Exp"),
219 - "Ernie4_5ForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-0.3B-PT",
220 - min_transformers_version="4.54"),
221 - "Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT",
222 - min_transformers_version="4.54"),
223 - "ExaoneForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct",
224 - trust_remote_code=True),
225 - "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B",
226 - min_transformers_version="4.54"),
227 - "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"), # noqa: E501
239 + "Ernie4_5ForCausalLM": _HfExamplesInfo(
240 + "baidu/ERNIE-4.5-0.3B-PT", min_transformers_version="4.54"
241 + ),
242 + "Ernie4_5_MoeForCausalLM": _HfExamplesInfo(
243 + "baidu/ERNIE-4.5-21B-A3B-PT", min_transformers_version="4.54"
244 + ),
245 + "ExaoneForCausalLM": _HfExamplesInfo(
246 + "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
247 + ),
248 + "Exaone4ForCausalLM": _HfExamplesInfo(
249 + "LGAI-EXAONE/EXAONE-4.0-32B", min_transformers_version="4.54"
250 + ),
251 + "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"),
252 "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
229 - "FalconH1ForCausalLM":_HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
253 + "FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
254 "GemmaForCausalLM": _HfExamplesInfo("google/gemma-1.1-2b-it"),
255 "Gemma2ForCausalLM": _HfExamplesInfo("google/gemma-2-9b"),
256 "Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
233 - "Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it",
234 - min_transformers_version="4.53"),
257 + "Gemma3nForCausalLM": _HfExamplesInfo(
258 + "google/gemma-3n-E2B-it", min_transformers_version="4.53"
259 + ),
260 "GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
261 "Glm4ForCausalLM": _HfExamplesInfo("zai-org/GLM-4-9B-0414"),
237 - "Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5",
238 - min_transformers_version="4.54"), # noqa: E501
239 - "GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2",
240 - {"alias": "gpt2"}),
241 - "GPTBigCodeForCausalLM": _HfExamplesInfo("bigcode/starcoder",
242 - extras={"tiny": "bigcode/tiny_starcoder_py"}, # noqa: E501
243 - min_transformers_version="4.55.1",
244 - transformers_version_reason="HF model broken in 4.55.0"), # noqa: E501
245 - "GPTJForCausalLM": _HfExamplesInfo("Milos/slovak-gpt-j-405M",
246 - {"6b": "EleutherAI/gpt-j-6b"}),
247 - "GPTNeoXForCausalLM": _HfExamplesInfo("EleutherAI/pythia-70m",
248 - {"1b": "EleutherAI/pythia-1.4b"}),
262 + "Glm4MoeForCausalLM": _HfExamplesInfo(
263 + "zai-org/GLM-4.5", min_transformers_version="4.54"
264 + ),
265 + "GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2", {"alias": "gpt2"}),
266 + "GPTBigCodeForCausalLM": _HfExamplesInfo(
267 + "bigcode/starcoder",
268 + extras={"tiny": "bigcode/tiny_starcoder_py"},
269 + min_transformers_version="4.55.1",
270 + transformers_version_reason="HF model broken in 4.55.0",
271 + ),
272 + "GPTJForCausalLM": _HfExamplesInfo(
273 + "Milos/slovak-gpt-j-405M", {"6b": "EleutherAI/gpt-j-6b"}
274 + ),
275 + "GPTNeoXForCausalLM": _HfExamplesInfo(
276 + "EleutherAI/pythia-70m", {"1b": "EleutherAI/pythia-1.4b"}
277 + ),
278 "GptOssForCausalLM": _HfExamplesInfo("lmsys/gpt-oss-20b-bf16"),
279 "GraniteForCausalLM": _HfExamplesInfo("ibm/PowerLM-3b"),
280 "GraniteMoeForCausalLM": _HfExamplesInfo("ibm/PowerMoE-3b"),
252 - "GraniteMoeHybridForCausalLM": _HfExamplesInfo("ibm-granite/granite-4.0-tiny-preview", # noqa: E501
253 - min_transformers_version="4.55.3"),
254 - "GraniteMoeSharedForCausalLM": _HfExamplesInfo("ibm-research/moe-7b-1b-active-shared-experts"), # noqa: E501
255 - "Grok1ModelForCausalLM": _HfExamplesInfo("hpcai-tech/grok-1",
256 - trust_remote_code=True),
257 - "HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct",
258 - trust_remote_code=True),
281 + "GraniteMoeHybridForCausalLM": _HfExamplesInfo(
282 + "ibm-granite/granite-4.0-tiny-preview",
283 + min_transformers_version="4.55.3",
284 + ),
285 + "GraniteMoeSharedForCausalLM": _HfExamplesInfo(
286 + "ibm-research/moe-7b-1b-active-shared-experts"
287 + ),
288 + "Grok1ModelForCausalLM": _HfExamplesInfo(
289 + "hpcai-tech/grok-1", trust_remote_code=True
290 + ),
291 + "HunYuanMoEV1ForCausalLM": _HfExamplesInfo(
292 + "tencent/Hunyuan-A13B-Instruct", trust_remote_code=True
293 + ),
294 # TODO: Remove is_available_online once their config.json is fixed
260 - "HunYuanDenseV1ForCausalLM":_HfExamplesInfo("tencent/Hunyuan-7B-Instruct-0124",
261 - trust_remote_code=True,
262 - is_available_online=False),
263 - "InternLMForCausalLM": _HfExamplesInfo("internlm/internlm-chat-7b",
264 - trust_remote_code=True),
265 - "InternLM2ForCausalLM": _HfExamplesInfo("internlm/internlm2-chat-7b",
266 - trust_remote_code=True),
267 - "InternLM2VEForCausalLM": _HfExamplesInfo("OpenGVLab/Mono-InternVL-2B",
268 - trust_remote_code=True),
269 - "InternLM3ForCausalLM": _HfExamplesInfo("internlm/internlm3-8b-instruct",
270 - trust_remote_code=True),
295 + "HunYuanDenseV1ForCausalLM": _HfExamplesInfo(
296 + "tencent/Hunyuan-7B-Instruct-0124",
297 + trust_remote_code=True,
298 + is_available_online=False,
299 + ),
300 + "InternLMForCausalLM": _HfExamplesInfo(
301 + "internlm/internlm-chat-7b", trust_remote_code=True
302 + ),
303 + "InternLM2ForCausalLM": _HfExamplesInfo(
304 + "internlm/internlm2-chat-7b", trust_remote_code=True
305 + ),
306 + "InternLM2VEForCausalLM": _HfExamplesInfo(
307 + "OpenGVLab/Mono-InternVL-2B", trust_remote_code=True
308 + ),
309 + "InternLM3ForCausalLM": _HfExamplesInfo(
310 + "internlm/internlm3-8b-instruct", trust_remote_code=True
311 + ),
312 "JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
272 - "JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini",
273 - min_transformers_version="4.55.3",
274 - extras={
275 - "tiny": "ai21labs/Jamba-tiny-dev",
276 - "random": "ai21labs/Jamba-tiny-random", # noqa: E501
277 - }),
278 - "Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B",
279 - min_transformers_version="4.54"),
280 - "LlamaForCausalLM": _HfExamplesInfo("meta-llama/Llama-3.2-1B-Instruct",
281 - extras={"guard": "meta-llama/Llama-Guard-3-1B", # noqa: E501
282 - "hermes": "NousResearch/Hermes-3-Llama-3.1-8B", # noqa: E501
283 - "fp8": "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8"}), # noqa: E501
284 - "LLaMAForCausalLM": _HfExamplesInfo("decapoda-research/llama-7b-hf",
285 - is_available_online=False),
286 - "Llama4ForCausalLM": _HfExamplesInfo("meta-llama/Llama-4-Scout-17B-16E-Instruct", # noqa: E501
287 - is_available_online=False),
288 - "LongcatFlashForCausalLM": _HfExamplesInfo
289 - ("meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True),
313 + "JambaForCausalLM": _HfExamplesInfo(
314 + "ai21labs/AI21-Jamba-1.5-Mini",
315 + min_transformers_version="4.55.3",
316 + extras={
317 + "tiny": "ai21labs/Jamba-tiny-dev",
318 + "random": "ai21labs/Jamba-tiny-random",
319 + },
320 + ),
321 + "Lfm2ForCausalLM": _HfExamplesInfo(
322 + "LiquidAI/LFM2-1.2B", min_transformers_version="4.54"
323 + ),
324 + "LlamaForCausalLM": _HfExamplesInfo(
325 + "meta-llama/Llama-3.2-1B-Instruct",
326 + extras={
327 + "guard": "meta-llama/Llama-Guard-3-1B",
328 + "hermes": "NousResearch/Hermes-3-Llama-3.1-8B",
329 + "fp8": "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
330 + },
331 + ),
332 + "LLaMAForCausalLM": _HfExamplesInfo(
333 + "decapoda-research/llama-7b-hf", is_available_online=False
334 + ),
335 + "Llama4ForCausalLM": _HfExamplesInfo(
336 + "meta-llama/Llama-4-Scout-17B-16E-Instruct",
337 + is_available_online=False,
338 + ),
339 + "LongcatFlashForCausalLM": _HfExamplesInfo(
340 + "meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
341 + ),
342 "MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
291 - "Mamba2ForCausalLM": _HfExamplesInfo("mistralai/Mamba-Codestral-7B-v0.1",
292 - min_transformers_version="4.55.3",
293 - extras={
294 - "random": "yujiepan/mamba2-codestral-v0.1-tiny-random", # noqa: E501
295 - }),
296 - "FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"), # noqa: E501
297 - "MiniCPMForCausalLM": _HfExamplesInfo("openbmb/MiniCPM-2B-sft-bf16",
298 - trust_remote_code=True),
299 - "MiniCPM3ForCausalLM": _HfExamplesInfo("openbmb/MiniCPM3-4B",
300 - trust_remote_code=True),
343 + "Mamba2ForCausalLM": _HfExamplesInfo(
344 + "mistralai/Mamba-Codestral-7B-v0.1",
345 + min_transformers_version="4.55.3",
346 + extras={
347 + "random": "yujiepan/mamba2-codestral-v0.1-tiny-random",
348 + },
349 + ),
350 + "FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"),
351 + "MiniCPMForCausalLM": _HfExamplesInfo(
352 + "openbmb/MiniCPM-2B-sft-bf16", trust_remote_code=True
353 + ),
354 + "MiniCPM3ForCausalLM": _HfExamplesInfo(
355 + "openbmb/MiniCPM3-4B", trust_remote_code=True
356 + ),
357 "MiniMaxForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01-hf"),
302 - "MiniMaxText01ForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01",
303 - trust_remote_code=True,
304 - revision="a59aa9cbc53b9fb8742ca4e9e1531b9802b6fdc3"), # noqa: E501
305 - "MiniMaxM1ForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-M1-40k",
306 - trust_remote_code=True),
358 + "MiniMaxText01ForCausalLM": _HfExamplesInfo(
359 + "MiniMaxAI/MiniMax-Text-01",
360 + trust_remote_code=True,
361 + revision="a59aa9cbc53b9fb8742ca4e9e1531b9802b6fdc3",
362 + ),
363 + "MiniMaxM1ForCausalLM": _HfExamplesInfo(
364 + "MiniMaxAI/MiniMax-M1-40k", trust_remote_code=True
365 + ),
366 "MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
308 - "MixtralForCausalLM": _HfExamplesInfo("mistralai/Mixtral-8x7B-Instruct-v0.1", # noqa: E501
309 - {"tiny": "TitanML/tiny-mixtral"}), # noqa: E501
367 + "MixtralForCausalLM": _HfExamplesInfo(
368 + "mistralai/Mixtral-8x7B-Instruct-v0.1",
369 + {"tiny": "TitanML/tiny-mixtral"},
370 + ),
371 "MptForCausalLM": _HfExamplesInfo("mpt", is_available_online=False),
372 "MPTForCausalLM": _HfExamplesInfo("mosaicml/mpt-7b"),
373 "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"),
313 - "NemotronHForCausalLM": _HfExamplesInfo("nvidia/Nemotron-H-8B-Base-8K",
314 - trust_remote_code=True),
374 + "NemotronHForCausalLM": _HfExamplesInfo(
375 + "nvidia/Nemotron-H-8B-Base-8K", trust_remote_code=True
376 + ),
377 "OlmoForCausalLM": _HfExamplesInfo("allenai/OLMo-1B-hf"),
378 "Olmo2ForCausalLM": _HfExamplesInfo("allenai/OLMo-2-0425-1B"),
379 "Olmo3ForCausalLM": _HfExamplesInfo("shanearora/2025-sep-a-base-model"),
380 "OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"),
319 - "OPTForCausalLM": _HfExamplesInfo("facebook/opt-125m",
320 - {"1b": "facebook/opt-iml-max-1.3b"}),
321 - "OrionForCausalLM": _HfExamplesInfo("OrionStarAI/Orion-14B-Chat",
322 - trust_remote_code=True),
381 + "OPTForCausalLM": _HfExamplesInfo(
382 + "facebook/opt-125m", {"1b": "facebook/opt-iml-max-1.3b"}
383 + ),
384 + "OrionForCausalLM": _HfExamplesInfo(
385 + "OrionStarAI/Orion-14B-Chat", trust_remote_code=True
386 + ),
387 "PersimmonForCausalLM": _HfExamplesInfo("adept/persimmon-8b-chat"),
388 "PhiForCausalLM": _HfExamplesInfo("microsoft/phi-2"),
389 "Phi3ForCausalLM": _HfExamplesInfo("microsoft/Phi-3-mini-4k-instruct"),
326 - "PhiMoEForCausalLM": _HfExamplesInfo("microsoft/Phi-3.5-MoE-instruct",
327 - trust_remote_code=True),
328 - "Plamo2ForCausalLM": _HfExamplesInfo("pfnet/plamo-2-1b",
329 - max_transformers_version="4.55.4",
330 - transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers", # noqa: E501
331 - trust_remote_code=True),
332 - "QWenLMHeadModel": _HfExamplesInfo("Qwen/Qwen-7B-Chat",
333 - max_transformers_version="4.53",
334 - transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers", # noqa: E501
335 - trust_remote_code=True),
336 - "Qwen2ForCausalLM": _HfExamplesInfo("Qwen/Qwen2-0.5B-Instruct",
337 - extras={"2.5": "Qwen/Qwen2.5-0.5B-Instruct"}), # noqa: E501
390 + "PhiMoEForCausalLM": _HfExamplesInfo(
391 + "microsoft/Phi-3.5-MoE-instruct", trust_remote_code=True
392 + ),
393 + "Plamo2ForCausalLM": _HfExamplesInfo(
394 + "pfnet/plamo-2-1b",
395 + max_transformers_version="4.55.4",
396 + transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers", # noqa: E501
397 + trust_remote_code=True,
398 + ),
399 + "QWenLMHeadModel": _HfExamplesInfo(
400 + "Qwen/Qwen-7B-Chat",
401 + max_transformers_version="4.53",
402 + transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers", # noqa: E501
403 + trust_remote_code=True,
404 + ),
405 + "Qwen2ForCausalLM": _HfExamplesInfo(
406 + "Qwen/Qwen2-0.5B-Instruct", extras={"2.5": "Qwen/Qwen2.5-0.5B-Instruct"}
407 + ),
408 "Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
409 "Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"),
410 "Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
341 - "Qwen3NextForCausalLM": _HfExamplesInfo("Qwen/Qwen3-Next-80B-A3B-Instruct",
342 - extras={"tiny-random": "tiny-random/qwen3-next-moe"}, # noqa: E501
343 - min_transformers_version="4.56.3"),
411 + "Qwen3NextForCausalLM": _HfExamplesInfo(
412 + "Qwen/Qwen3-Next-80B-A3B-Instruct",
413 + extras={"tiny-random": "tiny-random/qwen3-next-moe"},
414 + min_transformers_version="4.56.3",
415 + ),
416 "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b"),
345 - "SeedOssForCausalLM": _HfExamplesInfo("ByteDance-Seed/Seed-OSS-36B-Instruct", # noqa: E501
346 - trust_remote_code=True,
347 - is_available_online=False),
417 + "SeedOssForCausalLM": _HfExamplesInfo(
418 + "ByteDance-Seed/Seed-OSS-36B-Instruct",
419 + trust_remote_code=True,
420 + is_available_online=False,
421 + ),
422 "SmolLM3ForCausalLM": _HfExamplesInfo("HuggingFaceTB/SmolLM3-3B"),
349 - "StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b"), # noqa: E501
423 + "StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b"),
424 "StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t"),
425 "Starcoder2ForCausalLM": _HfExamplesInfo("bigcode/starcoder2-3b"),
352 - "Step3TextForCausalLM": _HfExamplesInfo("stepfun-ai/step3",
353 - trust_remote_code=True),
354 - "SolarForCausalLM": _HfExamplesInfo("upstage/solar-pro-preview-instruct",
355 - trust_remote_code=True),
356 - "TeleChat2ForCausalLM": _HfExamplesInfo("Tele-AI/TeleChat2-3B",
357 - trust_remote_code=True),
358 - "TeleFLMForCausalLM": _HfExamplesInfo("CofeAI/FLM-2-52B-Instruct-2407",
359 - trust_remote_code=True),
360 - "XverseForCausalLM": _HfExamplesInfo("xverse/XVERSE-7B-Chat",
361 - tokenizer="meta-llama/Llama-2-7b",
362 - trust_remote_code=True),
426 + "Step3TextForCausalLM": _HfExamplesInfo("stepfun-ai/step3", trust_remote_code=True),
427 + "SolarForCausalLM": _HfExamplesInfo(
428 + "upstage/solar-pro-preview-instruct", trust_remote_code=True
429 + ),
430 + "TeleChat2ForCausalLM": _HfExamplesInfo(
431 + "Tele-AI/TeleChat2-3B", trust_remote_code=True
432 + ),
433 + "TeleFLMForCausalLM": _HfExamplesInfo(
434 + "CofeAI/FLM-2-52B-Instruct-2407", trust_remote_code=True
435 + ),
436 + "XverseForCausalLM": _HfExamplesInfo(
437 + "xverse/XVERSE-7B-Chat",
438 + tokenizer="meta-llama/Llama-2-7b",
439 + trust_remote_code=True,
440 + ),
441 "Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
364 - "MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL",
365 - trust_remote_code=True),
442 + "MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL", trust_remote_code=True),
443 "Dots1ForCausalLM": _HfExamplesInfo("rednote-hilab/dots.llm1.inst"),
444 }
445
446 _EMBEDDING_EXAMPLE_MODELS = {
447 # [Text-only]
448 "BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
372 - "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"), # noqa: E501
449 + "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
450 "Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"),
451 "GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"),
375 - "GteModel": _HfExamplesInfo("Snowflake/snowflake-arctic-embed-m-v2.0",
376 - trust_remote_code=True),
377 - "GteNewModel": _HfExamplesInfo("Alibaba-NLP/gte-base-en-v1.5",
378 - trust_remote_code=True,
379 - hf_overrides={"architectures": ["GteNewModel"]}), # noqa: E501
380 - "InternLM2ForRewardModel": _HfExamplesInfo("internlm/internlm2-1_8b-reward",
381 - trust_remote_code=True),
382 - "JambaForSequenceClassification": _HfExamplesInfo("ai21labs/Jamba-tiny-reward-dev"), # noqa: E501
452 + "GteModel": _HfExamplesInfo(
453 + "Snowflake/snowflake-arctic-embed-m-v2.0", trust_remote_code=True
454 + ),
455 + "GteNewModel": _HfExamplesInfo(
456 + "Alibaba-NLP/gte-base-en-v1.5",
457 + trust_remote_code=True,
458 + hf_overrides={"architectures": ["GteNewModel"]},
459 + ),
460 + "InternLM2ForRewardModel": _HfExamplesInfo(
461 + "internlm/internlm2-1_8b-reward", trust_remote_code=True
462 + ),
463 + "JambaForSequenceClassification": _HfExamplesInfo("ai21labs/Jamba-tiny-reward-dev"),
464 "LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
465 "MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
385 - "ModernBertModel": _HfExamplesInfo("Alibaba-NLP/gte-modernbert-base",
386 - trust_remote_code=True),
387 - "NomicBertModel": _HfExamplesInfo("nomic-ai/nomic-embed-text-v2-moe",
388 - trust_remote_code=True), # noqa: E501
466 + "ModernBertModel": _HfExamplesInfo(
467 + "Alibaba-NLP/gte-modernbert-base", trust_remote_code=True
468 + ),
469 + "NomicBertModel": _HfExamplesInfo(
470 + "nomic-ai/nomic-embed-text-v2-moe", trust_remote_code=True
471 + ),
472 "Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
390 - "Qwen2ForRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-RM-72B",
391 - max_transformers_version="4.53",
392 - transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers"), # noqa: E501
393 - "Qwen2ForProcessRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-PRM-7B",
394 - max_transformers_version="4.53",
395 - transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers"), # noqa: E501
396 - "RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"), # noqa: E501
397 - "RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"), # noqa: E501
398 - "XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-small"), # noqa: E501
473 + "Qwen2ForRewardModel": _HfExamplesInfo(
474 + "Qwen/Qwen2.5-Math-RM-72B",
475 + max_transformers_version="4.53",
476 + transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers", # noqa: E501
477 + ),
478 + "Qwen2ForProcessRewardModel": _HfExamplesInfo(
479 + "Qwen/Qwen2.5-Math-PRM-7B",
480 + max_transformers_version="4.53",
481 + transformers_version_reason="HF model uses remote code that is not compatible with latest Transformers", # noqa: E501
482 + ),
483 + "RobertaModel": _HfExamplesInfo("sentence-transformers/stsb-roberta-base-v2"),
484 + "RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"),
485 + "XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-small"),
486 # [Multimodal]
487 "CLIPModel": _HfExamplesInfo("openai/clip-vit-base-patch32"),
488 "LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"),
402 - "Phi3VForCausalLM": _HfExamplesInfo("TIGER-Lab/VLM2Vec-Full",
403 - trust_remote_code=True),
404 - "Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"), # noqa: E501
405 - "PrithviGeoSpatialMAE": _HfExamplesInfo("ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11", # noqa: E501
406 - dtype=torch.float16,
407 - enforce_eager=True,
408 - skip_tokenizer_init=True,
409 - # This is to avoid the model
410 - # going OOM in CI
411 - max_num_seqs=32,
412 - ),
413 - "Terratorch": _HfExamplesInfo("ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11", # noqa: E501
414 - dtype=torch.float16,
415 - enforce_eager=True,
416 - skip_tokenizer_init=True,
417 - # This is to avoid the model going OOM in CI
418 - max_num_seqs=32,
419 - ),
489 + "Phi3VForCausalLM": _HfExamplesInfo(
490 + "TIGER-Lab/VLM2Vec-Full", trust_remote_code=True
491 + ),
492 + "Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"),
493 + "PrithviGeoSpatialMAE": _HfExamplesInfo(
494 + "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
495 + dtype=torch.float16,
496 + enforce_eager=True,
497 + skip_tokenizer_init=True,
498 + # This is to avoid the model
499 + # going OOM in CI
500 + max_num_seqs=32,
501 + ),
502 + "Terratorch": _HfExamplesInfo(
503 + "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
504 + dtype=torch.float16,
505 + enforce_eager=True,
506 + skip_tokenizer_init=True,
507 + # This is to avoid the model going OOM in CI
508 + max_num_seqs=32,
509 + ),
510 }
511
512 _SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
513 # [Decoder-only]
424 - "GPT2ForSequenceClassification": _HfExamplesInfo("nie3e/sentiment-polish-gpt2-small"), # noqa: E501
425 -
514 + "GPT2ForSequenceClassification": _HfExamplesInfo(
515 + "nie3e/sentiment-polish-gpt2-small"
516 + ),
517 # [Cross-encoder]
427 - "BertForSequenceClassification": _HfExamplesInfo("cross-encoder/ms-marco-MiniLM-L-6-v2"), # noqa: E501
518 + "BertForSequenceClassification": _HfExamplesInfo(
519 + "cross-encoder/ms-marco-MiniLM-L-6-v2"
520 + ),
521 "BertForTokenClassification": _HfExamplesInfo("boltuix/NeuroBERT-NER"),
429 - "GteNewForSequenceClassification": _HfExamplesInfo("Alibaba-NLP/gte-multilingual-reranker-base", # noqa: E501
430 - trust_remote_code=True,
431 - hf_overrides={
432 - "architectures": ["GteNewForSequenceClassification"]}),# noqa: E501
433 - "ModernBertForSequenceClassification": _HfExamplesInfo("Alibaba-NLP/gte-reranker-modernbert-base"), # noqa: E501
434 - "RobertaForSequenceClassification": _HfExamplesInfo("cross-encoder/quora-roberta-base"), # noqa: E501
435 - "XLMRobertaForSequenceClassification": _HfExamplesInfo("BAAI/bge-reranker-v2-m3"), # noqa: E501
522 + "GteNewForSequenceClassification": _HfExamplesInfo(
523 + "Alibaba-NLP/gte-multilingual-reranker-base",
524 + trust_remote_code=True,
525 + hf_overrides={"architectures": ["GteNewForSequenceClassification"]},
526 + ),
527 + "ModernBertForSequenceClassification": _HfExamplesInfo(
528 + "Alibaba-NLP/gte-reranker-modernbert-base"
529 + ),
530 + "RobertaForSequenceClassification": _HfExamplesInfo(
531 + "cross-encoder/quora-roberta-base"
532 + ),
533 + "XLMRobertaForSequenceClassification": _HfExamplesInfo("BAAI/bge-reranker-v2-m3"),
534 }
535
536 _AUTOMATIC_CONVERTED_MODELS = {
537 # Use as_seq_cls_model for automatic conversion
440 - "GemmaForSequenceClassification": _HfExamplesInfo("BAAI/bge-reranker-v2-gemma", # noqa: E501
441 - hf_overrides={"architectures": ["GemmaForSequenceClassification"], # noqa: E501
442 - "classifier_from_token": ["Yes"], # noqa: E501
443 - "method": "no_post_processing"}), # noqa: E501
444 - "LlamaForSequenceClassification": _HfExamplesInfo("Skywork/Skywork-Reward-V2-Llama-3.2-1B"), # noqa: E501
445 - "Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"), # noqa: E501
446 - "Qwen3ForSequenceClassification": _HfExamplesInfo("tomaarsen/Qwen3-Reranker-0.6B-seq-cls"), # noqa: E501
538 + "GemmaForSequenceClassification": _HfExamplesInfo(
539 + "BAAI/bge-reranker-v2-gemma",
540 + hf_overrides={
541 + "architectures": ["GemmaForSequenceClassification"],
542 + "classifier_from_token": ["Yes"],
543 + "method": "no_post_processing",
544 + },
545 + ),
546 + "LlamaForSequenceClassification": _HfExamplesInfo(
547 + "Skywork/Skywork-Reward-V2-Llama-3.2-1B"
548 + ),
549 + "Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"),
550 + "Qwen3ForSequenceClassification": _HfExamplesInfo(
551 + "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
552 + ),
553 }
554
555 _MULTIMODAL_EXAMPLE_MODELS = {
556 # [Decoder-only]
557 "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
452 - "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereForAI/aya-vision-8b"), # noqa: E501
453 - "Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b", # noqa: E501
454 - extras={"6b": "Salesforce/blip2-opt-6.7b"}), # noqa: E501
455 - "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"), # noqa: E501
456 - "Cohere2VisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/command-a-vision-07-2025"), # noqa: E501
457 - "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny", # noqa: E501
458 - extras={"fork": "Isotr0py/deepseek-vl2-tiny"}, # noqa: E501
459 - max_transformers_version="4.48", # noqa: E501
460 - transformers_version_reason="HF model is not compatible.", # noqa: E501
461 - hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]}), # noqa: E501
462 - "DotsOCRForCausalLM": _HfExamplesInfo("rednote-hilab/dots.ocr",
463 - trust_remote_code=True),
558 + "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereForAI/aya-vision-8b"),
559 + "Blip2ForConditionalGeneration": _HfExamplesInfo(
560 + "Salesforce/blip2-opt-2.7b",
561 + extras={"6b": "Salesforce/blip2-opt-6.7b"},
562 + ),
563 + "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"),
564 + "Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
565 + "CohereLabs/command-a-vision-07-2025"
566 + ),
567 + "DeepseekVLV2ForCausalLM": _HfExamplesInfo(
568 + "deepseek-ai/deepseek-vl2-tiny",
569 + extras={"fork": "Isotr0py/deepseek-vl2-tiny"},
570 + max_transformers_version="4.48",
571 + transformers_version_reason="HF model is not compatible.",
572 + hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
573 + ),
574 + "DotsOCRForCausalLM": _HfExamplesInfo(
575 + "rednote-hilab/dots.ocr", trust_remote_code=True
576 + ),
577 "Emu3ForConditionalGeneration": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
465 - "Ernie4_5_VLMoeForConditionalGeneration": _HfExamplesInfo("baidu/ERNIE-4.5-VL-28B-A3B-PT", # noqa: E501
466 - trust_remote_code=True),
578 + "Ernie4_5_VLMoeForConditionalGeneration": _HfExamplesInfo(
579 + "baidu/ERNIE-4.5-VL-28B-A3B-PT",
580 + trust_remote_code=True,
581 + ),
582 "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
583 "Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
469 - "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it", # noqa: E501
470 - min_transformers_version="4.53"),
471 - "GraniteSpeechForConditionalGeneration": _HfExamplesInfo("ibm-granite/granite-speech-3.3-2b"), # noqa: E501
472 - "GLM4VForCausalLM": _HfExamplesInfo("zai-org/glm-4v-9b",
473 - trust_remote_code=True,
474 - hf_overrides={"architectures": ["GLM4VForCausalLM"]}), # noqa: E501
475 - "Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"), # noqa: E501
476 - "Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V",
477 - min_transformers_version="4.56"), # noqa: E501
478 - "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m",
479 - trust_remote_code=True,
480 - extras={"2b": "h2oai/h2ovl-mississippi-2b"}, # noqa: E501
481 - max_transformers_version="4.48", # noqa: E501
482 - transformers_version_reason="HF model is not compatible."), # noqa: E501
483 - "HCXVisionForCausalLM": _HfExamplesInfo("naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B", # noqa: E501
484 - trust_remote_code=True),
485 - "Idefics3ForConditionalGeneration": _HfExamplesInfo("HuggingFaceM4/Idefics3-8B-Llama3", # noqa: E501
486 - {"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"}, # noqa: E501
487 - min_transformers_version="4.56",
488 - transformers_version_reason="HF model broken in 4.55"), # noqa: E501
489 - "InternS1ForConditionalGeneration": _HfExamplesInfo("internlm/Intern-S1",
490 - trust_remote_code=True), # noqa: E501
491 - "InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
492 - extras={"2B": "OpenGVLab/InternVL2-2B",
493 - "3.0": "OpenGVLab/InternVL3-1B", # noqa: E501
494 - "3.5-qwen3": "OpenGVLab/InternVL3_5-1B", # noqa: E501
495 - "3.5-qwen3moe": "OpenGVLab/InternVL3_5-30B-A3B", # noqa: E501
496 - "3.5-gptoss": "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview"}, # noqa: E501
497 - trust_remote_code=True),
498 - "InternVLForConditionalGeneration": _HfExamplesInfo("OpenGVLab/InternVL3-1B-hf"), # noqa: E501
499 - "KeyeForConditionalGeneration": _HfExamplesInfo("Kwai-Keye/Keye-VL-8B-Preview", # noqa: E501
500 - trust_remote_code=True),
501 - "KeyeVL1_5ForConditionalGeneration": _HfExamplesInfo("Kwai-Keye/Keye-VL-1_5-8B", # noqa: E501
502 - trust_remote_code=True),
503 - "KimiVLForConditionalGeneration": _HfExamplesInfo("moonshotai/Kimi-VL-A3B-Instruct", # noqa: E501
504 - extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"}, # noqa: E501
505 - trust_remote_code=True),
506 - "Llama4ForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-4-Scout-17B-16E-Instruct", # noqa: E501
507 - max_model_len=10240,
508 - extras={"llama-guard-4": "meta-llama/Llama-Guard-4-12B"}, # noqa: E501
509 - ),
510 - "LlavaForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-1.5-7b-hf",
511 - extras={"mistral": "mistral-community/pixtral-12b", # noqa: E501
512 - "mistral-fp8": "nm-testing/pixtral-12b-FP8-dynamic"}), # noqa: E501
513 - "LlavaNextForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-v1.6-mistral-7b-hf"), # noqa: E501
514 - "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo("llava-hf/LLaVA-NeXT-Video-7B-hf"), # noqa: E501
515 - "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
516 - "MantisForConditionalGeneration": _HfExamplesInfo("TIGER-Lab/Mantis-8B-siglip-llama3", # noqa: E501
517 - max_transformers_version="4.48", # noqa: E501
518 - transformers_version_reason="HF model is not compatible.", # noqa: E501
519 - hf_overrides={"architectures": ["MantisForConditionalGeneration"]}), # noqa: E501
520 - "MiDashengLMModel": _HfExamplesInfo("mispeech/midashenglm-7b",
521 - trust_remote_code=True),
522 - "MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6",
523 - trust_remote_code=True),
524 - "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
525 - extras={"2.6": "openbmb/MiniCPM-V-2_6", "4.0": "openbmb/MiniCPM-V-4", "4.5": "openbmb/MiniCPM-V-4_5"}, # noqa: E501
526 - trust_remote_code=True),
527 - "MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo("MiniMaxAI/MiniMax-VL-01", # noqa: E501
528 - trust_remote_code=True,
529 - v0_only=True),
530 - "Mistral3ForConditionalGeneration": _HfExamplesInfo("mistralai/Mistral-Small-3.1-24B-Instruct-2503", # noqa: E501
531 - extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"}), # noqa: E501
532 - "MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
533 - max_transformers_version="4.48",
534 - transformers_version_reason="Incorrectly-detected `tensorflow` import.", # noqa: E501
535 - extras={"olmo": "allenai/Molmo-7B-O-0924"}, # noqa: E501
536 - trust_remote_code=True),
537 - "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B",
538 - trust_remote_code=True),
539 - "Llama_Nemotron_Nano_VL" : _HfExamplesInfo("nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1", # noqa: E501
540 - trust_remote_code=True),
541 - "NemotronH_Nano_VL_V2": _HfExamplesInfo("nano_vl_dummy",
542 - is_available_online=False,
543 - trust_remote_code=True),
544 - "Ovis": _HfExamplesInfo("AIDC-AI/Ovis2-1B", trust_remote_code=True,
545 - max_transformers_version="4.53",
546 - transformers_version_reason="HF model is not compatible", # noqa: E501
547 - extras={"1.6-llama": "AIDC-AI/Ovis1.6-Llama3.2-3B",
548 - "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B"}), # noqa: E501
549 - "Ovis2_5": _HfExamplesInfo("AIDC-AI/Ovis2.5-2B",
550 - trust_remote_code=True),
551 - "PaliGemmaForConditionalGeneration": _HfExamplesInfo("google/paligemma-3b-mix-224", # noqa: E501
552 - extras={"v2": "google/paligemma2-3b-ft-docci-448"}), # noqa: E501
553 - "Phi3VForCausalLM": _HfExamplesInfo("microsoft/Phi-3-vision-128k-instruct",
554 - trust_remote_code=True,
555 - max_transformers_version="4.48",
556 - transformers_version_reason="Use of deprecated imports which have been removed.", # noqa: E501
557 - extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"}), # noqa: E501
558 - "Phi4MMForCausalLM": _HfExamplesInfo("microsoft/Phi-4-multimodal-instruct",
559 - trust_remote_code=True),
560 - "Phi4MultimodalForCausalLM": _HfExamplesInfo("microsoft/Phi-4-multimodal-instruct", # noqa: E501
561 - revision="refs/pr/70"),
562 - "PixtralForConditionalGeneration": _HfExamplesInfo("mistralai/Pixtral-12B-2409", # noqa: E501
563 - tokenizer_mode="mistral"),
564 - "QwenVLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen-VL",
565 - extras={"chat": "Qwen/Qwen-VL-Chat"}, # noqa: E501
566 - trust_remote_code=True,
567 - hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]}), # noqa: E501
568 - "Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
569 - "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
570 - "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501
571 - max_model_len=4096),
584 + "Gemma3nForConditionalGeneration": _HfExamplesInfo(
585 + "google/gemma-3n-E2B-it",
586 + min_transformers_version="4.53",
587 + ),
588 + "GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
589 + "ibm-granite/granite-speech-3.3-2b"
590 + ),
591 + "GLM4VForCausalLM": _HfExamplesInfo(
592 + "zai-org/glm-4v-9b",
593 + trust_remote_code=True,
594 + hf_overrides={"architectures": ["GLM4VForCausalLM"]},
595 + ),
596 + "Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"),
597 + "Glm4vMoeForConditionalGeneration": _HfExamplesInfo(
598 + "zai-org/GLM-4.5V", min_transformers_version="4.56"
599 + ),
600 + "H2OVLChatModel": _HfExamplesInfo(
601 + "h2oai/h2ovl-mississippi-800m",
602 + trust_remote_code=True,
603 + extras={"2b": "h2oai/h2ovl-mississippi-2b"},
604 + max_transformers_version="4.48",
605 + transformers_version_reason="HF model is not compatible.",
606 + ),
607 + "HCXVisionForCausalLM": _HfExamplesInfo(
608 + "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
609 + trust_remote_code=True,
610 + ),
611 + "Idefics3ForConditionalGeneration": _HfExamplesInfo(
612 + "HuggingFaceM4/Idefics3-8B-Llama3",
613 + {"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"},
614 + min_transformers_version="4.56",
615 + transformers_version_reason="HF model broken in 4.55",
616 + ),
617 + "InternS1ForConditionalGeneration": _HfExamplesInfo(
618 + "internlm/Intern-S1", trust_remote_code=True
619 + ),
620 + "InternVLChatModel": _HfExamplesInfo(
621 + "OpenGVLab/InternVL2-1B",
622 + extras={
623 + "2B": "OpenGVLab/InternVL2-2B",
624 + "3.0": "OpenGVLab/InternVL3-1B",
625 + "3.5-qwen3": "OpenGVLab/InternVL3_5-1B",
626 + "3.5-qwen3moe": "OpenGVLab/InternVL3_5-30B-A3B",
627 + "3.5-gptoss": "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview",
628 + },
629 + trust_remote_code=True,
630 + ),
631 + "InternVLForConditionalGeneration": _HfExamplesInfo("OpenGVLab/InternVL3-1B-hf"),
632 + "KeyeForConditionalGeneration": _HfExamplesInfo(
633 + "Kwai-Keye/Keye-VL-8B-Preview",
634 + trust_remote_code=True,
635 + ),
636 + "KeyeVL1_5ForConditionalGeneration": _HfExamplesInfo(
637 + "Kwai-Keye/Keye-VL-1_5-8B",
638 + trust_remote_code=True,
639 + ),
640 + "KimiVLForConditionalGeneration": _HfExamplesInfo(
641 + "moonshotai/Kimi-VL-A3B-Instruct",
642 + extras={"thinking": "moonshotai/Kimi-VL-A3B-Thinking"},
643 + trust_remote_code=True,
644 + ),
645 + "Llama4ForConditionalGeneration": _HfExamplesInfo(
646 + "meta-llama/Llama-4-Scout-17B-16E-Instruct",
647 + max_model_len=10240,
648 + extras={"llama-guard-4": "meta-llama/Llama-Guard-4-12B"},
649 + ),
650 + "LlavaForConditionalGeneration": _HfExamplesInfo(
651 + "llava-hf/llava-1.5-7b-hf",
652 + extras={
653 + "mistral": "mistral-community/pixtral-12b",
654 + "mistral-fp8": "nm-testing/pixtral-12b-FP8-dynamic",
655 + },
656 + ),
657 + "LlavaNextForConditionalGeneration": _HfExamplesInfo(
658 + "llava-hf/llava-v1.6-mistral-7b-hf"
659 + ),
660 + "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo(
661 + "llava-hf/LLaVA-NeXT-Video-7B-hf"
662 + ),
663 + "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo(
664 + "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
665 + ),
666 + "MantisForConditionalGeneration": _HfExamplesInfo(
667 + "TIGER-Lab/Mantis-8B-siglip-llama3",
668 + max_transformers_version="4.48",
669 + transformers_version_reason="HF model is not compatible.",
670 + hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
671 + ),
672 + "MiDashengLMModel": _HfExamplesInfo(
673 + "mispeech/midashenglm-7b", trust_remote_code=True
674 + ),
675 + "MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6", trust_remote_code=True),
676 + "MiniCPMV": _HfExamplesInfo(
677 + "openbmb/MiniCPM-Llama3-V-2_5",
678 + extras={
679 + "2.6": "openbmb/MiniCPM-V-2_6",
680 + "4.0": "openbmb/MiniCPM-V-4",
681 + "4.5": "openbmb/MiniCPM-V-4_5",
682 + },
683 + trust_remote_code=True,
684 + ),
685 + "MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
686 + "MiniMaxAI/MiniMax-VL-01",
687 + trust_remote_code=True,
688 + v0_only=True,
689 + ),
690 + "Mistral3ForConditionalGeneration": _HfExamplesInfo(
691 + "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
692 + extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"},
693 + ),
694 + "MolmoForCausalLM": _HfExamplesInfo(
695 + "allenai/Molmo-7B-D-0924",
696 + max_transformers_version="4.48",
697 + transformers_version_reason="Incorrectly-detected `tensorflow` import.",
698 + extras={"olmo": "allenai/Molmo-7B-O-0924"},
699 + trust_remote_code=True,
700 + ),
701 + "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B", trust_remote_code=True),
702 + "Llama_Nemotron_Nano_VL": _HfExamplesInfo(
703 + "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1",
704 + trust_remote_code=True,
705 + ),
706 + "NemotronH_Nano_VL_V2": _HfExamplesInfo(
707 + "nano_vl_dummy", is_available_online=False, trust_remote_code=True
708 + ),
709 + "Ovis": _HfExamplesInfo(
710 + "AIDC-AI/Ovis2-1B",
711 + trust_remote_code=True,
712 + max_transformers_version="4.53",
713 + transformers_version_reason="HF model is not compatible",
714 + extras={
715 + "1.6-llama": "AIDC-AI/Ovis1.6-Llama3.2-3B",
716 + "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B",
717 + },
718 + ),
719 + "Ovis2_5": _HfExamplesInfo("AIDC-AI/Ovis2.5-2B", trust_remote_code=True),
720 + "PaliGemmaForConditionalGeneration": _HfExamplesInfo(
721 + "google/paligemma-3b-mix-224",
722 + extras={"v2": "google/paligemma2-3b-ft-docci-448"},
723 + ),
724 + "Phi3VForCausalLM": _HfExamplesInfo(
725 + "microsoft/Phi-3-vision-128k-instruct",
726 + trust_remote_code=True,
727 + max_transformers_version="4.48",
728 + transformers_version_reason="Use of deprecated imports which have been removed.", # noqa: E501
729 + extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"},
730 + ),
731 + "Phi4MMForCausalLM": _HfExamplesInfo(
732 + "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True
733 + ),
734 + "Phi4MultimodalForCausalLM": _HfExamplesInfo(
735 + "microsoft/Phi-4-multimodal-instruct",
736 + revision="refs/pr/70",
737 + ),
738 + "PixtralForConditionalGeneration": _HfExamplesInfo(
739 + "mistralai/Pixtral-12B-2409",
740 + tokenizer_mode="mistral",
741 + ),
742 + "QwenVLForConditionalGeneration": _HfExamplesInfo(
743 + "Qwen/Qwen-VL",
744 + extras={"chat": "Qwen/Qwen-VL-Chat"},
745 + trust_remote_code=True,
746 + hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
747 + ),
748 + "Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
749 + "Qwen/Qwen2-Audio-7B-Instruct"
750 + ),
751 + "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"),
752 + "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo(
753 + "Qwen/Qwen2.5-VL-3B-Instruct",
754 + max_model_len=4096,
755 + ),
756 "Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),
573 - "Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"), # noqa: E501
574 - "Qwen3VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen3-VL-4B-Instruct", # noqa: E501
575 - max_model_len=4096,
576 - min_transformers_version="4.57",
577 - is_available_online=False),
578 - "Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen3-VL-30B-A3B-Instruct", # noqa: E501
579 - max_model_len=4096,
580 - min_transformers_version="4.57",
581 - is_available_online=False),
582 - "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B",
583 - trust_remote_code=True),
584 - "SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B",
585 - trust_remote_code=True),
586 - "SmolVLMForConditionalGeneration": _HfExamplesInfo("HuggingFaceTB/SmolVLM2-2.2B-Instruct", # noqa: E501
587 - min_transformers_version="4.56",
588 - transformers_version_reason="HF model broken in 4.55"), # noqa: E501
589 - "Step3VLForConditionalGeneration": _HfExamplesInfo("stepfun-ai/step3",
590 - trust_remote_code=True),
591 - "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b", # noqa: E501
592 - trust_remote_code=True),
593 - "TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"), # noqa: E501
594 - "Tarsier2ForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier2-Recap-7b", # noqa: E501
595 - hf_overrides={"architectures": ["Tarsier2ForConditionalGeneration"]}), # noqa: E501
757 + "Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"),
758 + "Qwen3VLForConditionalGeneration": _HfExamplesInfo(
759 + "Qwen/Qwen3-VL-4B-Instruct",
760 + max_model_len=4096,
761 + min_transformers_version="4.57",
762 + is_available_online=False,
763 + ),
764 + "Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
765 + "Qwen/Qwen3-VL-30B-A3B-Instruct",
766 + max_model_len=4096,
767 + min_transformers_version="4.57",
768 + is_available_online=False,
769 + ),
770 + "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
771 + "SkyworkR1VChatModel": _HfExamplesInfo(
772 + "Skywork/Skywork-R1V-38B", trust_remote_code=True
773 + ),
774 + "SmolVLMForConditionalGeneration": _HfExamplesInfo(
775 + "HuggingFaceTB/SmolVLM2-2.2B-Instruct",
776 + min_transformers_version="4.56",
777 + transformers_version_reason="HF model broken in 4.55",
778 + ),
779 + "Step3VLForConditionalGeneration": _HfExamplesInfo(
780 + "stepfun-ai/step3", trust_remote_code=True
781 + ),
782 + "UltravoxModel": _HfExamplesInfo(
783 + "fixie-ai/ultravox-v0_5-llama-3_2-1b",
784 + trust_remote_code=True,
785 + ),
786 + "TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"),
787 + "Tarsier2ForConditionalGeneration": _HfExamplesInfo(
788 + "omni-research/Tarsier2-Recap-7b",
789 + hf_overrides={"architectures": ["Tarsier2ForConditionalGeneration"]},
790 + ),
791 "VoxtralForConditionalGeneration": _HfExamplesInfo(
792 "mistralai/Voxtral-Mini-3B-2507",
793 min_transformers_version="4.54",
@@ -600,80 +795,120 @@ _MULTIMODAL_EXAMPLE_MODELS = {
795 is_available_online=False,
796 ),
797 # [Encoder-decoder]
603 - "WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501
798 + "WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"),
799 # [Cross-encoder]
605 - "JinaVLForRanking": _HfExamplesInfo("jinaai/jina-reranker-m0"), # noqa: E501
800 + "JinaVLForRanking": _HfExamplesInfo("jinaai/jina-reranker-m0"),
801 }
802
803
804 _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
610 - "MedusaModel": _HfExamplesInfo("JackFram/llama-68m",
611 - speculative_model="abhigoyal/vllm-medusa-llama-68m-random"), # noqa: E501
805 + "MedusaModel": _HfExamplesInfo(
806 + "JackFram/llama-68m", speculative_model="abhigoyal/vllm-medusa-llama-68m-random"
807 + ),
808 # Temporarily disabled.
809 # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
614 - # "MLPSpeculatorPreTrainedModel": _HfExamplesInfo("JackFram/llama-160m",
615 - # speculative_model="ibm-ai-platform/llama-160m-accelerator"), # noqa: E501
616 - "DeepSeekMTPModel": _HfExamplesInfo("luccafong/deepseek_mtp_main_random",
617 - speculative_model="luccafong/deepseek_mtp_draft_random", # noqa: E501
618 - trust_remote_code=True),
619 - "EagleDeepSeekMTPModel": _HfExamplesInfo("eagle618/deepseek-v3-random",
620 - speculative_model="eagle618/eagle-deepseek-v3-random", # noqa: E501
621 - trust_remote_code=True),
622 - "EagleLlamaForCausalLM": _HfExamplesInfo("meta-llama/Meta-Llama-3-8B-Instruct", # noqa: E501
623 - trust_remote_code=True,
624 - speculative_model="yuhuili/EAGLE-LLaMA3-Instruct-8B",
625 - tokenizer="meta-llama/Meta-Llama-3-8B-Instruct"), # noqa: E501
626 - "Eagle3LlamaForCausalLM": _HfExamplesInfo("meta-llama/Llama-3.1-8B-Instruct", # noqa: E501
627 - trust_remote_code=True,
628 - speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B", # noqa: E501
629 - tokenizer="meta-llama/Llama-3.1-8B-Instruct",
630 - use_original_num_layers=True,
631 - max_model_len=10240),
632 - "LlamaForCausalLMEagle3": _HfExamplesInfo("Qwen/Qwen3-8B", # noqa: E501
633 - trust_remote_code=True,
634 - speculative_model="AngelSlim/Qwen3-8B_eagle3", # noqa: E501
635 - tokenizer="Qwen/Qwen3-8B",
636 - use_original_num_layers=True),
810 + # "MLPSpeculatorPreTrainedModel": _HfExamplesInfo(
811 + # "JackFram/llama-160m",
812 + # speculative_model="ibm-ai-platform/llama-160m-accelerator"
813 + # ),
814 + "DeepSeekMTPModel": _HfExamplesInfo(
815 + "luccafong/deepseek_mtp_main_random",
816 + speculative_model="luccafong/deepseek_mtp_draft_random",
817 + trust_remote_code=True,
818 + ),
819 + "EagleDeepSeekMTPModel": _HfExamplesInfo(
820 + "eagle618/deepseek-v3-random",
821 + speculative_model="eagle618/eagle-deepseek-v3-random",
822 + trust_remote_code=True,
823 + ),
824 + "EagleLlamaForCausalLM": _HfExamplesInfo(
825 + "meta-llama/Meta-Llama-3-8B-Instruct",
826 + trust_remote_code=True,
827 + speculative_model="yuhuili/EAGLE-LLaMA3-Instruct-8B",
828 + tokenizer="meta-llama/Meta-Llama-3-8B-Instruct",
829 + ),
830 + "Eagle3LlamaForCausalLM": _HfExamplesInfo(
831 + "meta-llama/Llama-3.1-8B-Instruct",
832 + trust_remote_code=True,
833 + speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B",
834 + tokenizer="meta-llama/Llama-3.1-8B-Instruct",
835 + use_original_num_layers=True,
836 + max_model_len=10240,
837 + ),
838 + "LlamaForCausalLMEagle3": _HfExamplesInfo(
839 + "Qwen/Qwen3-8B",
840 + trust_remote_code=True,
841 + speculative_model="AngelSlim/Qwen3-8B_eagle3",
842 + tokenizer="Qwen/Qwen3-8B",
843 + use_original_num_layers=True,
844 + ),
845 "EagleLlama4ForCausalLM": _HfExamplesInfo(
846 "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
847 trust_remote_code=True,
848 speculative_model="morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
641 - tokenizer="meta-llama/Llama-4-Scout-17B-16E-Instruct"), # noqa: E501
642 - "EagleMiniCPMForCausalLM": _HfExamplesInfo("openbmb/MiniCPM-1B-sft-bf16",
643 - trust_remote_code=True,
644 - is_available_online=False,
645 - speculative_model="openbmb/MiniCPM-2B-sft-bf16",
646 - tokenizer="openbmb/MiniCPM-2B-sft-bf16"),
647 - "ErnieMTPModel": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT",
648 - trust_remote_code=True,
649 - speculative_model="baidu/ERNIE-4.5-21B-A3B-PT"),
650 - "Glm4MoeMTPModel": _HfExamplesInfo("zai-org/GLM-4.5",
651 - speculative_model="zai-org/GLM-4.5",
652 - min_transformers_version="4.56",
653 - is_available_online=False),
849 + tokenizer="meta-llama/Llama-4-Scout-17B-16E-Instruct",
850 + ),
851 + "EagleMiniCPMForCausalLM": _HfExamplesInfo(
852 + "openbmb/MiniCPM-1B-sft-bf16",
853 + trust_remote_code=True,
854 + is_available_online=False,
855 + speculative_model="openbmb/MiniCPM-2B-sft-bf16",
856 + tokenizer="openbmb/MiniCPM-2B-sft-bf16",
857 + ),
858 + "ErnieMTPModel": _HfExamplesInfo(
859 + "baidu/ERNIE-4.5-21B-A3B-PT",
860 + trust_remote_code=True,
861 + speculative_model="baidu/ERNIE-4.5-21B-A3B-PT",
862 + ),
863 + "Glm4MoeMTPModel": _HfExamplesInfo(
864 + "zai-org/GLM-4.5",
865 + speculative_model="zai-org/GLM-4.5",
866 + min_transformers_version="4.56",
867 + is_available_online=False,
868 + ),
869 "LongCatFlashMTPModel": _HfExamplesInfo(
870 "meituan-longcat/LongCat-Flash-Chat",
871 trust_remote_code=True,
657 - speculative_model="meituan-longcat/LongCat-Flash-Chat"),
658 - "MiMoMTPModel": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL",
659 - trust_remote_code=True,
660 - speculative_model="XiaomiMiMo/MiMo-7B-RL"),
872 + speculative_model="meituan-longcat/LongCat-Flash-Chat",
873 + ),
874 + "MiMoMTPModel": _HfExamplesInfo(
875 + "XiaomiMiMo/MiMo-7B-RL",
876 + trust_remote_code=True,
877 + speculative_model="XiaomiMiMo/MiMo-7B-RL",
878 + ),
879 "Eagle3Qwen2_5vlForCausalLM": _HfExamplesInfo(
880 "Qwen/Qwen2.5-VL-7B-Instruct",
663 - speculative_model="Rayzl/qwen2.5-vl-7b-eagle3-sgl"),
664 - "Qwen3NextMTP": _HfExamplesInfo("Qwen/Qwen3-Next-80B-A3B-Instruct",
665 - min_transformers_version="4.56.3"),
881 + speculative_model="Rayzl/qwen2.5-vl-7b-eagle3-sgl",
882 + ),
883 + "Qwen3NextMTP": _HfExamplesInfo(
884 + "Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
885 + ),
886 }
887
888 _TRANSFORMERS_BACKEND_MODELS = {
669 - "TransformersEmbeddingModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5", min_transformers_version="4.57.0.dev0"), # noqa: E501
670 - "TransformersForSequenceClassification": _HfExamplesInfo("papluca/xlm-roberta-base-language-detection", min_transformers_version="4.57.0.dev0"), # noqa: E501
671 - "TransformersForCausalLM": _HfExamplesInfo("hmellor/Ilama-3.2-1B", trust_remote_code=True), # noqa: E501
889 + "TransformersEmbeddingModel": _HfExamplesInfo(
890 + "BAAI/bge-base-en-v1.5", min_transformers_version="4.57.0.dev0"
891 + ),
892 + "TransformersForSequenceClassification": _HfExamplesInfo(
893 + "papluca/xlm-roberta-base-language-detection",
894 + min_transformers_version="4.57.0.dev0",
895 + ),
896 + "TransformersForCausalLM": _HfExamplesInfo(
897 + "hmellor/Ilama-3.2-1B", trust_remote_code=True
898 + ),
899 "TransformersForMultimodalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
673 - "TransformersMoEForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924", min_transformers_version="4.57.0.dev0"), # noqa: E501
674 - "TransformersMoEForMultimodalLM": _HfExamplesInfo("Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="4.57.0.dev0"), # noqa: E501
675 - "TransformersMoEEmbeddingModel": _HfExamplesInfo("Qwen/Qwen3-30B-A3B", min_transformers_version="4.57.0.dev0"), # noqa: E501
676 - "TransformersMoEForSequenceClassification": _HfExamplesInfo("Qwen/Qwen3-30B-A3B", min_transformers_version="4.57.0.dev0"), # noqa: E501
900 + "TransformersMoEForCausalLM": _HfExamplesInfo(
901 + "allenai/OLMoE-1B-7B-0924", min_transformers_version="4.57.0.dev0"
902 + ),
903 + "TransformersMoEForMultimodalLM": _HfExamplesInfo(
904 + "Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="4.57.0.dev0"
905 + ),
906 + "TransformersMoEEmbeddingModel": _HfExamplesInfo(
907 + "Qwen/Qwen3-30B-A3B", min_transformers_version="4.57.0.dev0"
908 + ),
909 + "TransformersMoEForSequenceClassification": _HfExamplesInfo(
910 + "Qwen/Qwen3-30B-A3B", min_transformers_version="4.57.0.dev0"
911 + ),
912 }
913
914 _EXAMPLE_MODELS = {
@@ -699,8 +934,9 @@ class HfExampleModels:
934 try:
935 return self.hf_models[model_arch]
936 except KeyError:
702 - raise ValueError(f"No example model defined for {model_arch}; "
703 - f"please update this file.") from None
937 + raise ValueError(
938 + f"No example model defined for {model_arch}; please update this file."
939 + ) from None
940
941 def find_hf_info(self, model_id: str) -> _HfExamplesInfo:
942 for info in self.hf_models.values():
@@ -712,8 +948,9 @@ class HfExampleModels:
948 if any(extra == model_id for extra in info.extras.values()):
949 return info
950
715 - raise ValueError(f"No example model defined for {model_id}; "
716 - f"please update this file.")
951 + raise ValueError(
952 + f"No example model defined for {model_id}; please update this file."
953 + )
954
955
956 HF_EXAMPLE_MODELS = HfExampleModels(_EXAMPLE_MODELS)
tests/multimodal/test_cache.py
+19 -20
@@ -71,25 +71,27 @@ def _dummy_items(
71 )
72
73
74 -# yapf: disable
74 @pytest.mark.parametrize(
75 ("item", "expected_size"),
76 [
77 (_dummy_item("a", {"a1": 100}), 100),
78 (_dummy_item("a", {"a1": 100, "a2": 110}), 210),
79 (_dummy_items({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}), 460), # noqa: E501
81 - (_dummy_items({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}).get_data(), 460), # noqa: E501
80 + (
81 + _dummy_items(
82 + {"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}
83 + ).get_data(),
84 + 460,
85 + ), # noqa: E501
86 ],
87 )
84 -# yapf: enable
88 def test_cache_item_size(item, expected_size):
89 cache = MultiModalCache.get_lru_cache(2048, type(item))
90
91 cache[""] = item
92 assert cache.currsize == expected_size
93
91 - prompt_update = PromptInsertion("dummy", "target", "insertion") \
92 - .resolve(0)
94 + prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
95
96 cache[""] = MultiModalProcessorCacheItem(item, [prompt_update])
97 assert cache.currsize == expected_size
@@ -106,9 +108,9 @@ def _create_vllm_config(
108 return VllmConfig(
109 model_config=ModelConfig(
110 model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
109 - mm_processor_cache_gb=mm_processor_cache_gb),
110 - parallel_config=ParallelConfig(
111 - data_parallel_size=1 if enable_ipc else 2),
111 + mm_processor_cache_gb=mm_processor_cache_gb,
112 + ),
113 + parallel_config=ParallelConfig(data_parallel_size=1 if enable_ipc else 2),
114 )
115
116
@@ -124,11 +126,9 @@ def _compare_caches(
126 seed: int = 0,
127 ):
128 cache_0_p0 = processor_cache_from_config(config_0, MULTIMODAL_REGISTRY)
127 - cache_0_p1 = engine_receiver_cache_from_config(config_0,
128 - MULTIMODAL_REGISTRY)
129 + cache_0_p1 = engine_receiver_cache_from_config(config_0, MULTIMODAL_REGISTRY)
130 cache_1_p0 = processor_cache_from_config(config_1, MULTIMODAL_REGISTRY)
130 - cache_1_p1 = engine_receiver_cache_from_config(config_1,
131 - MULTIMODAL_REGISTRY)
131 + cache_1_p1 = engine_receiver_cache_from_config(config_1, MULTIMODAL_REGISTRY)
132
133 cache_size_gb = max(
134 config_0.model_config.multimodal_config.mm_processor_cache_gb,
@@ -142,8 +142,7 @@ def _compare_caches(
142 for _ in range(int(item_capacity / hit_rate))
143 ]
144 all_hashes = [
145 - MultiModalHasher.hash_kwargs(item=item.get_data())
146 - for item in all_items
145 + MultiModalHasher.hash_kwargs(item=item.get_data()) for item in all_items
146 ]
147
148 # Should not be used since there is nothing to convert to text
@@ -162,7 +161,8 @@ def _compare_caches(
161 for _ in range(is_cached_calls_per_iter):
162 cache_0_p0.is_cached(selected_hashes)
163 cache_0_p0_out = [
165 - item for item, _ in cache_0_p0.get_and_update(
164 + item
165 + for item, _ in cache_0_p0.get_and_update(
166 [(item, prompt_update.content) for item in selected_items],
167 selected_hashes,
168 )
@@ -174,7 +174,8 @@ def _compare_caches(
174 for _ in range(is_cached_calls_per_iter):
175 cache_1_p0.is_cached(selected_hashes)
176 cache_1_p0_out = [
177 - item for item, _ in cache_1_p0.get_and_update(
177 + item
178 + for item, _ in cache_1_p0.get_and_update(
179 [(item, prompt_update.content) for item in selected_items],
180 selected_hashes,
181 )
@@ -183,14 +184,12 @@ def _compare_caches(
184 if cache_0_p1 is None:
185 cache_0_p1_out = cache_0_p0_out
186 else:
186 - cache_0_p1_out = cache_0_p1.get_and_update(cache_0_p0_out,
187 - selected_hashes)
187 + cache_0_p1_out = cache_0_p1.get_and_update(cache_0_p0_out, selected_hashes)
188
189 if cache_1_p1 is None:
190 cache_1_p1_out = cache_1_p0_out
191 else:
192 - cache_1_p1_out = cache_1_p1.get_and_update(cache_1_p0_out,
193 - selected_hashes)
192 + cache_1_p1_out = cache_1_p1.get_and_update(cache_1_p0_out, selected_hashes)
193
194 assert cache_0_p1_out == cache_1_p1_out, f"Failed at {it=}"
195
tests/multimodal/test_processing.py
+127 -99
@@ -9,9 +9,6 @@ import pytest
9
10 from vllm.config import ModelConfig
11 from vllm.multimodal import MULTIMODAL_REGISTRY
12 -
13 -# yapf conflicts with isort for this block
14 -# yapf: disable
12 from vllm.multimodal.processing import (
13 InputProcessingContext,
14 PlaceholderFeaturesInfo,
@@ -24,8 +21,6 @@ from vllm.multimodal.processing import (
21 iter_token_matches,
22 replace_token_matches,
23 )
27 -
28 -# yapf: enable
24 from vllm.multimodal.profiling import MultiModalProfiler
25 from vllm.transformers_utils.tokenizer import AnyTokenizer
26
@@ -34,7 +29,6 @@ from .utils import random_image
29 pytestmark = pytest.mark.cpu_test
30
31
37 -# yapf: disable
32 @pytest.mark.parametrize(
33 ("token_ids", "match_ids", "expected"),
34 [
@@ -44,34 +38,34 @@ pytestmark = pytest.mark.cpu_test
38 [32000, 32000, 32000],
39 [32000],
40 [
47 - { "start_idx": 0, "end_idx": 1 },
48 - { "start_idx": 1, "end_idx": 2 },
49 - { "start_idx": 2, "end_idx": 3 },
41 + {"start_idx": 0, "end_idx": 1},
42 + {"start_idx": 1, "end_idx": 2},
43 + {"start_idx": 2, "end_idx": 3},
44 ],
45 ),
46 (
47 [32000, 32000, 32000],
48 [32000, 32000],
55 - [{ "start_idx": 0, "end_idx": 2 }],
49 + [{"start_idx": 0, "end_idx": 2}],
50 ),
51 (
52 [32000, 32000, 32000],
53 [32000, 32000, 32000],
60 - [{ "start_idx": 0, "end_idx": 3 }],
54 + [{"start_idx": 0, "end_idx": 3}],
55 ),
56 (
57 [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
58 [28747, 32000],
59 [
66 - { "start_idx": 1, "end_idx": 3 },
67 - { "start_idx": 6, "end_idx": 8 },
60 + {"start_idx": 1, "end_idx": 3},
61 + {"start_idx": 6, "end_idx": 8},
62 ],
63 ),
64 (
65 [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
66 [28747, 32000, 32000, 32000],
67 [
74 - { "start_idx": 1, "end_idx": 5 },
68 + {"start_idx": 1, "end_idx": 5},
69 ],
70 ),
71 (
@@ -82,14 +76,13 @@ pytestmark = pytest.mark.cpu_test
76 ],
77 )
78 @pytest.mark.parametrize("start_idx", [0, 4, 8])
85 -# yapf: enable
79 def test_iter_token_matches(token_ids, match_ids, expected, start_idx):
87 - result = list(iter_token_matches(token_ids, match_ids,
88 - start_idx=start_idx))
80 + result = list(iter_token_matches(token_ids, match_ids, start_idx=start_idx))
81
82 # Manually constructed results
91 - assert [item._asdict() for item in result
92 - ] == [item for item in expected if item["start_idx"] >= start_idx]
83 + assert [item._asdict() for item in result] == [
84 + item for item in expected if item["start_idx"] >= start_idx
85 + ]
86
87 # Invariants
88 match_lens = [end - start for start, end in result]
@@ -97,7 +90,6 @@ def test_iter_token_matches(token_ids, match_ids, expected, start_idx):
90 assert all(match_len == len(match_ids) for match_len in match_lens)
91
92
100 -# yapf: disable
93 @pytest.mark.parametrize(
94 ("token_ids", "match_ids", "new_ids", "expected"),
95 [
@@ -141,7 +133,6 @@ def test_iter_token_matches(token_ids, match_ids, expected, start_idx):
133 ),
134 ],
135 )
144 -# yapf: enable
136 def test_replace_token_matches(token_ids, match_ids, new_ids, expected):
137 result = replace_token_matches(token_ids, match_ids, new_ids)
138
@@ -149,7 +140,6 @@ def test_replace_token_matches(token_ids, match_ids, new_ids, expected):
140 assert result == expected
141
142
152 -# yapf: disable
143 @pytest.mark.parametrize(
144 ("prompt", "target_by_key", "expected_by_key"),
145 [
@@ -166,11 +156,11 @@ def test_replace_token_matches(token_ids, match_ids, new_ids, expected):
156 "pattern_1": [],
157 "pattern_2": [],
158 "pattern_3": [
169 - { "start_idx": 0, "end_idx": 0 },
159 + {"start_idx": 0, "end_idx": 0},
160 ],
161 "pattern_4": [],
162 "pattern_5": [
173 - { "start_idx": 0, "end_idx": 0 },
163 + {"start_idx": 0, "end_idx": 0},
164 ],
165 },
166 ),
@@ -186,26 +176,26 @@ def test_replace_token_matches(token_ids, match_ids, new_ids, expected):
176 },
177 {
178 "pattern_1": [
189 - { "start_idx": 0, "end_idx": 1 },
190 - { "start_idx": 1, "end_idx": 2 },
191 - { "start_idx": 2, "end_idx": 3 },
192 - { "start_idx": 3, "end_idx": 4 },
179 + {"start_idx": 0, "end_idx": 1},
180 + {"start_idx": 1, "end_idx": 2},
181 + {"start_idx": 2, "end_idx": 3},
182 + {"start_idx": 3, "end_idx": 4},
183 ],
184 "pattern_2": [
195 - { "start_idx": 0, "end_idx": 2 },
196 - { "start_idx": 2, "end_idx": 4 },
185 + {"start_idx": 0, "end_idx": 2},
186 + {"start_idx": 2, "end_idx": 4},
187 ],
188 "pattern_3": [
199 - { "start_idx": 0, "end_idx": 3 },
189 + {"start_idx": 0, "end_idx": 3},
190 ],
191 "pattern_4": [
202 - { "start_idx": 0, "end_idx": 0 },
192 + {"start_idx": 0, "end_idx": 0},
193 ],
194 "pattern_5": [
205 - { "start_idx": 1, "end_idx": 1 },
195 + {"start_idx": 1, "end_idx": 1},
196 ],
197 "pattern_6": [
208 - { "start_idx": 4, "end_idx": 4 },
198 + {"start_idx": 4, "end_idx": 4},
199 ],
200 },
201 ),
@@ -221,26 +211,25 @@ def test_replace_token_matches(token_ids, match_ids, new_ids, expected):
211 },
212 {
213 "pattern_1": [
224 - { "start_idx": 1, "end_idx": 3 },
225 - { "start_idx": 6, "end_idx": 8 },
214 + {"start_idx": 1, "end_idx": 3},
215 + {"start_idx": 6, "end_idx": 8},
216 ],
217 "pattern_2": [
228 - { "start_idx": 1, "end_idx": 5 },
218 + {"start_idx": 1, "end_idx": 5},
219 ],
220 "pattern_3": [],
221 "pattern_4": [
232 - { "start_idx": 0, "end_idx": 0 },
222 + {"start_idx": 0, "end_idx": 0},
223 ],
224 "pattern_5": [],
225 "pattern_6": [
236 - { "start_idx": 10, "end_idx": 10 },
226 + {"start_idx": 10, "end_idx": 10},
227 ],
228 },
229 ),
230 ],
231 )
232 @pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
243 -# yapf: enable
233 def test_find_token_matches(
234 prompt,
235 target_by_key,
@@ -272,7 +261,6 @@ def test_find_token_matches(
261 } == expected_by_key
262
263
275 -# yapf: disable
264 @pytest.mark.parametrize(
265 ("prompt", "target_by_key", "expected_by_key"),
266 [
@@ -288,16 +276,16 @@ def test_find_token_matches(
276 "pattern_5": PromptIndexTargets.end(),
277 },
278 {
291 - "pattern_1": [{ "start_idx": 0, "end_idx": 0 }],
279 + "pattern_1": [{"start_idx": 0, "end_idx": 0}],
280 "pattern_2": [],
281 "pattern_3": [
294 - { "start_idx": 0, "end_idx": 0 },
282 + {"start_idx": 0, "end_idx": 0},
283 ],
284 "pattern_4": [],
285 "pattern_5": [
298 - { "start_idx": 0, "end_idx": 0 },
286 + {"start_idx": 0, "end_idx": 0},
287 ],
300 - }
288 + },
289 ),
290 (
291 "<image><image><image><image>",
@@ -311,26 +299,26 @@ def test_find_token_matches(
299 },
300 {
301 "pattern_1": [
314 - { "start_idx": 0, "end_idx": 7 },
315 - { "start_idx": 7, "end_idx": 14 },
316 - { "start_idx": 14, "end_idx": 21 },
317 - { "start_idx": 21, "end_idx": 28 },
302 + {"start_idx": 0, "end_idx": 7},
303 + {"start_idx": 7, "end_idx": 14},
304 + {"start_idx": 14, "end_idx": 21},
305 + {"start_idx": 21, "end_idx": 28},
306 ],
307 "pattern_2": [
320 - { "start_idx": 0, "end_idx": 14 },
321 - { "start_idx": 14, "end_idx": 28 },
308 + {"start_idx": 0, "end_idx": 14},
309 + {"start_idx": 14, "end_idx": 28},
310 ],
311 "pattern_3": [
324 - { "start_idx": 0, "end_idx": 21 },
312 + {"start_idx": 0, "end_idx": 21},
313 ],
314 "pattern_4": [
327 - { "start_idx": 0, "end_idx": 0 },
315 + {"start_idx": 0, "end_idx": 0},
316 ],
317 "pattern_5": [
330 - { "start_idx": 7, "end_idx": 7 },
318 + {"start_idx": 7, "end_idx": 7},
319 ],
320 "pattern_6": [
333 - { "start_idx": 28, "end_idx": 28 },
321 + {"start_idx": 28, "end_idx": 28},
322 ],
323 },
324 ),
@@ -346,21 +334,21 @@ def test_find_token_matches(
334 },
335 {
336 "pattern_1": [
349 - { "start_idx": 0, "end_idx": 13 },
350 - { "start_idx": 27, "end_idx": 40 },
337 + {"start_idx": 0, "end_idx": 13},
338 + {"start_idx": 27, "end_idx": 40},
339 ],
340 "pattern_2": [
353 - { "start_idx": 0, "end_idx": 27 },
341 + {"start_idx": 0, "end_idx": 27},
342 ],
343 "pattern_3": [],
344 "pattern_4": [
357 - { "start_idx": 0, "end_idx": 0 },
345 + {"start_idx": 0, "end_idx": 0},
346 ],
347 "pattern_5": [
360 - { "start_idx": 13, "end_idx": 13 },
348 + {"start_idx": 13, "end_idx": 13},
349 ],
350 "pattern_6": [
363 - { "start_idx": 48, "end_idx": 48 },
351 + {"start_idx": 48, "end_idx": 48},
352 ],
353 },
354 ),
@@ -374,22 +362,21 @@ def test_find_token_matches(
362 },
363 {
364 "pattern_1": [
377 - { "start_idx": 0, "end_idx": 9 },
378 - { "start_idx": 16, "end_idx": 25 },
365 + {"start_idx": 0, "end_idx": 9},
366 + {"start_idx": 16, "end_idx": 25},
367 ],
368 "pattern_2": [
381 - { "start_idx": 0, "end_idx": 16 },
382 - { "start_idx": 16, "end_idx": 32 },
369 + {"start_idx": 0, "end_idx": 16},
370 + {"start_idx": 16, "end_idx": 32},
371 ],
372 "pattern_3": [
385 - { "start_idx": 0, "end_idx": 25 },
373 + {"start_idx": 0, "end_idx": 25},
374 ],
375 },
376 ),
377 ],
378 )
379 @pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
392 -# yapf: enable
380 def test_find_text_matches(
381 prompt,
382 target_by_key,
@@ -421,7 +408,6 @@ def test_find_text_matches(
408 } == expected_by_key
409
410
424 -# yapf: disable
411 @pytest.mark.parametrize(
412 ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
413 [
@@ -549,9 +535,8 @@ def test_find_text_matches(
535 },
536 },
537 ),
552 - ]
538 + ],
539 )
554 -# yapf: enable
540 def test_find_update_text(
541 prompt,
542 target_by_key,
@@ -562,13 +547,15 @@ def test_find_update_text(
547 mock_tokenizer = cast(AnyTokenizer, object())
548
549 for (
565 - update_type,
566 - expected_by_mm_count,
550 + update_type,
551 + expected_by_mm_count,
552 ) in expected_by_update_type_mm_count.items():
553 for mm_count, expected in expected_by_mm_count.items():
554 mm_prompt_updates = {
570 - key: [[update_type(key, target, repl_by_key[key]).resolve(i)]
571 - for i in range(mm_count)]
555 + key: [
556 + [update_type(key, target, repl_by_key[key]).resolve(i)]
557 + for i in range(mm_count)
558 + ]
559 for key, target in target_by_key.items()
560 }
561
@@ -589,7 +576,6 @@ def test_find_update_text(
576 assert new_prompt == expected
577
578
592 -# yapf: disable
579 @pytest.mark.parametrize(
580 ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
581 [
@@ -615,8 +601,43 @@ def test_find_update_text(
601 {
602 PromptInsertion: {
603 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
618 - 1: [1, 9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918, 1550, 918, 1550], # noqa: E501
619 - 2: [1, 9833, 28747, 32000, 32000, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918, 1550, 918, 1550, 1550, 918, 1550], # noqa: E501
604 + 1: [
605 + 1,
606 + 9833,
607 + 28747,
608 + 32000,
609 + 32000,
610 + 32000,
611 + 9833,
612 + 28747,
613 + 32000,
614 + 32000,
615 + 918,
616 + 1550,
617 + 918,
618 + 1550,
619 + ], # noqa: E501
620 + 2: [
621 + 1,
622 + 9833,
623 + 28747,
624 + 32000,
625 + 32000,
626 + 32000,
627 + 32000,
628 + 32000,
629 + 9833,
630 + 28747,
631 + 32000,
632 + 32000,
633 + 918,
634 + 1550,
635 + 918,
636 + 1550,
637 + 1550,
638 + 918,
639 + 1550,
640 + ], # noqa: E501
641 },
642 PromptReplacement: {
643 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
@@ -719,9 +740,8 @@ def test_find_update_text(
740 },
741 },
742 ),
722 - ]
743 + ],
744 )
724 -# yapf: enable
745 def test_find_update_tokens(
746 prompt,
747 target_by_key,
@@ -732,13 +752,15 @@ def test_find_update_tokens(
752 mock_tokenizer = cast(AnyTokenizer, object())
753
754 for (
735 - update_type,
736 - expected_by_mm_count,
755 + update_type,
756 + expected_by_mm_count,
757 ) in expected_by_update_type_mm_count.items():
758 for mm_count, expected in expected_by_mm_count.items():
759 mm_prompt_updates = {
740 - key: [[update_type(key, target, repl_by_key[key]).resolve(i)]
741 - for i in range(mm_count)]
760 + key: [
761 + [update_type(key, target, repl_by_key[key]).resolve(i)]
762 + for i in range(mm_count)
763 + ]
764 for key, target in target_by_key.items()
765 }
766
@@ -759,7 +781,6 @@ def test_find_update_tokens(
781 assert new_prompt == expected
782
783
762 -# yapf: disable
784 @pytest.mark.parametrize(
785 "repl_by_key",
786 [
@@ -796,8 +817,7 @@ def test_find_update_tokens(
817 is_embed=None,
818 ),
819 ],
799 - }
800 -
820 + },
821 ),
822 (
823 [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
@@ -828,7 +848,7 @@ def test_find_update_tokens(
848 ),
849 ],
850 # No match for pattern_4 as it has lower priority than pattern_1
831 - }
851 + },
852 ),
853 (
854 [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
@@ -867,12 +887,11 @@ def test_find_update_tokens(
887 is_embed=None,
888 ),
889 ],
870 - }
890 + },
891 ),
872 - ]
892 + ],
893 )
894 @pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
875 -# yapf: enable
895 def test_find_mm_placeholders(
896 repl_by_key,
897 prompt,
@@ -899,8 +918,15 @@ def test_find_mm_placeholders(
918 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
919 @pytest.mark.parametrize(
920 ("limit", "num_supported", "is_valid"),
902 - [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
903 - (2, 1, False), (2, 2, True)],
921 + [
922 + (0, 0, True),
923 + (0, 1, True),
924 + (1, 0, False),
925 + (1, 1, True),
926 + (1, 2, True),
927 + (2, 1, False),
928 + (2, 2, True),
929 + ],
930 )
931 def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
932 limit_mm_per_prompt = {"image": limit}
@@ -930,8 +956,15 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
956 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
957 @pytest.mark.parametrize(
958 ("num_images", "limit", "is_valid"),
933 - [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
934 - (2, 1, False), (2, 2, True)],
959 + [
960 + (0, 0, True),
961 + (0, 1, True),
962 + (1, 0, False),
963 + (1, 1, True),
964 + (1, 2, True),
965 + (2, 1, False),
966 + (2, 2, True),
967 + ],
968 )
969 def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
970 limit_mm_per_prompt = {"image": limit}
@@ -966,7 +999,6 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
999
1000
1001 class DummyProcessor:
969 -
1002 def __init__(self, a: int = 0, b: int = 0) -> None:
1003 super().__init__()
1004
@@ -982,7 +1014,6 @@ class DummyProcessor:
1014 return dict(a=a, c=c)
1015
1016
985 -# yapf: disable
1017 @pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"]) # Dummy
1018 @pytest.mark.parametrize(
1019 ("config_kwargs", "inference_kwargs", "expected_kwargs"),
@@ -996,7 +1027,6 @@ class DummyProcessor:
1027 ({"b": 1, "c": 1}, {}, {"a": 0, "b": 1}),
1028 ],
1029 )
999 -# yapf: enable
1030 def test_hf_processor_init_kwargs(
1031 model_id,
1032 config_kwargs,
@@ -1020,7 +1050,6 @@ def test_hf_processor_init_kwargs(
1050 assert getattr(processor, k) == v
1051
1052
1023 -# yapf: disable
1053 @pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"]) # Dummy
1054 @pytest.mark.parametrize(
1055 ("config_kwargs", "inference_kwargs", "expected_kwargs"),
@@ -1034,7 +1063,6 @@ def test_hf_processor_init_kwargs(
1063 ({"b": 1, "c": 1}, {}, {"a": 0, "c": 1}),
1064 ],
1065 )
1037 -# yapf: enable
1066 def test_hf_processor_call_kwargs(
1067 model_id,
1068 config_kwargs,
tests/multimodal/test_utils.py
+13 -14
@@ -233,7 +233,6 @@ async def test_fetch_video_http_with_dynamic_loader(
233 assert metadata_sync["video_backend"] == "opencv_dynamic"
234
235
236 -# yapf: disable
236 @pytest.mark.parametrize(
237 "case",
238 [
@@ -264,7 +263,6 @@ async def test_fetch_video_http_with_dynamic_loader(
263 ("image", 0),
264 ],
265 ),
267 -
266 # Two modalities
267 ## Internally sorted
268 dict(
@@ -276,7 +274,7 @@ async def test_fetch_video_http_with_dynamic_loader(
274 "audio": [
275 PlaceholderRange(offset=0, length=2),
276 PlaceholderRange(offset=2, length=3),
279 - ]
277 + ],
278 },
279 expected_modality_idxs=[
280 ("audio", 0),
@@ -295,7 +293,7 @@ async def test_fetch_video_http_with_dynamic_loader(
293 "audio": [
294 PlaceholderRange(offset=5, length=2),
295 PlaceholderRange(offset=11, length=4),
298 - ]
296 + ],
297 },
298 expected_modality_idxs=[
299 ("image", 0),
@@ -314,7 +312,7 @@ async def test_fetch_video_http_with_dynamic_loader(
312 "audio": [
313 PlaceholderRange(offset=11, length=4),
314 PlaceholderRange(offset=5, length=2),
317 - ]
315 + ],
316 },
317 expected_modality_idxs=[
318 ("image", 1),
@@ -323,7 +321,6 @@ async def test_fetch_video_http_with_dynamic_loader(
321 ("audio", 0),
322 ],
323 ),
326 -
324 # Three modalities
325 ## Internally sorted
326 dict(
@@ -339,7 +336,7 @@ async def test_fetch_video_http_with_dynamic_loader(
336 PlaceholderRange(offset=3, length=4),
337 PlaceholderRange(offset=7, length=5),
338 PlaceholderRange(offset=12, length=6),
342 - ]
339 + ],
340 },
341 expected_modality_idxs=[
342 ("audio", 0),
@@ -363,7 +360,7 @@ async def test_fetch_video_http_with_dynamic_loader(
360 ],
361 "video": [
362 PlaceholderRange(offset=8, length=5),
366 - ]
363 + ],
364 },
365 expected_modality_idxs=[
366 ("image", 0),
@@ -386,7 +383,7 @@ async def test_fetch_video_http_with_dynamic_loader(
383 ],
384 "video": [
385 PlaceholderRange(offset=8, length=5),
389 - ]
386 + ],
387 },
388 expected_modality_idxs=[
389 ("image", 0),
@@ -398,7 +395,6 @@ async def test_fetch_video_http_with_dynamic_loader(
395 ),
396 ],
397 )
401 -# yapf: enable
398 def test_argsort_mm_positions(case):
399 mm_positions = case["mm_positions"]
400 expected_modality_idxs = case["expected_modality_idxs"]
@@ -413,13 +409,16 @@ def test_argsort_mm_positions(case):
409 @pytest.mark.parametrize("num_frames", [-1, 32, 1800])
410 async def test_allowed_media_domains(video_url: str, num_frames: int):
411 connector = MediaConnector(
416 - media_io_kwargs={"video": {
417 - "num_frames": num_frames,
418 - }},
412 + media_io_kwargs={
413 + "video": {
414 + "num_frames": num_frames,
415 + }
416 + },
417 allowed_media_domains=[
418 "www.bogotobogo.com",
419 "github.com",
422 - ])
420 + ],
421 + )
422
423 video_sync, metadata_sync = connector.fetch_video(video_url)
424 video_async, metadata_async = await connector.fetch_video_async(video_url)
tests/test_inputs.py
+52 -44
@@ -59,48 +59,52 @@ def test_parse_raw_single_batch_string_slice(inputs_slice: slice):
59 )
60
61
62 -# yapf: disable
63 -@pytest.mark.parametrize('mm_processor_kwargs,expected_mm_kwargs', [
64 - (None, [{}, {}]),
65 - ({}, [{}, {}]),
66 - ({"foo": 100}, [{"foo": 100}, {"foo": 100}]),
67 - ([{"foo": 100}, {"bar": 200}], [{"foo": 100}, {"bar": 200}]),
68 -])
69 -# yapf: enable
62 +@pytest.mark.parametrize(
63 + "mm_processor_kwargs,expected_mm_kwargs",
64 + [
65 + (None, [{}, {}]),
66 + ({}, [{}, {}]),
67 + ({"foo": 100}, [{"foo": 100}, {"foo": 100}]),
68 + ([{"foo": 100}, {"bar": 200}], [{"foo": 100}, {"bar": 200}]),
69 + ],
70 +)
71 def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
72 """Test mm_processor_kwargs init for zipping enc/dec prompts."""
72 - encoder_prompts = ['An encoder prompt', 'Another encoder prompt']
73 - decoder_prompts = ['A decoder prompt', 'Another decoder prompt']
74 - zipped_prompts = zip_enc_dec_prompts(encoder_prompts, decoder_prompts,
75 - mm_processor_kwargs)
73 + encoder_prompts = ["An encoder prompt", "Another encoder prompt"]
74 + decoder_prompts = ["A decoder prompt", "Another decoder prompt"]
75 + zipped_prompts = zip_enc_dec_prompts(
76 + encoder_prompts, decoder_prompts, mm_processor_kwargs
77 + )
78 assert len(zipped_prompts) == len(encoder_prompts) == len(decoder_prompts)
77 - for enc, dec, exp_kwargs, zipped in zip(encoder_prompts, decoder_prompts,
78 - expected_mm_kwargs,
79 - zipped_prompts):
79 + for enc, dec, exp_kwargs, zipped in zip(
80 + encoder_prompts, decoder_prompts, expected_mm_kwargs, zipped_prompts
81 + ):
82 assert isinstance(zipped, dict)
83 assert len(zipped.keys()) == 3
82 - assert zipped['encoder_prompt'] == enc
83 - assert zipped['decoder_prompt'] == dec
84 - assert zipped['mm_processor_kwargs'] == exp_kwargs
85 -
86 -
87 -@pytest.mark.parametrize("model_id", [
88 - "facebook/opt-125m",
89 -])
90 -@pytest.mark.parametrize("prompt", [
91 - {
92 - "prompt": "",
93 - "multi_modal_data": {
94 - "dummy": []
84 + assert zipped["encoder_prompt"] == enc
85 + assert zipped["decoder_prompt"] == dec
86 + assert zipped["mm_processor_kwargs"] == exp_kwargs
87 +
88 +
89 +@pytest.mark.parametrize(
90 + "model_id",
91 + [
92 + "facebook/opt-125m",
93 + ],
94 +)
95 +@pytest.mark.parametrize(
96 + "prompt",
97 + [
98 + {
99 + "prompt": "",
100 + "multi_modal_data": {"dummy": []},
101 },
96 - },
97 - {
98 - "prompt_token_ids": [],
99 - "multi_modal_data": {
100 - "dummy": []
102 + {
103 + "prompt_token_ids": [],
104 + "multi_modal_data": {"dummy": []},
105 },
102 - },
103 -])
106 + ],
107 +)
108 def test_preprocessor_text_no_mm_inputs(model_id, prompt):
109 model_config = ModelConfig(model=model_id)
110 tokenizer = init_tokenizer_from_configs(model_config)
@@ -110,15 +114,19 @@ def test_preprocessor_text_no_mm_inputs(model_id, prompt):
114 input_preprocessor.preprocess(prompt)
115
116
113 -@pytest.mark.parametrize("model_id", [
114 - "facebook/chameleon-7b",
115 -])
116 -@pytest.mark.parametrize("prompt", [
117 - "",
118 - {
119 - "prompt_token_ids": []
120 - },
121 -])
117 +@pytest.mark.parametrize(
118 + "model_id",
119 + [
120 + "facebook/chameleon-7b",
121 + ],
122 +)
123 +@pytest.mark.parametrize(
124 + "prompt",
125 + [
126 + "",
127 + {"prompt_token_ids": []},
128 + ],
129 +)
130 def test_preprocessor_always_mm_code_path(model_id, prompt):
131 model_config = ModelConfig(model=model_id)
132 tokenizer = init_tokenizer_from_configs(model_config)
tests/tpu/test_moe_pallas.py
-4
@@ -9,14 +9,10 @@ import pytest
9 import torch
10 import torch_xla
11
12 -# yapf conflicts with isort for this block
13 -# yapf: disable
12 from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe as pallas_moe
13 from vllm.model_executor.layers.fused_moe.moe_torch_iterative import (
14 fused_moe as torch_moe,
15 )
18 -
19 -# yapf: enable
16 from vllm.platforms import current_platform
17
18 if not current_platform.is_tpu():
tests/utils_/test_utils.py
+70 -61
@@ -388,7 +388,6 @@ def test_duplicate_dict_args(caplog_vllm, parser):
388 assert "-O.level" in caplog_vllm.text
389
390
391 -# yapf: enable
391 @pytest.mark.parametrize(
392 "callable,kw_name,requires_kw_only,allow_var_kwargs,is_supported",
393 [
@@ -408,7 +407,6 @@ def test_duplicate_dict_args(caplog_vllm, parser):
407 (lambda foo, **kwargs: None, "foo", True, True, False),
408 ],
409 )
411 -# yapf: disable
410 def test_supports_kw(
411 callable, kw_name, requires_kw_only, allow_var_kwargs, is_supported
412 ):
@@ -681,7 +679,6 @@ def test_lru_cache():
679 assert 6 in cache
680
681
684 -# yapf: disable
682 @pytest.mark.parametrize(
683 ("src_dtype", "tgt_dtype", "expected_result"),
684 [
@@ -715,12 +712,10 @@ def test_lru_cache():
712 (torch.complex64, torch.complex32, False),
713 ],
714 )
718 -# yapf: enable
715 def test_is_lossless_cast(src_dtype, tgt_dtype, expected_result):
716 assert is_lossless_cast(src_dtype, tgt_dtype) == expected_result
717
718
723 -# yapf: disable
719 @pytest.mark.parametrize(
720 ("dtypes", "expected_result"),
721 [
@@ -730,7 +725,6 @@ def test_is_lossless_cast(src_dtype, tgt_dtype, expected_result):
725 ([torch.bool, torch.int8, torch.float16, torch.complex32], torch.complex32), # noqa: E501
726 ],
727 )
733 -# yapf: enable
728 def test_common_broadcastable_dtype(dtypes, expected_result):
729 assert common_broadcastable_dtype(dtypes) == expected_result
730
@@ -775,7 +769,6 @@ def test_placeholder_module_error_handling():
769 _ = placeholder_attr.module
770
771
778 -# yapf: disable
772 @pytest.mark.parametrize(
773 "obj,key1,key2",
774 [
@@ -785,8 +778,8 @@ def test_placeholder_module_error_handling():
778 ({1: "a", 2: "b"}, 1, 3),
779 # Tests for both keys do not exist
780 ({1: "a", 2: "b"}, 3, 4),
788 - ])
789 -# yapf: enable
781 + ],
782 +)
783 def test_swap_dict_values(obj, key1, key2):
784 original_obj = obj.copy()
785 swap_dict_values(obj, key1, key2)
@@ -800,26 +793,30 @@ def test_swap_dict_values(obj, key1, key2):
793 assert key1 not in obj
794
795
803 -def test_model_specification(parser_with_config, cli_config_file,
804 - cli_config_file_with_model):
796 +def test_model_specification(
797 + parser_with_config, cli_config_file, cli_config_file_with_model
798 +):
799 # Test model in CLI takes precedence over config
800 args = parser_with_config.parse_args(
807 - ['serve', 'cli-model', '--config', cli_config_file_with_model])
808 - assert args.model_tag == 'cli-model'
809 - assert args.served_model_name == 'mymodel'
801 + ["serve", "cli-model", "--config", cli_config_file_with_model]
802 + )
803 + assert args.model_tag == "cli-model"
804 + assert args.served_model_name == "mymodel"
805
806 # Test model from config file works
812 - args = parser_with_config.parse_args([
813 - 'serve',
814 - '--config',
815 - cli_config_file_with_model,
816 - ])
817 - assert args.model == 'config-model'
818 - assert args.served_model_name == 'mymodel'
807 + args = parser_with_config.parse_args(
808 + [
809 + "serve",
810 + "--config",
811 + cli_config_file_with_model,
812 + ]
813 + )
814 + assert args.model == "config-model"
815 + assert args.served_model_name == "mymodel"
816
817 # Test no model specified anywhere raises error
818 with pytest.raises(ValueError, match="No model specified!"):
822 - parser_with_config.parse_args(['serve', '--config', cli_config_file])
819 + parser_with_config.parse_args(["serve", "--config", cli_config_file])
820
821 # Test using --model option raises error
822 # with pytest.raises(
@@ -833,47 +830,52 @@ def test_model_specification(parser_with_config, cli_config_file,
830 # Test using --model option back-compatibility
831 # (when back-compatibility ends, the above test should be uncommented
832 # and the below test should be removed)
836 - args = parser_with_config.parse_args([
837 - 'serve',
838 - '--tensor-parallel-size',
839 - '2',
840 - '--model',
841 - 'my-model',
842 - '--trust-remote-code',
843 - '--port',
844 - '8001',
845 - ])
833 + args = parser_with_config.parse_args(
834 + [
835 + "serve",
836 + "--tensor-parallel-size",
837 + "2",
838 + "--model",
839 + "my-model",
840 + "--trust-remote-code",
841 + "--port",
842 + "8001",
843 + ]
844 + )
845 assert args.model is None
846 assert args.tensor_parallel_size == 2
847 assert args.trust_remote_code is True
848 assert args.port == 8001
849
851 - args = parser_with_config.parse_args([
852 - 'serve',
853 - '--tensor-parallel-size=2',
854 - '--model=my-model',
855 - '--trust-remote-code',
856 - '--port=8001',
857 - ])
850 + args = parser_with_config.parse_args(
851 + [
852 + "serve",
853 + "--tensor-parallel-size=2",
854 + "--model=my-model",
855 + "--trust-remote-code",
856 + "--port=8001",
857 + ]
858 + )
859 assert args.model is None
860 assert args.tensor_parallel_size == 2
861 assert args.trust_remote_code is True
862 assert args.port == 8001
863
864 # Test other config values are preserved
864 - args = parser_with_config.parse_args([
865 - 'serve',
866 - 'cli-model',
867 - '--config',
868 - cli_config_file_with_model,
869 - ])
865 + args = parser_with_config.parse_args(
866 + [
867 + "serve",
868 + "cli-model",
869 + "--config",
870 + cli_config_file_with_model,
871 + ]
872 + )
873 assert args.tensor_parallel_size == 2
874 assert args.trust_remote_code is True
875 assert args.port == 12312
876
877
875 -@pytest.mark.parametrize("input", [(), ("abc", ), (None, ),
876 - (None, bool, [1, 2, 3])])
878 +@pytest.mark.parametrize("input", [(), ("abc",), (None,), (None, bool, [1, 2, 3])])
879 def test_sha256(input: tuple):
880 digest = sha256(input)
881 assert digest is not None
@@ -887,7 +889,7 @@ def test_sha256(input: tuple):
889 assert digest == sha256(input)
890
891 # hashing different input, returns different value
890 - assert digest != sha256(input + (1, ))
892 + assert digest != sha256(input + (1,))
893
894
895 @pytest.mark.parametrize(
@@ -897,7 +899,8 @@ def test_sha256(input: tuple):
899 ("tcp://127.0.0.1:5555", ("tcp", "127.0.0.1", "5555")),
900 ("tcp://[::1]:5555", ("tcp", "::1", "5555")), # IPv6 address
901 ("inproc://some_identifier", ("inproc", "some_identifier", "")),
900 - ])
902 + ],
903 +)
904 def test_split_zmq_path(path, expected):
905 assert split_zmq_path(path) == expected
906
@@ -909,7 +912,8 @@ def test_split_zmq_path(path, expected):
912 "tcp://127.0.0.1", # Missing port
913 "tcp://[::1]", # Missing port for IPv6
914 "tcp://:5555", # Missing host
912 - ])
915 + ],
916 +)
917 def test_split_zmq_path_invalid(invalid_path):
918 with pytest.raises(ValueError):
919 split_zmq_path(invalid_path)
@@ -931,8 +935,9 @@ def test_make_zmq_socket_ipv6():
935 zsock: zmq.Socket = make_zmq_socket(ctx, ipv6_path, socket_type)
936
937 # Verify that the IPV6 option is set
934 - assert zsock.getsockopt(
935 - zmq.IPV6) == 1, "IPV6 option should be enabled for IPv6 addresses"
938 + assert zsock.getsockopt(zmq.IPV6) == 1, (
939 + "IPV6 option should be enabled for IPv6 addresses"
940 + )
941
942 # Clean up
943 zsock.close()
@@ -1019,15 +1024,14 @@ def test_convert_ids_list_to_tokens():
1024 tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-1.5B-Instruct")
1025 token_ids = tokenizer.encode("Hello, world!")
1026 # token_ids = [9707, 11, 1879, 0]
1022 - assert tokenizer.convert_ids_to_tokens(token_ids) == [
1023 - 'Hello', ',', 'Ġworld', '!'
1024 - ]
1027 + assert tokenizer.convert_ids_to_tokens(token_ids) == ["Hello", ",", "Ġworld", "!"]
1028 tokens = convert_ids_list_to_tokens(tokenizer, token_ids)
1026 - assert tokens == ['Hello', ',', ' world', '!']
1029 + assert tokens == ["Hello", ",", " world", "!"]
1030
1031
1032 def test_current_stream_multithread():
1033 import threading
1034 +
1035 if not torch.cuda.is_available():
1036 pytest.skip("CUDA not available")
1037
@@ -1046,13 +1050,18 @@ def test_current_stream_multithread():
1050 child_thread.start()
1051
1052 try:
1049 - assert thread_stream_ready.wait(
1050 - timeout=5), "Child thread failed to enter stream context in time"
1053 + assert thread_stream_ready.wait(timeout=5), (
1054 + "Child thread failed to enter stream context in time"
1055 + )
1056
1057 main_current_stream = current_stream()
1058
1054 - assert main_current_stream != child_stream, "Main thread's current_stream was contaminated by child thread"
1055 - assert main_current_stream == main_default_stream, "Main thread's current_stream is not the default stream"
1059 + assert main_current_stream != child_stream, (
1060 + "Main thread's current_stream was contaminated by child thread"
1061 + )
1062 + assert main_current_stream == main_default_stream, (
1063 + "Main thread's current_stream is not the default stream"
1064 + )
1065
1066 # Notify child thread it can exit
1067 thread_can_exit.set()
@@ -1070,7 +1079,7 @@ def test_load_config_file(tmp_path):
1079 "enable-logging": True,
1080 "list-arg": ["item1", "item2"],
1081 "port": 12323,
1073 - "tensor-parallel-size": 4
1082 + "tensor-parallel-size": 4,
1083 }
1084
1085 # Write the configuration data to a temporary YAML file
tests/v1/core/test_kv_cache_utils.py
-5
@@ -16,9 +16,6 @@ from vllm.multimodal.inputs import (
16 from vllm.sampling_params import SamplingParams
17 from vllm.utils import GiB_bytes, sha256, sha256_cbor
18 from vllm.v1.core.kv_cache_manager import KVCacheManager
19 -
20 -# disable yapf here as it formats differently than isort such that both fail
21 -# yapf: disable
19 from vllm.v1.core.kv_cache_utils import (
20 BlockHash,
21 FreeKVCacheBlockQueue,
@@ -48,8 +45,6 @@ from vllm.v1.kv_cache_interface import (
45 from vllm.v1.metrics.stats import PrefixCacheStats
46 from vllm.v1.request import Request
47
51 -# yapf: enable
52 -
48 pytestmark = pytest.mark.cpu_test
49
50
tests/v1/logits_processors/test_correctness.py
-4
@@ -22,8 +22,6 @@ from vllm.config import VllmConfig
22 from vllm.platforms import current_platform
23 from vllm.sampling_params import SamplingParams
24 from vllm.utils import is_pin_memory_available
25 -
26 -# yapf: disable
25 from vllm.v1.sample.logits_processor import (
26 BatchUpdate,
27 BatchUpdateBuilder,
@@ -34,8 +32,6 @@ from vllm.v1.sample.logits_processor import (
32 MoveDirectionality,
33 build_logitsprocs,
34 )
37 -
38 -# yapf: enable
35 from vllm.v1.sample.metadata import SamplingMetadata
36
37 PIN_MEMORY_AVAILABLE = is_pin_memory_available()
tests/v1/logits_processors/test_custom_offline.py
-4
@@ -7,8 +7,6 @@ from typing import Union
7 import pytest
8
9 from tests.utils import create_new_process_for_each_test
10 -
11 -# yapf: disable
10 from tests.v1.logits_processors.utils import (
11 DUMMY_LOGITPROC_ARG,
12 DUMMY_LOGITPROC_FQCN,
@@ -24,8 +22,6 @@ from tests.v1.logits_processors.utils import (
22 prompts,
23 )
24 from tests.v1.logits_processors.utils import entry_points as fake_entry_points
27 -
28 -# yapf: enable
25 from vllm import LLM, SamplingParams
26 from vllm.v1.sample.logits_processor import (
27 STR_POOLING_REJECTS_LOGITSPROCS,
tests/v1/logits_processors/test_custom_online.py
-4
@@ -11,8 +11,6 @@ import pytest
11 import pytest_asyncio
12
13 from tests.utils import RemoteOpenAIServerCustom, create_new_process_for_each_test
14 -
15 -# yapf: disable
14 from tests.v1.logits_processors.utils import (
15 DUMMY_LOGITPROC_ARG,
16 DUMMY_LOGITPROC_FQCN,
@@ -25,8 +23,6 @@ from tests.v1.logits_processors.utils import (
23 )
24 from tests.v1.logits_processors.utils import entry_points as fake_entry_points
25
28 -# yapf: enable
29 -
26
27 def _server_with_logitproc_entrypoint(
28 env_dict: Optional[dict[str, str]],
vllm/distributed/kv_transfer/kv_connector/factory.py
-3
@@ -4,7 +4,6 @@
4 import importlib
5 from typing import TYPE_CHECKING, Callable
6
7 -# yapf: disable
7 import vllm.envs as envs
8 from vllm.distributed.kv_transfer.kv_connector.base import (
9 KVConnectorBase,
@@ -13,8 +12,6 @@ from vllm.distributed.kv_transfer.kv_connector.base import (
12 from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
13 from vllm.logger import init_logger
14
16 -# yapf: enable
17 -
15 if TYPE_CHECKING:
16 from vllm.config import VllmConfig
17 from vllm.config.kv_transfer import KVTransferConfig
vllm/engine/arg_utils.py
-3
@@ -1,7 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -# yapf: disable
4 import argparse
5 import copy
6 import dataclasses
@@ -88,8 +87,6 @@ from vllm.transformers_utils.utils import check_gguf_file
87 from vllm.utils import FlexibleArgumentParser, GiB_bytes, get_ip, is_in_ray_actor
88 from vllm.v1.sample.logits_processor import LogitsProcessor
89
91 -# yapf: enable
92 -
90 if TYPE_CHECKING:
91 from vllm.executor.executor_base import ExecutorBase
92 from vllm.model_executor.layers.quantization import QuantizationMethods
vllm/entrypoints/chat_utils.py
+1 -14
@@ -17,9 +17,6 @@ import jinja2.nodes
17 import jinja2.parser
18 import jinja2.sandbox
19 import transformers.utils.chat_template_utils as hf_chat_utils
20 -
21 -# yapf conflicts with isort for this block
22 -# yapf: disable
20 from openai.types.chat import (
21 ChatCompletionAssistantMessageParam,
22 ChatCompletionContentPartImageParam,
@@ -40,8 +37,6 @@ from openai.types.responses import ResponseInputImageParam
37 from openai_harmony import Message as OpenAIHarmonyMessage
38 from PIL import Image
39 from pydantic import BaseModel, ConfigDict, TypeAdapter
43 -
44 -# yapf: enable
40 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, ProcessorMixin
41
42 # pydantic needs the TypedDict from typing_extensions
@@ -52,11 +47,7 @@ from vllm.logger import init_logger
47 from vllm.model_executor.models import SupportsMultiModal
48 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict
49 from vllm.multimodal.utils import MediaConnector
55 -
56 -# yapf: disable
50 from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
58 -
59 -# yapf: enable
51 from vllm.transformers_utils.processor import cached_get_processor
52 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
53 from vllm.utils import random_uuid, supports_kw
@@ -317,11 +308,7 @@ def _is_var_or_elems_access(
308 ):
309 return _is_var_or_elems_access(node.node, varname, key)
310
320 - # yapf: disable
321 - return (
322 - _is_attr_access(node, varname, key) if key
323 - else _is_var_access(node, varname)
324 - ) # yapf: enable
311 + return _is_attr_access(node, varname, key) if key else _is_var_access(node, varname)
312
313
314 def _iter_nodes_assign_var_or_elems(root: jinja2.nodes.Node, varname: str):
vllm/entrypoints/llm.py
-5
@@ -39,9 +39,6 @@ from vllm.entrypoints.chat_utils import (
39 parse_chat_messages,
40 resolve_chat_template_content_format,
41 )
42 -
43 -# yapf conflicts with isort for this block
44 -# yapf: disable
42 from vllm.entrypoints.score_utils import (
43 ScoreContentPartParam,
44 ScoreMultiModalParam,
@@ -50,8 +47,6 @@ from vllm.entrypoints.score_utils import (
47 compress_token_type_ids,
48 get_score_prompt,
49 )
53 -
54 -# yapf: enable
50 from vllm.entrypoints.utils import _validate_truncation_size, log_non_default_args
51 from vllm.inputs import (
52 DataPrompt,
vllm/entrypoints/openai/api_server.py
-5
@@ -49,9 +49,6 @@ from vllm.entrypoints.chat_utils import (
49 from vllm.entrypoints.launcher import serve_http
50 from vllm.entrypoints.logger import RequestLogger
51 from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
52 -
53 -# yapf conflicts with isort for this block
54 -# yapf: disable
52 from vllm.entrypoints.openai.protocol import (
53 ChatCompletionRequest,
54 ChatCompletionResponse,
@@ -84,8 +81,6 @@ from vllm.entrypoints.openai.protocol import (
81 TranslationResponse,
82 UnloadLoRAAdapterRequest,
83 )
87 -
88 -# yapf: enable
84 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
85 from vllm.entrypoints.openai.serving_classification import ServingClassification
86 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
vllm/entrypoints/openai/protocol.py
-4
@@ -11,8 +11,6 @@ from typing import Annotated, Any, ClassVar, Generic, Literal, Optional, TypeVar
11 import regex as re
12 import torch
13 from fastapi import HTTPException, UploadFile
14 -
15 -# yapf: disable
14 from openai.types.chat.chat_completion_audio import (
15 ChatCompletionAudio as OpenAIChatCompletionAudio,
16 )
@@ -46,8 +44,6 @@ from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreated
44 from openai.types.responses import (
45 ResponseInProgressEvent as OpenAIResponseInProgressEvent,
46 )
49 -
50 -# yapf: enable
47 from openai.types.responses.response_reasoning_item import (
48 Content as ResponseReasoningTextContent,
49 )
vllm/entrypoints/openai/run_batch.py
-4
@@ -18,8 +18,6 @@ from vllm.config import VllmConfig
18 from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
19 from vllm.engine.protocol import EngineClient
20 from vllm.entrypoints.logger import RequestLogger
21 -
22 -# yapf: disable
21 from vllm.entrypoints.openai.protocol import (
22 BatchRequestInput,
23 BatchRequestOutput,
@@ -30,8 +28,6 @@ from vllm.entrypoints.openai.protocol import (
28 RerankResponse,
29 ScoreResponse,
30 )
33 -
34 -# yapf: enable
31 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
32 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
33 from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
vllm/entrypoints/openai/serving_chat.py
+5 -3
@@ -1733,13 +1733,15 @@ class OpenAIServingChat(OpenAIServing):
1733 is a tool call with arguments.
1734 """
1735
1736 - # yapf: disable
1736 return bool(
1737 # if there is a delta message that includes tool calls which
1738 # include a function that has arguments
1739 output.finish_reason is not None
1741 - and self.enable_auto_tools and self.tool_parser and delta_message
1742 - and delta_message.tool_calls and delta_message.tool_calls[0]
1740 + and self.enable_auto_tools
1741 + and self.tool_parser
1742 + and delta_message
1743 + and delta_message.tool_calls
1744 + and delta_message.tool_calls[0]
1745 and delta_message.tool_calls[0].function
1746 and delta_message.tool_calls[0].function.arguments is not None
1747 )
vllm/entrypoints/openai/serving_classification.py
-2
@@ -18,8 +18,6 @@ from vllm.entrypoints.openai.protocol import (
18 ErrorResponse,
19 UsageInfo,
20 )
21 -
22 -# yapf: enable
21 from vllm.entrypoints.openai.serving_engine import (
22 ClassificationServeContext,
23 OpenAIServing,
vllm/entrypoints/openai/serving_completion.py
-5
@@ -13,9 +13,6 @@ from fastapi import Request
13 from vllm.config import ModelConfig
14 from vllm.engine.protocol import EngineClient
15 from vllm.entrypoints.logger import RequestLogger
16 -
17 -# yapf conflicts with isort for this block
18 -# yapf: disable
16 from vllm.entrypoints.openai.protocol import (
17 CompletionLogProbs,
18 CompletionRequest,
@@ -29,8 +26,6 @@ from vllm.entrypoints.openai.protocol import (
26 UsageInfo,
27 )
28 from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
32 -
33 -# yapf: enable
29 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
30 from vllm.entrypoints.renderer import RenderConfig
31 from vllm.entrypoints.utils import get_max_tokens
vllm/entrypoints/openai/serving_embedding.py
-5
@@ -14,9 +14,6 @@ from vllm.config import ModelConfig
14 from vllm.engine.protocol import EngineClient
15 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
16 from vllm.entrypoints.logger import RequestLogger
17 -
18 -# yapf conflicts with isort for this docstring
19 -# yapf: disable
17 from vllm.entrypoints.openai.protocol import (
18 EmbeddingChatRequest,
19 EmbeddingCompletionRequest,
@@ -32,8 +29,6 @@ from vllm.entrypoints.openai.serving_engine import (
29 ServeContext,
30 TextTokensPrompt,
31 )
35 -
36 -# yapf: enable
32 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
33 from vllm.entrypoints.renderer import RenderConfig
34 from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
vllm/entrypoints/openai/serving_engine.py
-5
@@ -28,9 +28,6 @@ else:
28 import vllm.envs as envs
29 from vllm.config import ModelConfig
30 from vllm.engine.protocol import EngineClient
31 -
32 -# yapf conflicts with isort for this block
33 -# yapf: disable
31 from vllm.entrypoints.chat_utils import (
32 ChatCompletionMessageParam,
33 ChatTemplateContentFormatOption,
@@ -72,8 +69,6 @@ from vllm.entrypoints.openai.protocol import (
69 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
70 from vllm.entrypoints.openai.tool_parsers import ToolParser
71 from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig
75 -
76 -# yapf: enable
72 from vllm.inputs.data import PromptType
73 from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
74 from vllm.inputs.parse import PromptComponents, get_prompt_components
vllm/entrypoints/openai/serving_pooling.py
-4
@@ -17,8 +17,6 @@ from vllm.config import VllmConfig
17 from vllm.engine.protocol import EngineClient
18 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
19 from vllm.entrypoints.logger import RequestLogger
20 -
21 -# yapf: disable
20 from vllm.entrypoints.openai.protocol import (
21 ErrorResponse,
22 IOProcessorRequest,
@@ -30,8 +28,6 @@ from vllm.entrypoints.openai.protocol import (
28 PoolingResponseData,
29 UsageInfo,
30 )
33 -
34 -# yapf: enable
31 from vllm.entrypoints.openai.serving_engine import OpenAIServing
32 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
33 from vllm.entrypoints.renderer import RenderConfig
vllm/entrypoints/openai/serving_responses.py
-10
@@ -14,9 +14,6 @@ from typing import Callable, Final, Optional, Union
14
15 import jinja2
16 from fastapi import Request
17 -
18 -# yapf conflicts with isort for this block
19 -# yapf: disable
17 from openai.types.responses import (
18 ResponseCodeInterpreterCallCodeDeltaEvent,
19 ResponseCodeInterpreterCallCodeDoneEvent,
@@ -46,8 +43,6 @@ from openai.types.responses import (
43 response_text_delta_event,
44 )
45 from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
49 -
50 -# yapf: enable
46 from openai.types.responses.response_reasoning_item import (
47 Content as ResponseReasoningTextContent,
48 )
@@ -78,9 +73,6 @@ from vllm.entrypoints.harmony_utils import (
73 render_for_completion,
74 )
75 from vllm.entrypoints.logger import RequestLogger
81 -
82 -# yapf conflicts with isort for this block
83 -# yapf: disable
76 from vllm.entrypoints.openai.protocol import (
77 DeltaMessage,
78 ErrorResponse,
@@ -97,8 +89,6 @@ from vllm.entrypoints.openai.protocol import (
89 ResponseUsage,
90 StreamingResponsesResponse,
91 )
100 -
101 -# yapf: enable
92 from vllm.entrypoints.openai.serving_engine import OpenAIServing
93 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
94 from vllm.entrypoints.tool_server import ToolServer
vllm/entrypoints/openai/serving_score.py
-5
@@ -24,9 +24,6 @@ from vllm.entrypoints.openai.protocol import (
24 )
25 from vllm.entrypoints.openai.serving_engine import OpenAIServing
26 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
27 -
28 -# yapf conflicts with isort for this block
29 -# yapf: disable
27 from vllm.entrypoints.score_utils import (
28 ScoreContentPartParam,
29 ScoreMultiModalParam,
@@ -35,8 +32,6 @@ from vllm.entrypoints.score_utils import (
32 compress_token_type_ids,
33 get_score_prompt,
34 )
38 -
39 -# yapf: enable
35 from vllm.entrypoints.utils import _validate_truncation_size
36 from vllm.inputs.data import TokensPrompt
37 from vllm.logger import init_logger
vllm/entrypoints/openai/serving_tokenization.py
-5
@@ -10,9 +10,6 @@ from vllm.config import ModelConfig
10 from vllm.engine.protocol import EngineClient
11 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
12 from vllm.entrypoints.logger import RequestLogger
13 -
14 -# yapf conflicts with isort for this block
15 -# yapf: disable
13 from vllm.entrypoints.openai.protocol import (
14 DetokenizeRequest,
15 DetokenizeResponse,
@@ -22,8 +19,6 @@ from vllm.entrypoints.openai.protocol import (
19 TokenizeResponse,
20 TokenizerInfoResponse,
21 )
25 -
26 -# yapf: enable
22 from vllm.entrypoints.openai.serving_engine import OpenAIServing
23 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
24 from vllm.entrypoints.renderer import RenderConfig
vllm/executor/ray_distributed_executor.py
+1 -1
@@ -11,7 +11,7 @@ import cloudpickle
11 import msgspec
12
13 import vllm.envs as envs
14 -from vllm.executor.executor_base import DistributedExecutorBase # yapf: disable
14 +from vllm.executor.executor_base import DistributedExecutorBase
15 from vllm.executor.msgspec_utils import encode_hook
16 from vllm.executor.ray_utils import RayWorkerWrapper, initialize_ray_cluster, ray
17 from vllm.logger import init_logger
vllm/lora/layers/base_linear.py
+50 -36
@@ -8,8 +8,6 @@ from transformers import PretrainedConfig
8
9 from vllm.config.lora import LoRAConfig
10 from vllm.distributed.utils import divide
11 -
12 -# yapf: disable
11 from vllm.model_executor.layers.linear import (
12 ColumnParallelLinear,
13 LinearBase,
@@ -23,7 +21,6 @@ from .utils import _get_lora_device
21
22
23 class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
26 -
24 def __init__(self, base_layer: LinearBase):
25 super().__init__()
26 self.base_layer = base_layer
@@ -50,16 +47,20 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
47 lora_b_out_size = self.output_size
48
49 elif isinstance(self.base_layer, ColumnParallelLinear):
53 - lora_a_out_size = (lora_config.max_lora_rank if
54 - not lora_config.fully_sharded_loras else divide(
55 - lora_config.max_lora_rank, self.tp_size))
50 + lora_a_out_size = (
51 + lora_config.max_lora_rank
52 + if not lora_config.fully_sharded_loras
53 + else divide(lora_config.max_lora_rank, self.tp_size)
54 + )
55 lora_b_out_size = self.output_size
56
57 elif isinstance(self.base_layer, RowParallelLinear):
58 lora_a_out_size = lora_config.max_lora_rank
60 - lora_b_out_size = (self.output_size if
61 - not lora_config.fully_sharded_loras else divide(
62 - self.output_size, self.tp_size))
59 + lora_b_out_size = (
60 + self.output_size
61 + if not lora_config.fully_sharded_loras
62 + else divide(self.output_size, self.tp_size)
63 + )
64 else:
65 raise NotImplementedError
66
@@ -71,7 +72,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
72 self.input_size,
73 dtype=lora_config.lora_dtype,
74 device=self.device,
74 - ) for _ in range(self.n_slices))
75 + )
76 + for _ in range(self.n_slices)
77 + )
78 self.lora_b_stacked = tuple(
79 torch.zeros(
80 max_loras,
@@ -80,7 +83,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
83 lora_config.max_lora_rank,
84 dtype=lora_config.lora_dtype,
85 device=self.device,
83 - ) for _ in range(self.n_slices))
86 + )
87 + for _ in range(self.n_slices)
88 + )
89 if lora_config.bias_enabled:
90 lora_bias_out_size = lora_b_out_size
91 self.lora_bias_stacked = tuple(
@@ -90,8 +95,10 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
95 lora_bias_out_size,
96 dtype=lora_config.lora_dtype,
97 device=self.device,
93 - ) for _ in range(self.n_slices))
94 - self.output_slices = (self.lora_b_stacked[0].shape[2], )
98 + )
99 + for _ in range(self.n_slices)
100 + )
101 + self.output_slices = (self.lora_b_stacked[0].shape[2],)
102
103 def reset_lora(self, index: int):
104 for s_index in range(self.n_slices):
@@ -99,8 +106,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
106 self.lora_b_stacked[s_index][index] = 0
107 if self.lora_config.bias_enabled:
108 # Make mypy happy
102 - self.lora_bias_stacked = cast(tuple[torch.Tensor, ...],
103 - self.lora_bias_stacked)
109 + self.lora_bias_stacked = cast(
110 + tuple[torch.Tensor, ...], self.lora_bias_stacked
111 + )
112 self.lora_bias_stacked[s_index][index] = 0
113
114 def set_lora(
@@ -115,8 +123,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
123 # MergedColumnParallelLinearWithLoRA, all other linear LoRA layers
124 # store weights in a tuple of size 1. These two layers will
125 # override this function.
118 - assert (len(self.lora_a_stacked) == len(self.lora_b_stacked) ==
119 - self.n_slices == 1)
126 + assert (
127 + len(self.lora_a_stacked) == len(self.lora_b_stacked) == self.n_slices == 1
128 + )
129
130 self.reset_lora(index)
131 if self.tp_size > 1:
@@ -125,23 +134,24 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
134 if lora_bias is not None:
135 lora_bias = self.slice_bias(lora_bias)
136
128 - self.lora_a_stacked[0][index,
129 - 0, :lora_a.shape[0], :lora_a.shape[1]].copy_(
130 - lora_a, non_blocking=True)
131 - self.lora_b_stacked[0][index,
132 - 0, :lora_b.shape[0], :lora_b.shape[1]].copy_(
133 - lora_b, non_blocking=True)
137 + self.lora_a_stacked[0][index, 0, : lora_a.shape[0], : lora_a.shape[1]].copy_(
138 + lora_a, non_blocking=True
139 + )
140 + self.lora_b_stacked[0][index, 0, : lora_b.shape[0], : lora_b.shape[1]].copy_(
141 + lora_b, non_blocking=True
142 + )
143 if lora_bias is not None:
135 -
136 - self.lora_bias_stacked = cast(tuple[torch.Tensor, ...],
137 - self.lora_bias_stacked)
144 + self.lora_bias_stacked = cast(
145 + tuple[torch.Tensor, ...], self.lora_bias_stacked
146 + )
147 assert len(self.lora_bias_stacked)
139 - self.lora_bias_stacked[0][index, 0, :lora_bias.shape[0]].copy_(
140 - lora_bias, non_blocking=True)
148 + self.lora_bias_stacked[0][index, 0, : lora_bias.shape[0]].copy_(
149 + lora_bias, non_blocking=True
150 + )
151
142 - def apply(self,
143 - x: torch.Tensor,
144 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
152 + def apply(
153 + self, x: torch.Tensor, bias: Optional[torch.Tensor] = None
154 + ) -> torch.Tensor:
155 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
156
157 # In transformers backend, x and output have extra batch dimension like
@@ -151,10 +161,15 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
161 output = output.flatten(0, 1)
162 x = x.flatten(0, 1)
163
154 - lora_output: Optional[
155 - torch.Tensor] = self.punica_wrapper.add_lora_linear(
156 - output, x, self.lora_a_stacked, self.lora_b_stacked,
157 - self.lora_bias_stacked, 1.0, self.output_slices)
164 + lora_output: Optional[torch.Tensor] = self.punica_wrapper.add_lora_linear(
165 + output,
166 + x,
167 + self.lora_a_stacked,
168 + self.lora_b_stacked,
169 + self.lora_bias_stacked,
170 + 1.0,
171 + self.output_slices,
172 + )
173 if not current_platform.can_update_inplace():
174 output = lora_output
175
@@ -162,7 +177,6 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
177
178 @property
179 def weight(self) -> torch.Tensor:
165 -
180 # unquantizedLinear
181 if hasattr(self.base_layer, "weight"):
182 return self.base_layer.weight
vllm/lora/layers/row_parallel_linear.py
+18 -19
@@ -12,8 +12,6 @@ from vllm.distributed import (
12 split_tensor_along_last_dim,
13 tensor_model_parallel_all_reduce,
14 )
15 -
16 -# yapf: disable
15 from vllm.model_executor.layers.linear import RowParallelLinear
16 from vllm.platforms import current_platform
17
@@ -22,7 +20,6 @@ from .utils import _fully_sharded_can_replace, _not_fully_sharded_can_replace
20
21
22 class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
25 -
23 def __init__(self, base_layer: RowParallelLinear) -> None:
24 super().__init__(base_layer)
25
@@ -33,11 +30,10 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
30 self.n_slices = 1
31
32 def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
36 -
33 shard_size = self.input_size
34 start_idx = self.tp_rank * shard_size
35 end_idx = (self.tp_rank + 1) * shard_size
40 - lora_a = lora_a[:,start_idx:end_idx]
36 + lora_a = lora_a[:, start_idx:end_idx]
37 return lora_a
38
39 def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
@@ -66,7 +62,8 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
62 else:
63 # TODO: simplify code below
64 splitted_input = split_tensor_along_last_dim(
69 - input_, num_partitions=self.tp_size)
65 + input_, num_partitions=self.tp_size
66 + )
67 input_parallel = splitted_input[self.tp_rank].contiguous()
68
69 # Matrix multiply.
@@ -77,8 +74,11 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
74 output_ = output_parallel
75
76 if not self.base_layer.skip_bias_add:
80 - output = (output_ + self.base_layer.bias
81 - if self.base_layer.bias is not None else output_)
77 + output = (
78 + output_ + self.base_layer.bias
79 + if self.base_layer.bias is not None
80 + else output_
81 + )
82 output_bias = None
83 else:
84 output = output_
@@ -101,11 +101,11 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
101 return type(source_layer) is RowParallelLinear
102
103
104 -
104 # The following layer is based on the tensor parallelism strategy given in
105 # Y. Sheng et al., S-LoRA: Serving Thousands of Concurrent LoRA Adapters. 2023,
106 # https://arxiv.org/abs/2311.03285.
107
108 +
109 class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
110 """
111 Differs from RowParallelLinearWithLoRA by slicing the
@@ -120,28 +120,26 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
120 shard_size = self.lora_b_stacked[0].shape[2]
121 start_idx = self.tp_rank * shard_size
122 end_idx = (self.tp_rank + 1) * shard_size
123 - lora_b = lora_b[ start_idx:end_idx,:]
123 + lora_b = lora_b[start_idx:end_idx, :]
124 return lora_b
125
126 def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
127 if bias is None:
128 return bias
129 - self.lora_bias_stacked = cast(tuple[torch.Tensor, ...],
130 - self.lora_bias_stacked)
129 + self.lora_bias_stacked = cast(tuple[torch.Tensor, ...], self.lora_bias_stacked)
130 shard_size = self.lora_bias_stacked[0].shape[2]
131 start_idx = self.tp_rank * shard_size
132 end_idx = (self.tp_rank + 1) * shard_size
133 bias = bias[start_idx:end_idx]
134 return bias
135
137 - def apply(self,
138 - x: torch.Tensor,
139 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
136 + def apply(
137 + self, x: torch.Tensor, bias: Optional[torch.Tensor] = None
138 + ) -> torch.Tensor:
139 output = self.base_layer.quant_method.apply(self.base_layer, x)
140
141 x = x.view(-1, x.shape[-1])
143 - output, out_orig_shape = output.view(-1,
144 - output.shape[-1]), output.shape
142 + output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
143 buffer = torch.zeros(
144 (self.n_slices, x.shape[0], self.lora_a_stacked[0].shape[2]),
145 dtype=torch.float32,
@@ -149,10 +147,11 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
147 )
148
149 shrunk_buffer: Optional[torch.Tensor] = self.punica_wrapper.add_shrink(
152 - buffer, x, self.lora_a_stacked, 1.0)
150 + buffer, x, self.lora_a_stacked, 1.0
151 + )
152 if not current_platform.can_update_inplace():
153 buffer = shrunk_buffer
155 - if self.tp_size>1:
154 + if self.tp_size > 1:
155 buffer = tensor_model_parallel_all_reduce(buffer)
156
157 # following S-LoRA, allows the fusing of all_gather and all_reduce
vllm/lora/utils.py
-4
@@ -19,8 +19,6 @@ from vllm.config.lora import LoRAConfig
19 from vllm.logger import init_logger
20
21 # being imported for _all_lora_classes below
22 -# yapf conflicts with isort for this block
23 -# yapf: disable
22 from vllm.lora.layers import (
23 BaseLayerWithLoRA,
24 ColumnParallelLinearWithLoRA,
@@ -39,8 +37,6 @@ from vllm.lora.layers import (
37 )
38 from vllm.model_executor.layers.linear import LinearBase
39
42 -# yapf: enable
43 -
40 if TYPE_CHECKING:
41 from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
vllm/model_executor/layers/fused_moe/fused_moe.py
-4
@@ -14,8 +14,6 @@ import vllm.envs as envs
14 import vllm.model_executor.layers.fused_moe.modular_kernel as mk
15 from vllm import _custom_ops as ops
16 from vllm.logger import init_logger
17 -
18 -# yapf: disable
17 from vllm.model_executor.layers.fused_moe.config import (
18 FUSED_MOE_UNQUANTIZED_CONFIG,
19 FusedMoEQuantConfig,
@@ -25,8 +23,6 @@ from vllm.model_executor.layers.fused_moe.cutlass_moe import (
23 _valid_cutlass_block_scaled_grouped_gemm,
24 run_cutlass_block_scaled_fused_experts,
25 )
28 -
29 -# yapf: enable
26 from vllm.model_executor.layers.fused_moe.deep_gemm_moe import (
27 _valid_deep_gemm,
28 deep_gemm_moe_fp8,
vllm/model_executor/layers/fused_moe/layer.py
-4
@@ -24,8 +24,6 @@ from vllm.distributed.eplb.eplb_state import EplbState
24 from vllm.forward_context import ForwardContext, get_forward_context
25 from vllm.logger import init_logger
26 from vllm.model_executor.custom_op import CustomOp
27 -
28 -# yapf: disable
27 from vllm.model_executor.layers.fused_moe.config import (
28 FUSED_MOE_UNQUANTIZED_CONFIG,
29 FusedMoEConfig,
@@ -34,8 +32,6 @@ from vllm.model_executor.layers.fused_moe.config import (
32 biased_moe_quant_config,
33 )
34 from vllm.model_executor.layers.fused_moe.fused_moe import zero_experts_compute_triton
37 -
38 -# yapf: enable
35 from vllm.model_executor.layers.fused_moe.modular_kernel import (
36 FusedMoEActivationFormat,
37 FusedMoEModularKernel,
vllm/model_executor/layers/fused_moe/modular_kernel.py
+1 -1
@@ -10,7 +10,7 @@ import torch
10
11 import vllm.envs as envs
12 from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
13 -from vllm.model_executor.layers.fused_moe.utils import ( # yapf: disable
13 +from vllm.model_executor.layers.fused_moe.utils import (
14 _resize_cache,
15 count_expert_num_tokens,
16 )
vllm/model_executor/layers/linear.py
-4
@@ -24,8 +24,6 @@ from vllm.model_executor.layers.quantization.base_config import (
24 QuantizeMethodBase,
25 )
26 from vllm.model_executor.layers.utils import dispatch_unquantized_gemm
27 -
28 -# yapf: disable
27 from vllm.model_executor.parameter import (
28 BasevLLMParameter,
29 BlockQuantScaleParameter,
@@ -35,8 +33,6 @@ from vllm.model_executor.parameter import (
33 PerTensorScaleParameter,
34 RowvLLMParameter,
35 )
38 -
39 -# yapf: enable
36 from vllm.model_executor.utils import set_weight_attrs
37 from vllm.platforms import current_platform
38 from vllm.utils import GiB_bytes
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py
-5
@@ -17,17 +17,12 @@ from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
17 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
18 marlin_repeat_scales_on_all_ranks,
19 )
20 -
21 -# yapf conflicts with isort for this block
22 -# yapf: disable
20 from vllm.model_executor.parameter import (
21 BasevLLMParameter,
22 ChannelQuantScaleParameter,
23 GroupQuantScaleParameter,
24 PackedvLLMParameter,
25 )
29 -
30 -# yapf: enable
26 from vllm.scalar_type import scalar_types
27
28 logger = init_logger(__name__)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
-5
@@ -17,9 +17,6 @@ from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
17 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
18 marlin_repeat_scales_on_all_ranks,
19 )
20 -
21 -# yapf conflicts with isort for this block
22 -# yapf: disable
20 from vllm.model_executor.parameter import (
21 BasevLLMParameter,
22 ChannelQuantScaleParameter,
@@ -28,8 +25,6 @@ from vllm.model_executor.parameter import (
25 PackedvLLMParameter,
26 RowvLLMParameter,
27 )
31 -
32 -# yapf: enable
28 from vllm.scalar_type import scalar_types
29
30 logger = init_logger(__name__)
vllm/model_executor/model_loader/bitsandbytes_loader.py
-4
@@ -22,8 +22,6 @@ from vllm.distributed import (
22 get_tensor_model_parallel_rank,
23 get_tensor_model_parallel_world_size,
24 )
25 -
26 -# yapf: enable
25 from vllm.logger import init_logger
26 from vllm.model_executor.layers.fused_moe import FusedMoE
27 from vllm.model_executor.layers.linear import (
@@ -51,8 +49,6 @@ from vllm.model_executor.utils import (
49 )
50 from vllm.platforms import current_platform
51
54 -# yapf conflicts with isort for this block
55 -
52 logger = init_logger(__name__)
53
54
vllm/model_executor/models/aria.py
-3
@@ -39,13 +39,10 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
39 from vllm.sequence import IntermediateTensors
40 from vllm.utils.tensor_schema import TensorSchema, TensorShape
41
42 -# yapf: disable
42 from .idefics2_vision_model import Idefics2VisionConfig
43 from .idefics2_vision_model import (
44 Idefics2VisionTransformer as Idefics3VisionTransformer,
45 )
47 -
48 -# yapf: enable
46 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
47 from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel
48 from .utils import (
vllm/model_executor/models/gemma3_mm.py
-4
@@ -22,8 +22,6 @@ from vllm.multimodal.inputs import (
22 MultiModalKwargsItems,
23 )
24 from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
25 -
26 -# yapf: disable
25 from vllm.multimodal.processing import (
26 BaseMultiModalProcessor,
27 BaseProcessingInfo,
@@ -35,8 +33,6 @@ from vllm.multimodal.processing import (
33 PromptUpdateDetails,
34 replace_token_matches,
35 )
38 -
39 -# yapf: enable
36 from vllm.multimodal.profiling import BaseDummyInputsBuilder
37 from vllm.sequence import IntermediateTensors
38 from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/gemma3n_mm.py
+30 -21
@@ -6,14 +6,16 @@ from typing import Annotated, Any, Literal, Optional, Union, cast
6 import numpy as np
7 import torch
8
9 -# yapf: disable
9 from torch import nn
10 from transformers import AutoModel, BatchFeature
12 -from transformers.models.gemma3n import (Gemma3nAudioConfig,
13 - Gemma3nAudioFeatureExtractor,
14 - Gemma3nConfig, Gemma3nProcessor,
15 - Gemma3nTextConfig,
16 - Gemma3nVisionConfig)
11 +from transformers.models.gemma3n import (
12 + Gemma3nAudioConfig,
13 + Gemma3nAudioFeatureExtractor,
14 + Gemma3nConfig,
15 + Gemma3nProcessor,
16 + Gemma3nTextConfig,
17 + Gemma3nVisionConfig,
18 +)
19 from transformers.models.siglip import SiglipImageProcessorFast
20
21 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
@@ -22,25 +24,32 @@ from vllm.inputs.data import PromptType
24 from vllm.logger import init_logger
25 from vllm.model_executor.layers.layernorm import RMSNorm
26 from vllm.model_executor.layers.linear import RowParallelLinear
25 -from vllm.model_executor.layers.vocab_parallel_embedding import (
26 - VocabParallelEmbedding)
27 +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
28 from vllm.model_executor.models.gemma3n import Gemma3nForCausalLM
29 from vllm.model_executor.models.module_mapping import MultiModelKeys
30 from vllm.model_executor.models.whisper import ISO639_1_SUPPORTED_LANGS
31 from vllm.multimodal import MULTIMODAL_REGISTRY
31 -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
32 - MultiModalKwargsItems)
33 -from vllm.multimodal.parse import (ImageProcessorItems, MultiModalDataItems,
34 - MultiModalDataParser)
35 -from vllm.multimodal.processing import (BaseMultiModalProcessor,
36 - BaseProcessingInfo,
37 - MultiModalPromptUpdates,
38 - MultiModalPromptUpdatesApplyResult,
39 - PlaceholderFeaturesInfo,
40 - PromptReplacement, PromptUpdate,
41 - PromptUpdateDetails,
42 - replace_token_matches)
43 -# yapf: enable
32 +from vllm.multimodal.inputs import (
33 + MultiModalDataDict,
34 + MultiModalFieldConfig,
35 + MultiModalKwargsItems,
36 +)
37 +from vllm.multimodal.parse import (
38 + ImageProcessorItems,
39 + MultiModalDataItems,
40 + MultiModalDataParser,
41 +)
42 +from vllm.multimodal.processing import (
43 + BaseMultiModalProcessor,
44 + BaseProcessingInfo,
45 + MultiModalPromptUpdates,
46 + MultiModalPromptUpdatesApplyResult,
47 + PlaceholderFeaturesInfo,
48 + PromptReplacement,
49 + PromptUpdate,
50 + PromptUpdateDetails,
51 + replace_token_matches,
52 +)
53 from vllm.multimodal.profiling import BaseDummyInputsBuilder
54 from vllm.sequence import IntermediateTensors
55 from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/idefics3.py
-8
@@ -43,9 +43,6 @@ from vllm.multimodal.inputs import (
43 MultiModalKwargsItems,
44 )
45 from vllm.multimodal.parse import ImageProcessorItems, ImageSize
46 -
47 -# yapf conflicts with isort for this block
48 -# yapf: disable
46 from vllm.multimodal.processing import (
47 BaseMultiModalProcessor,
48 BaseProcessingInfo,
@@ -54,18 +51,13 @@ from vllm.multimodal.processing import (
51 PromptUpdate,
52 PromptUpdateDetails,
53 )
57 -
58 -# yapf: enable
54 from vllm.multimodal.profiling import BaseDummyInputsBuilder
55 from vllm.sequence import IntermediateTensors
56 from vllm.utils.tensor_schema import TensorSchema, TensorShape
57
63 -# yapf: disable
58 from .idefics2_vision_model import (
59 Idefics2VisionTransformer as Idefics3VisionTransformer,
60 )
67 -
68 -# yapf: enable
61 from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal
62 from .llama import LlamaModel
63 from .utils import AutoWeightsLoader, maybe_prefix
vllm/model_executor/models/phi3v.py
-5
@@ -45,9 +45,6 @@ from vllm.multimodal.parse import (
45 ImageSize,
46 MultiModalDataItems,
47 )
48 -
49 -# yapf conflicts with isort for this block
50 -# yapf: disable
48 from vllm.multimodal.processing import (
49 BaseMultiModalProcessor,
50 BaseProcessingInfo,
@@ -57,8 +54,6 @@ from vllm.multimodal.processing import (
54 PromptUpdate,
55 ResolvedPromptUpdate,
56 )
60 -
61 -# yapf: enable
57 from vllm.multimodal.profiling import BaseDummyInputsBuilder
58 from vllm.sequence import IntermediateTensors
59 from vllm.utils import is_list_of
vllm/model_executor/models/qwen2_5_vl.py
-4
@@ -52,16 +52,12 @@ from vllm.distributed import utils as dist_utils
52 from vllm.logger import init_logger
53 from vllm.model_executor.layers.activation import get_act_and_mul_fn
54 from vllm.model_executor.layers.layernorm import RMSNorm
55 -
56 -# yapf: disable
55 from vllm.model_executor.layers.linear import (
56 ColumnParallelLinear,
57 MergedColumnParallelLinear,
58 QKVParallelLinear,
59 RowParallelLinear,
60 )
63 -
64 -# yapf: enable
61 from vllm.model_executor.layers.quantization import QuantizationConfig
62 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
63 from vllm.model_executor.models.module_mapping import MultiModelKeys
vllm/model_executor/models/qwen3_next.py
-5
@@ -37,12 +37,7 @@ from vllm.model_executor.layers.fla.ops import (
37 fused_recurrent_gated_delta_rule,
38 )
39 from vllm.model_executor.layers.fused_moe import FusedMoE
40 -
41 -# yapf conflicts with isort for this block
42 -# yapf: disable
40 from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3NextRMSNorm
44 -
45 -# yapf: enable
41 from vllm.model_executor.layers.linear import (
42 ColumnParallelLinear,
43 QKVParallelLinear,
vllm/model_executor/models/registry.py
+132 -44
@@ -54,7 +54,6 @@ from .interfaces_base import (
54
55 logger = init_logger(__name__)
56
57 -# yapf: disable
57 _TEXT_GENERATION_MODELS = {
58 # [Decoder-only]
59 "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
@@ -106,8 +105,8 @@ _TEXT_GENERATION_MODELS = {
105 "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
106 "GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
107 "GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
109 - "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"), # noqa: E501
110 - "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"), # noqa: E501
108 + "GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"), # noqa: E501
109 + "GraniteMoeSharedForCausalLM": ("granitemoeshared", "GraniteMoeSharedForCausalLM"), # noqa: E501
110 "GritLM": ("gritlm", "GritLM"),
111 "Grok1ModelForCausalLM": ("grok1", "Grok1ForCausalLM"),
112 "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
@@ -127,7 +126,7 @@ _TEXT_GENERATION_MODELS = {
126 "LongcatFlashForCausalLM": ("longcat_flash", "LongcatFlashForCausalLM"),
127 "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
128 "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
130 - "FalconH1ForCausalLM":("falcon_h1", "FalconH1ForCausalLM"),
129 + "FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
130 "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
131 "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
132 "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
@@ -184,7 +183,8 @@ _EMBEDDING_MODELS = {
183 "LlamaModel": ("llama", "LlamaForCausalLM"),
184 **{
185 # Multiple models share the same architecture, so we include them all
187 - k: (mod, arch) for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
186 + k: (mod, arch)
187 + for k, (mod, arch) in _TEXT_GENERATION_MODELS.items()
188 if arch == "LlamaForCausalLM"
189 },
190 "MistralModel": ("llama", "LlamaForCausalLM"),
@@ -201,7 +201,10 @@ _EMBEDDING_MODELS = {
201 "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
202 # [Multimodal]
203 "CLIPModel": ("clip", "CLIPEmbeddingModel"),
204 - "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
204 + "LlavaNextForConditionalGeneration": (
205 + "llava_next",
206 + "LlavaNextForConditionalGeneration",
207 + ), # noqa: E501
208 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
209 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
210 # Technically Terratorch models work on images, both in
@@ -214,79 +217,150 @@ _EMBEDDING_MODELS = {
217 _CROSS_ENCODER_MODELS = {
218 "BertForSequenceClassification": ("bert", "BertForSequenceClassification"),
219 "BertForTokenClassification": ("bert", "BertForTokenClassification"),
217 - "GteNewForSequenceClassification": ("bert_with_rope",
218 - "GteNewForSequenceClassification"),
219 - "ModernBertForSequenceClassification": ("modernbert",
220 - "ModernBertForSequenceClassification"),
221 - "RobertaForSequenceClassification": ("roberta",
222 - "RobertaForSequenceClassification"),
223 - "XLMRobertaForSequenceClassification": ("roberta",
224 - "RobertaForSequenceClassification"),
220 + "GteNewForSequenceClassification": (
221 + "bert_with_rope",
222 + "GteNewForSequenceClassification",
223 + ),
224 + "ModernBertForSequenceClassification": (
225 + "modernbert",
226 + "ModernBertForSequenceClassification",
227 + ),
228 + "RobertaForSequenceClassification": ("roberta", "RobertaForSequenceClassification"),
229 + "XLMRobertaForSequenceClassification": (
230 + "roberta",
231 + "RobertaForSequenceClassification",
232 + ),
233 # [Auto-converted (see adapters.py)]
226 - "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"), # noqa: E501,
234 + "JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"), # noqa: E501,
235 }
236
237 _MULTIMODAL_MODELS = {
238 # [Decoder-only]
239 "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
232 - "AyaVisionForConditionalGeneration": ("aya_vision", "AyaVisionForConditionalGeneration"), # noqa: E501
240 + "AyaVisionForConditionalGeneration": (
241 + "aya_vision",
242 + "AyaVisionForConditionalGeneration",
243 + ), # noqa: E501
244 "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
234 - "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
235 - "Cohere2VisionForConditionalGeneration": ("cohere2_vision", "Cohere2VisionForConditionalGeneration"), # noqa: E501
245 + "ChameleonForConditionalGeneration": (
246 + "chameleon",
247 + "ChameleonForConditionalGeneration",
248 + ), # noqa: E501
249 + "Cohere2VisionForConditionalGeneration": (
250 + "cohere2_vision",
251 + "Cohere2VisionForConditionalGeneration",
252 + ), # noqa: E501
253 "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
254 "DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
238 - "Ernie4_5_VLMoeForConditionalGeneration": ("ernie45_vl", "Ernie4_5_VLMoeForConditionalGeneration"), # noqa: E501
255 + "Ernie4_5_VLMoeForConditionalGeneration": (
256 + "ernie45_vl",
257 + "Ernie4_5_VLMoeForConditionalGeneration",
258 + ), # noqa: E501
259 "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
260 "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"), # noqa: E501
241 - "Gemma3nForConditionalGeneration": ("gemma3n_mm", "Gemma3nForConditionalGeneration"), # noqa: E501
261 + "Gemma3nForConditionalGeneration": (
262 + "gemma3n_mm",
263 + "Gemma3nForConditionalGeneration",
264 + ), # noqa: E501
265 "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
266 "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"), # noqa: E501
267 "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"), # noqa: E501
245 - "GraniteSpeechForConditionalGeneration": ("granite_speech", "GraniteSpeechForConditionalGeneration"), # noqa: E501
268 + "GraniteSpeechForConditionalGeneration": (
269 + "granite_speech",
270 + "GraniteSpeechForConditionalGeneration",
271 + ), # noqa: E501
272 "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
273 "InternVLChatModel": ("internvl", "InternVLChatModel"),
274 "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
249 - "InternS1ForConditionalGeneration": ("interns1", "InternS1ForConditionalGeneration"), # noqa: E501
250 - "InternVLForConditionalGeneration": ("interns1", "InternS1ForConditionalGeneration"), # noqa: E501
251 - "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
252 - "SmolVLMForConditionalGeneration": ("smolvlm","SmolVLMForConditionalGeneration"), # noqa: E501
275 + "InternS1ForConditionalGeneration": (
276 + "interns1",
277 + "InternS1ForConditionalGeneration",
278 + ), # noqa: E501
279 + "InternVLForConditionalGeneration": (
280 + "interns1",
281 + "InternS1ForConditionalGeneration",
282 + ), # noqa: E501
283 + "Idefics3ForConditionalGeneration": (
284 + "idefics3",
285 + "Idefics3ForConditionalGeneration",
286 + ),
287 + "SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"), # noqa: E501
288 "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
254 - "KeyeVL1_5ForConditionalGeneration": ("keye_vl1_5", "KeyeVL1_5ForConditionalGeneration"), # noqa: E501
289 + "KeyeVL1_5ForConditionalGeneration": (
290 + "keye_vl1_5",
291 + "KeyeVL1_5ForConditionalGeneration",
292 + ), # noqa: E501
293 "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
294 "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"), # noqa: E501
295 "Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
296 "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"), # noqa: E501
297 "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
260 - "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
261 - "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
262 - "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"), # noqa: E501
298 + "LlavaNextForConditionalGeneration": (
299 + "llava_next",
300 + "LlavaNextForConditionalGeneration",
301 + ), # noqa: E501
302 + "LlavaNextVideoForConditionalGeneration": (
303 + "llava_next_video",
304 + "LlavaNextVideoForConditionalGeneration",
305 + ), # noqa: E501
306 + "LlavaOnevisionForConditionalGeneration": (
307 + "llava_onevision",
308 + "LlavaOnevisionForConditionalGeneration",
309 + ), # noqa: E501
310 "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"), # noqa: E501
311 "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
265 - "MiniMaxVL01ForConditionalGeneration": ("minimax_vl_01", "MiniMaxVL01ForConditionalGeneration"), # noqa: E501
312 + "MiniMaxVL01ForConditionalGeneration": (
313 + "minimax_vl_01",
314 + "MiniMaxVL01ForConditionalGeneration",
315 + ), # noqa: E501
316 "MiniCPMO": ("minicpmo", "MiniCPMO"),
317 "MiniCPMV": ("minicpmv", "MiniCPMV"),
268 - "Mistral3ForConditionalGeneration": ("mistral3", "Mistral3ForConditionalGeneration"), # noqa: E501
318 + "Mistral3ForConditionalGeneration": (
319 + "mistral3",
320 + "Mistral3ForConditionalGeneration",
321 + ), # noqa: E501
322 "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
323 "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
324 "Ovis": ("ovis", "Ovis"),
325 "Ovis2_5": ("ovis2_5", "Ovis2_5"),
273 - "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501
326 + "PaliGemmaForConditionalGeneration": (
327 + "paligemma",
328 + "PaliGemmaForConditionalGeneration",
329 + ), # noqa: E501
330 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
331 "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
332 "Phi4MultimodalForCausalLM": ("phi4_multimodal", "Phi4MultimodalForCausalLM"), # noqa: E501
333 "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
334 "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"), # noqa: E501
335 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
280 - "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"), # noqa: E501
281 - "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"), # noqa: E501
282 - "Qwen2_5OmniModel": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"), # noqa: E501
283 - "Qwen2_5OmniForConditionalGeneration": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"), # noqa: E501
336 + "Qwen2_5_VLForConditionalGeneration": (
337 + "qwen2_5_vl",
338 + "Qwen2_5_VLForConditionalGeneration",
339 + ), # noqa: E501
340 + "Qwen2AudioForConditionalGeneration": (
341 + "qwen2_audio",
342 + "Qwen2AudioForConditionalGeneration",
343 + ), # noqa: E501
344 + "Qwen2_5OmniModel": (
345 + "qwen2_5_omni_thinker",
346 + "Qwen2_5OmniThinkerForConditionalGeneration",
347 + ), # noqa: E501
348 + "Qwen2_5OmniForConditionalGeneration": (
349 + "qwen2_5_omni_thinker",
350 + "Qwen2_5OmniThinkerForConditionalGeneration",
351 + ), # noqa: E501
352 "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"), # noqa: E501
285 - "Qwen3VLMoeForConditionalGeneration": ("qwen3_vl_moe", "Qwen3VLMoeForConditionalGeneration"), # noqa: E501
353 + "Qwen3VLMoeForConditionalGeneration": (
354 + "qwen3_vl_moe",
355 + "Qwen3VLMoeForConditionalGeneration",
356 + ), # noqa: E501
357 "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
358 "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"), # noqa: E501
359 "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"), # noqa: E501
289 - "Tarsier2ForConditionalGeneration": ("qwen2_vl", "Tarsier2ForConditionalGeneration"), # noqa: E501
360 + "Tarsier2ForConditionalGeneration": (
361 + "qwen2_vl",
362 + "Tarsier2ForConditionalGeneration",
363 + ), # noqa: E501
364 "UltravoxModel": ("ultravox", "UltravoxModel"),
365 "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"), # noqa: E501
366 # [Encoder-decoder]
@@ -324,13 +398,27 @@ _TRANSFORMERS_BACKEND_MODELS = {
398 "TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
399 "TransformersForMultimodalLM": ("transformers", "TransformersForMultimodalLM"), # noqa: E501
400 "TransformersMoEForCausalLM": ("transformers_moe", "TransformersMoEForCausalLM"), # noqa: E501
327 - "TransformersMoEForMultimodalLM": ("transformers_moe", "TransformersMoEForMultimodalLM"), # noqa: E501
328 - "TransformersEmbeddingModel": ("transformers_pooling", "TransformersEmbeddingModel"), # noqa: E501
329 - "TransformersForSequenceClassification": ("transformers_pooling", "TransformersForSequenceClassification"), # noqa: E501
330 - "TransformersMoEForSequenceClassification": ("transformers_pooling", "TransformersMoEForSequenceClassification"), # noqa: E501
331 - "TransformersMoEEmbeddingModel": ("transformers_pooling", "TransformersMoEEmbeddingModel"), # noqa: E501
401 + "TransformersMoEForMultimodalLM": (
402 + "transformers_moe",
403 + "TransformersMoEForMultimodalLM",
404 + ), # noqa: E501
405 + "TransformersEmbeddingModel": (
406 + "transformers_pooling",
407 + "TransformersEmbeddingModel",
408 + ), # noqa: E501
409 + "TransformersForSequenceClassification": (
410 + "transformers_pooling",
411 + "TransformersForSequenceClassification",
412 + ), # noqa: E501
413 + "TransformersMoEForSequenceClassification": (
414 + "transformers_pooling",
415 + "TransformersMoEForSequenceClassification",
416 + ), # noqa: E501
417 + "TransformersMoEEmbeddingModel": (
418 + "transformers_pooling",
419 + "TransformersMoEEmbeddingModel",
420 + ), # noqa: E501
421 }
333 -# yapf: enable
422
423 _VLLM_MODELS = {
424 **_TEXT_GENERATION_MODELS,
vllm/model_executor/models/smolvlm.py
-3
@@ -8,13 +8,10 @@ from transformers import SmolVLMProcessor
8 from vllm.config import VllmConfig
9 from vllm.multimodal import MULTIMODAL_REGISTRY
10
11 -# yapf: disable
11 from .idefics3 import Idefics3DummyInputsBuilder as SmolVLMDummyInputsBuilder
12 from .idefics3 import Idefics3ForConditionalGeneration, Idefics3ProcessingInfo
13 from .idefics3 import Idefics3MultiModalProcessor as SmolVLMMultiModalProcessor
14
16 -# yapf: enable
17 -
15
16 class SmolVLMProcessingInfo(Idefics3ProcessingInfo):
17 def get_hf_processor(self, **kwargs: object) -> SmolVLMProcessor:
vllm/model_executor/models/voxtral.py
-4
@@ -32,11 +32,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
32 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
33 from vllm.model_executor.models import SupportsPP
34 from vllm.model_executor.models.module_mapping import MultiModelKeys
35 -
36 -# yapf: disable
35 from vllm.model_executor.models.whisper import WhisperEncoder
38 -
39 -# yapf: enable
36 from vllm.multimodal import MULTIMODAL_REGISTRY
37 from vllm.multimodal.inputs import (
38 MultiModalDataDict,
vllm/transformers_utils/chat_templates/registry.py
-2
@@ -28,7 +28,6 @@ def _get_minicpmv_chat_template_fallback(tokenizer_name_or_path: str) -> Optiona
28 return CHAT_TEMPLATES_DIR / "template_chatml.jinja"
29
30
31 -# yapf: disable
31 _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
32 "blip-2": CHAT_TEMPLATES_DIR / "template_blip2.jinja",
33 "clip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
@@ -39,7 +38,6 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
38 "paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
39 "qwen": _get_qwen_chat_template_fallback,
40 }
42 -# yapf: enable
41
42
43 def register_chat_template_fallback_path(
vllm/transformers_utils/configs/arctic.py
+1 -2
@@ -1,12 +1,11 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -# yapf: disable
4 # ruff: noqa: E501
5 # coding=utf-8
6 # Copied from
7 # https://huggingface.co/Snowflake/snowflake-arctic-instruct/blob/main/configuration_arctic.py
9 -""" Arctic model configuration"""
8 +"""Arctic model configuration"""
9
10 from dataclasses import asdict, dataclass
11 from typing import Any
vllm/transformers_utils/configs/nemotron_vl.py
+11 -7
@@ -1,7 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -# yapf: disable
4 # ruff: noqa: E501
5 # Adapted from
6 # https://huggingface.co/nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1/blob/main/configuration.py
@@ -16,7 +15,7 @@ from transformers.dynamic_module_utils import get_class_from_dynamic_module
15
16
17 class Nemotron_Nano_VL_Config(PretrainedConfig):
19 - model_type = 'Llama_Nemotron_Nano_VL'
18 + model_type = "Llama_Nemotron_Nano_VL"
19 is_composition = True
20
21 def __init__(
@@ -26,17 +25,22 @@ class Nemotron_Nano_VL_Config(PretrainedConfig):
25 force_image_size=None,
26 downsample_ratio=0.5,
27 template=None,
29 - ps_version='v1',
28 + ps_version="v1",
29 image_tag_type="internvl",
30 projector_hidden_size=4096,
31 vit_hidden_size=1280,
33 - **kwargs
32 + **kwargs,
33 ):
34 super().__init__(**kwargs)
35
36 if vision_config is not None:
38 - assert "auto_map" in vision_config and "AutoConfig" in vision_config["auto_map"]
39 - vision_auto_config = get_class_from_dynamic_module(*vision_config["auto_map"]["AutoConfig"].split("--")[::-1])
37 + assert (
38 + "auto_map" in vision_config
39 + and "AutoConfig" in vision_config["auto_map"]
40 + )
41 + vision_auto_config = get_class_from_dynamic_module(
42 + *vision_config["auto_map"]["AutoConfig"].split("--")[::-1]
43 + )
44 self.vision_config = vision_auto_config(**vision_config)
45 else:
46 self.vision_config = PretrainedConfig()
@@ -51,6 +55,6 @@ class Nemotron_Nano_VL_Config(PretrainedConfig):
55 self.downsample_ratio = downsample_ratio
56 self.template = template # TODO move out of here and into the tokenizer
57 self.ps_version = ps_version # Pixel shuffle version
54 - self.image_tag_type = image_tag_type # TODO: into the tokenizer too?
58 + self.image_tag_type = image_tag_type # TODO: into the tokenizer too?
59 self.projector_hidden_size = projector_hidden_size
60 self.vit_hidden_size = vit_hidden_size
vllm/transformers_utils/configs/ovis.py
+42 -36
@@ -1,7 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -# yapf: disable
4 # ruff: noqa: E501
5 # adapted from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_aimv2.py
6 # and https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py
@@ -70,34 +69,37 @@ class AIMv2Config(PretrainedConfig):
69 # Visual Tokenizer Configuration
70 # ----------------------------------------------------------------------
71 class BaseVisualTokenizerConfig(PretrainedConfig):
73 -
74 - def __init__(self,
75 - vocab_size=16384,
76 - tokenize_function="softmax",
77 - tau=1.0,
78 - depths=None,
79 - drop_cls_token=False,
80 - backbone_config: Optional[Union[PretrainedConfig,
81 - dict]] = None,
82 - hidden_stride: int = 1,
83 - **kwargs):
72 + def __init__(
73 + self,
74 + vocab_size=16384,
75 + tokenize_function="softmax",
76 + tau=1.0,
77 + depths=None,
78 + drop_cls_token=False,
79 + backbone_config: Optional[Union[PretrainedConfig, dict]] = None,
80 + hidden_stride: int = 1,
81 + **kwargs,
82 + ):
83 super().__init__(**kwargs)
84 self.vocab_size = vocab_size
85 self.tokenize_function = tokenize_function
86 self.tau = tau
87 if isinstance(depths, str):
89 - depths = [int(x) for x in depths.split('|')]
88 + depths = [int(x) for x in depths.split("|")]
89 self.depths = depths
90 self.backbone_kwargs = dict[str, Any]()
91 self.drop_cls_token = drop_cls_token
92 if backbone_config is not None:
94 - assert isinstance(backbone_config, (PretrainedConfig, dict)), \
93 + assert isinstance(backbone_config, (PretrainedConfig, dict)), (
94 f"expect `backbone_config` to be instance of PretrainedConfig or dict, but got {type(backbone_config)} type"
95 + )
96 if not isinstance(backbone_config, PretrainedConfig):
97 - model_type = backbone_config['model_type']
97 + model_type = backbone_config["model_type"]
98 if model_type != "aimv2":
99 - backbone_config.pop('model_type')
100 - backbone_config = AutoConfig.for_model(model_type, **backbone_config)
99 + backbone_config.pop("model_type")
100 + backbone_config = AutoConfig.for_model(
101 + model_type, **backbone_config
102 + )
103 else:
104 backbone_config = AIMv2Config(**backbone_config)
105 self.backbone_config = backbone_config
@@ -113,7 +115,7 @@ class Aimv2VisualTokenizerConfig(BaseVisualTokenizerConfig):
115 self.drop_cls_token = False
116 if self.depths:
117 assert len(self.depths) == 1
116 - self.backbone_kwargs['num_hidden_layers'] = self.depths[0]
118 + self.backbone_kwargs["num_hidden_layers"] = self.depths[0]
119
120
121 class SiglipVisualTokenizerConfig(BaseVisualTokenizerConfig):
@@ -125,7 +127,7 @@ class SiglipVisualTokenizerConfig(BaseVisualTokenizerConfig):
127 self.drop_cls_token = False
128 if self.depths:
129 assert len(self.depths) == 1
128 - self.backbone_kwargs['num_hidden_layers'] = self.depths[0]
130 + self.backbone_kwargs["num_hidden_layers"] = self.depths[0]
131
132
133 AutoConfig.register("siglip_visual_tokenizer", SiglipVisualTokenizerConfig)
@@ -138,35 +140,39 @@ AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig)
140 class OvisConfig(PretrainedConfig):
141 model_type = "ovis"
142
141 - def __init__(self,
142 - llm_config: Optional[Union[PretrainedConfig, dict]] = None,
143 - visual_tokenizer_config: Optional[Union[PretrainedConfig,
144 - dict]] = None,
145 - multimodal_max_length=8192,
146 - hidden_size=None,
147 - conversation_formatter_class=None,
148 - llm_attn_implementation=None,
149 - disable_tie_weight=False,
150 - **kwargs):
143 + def __init__(
144 + self,
145 + llm_config: Optional[Union[PretrainedConfig, dict]] = None,
146 + visual_tokenizer_config: Optional[Union[PretrainedConfig, dict]] = None,
147 + multimodal_max_length=8192,
148 + hidden_size=None,
149 + conversation_formatter_class=None,
150 + llm_attn_implementation=None,
151 + disable_tie_weight=False,
152 + **kwargs,
153 + ):
154 super().__init__(**kwargs)
155 if llm_config is not None:
153 - assert isinstance(llm_config, (PretrainedConfig, dict)), \
156 + assert isinstance(llm_config, (PretrainedConfig, dict)), (
157 f"expect `llm_config` to be instance of PretrainedConfig or dict, but got {type(llm_config)} type"
158 + )
159 if not isinstance(llm_config, PretrainedConfig):
156 - model_type = llm_config['model_type']
157 - llm_config.pop('model_type')
160 + model_type = llm_config["model_type"]
161 + llm_config.pop("model_type")
162 llm_config = AutoConfig.for_model(model_type, **llm_config)
163
164 # map llm_config to text_config
165 self.text_config = llm_config
166 if visual_tokenizer_config is not None:
163 - assert isinstance(visual_tokenizer_config, (PretrainedConfig, dict)), \
167 + assert isinstance(visual_tokenizer_config, (PretrainedConfig, dict)), (
168 f"expect `visual_tokenizer_config` to be instance of PretrainedConfig or dict, but got {type(visual_tokenizer_config)} type"
169 + )
170 if not isinstance(visual_tokenizer_config, PretrainedConfig):
166 - model_type = visual_tokenizer_config['model_type']
167 - visual_tokenizer_config.pop('model_type')
171 + model_type = visual_tokenizer_config["model_type"]
172 + visual_tokenizer_config.pop("model_type")
173 visual_tokenizer_config = AutoConfig.for_model(
169 - model_type, **visual_tokenizer_config)
174 + model_type, **visual_tokenizer_config
175 + )
176
177 self.visual_tokenizer_config = visual_tokenizer_config
178 self.multimodal_max_length = multimodal_max_length
vllm/transformers_utils/processors/deepseek_vl2.py
+81 -37
@@ -1,7 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -# yapf: disable
4 # ruff: noqa: E501
5 # coding=utf-8
6 # adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/ff23960c5cf9e6874b44be38af930cfb0ccbb620/deepseek_vl2/models/processing_deepseek_vl_v2.py
@@ -35,11 +34,12 @@ from transformers.processing_utils import ProcessorMixin
34
35
36 class ImageTransform:
38 -
39 - def __init__(self,
40 - mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
41 - std: tuple[float, float, float] = (0.5, 0.5, 0.5),
42 - normalize: bool = True):
37 + def __init__(
38 + self,
39 + mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
40 + std: tuple[float, float, float] = (0.5, 0.5, 0.5),
41 + normalize: bool = True,
42 + ):
43 self.mean = mean
44 self.std = std
45 self.normalize = normalize
@@ -77,7 +77,6 @@ class DeepseekVLV2Processor(ProcessorMixin):
77 ignore_id: int = -100,
78 **kwargs,
79 ):
80 -
80 self.candidate_resolutions = candidate_resolutions
81 self.image_size = candidate_resolutions[0][0]
82 self.patch_size = patch_size
@@ -86,13 +85,15 @@ class DeepseekVLV2Processor(ProcessorMixin):
85 self.normalize = normalize
86 self.downsample_ratio = downsample_ratio
87
89 - self.image_transform = ImageTransform(mean=image_mean, std=image_std, normalize=normalize)
88 + self.image_transform = ImageTransform(
89 + mean=image_mean, std=image_std, normalize=normalize
90 + )
91 self.tokenizer = tokenizer
91 - self.tokenizer.padding_side = 'left' # must set this，padding side with make a difference in batch inference
92 + self.tokenizer.padding_side = "left" # must set this，padding side with make a difference in batch inference
93
94 # add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id'
95 if tokenizer.pad_token is None:
95 - self.tokenizer.add_special_tokens({'pad_token': pad_token})
96 + self.tokenizer.add_special_tokens({"pad_token": pad_token})
97
98 # add image token
99 image_token_id = self.tokenizer.vocab.get(image_token)
@@ -104,7 +105,7 @@ class DeepseekVLV2Processor(ProcessorMixin):
105
106 # add five special tokens for grounding-related tasks
107 # <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|>
107 - special_tokens = ['<|ref|>', '<|/ref|>', '<|det|>', '<|/det|>', '<|grounding|>']
108 + special_tokens = ["<|ref|>", "<|/ref|>", "<|det|>", "<|/det|>", "<|grounding|>"]
109 special_tokens_dict = {"additional_special_tokens": special_tokens}
110 self.tokenizer.add_special_tokens(special_tokens_dict)
111
@@ -134,15 +135,19 @@ class DeepseekVLV2Processor(ProcessorMixin):
135
136 for width, height in self.candidate_resolutions:
137 scale = min(width / original_width, height / original_height)
137 - downscaled_width, downscaled_height = int(
138 - original_width * scale), int(original_height * scale)
139 - effective_resolution = min(downscaled_width * downscaled_height,
140 - original_width * original_height)
138 + downscaled_width, downscaled_height = (
139 + int(original_width * scale),
140 + int(original_height * scale),
141 + )
142 + effective_resolution = min(
143 + downscaled_width * downscaled_height, original_width * original_height
144 + )
145 wasted_resolution = (width * height) - effective_resolution
146
147 if effective_resolution > max_effective_resolution or (
144 - effective_resolution == max_effective_resolution
145 - and wasted_resolution < min_wasted_resolution):
148 + effective_resolution == max_effective_resolution
149 + and wasted_resolution < min_wasted_resolution
150 + ):
151 max_effective_resolution = effective_resolution
152 min_wasted_resolution = wasted_resolution
153 best_fit = (width, height)
@@ -198,12 +203,20 @@ class DeepseekVLV2Processor(ProcessorMixin):
203 - num_image_tokens (list[int]): the number of image tokens
204 """
205
201 - assert (prompt is not None and images is not None
202 - ), "prompt and images must be used at the same time."
206 + assert prompt is not None and images is not None, (
207 + "prompt and images must be used at the same time."
208 + )
209
210 sft_format = prompt
205 - tokenized_str, images_list, images_seq_mask, images_spatial_crop, num_image_tokens = self.tokenize_with_images(
206 - sft_format, images, bos=True, eos=True, cropping=len(images) <= 2)
211 + (
212 + tokenized_str,
213 + images_list,
214 + images_seq_mask,
215 + images_spatial_crop,
216 + num_image_tokens,
217 + ) = self.tokenize_with_images(
218 + sft_format, images, bos=True, eos=True, cropping=len(images) <= 2
219 + )
220 masked_tokenized_str = []
221 for token_index in tokenized_str:
222 if token_index != self.image_token_id:
@@ -211,17 +224,21 @@ class DeepseekVLV2Processor(ProcessorMixin):
224 else:
225 masked_tokenized_str.append(self.ignore_id)
226
214 - assert len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str), \
215 - (f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, "
216 - f"imags_seq_mask's length {len(images_seq_mask)}, are not equal")
227 + assert (
228 + len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str)
229 + ), (
230 + f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, "
231 + f"imags_seq_mask's length {len(images_seq_mask)}, are not equal"
232 + )
233
234 input_ids = torch.LongTensor(tokenized_str)
235 target_ids = torch.LongTensor(masked_tokenized_str)
236 images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)
237
238 # set input_ids < 0 | input_ids == self.image_token_id as ignore_id
223 - target_ids[(input_ids < 0) |
224 - (input_ids == self.image_token_id)] = self.ignore_id
239 + target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = (
240 + self.ignore_id
241 + )
242 input_ids[input_ids < 0] = self.pad_id
243
244 if inference_mode:
@@ -311,30 +328,50 @@ class DeepseekVLV2Processor(ProcessorMixin):
328 best_width, best_height = self.image_size, self.image_size
329
330 """process the global view"""
314 - global_view = ImageOps.pad(image, (self.image_size, self.image_size),
315 - color=tuple(int(x * 255) for x in self.image_transform.mean))
331 + global_view = ImageOps.pad(
332 + image,
333 + (self.image_size, self.image_size),
334 + color=tuple(int(x * 255) for x in self.image_transform.mean),
335 + )
336 images_list.append(self.image_transform(global_view))
337
338 """process the local views"""
319 - local_view = ImageOps.pad(image, (best_width, best_height),
320 - color=tuple(int(x * 255) for x in self.image_transform.mean))
339 + local_view = ImageOps.pad(
340 + image,
341 + (best_width, best_height),
342 + color=tuple(int(x * 255) for x in self.image_transform.mean),
343 + )
344 for i in range(0, best_height, self.image_size):
345 for j in range(0, best_width, self.image_size):
346 images_list.append(
324 - self.image_transform(local_view.crop((j, i, j + self.image_size, i + self.image_size))))
347 + self.image_transform(
348 + local_view.crop(
349 + (j, i, j + self.image_size, i + self.image_size)
350 + )
351 + )
352 + )
353
354 """record height / width crop num"""
327 - num_width_tiles, num_height_tiles = best_width // self.image_size, best_height // self.image_size
355 + num_width_tiles, num_height_tiles = (
356 + best_width // self.image_size,
357 + best_height // self.image_size,
358 + )
359 images_spatial_crop.append([num_width_tiles, num_height_tiles])
360
361 """add image tokens"""
331 - h = w = math.ceil((self.image_size // self.patch_size) / self.downsample_ratio)
362 + h = w = math.ceil(
363 + (self.image_size // self.patch_size) / self.downsample_ratio
364 + )
365 # global views tokens h * (w + 1), 1 is for line separator
366 tokenized_image = [self.image_token_id] * h * (w + 1)
367 # add a separator between global and local views
368 tokenized_image += [self.image_token_id]
369 # local views tokens, (num_height_tiles * h) * (num_width_tiles * w + 1)
337 - tokenized_image += [self.image_token_id] * (num_height_tiles * h) * (num_width_tiles * w + 1)
370 + tokenized_image += (
371 + [self.image_token_id]
372 + * (num_height_tiles * h)
373 + * (num_width_tiles * w + 1)
374 + )
375
376 tokenized_str += tokenized_image
377 images_seq_mask += [True] * len(tokenized_image)
@@ -353,10 +390,17 @@ class DeepseekVLV2Processor(ProcessorMixin):
390 tokenized_str = tokenized_str + [self.eos_id]
391 images_seq_mask = images_seq_mask + [False]
392
356 - assert len(tokenized_str) == len(
357 - images_seq_mask), f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}"
393 + assert len(tokenized_str) == len(images_seq_mask), (
394 + f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}"
395 + )
396
359 - return tokenized_str, images_list, images_seq_mask, images_spatial_crop, num_image_tokens
397 + return (
398 + tokenized_str,
399 + images_list,
400 + images_seq_mask,
401 + images_spatial_crop,
402 + num_image_tokens,
403 + )
404
405
406 AutoProcessor.register("DeepseekVLV2Processor", DeepseekVLV2Processor)
vllm/transformers_utils/processors/ovis.py
+79 -45
@@ -1,7 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -# yapf: disable
4 # ruff: noqa: E501
5 # coding=utf-8
6 # adapted from https://github.com/AIDC-AI/Ovis/blob/35ab51a1a1e3542fa6db260a1084cefbc8f164bb/ovis/vllm/processing_ovis.py
@@ -35,23 +34,24 @@ from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
34
35 from vllm.multimodal.image import convert_image_mode
36
38 -__all__ = ['OvisProcessor']
37 +__all__ = ["OvisProcessor"]
38 IGNORE_ID = -100
39
41 -class OvisProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
40 +
41 +class OvisProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
42 _defaults = {
43 "text_kwargs": {
44 "padding": False,
45 },
46 "images_kwargs": {
47 - 'max_partition':9,
48 - 'covering_threshold':0.9,
49 - 'convert_to_rgb':True,
50 - 'return_tensors':'pt'},
47 + "max_partition": 9,
48 + "covering_threshold": 0.9,
49 + "convert_to_rgb": True,
50 + "return_tensors": "pt",
51 + },
52 }
53
54
54 -
55 class OvisProcessor(ProcessorMixin):
56 r"""
57 Constructs an Ovis processor which wraps an Ovis image processor and a Qwen2 tokenizer into a single processor.
@@ -97,14 +97,16 @@ class OvisProcessor(ProcessorMixin):
97 "image_col_sep": -303,
98 "image_row_sep": -304,
99 "image_end": -305,
100 - 'image_pad': image_pad_token_id,
100 + "image_pad": image_pad_token_id,
101 }
102 return extra_special_tokens
103
104 def __call__(
105 self,
106 images: ImageInput = None,
107 - text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,
107 + text: Union[
108 + TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]
109 + ] = None,
110 **kwargs: Unpack[OvisProcessorKwargs],
111 ) -> BatchFeature:
112 """
@@ -169,7 +171,6 @@ class OvisProcessor(ProcessorMixin):
171
172 # Process text input
173 if text is not None:
172 -
174 if not isinstance(text, list):
175 text = [text]
176
@@ -178,7 +179,10 @@ class OvisProcessor(ProcessorMixin):
179 replaced_ids_list = []
180 idx = 0
181 for ids_tensor in tokenized_batched_text:
181 - if image_token_id in ids_tensor and "image_placeholders" in image_features:
182 + if (
183 + image_token_id in ids_tensor
184 + and "image_placeholders" in image_features
185 + ):
186 if idx < len(image_features["image_placeholders"]):
187 # Converts in list for ease of use
188 ids_list = ids_tensor.tolist()
@@ -188,7 +192,9 @@ class OvisProcessor(ProcessorMixin):
192 # replace placeholders
193 for i, token_id in enumerate(ids_list):
194 if token_id == image_token_id:
191 - placeholder_ids = image_features["image_placeholders"][idx]
195 + placeholder_ids = image_features["image_placeholders"][
196 + idx
197 + ]
198 new_ids.extend(placeholder_ids)
199 idx += 1
200 else:
@@ -198,7 +204,8 @@ class OvisProcessor(ProcessorMixin):
204 ids_tensor = torch.tensor(new_ids, dtype=torch.long)
205 else:
206 raise RuntimeError(
201 - 'Mismatch between the images you provided and the number of placeholder present in the text')
207 + "Mismatch between the images you provided and the number of placeholder present in the text"
208 + )
209
210 replaced_ids_list.append(ids_tensor)
211
@@ -217,7 +224,7 @@ class OvisProcessor(ProcessorMixin):
224 # Add image features if present
225 if image_features:
226 output["pixel_values"] = processed_images
220 - output['grids'] = grids
227 + output["grids"] = grids
228
229 return output
230
@@ -227,8 +234,10 @@ class OvisProcessor(ProcessorMixin):
234 def _tokenize_with_image_symbol(self, text_list: list[str]) -> torch.LongTensor:
235 batch_token_ids = []
236 for text in text_list:
230 - text_chunks = [self.tokenizer(chunk, add_special_tokens=False).input_ids for chunk in
231 - text.split(self.image_token)]
237 + text_chunks = [
238 + self.tokenizer(chunk, add_special_tokens=False).input_ids
239 + for chunk in text.split(self.image_token)
240 + ]
241 token_ids = []
242 num_chuck = len(text_chunks)
243 for i, chunk in enumerate(text_chunks):
@@ -240,50 +249,60 @@ class OvisProcessor(ProcessorMixin):
249
250 def get_image_size(self):
251 size = self.image_processor.size
243 - if 'shortest_edge' in size:
244 - width = height = size['shortest_edge']
252 + if "shortest_edge" in size:
253 + width = height = size["shortest_edge"]
254 elif "height" in size and "width" in size:
246 - width = size['width']
247 - height = size['height']
255 + width = size["width"]
256 + height = size["height"]
257 else:
249 - raise ValueError( "Can't parse image size from image_processor config.")
258 + raise ValueError("Can't parse image size from image_processor config.")
259 return height, width
260
261 def get_token_value(self, tok):
262 return self.extra_special_tokens[tok]
263
264 def construct_image_indicators(self, grid):
256 - image_placeholders = [self.get_token_value('image_start'),
257 - self.get_token_value('image_atom'),
258 - self.get_token_value('image_prefix')]
265 + image_placeholders = [
266 + self.get_token_value("image_start"),
267 + self.get_token_value("image_atom"),
268 + self.get_token_value("image_prefix"),
269 + ]
270 if grid[0] * grid[1] > 1:
271 for r in range(grid[0]):
272 for c in range(grid[1]):
262 - image_placeholders.append(self.get_token_value('image_atom') )
273 + image_placeholders.append(self.get_token_value("image_atom"))
274 if c < grid[1] - 1:
264 - image_placeholders.append(self.get_token_value('image_col_sep'))
275 + image_placeholders.append(self.get_token_value("image_col_sep"))
276 if r < grid[0] - 1:
266 - image_placeholders.append(self.get_token_value('image_row_sep'))
267 - image_placeholders.append(self.get_token_value('image_end'))
277 + image_placeholders.append(self.get_token_value("image_row_sep"))
278 + image_placeholders.append(self.get_token_value("image_end"))
279 return image_placeholders
280
281 def construct_image_placeholders(self, grid):
271 -
282 image_placeholders = self.construct_image_indicators(grid)
283
274 - image_atom_token_id = self.get_token_value('image_atom')
284 + image_atom_token_id = self.get_token_value("image_atom")
285 # Extract the padding token ID from tokenizer
276 - image_padding_token_id = self.get_token_value('image_pad')
286 + image_padding_token_id = self.get_token_value("image_pad")
287
288 # Create a new list with padding tokens inserted
289 padded_placeholder_tokens = []
290 for token in image_placeholders:
291 padded_placeholder_tokens.append(image_padding_token_id)
292 if token == image_atom_token_id:
283 - padded_placeholder_tokens.extend([image_padding_token_id] * self.image_segment_len)
293 + padded_placeholder_tokens.extend(
294 + [image_padding_token_id] * self.image_segment_len
295 + )
296 return padded_placeholder_tokens
297
286 - def preprocess_image(self, image: PIL.Image.Image, max_partition, covering_threshold, convert_to_rgb, return_tensors):
298 + def preprocess_image(
299 + self,
300 + image: PIL.Image.Image,
301 + max_partition,
302 + covering_threshold,
303 + convert_to_rgb,
304 + return_tensors,
305 + ):
306 def _preprocess(img: PIL.Image.Image, side):
307 # first resize and preprocess
308 w, h = img.size
@@ -296,19 +315,27 @@ class OvisProcessor(ProcessorMixin):
315 new_height = side
316 new_width = int(w / h * new_height)
317 new_size = dict(height=new_height, width=new_width)
299 - pixel_values = self.image_processor.preprocess(img, size=new_size, return_tensors=return_tensors)['pixel_values']
318 + pixel_values = self.image_processor.preprocess(
319 + img, size=new_size, return_tensors=return_tensors
320 + )["pixel_values"]
321
322 # then pad to square
302 - square_values = torch.zeros([1, 3, side, side], dtype=pixel_values.dtype, device=pixel_values.device)
323 + square_values = torch.zeros(
324 + [1, 3, side, side], dtype=pixel_values.dtype, device=pixel_values.device
325 + )
326 new_height, new_width = pixel_values.shape[2:]
327 if new_height == new_width:
328 square_values[:, :, :, :] = pixel_values
329 elif new_height > new_width:
330 from_index = (side - new_width) // 2
308 - square_values[:, :, :, from_index:from_index + new_width] = pixel_values
331 + square_values[:, :, :, from_index : from_index + new_width] = (
332 + pixel_values
333 + )
334 else:
335 from_index = (side - new_height) // 2
311 - square_values[:, :, from_index:from_index + new_height, :] = pixel_values
336 + square_values[:, :, from_index : from_index + new_height, :] = (
337 + pixel_values
338 + )
339
340 return square_values
341
@@ -350,7 +377,9 @@ class OvisProcessor(ProcessorMixin):
377 good_grids = []
378 for grid in candidate_grids:
379 partition = _partition(img, grid)
353 - covering_ratio = sum([_covering_area(*p, side) for p in partition]) / img_area
380 + covering_ratio = (
381 + sum([_covering_area(*p, side) for p in partition]) / img_area
382 + )
383 assert covering_ratio <= 1.0
384 all_grids.append((grid, covering_ratio))
385 if covering_ratio > covering_threshold:
@@ -358,18 +387,19 @@ class OvisProcessor(ProcessorMixin):
387
388 if len(good_grids) > 0:
389 # pick the good partition with minimum #sub_images and break the tie using covering_ratio
361 - return sorted(good_grids, key=lambda x: (x[0][0] * x[0][1], -x[1]))[0][0]
390 + return sorted(good_grids, key=lambda x: (x[0][0] * x[0][1], -x[1]))[0][
391 + 0
392 + ]
393 else:
394 # pick the partition with maximum covering_ratio and break the tie using #sub_images
395 return sorted(all_grids, key=lambda x: (-x[1], x[0][0] * x[0][1]))[0][0]
396
397 if convert_to_rgb:
367 - image = convert_image_mode(image, 'RGB')
368 -
398 + image = convert_image_mode(image, "RGB")
399
400 sides = self.get_image_size()
401 if sides[0] != sides[1]:
372 - raise ValueError('get_image_size() returns non-square size')
402 + raise ValueError("get_image_size() returns non-square size")
403 side = sides[0]
404 grid = _get_best_grid(image, side)
405 partition = _partition(image, grid)
@@ -405,14 +435,18 @@ class OvisProcessor(ProcessorMixin):
435 `list[str]`: The decoded text.
436 """
437 return self.tokenizer.batch_decode(
408 - generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False
438 + generated_outputs,
439 + skip_special_tokens=True,
440 + clean_up_tokenization_spaces=False,
441 )
442
443 @property
444 def model_input_names(self):
445 tokenizer_input_names = self.tokenizer.model_input_names
446 image_processor_input_names = self.image_processor.model_input_names
415 - names_from_processor = list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
447 + names_from_processor = list(
448 + dict.fromkeys(tokenizer_input_names + image_processor_input_names)
449 + )
450 return names_from_processor + ["second_per_grid_ts"]
451
452
vllm/v1/attention/backends/flashinfer.py
-5
@@ -40,9 +40,6 @@ from vllm.utils.flashinfer import (
40 supports_trtllm_attention,
41 use_trtllm_attention,
42 )
43 -
44 -# yapf conflicts with isort for this block
45 -# yapf: disable
43 from vllm.v1.attention.backends.utils import (
44 AttentionCGSupport,
45 AttentionMetadataBuilder,
@@ -52,8 +49,6 @@ from vllm.v1.attention.backends.utils import (
49 infer_global_hyperparameters,
50 split_decodes_and_prefills,
51 )
55 -
56 -# yapf: enable
52 from vllm.v1.kv_cache_interface import AttentionSpec
53
54 FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
vllm/v1/attention/backends/mla/rocm_aiter_mla.py
-5
@@ -11,9 +11,6 @@ from vllm.attention.backends.abstract import AttentionLayer
11 from vllm.attention.ops.rocm_aiter_mla import aiter_mla_decode_fwd
12 from vllm.config import VllmConfig
13 from vllm.utils import cdiv
14 -
15 -# yapf conflicts with isort for this docstring
16 -# yapf: disable
14 from vllm.v1.attention.backends.mla.common import (
15 MLACommonBackend,
16 MLACommonDecodeMetadata,
@@ -24,8 +21,6 @@ from vllm.v1.attention.backends.mla.common import (
21 from vllm.v1.attention.backends.utils import AttentionCGSupport
22 from vllm.v1.kv_cache_interface import AttentionSpec
23
27 -# yapf: enable
28 -
24
25 def is_aiter_mla_enabled() -> bool:
26 return envs.VLLM_ROCM_USE_AITER and envs.VLLM_ROCM_USE_AITER_MLA
vllm/v1/serial_utils.py
-4
@@ -18,8 +18,6 @@ from msgspec import msgpack
18
19 from vllm import envs
20 from vllm.logger import init_logger
21 -
22 -# yapf: disable
21 from vllm.multimodal.inputs import (
22 BaseMultiModalField,
23 MultiModalBatchedField,
@@ -32,8 +30,6 @@ from vllm.multimodal.inputs import (
30 MultiModalSharedField,
31 NestedTensors,
32 )
35 -
36 -# yapf: enable
33 from vllm.v1.engine import UtilityResult
34
35 logger = init_logger(__name__)
vllm/v1/worker/gpu_model_runner.py
-10
@@ -48,9 +48,6 @@ from vllm.model_executor.layers.mamba.abstract import MambaBase
48 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
49 from vllm.model_executor.model_loader import TensorizerLoader, get_model_loader
50 from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
51 -
52 -# yapf conflicts with isort for this block
53 -# yapf: disable
51 from vllm.model_executor.models.interfaces import (
52 SupportsMultiModal,
53 is_mixture_of_experts,
@@ -59,8 +56,6 @@ from vllm.model_executor.models.interfaces import (
56 supports_multimodal_pruning,
57 supports_transcription,
58 )
62 -
63 -# yapf: enable
59 from vllm.model_executor.models.interfaces_base import (
60 VllmModelForPooling,
61 is_pooling_model,
@@ -101,9 +96,6 @@ from vllm.v1.attention.backends.utils import (
96 split_attn_metadata,
97 )
98 from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
104 -
105 -# yapf conflicts with isort for this block
106 -# yapf: disable
99 from vllm.v1.kv_cache_interface import (
100 AttentionSpec,
101 ChunkedLocalAttentionSpec,
@@ -118,8 +110,6 @@ from vllm.v1.kv_cache_interface import (
110 SlidingWindowSpec,
111 UniformTypeKVCacheSpecs,
112 )
121 -
122 -# yapf: enable
113 from vllm.v1.outputs import (
114 EMPTY_MODEL_RUNNER_OUTPUT,
115 AsyncModelRunnerOutput,