A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Model] Officially support Emu3 with Transformers backend (#21319)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Jul 24, 2025 at 11:22 UTC
13abd0eaf924af1d27cf002b9d423087e05a2315
5 files changed
+29
-14
docs/models/supported_models.md
+6
@@ -623,6 +623,12 @@ Specified using `--task generate`.
623
| `TarsierForConditionalGeneration` | Tarsier | T + I<sup>E+</sup> | `omni-search/Tarsier-7b`, `omni-search/Tarsier-34b` | | ✅︎ | ✅︎ |
624
| `Tarsier2ForConditionalGeneration`<sup>^</sup> | Tarsier2 | T + I<sup>E+</sup> + V<sup>E+</sup> | `omni-research/Tarsier2-Recap-7b`, `omni-research/Tarsier2-7b-0115` | | ✅︎ | ✅︎ |
625
626
+Some models are supported only via the [Transformers backend](#transformers). The purpose of the table below is to acknowledge models which we officially support in this way. The logs will say that the Transformers backend is being used, and you will see no warning that this is fallback behaviour. This means that, if you have issues with any of the models listed below, please [make an issue](https://github.com/vllm-project/vllm/issues/new/choose) and we'll do our best to fix it!
627
+
628
+| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/distributed_serving.md) | [V1](gh-issue:8779) |
629
+|--------------|--------|--------|-------------------|-----------------------------|-----------------------------------------|---------------------|
630
+| `Emu3ForConditionalGeneration` | Emu3 | T + I | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ | ✅︎ |
631
+
632
<sup>^</sup> You need to set the architecture name via `--hf-overrides` to match the one in vLLM.
633
• For example, to use DeepSeek-VL2 series models:
634
`--hf-overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'`
tests/models/multimodal/test_mapping.py
+10
-7
@@ -23,18 +23,14 @@ def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
23
return ((name, torch.empty(0)) for name in weight_names)
24
25
26
-def create_model_dummy_weights(
27
- repo: str,
28
- model_arch: str,
29
-) -> Iterable[tuple[str, torch.Tensor]]:
26
+def create_dummy_model(repo: str, model_arch: str) -> PreTrainedModel:
27
"""
28
Create weights from a dummy meta deserialized hf model with name conversion
29
"""
30
model_cls: PreTrainedModel = getattr(transformers, model_arch)
31
config = AutoConfig.from_pretrained(repo)
32
with torch.device("meta"):
36
- model: PreTrainedModel = model_cls._from_config(config)
37
- return model.named_parameters()
33
+ return model_cls._from_config(config)
34
35
36
def model_architectures_for_test() -> list[str]:
@@ -70,14 +66,21 @@ def test_hf_model_weights_mapper(model_arch: str):
66
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
67
68
original_weights = create_repo_dummy_weights(model_id)
73
- hf_converted_weights = create_model_dummy_weights(model_id, model_arch)
69
+ hf_dummy_model = create_dummy_model(model_id, model_arch)
70
+ hf_converted_weights = hf_dummy_model.named_parameters()
71
+ hf_converted_buffers = hf_dummy_model.named_buffers()
72
mapper: WeightsMapper = model_cls.hf_to_vllm_mapper
73
74
mapped_original_weights = mapper.apply(original_weights)
75
mapped_hf_converted_weights = mapper.apply(hf_converted_weights)
76
+ mapped_hf_converted_buffers = mapper.apply(hf_converted_buffers)
77
78
ref_weight_names = set(map(lambda x: x[0], mapped_original_weights))
79
weight_names = set(map(lambda x: x[0], mapped_hf_converted_weights))
80
+ buffer_names = set(map(lambda x: x[0], mapped_hf_converted_buffers))
81
+
82
+ # Some checkpoints may have buffers, we ignore them for this test
83
+ ref_weight_names -= buffer_names
84
85
weights_missing = ref_weight_names - weight_names
86
weights_unmapped = weight_names - ref_weight_names
tests/models/registry.py
+3
-2
@@ -357,6 +357,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
357
max_transformers_version="4.48", # noqa: E501
358
transformers_version_reason="HF model is not compatible.", # noqa: E501
359
hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]}), # noqa: E501
360
+ "Emu3ForConditionalGeneration": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
361
"FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
362
"Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
363
"GraniteSpeechForConditionalGeneration": _HfExamplesInfo("ibm-granite/granite-speech-3.3-2b"), # noqa: E501
@@ -501,7 +502,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
502
speculative_model="XiaomiMiMo/MiMo-7B-RL")
503
}
504
504
-_TRANSFORMERS_MODELS = {
505
+_TRANSFORMERS_BACKEND_MODELS = {
506
"TransformersForCausalLM": _HfExamplesInfo("hmellor/Ilama-3.2-1B", trust_remote_code=True), # noqa: E501
507
"TransformersForMultimodalLM": _HfExamplesInfo("OpenGVLab/InternVL3-1B-hf"),
508
}
@@ -512,7 +513,7 @@ _EXAMPLE_MODELS = {
513
**_SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS,
514
**_MULTIMODAL_EXAMPLE_MODELS,
515
**_SPECULATIVE_DECODING_EXAMPLE_MODELS,
515
- **_TRANSFORMERS_MODELS,
516
+ **_TRANSFORMERS_BACKEND_MODELS,
517
}
518
519
vllm/model_executor/model_loader/utils.py
+3
-3
@@ -26,7 +26,7 @@ from vllm.model_executor.models.adapters import (as_embedding_model,
26
as_seq_cls_model)
27
from vllm.model_executor.models.interfaces import SupportsQuant
28
from vllm.model_executor.models.registry import (_PREVIOUSLY_SUPPORTED_MODELS,
29
- _TRANSFORMERS_MODELS)
29
+ _TRANSFORMERS_BACKEND_MODELS)
30
from vllm.utils import is_pin_memory_available
31
32
logger = init_logger(__name__)
@@ -178,7 +178,7 @@ def resolve_transformers_arch(model_config: ModelConfig,
178
"happen.")
179
180
for i, arch in enumerate(architectures):
181
- if arch in _TRANSFORMERS_MODELS:
181
+ if arch in _TRANSFORMERS_BACKEND_MODELS:
182
continue
183
184
if model_config.model_impl == ModelImpl.AUTO:
@@ -241,7 +241,7 @@ def get_model_architecture(
241
242
vllm_supported_archs = ModelRegistry.get_supported_archs()
243
is_supported = lambda arch: (arch in vllm_supported_archs and arch not in
244
- _TRANSFORMERS_MODELS)
244
+ _TRANSFORMERS_BACKEND_MODELS)
245
vllm_not_supported = not any(is_supported(arch) for arch in architectures)
246
247
if vllm_not_supported:
vllm/model_executor/models/registry.py
+7
-2
@@ -254,7 +254,11 @@ _SPECULATIVE_DECODING_MODELS = {
254
# "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
255
}
256
257
-_TRANSFORMERS_MODELS = {
257
+_TRANSFORMERS_SUPPORTED_MODELS = {
258
+ "Emu3ForConditionalGeneration": ("transformers", "TransformersForMultimodalLM"), # noqa: E501
259
+}
260
+
261
+_TRANSFORMERS_BACKEND_MODELS = {
262
"TransformersForMultimodalLM": ("transformers", "TransformersForMultimodalLM"), # noqa: E501
263
"TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
264
}
@@ -266,7 +270,8 @@ _VLLM_MODELS = {
270
**_CROSS_ENCODER_MODELS,
271
**_MULTIMODAL_MODELS,
272
**_SPECULATIVE_DECODING_MODELS,
269
- **_TRANSFORMERS_MODELS,
273
+ **_TRANSFORMERS_SUPPORTED_MODELS,
274
+ **_TRANSFORMERS_BACKEND_MODELS,
275
}
276
277
# This variable is used as the args for subprocess.run(). We