@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 972112d82f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Fix unable to load some models (#10312)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 15, 2024 at 08:55 UTC 972112d82f00e1396c0376cde78c083208b77127
13 files changed +339 -58
.buildkite/test-pipeline.yaml
+3 -2
@@ -313,14 +313,15 @@ steps:
313
314 ##### models test #####
315
316 -- label: Basic Models Test # 10min
316 +- label: Basic Models Test # 30min
317 source_file_dependencies:
318 - vllm/
319 - tests/models
320 commands:
321 - pip install -e ./plugins/vllm_add_dummy_model
322 - pytest -v -s models/test_oot_registration.py # it needs a clean process
323 - - pytest -v -s models/*.py --ignore=models/test_oot_registration.py
323 + - pytest -v -s models/test_registry.py
324 + - pytest -v -s models/test_initialization.py
325
326 - label: Decoder-only Language Models Test (Standard) # 18min
327 #mirror_hardwares: [amd]
tests/distributed/test_pipeline_parallel.py
+3 -3
@@ -166,14 +166,14 @@ TEXT_GENERATION_MODELS = {
166 "mistralai/Mixtral-8x7B-Instruct-v0.1": PPTestSettings.fast(tp_base=4),
167 "mosaicml/mpt-7b": PPTestSettings.fast(),
168 "nvidia/Minitron-8B-Base": PPTestSettings.fast(),
169 - "allenai/OLMoE-1B-7B-0924-Instruct": PPTestSettings.fast(),
169 "allenai/OLMo-1B-hf": PPTestSettings.fast(),
170 + "allenai/OLMoE-1B-7B-0924-Instruct": PPTestSettings.fast(),
171 "facebook/opt-iml-max-1.3b": PPTestSettings.fast(),
172 "OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(trust_remote_code=True),
173 + "adept/persimmon-8b-chat": PPTestSettings.fast(),
174 "microsoft/phi-2": PPTestSettings.fast(),
174 - "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True, load_format="dummy", hf_overrides='{"num_hidden_layers": 4, "hidden_size": 512, "intermediate_size": 800, "num_attention_heads": 4, "num_key_value_heads": 1}'), # noqa: E501
175 "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
176 - "adept/persimmon-8b-chat": PPTestSettings.fast(),
176 + "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True, load_format="dummy", hf_overrides='{"num_hidden_layers": 4, "hidden_size": 512, "intermediate_size": 800, "num_attention_heads": 4, "num_key_value_heads": 1}'), # noqa: E501
177 "Qwen/Qwen-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
178 "Qwen/Qwen2-7B-Instruct": PPTestSettings.fast(),
179 "Qwen/Qwen1.5-MoE-A2.7B-Chat": PPTestSettings.fast(),
tests/models/registry.py new
+212
@@ -0,0 +1,212 @@
1 +from dataclasses import dataclass, field
2 +from typing import AbstractSet, Mapping, Optional
3 +
4 +
5 +@dataclass(frozen=True)
6 +class _HfExamplesInfo:
7 + default: str
8 + """The default model to use for testing this architecture."""
9 +
10 + extras: Mapping[str, str] = field(default_factory=dict)
11 + """Extra models to use for testing this architecture."""
12 +
13 + tokenizer: Optional[str] = None
14 + """Set the tokenizer to load for this architecture."""
15 +
16 + tokenizer_mode: str = "auto"
17 + """Set the tokenizer type for this architecture."""
18 +
19 + speculative_model: Optional[str] = None
20 + """
21 + The default model to use for testing this architecture, which is only used
22 + for speculative decoding.
23 + """
24 +
25 + is_available_online: bool = True
26 + """
27 + Set this to ``False`` if the name of this architecture no longer exists on
28 + the HF repo. To maintain backwards compatibility, we have not removed them
29 + from the main model registry, so without this flag the registry tests will
30 + fail.
31 + """
32 +
33 + trust_remote_code: bool = False
34 + """The ``trust_remote_code`` level required to load the model."""
35 +
36 +
37 +# yapf: disable
38 +_TEXT_GENERATION_EXAMPLE_MODELS = {
39 + # [Decoder-only]
40 + "AquilaModel": _HfExamplesInfo("BAAI/AquilaChat-7B",
41 + trust_remote_code=True),
42 + "AquilaForCausalLM": _HfExamplesInfo("BAAI/AquilaChat2-7B",
43 + trust_remote_code=True),
44 + "ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct",
45 + trust_remote_code=True),
46 + "BaiChuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan-7B",
47 + trust_remote_code=True),
48 + "BaichuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan2-7B-chat",
49 + trust_remote_code=True),
50 + "BloomForCausalLM": _HfExamplesInfo("bigscience/bloomz-1b1"),
51 + # ChatGLMModel supports multimodal
52 + "CohereForCausalLM": _HfExamplesInfo("CohereForAI/c4ai-command-r-v01",
53 + trust_remote_code=True),
54 + "DbrxForCausalLM": _HfExamplesInfo("databricks/dbrx-instruct"),
55 + "DeciLMForCausalLM": _HfExamplesInfo("Deci/DeciLM-7B-instruct",
56 + trust_remote_code=True),
57 + "DeepseekForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-llm-7b-chat"),
58 + "DeepseekV2ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V2-Lite-Chat", # noqa: E501
59 + trust_remote_code=True),
60 + "ExaoneForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct"), # noqa: E501
61 + "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
62 + "GemmaForCausalLM": _HfExamplesInfo("google/gemma-2b"),
63 + "Gemma2ForCausalLM": _HfExamplesInfo("google/gemma-2-9b"),
64 + "GPT2LMHeadModel": _HfExamplesInfo("gpt2"),
65 + "GPTBigCodeForCausalLM": _HfExamplesInfo("bigcode/starcoder"),
66 + "GPTJForCausalLM": _HfExamplesInfo("EleutherAI/gpt-j-6b"),
67 + "GPTNeoXForCausalLM": _HfExamplesInfo("EleutherAI/pythia-160m"),
68 + "GraniteForCausalLM": _HfExamplesInfo("ibm/PowerLM-3b"),
69 + "GraniteMoeForCausalLM": _HfExamplesInfo("ibm/PowerMoE-3b"),
70 + "InternLMForCausalLM": _HfExamplesInfo("internlm/internlm-chat-7b",
71 + trust_remote_code=True),
72 + "InternLM2ForCausalLM": _HfExamplesInfo("internlm/internlm2-chat-7b",
73 + trust_remote_code=True),
74 + "InternLM2VEForCausalLM": _HfExamplesInfo("OpenGVLab/Mono-InternVL-2B",
75 + trust_remote_code=True),
76 + "JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
77 + "JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini"),
78 + "LlamaForCausalLM": _HfExamplesInfo("meta-llama/Meta-Llama-3-8B"),
79 + "LLaMAForCausalLM": _HfExamplesInfo("decapoda-research/llama-7b-hf",
80 + is_available_online=False),
81 + "MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
82 + "FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"), # noqa: E501
83 + "MiniCPMForCausalLM": _HfExamplesInfo("openbmb/MiniCPM-2B-sft-bf16",
84 + trust_remote_code=True),
85 + "MiniCPM3ForCausalLM": _HfExamplesInfo("openbmb/MiniCPM3-4B",
86 + trust_remote_code=True),
87 + "MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
88 + "MixtralForCausalLM": _HfExamplesInfo("mistralai/Mixtral-8x7B-Instruct-v0.1"), # noqa: E501
89 + "QuantMixtralForCausalLM": _HfExamplesInfo("mistral-community/Mixtral-8x22B-v0.1-AWQ"), # noqa: E501
90 + "MptForCausalLM": _HfExamplesInfo("mpt", is_available_online=False),
91 + "MPTForCausalLM": _HfExamplesInfo("mosaicml/mpt-7b"),
92 + "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"),
93 + "OlmoForCausalLM": _HfExamplesInfo("allenai/OLMo-1B-hf"),
94 + "OlmoeForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924-Instruct"),
95 + "OPTForCausalLM": _HfExamplesInfo("facebook/opt-iml-max-1.3b"),
96 + "OrionForCausalLM": _HfExamplesInfo("OrionStarAI/Orion-14B-Chat",
97 + trust_remote_code=True),
98 + "PersimmonForCausalLM": _HfExamplesInfo("adept/persimmon-8b-chat"),
99 + "PhiForCausalLM": _HfExamplesInfo("microsoft/phi-2"),
100 + "Phi3ForCausalLM": _HfExamplesInfo("microsoft/Phi-3-mini-4k-instruct"),
101 + "Phi3SmallForCausalLM": _HfExamplesInfo("microsoft/Phi-3-small-8k-instruct",
102 + trust_remote_code=True),
103 + "PhiMoEForCausalLM": _HfExamplesInfo("microsoft/Phi-3.5-MoE-instruct",
104 + trust_remote_code=True),
105 + # QWenLMHeadModel supports multimodal
106 + "Qwen2ForCausalLM": _HfExamplesInfo("Qwen/Qwen2-7B-Instruct"),
107 + "Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"),
108 + "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b",
109 + is_available_online=False),
110 + "StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b", # noqa: E501
111 + is_available_online=False),
112 + "StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t"),
113 + "Starcoder2ForCausalLM": _HfExamplesInfo("bigcode/starcoder2-3b"),
114 + "SolarForCausalLM": _HfExamplesInfo("upstage/solar-pro-preview-instruct"),
115 + "XverseForCausalLM": _HfExamplesInfo("xverse/XVERSE-7B-Chat",
116 + is_available_online=False,
117 + trust_remote_code=True),
118 + # [Encoder-decoder]
119 + "BartModel": _HfExamplesInfo("facebook/bart-base"),
120 + "BartForConditionalGeneration": _HfExamplesInfo("facebook/bart-large-cnn"),
121 + # Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
122 + # Therefore, we borrow the BartTokenizer from the original Bart model
123 + "Florence2ForConditionalGeneration": _HfExamplesInfo("microsoft/Florence-2-base", # noqa: E501
124 + tokenizer="facebook/bart-base",
125 + trust_remote_code=True), # noqa: E501
126 +}
127 +
128 +_EMBEDDING_EXAMPLE_MODELS = {
129 + # [Text-only]
130 + "BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
131 + "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
132 + "MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
133 + "Qwen2ForRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-RM-72B"),
134 + "Qwen2ForSequenceClassification": _HfExamplesInfo("jason9693/Qwen2.5-1.5B-apeach"), # noqa: E501
135 + # [Multimodal]
136 + "LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"),
137 + "Phi3VForCausalLM": _HfExamplesInfo("TIGER-Lab/VLM2Vec-Full",
138 + trust_remote_code=True),
139 + "Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"), # noqa: E501
140 +}
141 +
142 +_MULTIMODAL_EXAMPLE_MODELS = {
143 + # [Decoder-only]
144 + "Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b"), # noqa: E501
145 + "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"), # noqa: E501
146 + "ChatGLMModel": _HfExamplesInfo("THUDM/glm-4v-9b",
147 + extras={"text_only": "THUDM/chatglm3-6b"},
148 + trust_remote_code=True),
149 + "ChatGLMForConditionalGeneration": _HfExamplesInfo("chatglm2-6b",
150 + is_available_online=False),
151 + "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
152 + "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m"),
153 + "InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
154 + trust_remote_code=True),
155 + "Idefics3ForConditionalGeneration": _HfExamplesInfo("HuggingFaceM4/Idefics3-8B-Llama3"), # noqa: E501
156 + "LlavaForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-1.5-7b-hf",
157 + extras={"mistral": "mistral-community/pixtral-12b"}), # noqa: E501
158 + "LlavaNextForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-v1.6-mistral-7b-hf"), # noqa: E501
159 + "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo("llava-hf/LLaVA-NeXT-Video-7B-hf"), # noqa: E501
160 + "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
161 + "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
162 + trust_remote_code=True),
163 + "MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
164 + trust_remote_code=True),
165 + "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B",
166 + trust_remote_code=True),
167 + "PaliGemmaForConditionalGeneration": _HfExamplesInfo("google/paligemma-3b-pt-224"), # noqa: E501
168 + "Phi3VForCausalLM": _HfExamplesInfo("microsoft/Phi-3-vision-128k-instruct",
169 + trust_remote_code=True),
170 + "PixtralForConditionalGeneration": _HfExamplesInfo("mistralai/Pixtral-12B-2409", # noqa: E501
171 + tokenizer_mode="mistral"),
172 + "QWenLMHeadModel": _HfExamplesInfo("Qwen/Qwen-VL-Chat",
173 + extras={"text_only": "Qwen/Qwen-7B-Chat"}, # noqa: E501
174 + trust_remote_code=True),
175 + "Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
176 + "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
177 + "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3"),
178 + # [Encoder-decoder]
179 + "MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
180 +}
181 +
182 +_SPECULATIVE_DECODING_EXAMPLE_MODELS = {
183 + "EAGLEModel": _HfExamplesInfo("JackFram/llama-68m",
184 + speculative_model="abhigoyal/vllm-eagle-llama-68m-random"), # noqa: E501
185 + "MedusaModel": _HfExamplesInfo("JackFram/llama-68m",
186 + speculative_model="abhigoyal/vllm-medusa-llama-68m-random"), # noqa: E501
187 + "MLPSpeculatorPreTrainedModel": _HfExamplesInfo("JackFram/llama-160m",
188 + speculative_model="ibm-fms/llama-160m-accelerator"), # noqa: E501
189 +}
190 +
191 +_EXAMPLE_MODELS = {
192 + **_TEXT_GENERATION_EXAMPLE_MODELS,
193 + **_EMBEDDING_EXAMPLE_MODELS,
194 + **_MULTIMODAL_EXAMPLE_MODELS,
195 + **_SPECULATIVE_DECODING_EXAMPLE_MODELS,
196 +}
197 +
198 +
199 +class HfExampleModels:
200 + def __init__(self, hf_models: Mapping[str, _HfExamplesInfo]) -> None:
201 + super().__init__()
202 +
203 + self.hf_models = hf_models
204 +
205 + def get_supported_archs(self) -> AbstractSet[str]:
206 + return self.hf_models.keys()
207 +
208 + def get_hf_info(self, model_arch: str) -> _HfExamplesInfo:
209 + return self.hf_models[model_arch]
210 +
211 +
212 +HF_EXAMPLE_MODELS = HfExampleModels(_EXAMPLE_MODELS)
tests/models/test_initialization.py new
+55
@@ -0,0 +1,55 @@
1 +from unittest.mock import patch
2 +
3 +import pytest
4 +import transformers
5 +from transformers import PretrainedConfig
6 +
7 +from vllm import LLM
8 +
9 +from .registry import HF_EXAMPLE_MODELS
10 +
11 +
12 +@pytest.mark.parametrize("model_arch", HF_EXAMPLE_MODELS.get_supported_archs())
13 +def test_can_initialize(model_arch):
14 + if (model_arch == "Idefics3ForConditionalGeneration"
15 + and transformers.__version__ < "4.46.0"):
16 + pytest.skip(reason="Model introduced in HF >= 4.46.0")
17 +
18 + model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
19 + if not model_info.is_available_online:
20 + pytest.skip("Model is not available online")
21 +
22 + # Avoid OOM
23 + def hf_overrides(hf_config: PretrainedConfig) -> PretrainedConfig:
24 + if hasattr(hf_config, "text_config"):
25 + text_config: PretrainedConfig = hf_config.text_config
26 + else:
27 + text_config = hf_config
28 +
29 + text_config.update({
30 + "num_layers": 1,
31 + "num_hidden_layers": 1,
32 + "num_experts": 2,
33 + "num_experts_per_tok": 2,
34 + "num_local_experts": 2,
35 + })
36 +
37 + return hf_config
38 +
39 + # Avoid calling model.forward()
40 + def _initialize_kv_caches(self) -> None:
41 + self.cache_config.num_gpu_blocks = 0
42 + self.cache_config.num_cpu_blocks = 0
43 +
44 + with patch.object(LLM.get_engine_class(), "_initialize_kv_caches",
45 + _initialize_kv_caches):
46 + LLM(
47 + model_info.default,
48 + tokenizer=model_info.tokenizer,
49 + tokenizer_mode=model_info.tokenizer_mode,
50 + speculative_model=model_info.speculative_model,
51 + num_speculative_tokens=1 if model_info.speculative_model else None,
52 + trust_remote_code=model_info.trust_remote_code,
53 + load_format="dummy",
54 + hf_overrides=hf_overrides,
55 + )
tests/models/test_registry.py
+10
@@ -14,6 +14,7 @@ from vllm.model_executor.models.registry import (_EMBEDDING_MODELS,
14 from vllm.platforms import current_platform
15
16 from ..utils import fork_new_process_for_each_test
17 +from .registry import HF_EXAMPLE_MODELS
18
19
20 @pytest.mark.parametrize("model_arch", ModelRegistry.get_supported_archs())
@@ -73,3 +74,12 @@ def test_registry_is_pp(model_arch, is_pp, init_cuda):
74 "This model no longer initializes CUDA on import. "
75 "Please test using a different one.",
76 stacklevel=2)
77 +
78 +
79 +def test_hf_registry_coverage():
80 + untested_archs = (HF_EXAMPLE_MODELS.get_supported_archs() -
81 + set(ModelRegistry.get_supported_archs()))
82 +
83 + assert not untested_archs, (
84 + "Please add the following architectures to "
85 + f"`tests/models/registry.py`: {untested_archs}")
vllm/config.py
+26 -10
@@ -3,8 +3,8 @@ import enum
3 import json
4 import warnings
5 from dataclasses import dataclass, field, replace
6 -from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Final, List, Literal,
7 - Mapping, Optional, Set, Tuple, Type, Union)
6 +from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Dict, Final, List,
7 + Literal, Mapping, Optional, Set, Tuple, Type, Union)
8
9 import torch
10 from transformers import PretrainedConfig
@@ -20,7 +20,7 @@ from vllm.transformers_utils.config import (
20 get_hf_text_config, get_pooling_config,
21 get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
22 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
23 - print_warning_once)
23 + identity, print_warning_once)
24
25 if TYPE_CHECKING:
26 from ray.util.placement_group import PlacementGroup
@@ -44,6 +44,9 @@ TaskOption = Literal["auto", "generate", "embedding"]
44 # "draft" is only used internally for speculative decoding
45 _Task = Literal["generate", "embedding", "draft"]
46
47 +HfOverrides = Union[Dict[str, Any], Callable[[PretrainedConfig],
48 + PretrainedConfig]]
49 +
50
51 class ModelConfig:
52 """Configuration for the model.
@@ -115,7 +118,9 @@ class ModelConfig:
118 can not be gathered from the vllm arguments.
119 config_format: The config format which shall be loaded.
120 Defaults to 'auto' which defaults to 'hf'.
118 - hf_overrides: Arguments to be forwarded to the HuggingFace config.
121 + hf_overrides: If a dictionary, contains arguments to be forwarded to the
122 + HuggingFace config. If a callable, it is called to update the
123 + HuggingFace config.
124 mm_processor_kwargs: Arguments to be forwarded to the model's processor
125 for multi-modal data, e.g., image processor.
126 pooling_type: Used to configure the pooling method in the embedding
@@ -164,7 +169,7 @@ class ModelConfig:
169 override_neuron_config: Optional[Dict[str, Any]] = None,
170 config_format: ConfigFormat = ConfigFormat.AUTO,
171 chat_template_text_format: str = "string",
167 - hf_overrides: Optional[Dict[str, Any]] = None,
172 + hf_overrides: Optional[HfOverrides] = None,
173 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
174 pooling_type: Optional[str] = None,
175 pooling_norm: Optional[bool] = None,
@@ -182,15 +187,23 @@ class ModelConfig:
187
188 if hf_overrides is None:
189 hf_overrides = {}
190 +
191 + if callable(hf_overrides):
192 + hf_overrides_kw = {}
193 + hf_overrides_fn = hf_overrides
194 + else:
195 + hf_overrides_kw = hf_overrides
196 + hf_overrides_fn = identity
197 +
198 if rope_scaling is not None:
199 hf_override: Dict[str, Any] = {"rope_scaling": rope_scaling}
187 - hf_overrides.update(hf_override)
200 + hf_overrides_kw.update(hf_override)
201 msg = ("`--rope-scaling` will be removed in a future release. "
202 f"'Please instead use `--hf-overrides '{hf_override!r}'`")
203 warnings.warn(DeprecationWarning(msg), stacklevel=2)
204 if rope_theta is not None:
205 hf_override = {"rope_theta": rope_theta}
193 - hf_overrides.update(hf_override)
206 + hf_overrides_kw.update(hf_override)
207 msg = ("`--rope-theta` will be removed in a future release. "
208 f"'Please instead use `--hf-overrides '{hf_override!r}'`")
209 warnings.warn(DeprecationWarning(msg), stacklevel=2)
@@ -207,9 +220,12 @@ class ModelConfig:
220 self.max_logprobs = max_logprobs
221 self.disable_sliding_window = disable_sliding_window
222 self.skip_tokenizer_init = skip_tokenizer_init
210 - self.hf_config = get_config(self.model, trust_remote_code, revision,
211 - code_revision, config_format,
212 - **hf_overrides)
223 +
224 + hf_config = get_config(self.model, trust_remote_code, revision,
225 + code_revision, config_format, **hf_overrides_kw)
226 + hf_config = hf_overrides_fn(hf_config)
227 + self.hf_config = hf_config
228 +
229 self.hf_text_config = get_hf_text_config(self.hf_config)
230 self.encoder_config = self._get_encoder_config()
231 self.hf_image_processor_config = get_hf_image_processor_config(
vllm/engine/arg_utils.py
+4 -4
@@ -9,9 +9,9 @@ import torch
9
10 import vllm.envs as envs
11 from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
12 - DeviceConfig, LoadConfig, LoadFormat, LoRAConfig,
13 - ModelConfig, ObservabilityConfig, ParallelConfig,
14 - PromptAdapterConfig, SchedulerConfig,
12 + DeviceConfig, HfOverrides, LoadConfig, LoadFormat,
13 + LoRAConfig, ModelConfig, ObservabilityConfig,
14 + ParallelConfig, PromptAdapterConfig, SchedulerConfig,
15 SpeculativeConfig, TaskOption, TokenizerPoolConfig,
16 VllmConfig)
17 from vllm.executor.executor_base import ExecutorBase
@@ -128,7 +128,7 @@ class EngineArgs:
128 code_revision: Optional[str] = None
129 rope_scaling: Optional[Dict[str, Any]] = None
130 rope_theta: Optional[float] = None
131 - hf_overrides: Optional[Dict[str, Any]] = None
131 + hf_overrides: Optional[HfOverrides] = None
132 tokenizer_revision: Optional[str] = None
133 quantization: Optional[str] = None
134 enforce_eager: Optional[bool] = None
vllm/entrypoints/llm.py
+5 -3
@@ -9,7 +9,7 @@ from tqdm import tqdm
9 from vllm import envs
10 from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
11 BeamSearchSequence, get_beam_search_score)
12 -from vllm.engine.arg_utils import EngineArgs, TaskOption
12 +from vllm.engine.arg_utils import EngineArgs, HfOverrides, TaskOption
13 from vllm.engine.llm_engine import LLMEngine
14 from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
15 apply_hf_chat_template,
@@ -101,7 +101,9 @@ class LLM:
101 disable_custom_all_reduce: See :class:`~vllm.config.ParallelConfig`
102 disable_async_output_proc: Disable async output processing.
103 This may result in lower performance.
104 - hf_overrides: Arguments to be forwarded to the HuggingFace config.
104 + hf_overrides: If a dictionary, contains arguments to be forwarded to the
105 + HuggingFace config. If a callable, it is called to update the
106 + HuggingFace config.
107 **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
108 :ref:`engine_args`)
109
@@ -156,7 +158,7 @@ class LLM:
158 max_seq_len_to_capture: int = 8192,
159 disable_custom_all_reduce: bool = False,
160 disable_async_output_proc: bool = False,
159 - hf_overrides: Optional[dict] = None,
161 + hf_overrides: Optional[HfOverrides] = None,
162 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
163 # After positional args are removed, move this right below `model`
164 task: TaskOption = "auto",
vllm/model_executor/models/fuyu.py
+5 -2
@@ -41,7 +41,8 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
41 from vllm.utils import is_list_of
42
43 from .interfaces import SupportsMultiModal, SupportsPP
44 -from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
44 +from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
45 + merge_multimodal_embeddings)
46
47 # Cannot find the following 2 numbers from hf config.
48 _IMAGE_TOKEN_ID = 71011
@@ -245,7 +246,9 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
246 gather_output=True,
247 )
248 self.language_model = PersimmonForCausalLM(
248 - vllm_config.with_hf_config(config.text_config))
249 + vllm_config=vllm_config.with_hf_config(config.text_config),
250 + prefix=maybe_prefix(prefix, "language_model"),
251 + )
252 self.make_empty_intermediate_tensors = (
253 self.language_model.make_empty_intermediate_tensors)
254
vllm/model_executor/models/internlm2_ve.py
+1 -7
@@ -161,11 +161,5 @@ class InternLM2VEForCausalLM(InternLM2ForCausalLM):
161 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
162 super().__init__(vllm_config=vllm_config, prefix=prefix)
163
164 - config = vllm_config.model_config.hf_config
165 - cache_config = vllm_config.cache_config
166 - quant_config = vllm_config.quant_config
167 -
168 - self.model = InternLM2VEModel(config,
169 - cache_config,
170 - quant_config,
164 + self.model = InternLM2VEModel(vllm_config=vllm_config,
165 prefix=maybe_prefix(prefix, "model"))
vllm/model_executor/models/minicpmv.py
+8 -24
@@ -382,11 +382,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
382 instantiated.
383 """
384
385 - def __init__(
386 - self,
387 - vllm_config: VllmConfig,
388 - prefix: str = "",
389 - ):
385 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
386 config = vllm_config.model_config.hf_config
387 multimodal_config = vllm_config.model_config.multimodal_config
388 quant_config = vllm_config.quant_config
@@ -699,12 +695,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
695
696 class MiniCPMV2_0(MiniCPMVBaseModel):
697
702 - def __init__(
703 - self,
704 - vllm_config: VllmConfig,
705 - prefix: str = "",
706 - ):
707 - super().__init__(vllm_config)
698 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
699 + super().__init__(vllm_config=vllm_config, prefix=prefix)
700 assert self.version == (2, 0)
701
702 def init_llm(
@@ -857,12 +849,8 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
849 embedding_modules = {}
850 embedding_padding_modules = []
851
860 - def __init__(
861 - self,
862 - vllm_config: VllmConfig,
863 - prefix: str = "",
864 - ):
865 - super().__init__(vllm_config)
852 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
853 + super().__init__(vllm_config=vllm_config, prefix=prefix)
854 assert self.version == (2, 5)
855
856 def init_llm(
@@ -999,12 +987,8 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
987 embedding_modules = {}
988 embedding_padding_modules = []
989
1002 - def __init__(
1003 - self,
1004 - vllm_config: VllmConfig,
1005 - prefix: str = "",
1006 - ):
1007 - super().__init__(vllm_config)
990 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
991 + super().__init__(vllm_config=vllm_config, prefix=prefix)
992 assert self.version == (2, 6)
993
994 def init_llm(
@@ -1117,7 +1101,7 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
1101 embedding_modules = {}
1102 embedding_padding_modules = []
1103
1120 - def __new__(cls, vllm_config: VllmConfig, prefix: str = ""):
1104 + def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""):
1105 config = vllm_config.model_config.hf_config
1106 if not hasattr(config, "version"):
1107 if config.hidden_size == 2304 and config.query_num == 64:
vllm/model_executor/models/mlp_speculator.py
+1 -1
@@ -65,7 +65,7 @@ class MLPSpeculator(nn.Module):
65 https://huggingface.co/ibm-fms and https://huggingface.co/ibm-granite
66 """
67
68 - def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
68 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
69 super().__init__()
70 config = vllm_config.model_config.hf_config
71 self.n_predict = config.n_predict
vllm/model_executor/models/registry.py
+6 -2
@@ -1,3 +1,7 @@
1 +"""
2 +Whenever you add an architecture to this page, please also update
3 +`tests/models/registry.py` with example HuggingFace models for it.
4 +"""
5 import importlib
6 import os
7 import pickle
@@ -58,14 +62,14 @@ _TEXT_GENERATION_MODELS = {
62 "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
63 "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
64 "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
65 + "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
66 + "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
67 "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
68 "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
69 "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
70 # transformers's mpt class has lower case
71 "MptForCausalLM": ("mpt", "MPTForCausalLM"),
72 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
67 - "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
68 - "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
73 "NemotronForCausalLM": ("nemotron", "NemotronForCausalLM"),
74 "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
75 "OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),