A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Model] Allow loading from original Mistral format (#8168)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Patrick von Platen committed
Sep 7, 2024 at 01:02 UTC
29f49cd6e3d3c5658b92ea3e97138c1ab5cb6b30
7 files changed
+291
-81
tests/models/test_mistral.py
+40
@@ -41,3 +41,43 @@ def test_models(
41
name_0="hf",
42
name_1="vllm",
43
)
44
+
45
+
46
+@pytest.mark.parametrize("model", MODELS[1:])
47
+@pytest.mark.parametrize("dtype", ["bfloat16"])
48
+@pytest.mark.parametrize("max_tokens", [64])
49
+@pytest.mark.parametrize("num_logprobs", [5])
50
+def test_mistral_format(
51
+ vllm_runner,
52
+ example_prompts,
53
+ model: str,
54
+ dtype: str,
55
+ max_tokens: int,
56
+ num_logprobs: int,
57
+) -> None:
58
+ with vllm_runner(
59
+ model,
60
+ dtype=dtype,
61
+ tokenizer_mode="auto",
62
+ load_format="safetensors",
63
+ config_format="hf",
64
+ ) as hf_format_model:
65
+ hf_format_outputs = hf_format_model.generate_greedy_logprobs(
66
+ example_prompts, max_tokens, num_logprobs)
67
+
68
+ with vllm_runner(
69
+ model,
70
+ dtype=dtype,
71
+ tokenizer_mode="mistral",
72
+ load_format="mistral",
73
+ config_format="mistral",
74
+ ) as mistral_format_model:
75
+ mistral_format_outputs = mistral_format_model.generate_greedy_logprobs(
76
+ example_prompts, max_tokens, num_logprobs)
77
+
78
+ check_logprobs_close(
79
+ outputs_0_lst=hf_format_outputs,
80
+ outputs_1_lst=mistral_format_outputs,
81
+ name_0="hf",
82
+ name_1="mistral",
83
+ )
vllm/config.py
+33
-29
@@ -13,7 +13,7 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
from vllm.model_executor.models import ModelRegistry
14
from vllm.platforms import current_platform
15
from vllm.tracing import is_otel_available, otel_import_error_traceback
16
-from vllm.transformers_utils.config import (get_config,
16
+from vllm.transformers_utils.config import (ConfigFormat, get_config,
17
get_hf_image_processor_config,
18
get_hf_text_config)
19
from vllm.utils import (STR_NOT_IMPL_ENC_DEC_CUDAGRAPH, GiB_bytes,
@@ -121,35 +121,37 @@ class ModelConfig:
121
override default neuron config that are specific to Neuron devices,
122
this argument will be used to configure the neuron config that
123
can not be gathered from the vllm arguments.
124
+ config_format: The config format which shall be loaded.
125
+ Defaults to 'auto' which defaults to 'hf'.
126
"""
127
126
- def __init__(
127
- self,
128
- model: str,
129
- tokenizer: str,
130
- tokenizer_mode: str,
131
- trust_remote_code: bool,
132
- dtype: Union[str, torch.dtype],
133
- seed: int,
134
- revision: Optional[str] = None,
135
- code_revision: Optional[str] = None,
136
- rope_scaling: Optional[dict] = None,
137
- rope_theta: Optional[float] = None,
138
- tokenizer_revision: Optional[str] = None,
139
- max_model_len: Optional[int] = None,
140
- spec_target_max_model_len: Optional[int] = None,
141
- quantization: Optional[str] = None,
142
- quantization_param_path: Optional[str] = None,
143
- enforce_eager: Optional[bool] = None,
144
- max_context_len_to_capture: Optional[int] = None,
145
- max_seq_len_to_capture: Optional[int] = None,
146
- max_logprobs: int = 20,
147
- disable_sliding_window: bool = False,
148
- skip_tokenizer_init: bool = False,
149
- served_model_name: Optional[Union[str, List[str]]] = None,
150
- limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
151
- use_async_output_proc: bool = True,
152
- override_neuron_config: Optional[Dict[str, Any]] = None) -> None:
128
+ def __init__(self,
129
+ model: str,
130
+ tokenizer: str,
131
+ tokenizer_mode: str,
132
+ trust_remote_code: bool,
133
+ dtype: Union[str, torch.dtype],
134
+ seed: int,
135
+ revision: Optional[str] = None,
136
+ code_revision: Optional[str] = None,
137
+ rope_scaling: Optional[dict] = None,
138
+ rope_theta: Optional[float] = None,
139
+ tokenizer_revision: Optional[str] = None,
140
+ max_model_len: Optional[int] = None,
141
+ spec_target_max_model_len: Optional[int] = None,
142
+ quantization: Optional[str] = None,
143
+ quantization_param_path: Optional[str] = None,
144
+ enforce_eager: Optional[bool] = None,
145
+ max_context_len_to_capture: Optional[int] = None,
146
+ max_seq_len_to_capture: Optional[int] = None,
147
+ max_logprobs: int = 20,
148
+ disable_sliding_window: bool = False,
149
+ skip_tokenizer_init: bool = False,
150
+ served_model_name: Optional[Union[str, List[str]]] = None,
151
+ limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
152
+ use_async_output_proc: bool = True,
153
+ override_neuron_config: Optional[Dict[str, Any]] = None,
154
+ config_format: ConfigFormat = ConfigFormat.AUTO) -> None:
155
self.model = model
156
self.tokenizer = tokenizer
157
self.tokenizer_mode = tokenizer_mode
@@ -176,7 +178,8 @@ class ModelConfig:
178
self.skip_tokenizer_init = skip_tokenizer_init
179
180
self.hf_config = get_config(self.model, trust_remote_code, revision,
179
- code_revision, rope_scaling, rope_theta)
181
+ code_revision, rope_scaling, rope_theta,
182
+ config_format)
183
self.hf_text_config = get_hf_text_config(self.hf_config)
184
self.hf_image_processor_config = get_hf_image_processor_config(
185
self.model, revision)
@@ -746,6 +749,7 @@ class LoadFormat(str, enum.Enum):
749
SHARDED_STATE = "sharded_state"
750
GGUF = "gguf"
751
BITSANDBYTES = "bitsandbytes"
752
+ MISTRAL = "mistral"
753
754
755
@dataclass
vllm/engine/arg_utils.py
+16
-5
@@ -8,10 +8,10 @@ from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple,
8
import torch
9
10
import vllm.envs as envs
11
-from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
12
- EngineConfig, LoadConfig, LoadFormat, LoRAConfig,
13
- ModelConfig, ObservabilityConfig, ParallelConfig,
14
- PromptAdapterConfig, SchedulerConfig,
11
+from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
12
+ DeviceConfig, EngineConfig, LoadConfig, LoadFormat,
13
+ LoRAConfig, ModelConfig, ObservabilityConfig,
14
+ ParallelConfig, PromptAdapterConfig, SchedulerConfig,
15
SpeculativeConfig, TokenizerPoolConfig)
16
from vllm.executor.executor_base import ExecutorBase
17
from vllm.logger import init_logger
@@ -65,6 +65,7 @@ class EngineArgs:
65
trust_remote_code: bool = False
66
download_dir: Optional[str] = None
67
load_format: str = 'auto'
68
+ config_format: str = 'auto'
69
dtype: str = 'auto'
70
kv_cache_dtype: str = 'auto'
71
quantization_param_path: Optional[str] = None
@@ -234,6 +235,13 @@ class EngineArgs:
235
'section for more information.\n'
236
'* "bitsandbytes" will load the weights using bitsandbytes '
237
'quantization.\n')
238
+ parser.add_argument(
239
+ '--config-format',
240
+ default=EngineArgs.config_format,
241
+ choices=[f.value for f in ConfigFormat],
242
+ help='The format of the model config to load.\n\n'
243
+ '* "auto" will try to load the config in hf format '
244
+ 'if available else it will try to load in mistral format ')
245
parser.add_argument(
246
'--dtype',
247
type=str,
@@ -813,7 +821,10 @@ class EngineArgs:
821
served_model_name=self.served_model_name,
822
limit_mm_per_prompt=self.limit_mm_per_prompt,
823
use_async_output_proc=not self.disable_async_output_proc,
816
- override_neuron_config=self.override_neuron_config)
824
+ override_neuron_config=self.override_neuron_config,
825
+ config_format=self.config_format,
826
+ )
827
+
828
cache_config = CacheConfig(
829
block_size=self.block_size if self.device != "neuron" else
830
self.max_model_len, # neuron needs block_size = max_model_len
vllm/model_executor/model_loader/loader.py
+9
-3
@@ -17,6 +17,7 @@ import torch
17
from huggingface_hub import HfApi, hf_hub_download
18
from torch import nn
19
from transformers import AutoModelForCausalLM, PretrainedConfig
20
+from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
21
22
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
23
LoRAConfig, ModelConfig, MultiModalConfig,
@@ -241,12 +242,17 @@ class DefaultModelLoader(BaseModelLoader):
242
is_local = os.path.isdir(model_name_or_path)
243
load_format = self.load_config.load_format
244
use_safetensors = False
245
+ index_file = SAFE_WEIGHTS_INDEX_NAME
246
# Some quantized models use .pt files for storing the weights.
247
if load_format == LoadFormat.AUTO:
248
allow_patterns = ["*.safetensors", "*.bin"]
249
elif load_format == LoadFormat.SAFETENSORS:
250
use_safetensors = True
251
allow_patterns = ["*.safetensors"]
252
+ elif load_format == LoadFormat.MISTRAL:
253
+ use_safetensors = True
254
+ allow_patterns = ["consolidated*.safetensors"]
255
+ index_file = "consolidated.safetensors.index.json"
256
elif load_format == LoadFormat.PT:
257
allow_patterns = ["*.pt"]
258
elif load_format == LoadFormat.NPCACHE:
@@ -284,10 +290,10 @@ class DefaultModelLoader(BaseModelLoader):
290
# any files not found in the index.
291
if not is_local:
292
download_safetensors_index_file_from_hf(
287
- model_name_or_path, self.load_config.download_dir,
288
- revision)
293
+ model_name_or_path, index_file,
294
+ self.load_config.download_dir, revision)
295
hf_weights_files = filter_duplicate_safetensors_files(
290
- hf_weights_files, hf_folder)
296
+ hf_weights_files, hf_folder, index_file)
297
else:
298
hf_weights_files = filter_files_not_needed_for_inference(
299
hf_weights_files)
vllm/model_executor/model_loader/weight_utils.py
+11
-10
@@ -16,7 +16,6 @@ import torch
16
from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download
17
from safetensors.torch import load_file, safe_open, save_file
18
from tqdm.auto import tqdm
19
-from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
19
20
from vllm.config import LoadConfig, ModelConfig
21
from vllm.distributed import get_tensor_model_parallel_rank
@@ -251,6 +250,7 @@ def download_weights_from_hf(
250
251
def download_safetensors_index_file_from_hf(
252
model_name_or_path: str,
253
+ index_file: str,
254
cache_dir: Optional[str],
255
revision: Optional[str] = None,
256
) -> None:
@@ -269,36 +269,37 @@ def download_safetensors_index_file_from_hf(
269
# Download the safetensors index file.
270
hf_hub_download(
271
repo_id=model_name_or_path,
272
- filename=SAFE_WEIGHTS_INDEX_NAME,
272
+ filename=index_file,
273
cache_dir=cache_dir,
274
revision=revision,
275
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
276
)
277
# If file not found on remote or locally, we should not fail since
278
- # only some models will have SAFE_WEIGHTS_INDEX_NAME.
278
+ # only some models will have index_file.
279
except huggingface_hub.utils.EntryNotFoundError:
280
- logger.info("No %s found in remote.", SAFE_WEIGHTS_INDEX_NAME)
280
+ logger.info("No %s found in remote.", index_file)
281
except huggingface_hub.utils.LocalEntryNotFoundError:
282
- logger.info("No %s found in local cache.", SAFE_WEIGHTS_INDEX_NAME)
282
+ logger.info("No %s found in local cache.", index_file)
283
284
285
# For models like Mistral-7B-v0.3, there are both sharded
286
# safetensors files and a consolidated safetensors file.
287
# Passing both of these to the weight loader functionality breaks.
288
-# So, we use the SAFE_WEIGHTS_INDEX_NAME to
288
+# So, we use the index_file to
289
# look up which safetensors files should be used.
290
def filter_duplicate_safetensors_files(hf_weights_files: List[str],
291
- hf_folder: str) -> List[str]:
291
+ hf_folder: str,
292
+ index_file: str) -> List[str]:
293
# model.safetensors.index.json is a mapping from keys in the
294
# torch state_dict to safetensors file holding that weight.
294
- index_file_name = os.path.join(hf_folder, SAFE_WEIGHTS_INDEX_NAME)
295
+ index_file_name = os.path.join(hf_folder, index_file)
296
if not os.path.isfile(index_file_name):
297
return hf_weights_files
298
299
# Iterate through the weight_map (weight_name: safetensors files)
300
# to identify weights that we should use.
300
- with open(index_file_name) as index_file:
301
- weight_map = json.load(index_file)["weight_map"]
301
+ with open(index_file_name, "r") as f:
302
+ weight_map = json.load(f)["weight_map"]
303
weight_files_in_index = set()
304
for weight_name in weight_map:
305
weight_files_in_index.add(
vllm/model_executor/models/llama.py
+51
@@ -375,6 +375,25 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
375
"gate_proj": ("gate_up_proj", 0),
376
"up_proj": ("gate_up_proj", 1),
377
}
378
+ # Mistral/Llama models can also be loaded with --load-format mistral
379
+ # from consolidated.safetensors checkpoints
380
+ mistral_mapping = {
381
+ "layers": "model.layers",
382
+ "attention": "self_attn",
383
+ "wq": "q_proj",
384
+ "wk": "k_proj",
385
+ "wv": "v_proj",
386
+ "wo": "o_proj",
387
+ "attention_norm": "input_layernorm",
388
+ "feed_forward": "mlp",
389
+ "w1": "gate_proj",
390
+ "w2": "down_proj",
391
+ "w3": "up_proj",
392
+ "ffn_norm": "post_attention_layernorm",
393
+ "tok_embeddings": "model.embed_tokens",
394
+ "output": "lm_head",
395
+ "norm": "model.norm"
396
+ }
397
398
def __init__(
399
self,
@@ -472,6 +491,8 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
491
]
492
params_dict = dict(self.named_parameters())
493
for name, loaded_weight in weights:
494
+ name, loaded_weight = self.maybe_remap_mistral(name, loaded_weight)
495
+
496
if "rotary_emb.inv_freq" in name:
497
continue
498
if ("rotary_emb.cos_cached" in name
@@ -549,3 +570,33 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
570
else:
571
raise RuntimeError("Self attention has no KV cache scaling "
572
"factor attribute!")
573
+
574
+ # This function is used to remap the mistral format as
575
+ # used by Mistral and Llama <=2
576
+ def maybe_remap_mistral(
577
+ self, name: str,
578
+ loaded_weight: torch.Tensor) -> Tuple[str, torch.Tensor]:
579
+
580
+ def permute(w, n_heads):
581
+ attn_in = self.config.head_dim * n_heads
582
+ attn_out = self.config.hidden_size
583
+
584
+ return w.view(n_heads, attn_in // n_heads // 2, 2,
585
+ attn_out).transpose(1, 2).reshape(attn_in, attn_out)
586
+
587
+ mapping = self.mistral_mapping
588
+ modules = name.split(".")
589
+
590
+ # rotary embeds should be sliced
591
+ if "wk" in modules:
592
+ loaded_weight = permute(loaded_weight,
593
+ self.config.num_key_value_heads)
594
+ elif "wq" in modules:
595
+ loaded_weight = permute(loaded_weight,
596
+ self.config.num_attention_heads)
597
+
598
+ for item in modules:
599
+ if item in mapping and mapping[item] not in name:
600
+ name = name.replace(item, mapping[item])
601
+
602
+ return name, loaded_weight
vllm/transformers_utils/config.py
+131
-34
@@ -1,12 +1,16 @@
1
import contextlib
2
+import enum
3
+import json
4
from pathlib import Path
5
from typing import Any, Dict, Optional, Type, Union
6
7
+from huggingface_hub import file_exists, hf_hub_download
8
from transformers import GenerationConfig, PretrainedConfig
9
from transformers.models.auto.image_processing_auto import (
10
get_image_processor_config)
11
from transformers.models.auto.modeling_auto import (
12
MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)
13
+from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
14
15
from vllm.envs import VLLM_USE_MODELSCOPE
16
from vllm.logger import init_logger
@@ -27,6 +31,8 @@ if VLLM_USE_MODELSCOPE:
31
else:
32
from transformers import AutoConfig
33
34
+MISTRAL_CONFIG_NAME = "params.json"
35
+
36
logger = init_logger(__name__)
37
38
_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
@@ -53,6 +59,20 @@ for name, cls in _CONFIG_REGISTRY.items():
59
AutoConfig.register(name, cls)
60
61
62
+class ConfigFormat(str, enum.Enum):
63
+ AUTO = "auto"
64
+ HF = "hf"
65
+ MISTRAL = "mistral"
66
+
67
+
68
+def file_or_path_exists(model: Union[str, Path], config_name, revision,
69
+ token) -> bool:
70
+ if Path(model).exists():
71
+ return (Path(model) / config_name).is_file()
72
+
73
+ return file_exists(model, HF_CONFIG_NAME, revision=revision, token=token)
74
+
75
+
76
def get_config(
77
model: Union[str, Path],
78
trust_remote_code: bool,
@@ -60,45 +80,68 @@ def get_config(
80
code_revision: Optional[str] = None,
81
rope_scaling: Optional[dict] = None,
82
rope_theta: Optional[float] = None,
83
+ config_format: ConfigFormat = ConfigFormat.AUTO,
84
**kwargs,
85
) -> PretrainedConfig:
65
-
86
# Separate model folder from file path for GGUF models
87
+
88
is_gguf = check_gguf_file(model)
89
if is_gguf:
90
kwargs["gguf_file"] = Path(model).name
91
model = Path(model).parent
92
72
- config_dict, _ = PretrainedConfig.get_config_dict(
73
- model, revision=revision, code_revision=code_revision, **kwargs)
93
+ if config_format == ConfigFormat.AUTO:
94
+ if is_gguf or file_or_path_exists(model,
95
+ HF_CONFIG_NAME,
96
+ revision=revision,
97
+ token=kwargs.get("token")):
98
+ config_format = ConfigFormat.HF
99
+ elif file_or_path_exists(model,
100
+ MISTRAL_CONFIG_NAME,
101
+ revision=revision,
102
+ token=kwargs.get("token")):
103
+ config_format = ConfigFormat.MISTRAL
104
+ else:
105
+ raise ValueError(f"No supported config format found in {model}")
106
+
107
+ if config_format == ConfigFormat.HF:
108
+ config_dict, _ = PretrainedConfig.get_config_dict(
109
+ model, revision=revision, code_revision=code_revision, **kwargs)
110
+
111
+ # Use custom model class if it's in our registry
112
+ model_type = config_dict.get("model_type")
113
+ if model_type in _CONFIG_REGISTRY:
114
+ config_class = _CONFIG_REGISTRY[model_type]
115
+ config = config_class.from_pretrained(model,
116
+ revision=revision,
117
+ code_revision=code_revision)
118
+ else:
119
+ try:
120
+ config = AutoConfig.from_pretrained(
121
+ model,
122
+ trust_remote_code=trust_remote_code,
123
+ revision=revision,
124
+ code_revision=code_revision,
125
+ **kwargs,
126
+ )
127
+ except ValueError as e:
128
+ if (not trust_remote_code
129
+ and "requires you to execute the configuration file"
130
+ in str(e)):
131
+ err_msg = (
132
+ "Failed to load the model config. If the model "
133
+ "is a custom model not yet available in the "
134
+ "HuggingFace transformers library, consider setting "
135
+ "`trust_remote_code=True` in LLM or using the "
136
+ "`--trust-remote-code` flag in the CLI.")
137
+ raise RuntimeError(err_msg) from e
138
+ else:
139
+ raise e
140
75
- # Use custom model class if it's in our registry
76
- model_type = config_dict.get("model_type")
77
- if model_type in _CONFIG_REGISTRY:
78
- config_class = _CONFIG_REGISTRY[model_type]
79
- config = config_class.from_pretrained(model,
80
- revision=revision,
81
- code_revision=code_revision)
141
+ elif config_format == ConfigFormat.MISTRAL:
142
+ config = load_params_config(model, revision)
143
else:
83
- try:
84
- config = AutoConfig.from_pretrained(
85
- model,
86
- trust_remote_code=trust_remote_code,
87
- revision=revision,
88
- code_revision=code_revision,
89
- **kwargs)
90
- except ValueError as e:
91
- if (not trust_remote_code
92
- and "requires you to execute the configuration file"
93
- in str(e)):
94
- err_msg = (
95
- "Failed to load the model config. If the model is a custom "
96
- "model not yet available in the HuggingFace transformers "
97
- "library, consider setting `trust_remote_code=True` in LLM "
98
- "or using the `--trust-remote-code` flag in the CLI.")
99
- raise RuntimeError(err_msg) from e
100
- else:
101
- raise e
144
+ raise ValueError(f"Unsupported config format: {config_format}")
145
146
# Special architecture mapping check for GGUF models
147
if is_gguf:
@@ -108,16 +151,70 @@ def get_config(
151
model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
152
config.update({"architectures": [model_type]})
153
111
- for key, value in [("rope_scaling", rope_scaling),
112
- ("rope_theta", rope_theta)]:
154
+ for key, value in [
155
+ ("rope_scaling", rope_scaling),
156
+ ("rope_theta", rope_theta),
157
+ ]:
158
if value is not None:
114
- logger.info("Updating %s from %r to %r", key,
115
- getattr(config, key, None), value)
159
+ logger.info(
160
+ "Updating %s from %r to %r",
161
+ key,
162
+ getattr(config, key, None),
163
+ value,
164
+ )
165
config.update({key: value})
166
167
return config
168
169
170
+def load_params_config(model, revision) -> PretrainedConfig:
171
+ # This function loads a params.json config which
172
+ # should be used when loading models in mistral format
173
+
174
+ config_file_name = "params.json"
175
+
176
+ config_path = Path(model) / config_file_name
177
+
178
+ if not config_path.is_file():
179
+ config_path = Path(
180
+ hf_hub_download(model, config_file_name, revision=revision))
181
+
182
+ with open(config_path, "r") as file:
183
+ config_dict = json.load(file)
184
+
185
+ config_mapping = {
186
+ "dim": "hidden_size",
187
+ "norm_eps": "rms_norm_eps",
188
+ "n_kv_heads": "num_key_value_heads",
189
+ "n_layers": "num_hidden_layers",
190
+ "n_heads": "num_attention_heads",
191
+ "hidden_dim": "intermediate_size",
192
+ }
193
+
194
+ def recurse_elems(elem: Any):
195
+ if isinstance(elem, dict):
196
+ config_dict = {}
197
+ for key, value in elem.items():
198
+ key = config_mapping.get(key, key)
199
+ config_dict[key] = recurse_elems(value)
200
+ return PretrainedConfig(**config_dict)
201
+ else:
202
+ return elem
203
+
204
+ config_dict["model_type"] = config_dict.get("model_type", "transformer")
205
+ config_dict["hidden_act"] = config_dict.get("activation", "silu")
206
+ config_dict["tie_word_embeddings"] = config_dict.get(
207
+ "tie_embeddings", False)
208
+
209
+ if config_dict["model_type"] == "transformer":
210
+ if "moe" in config_dict:
211
+ config_dict["architectures"] = ["MixtralForCausalLM"]
212
+ else:
213
+ config_dict["architectures"] = ["MistralForCausalLM"]
214
+
215
+ return recurse_elems(config_dict)
216
+
217
+
218
def get_hf_image_processor_config(
219
model: Union[str, Path],
220
revision: Optional[str] = None,
@@ -134,7 +231,7 @@ def get_hf_image_processor_config(
231
232
def get_hf_text_config(config: PretrainedConfig):
233
"""Get the "sub" config relevant to llm for multi modal models.
137
- No op for pure text models.
234
+ No op for pure text models.
235
"""
236
if hasattr(config, "text_config"):
237
# The code operates under the assumption that text_config should have