A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 00:30 UTC
[Model][Ouro] Support Ouro Model (#27794)
Signed-off-by: yinfan.1024 <yinfan.1024@bytedance.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: yinfan.1024 <yinfan.1024@bytedance.com> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Fan Yin committed
Oct 30, 2025 at 22:34 UTC
9956aae4ead0906abe7a1840a503587cab2013c1
4 files changed
+521
docs/models/supported_models.md
+1
@@ -403,6 +403,7 @@ th {
403
| `OLMoEForCausalLM` | OLMoE | `allenai/OLMoE-1B-7B-0924`, `allenai/OLMoE-1B-7B-0924-Instruct`, etc. | | ✅︎ |
404
| `OPTForCausalLM` | OPT, OPT-IML | `facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc. | ✅︎ | ✅︎ |
405
| `OrionForCausalLM` | Orion | `OrionStarAI/Orion-14B-Base`, `OrionStarAI/Orion-14B-Chat`, etc. | | ✅︎ |
406
+| `OuroForCausalLM` | ouro | `ByteDance/Ouro-1.4B`, `ByteDance/Ouro-2.6B`, etc. | ✅︎ | |
407
| `PhiForCausalLM` | Phi | `microsoft/phi-1_5`, `microsoft/phi-2`, etc. | ✅︎ | ✅︎ |
408
| `Phi3ForCausalLM` | Phi-4, Phi-3 | `microsoft/Phi-4-mini-instruct`, `microsoft/Phi-4`, `microsoft/Phi-3-mini-4k-instruct`, `microsoft/Phi-3-mini-128k-instruct`, `microsoft/Phi-3-medium-128k-instruct`, etc. | ✅︎ | ✅︎ |
409
| `PhiMoEForCausalLM` | Phi-3.5-MoE | `microsoft/Phi-3.5-MoE-instruct`, etc. | ✅︎ | ✅︎ |
tests/models/registry.py
+1
@@ -369,6 +369,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
369
"OrionForCausalLM": _HfExamplesInfo(
370
"OrionStarAI/Orion-14B-Chat", trust_remote_code=True
371
),
372
+ "OuroForCausalLM": _HfExamplesInfo("ByteDance/Ouro-1.4B", trust_remote_code=True),
373
"PersimmonForCausalLM": _HfExamplesInfo("adept/persimmon-8b-chat"),
374
"PhiForCausalLM": _HfExamplesInfo("microsoft/phi-2"),
375
"Phi3ForCausalLM": _HfExamplesInfo("microsoft/Phi-3-mini-4k-instruct"),
vllm/model_executor/models/ouro.py
new
+518
@@ -0,0 +1,518 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
5
+# Adapted from
6
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2/modeling_qwen2.py
7
+# Copyright 2024 The Qwen team.
8
+# Copyright 2023 The vLLM team.
9
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
10
+#
11
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
12
+# and OPT implementations in this library. It has been modified from its
13
+# original forms to accommodate minor architectural differences compared
14
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
15
+#
16
+# Licensed under the Apache License, Version 2.0 (the "License");
17
+# you may not use this file except in compliance with the License.
18
+# You may obtain a copy of the License at
19
+#
20
+# http://www.apache.org/licenses/LICENSE-2.0
21
+#
22
+# Unless required by applicable law or agreed to in writing, software
23
+# distributed under the License is distributed on an "AS IS" BASIS,
24
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25
+# See the License for the specific language governing permissions and
26
+# limitations under the License.
27
+"""Inference-only Ouro model compatible with HuggingFace weights."""
28
+
29
+from collections.abc import Iterable
30
+from typing import Any
31
+
32
+import torch
33
+from torch import nn
34
+from transformers import PretrainedConfig
35
+
36
+from vllm.attention import Attention, AttentionType
37
+from vllm.compilation.decorators import support_torch_compile
38
+from vllm.config import CacheConfig, VllmConfig
39
+from vllm.distributed import get_tensor_model_parallel_world_size
40
+from vllm.model_executor.layers.activation import SiluAndMul
41
+from vllm.model_executor.layers.layernorm import RMSNorm
42
+from vllm.model_executor.layers.linear import (
43
+ MergedColumnParallelLinear,
44
+ QKVParallelLinear,
45
+ RowParallelLinear,
46
+)
47
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
48
+from vllm.model_executor.layers.quantization import QuantizationConfig
49
+from vllm.model_executor.layers.rotary_embedding import get_rope
50
+from vllm.model_executor.layers.vocab_parallel_embedding import (
51
+ ParallelLMHead,
52
+ VocabParallelEmbedding,
53
+)
54
+from vllm.model_executor.model_loader.weight_utils import (
55
+ default_weight_loader,
56
+ maybe_remap_kv_scale_name,
57
+)
58
+from vllm.sequence import IntermediateTensors
59
+
60
+from .interfaces import SupportsLoRA
61
+from .utils import (
62
+ AutoWeightsLoader,
63
+ extract_layer_index,
64
+ make_empty_intermediate_tensors_factory,
65
+ make_layers,
66
+ maybe_prefix,
67
+)
68
+
69
+
70
+class OuroMLP(nn.Module):
71
+ def __init__(
72
+ self,
73
+ hidden_size: int,
74
+ intermediate_size: int,
75
+ hidden_act: str,
76
+ quant_config: QuantizationConfig | None = None,
77
+ prefix: str = "",
78
+ ) -> None:
79
+ super().__init__()
80
+ self.gate_up_proj = MergedColumnParallelLinear(
81
+ hidden_size,
82
+ [intermediate_size] * 2,
83
+ bias=False,
84
+ quant_config=quant_config,
85
+ prefix=f"{prefix}.gate_up_proj",
86
+ )
87
+ self.down_proj = RowParallelLinear(
88
+ intermediate_size,
89
+ hidden_size,
90
+ bias=False,
91
+ quant_config=quant_config,
92
+ prefix=f"{prefix}.down_proj",
93
+ )
94
+ if hidden_act != "silu":
95
+ raise ValueError(
96
+ f"Unsupported activation: {hidden_act}. Only silu is supported for now."
97
+ )
98
+ self.act_fn = SiluAndMul()
99
+
100
+ def forward(self, x):
101
+ gate_up, _ = self.gate_up_proj(x)
102
+ x = self.act_fn(gate_up)
103
+ x, _ = self.down_proj(x)
104
+ return x
105
+
106
+
107
+class OuroAttention(nn.Module):
108
+ def __init__(
109
+ self,
110
+ config: PretrainedConfig,
111
+ hidden_size: int,
112
+ num_heads: int,
113
+ num_kv_heads: int,
114
+ max_position: int = 4096 * 32,
115
+ rope_theta: float = 10000,
116
+ cache_config: CacheConfig | None = None,
117
+ quant_config: QuantizationConfig | None = None,
118
+ rope_scaling: tuple | None = None,
119
+ prefix: str = "",
120
+ attn_type: str = AttentionType.DECODER,
121
+ dual_chunk_attention_config: dict[str, Any] | None = None,
122
+ ) -> None:
123
+ super().__init__()
124
+ self.hidden_size = hidden_size
125
+ tp_size = get_tensor_model_parallel_world_size()
126
+ self.total_num_heads = num_heads
127
+ assert self.total_num_heads % tp_size == 0
128
+ self.num_heads = self.total_num_heads // tp_size
129
+ self.total_num_kv_heads = num_kv_heads
130
+ if self.total_num_kv_heads >= tp_size:
131
+ # Number of KV heads is greater than TP size, so we partition
132
+ # the KV heads across multiple tensor parallel GPUs.
133
+ assert self.total_num_kv_heads % tp_size == 0
134
+ else:
135
+ # Number of KV heads is less than TP size, so we replicate
136
+ # the KV heads across multiple tensor parallel GPUs.
137
+ assert tp_size % self.total_num_kv_heads == 0
138
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
139
+ self.head_dim = hidden_size // self.total_num_heads
140
+ self.q_size = self.num_heads * self.head_dim
141
+ self.kv_size = self.num_kv_heads * self.head_dim
142
+ self.scaling = self.head_dim**-0.5
143
+ self.rope_theta = rope_theta
144
+ self.dual_chunk_attention_config = dual_chunk_attention_config
145
+
146
+ # Get total_ut_steps from config, default to 4 if not specified
147
+ total_ut_steps = getattr(config, "total_ut_steps", 4)
148
+
149
+ # Use total number of hidden layers instead of hardcoded 24
150
+ total_layers = config.num_hidden_layers
151
+
152
+ self.qkv_proj = QKVParallelLinear(
153
+ hidden_size,
154
+ self.head_dim,
155
+ self.total_num_heads,
156
+ self.total_num_kv_heads,
157
+ bias=False,
158
+ quant_config=quant_config,
159
+ prefix=f"{prefix}.qkv_proj",
160
+ )
161
+ self.o_proj = RowParallelLinear(
162
+ self.total_num_heads * self.head_dim,
163
+ hidden_size,
164
+ bias=False,
165
+ quant_config=quant_config,
166
+ prefix=f"{prefix}.o_proj",
167
+ )
168
+
169
+ self.rotary_emb = get_rope(
170
+ self.head_dim,
171
+ rotary_dim=self.head_dim,
172
+ max_position=max_position,
173
+ base=self.rope_theta,
174
+ rope_scaling=rope_scaling,
175
+ dual_chunk_attention_config=dual_chunk_attention_config,
176
+ )
177
+ self.attn = nn.ModuleList()
178
+ for ut_step in range(total_ut_steps):
179
+ base_layer_idx = extract_layer_index(prefix)
180
+ unique_layer_idx = ut_step * total_layers + base_layer_idx
181
+
182
+ unique_prefix = prefix.replace(
183
+ f"layers.{base_layer_idx}", f"layers.{unique_layer_idx}"
184
+ )
185
+
186
+ self.attn.append(
187
+ Attention(
188
+ self.num_heads,
189
+ self.head_dim,
190
+ self.scaling,
191
+ num_kv_heads=self.num_kv_heads,
192
+ cache_config=cache_config,
193
+ quant_config=quant_config,
194
+ attn_type=attn_type,
195
+ prefix=f"{unique_prefix}.attn",
196
+ **{
197
+ "layer_idx": unique_layer_idx,
198
+ "dual_chunk_attention_config": dual_chunk_attention_config,
199
+ }
200
+ if dual_chunk_attention_config
201
+ else {},
202
+ )
203
+ )
204
+
205
+ def forward(
206
+ self,
207
+ positions: torch.Tensor,
208
+ hidden_states: torch.Tensor,
209
+ current_ut: int,
210
+ ) -> torch.Tensor:
211
+ qkv, _ = self.qkv_proj(hidden_states)
212
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
213
+ q, k = self.rotary_emb(positions, q, k)
214
+ attn_output = self.attn[current_ut](q, k, v)
215
+ output, _ = self.o_proj(attn_output)
216
+ return output
217
+
218
+
219
+class OuroDecoderLayer(nn.Module):
220
+ def __init__(
221
+ self,
222
+ config: PretrainedConfig,
223
+ cache_config: CacheConfig | None = None,
224
+ quant_config: QuantizationConfig | None = None,
225
+ prefix: str = "",
226
+ ) -> None:
227
+ super().__init__()
228
+ self.hidden_size = config.hidden_size
229
+ # Requires transformers > 4.32.0
230
+ rope_theta = getattr(config, "rope_theta", 1000000)
231
+ rope_scaling = getattr(config, "rope_scaling", None)
232
+ dual_chunk_attention_config = getattr(
233
+ config, "dual_chunk_attention_config", None
234
+ )
235
+
236
+ if getattr(config, "is_causal", True):
237
+ attn_type = AttentionType.DECODER
238
+ else:
239
+ attn_type = AttentionType.ENCODER_ONLY
240
+
241
+ self.self_attn = OuroAttention(
242
+ config=config,
243
+ hidden_size=self.hidden_size,
244
+ num_heads=config.num_attention_heads,
245
+ max_position=config.max_position_embeddings,
246
+ num_kv_heads=config.num_key_value_heads,
247
+ rope_theta=rope_theta,
248
+ cache_config=cache_config,
249
+ quant_config=quant_config,
250
+ rope_scaling=rope_scaling,
251
+ prefix=f"{prefix}.self_attn",
252
+ attn_type=attn_type,
253
+ dual_chunk_attention_config=dual_chunk_attention_config,
254
+ )
255
+ self.mlp = OuroMLP(
256
+ hidden_size=self.hidden_size,
257
+ intermediate_size=config.intermediate_size,
258
+ hidden_act=config.hidden_act,
259
+ quant_config=quant_config,
260
+ prefix=f"{prefix}.mlp",
261
+ )
262
+ self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
263
+ self.input_layernorm_2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
264
+
265
+ self.post_attention_layernorm = RMSNorm(
266
+ config.hidden_size, eps=config.rms_norm_eps
267
+ )
268
+ self.post_attention_layernorm_2 = RMSNorm(
269
+ config.hidden_size, eps=config.rms_norm_eps
270
+ )
271
+
272
+ def forward(
273
+ self,
274
+ positions: torch.Tensor,
275
+ hidden_states: torch.Tensor,
276
+ current_ut: int,
277
+ residual: torch.Tensor | None = None,
278
+ ) -> tuple[torch.Tensor, torch.Tensor]:
279
+ if residual is None:
280
+ residual = hidden_states
281
+ hidden_states = self.input_layernorm(hidden_states)
282
+ else:
283
+ hidden_states, residual = self.input_layernorm(hidden_states, residual)
284
+ hidden_states = self.self_attn(
285
+ positions=positions, hidden_states=hidden_states, current_ut=current_ut
286
+ )
287
+ hidden_states = self.input_layernorm_2(hidden_states)
288
+
289
+ hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
290
+ hidden_states = self.mlp(hidden_states)
291
+ hidden_states = self.post_attention_layernorm_2(hidden_states)
292
+
293
+ return hidden_states, residual
294
+
295
+
296
+@support_torch_compile(
297
+ dynamic_arg_dims={
298
+ "input_ids": 0,
299
+ "positions": -1,
300
+ "intermediate_tensors": 0,
301
+ "inputs_embeds": 0,
302
+ }
303
+)
304
+class OuroModel(nn.Module):
305
+ def __init__(
306
+ self,
307
+ *,
308
+ vllm_config: VllmConfig,
309
+ prefix: str = "",
310
+ decoder_layer_type: type[nn.Module] = OuroDecoderLayer,
311
+ ):
312
+ super().__init__()
313
+
314
+ config = vllm_config.model_config.hf_config
315
+ cache_config = vllm_config.cache_config
316
+ quant_config = vllm_config.quant_config
317
+
318
+ # TODO (@robertgshaw2): see if this can be moved out
319
+ if cache_config.sliding_window is not None and hasattr(
320
+ config, "max_window_layers"
321
+ ):
322
+ assert config.max_window_layers == config.num_hidden_layers, (
323
+ "Sliding window for some but all layers is not supported. "
324
+ "This model uses sliding window but `max_window_layers` = {} "
325
+ "is less than `num_hidden_layers` = {}. Please open an issue "
326
+ "to discuss this feature.".format(
327
+ config.max_window_layers,
328
+ config.num_hidden_layers,
329
+ )
330
+ )
331
+
332
+ self.config = config
333
+ self.quant_config = quant_config
334
+ self.vocab_size = config.vocab_size
335
+
336
+ self.embed_tokens = VocabParallelEmbedding(
337
+ config.vocab_size,
338
+ config.hidden_size,
339
+ quant_config=quant_config,
340
+ prefix=f"{prefix}.embed_tokens",
341
+ )
342
+
343
+ # Use the provided decoder layer type or default to OuroDecoderLayer
344
+ decoder_layer_type = decoder_layer_type or OuroDecoderLayer
345
+ self.start_layer, self.end_layer, self.layers = make_layers(
346
+ config.num_hidden_layers,
347
+ lambda prefix: decoder_layer_type(
348
+ config=config,
349
+ cache_config=cache_config,
350
+ quant_config=quant_config,
351
+ prefix=prefix,
352
+ ),
353
+ prefix=f"{prefix}.layers",
354
+ )
355
+
356
+ self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
357
+ ["hidden_states", "residual"], config.hidden_size
358
+ )
359
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
360
+ self.early_exit_gate = RowParallelLinear(config.hidden_size, 1, bias=True)
361
+
362
+ self.total_ut_steps = getattr(self.config, "total_ut_steps", 4)
363
+
364
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
365
+ return self.embed_tokens(input_ids)
366
+
367
+ def forward(
368
+ self,
369
+ input_ids: torch.Tensor,
370
+ positions: torch.Tensor,
371
+ intermediate_tensors: IntermediateTensors | None = None,
372
+ inputs_embeds: torch.Tensor | None = None,
373
+ ) -> torch.Tensor | IntermediateTensors:
374
+ if inputs_embeds is not None:
375
+ hidden_states = inputs_embeds
376
+ else:
377
+ hidden_states = self.get_input_embeddings(input_ids)
378
+
379
+ for current_ut in range(self.total_ut_steps):
380
+ residual = None
381
+ for layer in self.layers[self.start_layer : self.end_layer]:
382
+ hidden_states, residual = layer(
383
+ positions, hidden_states, current_ut, residual
384
+ )
385
+ hidden_states, _ = self.norm(hidden_states, residual)
386
+ return hidden_states
387
+
388
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
389
+ stacked_params_mapping = [
390
+ # (param_name, shard_name, shard_id)
391
+ ("qkv_proj", "q_proj", "q"),
392
+ ("qkv_proj", "k_proj", "k"),
393
+ ("qkv_proj", "v_proj", "v"),
394
+ ("gate_up_proj", "gate_proj", 0),
395
+ ("gate_up_proj", "up_proj", 1),
396
+ ]
397
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
398
+ loaded_params: set[str] = set()
399
+ for name, loaded_weight in weights:
400
+ if "rotary_emb.inv_freq" in name:
401
+ continue
402
+ if self.quant_config is not None and (
403
+ scale_name := self.quant_config.get_cache_scale(name)
404
+ ):
405
+ # Loading kv cache quantization scales
406
+ param = params_dict[scale_name]
407
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
408
+ loaded_weight = (
409
+ loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
410
+ )
411
+ weight_loader(param, loaded_weight)
412
+ loaded_params.add(scale_name)
413
+ continue
414
+ for param_name, weight_name, shard_id in stacked_params_mapping:
415
+ if weight_name not in name:
416
+ continue
417
+ name = name.replace(weight_name, param_name)
418
+ # Skip loading extra bias for GPTQ models.
419
+ if name.endswith(".bias") and name not in params_dict:
420
+ continue
421
+ if name.endswith("scale"):
422
+ # Remapping the name of FP8 kv-scale.
423
+ name = maybe_remap_kv_scale_name(name, params_dict)
424
+ if name is None:
425
+ continue
426
+ param = params_dict[name]
427
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
428
+ if weight_loader == default_weight_loader:
429
+ weight_loader(param, loaded_weight)
430
+ else:
431
+ weight_loader(param, loaded_weight, shard_id)
432
+ break
433
+ else:
434
+ # Skip loading extra bias for GPTQ models.
435
+ if name.endswith(".bias") and name not in params_dict:
436
+ continue
437
+ # Remapping the name of FP8 kv-scale.
438
+ name = maybe_remap_kv_scale_name(name, params_dict)
439
+ if name is None:
440
+ continue
441
+ param = params_dict[name]
442
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
443
+ weight_loader(param, loaded_weight)
444
+ loaded_params.add(name)
445
+ return loaded_params
446
+
447
+
448
+class OuroForCausalLM(nn.Module, SupportsLoRA):
449
+ packed_modules_mapping = {
450
+ "qkv_proj": [
451
+ "q_proj",
452
+ "k_proj",
453
+ "v_proj",
454
+ ],
455
+ "gate_up_proj": [
456
+ "gate_proj",
457
+ "up_proj",
458
+ ],
459
+ }
460
+
461
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
462
+ super().__init__()
463
+ config = vllm_config.model_config.hf_config
464
+ quant_config = vllm_config.quant_config
465
+ lora_config = vllm_config.lora_config
466
+
467
+ self.config = config
468
+ self.lora_config = lora_config
469
+
470
+ self.quant_config = quant_config
471
+ self.model = OuroModel(
472
+ vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
473
+ )
474
+
475
+ if config.tie_word_embeddings:
476
+ self.lm_head = self.model.embed_tokens
477
+ else:
478
+ self.lm_head = ParallelLMHead(
479
+ config.vocab_size,
480
+ config.hidden_size,
481
+ quant_config=quant_config,
482
+ prefix=maybe_prefix(prefix, "lm_head"),
483
+ )
484
+
485
+ self.logits_processor = LogitsProcessor(config.vocab_size)
486
+
487
+ self.make_empty_intermediate_tensors = (
488
+ self.model.make_empty_intermediate_tensors
489
+ )
490
+
491
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
492
+ return self.model.get_input_embeddings(input_ids)
493
+
494
+ def forward(
495
+ self,
496
+ input_ids: torch.Tensor,
497
+ positions: torch.Tensor,
498
+ intermediate_tensors: IntermediateTensors | None = None,
499
+ inputs_embeds: torch.Tensor | None = None,
500
+ ) -> torch.Tensor | IntermediateTensors:
501
+ hidden_states = self.model(
502
+ input_ids, positions, intermediate_tensors, inputs_embeds
503
+ )
504
+ return hidden_states
505
+
506
+ def compute_logits(
507
+ self,
508
+ hidden_states: torch.Tensor,
509
+ ) -> torch.Tensor | None:
510
+ logits = self.logits_processor(self.lm_head, hidden_states)
511
+ return logits
512
+
513
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
514
+ loader = AutoWeightsLoader(
515
+ self,
516
+ skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
517
+ )
518
+ return loader.load_weights(weights)
vllm/model_executor/models/registry.py
+1
@@ -148,6 +148,7 @@ _TEXT_GENERATION_MODELS = {
148
"OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
149
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
150
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
151
+ "OuroForCausalLM": ("ouro", "OuroForCausalLM"),
152
"PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"),
153
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
154
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),