A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Model] support dots1 (#18254)
Signed-off-by: redmoe-moutain <agiredmoe@gmail.com>
redmoe-moutain committed
Jun 30, 2025 at 10:34 UTC
65b1cbb1381bf2301a2441fd988bbe88b4b7865e
4 files changed
+539
docs/models/supported_models.md
+1
@@ -329,6 +329,7 @@ Specified using `--task generate`.
329
| `DeepseekForCausalLM` | DeepSeek | `deepseek-ai/deepseek-llm-67b-base`, `deepseek-ai/deepseek-llm-7b-chat` etc. | | ✅︎ | ✅︎ |
330
| `DeepseekV2ForCausalLM` | DeepSeek-V2 | `deepseek-ai/DeepSeek-V2`, `deepseek-ai/DeepSeek-V2-Chat` etc. | | ✅︎ | ✅︎ |
331
| `DeepseekV3ForCausalLM` | DeepSeek-V3 | `deepseek-ai/DeepSeek-V3-Base`, `deepseek-ai/DeepSeek-V3` etc. | | ✅︎ | ✅︎ |
332
+| `Dots1ForCausalLM` | dots.llm1 | `rednote-hilab/dots.llm1.base`, `rednote-hilab/dots.llm1.inst` etc. | | ✅︎ | ✅︎ |
333
| `ExaoneForCausalLM` | EXAONE-3 | `LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
334
| `FalconForCausalLM` | Falcon | `tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc. | | ✅︎ | ✅︎ |
335
| `FalconMambaForCausalLM` | FalconMamba | `tiiuae/falcon-mamba-7b`, `tiiuae/falcon-mamba-7b-instruct`, etc. | | ✅︎ | ✅︎ |
tests/models/registry.py
+2
@@ -268,6 +268,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
268
"Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
269
"MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL",
270
trust_remote_code=True),
271
+ "Dots1ForCausalLM": _HfExamplesInfo("rednote-hilab/dots.llm1.inst",
272
+ min_transformers_version="4.53"),
273
# [Encoder-decoder]
274
"BartModel": _HfExamplesInfo("facebook/bart-base"),
275
"BartForConditionalGeneration": _HfExamplesInfo("facebook/bart-large-cnn"),
vllm/model_executor/models/dots1.py
new
+535
@@ -0,0 +1,535 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+# Adapted from
4
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
5
+# Copyright 2025 The rednote-hilab team.
6
+# Copyright 2023 The vLLM team.
7
+# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved.
8
+#
9
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
10
+# and OPT implementations in this library. It has been modified from its
11
+# original forms to accommodate minor architectural differences compared
12
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
13
+#
14
+# Licensed under the Apache License, Version 2.0 (the "License");
15
+# you may not use this file except in compliance with the License.
16
+# You may obtain a copy of the License at
17
+#
18
+# http://www.apache.org/licenses/LICENSE-2.0
19
+#
20
+# Unless required by applicable law or agreed to in writing, software
21
+# distributed under the License is distributed on an "AS IS" BASIS,
22
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23
+# See the License for the specific language governing permissions and
24
+# limitations under the License.
25
+"""Inference-only dots1 model."""
26
+from collections.abc import Iterable
27
+from typing import Any, Optional, Union
28
+
29
+import torch
30
+from torch import nn
31
+from transformers import PretrainedConfig
32
+
33
+from vllm.attention import Attention
34
+from vllm.compilation.decorators import support_torch_compile
35
+from vllm.config import CacheConfig, ModelConfig, VllmConfig
36
+from vllm.distributed import (get_pp_group,
37
+ get_tensor_model_parallel_world_size,
38
+ tensor_model_parallel_all_reduce)
39
+from vllm.model_executor.layers.activation import SiluAndMul
40
+from vllm.model_executor.layers.fused_moe import FusedMoE
41
+from vllm.model_executor.layers.layernorm import RMSNorm
42
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
43
+ QKVParallelLinear,
44
+ ReplicatedLinear,
45
+ RowParallelLinear)
46
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
47
+from vllm.model_executor.layers.quantization import QuantizationConfig
48
+from vllm.model_executor.layers.rotary_embedding import get_rope
49
+from vllm.model_executor.layers.vocab_parallel_embedding import (
50
+ ParallelLMHead, VocabParallelEmbedding)
51
+from vllm.model_executor.model_loader.weight_utils import (
52
+ default_weight_loader, maybe_remap_kv_scale_name)
53
+from vllm.model_executor.sampling_metadata import SamplingMetadata
54
+from vllm.sequence import IntermediateTensors
55
+
56
+from .interfaces import SupportsPP
57
+from .utils import (PPMissingLayer, is_pp_missing_parameter,
58
+ make_empty_intermediate_tensors_factory, make_layers,
59
+ maybe_prefix)
60
+
61
+
62
+class Dots1MLP(nn.Module):
63
+
64
+ def __init__(
65
+ self,
66
+ hidden_size: int,
67
+ intermediate_size: int,
68
+ hidden_act: str,
69
+ quant_config: Optional[QuantizationConfig] = None,
70
+ reduce_results: bool = True,
71
+ prefix: str = "",
72
+ ) -> None:
73
+ super().__init__()
74
+ self.gate_up_proj = MergedColumnParallelLinear(
75
+ hidden_size, [intermediate_size] * 2,
76
+ bias=False,
77
+ quant_config=quant_config,
78
+ prefix=f"{prefix}.gate_up_proj")
79
+ self.down_proj = RowParallelLinear(intermediate_size,
80
+ hidden_size,
81
+ bias=False,
82
+ quant_config=quant_config,
83
+ reduce_results=reduce_results,
84
+ prefix=f"{prefix}.down_proj")
85
+ if hidden_act != "silu":
86
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
87
+ "Only silu is supported for now.")
88
+ self.act_fn = SiluAndMul()
89
+
90
+ def forward(self, x):
91
+ gate_up, _ = self.gate_up_proj(x)
92
+ x = self.act_fn(gate_up)
93
+ x, _ = self.down_proj(x)
94
+ return x
95
+
96
+
97
+class Dots1MoE(nn.Module):
98
+
99
+ def __init__(
100
+ self,
101
+ config: PretrainedConfig,
102
+ quant_config: Optional[QuantizationConfig] = None,
103
+ prefix: str = "",
104
+ ):
105
+ super().__init__()
106
+ self.tp_size = get_tensor_model_parallel_world_size()
107
+ self.routed_scaling_factor = config.routed_scaling_factor
108
+ self.n_shared_experts = config.n_shared_experts
109
+
110
+ if config.hidden_act != "silu":
111
+ raise ValueError(f"Unsupported activation: {config.hidden_act}. "
112
+ "Only silu is supported for now.")
113
+
114
+ self.gate = ReplicatedLinear(config.hidden_size,
115
+ config.n_routed_experts,
116
+ bias=False,
117
+ quant_config=None,
118
+ prefix=f"{prefix}.gate")
119
+ if config.topk_method == "noaux_tc":
120
+ self.gate.e_score_correction_bias = (nn.Parameter(
121
+ torch.empty(config.n_routed_experts)))
122
+ else:
123
+ self.gate.e_score_correction_bias = None
124
+
125
+ self.experts = FusedMoE(
126
+ num_experts=config.n_routed_experts,
127
+ top_k=config.num_experts_per_tok,
128
+ hidden_size=config.hidden_size,
129
+ intermediate_size=config.moe_intermediate_size,
130
+ reduce_results=False,
131
+ renormalize=config.norm_topk_prob,
132
+ quant_config=quant_config,
133
+ use_grouped_topk=True,
134
+ num_expert_group=config.n_group,
135
+ topk_group=config.topk_group,
136
+ prefix=f"{prefix}.experts",
137
+ scoring_func=config.scoring_func,
138
+ e_score_correction_bias=self.gate.e_score_correction_bias)
139
+
140
+ if config.n_shared_experts is not None:
141
+ intermediate_size = (config.moe_intermediate_size *
142
+ config.n_shared_experts)
143
+ self.shared_experts = Dots1MLP(
144
+ hidden_size=config.hidden_size,
145
+ intermediate_size=intermediate_size,
146
+ hidden_act=config.hidden_act,
147
+ quant_config=quant_config,
148
+ reduce_results=False,
149
+ prefix=f"{prefix}.shared_experts",
150
+ )
151
+
152
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
153
+ num_tokens, hidden_dim = hidden_states.shape
154
+ hidden_states = hidden_states.view(-1, hidden_dim)
155
+ if self.n_shared_experts is not None:
156
+ shared_output = self.shared_experts(hidden_states)
157
+ router_logits, _ = self.gate(hidden_states)
158
+ final_hidden_states = self.experts(
159
+ hidden_states=hidden_states,
160
+ router_logits=router_logits) * self.routed_scaling_factor
161
+ if shared_output is not None:
162
+ final_hidden_states = final_hidden_states + shared_output
163
+ if self.tp_size > 1:
164
+ final_hidden_states = tensor_model_parallel_all_reduce(
165
+ final_hidden_states)
166
+ return final_hidden_states.view(num_tokens, hidden_dim)
167
+
168
+
169
+class Dots1Attention(nn.Module):
170
+
171
+ def __init__(
172
+ self,
173
+ hidden_size: int,
174
+ num_heads: int,
175
+ num_kv_heads: int,
176
+ config: PretrainedConfig,
177
+ rope_theta: float = 10000,
178
+ rope_scaling: Optional[dict[str, Any]] = None,
179
+ max_position_embeddings: int = 8192,
180
+ cache_config: Optional[CacheConfig] = None,
181
+ quant_config: Optional[QuantizationConfig] = None,
182
+ prefix: str = "",
183
+ ) -> None:
184
+ super().__init__()
185
+ self.hidden_size = hidden_size
186
+ tp_size = get_tensor_model_parallel_world_size()
187
+ self.total_num_heads = num_heads
188
+ assert self.total_num_heads % tp_size == 0
189
+ self.num_heads = self.total_num_heads // tp_size
190
+ self.total_num_kv_heads = num_kv_heads
191
+ if self.total_num_kv_heads >= tp_size:
192
+ # Number of KV heads is greater than TP size, so we partition
193
+ # the KV heads across multiple tensor parallel GPUs.
194
+ assert self.total_num_kv_heads % tp_size == 0
195
+ else:
196
+ # Number of KV heads is less than TP size, so we replicate
197
+ # the KV heads across multiple tensor parallel GPUs.
198
+ assert tp_size % self.total_num_kv_heads == 0
199
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
200
+ self.head_dim = getattr(config, "head_dim",
201
+ hidden_size // self.total_num_heads)
202
+ self.q_size = self.num_heads * self.head_dim
203
+ self.kv_size = self.num_kv_heads * self.head_dim
204
+ self.scaling = self.head_dim**-0.5
205
+ self.rope_theta = rope_theta
206
+ self.max_position_embeddings = max_position_embeddings
207
+ attention_bias = config.attention_bias
208
+
209
+ self.qkv_proj = QKVParallelLinear(
210
+ hidden_size,
211
+ self.head_dim,
212
+ self.total_num_heads,
213
+ self.total_num_kv_heads,
214
+ bias=attention_bias,
215
+ quant_config=quant_config,
216
+ )
217
+
218
+ self.o_proj = RowParallelLinear(
219
+ self.total_num_heads * self.head_dim,
220
+ hidden_size,
221
+ bias=False,
222
+ quant_config=quant_config,
223
+ )
224
+
225
+ self.rotary_emb = get_rope(
226
+ self.head_dim,
227
+ rotary_dim=self.head_dim,
228
+ max_position=max_position_embeddings,
229
+ base=rope_theta,
230
+ rope_scaling=rope_scaling,
231
+ )
232
+ self.attn = Attention(
233
+ self.num_heads,
234
+ self.head_dim,
235
+ self.scaling,
236
+ num_kv_heads=self.num_kv_heads,
237
+ cache_config=cache_config,
238
+ quant_config=quant_config,
239
+ prefix=f"{prefix}.attn",
240
+ )
241
+ self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
242
+ self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
243
+
244
+ def forward(self, positions: torch.Tensor,
245
+ hidden_states: torch.Tensor) -> torch.Tensor:
246
+ qkv, _ = self.qkv_proj(hidden_states)
247
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
248
+ q = self.q_norm(q.reshape(-1, self.num_heads,
249
+ self.head_dim)).reshape(q.shape)
250
+ k = self.k_norm(k.reshape(-1, self.num_kv_heads,
251
+ self.head_dim)).reshape(k.shape)
252
+ q, k = self.rotary_emb(positions, q, k)
253
+ attn_output = self.attn(q, k, v)
254
+ output, _ = self.o_proj(attn_output)
255
+ return output
256
+
257
+
258
+class Dots1DecoderLayer(nn.Module):
259
+
260
+ def __init__(
261
+ self,
262
+ config: PretrainedConfig,
263
+ prefix: str,
264
+ model_config: ModelConfig,
265
+ cache_config: Optional[CacheConfig] = None,
266
+ quant_config: Optional[QuantizationConfig] = None,
267
+ ) -> None:
268
+ super().__init__()
269
+ self.hidden_size = config.hidden_size
270
+ rope_theta = getattr(config, "rope_theta", 10000)
271
+ rope_scaling = getattr(config, "rope_scaling", None)
272
+ max_position_embeddings = getattr(config, "max_position_embeddings",
273
+ 8192)
274
+ layer_idx = int(prefix.split(sep='.')[-1])
275
+ self.layer_idx = layer_idx
276
+
277
+ self.self_attn = Dots1Attention(
278
+ hidden_size=self.hidden_size,
279
+ num_heads=config.num_attention_heads,
280
+ num_kv_heads=config.num_key_value_heads,
281
+ config=config,
282
+ rope_theta=rope_theta,
283
+ rope_scaling=rope_scaling,
284
+ max_position_embeddings=max_position_embeddings,
285
+ cache_config=cache_config,
286
+ quant_config=quant_config,
287
+ prefix=f"{prefix}.self_attn",
288
+ )
289
+ if (config.n_routed_experts is not None
290
+ and layer_idx >= config.first_k_dense_replace
291
+ and layer_idx % config.moe_layer_freq == 0):
292
+ self.mlp = Dots1MoE(config=config,
293
+ quant_config=quant_config,
294
+ prefix=f"{prefix}.mlp")
295
+ else:
296
+ self.mlp = Dots1MLP(
297
+ hidden_size=config.hidden_size,
298
+ intermediate_size=config.intermediate_size,
299
+ hidden_act=config.hidden_act,
300
+ quant_config=quant_config,
301
+ prefix=f"{prefix}.mlp",
302
+ )
303
+ self.input_layernorm = RMSNorm(config.hidden_size,
304
+ eps=config.rms_norm_eps)
305
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
306
+ eps=config.rms_norm_eps)
307
+ self.routed_scaling_factor = config.routed_scaling_factor
308
+
309
+ def forward(
310
+ self,
311
+ positions: torch.Tensor,
312
+ hidden_states: torch.Tensor,
313
+ residual: Optional[torch.Tensor],
314
+ ) -> torch.Tensor:
315
+ if residual is None:
316
+ residual = hidden_states
317
+ hidden_states = self.input_layernorm(hidden_states)
318
+ else:
319
+ hidden_states, residual = self.input_layernorm(
320
+ hidden_states, residual)
321
+ hidden_states = self.self_attn(positions=positions,
322
+ hidden_states=hidden_states)
323
+ hidden_states, residual = self.post_attention_layernorm(
324
+ hidden_states, residual)
325
+ hidden_states = self.mlp(hidden_states)
326
+ return hidden_states, residual
327
+
328
+
329
+class Dots1Model(nn.Module):
330
+
331
+ fall_back_to_pt_during_load = False
332
+
333
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
334
+ super().__init__()
335
+
336
+ config = vllm_config.model_config.hf_config
337
+ model_config = vllm_config.model_config
338
+ cache_config = vllm_config.cache_config
339
+ quant_config = vllm_config.quant_config
340
+ self.config = config
341
+
342
+ self.vocab_size = config.vocab_size
343
+
344
+ if get_pp_group().is_first_rank:
345
+ self.embed_tokens = VocabParallelEmbedding(
346
+ config.vocab_size,
347
+ config.hidden_size,
348
+ quant_config=quant_config,
349
+ prefix=f"{prefix}.embed_tokens")
350
+ else:
351
+ self.embed_tokens = PPMissingLayer()
352
+
353
+ self.start_layer, self.end_layer, self.layers = make_layers(
354
+ config.num_hidden_layers,
355
+ lambda prefix: Dots1DecoderLayer(
356
+ config,
357
+ prefix,
358
+ model_config=model_config,
359
+ cache_config=cache_config,
360
+ quant_config=quant_config,
361
+ ),
362
+ prefix=f"{prefix}.layers")
363
+
364
+ if get_pp_group().is_last_rank:
365
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
366
+ else:
367
+ self.norm = PPMissingLayer()
368
+ self.make_empty_intermediate_tensors = (
369
+ make_empty_intermediate_tensors_factory(
370
+ ["hidden_states", "residual"], config.hidden_size))
371
+
372
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
373
+ return self.embed_tokens(input_ids)
374
+
375
+ def forward(
376
+ self,
377
+ input_ids: torch.Tensor,
378
+ positions: torch.Tensor,
379
+ intermediate_tensors: Optional[IntermediateTensors],
380
+ inputs_embeds: Optional[torch.Tensor] = None,
381
+ ) -> Union[torch.Tensor, IntermediateTensors]:
382
+ if get_pp_group().is_first_rank:
383
+ if inputs_embeds is not None:
384
+ hidden_states = inputs_embeds
385
+ else:
386
+ hidden_states = self.get_input_embeddings(input_ids)
387
+ residual = None
388
+ else:
389
+ assert intermediate_tensors is not None
390
+ hidden_states = intermediate_tensors["hidden_states"]
391
+ residual = intermediate_tensors["residual"]
392
+ for layer in self.layers[self.start_layer:self.end_layer]:
393
+ hidden_states, residual = layer(
394
+ positions,
395
+ hidden_states,
396
+ residual,
397
+ )
398
+ if not get_pp_group().is_last_rank:
399
+ return IntermediateTensors({
400
+ "hidden_states": hidden_states,
401
+ "residual": residual
402
+ })
403
+ hidden_states, _ = self.norm(hidden_states, residual)
404
+ return hidden_states
405
+
406
+
407
+@support_torch_compile
408
+class Dots1ForCausalLM(nn.Module, SupportsPP):
409
+
410
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
411
+ super().__init__()
412
+ config = vllm_config.model_config.hf_config
413
+ quant_config = vllm_config.quant_config
414
+ self.config = config
415
+ self.quant_config = quant_config
416
+ self.model = Dots1Model(vllm_config=vllm_config,
417
+ prefix=maybe_prefix(prefix, "model"))
418
+ if get_pp_group().is_last_rank:
419
+ self.lm_head = ParallelLMHead(config.vocab_size,
420
+ config.hidden_size,
421
+ quant_config=quant_config)
422
+ else:
423
+ self.lm_head = PPMissingLayer()
424
+ self.logits_processor = LogitsProcessor(config.vocab_size)
425
+ self.make_empty_intermediate_tensors = (
426
+ self.model.make_empty_intermediate_tensors)
427
+
428
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
429
+ return self.model.get_input_embeddings(input_ids)
430
+
431
+ def forward(
432
+ self,
433
+ input_ids: torch.Tensor,
434
+ positions: torch.Tensor,
435
+ intermediate_tensors: Optional[IntermediateTensors] = None,
436
+ inputs_embeds: Optional[torch.Tensor] = None,
437
+ ) -> Union[torch.Tensor, IntermediateTensors]:
438
+ hidden_states = self.model(
439
+ input_ids,
440
+ positions,
441
+ intermediate_tensors,
442
+ inputs_embeds,
443
+ )
444
+ return hidden_states
445
+
446
+ def compute_logits(
447
+ self,
448
+ hidden_states: torch.Tensor,
449
+ sampling_metadata: SamplingMetadata,
450
+ ) -> Optional[torch.Tensor]:
451
+ logits = self.logits_processor(self.lm_head, hidden_states,
452
+ sampling_metadata)
453
+ return logits
454
+
455
+ def make_empty_intermediate_tensors(
456
+ self, batch_size: int, dtype: torch.dtype,
457
+ device: torch.device) -> IntermediateTensors:
458
+ return IntermediateTensors({
459
+ "hidden_states":
460
+ torch.zeros((batch_size, self.config.hidden_size),
461
+ dtype=dtype,
462
+ device=device),
463
+ "residual":
464
+ torch.zeros((batch_size, self.config.hidden_size),
465
+ dtype=dtype,
466
+ device=device),
467
+ })
468
+
469
+ def load_weights(self, weights: Iterable[tuple[str,
470
+ torch.Tensor]]) -> set[str]:
471
+ stacked_params_mapping = [
472
+ ("qkv_proj", "q_proj", "q"),
473
+ ("qkv_proj", "k_proj", "k"),
474
+ ("qkv_proj", "v_proj", "v"),
475
+ ("gate_up_proj", "gate_proj", 0),
476
+ ("gate_up_proj", "up_proj", 1),
477
+ ]
478
+
479
+ expert_params_mapping = FusedMoE.make_expert_params_mapping(
480
+ ckpt_gate_proj_name="gate_proj",
481
+ ckpt_down_proj_name="down_proj",
482
+ ckpt_up_proj_name="up_proj",
483
+ num_experts=self.config.n_routed_experts)
484
+
485
+ params_dict = dict(self.named_parameters())
486
+ loaded_params: set[str] = set()
487
+ for name, loaded_weight in weights:
488
+ if "rotary_emb.inv_freq" in name:
489
+ continue
490
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
491
+ if weight_name not in name:
492
+ continue
493
+ if (("mlp.experts." in name) and name not in params_dict):
494
+ continue
495
+ name = name.replace(weight_name, param_name)
496
+ if name.endswith(".bias") and name not in params_dict:
497
+ continue
498
+ if is_pp_missing_parameter(name, self):
499
+ continue
500
+ param = params_dict[name]
501
+ weight_loader = param.weight_loader
502
+ weight_loader(param, loaded_weight, shard_id)
503
+ break
504
+ else:
505
+ for mapping in expert_params_mapping:
506
+ param_name, weight_name, expert_id, shard_id = mapping
507
+ if weight_name not in name:
508
+ continue
509
+ name = name.replace(weight_name, param_name)
510
+
511
+ if is_pp_missing_parameter(name, self):
512
+ continue
513
+
514
+ param = params_dict[name]
515
+ weight_loader = param.weight_loader
516
+ weight_loader(param,
517
+ loaded_weight,
518
+ name,
519
+ shard_id=shard_id,
520
+ expert_id=expert_id)
521
+ break
522
+ else:
523
+ if name.endswith(".bias") and name not in params_dict:
524
+ continue
525
+ name = maybe_remap_kv_scale_name(name, params_dict)
526
+ if name is None:
527
+ continue
528
+ if is_pp_missing_parameter(name, self):
529
+ continue
530
+ param = params_dict[name]
531
+ weight_loader = getattr(param, "weight_loader",
532
+ default_weight_loader)
533
+ weight_loader(param, loaded_weight)
534
+ loaded_params.add(name)
535
+ return loaded_params
vllm/model_executor/models/registry.py
+1
@@ -52,6 +52,7 @@ _TEXT_GENERATION_MODELS = {
52
"DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
53
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
54
"DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
55
+ "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
56
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
57
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
58
"Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),