@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7f0be2aa24

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 21:00 UTC

[Model] Deepseek GGUF support (#13167)

Szymon Ożóg committed Feb 27, 2025 at 11:08 UTC 7f0be2aa24dfacf6f7aaf92411fdcd2f55ff8478

8 files changed +198 -10

docs/source/features/quantization/gguf.md

@@ -29,6 +29,13 @@ vllm serve ./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf --tokenizer TinyLlama/TinyLlam

29 We recommend using the tokenizer from base model instead of GGUF model. Because the tokenizer conversion from GGUF is time-consuming and unstable, especially for some models with large vocab size.

30 :::

32 +GGUF assumes that huggingface can convert the metadata to a config file. In case huggingface doesn't support your model you can manually create a config and pass it as hf-confing-path

33 +

34 +```console

35 +# If you model is not supported by huggingface you can manually provide a huggingface compatible config path

36 +vllm serve ./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf --tokenizer TinyLlama/TinyLlama-1.1B-Chat-v1.0 --hf-config-path Tinyllama/TInyLlama-1.1B-Chat-v1.0

37 +```

38 +

39 You can also use the GGUF model directly through the LLM entrypoint:

41 ```python

vllm/config.py

+6 -3

@@ -229,6 +229,7 @@ class ModelConfig:

229 trust_remote_code: bool,

230 dtype: Union[str, torch.dtype],

231 seed: int,

232 + hf_config_path: Optional[str] = None,

233 allowed_local_media_path: str = "",

234 revision: Optional[str] = None,

235 code_revision: Optional[str] = None,

@@ -259,6 +260,7 @@ class ModelConfig:

260 model_impl: Union[str, ModelImpl] = ModelImpl.AUTO,

261 ) -> None:

262 self.model = model

263 + self.hf_config_path = hf_config_path

264 self.tokenizer = tokenizer

265 self.tokenizer_mode = tokenizer_mode

266 self.trust_remote_code = trust_remote_code

@@ -321,8 +323,9 @@ class ModelConfig:

323 if self.enable_sleep_mode and not current_platform.is_cuda():

324 raise ValueError("Sleep mode is only supported on CUDA devices.")

325

324 - hf_config = get_config(self.model, trust_remote_code, revision,

325 - code_revision, config_format)

326 + hf_config = get_config(self.hf_config_path or self.model,

327 + trust_remote_code, revision, code_revision,

328 + config_format)

329

330 if hf_overrides_kw:

331 logger.info("Overriding HF config with %s", hf_overrides_kw)

@@ -947,7 +950,7 @@ class ModelConfig:

950 def try_get_generation_config(self) -> Dict[str, Any]:

951 if self.generation_config is None or self.generation_config == "auto":

952 config = try_get_generation_config(

950 - self.model,

953 + self.hf_config_path or self.model,

954 trust_remote_code=self.trust_remote_code,

955 revision=self.revision,

956 )

vllm/engine/arg_utils.py

@@ -93,6 +93,7 @@ class EngineArgs:

93 model: str = 'facebook/opt-125m'

94 served_model_name: Optional[Union[str, List[str]]] = None

95 tokenizer: Optional[str] = None

96 + hf_config_path: Optional[str] = None

97 task: TaskOption = "auto"

98 skip_tokenizer_init: bool = False

99 tokenizer_mode: str = 'auto'

@@ -262,6 +263,12 @@ class EngineArgs:

263 default=EngineArgs.tokenizer,

264 help='Name or path of the huggingface tokenizer to use. '

265 'If unspecified, model name or path will be used.')

266 + parser.add_argument(

267 + "--hf-config-path",

268 + type=nullable_str,

269 + default=EngineArgs.hf_config_path,

270 + help='Name or path of the huggingface config to use. '

271 + 'If unspecified, model name or path will be used.')

272 parser.add_argument(

273 '--skip-tokenizer-init',

274 action='store_true',

@@ -1076,6 +1083,7 @@ class EngineArgs:

1083

1084 return ModelConfig(

1085 model=self.model,

1086 + hf_config_path=self.hf_config_path,

1087 task=self.task,

1088 # We know this is not None because we set it in __post_init__

1089 tokenizer=cast(str, self.tokenizer),

vllm/model_executor/layers/fused_moe/layer.py

+21 -1

@@ -5,6 +5,7 @@ from enum import Enum

5 from typing import Callable, List, Optional, Tuple

7 import torch

8 +from torch.nn.parameter import UninitializedParameter

10 import vllm.envs as envs

11 from vllm.distributed import (get_tensor_model_parallel_rank,

@@ -514,7 +515,12 @@ class FusedMoE(torch.nn.Module):

515 # dimension intermediate_size_per_partition is used.

516 SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0}

517

517 - expert_data = param.data[expert_id]

518 + is_gguf_weight = getattr(param, "is_gguf_weight", False)

519 + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)

520 + if is_gguf_weight_type:

521 + param.weight_type = loaded_weight.item()

522 + param.data.copy_(loaded_weight)

523 + return

524

525 # is_transposed: if the dim to shard the weight

526 # should be flipped. Required by GPTQ, compressed-tensors

@@ -524,6 +530,20 @@ class FusedMoE(torch.nn.Module):

530 if is_transposed:

531 shard_dim = int(not shard_dim)

532

533 + full_load = len(loaded_weight.shape) == 3

534 + if full_load:

535 + shard_dim += 1

536 +

537 + # Materialize GGUF UninitializedParameter

538 + if is_gguf_weight and isinstance(param, UninitializedParameter):

539 + final_shape = list(loaded_weight.shape)

540 + if shard_id in ["w1", "w3"]:

541 + final_shape[1] *= 2

542 + final_shape[shard_dim] = final_shape[

543 + shard_dim] // get_tensor_model_parallel_world_size()

544 + param.materialize(final_shape, dtype=loaded_weight.dtype)

545 +

546 + expert_data = param.data if full_load else param.data[expert_id]

547 # Case input scale: input_scale loading is only supported for fp8

548 if "input_scale" in weight_name:

549 # this is needed for compressed-tensors only

vllm/model_executor/layers/linear.py

+14 -1

@@ -235,10 +235,23 @@ class ReplicatedLinear(LinearBase):

235 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):

236 # If the weight on disk does not have a shape, give it one

237 # (such scales for AutoFp8).

238 + # Special case for GGUF

239 +

240 + is_gguf_weight = getattr(param, "is_gguf_weight", False)

241 + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)

242 + if is_gguf_weight_type:

243 + param.weight_type = loaded_weight.item()

244 +

245 + # Materialize GGUF UninitializedParameter

246 + if is_gguf_weight and isinstance(param, UninitializedParameter):

247 + param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)

248 +

249 if len(loaded_weight.shape) == 0:

250 loaded_weight = loaded_weight.reshape(1)

251

241 - assert param.size() == loaded_weight.size()

252 + assert param.size() == loaded_weight.size(), (

253 + f"Tried to load weights of size {loaded_weight.size()}"

254 + f"to a parameter of size {param.size()}")

255 param.data.copy_(loaded_weight)

256

257 def forward(self,

vllm/model_executor/layers/quantization/gguf.py

+125 -2

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

3 -from typing import Any, Dict, List, Optional

3 +from typing import Any, Callable, Dict, List, Optional

5 import gguf

6 import torch

@@ -8,6 +8,9 @@ from gguf import GGMLQuantizationType as WeightType

8 from torch.nn.parameter import Parameter, UninitializedParameter

10 from vllm import _custom_ops as ops

11 +from vllm.model_executor.layers.activation import SiluAndMul

12 +from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,

13 + FusedMoEMethodBase)

14 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase

15 from vllm.model_executor.layers.quantization.base_config import (

16 QuantizationConfig, QuantizeMethodBase)

@@ -29,7 +32,7 @@ class GGUFConfig(QuantizationConfig):

32 return "gguf"

34 def get_supported_act_dtypes(self) -> List[torch.dtype]:

32 - return [torch.half, torch.bfloat16]

35 + return [torch.half]

37 @classmethod

38 def get_min_capability(cls) -> int:

@@ -49,6 +52,8 @@ class GGUFConfig(QuantizationConfig):

52 return GGUFLinearMethod(self)

53 elif isinstance(layer, VocabParallelEmbedding):

54 return GGUFEmbeddingMethod(self)

55 + elif isinstance(layer, FusedMoE):

56 + return GGUFMoEMethod(self)

57 return None

@@ -184,6 +189,124 @@ class GGUFLinearMethod(LinearMethodBase):

189 return out

190

191

192 +class GGUFMoEMethod(FusedMoEMethodBase):

193 + """MoE method for GGUF.

194 +

195 + Args:

196 + quant_config: The GGUF quantization config.

197 + """

198 +

199 + def __init__(self, quant_config: GGUFConfig):

200 + self.quant_config = quant_config

201 +

202 + def create_weights(self, layer: torch.nn.Module, num_experts: int,

203 + hidden_size: int, intermediate_size_per_partition: int,

204 + params_dtype: torch.dtype, **extra_weight_attrs):

205 +

206 + tensor_shape = (num_experts, 2 * intermediate_size_per_partition,

207 + hidden_size)

208 + #gate up proj

209 + w13_qweight = GGUFUninitializedParameter(requires_grad=False)

210 + set_weight_attrs(

211 + w13_qweight, {

212 + "input_dim": 1,

213 + "output_dim": 0,

214 + "tensor_shape": tensor_shape,

215 + "is_gguf_weight": True,

216 + "data_container": [],

217 + })

218 + set_weight_attrs(w13_qweight, extra_weight_attrs)

219 + layer.register_parameter("w13_qweight", w13_qweight)

220 +

221 + w13_qweight_type = Parameter(torch.empty(1, dtype=torch.uint8),

222 + requires_grad=False)

223 + set_weight_attrs(w13_qweight_type, {

224 + "is_gguf_weight_type": True,

225 + "weight_type": 0,

226 + "ignore_warning": True

227 + })

228 + set_weight_attrs(w13_qweight_type, extra_weight_attrs)

229 + layer.register_parameter("w13_qweight_type", w13_qweight_type)

230 +

231 + tensor_shape = (num_experts, intermediate_size_per_partition,

232 + hidden_size)

233 + #gate down proj

234 + w2_qweight = GGUFUninitializedParameter(requires_grad=False)

235 + set_weight_attrs(

236 + w2_qweight, {

237 + "input_dim": 1,

238 + "output_dim": 0,

239 + "tensor_shape": tensor_shape,

240 + "is_gguf_weight": True,

241 + "data_container": [],

242 + })

243 + set_weight_attrs(w2_qweight, extra_weight_attrs)

244 + layer.register_parameter("w2_qweight", w2_qweight)

245 +

246 + w2_qweight_type = Parameter(torch.empty(1, dtype=torch.uint8),

247 + requires_grad=False)

248 + set_weight_attrs(w2_qweight_type, {

249 + "is_gguf_weight_type": True,

250 + "weight_type": 0,

251 + "ignore_warning": True

252 + })

253 +

254 + set_weight_attrs(w2_qweight_type, extra_weight_attrs)

255 + layer.register_parameter("w2_qweight_type", w2_qweight_type)

256 + self.act = SiluAndMul()

257 +

258 + def apply(

259 + self,

260 + layer: torch.nn.Module,

261 + x: torch.Tensor,

262 + router_logits: torch.Tensor,

263 + top_k: int,

264 + renormalize: bool,

265 + use_grouped_topk: bool = False,

266 + topk_group: Optional[int] = None,

267 + num_expert_group: Optional[int] = None,

268 + global_num_experts: int = -1,

269 + expert_map: Optional[torch.Tensor] = None,

270 + custom_routing_function: Optional[Callable] = None,

271 + scoring_func: str = "softmax",

272 + e_score_correction_bias: Optional[torch.Tensor] = None,

273 + activation: str = "silu",

274 + ):

275 + assert activation == "silu", "Only SiLU activation is supported."

276 + topk_weights, topk_ids = FusedMoE.select_experts(

277 + hidden_states=x,

278 + router_logits=router_logits,

279 + use_grouped_topk=use_grouped_topk,

280 + top_k=top_k,

281 + renormalize=renormalize,

282 + topk_group=topk_group,

283 + num_expert_group=num_expert_group,

284 + custom_routing_function=custom_routing_function,

285 + scoring_func=scoring_func,

286 + e_score_correction_bias=e_score_correction_bias)

287 + final_hidden_states = torch.empty_like(x)

288 + for tok, (w, idx) in enumerate(zip(topk_weights, topk_ids)):

289 + inp = x[tok].reshape((1, ) + x.shape[1:])

290 + current_hidden_state = None

291 + for ww, ii in zip(w, idx):

292 + expert_up = layer.w13_qweight[ii]

293 +

294 + out = _fuse_mul_mat(inp, expert_up,

295 + layer.w13_qweight_type.weight_type)

296 + out = self.act(out)

297 +

298 + expert_down = layer.w2_qweight[ii]

299 + current_state = _fuse_mul_mat(

300 + out, expert_down,

301 + layer.w2_qweight_type.weight_type).mul_(ww)

302 + if current_hidden_state is None:

303 + current_hidden_state = current_state

304 + else:

305 + current_hidden_state.add_(current_state)

306 + final_hidden_states[tok] = current_hidden_state

307 + return final_hidden_states

308 +

309 +

310 class GGUFEmbeddingMethod(GGUFLinearMethod):

311 """Embedding method for GGUF.

312

vllm/model_executor/model_loader/loader.py

+17 -2

@@ -1245,9 +1245,24 @@ class GGUFModelLoader(BaseModelLoader):

1245 """

1246 config = model_config.hf_config

1247 model_type = config.model_type

1248 + gguf_to_hf_name_map = {}

1249 # hack: ggufs have a different name than transformers

1250 if model_type == "cohere":

1251 model_type = "command-r"

1252 + if model_type in ("deepseek_v3", "deepseek_v2"):

1253 + model_type = "deepseek2"

1254 + # GGUF layer map assumes that we will have a merged expert weights

1255 + # so we need to map them manually

1256 + for idx in range(config.num_hidden_layers):

1257 + gguf_to_hf_name_map[f"blk.{idx}.exp_probs_b.bias"] = \

1258 + f"model.layers.{idx}.mlp.gate.e_score_correction_bias"

1259 + gguf_to_hf_name_map[f"blk.{idx}.ffn_down_exps.weight"] = \

1260 + f"model.layers.{idx}.mlp.experts.0.down_proj.weight"

1261 + gguf_to_hf_name_map[f"blk.{idx}.ffn_gate_exps.weight"] = \

1262 + f"model.layers.{idx}.mlp.experts.0.gate_proj.weight"

1263 + gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = \

1264 + f"model.layers.{idx}.mlp.experts.0.up_proj.weight"

1265 +

1266 arch = None

1267 for key, value in gguf.MODEL_ARCH_NAMES.items():

1268 if value == model_type:

@@ -1258,10 +1273,10 @@ class GGUFModelLoader(BaseModelLoader):

1273 num_layers = config.num_hidden_layers

1274 name_map = gguf.get_tensor_name_map(arch, num_layers)

1275 with torch.device("meta"):

1261 - dummy_model = AutoModelForCausalLM.from_config(config)

1276 + dummy_model = AutoModelForCausalLM.from_config(

1277 + config, trust_remote_code=model_config.trust_remote_code)

1278 state_dict = dummy_model.state_dict()

1279

1264 - gguf_to_hf_name_map = {}

1280 for hf_name in state_dict:

1281 name, suffix = hf_name.rsplit(".", 1)

1282 gguf_name = name_map.get_name(name)

vllm/model_executor/model_loader/weight_utils.py

-1

@@ -496,7 +496,6 @@ def gguf_quant_weights_iterator(

496 weight = tensor.data

497 weight_type = tensor.tensor_type

498 name = gguf_to_hf_name_map[tensor.name]

499 -

499 if weight_type.name != "F32":

500 name = name.replace("weight", "qweight")

501 param = torch.tensor(weight)