A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix GPTQ model loading in Transformers backend (#25770)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Harry Mellor committed
Sep 27, 2025 at 13:18 UTC
ec152c8748d0b37da157fa6a99a75920822dc30d
3 files changed
+29
-10
tests/models/test_transformers.py
+7
-3
@@ -100,10 +100,9 @@ def test_distributed(
100
kwargs_test=kwargs)
101
102
103
-@pytest.mark.skipif(
104
- current_platform.is_rocm(),
105
- reason="bitsandbytes quantization is currently not supported in rocm.")
103
@pytest.mark.parametrize("model, quantization_kwargs", [
104
+ ("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {}),
105
+ ("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {}),
106
(
107
"meta-llama/Llama-3.2-1B-Instruct",
108
{
@@ -121,6 +120,11 @@ def test_quantization(
120
max_tokens: int,
121
num_logprobs: int,
122
) -> None:
123
+ if (current_platform.is_rocm()
124
+ and quantization_kwargs.get("quantization", "") == "bitsandbytes"):
125
+ pytest.skip(
126
+ "bitsandbytes quantization is currently not supported in rocm.")
127
+
128
with vllm_runner(
129
model, model_impl="auto", enforce_eager=True,
130
**quantization_kwargs) as vllm_model: # type: ignore[arg-type]
vllm/model_executor/models/transformers.py
+17
-5
@@ -447,7 +447,8 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
447
self.device_config: DeviceConfig = vllm_config.device_config
448
self.model_config: ModelConfig = vllm_config.model_config
449
self.parallel_config: ParallelConfig = vllm_config.parallel_config
450
- self.quant_config: QuantizationConfig = vllm_config.quant_config
450
+ self.quant_config: Optional[
451
+ QuantizationConfig] = vllm_config.quant_config
452
453
self.pp_group = get_pp_group()
454
self.pp_size = self.pp_group.world_size
@@ -456,7 +457,18 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
457
458
# Weights to skip in `self.load_weights`
459
self.skip_prefixes: list[str] = []
460
+ """Skip loading weights whose qualname starts with these prefixes."""
461
self.skip_substrs: list[str] = []
462
+ """Skip loading weights whose qualname contains these substrings."""
463
+ self.ignore_unexpected_prefixes: list[str] = []
464
+ """Ignore unexpected weights whose qualname starts with these prefixes.
465
+ """
466
+ self.ignore_unexpected_suffixes: list[str] = []
467
+ """Ignore unexpected weights whose qualname ends with these suffixes."""
468
+
469
+ # Skip loading extra bias for GPTQ models.
470
+ if self.quant_config and "gptq" in self.quant_config.get_name():
471
+ self.ignore_unexpected_suffixes.append(".bias")
472
473
# Set correct attn and init on "meta" to delay allocating GPU tensors
474
# TODO: @raushan, use the public `model.set_attn_implementation()`
@@ -563,9 +575,7 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
575
raise ValueError(
576
f"{type(self.model)} does not support tensor parallel. {tip}")
577
566
- def _tensor_parallel(module: nn.Module,
567
- prefix: str = "",
568
- tp_plan=None):
578
+ def _tensor_parallel(module: nn.Module, prefix: str, tp_plan=None):
579
tp_plan = tp_plan or {}
580
581
# If the current module is a PreTrainedModel, set the tp_plan for
@@ -597,7 +607,7 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
607
prefix=qual_name,
608
tp_plan=tp_plan)
609
600
- _tensor_parallel(self.model)
610
+ _tensor_parallel(self.model, prefix="model")
611
612
def create_attention_instances(
613
self,
@@ -696,6 +706,8 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
706
self,
707
skip_prefixes=self.skip_prefixes,
708
skip_substrs=self.skip_substrs,
709
+ ignore_unexpected_prefixes=self.ignore_unexpected_prefixes,
710
+ ignore_unexpected_suffixes=self.ignore_unexpected_suffixes,
711
)
712
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
713
vllm/model_executor/models/utils.py
+5
-2
@@ -109,6 +109,7 @@ class AutoWeightsLoader:
109
skip_prefixes: Optional[list[str]] = None,
110
skip_substrs: Optional[list[str]] = None,
111
ignore_unexpected_prefixes: Optional[list[str]] = None,
112
+ ignore_unexpected_suffixes: Optional[list[str]] = None,
113
) -> None:
114
super().__init__()
115
@@ -116,6 +117,7 @@ class AutoWeightsLoader:
117
self.skip_prefixes = skip_prefixes or []
118
self.skip_substrs = skip_substrs or []
119
self.ignore_unexpected_prefixes = ignore_unexpected_prefixes or []
120
+ self.ignore_unexpected_suffixes = ignore_unexpected_suffixes or []
121
# update default skip_substrs
122
self.skip_substrs += self.ROTARY_EMBEDS_UNUSED_WEIGHTS
123
@@ -149,8 +151,9 @@ class AutoWeightsLoader:
151
or any(substr in qualname for substr in self.skip_substrs))
152
153
def _can_ignore_unexpected(self, qualname: str) -> bool:
152
- return any(
153
- qualname.startswith(p) for p in self.ignore_unexpected_prefixes)
154
+ iup = (qualname.startswith(p) for p in self.ignore_unexpected_prefixes)
155
+ ius = (qualname.endswith(s) for s in self.ignore_unexpected_suffixes)
156
+ return any(iup) or any(ius)
157
158
def _load_param(
159
self,