A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix the bug related to loading GPTP INT3 weights. (#23328)
Signed-off-by: JunHowie <JunHowie@aliyun.com> Co-authored-by: JunHowie <JunHowie@aliyun.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Jun-Howie committed
Sep 1, 2025 at 13:39 UTC
acc1a6e10af7ec16787ffe8dc67ad36a05955024
1 file changed
+2
-1
vllm/model_executor/layers/quantization/utils/gptq_utils.py
+2
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from copy import deepcopy
4
+from fractions import Fraction
5
from typing import Optional, Union
6
7
import regex as re
@@ -29,7 +30,7 @@ def override_config(config: QuantizationConfig, prefix: str):
30
if isinstance(desc_act, bool):
31
config.desc_act = desc_act
32
32
- config.pack_factor = 32 // config.weight_bits # packed into int32
33
+ config.pack_factor = Fraction(32, config.weight_bits) # packed into int32
34
if config.get_name() == "gptq_marlin":
35
is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
36
if isinstance(is_sym, bool):