A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 21:45 UTC
[Bugfix] Support other quantization methods in glm41v (#36321)
Signed-off-by: g00887675/loganJane <g00887675/loganJane73@hotmail.com> Co-authored-by: g00887675/loganJane <g00887675/loganJane73@hotmail.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
LoganJane committed
Mar 11, 2026 at 17:48 UTC
545d18d81bf11761e51c2b11a006573c2ae366c1
1 file changed
+6
-1
vllm/model_executor/models/glm4_1v.py
+6
-1
@@ -63,6 +63,9 @@ from vllm.model_executor.layers.linear import (
63
RowParallelLinear,
64
)
65
from vllm.model_executor.layers.quantization import QuantizationConfig
66
+from vllm.model_executor.layers.quantization.compressed_tensors import (
67
+ compressed_tensors,
68
+)
69
from vllm.model_executor.layers.rotary_embedding import get_rope
70
from vllm.model_executor.layers.rotary_embedding.common import (
71
ApplyRotaryEmb,
@@ -280,7 +283,9 @@ class Glm4vVisionAttention(nn.Module):
283
bias=False,
284
quant_config=quant_config,
285
# Change qkv prefix to align with GLM-4.5V-FP8 quantization cfg
283
- prefix=f"{prefix}.qkv_proj" if quant_config else f"{prefix}.qkv",
286
+ prefix=f"{prefix}.qkv_proj"
287
+ if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig)
288
+ else f"{prefix}.qkv",
289
disable_tp=use_data_parallel,
290
)
291
self.proj = RowParallelLinear(