Add support for unquantized ngram in CT format (#59431)
Eldar Kurtić committed
Sep 30, 2026 at 18:08 UTC
c32513b99d29b3c6efee38df752a696bc2696c2f
1 file changed
+8
vllm/models/qwen4_exp/common/ngram_embedding.py
+8
@@ -23,6 +23,9 @@ from vllm.model_executor.layers.quantization.base_config import (
23
QuantizationConfig,
24
QuantizeMethodBase,
25
)
26
+from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
27
+ CompressedTensorsConfig,
28
+)
29
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
30
from vllm.model_executor.layers.quantization.modelopt import (
31
ModelOptMixedPrecisionConfig,
@@ -182,6 +185,11 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
185
quant_config, ModelOptQuantConfigBase
186
) and quant_config.is_layer_excluded(prefix):
187
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
188
+ if (
189
+ isinstance(quant_config, CompressedTensorsConfig)
190
+ and quant_config.get_scheme_dict(None, layer_name=prefix) is None
191
+ ):
192
+ return Qwen4ExpPLEUnquantizedEmbeddingMethod()
193
if not isinstance(quant_config, Fp8Config):
194
raise NotImplementedError(
195
"Qwen4Exp PLE embedding does not support quantization config "