A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Quantization] Bump compressed-tensors version; update NVFP4A16 test model (#19224)
Signed-off-by: Dipika Sikka <dipikasikka1@gmail.com>
Dipika Sikka committed
Jun 6, 2025 at 04:21 UTC
94870359cdce95cc1c56d7ae2730e8ac9be40049
3 files changed
+5
-6
requirements/common.txt
+1
-1
@@ -37,7 +37,7 @@ pyyaml
37
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
38
setuptools>=77.0.3,<80; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
39
einops # Required for Qwen2-VL.
40
-compressed-tensors == 0.9.4 # required for compressed-tensors
40
+compressed-tensors == 0.10.0 # required for compressed-tensors
41
depyf==0.18.0 # required for profiling and debugging with compilation config
42
cloudpickle # allows pickling lambda functions in model_executor/models/registry.py
43
watchfiles # required for http server to monitor the updates of TLS files
tests/quantization/test_compressed_tensors.py
+1
-2
@@ -651,10 +651,9 @@ def test_compressed_tensors_2of4_sparse_compressed(vllm_runner, args_2of4):
651
assert output
652
653
654
-@pytest.mark.skip(reason="Skip until the model config is updated")
654
def test_compressed_tensors_nvfp4a16(vllm_runner):
655
# run weight only example
657
- model = "nm-testing/TinyLlama-1.1B-Chat-v1.0-FP4"
656
+ model = "nm-testing/TinyLlama-1.1B-Chat-v1.0-NVFP4A16"
657
with vllm_runner(model, enforce_eager=True) as llm:
658
659
def check_model(model):
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+3
-3
@@ -222,15 +222,15 @@ class CompressedTensorsConfig(QuantizationConfig):
222
input_quant: BaseModel):
223
224
is_weight_only = weight_quant is not None and input_quant is None
225
- is_group_quant = (
226
- weight_quant.strategy == QuantizationStrategy.GROUP.value)
225
+ is_tensor_group_quant = (
226
+ weight_quant.strategy == QuantizationStrategy.TENSOR_GROUP.value)
227
is_symmetric = weight_quant.symmetric
228
229
is_group_size_16 = weight_quant.group_size == 16
230
is_float_type = weight_quant.type == QuantizationType.FLOAT
231
is_4_bits = weight_quant.num_bits == 4
232
233
- return (is_weight_only and is_group_quant and is_float_type
233
+ return (is_weight_only and is_tensor_group_quant and is_float_type
234
and is_4_bits and is_group_size_16 and is_symmetric)
235
236
def _is_static_tensor_w8a8(self, weight_quant: BaseModel,