@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5884c2b454
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc] Update to comply with the new `compressed-tensors` config (#5350)

Co-authored-by: Michael Goin <michael@neuralmagic.com>

Dipika Sikka committed Jun 9, 2024 at 23:49 UTC 5884c2b454d9a6e16646e949d7308a4cfae3ac12
4 files changed +19 -20
tests/quantization/test_compressed_tensors.py
+13 -7
@@ -5,15 +5,15 @@ Run `pytest tests/quantization/test_compressed_tensors.py`.
5
6 import torch
7
8 +from vllm import SamplingParams
9 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10 CompressedTensorsLinearMethod, CompressedTensorsW8A8DynamicToken,
11 CompressedTensorsW8A8StaticTensor)
12
13
14 def test_compressed_tensors_w8a8_static_setup(vllm_runner):
14 - model_path = "nm-testing/tinyllama-one-shot-static-quant-test-compressed"
15 - with vllm_runner(model_path, quantization="sparseml",
16 - enforce_eager=True) as llm:
15 + model_path = "nm-testing/tinyllama-oneshot-w8a8-static-v2"
16 + with vllm_runner(model_path, enforce_eager=True) as llm:
17 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
18 layer = model.model.layers[0]
19
@@ -40,11 +40,17 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner):
40 assert qkv_proj.input_scale.dtype is torch.float32
41
42
43 +def test_compressed_tensors_no_enforce_eager(vllm_runner):
44 + model_path = "nm-testing/tinyllama-oneshot-w8a8-static-v2"
45 + with vllm_runner(model_path) as llm:
46 + sampling_params = SamplingParams()
47 + output = llm.generate("Hello world!", sampling_params=sampling_params)
48 + assert output
49 +
50 +
51 def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
44 - model_path = "nm-testing/tinyllama-one-shot-dynamic-test"
45 - with vllm_runner(model_path,
46 - quantization="sparseml",
47 - enforce_eager=True,
52 + model_path = "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2"
53 + with vllm_runner(model_path, enforce_eager=True,
54 dtype=torch.float16) as llm:
55 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
56 layer = model.model.layers[0]
vllm/config.py
+2 -6
@@ -164,12 +164,8 @@ class ModelConfig:
164 def _parse_quant_hf_config(self):
165 quant_cfg = getattr(self.hf_config, "quantization_config", None)
166 if quant_cfg is None:
167 - # SparseML uses a "compression_config" with a "quantization_config".
168 - compression_cfg = getattr(self.hf_config, "compression_config",
169 - None)
170 - if compression_cfg is not None:
171 - quant_cfg = compression_cfg.get("quantization_config", None)
172 -
167 + # compress-tensors uses a "compression_config" key
168 + quant_cfg = getattr(self.hf_config, "compression_config", None)
169 return quant_cfg
170
171 def _verify_quantization(self) -> None:
vllm/model_executor/layers/quantization/__init__.py
+1 -1
@@ -31,7 +31,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
31 "gptq_marlin": GPTQMarlinConfig,
32 "gptq": GPTQConfig,
33 "squeezellm": SqueezeLLMConfig,
34 - "sparseml": CompressedTensorsConfig,
34 + "compressed-tensors": CompressedTensorsConfig,
35 "bitsandbytes": BitsAndBytesConfig,
36 }
37
vllm/model_executor/model_loader/weight_utils.py
+3 -6
@@ -122,12 +122,9 @@ def get_quant_config(model_config: ModelConfig,
122 hf_quant_config = getattr(model_config.hf_config, "quantization_config",
123 None)
124 if hf_quant_config is None:
125 - compression_config = getattr(model_config.hf_config,
126 - "compression_config", None)
127 - if compression_config is not None:
128 - hf_quant_config = compression_config.get("quantization_config",
129 - None)
130 -
125 + # compressed-tensors uses a compressions_config
126 + hf_quant_config = getattr(model_config.hf_config, "compression_config",
127 + None)
128 if hf_quant_config is not None:
129 return quant_cls.from_config(hf_quant_config)
130 # In case of bitsandbytes/QLoRA, get quant config from the adapter model.