A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc] Update to comply with the new `compressed-tensors` config (#5350)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Dipika Sikka committed
Jun 9, 2024 at 23:49 UTC
5884c2b454d9a6e16646e949d7308a4cfae3ac12
4 files changed
+19
-20
tests/quantization/test_compressed_tensors.py
+13
-7
@@ -5,15 +5,15 @@ Run `pytest tests/quantization/test_compressed_tensors.py`.
5
6
import torch
7
8
+from vllm import SamplingParams
9
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10
CompressedTensorsLinearMethod, CompressedTensorsW8A8DynamicToken,
11
CompressedTensorsW8A8StaticTensor)
12
13
14
def test_compressed_tensors_w8a8_static_setup(vllm_runner):
14
- model_path = "nm-testing/tinyllama-one-shot-static-quant-test-compressed"
15
- with vllm_runner(model_path, quantization="sparseml",
16
- enforce_eager=True) as llm:
15
+ model_path = "nm-testing/tinyllama-oneshot-w8a8-static-v2"
16
+ with vllm_runner(model_path, enforce_eager=True) as llm:
17
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
18
layer = model.model.layers[0]
19
@@ -40,11 +40,17 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner):
40
assert qkv_proj.input_scale.dtype is torch.float32
41
42
43
+def test_compressed_tensors_no_enforce_eager(vllm_runner):
44
+ model_path = "nm-testing/tinyllama-oneshot-w8a8-static-v2"
45
+ with vllm_runner(model_path) as llm:
46
+ sampling_params = SamplingParams()
47
+ output = llm.generate("Hello world!", sampling_params=sampling_params)
48
+ assert output
49
+
50
+
51
def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner):
44
- model_path = "nm-testing/tinyllama-one-shot-dynamic-test"
45
- with vllm_runner(model_path,
46
- quantization="sparseml",
47
- enforce_eager=True,
52
+ model_path = "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2"
53
+ with vllm_runner(model_path, enforce_eager=True,
54
dtype=torch.float16) as llm:
55
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
56
layer = model.model.layers[0]
vllm/config.py
+2
-6
@@ -164,12 +164,8 @@ class ModelConfig:
164
def _parse_quant_hf_config(self):
165
quant_cfg = getattr(self.hf_config, "quantization_config", None)
166
if quant_cfg is None:
167
- # SparseML uses a "compression_config" with a "quantization_config".
168
- compression_cfg = getattr(self.hf_config, "compression_config",
169
- None)
170
- if compression_cfg is not None:
171
- quant_cfg = compression_cfg.get("quantization_config", None)
172
-
167
+ # compress-tensors uses a "compression_config" key
168
+ quant_cfg = getattr(self.hf_config, "compression_config", None)
169
return quant_cfg
170
171
def _verify_quantization(self) -> None:
vllm/model_executor/layers/quantization/__init__.py
+1
-1
@@ -31,7 +31,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
31
"gptq_marlin": GPTQMarlinConfig,
32
"gptq": GPTQConfig,
33
"squeezellm": SqueezeLLMConfig,
34
- "sparseml": CompressedTensorsConfig,
34
+ "compressed-tensors": CompressedTensorsConfig,
35
"bitsandbytes": BitsAndBytesConfig,
36
}
37
vllm/model_executor/model_loader/weight_utils.py
+3
-6
@@ -122,12 +122,9 @@ def get_quant_config(model_config: ModelConfig,
122
hf_quant_config = getattr(model_config.hf_config, "quantization_config",
123
None)
124
if hf_quant_config is None:
125
- compression_config = getattr(model_config.hf_config,
126
- "compression_config", None)
127
- if compression_config is not None:
128
- hf_quant_config = compression_config.get("quantization_config",
129
- None)
130
-
125
+ # compressed-tensors uses a compressions_config
126
+ hf_quant_config = getattr(model_config.hf_config, "compression_config",
127
+ None)
128
if hf_quant_config is not None:
129
return quant_cls.from_config(hf_quant_config)
130
# In case of bitsandbytes/QLoRA, get quant config from the adapter model.