A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
Add: Support for Sparse24Bitmask Compressed Models
Rahul Tuli committed
Feb 5, 2025 at 15:30 UTC
3b2005e1db79efe2ea4587035eb2e7ced6e258cb
4 files changed
+503
-112
.buildkite/lm-eval-harness/configs/SparseLlama3.1_2of4_fp8_compressed.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash ./run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM -b "auto" -t 2
2
+model_name: "nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.6353
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.637
10
+limit: null
11
+num_fewshot: null
tests/quantization/test_compressed_tensors.py
+276
-56
@@ -3,6 +3,7 @@
3
4
Run `pytest tests/quantization/test_compressed_tensors.py`.
5
"""
6
+
7
from typing import Optional
8
9
import pytest
@@ -22,12 +23,30 @@ from vllm.platforms import current_platform
23
24
@pytest.mark.parametrize(
25
"model_args",
25
- [("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", "tensor",
26
- QuantizationType.INT, 2560, True),
27
- ("nm-testing/tinyllama-oneshot-w8-channel-a8-tensor", "channel",
28
- QuantizationType.INT, 2560, True),
29
- ("nm-testing/asym-w8w8-int8-static-per-tensor-tiny-llama", "tensor",
30
- QuantizationType.INT, 2560, False)])
26
+ [
27
+ (
28
+ "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change",
29
+ "tensor",
30
+ QuantizationType.INT,
31
+ 2560,
32
+ True,
33
+ ),
34
+ (
35
+ "nm-testing/tinyllama-oneshot-w8-channel-a8-tensor",
36
+ "channel",
37
+ QuantizationType.INT,
38
+ 2560,
39
+ True,
40
+ ),
41
+ (
42
+ "nm-testing/asym-w8w8-int8-static-per-tensor-tiny-llama",
43
+ "tensor",
44
+ QuantizationType.INT,
45
+ 2560,
46
+ False,
47
+ ),
48
+ ],
49
+)
50
def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
51
model_path, strategy, quant_type, shape_0, is_symmetric = model_args
52
with vllm_runner(model_path, enforce_eager=True) as llm:
@@ -85,21 +104,31 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
104
assert output
105
106
88
-@pytest.mark.parametrize("model_path", [
89
- "neuralmagic/Llama-3.2-1B-quantized.w8a8",
90
- "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Asym",
91
- "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym",
92
- "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym"
93
-])
107
+@pytest.mark.parametrize(
108
+ "model_path",
109
+ [
110
+ "neuralmagic/Llama-3.2-1B-quantized.w8a8",
111
+ "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Asym",
112
+ "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym",
113
+ "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym",
114
+ ],
115
+)
116
@pytest.mark.parametrize("max_tokens", [32])
117
@pytest.mark.parametrize("num_logprobs", [10])
96
-def test_compressed_tensors_w8a8_logprobs(hf_runner, vllm_runner,
97
- example_prompts, model_path,
98
- max_tokens, num_logprobs):
118
+def test_compressed_tensors_w8a8_logprobs(
119
+ hf_runner,
120
+ vllm_runner,
121
+ example_prompts,
122
+ model_path,
123
+ max_tokens,
124
+ num_logprobs,
125
+):
126
dtype = "bfloat16"
127
128
# skip language translation prompt for the static per tensor asym model
102
- if model_path == "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym": # noqa: E501
129
+ if (model_path ==
130
+ "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym"
131
+ ): # noqa: E501
132
example_prompts = example_prompts[0:-1]
133
134
with hf_runner(model_path, dtype=dtype) as hf_model:
@@ -125,13 +154,21 @@ def test_compressed_tensors_no_enforce_eager(vllm_runner):
154
assert output
155
156
128
-@pytest.mark.parametrize("model_args", [
129
- ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", "tensor"),
130
- ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2-asym", "tensor"),
131
- ("nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2", "channel"),
132
- ("nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2-asym",
133
- "channel"),
134
-])
157
+@pytest.mark.parametrize(
158
+ "model_args",
159
+ [
160
+ ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", "tensor"),
161
+ ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2-asym", "tensor"),
162
+ (
163
+ "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2",
164
+ "channel",
165
+ ),
166
+ (
167
+ "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2-asym",
168
+ "channel",
169
+ ),
170
+ ],
171
+)
172
def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args):
173
model_path, strategy = model_args
174
with vllm_runner(model_path, dtype=torch.float16) as llm:
@@ -156,9 +193,12 @@ def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args):
193
194
@pytest.mark.parametrize(
195
"wNa16_args",
159
- [("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None, 8),
160
- ("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128, 8),
161
- ("nm-testing/tinyllama-oneshot-w8a16-per-channel", "channel", None, 4)])
196
+ [
197
+ ("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None, 8),
198
+ ("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128, 8),
199
+ ("nm-testing/tinyllama-oneshot-w8a16-per-channel", "channel", None, 4),
200
+ ],
201
+)
202
def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
203
model, strategy, group, pack_factor = wNa16_args
204
with vllm_runner(model) as llm:
@@ -218,7 +258,8 @@ def test_compressed_tensors_fp8(vllm_runner):
258
CompressedTensorsLinearMethod)
259
assert isinstance(
260
qkv_proj.scheme,
221
- (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8))
261
+ (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8),
262
+ )
263
264
assert qkv_proj.input_scale.dtype is torch.float32
265
@@ -241,9 +282,14 @@ def test_compressed_tensors_kv_cache(vllm_runner):
282
assert output
283
284
244
-@pytest.mark.skipif(not sparse_cutlass_supported(),
245
- reason="Sparse FP8 is not yet supported on this GPU type.")
246
-def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy):
285
+@pytest.mark.skipif(
286
+ not sparse_cutlass_supported(),
287
+ reason="Sparse FP8 is not yet supported on this GPU type.",
288
+)
289
+def _test_2of4_quant_models(qkv_proj,
290
+ weight_strategy,
291
+ input_strategy,
292
+ format="dense"):
293
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
294
assert isinstance(qkv_proj.scheme, CompressedTensors24)
295
@@ -252,22 +298,39 @@ def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy):
298
assert qkv_proj.scheme.quantized
299
assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
300
sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
255
- assert sparsity_map.get("Linear").format == "dense"
301
+ assert sparsity_map.get("Linear").format == format
302
assert sparsity_map.get("Linear").sparsity_structure == "2:4"
303
304
259
-@pytest.mark.skipif(not current_platform.has_device_capability(90),
260
- reason="Sparse FP8 is not yet supported on this GPU type.")
261
-@pytest.mark.parametrize("args_2of4", [
262
- ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-2of4-testing", "channel",
263
- "token"),
264
- ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-Per-Tensor-testing",
265
- "channel", "tensor"),
266
- ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-testing", "tensor",
267
- "tensor"),
268
- ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-IA-Per-Tensor-Weight-testing",
269
- "tensor", "token"),
270
-])
305
+@pytest.mark.skipif(
306
+ not current_platform.has_device_capability(90),
307
+ reason="Sparse FP8 is not yet supported on this GPU type.",
308
+)
309
+@pytest.mark.parametrize(
310
+ "args_2of4",
311
+ [
312
+ (
313
+ "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-2of4-testing",
314
+ "channel",
315
+ "token",
316
+ ),
317
+ (
318
+ "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-Per-Tensor-testing",
319
+ "channel",
320
+ "tensor",
321
+ ),
322
+ (
323
+ "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-testing",
324
+ "tensor",
325
+ "tensor",
326
+ ),
327
+ (
328
+ "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-IA-Per-Tensor-Weight-testing",
329
+ "tensor",
330
+ "token",
331
+ ),
332
+ ],
333
+)
334
def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
335
model, weight_strategy, input_strategy = args_2of4
336
with vllm_runner(model) as llm:
@@ -286,16 +349,134 @@ def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
349
assert output
350
351
289
-@pytest.mark.skipif(not sparse_cutlass_supported(),
290
- reason="Sparse FP8 is not yet supported on this GPU type.")
291
-@pytest.mark.parametrize("args_2of4", [
292
- ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing",
293
- "channel", "token"),
294
- ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Static-testing", "tensor",
295
- "tensor"),
296
- ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Tensor-Weight-testing",
297
- "tensor", "token"),
298
-])
352
+@pytest.mark.skipif(
353
+ not current_platform.has_device_capability(90),
354
+ reason="Sparse FP8 is not yet supported on this GPU type.",
355
+)
356
+@pytest.mark.parametrize(
357
+ "args_2of4",
358
+ [
359
+ (
360
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM",
361
+ "channel",
362
+ "token",
363
+ ),
364
+ (
365
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_tensor_act_fp8-BitM",
366
+ "channel",
367
+ "tensor",
368
+ ),
369
+ (
370
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_per_tok_dyn_act_fp8-BitM",
371
+ "tensor",
372
+ "token",
373
+ ),
374
+ (
375
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_tensor_act_fp8-BitM",
376
+ "tensor",
377
+ "tensor",
378
+ ),
379
+ ],
380
+)
381
+def test_compressed_tensors_2of4_quant_fp8_compressed(vllm_runner, args_2of4):
382
+ model, weight_strategy, input_strategy = args_2of4
383
+ with vllm_runner(model) as llm:
384
+
385
+ def check_model(model):
386
+ layer = model.model.layers[0]
387
+
388
+ qkv_proj = layer.self_attn.qkv_proj
389
+ assert qkv_proj.scheme.weights_dtype == torch.float8_e4m3fn
390
+ _test_2of4_quant_models(
391
+ qkv_proj,
392
+ weight_strategy,
393
+ input_strategy,
394
+ format="sparse-24-bitmask",
395
+ )
396
+
397
+ llm.apply_model(check_model)
398
+
399
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
400
+ print(output)
401
+ assert output
402
+
403
+
404
+@pytest.mark.skipif(
405
+ not sparse_cutlass_supported(),
406
+ reason="cutlass is not yet supported on this GPU type.",
407
+)
408
+@pytest.mark.parametrize(
409
+ "args_2of4",
410
+ [
411
+ (
412
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_int8-BitM",
413
+ "channel",
414
+ "token",
415
+ ),
416
+ (
417
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_tensor_act_int8-BitM",
418
+ "channel",
419
+ "tensor",
420
+ ),
421
+ (
422
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_per_tok_dyn_act_int8-BitM",
423
+ "tensor",
424
+ "token",
425
+ ),
426
+ (
427
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_tensor_act_int8-BitM",
428
+ "tensor",
429
+ "tensor",
430
+ ),
431
+ ],
432
+)
433
+def test_compressed_tensors_2of4_quant_int8_compressed(vllm_runner, args_2of4):
434
+ model, weight_strategy, input_strategy = args_2of4
435
+ with vllm_runner(model) as llm:
436
+
437
+ def check_model(model):
438
+ layer = model.model.layers[0]
439
+
440
+ qkv_proj = layer.self_attn.qkv_proj
441
+ assert qkv_proj.scheme.weights_dtype == torch.int8
442
+ _test_2of4_quant_models(
443
+ qkv_proj,
444
+ weight_strategy,
445
+ input_strategy,
446
+ format="sparse-24-bitmask",
447
+ )
448
+
449
+ llm.apply_model(check_model)
450
+
451
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
452
+ print(output)
453
+ assert output
454
+
455
+
456
+@pytest.mark.skipif(
457
+ not sparse_cutlass_supported(),
458
+ reason="Sparse FP8 is not yet supported on this GPU type.",
459
+)
460
+@pytest.mark.parametrize(
461
+ "args_2of4",
462
+ [
463
+ (
464
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing",
465
+ "channel",
466
+ "token",
467
+ ),
468
+ (
469
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Static-testing",
470
+ "tensor",
471
+ "tensor",
472
+ ),
473
+ (
474
+ "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Tensor-Weight-testing",
475
+ "tensor",
476
+ "token",
477
+ ),
478
+ ],
479
+)
480
def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
481
model, weight_strategy, input_strategy = args_2of4
482
with vllm_runner(model) as llm:
@@ -317,10 +498,12 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
498
@pytest.mark.skip(reason="2of4 sparse w16a16 CUTLASS produces bad output.")
499
@pytest.mark.skipif(
500
not sparse_cutlass_supported(),
320
- reason="2of4 Sparse is not yet supported on this GPU type.")
501
+ reason="2of4 Sparse is not yet supported on this GPU type.",
502
+)
503
@pytest.mark.parametrize(
504
"args_2of4",
323
- [("nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor")])
505
+ [("nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor")],
506
+)
507
def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):
508
model = args_2of4
509
with vllm_runner(model) as llm:
@@ -337,7 +520,9 @@ def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):
520
assert qkv_proj.scheme.input_quant is None
521
assert not qkv_proj.scheme.quantized
522
assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
340
- sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
523
+ sparsity_map = (
524
+ qkv_proj.quant_method.quantization_config.sparsity_scheme_map
525
+ ) # noqa: E501
526
assert sparsity_map.get("Linear").format == "dense"
527
assert sparsity_map.get("Linear").sparsity_structure == "2:4"
528
@@ -346,3 +531,38 @@ def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):
531
output = llm.generate_greedy("Hello my name is", max_tokens=20)
532
print(output)
533
assert output
534
+
535
+
536
+@pytest.mark.skipif(
537
+ not sparse_cutlass_supported(),
538
+ reason="Cutlass is not yet supported on this GPU type.",
539
+)
540
+@pytest.mark.parametrize(
541
+ "args_2of4", [("nm-testing/llama2.c-stories42M-pruned2.4-compressed")])
542
+def test_compressed_tensors_2of4_sparse_compressed(vllm_runner, args_2of4):
543
+ model = args_2of4
544
+ with vllm_runner(model) as llm:
545
+
546
+ def check_model(model):
547
+ layer = model.model.layers[0]
548
+
549
+ qkv_proj = layer.self_attn.qkv_proj
550
+ assert isinstance(qkv_proj.quant_method,
551
+ CompressedTensorsLinearMethod)
552
+ assert isinstance(qkv_proj.scheme, CompressedTensors24)
553
+
554
+ assert qkv_proj.scheme.weight_quant is None
555
+ assert qkv_proj.scheme.input_quant is None
556
+ assert not qkv_proj.scheme.quantized
557
+ assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
558
+ sparsity_map = (
559
+ qkv_proj.quant_method.quantization_config.sparsity_scheme_map
560
+ ) # noqa: E501
561
+ assert sparsity_map.get("Linear").format == "sparse-24-bitmask"
562
+ assert sparsity_map.get("Linear").sparsity_structure == "2:4"
563
+
564
+ llm.apply_model(check_model)
565
+
566
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
567
+ print(output)
568
+ assert output
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+26
-8
@@ -417,15 +417,22 @@ class CompressedTensorsConfig(QuantizationConfig):
417
return None
418
# Have a valid sparsity scheme
419
# Validate layer is supported by Cutlass 2:4 Kernel
420
- scheme = CompressedTensors24(quantized=weight_quant is not None
421
- or input_quant is not None,
422
- weight_quant=weight_quant,
423
- input_quant=input_quant)
420
+ model_compression_config = (None if sparsity_scheme is None
421
+ or sparsity_scheme.format == "dense"
422
+ else self.config)
423
+
424
+ scheme = CompressedTensors24(
425
+ quantized=weight_quant is not None or input_quant is not None,
426
+ weight_quant=weight_quant,
427
+ input_quant=input_quant,
428
+ model_compression_config=model_compression_config,
429
+ )
430
elif weight_quant is None:
431
logger.warning_once("Acceleration for non-quantized schemes is "
432
"not supported by Compressed Tensors. "
433
"Falling back to UnquantizedLinearMethod")
434
return None
435
+
436
else:
437
# Find the quant_scheme
438
scheme = self._get_scheme_from_parts( # type: ignore
@@ -475,10 +482,21 @@ class CompressedTensorsConfig(QuantizationConfig):
482
:return: True if the layer is supported by the Cutlass 2:4 Kernel
483
False otherwise
484
"""
478
- is_valid_sparsity = (sparsity_scheme is not None
479
- and sparsity_scheme.sparsity_structure
480
- == SparsityStructure.TWO_FOUR.value
481
- and sparsity_scheme.format == "dense")
485
+ if sparsity_scheme is None:
486
+ return False
487
+
488
+ is_valid_sparsity_structure: bool = (
489
+ sparsity_scheme.sparsity_structure ==
490
+ SparsityStructure.TWO_FOUR.value)
491
+
492
+ valid_compressors = {
493
+ CompressionFormat.dense.value,
494
+ CompressionFormat.sparse_24_bitmask.value
495
+ }
496
+
497
+ is_valid_sparsity = (is_valid_sparsity_structure
498
+ and sparsity_scheme.format in valid_compressors)
499
+
500
if not is_valid_sparsity:
501
return False
502
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py
+190
-48
@@ -1,13 +1,17 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import Callable, List, Optional
3
+from typing import Any, Callable, Dict, List, Optional, Tuple
4
5
import torch
6
+from compressed_tensors import CompressionFormat, ModelCompressor
7
from compressed_tensors.quantization import (QuantizationArgs,
8
QuantizationStrategy,
9
QuantizationType)
10
+from compressed_tensors.utils import combine_shards
11
12
from vllm import _custom_ops as ops
13
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
14
+ QKVParallelLinear)
15
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
16
CompressedTensorsScheme)
17
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -22,26 +26,39 @@ __all__ = ["CompressedTensors24"]
26
27
class CompressedTensors24(CompressedTensorsScheme):
28
25
- def __init__(self,
26
- quantized: bool = False,
27
- weight_quant: Optional[QuantizationArgs] = None,
28
- input_quant: Optional[QuantizationArgs] = None):
29
-
29
+ def __init__(
30
+ self,
31
+ quantized: bool = False,
32
+ weight_quant: Optional[QuantizationArgs] = None,
33
+ input_quant: Optional[QuantizationArgs] = None,
34
+ model_compression_config: Optional[Dict[str, Any]] = None,
35
+ ):
36
self.quantized = quantized
37
self.weight_quant = weight_quant
38
self.input_quant = input_quant
39
+ self.model_compressor = (
40
+ ModelCompressor.from_compression_config(model_compression_config)
41
+ if model_compression_config is not None else None)
42
+ self.do_sparse_decompress = (
43
+ self.model_compressor is not None
44
+ and self.model_compressor.sparsity_config.format
45
+ == CompressionFormat.sparse_24_bitmask.value)
46
47
@classmethod
48
def get_min_capability(cls) -> int:
49
# Only cutlass 3.x kernels are implemented so far
50
return 90
51
39
- def create_weights(self, layer: torch.nn.Module, input_size: int,
40
- output_partition_sizes: List[int],
41
- input_size_per_partition: int,
42
- params_dtype: torch.dtype, weight_loader: Callable,
43
- **kwargs):
44
-
52
+ def create_weights(
53
+ self,
54
+ layer: torch.nn.Module,
55
+ input_size: int,
56
+ output_partition_sizes: List[int],
57
+ input_size_per_partition: int,
58
+ params_dtype: torch.dtype,
59
+ weight_loader: Callable,
60
+ **kwargs,
61
+ ):
62
if not sparse_cutlass_supported():
63
raise ValueError(
64
"Sparse CUTLASS not supported. vLLM must be built with "
@@ -49,16 +66,56 @@ class CompressedTensors24(CompressedTensorsScheme):
66
67
self.output_dtype = params_dtype
68
layer.logical_widths = output_partition_sizes
69
+ layer.input_size = input_size
70
+ layer.input_size_per_partition = input_size_per_partition
71
self.weights_dtype: torch.dtype = self._get_params_dtype(params_dtype)
72
73
# parameter to store uncompressed weight
55
- weight = ModelWeightParameter(data=torch.empty(
56
- sum(output_partition_sizes),
57
- input_size_per_partition,
58
- dtype=self.weights_dtype),
59
- input_dim=1,
60
- output_dim=0,
61
- weight_loader=weight_loader)
74
+ weight = ModelWeightParameter(
75
+ data=torch.empty(
76
+ sum(output_partition_sizes),
77
+ input_size_per_partition,
78
+ dtype=self.weights_dtype,
79
+ ),
80
+ input_dim=1,
81
+ output_dim=0,
82
+ weight_loader=weight_loader,
83
+ )
84
+ if self.do_sparse_decompress:
85
+ assert all(partition_size % 8 == 0
86
+ for partition_size in output_partition_sizes
87
+ ), "All partitions must be divisible by 8 for "
88
+ "2:4 sparse compressed models"
89
+
90
+ shape = BasevLLMParameter(
91
+ data=torch.empty(2, 1, dtype=torch.int64),
92
+ weight_loader=weight_loader,
93
+ )
94
+ compressed_weight = ModelWeightParameter(
95
+ data=torch.empty(
96
+ sum(output_partition_sizes),
97
+ input_size_per_partition // 2,
98
+ dtype=self.weights_dtype,
99
+ ),
100
+ input_dim=1,
101
+ output_dim=0,
102
+ weight_loader=weight_loader,
103
+ )
104
+
105
+ bitmask = ModelWeightParameter(
106
+ data=torch.empty(
107
+ sum(output_partition_sizes),
108
+ input_size_per_partition // 8,
109
+ dtype=torch.uint8,
110
+ ),
111
+ input_dim=1,
112
+ output_dim=0,
113
+ weight_loader=weight_loader,
114
+ )
115
+
116
+ layer.register_parameter("shape", shape)
117
+ layer.register_parameter("compressed", compressed_weight)
118
+ layer.register_parameter("bitmask", bitmask)
119
120
# Check if quantized, not just 2:4 Sparse
121
if self.quantized:
@@ -68,14 +125,16 @@ class CompressedTensors24(CompressedTensorsScheme):
125
data=torch.empty((sum(output_partition_sizes), 1),
126
dtype=torch.float32),
127
output_dim=0,
71
- weight_loader=weight_loader)
128
+ weight_loader=weight_loader,
129
+ )
130
else:
131
assert (self.weight_quant and self.weight_quant.strategy
132
== QuantizationStrategy.TENSOR.value)
133
weight_scale = PerTensorScaleParameter(
134
data=torch.empty(len(output_partition_sizes),
135
dtype=torch.float32),
78
- weight_loader=weight_loader)
136
+ weight_loader=weight_loader,
137
+ )
138
139
layer.register_parameter("weight_scale", weight_scale)
140
@@ -84,9 +143,10 @@ class CompressedTensors24(CompressedTensorsScheme):
143
# register input quant scale
144
assert (self.input_quant.strategy ==
145
QuantizationStrategy.TENSOR.value)
87
- input_scale = BasevLLMParameter(data=torch.empty(
88
- 1, dtype=torch.float32),
89
- weight_loader=weight_loader)
146
+ input_scale = BasevLLMParameter(
147
+ data=torch.empty(1, dtype=torch.float32),
148
+ weight_loader=weight_loader,
149
+ )
150
151
layer.register_parameter("input_scale", input_scale)
152
@@ -107,13 +167,25 @@ class CompressedTensors24(CompressedTensorsScheme):
167
"""
168
Compress weights after loading. Store compressed weight and meta
169
tensor
110
-
170
+
171
:post-condition: layer.w_compressed and layer.meta are
172
set to the compressed weight and meta tensor in the
173
format expected by the Cutlass kernels
174
:param layer: The layer with the weights to be processed
115
-
175
+
176
"""
177
+ if self.do_sparse_decompress:
178
+ layer.weight.data = self._decompress_bitmask_compressed_weight(
179
+ compressed=layer.compressed,
180
+ bitmask=layer.bitmask,
181
+ layer=layer,
182
+ )
183
+
184
+ # compressed and bitmask tensors
185
+ # are no longer needed after decompression
186
+ del layer.compressed
187
+ del layer.bitmask
188
+
189
# torch.compile workaround
190
if hasattr(layer, "input_scale"):
191
layer.input_scale = torch.nn.Parameter(layer.input_scale.data,
@@ -121,10 +193,13 @@ class CompressedTensors24(CompressedTensorsScheme):
193
194
if self.weight_quant:
195
if self.weight_quant.strategy == QuantizationStrategy.TENSOR.value:
124
- layer.weight_scale = torch.nn.Parameter(convert_to_channelwise(
125
- weight_scale=layer.weight_scale,
126
- logical_widths=layer.logical_widths),
127
- requires_grad=False)
196
+ layer.weight_scale = torch.nn.Parameter(
197
+ convert_to_channelwise(
198
+ weight_scale=layer.weight_scale,
199
+ logical_widths=layer.logical_widths,
200
+ ),
201
+ requires_grad=False,
202
+ )
203
else:
204
# torch.compile workaround
205
layer.weight_scale = torch.nn.Parameter(
@@ -134,20 +209,22 @@ class CompressedTensors24(CompressedTensorsScheme):
209
layer.weight = torch.nn.Parameter(w_compressed, requires_grad=False)
210
layer.meta = torch.nn.Parameter(meta, requires_grad=False)
211
137
- def apply_weights(self,
138
- layer: torch.nn.Module,
139
- x: torch.Tensor,
140
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
212
+ def apply_weights(
213
+ self,
214
+ layer: torch.nn.Module,
215
+ x: torch.Tensor,
216
+ bias: Optional[torch.Tensor] = None,
217
+ ) -> torch.Tensor:
218
"""
142
- Returns the output tensor for the layer with 2:4
219
+ Returns the output tensor for the layer with 2:4
220
sparse compressed weights, given the input tensor
221
and bias
222
146
- :param layer: The layer with 2:4 sparse compressed
223
+ :param layer: The layer with 2:4 sparse compressed
224
weights to be used for the computation
225
:param x: The input tensor to the layer
226
:param bias: The bias to be added to the output tensor
150
- :return: The output tensor of the layer
227
+ :return: The output tensor of the layer
228
"""
229
if self.quantized:
230
scale = None
@@ -171,13 +248,15 @@ class CompressedTensors24(CompressedTensorsScheme):
248
input_scale = layer.input_scale
249
q_input = x
250
174
- out = ops.cutlass_scaled_sparse_mm(a=q_input,
175
- bt_nzs=layer.weight,
176
- bt_meta=layer.meta,
177
- scale_a=input_scale,
178
- scale_b=layer.weight_scale,
179
- out_dtype=self.output_dtype,
180
- bias=bias)
251
+ out = ops.cutlass_scaled_sparse_mm(
252
+ a=q_input,
253
+ bt_nzs=layer.weight,
254
+ bt_meta=layer.meta,
255
+ scale_a=input_scale,
256
+ scale_b=layer.weight_scale,
257
+ out_dtype=self.output_dtype,
258
+ bias=bias,
259
+ )
260
assert out.is_contiguous()
261
return out
262
@@ -203,8 +282,71 @@ class CompressedTensors24(CompressedTensorsScheme):
282
283
raise ValueError("Quantization type not supported by Cutlass")
284
285
+ def _decompress_bitmask_compressed_weight(
286
+ self,
287
+ compressed: torch.Tensor,
288
+ bitmask: torch.Tensor,
289
+ layer: torch.nn.Module,
290
+ ) -> torch.Tensor:
291
+ """
292
+ Decompress a compressed 2:4 sparse weight tensor using the bitmask and
293
+ return the result.
294
+
295
+ This function also supports sharded decompression.
296
+
297
+ :param compressed: The 2:4 sparse weight tensor compressed using the
298
+ sparse-24-bitmask compressor. This is different from
299
+ `cutlass_sparse_compress` which uses a different scheme (2 bits for
300
+ every nonzero element that represent the coordinate within the block
301
+ of 4). The bitmask compression here uses a bitmask to indicate the
302
+ positions of non-zero elements.
303
+ :param bitmask: The 2:4 bitmask associated with the compressed weights,
304
+ representing the positions of non-zero elements in the compressed
305
+ tensor.
306
+ :param layer: The layer whose weights need to be processed after
307
+ loading.
308
+ :return: The decompressed 2:4 sparse weight tensor.
309
+ """
310
207
-def check_24(tensor):
208
- new_tensor = tensor.view(-1, 4)
209
- zero_counts = (new_tensor == 0).sum(dim=1)
210
- return (zero_counts >= 2).all().item()
311
+ sparsity_compressor = self.model_compressor.sparsity_compressor
312
+
313
+ def _process_split(
314
+ bitmask_compressed_weight: torch.Tensor,
315
+ shape,
316
+ bitmask: torch.Tensor,
317
+ ) -> torch.Tensor:
318
+ weight_data = dict(
319
+ compressed=bitmask_compressed_weight,
320
+ shape=shape,
321
+ bitmask=bitmask,
322
+ )
323
+ return sparsity_compressor.decompress_weight(weight_data)
324
+
325
+ split_weights: List[torch.Tensor] = []
326
+ split_bitmask: List[torch.Tensor] = []
327
+ split_shape: List[Tuple[int, int]] = []
328
+
329
+ if isinstance(layer, (QKVParallelLinear, MergedColumnParallelLinear)):
330
+ split_weights = torch.split(compressed, layer.logical_widths)
331
+ split_bitmask = torch.split(bitmask, layer.logical_widths)
332
+ split_shape = [(out, layer.input_size_per_partition)
333
+ for out in layer.logical_widths]
334
+
335
+ if split_weights:
336
+ decompressed_shards = [
337
+ _process_split(compressed_weight, shape, bitmask)
338
+ for compressed_weight, shape, bitmask in zip(
339
+ split_weights, split_shape, split_bitmask)
340
+ ]
341
+ decompressed = combine_shards(decompressed_shards)
342
+ else:
343
+ decompressed = sparsity_compressor.decompress_weight(
344
+ dict(
345
+ compressed=compressed,
346
+ shape=(
347
+ layer.logical_widths[0],
348
+ layer.input_size_per_partition,
349
+ ),
350
+ bitmask=bitmask,
351
+ ))
352
+ return decompressed