@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b2005e1db
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

Add: Support for Sparse24Bitmask Compressed Models

Rahul Tuli committed Feb 5, 2025 at 15:30 UTC 3b2005e1db79efe2ea4587035eb2e7ced6e258cb
4 files changed +503 -112
.buildkite/lm-eval-harness/configs/SparseLlama3.1_2of4_fp8_compressed.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash ./run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM -b "auto" -t 2
2 +model_name: "nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.6353
8 + - name: "exact_match,flexible-extract"
9 + value: 0.637
10 +limit: null
11 +num_fewshot: null
tests/quantization/test_compressed_tensors.py
+276 -56
@@ -3,6 +3,7 @@
3
4 Run `pytest tests/quantization/test_compressed_tensors.py`.
5 """
6 +
7 from typing import Optional
8
9 import pytest
@@ -22,12 +23,30 @@ from vllm.platforms import current_platform
23
24 @pytest.mark.parametrize(
25 "model_args",
25 - [("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", "tensor",
26 - QuantizationType.INT, 2560, True),
27 - ("nm-testing/tinyllama-oneshot-w8-channel-a8-tensor", "channel",
28 - QuantizationType.INT, 2560, True),
29 - ("nm-testing/asym-w8w8-int8-static-per-tensor-tiny-llama", "tensor",
30 - QuantizationType.INT, 2560, False)])
26 + [
27 + (
28 + "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change",
29 + "tensor",
30 + QuantizationType.INT,
31 + 2560,
32 + True,
33 + ),
34 + (
35 + "nm-testing/tinyllama-oneshot-w8-channel-a8-tensor",
36 + "channel",
37 + QuantizationType.INT,
38 + 2560,
39 + True,
40 + ),
41 + (
42 + "nm-testing/asym-w8w8-int8-static-per-tensor-tiny-llama",
43 + "tensor",
44 + QuantizationType.INT,
45 + 2560,
46 + False,
47 + ),
48 + ],
49 +)
50 def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
51 model_path, strategy, quant_type, shape_0, is_symmetric = model_args
52 with vllm_runner(model_path, enforce_eager=True) as llm:
@@ -85,21 +104,31 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
104 assert output
105
106
88 -@pytest.mark.parametrize("model_path", [
89 - "neuralmagic/Llama-3.2-1B-quantized.w8a8",
90 - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Asym",
91 - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym",
92 - "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym"
93 -])
107 +@pytest.mark.parametrize(
108 + "model_path",
109 + [
110 + "neuralmagic/Llama-3.2-1B-quantized.w8a8",
111 + "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Asym",
112 + "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym",
113 + "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym",
114 + ],
115 +)
116 @pytest.mark.parametrize("max_tokens", [32])
117 @pytest.mark.parametrize("num_logprobs", [10])
96 -def test_compressed_tensors_w8a8_logprobs(hf_runner, vllm_runner,
97 - example_prompts, model_path,
98 - max_tokens, num_logprobs):
118 +def test_compressed_tensors_w8a8_logprobs(
119 + hf_runner,
120 + vllm_runner,
121 + example_prompts,
122 + model_path,
123 + max_tokens,
124 + num_logprobs,
125 +):
126 dtype = "bfloat16"
127
128 # skip language translation prompt for the static per tensor asym model
102 - if model_path == "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym": # noqa: E501
129 + if (model_path ==
130 + "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym"
131 + ): # noqa: E501
132 example_prompts = example_prompts[0:-1]
133
134 with hf_runner(model_path, dtype=dtype) as hf_model:
@@ -125,13 +154,21 @@ def test_compressed_tensors_no_enforce_eager(vllm_runner):
154 assert output
155
156
128 -@pytest.mark.parametrize("model_args", [
129 - ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", "tensor"),
130 - ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2-asym", "tensor"),
131 - ("nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2", "channel"),
132 - ("nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2-asym",
133 - "channel"),
134 -])
157 +@pytest.mark.parametrize(
158 + "model_args",
159 + [
160 + ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", "tensor"),
161 + ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2-asym", "tensor"),
162 + (
163 + "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2",
164 + "channel",
165 + ),
166 + (
167 + "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2-asym",
168 + "channel",
169 + ),
170 + ],
171 +)
172 def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args):
173 model_path, strategy = model_args
174 with vllm_runner(model_path, dtype=torch.float16) as llm:
@@ -156,9 +193,12 @@ def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args):
193
194 @pytest.mark.parametrize(
195 "wNa16_args",
159 - [("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None, 8),
160 - ("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128, 8),
161 - ("nm-testing/tinyllama-oneshot-w8a16-per-channel", "channel", None, 4)])
196 + [
197 + ("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None, 8),
198 + ("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128, 8),
199 + ("nm-testing/tinyllama-oneshot-w8a16-per-channel", "channel", None, 4),
200 + ],
201 +)
202 def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
203 model, strategy, group, pack_factor = wNa16_args
204 with vllm_runner(model) as llm:
@@ -218,7 +258,8 @@ def test_compressed_tensors_fp8(vllm_runner):
258 CompressedTensorsLinearMethod)
259 assert isinstance(
260 qkv_proj.scheme,
221 - (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8))
261 + (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8),
262 + )
263
264 assert qkv_proj.input_scale.dtype is torch.float32
265
@@ -241,9 +282,14 @@ def test_compressed_tensors_kv_cache(vllm_runner):
282 assert output
283
284
244 -@pytest.mark.skipif(not sparse_cutlass_supported(),
245 - reason="Sparse FP8 is not yet supported on this GPU type.")
246 -def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy):
285 +@pytest.mark.skipif(
286 + not sparse_cutlass_supported(),
287 + reason="Sparse FP8 is not yet supported on this GPU type.",
288 +)
289 +def _test_2of4_quant_models(qkv_proj,
290 + weight_strategy,
291 + input_strategy,
292 + format="dense"):
293 assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
294 assert isinstance(qkv_proj.scheme, CompressedTensors24)
295
@@ -252,22 +298,39 @@ def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy):
298 assert qkv_proj.scheme.quantized
299 assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
300 sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
255 - assert sparsity_map.get("Linear").format == "dense"
301 + assert sparsity_map.get("Linear").format == format
302 assert sparsity_map.get("Linear").sparsity_structure == "2:4"
303
304
259 -@pytest.mark.skipif(not current_platform.has_device_capability(90),
260 - reason="Sparse FP8 is not yet supported on this GPU type.")
261 -@pytest.mark.parametrize("args_2of4", [
262 - ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-2of4-testing", "channel",
263 - "token"),
264 - ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-Per-Tensor-testing",
265 - "channel", "tensor"),
266 - ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-testing", "tensor",
267 - "tensor"),
268 - ("nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-IA-Per-Tensor-Weight-testing",
269 - "tensor", "token"),
270 -])
305 +@pytest.mark.skipif(
306 + not current_platform.has_device_capability(90),
307 + reason="Sparse FP8 is not yet supported on this GPU type.",
308 +)
309 +@pytest.mark.parametrize(
310 + "args_2of4",
311 + [
312 + (
313 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-2of4-testing",
314 + "channel",
315 + "token",
316 + ),
317 + (
318 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-Per-Tensor-testing",
319 + "channel",
320 + "tensor",
321 + ),
322 + (
323 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Static-testing",
324 + "tensor",
325 + "tensor",
326 + ),
327 + (
328 + "nm-testing/Meta-Llama-3-8B-Instruct-FP8-Dynamic-IA-Per-Tensor-Weight-testing",
329 + "tensor",
330 + "token",
331 + ),
332 + ],
333 +)
334 def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
335 model, weight_strategy, input_strategy = args_2of4
336 with vllm_runner(model) as llm:
@@ -286,16 +349,134 @@ def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
349 assert output
350
351
289 -@pytest.mark.skipif(not sparse_cutlass_supported(),
290 - reason="Sparse FP8 is not yet supported on this GPU type.")
291 -@pytest.mark.parametrize("args_2of4", [
292 - ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing",
293 - "channel", "token"),
294 - ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Static-testing", "tensor",
295 - "tensor"),
296 - ("nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Tensor-Weight-testing",
297 - "tensor", "token"),
298 -])
352 +@pytest.mark.skipif(
353 + not current_platform.has_device_capability(90),
354 + reason="Sparse FP8 is not yet supported on this GPU type.",
355 +)
356 +@pytest.mark.parametrize(
357 + "args_2of4",
358 + [
359 + (
360 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM",
361 + "channel",
362 + "token",
363 + ),
364 + (
365 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_tensor_act_fp8-BitM",
366 + "channel",
367 + "tensor",
368 + ),
369 + (
370 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_per_tok_dyn_act_fp8-BitM",
371 + "tensor",
372 + "token",
373 + ),
374 + (
375 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_tensor_act_fp8-BitM",
376 + "tensor",
377 + "tensor",
378 + ),
379 + ],
380 +)
381 +def test_compressed_tensors_2of4_quant_fp8_compressed(vllm_runner, args_2of4):
382 + model, weight_strategy, input_strategy = args_2of4
383 + with vllm_runner(model) as llm:
384 +
385 + def check_model(model):
386 + layer = model.model.layers[0]
387 +
388 + qkv_proj = layer.self_attn.qkv_proj
389 + assert qkv_proj.scheme.weights_dtype == torch.float8_e4m3fn
390 + _test_2of4_quant_models(
391 + qkv_proj,
392 + weight_strategy,
393 + input_strategy,
394 + format="sparse-24-bitmask",
395 + )
396 +
397 + llm.apply_model(check_model)
398 +
399 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
400 + print(output)
401 + assert output
402 +
403 +
404 +@pytest.mark.skipif(
405 + not sparse_cutlass_supported(),
406 + reason="cutlass is not yet supported on this GPU type.",
407 +)
408 +@pytest.mark.parametrize(
409 + "args_2of4",
410 + [
411 + (
412 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_int8-BitM",
413 + "channel",
414 + "token",
415 + ),
416 + (
417 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-chnl_wts_tensor_act_int8-BitM",
418 + "channel",
419 + "tensor",
420 + ),
421 + (
422 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_per_tok_dyn_act_int8-BitM",
423 + "tensor",
424 + "token",
425 + ),
426 + (
427 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-gsm8k-pruned.2of4-tensor_wts_tensor_act_int8-BitM",
428 + "tensor",
429 + "tensor",
430 + ),
431 + ],
432 +)
433 +def test_compressed_tensors_2of4_quant_int8_compressed(vllm_runner, args_2of4):
434 + model, weight_strategy, input_strategy = args_2of4
435 + with vllm_runner(model) as llm:
436 +
437 + def check_model(model):
438 + layer = model.model.layers[0]
439 +
440 + qkv_proj = layer.self_attn.qkv_proj
441 + assert qkv_proj.scheme.weights_dtype == torch.int8
442 + _test_2of4_quant_models(
443 + qkv_proj,
444 + weight_strategy,
445 + input_strategy,
446 + format="sparse-24-bitmask",
447 + )
448 +
449 + llm.apply_model(check_model)
450 +
451 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
452 + print(output)
453 + assert output
454 +
455 +
456 +@pytest.mark.skipif(
457 + not sparse_cutlass_supported(),
458 + reason="Sparse FP8 is not yet supported on this GPU type.",
459 +)
460 +@pytest.mark.parametrize(
461 + "args_2of4",
462 + [
463 + (
464 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Channel-Weight-testing",
465 + "channel",
466 + "token",
467 + ),
468 + (
469 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Static-testing",
470 + "tensor",
471 + "tensor",
472 + ),
473 + (
474 + "nm-testing/TinyLlama-1.1B-Chat-v1.0-INT8-Dynamic-IA-Per-Tensor-Weight-testing",
475 + "tensor",
476 + "token",
477 + ),
478 + ],
479 +)
480 def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
481 model, weight_strategy, input_strategy = args_2of4
482 with vllm_runner(model) as llm:
@@ -317,10 +498,12 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
498 @pytest.mark.skip(reason="2of4 sparse w16a16 CUTLASS produces bad output.")
499 @pytest.mark.skipif(
500 not sparse_cutlass_supported(),
320 - reason="2of4 Sparse is not yet supported on this GPU type.")
501 + reason="2of4 Sparse is not yet supported on this GPU type.",
502 +)
503 @pytest.mark.parametrize(
504 "args_2of4",
323 - [("nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor")])
505 + [("nm-testing/TinyLlama-1.1B-Chat-v1.0-2of4-Sparse-Dense-Compressor")],
506 +)
507 def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):
508 model = args_2of4
509 with vllm_runner(model) as llm:
@@ -337,7 +520,9 @@ def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):
520 assert qkv_proj.scheme.input_quant is None
521 assert not qkv_proj.scheme.quantized
522 assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
340 - sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
523 + sparsity_map = (
524 + qkv_proj.quant_method.quantization_config.sparsity_scheme_map
525 + ) # noqa: E501
526 assert sparsity_map.get("Linear").format == "dense"
527 assert sparsity_map.get("Linear").sparsity_structure == "2:4"
528
@@ -346,3 +531,38 @@ def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):
531 output = llm.generate_greedy("Hello my name is", max_tokens=20)
532 print(output)
533 assert output
534 +
535 +
536 +@pytest.mark.skipif(
537 + not sparse_cutlass_supported(),
538 + reason="Cutlass is not yet supported on this GPU type.",
539 +)
540 +@pytest.mark.parametrize(
541 + "args_2of4", [("nm-testing/llama2.c-stories42M-pruned2.4-compressed")])
542 +def test_compressed_tensors_2of4_sparse_compressed(vllm_runner, args_2of4):
543 + model = args_2of4
544 + with vllm_runner(model) as llm:
545 +
546 + def check_model(model):
547 + layer = model.model.layers[0]
548 +
549 + qkv_proj = layer.self_attn.qkv_proj
550 + assert isinstance(qkv_proj.quant_method,
551 + CompressedTensorsLinearMethod)
552 + assert isinstance(qkv_proj.scheme, CompressedTensors24)
553 +
554 + assert qkv_proj.scheme.weight_quant is None
555 + assert qkv_proj.scheme.input_quant is None
556 + assert not qkv_proj.scheme.quantized
557 + assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
558 + sparsity_map = (
559 + qkv_proj.quant_method.quantization_config.sparsity_scheme_map
560 + ) # noqa: E501
561 + assert sparsity_map.get("Linear").format == "sparse-24-bitmask"
562 + assert sparsity_map.get("Linear").sparsity_structure == "2:4"
563 +
564 + llm.apply_model(check_model)
565 +
566 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
567 + print(output)
568 + assert output
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+26 -8
@@ -417,15 +417,22 @@ class CompressedTensorsConfig(QuantizationConfig):
417 return None
418 # Have a valid sparsity scheme
419 # Validate layer is supported by Cutlass 2:4 Kernel
420 - scheme = CompressedTensors24(quantized=weight_quant is not None
421 - or input_quant is not None,
422 - weight_quant=weight_quant,
423 - input_quant=input_quant)
420 + model_compression_config = (None if sparsity_scheme is None
421 + or sparsity_scheme.format == "dense"
422 + else self.config)
423 +
424 + scheme = CompressedTensors24(
425 + quantized=weight_quant is not None or input_quant is not None,
426 + weight_quant=weight_quant,
427 + input_quant=input_quant,
428 + model_compression_config=model_compression_config,
429 + )
430 elif weight_quant is None:
431 logger.warning_once("Acceleration for non-quantized schemes is "
432 "not supported by Compressed Tensors. "
433 "Falling back to UnquantizedLinearMethod")
434 return None
435 +
436 else:
437 # Find the quant_scheme
438 scheme = self._get_scheme_from_parts( # type: ignore
@@ -475,10 +482,21 @@ class CompressedTensorsConfig(QuantizationConfig):
482 :return: True if the layer is supported by the Cutlass 2:4 Kernel
483 False otherwise
484 """
478 - is_valid_sparsity = (sparsity_scheme is not None
479 - and sparsity_scheme.sparsity_structure
480 - == SparsityStructure.TWO_FOUR.value
481 - and sparsity_scheme.format == "dense")
485 + if sparsity_scheme is None:
486 + return False
487 +
488 + is_valid_sparsity_structure: bool = (
489 + sparsity_scheme.sparsity_structure ==
490 + SparsityStructure.TWO_FOUR.value)
491 +
492 + valid_compressors = {
493 + CompressionFormat.dense.value,
494 + CompressionFormat.sparse_24_bitmask.value
495 + }
496 +
497 + is_valid_sparsity = (is_valid_sparsity_structure
498 + and sparsity_scheme.format in valid_compressors)
499 +
500 if not is_valid_sparsity:
501 return False
502
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_24.py
+190 -48
@@ -1,13 +1,17 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from typing import Callable, List, Optional
3 +from typing import Any, Callable, Dict, List, Optional, Tuple
4
5 import torch
6 +from compressed_tensors import CompressionFormat, ModelCompressor
7 from compressed_tensors.quantization import (QuantizationArgs,
8 QuantizationStrategy,
9 QuantizationType)
10 +from compressed_tensors.utils import combine_shards
11
12 from vllm import _custom_ops as ops
13 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
14 + QKVParallelLinear)
15 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
16 CompressedTensorsScheme)
17 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -22,26 +26,39 @@ __all__ = ["CompressedTensors24"]
26
27 class CompressedTensors24(CompressedTensorsScheme):
28
25 - def __init__(self,
26 - quantized: bool = False,
27 - weight_quant: Optional[QuantizationArgs] = None,
28 - input_quant: Optional[QuantizationArgs] = None):
29 -
29 + def __init__(
30 + self,
31 + quantized: bool = False,
32 + weight_quant: Optional[QuantizationArgs] = None,
33 + input_quant: Optional[QuantizationArgs] = None,
34 + model_compression_config: Optional[Dict[str, Any]] = None,
35 + ):
36 self.quantized = quantized
37 self.weight_quant = weight_quant
38 self.input_quant = input_quant
39 + self.model_compressor = (
40 + ModelCompressor.from_compression_config(model_compression_config)
41 + if model_compression_config is not None else None)
42 + self.do_sparse_decompress = (
43 + self.model_compressor is not None
44 + and self.model_compressor.sparsity_config.format
45 + == CompressionFormat.sparse_24_bitmask.value)
46
47 @classmethod
48 def get_min_capability(cls) -> int:
49 # Only cutlass 3.x kernels are implemented so far
50 return 90
51
39 - def create_weights(self, layer: torch.nn.Module, input_size: int,
40 - output_partition_sizes: List[int],
41 - input_size_per_partition: int,
42 - params_dtype: torch.dtype, weight_loader: Callable,
43 - **kwargs):
44 -
52 + def create_weights(
53 + self,
54 + layer: torch.nn.Module,
55 + input_size: int,
56 + output_partition_sizes: List[int],
57 + input_size_per_partition: int,
58 + params_dtype: torch.dtype,
59 + weight_loader: Callable,
60 + **kwargs,
61 + ):
62 if not sparse_cutlass_supported():
63 raise ValueError(
64 "Sparse CUTLASS not supported. vLLM must be built with "
@@ -49,16 +66,56 @@ class CompressedTensors24(CompressedTensorsScheme):
66
67 self.output_dtype = params_dtype
68 layer.logical_widths = output_partition_sizes
69 + layer.input_size = input_size
70 + layer.input_size_per_partition = input_size_per_partition
71 self.weights_dtype: torch.dtype = self._get_params_dtype(params_dtype)
72
73 # parameter to store uncompressed weight
55 - weight = ModelWeightParameter(data=torch.empty(
56 - sum(output_partition_sizes),
57 - input_size_per_partition,
58 - dtype=self.weights_dtype),
59 - input_dim=1,
60 - output_dim=0,
61 - weight_loader=weight_loader)
74 + weight = ModelWeightParameter(
75 + data=torch.empty(
76 + sum(output_partition_sizes),
77 + input_size_per_partition,
78 + dtype=self.weights_dtype,
79 + ),
80 + input_dim=1,
81 + output_dim=0,
82 + weight_loader=weight_loader,
83 + )
84 + if self.do_sparse_decompress:
85 + assert all(partition_size % 8 == 0
86 + for partition_size in output_partition_sizes
87 + ), "All partitions must be divisible by 8 for "
88 + "2:4 sparse compressed models"
89 +
90 + shape = BasevLLMParameter(
91 + data=torch.empty(2, 1, dtype=torch.int64),
92 + weight_loader=weight_loader,
93 + )
94 + compressed_weight = ModelWeightParameter(
95 + data=torch.empty(
96 + sum(output_partition_sizes),
97 + input_size_per_partition // 2,
98 + dtype=self.weights_dtype,
99 + ),
100 + input_dim=1,
101 + output_dim=0,
102 + weight_loader=weight_loader,
103 + )
104 +
105 + bitmask = ModelWeightParameter(
106 + data=torch.empty(
107 + sum(output_partition_sizes),
108 + input_size_per_partition // 8,
109 + dtype=torch.uint8,
110 + ),
111 + input_dim=1,
112 + output_dim=0,
113 + weight_loader=weight_loader,
114 + )
115 +
116 + layer.register_parameter("shape", shape)
117 + layer.register_parameter("compressed", compressed_weight)
118 + layer.register_parameter("bitmask", bitmask)
119
120 # Check if quantized, not just 2:4 Sparse
121 if self.quantized:
@@ -68,14 +125,16 @@ class CompressedTensors24(CompressedTensorsScheme):
125 data=torch.empty((sum(output_partition_sizes), 1),
126 dtype=torch.float32),
127 output_dim=0,
71 - weight_loader=weight_loader)
128 + weight_loader=weight_loader,
129 + )
130 else:
131 assert (self.weight_quant and self.weight_quant.strategy
132 == QuantizationStrategy.TENSOR.value)
133 weight_scale = PerTensorScaleParameter(
134 data=torch.empty(len(output_partition_sizes),
135 dtype=torch.float32),
78 - weight_loader=weight_loader)
136 + weight_loader=weight_loader,
137 + )
138
139 layer.register_parameter("weight_scale", weight_scale)
140
@@ -84,9 +143,10 @@ class CompressedTensors24(CompressedTensorsScheme):
143 # register input quant scale
144 assert (self.input_quant.strategy ==
145 QuantizationStrategy.TENSOR.value)
87 - input_scale = BasevLLMParameter(data=torch.empty(
88 - 1, dtype=torch.float32),
89 - weight_loader=weight_loader)
146 + input_scale = BasevLLMParameter(
147 + data=torch.empty(1, dtype=torch.float32),
148 + weight_loader=weight_loader,
149 + )
150
151 layer.register_parameter("input_scale", input_scale)
152
@@ -107,13 +167,25 @@ class CompressedTensors24(CompressedTensorsScheme):
167 """
168 Compress weights after loading. Store compressed weight and meta
169 tensor
110 -
170 +
171 :post-condition: layer.w_compressed and layer.meta are
172 set to the compressed weight and meta tensor in the
173 format expected by the Cutlass kernels
174 :param layer: The layer with the weights to be processed
115 -
175 +
176 """
177 + if self.do_sparse_decompress:
178 + layer.weight.data = self._decompress_bitmask_compressed_weight(
179 + compressed=layer.compressed,
180 + bitmask=layer.bitmask,
181 + layer=layer,
182 + )
183 +
184 + # compressed and bitmask tensors
185 + # are no longer needed after decompression
186 + del layer.compressed
187 + del layer.bitmask
188 +
189 # torch.compile workaround
190 if hasattr(layer, "input_scale"):
191 layer.input_scale = torch.nn.Parameter(layer.input_scale.data,
@@ -121,10 +193,13 @@ class CompressedTensors24(CompressedTensorsScheme):
193
194 if self.weight_quant:
195 if self.weight_quant.strategy == QuantizationStrategy.TENSOR.value:
124 - layer.weight_scale = torch.nn.Parameter(convert_to_channelwise(
125 - weight_scale=layer.weight_scale,
126 - logical_widths=layer.logical_widths),
127 - requires_grad=False)
196 + layer.weight_scale = torch.nn.Parameter(
197 + convert_to_channelwise(
198 + weight_scale=layer.weight_scale,
199 + logical_widths=layer.logical_widths,
200 + ),
201 + requires_grad=False,
202 + )
203 else:
204 # torch.compile workaround
205 layer.weight_scale = torch.nn.Parameter(
@@ -134,20 +209,22 @@ class CompressedTensors24(CompressedTensorsScheme):
209 layer.weight = torch.nn.Parameter(w_compressed, requires_grad=False)
210 layer.meta = torch.nn.Parameter(meta, requires_grad=False)
211
137 - def apply_weights(self,
138 - layer: torch.nn.Module,
139 - x: torch.Tensor,
140 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
212 + def apply_weights(
213 + self,
214 + layer: torch.nn.Module,
215 + x: torch.Tensor,
216 + bias: Optional[torch.Tensor] = None,
217 + ) -> torch.Tensor:
218 """
142 - Returns the output tensor for the layer with 2:4
219 + Returns the output tensor for the layer with 2:4
220 sparse compressed weights, given the input tensor
221 and bias
222
146 - :param layer: The layer with 2:4 sparse compressed
223 + :param layer: The layer with 2:4 sparse compressed
224 weights to be used for the computation
225 :param x: The input tensor to the layer
226 :param bias: The bias to be added to the output tensor
150 - :return: The output tensor of the layer
227 + :return: The output tensor of the layer
228 """
229 if self.quantized:
230 scale = None
@@ -171,13 +248,15 @@ class CompressedTensors24(CompressedTensorsScheme):
248 input_scale = layer.input_scale
249 q_input = x
250
174 - out = ops.cutlass_scaled_sparse_mm(a=q_input,
175 - bt_nzs=layer.weight,
176 - bt_meta=layer.meta,
177 - scale_a=input_scale,
178 - scale_b=layer.weight_scale,
179 - out_dtype=self.output_dtype,
180 - bias=bias)
251 + out = ops.cutlass_scaled_sparse_mm(
252 + a=q_input,
253 + bt_nzs=layer.weight,
254 + bt_meta=layer.meta,
255 + scale_a=input_scale,
256 + scale_b=layer.weight_scale,
257 + out_dtype=self.output_dtype,
258 + bias=bias,
259 + )
260 assert out.is_contiguous()
261 return out
262
@@ -203,8 +282,71 @@ class CompressedTensors24(CompressedTensorsScheme):
282
283 raise ValueError("Quantization type not supported by Cutlass")
284
285 + def _decompress_bitmask_compressed_weight(
286 + self,
287 + compressed: torch.Tensor,
288 + bitmask: torch.Tensor,
289 + layer: torch.nn.Module,
290 + ) -> torch.Tensor:
291 + """
292 + Decompress a compressed 2:4 sparse weight tensor using the bitmask and
293 + return the result.
294 +
295 + This function also supports sharded decompression.
296 +
297 + :param compressed: The 2:4 sparse weight tensor compressed using the
298 + sparse-24-bitmask compressor. This is different from
299 + `cutlass_sparse_compress` which uses a different scheme (2 bits for
300 + every nonzero element that represent the coordinate within the block
301 + of 4). The bitmask compression here uses a bitmask to indicate the
302 + positions of non-zero elements.
303 + :param bitmask: The 2:4 bitmask associated with the compressed weights,
304 + representing the positions of non-zero elements in the compressed
305 + tensor.
306 + :param layer: The layer whose weights need to be processed after
307 + loading.
308 + :return: The decompressed 2:4 sparse weight tensor.
309 + """
310
207 -def check_24(tensor):
208 - new_tensor = tensor.view(-1, 4)
209 - zero_counts = (new_tensor == 0).sum(dim=1)
210 - return (zero_counts >= 2).all().item()
311 + sparsity_compressor = self.model_compressor.sparsity_compressor
312 +
313 + def _process_split(
314 + bitmask_compressed_weight: torch.Tensor,
315 + shape,
316 + bitmask: torch.Tensor,
317 + ) -> torch.Tensor:
318 + weight_data = dict(
319 + compressed=bitmask_compressed_weight,
320 + shape=shape,
321 + bitmask=bitmask,
322 + )
323 + return sparsity_compressor.decompress_weight(weight_data)
324 +
325 + split_weights: List[torch.Tensor] = []
326 + split_bitmask: List[torch.Tensor] = []
327 + split_shape: List[Tuple[int, int]] = []
328 +
329 + if isinstance(layer, (QKVParallelLinear, MergedColumnParallelLinear)):
330 + split_weights = torch.split(compressed, layer.logical_widths)
331 + split_bitmask = torch.split(bitmask, layer.logical_widths)
332 + split_shape = [(out, layer.input_size_per_partition)
333 + for out in layer.logical_widths]
334 +
335 + if split_weights:
336 + decompressed_shards = [
337 + _process_split(compressed_weight, shape, bitmask)
338 + for compressed_weight, shape, bitmask in zip(
339 + split_weights, split_shape, split_bitmask)
340 + ]
341 + decompressed = combine_shards(decompressed_shards)
342 + else:
343 + decompressed = sparsity_compressor.decompress_weight(
344 + dict(
345 + compressed=compressed,
346 + shape=(
347 + layer.logical_widths[0],
348 + layer.input_size_per_partition,
349 + ),
350 + bitmask=bitmask,
351 + ))
352 + return decompressed