@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 23ec72fa03
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[CI/Build][REDO] Add is_quant_method_supported to control quantization test configurations (#5466)

Michael Goin committed Jun 13, 2024 at 11:18 UTC 23ec72fa032b3d81a5ea9eb0f7c607f1d6e7949a
8 files changed +32 -71
tests/models/test_aqlm.py
+2 -11
@@ -4,17 +4,8 @@ Run `pytest tests/models/test_aqlm.py`.
4 """
5
6 import pytest
7 -import torch
7
9 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
10 -
11 -aqlm_not_supported = True
12 -
13 -if torch.cuda.is_available():
14 - capability = torch.cuda.get_device_capability()
15 - capability = capability[0] * 10 + capability[1]
16 - aqlm_not_supported = (capability <
17 - QUANTIZATION_METHODS["aqlm"].get_min_capability())
8 +from tests.quantization.utils import is_quant_method_supported
9
10 # In this test we hardcode prompts and generations for the model so we don't
11 # need to require the AQLM package as a dependency
@@ -67,7 +58,7 @@ ground_truth_generations = [
58 ]
59
60
70 -@pytest.mark.skipif(aqlm_not_supported,
61 +@pytest.mark.skipif(not is_quant_method_supported("aqlm"),
62 reason="AQLM is not supported on this GPU type.")
63 @pytest.mark.parametrize("model", ["ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf"])
64 @pytest.mark.parametrize("dtype", ["half"])
tests/models/test_fp8.py
+2 -10
@@ -8,8 +8,8 @@ import pytest
8 import torch
9 from transformers import AutoTokenizer
10
11 +from tests.quantization.utils import is_quant_method_supported
12 from vllm import LLM, SamplingParams
12 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
14 os.environ["TOKENIZERS_PARALLELISM"] = "true"
15
@@ -67,16 +67,8 @@ EXPECTED_STRS_MAP = {
67 },
68 }
69
70 -fp8_not_supported = True
70
72 -if torch.cuda.is_available():
73 - capability = torch.cuda.get_device_capability()
74 - capability = capability[0] * 10 + capability[1]
75 - fp8_not_supported = (capability <
76 - QUANTIZATION_METHODS["fp8"].get_min_capability())
77 -
78 -
79 -@pytest.mark.skipif(fp8_not_supported,
71 +@pytest.mark.skipif(not is_quant_method_supported("fp8"),
72 reason="fp8 is not supported on this GPU type.")
73 @pytest.mark.parametrize("model_name", MODELS)
74 @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
tests/models/test_gptq_marlin.py
+2 -11
@@ -11,9 +11,8 @@ Run `pytest tests/models/test_gptq_marlin.py`.
11 import os
12
13 import pytest
14 -import torch
14
16 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
15 +from tests.quantization.utils import is_quant_method_supported
16 from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
17
18 from .utils import check_logprobs_close
@@ -22,14 +21,6 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
21
22 MAX_MODEL_LEN = 1024
23
25 -gptq_marlin_not_supported = True
26 -
27 -if torch.cuda.is_available():
28 - capability = torch.cuda.get_device_capability()
29 - capability = capability[0] * 10 + capability[1]
30 - gptq_marlin_not_supported = (
31 - capability < QUANTIZATION_METHODS["gptq_marlin"].get_min_capability())
32 -
24 MODELS = [
25 # act_order==False, group_size=channelwise
26 ("robertgshaw2/zephyr-7b-beta-channelwise-gptq", "main"),
@@ -53,7 +44,7 @@ MODELS = [
44
45
46 @pytest.mark.flaky(reruns=3)
56 -@pytest.mark.skipif(gptq_marlin_not_supported,
47 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
48 reason="gptq_marlin is not supported on this GPU type.")
49 @pytest.mark.parametrize("model", MODELS)
50 @pytest.mark.parametrize("dtype", ["half", "bfloat16"])
tests/models/test_gptq_marlin_24.py
+2 -11
@@ -9,18 +9,9 @@ Run `pytest tests/models/test_marlin_24.py`.
9 from dataclasses import dataclass
10
11 import pytest
12 -import torch
12
13 from tests.models.utils import check_logprobs_close
15 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 -
17 -marlin_not_supported = True
18 -
19 -if torch.cuda.is_available():
20 - capability = torch.cuda.get_device_capability()
21 - capability = capability[0] * 10 + capability[1]
22 - marlin_not_supported = (
23 - capability < QUANTIZATION_METHODS["marlin"].get_min_capability())
14 +from tests.quantization.utils import is_quant_method_supported
15
16
17 @dataclass
@@ -47,7 +38,7 @@ model_pairs = [
38
39
40 @pytest.mark.flaky(reruns=2)
50 -@pytest.mark.skipif(marlin_not_supported,
41 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin_24"),
42 reason="Marlin24 is not supported on this GPU type.")
43 @pytest.mark.parametrize("model_pair", model_pairs)
44 @pytest.mark.parametrize("dtype", ["half"])
tests/models/test_marlin.py
+2 -11
@@ -13,20 +13,11 @@ Run `pytest tests/models/test_marlin.py`.
13 from dataclasses import dataclass
14
15 import pytest
16 -import torch
16
18 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
17 +from tests.quantization.utils import is_quant_method_supported
18
19 from .utils import check_logprobs_close
20
22 -marlin_not_supported = True
23 -
24 -if torch.cuda.is_available():
25 - capability = torch.cuda.get_device_capability()
26 - capability = capability[0] * 10 + capability[1]
27 - marlin_not_supported = (
28 - capability < QUANTIZATION_METHODS["marlin"].get_min_capability())
29 -
21
22 @dataclass
23 class ModelPair:
@@ -45,7 +36,7 @@ model_pairs = [
36
37
38 @pytest.mark.flaky(reruns=2)
48 -@pytest.mark.skipif(marlin_not_supported,
39 +@pytest.mark.skipif(not is_quant_method_supported("marlin"),
40 reason="Marlin is not supported on this GPU type.")
41 @pytest.mark.parametrize("model_pair", model_pairs)
42 @pytest.mark.parametrize("dtype", ["half"])
tests/quantization/test_bitsandbytes.py
+3 -7
@@ -5,16 +5,12 @@ Run `pytest tests/quantization/test_bitsandbytes.py`.
5 import pytest
6 import torch
7
8 +from tests.quantization.utils import is_quant_method_supported
9 from vllm import SamplingParams
9 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
10
11 -capability = torch.cuda.get_device_capability()
12 -capability = capability[0] * 10 + capability[1]
11
14 -
15 -@pytest.mark.skipif(
16 - capability < QUANTIZATION_METHODS['bitsandbytes'].get_min_capability(),
17 - reason='bitsandbytes is not supported on this GPU type.')
12 +@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
13 + reason='bitsandbytes is not supported on this GPU type.')
14 def test_load_bnb_model(vllm_runner) -> None:
15 with vllm_runner('huggyllama/llama-7b',
16 quantization='bitsandbytes',
tests/quantization/test_fp8.py
+5 -10
@@ -5,17 +5,13 @@ Run `pytest tests/quantization/test_fp8.py --forked`.
5 import pytest
6 import torch
7
8 +from tests.quantization.utils import is_quant_method_supported
9 from vllm._custom_ops import scaled_fp8_quant
9 -from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
10 from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
11
12 -capability = torch.cuda.get_device_capability()
13 -capability = capability[0] * 10 + capability[1]
12
15 -
16 -@pytest.mark.skipif(
17 - capability < QUANTIZATION_METHODS["fp8"].get_min_capability(),
18 - reason="FP8 is not supported on this GPU type.")
13 +@pytest.mark.skipif(not is_quant_method_supported("fp8"),
14 + reason="FP8 is not supported on this GPU type.")
15 def test_load_fp16_model(vllm_runner) -> None:
16 with vllm_runner("facebook/opt-125m", quantization="fp8") as llm:
17
@@ -25,9 +21,8 @@ def test_load_fp16_model(vllm_runner) -> None:
21 assert fc1.weight.dtype == torch.float8_e4m3fn
22
23
28 -@pytest.mark.skipif(
29 - capability < QUANTIZATION_METHODS["fp8"].get_min_capability(),
30 - reason="FP8 is not supported on this GPU type.")
24 +@pytest.mark.skipif(not is_quant_method_supported("fp8"),
25 + reason="FP8 is not supported on this GPU type.")
26 @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
27 def test_scaled_fp8_quant(dtype) -> None:
28
tests/quantization/utils.py new
+14
@@ -0,0 +1,14 @@
1 +import torch
2 +
3 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
4 +
5 +
6 +def is_quant_method_supported(quant_method: str) -> bool:
7 + # Currently, all quantization methods require Nvidia or AMD GPUs
8 + if not torch.cuda.is_available():
9 + return False
10 +
11 + capability = torch.cuda.get_device_capability()
12 + capability = capability[0] * 10 + capability[1]
13 + return (capability <
14 + QUANTIZATION_METHODS[quant_method].get_min_capability())