A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[misc][ci] fix quant test (#8449)
youkaichao committed
Sep 13, 2024 at 02:20 UTC
a2469127db6144eedb38d0b505287c0044e4ce06
2 files changed
+21
-15
tests/quantization/test_bitsandbytes.py
+20
-12
@@ -10,6 +10,8 @@ import torch
10
11
from tests.quantization.utils import is_quant_method_supported
12
13
+from ..utils import fork_new_process_for_each_test
14
+
15
models_4bit_to_test = [
16
('huggyllama/llama-7b', 'quantize model inflight'),
17
]
@@ -29,6 +31,7 @@ models_pre_quant_8bit_to_test = [
31
@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
32
reason='bitsandbytes is not supported on this GPU type.')
33
@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
34
+@fork_new_process_for_each_test
35
def test_load_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
36
model_name, description) -> None:
37
@@ -41,6 +44,7 @@ def test_load_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
44
reason='bitsandbytes is not supported on this GPU type.')
45
@pytest.mark.parametrize("model_name, description",
46
models_pre_qaunt_4bit_to_test)
47
+@fork_new_process_for_each_test
48
def test_load_pre_quant_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
49
model_name, description) -> None:
50
@@ -52,6 +56,7 @@ def test_load_pre_quant_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
56
reason='bitsandbytes is not supported on this GPU type.')
57
@pytest.mark.parametrize("model_name, description",
58
models_pre_quant_8bit_to_test)
59
+@fork_new_process_for_each_test
60
def test_load_8bit_bnb_model(hf_runner, vllm_runner, example_prompts,
61
model_name, description) -> None:
62
@@ -77,18 +82,8 @@ def validate_generated_texts(hf_runner,
82
model_name,
83
hf_model_kwargs=None):
84
80
- if hf_model_kwargs is None:
81
- hf_model_kwargs = {}
82
-
83
- # Run with HF runner
84
- with hf_runner(model_name, model_kwargs=hf_model_kwargs) as llm:
85
- hf_outputs = llm.generate_greedy(prompts, 8)
86
- hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")
87
-
88
- # Clean up the GPU memory for the next test
89
- torch.cuda.synchronize()
90
- gc.collect()
91
- torch.cuda.empty_cache()
85
+ # NOTE: run vLLM first, as it requires a clean process
86
+ # when using distributed inference
87
88
#Run with vLLM runner
89
with vllm_runner(model_name,
@@ -104,6 +99,19 @@ def validate_generated_texts(hf_runner,
99
gc.collect()
100
torch.cuda.empty_cache()
101
102
+ if hf_model_kwargs is None:
103
+ hf_model_kwargs = {}
104
+
105
+ # Run with HF runner
106
+ with hf_runner(model_name, model_kwargs=hf_model_kwargs) as llm:
107
+ hf_outputs = llm.generate_greedy(prompts, 8)
108
+ hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")
109
+
110
+ # Clean up the GPU memory for the next test
111
+ torch.cuda.synchronize()
112
+ gc.collect()
113
+ torch.cuda.empty_cache()
114
+
115
# Compare the generated strings
116
for hf_log, vllm_log in zip(hf_logs, vllm_logs):
117
hf_str = hf_log["generated_text"]
tests/quantization/utils.py
+1
-3
@@ -1,12 +1,10 @@
1
-import torch
2
-
1
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
2
from vllm.platforms import current_platform
3
4
5
def is_quant_method_supported(quant_method: str) -> bool:
6
# Currently, all quantization methods require Nvidia or AMD GPUs
9
- if not torch.cuda.is_available():
7
+ if not (current_platform.is_cuda() or current_platform.is_rocm()):
8
return False
9
10
capability = current_platform.get_device_capability()