@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a2469127db
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[misc][ci] fix quant test (#8449)

youkaichao committed Sep 13, 2024 at 02:20 UTC a2469127db6144eedb38d0b505287c0044e4ce06
2 files changed +21 -15
tests/quantization/test_bitsandbytes.py
+20 -12
@@ -10,6 +10,8 @@ import torch
10
11 from tests.quantization.utils import is_quant_method_supported
12
13 +from ..utils import fork_new_process_for_each_test
14 +
15 models_4bit_to_test = [
16 ('huggyllama/llama-7b', 'quantize model inflight'),
17 ]
@@ -29,6 +31,7 @@ models_pre_quant_8bit_to_test = [
31 @pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
32 reason='bitsandbytes is not supported on this GPU type.')
33 @pytest.mark.parametrize("model_name, description", models_4bit_to_test)
34 +@fork_new_process_for_each_test
35 def test_load_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
36 model_name, description) -> None:
37
@@ -41,6 +44,7 @@ def test_load_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
44 reason='bitsandbytes is not supported on this GPU type.')
45 @pytest.mark.parametrize("model_name, description",
46 models_pre_qaunt_4bit_to_test)
47 +@fork_new_process_for_each_test
48 def test_load_pre_quant_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
49 model_name, description) -> None:
50
@@ -52,6 +56,7 @@ def test_load_pre_quant_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
56 reason='bitsandbytes is not supported on this GPU type.')
57 @pytest.mark.parametrize("model_name, description",
58 models_pre_quant_8bit_to_test)
59 +@fork_new_process_for_each_test
60 def test_load_8bit_bnb_model(hf_runner, vllm_runner, example_prompts,
61 model_name, description) -> None:
62
@@ -77,18 +82,8 @@ def validate_generated_texts(hf_runner,
82 model_name,
83 hf_model_kwargs=None):
84
80 - if hf_model_kwargs is None:
81 - hf_model_kwargs = {}
82 -
83 - # Run with HF runner
84 - with hf_runner(model_name, model_kwargs=hf_model_kwargs) as llm:
85 - hf_outputs = llm.generate_greedy(prompts, 8)
86 - hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")
87 -
88 - # Clean up the GPU memory for the next test
89 - torch.cuda.synchronize()
90 - gc.collect()
91 - torch.cuda.empty_cache()
85 + # NOTE: run vLLM first, as it requires a clean process
86 + # when using distributed inference
87
88 #Run with vLLM runner
89 with vllm_runner(model_name,
@@ -104,6 +99,19 @@ def validate_generated_texts(hf_runner,
99 gc.collect()
100 torch.cuda.empty_cache()
101
102 + if hf_model_kwargs is None:
103 + hf_model_kwargs = {}
104 +
105 + # Run with HF runner
106 + with hf_runner(model_name, model_kwargs=hf_model_kwargs) as llm:
107 + hf_outputs = llm.generate_greedy(prompts, 8)
108 + hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")
109 +
110 + # Clean up the GPU memory for the next test
111 + torch.cuda.synchronize()
112 + gc.collect()
113 + torch.cuda.empty_cache()
114 +
115 # Compare the generated strings
116 for hf_log, vllm_log in zip(hf_logs, vllm_logs):
117 hf_str = hf_log["generated_text"]
tests/quantization/utils.py
+1 -3
@@ -1,12 +1,10 @@
1 -import torch
2 -
1 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
2 from vllm.platforms import current_platform
3
4
5 def is_quant_method_supported(quant_method: str) -> bool:
6 # Currently, all quantization methods require Nvidia or AMD GPUs
9 - if not torch.cuda.is_available():
7 + if not (current_platform.is_cuda() or current_platform.is_rocm()):
8 return False
9
10 capability = current_platform.get_device_capability()