A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[CI/Build] Add test decorator for minimum GPU memory (#8925)
Cyrus Leung committed
Sep 29, 2024 at 10:50 UTC
26a68d5d7e7dd47c7d8538a326493c8a171f5016
14 files changed
+117
-73
tests/lora/test_baichuan.py
+4
-5
@@ -63,12 +63,11 @@ def test_baichuan_lora(baichuan_lora_files):
63
assert output2[i] == expected_lora_output[i]
64
65
66
-@pytest.mark.skip("Requires multiple GPUs")
66
@pytest.mark.parametrize("fully_sharded", [True, False])
68
-def test_baichuan_tensor_parallel_equality(baichuan_lora_files, fully_sharded):
69
- # Cannot use as it will initialize torch.cuda too early...
70
- # if torch.cuda.device_count() < 4:
71
- # pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
67
+def test_baichuan_tensor_parallel_equality(baichuan_lora_files,
68
+ num_gpus_available, fully_sharded):
69
+ if num_gpus_available < 4:
70
+ pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
71
72
llm_tp1 = vllm.LLM(MODEL_PATH,
73
enable_lora=True,
tests/lora/test_quant_model.py
+8
-9
@@ -71,10 +71,10 @@ def do_sample(llm: vllm.LLM,
71
72
@pytest.mark.parametrize("model", MODELS)
73
@pytest.mark.parametrize("tp_size", [1])
74
-def test_quant_model_lora(tinyllama_lora_files, model, tp_size):
75
- # Cannot use as it will initialize torch.cuda too early...
76
- # if torch.cuda.device_count() < tp_size:
77
- # pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
74
+def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
75
+ tp_size):
76
+ if num_gpus_available < tp_size:
77
+ pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
78
79
llm = vllm.LLM(
80
model=model.model_path,
@@ -164,11 +164,10 @@ def test_quant_model_lora(tinyllama_lora_files, model, tp_size):
164
165
166
@pytest.mark.parametrize("model", MODELS)
167
-@pytest.mark.skip("Requires multiple GPUs")
168
-def test_quant_model_tp_equality(tinyllama_lora_files, model):
169
- # Cannot use as it will initialize torch.cuda too early...
170
- # if torch.cuda.device_count() < 2:
171
- # pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
167
+def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
168
+ model):
169
+ if num_gpus_available < 2:
170
+ pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
171
172
llm_tp1 = vllm.LLM(
173
model=model.model_path,
tests/models/decoder_only/language/test_phimoe.py
+2
-11
@@ -7,6 +7,7 @@ import torch
7
8
from vllm.utils import is_cpu
9
10
+from ....utils import large_gpu_test
11
from ...utils import check_logprobs_close
12
13
MODELS = [
@@ -69,20 +70,10 @@ def test_phimoe_routing_function():
70
assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
71
72
72
-def get_gpu_memory():
73
- try:
74
- props = torch.cuda.get_device_properties(torch.cuda.current_device())
75
- gpu_memory = props.total_memory / (1024**3)
76
- return gpu_memory
77
- except Exception:
78
- return 0
79
-
80
-
73
@pytest.mark.skipif(condition=is_cpu(),
74
reason="This test takes a lot time to run on CPU, "
75
"and vllm CI's disk space is not enough for this model.")
84
-@pytest.mark.skipif(condition=get_gpu_memory() < 100,
85
- reason="Skip this test if GPU memory is insufficient.")
76
+@large_gpu_test(min_gb=80)
77
@pytest.mark.parametrize("model", MODELS)
78
@pytest.mark.parametrize("dtype", ["bfloat16"])
79
@pytest.mark.parametrize("max_tokens", [64])
tests/models/decoder_only/vision_language/test_llava_onevision.py
+4
-9
@@ -11,6 +11,7 @@ from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11
12
from ....conftest import (VIDEO_ASSETS, HfRunner, PromptImageInput, VllmRunner,
13
_VideoAssets)
14
+from ....utils import large_gpu_test
15
from ...utils import check_logprobs_close
16
17
# Video test
@@ -164,9 +165,7 @@ def run_video_test(
165
)
166
167
167
-@pytest.mark.skip(
168
- reason=
169
- "Model is too big, test passed on L40 locally but will OOM on CI machine.")
168
+@large_gpu_test(min_gb=48)
169
@pytest.mark.parametrize("model", models)
170
@pytest.mark.parametrize(
171
"size_factors",
@@ -210,9 +209,7 @@ def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
209
)
210
211
213
-@pytest.mark.skip(
214
- reason=
215
- "Model is too big, test passed on L40 locally but will OOM on CI machine.")
212
+@large_gpu_test(min_gb=48)
213
@pytest.mark.parametrize("model", models)
214
@pytest.mark.parametrize(
215
"sizes",
@@ -306,9 +303,7 @@ def run_image_test(
303
)
304
305
309
-@pytest.mark.skip(
310
- reason=
311
- "Model is too big, test passed on L40 locally but will OOM on CI machine.")
306
+@large_gpu_test(min_gb=48)
307
@pytest.mark.parametrize("model", models)
308
@pytest.mark.parametrize("dtype", ["half"])
309
@pytest.mark.parametrize("max_tokens", [128])
tests/models/decoder_only/vision_language/test_pixtral.py
+3
-9
@@ -17,7 +17,7 @@ from vllm import EngineArgs, LLMEngine, SamplingParams, TokensPrompt
17
from vllm.multimodal import MultiModalDataBuiltins
18
from vllm.sequence import Logprob, SampleLogprobs
19
20
-from ....utils import VLLM_PATH
20
+from ....utils import VLLM_PATH, large_gpu_test
21
from ...utils import check_logprobs_close
22
23
if TYPE_CHECKING:
@@ -121,10 +121,7 @@ def load_outputs_w_logprobs(filename: "StrPath") -> OutputsLogprobs:
121
for tokens, text, logprobs in json_data]
122
123
124
-@pytest.mark.skip(
125
- reason=
126
- "Model is too big, test passed on A100 locally but will OOM on CI machine."
127
-)
124
+@large_gpu_test(min_gb=80)
125
@pytest.mark.parametrize("model", MODELS)
126
@pytest.mark.parametrize("max_model_len", MAX_MODEL_LEN)
127
@pytest.mark.parametrize("dtype", ["bfloat16"])
@@ -157,10 +154,7 @@ def test_chat(
154
name_1="output")
155
156
160
-@pytest.mark.skip(
161
- reason=
162
- "Model is too big, test passed on A100 locally but will OOM on CI machine."
163
-)
157
+@large_gpu_test(min_gb=80)
158
@pytest.mark.parametrize("model", MODELS)
159
@pytest.mark.parametrize("dtype", ["bfloat16"])
160
def test_model_engine(vllm_runner, model: str, dtype: str) -> None:
tests/models/encoder_decoder/vision_language/test_mllama.py
+20
-22
@@ -9,6 +9,7 @@ from vllm.sequence import SampleLogprobs
9
10
from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
11
_ImageAssets)
12
+from ....utils import large_gpu_test
13
from ...utils import check_logprobs_close
14
15
_LIMIT_IMAGE_PER_PROMPT = 1
@@ -227,29 +228,26 @@ def _run_test(
228
)
229
230
230
-SIZES = [
231
- # Text only
232
- [],
233
- # Single-size
234
- [(512, 512)],
235
- # Single-size, batched
236
- [(512, 512), (512, 512), (512, 512)],
237
- # Multi-size, batched
238
- [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
239
- (1024, 1024), (512, 1536), (512, 2028)],
240
- # Multi-size, batched, including text only
241
- [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
242
- (1024, 1024), (512, 1536), (512, 2028), None],
243
- # mllama has 8 possible aspect ratios, carefully set the sizes
244
- # to cover all of them
245
-]
246
-
247
-
248
-@pytest.mark.skip(
249
- reason=
250
- "Model is too big, test passed on L40 locally but will OOM on CI machine.")
231
+@large_gpu_test(min_gb=48)
232
@pytest.mark.parametrize("model", models)
252
-@pytest.mark.parametrize("sizes", SIZES)
233
+@pytest.mark.parametrize(
234
+ "sizes",
235
+ [
236
+ # Text only
237
+ [],
238
+ # Single-size
239
+ [(512, 512)],
240
+ # Single-size, batched
241
+ [(512, 512), (512, 512), (512, 512)],
242
+ # Multi-size, batched
243
+ [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
244
+ (1024, 1024), (512, 1536), (512, 2028)],
245
+ # Multi-size, batched, including text only
246
+ [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
247
+ (1024, 1024), (512, 1536), (512, 2028), None],
248
+ # mllama has 8 possible aspect ratios, carefully set the sizes
249
+ # to cover all of them
250
+ ])
251
@pytest.mark.parametrize("dtype", ["bfloat16"])
252
@pytest.mark.parametrize("max_tokens", [128])
253
@pytest.mark.parametrize("num_logprobs", [5])
tests/utils.py
+33
-2
@@ -24,8 +24,8 @@ from vllm.engine.arg_utils import AsyncEngineArgs
24
from vllm.entrypoints.openai.cli_args import make_arg_parser
25
from vllm.model_executor.model_loader.loader import get_model_loader
26
from vllm.platforms import current_platform
27
-from vllm.utils import (FlexibleArgumentParser, cuda_device_count_stateless,
28
- get_open_port, is_hip)
27
+from vllm.utils import (FlexibleArgumentParser, GB_bytes,
28
+ cuda_device_count_stateless, get_open_port, is_hip)
29
30
if current_platform.is_rocm():
31
from amdsmi import (amdsmi_get_gpu_vram_usage,
@@ -455,6 +455,37 @@ def fork_new_process_for_each_test(
455
return wrapper
456
457
458
+def large_gpu_test(*, min_gb: int):
459
+ """
460
+ Decorate a test to be skipped if no GPU is available or it does not have
461
+ sufficient memory.
462
+
463
+ Currently, the CI machine uses L4 GPU which has 24 GB VRAM.
464
+ """
465
+ try:
466
+ if current_platform.is_cpu():
467
+ memory_gb = 0
468
+ else:
469
+ memory_gb = current_platform.get_device_total_memory() / GB_bytes
470
+ except Exception as e:
471
+ warnings.warn(
472
+ f"An error occurred when finding the available memory: {e}",
473
+ stacklevel=2,
474
+ )
475
+
476
+ memory_gb = 0
477
+
478
+ test_skipif = pytest.mark.skipif(
479
+ memory_gb < min_gb,
480
+ reason=f"Need at least {memory_gb}GB GPU memory to run the test.",
481
+ )
482
+
483
+ def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:
484
+ return test_skipif(fork_new_process_for_each_test(f))
485
+
486
+ return wrapper
487
+
488
+
489
def multi_gpu_test(*, num_gpus: int):
490
"""
491
Decorate a test to be run only when multiple GPUs are available.
vllm/platforms/cpu.py
+5
@@ -1,3 +1,4 @@
1
+import psutil
2
import torch
3
4
from .interface import Platform, PlatformEnum
@@ -10,6 +11,10 @@ class CpuPlatform(Platform):
11
def get_device_name(cls, device_id: int = 0) -> str:
12
return "cpu"
13
14
+ @classmethod
15
+ def get_device_total_memory(cls, device_id: int = 0) -> int:
16
+ return psutil.virtual_memory().total
17
+
18
@classmethod
19
def inference_mode(cls):
20
return torch.no_grad()
vllm/platforms/cuda.py
+12
@@ -59,6 +59,13 @@ def get_physical_device_name(device_id: int = 0) -> str:
59
return pynvml.nvmlDeviceGetName(handle)
60
61
62
+@lru_cache(maxsize=8)
63
+@with_nvml_context
64
+def get_physical_device_total_memory(device_id: int = 0) -> int:
65
+ handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
66
+ return int(pynvml.nvmlDeviceGetMemoryInfo(handle).total)
67
+
68
+
69
@with_nvml_context
70
def warn_if_different_devices():
71
device_ids: int = pynvml.nvmlDeviceGetCount()
@@ -107,6 +114,11 @@ class CudaPlatform(Platform):
114
physical_device_id = device_id_to_physical_device_id(device_id)
115
return get_physical_device_name(physical_device_id)
116
117
+ @classmethod
118
+ def get_device_total_memory(cls, device_id: int = 0) -> int:
119
+ physical_device_id = device_id_to_physical_device_id(device_id)
120
+ return get_physical_device_total_memory(physical_device_id)
121
+
122
@classmethod
123
@with_nvml_context
124
def is_full_nvlink(cls, physical_device_ids: List[int]) -> bool:
vllm/platforms/interface.py
+6
@@ -85,6 +85,12 @@ class Platform:
85
86
@classmethod
87
def get_device_name(cls, device_id: int = 0) -> str:
88
+ """Get the name of a device."""
89
+ raise NotImplementedError
90
+
91
+ @classmethod
92
+ def get_device_total_memory(cls, device_id: int = 0) -> int:
93
+ """Get the total memory of a device in bytes."""
94
raise NotImplementedError
95
96
@classmethod
vllm/platforms/rocm.py
+5
@@ -29,3 +29,8 @@ class RocmPlatform(Platform):
29
@lru_cache(maxsize=8)
30
def get_device_name(cls, device_id: int = 0) -> str:
31
return torch.cuda.get_device_name(device_id)
32
+
33
+ @classmethod
34
+ def get_device_total_memory(cls, device_id: int = 0) -> int:
35
+ device_props = torch.cuda.get_device_properties(device_id)
36
+ return device_props.total_memory
vllm/platforms/tpu.py
+4
@@ -10,6 +10,10 @@ class TpuPlatform(Platform):
10
def get_device_name(cls, device_id: int = 0) -> str:
11
raise NotImplementedError
12
13
+ @classmethod
14
+ def get_device_total_memory(cls, device_id: int = 0) -> int:
15
+ raise NotImplementedError
16
+
17
@classmethod
18
def inference_mode(cls):
19
return torch.no_grad()
vllm/platforms/xpu.py
+8
-6
@@ -8,13 +8,15 @@ class XPUPlatform(Platform):
8
9
@staticmethod
10
def get_device_capability(device_id: int = 0) -> DeviceCapability:
11
- return DeviceCapability(major=int(
12
- torch.xpu.get_device_capability(device_id)['version'].split('.')
13
- [0]),
14
- minor=int(
15
- torch.xpu.get_device_capability(device_id)
16
- ['version'].split('.')[1]))
11
+ major, minor, *_ = torch.xpu.get_device_capability(
12
+ device_id)['version'].split('.')
13
+ return DeviceCapability(major=int(major), minor=int(minor))
14
15
@staticmethod
16
def get_device_name(device_id: int = 0) -> str:
17
return torch.xpu.get_device_name(device_id)
18
+
19
+ @classmethod
20
+ def get_device_total_memory(cls, device_id: int = 0) -> int:
21
+ device_props = torch.xpu.get_device_properties(device_id)
22
+ return device_props.total_memory
vllm/utils.py
+3
@@ -119,6 +119,9 @@ STR_XFORMERS_ATTN_VAL: str = "XFORMERS"
119
STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
120
STR_INVALID_VAL: str = "INVALID"
121
122
+GB_bytes = 1_000_000_000
123
+"""The number of bytes in one gigabyte (GB)."""
124
+
125
GiB_bytes = 1 << 30
126
"""The number of bytes in one gibibyte (GiB)."""
127