@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 26a68d5d7e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[CI/Build] Add test decorator for minimum GPU memory (#8925)

Cyrus Leung committed Sep 29, 2024 at 10:50 UTC 26a68d5d7e7dd47c7d8538a326493c8a171f5016
14 files changed +117 -73
tests/lora/test_baichuan.py
+4 -5
@@ -63,12 +63,11 @@ def test_baichuan_lora(baichuan_lora_files):
63 assert output2[i] == expected_lora_output[i]
64
65
66 -@pytest.mark.skip("Requires multiple GPUs")
66 @pytest.mark.parametrize("fully_sharded", [True, False])
68 -def test_baichuan_tensor_parallel_equality(baichuan_lora_files, fully_sharded):
69 - # Cannot use as it will initialize torch.cuda too early...
70 - # if torch.cuda.device_count() < 4:
71 - # pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
67 +def test_baichuan_tensor_parallel_equality(baichuan_lora_files,
68 + num_gpus_available, fully_sharded):
69 + if num_gpus_available < 4:
70 + pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
71
72 llm_tp1 = vllm.LLM(MODEL_PATH,
73 enable_lora=True,
tests/lora/test_quant_model.py
+8 -9
@@ -71,10 +71,10 @@ def do_sample(llm: vllm.LLM,
71
72 @pytest.mark.parametrize("model", MODELS)
73 @pytest.mark.parametrize("tp_size", [1])
74 -def test_quant_model_lora(tinyllama_lora_files, model, tp_size):
75 - # Cannot use as it will initialize torch.cuda too early...
76 - # if torch.cuda.device_count() < tp_size:
77 - # pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
74 +def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
75 + tp_size):
76 + if num_gpus_available < tp_size:
77 + pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
78
79 llm = vllm.LLM(
80 model=model.model_path,
@@ -164,11 +164,10 @@ def test_quant_model_lora(tinyllama_lora_files, model, tp_size):
164
165
166 @pytest.mark.parametrize("model", MODELS)
167 -@pytest.mark.skip("Requires multiple GPUs")
168 -def test_quant_model_tp_equality(tinyllama_lora_files, model):
169 - # Cannot use as it will initialize torch.cuda too early...
170 - # if torch.cuda.device_count() < 2:
171 - # pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
167 +def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
168 + model):
169 + if num_gpus_available < 2:
170 + pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
171
172 llm_tp1 = vllm.LLM(
173 model=model.model_path,
tests/models/decoder_only/language/test_phimoe.py
+2 -11
@@ -7,6 +7,7 @@ import torch
7
8 from vllm.utils import is_cpu
9
10 +from ....utils import large_gpu_test
11 from ...utils import check_logprobs_close
12
13 MODELS = [
@@ -69,20 +70,10 @@ def test_phimoe_routing_function():
70 assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
71
72
72 -def get_gpu_memory():
73 - try:
74 - props = torch.cuda.get_device_properties(torch.cuda.current_device())
75 - gpu_memory = props.total_memory / (1024**3)
76 - return gpu_memory
77 - except Exception:
78 - return 0
79 -
80 -
73 @pytest.mark.skipif(condition=is_cpu(),
74 reason="This test takes a lot time to run on CPU, "
75 "and vllm CI's disk space is not enough for this model.")
84 -@pytest.mark.skipif(condition=get_gpu_memory() < 100,
85 - reason="Skip this test if GPU memory is insufficient.")
76 +@large_gpu_test(min_gb=80)
77 @pytest.mark.parametrize("model", MODELS)
78 @pytest.mark.parametrize("dtype", ["bfloat16"])
79 @pytest.mark.parametrize("max_tokens", [64])
tests/models/decoder_only/vision_language/test_llava_onevision.py
+4 -9
@@ -11,6 +11,7 @@ from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11
12 from ....conftest import (VIDEO_ASSETS, HfRunner, PromptImageInput, VllmRunner,
13 _VideoAssets)
14 +from ....utils import large_gpu_test
15 from ...utils import check_logprobs_close
16
17 # Video test
@@ -164,9 +165,7 @@ def run_video_test(
165 )
166
167
167 -@pytest.mark.skip(
168 - reason=
169 - "Model is too big, test passed on L40 locally but will OOM on CI machine.")
168 +@large_gpu_test(min_gb=48)
169 @pytest.mark.parametrize("model", models)
170 @pytest.mark.parametrize(
171 "size_factors",
@@ -210,9 +209,7 @@ def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
209 )
210
211
213 -@pytest.mark.skip(
214 - reason=
215 - "Model is too big, test passed on L40 locally but will OOM on CI machine.")
212 +@large_gpu_test(min_gb=48)
213 @pytest.mark.parametrize("model", models)
214 @pytest.mark.parametrize(
215 "sizes",
@@ -306,9 +303,7 @@ def run_image_test(
303 )
304
305
309 -@pytest.mark.skip(
310 - reason=
311 - "Model is too big, test passed on L40 locally but will OOM on CI machine.")
306 +@large_gpu_test(min_gb=48)
307 @pytest.mark.parametrize("model", models)
308 @pytest.mark.parametrize("dtype", ["half"])
309 @pytest.mark.parametrize("max_tokens", [128])
tests/models/decoder_only/vision_language/test_pixtral.py
+3 -9
@@ -17,7 +17,7 @@ from vllm import EngineArgs, LLMEngine, SamplingParams, TokensPrompt
17 from vllm.multimodal import MultiModalDataBuiltins
18 from vllm.sequence import Logprob, SampleLogprobs
19
20 -from ....utils import VLLM_PATH
20 +from ....utils import VLLM_PATH, large_gpu_test
21 from ...utils import check_logprobs_close
22
23 if TYPE_CHECKING:
@@ -121,10 +121,7 @@ def load_outputs_w_logprobs(filename: "StrPath") -> OutputsLogprobs:
121 for tokens, text, logprobs in json_data]
122
123
124 -@pytest.mark.skip(
125 - reason=
126 - "Model is too big, test passed on A100 locally but will OOM on CI machine."
127 -)
124 +@large_gpu_test(min_gb=80)
125 @pytest.mark.parametrize("model", MODELS)
126 @pytest.mark.parametrize("max_model_len", MAX_MODEL_LEN)
127 @pytest.mark.parametrize("dtype", ["bfloat16"])
@@ -157,10 +154,7 @@ def test_chat(
154 name_1="output")
155
156
160 -@pytest.mark.skip(
161 - reason=
162 - "Model is too big, test passed on A100 locally but will OOM on CI machine."
163 -)
157 +@large_gpu_test(min_gb=80)
158 @pytest.mark.parametrize("model", MODELS)
159 @pytest.mark.parametrize("dtype", ["bfloat16"])
160 def test_model_engine(vllm_runner, model: str, dtype: str) -> None:
tests/models/encoder_decoder/vision_language/test_mllama.py
+20 -22
@@ -9,6 +9,7 @@ from vllm.sequence import SampleLogprobs
9
10 from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
11 _ImageAssets)
12 +from ....utils import large_gpu_test
13 from ...utils import check_logprobs_close
14
15 _LIMIT_IMAGE_PER_PROMPT = 1
@@ -227,29 +228,26 @@ def _run_test(
228 )
229
230
230 -SIZES = [
231 - # Text only
232 - [],
233 - # Single-size
234 - [(512, 512)],
235 - # Single-size, batched
236 - [(512, 512), (512, 512), (512, 512)],
237 - # Multi-size, batched
238 - [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
239 - (1024, 1024), (512, 1536), (512, 2028)],
240 - # Multi-size, batched, including text only
241 - [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
242 - (1024, 1024), (512, 1536), (512, 2028), None],
243 - # mllama has 8 possible aspect ratios, carefully set the sizes
244 - # to cover all of them
245 -]
246 -
247 -
248 -@pytest.mark.skip(
249 - reason=
250 - "Model is too big, test passed on L40 locally but will OOM on CI machine.")
231 +@large_gpu_test(min_gb=48)
232 @pytest.mark.parametrize("model", models)
252 -@pytest.mark.parametrize("sizes", SIZES)
233 +@pytest.mark.parametrize(
234 + "sizes",
235 + [
236 + # Text only
237 + [],
238 + # Single-size
239 + [(512, 512)],
240 + # Single-size, batched
241 + [(512, 512), (512, 512), (512, 512)],
242 + # Multi-size, batched
243 + [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
244 + (1024, 1024), (512, 1536), (512, 2028)],
245 + # Multi-size, batched, including text only
246 + [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
247 + (1024, 1024), (512, 1536), (512, 2028), None],
248 + # mllama has 8 possible aspect ratios, carefully set the sizes
249 + # to cover all of them
250 + ])
251 @pytest.mark.parametrize("dtype", ["bfloat16"])
252 @pytest.mark.parametrize("max_tokens", [128])
253 @pytest.mark.parametrize("num_logprobs", [5])
tests/utils.py
+33 -2
@@ -24,8 +24,8 @@ from vllm.engine.arg_utils import AsyncEngineArgs
24 from vllm.entrypoints.openai.cli_args import make_arg_parser
25 from vllm.model_executor.model_loader.loader import get_model_loader
26 from vllm.platforms import current_platform
27 -from vllm.utils import (FlexibleArgumentParser, cuda_device_count_stateless,
28 - get_open_port, is_hip)
27 +from vllm.utils import (FlexibleArgumentParser, GB_bytes,
28 + cuda_device_count_stateless, get_open_port, is_hip)
29
30 if current_platform.is_rocm():
31 from amdsmi import (amdsmi_get_gpu_vram_usage,
@@ -455,6 +455,37 @@ def fork_new_process_for_each_test(
455 return wrapper
456
457
458 +def large_gpu_test(*, min_gb: int):
459 + """
460 + Decorate a test to be skipped if no GPU is available or it does not have
461 + sufficient memory.
462 +
463 + Currently, the CI machine uses L4 GPU which has 24 GB VRAM.
464 + """
465 + try:
466 + if current_platform.is_cpu():
467 + memory_gb = 0
468 + else:
469 + memory_gb = current_platform.get_device_total_memory() / GB_bytes
470 + except Exception as e:
471 + warnings.warn(
472 + f"An error occurred when finding the available memory: {e}",
473 + stacklevel=2,
474 + )
475 +
476 + memory_gb = 0
477 +
478 + test_skipif = pytest.mark.skipif(
479 + memory_gb < min_gb,
480 + reason=f"Need at least {memory_gb}GB GPU memory to run the test.",
481 + )
482 +
483 + def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:
484 + return test_skipif(fork_new_process_for_each_test(f))
485 +
486 + return wrapper
487 +
488 +
489 def multi_gpu_test(*, num_gpus: int):
490 """
491 Decorate a test to be run only when multiple GPUs are available.
vllm/platforms/cpu.py
+5
@@ -1,3 +1,4 @@
1 +import psutil
2 import torch
3
4 from .interface import Platform, PlatformEnum
@@ -10,6 +11,10 @@ class CpuPlatform(Platform):
11 def get_device_name(cls, device_id: int = 0) -> str:
12 return "cpu"
13
14 + @classmethod
15 + def get_device_total_memory(cls, device_id: int = 0) -> int:
16 + return psutil.virtual_memory().total
17 +
18 @classmethod
19 def inference_mode(cls):
20 return torch.no_grad()
vllm/platforms/cuda.py
+12
@@ -59,6 +59,13 @@ def get_physical_device_name(device_id: int = 0) -> str:
59 return pynvml.nvmlDeviceGetName(handle)
60
61
62 +@lru_cache(maxsize=8)
63 +@with_nvml_context
64 +def get_physical_device_total_memory(device_id: int = 0) -> int:
65 + handle = pynvml.nvmlDeviceGetHandleByIndex(device_id)
66 + return int(pynvml.nvmlDeviceGetMemoryInfo(handle).total)
67 +
68 +
69 @with_nvml_context
70 def warn_if_different_devices():
71 device_ids: int = pynvml.nvmlDeviceGetCount()
@@ -107,6 +114,11 @@ class CudaPlatform(Platform):
114 physical_device_id = device_id_to_physical_device_id(device_id)
115 return get_physical_device_name(physical_device_id)
116
117 + @classmethod
118 + def get_device_total_memory(cls, device_id: int = 0) -> int:
119 + physical_device_id = device_id_to_physical_device_id(device_id)
120 + return get_physical_device_total_memory(physical_device_id)
121 +
122 @classmethod
123 @with_nvml_context
124 def is_full_nvlink(cls, physical_device_ids: List[int]) -> bool:
vllm/platforms/interface.py
+6
@@ -85,6 +85,12 @@ class Platform:
85
86 @classmethod
87 def get_device_name(cls, device_id: int = 0) -> str:
88 + """Get the name of a device."""
89 + raise NotImplementedError
90 +
91 + @classmethod
92 + def get_device_total_memory(cls, device_id: int = 0) -> int:
93 + """Get the total memory of a device in bytes."""
94 raise NotImplementedError
95
96 @classmethod
vllm/platforms/rocm.py
+5
@@ -29,3 +29,8 @@ class RocmPlatform(Platform):
29 @lru_cache(maxsize=8)
30 def get_device_name(cls, device_id: int = 0) -> str:
31 return torch.cuda.get_device_name(device_id)
32 +
33 + @classmethod
34 + def get_device_total_memory(cls, device_id: int = 0) -> int:
35 + device_props = torch.cuda.get_device_properties(device_id)
36 + return device_props.total_memory
vllm/platforms/tpu.py
+4
@@ -10,6 +10,10 @@ class TpuPlatform(Platform):
10 def get_device_name(cls, device_id: int = 0) -> str:
11 raise NotImplementedError
12
13 + @classmethod
14 + def get_device_total_memory(cls, device_id: int = 0) -> int:
15 + raise NotImplementedError
16 +
17 @classmethod
18 def inference_mode(cls):
19 return torch.no_grad()
vllm/platforms/xpu.py
+8 -6
@@ -8,13 +8,15 @@ class XPUPlatform(Platform):
8
9 @staticmethod
10 def get_device_capability(device_id: int = 0) -> DeviceCapability:
11 - return DeviceCapability(major=int(
12 - torch.xpu.get_device_capability(device_id)['version'].split('.')
13 - [0]),
14 - minor=int(
15 - torch.xpu.get_device_capability(device_id)
16 - ['version'].split('.')[1]))
11 + major, minor, *_ = torch.xpu.get_device_capability(
12 + device_id)['version'].split('.')
13 + return DeviceCapability(major=int(major), minor=int(minor))
14
15 @staticmethod
16 def get_device_name(device_id: int = 0) -> str:
17 return torch.xpu.get_device_name(device_id)
18 +
19 + @classmethod
20 + def get_device_total_memory(cls, device_id: int = 0) -> int:
21 + device_props = torch.xpu.get_device_properties(device_id)
22 + return device_props.total_memory
vllm/utils.py
+3
@@ -119,6 +119,9 @@ STR_XFORMERS_ATTN_VAL: str = "XFORMERS"
119 STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
120 STR_INVALID_VAL: str = "INVALID"
121
122 +GB_bytes = 1_000_000_000
123 +"""The number of bytes in one gigabyte (GB)."""
124 +
125 GiB_bytes = 1 << 30
126 """The number of bytes in one gibibyte (GiB)."""
127