A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[ci][distributed] fix device count call
[ci][distributed] fix some cuda init that makes it necessary to use spawn (#5991)
youkaichao committed
Jun 30, 2024 at 01:06 UTC
2be6955a3fd596b33be92a2927f55ee0779a4690
6 files changed
+83
-51
.buildkite/test-pipeline.yaml
+1
-11
@@ -45,9 +45,6 @@ steps:
45
num_gpus: 2
46
commands:
47
- bash ../.buildkite/download-images.sh
48
- # FIXIT: find out which code initialize cuda before running the test
49
- # before the fix, we need to use spawn to test it
50
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
48
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
49
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
50
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
@@ -60,8 +57,7 @@ steps:
57
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
58
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
59
- TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
63
- # FIXIT: find out why TP is failing with mp backend on phi3-v
64
- # - TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
60
+ - TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
61
- pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
62
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
63
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
@@ -71,9 +67,6 @@ steps:
67
working_dir: "/vllm-workspace/tests"
68
num_gpus: 4
69
commands:
74
- # FIXIT: find out which code initialize cuda before running the test
75
- # before the fix, we need to use spawn to test it
76
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
70
- pytest -v -s distributed/test_pynccl.py
71
# We want to test that models which use 2 GPUs work with 4 GPUs, which is why we duplicate them here.
72
# See https://github.com/vllm-project/vllm/pull/5473#issuecomment-2166601837 for context.
@@ -225,9 +218,6 @@ steps:
218
gpu: a100
219
num_gpus: 4
220
commands:
228
- # FIXIT: find out which code initialize cuda before running the test
229
- # before the fix, we need to use spawn to test it
230
- - export VLLM_WORKER_MULTIPROC_METHOD=spawn
221
# NOTE: don't test llama model here, it seems hf implementation is buggy
222
# see https://github.com/vllm-project/vllm/pull/5689 for details
223
- pytest -v -s distributed/test_custom_all_reduce.py
tests/conftest.py
+16
-5
@@ -5,8 +5,8 @@ from collections import UserList
5
from dataclasses import dataclass
6
from functools import cached_property
7
from pathlib import Path
8
-from typing import (Any, Dict, List, Literal, Optional, Tuple, TypedDict,
9
- TypeVar)
8
+from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple,
9
+ TypedDict, TypeVar)
10
11
import pytest
12
import torch
@@ -14,7 +14,7 @@ import torch.nn as nn
14
import torch.nn.functional as F
15
from PIL import Image
16
from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
17
- AutoProcessor, AutoTokenizer, BatchEncoding)
17
+ AutoTokenizer, BatchEncoding)
18
19
from vllm import LLM, SamplingParams
20
from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
@@ -22,8 +22,12 @@ from vllm.distributed import (destroy_distributed_environment,
22
destroy_model_parallel)
23
from vllm.inputs import TextPrompt
24
from vllm.logger import init_logger
25
-from vllm.multimodal import MultiModalData
26
-from vllm.multimodal.image import ImageFeatureData, ImagePixelData
25
+
26
+if TYPE_CHECKING:
27
+ from vllm.multimodal import MultiModalData
28
+else:
29
+ # it will call torch.cuda.device_count()
30
+ MultiModalData = None
31
from vllm.sequence import SampleLogprobs
32
from vllm.utils import cuda_device_count_stateless, is_cpu
33
@@ -63,6 +67,10 @@ class ImageAsset:
67
return self.pil_image
68
69
def for_vllm(self, vision_config: VisionLanguageConfig) -> MultiModalData:
70
+ # don't put this import at the top level
71
+ # it will call torch.cuda.device_count()
72
+ from vllm.multimodal.image import ImageFeatureData # noqa: F401
73
+ from vllm.multimodal.image import ImagePixelData
74
image_input_type = vision_config.image_input_type
75
ImageInputType = VisionLanguageConfig.ImageInputType
76
@@ -216,6 +224,9 @@ class HfRunner:
224
)
225
226
try:
227
+ # don't put this import at the top level
228
+ # it will call torch.cuda.device_count()
229
+ from transformers import AutoProcessor # noqa: F401
230
self.processor = AutoProcessor.from_pretrained(
231
model_name,
232
torch_dtype=torch_dtype,
tests/distributed/test_basic_distributed_correctness.py
+10
-5
@@ -15,7 +15,8 @@ TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
15
import os
16
17
import pytest
18
-import torch
18
+
19
+from vllm.utils import cuda_device_count_stateless
20
21
from ..models.utils import check_outputs_equal
22
@@ -25,7 +26,7 @@ MODELS = [
26
DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
27
28
28
-@pytest.mark.skipif(torch.cuda.device_count() < 2,
29
+@pytest.mark.skipif(cuda_device_count_stateless() < 2,
30
reason="Need at least 2 GPUs to run the test.")
31
@pytest.mark.parametrize("model", MODELS)
32
@pytest.mark.parametrize("dtype", ["half"])
@@ -40,9 +41,10 @@ def test_models(
41
) -> None:
42
distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
43
43
- with hf_runner(model, dtype=dtype) as hf_model:
44
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
45
-
44
+ # NOTE: take care of the order. run vLLM first, and then run HF.
45
+ # vLLM needs a fresh new process without cuda initialization.
46
+ # if we run HF first, the cuda initialization will be done and it
47
+ # will hurt multiprocessing backend with fork method (the default method).
48
with vllm_runner(model,
49
dtype=dtype,
50
tensor_parallel_size=2,
@@ -50,6 +52,9 @@ def test_models(
52
) as vllm_model:
53
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54
55
+ with hf_runner(model, dtype=dtype) as hf_model:
56
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
57
+
58
check_outputs_equal(
59
outputs_0_lst=hf_outputs,
60
outputs_1_lst=vllm_outputs,
tests/distributed/test_chunked_prefill_distributed.py
+10
-4
@@ -14,7 +14,8 @@ TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
14
import os
15
16
import pytest
17
-import torch
17
+
18
+from vllm.utils import cuda_device_count_stateless
19
20
from ..models.utils import check_outputs_equal
21
@@ -24,7 +25,7 @@ MODELS = [
25
DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
26
27
27
-@pytest.mark.skipif(torch.cuda.device_count() < 2,
28
+@pytest.mark.skipif(cuda_device_count_stateless() < 2,
29
reason="Need at least 2 GPUs to run the test.")
30
@pytest.mark.parametrize("model", MODELS)
31
@pytest.mark.parametrize("dtype", ["half"])
@@ -47,8 +48,10 @@ def test_models(
48
enable_chunked_prefill = True
49
max_num_batched_tokens = chunked_prefill_token_size
50
50
- with hf_runner(model, dtype=dtype) as hf_model:
51
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
51
+ # NOTE: take care of the order. run vLLM first, and then run HF.
52
+ # vLLM needs a fresh new process without cuda initialization.
53
+ # if we run HF first, the cuda initialization will be done and it
54
+ # will hurt multiprocessing backend with fork method (the default method).
55
56
with vllm_runner(
57
model,
@@ -61,6 +64,9 @@ def test_models(
64
) as vllm_model:
65
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
66
67
+ with hf_runner(model, dtype=dtype) as hf_model:
68
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
69
+
70
check_outputs_equal(
71
outputs_0_lst=hf_outputs,
72
outputs_1_lst=vllm_outputs,
tests/models/test_llava.py
+20
-10
@@ -88,17 +88,11 @@ def run_test(
88
"""
89
model_id, vlm_config = model_and_config
90
hf_images = [asset.for_hf() for asset in image_assets]
91
- vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
91
93
- with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
94
- hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
95
- max_tokens,
96
- images=hf_images)
97
-
98
- vllm_image_prompts = [
99
- p.replace("<image>", "<image>" * vlm_config.image_feature_size)
100
- for p in HF_IMAGE_PROMPTS
101
- ]
92
+ # NOTE: take care of the order. run vLLM first, and then run HF.
93
+ # vLLM needs a fresh new process without cuda initialization.
94
+ # if we run HF first, the cuda initialization will be done and it
95
+ # will hurt multiprocessing backend with fork method (the default method).
96
97
with vllm_runner(model_id,
98
dtype=dtype,
@@ -106,10 +100,26 @@ def run_test(
100
distributed_executor_backend=distributed_executor_backend,
101
enforce_eager=True,
102
**vlm_config.as_cli_args_dict()) as vllm_model:
103
+
104
+ # NOTE: `asset.for_vllm` will call `torch.cuda.device_count()`
105
+ # we must put it inside the vllm_runner context manager
106
+ # i.e. after creating vLLM instance.
107
+ vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
108
+
109
+ vllm_image_prompts = [
110
+ p.replace("<image>", "<image>" * vlm_config.image_feature_size)
111
+ for p in HF_IMAGE_PROMPTS
112
+ ]
113
+
114
vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
115
max_tokens,
116
images=vllm_images)
117
118
+ with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
119
+ hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
120
+ max_tokens,
121
+ images=hf_images)
122
+
123
check_outputs_equal(
124
hf_outputs,
125
[
tests/models/test_phi3v.py
+26
-16
@@ -96,23 +96,11 @@ def run_test(
96
"""
97
model_id, vlm_config = model_and_config
98
hf_images = [asset.for_hf() for asset in image_assets]
99
- vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
99
101
- # use eager mode for hf runner, since phi3_v didn't work with flash_attn
102
- hf_model_kwargs = {"_attn_implementation": "eager"}
103
- with hf_runner(model_id, dtype=dtype,
104
- model_kwargs=hf_model_kwargs) as hf_model:
105
- hf_outputs = hf_model.generate_greedy(
106
- HF_IMAGE_PROMPTS,
107
- max_tokens,
108
- images=hf_images,
109
- eos_token_id=hf_model.processor.tokenizer.eos_token_id)
110
-
111
- vllm_image_prompts = [
112
- p.replace("<|image_1|>",
113
- "<|image|>" * vlm_config.image_feature_size + "<s>")
114
- for p in HF_IMAGE_PROMPTS
115
- ]
100
+ # NOTE: take care of the order. run vLLM first, and then run HF.
101
+ # vLLM needs a fresh new process without cuda initialization.
102
+ # if we run HF first, the cuda initialization will be done and it
103
+ # will hurt multiprocessing backend with fork method (the default method).
104
105
with vllm_runner(model_id,
106
max_model_len=2048,
@@ -121,10 +109,32 @@ def run_test(
109
enforce_eager=True,
110
distributed_executor_backend=distributed_executor_backend,
111
**vlm_config.as_cli_args_dict()) as vllm_model:
112
+ # NOTE: `asset.for_vllm` will call `torch.cuda.device_count()`
113
+ # we must put it inside the vllm_runner context manager
114
+ # i.e. after creating vLLM instance.
115
+
116
+ vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
117
+
118
+ vllm_image_prompts = [
119
+ p.replace("<|image_1|>",
120
+ "<|image|>" * vlm_config.image_feature_size + "<s>")
121
+ for p in HF_IMAGE_PROMPTS
122
+ ]
123
+
124
vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
125
max_tokens,
126
images=vllm_images)
127
128
+ # use eager mode for hf runner, since phi3_v didn't work with flash_attn
129
+ hf_model_kwargs = {"_attn_implementation": "eager"}
130
+ with hf_runner(model_id, dtype=dtype,
131
+ model_kwargs=hf_model_kwargs) as hf_model:
132
+ hf_outputs = hf_model.generate_greedy(
133
+ HF_IMAGE_PROMPTS,
134
+ max_tokens,
135
+ images=hf_images,
136
+ eos_token_id=hf_model.processor.tokenizer.eos_token_id)
137
+
138
check_outputs_equal(
139
hf_outputs,
140
[