@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2be6955a3f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ci][distributed] fix device count call

[ci][distributed] fix some cuda init that makes it necessary to use spawn (#5991)

youkaichao committed Jun 30, 2024 at 01:06 UTC 2be6955a3fd596b33be92a2927f55ee0779a4690
6 files changed +83 -51
.buildkite/test-pipeline.yaml
+1 -11
@@ -45,9 +45,6 @@ steps:
45 num_gpus: 2
46 commands:
47 - bash ../.buildkite/download-images.sh
48 - # FIXIT: find out which code initialize cuda before running the test
49 - # before the fix, we need to use spawn to test it
50 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn
48 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
49 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
50 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
@@ -60,8 +57,7 @@ steps:
57 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
58 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
59 - TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
63 - # FIXIT: find out why TP is failing with mp backend on phi3-v
64 - # - TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
60 + - TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
61 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
62 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
63 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
@@ -71,9 +67,6 @@ steps:
67 working_dir: "/vllm-workspace/tests"
68 num_gpus: 4
69 commands:
74 - # FIXIT: find out which code initialize cuda before running the test
75 - # before the fix, we need to use spawn to test it
76 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn
70 - pytest -v -s distributed/test_pynccl.py
71 # We want to test that models which use 2 GPUs work with 4 GPUs, which is why we duplicate them here.
72 # See https://github.com/vllm-project/vllm/pull/5473#issuecomment-2166601837 for context.
@@ -225,9 +218,6 @@ steps:
218 gpu: a100
219 num_gpus: 4
220 commands:
228 - # FIXIT: find out which code initialize cuda before running the test
229 - # before the fix, we need to use spawn to test it
230 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn
221 # NOTE: don't test llama model here, it seems hf implementation is buggy
222 # see https://github.com/vllm-project/vllm/pull/5689 for details
223 - pytest -v -s distributed/test_custom_all_reduce.py
tests/conftest.py
+16 -5
@@ -5,8 +5,8 @@ from collections import UserList
5 from dataclasses import dataclass
6 from functools import cached_property
7 from pathlib import Path
8 -from typing import (Any, Dict, List, Literal, Optional, Tuple, TypedDict,
9 - TypeVar)
8 +from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple,
9 + TypedDict, TypeVar)
10
11 import pytest
12 import torch
@@ -14,7 +14,7 @@ import torch.nn as nn
14 import torch.nn.functional as F
15 from PIL import Image
16 from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
17 - AutoProcessor, AutoTokenizer, BatchEncoding)
17 + AutoTokenizer, BatchEncoding)
18
19 from vllm import LLM, SamplingParams
20 from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
@@ -22,8 +22,12 @@ from vllm.distributed import (destroy_distributed_environment,
22 destroy_model_parallel)
23 from vllm.inputs import TextPrompt
24 from vllm.logger import init_logger
25 -from vllm.multimodal import MultiModalData
26 -from vllm.multimodal.image import ImageFeatureData, ImagePixelData
25 +
26 +if TYPE_CHECKING:
27 + from vllm.multimodal import MultiModalData
28 +else:
29 + # it will call torch.cuda.device_count()
30 + MultiModalData = None
31 from vllm.sequence import SampleLogprobs
32 from vllm.utils import cuda_device_count_stateless, is_cpu
33
@@ -63,6 +67,10 @@ class ImageAsset:
67 return self.pil_image
68
69 def for_vllm(self, vision_config: VisionLanguageConfig) -> MultiModalData:
70 + # don't put this import at the top level
71 + # it will call torch.cuda.device_count()
72 + from vllm.multimodal.image import ImageFeatureData # noqa: F401
73 + from vllm.multimodal.image import ImagePixelData
74 image_input_type = vision_config.image_input_type
75 ImageInputType = VisionLanguageConfig.ImageInputType
76
@@ -216,6 +224,9 @@ class HfRunner:
224 )
225
226 try:
227 + # don't put this import at the top level
228 + # it will call torch.cuda.device_count()
229 + from transformers import AutoProcessor # noqa: F401
230 self.processor = AutoProcessor.from_pretrained(
231 model_name,
232 torch_dtype=torch_dtype,
tests/distributed/test_basic_distributed_correctness.py
+10 -5
@@ -15,7 +15,8 @@ TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
15 import os
16
17 import pytest
18 -import torch
18 +
19 +from vllm.utils import cuda_device_count_stateless
20
21 from ..models.utils import check_outputs_equal
22
@@ -25,7 +26,7 @@ MODELS = [
26 DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
27
28
28 -@pytest.mark.skipif(torch.cuda.device_count() < 2,
29 +@pytest.mark.skipif(cuda_device_count_stateless() < 2,
30 reason="Need at least 2 GPUs to run the test.")
31 @pytest.mark.parametrize("model", MODELS)
32 @pytest.mark.parametrize("dtype", ["half"])
@@ -40,9 +41,10 @@ def test_models(
41 ) -> None:
42 distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
43
43 - with hf_runner(model, dtype=dtype) as hf_model:
44 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
45 -
44 + # NOTE: take care of the order. run vLLM first, and then run HF.
45 + # vLLM needs a fresh new process without cuda initialization.
46 + # if we run HF first, the cuda initialization will be done and it
47 + # will hurt multiprocessing backend with fork method (the default method).
48 with vllm_runner(model,
49 dtype=dtype,
50 tensor_parallel_size=2,
@@ -50,6 +52,9 @@ def test_models(
52 ) as vllm_model:
53 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54
55 + with hf_runner(model, dtype=dtype) as hf_model:
56 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
57 +
58 check_outputs_equal(
59 outputs_0_lst=hf_outputs,
60 outputs_1_lst=vllm_outputs,
tests/distributed/test_chunked_prefill_distributed.py
+10 -4
@@ -14,7 +14,8 @@ TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
14 import os
15
16 import pytest
17 -import torch
17 +
18 +from vllm.utils import cuda_device_count_stateless
19
20 from ..models.utils import check_outputs_equal
21
@@ -24,7 +25,7 @@ MODELS = [
25 DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
26
27
27 -@pytest.mark.skipif(torch.cuda.device_count() < 2,
28 +@pytest.mark.skipif(cuda_device_count_stateless() < 2,
29 reason="Need at least 2 GPUs to run the test.")
30 @pytest.mark.parametrize("model", MODELS)
31 @pytest.mark.parametrize("dtype", ["half"])
@@ -47,8 +48,10 @@ def test_models(
48 enable_chunked_prefill = True
49 max_num_batched_tokens = chunked_prefill_token_size
50
50 - with hf_runner(model, dtype=dtype) as hf_model:
51 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
51 + # NOTE: take care of the order. run vLLM first, and then run HF.
52 + # vLLM needs a fresh new process without cuda initialization.
53 + # if we run HF first, the cuda initialization will be done and it
54 + # will hurt multiprocessing backend with fork method (the default method).
55
56 with vllm_runner(
57 model,
@@ -61,6 +64,9 @@ def test_models(
64 ) as vllm_model:
65 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
66
67 + with hf_runner(model, dtype=dtype) as hf_model:
68 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
69 +
70 check_outputs_equal(
71 outputs_0_lst=hf_outputs,
72 outputs_1_lst=vllm_outputs,
tests/models/test_llava.py
+20 -10
@@ -88,17 +88,11 @@ def run_test(
88 """
89 model_id, vlm_config = model_and_config
90 hf_images = [asset.for_hf() for asset in image_assets]
91 - vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
91
93 - with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
94 - hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
95 - max_tokens,
96 - images=hf_images)
97 -
98 - vllm_image_prompts = [
99 - p.replace("<image>", "<image>" * vlm_config.image_feature_size)
100 - for p in HF_IMAGE_PROMPTS
101 - ]
92 + # NOTE: take care of the order. run vLLM first, and then run HF.
93 + # vLLM needs a fresh new process without cuda initialization.
94 + # if we run HF first, the cuda initialization will be done and it
95 + # will hurt multiprocessing backend with fork method (the default method).
96
97 with vllm_runner(model_id,
98 dtype=dtype,
@@ -106,10 +100,26 @@ def run_test(
100 distributed_executor_backend=distributed_executor_backend,
101 enforce_eager=True,
102 **vlm_config.as_cli_args_dict()) as vllm_model:
103 +
104 + # NOTE: `asset.for_vllm` will call `torch.cuda.device_count()`
105 + # we must put it inside the vllm_runner context manager
106 + # i.e. after creating vLLM instance.
107 + vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
108 +
109 + vllm_image_prompts = [
110 + p.replace("<image>", "<image>" * vlm_config.image_feature_size)
111 + for p in HF_IMAGE_PROMPTS
112 + ]
113 +
114 vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
115 max_tokens,
116 images=vllm_images)
117
118 + with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
119 + hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
120 + max_tokens,
121 + images=hf_images)
122 +
123 check_outputs_equal(
124 hf_outputs,
125 [
tests/models/test_phi3v.py
+26 -16
@@ -96,23 +96,11 @@ def run_test(
96 """
97 model_id, vlm_config = model_and_config
98 hf_images = [asset.for_hf() for asset in image_assets]
99 - vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
99
101 - # use eager mode for hf runner, since phi3_v didn't work with flash_attn
102 - hf_model_kwargs = {"_attn_implementation": "eager"}
103 - with hf_runner(model_id, dtype=dtype,
104 - model_kwargs=hf_model_kwargs) as hf_model:
105 - hf_outputs = hf_model.generate_greedy(
106 - HF_IMAGE_PROMPTS,
107 - max_tokens,
108 - images=hf_images,
109 - eos_token_id=hf_model.processor.tokenizer.eos_token_id)
110 -
111 - vllm_image_prompts = [
112 - p.replace("<|image_1|>",
113 - "<|image|>" * vlm_config.image_feature_size + "<s>")
114 - for p in HF_IMAGE_PROMPTS
115 - ]
100 + # NOTE: take care of the order. run vLLM first, and then run HF.
101 + # vLLM needs a fresh new process without cuda initialization.
102 + # if we run HF first, the cuda initialization will be done and it
103 + # will hurt multiprocessing backend with fork method (the default method).
104
105 with vllm_runner(model_id,
106 max_model_len=2048,
@@ -121,10 +109,32 @@ def run_test(
109 enforce_eager=True,
110 distributed_executor_backend=distributed_executor_backend,
111 **vlm_config.as_cli_args_dict()) as vllm_model:
112 + # NOTE: `asset.for_vllm` will call `torch.cuda.device_count()`
113 + # we must put it inside the vllm_runner context manager
114 + # i.e. after creating vLLM instance.
115 +
116 + vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
117 +
118 + vllm_image_prompts = [
119 + p.replace("<|image_1|>",
120 + "<|image|>" * vlm_config.image_feature_size + "<s>")
121 + for p in HF_IMAGE_PROMPTS
122 + ]
123 +
124 vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
125 max_tokens,
126 images=vllm_images)
127
128 + # use eager mode for hf runner, since phi3_v didn't work with flash_attn
129 + hf_model_kwargs = {"_attn_implementation": "eager"}
130 + with hf_runner(model_id, dtype=dtype,
131 + model_kwargs=hf_model_kwargs) as hf_model:
132 + hf_outputs = hf_model.generate_greedy(
133 + HF_IMAGE_PROMPTS,
134 + max_tokens,
135 + images=hf_images,
136 + eos_token_id=hf_model.processor.tokenizer.eos_token_id)
137 +
138 check_outputs_equal(
139 hf_outputs,
140 [