@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 99397da534
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[CI/Build] Add TP test for vision models (#5892)

Cyrus Leung committed Jun 29, 2024 at 23:45 UTC 99397da5349226c553debfd37469a6de724d3f24
9 files changed +131 -27
.buildkite/test-pipeline.yaml
+5
@@ -44,6 +44,7 @@ steps:
44 working_dir: "/vllm-workspace/tests"
45 num_gpus: 2
46 commands:
47 + - bash ../.buildkite/download-images.sh
48 # FIXIT: find out which code initialize cuda before running the test
49 # before the fix, we need to use spawn to test it
50 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
@@ -52,10 +53,14 @@ steps:
53 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
54 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
55 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
56 + - TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
57 + - TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
58 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
59 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
60 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
61 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_chunked_prefill_distributed.py
62 + - TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
63 + - TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_multimodal_broadcast.py
64 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
65 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
66 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
tests/distributed/test_multimodal_broadcast.py new
+51
@@ -0,0 +1,51 @@
1 +"""Compare the outputs of HF and distributed vLLM when using greedy sampling.
2 +The second test will hang if more than one test is run per command, so we need
3 +to run the tests one by one. The solution is to pass arguments (model name) by
4 +environment variables.
5 +
6 +Run:
7 +```sh
8 +TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf \
9 + test_multimodal_broadcast.py
10 +TEST_DIST_MODEL=microsoft/Phi-3-vision-128k-instruct \
11 + test_multimodal_broadcast.py
12 +```
13 +"""
14 +import os
15 +
16 +import pytest
17 +
18 +from vllm.utils import cuda_device_count_stateless
19 +
20 +model = os.environ["TEST_DIST_MODEL"]
21 +
22 +if model.startswith("llava-hf/llava"):
23 + from ..models.test_llava import model_and_vl_config, run_test
24 +elif model.startswith("microsoft/Phi-3-vision"):
25 + from ..models.test_phi3v import model_and_vl_config, run_test
26 +else:
27 + raise NotImplementedError(f"Unsupported model: {model}")
28 +
29 +
30 +@pytest.mark.parametrize("tensor_parallel_size", [2])
31 +@pytest.mark.parametrize("dtype", ["half"])
32 +@pytest.mark.parametrize("max_tokens", [128])
33 +def test_models(hf_runner, vllm_runner, image_assets,
34 + tensor_parallel_size: int, dtype: str,
35 + max_tokens: int) -> None:
36 + if cuda_device_count_stateless() < tensor_parallel_size:
37 + pytest.skip(
38 + f"Need at least {tensor_parallel_size} GPUs to run the test.")
39 +
40 + distributed_executor_backend = os.getenv("DISTRIBUTED_EXECUTOR_BACKEND")
41 +
42 + run_test(
43 + hf_runner,
44 + vllm_runner,
45 + image_assets,
46 + model_and_config=model_and_vl_config[0],
47 + dtype=dtype,
48 + max_tokens=max_tokens,
49 + tensor_parallel_size=tensor_parallel_size,
50 + distributed_executor_backend=distributed_executor_backend,
51 + )
tests/models/test_llava.py
+31 -8
@@ -1,11 +1,11 @@
1 -from typing import List, Tuple
1 +from typing import List, Optional, Tuple, Type
2
3 import pytest
4 from transformers import AutoTokenizer
5
6 from vllm.config import VisionLanguageConfig
7
8 -from ..conftest import IMAGE_ASSETS
8 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9
10 pytestmark = pytest.mark.vlm
11
@@ -65,12 +65,17 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
65 return hf_output_ids, hf_output_str
66
67
68 -# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
69 -@pytest.mark.parametrize("model_and_config", model_and_vl_config)
70 -@pytest.mark.parametrize("dtype", ["half"])
71 -@pytest.mark.parametrize("max_tokens", [128])
72 -def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
73 - dtype: str, max_tokens: int) -> None:
68 +def run_test(
69 + hf_runner: Type[HfRunner],
70 + vllm_runner: Type[VllmRunner],
71 + image_assets: _ImageAssets,
72 + model_and_config: Tuple[str, VisionLanguageConfig],
73 + *,
74 + dtype: str,
75 + max_tokens: int,
76 + tensor_parallel_size: int,
77 + distributed_executor_backend: Optional[str] = None,
78 +):
79 """Inference result should be the same between hf and vllm.
80
81 All the image fixtures for the test is under tests/images.
@@ -96,6 +101,8 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
101
102 with vllm_runner(model_id,
103 dtype=dtype,
104 + tensor_parallel_size=tensor_parallel_size,
105 + distributed_executor_backend=distributed_executor_backend,
106 enforce_eager=True,
107 **vlm_config.as_cli_args_dict()) as vllm_model:
108 vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
@@ -110,3 +117,19 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
117 f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
118 assert hf_output_ids == vllm_output_ids, (
119 f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
120 +
121 +
122 +@pytest.mark.parametrize("model_and_config", model_and_vl_config)
123 +@pytest.mark.parametrize("dtype", ["half"])
124 +@pytest.mark.parametrize("max_tokens", [128])
125 +def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
126 + dtype: str, max_tokens: int) -> None:
127 + run_test(
128 + hf_runner,
129 + vllm_runner,
130 + image_assets,
131 + model_and_config,
132 + dtype=dtype,
133 + max_tokens=max_tokens,
134 + tensor_parallel_size=1,
135 + )
tests/models/test_phi3v.py
+36 -13
@@ -1,4 +1,4 @@
1 -from typing import List, Tuple
1 +from typing import List, Optional, Tuple, Type
2
3 import pytest
4 from transformers import AutoTokenizer
@@ -6,7 +6,7 @@ from transformers import AutoTokenizer
6 from vllm.config import VisionLanguageConfig
7 from vllm.utils import is_cpu
8
9 -from ..conftest import IMAGE_ASSETS
9 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10
11 pytestmark = pytest.mark.vlm
12
@@ -73,17 +73,17 @@ if is_cpu():
73 target_dtype = "bfloat16"
74
75
76 -# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
77 -# Since we use _attn_implementation="eager" for hf_runner, here is
78 -# numeric difference for longer context and test can't pass
79 -@pytest.mark.xfail(
80 - reason="Inconsistent image processor being used due to lack "
81 - "of support for dynamic image token replacement")
82 -@pytest.mark.parametrize("model_and_config", model_and_vl_config)
83 -@pytest.mark.parametrize("dtype", [target_dtype])
84 -@pytest.mark.parametrize("max_tokens", [128])
85 -def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
86 - dtype: str, max_tokens: int) -> None:
76 +def run_test(
77 + hf_runner: Type[HfRunner],
78 + vllm_runner: Type[VllmRunner],
79 + image_assets: _ImageAssets,
80 + model_and_config: Tuple[str, VisionLanguageConfig],
81 + *,
82 + dtype: str,
83 + max_tokens: int,
84 + tensor_parallel_size: int,
85 + distributed_executor_backend: Optional[str] = None,
86 +):
87 """Inference result should be the same between hf and vllm.
88
89 All the image fixtures for the test is under tests/images.
@@ -116,7 +116,9 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
116 with vllm_runner(model_id,
117 max_model_len=2048,
118 dtype=dtype,
119 + tensor_parallel_size=tensor_parallel_size,
120 enforce_eager=True,
121 + distributed_executor_backend=distributed_executor_backend,
122 **vlm_config.as_cli_args_dict()) as vllm_model:
123 vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
124 max_tokens,
@@ -130,3 +132,24 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
132 f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
133 assert hf_output_ids == vllm_output_ids, (
134 f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
135 +
136 +
137 +# Since we use _attn_implementation="eager" for hf_runner, here is
138 +# numeric difference for longer context and test can't pass
139 +@pytest.mark.xfail(
140 + reason="Inconsistent image processor being used due to lack "
141 + "of support for dynamic image token replacement")
142 +@pytest.mark.parametrize("model_and_config", model_and_vl_config)
143 +@pytest.mark.parametrize("dtype", [target_dtype])
144 +@pytest.mark.parametrize("max_tokens", [128])
145 +def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
146 + dtype: str, max_tokens: int) -> None:
147 + run_test(
148 + hf_runner,
149 + vllm_runner,
150 + image_assets,
151 + model_and_config,
152 + dtype=dtype,
153 + max_tokens=max_tokens,
154 + tensor_parallel_size=1,
155 + )
vllm/distributed/device_communicators/shm_broadcast.py
+1
@@ -268,6 +268,7 @@ class ShmRingBufferIO:
268 else:
269 return self.dequeue()
270
271 + @staticmethod
272 def create_from_process_group(pg: ProcessGroup,
273 max_chunk_bytes,
274 max_chunks,
vllm/distributed/parallel_state.py
+3 -1
@@ -194,7 +194,7 @@ class GroupCoordinator:
194 self.shm_broadcaster: Optional[ShmRingBufferIO] = None
195 if self.world_size > 1 and is_in_the_same_node(self.cpu_group):
196 self.shm_broadcaster = ShmRingBufferIO.create_from_process_group(
197 - self.cpu_group, 1 << 20, 6)
197 + self.cpu_group, 1 << 22, 6)
198
199 @property
200 def first_rank(self):
@@ -690,6 +690,8 @@ class GroupCoordinator:
690 self.pynccl_comm = None
691 if self.ca_comm is not None:
692 self.ca_comm = None
693 + if self.shm_broadcaster is not None:
694 + self.shm_broadcaster = None
695
696
697 _WORLD: Optional[GroupCoordinator] = None
vllm/model_executor/models/llava.py
+1 -1
@@ -219,7 +219,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
219
220 # NOTE: we skip the step to select the vision feature layer since
221 # this is already done inside the vision tower
222 - image_features = vision_tower(pixel_values.to(vision_tower.device),
222 + image_features = vision_tower(pixel_values,
223 self.config.vision_feature_layer)
224
225 return self._select_image_features(
vllm/model_executor/models/llava_next.py
+1 -1
@@ -301,7 +301,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
301
302 # NOTE: we skip the step to select the vision feature layer since
303 # this is already done inside the vision tower
304 - image_features = vision_tower(pixel_values.to(vision_tower.device),
304 + image_features = vision_tower(pixel_values,
305 self.config.vision_feature_layer)
306
307 return self._select_image_features(
vllm/model_executor/models/phi3v.py
+2 -3
@@ -157,7 +157,6 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
157
158 select = False
159
160 - target_device = self.img_projection[0].bias.device
160 target_dtype = self.img_projection[0].bias.dtype
161
162 if len(positions.tolist()) > 0:
@@ -231,7 +230,7 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
230 img_set_tensor = []
231 for _output_img in output_imgs:
232 img_feature_proj = self.img_projection(
234 - _output_img.to(target_device, target_dtype))
233 + _output_img.to(target_dtype))
234 img_set_tensor.append(img_feature_proj)
235 select = True
236
@@ -245,7 +244,7 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
244 hidden_states[positions[idx, 0],
245 positions[idx, 1]:positions[idx, 1] +
246 cnt] = (img_set_tensor[i].to(
248 - hidden_states.device, hidden_states.dtype))
247 + hidden_states.dtype))
248 idx += cnt
249
250 return hidden_states.squeeze(0)