A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Fix InternVL2 vision embeddings process with pipeline parallel (#8299)
Isotr0py committed
Sep 11, 2024 at 10:11 UTC
1230263e161caa9fd698e109d33437950769ec09
2 files changed
+10
-3
tests/distributed/test_pipeline_parallel.py
+8
-2
@@ -32,7 +32,9 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
32
(1, 4, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
33
(2, 2, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
34
(2, 2, 0, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
35
- (2, 2, 1, 1, 1, "internlm/internlm2_5-7b-chat", "ray"),
35
+ (1, 2, 1, 1, 1, "OpenGVLab/InternVL2-1B", "ray"),
36
+ (1, 2, 1, 1, 1, "OpenGVLab/InternVL2-2B", "ray"),
37
+ (1, 2, 1, 0, 1, "OpenGVLab/InternVL2-4B", "ray"),
38
],
39
)
40
@fork_new_process_for_each_test
@@ -46,6 +48,8 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL,
48
# use half precision for speed and memory savings in CI environment
49
"--dtype",
50
"float16",
51
+ "--max-model-len",
52
+ "8192",
53
"--pipeline-parallel-size",
54
str(PP_SIZE),
55
"--tensor-parallel-size",
@@ -62,7 +66,9 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL,
66
tp_args = [
67
# use half precision for speed and memory savings in CI environment
68
"--dtype",
65
- "bfloat16",
69
+ "float16",
70
+ "--max-model-len",
71
+ "8192",
72
"--tensor-parallel-size",
73
str(max(TP_SIZE, 2)), # We only use 2 GPUs in the CI.
74
"--distributed-executor-backend",
vllm/model_executor/models/internvl.py
+2
-1
@@ -17,6 +17,7 @@ from transformers import PretrainedConfig
17
18
from vllm.attention import AttentionMetadata
19
from vllm.config import CacheConfig, MultiModalConfig
20
+from vllm.distributed import get_pp_group
21
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
22
from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -480,7 +481,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
481
**kwargs: object,
482
) -> SamplerOutput:
483
image_input = self._parse_and_validate_image_input(**kwargs)
483
- if image_input is not None:
484
+ if image_input is not None and get_pp_group().is_first_rank:
485
inputs_embeds = self.language_model.model.get_input_embeddings(
486
input_ids)
487
vision_embeddings = self._process_image_input(image_input)