@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1230263e16
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Fix InternVL2 vision embeddings process with pipeline parallel (#8299)

Isotr0py committed Sep 11, 2024 at 10:11 UTC 1230263e161caa9fd698e109d33437950769ec09
2 files changed +10 -3
tests/distributed/test_pipeline_parallel.py
+8 -2
@@ -32,7 +32,9 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
32 (1, 4, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
33 (2, 2, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
34 (2, 2, 0, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
35 - (2, 2, 1, 1, 1, "internlm/internlm2_5-7b-chat", "ray"),
35 + (1, 2, 1, 1, 1, "OpenGVLab/InternVL2-1B", "ray"),
36 + (1, 2, 1, 1, 1, "OpenGVLab/InternVL2-2B", "ray"),
37 + (1, 2, 1, 0, 1, "OpenGVLab/InternVL2-4B", "ray"),
38 ],
39 )
40 @fork_new_process_for_each_test
@@ -46,6 +48,8 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL,
48 # use half precision for speed and memory savings in CI environment
49 "--dtype",
50 "float16",
51 + "--max-model-len",
52 + "8192",
53 "--pipeline-parallel-size",
54 str(PP_SIZE),
55 "--tensor-parallel-size",
@@ -62,7 +66,9 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL,
66 tp_args = [
67 # use half precision for speed and memory savings in CI environment
68 "--dtype",
65 - "bfloat16",
69 + "float16",
70 + "--max-model-len",
71 + "8192",
72 "--tensor-parallel-size",
73 str(max(TP_SIZE, 2)), # We only use 2 GPUs in the CI.
74 "--distributed-executor-backend",
vllm/model_executor/models/internvl.py
+2 -1
@@ -17,6 +17,7 @@ from transformers import PretrainedConfig
17
18 from vllm.attention import AttentionMetadata
19 from vllm.config import CacheConfig, MultiModalConfig
20 +from vllm.distributed import get_pp_group
21 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
22 from vllm.model_executor.layers.quantization import QuantizationConfig
23 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -480,7 +481,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
481 **kwargs: object,
482 ) -> SamplerOutput:
483 image_input = self._parse_and_validate_image_input(**kwargs)
483 - if image_input is not None:
484 + if image_input is not None and get_pp_group().is_first_rank:
485 inputs_embeds = self.language_model.model.get_input_embeddings(
486 input_ids)
487 vision_embeddings = self._process_image_input(image_input)