A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Model] Support pp for qwen2-vl (#8696)
Yanyi Liu committed
Sep 23, 2024 at 21:46 UTC
a79e5229843e2800956956d0668b1b4858dbb61e
4 files changed
+46
-14
tests/distributed/test_pipeline_parallel.py
+8
@@ -8,6 +8,8 @@ WARNING: This test runs in both single-node (4 GPUs) and multi-node
8
import os
9
10
import pytest
11
+from packaging import version
12
+from transformers import __version__ as transformers_version
13
14
from vllm.logger import init_logger
15
@@ -37,6 +39,7 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
39
(1, 2, 1, 1, 1, "OpenGVLab/InternVL2-1B", "mp"),
40
(1, 2, 1, 1, 1, "OpenGVLab/InternVL2-2B", "mp"),
41
(1, 2, 1, 0, 1, "OpenGVLab/InternVL2-4B", "mp"),
42
+ (1, 2, 0, 1, 0, "Qwen/Qwen2-VL-2B-Instruct", "mp")
43
],
44
)
45
@fork_new_process_for_each_test
@@ -46,6 +49,11 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL,
49
pytest.skip("Skipping multi-node pipeline parallel test for "
50
"multiprocessing distributed backend")
51
52
+ # Skip tests that require transformers>=4.45.0
53
+ if "Qwen2-VL" in MODEL_NAME and version.parse(
54
+ transformers_version) < version.parse("4.45.0.dev0"):
55
+ pytest.skip("This test requires transformers>=4.45.0")
56
+
57
pp_args = [
58
# use half precision for speed and memory savings in CI environment
59
"--dtype",
vllm/config.py
+1
@@ -51,6 +51,7 @@ _PP_SUPPORTED_MODELS = [
51
"Qwen2ForCausalLM",
52
"Qwen2MoeForCausalLM",
53
"QWenLMHeadModel",
54
+ "Qwen2VLForConditionalGeneration",
55
]
56
57
vllm/model_executor/models/qwen2.py
+15
-7
@@ -49,7 +49,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
49
from vllm.sequence import IntermediateTensors
50
51
from .interfaces import SupportsLoRA
52
-from .utils import is_pp_missing_parameter, make_layers
52
+from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
53
54
55
class Qwen2MLP(nn.Module):
@@ -235,11 +235,16 @@ class Qwen2Model(nn.Module):
235
self.padding_idx = config.pad_token_id
236
self.vocab_size = config.vocab_size
237
238
- self.embed_tokens = VocabParallelEmbedding(
239
- config.vocab_size,
240
- config.hidden_size,
241
- quant_config=quant_config,
242
- )
238
+ if get_pp_group().is_first_rank or (config.tie_word_embeddings
239
+ and get_pp_group().is_last_rank):
240
+ self.embed_tokens = VocabParallelEmbedding(
241
+ config.vocab_size,
242
+ config.hidden_size,
243
+ quant_config=quant_config,
244
+ )
245
+ else:
246
+ self.embed_tokens = PPMissingLayer()
247
+
248
self.start_layer, self.end_layer, self.layers = make_layers(
249
config.num_hidden_layers,
250
lambda prefix: Qwen2DecoderLayer(config=config,
@@ -248,7 +253,10 @@ class Qwen2Model(nn.Module):
253
prefix=f"{prefix}.layers",
254
)
255
251
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
256
+ if get_pp_group().is_last_rank:
257
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
258
+ else:
259
+ self.norm = PPMissingLayer()
260
261
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
262
return self.embed_tokens(input_ids)
vllm/model_executor/models/qwen2_vl.py
+22
-7
@@ -45,7 +45,7 @@ from vllm.attention import AttentionMetadata
45
from vllm.attention.selector import (_Backend, backend_name_to_enum,
46
get_global_forced_attn_backend)
47
from vllm.config import CacheConfig, MultiModalConfig
48
-from vllm.distributed import parallel_state
48
+from vllm.distributed import get_pp_group, parallel_state
49
from vllm.distributed import utils as dist_utils
50
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
51
from vllm.logger import init_logger
@@ -68,6 +68,9 @@ from vllm.platforms import current_platform
68
from vllm.sequence import IntermediateTensors, SequenceData
69
from vllm.transformers_utils.processor import get_processor
70
71
+from .utils import (PPMissingLayer, is_pp_missing_parameter,
72
+ make_empty_intermediate_tensors_factory)
73
+
74
logger = init_logger(__name__)
75
76
# === Vision Inputs === #
@@ -856,15 +859,21 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
859
860
self.model = Qwen2Model(config, cache_config, quant_config)
861
859
- if config.tie_word_embeddings:
860
- self.lm_head = self.model.embed_tokens
862
+ if get_pp_group().is_last_rank:
863
+ if config.tie_word_embeddings:
864
+ self.lm_head = self.model.embed_tokens
865
+ else:
866
+ self.lm_head = ParallelLMHead(config.vocab_size,
867
+ config.hidden_size,
868
+ quant_config=quant_config)
869
else:
862
- self.lm_head = ParallelLMHead(config.vocab_size,
863
- config.hidden_size,
864
- quant_config=quant_config)
870
+ self.lm_head = PPMissingLayer()
871
872
self.logits_processor = LogitsProcessor(config.vocab_size)
873
self.sampler = Sampler()
874
+ self.make_empty_intermediate_tensors = (
875
+ make_empty_intermediate_tensors_factory(
876
+ ["hidden_states", "residual"], config.hidden_size))
877
878
def _validate_and_reshape_mm_tensor(self,
879
mm_input: Union[torch.Tensor,
@@ -979,7 +988,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
988
image_input = self._parse_and_validate_image_input(**kwargs)
989
video_input = self._parse_and_validate_video_input(**kwargs)
990
982
- if image_input is None and video_input is None:
991
+ if (image_input is None
992
+ and video_input is None) or not get_pp_group().is_first_rank:
993
inputs_embeds = None
994
else:
995
if getattr(self.config, "rope_scaling", {}).get("type",
@@ -1015,6 +1025,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
1025
positions=positions,
1026
kv_caches=kv_caches,
1027
attn_metadata=attn_metadata,
1028
+ intermediate_tensors=intermediate_tensors,
1029
inputs_embeds=inputs_embeds,
1030
)
1031
return hidden_states
@@ -1055,6 +1066,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
1066
# Skip loading extra bias for GPTQ models.
1067
if name.endswith(".bias") and name not in params_dict:
1068
continue
1069
+ if is_pp_missing_parameter(name, self):
1070
+ continue
1071
param = params_dict[name]
1072
weight_loader = param.weight_loader
1073
weight_loader(param, loaded_weight, shard_id)
@@ -1081,6 +1094,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
1094
# Skip loading extra bias for GPTQ models.
1095
if name.endswith(".bias") and name not in params_dict:
1096
continue
1097
+ if is_pp_missing_parameter(name, self):
1098
+ continue
1099
param = params_dict[name]
1100
except KeyError:
1101
print(params_dict.keys())