@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a79e522984
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Model] Support pp for qwen2-vl (#8696)

Yanyi Liu committed Sep 23, 2024 at 21:46 UTC a79e5229843e2800956956d0668b1b4858dbb61e
4 files changed +46 -14
tests/distributed/test_pipeline_parallel.py
+8
@@ -8,6 +8,8 @@ WARNING: This test runs in both single-node (4 GPUs) and multi-node
8 import os
9
10 import pytest
11 +from packaging import version
12 +from transformers import __version__ as transformers_version
13
14 from vllm.logger import init_logger
15
@@ -37,6 +39,7 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
39 (1, 2, 1, 1, 1, "OpenGVLab/InternVL2-1B", "mp"),
40 (1, 2, 1, 1, 1, "OpenGVLab/InternVL2-2B", "mp"),
41 (1, 2, 1, 0, 1, "OpenGVLab/InternVL2-4B", "mp"),
42 + (1, 2, 0, 1, 0, "Qwen/Qwen2-VL-2B-Instruct", "mp")
43 ],
44 )
45 @fork_new_process_for_each_test
@@ -46,6 +49,11 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL,
49 pytest.skip("Skipping multi-node pipeline parallel test for "
50 "multiprocessing distributed backend")
51
52 + # Skip tests that require transformers>=4.45.0
53 + if "Qwen2-VL" in MODEL_NAME and version.parse(
54 + transformers_version) < version.parse("4.45.0.dev0"):
55 + pytest.skip("This test requires transformers>=4.45.0")
56 +
57 pp_args = [
58 # use half precision for speed and memory savings in CI environment
59 "--dtype",
vllm/config.py
+1
@@ -51,6 +51,7 @@ _PP_SUPPORTED_MODELS = [
51 "Qwen2ForCausalLM",
52 "Qwen2MoeForCausalLM",
53 "QWenLMHeadModel",
54 + "Qwen2VLForConditionalGeneration",
55 ]
56
57
vllm/model_executor/models/qwen2.py
+15 -7
@@ -49,7 +49,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
49 from vllm.sequence import IntermediateTensors
50
51 from .interfaces import SupportsLoRA
52 -from .utils import is_pp_missing_parameter, make_layers
52 +from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
53
54
55 class Qwen2MLP(nn.Module):
@@ -235,11 +235,16 @@ class Qwen2Model(nn.Module):
235 self.padding_idx = config.pad_token_id
236 self.vocab_size = config.vocab_size
237
238 - self.embed_tokens = VocabParallelEmbedding(
239 - config.vocab_size,
240 - config.hidden_size,
241 - quant_config=quant_config,
242 - )
238 + if get_pp_group().is_first_rank or (config.tie_word_embeddings
239 + and get_pp_group().is_last_rank):
240 + self.embed_tokens = VocabParallelEmbedding(
241 + config.vocab_size,
242 + config.hidden_size,
243 + quant_config=quant_config,
244 + )
245 + else:
246 + self.embed_tokens = PPMissingLayer()
247 +
248 self.start_layer, self.end_layer, self.layers = make_layers(
249 config.num_hidden_layers,
250 lambda prefix: Qwen2DecoderLayer(config=config,
@@ -248,7 +253,10 @@ class Qwen2Model(nn.Module):
253 prefix=f"{prefix}.layers",
254 )
255
251 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
256 + if get_pp_group().is_last_rank:
257 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
258 + else:
259 + self.norm = PPMissingLayer()
260
261 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
262 return self.embed_tokens(input_ids)
vllm/model_executor/models/qwen2_vl.py
+22 -7
@@ -45,7 +45,7 @@ from vllm.attention import AttentionMetadata
45 from vllm.attention.selector import (_Backend, backend_name_to_enum,
46 get_global_forced_attn_backend)
47 from vllm.config import CacheConfig, MultiModalConfig
48 -from vllm.distributed import parallel_state
48 +from vllm.distributed import get_pp_group, parallel_state
49 from vllm.distributed import utils as dist_utils
50 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
51 from vllm.logger import init_logger
@@ -68,6 +68,9 @@ from vllm.platforms import current_platform
68 from vllm.sequence import IntermediateTensors, SequenceData
69 from vllm.transformers_utils.processor import get_processor
70
71 +from .utils import (PPMissingLayer, is_pp_missing_parameter,
72 + make_empty_intermediate_tensors_factory)
73 +
74 logger = init_logger(__name__)
75
76 # === Vision Inputs === #
@@ -856,15 +859,21 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
859
860 self.model = Qwen2Model(config, cache_config, quant_config)
861
859 - if config.tie_word_embeddings:
860 - self.lm_head = self.model.embed_tokens
862 + if get_pp_group().is_last_rank:
863 + if config.tie_word_embeddings:
864 + self.lm_head = self.model.embed_tokens
865 + else:
866 + self.lm_head = ParallelLMHead(config.vocab_size,
867 + config.hidden_size,
868 + quant_config=quant_config)
869 else:
862 - self.lm_head = ParallelLMHead(config.vocab_size,
863 - config.hidden_size,
864 - quant_config=quant_config)
870 + self.lm_head = PPMissingLayer()
871
872 self.logits_processor = LogitsProcessor(config.vocab_size)
873 self.sampler = Sampler()
874 + self.make_empty_intermediate_tensors = (
875 + make_empty_intermediate_tensors_factory(
876 + ["hidden_states", "residual"], config.hidden_size))
877
878 def _validate_and_reshape_mm_tensor(self,
879 mm_input: Union[torch.Tensor,
@@ -979,7 +988,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
988 image_input = self._parse_and_validate_image_input(**kwargs)
989 video_input = self._parse_and_validate_video_input(**kwargs)
990
982 - if image_input is None and video_input is None:
991 + if (image_input is None
992 + and video_input is None) or not get_pp_group().is_first_rank:
993 inputs_embeds = None
994 else:
995 if getattr(self.config, "rope_scaling", {}).get("type",
@@ -1015,6 +1025,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
1025 positions=positions,
1026 kv_caches=kv_caches,
1027 attn_metadata=attn_metadata,
1028 + intermediate_tensors=intermediate_tensors,
1029 inputs_embeds=inputs_embeds,
1030 )
1031 return hidden_states
@@ -1055,6 +1066,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
1066 # Skip loading extra bias for GPTQ models.
1067 if name.endswith(".bias") and name not in params_dict:
1068 continue
1069 + if is_pp_missing_parameter(name, self):
1070 + continue
1071 param = params_dict[name]
1072 weight_loader = param.weight_loader
1073 weight_loader(param, loaded_weight, shard_id)
@@ -1081,6 +1094,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
1094 # Skip loading extra bias for GPTQ models.
1095 if name.endswith(".bias") and name not in params_dict:
1096 continue
1097 + if is_pp_missing_parameter(name, self):
1098 + continue
1099 param = params_dict[name]
1100 except KeyError:
1101 print(params_dict.keys())