@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8685ba1a1e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

Inclusion of InternVLChatModel In PP_SUPPORTED_MODELS(Pipeline Parallelism) (#7860)

manikandan.tm@zucisystems.com committed Sep 5, 2024 at 17:03 UTC 8685ba1a1ec08d2c14df924b6e2b499be14405e7
6 files changed +90 -35
tests/distributed/test_pipeline_parallel.py
+22 -16
@@ -18,23 +18,26 @@ logger = init_logger("test_pipeline_parallel")
18 VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
19
20
21 -@pytest.mark.parametrize(("TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, "
22 - "MODEL_NAME, DIST_BACKEND"),
23 - [
24 - (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
25 - (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
26 - (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
27 - (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
28 - (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
29 - (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
30 - (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
31 - (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
32 - (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
33 - (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
34 - ])
21 +@pytest.mark.parametrize(
22 + ("TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, TRUST_REMOTE_CODE, "
23 + "MODEL_NAME, DIST_BACKEND"),
24 + [
25 + (2, 2, 0, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
26 + (2, 2, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
27 + (1, 3, 0, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
28 + (1, 4, 0, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
29 + (1, 4, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
30 + (1, 3, 0, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
31 + (1, 4, 0, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
32 + (1, 4, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
33 + (2, 2, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
34 + (2, 2, 0, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
35 + (2, 2, 1, 1, 1, "internlm/internlm2_5-7b-chat", "ray"),
36 + ],
37 +)
38 @fork_new_process_for_each_test
36 -def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
37 - DIST_BACKEND):
39 +def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL,
40 + TRUST_REMOTE_CODE, MODEL_NAME, DIST_BACKEND):
41 if VLLM_MULTI_NODE and DIST_BACKEND == "mp":
42 pytest.skip("Skipping multi-node pipeline parallel test for "
43 "multiprocessing distributed backend")
@@ -71,6 +74,9 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
74 if EAGER_MODE:
75 pp_args.append("--enforce-eager")
76 tp_args.append("--enforce-eager")
77 + if TRUST_REMOTE_CODE:
78 + pp_args.append("--trust-remote-code")
79 + tp_args.append("--trust-remote-code")
80 pp_env = None
81 if (DIST_BACKEND == "ray" and TP_SIZE == 2 and PP_SIZE == 2
82 and CHUNKED_PREFILL):
tests/utils.py
+6 -1
@@ -178,7 +178,12 @@ def compare_two_settings(model: str,
178 env2: The second set of environment variables to pass to the API server.
179 """
180
181 - tokenizer = AutoTokenizer.from_pretrained(model)
181 + trust_remote_code = "--trust-remote-code"
182 + if trust_remote_code in arg1 or trust_remote_code in arg2:
183 + tokenizer = AutoTokenizer.from_pretrained(model,
184 + trust_remote_code=True)
185 + else:
186 + tokenizer = AutoTokenizer.from_pretrained(model)
187
188 prompt = "Hello, my name is"
189 token_ids = tokenizer(prompt)["input_ids"]
vllm/config.py
+5 -3
@@ -35,18 +35,20 @@ _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
35 _MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 4096
36
37 _PP_SUPPORTED_MODELS = [
38 - "AquilaModel",
38 "AquilaForCausalLM",
39 + "AquilaModel",
40 "DeepseekV2ForCausalLM",
41 + "GPT2LMHeadModel",
42 + "InternLM2ForCausalLM",
43 "InternLMForCausalLM",
44 + "InternVLChatModel",
45 "JAISLMHeadModel",
46 "LlamaForCausalLM",
47 "LLaMAForCausalLM",
48 "MistralForCausalLM",
46 - "Phi3ForCausalLM",
47 - "GPT2LMHeadModel",
49 "MixtralForCausalLM",
50 "NemotronForCausalLM",
51 + "Phi3ForCausalLM",
52 "Qwen2ForCausalLM",
53 "Qwen2MoeForCausalLM",
54 "QWenLMHeadModel",
vllm/model_executor/models/internlm2.py
+38 -14
@@ -1,6 +1,6 @@
1 # -*- coding: utf-8 -*-
2 from functools import partial
3 -from typing import Any, Dict, Iterable, List, Optional, Tuple
3 +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
4
5 import torch
6 from torch import nn
@@ -8,7 +8,7 @@ from transformers import PretrainedConfig
8
9 from vllm.attention import Attention, AttentionMetadata
10 from vllm.config import CacheConfig
11 -from vllm.distributed import (get_tensor_model_parallel_rank,
11 +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
12 get_tensor_model_parallel_world_size,
13 split_tensor_along_last_dim,
14 tensor_model_parallel_all_gather)
@@ -28,6 +28,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28 from vllm.model_executor.sampling_metadata import SamplingMetadata
29 from vllm.sequence import IntermediateTensors
30
31 +from .utils import (is_pp_missing_parameter,
32 + make_empty_intermediate_tensors_factory, make_layers)
33 +
34
35 class InternLM2MLP(nn.Module):
36
@@ -234,6 +237,7 @@ class InternLM2Model(nn.Module):
237 config: PretrainedConfig,
238 cache_config: Optional[CacheConfig] = None,
239 quant_config: Optional[QuantizationConfig] = None,
240 + prefix: str = "",
241 ) -> None:
242 super().__init__()
243 self.config = config
@@ -243,11 +247,15 @@ class InternLM2Model(nn.Module):
247 config.vocab_size,
248 config.hidden_size,
249 )
246 - self.layers = nn.ModuleList([
247 - InternLMDecoderLayer(config, cache_config, quant_config)
248 - for _ in range(config.num_hidden_layers)
249 - ])
250 + self.start_layer, self.end_layer, self.layers = make_layers(
251 + config.num_hidden_layers,
252 + lambda prefix: InternLMDecoderLayer(config, cache_config,
253 + quant_config),
254 + prefix=f"{prefix}.layers")
255 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
256 + self.make_empty_intermediate_tensors = (
257 + make_empty_intermediate_tensors_factory(
258 + ["hidden_states", "residual"], config.hidden_size))
259
260 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
261 return self.tok_embeddings(input_ids)
@@ -260,21 +268,31 @@ class InternLM2Model(nn.Module):
268 attn_metadata: AttentionMetadata,
269 intermediate_tensors: IntermediateTensors = None,
270 inputs_embeds: Optional[torch.Tensor] = None,
263 - ) -> torch.Tensor:
264 - if inputs_embeds is not None:
265 - hidden_states = inputs_embeds
271 + ) -> Union[torch.Tensor, IntermediateTensors]:
272 + if get_pp_group().is_first_rank:
273 + if inputs_embeds is not None:
274 + hidden_states = inputs_embeds
275 + else:
276 + hidden_states = self.tok_embeddings(input_ids)
277 + residual = None
278 else:
267 - hidden_states = self.tok_embeddings(input_ids)
268 - residual = None
269 - for i in range(len(self.layers)):
279 + assert intermediate_tensors is not None
280 + hidden_states = intermediate_tensors["hidden_states"]
281 + residual = intermediate_tensors["residual"]
282 + for i in range(self.start_layer, self.end_layer):
283 layer = self.layers[i]
284 hidden_states, residual = layer(
285 positions,
286 hidden_states,
274 - kv_caches[i],
287 + kv_caches[i - self.start_layer],
288 attn_metadata,
289 residual,
290 )
291 + if not get_pp_group().is_last_rank:
292 + return IntermediateTensors({
293 + "hidden_states": hidden_states,
294 + "residual": residual
295 + })
296 hidden_states, _ = self.norm(hidden_states, residual)
297 return hidden_states
298
@@ -298,6 +316,8 @@ class InternLM2ForCausalLM(nn.Module):
316 self.output.weight = self.model.tok_embeddings.weight
317 self.logits_processor = LogitsProcessor(config.vocab_size)
318 self.sampler = Sampler()
319 + self.make_empty_intermediate_tensors = (
320 + self.model.make_empty_intermediate_tensors)
321
322 def forward(
323 self,
@@ -308,7 +328,7 @@ class InternLM2ForCausalLM(nn.Module):
328 intermediate_tensors: IntermediateTensors,
329 ) -> torch.Tensor:
330 hidden_states = self.model(input_ids, positions, kv_caches,
311 - attn_metadata)
331 + attn_metadata, intermediate_tensors)
332 return hidden_states
333
334 def compute_logits(
@@ -345,6 +365,8 @@ class InternLM2ForCausalLM(nn.Module):
365 # Skip loading extra bias for GPTQ models.
366 if name.endswith(".bias") and name not in params_dict:
367 continue
368 + if is_pp_missing_parameter(name, self):
369 + continue
370 param = params_dict[name]
371 weight_loader = param.weight_loader
372 weight_loader(param, loaded_weight, shard_id)
@@ -353,6 +375,8 @@ class InternLM2ForCausalLM(nn.Module):
375 # Skip loading extra bias for GPTQ models.
376 if name.endswith(".bias") and name not in params_dict:
377 continue
378 + if is_pp_missing_parameter(name, self):
379 + continue
380 param = params_dict[name]
381 weight_loader = getattr(param, "weight_loader",
382 default_weight_loader)
vllm/model_executor/models/internvl.py
+3 -1
@@ -341,6 +341,8 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
341 nn.Linear(llm_hidden_size, llm_hidden_size))
342
343 self.img_context_token_id = None
344 + self.make_empty_intermediate_tensors = (
345 + self.language_model.make_empty_intermediate_tensors)
346
347 def pixel_shuffle(self, x, scale_factor=0.5):
348 n, w, h, c = x.size()
@@ -461,7 +463,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
463 positions,
464 kv_caches,
465 attn_metadata,
464 - None,
466 + intermediate_tensors,
467 inputs_embeds=inputs_embeds)
468 return hidden_states
469
vllm/model_executor/models/utils.py
+16
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
12 from vllm.model_executor.model_loader.loader import build_model
13 from vllm.model_executor.models import ModelRegistry
14 from vllm.multimodal.base import NestedTensors
15 +from vllm.sequence import IntermediateTensors
16 from vllm.utils import is_pin_memory_available
17
18
@@ -279,3 +280,18 @@ def is_pp_missing_parameter(name: str, model: torch.nn.Module) -> bool:
280 if name.startswith(missing_layer_name):
281 return True
282 return False
283 +
284 +
285 +def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int):
286 +
287 + def make_empty_intermediate_tensors(
288 + batch_size: int, dtype: torch.dtype,
289 + device: torch.device) -> IntermediateTensors:
290 + return IntermediateTensors({
291 + key: torch.zeros((batch_size, hidden_size),
292 + dtype=dtype,
293 + device=device)
294 + for key in keys
295 + })
296 +
297 + return make_empty_intermediate_tensors