A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
Inclusion of InternVLChatModel In PP_SUPPORTED_MODELS(Pipeline Parallelism) (#7860)
manikandan.tm@zucisystems.com committed
Sep 5, 2024 at 17:03 UTC
8685ba1a1ec08d2c14df924b6e2b499be14405e7
6 files changed
+90
-35
tests/distributed/test_pipeline_parallel.py
+22
-16
@@ -18,23 +18,26 @@ logger = init_logger("test_pipeline_parallel")
18
VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
19
20
21
-@pytest.mark.parametrize(("TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, "
22
- "MODEL_NAME, DIST_BACKEND"),
23
- [
24
- (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
25
- (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
26
- (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
27
- (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
28
- (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
29
- (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
30
- (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
31
- (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
32
- (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
33
- (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
34
- ])
21
+@pytest.mark.parametrize(
22
+ ("TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, TRUST_REMOTE_CODE, "
23
+ "MODEL_NAME, DIST_BACKEND"),
24
+ [
25
+ (2, 2, 0, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
26
+ (2, 2, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
27
+ (1, 3, 0, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
28
+ (1, 4, 0, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
29
+ (1, 4, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
30
+ (1, 3, 0, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
31
+ (1, 4, 0, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
32
+ (1, 4, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
33
+ (2, 2, 1, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
34
+ (2, 2, 0, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
35
+ (2, 2, 1, 1, 1, "internlm/internlm2_5-7b-chat", "ray"),
36
+ ],
37
+)
38
@fork_new_process_for_each_test
36
-def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
37
- DIST_BACKEND):
39
+def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL,
40
+ TRUST_REMOTE_CODE, MODEL_NAME, DIST_BACKEND):
41
if VLLM_MULTI_NODE and DIST_BACKEND == "mp":
42
pytest.skip("Skipping multi-node pipeline parallel test for "
43
"multiprocessing distributed backend")
@@ -71,6 +74,9 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
74
if EAGER_MODE:
75
pp_args.append("--enforce-eager")
76
tp_args.append("--enforce-eager")
77
+ if TRUST_REMOTE_CODE:
78
+ pp_args.append("--trust-remote-code")
79
+ tp_args.append("--trust-remote-code")
80
pp_env = None
81
if (DIST_BACKEND == "ray" and TP_SIZE == 2 and PP_SIZE == 2
82
and CHUNKED_PREFILL):
tests/utils.py
+6
-1
@@ -178,7 +178,12 @@ def compare_two_settings(model: str,
178
env2: The second set of environment variables to pass to the API server.
179
"""
180
181
- tokenizer = AutoTokenizer.from_pretrained(model)
181
+ trust_remote_code = "--trust-remote-code"
182
+ if trust_remote_code in arg1 or trust_remote_code in arg2:
183
+ tokenizer = AutoTokenizer.from_pretrained(model,
184
+ trust_remote_code=True)
185
+ else:
186
+ tokenizer = AutoTokenizer.from_pretrained(model)
187
188
prompt = "Hello, my name is"
189
token_ids = tokenizer(prompt)["input_ids"]
vllm/config.py
+5
-3
@@ -35,18 +35,20 @@ _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
35
_MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 4096
36
37
_PP_SUPPORTED_MODELS = [
38
- "AquilaModel",
38
"AquilaForCausalLM",
39
+ "AquilaModel",
40
"DeepseekV2ForCausalLM",
41
+ "GPT2LMHeadModel",
42
+ "InternLM2ForCausalLM",
43
"InternLMForCausalLM",
44
+ "InternVLChatModel",
45
"JAISLMHeadModel",
46
"LlamaForCausalLM",
47
"LLaMAForCausalLM",
48
"MistralForCausalLM",
46
- "Phi3ForCausalLM",
47
- "GPT2LMHeadModel",
49
"MixtralForCausalLM",
50
"NemotronForCausalLM",
51
+ "Phi3ForCausalLM",
52
"Qwen2ForCausalLM",
53
"Qwen2MoeForCausalLM",
54
"QWenLMHeadModel",
vllm/model_executor/models/internlm2.py
+38
-14
@@ -1,6 +1,6 @@
1
# -*- coding: utf-8 -*-
2
from functools import partial
3
-from typing import Any, Dict, Iterable, List, Optional, Tuple
3
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
4
5
import torch
6
from torch import nn
@@ -8,7 +8,7 @@ from transformers import PretrainedConfig
8
9
from vllm.attention import Attention, AttentionMetadata
10
from vllm.config import CacheConfig
11
-from vllm.distributed import (get_tensor_model_parallel_rank,
11
+from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
12
get_tensor_model_parallel_world_size,
13
split_tensor_along_last_dim,
14
tensor_model_parallel_all_gather)
@@ -28,6 +28,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
29
from vllm.sequence import IntermediateTensors
30
31
+from .utils import (is_pp_missing_parameter,
32
+ make_empty_intermediate_tensors_factory, make_layers)
33
+
34
35
class InternLM2MLP(nn.Module):
36
@@ -234,6 +237,7 @@ class InternLM2Model(nn.Module):
237
config: PretrainedConfig,
238
cache_config: Optional[CacheConfig] = None,
239
quant_config: Optional[QuantizationConfig] = None,
240
+ prefix: str = "",
241
) -> None:
242
super().__init__()
243
self.config = config
@@ -243,11 +247,15 @@ class InternLM2Model(nn.Module):
247
config.vocab_size,
248
config.hidden_size,
249
)
246
- self.layers = nn.ModuleList([
247
- InternLMDecoderLayer(config, cache_config, quant_config)
248
- for _ in range(config.num_hidden_layers)
249
- ])
250
+ self.start_layer, self.end_layer, self.layers = make_layers(
251
+ config.num_hidden_layers,
252
+ lambda prefix: InternLMDecoderLayer(config, cache_config,
253
+ quant_config),
254
+ prefix=f"{prefix}.layers")
255
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
256
+ self.make_empty_intermediate_tensors = (
257
+ make_empty_intermediate_tensors_factory(
258
+ ["hidden_states", "residual"], config.hidden_size))
259
260
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
261
return self.tok_embeddings(input_ids)
@@ -260,21 +268,31 @@ class InternLM2Model(nn.Module):
268
attn_metadata: AttentionMetadata,
269
intermediate_tensors: IntermediateTensors = None,
270
inputs_embeds: Optional[torch.Tensor] = None,
263
- ) -> torch.Tensor:
264
- if inputs_embeds is not None:
265
- hidden_states = inputs_embeds
271
+ ) -> Union[torch.Tensor, IntermediateTensors]:
272
+ if get_pp_group().is_first_rank:
273
+ if inputs_embeds is not None:
274
+ hidden_states = inputs_embeds
275
+ else:
276
+ hidden_states = self.tok_embeddings(input_ids)
277
+ residual = None
278
else:
267
- hidden_states = self.tok_embeddings(input_ids)
268
- residual = None
269
- for i in range(len(self.layers)):
279
+ assert intermediate_tensors is not None
280
+ hidden_states = intermediate_tensors["hidden_states"]
281
+ residual = intermediate_tensors["residual"]
282
+ for i in range(self.start_layer, self.end_layer):
283
layer = self.layers[i]
284
hidden_states, residual = layer(
285
positions,
286
hidden_states,
274
- kv_caches[i],
287
+ kv_caches[i - self.start_layer],
288
attn_metadata,
289
residual,
290
)
291
+ if not get_pp_group().is_last_rank:
292
+ return IntermediateTensors({
293
+ "hidden_states": hidden_states,
294
+ "residual": residual
295
+ })
296
hidden_states, _ = self.norm(hidden_states, residual)
297
return hidden_states
298
@@ -298,6 +316,8 @@ class InternLM2ForCausalLM(nn.Module):
316
self.output.weight = self.model.tok_embeddings.weight
317
self.logits_processor = LogitsProcessor(config.vocab_size)
318
self.sampler = Sampler()
319
+ self.make_empty_intermediate_tensors = (
320
+ self.model.make_empty_intermediate_tensors)
321
322
def forward(
323
self,
@@ -308,7 +328,7 @@ class InternLM2ForCausalLM(nn.Module):
328
intermediate_tensors: IntermediateTensors,
329
) -> torch.Tensor:
330
hidden_states = self.model(input_ids, positions, kv_caches,
311
- attn_metadata)
331
+ attn_metadata, intermediate_tensors)
332
return hidden_states
333
334
def compute_logits(
@@ -345,6 +365,8 @@ class InternLM2ForCausalLM(nn.Module):
365
# Skip loading extra bias for GPTQ models.
366
if name.endswith(".bias") and name not in params_dict:
367
continue
368
+ if is_pp_missing_parameter(name, self):
369
+ continue
370
param = params_dict[name]
371
weight_loader = param.weight_loader
372
weight_loader(param, loaded_weight, shard_id)
@@ -353,6 +375,8 @@ class InternLM2ForCausalLM(nn.Module):
375
# Skip loading extra bias for GPTQ models.
376
if name.endswith(".bias") and name not in params_dict:
377
continue
378
+ if is_pp_missing_parameter(name, self):
379
+ continue
380
param = params_dict[name]
381
weight_loader = getattr(param, "weight_loader",
382
default_weight_loader)
vllm/model_executor/models/internvl.py
+3
-1
@@ -341,6 +341,8 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
341
nn.Linear(llm_hidden_size, llm_hidden_size))
342
343
self.img_context_token_id = None
344
+ self.make_empty_intermediate_tensors = (
345
+ self.language_model.make_empty_intermediate_tensors)
346
347
def pixel_shuffle(self, x, scale_factor=0.5):
348
n, w, h, c = x.size()
@@ -461,7 +463,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
463
positions,
464
kv_caches,
465
attn_metadata,
464
- None,
466
+ intermediate_tensors,
467
inputs_embeds=inputs_embeds)
468
return hidden_states
469
vllm/model_executor/models/utils.py
+16
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
12
from vllm.model_executor.model_loader.loader import build_model
13
from vllm.model_executor.models import ModelRegistry
14
from vllm.multimodal.base import NestedTensors
15
+from vllm.sequence import IntermediateTensors
16
from vllm.utils import is_pin_memory_available
17
18
@@ -279,3 +280,18 @@ def is_pp_missing_parameter(name: str, model: torch.nn.Module) -> bool:
280
if name.startswith(missing_layer_name):
281
return True
282
return False
283
+
284
+
285
+def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int):
286
+
287
+ def make_empty_intermediate_tensors(
288
+ batch_size: int, dtype: torch.dtype,
289
+ device: torch.device) -> IntermediateTensors:
290
+ return IntermediateTensors({
291
+ key: torch.zeros((batch_size, hidden_size),
292
+ dtype=dtype,
293
+ device=device)
294
+ for key in keys
295
+ })
296
+
297
+ return make_empty_intermediate_tensors