A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Model] Pipeline parallel support for Qwen2 (#6924)
xuyi committed
Aug 1, 2024 at 09:49 UTC
1d2e7fb73f1205ae03e4ee3bcd3de566733bf582
3 files changed
+101
-27
vllm/config.py
+2
@@ -40,6 +40,8 @@ _PP_SUPPORTED_MODELS = [
40
"GPT2LMHeadModel",
41
"MixtralForCausalLM",
42
"NemotronForCausalLM",
43
+ "Qwen2ForCausalLM",
44
+ "Qwen2MoeForCausalLM",
45
]
46
47
vllm/model_executor/models/qwen2.py
+45
-12
@@ -30,7 +30,7 @@ from transformers import Qwen2Config
30
31
from vllm.attention import Attention, AttentionMetadata
32
from vllm.config import CacheConfig, LoRAConfig
33
-from vllm.distributed import get_tensor_model_parallel_world_size
33
+from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.layernorm import RMSNorm
36
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -49,6 +49,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
49
from vllm.sequence import IntermediateTensors, SamplerOutput
50
51
from .interfaces import SupportsLoRA
52
+from .utils import is_pp_missing_parameter, make_layers
53
54
55
class Qwen2MLP(nn.Module):
@@ -227,6 +228,7 @@ class Qwen2Model(nn.Module):
228
config: Qwen2Config,
229
cache_config: Optional[CacheConfig] = None,
230
quant_config: Optional[QuantizationConfig] = None,
231
+ prefix: str = "",
232
) -> None:
233
super().__init__()
234
self.config = config
@@ -237,10 +239,14 @@ class Qwen2Model(nn.Module):
239
config.vocab_size,
240
config.hidden_size,
241
)
240
- self.layers = nn.ModuleList([
241
- Qwen2DecoderLayer(config, cache_config, quant_config)
242
- for _ in range(config.num_hidden_layers)
243
- ])
242
+ self.start_layer, self.end_layer, self.layers = make_layers(
243
+ config.num_hidden_layers,
244
+ lambda prefix: Qwen2DecoderLayer(config=config,
245
+ cache_config=cache_config,
246
+ quant_config=quant_config),
247
+ prefix=f"{prefix}.layers",
248
+ )
249
+
250
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
251
252
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -255,20 +261,30 @@ class Qwen2Model(nn.Module):
261
intermediate_tensors: Optional[IntermediateTensors] = None,
262
inputs_embeds: Optional[torch.Tensor] = None,
263
) -> torch.Tensor:
258
- if inputs_embeds is not None:
259
- hidden_states = inputs_embeds
264
+ if get_pp_group().is_first_rank:
265
+ if inputs_embeds is not None:
266
+ hidden_states = inputs_embeds
267
+ else:
268
+ hidden_states = self.embed_tokens(input_ids)
269
+ residual = None
270
else:
261
- hidden_states = self.embed_tokens(input_ids)
262
- residual = None
263
- for i in range(len(self.layers)):
271
+ assert intermediate_tensors is not None
272
+ hidden_states = intermediate_tensors["hidden_states"]
273
+ residual = intermediate_tensors["residual"]
274
+ for i in range(self.start_layer, self.end_layer):
275
layer = self.layers[i]
276
hidden_states, residual = layer(
277
positions,
278
hidden_states,
268
- kv_caches[i],
279
+ kv_caches[i - self.start_layer],
280
attn_metadata,
281
residual,
282
)
283
+ if not get_pp_group().is_last_rank:
284
+ return IntermediateTensors({
285
+ "hidden_states": hidden_states,
286
+ "residual": residual
287
+ })
288
hidden_states, _ = self.norm(hidden_states, residual)
289
return hidden_states
290
@@ -351,6 +367,20 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
367
sampling_metadata)
368
return logits
369
370
+ def make_empty_intermediate_tensors(
371
+ self, batch_size: int, dtype: torch.dtype,
372
+ device: torch.device) -> IntermediateTensors:
373
+ return IntermediateTensors({
374
+ "hidden_states":
375
+ torch.zeros((batch_size, self.config.hidden_size),
376
+ dtype=dtype,
377
+ device=device),
378
+ "residual":
379
+ torch.zeros((batch_size, self.config.hidden_size),
380
+ dtype=dtype,
381
+ device=device),
382
+ })
383
+
384
def sample(
385
self,
386
logits: torch.Tensor,
@@ -381,6 +411,8 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
411
# Skip loading extra bias for GPTQ models.
412
if name.endswith(".bias") and name not in params_dict:
413
continue
414
+ if is_pp_missing_parameter(name, self):
415
+ continue
416
param = params_dict[name]
417
weight_loader = param.weight_loader
418
weight_loader(param, loaded_weight, shard_id)
@@ -393,7 +425,8 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
425
name = maybe_remap_kv_scale_name(name, params_dict)
426
if name is None:
427
continue
396
-
428
+ if is_pp_missing_parameter(name, self):
429
+ continue
430
param = params_dict[name]
431
weight_loader = getattr(param, "weight_loader",
432
default_weight_loader)
vllm/model_executor/models/qwen2_moe.py
+54
-15
@@ -31,7 +31,8 @@ from transformers import PretrainedConfig
31
32
from vllm.attention import Attention, AttentionMetadata
33
from vllm.config import CacheConfig
34
-from vllm.distributed import (get_tensor_model_parallel_world_size,
34
+from vllm.distributed import (get_pp_group,
35
+ get_tensor_model_parallel_world_size,
36
tensor_model_parallel_all_reduce)
37
from vllm.model_executor.layers.activation import SiluAndMul
38
from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -52,6 +53,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
53
from vllm.sequence import IntermediateTensors, SamplerOutput
54
from vllm.utils import print_warning_once
55
56
+from .utils import is_pp_missing_parameter, make_layers
57
+
58
59
class Qwen2MoeMLP(nn.Module):
60
@@ -315,6 +318,7 @@ class Qwen2MoeModel(nn.Module):
318
config: PretrainedConfig,
319
cache_config: Optional[CacheConfig] = None,
320
quant_config: Optional[QuantizationConfig] = None,
321
+ prefix: str = "",
322
) -> None:
323
super().__init__()
324
self.padding_idx = config.pad_token_id
@@ -324,13 +328,15 @@ class Qwen2MoeModel(nn.Module):
328
config.vocab_size,
329
config.hidden_size,
330
)
327
- self.layers = nn.ModuleList([
328
- Qwen2MoeDecoderLayer(config,
329
- layer_idx,
330
- cache_config,
331
- quant_config=quant_config)
332
- for layer_idx in range(config.num_hidden_layers)
333
- ])
331
+ self.start_layer, self.end_layer, self.layers = make_layers(
332
+ config.num_hidden_layers,
333
+ lambda prefix: Qwen2MoeDecoderLayer(config=config,
334
+ layer_idx=int(
335
+ prefix.split(".")[-1]),
336
+ cache_config=cache_config,
337
+ quant_config=quant_config),
338
+ prefix=f"{prefix}.layers",
339
+ )
340
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
341
342
def forward(
@@ -339,14 +345,25 @@ class Qwen2MoeModel(nn.Module):
345
positions: torch.Tensor,
346
kv_caches: List[torch.Tensor],
347
attn_metadata: AttentionMetadata,
348
+ intermediate_tensors: Optional[IntermediateTensors] = None,
349
) -> torch.Tensor:
343
- hidden_states = self.embed_tokens(input_ids)
344
- residual = None
345
- for i in range(len(self.layers)):
350
+ if get_pp_group().is_first_rank:
351
+ hidden_states = self.embed_tokens(input_ids)
352
+ residual = None
353
+ else:
354
+ assert intermediate_tensors is not None
355
+ hidden_states = intermediate_tensors["hidden_states"]
356
+ residual = intermediate_tensors["residual"]
357
+ for i in range(self.start_layer, self.end_layer):
358
layer = self.layers[i]
359
hidden_states, residual = layer(positions, hidden_states,
348
- kv_caches[i], attn_metadata,
349
- residual)
360
+ kv_caches[i - self.start_layer],
361
+ attn_metadata, residual)
362
+ if not get_pp_group().is_last_rank:
363
+ return IntermediateTensors({
364
+ "hidden_states": hidden_states,
365
+ "residual": residual
366
+ })
367
hidden_states, _ = self.norm(hidden_states, residual)
368
return hidden_states
369
@@ -380,7 +397,7 @@ class Qwen2MoeForCausalLM(nn.Module):
397
intermediate_tensors: Optional[IntermediateTensors] = None,
398
) -> torch.Tensor:
399
hidden_states = self.model(input_ids, positions, kv_caches,
383
- attn_metadata)
400
+ attn_metadata, intermediate_tensors)
401
return hidden_states
402
403
def compute_logits(self, hidden_states: torch.Tensor,
@@ -389,6 +406,20 @@ class Qwen2MoeForCausalLM(nn.Module):
406
sampling_metadata)
407
return logits
408
409
+ def make_empty_intermediate_tensors(
410
+ self, batch_size: int, dtype: torch.dtype,
411
+ device: torch.device) -> IntermediateTensors:
412
+ return IntermediateTensors({
413
+ "hidden_states":
414
+ torch.zeros((batch_size, self.config.hidden_size),
415
+ dtype=dtype,
416
+ device=device),
417
+ "residual":
418
+ torch.zeros((batch_size, self.config.hidden_size),
419
+ dtype=dtype,
420
+ device=device),
421
+ })
422
+
423
def sample(
424
self,
425
logits: Optional[torch.Tensor],
@@ -435,6 +466,9 @@ class Qwen2MoeForCausalLM(nn.Module):
466
# Skip loading extra bias for GPTQ models.
467
if name.endswith(".bias") and name not in params_dict:
468
continue
469
+ # Skip layers on other devices.
470
+ if is_pp_missing_parameter(name, self):
471
+ continue
472
if name not in params_dict:
473
continue
474
@@ -448,6 +482,9 @@ class Qwen2MoeForCausalLM(nn.Module):
482
if weight_name not in name:
483
continue
484
name = name.replace(weight_name, param_name)
485
+ # Skip layers on other devices.
486
+ if is_pp_missing_parameter(name, self):
487
+ continue
488
param = params_dict[name]
489
weight_loader = param.weight_loader
490
weight_loader(param,
@@ -460,6 +497,9 @@ class Qwen2MoeForCausalLM(nn.Module):
497
# Skip loading extra bias for GPTQ models.
498
if name.endswith(".bias") and name not in params_dict:
499
continue
500
+ # Skip layers on other devices.
501
+ if is_pp_missing_parameter(name, self):
502
+ continue
503
# Remapping the name of FP8 kv-scale.
504
if name.endswith("kv_scale"):
505
remapped_kv_scale_name = name.replace(
@@ -474,7 +514,6 @@ class Qwen2MoeForCausalLM(nn.Module):
514
continue
515
else:
516
name = remapped_kv_scale_name
477
-
517
param = params_dict[name]
518
weight_loader = getattr(param, "weight_loader",
519
default_weight_loader)