@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1d2e7fb73f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Model] Pipeline parallel support for Qwen2 (#6924)

xuyi committed Aug 1, 2024 at 09:49 UTC 1d2e7fb73f1205ae03e4ee3bcd3de566733bf582
3 files changed +101 -27
vllm/config.py
+2
@@ -40,6 +40,8 @@ _PP_SUPPORTED_MODELS = [
40 "GPT2LMHeadModel",
41 "MixtralForCausalLM",
42 "NemotronForCausalLM",
43 + "Qwen2ForCausalLM",
44 + "Qwen2MoeForCausalLM",
45 ]
46
47
vllm/model_executor/models/qwen2.py
+45 -12
@@ -30,7 +30,7 @@ from transformers import Qwen2Config
30
31 from vllm.attention import Attention, AttentionMetadata
32 from vllm.config import CacheConfig, LoRAConfig
33 -from vllm.distributed import get_tensor_model_parallel_world_size
33 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.layernorm import RMSNorm
36 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -49,6 +49,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
49 from vllm.sequence import IntermediateTensors, SamplerOutput
50
51 from .interfaces import SupportsLoRA
52 +from .utils import is_pp_missing_parameter, make_layers
53
54
55 class Qwen2MLP(nn.Module):
@@ -227,6 +228,7 @@ class Qwen2Model(nn.Module):
228 config: Qwen2Config,
229 cache_config: Optional[CacheConfig] = None,
230 quant_config: Optional[QuantizationConfig] = None,
231 + prefix: str = "",
232 ) -> None:
233 super().__init__()
234 self.config = config
@@ -237,10 +239,14 @@ class Qwen2Model(nn.Module):
239 config.vocab_size,
240 config.hidden_size,
241 )
240 - self.layers = nn.ModuleList([
241 - Qwen2DecoderLayer(config, cache_config, quant_config)
242 - for _ in range(config.num_hidden_layers)
243 - ])
242 + self.start_layer, self.end_layer, self.layers = make_layers(
243 + config.num_hidden_layers,
244 + lambda prefix: Qwen2DecoderLayer(config=config,
245 + cache_config=cache_config,
246 + quant_config=quant_config),
247 + prefix=f"{prefix}.layers",
248 + )
249 +
250 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
251
252 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -255,20 +261,30 @@ class Qwen2Model(nn.Module):
261 intermediate_tensors: Optional[IntermediateTensors] = None,
262 inputs_embeds: Optional[torch.Tensor] = None,
263 ) -> torch.Tensor:
258 - if inputs_embeds is not None:
259 - hidden_states = inputs_embeds
264 + if get_pp_group().is_first_rank:
265 + if inputs_embeds is not None:
266 + hidden_states = inputs_embeds
267 + else:
268 + hidden_states = self.embed_tokens(input_ids)
269 + residual = None
270 else:
261 - hidden_states = self.embed_tokens(input_ids)
262 - residual = None
263 - for i in range(len(self.layers)):
271 + assert intermediate_tensors is not None
272 + hidden_states = intermediate_tensors["hidden_states"]
273 + residual = intermediate_tensors["residual"]
274 + for i in range(self.start_layer, self.end_layer):
275 layer = self.layers[i]
276 hidden_states, residual = layer(
277 positions,
278 hidden_states,
268 - kv_caches[i],
279 + kv_caches[i - self.start_layer],
280 attn_metadata,
281 residual,
282 )
283 + if not get_pp_group().is_last_rank:
284 + return IntermediateTensors({
285 + "hidden_states": hidden_states,
286 + "residual": residual
287 + })
288 hidden_states, _ = self.norm(hidden_states, residual)
289 return hidden_states
290
@@ -351,6 +367,20 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
367 sampling_metadata)
368 return logits
369
370 + def make_empty_intermediate_tensors(
371 + self, batch_size: int, dtype: torch.dtype,
372 + device: torch.device) -> IntermediateTensors:
373 + return IntermediateTensors({
374 + "hidden_states":
375 + torch.zeros((batch_size, self.config.hidden_size),
376 + dtype=dtype,
377 + device=device),
378 + "residual":
379 + torch.zeros((batch_size, self.config.hidden_size),
380 + dtype=dtype,
381 + device=device),
382 + })
383 +
384 def sample(
385 self,
386 logits: torch.Tensor,
@@ -381,6 +411,8 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
411 # Skip loading extra bias for GPTQ models.
412 if name.endswith(".bias") and name not in params_dict:
413 continue
414 + if is_pp_missing_parameter(name, self):
415 + continue
416 param = params_dict[name]
417 weight_loader = param.weight_loader
418 weight_loader(param, loaded_weight, shard_id)
@@ -393,7 +425,8 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
425 name = maybe_remap_kv_scale_name(name, params_dict)
426 if name is None:
427 continue
396 -
428 + if is_pp_missing_parameter(name, self):
429 + continue
430 param = params_dict[name]
431 weight_loader = getattr(param, "weight_loader",
432 default_weight_loader)
vllm/model_executor/models/qwen2_moe.py
+54 -15
@@ -31,7 +31,8 @@ from transformers import PretrainedConfig
31
32 from vllm.attention import Attention, AttentionMetadata
33 from vllm.config import CacheConfig
34 -from vllm.distributed import (get_tensor_model_parallel_world_size,
34 +from vllm.distributed import (get_pp_group,
35 + get_tensor_model_parallel_world_size,
36 tensor_model_parallel_all_reduce)
37 from vllm.model_executor.layers.activation import SiluAndMul
38 from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -52,6 +53,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
53 from vllm.sequence import IntermediateTensors, SamplerOutput
54 from vllm.utils import print_warning_once
55
56 +from .utils import is_pp_missing_parameter, make_layers
57 +
58
59 class Qwen2MoeMLP(nn.Module):
60
@@ -315,6 +318,7 @@ class Qwen2MoeModel(nn.Module):
318 config: PretrainedConfig,
319 cache_config: Optional[CacheConfig] = None,
320 quant_config: Optional[QuantizationConfig] = None,
321 + prefix: str = "",
322 ) -> None:
323 super().__init__()
324 self.padding_idx = config.pad_token_id
@@ -324,13 +328,15 @@ class Qwen2MoeModel(nn.Module):
328 config.vocab_size,
329 config.hidden_size,
330 )
327 - self.layers = nn.ModuleList([
328 - Qwen2MoeDecoderLayer(config,
329 - layer_idx,
330 - cache_config,
331 - quant_config=quant_config)
332 - for layer_idx in range(config.num_hidden_layers)
333 - ])
331 + self.start_layer, self.end_layer, self.layers = make_layers(
332 + config.num_hidden_layers,
333 + lambda prefix: Qwen2MoeDecoderLayer(config=config,
334 + layer_idx=int(
335 + prefix.split(".")[-1]),
336 + cache_config=cache_config,
337 + quant_config=quant_config),
338 + prefix=f"{prefix}.layers",
339 + )
340 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
341
342 def forward(
@@ -339,14 +345,25 @@ class Qwen2MoeModel(nn.Module):
345 positions: torch.Tensor,
346 kv_caches: List[torch.Tensor],
347 attn_metadata: AttentionMetadata,
348 + intermediate_tensors: Optional[IntermediateTensors] = None,
349 ) -> torch.Tensor:
343 - hidden_states = self.embed_tokens(input_ids)
344 - residual = None
345 - for i in range(len(self.layers)):
350 + if get_pp_group().is_first_rank:
351 + hidden_states = self.embed_tokens(input_ids)
352 + residual = None
353 + else:
354 + assert intermediate_tensors is not None
355 + hidden_states = intermediate_tensors["hidden_states"]
356 + residual = intermediate_tensors["residual"]
357 + for i in range(self.start_layer, self.end_layer):
358 layer = self.layers[i]
359 hidden_states, residual = layer(positions, hidden_states,
348 - kv_caches[i], attn_metadata,
349 - residual)
360 + kv_caches[i - self.start_layer],
361 + attn_metadata, residual)
362 + if not get_pp_group().is_last_rank:
363 + return IntermediateTensors({
364 + "hidden_states": hidden_states,
365 + "residual": residual
366 + })
367 hidden_states, _ = self.norm(hidden_states, residual)
368 return hidden_states
369
@@ -380,7 +397,7 @@ class Qwen2MoeForCausalLM(nn.Module):
397 intermediate_tensors: Optional[IntermediateTensors] = None,
398 ) -> torch.Tensor:
399 hidden_states = self.model(input_ids, positions, kv_caches,
383 - attn_metadata)
400 + attn_metadata, intermediate_tensors)
401 return hidden_states
402
403 def compute_logits(self, hidden_states: torch.Tensor,
@@ -389,6 +406,20 @@ class Qwen2MoeForCausalLM(nn.Module):
406 sampling_metadata)
407 return logits
408
409 + def make_empty_intermediate_tensors(
410 + self, batch_size: int, dtype: torch.dtype,
411 + device: torch.device) -> IntermediateTensors:
412 + return IntermediateTensors({
413 + "hidden_states":
414 + torch.zeros((batch_size, self.config.hidden_size),
415 + dtype=dtype,
416 + device=device),
417 + "residual":
418 + torch.zeros((batch_size, self.config.hidden_size),
419 + dtype=dtype,
420 + device=device),
421 + })
422 +
423 def sample(
424 self,
425 logits: Optional[torch.Tensor],
@@ -435,6 +466,9 @@ class Qwen2MoeForCausalLM(nn.Module):
466 # Skip loading extra bias for GPTQ models.
467 if name.endswith(".bias") and name not in params_dict:
468 continue
469 + # Skip layers on other devices.
470 + if is_pp_missing_parameter(name, self):
471 + continue
472 if name not in params_dict:
473 continue
474
@@ -448,6 +482,9 @@ class Qwen2MoeForCausalLM(nn.Module):
482 if weight_name not in name:
483 continue
484 name = name.replace(weight_name, param_name)
485 + # Skip layers on other devices.
486 + if is_pp_missing_parameter(name, self):
487 + continue
488 param = params_dict[name]
489 weight_loader = param.weight_loader
490 weight_loader(param,
@@ -460,6 +497,9 @@ class Qwen2MoeForCausalLM(nn.Module):
497 # Skip loading extra bias for GPTQ models.
498 if name.endswith(".bias") and name not in params_dict:
499 continue
500 + # Skip layers on other devices.
501 + if is_pp_missing_parameter(name, self):
502 + continue
503 # Remapping the name of FP8 kv-scale.
504 if name.endswith("kv_scale"):
505 remapped_kv_scale_name = name.replace(
@@ -474,7 +514,6 @@ class Qwen2MoeForCausalLM(nn.Module):
514 continue
515 else:
516 name = remapped_kv_scale_name
477 -
517 param = params_dict[name]
518 weight_loader = getattr(param, "weight_loader",
519 default_weight_loader)