@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1d094fd7c0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Distributed][PP] only create embedding & lm head when necessary (#6455)

original title: [Distributed][Model] Rank-based Component Creation for Pipeline Parallelism Memory Optimization

Wushi Dong committed Jul 16, 2024 at 19:20 UTC 1d094fd7c0d72e7eaf3a2b65fff9f7de8d812a49
1 file changed +38 -27
vllm/model_executor/models/llama.py
+38 -27
@@ -50,7 +50,7 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
50 from vllm.utils import is_hip
51
52 from .interfaces import SupportsLoRA
53 -from .utils import is_pp_missing_parameter, make_layers
53 +from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
54
55
56 class LlamaMLP(nn.Module):
@@ -257,17 +257,24 @@ class LlamaModel(nn.Module):
257 (lora_config.max_loras or 1)) if lora_config else 0
258 self.vocab_size = config.vocab_size + lora_vocab
259 self.org_vocab_size = config.vocab_size
260 - self.embed_tokens = VocabParallelEmbedding(
261 - self.vocab_size,
262 - config.hidden_size,
263 - org_num_embeddings=config.vocab_size,
264 - )
260 + if get_pp_group().is_first_rank or (config.tie_word_embeddings
261 + and get_pp_group().is_last_rank):
262 + self.embed_tokens = VocabParallelEmbedding(
263 + self.vocab_size,
264 + config.hidden_size,
265 + org_num_embeddings=config.vocab_size,
266 + )
267 + else:
268 + self.embed_tokens = PPMissingLayer()
269 self.start_layer, self.end_layer, self.layers = make_layers(
270 config.num_hidden_layers,
271 lambda: LlamaDecoderLayer(config=config,
272 cache_config=cache_config,
273 quant_config=quant_config))
270 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
274 + if get_pp_group().is_last_rank:
275 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
276 + else:
277 + self.norm = PPMissingLayer()
278
279 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
280 return self.embed_tokens(input_ids)
@@ -360,26 +367,30 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
367 cache_config,
368 quant_config,
369 lora_config=lora_config)
363 - self.unpadded_vocab_size = config.vocab_size
364 - if lora_config:
365 - self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
366 - self.lm_head = ParallelLMHead(
367 - self.unpadded_vocab_size,
368 - config.hidden_size,
369 - org_num_embeddings=config.vocab_size,
370 - padding_size=DEFAULT_VOCAB_PADDING_SIZE
371 - # We need bigger padding if using lora for kernel
372 - # compatibility
373 - if not lora_config else lora_config.lora_vocab_padding_size,
374 - quant_config=quant_config,
375 - )
376 - if config.tie_word_embeddings:
377 - self.lm_head.weight = self.model.embed_tokens.weight
378 -
379 - logit_scale = getattr(config, "logit_scale", 1.0)
380 - self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
381 - config.vocab_size, logit_scale)
382 - self.sampler = Sampler()
370 + if get_pp_group().is_last_rank:
371 + self.unpadded_vocab_size = config.vocab_size
372 + if lora_config:
373 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
374 + self.lm_head = ParallelLMHead(
375 + self.unpadded_vocab_size,
376 + config.hidden_size,
377 + org_num_embeddings=config.vocab_size,
378 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
379 + # We need bigger padding if using lora for kernel
380 + # compatibility
381 + if not lora_config else lora_config.lora_vocab_padding_size,
382 + quant_config=quant_config,
383 + )
384 + if config.tie_word_embeddings:
385 + self.lm_head.weight = self.model.embed_tokens.weight
386 +
387 + logit_scale = getattr(config, "logit_scale", 1.0)
388 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
389 + config.vocab_size,
390 + logit_scale)
391 + self.sampler = Sampler()
392 + else:
393 + self.lm_head = PPMissingLayer()
394
395 def forward(
396 self,