@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f4fc7337bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] support `tie_word_embeddings` for all models (#5724)

Zijian Hu committed Aug 19, 2024 at 20:00 UTC f4fc7337bfaf5f10b8da4ba547e4009179348a26
30 files changed +90 -16
vllm/model_executor/models/arctic.py
+2
@@ -414,6 +414,8 @@ class ArcticForCausalLM(nn.Module):
414 config.hidden_size,
415 quant_config=quant_config,
416 )
417 + if self.config.tie_word_embeddings:
418 + self.lm_head.weight = self.model.embed_tokens.weight
419 self.num_experts = config.num_local_experts
420 self.num_experts_per_tok = config.num_experts_per_tok
421 self.unpadded_vocab_size = config.vocab_size
vllm/model_executor/models/baichuan.py
+2
@@ -331,6 +331,8 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
331 self.lm_head = ParallelLMHead(config.vocab_size,
332 config.hidden_size,
333 quant_config=quant_config)
334 + if self.config.tie_word_embeddings:
335 + self.lm_head.weight = self.model.embed_tokens.weight
336 self.logits_processor = LogitsProcessor(config.vocab_size)
337 self.sampler = Sampler()
338
vllm/model_executor/models/bart.py
+2
@@ -821,6 +821,8 @@ class BartForConditionalGeneration(nn.Module):
821 lora_config: Optional[LoRAConfig] = None):
822
823 super().__init__()
824 + # currently all existing BART models have `tie_word_embeddings` enabled
825 + assert config.tie_word_embeddings
826 self.config = config
827 self.model = BartModel(config,
828 cache_config,
vllm/model_executor/models/blip2.py
+3
@@ -494,6 +494,9 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
494
495 super().__init__()
496
497 + # currently all existing BLIP-2 models have `tie_word_embeddings`
498 + # enabled
499 + assert config.tie_word_embeddings
500 self.config = config
501 self.multimodal_config = multimodal_config
502
vllm/model_executor/models/bloom.py
+7 -2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 - VocabParallelEmbedding)
39 + ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -276,7 +276,12 @@ class BloomForCausalLM(nn.Module):
276 self.config = config
277 self.quant_config = quant_config
278 self.transformer = BloomModel(config, cache_config, quant_config)
279 - self.lm_head = self.transformer.word_embeddings
279 + if self.config.tie_word_embeddings:
280 + self.lm_head = self.transformer.word_embeddings
281 + else:
282 + self.lm_head = ParallelLMHead(self.config.vocab_size,
283 + self.config.hidden_size)
284 +
285 self.logits_processor = LogitsProcessor(config.vocab_size)
286 self.sampler = Sampler()
287
vllm/model_executor/models/chatglm.py
+3
@@ -356,6 +356,9 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
356 self.max_position_embeddings = getattr(config, "max_sequence_length",
357 8192)
358 self.transformer = ChatGLMModel(config, cache_config, quant_config)
359 + if self.config.tie_word_embeddings:
360 + self.transformer.output_layer.weight = (
361 + self.transformer.embedding.weight)
362 self.lm_head = self.transformer.output_layer
363 self.logits_processor = LogitsProcessor(config.padded_vocab_size)
364 self.sampler = Sampler()
vllm/model_executor/models/commandr.py
+3
@@ -321,6 +321,9 @@ class CohereForCausalLM(nn.Module):
321 ) -> None:
322 super().__init__()
323 self.config = config
324 + # currently all existing command R models have `tie_word_embeddings`
325 + # enabled
326 + assert config.tie_word_embeddings
327 self.unpadded_vocab_size = config.vocab_size
328 if lora_config:
329 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/dbrx.py
+3
@@ -362,6 +362,9 @@ class DbrxForCausalLM(nn.Module):
362 ):
363 super().__init__()
364 self.config = config
365 + if config.tie_word_embeddings:
366 + raise ValueError(
367 + "tie_word_embeddings is not supported for Dbrx models.")
368 self.quant_config = quant_config
369 self.unpadded_vocab_size = config.vocab_size
370 self.transformer = DbrxModel(config, cache_config, quant_config)
vllm/model_executor/models/deepseek.py
+2
@@ -380,6 +380,8 @@ class DeepseekForCausalLM(nn.Module):
380 self.lm_head = ParallelLMHead(config.vocab_size,
381 config.hidden_size,
382 quant_config=quant_config)
383 + if self.config.tie_word_embeddings:
384 + self.lm_head.weight = self.model.embed_tokens.weight
385 self.logits_processor = LogitsProcessor(config.vocab_size)
386 self.sampler = Sampler()
387
vllm/model_executor/models/gemma.py
+2
@@ -331,6 +331,8 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
331 super().__init__()
332
333 self.config = config
334 + # currently all existing Gemma models have `tie_word_embeddings` enabled
335 + assert config.tie_word_embeddings
336 self.lora_config = lora_config
337
338 self.quant_config = quant_config
vllm/model_executor/models/gemma2.py
+2
@@ -323,6 +323,8 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
323 del lora_config # Unused.
324 super().__init__()
325 self.config = config
326 + # currently all existing Gemma models have `tie_word_embeddings` enabled
327 + assert config.tie_word_embeddings
328 self.quant_config = quant_config
329 self.model = Gemma2Model(config, cache_config, quant_config)
330 self.logits_processor = LogitsProcessor(
vllm/model_executor/models/gpt2.py
+6 -2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 - VocabParallelEmbedding)
39 + ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -249,7 +249,11 @@ class GPT2LMHeadModel(nn.Module):
249 cache_config,
250 quant_config,
251 prefix="transformer")
252 - self.lm_head = self.transformer.wte
252 + if self.config.tie_word_embeddings:
253 + self.lm_head = self.transformer.wte
254 + else:
255 + self.lm_head = ParallelLMHead(self.config.vocab_size,
256 + self.config.hidden_size)
257 self.logits_processor = LogitsProcessor(config.vocab_size)
258 self.sampler = Sampler()
259
vllm/model_executor/models/gpt_bigcode.py
+8 -2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 - VocabParallelEmbedding)
39 + ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -259,7 +259,13 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
259 self.quant_config = quant_config
260 self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
261 lora_config)
262 - self.lm_head = self.transformer.wte
262 + if self.config.tie_word_embeddings:
263 + self.lm_head = self.transformer.wte
264 + else:
265 + self.lm_head = ParallelLMHead(
266 + self.transformer.vocab_size,
267 + self.transformer.embed_dim,
268 + org_num_embeddings=self.config.vocab_size)
269 self.unpadded_vocab_size = config.vocab_size
270 if lora_config:
271 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/gpt_neox.py
+3 -1
@@ -230,7 +230,7 @@ class GPTNeoXForCausalLM(nn.Module):
230
231 def __init__(
232 self,
233 - config,
233 + config: GPTNeoXConfig,
234 cache_config: Optional[CacheConfig] = None,
235 quant_config: Optional[QuantizationConfig] = None,
236 ):
@@ -243,6 +243,8 @@ class GPTNeoXForCausalLM(nn.Module):
243 config.hidden_size,
244 quant_config=quant_config,
245 )
246 + if self.config.tie_word_embeddings:
247 + self.embed_out.weight = self.gpt_neox.embed_in.weight
248 self.logits_processor = LogitsProcessor(config.vocab_size)
249 self.sampler = Sampler()
250
vllm/model_executor/models/internlm2.py
+2
@@ -264,6 +264,8 @@ class InternLM2ForCausalLM(nn.Module):
264 self.output = ParallelLMHead(config.vocab_size,
265 config.hidden_size,
266 quant_config=quant_config)
267 + if self.config.tie_word_embeddings:
268 + self.output.weight = self.model.tok_embeddings.weight
269 self.logits_processor = LogitsProcessor(config.vocab_size)
270 self.sampler = Sampler()
271
vllm/model_executor/models/jais.py
+6 -2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.quantization.base_config import (
37 QuantizationConfig)
38 from vllm.model_executor.layers.sampler import Sampler
39 from vllm.model_executor.layers.vocab_parallel_embedding import (
40 - VocabParallelEmbedding)
40 + ParallelLMHead, VocabParallelEmbedding)
41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42 from vllm.model_executor.sampling_metadata import SamplingMetadata
43 from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -291,7 +291,11 @@ class JAISLMHeadModel(nn.Module):
291 self.config = config
292 self.quant_config = quant_config
293 self.transformer = JAISModel(config, cache_config, quant_config)
294 - self.lm_head = self.transformer.wte
294 + if self.config.tie_word_embeddings:
295 + self.lm_head = self.transformer.wte
296 + else:
297 + self.lm_head = ParallelLMHead(self.config.vocab_size,
298 + self.config.hidden_size)
299 if hasattr(config, "width_scale"):
300 self.output_logits_scale = config.width_scale
301 else:
vllm/model_executor/models/llava.py
+2 -2
@@ -313,7 +313,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
313 278, 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566, 29901]`.
314
315 To reserve space in KV cache, we have to insert placeholder tokens
316 - before they are inputted to the model, so the input processor prepends
316 + before they are inputted to the model, so the input processor prepends
317 additional image tokens (denoted as `32000`), resulting in:
318 `[1, 3148, 1001, 29901, 29871, 32000, ..., 32000, 29871, 13, 5618,
319 29915, 29879, 278, 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566,
@@ -331,7 +331,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
331 input_ids: Flattened (concatenated) input_ids corresponding to a
332 batch.
333 pixel_values: The pixels in each input image.
334 -
334 +
335 See also:
336 :class:`LlavaImageInputs`
337 """
vllm/model_executor/models/llava_next.py
+2 -2
@@ -545,7 +545,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
545 9047, 13566, 29901]`.
546
547 To reserve space in KV cache, we have to insert placeholder tokens
548 - before they are inputted to the model, so the input processor prepends
548 + before they are inputted to the model, so the input processor prepends
549 additional image tokens (denoted as `32000`), resulting in:
550 `[1, 319, 13563, 1546, 263, 12758, 5199, 322, 385, 23116, 21082, 20255,
551 29889, 450, 20255, 4076, 8444, 29892, 13173, 29892, 322, 1248, 568,
@@ -566,7 +566,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
566 batch.
567 pixel_values: The pixels in each grid patch for each input image.
568 image_sizes: The original `(height, width)` for each input image.
569 -
569 +
570 See also:
571 :class:`LlavaNextImageInputs`
572 """
vllm/model_executor/models/minicpmv.py
+4
@@ -496,6 +496,10 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
496 quant_config: Optional[QuantizationConfig] = None,
497 ):
498 super().__init__()
499 + # All MiniCPM-V models disable `tie_word_embeddings` but
500 + # `PretrainedConfig.tie_word_embeddings` defaults to True; we cannot
501 + # check `tie_word_embeddings` until vLLM integrate MiniCPM-V model
502 + # and config class
503 self.config = config
504 self.multimodal_config = multimodal_config
505
vllm/model_executor/models/mixtral.py
+2
@@ -359,6 +359,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
359 if not lora_config else lora_config.lora_vocab_padding_size,
360 quant_config=quant_config,
361 )
362 + if self.config.tie_word_embeddings:
363 + self.lm_head.weight = self.model.embed_tokens.weight
364 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
365 config.vocab_size)
366 self.sampler = Sampler()
vllm/model_executor/models/mixtral_quant.py
+2
@@ -347,6 +347,8 @@ class MixtralForCausalLM(nn.Module):
347 self.lm_head = ParallelLMHead(config.vocab_size,
348 config.hidden_size,
349 quant_config=quant_config)
350 + if self.config.tie_word_embeddings:
351 + self.lm_head.weight = self.model.embed_tokens.weight
352 self.logits_processor = LogitsProcessor(config.vocab_size)
353 self.sampler = Sampler()
354
vllm/model_executor/models/opt.py
+6 -2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 - VocabParallelEmbedding)
39 + ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -307,7 +307,11 @@ class OPTForCausalLM(nn.Module):
307 self.config = config
308 self.quant_config = quant_config
309 self.model = OPTModel(config, cache_config, quant_config)
310 - self.lm_head = self.model.decoder.embed_tokens
310 + if self.config.tie_word_embeddings:
311 + self.lm_head = self.model.decoder.embed_tokens
312 + else:
313 + self.lm_head = ParallelLMHead(config.vocab_size,
314 + config.word_embed_proj_dim)
315 self.logits_processor = LogitsProcessor(config.vocab_size)
316 self.sampler = Sampler()
317
vllm/model_executor/models/orion.py
+2
@@ -262,6 +262,8 @@ class OrionForCausalLM(nn.Module):
262 self.lm_head = ParallelLMHead(config.vocab_size,
263 config.hidden_size,
264 quant_config=quant_config)
265 + if self.config.tie_word_embeddings:
266 + self.lm_head.weight = self.model.embed_tokens.weight
267 self.logits_processor = LogitsProcessor(config.vocab_size)
268 self.sampler = Sampler()
269
vllm/model_executor/models/phi.py
+2
@@ -260,6 +260,8 @@ class PhiForCausalLM(nn.Module, SupportsLoRA):
260 super().__init__()
261
262 self.config = config
263 + # lm_head use bias, cannot share word embeddings
264 + assert not config.tie_word_embeddings
265 self.lora_config = lora_config
266
267 self.quant_config = quant_config
vllm/model_executor/models/phi3_small.py
+2 -1
@@ -368,6 +368,8 @@ class Phi3SmallForCausalLM(nn.Module):
368 padding_size=DEFAULT_VOCAB_PADDING_SIZE,
369 quant_config=quant_config,
370 )
371 + if self.config.tie_word_embeddings:
372 + self.lm_head.weight = self.model.embed_tokens.weight
373 self.logits_processor = LogitsProcessor(config.vocab_size)
374 self.sampler = Sampler()
375
@@ -449,4 +451,3 @@ class Phi3SmallForCausalLM(nn.Module):
451 weight_loader = getattr(param, "weight_loader",
452 default_weight_loader)
453 weight_loader(param, loaded_weight)
452 - self.lm_head.weight.data.copy_(self.model.embed_tokens.weight.data)
vllm/model_executor/models/phi3v.py
+2
@@ -477,6 +477,8 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
477 self.lm_head = ParallelLMHead(config.vocab_size,
478 config.hidden_size,
479 quant_config=quant_config)
480 + if self.config.tie_word_embeddings:
481 + self.lm_head.weight = self.model.embed_tokens.weight
482 self.logits_processor = LogitsProcessor(config.vocab_size)
483 self.sampler = Sampler()
484
vllm/model_executor/models/qwen.py
+2
@@ -252,6 +252,8 @@ class QWenLMHeadModel(nn.Module):
252 self.lm_head = ParallelLMHead(config.vocab_size,
253 config.hidden_size,
254 quant_config=quant_config)
255 + if self.config.tie_word_embeddings:
256 + self.lm_head.weight = self.transformer.wte.weight
257 self.logits_processor = LogitsProcessor(config.vocab_size)
258 self.sampler = Sampler()
259
vllm/model_executor/models/qwen2_moe.py
+2
@@ -385,6 +385,8 @@ class Qwen2MoeForCausalLM(nn.Module):
385 self.lm_head = ParallelLMHead(config.vocab_size,
386 config.hidden_size,
387 quant_config=quant_config)
388 + if self.config.tie_word_embeddings:
389 + self.lm_head.weight = self.model.embed_tokens.weight
390 self.logits_processor = LogitsProcessor(config.vocab_size)
391 self.sampler = Sampler()
392
vllm/model_executor/models/stablelm.py
+2
@@ -243,6 +243,8 @@ class StablelmForCausalLM(nn.Module):
243 self.lm_head = ParallelLMHead(config.vocab_size,
244 config.hidden_size,
245 quant_config=quant_config)
246 + if self.config.tie_word_embeddings:
247 + self.lm_head.weight = self.model.embed_tokens.weight
248 self.logits_processor = LogitsProcessor(config.vocab_size)
249 self.sampler = Sampler()
250
vllm/model_executor/models/xverse.py
+2
@@ -313,6 +313,8 @@ class XverseForCausalLM(nn.Module, SupportsLoRA):
313 self.lm_head = ParallelLMHead(config.vocab_size,
314 config.hidden_size,
315 quant_config=quant_config)
316 + if self.config.tie_word_embeddings:
317 + self.lm_head.weight = self.model.embed_tokens.weight
318 self.logits_processor = LogitsProcessor(config.vocab_size)
319 self.sampler = Sampler()
320