A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] support `tie_word_embeddings` for all models (#5724)
Zijian Hu committed
Aug 19, 2024 at 20:00 UTC
f4fc7337bfaf5f10b8da4ba547e4009179348a26
30 files changed
+90
-16
vllm/model_executor/models/arctic.py
+2
@@ -414,6 +414,8 @@ class ArcticForCausalLM(nn.Module):
414
config.hidden_size,
415
quant_config=quant_config,
416
)
417
+ if self.config.tie_word_embeddings:
418
+ self.lm_head.weight = self.model.embed_tokens.weight
419
self.num_experts = config.num_local_experts
420
self.num_experts_per_tok = config.num_experts_per_tok
421
self.unpadded_vocab_size = config.vocab_size
vllm/model_executor/models/baichuan.py
+2
@@ -331,6 +331,8 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
331
self.lm_head = ParallelLMHead(config.vocab_size,
332
config.hidden_size,
333
quant_config=quant_config)
334
+ if self.config.tie_word_embeddings:
335
+ self.lm_head.weight = self.model.embed_tokens.weight
336
self.logits_processor = LogitsProcessor(config.vocab_size)
337
self.sampler = Sampler()
338
vllm/model_executor/models/bart.py
+2
@@ -821,6 +821,8 @@ class BartForConditionalGeneration(nn.Module):
821
lora_config: Optional[LoRAConfig] = None):
822
823
super().__init__()
824
+ # currently all existing BART models have `tie_word_embeddings` enabled
825
+ assert config.tie_word_embeddings
826
self.config = config
827
self.model = BartModel(config,
828
cache_config,
vllm/model_executor/models/blip2.py
+3
@@ -494,6 +494,9 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
494
495
super().__init__()
496
497
+ # currently all existing BLIP-2 models have `tie_word_embeddings`
498
+ # enabled
499
+ assert config.tie_word_embeddings
500
self.config = config
501
self.multimodal_config = multimodal_config
502
vllm/model_executor/models/bloom.py
+7
-2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
- VocabParallelEmbedding)
39
+ ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -276,7 +276,12 @@ class BloomForCausalLM(nn.Module):
276
self.config = config
277
self.quant_config = quant_config
278
self.transformer = BloomModel(config, cache_config, quant_config)
279
- self.lm_head = self.transformer.word_embeddings
279
+ if self.config.tie_word_embeddings:
280
+ self.lm_head = self.transformer.word_embeddings
281
+ else:
282
+ self.lm_head = ParallelLMHead(self.config.vocab_size,
283
+ self.config.hidden_size)
284
+
285
self.logits_processor = LogitsProcessor(config.vocab_size)
286
self.sampler = Sampler()
287
vllm/model_executor/models/chatglm.py
+3
@@ -356,6 +356,9 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
356
self.max_position_embeddings = getattr(config, "max_sequence_length",
357
8192)
358
self.transformer = ChatGLMModel(config, cache_config, quant_config)
359
+ if self.config.tie_word_embeddings:
360
+ self.transformer.output_layer.weight = (
361
+ self.transformer.embedding.weight)
362
self.lm_head = self.transformer.output_layer
363
self.logits_processor = LogitsProcessor(config.padded_vocab_size)
364
self.sampler = Sampler()
vllm/model_executor/models/commandr.py
+3
@@ -321,6 +321,9 @@ class CohereForCausalLM(nn.Module):
321
) -> None:
322
super().__init__()
323
self.config = config
324
+ # currently all existing command R models have `tie_word_embeddings`
325
+ # enabled
326
+ assert config.tie_word_embeddings
327
self.unpadded_vocab_size = config.vocab_size
328
if lora_config:
329
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/dbrx.py
+3
@@ -362,6 +362,9 @@ class DbrxForCausalLM(nn.Module):
362
):
363
super().__init__()
364
self.config = config
365
+ if config.tie_word_embeddings:
366
+ raise ValueError(
367
+ "tie_word_embeddings is not supported for Dbrx models.")
368
self.quant_config = quant_config
369
self.unpadded_vocab_size = config.vocab_size
370
self.transformer = DbrxModel(config, cache_config, quant_config)
vllm/model_executor/models/deepseek.py
+2
@@ -380,6 +380,8 @@ class DeepseekForCausalLM(nn.Module):
380
self.lm_head = ParallelLMHead(config.vocab_size,
381
config.hidden_size,
382
quant_config=quant_config)
383
+ if self.config.tie_word_embeddings:
384
+ self.lm_head.weight = self.model.embed_tokens.weight
385
self.logits_processor = LogitsProcessor(config.vocab_size)
386
self.sampler = Sampler()
387
vllm/model_executor/models/gemma.py
+2
@@ -331,6 +331,8 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
331
super().__init__()
332
333
self.config = config
334
+ # currently all existing Gemma models have `tie_word_embeddings` enabled
335
+ assert config.tie_word_embeddings
336
self.lora_config = lora_config
337
338
self.quant_config = quant_config
vllm/model_executor/models/gemma2.py
+2
@@ -323,6 +323,8 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
323
del lora_config # Unused.
324
super().__init__()
325
self.config = config
326
+ # currently all existing Gemma models have `tie_word_embeddings` enabled
327
+ assert config.tie_word_embeddings
328
self.quant_config = quant_config
329
self.model = Gemma2Model(config, cache_config, quant_config)
330
self.logits_processor = LogitsProcessor(
vllm/model_executor/models/gpt2.py
+6
-2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
- VocabParallelEmbedding)
39
+ ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -249,7 +249,11 @@ class GPT2LMHeadModel(nn.Module):
249
cache_config,
250
quant_config,
251
prefix="transformer")
252
- self.lm_head = self.transformer.wte
252
+ if self.config.tie_word_embeddings:
253
+ self.lm_head = self.transformer.wte
254
+ else:
255
+ self.lm_head = ParallelLMHead(self.config.vocab_size,
256
+ self.config.hidden_size)
257
self.logits_processor = LogitsProcessor(config.vocab_size)
258
self.sampler = Sampler()
259
vllm/model_executor/models/gpt_bigcode.py
+8
-2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
- VocabParallelEmbedding)
39
+ ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -259,7 +259,13 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
259
self.quant_config = quant_config
260
self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
261
lora_config)
262
- self.lm_head = self.transformer.wte
262
+ if self.config.tie_word_embeddings:
263
+ self.lm_head = self.transformer.wte
264
+ else:
265
+ self.lm_head = ParallelLMHead(
266
+ self.transformer.vocab_size,
267
+ self.transformer.embed_dim,
268
+ org_num_embeddings=self.config.vocab_size)
269
self.unpadded_vocab_size = config.vocab_size
270
if lora_config:
271
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/gpt_neox.py
+3
-1
@@ -230,7 +230,7 @@ class GPTNeoXForCausalLM(nn.Module):
230
231
def __init__(
232
self,
233
- config,
233
+ config: GPTNeoXConfig,
234
cache_config: Optional[CacheConfig] = None,
235
quant_config: Optional[QuantizationConfig] = None,
236
):
@@ -243,6 +243,8 @@ class GPTNeoXForCausalLM(nn.Module):
243
config.hidden_size,
244
quant_config=quant_config,
245
)
246
+ if self.config.tie_word_embeddings:
247
+ self.embed_out.weight = self.gpt_neox.embed_in.weight
248
self.logits_processor = LogitsProcessor(config.vocab_size)
249
self.sampler = Sampler()
250
vllm/model_executor/models/internlm2.py
+2
@@ -264,6 +264,8 @@ class InternLM2ForCausalLM(nn.Module):
264
self.output = ParallelLMHead(config.vocab_size,
265
config.hidden_size,
266
quant_config=quant_config)
267
+ if self.config.tie_word_embeddings:
268
+ self.output.weight = self.model.tok_embeddings.weight
269
self.logits_processor = LogitsProcessor(config.vocab_size)
270
self.sampler = Sampler()
271
vllm/model_executor/models/jais.py
+6
-2
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.quantization.base_config import (
37
QuantizationConfig)
38
from vllm.model_executor.layers.sampler import Sampler
39
from vllm.model_executor.layers.vocab_parallel_embedding import (
40
- VocabParallelEmbedding)
40
+ ParallelLMHead, VocabParallelEmbedding)
41
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42
from vllm.model_executor.sampling_metadata import SamplingMetadata
43
from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -291,7 +291,11 @@ class JAISLMHeadModel(nn.Module):
291
self.config = config
292
self.quant_config = quant_config
293
self.transformer = JAISModel(config, cache_config, quant_config)
294
- self.lm_head = self.transformer.wte
294
+ if self.config.tie_word_embeddings:
295
+ self.lm_head = self.transformer.wte
296
+ else:
297
+ self.lm_head = ParallelLMHead(self.config.vocab_size,
298
+ self.config.hidden_size)
299
if hasattr(config, "width_scale"):
300
self.output_logits_scale = config.width_scale
301
else:
vllm/model_executor/models/llava.py
+2
-2
@@ -313,7 +313,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
313
278, 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566, 29901]`.
314
315
To reserve space in KV cache, we have to insert placeholder tokens
316
- before they are inputted to the model, so the input processor prepends
316
+ before they are inputted to the model, so the input processor prepends
317
additional image tokens (denoted as `32000`), resulting in:
318
`[1, 3148, 1001, 29901, 29871, 32000, ..., 32000, 29871, 13, 5618,
319
29915, 29879, 278, 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566,
@@ -331,7 +331,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal):
331
input_ids: Flattened (concatenated) input_ids corresponding to a
332
batch.
333
pixel_values: The pixels in each input image.
334
-
334
+
335
See also:
336
:class:`LlavaImageInputs`
337
"""
vllm/model_executor/models/llava_next.py
+2
-2
@@ -545,7 +545,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
545
9047, 13566, 29901]`.
546
547
To reserve space in KV cache, we have to insert placeholder tokens
548
- before they are inputted to the model, so the input processor prepends
548
+ before they are inputted to the model, so the input processor prepends
549
additional image tokens (denoted as `32000`), resulting in:
550
`[1, 319, 13563, 1546, 263, 12758, 5199, 322, 385, 23116, 21082, 20255,
551
29889, 450, 20255, 4076, 8444, 29892, 13173, 29892, 322, 1248, 568,
@@ -566,7 +566,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
566
batch.
567
pixel_values: The pixels in each grid patch for each input image.
568
image_sizes: The original `(height, width)` for each input image.
569
-
569
+
570
See also:
571
:class:`LlavaNextImageInputs`
572
"""
vllm/model_executor/models/minicpmv.py
+4
@@ -496,6 +496,10 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
496
quant_config: Optional[QuantizationConfig] = None,
497
):
498
super().__init__()
499
+ # All MiniCPM-V models disable `tie_word_embeddings` but
500
+ # `PretrainedConfig.tie_word_embeddings` defaults to True; we cannot
501
+ # check `tie_word_embeddings` until vLLM integrate MiniCPM-V model
502
+ # and config class
503
self.config = config
504
self.multimodal_config = multimodal_config
505
vllm/model_executor/models/mixtral.py
+2
@@ -359,6 +359,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
359
if not lora_config else lora_config.lora_vocab_padding_size,
360
quant_config=quant_config,
361
)
362
+ if self.config.tie_word_embeddings:
363
+ self.lm_head.weight = self.model.embed_tokens.weight
364
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
365
config.vocab_size)
366
self.sampler = Sampler()
vllm/model_executor/models/mixtral_quant.py
+2
@@ -347,6 +347,8 @@ class MixtralForCausalLM(nn.Module):
347
self.lm_head = ParallelLMHead(config.vocab_size,
348
config.hidden_size,
349
quant_config=quant_config)
350
+ if self.config.tie_word_embeddings:
351
+ self.lm_head.weight = self.model.embed_tokens.weight
352
self.logits_processor = LogitsProcessor(config.vocab_size)
353
self.sampler = Sampler()
354
vllm/model_executor/models/opt.py
+6
-2
@@ -36,7 +36,7 @@ from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
- VocabParallelEmbedding)
39
+ ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -307,7 +307,11 @@ class OPTForCausalLM(nn.Module):
307
self.config = config
308
self.quant_config = quant_config
309
self.model = OPTModel(config, cache_config, quant_config)
310
- self.lm_head = self.model.decoder.embed_tokens
310
+ if self.config.tie_word_embeddings:
311
+ self.lm_head = self.model.decoder.embed_tokens
312
+ else:
313
+ self.lm_head = ParallelLMHead(config.vocab_size,
314
+ config.word_embed_proj_dim)
315
self.logits_processor = LogitsProcessor(config.vocab_size)
316
self.sampler = Sampler()
317
vllm/model_executor/models/orion.py
+2
@@ -262,6 +262,8 @@ class OrionForCausalLM(nn.Module):
262
self.lm_head = ParallelLMHead(config.vocab_size,
263
config.hidden_size,
264
quant_config=quant_config)
265
+ if self.config.tie_word_embeddings:
266
+ self.lm_head.weight = self.model.embed_tokens.weight
267
self.logits_processor = LogitsProcessor(config.vocab_size)
268
self.sampler = Sampler()
269
vllm/model_executor/models/phi.py
+2
@@ -260,6 +260,8 @@ class PhiForCausalLM(nn.Module, SupportsLoRA):
260
super().__init__()
261
262
self.config = config
263
+ # lm_head use bias, cannot share word embeddings
264
+ assert not config.tie_word_embeddings
265
self.lora_config = lora_config
266
267
self.quant_config = quant_config
vllm/model_executor/models/phi3_small.py
+2
-1
@@ -368,6 +368,8 @@ class Phi3SmallForCausalLM(nn.Module):
368
padding_size=DEFAULT_VOCAB_PADDING_SIZE,
369
quant_config=quant_config,
370
)
371
+ if self.config.tie_word_embeddings:
372
+ self.lm_head.weight = self.model.embed_tokens.weight
373
self.logits_processor = LogitsProcessor(config.vocab_size)
374
self.sampler = Sampler()
375
@@ -449,4 +451,3 @@ class Phi3SmallForCausalLM(nn.Module):
451
weight_loader = getattr(param, "weight_loader",
452
default_weight_loader)
453
weight_loader(param, loaded_weight)
452
- self.lm_head.weight.data.copy_(self.model.embed_tokens.weight.data)
vllm/model_executor/models/phi3v.py
+2
@@ -477,6 +477,8 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
477
self.lm_head = ParallelLMHead(config.vocab_size,
478
config.hidden_size,
479
quant_config=quant_config)
480
+ if self.config.tie_word_embeddings:
481
+ self.lm_head.weight = self.model.embed_tokens.weight
482
self.logits_processor = LogitsProcessor(config.vocab_size)
483
self.sampler = Sampler()
484
vllm/model_executor/models/qwen.py
+2
@@ -252,6 +252,8 @@ class QWenLMHeadModel(nn.Module):
252
self.lm_head = ParallelLMHead(config.vocab_size,
253
config.hidden_size,
254
quant_config=quant_config)
255
+ if self.config.tie_word_embeddings:
256
+ self.lm_head.weight = self.transformer.wte.weight
257
self.logits_processor = LogitsProcessor(config.vocab_size)
258
self.sampler = Sampler()
259
vllm/model_executor/models/qwen2_moe.py
+2
@@ -385,6 +385,8 @@ class Qwen2MoeForCausalLM(nn.Module):
385
self.lm_head = ParallelLMHead(config.vocab_size,
386
config.hidden_size,
387
quant_config=quant_config)
388
+ if self.config.tie_word_embeddings:
389
+ self.lm_head.weight = self.model.embed_tokens.weight
390
self.logits_processor = LogitsProcessor(config.vocab_size)
391
self.sampler = Sampler()
392
vllm/model_executor/models/stablelm.py
+2
@@ -243,6 +243,8 @@ class StablelmForCausalLM(nn.Module):
243
self.lm_head = ParallelLMHead(config.vocab_size,
244
config.hidden_size,
245
quant_config=quant_config)
246
+ if self.config.tie_word_embeddings:
247
+ self.lm_head.weight = self.model.embed_tokens.weight
248
self.logits_processor = LogitsProcessor(config.vocab_size)
249
self.sampler = Sampler()
250
vllm/model_executor/models/xverse.py
+2
@@ -313,6 +313,8 @@ class XverseForCausalLM(nn.Module, SupportsLoRA):
313
self.lm_head = ParallelLMHead(config.vocab_size,
314
config.hidden_size,
315
quant_config=quant_config)
316
+ if self.config.tie_word_embeddings:
317
+ self.lm_head.weight = self.model.embed_tokens.weight
318
self.logits_processor = LogitsProcessor(config.vocab_size)
319
self.sampler = Sampler()
320