A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[CORE] Quantized lm-head Framework (#4442)
Co-authored-by: Robert Shaw <rshaw@neuralmagic.com> Co-authored-by: ZX <zx@lbx.dev>
Qubitium-ModelCloud committed
Jul 3, 2024 at 06:25 UTC
ee93f4f92acbd9759a9af80747bc2a4459f07639
48 files changed
+268
-121
tests/lora/test_layers.py
+5
-5
@@ -475,10 +475,10 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
475
476
lora_result = lora_logits_processor._get_logits(
477
hidden_states=torch.cat(inputs),
478
- embedding=linear.weight,
478
+ lm_head=linear,
479
embedding_bias=None)
480
481
- original_weight = linear.weight.clone()
481
+ original_lm_head = deepcopy(linear)
482
483
linear.weight[logits_processor.
484
org_vocab_size:logits_processor.org_vocab_size +
@@ -490,7 +490,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
490
for input_, lora_id in zip(inputs, prompt_mapping):
491
lora = lora_dict[lora_id]
492
result = logits_processor._get_logits(hidden_states=input_,
493
- embedding=linear.weight,
493
+ lm_head=linear,
494
embedding_bias=None)
495
result[:, vocab_size + embeddings_tensor_len:] = float("-inf")
496
result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
@@ -519,11 +519,11 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
519
520
lora_result = lora_logits_processor._get_logits(
521
hidden_states=torch.cat(inputs),
522
- embedding=original_weight,
522
+ lm_head=original_lm_head,
523
embedding_bias=None)[:, :vocab_size]
524
expected_result = logits_processor._get_logits(
525
hidden_states=torch.cat(inputs),
526
- embedding=original_weight,
526
+ lm_head=original_lm_head,
527
embedding_bias=None)
528
529
rtol, atol = TOLERANCES[lora_result.dtype]
tests/quantization/test_lm_head.py
new
+45
@@ -0,0 +1,45 @@
1
+"""Tests whether gptq models with quantized lm_head can be loaded.
2
+
3
+Run `pytest tests/quantization/test_quant_lm_head_true.py --forked`.
4
+"""
5
+from typing import Tuple
6
+
7
+import pytest
8
+import torch
9
+
10
+from vllm.model_executor.layers.linear import UnquantizedLinearMethod
11
+from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
12
+from vllm.model_executor.layers.quantization.gptq_marlin import (
13
+ GPTQMarlinLinearMethod)
14
+from vllm.model_executor.layers.quantization.marlin import MarlinLinearMethod
15
+
16
+PROMPT = "On the surface of Mars, we found"
17
+
18
+MODELS_QUANT = [(
19
+ "LnL-AI/TinyLlama-1.1B-intermediate-step-1341k-3T-autoround-lm_head-symFalse",
20
+ True), ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", False),
21
+ ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", False)]
22
+
23
+
24
+@pytest.mark.parametrize("model_lm_head_quant", MODELS_QUANT)
25
+def test_lm_head(
26
+ vllm_runner,
27
+ model_lm_head_quant: Tuple[str, bool],
28
+) -> None:
29
+ model, lm_head_quantized = model_lm_head_quant
30
+ vllm_model = vllm_runner(model, dtype=torch.float16, max_model_len=2048)
31
+
32
+ lm_head_layer = (vllm_model.model.llm_engine.model_executor.driver_worker.
33
+ model_runner.model.lm_head)
34
+
35
+ if lm_head_quantized:
36
+ assert isinstance(
37
+ lm_head_layer.linear_method,
38
+ (GPTQLinearMethod, GPTQMarlinLinearMethod, MarlinLinearMethod))
39
+ else:
40
+ assert isinstance(lm_head_layer.linear_method, UnquantizedLinearMethod)
41
+
42
+ print(
43
+ vllm_model.generate_greedy(prompts=["Hello my name is"],
44
+ max_tokens=10)[0][1])
45
+ del vllm_model
tests/spec_decode/e2e/test_mlp_correctness.py
+1
-1
@@ -34,7 +34,7 @@ SPEC_MODEL = "ibm-granite/granite-3b-code-instruct-accelerator"
34
MAX_SPEC_TOKENS = 5
35
36
# precision
37
-PRECISION = "float16"
37
+PRECISION = "float32"
38
39
40
@pytest.mark.parametrize(
tests/test_logits_processor.py
+1
-1
@@ -83,7 +83,7 @@ def test_logits_processors(seed: int, device: str):
83
device=device,
84
pin_memory=is_pin_memory_available())
85
logits_processor_output = logits_processor(
86
- embedding=None,
86
+ lm_head=None,
87
hidden_states=input_tensor,
88
sampling_metadata=sampling_metadata)
89
vllm/lora/layers.py
+2
-2
@@ -1172,11 +1172,11 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1172
def _get_logits(
1173
self,
1174
hidden_states: torch.Tensor,
1175
- embedding: torch.Tensor,
1175
+ lm_head: VocabParallelEmbedding,
1176
embedding_bias: Optional[torch.Tensor] = None,
1177
) -> Optional[torch.Tensor]:
1178
# Get the logits for the next tokens.
1179
- logits = torch.matmul(hidden_states, embedding.t())
1179
+ logits = lm_head.linear_method.apply(lm_head, hidden_states)
1180
if embedding_bias is not None:
1181
logits += embedding_bias
1182
logits = tensor_model_parallel_gather(logits)
vllm/model_executor/layers/logits_processor.py
+9
-7
@@ -6,6 +6,8 @@ import torch
6
import torch.nn as nn
7
8
from vllm.distributed import tensor_model_parallel_gather
9
+from vllm.model_executor.layers.vocab_parallel_embedding import (
10
+ VocabParallelEmbedding)
11
from vllm.model_executor.sampling_metadata import SamplingMetadata
12
13
@@ -40,7 +42,7 @@ class LogitsProcessor(nn.Module):
42
43
def forward(
44
self,
43
- embedding: torch.Tensor,
45
+ lm_head: VocabParallelEmbedding,
46
hidden_states: torch.Tensor,
47
sampling_metadata: SamplingMetadata,
48
embedding_bias: Optional[torch.Tensor] = None,
@@ -52,8 +54,7 @@ class LogitsProcessor(nn.Module):
54
sampling_metadata)
55
56
# Get the logits for the next tokens.
55
- logits = self._get_logits(hidden_states, embedding, embedding_bias)
56
-
57
+ logits = self._get_logits(hidden_states, lm_head, embedding_bias)
58
if logits is not None:
59
if self.soft_cap is not None:
60
logits = logits / self.soft_cap
@@ -68,12 +69,13 @@ class LogitsProcessor(nn.Module):
69
70
return logits
71
71
- def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor,
72
+ def _get_logits(self, hidden_states: torch.Tensor,
73
+ lm_head: VocabParallelEmbedding,
74
embedding_bias: Optional[torch.Tensor]) -> torch.Tensor:
75
# Get the logits for the next tokens.
74
- logits = torch.matmul(hidden_states, embedding.t())
75
- if embedding_bias is not None:
76
- logits += embedding_bias
76
+ logits = lm_head.linear_method.apply(lm_head,
77
+ hidden_states,
78
+ bias=embedding_bias)
79
logits = tensor_model_parallel_gather(logits)
80
# Remove paddings in vocab (if any).
81
if logits is not None:
vllm/model_executor/layers/quantization/base_config.py
+9
@@ -87,6 +87,15 @@ class QuantizationConfig(ABC):
87
raise ValueError(f"Cannot find any of {keys} in the model's "
88
"quantization config.")
89
90
+ @staticmethod
91
+ def get_from_keys_or(config: Dict[str, Any], keys: List[str],
92
+ default: Any) -> Any:
93
+ """Get a optional value from the model's quantization config."""
94
+ try:
95
+ return QuantizationConfig.get_from_keys(config, keys)
96
+ except ValueError:
97
+ return default
98
+
99
@abstractmethod
100
def get_quant_method(
101
self, layer: torch.nn.Module) -> Optional[QuantizeMethodBase]:
vllm/model_executor/layers/quantization/gptq.py
+10
-3
@@ -10,6 +10,7 @@ from vllm import _custom_ops as ops
10
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
11
from vllm.model_executor.layers.quantization.base_config import (
12
QuantizationConfig)
13
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
14
from vllm.model_executor.utils import set_weight_attrs
15
16
@@ -24,10 +25,12 @@ class GPTQConfig(QuantizationConfig):
25
weight_bits: int,
26
group_size: int,
27
desc_act: bool,
28
+ lm_head_quantized: bool,
29
) -> None:
30
self.weight_bits = weight_bits
31
self.group_size = group_size
32
self.desc_act = desc_act
33
+ self.lm_head_quantized = lm_head_quantized
34
self.pack_factor = Fraction(32, self.weight_bits)
35
if self.weight_bits not in [2, 3, 4, 8]:
36
raise ValueError(
@@ -37,7 +40,8 @@ class GPTQConfig(QuantizationConfig):
40
def __repr__(self) -> str:
41
return (f"GPTQConfig(weight_bits={self.weight_bits}, "
42
f"group_size={self.group_size}, "
40
- f"desc_act={self.desc_act})")
43
+ f"desc_act={self.desc_act}),"
44
+ f"lm_head_quantized={self.lm_head_quantized}")
45
46
@classmethod
47
def get_name(cls) -> str:
@@ -61,11 +65,14 @@ class GPTQConfig(QuantizationConfig):
65
weight_bits = cls.get_from_keys(config, ["bits"])
66
group_size = cls.get_from_keys(config, ["group_size"])
67
desc_act = cls.get_from_keys(config, ["desc_act"])
64
- return cls(weight_bits, group_size, desc_act)
68
+ lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
69
+ default=False)
70
+ return cls(weight_bits, group_size, desc_act, lm_head_quantized)
71
72
def get_quant_method(
73
self, layer: torch.nn.Module) -> Optional["GPTQLinearMethod"]:
68
- if isinstance(layer, LinearBase):
74
+ if (isinstance(layer, LinearBase) or
75
+ (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
76
return GPTQLinearMethod(self)
77
return None
78
vllm/model_executor/layers/quantization/gptq_marlin.py
+11
-4
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11
set_weight_attrs)
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
15
from vllm.utils import get_device_capability_stateless
16
17
logger = init_logger(__name__)
@@ -59,7 +60,7 @@ class GPTQMarlinConfig(QuantizationConfig):
60
"""Config class for GPTQ Marlin"""
61
62
def __init__(self, weight_bits: int, group_size: int, desc_act: bool,
62
- is_sym: bool) -> None:
63
+ is_sym: bool, lm_head_quantized: bool) -> None:
64
if desc_act and group_size == -1:
65
# In this case, act_order == True is the same as act_order == False
66
# (since we have only one group per output channel)
@@ -69,6 +70,7 @@ class GPTQMarlinConfig(QuantizationConfig):
70
self.group_size = group_size
71
self.desc_act = desc_act
72
self.is_sym = is_sym
73
+ self.lm_head_quantized = lm_head_quantized
74
75
# Verify
76
if self.weight_bits not in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
@@ -96,7 +98,8 @@ class GPTQMarlinConfig(QuantizationConfig):
98
def __repr__(self) -> str:
99
return (f"GPTQMarlinConfig(weight_bits={self.weight_bits}, "
100
f"group_size={self.group_size}, "
99
- f"desc_act={self.desc_act})")
101
+ f"desc_act={self.desc_act}, "
102
+ f"lm_head_quantized={self.lm_head_quantized})")
103
104
@classmethod
105
def get_name(cls) -> str:
@@ -120,7 +123,10 @@ class GPTQMarlinConfig(QuantizationConfig):
123
group_size = cls.get_from_keys(config, ["group_size"])
124
desc_act = cls.get_from_keys(config, ["desc_act"])
125
is_sym = cls.get_from_keys(config, ["sym"])
123
- return cls(weight_bits, group_size, desc_act, is_sym)
126
+ lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
127
+ default=False)
128
+ return cls(weight_bits, group_size, desc_act, is_sym,
129
+ lm_head_quantized)
130
131
@classmethod
132
def override_quantization_method(cls, hf_quant_cfg,
@@ -145,7 +151,8 @@ class GPTQMarlinConfig(QuantizationConfig):
151
def get_quant_method(
152
self,
153
layer: torch.nn.Module) -> Optional["GPTQMarlinLinearMethod"]:
148
- if isinstance(layer, LinearBase):
154
+ if (isinstance(layer, LinearBase) or
155
+ (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
156
return GPTQMarlinLinearMethod(self)
157
return None
158
vllm/model_executor/layers/quantization/marlin.py
+10
-3
@@ -8,6 +8,7 @@ from vllm.logger import init_logger
8
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9
from vllm.model_executor.layers.quantization.base_config import (
10
QuantizationConfig)
11
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
12
from vllm.model_executor.utils import set_weight_attrs
13
14
logger = init_logger(__name__)
@@ -22,9 +23,11 @@ class MarlinConfig(QuantizationConfig):
23
def __init__(
24
self,
25
group_size: int,
26
+ lm_head_quantized: bool,
27
) -> None:
28
# Group size for the quantization.
29
self.group_size = group_size
30
+ self.lm_head_quantized = lm_head_quantized
31
if self.group_size != 128 and self.group_size != -1:
32
raise ValueError(
33
"Currently, only group size 128 and -1 (channelwise) "
@@ -51,7 +54,8 @@ class MarlinConfig(QuantizationConfig):
54
self.perm_len = 1024
55
56
def __repr__(self) -> str:
54
- return f"MarlinConfig(group_size={self.group_size})"
57
+ return (f"MarlinConfig(group_size={self.group_size}, "
58
+ f"lm_head_quantized={self.lm_head_quantized})")
59
60
@classmethod
61
def get_name(cls) -> str:
@@ -73,7 +77,9 @@ class MarlinConfig(QuantizationConfig):
77
@classmethod
78
def from_config(cls, config: Dict[str, Any]) -> "MarlinConfig":
79
group_size = cls.get_from_keys(config, ["group_size"])
76
- return cls(group_size)
80
+ lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
81
+ default=False)
82
+ return cls(group_size, lm_head_quantized)
83
84
@classmethod
85
def override_quantization_method(cls, hf_quant_cfg,
@@ -96,7 +102,8 @@ class MarlinConfig(QuantizationConfig):
102
103
def get_quant_method(
104
self, layer: torch.nn.Module) -> Optional["MarlinLinearMethod"]:
99
- if isinstance(layer, LinearBase):
105
+ if (isinstance(layer, LinearBase) or
106
+ (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
107
return MarlinLinearMethod(self)
108
return None
109
vllm/model_executor/layers/vocab_parallel_embedding.py
+53
-17
@@ -8,6 +8,9 @@ from torch.nn.parameter import Parameter
8
from vllm.distributed import (divide, get_tensor_model_parallel_rank,
9
get_tensor_model_parallel_world_size,
10
tensor_model_parallel_all_reduce)
11
+from vllm.model_executor.layers.linear import UnquantizedLinearMethod
12
+from vllm.model_executor.layers.quantization.base_config import (
13
+ QuantizationConfig, QuantizeMethodBase)
14
from vllm.model_executor.utils import set_weight_attrs
15
16
DEFAULT_VOCAB_PADDING_SIZE = 64
@@ -157,6 +160,7 @@ class VocabParallelEmbedding(torch.nn.Module):
160
params_dtype: type of the parameters.
161
org_num_embeddings: original vocabulary size (without LoRA).
162
padding_size: padding size for the vocabulary.
163
+ quant_config: quant config for the layer
164
""" # noqa: E501
165
166
def __init__(self,
@@ -164,7 +168,8 @@ class VocabParallelEmbedding(torch.nn.Module):
168
embedding_dim: int,
169
params_dtype: Optional[torch.dtype] = None,
170
org_num_embeddings: Optional[int] = None,
167
- padding_size: int = DEFAULT_VOCAB_PADDING_SIZE):
171
+ padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
172
+ quant_config: Optional[QuantizationConfig] = None):
173
super().__init__()
174
175
# Keep the input dimensions.
@@ -187,6 +192,14 @@ class VocabParallelEmbedding(torch.nn.Module):
192
self.org_vocab_size, tp_rank,
193
self.tp_size)
194
self.embedding_dim = embedding_dim
195
+
196
+ linear_method = None
197
+ if quant_config is not None:
198
+ linear_method = quant_config.get_quant_method(self)
199
+ if linear_method is None:
200
+ linear_method = UnquantizedLinearMethod()
201
+ self.linear_method: QuantizeMethodBase = linear_method
202
+
203
if params_dtype is None:
204
params_dtype = torch.get_default_dtype()
205
# Divide the weight matrix along the vocaburaly dimension.
@@ -201,14 +214,14 @@ class VocabParallelEmbedding(torch.nn.Module):
214
self.num_added_embeddings_per_partition = (
215
self.shard_indices.added_vocab_end_index -
216
self.shard_indices.added_vocab_start_index)
204
- self.weight = Parameter(
205
- torch.empty(self.num_embeddings_per_partition,
206
- self.embedding_dim,
207
- dtype=params_dtype))
208
- set_weight_attrs(self.weight, {
209
- "parallel_dim": 0,
210
- "weight_loader": self.weight_loader
211
- })
217
+
218
+ self.linear_method.create_weights(self,
219
+ self.embedding_dim,
220
+ [self.num_embeddings_per_partition],
221
+ self.embedding_dim,
222
+ self.num_embeddings_padded,
223
+ params_dtype=params_dtype,
224
+ weight_loader=self.weight_loader)
225
226
@classmethod
227
def _get_indices(cls, vocab_size_padded: int, org_vocab_size_padded: int,
@@ -288,10 +301,32 @@ class VocabParallelEmbedding(torch.nn.Module):
301
return ret
302
303
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
291
- parallel_dim = param.parallel_dim
292
- assert loaded_weight.shape[parallel_dim] == self.org_vocab_size
293
- loaded_weight = loaded_weight[self.shard_indices.org_vocab_start_index:
294
- self.shard_indices.org_vocab_end_index]
304
+ output_dim = getattr(param, "output_dim", None)
305
+ packed_dim = getattr(param, "packed_dim", None)
306
+
307
+ # If parameter does not have output dim, then it should
308
+ # be copied onto all gpus (e.g. g_idx for act_order gptq).
309
+ if output_dim is None:
310
+ assert param.data.shape == loaded_weight.shape
311
+ param.data.copy_(loaded_weight)
312
+ return
313
+
314
+ # Shard indexes for loading the weight
315
+ start_idx = self.shard_indices.org_vocab_start_index
316
+ shard_size = self.shard_indices.org_vocab_end_index - start_idx
317
+
318
+ # If param packed on the same dim we are sharding on, then
319
+ # need to adjust offsets of loaded weight by pack_factor.
320
+ if packed_dim is not None and packed_dim == output_dim:
321
+ assert loaded_weight.shape[output_dim] == (self.org_vocab_size //
322
+ param.pack_factor)
323
+ start_idx = start_idx // param.pack_factor
324
+ shard_size = shard_size // param.pack_factor
325
+ else:
326
+ assert loaded_weight.shape[output_dim] == self.org_vocab_size
327
+
328
+ # Copy the data.
329
+ loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
330
param[:loaded_weight.shape[0]].data.copy_(loaded_weight)
331
param[loaded_weight.shape[0]:].data.fill_(0)
332
@@ -346,16 +381,17 @@ class ParallelLMHead(VocabParallelEmbedding):
381
bias: bool = False,
382
params_dtype: Optional[torch.dtype] = None,
383
org_num_embeddings: Optional[int] = None,
349
- padding_size: int = DEFAULT_VOCAB_PADDING_SIZE):
384
+ padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
385
+ quant_config: Optional[QuantizationConfig] = None):
386
super().__init__(num_embeddings, embedding_dim, params_dtype,
351
- org_num_embeddings, padding_size)
387
+ org_num_embeddings, padding_size, quant_config)
388
if bias:
389
self.bias = Parameter(
390
torch.empty(self.num_embeddings_per_partition,
391
dtype=params_dtype))
392
set_weight_attrs(self.bias, {
357
- "parallel_dim": 0,
358
- "weight_loader": self.weight_loader
393
+ "output_dim": 0,
394
+ "weight_loader": self.weight_loader,
395
})
396
else:
397
self.register_parameter("bias", None)
vllm/model_executor/models/arctic.py
+2
-1
@@ -412,6 +412,7 @@ class ArcticForCausalLM(nn.Module):
412
self.lm_head = ParallelLMHead(
413
self.vocab_size,
414
config.hidden_size,
415
+ quant_config=quant_config,
416
)
417
self.num_experts = config.num_local_experts
418
self.num_experts_per_tok = config.num_experts_per_tok
@@ -434,7 +435,7 @@ class ArcticForCausalLM(nn.Module):
435
436
def compute_logits(self, hidden_states: torch.Tensor,
437
sampling_metadata: SamplingMetadata) -> torch.Tensor:
437
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
438
+ logits = self.logits_processor(self.lm_head, hidden_states,
439
sampling_metadata)
440
return logits
441
vllm/model_executor/models/baichuan.py
+4
-2
@@ -328,7 +328,9 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
328
self.quant_config = quant_config
329
self.model = BaiChuanModel(config, position_embedding, cache_config,
330
quant_config)
331
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
331
+ self.lm_head = ParallelLMHead(config.vocab_size,
332
+ config.hidden_size,
333
+ quant_config=quant_config)
334
self.logits_processor = LogitsProcessor(config.vocab_size)
335
self.sampler = Sampler()
336
@@ -346,7 +348,7 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
348
349
def compute_logits(self, hidden_states: torch.Tensor,
350
sampling_metadata: SamplingMetadata) -> torch.Tensor:
349
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
351
+ logits = self.logits_processor(self.lm_head, hidden_states,
352
sampling_metadata)
353
return logits
354
vllm/model_executor/models/bloom.py
+2
-2
@@ -276,7 +276,7 @@ class BloomForCausalLM(nn.Module):
276
self.config = config
277
self.quant_config = quant_config
278
self.transformer = BloomModel(config, cache_config, quant_config)
279
- self.lm_head_weight = self.transformer.word_embeddings.weight
279
+ self.lm_head = self.transformer.word_embeddings
280
self.logits_processor = LogitsProcessor(config.vocab_size)
281
self.sampler = Sampler()
282
@@ -294,7 +294,7 @@ class BloomForCausalLM(nn.Module):
294
295
def compute_logits(self, hidden_states: torch.Tensor,
296
sampling_metadata: SamplingMetadata) -> torch.Tensor:
297
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
297
+ logits = self.logits_processor(self.lm_head, hidden_states,
298
sampling_metadata)
299
return logits
300
vllm/model_executor/models/chatglm.py
+4
-3
@@ -303,7 +303,8 @@ class ChatGLMModel(nn.Module):
303
self.encoder = GLMTransformer(config, cache_config, quant_config)
304
305
self.output_layer = ParallelLMHead(config.padded_vocab_size,
306
- config.hidden_size)
306
+ config.hidden_size,
307
+ quant_config=quant_config)
308
309
def forward(
310
self,
@@ -355,7 +356,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
356
self.max_position_embeddings = getattr(config, "max_sequence_length",
357
8192)
358
self.transformer = ChatGLMModel(config, cache_config, quant_config)
358
- self.lm_head_weight = self.transformer.output_layer.weight
359
+ self.lm_head = self.transformer.output_layer
360
self.logits_processor = LogitsProcessor(config.padded_vocab_size)
361
self.sampler = Sampler()
362
@@ -373,7 +374,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
374
375
def compute_logits(self, hidden_states: torch.Tensor,
376
sampling_metadata: SamplingMetadata) -> torch.Tensor:
376
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
377
+ logits = self.logits_processor(self.lm_head, hidden_states,
378
sampling_metadata)
379
return logits
380
vllm/model_executor/models/commandr.py
+4
-4
@@ -363,12 +363,12 @@ class CohereForCausalLM(nn.Module):
363
sampling_metadata: SamplingMetadata) -> torch.Tensor:
364
is_not_lora = hasattr(self.model.embed_tokens, 'weight')
365
if is_not_lora:
366
- embedding_weights = self.model.embed_tokens.weight
366
+ logits = self.logits_processor(self.model.embed_tokens,
367
+ hidden_states, sampling_metadata)
368
else:
368
- embedding_weights = self.model.embed_tokens.base_layer.weight
369
+ logits = self.logits_processor(self.model.embed_tokens.base_layer,
370
+ hidden_states, sampling_metadata)
371
370
- logits = self.logits_processor(embedding_weights, hidden_states,
371
- sampling_metadata)
372
return logits
373
374
def sample(
vllm/model_executor/models/dbrx.py
+2
-1
@@ -370,6 +370,7 @@ class DbrxForCausalLM(nn.Module):
370
config.d_model,
371
org_num_embeddings=config.vocab_size,
372
padding_size=DEFAULT_VOCAB_PADDING_SIZE,
373
+ quant_config=quant_config,
374
)
375
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
376
config.vocab_size)
@@ -389,7 +390,7 @@ class DbrxForCausalLM(nn.Module):
390
391
def compute_logits(self, hidden_states: torch.Tensor,
392
sampling_metadata: SamplingMetadata) -> torch.Tensor:
392
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
393
+ logits = self.logits_processor(self.lm_head, hidden_states,
394
sampling_metadata)
395
return logits
396
vllm/model_executor/models/deepseek.py
+4
-2
@@ -377,7 +377,9 @@ class DeepseekForCausalLM(nn.Module):
377
self.config = config
378
self.quant_config = quant_config
379
self.model = DeepseekModel(config, cache_config, quant_config)
380
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
380
+ self.lm_head = ParallelLMHead(config.vocab_size,
381
+ config.hidden_size,
382
+ quant_config=quant_config)
383
self.logits_processor = LogitsProcessor(config.vocab_size)
384
self.sampler = Sampler()
385
@@ -395,7 +397,7 @@ class DeepseekForCausalLM(nn.Module):
397
398
def compute_logits(self, hidden_states: torch.Tensor,
399
sampling_metadata: SamplingMetadata) -> torch.Tensor:
398
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
400
+ logits = self.logits_processor(self.lm_head, hidden_states,
401
sampling_metadata)
402
return logits
403
vllm/model_executor/models/deepseek_v2.py
+4
-2
@@ -465,7 +465,9 @@ class DeepseekV2ForCausalLM(nn.Module):
465
self.config = config
466
self.quant_config = quant_config
467
self.model = DeepseekV2Model(config, cache_config, quant_config)
468
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
468
+ self.lm_head = ParallelLMHead(config.vocab_size,
469
+ config.hidden_size,
470
+ quant_config=quant_config)
471
self.logits_processor = LogitsProcessor(config.vocab_size)
472
self.sampler = Sampler()
473
@@ -483,7 +485,7 @@ class DeepseekV2ForCausalLM(nn.Module):
485
486
def compute_logits(self, hidden_states: torch.Tensor,
487
sampling_metadata: SamplingMetadata) -> torch.Tensor:
486
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
488
+ logits = self.logits_processor(self.lm_head, hidden_states,
489
sampling_metadata)
490
return logits
491
vllm/model_executor/models/falcon.py
+3
-3
@@ -394,13 +394,13 @@ class FalconForCausalLM(nn.Module):
394
if config.tie_word_embeddings is not None
395
else True)
396
if self.tie_word_embeddings:
397
- self.lm_head_weight = self.transformer.word_embeddings.weight
397
+ self.lm_head = self.transformer.word_embeddings
398
else:
399
self.lm_head = ParallelLMHead(
400
config.vocab_size,
401
config.hidden_size,
402
+ quant_config=quant_config,
403
)
403
- self.lm_head_weight = self.lm_head.weight
404
self.logits_processor = LogitsProcessor(config.vocab_size)
405
self.sampler = Sampler()
406
@@ -422,7 +422,7 @@ class FalconForCausalLM(nn.Module):
422
423
def compute_logits(self, hidden_states: torch.Tensor,
424
sampling_metadata: SamplingMetadata) -> torch.Tensor:
425
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
425
+ logits = self.logits_processor(self.lm_head, hidden_states,
426
sampling_metadata)
427
return logits
428
vllm/model_executor/models/gemma.py
+2
-2
@@ -347,8 +347,8 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
347
348
def compute_logits(self, hidden_states: torch.Tensor,
349
sampling_metadata: SamplingMetadata) -> torch.Tensor:
350
- logits = self.logits_processor(self.model.embed_tokens.weight,
351
- hidden_states, sampling_metadata)
350
+ logits = self.logits_processor(self.model.embed_tokens, hidden_states,
351
+ sampling_metadata)
352
return logits
353
354
def sample(
vllm/model_executor/models/gemma2.py
+2
-2
@@ -346,8 +346,8 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
346
347
def compute_logits(self, hidden_states: torch.Tensor,
348
sampling_metadata: SamplingMetadata) -> torch.Tensor:
349
- logits = self.logits_processor(self.model.embed_tokens.weight,
350
- hidden_states, sampling_metadata)
349
+ logits = self.logits_processor(self.model.embed_tokens, hidden_states,
350
+ sampling_metadata)
351
return logits
352
353
def sample(
vllm/model_executor/models/gpt2.py
+2
-2
@@ -238,7 +238,7 @@ class GPT2LMHeadModel(nn.Module):
238
self.config = config
239
self.quant_config = quant_config
240
self.transformer = GPT2Model(config, cache_config, quant_config)
241
- self.lm_head_weight = self.transformer.wte.weight
241
+ self.lm_head = self.transformer.wte
242
self.logits_processor = LogitsProcessor(config.vocab_size)
243
self.sampler = Sampler()
244
@@ -256,7 +256,7 @@ class GPT2LMHeadModel(nn.Module):
256
257
def compute_logits(self, hidden_states: torch.Tensor,
258
sampling_metadata: SamplingMetadata) -> torch.Tensor:
259
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
259
+ logits = self.logits_processor(self.lm_head, hidden_states,
260
sampling_metadata)
261
return logits
262
vllm/model_executor/models/gpt_bigcode.py
+2
-2
@@ -259,7 +259,7 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
259
self.quant_config = quant_config
260
self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
261
lora_config)
262
- self.lm_head_weight = self.transformer.wte.weight
262
+ self.lm_head = self.transformer.wte
263
self.unpadded_vocab_size = config.vocab_size
264
if lora_config:
265
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
@@ -281,7 +281,7 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
281
282
def compute_logits(self, hidden_states: torch.Tensor,
283
sampling_metadata: SamplingMetadata) -> torch.Tensor:
284
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
284
+ logits = self.logits_processor(self.lm_head, hidden_states,
285
sampling_metadata)
286
return logits
287
vllm/model_executor/models/gpt_j.py
+2
-1
@@ -229,6 +229,7 @@ class GPTJForCausalLM(nn.Module):
229
config.vocab_size,
230
config.n_embd,
231
bias=True,
232
+ quant_config=quant_config,
233
)
234
self.logits_processor = LogitsProcessor(config.vocab_size)
235
self.sampler = Sampler()
@@ -247,7 +248,7 @@ class GPTJForCausalLM(nn.Module):
248
249
def compute_logits(self, hidden_states: torch.Tensor,
250
sampling_metadata: SamplingMetadata) -> torch.Tensor:
250
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
251
+ logits = self.logits_processor(self.lm_head, hidden_states,
252
sampling_metadata, self.lm_head.bias)
253
return logits
254
vllm/model_executor/models/gpt_neox.py
+2
-1
@@ -241,6 +241,7 @@ class GPTNeoXForCausalLM(nn.Module):
241
self.embed_out = ParallelLMHead(
242
config.vocab_size,
243
config.hidden_size,
244
+ quant_config=quant_config,
245
)
246
self.logits_processor = LogitsProcessor(config.vocab_size)
247
self.sampler = Sampler()
@@ -259,7 +260,7 @@ class GPTNeoXForCausalLM(nn.Module):
260
261
def compute_logits(self, hidden_states: torch.Tensor,
262
sampling_metadata: SamplingMetadata) -> torch.Tensor:
262
- logits = self.logits_processor(self.embed_out.weight, hidden_states,
263
+ logits = self.logits_processor(self.embed_out, hidden_states,
264
sampling_metadata)
265
return logits
266
vllm/model_executor/models/internlm2.py
+4
-2
@@ -253,7 +253,9 @@ class InternLM2ForCausalLM(nn.Module):
253
self.config = config
254
self.quant_config = quant_config
255
self.model = InternLM2Model(config, cache_config, quant_config)
256
- self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
256
+ self.output = ParallelLMHead(config.vocab_size,
257
+ config.hidden_size,
258
+ quant_config=quant_config)
259
self.logits_processor = LogitsProcessor(config.vocab_size)
260
self.sampler = Sampler()
261
@@ -271,7 +273,7 @@ class InternLM2ForCausalLM(nn.Module):
273
274
def compute_logits(self, hidden_states: torch.Tensor,
275
sampling_metadata: SamplingMetadata) -> torch.Tensor:
274
- logits = self.logits_processor(self.output.weight, hidden_states,
276
+ logits = self.logits_processor(self.output, hidden_states,
277
sampling_metadata)
278
return logits
279
vllm/model_executor/models/jais.py
+2
-2
@@ -273,7 +273,7 @@ class JAISLMHeadModel(nn.Module):
273
self.config = config
274
self.quant_config = quant_config
275
self.transformer = JAISModel(config, cache_config, quant_config)
276
- self.lm_head_weight = self.transformer.wte.weight
276
+ self.lm_head = self.transformer.wte
277
if hasattr(config, "width_scale"):
278
self.output_logits_scale = config.width_scale
279
else:
@@ -297,7 +297,7 @@ class JAISLMHeadModel(nn.Module):
297
298
def compute_logits(self, hidden_states: torch.Tensor,
299
sampling_metadata: SamplingMetadata) -> torch.Tensor:
300
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
300
+ logits = self.logits_processor(self.lm_head, hidden_states,
301
sampling_metadata)
302
return logits
303
vllm/model_executor/models/llama.py
+2
-1
@@ -380,6 +380,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
380
# We need bigger padding if using lora for kernel
381
# compatibility
382
if not lora_config else lora_config.lora_vocab_padding_size,
383
+ quant_config=quant_config,
384
)
385
if config.tie_word_embeddings:
386
self.lm_head.weight = self.model.embed_tokens.weight
@@ -403,7 +404,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
404
405
def compute_logits(self, hidden_states: torch.Tensor,
406
sampling_metadata: SamplingMetadata) -> torch.Tensor:
406
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
407
+ logits = self.logits_processor(self.lm_head, hidden_states,
408
sampling_metadata)
409
return logits
410
vllm/model_executor/models/llava.py
+3
-2
@@ -125,7 +125,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
125
self.lm_head = ParallelLMHead(
126
self.unpadded_vocab_size,
127
config.text_config.hidden_size,
128
- org_num_embeddings=self.language_model.org_vocab_size)
128
+ org_num_embeddings=self.language_model.org_vocab_size,
129
+ quant_config=quant_config)
130
logit_scale = getattr(config, "logit_scale", 1.0)
131
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
132
config.vocab_size, logit_scale)
@@ -255,7 +256,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
256
257
def compute_logits(self, hidden_states: torch.Tensor,
258
sampling_metadata: SamplingMetadata) -> torch.Tensor:
258
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
259
+ logits = self.logits_processor(self.lm_head, hidden_states,
260
sampling_metadata)
261
return logits
262
vllm/model_executor/models/llava_next.py
+3
-2
@@ -186,7 +186,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
186
self.lm_head = ParallelLMHead(
187
self.unpadded_vocab_size,
188
config.text_config.hidden_size,
189
- org_num_embeddings=self.language_model.org_vocab_size)
189
+ org_num_embeddings=self.language_model.org_vocab_size,
190
+ quant_config=quant_config)
191
logit_scale = getattr(config, "logit_scale", 1.0)
192
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
193
config.vocab_size, logit_scale)
@@ -438,7 +439,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
439
440
def compute_logits(self, hidden_states: torch.Tensor,
441
sampling_metadata: SamplingMetadata) -> torch.Tensor:
441
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
442
+ logits = self.logits_processor(self.lm_head, hidden_states,
443
sampling_metadata)
444
return logits
445
vllm/model_executor/models/minicpm.py
+4
-3
@@ -449,6 +449,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
449
# We need bigger padding if using lora for kernel
450
# compatibility
451
if not lora_config else lora_config.lora_vocab_padding_size,
452
+ quant_config=quant_config,
453
)
454
self.scale_width = self.config.hidden_size / self.config.dim_model_base
455
@@ -472,10 +473,10 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
473
sampling_metadata: SamplingMetadata) -> torch.Tensor:
474
hidden_states = hidden_states / self.scale_width
475
if self.config.tie_word_embeddings:
475
- lm_head_weight = self.model.embed_tokens.weight
476
+ lm_head = self.model.embed_tokens
477
else:
477
- lm_head_weight = self.lm_head.weight
478
- logits = self.logits_processor(lm_head_weight, hidden_states,
478
+ lm_head = self.lm_head
479
+ logits = self.logits_processor(lm_head, hidden_states,
480
sampling_metadata)
481
return logits
482
vllm/model_executor/models/mixtral.py
+2
-1
@@ -331,6 +331,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
331
# We need bigger padding if using lora for kernel
332
# compatibility
333
if not lora_config else lora_config.lora_vocab_padding_size,
334
+ quant_config=quant_config,
335
)
336
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
337
config.vocab_size)
@@ -350,7 +351,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
351
352
def compute_logits(self, hidden_states: torch.Tensor,
353
sampling_metadata: SamplingMetadata) -> torch.Tensor:
353
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
354
+ logits = self.logits_processor(self.lm_head, hidden_states,
355
sampling_metadata)
356
return logits
357
vllm/model_executor/models/mixtral_quant.py
+4
-2
@@ -344,7 +344,9 @@ class MixtralForCausalLM(nn.Module):
344
self.config = config
345
self.quant_config = quant_config
346
self.model = MixtralModel(config, cache_config, quant_config)
347
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
347
+ self.lm_head = ParallelLMHead(config.vocab_size,
348
+ config.hidden_size,
349
+ quant_config=quant_config)
350
self.logits_processor = LogitsProcessor(config.vocab_size)
351
self.sampler = Sampler()
352
@@ -362,7 +364,7 @@ class MixtralForCausalLM(nn.Module):
364
365
def compute_logits(self, hidden_states: torch.Tensor,
366
sampling_metadata: SamplingMetadata) -> torch.Tensor:
365
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
367
+ logits = self.logits_processor(self.lm_head, hidden_states,
368
sampling_metadata)
369
return logits
370
vllm/model_executor/models/mlp_speculator.py
+4
-4
@@ -8,7 +8,7 @@ from vllm.model_executor import SamplingMetadata
8
from vllm.model_executor.layers.logits_processor import LogitsProcessor
9
from vllm.model_executor.layers.sampler import Sampler
10
from vllm.model_executor.layers.vocab_parallel_embedding import (
11
- VocabParallelEmbedding)
11
+ ParallelLMHead, VocabParallelEmbedding)
12
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13
from vllm.sequence import SamplerOutput
14
from vllm.transformers_utils.configs import MLPSpeculatorConfig
@@ -87,7 +87,7 @@ class MLPSpeculator(nn.Module):
87
self.proj = nn.ModuleList([proj_first] + [proj_tied] *
88
(self.max_speculative_tokens - 1))
89
90
- head = nn.Linear(self.inner_dim, self.vocab_size, bias=False)
90
+ head = ParallelLMHead(self.vocab_size, self.inner_dim, bias=False)
91
self.head = nn.ModuleList([head] * self.max_speculative_tokens)
92
93
ln = MLPSpeculatorLayerNorm(self.inner_dim,
@@ -169,8 +169,8 @@ class MLPSpeculator(nn.Module):
169
# TODO: not yet supporting top_k_tokens_per_head
170
previous_hidden_states = states
171
172
- logits = self.logits_processor(self.head[head_index].weight,
173
- states, sampling_metadata)
172
+ logits = self.logits_processor(self.head[head_index], states,
173
+ sampling_metadata)
174
175
output = self.sampler(logits.flatten(0, 1), sampling_metadata)
176
last_tokens = output.sampled_token_ids
vllm/model_executor/models/mpt.py
+2
-2
@@ -263,7 +263,7 @@ class MPTForCausalLM(nn.Module):
263
self.quant_config = quant_config
264
265
self.transformer = MPTModel(config, cache_config, quant_config)
266
- self.lm_head_weight = self.transformer.wte.weight
266
+ self.lm_head = self.transformer.wte
267
self.logits_processor = LogitsProcessor(config.vocab_size)
268
self.sampler = Sampler()
269
@@ -281,7 +281,7 @@ class MPTForCausalLM(nn.Module):
281
282
def compute_logits(self, hidden_states: torch.Tensor,
283
sampling_metadata: SamplingMetadata) -> torch.Tensor:
284
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
284
+ logits = self.logits_processor(self.lm_head, hidden_states,
285
sampling_metadata)
286
return logits
287
vllm/model_executor/models/olmo.py
+3
-3
@@ -283,15 +283,15 @@ class OlmoForCausalLM(nn.Module):
283
self.config = config
284
self.model = OlmoModel(config, cache_config, quant_config)
285
if config.tie_word_embeddings:
286
- self.lm_head_weight = self.model.embed_tokens.weight
286
+ self.lm_head = self.model.embed_tokens
287
else:
288
self.unpadded_vocab_size = config.vocab_size
289
self.lm_head = ParallelLMHead(
290
self.unpadded_vocab_size,
291
config.hidden_size,
292
org_num_embeddings=config.vocab_size,
293
+ quant_config=quant_config,
294
)
294
- self.lm_head_weight = self.lm_head.weight
295
self.logits_processor = LogitsProcessor(config.vocab_size)
296
self.sampler = Sampler()
297
@@ -313,7 +313,7 @@ class OlmoForCausalLM(nn.Module):
313
314
def compute_logits(self, hidden_states: torch.Tensor,
315
sampling_metadata: SamplingMetadata) -> torch.Tensor:
316
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
316
+ logits = self.logits_processor(self.lm_head, hidden_states,
317
sampling_metadata)
318
return logits
319
vllm/model_executor/models/opt.py
+2
-2
@@ -294,7 +294,7 @@ class OPTForCausalLM(nn.Module):
294
self.config = config
295
self.quant_config = quant_config
296
self.model = OPTModel(config, cache_config, quant_config)
297
- self.lm_head_weight = self.model.decoder.embed_tokens.weight
297
+ self.lm_head = self.model.decoder.embed_tokens
298
self.logits_processor = LogitsProcessor(config.vocab_size)
299
self.sampler = Sampler()
300
@@ -312,7 +312,7 @@ class OPTForCausalLM(nn.Module):
312
313
def compute_logits(self, hidden_states: torch.Tensor,
314
sampling_metadata: SamplingMetadata) -> torch.Tensor:
315
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
315
+ logits = self.logits_processor(self.lm_head, hidden_states,
316
sampling_metadata)
317
return logits
318
vllm/model_executor/models/orion.py
+4
-2
@@ -259,7 +259,9 @@ class OrionForCausalLM(nn.Module):
259
self.config = config
260
self.quant_config = quant_config
261
self.model = OrionModel(config, cache_config, quant_config)
262
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
262
+ self.lm_head = ParallelLMHead(config.vocab_size,
263
+ config.hidden_size,
264
+ quant_config=quant_config)
265
self.logits_processor = LogitsProcessor(config.vocab_size)
266
self.sampler = Sampler()
267
@@ -277,7 +279,7 @@ class OrionForCausalLM(nn.Module):
279
280
def compute_logits(self, hidden_states: torch.Tensor,
281
sampling_metadata: SamplingMetadata) -> torch.Tensor:
280
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
282
+ logits = self.logits_processor(self.lm_head, hidden_states,
283
sampling_metadata)
284
return logits
285
vllm/model_executor/models/phi.py
+3
-2
@@ -268,7 +268,8 @@ class PhiForCausalLM(nn.Module, SupportsLoRA):
268
269
self.lm_head = ParallelLMHead(config.vocab_size,
270
config.hidden_size,
271
- bias=True)
271
+ bias=True,
272
+ quant_config=quant_config)
273
self.logits_processor = LogitsProcessor(config.vocab_size)
274
self.sampler = Sampler()
275
@@ -287,7 +288,7 @@ class PhiForCausalLM(nn.Module, SupportsLoRA):
288
289
def compute_logits(self, hidden_states: torch.Tensor,
290
sampling_metadata: SamplingMetadata) -> torch.Tensor:
290
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
291
+ logits = self.logits_processor(self.lm_head, hidden_states,
292
sampling_metadata, self.lm_head.bias)
293
return logits
294
vllm/model_executor/models/phi3_small.py
+2
-1
@@ -366,6 +366,7 @@ class Phi3SmallForCausalLM(nn.Module):
366
config.hidden_size,
367
org_num_embeddings=config.vocab_size,
368
padding_size=DEFAULT_VOCAB_PADDING_SIZE,
369
+ quant_config=quant_config,
370
)
371
self.logits_processor = LogitsProcessor(config.vocab_size)
372
self.sampler = Sampler()
@@ -400,7 +401,7 @@ class Phi3SmallForCausalLM(nn.Module):
401
402
def compute_logits(self, hidden_states: torch.Tensor,
403
sampling_metadata: SamplingMetadata) -> torch.Tensor:
403
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
404
+ logits = self.logits_processor(self.lm_head, hidden_states,
405
sampling_metadata)
406
if self.dummy_token_indices is not None and logits is not None:
407
logits.index_fill_(-1, self.dummy_token_indices, -torch.inf)
vllm/model_executor/models/phi3v.py
+4
-2
@@ -365,7 +365,9 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
365
self.model = LlamaModel(config, cache_config, quant_config)
366
self.vision_embed_tokens = Phi3HDImageEmbedding(
367
vlm_config, config, self.model.embed_tokens)
368
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
368
+ self.lm_head = ParallelLMHead(config.vocab_size,
369
+ config.hidden_size,
370
+ quant_config=quant_config)
371
self.logits_processor = LogitsProcessor(config.vocab_size)
372
self.sampler = Sampler()
373
@@ -409,7 +411,7 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
411
412
def compute_logits(self, hidden_states: torch.Tensor,
413
sampling_metadata: SamplingMetadata) -> torch.Tensor:
412
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
414
+ logits = self.logits_processor(self.lm_head, hidden_states,
415
sampling_metadata)
416
return logits
417
vllm/model_executor/models/qwen.py
+4
-2
@@ -235,7 +235,9 @@ class QWenLMHeadModel(nn.Module):
235
self.config = config
236
self.quant_config = quant_config
237
self.transformer = QWenModel(config, cache_config, quant_config)
238
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
238
+ self.lm_head = ParallelLMHead(config.vocab_size,
239
+ config.hidden_size,
240
+ quant_config=quant_config)
241
self.logits_processor = LogitsProcessor(config.vocab_size)
242
self.sampler = Sampler()
243
@@ -253,7 +255,7 @@ class QWenLMHeadModel(nn.Module):
255
256
def compute_logits(self, hidden_states: torch.Tensor,
257
sampling_metadata: SamplingMetadata) -> torch.Tensor:
256
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
258
+ logits = self.logits_processor(self.lm_head, hidden_states,
259
sampling_metadata)
260
return logits
261
vllm/model_executor/models/qwen2.py
+4
-4
@@ -316,11 +316,11 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
316
self.model = Qwen2Model(config, cache_config, quant_config)
317
318
if config.tie_word_embeddings:
319
- self.lm_head_weight = self.model.embed_tokens.weight
319
+ self.lm_head = self.model.embed_tokens
320
else:
321
self.lm_head = ParallelLMHead(config.vocab_size,
322
- config.hidden_size)
323
- self.lm_head_weight = self.lm_head.weight
322
+ config.hidden_size,
323
+ quant_config=quant_config)
324
325
self.logits_processor = LogitsProcessor(config.vocab_size)
326
self.sampler = Sampler()
@@ -339,7 +339,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
339
340
def compute_logits(self, hidden_states: torch.Tensor,
341
sampling_metadata: SamplingMetadata) -> torch.Tensor:
342
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
342
+ logits = self.logits_processor(self.lm_head, hidden_states,
343
sampling_metadata)
344
return logits
345
vllm/model_executor/models/qwen2_moe.py
+4
-2
@@ -362,7 +362,9 @@ class Qwen2MoeForCausalLM(nn.Module):
362
self.config = config
363
self.quant_config = quant_config
364
self.model = Qwen2MoeModel(config, cache_config, quant_config)
365
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
365
+ self.lm_head = ParallelLMHead(config.vocab_size,
366
+ config.hidden_size,
367
+ quant_config=quant_config)
368
self.logits_processor = LogitsProcessor(config.vocab_size)
369
self.sampler = Sampler()
370
@@ -380,7 +382,7 @@ class Qwen2MoeForCausalLM(nn.Module):
382
383
def compute_logits(self, hidden_states: torch.Tensor,
384
sampling_metadata: SamplingMetadata) -> torch.Tensor:
383
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
385
+ logits = self.logits_processor(self.lm_head, hidden_states,
386
sampling_metadata)
387
return logits
388
vllm/model_executor/models/stablelm.py
+4
-2
@@ -240,7 +240,9 @@ class StablelmForCausalLM(nn.Module):
240
self.config = config
241
self.quant_config = quant_config
242
self.model = StableLMEpochModel(config, cache_config, quant_config)
243
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
243
+ self.lm_head = ParallelLMHead(config.vocab_size,
244
+ config.hidden_size,
245
+ quant_config=quant_config)
246
self.logits_processor = LogitsProcessor(config.vocab_size)
247
self.sampler = Sampler()
248
@@ -258,7 +260,7 @@ class StablelmForCausalLM(nn.Module):
260
261
def compute_logits(self, hidden_states: torch.Tensor,
262
sampling_metadata: SamplingMetadata) -> torch.Tensor:
261
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
263
+ logits = self.logits_processor(self.lm_head, hidden_states,
264
sampling_metadata)
265
return logits
266
vllm/model_executor/models/starcoder2.py
+3
-3
@@ -242,7 +242,7 @@ class Starcoder2ForCausalLM(nn.Module):
242
self.vocab_size = config.vocab_size
243
self.unpadded_vocab_size = config.vocab_size
244
if config.tie_word_embeddings:
245
- self.lm_head_weight = self.model.embed_tokens.weight
245
+ self.lm_head = self.model.embed_tokens
246
else:
247
self.unpadded_vocab_size = config.vocab_size
248
self.lm_head = ParallelLMHead(
@@ -250,8 +250,8 @@ class Starcoder2ForCausalLM(nn.Module):
250
config.hidden_size,
251
org_num_embeddings=config.vocab_size,
252
padding_size=DEFAULT_VOCAB_PADDING_SIZE,
253
+ quant_config=quant_config,
254
)
254
- self.lm_head_weight = self.lm_head.weight
255
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
256
config.vocab_size)
257
self.sampler = Sampler()
@@ -270,7 +270,7 @@ class Starcoder2ForCausalLM(nn.Module):
270
271
def compute_logits(self, hidden_states: torch.Tensor,
272
sampling_metadata: SamplingMetadata) -> torch.Tensor:
273
- logits = self.logits_processor(self.lm_head_weight, hidden_states,
273
+ logits = self.logits_processor(self.lm_head, hidden_states,
274
sampling_metadata)
275
return logits
276
vllm/model_executor/models/xverse.py
+4
-2
@@ -310,7 +310,9 @@ class XverseForCausalLM(nn.Module, SupportsLoRA):
310
311
self.quant_config = quant_config
312
self.model = XverseModel(config, cache_config, quant_config)
313
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
313
+ self.lm_head = ParallelLMHead(config.vocab_size,
314
+ config.hidden_size,
315
+ quant_config=quant_config)
316
self.logits_processor = LogitsProcessor(config.vocab_size)
317
self.sampler = Sampler()
318
@@ -328,7 +330,7 @@ class XverseForCausalLM(nn.Module, SupportsLoRA):
330
331
def compute_logits(self, hidden_states: torch.Tensor,
332
sampling_metadata: SamplingMetadata) -> torch.Tensor:
331
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
333
+ logits = self.logits_processor(self.lm_head, hidden_states,
334
sampling_metadata)
335
return logits
336