@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ee93f4f92a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[CORE] Quantized lm-head Framework (#4442)

Co-authored-by: Robert Shaw <rshaw@neuralmagic.com> Co-authored-by: ZX <zx@lbx.dev>

Qubitium-ModelCloud committed Jul 3, 2024 at 06:25 UTC ee93f4f92acbd9759a9af80747bc2a4459f07639
48 files changed +268 -121
tests/lora/test_layers.py
+5 -5
@@ -475,10 +475,10 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
475
476 lora_result = lora_logits_processor._get_logits(
477 hidden_states=torch.cat(inputs),
478 - embedding=linear.weight,
478 + lm_head=linear,
479 embedding_bias=None)
480
481 - original_weight = linear.weight.clone()
481 + original_lm_head = deepcopy(linear)
482
483 linear.weight[logits_processor.
484 org_vocab_size:logits_processor.org_vocab_size +
@@ -490,7 +490,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
490 for input_, lora_id in zip(inputs, prompt_mapping):
491 lora = lora_dict[lora_id]
492 result = logits_processor._get_logits(hidden_states=input_,
493 - embedding=linear.weight,
493 + lm_head=linear,
494 embedding_bias=None)
495 result[:, vocab_size + embeddings_tensor_len:] = float("-inf")
496 result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
@@ -519,11 +519,11 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
519
520 lora_result = lora_logits_processor._get_logits(
521 hidden_states=torch.cat(inputs),
522 - embedding=original_weight,
522 + lm_head=original_lm_head,
523 embedding_bias=None)[:, :vocab_size]
524 expected_result = logits_processor._get_logits(
525 hidden_states=torch.cat(inputs),
526 - embedding=original_weight,
526 + lm_head=original_lm_head,
527 embedding_bias=None)
528
529 rtol, atol = TOLERANCES[lora_result.dtype]
tests/quantization/test_lm_head.py new
+45
@@ -0,0 +1,45 @@
1 +"""Tests whether gptq models with quantized lm_head can be loaded.
2 +
3 +Run `pytest tests/quantization/test_quant_lm_head_true.py --forked`.
4 +"""
5 +from typing import Tuple
6 +
7 +import pytest
8 +import torch
9 +
10 +from vllm.model_executor.layers.linear import UnquantizedLinearMethod
11 +from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
12 +from vllm.model_executor.layers.quantization.gptq_marlin import (
13 + GPTQMarlinLinearMethod)
14 +from vllm.model_executor.layers.quantization.marlin import MarlinLinearMethod
15 +
16 +PROMPT = "On the surface of Mars, we found"
17 +
18 +MODELS_QUANT = [(
19 + "LnL-AI/TinyLlama-1.1B-intermediate-step-1341k-3T-autoround-lm_head-symFalse",
20 + True), ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", False),
21 + ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", False)]
22 +
23 +
24 +@pytest.mark.parametrize("model_lm_head_quant", MODELS_QUANT)
25 +def test_lm_head(
26 + vllm_runner,
27 + model_lm_head_quant: Tuple[str, bool],
28 +) -> None:
29 + model, lm_head_quantized = model_lm_head_quant
30 + vllm_model = vllm_runner(model, dtype=torch.float16, max_model_len=2048)
31 +
32 + lm_head_layer = (vllm_model.model.llm_engine.model_executor.driver_worker.
33 + model_runner.model.lm_head)
34 +
35 + if lm_head_quantized:
36 + assert isinstance(
37 + lm_head_layer.linear_method,
38 + (GPTQLinearMethod, GPTQMarlinLinearMethod, MarlinLinearMethod))
39 + else:
40 + assert isinstance(lm_head_layer.linear_method, UnquantizedLinearMethod)
41 +
42 + print(
43 + vllm_model.generate_greedy(prompts=["Hello my name is"],
44 + max_tokens=10)[0][1])
45 + del vllm_model
tests/spec_decode/e2e/test_mlp_correctness.py
+1 -1
@@ -34,7 +34,7 @@ SPEC_MODEL = "ibm-granite/granite-3b-code-instruct-accelerator"
34 MAX_SPEC_TOKENS = 5
35
36 # precision
37 -PRECISION = "float16"
37 +PRECISION = "float32"
38
39
40 @pytest.mark.parametrize(
tests/test_logits_processor.py
+1 -1
@@ -83,7 +83,7 @@ def test_logits_processors(seed: int, device: str):
83 device=device,
84 pin_memory=is_pin_memory_available())
85 logits_processor_output = logits_processor(
86 - embedding=None,
86 + lm_head=None,
87 hidden_states=input_tensor,
88 sampling_metadata=sampling_metadata)
89
vllm/lora/layers.py
+2 -2
@@ -1172,11 +1172,11 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1172 def _get_logits(
1173 self,
1174 hidden_states: torch.Tensor,
1175 - embedding: torch.Tensor,
1175 + lm_head: VocabParallelEmbedding,
1176 embedding_bias: Optional[torch.Tensor] = None,
1177 ) -> Optional[torch.Tensor]:
1178 # Get the logits for the next tokens.
1179 - logits = torch.matmul(hidden_states, embedding.t())
1179 + logits = lm_head.linear_method.apply(lm_head, hidden_states)
1180 if embedding_bias is not None:
1181 logits += embedding_bias
1182 logits = tensor_model_parallel_gather(logits)
vllm/model_executor/layers/logits_processor.py
+9 -7
@@ -6,6 +6,8 @@ import torch
6 import torch.nn as nn
7
8 from vllm.distributed import tensor_model_parallel_gather
9 +from vllm.model_executor.layers.vocab_parallel_embedding import (
10 + VocabParallelEmbedding)
11 from vllm.model_executor.sampling_metadata import SamplingMetadata
12
13
@@ -40,7 +42,7 @@ class LogitsProcessor(nn.Module):
42
43 def forward(
44 self,
43 - embedding: torch.Tensor,
45 + lm_head: VocabParallelEmbedding,
46 hidden_states: torch.Tensor,
47 sampling_metadata: SamplingMetadata,
48 embedding_bias: Optional[torch.Tensor] = None,
@@ -52,8 +54,7 @@ class LogitsProcessor(nn.Module):
54 sampling_metadata)
55
56 # Get the logits for the next tokens.
55 - logits = self._get_logits(hidden_states, embedding, embedding_bias)
56 -
57 + logits = self._get_logits(hidden_states, lm_head, embedding_bias)
58 if logits is not None:
59 if self.soft_cap is not None:
60 logits = logits / self.soft_cap
@@ -68,12 +69,13 @@ class LogitsProcessor(nn.Module):
69
70 return logits
71
71 - def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor,
72 + def _get_logits(self, hidden_states: torch.Tensor,
73 + lm_head: VocabParallelEmbedding,
74 embedding_bias: Optional[torch.Tensor]) -> torch.Tensor:
75 # Get the logits for the next tokens.
74 - logits = torch.matmul(hidden_states, embedding.t())
75 - if embedding_bias is not None:
76 - logits += embedding_bias
76 + logits = lm_head.linear_method.apply(lm_head,
77 + hidden_states,
78 + bias=embedding_bias)
79 logits = tensor_model_parallel_gather(logits)
80 # Remove paddings in vocab (if any).
81 if logits is not None:
vllm/model_executor/layers/quantization/base_config.py
+9
@@ -87,6 +87,15 @@ class QuantizationConfig(ABC):
87 raise ValueError(f"Cannot find any of {keys} in the model's "
88 "quantization config.")
89
90 + @staticmethod
91 + def get_from_keys_or(config: Dict[str, Any], keys: List[str],
92 + default: Any) -> Any:
93 + """Get a optional value from the model's quantization config."""
94 + try:
95 + return QuantizationConfig.get_from_keys(config, keys)
96 + except ValueError:
97 + return default
98 +
99 @abstractmethod
100 def get_quant_method(
101 self, layer: torch.nn.Module) -> Optional[QuantizeMethodBase]:
vllm/model_executor/layers/quantization/gptq.py
+10 -3
@@ -10,6 +10,7 @@ from vllm import _custom_ops as ops
10 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig)
13 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
14 from vllm.model_executor.utils import set_weight_attrs
15
16
@@ -24,10 +25,12 @@ class GPTQConfig(QuantizationConfig):
25 weight_bits: int,
26 group_size: int,
27 desc_act: bool,
28 + lm_head_quantized: bool,
29 ) -> None:
30 self.weight_bits = weight_bits
31 self.group_size = group_size
32 self.desc_act = desc_act
33 + self.lm_head_quantized = lm_head_quantized
34 self.pack_factor = Fraction(32, self.weight_bits)
35 if self.weight_bits not in [2, 3, 4, 8]:
36 raise ValueError(
@@ -37,7 +40,8 @@ class GPTQConfig(QuantizationConfig):
40 def __repr__(self) -> str:
41 return (f"GPTQConfig(weight_bits={self.weight_bits}, "
42 f"group_size={self.group_size}, "
40 - f"desc_act={self.desc_act})")
43 + f"desc_act={self.desc_act}),"
44 + f"lm_head_quantized={self.lm_head_quantized}")
45
46 @classmethod
47 def get_name(cls) -> str:
@@ -61,11 +65,14 @@ class GPTQConfig(QuantizationConfig):
65 weight_bits = cls.get_from_keys(config, ["bits"])
66 group_size = cls.get_from_keys(config, ["group_size"])
67 desc_act = cls.get_from_keys(config, ["desc_act"])
64 - return cls(weight_bits, group_size, desc_act)
68 + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
69 + default=False)
70 + return cls(weight_bits, group_size, desc_act, lm_head_quantized)
71
72 def get_quant_method(
73 self, layer: torch.nn.Module) -> Optional["GPTQLinearMethod"]:
68 - if isinstance(layer, LinearBase):
74 + if (isinstance(layer, LinearBase) or
75 + (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
76 return GPTQLinearMethod(self)
77 return None
78
vllm/model_executor/layers/quantization/gptq_marlin.py
+11 -4
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11 set_weight_attrs)
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
15 from vllm.utils import get_device_capability_stateless
16
17 logger = init_logger(__name__)
@@ -59,7 +60,7 @@ class GPTQMarlinConfig(QuantizationConfig):
60 """Config class for GPTQ Marlin"""
61
62 def __init__(self, weight_bits: int, group_size: int, desc_act: bool,
62 - is_sym: bool) -> None:
63 + is_sym: bool, lm_head_quantized: bool) -> None:
64 if desc_act and group_size == -1:
65 # In this case, act_order == True is the same as act_order == False
66 # (since we have only one group per output channel)
@@ -69,6 +70,7 @@ class GPTQMarlinConfig(QuantizationConfig):
70 self.group_size = group_size
71 self.desc_act = desc_act
72 self.is_sym = is_sym
73 + self.lm_head_quantized = lm_head_quantized
74
75 # Verify
76 if self.weight_bits not in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
@@ -96,7 +98,8 @@ class GPTQMarlinConfig(QuantizationConfig):
98 def __repr__(self) -> str:
99 return (f"GPTQMarlinConfig(weight_bits={self.weight_bits}, "
100 f"group_size={self.group_size}, "
99 - f"desc_act={self.desc_act})")
101 + f"desc_act={self.desc_act}, "
102 + f"lm_head_quantized={self.lm_head_quantized})")
103
104 @classmethod
105 def get_name(cls) -> str:
@@ -120,7 +123,10 @@ class GPTQMarlinConfig(QuantizationConfig):
123 group_size = cls.get_from_keys(config, ["group_size"])
124 desc_act = cls.get_from_keys(config, ["desc_act"])
125 is_sym = cls.get_from_keys(config, ["sym"])
123 - return cls(weight_bits, group_size, desc_act, is_sym)
126 + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
127 + default=False)
128 + return cls(weight_bits, group_size, desc_act, is_sym,
129 + lm_head_quantized)
130
131 @classmethod
132 def override_quantization_method(cls, hf_quant_cfg,
@@ -145,7 +151,8 @@ class GPTQMarlinConfig(QuantizationConfig):
151 def get_quant_method(
152 self,
153 layer: torch.nn.Module) -> Optional["GPTQMarlinLinearMethod"]:
148 - if isinstance(layer, LinearBase):
154 + if (isinstance(layer, LinearBase) or
155 + (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
156 return GPTQMarlinLinearMethod(self)
157 return None
158
vllm/model_executor/layers/quantization/marlin.py
+10 -3
@@ -8,6 +8,7 @@ from vllm.logger import init_logger
8 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9 from vllm.model_executor.layers.quantization.base_config import (
10 QuantizationConfig)
11 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
12 from vllm.model_executor.utils import set_weight_attrs
13
14 logger = init_logger(__name__)
@@ -22,9 +23,11 @@ class MarlinConfig(QuantizationConfig):
23 def __init__(
24 self,
25 group_size: int,
26 + lm_head_quantized: bool,
27 ) -> None:
28 # Group size for the quantization.
29 self.group_size = group_size
30 + self.lm_head_quantized = lm_head_quantized
31 if self.group_size != 128 and self.group_size != -1:
32 raise ValueError(
33 "Currently, only group size 128 and -1 (channelwise) "
@@ -51,7 +54,8 @@ class MarlinConfig(QuantizationConfig):
54 self.perm_len = 1024
55
56 def __repr__(self) -> str:
54 - return f"MarlinConfig(group_size={self.group_size})"
57 + return (f"MarlinConfig(group_size={self.group_size}, "
58 + f"lm_head_quantized={self.lm_head_quantized})")
59
60 @classmethod
61 def get_name(cls) -> str:
@@ -73,7 +77,9 @@ class MarlinConfig(QuantizationConfig):
77 @classmethod
78 def from_config(cls, config: Dict[str, Any]) -> "MarlinConfig":
79 group_size = cls.get_from_keys(config, ["group_size"])
76 - return cls(group_size)
80 + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
81 + default=False)
82 + return cls(group_size, lm_head_quantized)
83
84 @classmethod
85 def override_quantization_method(cls, hf_quant_cfg,
@@ -96,7 +102,8 @@ class MarlinConfig(QuantizationConfig):
102
103 def get_quant_method(
104 self, layer: torch.nn.Module) -> Optional["MarlinLinearMethod"]:
99 - if isinstance(layer, LinearBase):
105 + if (isinstance(layer, LinearBase) or
106 + (isinstance(layer, ParallelLMHead) and self.lm_head_quantized)):
107 return MarlinLinearMethod(self)
108 return None
109
vllm/model_executor/layers/vocab_parallel_embedding.py
+53 -17
@@ -8,6 +8,9 @@ from torch.nn.parameter import Parameter
8 from vllm.distributed import (divide, get_tensor_model_parallel_rank,
9 get_tensor_model_parallel_world_size,
10 tensor_model_parallel_all_reduce)
11 +from vllm.model_executor.layers.linear import UnquantizedLinearMethod
12 +from vllm.model_executor.layers.quantization.base_config import (
13 + QuantizationConfig, QuantizeMethodBase)
14 from vllm.model_executor.utils import set_weight_attrs
15
16 DEFAULT_VOCAB_PADDING_SIZE = 64
@@ -157,6 +160,7 @@ class VocabParallelEmbedding(torch.nn.Module):
160 params_dtype: type of the parameters.
161 org_num_embeddings: original vocabulary size (without LoRA).
162 padding_size: padding size for the vocabulary.
163 + quant_config: quant config for the layer
164 """ # noqa: E501
165
166 def __init__(self,
@@ -164,7 +168,8 @@ class VocabParallelEmbedding(torch.nn.Module):
168 embedding_dim: int,
169 params_dtype: Optional[torch.dtype] = None,
170 org_num_embeddings: Optional[int] = None,
167 - padding_size: int = DEFAULT_VOCAB_PADDING_SIZE):
171 + padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
172 + quant_config: Optional[QuantizationConfig] = None):
173 super().__init__()
174
175 # Keep the input dimensions.
@@ -187,6 +192,14 @@ class VocabParallelEmbedding(torch.nn.Module):
192 self.org_vocab_size, tp_rank,
193 self.tp_size)
194 self.embedding_dim = embedding_dim
195 +
196 + linear_method = None
197 + if quant_config is not None:
198 + linear_method = quant_config.get_quant_method(self)
199 + if linear_method is None:
200 + linear_method = UnquantizedLinearMethod()
201 + self.linear_method: QuantizeMethodBase = linear_method
202 +
203 if params_dtype is None:
204 params_dtype = torch.get_default_dtype()
205 # Divide the weight matrix along the vocaburaly dimension.
@@ -201,14 +214,14 @@ class VocabParallelEmbedding(torch.nn.Module):
214 self.num_added_embeddings_per_partition = (
215 self.shard_indices.added_vocab_end_index -
216 self.shard_indices.added_vocab_start_index)
204 - self.weight = Parameter(
205 - torch.empty(self.num_embeddings_per_partition,
206 - self.embedding_dim,
207 - dtype=params_dtype))
208 - set_weight_attrs(self.weight, {
209 - "parallel_dim": 0,
210 - "weight_loader": self.weight_loader
211 - })
217 +
218 + self.linear_method.create_weights(self,
219 + self.embedding_dim,
220 + [self.num_embeddings_per_partition],
221 + self.embedding_dim,
222 + self.num_embeddings_padded,
223 + params_dtype=params_dtype,
224 + weight_loader=self.weight_loader)
225
226 @classmethod
227 def _get_indices(cls, vocab_size_padded: int, org_vocab_size_padded: int,
@@ -288,10 +301,32 @@ class VocabParallelEmbedding(torch.nn.Module):
301 return ret
302
303 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
291 - parallel_dim = param.parallel_dim
292 - assert loaded_weight.shape[parallel_dim] == self.org_vocab_size
293 - loaded_weight = loaded_weight[self.shard_indices.org_vocab_start_index:
294 - self.shard_indices.org_vocab_end_index]
304 + output_dim = getattr(param, "output_dim", None)
305 + packed_dim = getattr(param, "packed_dim", None)
306 +
307 + # If parameter does not have output dim, then it should
308 + # be copied onto all gpus (e.g. g_idx for act_order gptq).
309 + if output_dim is None:
310 + assert param.data.shape == loaded_weight.shape
311 + param.data.copy_(loaded_weight)
312 + return
313 +
314 + # Shard indexes for loading the weight
315 + start_idx = self.shard_indices.org_vocab_start_index
316 + shard_size = self.shard_indices.org_vocab_end_index - start_idx
317 +
318 + # If param packed on the same dim we are sharding on, then
319 + # need to adjust offsets of loaded weight by pack_factor.
320 + if packed_dim is not None and packed_dim == output_dim:
321 + assert loaded_weight.shape[output_dim] == (self.org_vocab_size //
322 + param.pack_factor)
323 + start_idx = start_idx // param.pack_factor
324 + shard_size = shard_size // param.pack_factor
325 + else:
326 + assert loaded_weight.shape[output_dim] == self.org_vocab_size
327 +
328 + # Copy the data.
329 + loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
330 param[:loaded_weight.shape[0]].data.copy_(loaded_weight)
331 param[loaded_weight.shape[0]:].data.fill_(0)
332
@@ -346,16 +381,17 @@ class ParallelLMHead(VocabParallelEmbedding):
381 bias: bool = False,
382 params_dtype: Optional[torch.dtype] = None,
383 org_num_embeddings: Optional[int] = None,
349 - padding_size: int = DEFAULT_VOCAB_PADDING_SIZE):
384 + padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
385 + quant_config: Optional[QuantizationConfig] = None):
386 super().__init__(num_embeddings, embedding_dim, params_dtype,
351 - org_num_embeddings, padding_size)
387 + org_num_embeddings, padding_size, quant_config)
388 if bias:
389 self.bias = Parameter(
390 torch.empty(self.num_embeddings_per_partition,
391 dtype=params_dtype))
392 set_weight_attrs(self.bias, {
357 - "parallel_dim": 0,
358 - "weight_loader": self.weight_loader
393 + "output_dim": 0,
394 + "weight_loader": self.weight_loader,
395 })
396 else:
397 self.register_parameter("bias", None)
vllm/model_executor/models/arctic.py
+2 -1
@@ -412,6 +412,7 @@ class ArcticForCausalLM(nn.Module):
412 self.lm_head = ParallelLMHead(
413 self.vocab_size,
414 config.hidden_size,
415 + quant_config=quant_config,
416 )
417 self.num_experts = config.num_local_experts
418 self.num_experts_per_tok = config.num_experts_per_tok
@@ -434,7 +435,7 @@ class ArcticForCausalLM(nn.Module):
435
436 def compute_logits(self, hidden_states: torch.Tensor,
437 sampling_metadata: SamplingMetadata) -> torch.Tensor:
437 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
438 + logits = self.logits_processor(self.lm_head, hidden_states,
439 sampling_metadata)
440 return logits
441
vllm/model_executor/models/baichuan.py
+4 -2
@@ -328,7 +328,9 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
328 self.quant_config = quant_config
329 self.model = BaiChuanModel(config, position_embedding, cache_config,
330 quant_config)
331 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
331 + self.lm_head = ParallelLMHead(config.vocab_size,
332 + config.hidden_size,
333 + quant_config=quant_config)
334 self.logits_processor = LogitsProcessor(config.vocab_size)
335 self.sampler = Sampler()
336
@@ -346,7 +348,7 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
348
349 def compute_logits(self, hidden_states: torch.Tensor,
350 sampling_metadata: SamplingMetadata) -> torch.Tensor:
349 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
351 + logits = self.logits_processor(self.lm_head, hidden_states,
352 sampling_metadata)
353 return logits
354
vllm/model_executor/models/bloom.py
+2 -2
@@ -276,7 +276,7 @@ class BloomForCausalLM(nn.Module):
276 self.config = config
277 self.quant_config = quant_config
278 self.transformer = BloomModel(config, cache_config, quant_config)
279 - self.lm_head_weight = self.transformer.word_embeddings.weight
279 + self.lm_head = self.transformer.word_embeddings
280 self.logits_processor = LogitsProcessor(config.vocab_size)
281 self.sampler = Sampler()
282
@@ -294,7 +294,7 @@ class BloomForCausalLM(nn.Module):
294
295 def compute_logits(self, hidden_states: torch.Tensor,
296 sampling_metadata: SamplingMetadata) -> torch.Tensor:
297 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
297 + logits = self.logits_processor(self.lm_head, hidden_states,
298 sampling_metadata)
299 return logits
300
vllm/model_executor/models/chatglm.py
+4 -3
@@ -303,7 +303,8 @@ class ChatGLMModel(nn.Module):
303 self.encoder = GLMTransformer(config, cache_config, quant_config)
304
305 self.output_layer = ParallelLMHead(config.padded_vocab_size,
306 - config.hidden_size)
306 + config.hidden_size,
307 + quant_config=quant_config)
308
309 def forward(
310 self,
@@ -355,7 +356,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
356 self.max_position_embeddings = getattr(config, "max_sequence_length",
357 8192)
358 self.transformer = ChatGLMModel(config, cache_config, quant_config)
358 - self.lm_head_weight = self.transformer.output_layer.weight
359 + self.lm_head = self.transformer.output_layer
360 self.logits_processor = LogitsProcessor(config.padded_vocab_size)
361 self.sampler = Sampler()
362
@@ -373,7 +374,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
374
375 def compute_logits(self, hidden_states: torch.Tensor,
376 sampling_metadata: SamplingMetadata) -> torch.Tensor:
376 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
377 + logits = self.logits_processor(self.lm_head, hidden_states,
378 sampling_metadata)
379 return logits
380
vllm/model_executor/models/commandr.py
+4 -4
@@ -363,12 +363,12 @@ class CohereForCausalLM(nn.Module):
363 sampling_metadata: SamplingMetadata) -> torch.Tensor:
364 is_not_lora = hasattr(self.model.embed_tokens, 'weight')
365 if is_not_lora:
366 - embedding_weights = self.model.embed_tokens.weight
366 + logits = self.logits_processor(self.model.embed_tokens,
367 + hidden_states, sampling_metadata)
368 else:
368 - embedding_weights = self.model.embed_tokens.base_layer.weight
369 + logits = self.logits_processor(self.model.embed_tokens.base_layer,
370 + hidden_states, sampling_metadata)
371
370 - logits = self.logits_processor(embedding_weights, hidden_states,
371 - sampling_metadata)
372 return logits
373
374 def sample(
vllm/model_executor/models/dbrx.py
+2 -1
@@ -370,6 +370,7 @@ class DbrxForCausalLM(nn.Module):
370 config.d_model,
371 org_num_embeddings=config.vocab_size,
372 padding_size=DEFAULT_VOCAB_PADDING_SIZE,
373 + quant_config=quant_config,
374 )
375 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
376 config.vocab_size)
@@ -389,7 +390,7 @@ class DbrxForCausalLM(nn.Module):
390
391 def compute_logits(self, hidden_states: torch.Tensor,
392 sampling_metadata: SamplingMetadata) -> torch.Tensor:
392 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
393 + logits = self.logits_processor(self.lm_head, hidden_states,
394 sampling_metadata)
395 return logits
396
vllm/model_executor/models/deepseek.py
+4 -2
@@ -377,7 +377,9 @@ class DeepseekForCausalLM(nn.Module):
377 self.config = config
378 self.quant_config = quant_config
379 self.model = DeepseekModel(config, cache_config, quant_config)
380 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
380 + self.lm_head = ParallelLMHead(config.vocab_size,
381 + config.hidden_size,
382 + quant_config=quant_config)
383 self.logits_processor = LogitsProcessor(config.vocab_size)
384 self.sampler = Sampler()
385
@@ -395,7 +397,7 @@ class DeepseekForCausalLM(nn.Module):
397
398 def compute_logits(self, hidden_states: torch.Tensor,
399 sampling_metadata: SamplingMetadata) -> torch.Tensor:
398 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
400 + logits = self.logits_processor(self.lm_head, hidden_states,
401 sampling_metadata)
402 return logits
403
vllm/model_executor/models/deepseek_v2.py
+4 -2
@@ -465,7 +465,9 @@ class DeepseekV2ForCausalLM(nn.Module):
465 self.config = config
466 self.quant_config = quant_config
467 self.model = DeepseekV2Model(config, cache_config, quant_config)
468 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
468 + self.lm_head = ParallelLMHead(config.vocab_size,
469 + config.hidden_size,
470 + quant_config=quant_config)
471 self.logits_processor = LogitsProcessor(config.vocab_size)
472 self.sampler = Sampler()
473
@@ -483,7 +485,7 @@ class DeepseekV2ForCausalLM(nn.Module):
485
486 def compute_logits(self, hidden_states: torch.Tensor,
487 sampling_metadata: SamplingMetadata) -> torch.Tensor:
486 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
488 + logits = self.logits_processor(self.lm_head, hidden_states,
489 sampling_metadata)
490 return logits
491
vllm/model_executor/models/falcon.py
+3 -3
@@ -394,13 +394,13 @@ class FalconForCausalLM(nn.Module):
394 if config.tie_word_embeddings is not None
395 else True)
396 if self.tie_word_embeddings:
397 - self.lm_head_weight = self.transformer.word_embeddings.weight
397 + self.lm_head = self.transformer.word_embeddings
398 else:
399 self.lm_head = ParallelLMHead(
400 config.vocab_size,
401 config.hidden_size,
402 + quant_config=quant_config,
403 )
403 - self.lm_head_weight = self.lm_head.weight
404 self.logits_processor = LogitsProcessor(config.vocab_size)
405 self.sampler = Sampler()
406
@@ -422,7 +422,7 @@ class FalconForCausalLM(nn.Module):
422
423 def compute_logits(self, hidden_states: torch.Tensor,
424 sampling_metadata: SamplingMetadata) -> torch.Tensor:
425 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
425 + logits = self.logits_processor(self.lm_head, hidden_states,
426 sampling_metadata)
427 return logits
428
vllm/model_executor/models/gemma.py
+2 -2
@@ -347,8 +347,8 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
347
348 def compute_logits(self, hidden_states: torch.Tensor,
349 sampling_metadata: SamplingMetadata) -> torch.Tensor:
350 - logits = self.logits_processor(self.model.embed_tokens.weight,
351 - hidden_states, sampling_metadata)
350 + logits = self.logits_processor(self.model.embed_tokens, hidden_states,
351 + sampling_metadata)
352 return logits
353
354 def sample(
vllm/model_executor/models/gemma2.py
+2 -2
@@ -346,8 +346,8 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
346
347 def compute_logits(self, hidden_states: torch.Tensor,
348 sampling_metadata: SamplingMetadata) -> torch.Tensor:
349 - logits = self.logits_processor(self.model.embed_tokens.weight,
350 - hidden_states, sampling_metadata)
349 + logits = self.logits_processor(self.model.embed_tokens, hidden_states,
350 + sampling_metadata)
351 return logits
352
353 def sample(
vllm/model_executor/models/gpt2.py
+2 -2
@@ -238,7 +238,7 @@ class GPT2LMHeadModel(nn.Module):
238 self.config = config
239 self.quant_config = quant_config
240 self.transformer = GPT2Model(config, cache_config, quant_config)
241 - self.lm_head_weight = self.transformer.wte.weight
241 + self.lm_head = self.transformer.wte
242 self.logits_processor = LogitsProcessor(config.vocab_size)
243 self.sampler = Sampler()
244
@@ -256,7 +256,7 @@ class GPT2LMHeadModel(nn.Module):
256
257 def compute_logits(self, hidden_states: torch.Tensor,
258 sampling_metadata: SamplingMetadata) -> torch.Tensor:
259 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
259 + logits = self.logits_processor(self.lm_head, hidden_states,
260 sampling_metadata)
261 return logits
262
vllm/model_executor/models/gpt_bigcode.py
+2 -2
@@ -259,7 +259,7 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
259 self.quant_config = quant_config
260 self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
261 lora_config)
262 - self.lm_head_weight = self.transformer.wte.weight
262 + self.lm_head = self.transformer.wte
263 self.unpadded_vocab_size = config.vocab_size
264 if lora_config:
265 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
@@ -281,7 +281,7 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
281
282 def compute_logits(self, hidden_states: torch.Tensor,
283 sampling_metadata: SamplingMetadata) -> torch.Tensor:
284 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
284 + logits = self.logits_processor(self.lm_head, hidden_states,
285 sampling_metadata)
286 return logits
287
vllm/model_executor/models/gpt_j.py
+2 -1
@@ -229,6 +229,7 @@ class GPTJForCausalLM(nn.Module):
229 config.vocab_size,
230 config.n_embd,
231 bias=True,
232 + quant_config=quant_config,
233 )
234 self.logits_processor = LogitsProcessor(config.vocab_size)
235 self.sampler = Sampler()
@@ -247,7 +248,7 @@ class GPTJForCausalLM(nn.Module):
248
249 def compute_logits(self, hidden_states: torch.Tensor,
250 sampling_metadata: SamplingMetadata) -> torch.Tensor:
250 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
251 + logits = self.logits_processor(self.lm_head, hidden_states,
252 sampling_metadata, self.lm_head.bias)
253 return logits
254
vllm/model_executor/models/gpt_neox.py
+2 -1
@@ -241,6 +241,7 @@ class GPTNeoXForCausalLM(nn.Module):
241 self.embed_out = ParallelLMHead(
242 config.vocab_size,
243 config.hidden_size,
244 + quant_config=quant_config,
245 )
246 self.logits_processor = LogitsProcessor(config.vocab_size)
247 self.sampler = Sampler()
@@ -259,7 +260,7 @@ class GPTNeoXForCausalLM(nn.Module):
260
261 def compute_logits(self, hidden_states: torch.Tensor,
262 sampling_metadata: SamplingMetadata) -> torch.Tensor:
262 - logits = self.logits_processor(self.embed_out.weight, hidden_states,
263 + logits = self.logits_processor(self.embed_out, hidden_states,
264 sampling_metadata)
265 return logits
266
vllm/model_executor/models/internlm2.py
+4 -2
@@ -253,7 +253,9 @@ class InternLM2ForCausalLM(nn.Module):
253 self.config = config
254 self.quant_config = quant_config
255 self.model = InternLM2Model(config, cache_config, quant_config)
256 - self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
256 + self.output = ParallelLMHead(config.vocab_size,
257 + config.hidden_size,
258 + quant_config=quant_config)
259 self.logits_processor = LogitsProcessor(config.vocab_size)
260 self.sampler = Sampler()
261
@@ -271,7 +273,7 @@ class InternLM2ForCausalLM(nn.Module):
273
274 def compute_logits(self, hidden_states: torch.Tensor,
275 sampling_metadata: SamplingMetadata) -> torch.Tensor:
274 - logits = self.logits_processor(self.output.weight, hidden_states,
276 + logits = self.logits_processor(self.output, hidden_states,
277 sampling_metadata)
278 return logits
279
vllm/model_executor/models/jais.py
+2 -2
@@ -273,7 +273,7 @@ class JAISLMHeadModel(nn.Module):
273 self.config = config
274 self.quant_config = quant_config
275 self.transformer = JAISModel(config, cache_config, quant_config)
276 - self.lm_head_weight = self.transformer.wte.weight
276 + self.lm_head = self.transformer.wte
277 if hasattr(config, "width_scale"):
278 self.output_logits_scale = config.width_scale
279 else:
@@ -297,7 +297,7 @@ class JAISLMHeadModel(nn.Module):
297
298 def compute_logits(self, hidden_states: torch.Tensor,
299 sampling_metadata: SamplingMetadata) -> torch.Tensor:
300 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
300 + logits = self.logits_processor(self.lm_head, hidden_states,
301 sampling_metadata)
302 return logits
303
vllm/model_executor/models/llama.py
+2 -1
@@ -380,6 +380,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
380 # We need bigger padding if using lora for kernel
381 # compatibility
382 if not lora_config else lora_config.lora_vocab_padding_size,
383 + quant_config=quant_config,
384 )
385 if config.tie_word_embeddings:
386 self.lm_head.weight = self.model.embed_tokens.weight
@@ -403,7 +404,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
404
405 def compute_logits(self, hidden_states: torch.Tensor,
406 sampling_metadata: SamplingMetadata) -> torch.Tensor:
406 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
407 + logits = self.logits_processor(self.lm_head, hidden_states,
408 sampling_metadata)
409 return logits
410
vllm/model_executor/models/llava.py
+3 -2
@@ -125,7 +125,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
125 self.lm_head = ParallelLMHead(
126 self.unpadded_vocab_size,
127 config.text_config.hidden_size,
128 - org_num_embeddings=self.language_model.org_vocab_size)
128 + org_num_embeddings=self.language_model.org_vocab_size,
129 + quant_config=quant_config)
130 logit_scale = getattr(config, "logit_scale", 1.0)
131 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
132 config.vocab_size, logit_scale)
@@ -255,7 +256,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
256
257 def compute_logits(self, hidden_states: torch.Tensor,
258 sampling_metadata: SamplingMetadata) -> torch.Tensor:
258 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
259 + logits = self.logits_processor(self.lm_head, hidden_states,
260 sampling_metadata)
261 return logits
262
vllm/model_executor/models/llava_next.py
+3 -2
@@ -186,7 +186,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
186 self.lm_head = ParallelLMHead(
187 self.unpadded_vocab_size,
188 config.text_config.hidden_size,
189 - org_num_embeddings=self.language_model.org_vocab_size)
189 + org_num_embeddings=self.language_model.org_vocab_size,
190 + quant_config=quant_config)
191 logit_scale = getattr(config, "logit_scale", 1.0)
192 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
193 config.vocab_size, logit_scale)
@@ -438,7 +439,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
439
440 def compute_logits(self, hidden_states: torch.Tensor,
441 sampling_metadata: SamplingMetadata) -> torch.Tensor:
441 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
442 + logits = self.logits_processor(self.lm_head, hidden_states,
443 sampling_metadata)
444 return logits
445
vllm/model_executor/models/minicpm.py
+4 -3
@@ -449,6 +449,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
449 # We need bigger padding if using lora for kernel
450 # compatibility
451 if not lora_config else lora_config.lora_vocab_padding_size,
452 + quant_config=quant_config,
453 )
454 self.scale_width = self.config.hidden_size / self.config.dim_model_base
455
@@ -472,10 +473,10 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
473 sampling_metadata: SamplingMetadata) -> torch.Tensor:
474 hidden_states = hidden_states / self.scale_width
475 if self.config.tie_word_embeddings:
475 - lm_head_weight = self.model.embed_tokens.weight
476 + lm_head = self.model.embed_tokens
477 else:
477 - lm_head_weight = self.lm_head.weight
478 - logits = self.logits_processor(lm_head_weight, hidden_states,
478 + lm_head = self.lm_head
479 + logits = self.logits_processor(lm_head, hidden_states,
480 sampling_metadata)
481 return logits
482
vllm/model_executor/models/mixtral.py
+2 -1
@@ -331,6 +331,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
331 # We need bigger padding if using lora for kernel
332 # compatibility
333 if not lora_config else lora_config.lora_vocab_padding_size,
334 + quant_config=quant_config,
335 )
336 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
337 config.vocab_size)
@@ -350,7 +351,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
351
352 def compute_logits(self, hidden_states: torch.Tensor,
353 sampling_metadata: SamplingMetadata) -> torch.Tensor:
353 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
354 + logits = self.logits_processor(self.lm_head, hidden_states,
355 sampling_metadata)
356 return logits
357
vllm/model_executor/models/mixtral_quant.py
+4 -2
@@ -344,7 +344,9 @@ class MixtralForCausalLM(nn.Module):
344 self.config = config
345 self.quant_config = quant_config
346 self.model = MixtralModel(config, cache_config, quant_config)
347 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
347 + self.lm_head = ParallelLMHead(config.vocab_size,
348 + config.hidden_size,
349 + quant_config=quant_config)
350 self.logits_processor = LogitsProcessor(config.vocab_size)
351 self.sampler = Sampler()
352
@@ -362,7 +364,7 @@ class MixtralForCausalLM(nn.Module):
364
365 def compute_logits(self, hidden_states: torch.Tensor,
366 sampling_metadata: SamplingMetadata) -> torch.Tensor:
365 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
367 + logits = self.logits_processor(self.lm_head, hidden_states,
368 sampling_metadata)
369 return logits
370
vllm/model_executor/models/mlp_speculator.py
+4 -4
@@ -8,7 +8,7 @@ from vllm.model_executor import SamplingMetadata
8 from vllm.model_executor.layers.logits_processor import LogitsProcessor
9 from vllm.model_executor.layers.sampler import Sampler
10 from vllm.model_executor.layers.vocab_parallel_embedding import (
11 - VocabParallelEmbedding)
11 + ParallelLMHead, VocabParallelEmbedding)
12 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13 from vllm.sequence import SamplerOutput
14 from vllm.transformers_utils.configs import MLPSpeculatorConfig
@@ -87,7 +87,7 @@ class MLPSpeculator(nn.Module):
87 self.proj = nn.ModuleList([proj_first] + [proj_tied] *
88 (self.max_speculative_tokens - 1))
89
90 - head = nn.Linear(self.inner_dim, self.vocab_size, bias=False)
90 + head = ParallelLMHead(self.vocab_size, self.inner_dim, bias=False)
91 self.head = nn.ModuleList([head] * self.max_speculative_tokens)
92
93 ln = MLPSpeculatorLayerNorm(self.inner_dim,
@@ -169,8 +169,8 @@ class MLPSpeculator(nn.Module):
169 # TODO: not yet supporting top_k_tokens_per_head
170 previous_hidden_states = states
171
172 - logits = self.logits_processor(self.head[head_index].weight,
173 - states, sampling_metadata)
172 + logits = self.logits_processor(self.head[head_index], states,
173 + sampling_metadata)
174
175 output = self.sampler(logits.flatten(0, 1), sampling_metadata)
176 last_tokens = output.sampled_token_ids
vllm/model_executor/models/mpt.py
+2 -2
@@ -263,7 +263,7 @@ class MPTForCausalLM(nn.Module):
263 self.quant_config = quant_config
264
265 self.transformer = MPTModel(config, cache_config, quant_config)
266 - self.lm_head_weight = self.transformer.wte.weight
266 + self.lm_head = self.transformer.wte
267 self.logits_processor = LogitsProcessor(config.vocab_size)
268 self.sampler = Sampler()
269
@@ -281,7 +281,7 @@ class MPTForCausalLM(nn.Module):
281
282 def compute_logits(self, hidden_states: torch.Tensor,
283 sampling_metadata: SamplingMetadata) -> torch.Tensor:
284 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
284 + logits = self.logits_processor(self.lm_head, hidden_states,
285 sampling_metadata)
286 return logits
287
vllm/model_executor/models/olmo.py
+3 -3
@@ -283,15 +283,15 @@ class OlmoForCausalLM(nn.Module):
283 self.config = config
284 self.model = OlmoModel(config, cache_config, quant_config)
285 if config.tie_word_embeddings:
286 - self.lm_head_weight = self.model.embed_tokens.weight
286 + self.lm_head = self.model.embed_tokens
287 else:
288 self.unpadded_vocab_size = config.vocab_size
289 self.lm_head = ParallelLMHead(
290 self.unpadded_vocab_size,
291 config.hidden_size,
292 org_num_embeddings=config.vocab_size,
293 + quant_config=quant_config,
294 )
294 - self.lm_head_weight = self.lm_head.weight
295 self.logits_processor = LogitsProcessor(config.vocab_size)
296 self.sampler = Sampler()
297
@@ -313,7 +313,7 @@ class OlmoForCausalLM(nn.Module):
313
314 def compute_logits(self, hidden_states: torch.Tensor,
315 sampling_metadata: SamplingMetadata) -> torch.Tensor:
316 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
316 + logits = self.logits_processor(self.lm_head, hidden_states,
317 sampling_metadata)
318 return logits
319
vllm/model_executor/models/opt.py
+2 -2
@@ -294,7 +294,7 @@ class OPTForCausalLM(nn.Module):
294 self.config = config
295 self.quant_config = quant_config
296 self.model = OPTModel(config, cache_config, quant_config)
297 - self.lm_head_weight = self.model.decoder.embed_tokens.weight
297 + self.lm_head = self.model.decoder.embed_tokens
298 self.logits_processor = LogitsProcessor(config.vocab_size)
299 self.sampler = Sampler()
300
@@ -312,7 +312,7 @@ class OPTForCausalLM(nn.Module):
312
313 def compute_logits(self, hidden_states: torch.Tensor,
314 sampling_metadata: SamplingMetadata) -> torch.Tensor:
315 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
315 + logits = self.logits_processor(self.lm_head, hidden_states,
316 sampling_metadata)
317 return logits
318
vllm/model_executor/models/orion.py
+4 -2
@@ -259,7 +259,9 @@ class OrionForCausalLM(nn.Module):
259 self.config = config
260 self.quant_config = quant_config
261 self.model = OrionModel(config, cache_config, quant_config)
262 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
262 + self.lm_head = ParallelLMHead(config.vocab_size,
263 + config.hidden_size,
264 + quant_config=quant_config)
265 self.logits_processor = LogitsProcessor(config.vocab_size)
266 self.sampler = Sampler()
267
@@ -277,7 +279,7 @@ class OrionForCausalLM(nn.Module):
279
280 def compute_logits(self, hidden_states: torch.Tensor,
281 sampling_metadata: SamplingMetadata) -> torch.Tensor:
280 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
282 + logits = self.logits_processor(self.lm_head, hidden_states,
283 sampling_metadata)
284 return logits
285
vllm/model_executor/models/phi.py
+3 -2
@@ -268,7 +268,8 @@ class PhiForCausalLM(nn.Module, SupportsLoRA):
268
269 self.lm_head = ParallelLMHead(config.vocab_size,
270 config.hidden_size,
271 - bias=True)
271 + bias=True,
272 + quant_config=quant_config)
273 self.logits_processor = LogitsProcessor(config.vocab_size)
274 self.sampler = Sampler()
275
@@ -287,7 +288,7 @@ class PhiForCausalLM(nn.Module, SupportsLoRA):
288
289 def compute_logits(self, hidden_states: torch.Tensor,
290 sampling_metadata: SamplingMetadata) -> torch.Tensor:
290 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
291 + logits = self.logits_processor(self.lm_head, hidden_states,
292 sampling_metadata, self.lm_head.bias)
293 return logits
294
vllm/model_executor/models/phi3_small.py
+2 -1
@@ -366,6 +366,7 @@ class Phi3SmallForCausalLM(nn.Module):
366 config.hidden_size,
367 org_num_embeddings=config.vocab_size,
368 padding_size=DEFAULT_VOCAB_PADDING_SIZE,
369 + quant_config=quant_config,
370 )
371 self.logits_processor = LogitsProcessor(config.vocab_size)
372 self.sampler = Sampler()
@@ -400,7 +401,7 @@ class Phi3SmallForCausalLM(nn.Module):
401
402 def compute_logits(self, hidden_states: torch.Tensor,
403 sampling_metadata: SamplingMetadata) -> torch.Tensor:
403 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
404 + logits = self.logits_processor(self.lm_head, hidden_states,
405 sampling_metadata)
406 if self.dummy_token_indices is not None and logits is not None:
407 logits.index_fill_(-1, self.dummy_token_indices, -torch.inf)
vllm/model_executor/models/phi3v.py
+4 -2
@@ -365,7 +365,9 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
365 self.model = LlamaModel(config, cache_config, quant_config)
366 self.vision_embed_tokens = Phi3HDImageEmbedding(
367 vlm_config, config, self.model.embed_tokens)
368 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
368 + self.lm_head = ParallelLMHead(config.vocab_size,
369 + config.hidden_size,
370 + quant_config=quant_config)
371 self.logits_processor = LogitsProcessor(config.vocab_size)
372 self.sampler = Sampler()
373
@@ -409,7 +411,7 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
411
412 def compute_logits(self, hidden_states: torch.Tensor,
413 sampling_metadata: SamplingMetadata) -> torch.Tensor:
412 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
414 + logits = self.logits_processor(self.lm_head, hidden_states,
415 sampling_metadata)
416 return logits
417
vllm/model_executor/models/qwen.py
+4 -2
@@ -235,7 +235,9 @@ class QWenLMHeadModel(nn.Module):
235 self.config = config
236 self.quant_config = quant_config
237 self.transformer = QWenModel(config, cache_config, quant_config)
238 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
238 + self.lm_head = ParallelLMHead(config.vocab_size,
239 + config.hidden_size,
240 + quant_config=quant_config)
241 self.logits_processor = LogitsProcessor(config.vocab_size)
242 self.sampler = Sampler()
243
@@ -253,7 +255,7 @@ class QWenLMHeadModel(nn.Module):
255
256 def compute_logits(self, hidden_states: torch.Tensor,
257 sampling_metadata: SamplingMetadata) -> torch.Tensor:
256 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
258 + logits = self.logits_processor(self.lm_head, hidden_states,
259 sampling_metadata)
260 return logits
261
vllm/model_executor/models/qwen2.py
+4 -4
@@ -316,11 +316,11 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
316 self.model = Qwen2Model(config, cache_config, quant_config)
317
318 if config.tie_word_embeddings:
319 - self.lm_head_weight = self.model.embed_tokens.weight
319 + self.lm_head = self.model.embed_tokens
320 else:
321 self.lm_head = ParallelLMHead(config.vocab_size,
322 - config.hidden_size)
323 - self.lm_head_weight = self.lm_head.weight
322 + config.hidden_size,
323 + quant_config=quant_config)
324
325 self.logits_processor = LogitsProcessor(config.vocab_size)
326 self.sampler = Sampler()
@@ -339,7 +339,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
339
340 def compute_logits(self, hidden_states: torch.Tensor,
341 sampling_metadata: SamplingMetadata) -> torch.Tensor:
342 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
342 + logits = self.logits_processor(self.lm_head, hidden_states,
343 sampling_metadata)
344 return logits
345
vllm/model_executor/models/qwen2_moe.py
+4 -2
@@ -362,7 +362,9 @@ class Qwen2MoeForCausalLM(nn.Module):
362 self.config = config
363 self.quant_config = quant_config
364 self.model = Qwen2MoeModel(config, cache_config, quant_config)
365 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
365 + self.lm_head = ParallelLMHead(config.vocab_size,
366 + config.hidden_size,
367 + quant_config=quant_config)
368 self.logits_processor = LogitsProcessor(config.vocab_size)
369 self.sampler = Sampler()
370
@@ -380,7 +382,7 @@ class Qwen2MoeForCausalLM(nn.Module):
382
383 def compute_logits(self, hidden_states: torch.Tensor,
384 sampling_metadata: SamplingMetadata) -> torch.Tensor:
383 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
385 + logits = self.logits_processor(self.lm_head, hidden_states,
386 sampling_metadata)
387 return logits
388
vllm/model_executor/models/stablelm.py
+4 -2
@@ -240,7 +240,9 @@ class StablelmForCausalLM(nn.Module):
240 self.config = config
241 self.quant_config = quant_config
242 self.model = StableLMEpochModel(config, cache_config, quant_config)
243 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
243 + self.lm_head = ParallelLMHead(config.vocab_size,
244 + config.hidden_size,
245 + quant_config=quant_config)
246 self.logits_processor = LogitsProcessor(config.vocab_size)
247 self.sampler = Sampler()
248
@@ -258,7 +260,7 @@ class StablelmForCausalLM(nn.Module):
260
261 def compute_logits(self, hidden_states: torch.Tensor,
262 sampling_metadata: SamplingMetadata) -> torch.Tensor:
261 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
263 + logits = self.logits_processor(self.lm_head, hidden_states,
264 sampling_metadata)
265 return logits
266
vllm/model_executor/models/starcoder2.py
+3 -3
@@ -242,7 +242,7 @@ class Starcoder2ForCausalLM(nn.Module):
242 self.vocab_size = config.vocab_size
243 self.unpadded_vocab_size = config.vocab_size
244 if config.tie_word_embeddings:
245 - self.lm_head_weight = self.model.embed_tokens.weight
245 + self.lm_head = self.model.embed_tokens
246 else:
247 self.unpadded_vocab_size = config.vocab_size
248 self.lm_head = ParallelLMHead(
@@ -250,8 +250,8 @@ class Starcoder2ForCausalLM(nn.Module):
250 config.hidden_size,
251 org_num_embeddings=config.vocab_size,
252 padding_size=DEFAULT_VOCAB_PADDING_SIZE,
253 + quant_config=quant_config,
254 )
254 - self.lm_head_weight = self.lm_head.weight
255 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
256 config.vocab_size)
257 self.sampler = Sampler()
@@ -270,7 +270,7 @@ class Starcoder2ForCausalLM(nn.Module):
270
271 def compute_logits(self, hidden_states: torch.Tensor,
272 sampling_metadata: SamplingMetadata) -> torch.Tensor:
273 - logits = self.logits_processor(self.lm_head_weight, hidden_states,
273 + logits = self.logits_processor(self.lm_head, hidden_states,
274 sampling_metadata)
275 return logits
276
vllm/model_executor/models/xverse.py
+4 -2
@@ -310,7 +310,9 @@ class XverseForCausalLM(nn.Module, SupportsLoRA):
310
311 self.quant_config = quant_config
312 self.model = XverseModel(config, cache_config, quant_config)
313 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
313 + self.lm_head = ParallelLMHead(config.vocab_size,
314 + config.hidden_size,
315 + quant_config=quant_config)
316 self.logits_processor = LogitsProcessor(config.vocab_size)
317 self.sampler = Sampler()
318
@@ -328,7 +330,7 @@ class XverseForCausalLM(nn.Module, SupportsLoRA):
330
331 def compute_logits(self, hidden_states: torch.Tensor,
332 sampling_metadata: SamplingMetadata) -> torch.Tensor:
331 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
333 + logits = self.logits_processor(self.lm_head, hidden_states,
334 sampling_metadata)
335 return logits
336