A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Misc] Update compressed tensors lifecycle to remove `prefix` from `create_weights` (#7825)
Dipika Sikka committed
Aug 26, 2024 at 20:09 UTC
015e6cc252450920411a6779fbb9e631c3698de1
4 files changed
+17
-75
vllm/model_executor/layers/linear.py
+3
-6
@@ -208,8 +208,7 @@ class ReplicatedLinear(LinearBase):
208
self.input_size,
209
self.output_size,
210
self.params_dtype,
211
- weight_loader=self.weight_loader,
212
- prefix=prefix)
211
+ weight_loader=self.weight_loader)
212
213
if bias:
214
self.bias = Parameter(
@@ -307,8 +306,7 @@ class ColumnParallelLinear(LinearBase):
306
params_dtype=self.params_dtype,
307
weight_loader=(
308
self.weight_loader_v2 if self.quant_method.__class__.__name__
310
- in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader),
311
- prefix=prefix)
309
+ in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader))
310
if bias:
311
self.bias = Parameter(
312
torch.empty(self.output_size_per_partition,
@@ -976,8 +974,7 @@ class RowParallelLinear(LinearBase):
974
params_dtype=self.params_dtype,
975
weight_loader=(
976
self.weight_loader_v2 if self.quant_method.__class__.__name__
979
- in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader),
980
- prefix=prefix)
977
+ in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader))
978
if not reduce_results and (bias and not skip_bias_add):
979
raise ValueError("When not reduce the results, adding bias to the "
980
"results can lead to incorrect results")
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+14
-18
@@ -3,15 +3,15 @@ from typing import Any, Dict, List, Optional
3
import torch
4
from pydantic import BaseModel
5
6
-from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
6
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
7
+ UnquantizedLinearMethod)
8
from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
9
QuantizationConfig, QuantizeMethodBase)
10
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
11
W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11
- CompressedTensorsScheme, CompressedTensorsUnquantized,
12
- CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8Fp8,
13
- CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
14
- CompressedTensorsWNA16)
12
+ CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
13
+ CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
14
+ CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
15
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
16
CompressionFormat, QuantizationArgs, QuantizationStrategy,
17
QuantizationType, find_matched_target, is_activation_quantization_format,
@@ -52,15 +52,20 @@ class CompressedTensorsConfig(QuantizationConfig):
52
def get_name(self) -> str:
53
return "compressed_tensors"
54
55
- # TODO (@robertgshaw2-neuralmagic): do layer skipping though here
56
- # rather than though create_weights to match other methods
55
def get_quant_method(
56
self,
57
layer: torch.nn.Module,
58
prefix: str,
59
) -> Optional["QuantizeMethodBase"]:
60
from vllm.attention.layer import Attention # Avoid circular import
61
+
62
+ # Check if the layer is skipped for quantization.
63
+ # TODO (@robertgshaw2): support module names
64
+ if should_ignore_layer(prefix, ignore=self.ignore):
65
+ return UnquantizedLinearMethod()
66
if isinstance(layer, LinearBase):
67
+ scheme = self.get_scheme(layer=layer, layer_name=prefix)
68
+ layer.scheme = scheme
69
return CompressedTensorsLinearMethod(self)
70
if isinstance(layer, Attention):
71
return CompressedTensorsKVCacheMethod(self)
@@ -281,15 +286,11 @@ class CompressedTensorsConfig(QuantizationConfig):
286
to select the CompressedTensorsScheme used for infernece.
287
"""
288
284
- # Check if the layer is skipped for quantization.
285
- # TODO (@robertgshaw2): support module names
286
- if should_ignore_layer(layer_name, ignore=self.ignore):
287
- return CompressedTensorsUnquantized()
288
-
289
# Find the "target" in the compressed-tensors config
290
# that our layer conforms to.
291
# TODO (@robertgshaw): add compressed-tensors as dep
292
# so we do not have to re-write these functions
293
+ # need to make accelerate optional in ct to do this
294
matched_target = find_matched_target(
295
layer_name=layer_name,
296
module=layer,
@@ -327,10 +328,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
328
details
329
"""
330
weight_loader = extra_weight_attrs.get("weight_loader")
330
- layer_name = extra_weight_attrs.get("prefix")
331
-
332
- scheme = self.quantization_config.get_scheme(layer, layer_name)
333
- scheme.create_weights(
331
+ layer.scheme.create_weights(
332
layer=layer,
333
input_size=input_size,
334
input_size_per_partition=input_size_per_partition,
@@ -339,8 +337,6 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
337
params_dtype=params_dtype,
338
weight_loader=weight_loader)
339
342
- layer.scheme = scheme
343
-
340
def apply(self,
341
layer: torch.nn.Module,
342
x: torch.Tensor,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
-2
@@ -1,5 +1,4 @@
1
from .compressed_tensors_scheme import CompressedTensorsScheme
2
-from .compressed_tensors_unquantized import CompressedTensorsUnquantized
2
from .compressed_tensors_w4a16_24 import (W4A16SPARSE24_SUPPORTED_BITS,
3
CompressedTensorsW4A16Sparse24)
4
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
@@ -10,7 +9,6 @@ from .compressed_tensors_wNa16 import (WNA16_SUPPORTED_BITS,
9
10
__all__ = [
11
"CompressedTensorsScheme",
13
- "CompressedTensorsUnquantized",
12
"CompressedTensorsWNA16",
13
"CompressedTensorsW8A16Fp8",
14
"CompressedTensorsW4A16Sparse24",
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
deleted
-49
@@ -1,49 +0,0 @@
1
-from typing import Callable, List, Optional
2
-
3
-import torch
4
-import torch.nn.functional as F
5
-
6
-from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7
- CompressedTensorsScheme)
8
-from vllm.model_executor.parameter import ModelWeightParameter
9
-
10
-__all__ = ["CompressedTensorsUnquantized"]
11
-
12
-
13
-class CompressedTensorsUnquantized(CompressedTensorsScheme):
14
- """
15
- Implements the scheme for all layers which are ignored
16
- in the CompressedTensors config. The input and loaded weight are used
17
- in a linear transformation.
18
- """
19
-
20
- @classmethod
21
- def get_min_capability(cls) -> int:
22
- # volta and up
23
- return 70
24
-
25
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
26
- # required by torch.compile to be torch.nn.Parameter
27
- layer.weight = torch.nn.Parameter(layer.weight.data,
28
- requires_grad=False)
29
-
30
- def create_weights(self, layer: torch.nn.Module,
31
- output_partition_sizes: List[int],
32
- input_size_per_partition: int,
33
- params_dtype: torch.dtype, weight_loader: Callable,
34
- **kwargs):
35
-
36
- weight = ModelWeightParameter(data=torch.empty(
37
- sum(output_partition_sizes),
38
- input_size_per_partition,
39
- dtype=params_dtype),
40
- input_dim=1,
41
- output_dim=0,
42
- weight_loader=weight_loader)
43
-
44
- layer.register_parameter("weight", weight)
45
-
46
- def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
47
- bias: Optional[torch.Tensor]) -> torch.Tensor:
48
-
49
- return F.linear(x, layer.weight, bias)