@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 015e6cc252
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Misc] Update compressed tensors lifecycle to remove `prefix` from `create_weights` (#7825)

Dipika Sikka committed Aug 26, 2024 at 20:09 UTC 015e6cc252450920411a6779fbb9e631c3698de1
4 files changed +17 -75
vllm/model_executor/layers/linear.py
+3 -6
@@ -208,8 +208,7 @@ class ReplicatedLinear(LinearBase):
208 self.input_size,
209 self.output_size,
210 self.params_dtype,
211 - weight_loader=self.weight_loader,
212 - prefix=prefix)
211 + weight_loader=self.weight_loader)
212
213 if bias:
214 self.bias = Parameter(
@@ -307,8 +306,7 @@ class ColumnParallelLinear(LinearBase):
306 params_dtype=self.params_dtype,
307 weight_loader=(
308 self.weight_loader_v2 if self.quant_method.__class__.__name__
310 - in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader),
311 - prefix=prefix)
309 + in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader))
310 if bias:
311 self.bias = Parameter(
312 torch.empty(self.output_size_per_partition,
@@ -976,8 +974,7 @@ class RowParallelLinear(LinearBase):
974 params_dtype=self.params_dtype,
975 weight_loader=(
976 self.weight_loader_v2 if self.quant_method.__class__.__name__
979 - in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader),
980 - prefix=prefix)
977 + in WEIGHT_LOADER_V2_SUPPORTED else self.weight_loader))
978 if not reduce_results and (bias and not skip_bias_add):
979 raise ValueError("When not reduce the results, adding bias to the "
980 "results can lead to incorrect results")
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+14 -18
@@ -3,15 +3,15 @@ from typing import Any, Dict, List, Optional
3 import torch
4 from pydantic import BaseModel
5
6 -from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
6 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
7 + UnquantizedLinearMethod)
8 from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
9 QuantizationConfig, QuantizeMethodBase)
10 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
11 W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11 - CompressedTensorsScheme, CompressedTensorsUnquantized,
12 - CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8Fp8,
13 - CompressedTensorsW8A8Int8, CompressedTensorsW8A16Fp8,
14 - CompressedTensorsWNA16)
12 + CompressedTensorsScheme, CompressedTensorsW4A16Sparse24,
13 + CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
14 + CompressedTensorsW8A16Fp8, CompressedTensorsWNA16)
15 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
16 CompressionFormat, QuantizationArgs, QuantizationStrategy,
17 QuantizationType, find_matched_target, is_activation_quantization_format,
@@ -52,15 +52,20 @@ class CompressedTensorsConfig(QuantizationConfig):
52 def get_name(self) -> str:
53 return "compressed_tensors"
54
55 - # TODO (@robertgshaw2-neuralmagic): do layer skipping though here
56 - # rather than though create_weights to match other methods
55 def get_quant_method(
56 self,
57 layer: torch.nn.Module,
58 prefix: str,
59 ) -> Optional["QuantizeMethodBase"]:
60 from vllm.attention.layer import Attention # Avoid circular import
61 +
62 + # Check if the layer is skipped for quantization.
63 + # TODO (@robertgshaw2): support module names
64 + if should_ignore_layer(prefix, ignore=self.ignore):
65 + return UnquantizedLinearMethod()
66 if isinstance(layer, LinearBase):
67 + scheme = self.get_scheme(layer=layer, layer_name=prefix)
68 + layer.scheme = scheme
69 return CompressedTensorsLinearMethod(self)
70 if isinstance(layer, Attention):
71 return CompressedTensorsKVCacheMethod(self)
@@ -281,15 +286,11 @@ class CompressedTensorsConfig(QuantizationConfig):
286 to select the CompressedTensorsScheme used for infernece.
287 """
288
284 - # Check if the layer is skipped for quantization.
285 - # TODO (@robertgshaw2): support module names
286 - if should_ignore_layer(layer_name, ignore=self.ignore):
287 - return CompressedTensorsUnquantized()
288 -
289 # Find the "target" in the compressed-tensors config
290 # that our layer conforms to.
291 # TODO (@robertgshaw): add compressed-tensors as dep
292 # so we do not have to re-write these functions
293 + # need to make accelerate optional in ct to do this
294 matched_target = find_matched_target(
295 layer_name=layer_name,
296 module=layer,
@@ -327,10 +328,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
328 details
329 """
330 weight_loader = extra_weight_attrs.get("weight_loader")
330 - layer_name = extra_weight_attrs.get("prefix")
331 -
332 - scheme = self.quantization_config.get_scheme(layer, layer_name)
333 - scheme.create_weights(
331 + layer.scheme.create_weights(
332 layer=layer,
333 input_size=input_size,
334 input_size_per_partition=input_size_per_partition,
@@ -339,8 +337,6 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
337 params_dtype=params_dtype,
338 weight_loader=weight_loader)
339
342 - layer.scheme = scheme
343 -
340 def apply(self,
341 layer: torch.nn.Module,
342 x: torch.Tensor,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
-2
@@ -1,5 +1,4 @@
1 from .compressed_tensors_scheme import CompressedTensorsScheme
2 -from .compressed_tensors_unquantized import CompressedTensorsUnquantized
2 from .compressed_tensors_w4a16_24 import (W4A16SPARSE24_SUPPORTED_BITS,
3 CompressedTensorsW4A16Sparse24)
4 from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
@@ -10,7 +9,6 @@ from .compressed_tensors_wNa16 import (WNA16_SUPPORTED_BITS,
9
10 __all__ = [
11 "CompressedTensorsScheme",
13 - "CompressedTensorsUnquantized",
12 "CompressedTensorsWNA16",
13 "CompressedTensorsW8A16Fp8",
14 "CompressedTensorsW4A16Sparse24",
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py deleted
-49
@@ -1,49 +0,0 @@
1 -from typing import Callable, List, Optional
2 -
3 -import torch
4 -import torch.nn.functional as F
5 -
6 -from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
7 - CompressedTensorsScheme)
8 -from vllm.model_executor.parameter import ModelWeightParameter
9 -
10 -__all__ = ["CompressedTensorsUnquantized"]
11 -
12 -
13 -class CompressedTensorsUnquantized(CompressedTensorsScheme):
14 - """
15 - Implements the scheme for all layers which are ignored
16 - in the CompressedTensors config. The input and loaded weight are used
17 - in a linear transformation.
18 - """
19 -
20 - @classmethod
21 - def get_min_capability(cls) -> int:
22 - # volta and up
23 - return 70
24 -
25 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
26 - # required by torch.compile to be torch.nn.Parameter
27 - layer.weight = torch.nn.Parameter(layer.weight.data,
28 - requires_grad=False)
29 -
30 - def create_weights(self, layer: torch.nn.Module,
31 - output_partition_sizes: List[int],
32 - input_size_per_partition: int,
33 - params_dtype: torch.dtype, weight_loader: Callable,
34 - **kwargs):
35 -
36 - weight = ModelWeightParameter(data=torch.empty(
37 - sum(output_partition_sizes),
38 - input_size_per_partition,
39 - dtype=params_dtype),
40 - input_dim=1,
41 - output_dim=0,
42 - weight_loader=weight_loader)
43 -
44 - layer.register_parameter("weight", weight)
45 -
46 - def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
47 - bias: Optional[torch.Tensor]) -> torch.Tensor:
48 -
49 - return F.linear(x, layer.weight, bias)