@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 86a677de42
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[misc] update tpu int8 to use new vLLM Parameters (#7973)

Dipika Sikka committed Aug 29, 2024 at 16:46 UTC 86a677de42e83940c4fd55daa0f48d974e5e2c53
2 files changed +13 -11
vllm/model_executor/layers/linear.py
+2 -1
@@ -23,7 +23,8 @@ logger = init_logger(__name__)
23 WEIGHT_LOADER_V2_SUPPORTED = [
24 "CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
25 "AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
26 - "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod"
26 + "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
27 + "TPUInt8LinearMethod"
28 ]
29
30
vllm/model_executor/layers/quantization/tpu_int8.py
+11 -10
@@ -7,7 +7,7 @@ from torch.nn.parameter import Parameter
7 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 from vllm.model_executor.layers.quantization.base_config import (
9 QuantizationConfig)
10 -from vllm.model_executor.utils import set_weight_attrs
10 +from vllm.model_executor.parameter import ModelWeightParameter
11
12 ACTIVATION_SCHEMES = ["none"]
13
@@ -64,16 +64,16 @@ class TPUInt8LinearMethod(LinearMethodBase):
64 output_partition_sizes: List[int], input_size: int,
65 output_size: int, params_dtype: torch.dtype,
66 **extra_weight_attrs):
67 - weight = Parameter(torch.empty(sum(output_partition_sizes),
68 - input_size_per_partition,
69 - dtype=params_dtype),
70 - requires_grad=False)
67 +
68 + weight_loader = extra_weight_attrs.get("weight_loader")
69 + weight = ModelWeightParameter(data=torch.empty(
70 + sum(output_partition_sizes),
71 + input_size_per_partition,
72 + dtype=params_dtype),
73 + input_dim=1,
74 + output_dim=0,
75 + weight_loader=weight_loader)
76 layer.register_parameter("weight", weight)
72 - set_weight_attrs(weight, {
73 - **extra_weight_attrs,
74 - "input_dim": 1,
75 - "output_dim": 0,
76 - })
77
78 def _quantize_weight(
79 self, weight: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -92,6 +92,7 @@ class TPUInt8LinearMethod(LinearMethodBase):
92 return qweight, qscale
93
94 def process_weights_after_loading(self, layer: Module) -> None:
95 + layer.weight = Parameter(layer.weight.data, requires_grad=False)
96 device = layer.weight.device
97 qweight, qscale = self._quantize_weight(layer.weight)
98 qweight = qweight.to(device)