A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[misc] update tpu int8 to use new vLLM Parameters (#7973)
Dipika Sikka committed
Aug 29, 2024 at 16:46 UTC
86a677de42e83940c4fd55daa0f48d974e5e2c53
2 files changed
+13
-11
vllm/model_executor/layers/linear.py
+2
-1
@@ -23,7 +23,8 @@ logger = init_logger(__name__)
23
WEIGHT_LOADER_V2_SUPPORTED = [
24
"CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
25
"AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
26
- "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod"
26
+ "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
27
+ "TPUInt8LinearMethod"
28
]
29
30
vllm/model_executor/layers/quantization/tpu_int8.py
+11
-10
@@ -7,7 +7,7 @@ from torch.nn.parameter import Parameter
7
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8
from vllm.model_executor.layers.quantization.base_config import (
9
QuantizationConfig)
10
-from vllm.model_executor.utils import set_weight_attrs
10
+from vllm.model_executor.parameter import ModelWeightParameter
11
12
ACTIVATION_SCHEMES = ["none"]
13
@@ -64,16 +64,16 @@ class TPUInt8LinearMethod(LinearMethodBase):
64
output_partition_sizes: List[int], input_size: int,
65
output_size: int, params_dtype: torch.dtype,
66
**extra_weight_attrs):
67
- weight = Parameter(torch.empty(sum(output_partition_sizes),
68
- input_size_per_partition,
69
- dtype=params_dtype),
70
- requires_grad=False)
67
+
68
+ weight_loader = extra_weight_attrs.get("weight_loader")
69
+ weight = ModelWeightParameter(data=torch.empty(
70
+ sum(output_partition_sizes),
71
+ input_size_per_partition,
72
+ dtype=params_dtype),
73
+ input_dim=1,
74
+ output_dim=0,
75
+ weight_loader=weight_loader)
76
layer.register_parameter("weight", weight)
72
- set_weight_attrs(weight, {
73
- **extra_weight_attrs,
74
- "input_dim": 1,
75
- "output_dim": 0,
76
- })
77
78
def _quantize_weight(
79
self, weight: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -92,6 +92,7 @@ class TPUInt8LinearMethod(LinearMethodBase):
92
return qweight, qscale
93
94
def process_weights_after_loading(self, layer: Module) -> None:
95
+ layer.weight = Parameter(layer.weight.data, requires_grad=False)
96
device = layer.weight.device
97
qweight, qscale = self._quantize_weight(layer.weight)
98
qweight = qweight.to(device)