A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 13:30 UTC
[CPU] Migrate cpu_awq into awq_marlin (#43841)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
May 28, 2026 at 22:36 UTC
20d69d100a967d51788bd852169afb516ba18434
7 files changed
+112
-401
.buildkite/hardware_tests/cpu.yaml
+3
-4
@@ -64,7 +64,7 @@ steps:
64
- vllm/v1/worker/gpu/
65
commands:
66
- |
67
- bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
67
+ bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
68
uv pip install git+https://github.com/triton-lang/triton-cpu.git@270e696d
69
VLLM_USE_V2_MODEL_RUNNER=1 pytest -x -v -s tests/models/language/generation/test_granite.py -m cpu_model"
70
@@ -74,11 +74,10 @@ steps:
74
no_plugin: true
75
source_file_dependencies:
76
- csrc/cpu/
77
- - vllm/model_executor/layers/quantization/cpu_wna16.py
77
- vllm/model_executor/layers/quantization/auto_gptq.py
78
- vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
80
- - vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py
81
- - vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py
79
+ - vllm/model_executor/kernels/linear/mixed_precision/cpu.py
80
+ - vllm/model_executor/kernels/linear/scaled_mm/cpu.py
81
- vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
82
- tests/quantization/test_compressed_tensors.py
83
- tests/quantization/test_cpu_wna16.py
vllm/config/model.py
-1
@@ -1028,7 +1028,6 @@ class ModelConfig:
1028
"mxfp4",
1029
"gpt_oss_mxfp4",
1030
"deepseek_v4_fp8",
1031
- "cpu_awq",
1031
"humming",
1032
"gguf",
1033
]
vllm/model_executor/kernels/linear/mixed_precision/cpu.py
+108
-37
@@ -4,6 +4,7 @@
4
import torch
5
6
from vllm import _custom_ops as ops
7
+from vllm import envs
8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
9
pack_quantized_values_into_int32,
10
unpack_quantized_values_into_int32,
@@ -58,18 +59,11 @@ class CPUWNA16LinearKernel(MPLinearKernel):
59
return True, None
60
61
# note assumes that
61
- # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0} (marlin)
62
- # or: {input_dim = 1, output_dim = 0, packed_dim = 1} (CT)
63
- # `weight_scale` is: {input_dim = 0, output_dim = 1} (marlin)
64
- # or: {input_dim = 1, output_dim = 0} (CT)
65
- # `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1} (marlin)
66
- # or: {input_dim = 1, output_dim = 0, packed_dim = 0} (CT)
67
- def _process_gptq_weights(self, layer: torch.nn.Module):
62
+ # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
63
+ # `weight_scale` is: {input_dim = 0, output_dim = 1}
64
+ # `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1}
65
+ def _process_gptq_weights_w4a16(self, layer: torch.nn.Module):
66
packed_weight = getattr(layer, self.w_q_name)
69
- assert packed_weight.input_dim == packed_weight.packed_dim
70
- is_ct_format = packed_weight.input_dim == 1
71
- if is_ct_format:
72
- packed_weight = packed_weight.t()
67
bits = self.config.weight_type.mantissa
68
pack_factor = 32 // bits
69
p_w_k, _ = packed_weight.size()
@@ -92,14 +86,59 @@ class CPUWNA16LinearKernel(MPLinearKernel):
86
)
87
getattr(layer, self.w_q_name).data = weight
88
95
- # transpose scale, zp for CT format
96
- if is_ct_format:
97
- scales = getattr(layer, self.w_s_name)
98
- scales.data = scales.t().contiguous()
99
- if self.config.zero_points:
100
- assert self.w_zp_name
101
- zp = getattr(layer, self.w_zp_name)
102
- zp.data = zp.t().contiguous()
89
+ # note assumes that
90
+ # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
91
+ # `weight_scale` is: {input_dim = 0, output_dim = 1}
92
+ # `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1}
93
+ def _process_gptq_weights_w4a8(self, layer: torch.nn.Module):
94
+ packed_weight = getattr(layer, self.w_q_name)
95
+ scales = getattr(layer, self.w_s_name)
96
+ group_num = scales.data.size(0)
97
+ zp_output_size = scales.data.size(1) // 8
98
+ if self.config.zero_points:
99
+ assert self.w_zp_name
100
+ packed_zp = getattr(layer, self.w_zp_name)
101
+ else:
102
+ # w4a8 kernel always requires zp, allocate a fake zp
103
+ assert self.w_zp_name
104
+ packed_zp = torch.nn.Parameter(
105
+ torch.ones(group_num, zp_output_size, dtype=torch.int32) * -2004318072,
106
+ requires_grad=False,
107
+ )
108
+ setattr(layer, self.w_zp_name, packed_zp)
109
+
110
+ # FIXME: some bugs in convert_weight_packed_scale_zp with GPTQ format,
111
+ # repack to AWQ weight
112
+ weight = unpack_quantized_values_into_int32(
113
+ packed_weight, self.config.weight_type, 0
114
+ )
115
+ input_size, output_size = weight.size()
116
+ weight = weight.view(input_size, output_size // 8, 8)
117
+ weight = weight[:, :, (0, 2, 4, 6, 1, 3, 5, 7)].reshape(input_size, output_size)
118
+ weight = pack_quantized_values_into_int32(
119
+ weight, self.config.weight_type, 1
120
+ ).contiguous()
121
+
122
+ zp = unpack_quantized_values_into_int32(packed_zp, self.config.weight_type, 1)
123
+ zp = zp.view(group_num, output_size // 8, 8)
124
+ zp = zp[:, :, (0, 2, 4, 6, 1, 3, 5, 7)].reshape(group_num, output_size)
125
+ zp = pack_quantized_values_into_int32(
126
+ zp, self.config.weight_type, 1
127
+ ).contiguous()
128
+
129
+ blocked_w, blocked_zp, blocked_s = ops.convert_weight_packed_scale_zp(
130
+ weight,
131
+ zp,
132
+ scales.data,
133
+ ops.CPUQuantAlgo.AWQ,
134
+ )
135
+
136
+ if layer.bias is not None:
137
+ layer.bias.data = layer.bias.float()
138
+
139
+ packed_weight.data = blocked_w
140
+ scales.data = blocked_s
141
+ packed_zp.data = blocked_zp
142
143
def process_weights_after_loading(self, layer: torch.nn.Module):
144
if (not self.config.zero_points) and (self.w_zp_name is not None):
@@ -108,16 +147,39 @@ class CPUWNA16LinearKernel(MPLinearKernel):
147
if (not self.config.has_g_idx) and (self.w_gidx_name is not None):
148
setattr(layer, self.w_gidx_name, None)
149
111
- w_input_dim = getattr(layer, self.w_q_name).input_dim
112
- w_pack_dim = getattr(layer, self.w_q_name).packed_dim
113
- quant_method = "gptq" if w_pack_dim == w_input_dim else "awq"
150
+ weights = getattr(layer, self.w_q_name)
151
+ # Require GPTQ pack format
152
+ assert weights.input_dim == weights.packed_dim
153
+
154
+ # Weights in CT format is [output_size, input_size]
155
+ if weights.input_dim == 1:
156
+ weights.data = weights.t()
157
+
158
+ # Scales in CT format is [output_size, group_num]
159
+ scales = getattr(layer, self.w_s_name)
160
+ if scales.output_dim == 0:
161
+ scales.data = scales.t().contiguous()
162
+
163
+ # Zero points in CT format is [output_size, group_num]
164
+ # Zero points in awq_marlin format is [output_size, group_num]
165
+ if self.config.zero_points:
166
+ assert self.w_zp_name
167
+ zp = getattr(layer, self.w_zp_name)
168
+ if zp.output_dim == 0:
169
+ zp.data = zp.t().contiguous()
170
115
- if quant_method == "gptq":
116
- # GPTQ
117
- self._process_gptq_weights(layer)
171
+ layer.use_w4a8 = (
172
+ envs.VLLM_CPU_INT4_W4A8
173
+ and not self.config.has_g_idx
174
+ and self.config.act_type == torch.bfloat16
175
+ and torch.cpu._is_amx_tile_supported()
176
+ )
177
+ # layer.use_w4a8 = False
178
+ # AWQ format will be converted to GPTQ format in `AWQMarlinLinearMethod`
179
+ if layer.use_w4a8:
180
+ self._process_gptq_weights_w4a8(layer)
181
else:
119
- # AWQ
120
- raise NotImplementedError("AWQ is not supported in CPUWNA16LinearKernel")
182
+ self._process_gptq_weights_w4a16(layer)
183
184
def apply_weights(
185
self,
@@ -126,16 +188,25 @@ class CPUWNA16LinearKernel(MPLinearKernel):
188
bias: torch.Tensor | None = None,
189
) -> torch.Tensor:
190
w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
129
- x = ops.cpu_gemm_wna16(
130
- input=x,
131
- q_weight=w_q,
132
- scales=w_s,
133
- zeros=w_zp,
134
- g_idx=w_gidx,
135
- bias=bias,
136
- pack_factor=8, # 32 // 4
137
- isa_hint=layer.isa_hint,
138
- )
191
+ if layer.use_w4a8:
192
+ x = ops.int4_scaled_mm_cpu(
193
+ x=x,
194
+ w=w_q,
195
+ w_zeros=w_zp,
196
+ w_scales=w_s,
197
+ bias=bias,
198
+ )
199
+ else:
200
+ x = ops.cpu_gemm_wna16(
201
+ input=x,
202
+ q_weight=w_q,
203
+ scales=w_s,
204
+ zeros=w_zp,
205
+ g_idx=w_gidx,
206
+ bias=bias,
207
+ pack_factor=8, # 32 // 4
208
+ isa_hint=layer.isa_hint,
209
+ )
210
return x
211
212
vllm/model_executor/layers/quantization/__init__.py
-3
@@ -34,7 +34,6 @@ QuantizationMethods = Literal[
34
"mxfp4",
35
"gpt_oss_mxfp4",
36
"deepseek_v4_fp8",
37
- "cpu_awq",
37
"online",
38
# Below are online quant shorthand names (see vllm.config.quantization).
39
# Listed here as strings to avoid a circular import; kept in sync with
@@ -122,7 +121,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
121
from .compressed_tensors.compressed_tensors import (
122
CompressedTensorsConfig,
123
)
125
- from .cpu_wna16 import CPUAWQConfig
124
from .experts_int8 import ExpertsInt8Config
125
from .fbgemm_fp8 import FBGEMMFp8Config
126
from .fp8 import Fp8Config
@@ -166,7 +164,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
164
"mxfp4": Mxfp4Config,
165
"gpt_oss_mxfp4": GptOssMxfp4Config,
166
"deepseek_v4_fp8": DeepseekV4FP8Config,
169
- "cpu_awq": CPUAWQConfig,
167
"humming": HummingConfig,
168
"online": OnlineQuantizationConfig,
169
}
vllm/model_executor/layers/quantization/awq_marlin.py
+1
-1
@@ -332,7 +332,7 @@ class AWQMarlinConfig(QuantizationConfig):
332
group_size = quant_config.get("group_size")
333
zero_point = quant_config.get("zero_point")
334
335
- if not current_platform.is_cuda_alike():
335
+ if not (current_platform.is_cuda_alike() or current_platform.is_cpu()):
336
return False
337
338
if quant_method != "awq":
vllm/model_executor/layers/quantization/cpu_wna16.py
deleted
-354
@@ -1,354 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from typing import Any
5
-
6
-import torch
7
-from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
8
-from transformers import PretrainedConfig
9
-
10
-import vllm.envs as envs
11
-from vllm import _custom_ops as ops
12
-from vllm.logger import init_logger
13
-from vllm.model_executor.layers.linear import (
14
- LinearBase,
15
- LinearMethodBase,
16
- UnquantizedLinearMethod,
17
-)
18
-from vllm.model_executor.layers.quantization import QuantizationMethods
19
-from vllm.model_executor.layers.quantization.base_config import (
20
- QuantizationConfig,
21
- QuantizeMethodBase,
22
-)
23
-from vllm.model_executor.layers.quantization.utils.quant_utils import (
24
- is_layer_skipped,
25
- pack_cols,
26
- unpack_cols,
27
-)
28
-from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
29
-from vllm.model_executor.models.utils import WeightsMapper
30
-from vllm.model_executor.parameter import (
31
- GroupQuantScaleParameter,
32
- PackedvLLMParameter,
33
-)
34
-from vllm.platforms import current_platform
35
-from vllm.transformers_utils.config import get_safetensors_params_metadata
36
-
37
-logger = init_logger(__name__)
38
-
39
-
40
-class CPUAWQConfig(QuantizationConfig):
41
- """Config class for CPU AWQ"""
42
-
43
- def __init__(
44
- self,
45
- weight_bits: int,
46
- group_size: int,
47
- zero_point: bool,
48
- lm_head_quantized: bool,
49
- modules_to_not_convert: list[str] | None,
50
- full_config: dict[str, Any],
51
- ) -> None:
52
- super().__init__()
53
- assert weight_bits == 4
54
- self.pack_factor = 32 // weight_bits # packed into int32
55
- self.group_size = group_size
56
- self.zero_point = zero_point
57
- self.lm_head_quantized = lm_head_quantized
58
- self.weight_bits = weight_bits
59
- self.modules_to_not_convert = modules_to_not_convert or []
60
- self.full_config = full_config
61
-
62
- def __repr__(self) -> str:
63
- return (
64
- f"AWQMarlinConfig("
65
- f"group_size={self.group_size}, "
66
- f"zero_point={self.zero_point}, "
67
- f"lm_head_quantized={self.lm_head_quantized}, "
68
- f"modules_to_not_convert={self.modules_to_not_convert})"
69
- )
70
-
71
- @classmethod
72
- def get_name(cls) -> "QuantizationMethods":
73
- return "cpu_awq"
74
-
75
- @classmethod
76
- def get_supported_act_dtypes(cls) -> list[torch.dtype]:
77
- return [torch.half, torch.bfloat16]
78
-
79
- @classmethod
80
- def get_min_capability(cls) -> int:
81
- return -1
82
-
83
- @classmethod
84
- def get_config_filenames(cls) -> list[str]:
85
- return ["quantize_config.json"]
86
-
87
- @classmethod
88
- def from_config(cls, config: dict[str, Any]) -> "CPUAWQConfig":
89
- weight_bits = cls.get_from_keys(config, ["bits"])
90
- group_size = cls.get_from_keys(config, ["group_size"])
91
- zero_point = cls.get_from_keys(config, ["zero_point"])
92
- lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False)
93
- modules_to_not_convert = cls.get_from_keys_or(
94
- config, ["modules_to_not_convert"], None
95
- )
96
- return cls(
97
- weight_bits,
98
- group_size,
99
- zero_point,
100
- lm_head_quantized,
101
- modules_to_not_convert,
102
- config,
103
- )
104
-
105
- @classmethod
106
- def override_quantization_method(
107
- cls, hf_quant_cfg, user_quant, hf_config=None
108
- ) -> "QuantizationMethods | None":
109
- quant_method = hf_quant_cfg.get("quant_method", "").lower()
110
- if current_platform.is_cpu() and (quant_method == "awq"):
111
- return cls.get_name()
112
- return None
113
-
114
- def get_quant_method(
115
- self, layer: torch.nn.Module, prefix: str
116
- ) -> "QuantizeMethodBase | None":
117
- if isinstance(layer, LinearBase) or (
118
- isinstance(layer, ParallelLMHead) and self.lm_head_quantized
119
- ):
120
- if is_layer_skipped(
121
- prefix,
122
- self.modules_to_not_convert,
123
- self.packed_modules_mapping,
124
- skip_with_substr=True,
125
- ):
126
- return UnquantizedLinearMethod()
127
- return CPUAWQLinearMethod(self)
128
- return None
129
-
130
- def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
131
- if self.modules_to_not_convert:
132
- self.modules_to_not_convert = hf_to_vllm_mapper.apply_list(
133
- self.modules_to_not_convert
134
- )
135
-
136
- def maybe_update_config(
137
- self,
138
- model_name: str,
139
- hf_config: PretrainedConfig | None = None,
140
- revision: str | None = None,
141
- ):
142
- if self.modules_to_not_convert:
143
- return
144
-
145
- unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]
146
- metadata = get_safetensors_params_metadata(model_name, revision=revision)
147
- layers = {param_name.rsplit(".", 1)[0] for param_name in metadata}
148
- quant_layers: set[str] = {
149
- param_name.rsplit(".", 1)[0]
150
- for param_name, info in metadata.items()
151
- if (dtype := info.get("dtype", None))
152
- and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes
153
- }
154
- self.modules_to_not_convert = list(layers - quant_layers)
155
-
156
-
157
-class CPUAWQLinearMethod(LinearMethodBase):
158
- """Linear method for CPU AWQ.
159
-
160
- Args:
161
- quant_config: The CPU AWQ quantization config.
162
- """
163
-
164
- def __init__(self, quant_config: CPUAWQConfig) -> None:
165
- self.quant_config = quant_config
166
- assert self.quant_config.zero_point
167
-
168
- def create_weights(
169
- self,
170
- layer: torch.nn.Module,
171
- input_size_per_partition: int,
172
- output_partition_sizes: list[int],
173
- input_size: int,
174
- output_size: int,
175
- params_dtype: torch.dtype,
176
- **extra_weight_attrs,
177
- ) -> None:
178
- del output_size
179
- output_size_per_partition = sum(output_partition_sizes)
180
- weight_loader = extra_weight_attrs.get("weight_loader")
181
-
182
- # Normalize group_size
183
- if self.quant_config.group_size != -1:
184
- group_size = self.quant_config.group_size
185
- else:
186
- group_size = input_size
187
-
188
- qweight = PackedvLLMParameter(
189
- data=torch.empty(
190
- input_size_per_partition,
191
- output_size_per_partition // self.quant_config.pack_factor,
192
- dtype=torch.int32,
193
- ),
194
- input_dim=0,
195
- output_dim=1,
196
- packed_dim=1,
197
- packed_factor=self.quant_config.pack_factor,
198
- weight_loader=weight_loader,
199
- )
200
-
201
- num_groups = input_size_per_partition // group_size
202
-
203
- qzeros = PackedvLLMParameter(
204
- data=torch.empty(
205
- num_groups,
206
- output_size_per_partition // self.quant_config.pack_factor,
207
- dtype=torch.int32,
208
- ),
209
- input_dim=0,
210
- output_dim=1,
211
- packed_dim=1,
212
- packed_factor=self.quant_config.pack_factor,
213
- weight_loader=weight_loader,
214
- )
215
-
216
- scales = GroupQuantScaleParameter(
217
- data=torch.empty(
218
- num_groups,
219
- output_size_per_partition,
220
- dtype=params_dtype,
221
- ),
222
- input_dim=0,
223
- output_dim=1,
224
- weight_loader=weight_loader,
225
- )
226
-
227
- layer.register_parameter("qweight", qweight)
228
- layer.register_parameter("qzeros", qzeros)
229
- layer.register_parameter("scales", scales)
230
-
231
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
232
- layer.use_w4a8 = envs.VLLM_CPU_INT4_W4A8 and torch.cpu._is_amx_tile_supported()
233
- if layer.use_w4a8:
234
- self._process_weights_sglang_int4(layer)
235
- else:
236
- self._process_weights_woq(layer)
237
-
238
- def _process_weights_woq(self, layer: torch.nn.Module) -> None:
239
- """Original WOQ int4 repack path."""
240
- packed_weight = layer.qweight.data
241
- packed_zeros = layer.qzeros.data
242
- group_num = packed_zeros.size(0)
243
- bits = self.quant_config.weight_bits
244
- pack_factor = int(self.quant_config.pack_factor)
245
- input_size, packed_output_size = packed_weight.size()
246
- output_size = packed_output_size * pack_factor
247
- isa_hint = _get_isa_hint(layer.scales.dtype)
248
- layer.isa_hint = isa_hint
249
-
250
- interleave_map = (0, 4, 1, 5, 2, 6, 3, 7)
251
- weight = unpack_cols(
252
- packed_weight,
253
- bits,
254
- input_size,
255
- output_size,
256
- )
257
- zeros = unpack_cols(
258
- packed_zeros,
259
- bits,
260
- group_num,
261
- output_size,
262
- )
263
- weight = (
264
- weight.view(input_size, -1, pack_factor)[:, :, interleave_map]
265
- .reshape(input_size, output_size)
266
- .contiguous()
267
- )
268
- zeros = (
269
- zeros.view(group_num, -1, pack_factor)[:, :, interleave_map]
270
- .reshape(group_num, output_size)
271
- .contiguous()
272
- )
273
-
274
- zeros = pack_cols(zeros, bits, group_num, output_size).contiguous()
275
- weight = pack_cols(weight, bits, input_size, output_size)
276
- weight = (
277
- weight.view(input_size, -1, 16 // pack_factor)
278
- .permute(1, 0, 2)
279
- .reshape(-1, input_size * 16 // pack_factor)
280
- .contiguous()
281
- )
282
- layer.qweight.data = weight
283
- layer.qzeros.data = zeros
284
-
285
- def _process_weights_sglang_int4(self, layer: torch.nn.Module) -> None:
286
- """SGLang INT4 W4A8 path: pack int4 weights with VNNI reordering."""
287
- packed_weight = layer.qweight.data
288
- packed_zeros = layer.qzeros.data
289
- scales = layer.scales.data
290
- blocked_w, blocked_zp, blocked_s = ops.convert_weight_packed_scale_zp(
291
- packed_weight,
292
- packed_zeros,
293
- scales,
294
- ops.CPUQuantAlgo.AWQ,
295
- )
296
-
297
- layer.packed_weight = blocked_w
298
- layer.packed_qzeros = blocked_zp
299
- layer.packed_scales = blocked_s
300
- layer.qweight = None
301
- layer.qzeros = None
302
- layer.scales = None
303
-
304
- def apply(
305
- self,
306
- layer: torch.nn.Module,
307
- x: torch.Tensor,
308
- bias: torch.Tensor | None = None,
309
- ) -> torch.Tensor:
310
- if layer.use_w4a8:
311
- return self._apply_sglang_int4(layer, x, bias)
312
- return self._apply_woq(layer, x, bias)
313
-
314
- def _apply_woq(
315
- self,
316
- layer: torch.nn.Module,
317
- x: torch.Tensor,
318
- bias: torch.Tensor | None = None,
319
- ) -> torch.Tensor:
320
- """Original WOQ int4 GEMM path."""
321
- x = ops.cpu_gemm_wna16(
322
- input=x,
323
- q_weight=layer.qweight,
324
- scales=layer.scales,
325
- zeros=layer.qzeros,
326
- g_idx=None,
327
- bias=bias,
328
- pack_factor=8,
329
- isa_hint=layer.isa_hint,
330
- )
331
- return x
332
-
333
- def _apply_sglang_int4(
334
- self,
335
- layer: torch.nn.Module,
336
- x: torch.Tensor,
337
- bias: torch.Tensor | None = None,
338
- ) -> torch.Tensor:
339
- """SGLang INT4 W4A8 GEMM path."""
340
- return ops.int4_scaled_mm_cpu(
341
- x,
342
- layer.packed_weight,
343
- layer.packed_qzeros,
344
- layer.packed_scales,
345
- bias,
346
- )
347
-
348
-
349
-def _get_isa_hint(dtype: torch.dtype) -> str:
350
- supports_amx = torch.cpu._is_amx_tile_supported()
351
- if supports_amx and dtype in (torch.bfloat16,):
352
- return "amx"
353
- else:
354
- return "vec"
vllm/v1/metrics/perf.py
-1
@@ -72,7 +72,6 @@ _QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = {
72
"quark": 0.5,
73
"moe_wna16": 0.5,
74
"inc": 0.5,
75
- "cpu_awq": 0.5,
75
"experts_int8": 1,
76
}
77