A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 21:48 UTC
[Quantization][Deprecation] Remove `DeepSpeedFp8` (#32679)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
Robert Shaw committed
Jan 21, 2026 at 09:32 UTC
cea3c754c461ec3792e53234042a8896253bd1d7
5 files changed
+19
-284
examples/offline_inference/load_sharded_state.py
-2
@@ -9,14 +9,12 @@ Example usage:
9
10
python save_sharded_state.py \
11
--model /path/to/load \
12
- --quantization deepspeedfp \
12
--tensor-parallel-size 8 \
13
--output /path/to/save/sharded/model
14
15
python load_sharded_state.py \
16
--model /path/to/saved/sharded/model \
17
--load-format sharded_state \
19
- --quantization deepspeedfp \
18
--tensor-parallel-size 8 \
19
--prompt "Hello, my name is" \
20
--max-tokens 50
examples/offline_inference/save_sharded_state.py
-2
@@ -9,7 +9,6 @@ Example usage:
9
10
python save_sharded_state.py \
11
--model /path/to/load \
12
- --quantization deepspeedfp \
12
--tensor-parallel-size 8 \
13
--output /path/to/save
14
@@ -18,7 +17,6 @@ Then, the model can be loaded with
17
llm = LLM(
18
model="/path/to/save",
19
load_format="sharded_state",
21
- quantization="deepspeedfp",
20
tensor_parallel_size=8,
21
)
22
"""
vllm/model_executor/layers/quantization/__init__.py
-4
@@ -11,7 +11,6 @@ logger = init_logger(__name__)
11
12
QuantizationMethods = Literal[
13
"awq",
14
- "deepspeedfp",
14
"fp8",
15
"ptpc_fp8",
16
"fbgemm_fp8",
@@ -42,7 +41,6 @@ QuantizationMethods = Literal[
41
QUANTIZATION_METHODS: list[str] = list(get_args(QuantizationMethods))
42
43
DEPRECATED_QUANTIZATION_METHODS = [
45
- "deepspeedfp",
44
"tpu_int8",
45
"ptpc_fp8",
46
"fbgemm_fp8",
@@ -126,7 +124,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
124
CompressedTensorsConfig,
125
)
126
from .cpu_wna16 import CPUAWQConfig
129
- from .deepspeedfp import DeepSpeedFPConfig
127
from .experts_int8 import ExpertsInt8Config
128
from .fbgemm_fp8 import FBGEMMFp8Config
129
from .fp8 import Fp8Config
@@ -149,7 +146,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
146
147
method_to_config: dict[str, type[QuantizationConfig]] = {
148
"awq": AWQConfig,
152
- "deepspeedfp": DeepSpeedFPConfig,
149
"fp8": Fp8Config,
150
"fbgemm_fp8": FBGEMMFp8Config,
151
"fp_quant": FPQuantConfig,
vllm/model_executor/layers/quantization/deepspeedfp.py
deleted
-218
@@ -1,218 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from typing import Any, Optional
5
-
6
-import torch
7
-import torch.nn as nn
8
-import torch.nn.functional as F
9
-from packaging import version
10
-
11
-from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12
-from vllm.model_executor.layers.quantization import (
13
- QuantizationConfig,
14
- QuantizationMethods,
15
-)
16
-from vllm.model_executor.utils import set_weight_attrs
17
-
18
-
19
-class DeepSpeedFPConfig(QuantizationConfig):
20
- """Config for DeepSpeed FP quantizer. It supports fp6 and fp8.
21
-
22
- Args:
23
- weight_bits: the target quantization bits, 6 or 8.
24
- group_size: group size for quantizaiton, default to 128.
25
- """
26
-
27
- def __init__(
28
- self,
29
- weight_bits: int = 8,
30
- group_size: int = 512,
31
- ) -> None:
32
- super().__init__()
33
- self.weight_bits = weight_bits
34
- self.group_size = group_size
35
- self.valid_types = [torch.bfloat16, torch.float16]
36
-
37
- if self.weight_bits not in (6, 8):
38
- raise ValueError(
39
- "Currently, only 6-bit or 8-bit weight quantization are "
40
- f"supported for DeepSpeed FP quantizaiton, but got "
41
- f"{self.weight_bits} bits."
42
- )
43
-
44
- def __repr__(self) -> str:
45
- return (
46
- f"DeepSpeedFPConfig(weight_bits={self.weight_bits}), "
47
- f"group_size={self.group_size}"
48
- )
49
-
50
- @classmethod
51
- def get_name(cls) -> QuantizationMethods:
52
- return "deepspeedfp"
53
-
54
- @classmethod
55
- def from_config(cls, config: dict[str, Any]) -> "DeepSpeedFPConfig":
56
- weight_bits = cls.get_from_keys(config, ["bits"])
57
- group_size = cls.get_from_keys(config, ["group_size"])
58
- return cls(weight_bits=weight_bits, group_size=group_size)
59
-
60
- def get_linear_method(self) -> "DeepSpeedFPLinearMethod":
61
- return DeepSpeedFPLinearMethod(self)
62
-
63
- @classmethod
64
- def get_supported_act_dtypes(cls) -> list[torch.dtype]:
65
- return [torch.half, torch.bfloat16]
66
-
67
- @classmethod
68
- # Need to figure it out
69
- def get_min_capability(cls) -> int:
70
- return 60
71
-
72
- @staticmethod
73
- def get_config_filenames() -> list[str]:
74
- return [
75
- "quant_config.json",
76
- "quantize_config.json",
77
- ]
78
-
79
- def get_quant_method(
80
- self, layer: torch.nn.Module, prefix: str
81
- ) -> Optional["DeepSpeedFPLinearMethod"]:
82
- if isinstance(layer, LinearBase):
83
- return DeepSpeedFPLinearMethod(self)
84
- return None
85
-
86
-
87
-class DeepSpeedFPLinearMethod(LinearMethodBase):
88
- """Linear method for DeepSpeedFP quantizer.
89
-
90
- Args:
91
- quant_config: the DeepSpeedFP quantization config.
92
- """
93
-
94
- def __init__(self, quant_config: DeepSpeedFPConfig):
95
- self.quant_config = quant_config
96
- self.weight = None
97
-
98
- def create_weights(
99
- self,
100
- layer: torch.nn.Module,
101
- input_size_per_partition: int,
102
- output_partition_sizes: list[int],
103
- input_size: int,
104
- output_size: int,
105
- params_dtype: torch.dtype,
106
- weight_loader=None,
107
- **extra_weight_attrs,
108
- ):
109
- del output_size
110
- del input_size
111
- output_size_per_partition = sum(output_partition_sizes)
112
- weight = DeepSpeedFPParameter(
113
- torch.Size((output_size_per_partition, input_size_per_partition)),
114
- params_dtype=params_dtype,
115
- quant_config=self.quant_config,
116
- )
117
- set_weight_attrs(
118
- weight,
119
- {
120
- "input_dim": 1,
121
- "output_dim": 0,
122
- },
123
- )
124
- layer.register_parameter("weight", weight)
125
-
126
- def quant_weight_loader(param, loaded_weight, *args, **kwargs):
127
- # Calls the original weight loader (if any), quantizes the result,
128
- # and then loads the quantized parameter.
129
- if weight_loader is not None:
130
- orig_param_data = param.data
131
- param.data = param.ds_dequantize()
132
- weight_loader(param, loaded_weight, *args, **kwargs)
133
- param.data, loaded_weight = orig_param_data, param.data
134
- param.ds_quantize_(loaded_weight.cuda())
135
-
136
- extra_weight_attrs["weight_loader"] = quant_weight_loader
137
- set_weight_attrs(weight, extra_weight_attrs)
138
-
139
- def apply(
140
- self,
141
- layer: torch.nn.Module,
142
- x: torch.Tensor,
143
- bias: torch.Tensor | None = None,
144
- ) -> torch.Tensor:
145
- weight = layer.weight
146
- y = weight.ds_dequantize()
147
- return F.linear(x, y, bias)
148
-
149
-
150
-class DeepSpeedFPParameter(nn.Parameter):
151
- """
152
- DeepSpeedFP quantized parameter class that implements fp8/fp6
153
- quantization deepspeed. Weights are stored in quantized form on
154
- GPUs, and can be dequantized on-the-fly when needed by the model.
155
- """
156
-
157
- def __new__(
158
- cls,
159
- orig_shape: torch.Size,
160
- params_dtype: torch.dtype,
161
- quant_config: DeepSpeedFPConfig,
162
- ):
163
- try:
164
- import deepspeed
165
-
166
- if version.parse(deepspeed.__version__) < version.parse("0.14.2"):
167
- raise ImportError(
168
- "deepspeed version is wrong. Please install deepspeed>=0.14.2."
169
- )
170
- from deepspeed.ops.fp_quantizer import FP_Quantize
171
- except ImportError as err:
172
- raise ImportError(
173
- "Please install deepspeed>=0.14.2 via "
174
- "`pip install deepspeed>=0.14.2` to use "
175
- "deepspeedfp quantizer."
176
- ) from err
177
- data = torch.empty(
178
- (
179
- orig_shape.numel() // quant_config.group_size,
180
- quant_config.group_size * quant_config.weight_bits // 8 + 4,
181
- ),
182
- dtype=torch.int8,
183
- )
184
- self = torch.Tensor._make_subclass(cls, data, data.requires_grad)
185
- self.orig_shape = orig_shape
186
- self.quant_config = quant_config
187
- self.fp_quantizer = FP_Quantize(group_size=quant_config.group_size)
188
- self.fp_quantizer.orig_shape = orig_shape
189
- self.fp_quantizer.orig_dtype = params_dtype
190
- return self
191
-
192
- def ds_quantize_(self, tensor: torch.Tensor):
193
- assert tensor.device.type == "cuda" and tensor.dtype != torch.int8
194
- return self.data.copy_(
195
- self.fp_quantizer.quantize(
196
- tensor.data,
197
- q_bits=self.quant_config.weight_bits,
198
- )
199
- )
200
-
201
- def ds_dequantize(self, fp_out=None) -> torch.Tensor:
202
- """
203
- Return a tensor containing the dequantized weights of this parameter.
204
- """
205
- assert self.data.device.type == "cuda" and self.data.dtype == torch.int8
206
- return self.fp_quantizer.dequantize(
207
- self.data, fp_out=fp_out, q_bits=self.quant_config.weight_bits
208
- )
209
-
210
- def ds_selective_dequantize(self, indices, fp_out=None) -> torch.Tensor:
211
- """
212
- Return a tensor where only the weights at `indices` are dequantized
213
- (to save HBM -> SRAM bandwidth).
214
- """
215
- assert self.data.device.type == "cuda" and self.data.dtype == torch.int8
216
- return self.fp_quantizer.selective_dequantize(
217
- self.data, indices, fp_out=fp_out, q_bits=self.quant_config.weight_bits
218
- )
vllm/model_executor/models/arctic.py
+19
-58
@@ -29,10 +29,6 @@ from vllm.model_executor.layers.linear import (
29
)
30
from vllm.model_executor.layers.logits_processor import LogitsProcessor
31
from vllm.model_executor.layers.quantization import QuantizationConfig
32
-from vllm.model_executor.layers.quantization.deepspeedfp import (
33
- DeepSpeedFPConfig,
34
- DeepSpeedFPParameter,
35
-)
32
from vllm.model_executor.layers.rotary_embedding import get_rope
33
from vllm.model_executor.layers.vocab_parallel_embedding import (
34
ParallelLMHead,
@@ -128,7 +124,6 @@ class ArcticMoE(nn.Module):
124
self.intermediate_size = config.intermediate_size // self.tp_size
125
126
self.is_moe_layer = (layer_id + 1) % config.moe_layer_frequency == 0
131
- self.is_quant = isinstance(quant_config, DeepSpeedFPConfig)
127
self.reduce_results = reduce_results
128
# Some other parameters
129
if params_dtype is None:
@@ -151,40 +146,24 @@ class ArcticMoE(nn.Module):
146
quant_config=quant_config,
147
prefix=f"{prefix}.gate",
148
)
154
- if self.is_quant:
155
- self.ws = DeepSpeedFPParameter(
156
- torch.Size(
157
- (self.num_experts, 2 * self.intermediate_size, self.hidden_size)
158
- ),
159
- params_dtype=params_dtype,
160
- quant_config=quant_config,
161
- )
162
- self.w2s = DeepSpeedFPParameter(
163
- torch.Size(
164
- (self.num_experts, self.hidden_size, self.intermediate_size)
165
- ),
166
- params_dtype=params_dtype,
167
- quant_config=quant_config,
168
- )
169
- else:
170
- self.ws = nn.Parameter(
171
- torch.empty(
172
- self.num_experts,
173
- 2 * self.intermediate_size,
174
- self.hidden_size,
175
- device=current_platform.device_type,
176
- dtype=self.params_dtype,
177
- )
149
+ self.ws = nn.Parameter(
150
+ torch.empty(
151
+ self.num_experts,
152
+ 2 * self.intermediate_size,
153
+ self.hidden_size,
154
+ device=current_platform.device_type,
155
+ dtype=self.params_dtype,
156
)
179
- self.w2s = nn.Parameter(
180
- torch.empty(
181
- self.num_experts,
182
- self.hidden_size,
183
- self.intermediate_size,
184
- device=current_platform.device_type,
185
- dtype=self.params_dtype,
186
- )
157
+ )
158
+ self.w2s = nn.Parameter(
159
+ torch.empty(
160
+ self.num_experts,
161
+ self.hidden_size,
162
+ self.intermediate_size,
163
+ device=current_platform.device_type,
164
+ dtype=self.params_dtype,
165
)
166
+ )
167
set_weight_attrs(
168
self.ws,
169
{
@@ -206,7 +185,7 @@ class ArcticMoE(nn.Module):
185
expert_id: int,
186
):
187
tp_rank = get_tensor_model_parallel_rank()
209
- param_data = param.ds_dequantize() if self.is_quant else param.data
188
+ param_data = param.data
189
shard_size = self.intermediate_size
190
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
191
if weight_name.endswith("w1.weight"):
@@ -217,8 +196,6 @@ class ArcticMoE(nn.Module):
196
]
197
if weight_name.endswith("w2.weight"):
198
param_data[expert_id, :, :] = loaded_weight[:, shard]
220
- if self.is_quant:
221
- param.ds_quantize_(param_data)
199
200
def local_moe_fused(self, hidden_states: torch.Tensor) -> torch.Tensor:
201
num_tokens, hidden_size = hidden_states.shape
@@ -229,26 +206,10 @@ class ArcticMoE(nn.Module):
206
topk_weights, topk_ids, token_expert_indices = fused_topk(
207
hidden_states, router_logits, self.top_k, renormalize=do_normalize
208
)
232
- # topk_ids: (num_tokens, k)
233
- if self.is_quant:
234
- if 2 * num_tokens <= self.num_experts:
235
- # If much fewer tokens than experts, use selective dequantize.
236
- ws_dequantized = self.ws.ds_selective_dequantize(topk_ids.flatten())
237
- w2s_dequantized = self.w2s.ds_selective_dequantize(topk_ids.flatten())
238
- # We gathered the experts to the tokens so update the mapping.
239
- topk_ids = torch.arange(
240
- 0,
241
- topk_ids.numel(),
242
- device=topk_ids.device,
243
- ).reshape(topk_ids.shape)
244
- else:
245
- ws_dequantized = self.ws.ds_dequantize()
246
- w2s_dequantized = self.w2s.ds_dequantize()
247
-
209
final_hidden_states = fused_experts(
210
hidden_states,
250
- ws_dequantized if self.is_quant else self.ws,
251
- w2s_dequantized if self.is_quant else self.w2s,
211
+ self.ws,
212
+ self.w2s,
213
topk_weights,
214
topk_ids,
215
inplace=True,