A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 18:30 UTC
[Online Quant] [QeRL] Minor code cleanup (#38574)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
Kyle Sayers committed
Mar 31, 2026 at 10:56 UTC
5869f69c5f543c4b3cbd460a30c10eb97e5b3ca0
2 files changed
+2
-65
vllm/model_executor/layers/quantization/mxfp8.py
+1
-64
@@ -31,7 +31,6 @@ from vllm.model_executor.layers.quantization.fp8 import (
31
Fp8KVCacheMethod,
32
Fp8OnlineLinearMethod,
33
Fp8OnlineMoEMethod,
34
- _copy_missing_attrs,
34
)
35
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
36
MXFP8_BLOCK_SIZE,
@@ -43,11 +42,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
42
from vllm.model_executor.layers.quantization.utils.quant_utils import (
43
is_layer_skipped,
44
)
46
-from vllm.model_executor.model_loader.weight_utils import (
47
- initialize_single_dummy_weight,
48
-)
49
-from vllm.model_executor.parameter import ModelWeightParameter
50
-from vllm.model_executor.utils import replace_parameter, set_weight_attrs
45
+from vllm.model_executor.utils import replace_parameter
46
from vllm.platforms import current_platform
47
48
logger = init_logger(__name__)
@@ -183,17 +178,6 @@ class Mxfp8OnlineLinearMethod(Fp8OnlineLinearMethod):
178
if getattr(layer, "_already_called_process_weights_after_loading", False):
179
return
180
186
- if layer.weight.device == torch.device("meta"):
187
- weight = ModelWeightParameter(
188
- data=torch.empty_like(layer.weight, device=layer._load_device),
189
- input_dim=1,
190
- output_dim=0,
191
- weight_loader=layer.weight.weight_loader,
192
- )
193
- _copy_missing_attrs(layer.weight, weight)
194
- layer.register_parameter("weight", weight)
195
- initialize_single_dummy_weight(layer.weight)
196
-
181
weight_fp8, weight_scale = mxfp8_e4m3_quantize(layer.weight.contiguous())
182
183
if self.mxfp8_linear.backend == Mxfp8LinearBackend.FLASHINFER_CUTLASS:
@@ -265,28 +249,6 @@ class Mxfp8OnlineMoEMethod(Fp8OnlineMoEMethod):
249
**extra_weight_attrs,
250
)
251
268
- w13_weight_scale = torch.nn.Parameter(
269
- torch.zeros(
270
- num_experts,
271
- 2 * intermediate_size_per_partition,
272
- hidden_size // MXFP8_BLOCK_SIZE,
273
- dtype=torch.uint8,
274
- ),
275
- requires_grad=False,
276
- )
277
- w2_weight_scale = torch.nn.Parameter(
278
- torch.zeros(
279
- num_experts,
280
- hidden_size,
281
- intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
282
- dtype=torch.uint8,
283
- ),
284
- requires_grad=False,
285
- )
286
- layer.register_parameter("w13_weight_scale", w13_weight_scale)
287
- layer.register_parameter("w2_weight_scale", w2_weight_scale)
288
- set_weight_attrs(w13_weight_scale, extra_weight_attrs)
289
- set_weight_attrs(w2_weight_scale, extra_weight_attrs)
252
layer.weight_block_size = [1, MXFP8_BLOCK_SIZE]
253
254
def _quantize_mxfp8_moe_weight(
@@ -309,34 +271,9 @@ class Mxfp8OnlineMoEMethod(Fp8OnlineMoEMethod):
271
if getattr(layer, "_already_called_process_weights_after_loading", False):
272
return
273
312
- if layer.w13_weight.device == torch.device("meta"):
313
- w13_weight = torch.nn.Parameter(
314
- torch.empty_like(layer.w13_weight, device=layer._load_device),
315
- requires_grad=False,
316
- )
317
- set_weight_attrs(
318
- w13_weight, {"weight_loader": layer.w13_weight.weight_loader}
319
- )
320
- _copy_missing_attrs(layer.w13_weight, w13_weight)
321
- layer.register_parameter("w13_weight", w13_weight)
322
- initialize_single_dummy_weight(layer.w13_weight)
323
- if layer.w2_weight.device == torch.device("meta"):
324
- w2_weight = torch.nn.Parameter(
325
- torch.empty_like(layer.w2_weight, device=layer._load_device),
326
- requires_grad=False,
327
- )
328
- set_weight_attrs(
329
- w2_weight, {"weight_loader": layer.w2_weight.weight_loader}
330
- )
331
- _copy_missing_attrs(layer.w2_weight, w2_weight)
332
- layer.register_parameter("w2_weight", w2_weight)
333
- initialize_single_dummy_weight(layer.w2_weight)
334
-
274
fp8_dtype = current_platform.fp8_dtype()
275
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
276
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
338
- w13_scale = layer.w13_weight_scale
339
- w2_scale = layer.w2_weight_scale
277
layer.w13_input_scale = None
278
layer.w2_input_scale = None
279
vllm/model_executor/model_loader/reload/layerwise.py
+1
-1
@@ -230,7 +230,7 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
230
continue
231
232
# reloading: place kernel tensors back as a fallback
233
- else:
233
+ elif info.load_numel_total > 0: # type: ignore[operator]
234
logger.warning("%s: Failed to load weights", layer.__class__.__name__)
235
_place_kernel_tensors(layer, info)
236