[Bugfix] Avoid repeated dummy initialization and random CPU Engram fills (#56682)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Woosuk Kwon committed
Sep 13, 2026 at 02:23 UTC
e7a3963339654aa18b713f9ff233f129af9c7c8b
3 files changed
+14
-2
vllm/model_executor/model_loader/dummy_loader.py
+6
-2
@@ -14,7 +14,6 @@ from vllm.model_executor.model_loader.reload.meta import materialize_layer
14
from vllm.model_executor.model_loader.reload.types import LayerReloadingInfo
15
from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
16
from vllm.model_executor.model_loader.weight_utils import (
17
- initialize_dummy_weights,
17
initialize_single_dummy_weight,
18
)
19
@@ -41,7 +40,12 @@ class DummyModelLoader(BaseModelLoader):
40
else:
41
# NOTE(woosuk): For accurate performance evaluation, we assign
42
# random values to the weights.
44
- initialize_dummy_weights(layer, model_config)
43
+ # Initialize only local tensors: state_dict() would recursively
44
+ # initialize descendants that model.modules() also visits.
45
+ # Exclude nonpersistent buffers, matching state_dict().
46
+ for name, tensor in get_layer_tensors(layer).items():
47
+ if name not in layer._non_persistent_buffers_set:
48
+ initialize_single_dummy_weight(tensor)
49
50
def _process_online_quant_layer(
51
self,
vllm/model_executor/model_loader/weight_utils.py
+4
@@ -1341,6 +1341,10 @@ def initialize_single_dummy_weight(
1341
if param.device.type == "meta":
1342
return # deferred to finalize_layerwise_processing (e.g. online quant)
1343
1344
+ if (dummy_weight_value := getattr(param, "dummy_weight_value", None)) is not None:
1345
+ param.fill_(dummy_weight_value)
1346
+ return
1347
+
1348
if not torch.is_floating_point(param):
1349
if current_platform.is_rocm():
1350
# On ROCm, integer params (e.g. GPTQ qweight/qzeros) are left
vllm/models/deepseek_v4_1/common/engram.py
+4
@@ -687,6 +687,10 @@ class ParallelEngramEmbedding(nn.Module):
687
},
688
)
689
if cpu_offload:
690
+ # Constant dummy values avoid randomizing huge CPU lookup tables.
691
+ set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
692
+ # The ue8m0 encoding of scale 1.0 is exponent byte 127.
693
+ set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
694
logger.info(
695
"Engram table offloaded to pinned host memory: %d rows x %d, "
696
"%.2f GiB per rank",