A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[misc][distributed] add seed to dummy weights (#6491)
youkaichao committed
Jul 16, 2024 at 19:16 UTC
ce37be7ba05f21f2a8f3b792034195305ebe1b27
1 file changed
+11
-2
vllm/model_executor/model_loader/weight_utils.py
+11
-2
@@ -440,6 +440,7 @@ def initialize_dummy_weights(
440
model: torch.nn.Module,
441
low: float = -1e-3,
442
high: float = 1e-3,
443
+ seed: int = 1234,
444
) -> None:
445
"""Initialize model weights with random values.
446
@@ -447,17 +448,25 @@ def initialize_dummy_weights(
448
measurements. Additionally, the model weights should not cause NaNs in the
449
forward pass. We empirically found that initializing the weights with
450
values between -1e-3 and 1e-3 works well for most models.
451
+
452
+ We use per-parameter random seed, so that dummy weights are consistent,
453
+ even if the model is partitioned across multiple devices. When the seed
454
+ is fixed, the random values generated by this function only depends on
455
+ the parameter's number of elements and its data type.
456
"""
457
for param in model.state_dict().values():
458
if torch.is_floating_point(param):
459
+ generator = torch.Generator(device=param.data.device)
460
+ generator.manual_seed(seed)
461
if torch.finfo(param.data.dtype).bits < 16:
462
# uniform_ doesn't support < 16-bit datatypes (FP8)
463
dtype = param.data.dtype
464
tmp_param = param.data.to(torch.float16)
457
- tmp_param = tmp_param.uniform_(low, high).to(dtype)
465
+ tmp_param = tmp_param.uniform_(low, high,
466
+ generator=generator).to(dtype)
467
param.data.copy_(tmp_param)
468
else:
460
- param.uniform_(low, high)
469
+ param.uniform_(low, high, generator=generator)
470
471
472
def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]: