@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ce37be7ba0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[misc][distributed] add seed to dummy weights (#6491)

youkaichao committed Jul 16, 2024 at 19:16 UTC ce37be7ba05f21f2a8f3b792034195305ebe1b27
1 file changed +11 -2
vllm/model_executor/model_loader/weight_utils.py
+11 -2
@@ -440,6 +440,7 @@ def initialize_dummy_weights(
440 model: torch.nn.Module,
441 low: float = -1e-3,
442 high: float = 1e-3,
443 + seed: int = 1234,
444 ) -> None:
445 """Initialize model weights with random values.
446
@@ -447,17 +448,25 @@ def initialize_dummy_weights(
448 measurements. Additionally, the model weights should not cause NaNs in the
449 forward pass. We empirically found that initializing the weights with
450 values between -1e-3 and 1e-3 works well for most models.
451 +
452 + We use per-parameter random seed, so that dummy weights are consistent,
453 + even if the model is partitioned across multiple devices. When the seed
454 + is fixed, the random values generated by this function only depends on
455 + the parameter's number of elements and its data type.
456 """
457 for param in model.state_dict().values():
458 if torch.is_floating_point(param):
459 + generator = torch.Generator(device=param.data.device)
460 + generator.manual_seed(seed)
461 if torch.finfo(param.data.dtype).bits < 16:
462 # uniform_ doesn't support < 16-bit datatypes (FP8)
463 dtype = param.data.dtype
464 tmp_param = param.data.to(torch.float16)
457 - tmp_param = tmp_param.uniform_(low, high).to(dtype)
465 + tmp_param = tmp_param.uniform_(low, high,
466 + generator=generator).to(dtype)
467 param.data.copy_(tmp_param)
468 else:
460 - param.uniform_(low, high)
469 + param.uniform_(low, high, generator=generator)
470
471
472 def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]: