@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f0eecee610
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] Fix dummy weight for fp8 (#4916)

Allow dummy load format for fp8, torch.uniform_ doesn't support FP8 at the moment Co-authored-by: Mor Zusman <morz@ai21.com>

Mor Zusman committed May 20, 2024 at 21:44 UTC f0eecee6106774e1e0f9b31c7438cde77654df52
1 file changed +8 -1
vllm/model_executor/model_loader/weight_utils.py
+8 -1
@@ -369,4 +369,11 @@ def initialize_dummy_weights(
369 """
370 for param in model.state_dict().values():
371 if torch.is_floating_point(param):
372 - param.data.uniform_(low, high)
372 + if torch.finfo(param.data.dtype).bits < 16:
373 + # uniform_ doesn't support < 16-bit datatypes (FP8)
374 + dtype = param.data.dtype
375 + tmp_param = param.data.to(torch.float16)
376 + tmp_param = tmp_param.uniform_(low, high).to(dtype)
377 + param.data.copy_(tmp_param)
378 + else:
379 + param.uniform_(low, high)