A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix] Fix dummy weight for fp8 (#4916)
Allow dummy load format for fp8, torch.uniform_ doesn't support FP8 at the moment Co-authored-by: Mor Zusman <morz@ai21.com>
Mor Zusman committed
May 20, 2024 at 21:44 UTC
f0eecee6106774e1e0f9b31c7438cde77654df52
1 file changed
+8
-1
vllm/model_executor/model_loader/weight_utils.py
+8
-1
@@ -369,4 +369,11 @@ def initialize_dummy_weights(
369
"""
370
for param in model.state_dict().values():
371
if torch.is_floating_point(param):
372
- param.data.uniform_(low, high)
372
+ if torch.finfo(param.data.dtype).bits < 16:
373
+ # uniform_ doesn't support < 16-bit datatypes (FP8)
374
+ dtype = param.data.dtype
375
+ tmp_param = param.data.to(torch.float16)
376
+ tmp_param = tmp_param.uniform_(low, high).to(dtype)
377
+ param.data.copy_(tmp_param)
378
+ else:
379
+ param.uniform_(low, high)