A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix][TPU] Do not use torch.Generator for TPUs (#6981)
Woosuk Kwon committed
Jul 31, 2024 at 18:50 UTC
23993a7997ff927decaca60281871d5fdab11334
1 file changed
+6
vllm/model_executor/model_loader/weight_utils.py
+6
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
22
from vllm.model_executor.layers.quantization import (QuantizationConfig,
23
get_quantization_config)
24
from vllm.model_executor.layers.quantization.schema import QuantParamSchema
25
+from vllm.platforms import current_platform
26
from vllm.utils import print_warning_once
27
28
logger = init_logger(__name__)
@@ -490,6 +491,11 @@ def initialize_dummy_weights(
491
"""
492
for param in model.state_dict().values():
493
if torch.is_floating_point(param):
494
+ if current_platform.is_tpu():
495
+ # XLA device does not support torch.Generator()
496
+ param.uniform_(low, high)
497
+ continue
498
+
499
generator = torch.Generator(device=param.data.device)
500
generator.manual_seed(seed)
501
if torch.finfo(param.data.dtype).bits < 16: