@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 23993a7997
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix][TPU] Do not use torch.Generator for TPUs (#6981)

Woosuk Kwon committed Jul 31, 2024 at 18:50 UTC 23993a7997ff927decaca60281871d5fdab11334
1 file changed +6
vllm/model_executor/model_loader/weight_utils.py
+6
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
22 from vllm.model_executor.layers.quantization import (QuantizationConfig,
23 get_quantization_config)
24 from vllm.model_executor.layers.quantization.schema import QuantParamSchema
25 +from vllm.platforms import current_platform
26 from vllm.utils import print_warning_once
27
28 logger = init_logger(__name__)
@@ -490,6 +491,11 @@ def initialize_dummy_weights(
491 """
492 for param in model.state_dict().values():
493 if torch.is_floating_point(param):
494 + if current_platform.is_tpu():
495 + # XLA device does not support torch.Generator()
496 + param.uniform_(low, high)
497 + continue
498 +
499 generator = torch.Generator(device=param.data.device)
500 generator.manual_seed(seed)
501 if torch.finfo(param.data.dtype).bits < 16: