@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a044754bd

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[XPU] Delay BF16 check to worker init for spawn compatibility (#22979)

Signed-off-by: chzhang <chaojun.zhang@intel.com>

Chaojun Zhang committed Aug 26, 2025 at 04:09 UTC 8a044754bd083671e4bb09a68b1edae9610dfccc

6 files changed +60 -47

vllm/platforms/cuda.py

+20

@@ -518,6 +518,26 @@ class CudaPlatformBase(Platform):

518 supported = True

519 return supported

520

521 + @classmethod

522 + def check_if_supports_dtype(cls, torch_dtype: torch.dtype):

523 + if torch_dtype == torch.bfloat16: # noqa: SIM102

524 + if not cls.has_device_capability(80):

525 + capability = cls.get_device_capability()

526 + gpu_name = cls.get_device_name()

527 +

528 + if capability is None:

529 + compute_str = "does not have a compute capability"

530 + else:

531 + version_str = capability.as_version_str()

532 + compute_str = f"has compute capability {version_str}"

533 +

534 + raise ValueError(

535 + "Bfloat16 is only supported on GPUs "

536 + "with compute capability of at least 8.0. "

537 + f"Your {gpu_name} GPU {compute_str}. "

538 + "You can use float16 instead by explicitly setting the "

539 + "`dtype` flag in CLI, for example: --dtype=half.")

540 +

541

542 # NVML utils

543 # Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,

vllm/platforms/interface.py

@@ -572,6 +572,13 @@ class Platform:

572 """

573 return False

574

575 + @classmethod

576 + def check_if_supports_dtype(cls, torch_dtype: torch.dtype):

577 + """

578 + Check if the dtype is supported by the current platform.

579 + """

580 + raise NotImplementedError

581 +

582

583 class UnspecifiedPlatform(Platform):

584 _enum = PlatformEnum.UNSPECIFIED

vllm/platforms/rocm.py

+20

@@ -462,3 +462,23 @@ class RocmPlatform(Platform):

462 def is_kv_cache_dtype_supported(cls, kv_cache_dtype: str,

463 model_config: "ModelConfig") -> bool:

464 return True

465 +

466 + @classmethod

467 + def check_if_supports_dtype(cls, torch_dtype: torch.dtype):

468 + if torch_dtype == torch.bfloat16: # noqa: SIM102

469 + if not cls.has_device_capability(80):

470 + capability = cls.get_device_capability()

471 + gpu_name = cls.get_device_name()

472 +

473 + if capability is None:

474 + compute_str = "does not have a compute capability"

475 + else:

476 + version_str = capability.as_version_str()

477 + compute_str = f"has compute capability {version_str}"

478 +

479 + raise ValueError(

480 + "Bfloat16 is only supported on GPUs "

481 + "with compute capability of at least 8.0. "

482 + f"Your {gpu_name} GPU {compute_str}. "

483 + "You can use float16 instead by explicitly setting the "

484 + "`dtype` flag in CLI, for example: --dtype=half.")

vllm/platforms/xpu.py

+11 -26

@@ -97,13 +97,6 @@ class XPUPlatform(Platform):

97 from vllm.config import CompilationLevel

98 vllm_config.compilation_config.level = CompilationLevel.NO_COMPILATION # noqa: E501

100 - # Instances created using VllmConfig() typically have model_config as

101 - # None by default. The modification involves adding a check to prevent

102 - # potential null exceptions check and update model config.

103 - if model_config is not None and model_config.dtype == torch.bfloat16 \

104 - and not cls.device_support_bf16():

105 - model_config.dtype = torch.float16

106 -

100 # lazy import to avoid circular import

101 from vllm.config import CUDAGraphMode

102 compilation_config = vllm_config.compilation_config

@@ -162,30 +155,11 @@ class XPUPlatform(Platform):

155 torch.xpu.reset_peak_memory_stats(device)

156 return torch.xpu.max_memory_allocated(device)

157

165 - @classmethod

166 - def device_support_bf16(cls) -> bool:

167 - device_name = cls.get_device_name().lower()

168 - if cls.is_client_gpu_a770():

169 - logger.warning("Intel Arc A770 have bfloat16 accuracy known issue,"

170 - " fallback to float16")

171 - return False

172 - else:

173 - logger.info(

174 - "Device name %s supports bfloat16. Please file an issue "

175 - "if you encounter any accuracy problems with bfloat16.",

176 - device_name)

177 - return True

178 -

158 @classmethod

159 def is_data_center_gpu(cls) -> bool:

160 device_name = cls.get_device_name().lower()

161 return device_name.count("data center gpu") > 0

162

184 - @classmethod

185 - def is_client_gpu_a770(cls) -> bool:

186 - device_name = cls.get_device_name().lower()

187 - return device_name.count("a770") > 0

188 -

163 @classmethod

164 def get_device_communicator_cls(cls) -> str:

165 return "vllm.distributed.device_communicators.xpu_communicator.XpuCommunicator" # noqa

@@ -197,3 +171,14 @@ class XPUPlatform(Platform):

171 @classmethod

172 def device_count(cls) -> int:

173 return torch.xpu.device_count()

174 +

175 + @classmethod

176 + def check_if_supports_dtype(cls, torch_dtype: torch.dtype):

177 + if torch_dtype == torch.bfloat16: # noqa: SIM102

178 + device_name = cls.get_device_name().lower()

179 + # client gpu a770

180 + if device_name.count("a770") > 0:

181 + raise ValueError(

182 + "Intel Arc A770 have bfloat16 accuracy known issue. "

183 + "You can use float16 instead by explicitly setting the "

184 + "`dtype` flag in CLI, for example: --dtype=half.")

vllm/v1/worker/gpu_worker.py

+1 -21

@@ -167,7 +167,7 @@ class Worker(WorkerBase):

167 self.device = torch.device(f"cuda:{self.local_rank}")

168 current_platform.set_device(self.device)

169

170 - _check_if_gpu_supports_dtype(self.model_config.dtype)

170 + current_platform.check_if_supports_dtype(self.model_config.dtype)

171 gc.collect()

172 torch.cuda.empty_cache()

173

@@ -612,23 +612,3 @@ def init_worker_distributed_environment(

612 parallel_config.pipeline_parallel_size)

613

614 ensure_kv_transfer_initialized(vllm_config)

615 -

616 -

617 -def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):

618 - # Check if the GPU supports the dtype.

619 - if torch_dtype == torch.bfloat16: # noqa: SIM102

620 - if not current_platform.has_device_capability(80):

621 - capability = current_platform.get_device_capability()

622 - gpu_name = current_platform.get_device_name()

623 -

624 - if capability is None:

625 - compute_str = "does not have a compute capability"

626 - else:

627 - version_str = capability.as_version_str()

628 - compute_str = f"has compute capability {version_str}"

629 -

630 - raise ValueError(

631 - "Bfloat16 is only supported on GPUs with compute capability "

632 - f"of at least 8.0. Your {gpu_name} GPU {compute_str}. "

633 - "You can use float16 instead by explicitly setting the "

634 - "`dtype` flag in CLI, for example: --dtype=half.")

vllm/v1/worker/xpu_worker.py

@@ -145,6 +145,7 @@ class XPUWorker(Worker):

145 ):

146 self.device = torch.device(f"xpu:{self.local_rank}")

147 current_platform.set_device(self.device)

148 + current_platform.check_if_supports_dtype(self.model_config.dtype)

149 torch.xpu.empty_cache()

150 self.init_gpu_memory = torch.xpu.get_device_properties(

151 self.local_rank).total_memory