@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ee92b58b3a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Move bfloat16 check to worker (#1259)

Antoni Baum committed Oct 7, 2023 at 22:10 UTC ee92b58b3ae28a52c2500674de721e7a718432d8
2 files changed +14 -9
vllm/config.py
-9
@@ -345,15 +345,6 @@ def _get_and_verify_dtype(
345 # Casting between float16 and bfloat16 is allowed with a warning.
346 logger.warning(f"Casting {config_dtype} to {torch_dtype}.")
347
348 - # Check if the GPU supports the dtype.
349 - if torch_dtype == torch.bfloat16:
350 - compute_capability = torch.cuda.get_device_capability()
351 - if compute_capability[0] < 8:
352 - gpu_name = torch.cuda.get_device_name()
353 - raise ValueError(
354 - "Bfloat16 is only supported on GPUs with compute capability "
355 - f"of at least 8.0. Your {gpu_name} GPU has compute capability "
356 - f"{compute_capability[0]}.{compute_capability[1]}.")
348 return torch_dtype
349
350
vllm/worker/worker.py
+14
@@ -59,6 +59,8 @@ class Worker:
59 raise ValueError("Invalid or unspecified rank.")
60 torch.cuda.set_device(self.device)
61
62 + _check_if_gpu_supports_dtype(self.model_config.dtype)
63 +
64 # Initialize the distributed environment.
65 _init_distributed_environment(self.parallel_config, self.rank,
66 self.distributed_init_method)
@@ -385,3 +387,15 @@ def _check_if_can_support_max_seq_len(max_seq_len: int,
387 f"(required shared memory {required_shared_mem} > "
388 f"available shared memory {max_shared_mem}). "
389 "This will be fixed in a future release.")
390 +
391 +
392 +def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
393 + # Check if the GPU supports the dtype.
394 + if torch_dtype == torch.bfloat16:
395 + compute_capability = torch.cuda.get_device_capability()
396 + if compute_capability[0] < 8:
397 + gpu_name = torch.cuda.get_device_name()
398 + raise ValueError(
399 + "Bfloat16 is only supported on GPUs with compute capability "
400 + f"of at least 8.0. Your {gpu_name} GPU has compute capability "
401 + f"{compute_capability[0]}.{compute_capability[1]}.")