A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Move bfloat16 check to worker (#1259)
Antoni Baum committed
Oct 7, 2023 at 22:10 UTC
ee92b58b3ae28a52c2500674de721e7a718432d8
2 files changed
+14
-9
vllm/config.py
-9
@@ -345,15 +345,6 @@ def _get_and_verify_dtype(
345
# Casting between float16 and bfloat16 is allowed with a warning.
346
logger.warning(f"Casting {config_dtype} to {torch_dtype}.")
347
348
- # Check if the GPU supports the dtype.
349
- if torch_dtype == torch.bfloat16:
350
- compute_capability = torch.cuda.get_device_capability()
351
- if compute_capability[0] < 8:
352
- gpu_name = torch.cuda.get_device_name()
353
- raise ValueError(
354
- "Bfloat16 is only supported on GPUs with compute capability "
355
- f"of at least 8.0. Your {gpu_name} GPU has compute capability "
356
- f"{compute_capability[0]}.{compute_capability[1]}.")
348
return torch_dtype
349
350
vllm/worker/worker.py
+14
@@ -59,6 +59,8 @@ class Worker:
59
raise ValueError("Invalid or unspecified rank.")
60
torch.cuda.set_device(self.device)
61
62
+ _check_if_gpu_supports_dtype(self.model_config.dtype)
63
+
64
# Initialize the distributed environment.
65
_init_distributed_environment(self.parallel_config, self.rank,
66
self.distributed_init_method)
@@ -385,3 +387,15 @@ def _check_if_can_support_max_seq_len(max_seq_len: int,
387
f"(required shared memory {required_shared_mem} > "
388
f"available shared memory {max_shared_mem}). "
389
"This will be fixed in a future release.")
390
+
391
+
392
+def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
393
+ # Check if the GPU supports the dtype.
394
+ if torch_dtype == torch.bfloat16:
395
+ compute_capability = torch.cuda.get_device_capability()
396
+ if compute_capability[0] < 8:
397
+ gpu_name = torch.cuda.get_device_name()
398
+ raise ValueError(
399
+ "Bfloat16 is only supported on GPUs with compute capability "
400
+ f"of at least 8.0. Your {gpu_name} GPU has compute capability "
401
+ f"{compute_capability[0]}.{compute_capability[1]}.")