A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Suggest using dtype=half when OOM.
Ben committed
Jan 13, 2024 at 04:33 UTC
cb7a1c1cbf7c07e072df29844fb7a51a01344392
1 file changed
+3
-1
vllm/worker/worker.py
+3
-1
@@ -239,4 +239,6 @@ def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
239
raise ValueError(
240
"Bfloat16 is only supported on GPUs with compute capability "
241
f"of at least 8.0. Your {gpu_name} GPU has compute capability "
242
- f"{compute_capability[0]}.{compute_capability[1]}.")
242
+ f"{compute_capability[0]}.{compute_capability[1]}."
243
+ f" You can explicitly specify the data type by using the --dtype option, for example: --dtype=half."
244
+ )