@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cb7a1c1cbf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Suggest using dtype=half when OOM.

Ben committed Jan 13, 2024 at 04:33 UTC cb7a1c1cbf7c07e072df29844fb7a51a01344392
1 file changed +3 -1
vllm/worker/worker.py
+3 -1
@@ -239,4 +239,6 @@ def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
239 raise ValueError(
240 "Bfloat16 is only supported on GPUs with compute capability "
241 f"of at least 8.0. Your {gpu_name} GPU has compute capability "
242 - f"{compute_capability[0]}.{compute_capability[1]}.")
242 + f"{compute_capability[0]}.{compute_capability[1]}."
243 + f" You can explicitly specify the data type by using the --dtype option, for example: --dtype=half."
244 + )