@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 26908554b2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Doc] Remove float32 choice from --lora-dtype (#10348)

Signed-off-by: Xin Yang <xyang19@gmail.com>

Xin Yang committed Nov 15, 2024 at 02:22 UTC 26908554b2ecc8f76fa57942566629ec5713ef5b
1 file changed +1 -1
vllm/engine/arg_utils.py
+1 -1
@@ -601,7 +601,7 @@ class EngineArgs:
601 '--lora-dtype',
602 type=str,
603 default=EngineArgs.lora_dtype,
604 - choices=['auto', 'float16', 'bfloat16', 'float32'],
604 + choices=['auto', 'float16', 'bfloat16'],
605 help=('Data type for LoRA. If auto, will default to '
606 'base model dtype.'))
607 parser.add_argument(