@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 00e6a2dc53
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] fix lora_dtype value type in arg_utils.py (#5398)

Ali Panahi committed Jun 11, 2024 at 10:40 UTC 00e6a2dc535c89ac7c92551ef9b92acd8664df02
1 file changed +1 -1
vllm/engine/arg_utils.py
+1 -1
@@ -68,7 +68,7 @@ class EngineArgs:
68 fully_sharded_loras: bool = False
69 lora_extra_vocab_size: int = 256
70 long_lora_scaling_factors: Optional[Tuple[float]] = None
71 - lora_dtype = 'auto'
71 + lora_dtype: str = 'auto'
72 max_cpu_loras: Optional[int] = None
73 device: str = 'auto'
74 ray_workers_use_nsight: bool = False