@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dad961ef5c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] fix lora_dtype value type in arg_utils.py - part 2 (#5428)

Ali Panahi committed Aug 19, 2024 at 13:47 UTC dad961ef5ca3893b78224323ec943dce9f52f868
1 file changed +3 -1
vllm/engine/arg_utils.py
+3 -1
@@ -5,6 +5,8 @@ from dataclasses import dataclass
5 from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Type,
6 Union)
7
8 +import torch
9 +
10 import vllm.envs as envs
11 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
12 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
@@ -113,7 +115,7 @@ class EngineArgs:
115 fully_sharded_loras: bool = False
116 lora_extra_vocab_size: int = 256
117 long_lora_scaling_factors: Optional[Tuple[float]] = None
116 - lora_dtype: str = 'auto'
118 + lora_dtype: Optional[Union[str, torch.dtype]] = 'auto'
119 max_cpu_loras: Optional[int] = None
120 device: str = 'auto'
121 num_scheduler_steps: int = 1