A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] Use LoadFormat values for `vllm serve --load-format` (#7784)
Michael Goin committed
Aug 22, 2024 at 14:37 UTC
15310b5101963818b76f1821e93887cb22f0aea6
1 file changed
+3
-6
vllm/engine/arg_utils.py
+3
-6
@@ -9,8 +9,8 @@ import torch
9
10
import vllm.envs as envs
11
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
12
- EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
13
- ObservabilityConfig, ParallelConfig,
12
+ EngineConfig, LoadConfig, LoadFormat, LoRAConfig,
13
+ ModelConfig, ObservabilityConfig, ParallelConfig,
14
PromptAdapterConfig, SchedulerConfig,
15
SpeculativeConfig, TokenizerPoolConfig)
16
from vllm.executor.executor_base import ExecutorBase
@@ -214,10 +214,7 @@ class EngineArgs:
214
'--load-format',
215
type=str,
216
default=EngineArgs.load_format,
217
- choices=[
218
- 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer',
219
- 'bitsandbytes'
220
- ],
217
+ choices=[f.value for f in LoadFormat],
218
help='The format of the model weights to load.\n\n'
219
'* "auto" will try to load the weights in the safetensors format '
220
'and fall back to the pytorch bin format if safetensors format '