@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 15310b5101
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] Use LoadFormat values for `vllm serve --load-format` (#7784)

Michael Goin committed Aug 22, 2024 at 14:37 UTC 15310b5101963818b76f1821e93887cb22f0aea6
1 file changed +3 -6
vllm/engine/arg_utils.py
+3 -6
@@ -9,8 +9,8 @@ import torch
9
10 import vllm.envs as envs
11 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
12 - EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
13 - ObservabilityConfig, ParallelConfig,
12 + EngineConfig, LoadConfig, LoadFormat, LoRAConfig,
13 + ModelConfig, ObservabilityConfig, ParallelConfig,
14 PromptAdapterConfig, SchedulerConfig,
15 SpeculativeConfig, TokenizerPoolConfig)
16 from vllm.executor.executor_base import ExecutorBase
@@ -214,10 +214,7 @@ class EngineArgs:
214 '--load-format',
215 type=str,
216 default=EngineArgs.load_format,
217 - choices=[
218 - 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'tensorizer',
219 - 'bitsandbytes'
220 - ],
217 + choices=[f.value for f in LoadFormat],
218 help='The format of the model weights to load.\n\n'
219 '* "auto" will try to load the weights in the safetensors format '
220 'and fall back to the pytorch bin format if safetensors format '