@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 30fb0956df
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Minor] Add more detailed explanation on `quantization` argument (#2145)

Woosuk Kwon committed Dec 17, 2023 at 01:56 UTC 30fb0956dfee6113765ccb527f4f06703c75bf47
2 files changed +10 -4
vllm/engine/arg_utils.py
+6 -1
@@ -183,7 +183,12 @@ class EngineArgs:
183 type=str,
184 choices=['awq', 'gptq', 'squeezellm', None],
185 default=None,
186 - help='Method used to quantize the weights')
186 + help='Method used to quantize the weights. If '
187 + 'None, we first check the `quantization_config` '
188 + 'attribute in the model config file. If that is '
189 + 'None, we assume the model weights are not '
190 + 'quantized and use `dtype` to determine the data '
191 + 'type of the weights.')
192 parser.add_argument('--enforce-eager',
193 action='store_true',
194 help='Always use eager-mode PyTorch. If False, '
vllm/entrypoints/llm.py
+4 -3
@@ -38,9 +38,10 @@ class LLM:
38 However, if the `torch_dtype` in the config is `float32`, we will
39 use `float16` instead.
40 quantization: The method used to quantize the model weights. Currently,
41 - we support "awq", "gptq" and "squeezellm". If None, we assume the
42 - model weights are not quantized and use `dtype` to determine the
43 - data type of the weights.
41 + we support "awq", "gptq" and "squeezellm". If None, we first check
42 + the `quantization_config` attribute in the model config file. If
43 + that is None, we assume the model weights are not quantized and use
44 + `dtype` to determine the data type of the weights.
45 revision: The specific model version to use. It can be a branch name,
46 a tag name, or a commit id.
47 tokenizer_revision: The specific tokenizer version to use. It can be a