A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
added support for quantize on LLM module (#1080)
orellavie1212 committed
Sep 18, 2023 at 21:04 UTC
fbe66e1d0b8d1445cb3204150afac74ab075e559
1 file changed
+5
vllm/entrypoints/llm.py
+5
@@ -38,6 +38,9 @@ class LLM:
38
However, if the `torch_dtype` in the config is `float32`, we will
39
use `float16` instead.
40
seed: The seed to initialize the random number generator for sampling.
41
+ quantization: The method used to quantize the model weights. Currently,
42
+ we support "awq". If None, we assume the model weights are not
43
+ quantized and use `dtype` to determine the data type of the weights.
44
revision: The specific model version to use. It can be a branch name,
45
a tag name, or a commit id.
46
"""
@@ -51,6 +54,7 @@ class LLM:
54
tensor_parallel_size: int = 1,
55
dtype: str = "auto",
56
seed: int = 0,
57
+ quantization: Optional[str] = None,
58
**kwargs,
59
) -> None:
60
if "disable_log_stats" not in kwargs:
@@ -63,6 +67,7 @@ class LLM:
67
tensor_parallel_size=tensor_parallel_size,
68
dtype=dtype,
69
seed=seed,
70
+ quantization=quantization,
71
**kwargs,
72
)
73
self.llm_engine = LLMEngine.from_engine_args(engine_args)