@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fbe66e1d0b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

added support for quantize on LLM module (#1080)

orellavie1212 committed Sep 18, 2023 at 21:04 UTC fbe66e1d0b8d1445cb3204150afac74ab075e559
1 file changed +5
vllm/entrypoints/llm.py
+5
@@ -38,6 +38,9 @@ class LLM:
38 However, if the `torch_dtype` in the config is `float32`, we will
39 use `float16` instead.
40 seed: The seed to initialize the random number generator for sampling.
41 + quantization: The method used to quantize the model weights. Currently,
42 + we support "awq". If None, we assume the model weights are not
43 + quantized and use `dtype` to determine the data type of the weights.
44 revision: The specific model version to use. It can be a branch name,
45 a tag name, or a commit id.
46 """
@@ -51,6 +54,7 @@ class LLM:
54 tensor_parallel_size: int = 1,
55 dtype: str = "auto",
56 seed: int = 0,
57 + quantization: Optional[str] = None,
58 **kwargs,
59 ) -> None:
60 if "disable_log_stats" not in kwargs:
@@ -63,6 +67,7 @@ class LLM:
67 tensor_parallel_size=tensor_parallel_size,
68 dtype=dtype,
69 seed=seed,
70 + quantization=quantization,
71 **kwargs,
72 )
73 self.llm_engine = LLMEngine.from_engine_args(engine_args)