A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add gpu_memory_utilization and swap_space to LLM (#1090)
Woosuk Kwon committed
Sep 19, 2023 at 22:16 UTC
bc0644574ca12d754a031596bdcfe8e1f0e6ab39
1 file changed
+19
-3
vllm/entrypoints/llm.py
+19
-3
@@ -37,12 +37,22 @@ class LLM:
37
the `torch_dtype` attribute specified in the model config file.
38
However, if the `torch_dtype` in the config is `float32`, we will
39
use `float16` instead.
40
- seed: The seed to initialize the random number generator for sampling.
40
quantization: The method used to quantize the model weights. Currently,
41
we support "awq". If None, we assume the model weights are not
42
quantized and use `dtype` to determine the data type of the weights.
43
revision: The specific model version to use. It can be a branch name,
44
a tag name, or a commit id.
45
+ seed: The seed to initialize the random number generator for sampling.
46
+ gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
47
+ reserve for the model weights, activations, and KV cache. Higher
48
+ values will increase the KV cache size and thus improve the model's
49
+ throughput. However, if the value is too high, it may cause out-of-
50
+ memory (OOM) errors.
51
+ swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
52
+ This can be used for temporarily storing the states of the requests
53
+ when their `best_of` sampling parameters are larger than 1. If all
54
+ requests will have `best_of=1`, you can safely set this to 0.
55
+ Otherwise, too small values may cause out-of-memory (OOM) errors.
56
"""
57
58
def __init__(
@@ -53,8 +63,11 @@ class LLM:
63
trust_remote_code: bool = False,
64
tensor_parallel_size: int = 1,
65
dtype: str = "auto",
56
- seed: int = 0,
66
quantization: Optional[str] = None,
67
+ revision: Optional[str] = None,
68
+ seed: int = 0,
69
+ gpu_memory_utilization: float = 0.9,
70
+ swap_space: int = 4,
71
**kwargs,
72
) -> None:
73
if "disable_log_stats" not in kwargs:
@@ -66,8 +79,11 @@ class LLM:
79
trust_remote_code=trust_remote_code,
80
tensor_parallel_size=tensor_parallel_size,
81
dtype=dtype,
69
- seed=seed,
82
quantization=quantization,
83
+ revision=revision,
84
+ seed=seed,
85
+ gpu_memory_utilization=gpu_memory_utilization,
86
+ swap_space=swap_space,
87
**kwargs,
88
)
89
self.llm_engine = LLMEngine.from_engine_args(engine_args)