@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc0644574c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add gpu_memory_utilization and swap_space to LLM (#1090)

Woosuk Kwon committed Sep 19, 2023 at 22:16 UTC bc0644574ca12d754a031596bdcfe8e1f0e6ab39
1 file changed +19 -3
vllm/entrypoints/llm.py
+19 -3
@@ -37,12 +37,22 @@ class LLM:
37 the `torch_dtype` attribute specified in the model config file.
38 However, if the `torch_dtype` in the config is `float32`, we will
39 use `float16` instead.
40 - seed: The seed to initialize the random number generator for sampling.
40 quantization: The method used to quantize the model weights. Currently,
41 we support "awq". If None, we assume the model weights are not
42 quantized and use `dtype` to determine the data type of the weights.
43 revision: The specific model version to use. It can be a branch name,
44 a tag name, or a commit id.
45 + seed: The seed to initialize the random number generator for sampling.
46 + gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
47 + reserve for the model weights, activations, and KV cache. Higher
48 + values will increase the KV cache size and thus improve the model's
49 + throughput. However, if the value is too high, it may cause out-of-
50 + memory (OOM) errors.
51 + swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
52 + This can be used for temporarily storing the states of the requests
53 + when their `best_of` sampling parameters are larger than 1. If all
54 + requests will have `best_of=1`, you can safely set this to 0.
55 + Otherwise, too small values may cause out-of-memory (OOM) errors.
56 """
57
58 def __init__(
@@ -53,8 +63,11 @@ class LLM:
63 trust_remote_code: bool = False,
64 tensor_parallel_size: int = 1,
65 dtype: str = "auto",
56 - seed: int = 0,
66 quantization: Optional[str] = None,
67 + revision: Optional[str] = None,
68 + seed: int = 0,
69 + gpu_memory_utilization: float = 0.9,
70 + swap_space: int = 4,
71 **kwargs,
72 ) -> None:
73 if "disable_log_stats" not in kwargs:
@@ -66,8 +79,11 @@ class LLM:
79 trust_remote_code=trust_remote_code,
80 tensor_parallel_size=tensor_parallel_size,
81 dtype=dtype,
69 - seed=seed,
82 quantization=quantization,
83 + revision=revision,
84 + seed=seed,
85 + gpu_memory_utilization=gpu_memory_utilization,
86 + swap_space=swap_space,
87 **kwargs,
88 )
89 self.llm_engine = LLMEngine.from_engine_args(engine_args)