@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 995a148575
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[doc]Update config docstring (#10732)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>

wangxiyuan committed Dec 2, 2024 at 12:14 UTC 995a148575aaacc7889ff0d29a96195c329422ab
1 file changed +12 -1
vllm/config.py
+12 -1
@@ -91,6 +91,8 @@ class ModelConfig:
91 the default version.
92 max_model_len: Maximum length of a sequence (including prompt and
93 output). If None, will be derived from the model.
94 + spec_target_max_model_len: Specify the the maximum length for spec
95 + decoding draft models.
96 quantization: Quantization method that was used to quantize the model
97 weights. If None, we assume the model weights are not quantized.
98 quantization_param_path: Path to JSON file containing scaling factors.
@@ -107,6 +109,7 @@ class ModelConfig:
109 to eager mode. Additionally for encoder-decoder models, if the
110 sequence length of the encoder input is larger than this, we fall
111 back to the eager mode.
112 + max_logprobs: Maximum number of log probabilities. Defaults to 20.
113 disable_sliding_window: Whether to disable sliding window. If True,
114 we will disable the sliding window functionality of the model.
115 If the model does not support sliding window, this argument is
@@ -119,6 +122,8 @@ class ModelConfig:
122 the model name will be the same as `model`.
123 limit_mm_per_prompt: Maximum number of data items per modality
124 per prompt. Only applicable for multimodal models.
125 + use_async_output_proc: Whether to use async output processor.
126 + Defaults to True.
127 config_format: The config format which shall be loaded.
128 Defaults to 'auto' which defaults to 'hf'.
129 hf_overrides: If a dictionary, contains arguments to be forwarded to the
@@ -130,7 +135,7 @@ class ModelConfig:
135 override default neuron config that are specific to Neuron devices,
136 this argument will be used to configure the neuron config that
137 can not be gathered from the vllm arguments.
133 - override_pooling_config: Initialize non default pooling config or
138 + override_pooler_config: Initialize non default pooling config or
139 override default pooling config for the embedding model.
140 """
141
@@ -734,8 +739,13 @@ class CacheConfig:
739 vLLM execution.
740 swap_space: Size of the CPU swap space per GPU (in GiB).
741 cache_dtype: Data type for kv cache storage.
742 + is_attention_free: Whether the model is attention-free.
743 num_gpu_blocks_override: Number of GPU blocks to use. This overrides the
744 profiled num_gpu_blocks if specified. Does nothing if None.
745 + sliding_window: Sliding window size for the KV cache. Can not work with
746 + prefix caching enabled.
747 + enable_prefix_caching: Whether to enable prefix caching.
748 + cpu_offload_gb: Size of the CPU offload buffer in GiB.
749 """
750
751 def __init__(
@@ -904,6 +914,7 @@ class LoadConfig:
914 "tensorizer" will use CoreWeave's tensorizer library for
915 fast weight loading.
916 "bitsandbytes" will load nf4 type weights.
917 + model_loader_extra_config: The extra config for the model loader.
918 ignore_patterns: The list of patterns to ignore when loading the model.
919 Default to "original/**/*" to avoid repeated loading of llama's
920 checkpoints.