A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Frontend] Support override generation config in args (#12409)
Signed-off-by: liuyanyi <wolfsonliu@163.com>
Yanyi Liu committed
Jan 29, 2025 at 17:41 UTC
ff7424f491935a1b4737bcc1570de0d616fc22f3
3 files changed
+100
-8
tests/test_config.py
+70
@@ -281,3 +281,73 @@ def test_uses_mrope(model_id, uses_mrope):
281
)
282
283
assert config.uses_mrope == uses_mrope
284
+
285
+
286
+def test_generation_config_loading():
287
+ model_id = "Qwen/Qwen2.5-1.5B-Instruct"
288
+
289
+ # When set generation_config to None, the default generation config
290
+ # will not be loaded.
291
+ model_config = ModelConfig(model_id,
292
+ task="auto",
293
+ tokenizer=model_id,
294
+ tokenizer_mode="auto",
295
+ trust_remote_code=False,
296
+ seed=0,
297
+ dtype="float16",
298
+ generation_config=None)
299
+ assert model_config.get_diff_sampling_param() == {}
300
+
301
+ # When set generation_config to "auto", the default generation config
302
+ # should be loaded.
303
+ model_config = ModelConfig(model_id,
304
+ task="auto",
305
+ tokenizer=model_id,
306
+ tokenizer_mode="auto",
307
+ trust_remote_code=False,
308
+ seed=0,
309
+ dtype="float16",
310
+ generation_config="auto")
311
+
312
+ correct_generation_config = {
313
+ "repetition_penalty": 1.1,
314
+ "temperature": 0.7,
315
+ "top_p": 0.8,
316
+ "top_k": 20,
317
+ }
318
+
319
+ assert model_config.get_diff_sampling_param() == correct_generation_config
320
+
321
+ # The generation config could be overridden by the user.
322
+ override_generation_config = {"temperature": 0.5, "top_k": 5}
323
+
324
+ model_config = ModelConfig(
325
+ model_id,
326
+ task="auto",
327
+ tokenizer=model_id,
328
+ tokenizer_mode="auto",
329
+ trust_remote_code=False,
330
+ seed=0,
331
+ dtype="float16",
332
+ generation_config="auto",
333
+ override_generation_config=override_generation_config)
334
+
335
+ override_result = correct_generation_config.copy()
336
+ override_result.update(override_generation_config)
337
+
338
+ assert model_config.get_diff_sampling_param() == override_result
339
+
340
+ # When generation_config is set to None and override_generation_config
341
+ # is set, the override_generation_config should be used directly.
342
+ model_config = ModelConfig(
343
+ model_id,
344
+ task="auto",
345
+ tokenizer=model_id,
346
+ tokenizer_mode="auto",
347
+ trust_remote_code=False,
348
+ seed=0,
349
+ dtype="float16",
350
+ generation_config=None,
351
+ override_generation_config=override_generation_config)
352
+
353
+ assert model_config.get_diff_sampling_param() == override_generation_config
vllm/config.py
+11
-2
@@ -165,6 +165,8 @@ class ModelConfig:
165
`logits_processors` extra completion argument. Defaults to None,
166
which allows no processors.
167
generation_config: Configuration parameter file for generation.
168
+ override_generation_config: Override the generation config with the
169
+ given config.
170
"""
171
172
def compute_hash(self) -> str:
@@ -225,6 +227,7 @@ class ModelConfig:
227
logits_processor_pattern: Optional[str] = None,
228
generation_config: Optional[str] = None,
229
enable_sleep_mode: bool = False,
230
+ override_generation_config: Optional[Dict[str, Any]] = None,
231
) -> None:
232
self.model = model
233
self.tokenizer = tokenizer
@@ -368,6 +371,7 @@ class ModelConfig:
371
self.logits_processor_pattern = logits_processor_pattern
372
373
self.generation_config = generation_config
374
+ self.override_generation_config = override_generation_config or {}
375
376
self._verify_quantization()
377
self._verify_cuda_graph()
@@ -904,8 +908,13 @@ class ModelConfig:
908
"""
909
if self.generation_config is None:
910
# When generation_config is not set
907
- return {}
908
- config = self.try_get_generation_config()
911
+ config = {}
912
+ else:
913
+ config = self.try_get_generation_config()
914
+
915
+ # Overriding with given generation config
916
+ config.update(self.override_generation_config)
917
+
918
available_params = [
919
"repetition_penalty",
920
"temperature",
vllm/engine/arg_utils.py
+19
-6
@@ -195,6 +195,7 @@ class EngineArgs:
195
kv_transfer_config: Optional[KVTransferConfig] = None
196
197
generation_config: Optional[str] = None
198
+ override_generation_config: Optional[Dict[str, Any]] = None
199
enable_sleep_mode: bool = False
200
201
calculate_kv_scales: Optional[bool] = None
@@ -936,12 +937,23 @@ class EngineArgs:
937
type=nullable_str,
938
default=None,
939
help="The folder path to the generation config. "
939
- "Defaults to None, will use the default generation config in vLLM. "
940
- "If set to 'auto', the generation config will be automatically "
941
- "loaded from model. If set to a folder path, the generation config "
942
- "will be loaded from the specified folder path. If "
943
- "`max_new_tokens` is specified, then it sets a server-wide limit "
944
- "on the number of output tokens for all requests.")
940
+ "Defaults to None, no generation config is loaded, vLLM defaults "
941
+ "will be used. If set to 'auto', the generation config will be "
942
+ "loaded from model path. If set to a folder path, the generation "
943
+ "config will be loaded from the specified folder path. If "
944
+ "`max_new_tokens` is specified in generation config, then "
945
+ "it sets a server-wide limit on the number of output tokens "
946
+ "for all requests.")
947
+
948
+ parser.add_argument(
949
+ "--override-generation-config",
950
+ type=json.loads,
951
+ default=None,
952
+ help="Overrides or sets generation config in JSON format. "
953
+ "e.g. ``{\"temperature\": 0.5}``. If used with "
954
+ "--generation-config=auto, the override parameters will be merged "
955
+ "with the default config from the model. If generation-config is "
956
+ "None, only the override parameters are used.")
957
958
parser.add_argument("--enable-sleep-mode",
959
action="store_true",
@@ -1002,6 +1014,7 @@ class EngineArgs:
1014
override_pooler_config=self.override_pooler_config,
1015
logits_processor_pattern=self.logits_processor_pattern,
1016
generation_config=self.generation_config,
1017
+ override_generation_config=self.override_generation_config,
1018
enable_sleep_mode=self.enable_sleep_mode,
1019
)
1020