A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
Default to `generation_config` from model (#12622)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 8, 2025 at 07:46 UTC
47512b3200156cc0db4a62791d2f7fefcaecb6ad
7 files changed
+27
-26
tests/entrypoints/openai/correctness/test_lmeval.py
+1
-1
@@ -20,7 +20,7 @@ NUM_CONCURRENT = 500
20
TASK = "gsm8k"
21
FILTER = "exact_match,strict-match"
22
RTOL = 0.03
23
-EXPECTED_VALUE = 0.58
23
+EXPECTED_VALUE = 0.54
24
DEFAULT_ARGS = ["--max-model-len", "4096", "--disable-log-requests"]
25
MORE_ARGS_LIST = [
26
[], # Default
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -38,6 +38,7 @@ class MockModelConfig:
38
diff_sampling_param: Optional[dict] = None
39
allowed_local_media_path: str = ""
40
encoder_config = None
41
+ generation_config: str = "auto"
42
43
def get_diff_sampling_param(self):
44
return self.diff_sampling_param or {}
tests/test_config.py
+4
-4
@@ -289,7 +289,7 @@ def test_uses_mrope(model_id, uses_mrope):
289
def test_generation_config_loading():
290
model_id = "Qwen/Qwen2.5-1.5B-Instruct"
291
292
- # When set generation_config to None, the default generation config
292
+ # When set generation_config to "vllm", the default generation config
293
# will not be loaded.
294
model_config = ModelConfig(model_id,
295
task="auto",
@@ -298,7 +298,7 @@ def test_generation_config_loading():
298
trust_remote_code=False,
299
seed=0,
300
dtype="float16",
301
- generation_config=None)
301
+ generation_config="vllm")
302
assert model_config.get_diff_sampling_param() == {}
303
304
# When set generation_config to "auto", the default generation config
@@ -340,7 +340,7 @@ def test_generation_config_loading():
340
341
assert model_config.get_diff_sampling_param() == override_result
342
343
- # When generation_config is set to None and override_generation_config
343
+ # When generation_config is set to "vllm" and override_generation_config
344
# is set, the override_generation_config should be used directly.
345
model_config = ModelConfig(
346
model_id,
@@ -350,7 +350,7 @@ def test_generation_config_loading():
350
trust_remote_code=False,
351
seed=0,
352
dtype="float16",
353
- generation_config=None,
353
+ generation_config="vllm",
354
override_generation_config=override_generation_config)
355
356
assert model_config.get_diff_sampling_param() == override_generation_config
vllm/config.py
+6
-9
@@ -255,7 +255,7 @@ class ModelConfig:
255
override_neuron_config: Optional[dict[str, Any]] = None,
256
override_pooler_config: Optional["PoolerConfig"] = None,
257
logits_processor_pattern: Optional[str] = None,
258
- generation_config: Optional[str] = None,
258
+ generation_config: str = "auto",
259
enable_sleep_mode: bool = False,
260
override_generation_config: Optional[dict[str, Any]] = None,
261
model_impl: Union[str, ModelImpl] = ModelImpl.AUTO,
@@ -951,7 +951,7 @@ class ModelConfig:
951
return self.multimodal_config
952
953
def try_get_generation_config(self) -> dict[str, Any]:
954
- if self.generation_config is None or self.generation_config == "auto":
954
+ if self.generation_config in ("auto", "vllm"):
955
config = try_get_generation_config(
956
self.hf_config_path or self.model,
957
trust_remote_code=self.trust_remote_code,
@@ -971,17 +971,14 @@ class ModelConfig:
971
def get_diff_sampling_param(self) -> dict[str, Any]:
972
"""
973
This method returns a dictionary containing the parameters
974
- that differ from the default sampling parameters, but only
975
- if `generation_config` is set. If `generation_config` is not
976
- set, an empty dictionary is returned.
974
+ that differ from the default sampling parameters. If
975
+ `generation_config` is `"vllm"`, an empty dictionary is returned.
976
977
Returns:
978
dict[str, Any]: A dictionary with the differing sampling
980
- parameters if `generation_config` is set, otherwise an
981
- empty dictionary.
979
+ parameters, if `generation_config` is `"vllm"` an empty dictionary.
980
"""
983
- if self.generation_config is None:
984
- # When generation_config is not set
981
+ if self.generation_config == "vllm":
982
config = {}
983
else:
984
config = self.try_get_generation_config()
vllm/engine/arg_utils.py
+7
-7
@@ -207,7 +207,7 @@ class EngineArgs:
207
208
kv_transfer_config: Optional[KVTransferConfig] = None
209
210
- generation_config: Optional[str] = None
210
+ generation_config: Optional[str] = "auto"
211
override_generation_config: Optional[Dict[str, Any]] = None
212
enable_sleep_mode: bool = False
213
model_impl: str = "auto"
@@ -1034,13 +1034,13 @@ class EngineArgs:
1034
parser.add_argument(
1035
"--generation-config",
1036
type=nullable_str,
1037
- default=None,
1037
+ default="auto",
1038
help="The folder path to the generation config. "
1039
- "Defaults to None, no generation config is loaded, vLLM defaults "
1040
- "will be used. If set to 'auto', the generation config will be "
1041
- "loaded from model path. If set to a folder path, the generation "
1042
- "config will be loaded from the specified folder path. If "
1043
- "`max_new_tokens` is specified in generation config, then "
1039
+ "Defaults to 'auto', the generation config will be loaded from "
1040
+ "model path. If set to 'vllm', no generation config is loaded, "
1041
+ "vLLM defaults will be used. If set to a folder path, the "
1042
+ "generation config will be loaded from the specified folder path. "
1043
+ "If `max_new_tokens` is specified in generation config, then "
1044
"it sets a server-wide limit on the number of output tokens "
1045
"for all requests.")
1046
vllm/entrypoints/openai/serving_chat.py
+4
-2
@@ -109,8 +109,10 @@ class OpenAIServingChat(OpenAIServing):
109
self.default_sampling_params = (
110
self.model_config.get_diff_sampling_param())
111
if self.default_sampling_params:
112
- logger.info("Overwriting default chat sampling param with: %s",
113
- self.default_sampling_params)
112
+ source = self.model_config.generation_config
113
+ source = "model" if source == "auto" else source
114
+ logger.info("Using default chat sampling params from %s: %s",
115
+ source, self.default_sampling_params)
116
117
async def create_chat_completion(
118
self,
vllm/entrypoints/openai/serving_completion.py
+4
-3
@@ -55,9 +55,10 @@ class OpenAIServingCompletion(OpenAIServing):
55
self.default_sampling_params = (
56
self.model_config.get_diff_sampling_param())
57
if self.default_sampling_params:
58
- logger.info(
59
- "Overwriting default completion sampling param with: %s",
60
- self.default_sampling_params)
58
+ source = self.model_config.generation_config
59
+ source = "model" if source == "auto" else source
60
+ logger.info("Using default completion sampling params from %s: %s",
61
+ source, self.default_sampling_params)
62
63
async def create_completion(
64
self,