fix(config): apply presence_penalty/frequency_penalty from override-generation-config (#50769)
Signed-off-by: Chenglun Hu <chenglunhu@gmail.com> Signed-off-by: hclsys <chenglunhu@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
hcl committed
Sep 24, 2026 at 23:48 UTC
a7d203efe549de57d8289ad15dc4fdeccf1179be
5 files changed
+150
-60
tests/config/test_config_generation.py
+57
@@ -109,3 +109,60 @@ def test_unrecognized_env(monkeypatch):
109
fail_on_environ_validation=True,
110
)
111
engine_args.create_engine_config()
112
+
113
+
114
+def test_get_diff_sampling_param_includes_penalties():
115
+ """``--override-generation-config`` must apply presence_penalty and
116
+ frequency_penalty as server-side defaults, like the other sampling
117
+ params. Regression test for the whitelist silently dropping them.
118
+
119
+ Called unbound with a lightweight stand-in so no model is loaded:
120
+ generation_config="vllm" makes the method skip try_get_generation_config
121
+ and read only override_generation_config.
122
+ """
123
+ from types import SimpleNamespace
124
+
125
+ from vllm.config.model import ModelConfig
126
+
127
+ fake = SimpleNamespace(
128
+ generation_config="vllm",
129
+ override_generation_config={
130
+ "presence_penalty": 1.5,
131
+ "frequency_penalty": 0.5,
132
+ "temperature": 0.6,
133
+ },
134
+ )
135
+
136
+ diff = ModelConfig.get_diff_sampling_param(fake) # type: ignore[arg-type]
137
+
138
+ assert diff["presence_penalty"] == 1.5
139
+ assert diff["frequency_penalty"] == 0.5
140
+ assert diff["temperature"] == 0.6
141
+
142
+
143
+def test_generation_config_penalties_survive_diff_sampling_filter():
144
+ """``generation_config.json`` penalties must survive the diff-sampling
145
+ whitelist (regression for #50767, auto path).
146
+
147
+ Covers the ``generation_config="auto"`` branch, where the override
148
+ dict is empty and the values come from ``try_get_generation_config``.
149
+ """
150
+ from types import SimpleNamespace
151
+
152
+ from vllm.config.model import ModelConfig
153
+
154
+ fake = SimpleNamespace(
155
+ generation_config="auto",
156
+ override_generation_config={},
157
+ try_get_generation_config=lambda: {
158
+ "presence_penalty": 1.1,
159
+ "frequency_penalty": 0.7,
160
+ "temperature": 0.9,
161
+ },
162
+ )
163
+
164
+ diff = ModelConfig.get_diff_sampling_param(fake) # type: ignore[arg-type]
165
+
166
+ assert diff["presence_penalty"] == 1.1
167
+ assert diff["frequency_penalty"] == 0.7
168
+ assert diff["temperature"] == 0.9
tests/entrypoints/openai/test_penalty_default_resolution.py
new
+65
@@ -0,0 +1,65 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+"""Unit tests for presence_penalty / frequency_penalty resolution from
5
+default_sampling_params in ChatCompletionRequest and CompletionRequest.
6
+
7
+Regression test for https://github.com/vllm-project/vllm/issues/50767:
8
+these two penalties defaulted to 0.0 (not None) and to_sampling_params()
9
+forwarded them straight through, so server-side defaults coming from
10
+--override-generation-config / generation_config.json were silently
11
+discarded on the /v1/chat/completions and /v1/completions endpoints.
12
+"""
13
+
14
+from vllm.entrypoints.openai.chat_completion.protocol import (
15
+ ChatCompletionRequest,
16
+)
17
+from vllm.entrypoints.openai.completion.protocol import (
18
+ CompletionRequest,
19
+)
20
+
21
+_DEFAULTS = {"presence_penalty": 1.5, "frequency_penalty": 0.5}
22
+
23
+
24
+def _chat(**kwargs):
25
+ return ChatCompletionRequest(
26
+ model="test-model",
27
+ messages=[{"role": "user", "content": "hi"}],
28
+ **kwargs,
29
+ )
30
+
31
+
32
+class TestChatCompletionPenaltyDefaults:
33
+ def test_defaults_applied_when_client_omits(self):
34
+ """Server-default penalties are applied when the client sends none."""
35
+ sp = _chat().to_sampling_params(100, _DEFAULTS)
36
+ assert sp.presence_penalty == 1.5
37
+ assert sp.frequency_penalty == 0.5
38
+
39
+ def test_client_value_overrides_default(self):
40
+ """An explicit client penalty wins over the server default."""
41
+ sp = _chat(presence_penalty=0.2).to_sampling_params(100, _DEFAULTS)
42
+ assert sp.presence_penalty == 0.2
43
+ assert sp.frequency_penalty == 0.5
44
+
45
+ def test_falls_back_to_zero_without_default(self):
46
+ """Without a server default the neutral 0.0 is used (no regression)."""
47
+ sp = _chat().to_sampling_params(100, {})
48
+ assert sp.presence_penalty == 0.0
49
+ assert sp.frequency_penalty == 0.0
50
+
51
+
52
+class TestCompletionPenaltyDefaults:
53
+ def test_defaults_applied_when_client_omits(self):
54
+ sp = CompletionRequest(model="test-model", prompt="hi").to_sampling_params(
55
+ 16, _DEFAULTS
56
+ )
57
+ assert sp.presence_penalty == 1.5
58
+ assert sp.frequency_penalty == 0.5
59
+
60
+ def test_client_value_overrides_default(self):
61
+ sp = CompletionRequest(
62
+ model="test-model", prompt="hi", frequency_penalty=0.9
63
+ ).to_sampling_params(16, _DEFAULTS)
64
+ assert sp.presence_penalty == 1.5
65
+ assert sp.frequency_penalty == 0.9
vllm/config/model.py
+2
@@ -1822,6 +1822,8 @@ class ModelConfig:
1822
1823
available_params = [
1824
"repetition_penalty",
1825
+ "presence_penalty",
1826
+ "frequency_penalty",
1827
"temperature",
1828
"top_k",
1829
"top_p",
vllm/entrypoints/openai/chat_completion/protocol.py
+13
-30
@@ -215,7 +215,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
215
# https://platform.openai.com/docs/api-reference/chat/create
216
messages: list[ChatCompletionMessageParam]
217
model: str | None = None
218
- frequency_penalty: float | None = 0.0
218
+ frequency_penalty: float | None = None
219
logit_bias: dict[str, float] | None = None
220
logprobs: bool | None = False
221
top_logprobs: int | None = 0
@@ -226,7 +226,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
226
)
227
max_completion_tokens: int | None = None
228
n: int | None = 1
229
- presence_penalty: float | None = 0.0
229
+ presence_penalty: float | None = None
230
response_format: AnyResponseFormat | None = None
231
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
232
stop: StopParam = []
@@ -624,6 +624,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
624
# Default sampling parameters for chat completion requests
625
_DEFAULT_SAMPLING_PARAMS: dict = {
626
"repetition_penalty": 1.0,
627
+ "presence_penalty": 0.0,
628
+ "frequency_penalty": 0.0,
629
"temperature": 1.0,
630
"top_p": 1.0,
631
"top_k": 0,
@@ -661,28 +663,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
663
max_tokens: int,
664
default_sampling_params: dict,
665
) -> SamplingParams:
664
- # Default parameters
665
- if (repetition_penalty := self.repetition_penalty) is None:
666
- repetition_penalty = default_sampling_params.get(
667
- "repetition_penalty",
668
- self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
669
- )
670
- if (temperature := self.temperature) is None:
671
- temperature = default_sampling_params.get(
672
- "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
673
- )
674
- if (top_p := self.top_p) is None:
675
- top_p = default_sampling_params.get(
676
- "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
677
- )
678
- if (top_k := self.top_k) is None:
679
- top_k = default_sampling_params.get(
680
- "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
681
- )
682
- if (min_p := self.min_p) is None:
683
- min_p = default_sampling_params.get(
684
- "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
666
+ # Priority: user -> server default -> OpenAI default
667
+ sampling_params = {
668
+ name: (
669
+ value
670
+ if (value := getattr(self, name)) is not None
671
+ else default_sampling_params.get(name, default)
672
)
673
+ for name, default in self._DEFAULT_SAMPLING_PARAMS.items()
674
+ }
675
676
# Merge server-default stop_token_ids (e.g., model-specific tokens
677
# like </call> for gpt-oss) with any request-specified ones
@@ -709,14 +698,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
698
extra_args["ec_transfer_params"] = self.ec_transfer_params
699
return SamplingParams.from_optional(
700
n=self.n,
712
- presence_penalty=self.presence_penalty,
713
- frequency_penalty=self.frequency_penalty,
714
- repetition_penalty=repetition_penalty,
715
- temperature=temperature,
701
+ **sampling_params,
702
watermarking=self.watermarking,
717
- top_p=top_p,
718
- top_k=top_k,
719
- min_p=min_p,
703
seed=self.seed,
704
stop=self.stop,
705
stop_token_ids=stop_token_ids,
vllm/entrypoints/openai/completion/protocol.py
+13
-30
@@ -55,12 +55,12 @@ class CompletionRequest(OpenAIBaseModel):
55
| None
56
) = None
57
echo: bool | None = False
58
- frequency_penalty: float | None = 0.0
58
+ frequency_penalty: float | None = None
59
logit_bias: dict[str, float] | None = None
60
logprobs: int | None = None
61
max_tokens: int | None = 16
62
n: int = 1
63
- presence_penalty: float | None = 0.0
63
+ presence_penalty: float | None = None
64
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
65
stop: StopParam = []
66
stream: bool | None = False
@@ -278,6 +278,8 @@ class CompletionRequest(OpenAIBaseModel):
278
# Default sampling parameters for completion requests
279
_DEFAULT_SAMPLING_PARAMS: dict = {
280
"repetition_penalty": 1.0,
281
+ "presence_penalty": 0.0,
282
+ "frequency_penalty": 0.0,
283
"temperature": 1.0,
284
"top_p": 1.0,
285
"top_k": 0,
@@ -321,28 +323,15 @@ class CompletionRequest(OpenAIBaseModel):
323
if default_sampling_params is None:
324
default_sampling_params = {}
325
324
- # Default parameters
325
- if (repetition_penalty := self.repetition_penalty) is None:
326
- repetition_penalty = default_sampling_params.get(
327
- "repetition_penalty",
328
- self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
329
- )
330
- if (temperature := self.temperature) is None:
331
- temperature = default_sampling_params.get(
332
- "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
333
- )
334
- if (top_p := self.top_p) is None:
335
- top_p = default_sampling_params.get(
336
- "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
337
- )
338
- if (top_k := self.top_k) is None:
339
- top_k = default_sampling_params.get(
340
- "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
341
- )
342
- if (min_p := self.min_p) is None:
343
- min_p = default_sampling_params.get(
344
- "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
326
+ # Priority: user -> server default -> OpenAI default
327
+ sampling_params = {
328
+ name: (
329
+ value
330
+ if (value := getattr(self, name)) is not None
331
+ else default_sampling_params.get(name, default)
332
)
333
+ for name, default in self._DEFAULT_SAMPLING_PARAMS.items()
334
+ }
335
336
# Merge server-default stop_token_ids (e.g., model-specific tokens
337
# like </call> for gpt-oss) with any request-specified ones
@@ -371,14 +360,8 @@ class CompletionRequest(OpenAIBaseModel):
360
extra_args["ec_transfer_params"] = self.ec_transfer_params
361
return SamplingParams.from_optional(
362
n=self.n,
374
- presence_penalty=self.presence_penalty,
375
- frequency_penalty=self.frequency_penalty,
376
- repetition_penalty=repetition_penalty,
377
- temperature=temperature,
363
+ **sampling_params,
364
watermarking=self.watermarking,
379
- top_p=top_p,
380
- top_k=top_k,
381
- min_p=min_p,
365
seed=self.seed,
366
stop=self.stop,
367
stop_token_ids=stop_token_ids,