A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 12:30 UTC
[responsesAPI]add extra body parameters (#30532)
Signed-off-by: Ri0S <aa248424@gmail.com>
RioS committed
Dec 15, 2025 at 04:25 UTC
9ccbf6b692e0e39995b063a8381a322097cff5e0
1 file changed
+9
vllm/entrypoints/openai/protocol.py
+9
@@ -320,6 +320,7 @@ class ResponsesRequest(OpenAIBaseModel):
320
max_tool_calls: int | None = None
321
metadata: Metadata | None = None
322
model: str | None = None
323
+ logit_bias: dict[str, float] | None = None
324
parallel_tool_calls: bool | None = True
325
previous_response_id: str | None = None
326
prompt: ResponsePrompt | None = None
@@ -333,6 +334,7 @@ class ResponsesRequest(OpenAIBaseModel):
334
tools: list[Tool] = Field(default_factory=list)
335
top_logprobs: int | None = 0
336
top_p: float | None = None
337
+ top_k: int | None = None
338
truncation: Literal["auto", "disabled"] | None = "disabled"
339
user: str | None = None
340
@@ -387,6 +389,7 @@ class ResponsesRequest(OpenAIBaseModel):
389
_DEFAULT_SAMPLING_PARAMS = {
390
"temperature": 1.0,
391
"top_p": 1.0,
392
+ "top_k": 0,
393
}
394
395
def to_sampling_params(
@@ -408,6 +411,10 @@ class ResponsesRequest(OpenAIBaseModel):
411
top_p = default_sampling_params.get(
412
"top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
413
)
414
+ if (top_k := self.top_k) is None:
415
+ top_k = default_sampling_params.get(
416
+ "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
417
+ )
418
stop_token_ids = default_sampling_params.get("stop_token_ids")
419
420
# Structured output
@@ -428,6 +435,7 @@ class ResponsesRequest(OpenAIBaseModel):
435
return SamplingParams.from_optional(
436
temperature=temperature,
437
top_p=top_p,
438
+ top_k=top_k,
439
max_tokens=max_tokens,
440
logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
441
stop_token_ids=stop_token_ids,
@@ -435,6 +443,7 @@ class ResponsesRequest(OpenAIBaseModel):
443
RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
444
),
445
structured_outputs=structured_outputs,
446
+ logit_bias=self.logit_bias,
447
)
448
449
def is_include_output_logprobs(self) -> bool: