A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Frontend] Expose custom args in OpenAI APIs (#16862)
Signed-off-by: Andrew Feldman <afeldman@neuralmagic.com> Signed-off-by: Andrew Feldman <afeldman@redhat.com> Co-authored-by: Nick Hill <nhill@redhat.com>
afeldman-nm committed
Jun 18, 2025 at 20:41 UTC
dfada85eee0a24b982525cef3723498c016f1122
3 files changed
+44
-14
benchmarks/kernels/benchmark_moe_align_block_size.py
+1
-1
@@ -4,12 +4,12 @@ import argparse
4
import itertools
5
6
import torch
7
-import triton
7
8
from vllm import _custom_ops as ops
9
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
10
moe_align_block_size_triton,
11
)
12
+from vllm.triton_utils import triton
13
14
15
def get_topk_ids(num_tokens: int, num_experts: int, topk: int) -> torch.Tensor:
vllm/entrypoints/openai/protocol.py
+41
-11
@@ -326,8 +326,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
326
)
327
chat_template_kwargs: Optional[dict[str, Any]] = Field(
328
default=None,
329
- description=("Additional kwargs to pass to the template renderer. "
330
- "Will be accessible by the chat template."),
329
+ description=(
330
+ "Additional keyword args to pass to the template renderer. "
331
+ "Will be accessible by the chat template."),
332
)
333
mm_processor_kwargs: Optional[dict[str, Any]] = Field(
334
default=None,
@@ -414,6 +415,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
415
default=None,
416
description="KVTransfer parameters used for disaggregated serving.")
417
418
+ vllm_xargs: Optional[dict[str, Union[str, int, float]]] = Field(
419
+ default=None,
420
+ description=("Additional request parameters with string or "
421
+ "numeric values, used by custom extensions."),
422
+ )
423
+
424
# --8<-- [end:chat-completion-extra-params]
425
426
# Default sampling parameters for chat completion requests
@@ -523,6 +530,10 @@ class ChatCompletionRequest(OpenAIBaseModel):
530
structural_tag=self.structural_tag,
531
)
532
533
+ extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
534
+ if self.kv_transfer_params:
535
+ # Pass in kv_transfer_params via extra_args
536
+ extra_args["kv_transfer_params"] = self.kv_transfer_params
537
return SamplingParams.from_optional(
538
n=self.n,
539
best_of=self.best_of,
@@ -553,8 +564,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
564
logit_bias=self.logit_bias,
565
bad_words= self.bad_words,
566
allowed_token_ids=self.allowed_token_ids,
556
- extra_args=({"kv_transfer_params": self.kv_transfer_params}
557
- if self.kv_transfer_params else None))
567
+ extra_args=extra_args or None,
568
+ )
569
570
def _get_guided_json_from_tool(
571
self) -> Optional[Union[str, dict, BaseModel]]:
@@ -871,6 +882,12 @@ class CompletionRequest(OpenAIBaseModel):
882
default=None,
883
description="KVTransfer parameters used for disaggregated serving.")
884
885
+ vllm_xargs: Optional[dict[str, Union[str, int, float]]] = Field(
886
+ default=None,
887
+ description=("Additional request parameters with string or "
888
+ "numeric values, used by custom extensions."),
889
+ )
890
+
891
# --8<-- [end:completion-extra-params]
892
893
# Default sampling parameters for completion requests
@@ -968,6 +985,10 @@ class CompletionRequest(OpenAIBaseModel):
985
whitespace_pattern=self.guided_whitespace_pattern,
986
)
987
988
+ extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
989
+ if self.kv_transfer_params:
990
+ # Pass in kv_transfer_params via extra_args
991
+ extra_args["kv_transfer_params"] = self.kv_transfer_params
992
return SamplingParams.from_optional(
993
n=self.n,
994
best_of=self.best_of,
@@ -997,8 +1018,8 @@ class CompletionRequest(OpenAIBaseModel):
1018
guided_decoding=guided_decoding,
1019
logit_bias=self.logit_bias,
1020
allowed_token_ids=self.allowed_token_ids,
1000
- extra_args=({"kv_transfer_params": self.kv_transfer_params}
1001
- if self.kv_transfer_params else None))
1021
+ extra_args=extra_args or None,
1022
+ )
1023
1024
@model_validator(mode="before")
1025
@classmethod
@@ -1117,8 +1138,9 @@ class EmbeddingChatRequest(OpenAIBaseModel):
1138
)
1139
chat_template_kwargs: Optional[dict[str, Any]] = Field(
1140
default=None,
1120
- description=("Additional kwargs to pass to the template renderer. "
1121
- "Will be accessible by the chat template."),
1141
+ description=(
1142
+ "Additional keyword args to pass to the template renderer. "
1143
+ "Will be accessible by the chat template."),
1144
)
1145
mm_processor_kwargs: Optional[dict[str, Any]] = Field(
1146
default=None,
@@ -1623,8 +1645,9 @@ class TokenizeChatRequest(OpenAIBaseModel):
1645
)
1646
chat_template_kwargs: Optional[dict[str, Any]] = Field(
1647
default=None,
1626
- description=("Additional kwargs to pass to the template renderer. "
1627
- "Will be accessible by the chat template."),
1648
+ description=(
1649
+ "Additional keyword args to pass to the template renderer. "
1650
+ "Will be accessible by the chat template."),
1651
)
1652
mm_processor_kwargs: Optional[dict[str, Any]] = Field(
1653
default=None,
@@ -1736,6 +1759,12 @@ class TranscriptionRequest(OpenAIBaseModel):
1759
# Flattened stream option to simplify form data.
1760
stream_include_usage: Optional[bool] = False
1761
stream_continuous_usage_stats: Optional[bool] = False
1762
+
1763
+ vllm_xargs: Optional[dict[str, Union[str, int, float]]] = Field(
1764
+ default=None,
1765
+ description=("Additional request parameters with string or "
1766
+ "numeric values, used by custom extensions."),
1767
+ )
1768
# --8<-- [end:transcription-extra-params]
1769
1770
# --8<-- [start:transcription-sampling-params]
@@ -1823,7 +1852,8 @@ class TranscriptionRequest(OpenAIBaseModel):
1852
presence_penalty=self.presence_penalty,
1853
output_kind=RequestOutputKind.DELTA
1854
if self.stream \
1826
- else RequestOutputKind.FINAL_ONLY)
1855
+ else RequestOutputKind.FINAL_ONLY,
1856
+ extra_args=self.vllm_xargs)
1857
1858
@model_validator(mode="before")
1859
@classmethod
vllm/sampling_params.py
+2
-2
@@ -198,8 +198,8 @@ class SamplingParams(
198
processor which only retains scores for the given token ids.
199
Defaults to None.
200
extra_args: Arbitrary additional args, that can be used by custom
201
- sampling implementations. Not used by any in-tree sampling
202
- implementations.
201
+ sampling implementations, plugins, etc. Not used by any in-tree
202
+ sampling implementations.
203
"""
204
205
n: int = 1