@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a4113b035c

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 03:30 UTC

[Platform] Add custom default max tokens (#18557)

Signed-off-by: Gabriel Marinho <gmarinho@ibm.com>

Gabriel Marinho committed Jul 3, 2025 at 23:50 UTC a4113b035cd4ee90ec02de0658b75d47f0007ede

5 files changed +59 -60

vllm/entrypoints/openai/protocol.py

+10 -49

@@ -229,7 +229,6 @@ class ChatCompletionRequest(OpenAIBaseModel):

229 logit_bias: Optional[dict[str, float]] = None

230 logprobs: Optional[bool] = False

231 top_logprobs: Optional[int] = 0

232 - # TODO(#9845): remove max_tokens when field is removed from OpenAI API

232 max_tokens: Optional[int] = Field(

233 default=None,

234 deprecated=

@@ -433,23 +432,10 @@ class ChatCompletionRequest(OpenAIBaseModel):

432 }

433

434 def to_beam_search_params(

436 - self,

437 - default_max_tokens: int,

438 - default_sampling_params: Optional[dict] = None

439 - ) -> BeamSearchParams:

440 - # TODO(#9845): remove max_tokens when field is removed from OpenAI API

441 - max_tokens = self.max_completion_tokens or self.max_tokens

435 + self, max_tokens: int,

436 + default_sampling_params: dict) -> BeamSearchParams:

437

443 - if default_sampling_params is None:

444 - default_sampling_params = {}

438 n = self.n if self.n is not None else 1

446 -

447 - # Use minimum of context window, user request & server limit.

448 - max_tokens = min(

449 - val for val in (default_max_tokens, max_tokens,

450 - default_sampling_params.get("max_tokens", None))

451 - if val is not None)

452 -

439 if (temperature := self.temperature) is None:

440 temperature = default_sampling_params.get(

441 "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])

@@ -465,21 +451,10 @@ class ChatCompletionRequest(OpenAIBaseModel):

451

452 def to_sampling_params(

453 self,

468 - default_max_tokens: int,

454 + max_tokens: int,

455 logits_processor_pattern: Optional[str],

470 - default_sampling_params: Optional[dict] = None,

456 + default_sampling_params: dict,

457 ) -> SamplingParams:

472 - # TODO(#9845): remove max_tokens when field is removed from OpenAI API

473 - max_tokens = self.max_completion_tokens or self.max_tokens

474 -

475 - if default_sampling_params is None:

476 - default_sampling_params = {}

477 -

478 - # Use minimum of context window, user request & server limit.

479 - max_tokens = min(

480 - val for val in (default_max_tokens, max_tokens,

481 - default_sampling_params.get("max_tokens", None))

482 - if val is not None)

458

459 # Default parameters

460 if (repetition_penalty := self.repetition_penalty) is None:

@@ -898,22 +873,15 @@ class CompletionRequest(OpenAIBaseModel):

873 }

874

875 def to_beam_search_params(

901 - self,

902 - default_max_tokens: int,

903 - default_sampling_params: Optional[dict] = None

876 + self,

877 + max_tokens: int,

878 + default_sampling_params: Optional[dict] = None,

879 ) -> BeamSearchParams:

905 - max_tokens = self.max_tokens

880

881 if default_sampling_params is None:

882 default_sampling_params = {}

883 n = self.n if self.n is not None else 1

884

911 - # Use minimum of context window, user request & server limit.

912 - max_tokens = min(

913 - val for val in (default_max_tokens, max_tokens,

914 - default_sampling_params.get("max_tokens", None))

915 - if val is not None)

916 -

885 if (temperature := self.temperature) is None:

886 temperature = default_sampling_params.get("temperature", 1.0)

887

@@ -928,21 +896,14 @@ class CompletionRequest(OpenAIBaseModel):

896

897 def to_sampling_params(

898 self,

931 - default_max_tokens: int,

899 + max_tokens: int,

900 logits_processor_pattern: Optional[str],

901 default_sampling_params: Optional[dict] = None,

902 ) -> SamplingParams:

935 - max_tokens = self.max_tokens

903

904 if default_sampling_params is None:

905 default_sampling_params = {}

906

940 - # Use minimum of context window, user request & server limit.

941 - max_tokens = min(

942 - val for val in (default_max_tokens, max_tokens,

943 - default_sampling_params.get("max_tokens", None))

944 - if val is not None)

945 -

907 # Default parameters

908 if (repetition_penalty := self.repetition_penalty) is None:

909 repetition_penalty = default_sampling_params.get(

@@ -1813,7 +1774,7 @@ class TranscriptionRequest(OpenAIBaseModel):

1774 self,

1775 default_max_tokens: int,

1776 default_sampling_params: Optional[dict] = None) -> SamplingParams:

1816 - # TODO(#9845): remove max_tokens when field is removed from OpenAI API

1777 +

1778 max_tokens = default_max_tokens

1779

1780 if default_sampling_params is None:

@@ -2029,7 +1990,7 @@ class TranslationRequest(OpenAIBaseModel):

1990 self,

1991 default_max_tokens: int,

1992 default_sampling_params: Optional[dict] = None) -> SamplingParams:

2032 - # TODO(#9845): remove max_tokens when field is removed from OpenAI API

1993 +

1994 max_tokens = default_max_tokens

1995

1996 if default_sampling_params is None:

vllm/entrypoints/openai/serving_chat.py

+13 -5

@@ -34,6 +34,7 @@ from vllm.entrypoints.openai.serving_models import OpenAIServingModels

34 from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager

35 from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (

36 MistralToolCall)

37 +from vllm.entrypoints.utils import get_max_tokens

38 from vllm.logger import init_logger

39 from vllm.outputs import CompletionOutput, RequestOutput

40 from vllm.reasoning import ReasoningParser, ReasoningParserManager

@@ -233,15 +234,22 @@ class OpenAIServingChat(OpenAIServing):

234 try:

235 for i, engine_prompt in enumerate(engine_prompts):

236 sampling_params: Union[SamplingParams, BeamSearchParams]

236 - default_max_tokens = self.max_model_len - len(

237 - engine_prompt["prompt_token_ids"])

237 +

238 + if self.default_sampling_params is None:

239 + self.default_sampling_params = {}

240 +

241 + max_tokens = get_max_tokens(

242 + max_model_len=self.max_model_len,

243 + request=request,

244 + input_length=len(engine_prompt["prompt_token_ids"]),

245 + default_sampling_params=self.default_sampling_params)

246 +

247 if request.use_beam_search:

248 sampling_params = request.to_beam_search_params(

240 - default_max_tokens, self.default_sampling_params)

249 + max_tokens, self.default_sampling_params)

250 else:

251 sampling_params = request.to_sampling_params(

243 - default_max_tokens,

244 - self.model_config.logits_processor_pattern,

252 + max_tokens, self.model_config.logits_processor_pattern,

253 self.default_sampling_params)

254

255 self._log_inputs(request_id,

vllm/entrypoints/openai/serving_completion.py

+12 -4

@@ -33,6 +33,7 @@ from vllm.entrypoints.openai.serving_engine import (OpenAIServing,

33 is_text_tokens_prompt)

34 # yapf: enable

35 from vllm.entrypoints.openai.serving_models import OpenAIServingModels

36 +from vllm.entrypoints.utils import get_max_tokens

37 from vllm.inputs.data import (EmbedsPrompt, TokensPrompt, is_embeds_prompt,

38 is_tokens_prompt)

39 from vllm.logger import init_logger

@@ -160,15 +161,22 @@ class OpenAIServingCompletion(OpenAIServing):

161 input_length = len(engine_prompt["prompt_token_ids"])

162 else:

163 assert_never(engine_prompt)

163 - default_max_tokens = self.max_model_len - input_length

164 +

165 + if self.default_sampling_params is None:

166 + self.default_sampling_params = {}

167 +

168 + max_tokens = get_max_tokens(

169 + max_model_len=self.max_model_len,

170 + request=request,

171 + input_length=input_length,

172 + default_sampling_params=self.default_sampling_params)

173

174 if request.use_beam_search:

175 sampling_params = request.to_beam_search_params(

167 - default_max_tokens, self.default_sampling_params)

176 + max_tokens, self.default_sampling_params)

177 else:

178 sampling_params = request.to_sampling_params(

170 - default_max_tokens,

171 - self.model_config.logits_processor_pattern,

179 + max_tokens, self.model_config.logits_processor_pattern,

180 self.default_sampling_params)

181

182 request_id_item = f"{request_id}-{i}"

vllm/entrypoints/utils.py

+20 -2

@@ -5,13 +5,17 @@ import argparse

5 import asyncio

6 import functools

7 import os

8 -from typing import Any, Optional

8 +import sys

9 +from typing import Any, Optional, Union

11 from fastapi import Request

12 from fastapi.responses import JSONResponse, StreamingResponse

13 from starlette.background import BackgroundTask, BackgroundTasks

15 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,

16 + CompletionRequest)

17 from vllm.logger import init_logger

18 +from vllm.platforms import current_platform

20 logger = init_logger(__name__)

@@ -181,7 +185,6 @@ def _validate_truncation_size(

185

186 def show_filtered_argument_or_group_from_help(parser: argparse.ArgumentParser,

187 subcommand_name: list[str]):

184 - import sys

188

189 # Only handle --help=<keyword> for the current subcommand.

190 # Since subparser_init() runs for all subcommands during CLI setup,

@@ -242,3 +245,18 @@ def show_filtered_argument_or_group_from_help(parser: argparse.ArgumentParser,

245 print(f"\nNo group or parameter matching '{search_keyword}'")

246 print("Tip: use `--help=listgroup` to view all groups.")

247 sys.exit(1)

248 +

249 +

250 +def get_max_tokens(max_model_len: int, request: Union[ChatCompletionRequest,

251 + CompletionRequest],

252 + input_length: int, default_sampling_params: dict) -> int:

253 +

254 + max_tokens = getattr(request, "max_completion_tokens",

255 + None) or request.max_tokens

256 + default_max_tokens = max_model_len - input_length

257 + max_output_tokens = current_platform.get_max_output_tokens(input_length)

258 +

259 + return min(val

260 + for val in (default_max_tokens, max_tokens, max_output_tokens,

261 + default_sampling_params.get("max_tokens"))

262 + if val is not None)

vllm/platforms/interface.py

@@ -4,6 +4,7 @@ import enum

4 import os

5 import platform

6 import random

7 +import sys

8 from datetime import timedelta

9 from platform import uname

10 from typing import TYPE_CHECKING, NamedTuple, Optional, Union

@@ -164,6 +165,9 @@ class Platform:

165 def is_out_of_tree(self) -> bool:

166 return self._enum == PlatformEnum.OOT

167

168 + def get_max_output_tokens(self, prompt_len: int) -> int:

169 + return sys.maxsize

170 +

171 def is_cuda_alike(self) -> bool:

172 """Stateless version of [torch.cuda.is_available][]."""

173 return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)