A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 17:45 UTC
[Misc] Simplify `get_max_tokens` (#34036)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 7, 2026 at 16:59 UTC
11a4c9d30d205ee4cb8e9a92e34834fc6585fce0
5 files changed
+8
-30
vllm/entrypoints/openai/chat_completion/serving.py
+3
-1
@@ -388,7 +388,9 @@ class OpenAIServingChat(OpenAIServing):
388
389
max_tokens = get_max_tokens(
390
self.max_model_len,
391
- request,
391
+ request.max_completion_tokens
392
+ if request.max_completion_tokens is not None
393
+ else request.max_tokens,
394
self._extract_prompt_len(engine_prompt),
395
self.default_sampling_params,
396
)
vllm/entrypoints/openai/completion/serving.py
+1
-1
@@ -164,7 +164,7 @@ class OpenAIServingCompletion(OpenAIServing):
164
165
max_tokens = get_max_tokens(
166
self.max_model_len,
167
- request,
167
+ request.max_tokens,
168
self._extract_prompt_len(engine_prompt),
169
self.default_sampling_params,
170
)
vllm/entrypoints/openai/engine/serving.py
+1
-1
@@ -1176,7 +1176,7 @@ class OpenAIServing:
1176
1177
sampling_params.max_tokens = get_max_tokens(
1178
self.max_model_len,
1179
- context.request,
1179
+ context.request.max_output_tokens,
1180
self._extract_prompt_len(engine_prompt),
1181
self.default_sampling_params, # type: ignore
1182
)
vllm/entrypoints/openai/responses/serving.py
+1
-1
@@ -441,7 +441,7 @@ class OpenAIServingResponses(OpenAIServing):
441
442
default_max_tokens = get_max_tokens(
443
self.max_model_len,
444
- request,
444
+ request.max_output_tokens,
445
self._extract_prompt_len(engine_prompt),
446
self.default_sampling_params,
447
)
vllm/entrypoints/utils.py
+2
-26
@@ -22,23 +22,11 @@ from vllm.platforms import current_platform
22
from vllm.utils.argparse_utils import FlexibleArgumentParser
23
24
if TYPE_CHECKING:
25
- from vllm.entrypoints.openai.chat_completion.protocol import (
26
- ChatCompletionRequest,
27
- )
28
- from vllm.entrypoints.openai.completion.protocol import (
29
- CompletionRequest,
30
- )
31
- from vllm.entrypoints.openai.engine.protocol import (
32
- StreamOptions,
33
- )
25
+ from vllm.entrypoints.openai.engine.protocol import StreamOptions
26
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
35
- from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
27
else:
37
- ChatCompletionRequest = object
38
- CompletionRequest = object
28
StreamOptions = object
29
LoRAModulePath = object
41
- ResponsesRequest = object
30
31
32
logger = init_logger(__name__)
@@ -186,22 +174,10 @@ def cli_env_setup():
174
175
def get_max_tokens(
176
max_model_len: int,
189
- request: "CompletionRequest | ChatCompletionRequest | ResponsesRequest",
177
+ max_tokens: int | None,
178
input_length: int,
179
default_sampling_params: dict,
180
) -> int:
193
- # NOTE: Avoid isinstance() for better efficiency
194
- max_tokens: int | None = None
195
- if max_tokens is None:
196
- # ChatCompletionRequest
197
- max_tokens = getattr(request, "max_completion_tokens", None)
198
- if max_tokens is None:
199
- # ResponsesRequest
200
- max_tokens = getattr(request, "max_output_tokens", None)
201
- if max_tokens is None:
202
- # CompletionRequest (also a fallback for ChatCompletionRequest)
203
- max_tokens = getattr(request, "max_tokens", None)
204
-
181
default_max_tokens = max_model_len - input_length
182
max_output_tokens = current_platform.get_max_output_tokens(input_length)
183