A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix(async): Add support for truncate_prompt_tokens in AsyncLLM (#23800)
Raghavan committed
Aug 29, 2025 at 11:25 UTC
05d839c19e9582d62c860686678bac68240d7254
1 file changed
+20
vllm/v1/engine/async_llm.py
+20
@@ -15,6 +15,7 @@ import vllm.envs as envs
15
from vllm.config import ModelConfig, VllmConfig
16
from vllm.engine.arg_utils import AsyncEngineArgs
17
from vllm.engine.protocol import EngineClient
18
+from vllm.entrypoints.utils import _validate_truncation_size
19
from vllm.envs import VLLM_V1_OUTPUT_PROC_CHUNK_SIZE
20
from vllm.inputs import PromptType
21
from vllm.inputs.preprocess import InputPreprocessor
@@ -348,6 +349,15 @@ class AsyncLLM(EngineClient):
349
# to handle startup failure gracefully in the OpenAI server.
350
self._run_output_handler()
351
352
+ tokenization_kwargs: dict[str, Any] = {}
353
+ truncate_prompt_tokens = sampling_params.truncate_prompt_tokens
354
+
355
+ _validate_truncation_size(
356
+ self.model_config.max_model_len,
357
+ truncate_prompt_tokens,
358
+ tokenization_kwargs,
359
+ )
360
+
361
q = await self.add_request(
362
request_id,
363
prompt,
@@ -355,6 +365,7 @@ class AsyncLLM(EngineClient):
365
lora_request=lora_request,
366
trace_headers=trace_headers,
367
priority=priority,
368
+ tokenization_kwargs=tokenization_kwargs,
369
data_parallel_rank=data_parallel_rank,
370
)
371
@@ -481,6 +492,7 @@ class AsyncLLM(EngineClient):
492
lora_request: Optional[LoRARequest] = None,
493
trace_headers: Optional[Mapping[str, str]] = None,
494
priority: int = 0,
495
+ truncate_prompt_tokens: Optional[int] = None,
496
tokenization_kwargs: Optional[dict[str, Any]] = None,
497
) -> AsyncGenerator[PoolingRequestOutput, None]:
498
"""
@@ -503,6 +515,14 @@ class AsyncLLM(EngineClient):
515
# to handle startup failure gracefully in the OpenAI server.
516
self._run_output_handler()
517
518
+ if tokenization_kwargs is None:
519
+ tokenization_kwargs = dict[str, Any]()
520
+ _validate_truncation_size(
521
+ self.model_config.max_model_len,
522
+ truncate_prompt_tokens,
523
+ tokenization_kwargs,
524
+ )
525
+
526
q = await self.add_request(
527
request_id,
528
prompt,