@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 43e7593031

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 22:30 UTC

Support tokenization_kwargs override (#29794)

Signed-off-by: piood <2477084691@qq.com>

Yu Jiaqi committed Dec 6, 2025 at 17:12 UTC 43e75930318e1cb101fb6178876b027634f91f71

3 files changed +49 -8

tests/conftest.py

+13 -4

@@ -405,6 +405,7 @@ class HfRunner:

405 images: PromptImageInput | None = None,

406 videos: PromptVideoInput | None = None,

407 audios: PromptAudioInput | None = None,

408 + tokenization_kwargs: dict[str, Any] | None = None,

409 ) -> list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]]:

410 if images is not None:

411 assert len(prompts) == len(images)

@@ -418,10 +419,18 @@ class HfRunner:

419 all_inputs: list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]] = []

420 for i, prompt in enumerate(prompts):

421 if isinstance(prompt, str):

421 - processor_kwargs: dict[str, Any] = {

422 - "text": prompt,

423 - "return_tensors": "pt",

424 - }

422 + # Create a copy to avoid modifying the original dict

423 + processor_kwargs = (

424 + tokenization_kwargs.copy()

425 + if tokenization_kwargs is not None

426 + else {}

427 + )

428 + processor_kwargs.update(

429 + {

430 + "text": prompt,

431 + "return_tensors": "pt",

432 + }

433 + )

434 if images is not None and (image := images[i]) is not None:

435 processor_kwargs["images"] = image

436 if videos is not None and (video := videos[i]) is not None:

tests/models/multimodal/pooling/test_siglip.py

+16 -2

@@ -1,6 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 +from typing import Any

5 +

6 import pytest

7 from transformers import SiglipModel

@@ -35,7 +37,11 @@ def _run_test(

37 model: str,

38 *,

39 dtype: str,

40 + tokenization_kwargs: dict[str, Any] | None = None,

41 ) -> None:

42 + if tokenization_kwargs is None:

43 + tokenization_kwargs = {}

44 +

45 with vllm_runner(

46 model,

47 runner="pooling",

@@ -44,10 +50,14 @@ def _run_test(

50 max_model_len=64,

51 gpu_memory_utilization=0.7,

52 ) as vllm_model:

47 - vllm_outputs = vllm_model.embed(input_texts, images=input_images)

53 + vllm_outputs = vllm_model.embed(

54 + input_texts, images=input_images, tokenization_kwargs=tokenization_kwargs

55 + )

57 with hf_runner(model, dtype=dtype, auto_cls=SiglipModel) as hf_model:

50 - all_inputs = hf_model.get_inputs(input_texts, images=input_images)

58 + all_inputs = hf_model.get_inputs(

59 + input_texts, images=input_images, tokenization_kwargs=tokenization_kwargs

60 + )

62 all_outputs = []

63 for inputs in all_inputs:

@@ -94,6 +104,10 @@ def test_models_text(

104 input_images, # type: ignore

105 model,

106 dtype=dtype,

107 + tokenization_kwargs={

108 + "padding": "max_length",

109 + "max_length": 64,

110 + }, # siglip2 was trained with this padding setting.

111 )

112

113

vllm/entrypoints/llm.py

+20 -2

@@ -1076,6 +1076,7 @@ class LLM:

1076 params=pooling_params,

1077 use_tqdm=use_tqdm,

1078 lora_request=lora_request,

1079 + tokenization_kwargs=tokenization_kwargs,

1080 )

1081

1082 outputs = self._run_engine(use_tqdm=use_tqdm)

@@ -1113,6 +1114,7 @@ class LLM:

1114 use_tqdm: bool | Callable[..., tqdm] = True,

1115 pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,

1116 lora_request: list[LoRARequest] | LoRARequest | None = None,

1117 + tokenization_kwargs: dict[str, Any] | None = None,

1118 ) -> list[EmbeddingRequestOutput]:

1119 """

1120 Generate an embedding vector for each prompt.

@@ -1150,6 +1152,7 @@ class LLM:

1152 pooling_params=pooling_params,

1153 lora_request=lora_request,

1154 pooling_task="embed",

1155 + tokenization_kwargs=tokenization_kwargs,

1156 )

1157

1158 return [EmbeddingRequestOutput.from_base(item) for item in items]

@@ -1161,6 +1164,7 @@ class LLM:

1164 use_tqdm: bool | Callable[..., tqdm] = True,

1165 pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,

1166 lora_request: list[LoRARequest] | LoRARequest | None = None,

1167 + tokenization_kwargs: dict[str, Any] | None = None,

1168 ) -> list[ClassificationRequestOutput]:

1169 """

1170 Generate class logits for each prompt.

@@ -1196,6 +1200,7 @@ class LLM:

1200 pooling_params=pooling_params,

1201 lora_request=lora_request,

1202 pooling_task="classify",

1203 + tokenization_kwargs=tokenization_kwargs,

1204 )

1205

1206 return [ClassificationRequestOutput.from_base(item) for item in items]

@@ -1209,6 +1214,7 @@ class LLM:

1214 use_tqdm: bool | Callable[..., tqdm] = True,

1215 pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,

1216 lora_request: list[LoRARequest] | LoRARequest | None = None,

1217 + tokenization_kwargs: dict[str, Any] | None = None,

1218 ) -> list[PoolingRequestOutput]:

1219 """

1220 Generate rewards for each prompt.

@@ -1236,6 +1242,7 @@ class LLM:

1242 pooling_params=pooling_params,

1243 truncate_prompt_tokens=truncate_prompt_tokens,

1244 pooling_task="token_classify",

1245 + tokenization_kwargs=tokenization_kwargs,

1246 )

1247

1248 def _embedding_score(

@@ -1247,6 +1254,7 @@ class LLM:

1254 use_tqdm: bool | Callable[..., tqdm] = True,

1255 pooling_params: PoolingParams | None = None,

1256 lora_request: list[LoRARequest] | LoRARequest | None = None,

1257 + tokenization_kwargs: dict[str, Any] | None = None,

1258 ) -> list[ScoringRequestOutput]:

1259 encoded_output: list[PoolingRequestOutput] = self.encode(

1260 text_1 + text_2,

@@ -1255,6 +1263,7 @@ class LLM:

1263 lora_request=lora_request,

1264 pooling_params=pooling_params,

1265 pooling_task="embed",

1266 + tokenization_kwargs=tokenization_kwargs,

1267 )

1268

1269 encoded_output_1: list[PoolingRequestOutput] = encoded_output[0 : len(text_1)]

@@ -1279,6 +1288,7 @@ class LLM:

1288 use_tqdm: bool | Callable[..., tqdm] = True,

1289 pooling_params: PoolingParams | None = None,

1290 lora_request: list[LoRARequest] | LoRARequest | None = None,

1291 + tokenization_kwargs: dict[str, Any] | None = None,

1292 ) -> list[ScoringRequestOutput]:

1293 model_config = self.model_config

1294

@@ -1294,7 +1304,8 @@ class LLM:

1304 pooling_params.verify("score", model_config)

1305 pooling_params_list = list[PoolingParams]()

1306

1297 - tokenization_kwargs: dict[str, Any] = {}

1307 + local_kwargs = tokenization_kwargs or {}

1308 + tokenization_kwargs = local_kwargs.copy()

1309

1310 _validate_truncation_size(

1311 model_config.max_model_len, truncate_prompt_tokens, tokenization_kwargs

@@ -1557,6 +1568,7 @@ class LLM:

1568 use_tqdm: bool | Callable[..., tqdm] = True,

1569 lora_request: Sequence[LoRARequest] | LoRARequest | None,

1570 priority: list[int] | None = None,

1571 + tokenization_kwargs: dict[str, Any] | None = None,

1572 ) -> None:

1573 if isinstance(prompts, (str, dict)):

1574 # Convert a single prompt to a list.

@@ -1602,6 +1614,7 @@ class LLM:

1614 if isinstance(lora_request, Sequence)

1615 else lora_request,

1616 priority=priority[i] if priority else 0,

1617 + tokenization_kwargs=tokenization_kwargs,

1618 )

1619 added_request_ids.append(request_id)

1620 except Exception as e:

@@ -1665,9 +1678,12 @@ class LLM:

1678 *,

1679 lora_request: LoRARequest | None,

1680 priority: int,

1681 + tokenization_kwargs: dict[str, Any] | None = None,

1682 ) -> tuple[EngineCoreRequest, dict[str, Any]]:

1683 """Use the Processor to process inputs for LLMEngine."""

1670 - tokenization_kwargs: dict[str, Any] = {}

1684 +

1685 + local_kwargs = tokenization_kwargs or {}

1686 + tokenization_kwargs = local_kwargs.copy()

1687 _validate_truncation_size(

1688 self.model_config.max_model_len,

1689 params.truncate_prompt_tokens,

@@ -1690,6 +1706,7 @@ class LLM:

1706 params: SamplingParams | PoolingParams,

1707 lora_request: LoRARequest | None = None,

1708 priority: int = 0,

1709 + tokenization_kwargs: dict[str, Any] | None = None,

1710 ) -> str:

1711 prompt_text, _, _ = get_prompt_components(prompt)

1712 request_id = str(next(self.request_counter))

@@ -1700,6 +1717,7 @@ class LLM:

1717 params,

1718 lora_request=lora_request,

1719 priority=priority,

1720 + tokenization_kwargs=tokenization_kwargs,

1721 )

1722

1723 self.llm_engine.add_request(