@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5b8077b8ac

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Fix wrong truncate_prompt_tokens type hint (#22761)

Signed-off-by: Gabriel Marinho <gmarinho@ibm.com> Signed-off-by: Gabriel Marinho <104592062+gmarinho2@users.noreply.github.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Max de Bayser <mbayser@br.ibm.com>

Gabriel Marinho committed Aug 30, 2025 at 17:39 UTC 5b8077b8ac42625a3465ad1f885e409d33e0e42e

14 files changed +101 -102

vllm/entrypoints/llm.py

+18 -23

@@ -51,7 +51,7 @@ from vllm.tasks import PoolingTask

51 from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,

52 get_cached_tokenizer)

53 from vllm.usage.usage_lib import UsageContext

54 -from vllm.utils import Counter, Device, is_list_of

54 +from vllm.utils import Counter, Device, as_iter, is_list_of

55 from vllm.v1.sample.logits_processor import LogitsProcessor

57 if TYPE_CHECKING:

@@ -364,14 +364,6 @@ class LLM:

364 # Use default sampling params.

365 sampling_params = self.get_default_sampling_params()

366

367 - tokenization_kwargs: dict[str, Any] = {}

368 - truncate_prompt_tokens = None

369 - if isinstance(sampling_params, SamplingParams):

370 - truncate_prompt_tokens = sampling_params.truncate_prompt_tokens

371 -

372 - _validate_truncation_size(model_config.max_model_len,

373 - truncate_prompt_tokens, tokenization_kwargs)

374 -

367 # Add any modality specific loras to the corresponding prompts

368 lora_request = self._get_modality_specific_lora_reqs(

369 prompts, lora_request)

@@ -381,7 +373,6 @@ class LLM:

373 params=sampling_params,

374 use_tqdm=use_tqdm,

375 lora_request=lora_request,

384 - tokenization_kwargs=tokenization_kwargs,

376 priority=priority,

377 )

378

@@ -871,6 +862,8 @@ class LLM:

862 If `False`, no progress bar is created.

863 lora_request: LoRA request to use for generation, if any.

864 pooling_task: Override the pooling task to use.

865 + tokenization_kwargs: overrides tokenization_kwargs set in

866 + pooling_params

867

868 Returns:

869 A list of `PoolingRequestOutput` objects containing the

@@ -916,24 +909,17 @@ class LLM:

909 # Use default pooling params.

910 pooling_params = PoolingParams()

911

919 - if isinstance(pooling_params, PoolingParams):

920 - pooling_params.verify(pooling_task, model_config)

921 - else:

922 - for pooling_param in pooling_params:

923 - pooling_param.verify(pooling_task, model_config)

924 -

925 - if tokenization_kwargs is None:

926 - tokenization_kwargs = dict[str, Any]()

927 - _validate_truncation_size(model_config.max_model_len,

928 - truncate_prompt_tokens,

929 - tokenization_kwargs)

912 + for param in as_iter(pooling_params):

913 + param.verify(pooling_task, model_config)

914 + # for backwards compatibility

915 + if truncate_prompt_tokens is not None:

916 + param.truncate_prompt_tokens = truncate_prompt_tokens

917

918 self._validate_and_add_requests(

919 prompts=prompts,

920 params=pooling_params,

921 use_tqdm=use_tqdm,

922 lora_request=lora_request,

936 - tokenization_kwargs=tokenization_kwargs,

923 )

924

925 outputs = self._run_engine(use_tqdm=use_tqdm)

@@ -1385,7 +1371,6 @@ class LLM:

1371 *,

1372 use_tqdm: Union[bool, Callable[..., tqdm]] = True,

1373 lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],

1388 - tokenization_kwargs: Optional[dict[str, Any]] = None,

1374 priority: Optional[list[int]] = None,

1375 ) -> None:

1376 if isinstance(prompts, (str, dict)):

@@ -1412,7 +1397,17 @@ class LLM:

1397 tqdm_func = use_tqdm if callable(use_tqdm) else tqdm

1398 it = tqdm_func(it, desc="Adding requests")

1399

1400 + model_config = self.llm_engine.model_config

1401 +

1402 for i, prompt in enumerate(it):

1403 +

1404 + param = params[i] if isinstance(params, Sequence) else params

1405 +

1406 + tokenization_kwargs: dict[str, Any] = {}

1407 + _validate_truncation_size(model_config.max_model_len,

1408 + param.truncate_prompt_tokens,

1409 + tokenization_kwargs)

1410 +

1411 self._add_request(

1412 prompt,

1413 params[i] if isinstance(params, Sequence) else params,

vllm/entrypoints/openai/protocol.py

+19 -9

@@ -452,7 +452,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

452 min_tokens: int = 0

453 skip_special_tokens: bool = True

454 spaces_between_special_tokens: bool = True

455 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

455 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None

456 prompt_logprobs: Optional[int] = None

457 allowed_token_ids: Optional[list[int]] = None

458 bad_words: list[str] = Field(default_factory=list)

@@ -995,7 +995,7 @@ class CompletionRequest(OpenAIBaseModel):

995 min_tokens: int = 0

996 skip_special_tokens: bool = True

997 spaces_between_special_tokens: bool = True

998 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

998 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None

999 allowed_token_ids: Optional[list[int]] = None

1000 prompt_logprobs: Optional[int] = None

1001 # --8<-- [end:completion-sampling-params]

@@ -1325,8 +1325,10 @@ class EmbeddingCompletionRequest(OpenAIBaseModel):

1325 # --8<-- [end:embedding-extra-params]

1326

1327 def to_pooling_params(self):

1328 - return PoolingParams(dimensions=self.dimensions,

1329 - normalize=self.normalize)

1328 + return PoolingParams(

1329 + truncate_prompt_tokens=self.truncate_prompt_tokens,

1330 + dimensions=self.dimensions,

1331 + normalize=self.normalize)

1332

1333

1334 class EmbeddingChatRequest(OpenAIBaseModel):

@@ -1393,8 +1395,10 @@ class EmbeddingChatRequest(OpenAIBaseModel):

1395 return data

1396

1397 def to_pooling_params(self):

1396 - return PoolingParams(dimensions=self.dimensions,

1397 - normalize=self.normalize)

1398 + return PoolingParams(

1399 + truncate_prompt_tokens=self.truncate_prompt_tokens,

1400 + dimensions=self.dimensions,

1401 + normalize=self.normalize)

1402

1403

1404 EmbeddingRequest = Union[EmbeddingCompletionRequest, EmbeddingChatRequest]

@@ -1430,7 +1434,9 @@ class ScoreRequest(OpenAIBaseModel):

1434 # --8<-- [end:score-extra-params]

1435

1436 def to_pooling_params(self):

1433 - return PoolingParams(activation=self.activation)

1437 + return PoolingParams(

1438 + truncate_prompt_tokens=self.truncate_prompt_tokens,

1439 + activation=self.activation)

1440

1441

1442 class RerankRequest(OpenAIBaseModel):

@@ -1460,7 +1466,9 @@ class RerankRequest(OpenAIBaseModel):

1466 # --8<-- [end:rerank-extra-params]

1467

1468 def to_pooling_params(self):

1463 - return PoolingParams(activation=self.activation)

1469 + return PoolingParams(

1470 + truncate_prompt_tokens=self.truncate_prompt_tokens,

1471 + activation=self.activation)

1472

1473

1474 class RerankDocument(BaseModel):

@@ -1618,7 +1626,9 @@ class ClassificationRequest(OpenAIBaseModel):

1626 # --8<-- [end:classification-extra-params]

1627

1628 def to_pooling_params(self):

1621 - return PoolingParams(activation=self.activation)

1629 + return PoolingParams(

1630 + truncate_prompt_tokens=self.truncate_prompt_tokens,

1631 + activation=self.activation)

1632

1633

1634 class ClassificationData(OpenAIBaseModel):

vllm/entrypoints/openai/serving_chat.py

-1

@@ -237,7 +237,6 @@ class OpenAIServingChat(OpenAIServing):

237 documents=request.documents,

238 chat_template_kwargs=request.chat_template_kwargs,

239 tool_parser=tool_parser,

240 - truncate_prompt_tokens=request.truncate_prompt_tokens,

240 add_special_tokens=request.add_special_tokens,

241 )

242 else:

vllm/entrypoints/openai/serving_classification.py

-13

@@ -61,7 +61,6 @@ class ClassificationMixin(OpenAIServing):

61 ctx.request,

62 ctx.tokenizer,

63 ctx.request.input,

64 - truncate_prompt_tokens=ctx.request.truncate_prompt_tokens,

64 )

66 return None

@@ -157,18 +156,6 @@ class ServingClassification(ClassificationMixin):

156

157 return await super().handle(ctx) # type: ignore

158

160 - @override

161 - def _validate_request(

162 - self,

163 - ctx: ClassificationServeContext,

164 - ) -> Optional[ErrorResponse]:

165 - if error := super()._validate_request(ctx):

166 - return error

167 -

168 - ctx.truncate_prompt_tokens = ctx.request.truncate_prompt_tokens

169 -

170 - return None

171 -

159 @override

160 def _create_pooling_params(

161 self,

vllm/entrypoints/openai/serving_completion.py

-1

@@ -137,7 +137,6 @@ class OpenAIServingCompletion(OpenAIServing):

137 request,

138 tokenizer,

139 request.prompt,

140 - truncate_prompt_tokens=request.truncate_prompt_tokens,

140 add_special_tokens=request.add_special_tokens,

141 )

142 except ValueError as e:

vllm/entrypoints/openai/serving_embedding.py

-14

@@ -97,7 +97,6 @@ class EmbeddingMixin(OpenAIServing):

97 # so there is no need to append extra tokens to the input

98 add_generation_prompt=False,

99 continue_final_message=False,

100 - truncate_prompt_tokens=ctx.truncate_prompt_tokens,

100 add_special_tokens=ctx.request.add_special_tokens,

101 )

102 else:

@@ -106,7 +105,6 @@ class EmbeddingMixin(OpenAIServing):

105 ctx.request,

106 tokenizer,

107 ctx.request.input,

109 - truncate_prompt_tokens=ctx.truncate_prompt_tokens,

108 add_special_tokens=ctx.request.add_special_tokens,

109 )

110 return None

@@ -631,18 +629,6 @@ class OpenAIServingEmbedding(EmbeddingMixin):

629

630 return await super().handle(ctx) # type: ignore

631

634 - @override

635 - def _validate_request(

636 - self,

637 - ctx: ServeContext[EmbeddingRequest],

638 - ) -> Optional[ErrorResponse]:

639 - if error := super()._validate_request(ctx):

640 - return error

641 -

642 - ctx.truncate_prompt_tokens = ctx.request.truncate_prompt_tokens

643 -

644 - return None

645 -

632 @override

633 def _create_pooling_params(

634 self,

vllm/entrypoints/openai/serving_engine.py

+19 -28

@@ -165,7 +165,6 @@ class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, BaseModel,

165

166 # Shared across most requests

167 tokenizer: Optional[AnyTokenizer] = None

168 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

168

169 # `protected_namespaces` resolves Pydantic v2's warning

170 # on conflict with protected namespace "model_"

@@ -297,14 +296,12 @@ class OpenAIServing:

296 truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens",

297 None)

298

300 - if truncate_prompt_tokens is not None:

301 - if truncate_prompt_tokens <= self.max_model_len:

302 - ctx.truncate_prompt_tokens = truncate_prompt_tokens

303 - else:

304 - return self.create_error_response(

305 - "truncate_prompt_tokens value is "

306 - "greater than max_model_len."

307 - " Please, select a smaller truncation size.")

299 + if truncate_prompt_tokens is not None and \

300 + truncate_prompt_tokens > self.max_model_len:

301 + return self.create_error_response(

302 + "truncate_prompt_tokens value is "

303 + "greater than max_model_len."

304 + " Please, select a smaller truncation size.")

305 return None

306

307 def _create_pooling_params(

@@ -528,7 +525,6 @@ class OpenAIServing:

525 request: AnyRequest,

526 prompt: str,

527 tokenizer: AnyTokenizer,

531 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]],

528 add_special_tokens: bool,

529 ) -> TextTokensPrompt:

530 async_tokenizer = self._get_async_tokenizer(tokenizer)

@@ -538,6 +534,9 @@ class OpenAIServing:

534 "do_lower_case", False)):

535 prompt = prompt.lower()

536

537 + truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens",

538 + None)

539 +

540 if truncate_prompt_tokens is None:

541 encoded = await async_tokenizer(

542 prompt, add_special_tokens=add_special_tokens)

@@ -565,8 +564,10 @@ class OpenAIServing:

564 request: AnyRequest,

565 prompt_ids: list[int],

566 tokenizer: Optional[AnyTokenizer],

568 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]],

567 ) -> TextTokensPrompt:

568 + truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens",

569 + None)

570 +

571 if truncate_prompt_tokens is None:

572 input_ids = prompt_ids

573 elif truncate_prompt_tokens < 0:

@@ -652,7 +653,6 @@ class OpenAIServing:

653 request: AnyRequest,

654 tokenizer: AnyTokenizer,

655 prompt_input: Union[str, list[int]],

655 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,

656 add_special_tokens: bool = True,

657 ) -> TextTokensPrompt:

658 """

@@ -664,7 +664,6 @@ class OpenAIServing:

664 request,

665 tokenizer,

666 [prompt_input],

667 - truncate_prompt_tokens=truncate_prompt_tokens,

667 add_special_tokens=add_special_tokens,

668 ):

669 return result

@@ -675,7 +674,6 @@ class OpenAIServing:

674 request: AnyRequest,

675 tokenizer: AnyTokenizer,

676 prompt_inputs: Iterable[Union[str, list[int]]],

678 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,

677 add_special_tokens: bool = True,

678 ) -> AsyncGenerator[TextTokensPrompt, None]:

679 """

@@ -689,7 +687,6 @@ class OpenAIServing:

687 request,

688 prompt=prompt,

689 tokenizer=tokenizer,

692 - truncate_prompt_tokens=truncate_prompt_tokens,

690 add_special_tokens=add_special_tokens,

691 )

692 else:

@@ -697,7 +694,6 @@ class OpenAIServing:

694 request,

695 prompt_ids=prompt,

696 tokenizer=tokenizer,

700 - truncate_prompt_tokens=truncate_prompt_tokens,

697 )

698

699 async def _tokenize_prompt_input_or_inputs_async(

@@ -706,7 +702,6 @@ class OpenAIServing:

702 tokenizer: Optional[AnyTokenizer],

703 input_or_inputs: Optional[Union[str, list[str], list[int],

704 list[list[int]]]],

709 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,

705 add_special_tokens: bool = True,

706 ) -> tuple[list[TextTokensPrompt], list[EmbedsPrompt]]:

707 """

@@ -719,6 +714,12 @@ class OpenAIServing:

714 inputs_embeds = list[EmbedsPrompt]()

715 inputs_text = list[TextTokensPrompt]()

716

717 + truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens",

718 + None)

719 +

720 + if (truncate_prompt_tokens or 0) < 0:

721 + truncate_prompt_tokens = self.max_model_len

722 +

723 if (isinstance(request, CompletionRequest)

724 and request.prompt_embeds is not None):

725 inputs_embeds.extend(

@@ -748,14 +749,10 @@ class OpenAIServing:

749 request,

750 prompt_input["content"],

751 tokenizer=tokenizer,

751 - truncate_prompt_tokens=truncate_prompt_tokens,

752 add_special_tokens=add_special_tokens)

753 else:

754 task = self._normalize_prompt_tokens_to_input(

755 - request,

756 - prompt_input["content"],

757 - tokenizer=tokenizer,

758 - truncate_prompt_tokens=truncate_prompt_tokens)

755 + request, prompt_input["content"], tokenizer=tokenizer)

756 tasks.append(task)

757

758 # Wait for all tokenization tasks to complete

@@ -772,7 +769,6 @@ class OpenAIServing:

769 TokenizeCompletionRequest],

770 tokenizer: Optional[AnyTokenizer],

771 input_or_inputs: Union[str, list[str], list[int], list[list[int]]],

775 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = ...,

772 add_special_tokens: bool = ...,

773 ) -> tuple[list[TextTokensPrompt], list[EngineTokensPrompt]]:

774 ...

@@ -784,7 +780,6 @@ class OpenAIServing:

780 tokenizer: Optional[AnyTokenizer],

781 input_or_inputs: Optional[Union[str, list[str], list[int],

782 list[list[int]]]],

787 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = ...,

783 add_special_tokens: bool = ...,

784 ) -> tuple[list[Union[TextTokensPrompt, EmbedsPrompt]], list[Union[

785 EngineTokensPrompt, EngineEmbedsPrompt]]]:

@@ -796,7 +791,6 @@ class OpenAIServing:

791 tokenizer: Optional[AnyTokenizer],

792 input_or_inputs: Optional[Union[str, list[str], list[int],

793 list[list[int]]]],

799 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None,

794 add_special_tokens: bool = True,

795 ) -> tuple[Union[list[TextTokensPrompt], list[Union[

796 TextTokensPrompt, EmbedsPrompt]]], Union[

@@ -813,7 +807,6 @@ class OpenAIServing:

807 request,

808 tokenizer,

809 input_or_inputs,

816 - truncate_prompt_tokens=truncate_prompt_tokens,

810 add_special_tokens=add_special_tokens,

811 )

812

@@ -866,7 +859,6 @@ class OpenAIServing:

859 documents: Optional[list[dict[str, str]]] = None,

860 chat_template_kwargs: Optional[dict[str, Any]] = None,

861 tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None,

869 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,

862 add_special_tokens: bool = False,

863 ) -> tuple[list[ConversationMessage], Sequence[RequestPrompt],

864 list[EngineTokensPrompt]]:

@@ -941,7 +933,6 @@ class OpenAIServing:

933 request,

934 tokenizer,

935 request_prompt,

944 - truncate_prompt_tokens=truncate_prompt_tokens,

936 add_special_tokens=add_special_tokens,

937 )

938 else:

vllm/entrypoints/openai/serving_pooling.py

-2

@@ -120,7 +120,6 @@ class OpenAIServingPooling(OpenAIServing):

120 # so there is no need to append extra tokens to the input

121 add_generation_prompt=False,

122 continue_final_message=False,

123 - truncate_prompt_tokens=truncate_prompt_tokens,

123 add_special_tokens=request.add_special_tokens,

124 )

125 else:

@@ -129,7 +128,6 @@ class OpenAIServingPooling(OpenAIServing):

128 request,

129 tokenizer,

130 request.input,

132 - truncate_prompt_tokens=truncate_prompt_tokens,

131 add_special_tokens=request.add_special_tokens,

132 )

133 except (ValueError, TypeError, jinja2.TemplateError) as e:

vllm/entrypoints/openai/serving_score.py

+3 -3

@@ -266,12 +266,14 @@ class ServingScores(OpenAIServing):

266 request: Union[ScoreRequest, RerankRequest],

267 request_id: str,

268 raw_request: Optional[Request] = None,

269 - truncate_prompt_tokens: Optional[int] = None,

269 ) -> Union[list[PoolingRequestOutput], ErrorResponse]:

270 lora_request = self._maybe_get_adapters(request)

271

272 tokenizer = await self.engine_client.get_tokenizer(lora_request)

273

274 + truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens",

275 + None)

276 +

277 tokenization_kwargs: dict[str, Any] = {}

278 _validate_truncation_size(self.max_model_len, truncate_prompt_tokens,

279 tokenization_kwargs)

@@ -343,7 +345,6 @@ class ServingScores(OpenAIServing):

345 request,

346 request_id,

347 raw_request,

346 - request.truncate_prompt_tokens,

348 )

349 if isinstance(final_res_batch, ErrorResponse):

350 return final_res_batch

@@ -391,7 +392,6 @@ class ServingScores(OpenAIServing):

392 request,

393 request_id,

394 raw_request,

394 - request.truncate_prompt_tokens,

395 )

396 if isinstance(final_res_batch, ErrorResponse):

397 return final_res_batch

vllm/inputs/preprocess.py

+20 -2

@@ -346,6 +346,22 @@ class InputPreprocessor:

346 ) -> EmbedsInputs:

347 return self._process_embeds(parsed_content)

348

349 + def _truncate_inputs(

350 + self,

351 + inputs: list[int],

352 + tokenization_kwargs: Optional[dict[str, Any]] = None) -> list[int]:

353 +

354 + if not tokenization_kwargs or "truncation" not in \

355 + tokenization_kwargs or self.tokenizer is None:

356 + return inputs

357 +

358 + max_length = tokenization_kwargs["max_length"]

359 +

360 + if self.tokenizer.truncation_side == "left":

361 + return inputs[-max_length:]

362 + else:

363 + return inputs[:max_length]

364 +

365 def _process_tokens(

366 self,

367 parsed_content: TokensPrompt,

@@ -354,7 +370,8 @@ class InputPreprocessor:

370 *,

371 mm_hash_overrides: Optional[dict[str, list[str]]] = None,

372 ) -> Union[TokenInputs, MultiModalInputs]:

357 - prompt_token_ids = parsed_content["prompt_token_ids"]

373 + prompt_token_ids = self._truncate_inputs(

374 + parsed_content["prompt_token_ids"], tokenization_kwargs)

375

376 inputs: Union[TokenInputs, MultiModalInputs]

377 if multi_modal_data := parsed_content.get("multi_modal_data"):

@@ -382,7 +399,8 @@ class InputPreprocessor:

399 *,

400 mm_hash_overrides: Optional[dict[str, list[str]]] = None,

401 ) -> Union[TokenInputs, MultiModalInputs]:

385 - prompt_token_ids = parsed_content["prompt_token_ids"]

402 + prompt_token_ids = self._truncate_inputs(

403 + parsed_content["prompt_token_ids"], tokenization_kwargs)

404

405 inputs: Union[TokenInputs, MultiModalInputs]

406 if multi_modal_data := parsed_content.get("multi_modal_data"):

vllm/pooling_params.py

+6 -1

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 from copy import deepcopy

5 -from typing import TYPE_CHECKING, Any, Optional

5 +from typing import TYPE_CHECKING, Annotated, Any, Optional

7 import msgspec

@@ -27,6 +27,11 @@ class PoolingParams(

27 the classification outputs.

28 softmax: Whether to apply softmax to the reward outputs.

29 """

30 + truncate_prompt_tokens: Optional[Annotated[int,

31 + msgspec.Meta(ge=-1)]] = None

32 + """If set to -1, will use the truncation size supported by the model. If

33 + set to an integer k, will use only the last k tokens from the prompt

34 + (i.e., left truncation). If set to `None`, truncation is disabled."""

36 ## for embeddings models

37 dimensions: Optional[int] = None

vllm/sampling_params.py

+9 -5

@@ -182,7 +182,8 @@ class SamplingParams(

182 optionally prompt tokens as a first argument."""

183 include_stop_str_in_output: bool = False

184 """Whether to include the stop strings in output text."""

185 - truncate_prompt_tokens: Optional[Annotated[int, msgspec.Meta(ge=1)]] = None

185 + truncate_prompt_tokens: Optional[Annotated[int,

186 + msgspec.Meta(ge=-1)]] = None

187 """If set to -1, will use the truncation size supported by the model. If

188 set to an integer k, will use only the last k tokens from the prompt

189 (i.e., left truncation). If set to `None`, truncation is disabled."""

@@ -241,7 +242,8 @@ class SamplingParams(

242 spaces_between_special_tokens: bool = True,

243 logits_processors: Optional[list[LogitsProcessor]] = None,

244 truncate_prompt_tokens: Optional[Annotated[int,

244 - msgspec.Meta(ge=1)]] = None,

245 + msgspec.Meta(

246 + ge=-1)]] = None,

247 output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,

248 guided_decoding: Optional[GuidedDecodingParams] = None,

249 logit_bias: Optional[Union[dict[int, float], dict[str, float]]] = None,

@@ -411,9 +413,11 @@ class SamplingParams(

413 raise ValueError(f"prompt_logprobs must be non-negative, got "

414 f"{self.prompt_logprobs}.")

415 if (self.truncate_prompt_tokens is not None

414 - and self.truncate_prompt_tokens < 1):

415 - raise ValueError(f"truncate_prompt_tokens must be >= 1, "

416 - f"got {self.truncate_prompt_tokens}")

416 + and (self.truncate_prompt_tokens == 0

417 + or self.truncate_prompt_tokens < -1)):

418 + raise ValueError(

419 + f"truncate_prompt_tokens must be an integer >= 1 or -1, "

420 + f"got {self.truncate_prompt_tokens}")

421 assert isinstance(self.stop_token_ids, list)

422 if not all(isinstance(st_id, int) for st_id in self.stop_token_ids):

423 raise ValueError(f"stop_token_ids must contain only integers, "

vllm/transformers_utils/tokenizer_group.py

@@ -23,6 +23,7 @@ class TokenizerGroup:

23 self.tokenizer_config = tokenizer_config

24 self.enable_lora = enable_lora

25 self.max_input_length = max_input_length

26 + self.truncation_side = tokenizer_config.get("truncation_side", "left")

27 self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)

28 max_loras = tokenizer_config.get("max_loras", 0)

29 self.lora_tokenizers = LRUCache[int, AnyTokenizer](

vllm/utils/__init__.py

@@ -1328,6 +1328,12 @@ def as_list(maybe_list: Iterable[T]) -> list[T]:

1328 return maybe_list if isinstance(maybe_list, list) else list(maybe_list)

1329

1330

1331 +def as_iter(obj: Union[T, Iterable[T]]) -> Iterable[T]:

1332 + if isinstance(obj, str) or not isinstance(obj, Iterable):

1333 + obj = [obj]

1334 + return obj

1335 +

1336 +

1337 # `collections` helpers

1338 def is_list_of(

1339 value: object,