A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 20:45 UTC
[Frontend] Split the offline inference APIs and utils. (#43553)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
wang.yuqi committed
May 26, 2026 at 13:20 UTC
d5cf7b4a2c04b7f3661907ecb610ccc977058c9f
5 files changed
+642
-713
tests/entrypoints/llm/test_mm_processor_kwargs.py
+2
-2
@@ -192,7 +192,7 @@ def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer(
192
llm.renderer = renderer
193
194
monkeypatch.setattr(
195
- "vllm.entrypoints.llm.parse_model_prompt",
195
+ "vllm.entrypoints.offline_utils.parse_model_prompt",
196
lambda _model_config, parsed_prompt: parsed_prompt,
197
)
198
@@ -226,7 +226,7 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override(
226
llm.renderer = renderer
227
228
monkeypatch.setattr(
229
- "vllm.entrypoints.llm.parse_model_prompt",
229
+ "vllm.entrypoints.offline_utils.parse_model_prompt",
230
lambda _model_config, parsed_prompt: parsed_prompt,
231
)
232
vllm/entrypoints/generate/beam_search/offline.py
+3
-47
@@ -2,17 +2,13 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import itertools
5
-from abc import ABC, abstractmethod
6
-from collections.abc import Callable, Iterable, Sequence
7
-from typing import Any
5
6
from tqdm import tqdm
7
11
-from vllm import PromptType, RequestOutput, TextPrompt, TokensPrompt
12
-from vllm.inputs import EngineInput
8
+from vllm import RequestOutput, TextPrompt, TokensPrompt
9
+from vllm.entrypoints.offline_utils import OfflineInferenceMixin
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
15
-from vllm.renderers import BaseRenderer
12
from vllm.sampling_params import BeamSearchParams, SamplingParams
13
14
from .utils import (
@@ -25,11 +21,9 @@ from .utils import (
21
logger = init_logger(__name__)
22
23
28
-class BeamSearchOfflineMixin(ABC):
24
+class BeamSearchOfflineMixin(OfflineInferenceMixin):
25
"""Offline inference for beam search"""
26
31
- renderer: BaseRenderer
32
-
27
def beam_search(
28
self,
29
prompts: list[TokensPrompt | TextPrompt],
@@ -186,41 +180,3 @@ class BeamSearchOfflineMixin(ABC):
180
outputs.append(BeamSearchOutput(sequences=best_beams))
181
182
return outputs
189
-
190
- @abstractmethod
191
- def _preprocess_cmpl(
192
- self,
193
- prompts: Sequence[PromptType],
194
- tokenization_kwargs: dict[str, Any] | None = None,
195
- mm_processor_kwargs: dict[str, Any] | None = None,
196
- ) -> Sequence[EngineInput]:
197
- raise NotImplementedError
198
-
199
- @abstractmethod
200
- def _lora_request_to_seq(
201
- self,
202
- lora_request: LoRARequest | None | Sequence[LoRARequest | None],
203
- num_requests: int,
204
- ) -> Sequence[LoRARequest | None]:
205
- raise NotImplementedError
206
-
207
- @abstractmethod
208
- def _params_to_seq(
209
- self,
210
- params: SamplingParams,
211
- num_requests: int,
212
- ) -> Sequence[SamplingParams]:
213
- raise NotImplementedError
214
-
215
- @abstractmethod
216
- def _render_and_run_requests(
217
- self,
218
- prompts: Iterable[EngineInput],
219
- params: Sequence[SamplingParams],
220
- output_type: type[RequestOutput],
221
- *,
222
- lora_requests: Sequence[LoRARequest | None] | None = None,
223
- priorities: Sequence[int] | None = None,
224
- use_tqdm: bool | Callable[..., tqdm] = True,
225
- ):
226
- raise NotImplementedError
vllm/entrypoints/llm.py
+7
-596
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from collections.abc import Callable, Iterable, Sequence
4
+from collections.abc import Callable, Sequence
5
from pathlib import Path
6
from typing import TYPE_CHECKING, Any
7
@@ -9,7 +9,7 @@ import cloudpickle
9
import torch.nn as nn
10
from pydantic import ValidationError
11
from tqdm.auto import tqdm
12
-from typing_extensions import TypeVar, overload
12
+from typing_extensions import overload
13
14
from vllm.config import (
15
AttentionConfig,
@@ -41,47 +41,29 @@ from vllm.entrypoints.chat_utils import (
41
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
42
from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
43
from vllm.entrypoints.utils import log_non_default_args
44
-from vllm.inputs import (
45
- EngineInput,
46
- PromptType,
47
-)
44
+from vllm.inputs import PromptType
45
from vllm.logger import init_logger
46
from vllm.lora.request import LoRARequest
47
from vllm.model_executor.layers.quantization import QuantizationMethods
48
from vllm.outputs import PoolingRequestOutput, RequestOutput
49
from vllm.platforms import current_platform
53
-from vllm.pooling_params import PoolingParams
54
-from vllm.renderers import ChatParams, merge_kwargs
55
-from vllm.renderers.inputs.preprocess import (
56
- conversation_to_seq,
57
- parse_model_prompt,
58
- prompt_to_seq,
59
-)
60
-from vllm.sampling_params import RequestOutputKind, SamplingParams
50
+from vllm.sampling_params import SamplingParams
51
from vllm.tokenizers import TokenizerLike
52
from vllm.usage.usage_lib import UsageContext
53
from vllm.utils.counter import Counter
64
-from vllm.utils.mistral import is_mistral_tokenizer
65
-from vllm.utils.tqdm_utils import maybe_tqdm
54
from vllm.v1.engine import PauseMode
55
from vllm.v1.engine.llm_engine import LLMEngine
56
from vllm.v1.sample.logits_processor import LogitsProcessor
57
58
+from .offline_utils import _O, _R, OfflineInferenceMixin
59
+
60
if TYPE_CHECKING:
61
from vllm.v1.metrics.reader import Metric
62
63
logger = init_logger(__name__)
64
75
-_O = TypeVar(
76
- "_O",
77
- bound=RequestOutput | PoolingRequestOutput,
78
- default=RequestOutput | PoolingRequestOutput,
79
-)
80
-_P = TypeVar("_P", bound=SamplingParams | PoolingParams | None)
81
-_R = TypeVar("_R", default=Any)
65
83
-
84
-class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin):
66
+class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
67
"""An LLM for generating texts from given prompts and sampling parameters.
68
69
This class includes a tokenizer, a language model (possibly distributed
@@ -585,58 +567,6 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin):
567
568
return self._run_engine(output_type, use_tqdm=use_tqdm)
569
588
- def _resolve_mm_lora(
589
- self,
590
- prompt: EngineInput,
591
- lora_request: LoRARequest | None,
592
- ) -> LoRARequest | None:
593
- if prompt["type"] != "multimodal":
594
- return lora_request
595
-
596
- lora_config = self.llm_engine.vllm_config.lora_config
597
- default_mm_loras = None if lora_config is None else lora_config.default_mm_loras
598
- if not default_mm_loras:
599
- return lora_request
600
-
601
- prompt_modalities = prompt["mm_placeholders"].keys()
602
- intersection = set(prompt_modalities).intersection(default_mm_loras.keys())
603
- if not intersection:
604
- return lora_request
605
-
606
- if len(intersection) > 1:
607
- # TODO: Would be nice to be able to have multiple loras per prompt
608
- logger.warning(
609
- "Multiple modality specific loras were registered and would be "
610
- "used by a single prompt consuming several modalities; "
611
- "currently we only support one lora per request; as such, "
612
- "lora(s) registered with modalities: %s will be skipped",
613
- intersection,
614
- )
615
- return lora_request
616
-
617
- # Build the LoRA request; the ID of the default mm lora is the
618
- # index of the modality name sorted alphabetically + 1.
619
- modality_name = intersection.pop()
620
- modality_lora_path = default_mm_loras[modality_name]
621
- modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1
622
-
623
- # If we have a collision, warn if there is a collision,
624
- # but always send the explicitly provided request.
625
- if lora_request:
626
- if lora_request.lora_int_id != modality_lora_id:
627
- logger.warning(
628
- "A modality with a registered lora and a lora_request "
629
- "with a different ID were provided; falling back to the "
630
- "lora_request as we only apply one LoRARequest per prompt"
631
- )
632
- return lora_request
633
-
634
- return LoRARequest(
635
- modality_name,
636
- modality_lora_id,
637
- modality_lora_path,
638
- )
639
-
570
def collective_rpc(
571
self,
572
method: str | Callable[..., _R],
@@ -682,139 +612,6 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin):
612
"""
613
return self.llm_engine.apply_model(func)
614
685
- def _preprocess_cmpl(
686
- self,
687
- prompts: Sequence[PromptType],
688
- tokenization_kwargs: dict[str, Any] | None = None,
689
- mm_processor_kwargs: dict[str, Any] | None = None,
690
- ) -> Sequence[EngineInput]:
691
- """
692
- Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
693
- a format that can be passed to `_add_request`.
694
-
695
- Refer to [LLM.generate][] for a complete description of the arguments.
696
-
697
- Returns:
698
- A list of `EngineInput` objects ready to be passed into LLMEngine.
699
- """
700
- renderer = self.renderer
701
- model_config = self.model_config
702
-
703
- parsed_prompts = [
704
- parse_model_prompt(model_config, prompt) for prompt in prompts
705
- ]
706
- tok_params = renderer.default_cmpl_tok_params.with_kwargs(
707
- **(tokenization_kwargs or {})
708
- )
709
- prompt_extras = (
710
- None
711
- if mm_processor_kwargs is None
712
- else {"mm_processor_kwargs": mm_processor_kwargs}
713
- )
714
-
715
- return renderer.render_cmpl(
716
- parsed_prompts,
717
- tok_params,
718
- prompt_extras=prompt_extras,
719
- )
720
-
721
- def _preprocess_cmpl_one(
722
- self,
723
- prompt: PromptType,
724
- tokenization_kwargs: dict[str, Any] | None = None,
725
- mm_processor_kwargs: dict[str, Any] | None = None,
726
- ) -> EngineInput:
727
- (engine_input,) = self._preprocess_cmpl(
728
- [prompt],
729
- tokenization_kwargs,
730
- mm_processor_kwargs=mm_processor_kwargs,
731
- )
732
- return engine_input
733
-
734
- def _preprocess_chat(
735
- self,
736
- conversations: Sequence[list[ChatCompletionMessageParam]],
737
- chat_template: str | None = None,
738
- chat_template_content_format: ChatTemplateContentFormatOption = "auto",
739
- chat_template_kwargs: dict[str, Any] | None = None,
740
- add_generation_prompt: bool = True,
741
- continue_final_message: bool = False,
742
- tools: list[dict[str, Any]] | None = None,
743
- tokenization_kwargs: dict[str, Any] | None = None,
744
- mm_processor_kwargs: dict[str, Any] | None = None,
745
- ) -> Sequence[EngineInput]:
746
- """
747
- Convert a list of conversations into prompts so that they can then
748
- be used as input for other LLM APIs.
749
-
750
- Refer to [LLM.chat][] for a complete description of the arguments.
751
-
752
- Returns:
753
- A list of `EngineInput` objects ready to be passed into LLMEngine.
754
- """
755
- renderer = self.renderer
756
-
757
- chat_params = ChatParams(
758
- chat_template=chat_template,
759
- chat_template_content_format=chat_template_content_format,
760
- chat_template_kwargs=merge_kwargs(
761
- chat_template_kwargs,
762
- dict(
763
- add_generation_prompt=add_generation_prompt,
764
- continue_final_message=continue_final_message,
765
- tools=tools,
766
- tokenize=(
767
- is_mistral_tokenizer(renderer.tokenizer)
768
- or self.model_config.enable_prompt_embeds
769
- ),
770
- ),
771
- ),
772
- mm_processor_kwargs=mm_processor_kwargs,
773
- )
774
- tok_params = renderer.default_chat_tok_params.with_kwargs(
775
- **(tokenization_kwargs or {})
776
- )
777
- prompt_extras = (
778
- None
779
- if mm_processor_kwargs is None
780
- else {"mm_processor_kwargs": mm_processor_kwargs}
781
- )
782
-
783
- _, engine_inputs = renderer.render_chat(
784
- conversations,
785
- chat_params,
786
- tok_params,
787
- prompt_extras=prompt_extras,
788
- )
789
-
790
- return engine_inputs
791
-
792
- def _preprocess_chat_one(
793
- self,
794
- conversation: list[ChatCompletionMessageParam],
795
- chat_template: str | None = None,
796
- chat_template_content_format: ChatTemplateContentFormatOption = "auto",
797
- chat_template_kwargs: dict[str, Any] | None = None,
798
- add_generation_prompt: bool = True,
799
- continue_final_message: bool = False,
800
- tools: list[dict[str, Any]] | None = None,
801
- tokenization_kwargs: dict[str, Any] | None = None,
802
- mm_processor_kwargs: dict[str, Any] | None = None,
803
- ) -> EngineInput:
804
- (engine_input,) = self._preprocess_chat(
805
- [conversation],
806
- chat_template=chat_template,
807
- chat_template_content_format=chat_template_content_format,
808
- chat_template_kwargs=chat_template_kwargs,
809
- add_generation_prompt=add_generation_prompt,
810
- continue_final_message=continue_final_message,
811
- tools=tools,
812
- tokenization_kwargs=tokenization_kwargs,
813
- mm_processor_kwargs=mm_processor_kwargs,
814
- )
815
-
816
- return engine_input
817
-
615
def chat(
616
self,
617
messages: list[ChatCompletionMessageParam]
@@ -1058,392 +855,6 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin):
855
"""
856
return self.llm_engine.get_metrics()
857
1061
- def _params_to_seq(
1062
- self,
1063
- params: _P | Sequence[_P],
1064
- num_requests: int,
1065
- ) -> Sequence[_P]:
1066
- if isinstance(params, Sequence):
1067
- if len(params) != num_requests:
1068
- raise ValueError(
1069
- f"The lengths of prompts ({num_requests}) "
1070
- f"and params ({len(params)}) must be the same."
1071
- )
1072
-
1073
- return params
1074
-
1075
- return [params] * num_requests
1076
-
1077
- def _lora_request_to_seq(
1078
- self,
1079
- lora_request: LoRARequest | None | Sequence[LoRARequest | None],
1080
- num_requests: int,
1081
- ) -> Sequence[LoRARequest | None]:
1082
- if isinstance(lora_request, Sequence):
1083
- if len(lora_request) != num_requests:
1084
- raise ValueError(
1085
- f"The lengths of prompts ({num_requests}) "
1086
- f"and lora_request ({len(lora_request)}) must be the same."
1087
- )
1088
-
1089
- return lora_request
1090
-
1091
- return [lora_request] * num_requests
1092
-
1093
- def _priority_to_seq(
1094
- self,
1095
- priority: list[int] | None,
1096
- num_requests: int,
1097
- ) -> Sequence[int]:
1098
- if priority is not None:
1099
- if len(priority) != num_requests:
1100
- raise ValueError(
1101
- f"The lengths of prompts ({num_requests}) "
1102
- f"and priority ({len(priority)}) must be the same."
1103
- )
1104
-
1105
- return priority
1106
-
1107
- return [0] * num_requests
1108
-
1109
- def _add_completion_requests(
1110
- self,
1111
- prompts: PromptType | Sequence[PromptType],
1112
- params: SamplingParams
1113
- | PoolingParams
1114
- | Sequence[SamplingParams | PoolingParams],
1115
- *,
1116
- use_tqdm: bool | Callable[..., tqdm] = True,
1117
- lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
1118
- priority: list[int] | None = None,
1119
- tokenization_kwargs: dict[str, Any] | None = None,
1120
- mm_processor_kwargs: dict[str, Any] | None = None,
1121
- ) -> list[str]:
1122
- seq_prompts = prompt_to_seq(prompts)
1123
- seq_params = self._params_to_seq(params, len(seq_prompts))
1124
- seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_prompts))
1125
- seq_priority = self._priority_to_seq(priority, len(seq_prompts))
1126
-
1127
- return self._render_and_add_requests(
1128
- prompts=(
1129
- self._preprocess_cmpl_one(
1130
- prompt,
1131
- tokenization_kwargs,
1132
- mm_processor_kwargs=mm_processor_kwargs,
1133
- )
1134
- for prompt in maybe_tqdm(
1135
- seq_prompts,
1136
- use_tqdm=use_tqdm,
1137
- desc="Rendering prompts",
1138
- )
1139
- ),
1140
- params=seq_params,
1141
- lora_requests=seq_lora_requests,
1142
- priorities=seq_priority,
1143
- )
1144
-
1145
- def _run_completion(
1146
- self,
1147
- prompts: PromptType | Sequence[PromptType],
1148
- params: SamplingParams
1149
- | PoolingParams
1150
- | Sequence[SamplingParams | PoolingParams],
1151
- output_type: type[_O],
1152
- *,
1153
- use_tqdm: bool | Callable[..., tqdm] = True,
1154
- lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
1155
- priority: list[int] | None = None,
1156
- tokenization_kwargs: dict[str, Any] | None = None,
1157
- mm_processor_kwargs: dict[str, Any] | None = None,
1158
- ):
1159
- self._add_completion_requests(
1160
- prompts=prompts,
1161
- params=params,
1162
- use_tqdm=use_tqdm,
1163
- lora_request=lora_request,
1164
- priority=priority,
1165
- tokenization_kwargs=tokenization_kwargs,
1166
- mm_processor_kwargs=mm_processor_kwargs,
1167
- )
1168
- return self._run_engine(use_tqdm=use_tqdm, output_type=output_type)
1169
-
1170
- def _run_chat(
1171
- self,
1172
- messages: list[ChatCompletionMessageParam]
1173
- | Sequence[list[ChatCompletionMessageParam]],
1174
- params: SamplingParams
1175
- | PoolingParams
1176
- | Sequence[SamplingParams | PoolingParams],
1177
- output_type: type[_O],
1178
- *,
1179
- use_tqdm: bool | Callable[..., tqdm] = True,
1180
- lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
1181
- chat_template: str | None = None,
1182
- chat_template_content_format: ChatTemplateContentFormatOption = "auto",
1183
- add_generation_prompt: bool = True,
1184
- continue_final_message: bool = False,
1185
- tools: list[dict[str, Any]] | None = None,
1186
- chat_template_kwargs: dict[str, Any] | None = None,
1187
- tokenization_kwargs: dict[str, Any] | None = None,
1188
- mm_processor_kwargs: dict[str, Any] | None = None,
1189
- ):
1190
- self._add_chat_requests(
1191
- messages=messages,
1192
- params=params,
1193
- use_tqdm=use_tqdm,
1194
- lora_request=lora_request,
1195
- chat_template=chat_template,
1196
- chat_template_content_format=chat_template_content_format,
1197
- chat_template_kwargs=chat_template_kwargs,
1198
- add_generation_prompt=add_generation_prompt,
1199
- continue_final_message=continue_final_message,
1200
- tools=tools,
1201
- tokenization_kwargs=tokenization_kwargs,
1202
- mm_processor_kwargs=mm_processor_kwargs,
1203
- )
1204
- return self._run_engine(output_type=output_type, use_tqdm=use_tqdm)
1205
-
1206
- def _add_chat_requests(
1207
- self,
1208
- messages: list[ChatCompletionMessageParam]
1209
- | Sequence[list[ChatCompletionMessageParam]],
1210
- params: SamplingParams
1211
- | PoolingParams
1212
- | Sequence[SamplingParams | PoolingParams],
1213
- *,
1214
- use_tqdm: bool | Callable[..., tqdm] = True,
1215
- lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
1216
- priority: list[int] | None = None,
1217
- chat_template: str | None = None,
1218
- chat_template_content_format: ChatTemplateContentFormatOption = "auto",
1219
- add_generation_prompt: bool = True,
1220
- continue_final_message: bool = False,
1221
- tools: list[dict[str, Any]] | None = None,
1222
- chat_template_kwargs: dict[str, Any] | None = None,
1223
- tokenization_kwargs: dict[str, Any] | None = None,
1224
- mm_processor_kwargs: dict[str, Any] | None = None,
1225
- ) -> list[str]:
1226
- seq_convs = conversation_to_seq(messages)
1227
- seq_params = self._params_to_seq(params, len(seq_convs))
1228
- seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_convs))
1229
- seq_priority = self._priority_to_seq(priority, len(seq_convs))
1230
-
1231
- # When thinking is enabled or tools are provided, and the model
1232
- # uses special tokens for structured output (e.g. Gemma4's
1233
- # <|channel>, <|tool_call>, <|"|>), automatically set
1234
- # skip_special_tokens=False so these tokens are preserved in
1235
- # output.text for downstream parsing.
1236
- needs_parsing = (
1237
- chat_template_kwargs and chat_template_kwargs.get("enable_thinking")
1238
- ) or tools
1239
- if needs_parsing:
1240
- self._adjust_params_for_parsing(seq_params)
1241
-
1242
- return self._render_and_add_requests(
1243
- prompts=(
1244
- self._preprocess_chat_one(
1245
- conversation,
1246
- chat_template=chat_template,
1247
- chat_template_content_format=chat_template_content_format,
1248
- chat_template_kwargs=chat_template_kwargs,
1249
- add_generation_prompt=add_generation_prompt,
1250
- continue_final_message=continue_final_message,
1251
- tools=tools,
1252
- tokenization_kwargs=tokenization_kwargs,
1253
- mm_processor_kwargs=mm_processor_kwargs,
1254
- )
1255
- for conversation in maybe_tqdm(
1256
- seq_convs,
1257
- use_tqdm=use_tqdm,
1258
- desc="Rendering conversations",
1259
- )
1260
- ),
1261
- params=seq_params,
1262
- lora_requests=seq_lora_requests,
1263
- priorities=seq_priority,
1264
- )
1265
-
1266
- def _adjust_params_for_parsing(
1267
- self, params: Sequence[SamplingParams | PoolingParams]
1268
- ) -> None:
1269
- """Set ``skip_special_tokens=False`` when the model encodes
1270
- structured output syntax as special tokens.
1271
-
1272
- Models like Gemma4 register thinking delimiters
1273
- (``<|channel>``/``<channel|>``) and tool call tokens
1274
- (``<|tool_call>``/``<tool_call|>``/``<|"|>``) as special tokens.
1275
- The default ``skip_special_tokens=True`` strips them from
1276
- ``output.text``, breaking parsing of both reasoning blocks and
1277
- tool calls.
1278
-
1279
- This is a no-op for models whose structured tokens are regular
1280
- text tokens (e.g. DeepSeek's ``<think>``/``</think>``).
1281
- """
1282
- # The offline API currently lacks a unified rendering pipeline.
1283
- # Until the planned Renderer refactor is complete, we hardcode
1284
- # this token preservation logic specifically for Gemma4 models
1285
- # to avoid regressions on other models.
1286
- hf_config = getattr(self.model_config, "hf_config", None)
1287
- architectures = getattr(hf_config, "architectures", [])
1288
-
1289
- if any("Gemma4" in arch for arch in architectures):
1290
- tokenizer = self.renderer.get_tokenizer()
1291
- vocab = tokenizer.get_vocab()
1292
- special_ids = set(getattr(tokenizer, "all_special_ids", []))
1293
-
1294
- # Tokens used for thinking delimiters and tool call syntax
1295
- # that some models (Gemma4) register as special tokens.
1296
- structured_tokens = (
1297
- "<|channel>",
1298
- "<channel|>", # thinking delimiters
1299
- "<|tool_call>",
1300
- "<tool_call|>", # tool call delimiters
1301
- '<|"|>', # string quoting in tool args
1302
- )
1303
- needs_special = any(
1304
- vocab.get(tok) in special_ids
1305
- for tok in structured_tokens
1306
- if tok in vocab
1307
- )
1308
- if needs_special:
1309
- for sp in params:
1310
- if isinstance(sp, SamplingParams) and sp.skip_special_tokens:
1311
- sp.skip_special_tokens = False
1312
-
1313
- def _render_and_run_requests(
1314
- self,
1315
- prompts: Iterable[EngineInput],
1316
- params: Sequence[SamplingParams | PoolingParams],
1317
- output_type: type[_O],
1318
- *,
1319
- lora_requests: Sequence[LoRARequest | None] | None = None,
1320
- priorities: Sequence[int] | None = None,
1321
- use_tqdm: bool | Callable[..., tqdm] = True,
1322
- ):
1323
- if isinstance(prompts, (list, tuple)):
1324
- logger.warning_once(
1325
- "Rendering all prompts before adding them to the engine "
1326
- "is less efficient than performing both on the same prompt "
1327
- "before processing the next prompt. You should instead pass "
1328
- "a generator that renders one prompt per iteration, as that allows "
1329
- "engine execution to begin for the first prompt while processing "
1330
- "the next prompt."
1331
- )
1332
-
1333
- self._render_and_add_requests(
1334
- prompts=prompts,
1335
- params=params,
1336
- lora_requests=lora_requests,
1337
- priorities=priorities,
1338
- )
1339
-
1340
- return self._run_engine(output_type, use_tqdm=use_tqdm)
1341
-
1342
- def _render_and_add_requests(
1343
- self,
1344
- prompts: Iterable[EngineInput],
1345
- params: Sequence[SamplingParams | PoolingParams],
1346
- *,
1347
- lora_requests: Sequence[LoRARequest | None] | None = None,
1348
- priorities: Sequence[int] | None = None,
1349
- ) -> list[str]:
1350
- added_request_ids: list[str] = []
1351
-
1352
- try:
1353
- for i, prompt in enumerate(prompts):
1354
- request_id = self._add_request(
1355
- prompt,
1356
- params[i],
1357
- lora_request=self._resolve_mm_lora(
1358
- prompt,
1359
- None if lora_requests is None else lora_requests[i],
1360
- ),
1361
- priority=0 if priorities is None else priorities[i],
1362
- )
1363
- added_request_ids.append(request_id)
1364
- except Exception as e:
1365
- if added_request_ids:
1366
- self.llm_engine.abort_request(added_request_ids, internal=True)
1367
- raise e
1368
-
1369
- return added_request_ids
1370
-
1371
- def _add_request(
1372
- self,
1373
- prompt: EngineInput,
1374
- params: SamplingParams | PoolingParams,
1375
- lora_request: LoRARequest | None = None,
1376
- priority: int = 0,
1377
- ) -> str:
1378
- if isinstance(params, SamplingParams):
1379
- # We only care about the final output
1380
- params.output_kind = RequestOutputKind.FINAL_ONLY
1381
-
1382
- request_id = str(next(self.request_counter))
1383
-
1384
- return self.llm_engine.add_request(
1385
- request_id,
1386
- prompt,
1387
- params,
1388
- lora_request=lora_request,
1389
- priority=priority,
1390
- )
1391
-
1392
- def _run_engine(
1393
- self,
1394
- output_type: type[_O] | tuple[type[_O], ...],
1395
- *,
1396
- use_tqdm: bool | Callable[..., tqdm] = True,
1397
- ) -> list[_O]:
1398
- # Initialize tqdm.
1399
- if use_tqdm:
1400
- num_requests = self.llm_engine.get_num_unfinished_requests()
1401
- tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
1402
- pbar = tqdm_func(
1403
- total=num_requests,
1404
- desc="Processed prompts",
1405
- dynamic_ncols=True,
1406
- postfix=(f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s"),
1407
- )
1408
-
1409
- # Run the engine.
1410
- outputs: list[_O] = []
1411
- total_in_toks = 0
1412
- total_out_toks = 0
1413
- while self.llm_engine.has_unfinished_requests():
1414
- step_outputs = self.llm_engine.step()
1415
- for output in step_outputs:
1416
- assert isinstance(output, output_type)
1417
- if output.finished:
1418
- outputs.append(output) # type: ignore[arg-type]
1419
- if use_tqdm:
1420
- if isinstance(output, RequestOutput):
1421
- # Calculate tokens only for RequestOutput
1422
- n = len(output.outputs)
1423
- assert output.prompt_token_ids is not None
1424
- total_in_toks += len(output.prompt_token_ids) * n
1425
- in_spd = total_in_toks / pbar.format_dict["elapsed"]
1426
- total_out_toks += sum(
1427
- len(stp.token_ids) for stp in output.outputs
1428
- )
1429
- out_spd = total_out_toks / pbar.format_dict["elapsed"]
1430
- pbar.postfix = (
1431
- f"est. speed input: {in_spd:.2f} toks/s, "
1432
- f"output: {out_spd:.2f} toks/s"
1433
- )
1434
- pbar.update(n)
1435
- else:
1436
- pbar.update(1)
1437
- if pbar.n == num_requests:
1438
- pbar.refresh()
1439
-
1440
- if use_tqdm:
1441
- pbar.close()
1442
- # Sort the outputs by request ID.
1443
- # This is necessary because some requests may be finished earlier than
1444
- # its previous requests.
1445
- return sorted(outputs, key=lambda x: int(x.request_id))
1446
-
858
def init_weight_transfer_engine(
859
self, request: WeightTransferInitRequest | dict
860
) -> None:
vllm/entrypoints/offline_utils.py
new
+626
@@ -0,0 +1,626 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from collections.abc import Callable, Iterable, Sequence
5
+from typing import Any
6
+
7
+from tqdm import tqdm
8
+from typing_extensions import TypeVar
9
+
10
+from vllm import (
11
+ PoolingParams,
12
+ PoolingRequestOutput,
13
+ PromptType,
14
+ RequestOutput,
15
+ SamplingParams,
16
+)
17
+from vllm.config import ModelConfig
18
+from vllm.entrypoints.chat_utils import (
19
+ ChatCompletionMessageParam,
20
+ ChatTemplateContentFormatOption,
21
+)
22
+from vllm.inputs import EngineInput
23
+from vllm.logger import init_logger
24
+from vllm.lora.request import LoRARequest
25
+from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs
26
+from vllm.renderers.inputs.preprocess import (
27
+ conversation_to_seq,
28
+ parse_model_prompt,
29
+ prompt_to_seq,
30
+)
31
+from vllm.sampling_params import RequestOutputKind
32
+from vllm.utils.counter import Counter
33
+from vllm.utils.mistral import is_mistral_tokenizer
34
+from vllm.utils.tqdm_utils import maybe_tqdm
35
+from vllm.v1.engine.llm_engine import LLMEngine
36
+
37
+logger = init_logger(__name__)
38
+
39
+
40
+_P = TypeVar("_P", bound=SamplingParams | PoolingParams | None)
41
+_O = TypeVar(
42
+ "_O",
43
+ bound=RequestOutput | PoolingRequestOutput,
44
+ default=RequestOutput | PoolingRequestOutput,
45
+)
46
+_R = TypeVar("_R", default=Any)
47
+
48
+
49
+class OfflineInferenceMixin:
50
+ """Offline inference utils"""
51
+
52
+ request_counter: Counter
53
+ renderer: BaseRenderer
54
+ llm_engine: "LLMEngine"
55
+ model_config: ModelConfig
56
+
57
+ def _resolve_mm_lora(
58
+ self,
59
+ prompt: EngineInput,
60
+ lora_request: LoRARequest | None,
61
+ ) -> LoRARequest | None:
62
+ if prompt["type"] != "multimodal":
63
+ return lora_request
64
+
65
+ lora_config = self.llm_engine.vllm_config.lora_config
66
+ default_mm_loras = None if lora_config is None else lora_config.default_mm_loras
67
+ if not default_mm_loras:
68
+ return lora_request
69
+
70
+ prompt_modalities = prompt["mm_placeholders"].keys()
71
+ intersection = set(prompt_modalities).intersection(default_mm_loras.keys())
72
+ if not intersection:
73
+ return lora_request
74
+
75
+ if len(intersection) > 1:
76
+ # TODO: Would be nice to be able to have multiple loras per prompt
77
+ logger.warning(
78
+ "Multiple modality specific loras were registered and would be "
79
+ "used by a single prompt consuming several modalities; "
80
+ "currently we only support one lora per request; as such, "
81
+ "lora(s) registered with modalities: %s will be skipped",
82
+ intersection,
83
+ )
84
+ return lora_request
85
+
86
+ # Build the LoRA request; the ID of the default mm lora is the
87
+ # index of the modality name sorted alphabetically + 1.
88
+ modality_name = intersection.pop()
89
+ modality_lora_path = default_mm_loras[modality_name]
90
+ modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1
91
+
92
+ # If we have a collision, warn if there is a collision,
93
+ # but always send the explicitly provided request.
94
+ if lora_request:
95
+ if lora_request.lora_int_id != modality_lora_id:
96
+ logger.warning(
97
+ "A modality with a registered lora and a lora_request "
98
+ "with a different ID were provided; falling back to the "
99
+ "lora_request as we only apply one LoRARequest per prompt"
100
+ )
101
+ return lora_request
102
+
103
+ return LoRARequest(
104
+ modality_name,
105
+ modality_lora_id,
106
+ modality_lora_path,
107
+ )
108
+
109
+ def _preprocess_cmpl(
110
+ self,
111
+ prompts: Sequence[PromptType],
112
+ tokenization_kwargs: dict[str, Any] | None = None,
113
+ mm_processor_kwargs: dict[str, Any] | None = None,
114
+ ) -> Sequence[EngineInput]:
115
+ """
116
+ Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
117
+ a format that can be passed to `_add_request`.
118
+
119
+ Refer to [LLM.generate][] for a complete description of the arguments.
120
+
121
+ Returns:
122
+ A list of `EngineInput` objects ready to be passed into LLMEngine.
123
+ """
124
+ renderer = self.renderer
125
+ model_config = self.model_config
126
+
127
+ parsed_prompts = [
128
+ parse_model_prompt(model_config, prompt) for prompt in prompts
129
+ ]
130
+ tok_params = renderer.default_cmpl_tok_params.with_kwargs(
131
+ **(tokenization_kwargs or {})
132
+ )
133
+ prompt_extras = (
134
+ None
135
+ if mm_processor_kwargs is None
136
+ else {"mm_processor_kwargs": mm_processor_kwargs}
137
+ )
138
+
139
+ return renderer.render_cmpl(
140
+ parsed_prompts,
141
+ tok_params,
142
+ prompt_extras=prompt_extras,
143
+ )
144
+
145
+ def _preprocess_cmpl_one(
146
+ self,
147
+ prompt: PromptType,
148
+ tokenization_kwargs: dict[str, Any] | None = None,
149
+ mm_processor_kwargs: dict[str, Any] | None = None,
150
+ ) -> EngineInput:
151
+ (engine_input,) = self._preprocess_cmpl(
152
+ [prompt],
153
+ tokenization_kwargs,
154
+ mm_processor_kwargs=mm_processor_kwargs,
155
+ )
156
+ return engine_input
157
+
158
+ def _preprocess_chat(
159
+ self,
160
+ conversations: Sequence[list[ChatCompletionMessageParam]],
161
+ chat_template: str | None = None,
162
+ chat_template_content_format: ChatTemplateContentFormatOption = "auto",
163
+ chat_template_kwargs: dict[str, Any] | None = None,
164
+ add_generation_prompt: bool = True,
165
+ continue_final_message: bool = False,
166
+ tools: list[dict[str, Any]] | None = None,
167
+ tokenization_kwargs: dict[str, Any] | None = None,
168
+ mm_processor_kwargs: dict[str, Any] | None = None,
169
+ ) -> Sequence[EngineInput]:
170
+ """
171
+ Convert a list of conversations into prompts so that they can then
172
+ be used as input for other LLM APIs.
173
+
174
+ Refer to [LLM.chat][] for a complete description of the arguments.
175
+
176
+ Returns:
177
+ A list of `EngineInput` objects ready to be passed into LLMEngine.
178
+ """
179
+ renderer = self.renderer
180
+
181
+ chat_params = ChatParams(
182
+ chat_template=chat_template,
183
+ chat_template_content_format=chat_template_content_format,
184
+ chat_template_kwargs=merge_kwargs(
185
+ chat_template_kwargs,
186
+ dict(
187
+ add_generation_prompt=add_generation_prompt,
188
+ continue_final_message=continue_final_message,
189
+ tools=tools,
190
+ tokenize=(
191
+ is_mistral_tokenizer(renderer.tokenizer)
192
+ or self.model_config.enable_prompt_embeds
193
+ ),
194
+ ),
195
+ ),
196
+ mm_processor_kwargs=mm_processor_kwargs,
197
+ )
198
+ tok_params = renderer.default_chat_tok_params.with_kwargs(
199
+ **(tokenization_kwargs or {})
200
+ )
201
+ prompt_extras = (
202
+ None
203
+ if mm_processor_kwargs is None
204
+ else {"mm_processor_kwargs": mm_processor_kwargs}
205
+ )
206
+
207
+ _, engine_inputs = renderer.render_chat(
208
+ conversations,
209
+ chat_params,
210
+ tok_params,
211
+ prompt_extras=prompt_extras,
212
+ )
213
+
214
+ return engine_inputs
215
+
216
+ def _preprocess_chat_one(
217
+ self,
218
+ conversation: list[ChatCompletionMessageParam],
219
+ chat_template: str | None = None,
220
+ chat_template_content_format: ChatTemplateContentFormatOption = "auto",
221
+ chat_template_kwargs: dict[str, Any] | None = None,
222
+ add_generation_prompt: bool = True,
223
+ continue_final_message: bool = False,
224
+ tools: list[dict[str, Any]] | None = None,
225
+ tokenization_kwargs: dict[str, Any] | None = None,
226
+ mm_processor_kwargs: dict[str, Any] | None = None,
227
+ ) -> EngineInput:
228
+ (engine_input,) = self._preprocess_chat(
229
+ [conversation],
230
+ chat_template=chat_template,
231
+ chat_template_content_format=chat_template_content_format,
232
+ chat_template_kwargs=chat_template_kwargs,
233
+ add_generation_prompt=add_generation_prompt,
234
+ continue_final_message=continue_final_message,
235
+ tools=tools,
236
+ tokenization_kwargs=tokenization_kwargs,
237
+ mm_processor_kwargs=mm_processor_kwargs,
238
+ )
239
+
240
+ return engine_input
241
+
242
+ def _params_to_seq(
243
+ self,
244
+ params: _P | Sequence[_P],
245
+ num_requests: int,
246
+ ) -> Sequence[_P]:
247
+ if isinstance(params, Sequence):
248
+ if len(params) != num_requests:
249
+ raise ValueError(
250
+ f"The lengths of prompts ({num_requests}) "
251
+ f"and params ({len(params)}) must be the same."
252
+ )
253
+
254
+ return params
255
+
256
+ return [params] * num_requests
257
+
258
+ def _lora_request_to_seq(
259
+ self,
260
+ lora_request: LoRARequest | None | Sequence[LoRARequest | None],
261
+ num_requests: int,
262
+ ) -> Sequence[LoRARequest | None]:
263
+ if isinstance(lora_request, Sequence):
264
+ if len(lora_request) != num_requests:
265
+ raise ValueError(
266
+ f"The lengths of prompts ({num_requests}) "
267
+ f"and lora_request ({len(lora_request)}) must be the same."
268
+ )
269
+
270
+ return lora_request
271
+
272
+ return [lora_request] * num_requests
273
+
274
+ def _priority_to_seq(
275
+ self,
276
+ priority: list[int] | None,
277
+ num_requests: int,
278
+ ) -> Sequence[int]:
279
+ if priority is not None:
280
+ if len(priority) != num_requests:
281
+ raise ValueError(
282
+ f"The lengths of prompts ({num_requests}) "
283
+ f"and priority ({len(priority)}) must be the same."
284
+ )
285
+
286
+ return priority
287
+
288
+ return [0] * num_requests
289
+
290
+ def _add_completion_requests(
291
+ self,
292
+ prompts: PromptType | Sequence[PromptType],
293
+ params: SamplingParams
294
+ | PoolingParams
295
+ | Sequence[SamplingParams | PoolingParams],
296
+ *,
297
+ use_tqdm: bool | Callable[..., tqdm] = True,
298
+ lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
299
+ priority: list[int] | None = None,
300
+ tokenization_kwargs: dict[str, Any] | None = None,
301
+ mm_processor_kwargs: dict[str, Any] | None = None,
302
+ ) -> list[str]:
303
+ seq_prompts = prompt_to_seq(prompts)
304
+ seq_params = self._params_to_seq(params, len(seq_prompts))
305
+ seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_prompts))
306
+ seq_priority = self._priority_to_seq(priority, len(seq_prompts))
307
+
308
+ return self._render_and_add_requests(
309
+ prompts=(
310
+ self._preprocess_cmpl_one(
311
+ prompt,
312
+ tokenization_kwargs,
313
+ mm_processor_kwargs=mm_processor_kwargs,
314
+ )
315
+ for prompt in maybe_tqdm(
316
+ seq_prompts,
317
+ use_tqdm=use_tqdm,
318
+ desc="Rendering prompts",
319
+ )
320
+ ),
321
+ params=seq_params,
322
+ lora_requests=seq_lora_requests,
323
+ priorities=seq_priority,
324
+ )
325
+
326
+ def _run_completion(
327
+ self,
328
+ prompts: PromptType | Sequence[PromptType],
329
+ params: SamplingParams
330
+ | PoolingParams
331
+ | Sequence[SamplingParams | PoolingParams],
332
+ output_type: type[_O],
333
+ *,
334
+ use_tqdm: bool | Callable[..., tqdm] = True,
335
+ lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
336
+ priority: list[int] | None = None,
337
+ tokenization_kwargs: dict[str, Any] | None = None,
338
+ mm_processor_kwargs: dict[str, Any] | None = None,
339
+ ):
340
+ self._add_completion_requests(
341
+ prompts=prompts,
342
+ params=params,
343
+ use_tqdm=use_tqdm,
344
+ lora_request=lora_request,
345
+ priority=priority,
346
+ tokenization_kwargs=tokenization_kwargs,
347
+ mm_processor_kwargs=mm_processor_kwargs,
348
+ )
349
+ return self._run_engine(use_tqdm=use_tqdm, output_type=output_type)
350
+
351
+ def _run_chat(
352
+ self,
353
+ messages: list[ChatCompletionMessageParam]
354
+ | Sequence[list[ChatCompletionMessageParam]],
355
+ params: SamplingParams
356
+ | PoolingParams
357
+ | Sequence[SamplingParams | PoolingParams],
358
+ output_type: type[_O],
359
+ *,
360
+ use_tqdm: bool | Callable[..., tqdm] = True,
361
+ lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
362
+ chat_template: str | None = None,
363
+ chat_template_content_format: ChatTemplateContentFormatOption = "auto",
364
+ add_generation_prompt: bool = True,
365
+ continue_final_message: bool = False,
366
+ tools: list[dict[str, Any]] | None = None,
367
+ chat_template_kwargs: dict[str, Any] | None = None,
368
+ tokenization_kwargs: dict[str, Any] | None = None,
369
+ mm_processor_kwargs: dict[str, Any] | None = None,
370
+ ):
371
+ self._add_chat_requests(
372
+ messages=messages,
373
+ params=params,
374
+ use_tqdm=use_tqdm,
375
+ lora_request=lora_request,
376
+ chat_template=chat_template,
377
+ chat_template_content_format=chat_template_content_format,
378
+ chat_template_kwargs=chat_template_kwargs,
379
+ add_generation_prompt=add_generation_prompt,
380
+ continue_final_message=continue_final_message,
381
+ tools=tools,
382
+ tokenization_kwargs=tokenization_kwargs,
383
+ mm_processor_kwargs=mm_processor_kwargs,
384
+ )
385
+ return self._run_engine(output_type=output_type, use_tqdm=use_tqdm)
386
+
387
+ def _add_chat_requests(
388
+ self,
389
+ messages: list[ChatCompletionMessageParam]
390
+ | Sequence[list[ChatCompletionMessageParam]],
391
+ params: SamplingParams
392
+ | PoolingParams
393
+ | Sequence[SamplingParams | PoolingParams],
394
+ *,
395
+ use_tqdm: bool | Callable[..., tqdm] = True,
396
+ lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
397
+ priority: list[int] | None = None,
398
+ chat_template: str | None = None,
399
+ chat_template_content_format: ChatTemplateContentFormatOption = "auto",
400
+ add_generation_prompt: bool = True,
401
+ continue_final_message: bool = False,
402
+ tools: list[dict[str, Any]] | None = None,
403
+ chat_template_kwargs: dict[str, Any] | None = None,
404
+ tokenization_kwargs: dict[str, Any] | None = None,
405
+ mm_processor_kwargs: dict[str, Any] | None = None,
406
+ ) -> list[str]:
407
+ seq_convs = conversation_to_seq(messages)
408
+ seq_params = self._params_to_seq(params, len(seq_convs))
409
+ seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_convs))
410
+ seq_priority = self._priority_to_seq(priority, len(seq_convs))
411
+
412
+ # When thinking is enabled or tools are provided, and the model
413
+ # uses special tokens for structured output (e.g. Gemma4's
414
+ # <|channel>, <|tool_call>, <|"|>), automatically set
415
+ # skip_special_tokens=False so these tokens are preserved in
416
+ # output.text for downstream parsing.
417
+ needs_parsing = (
418
+ chat_template_kwargs and chat_template_kwargs.get("enable_thinking")
419
+ ) or tools
420
+ if needs_parsing:
421
+ self._adjust_params_for_parsing(seq_params)
422
+
423
+ return self._render_and_add_requests(
424
+ prompts=(
425
+ self._preprocess_chat_one(
426
+ conversation,
427
+ chat_template=chat_template,
428
+ chat_template_content_format=chat_template_content_format,
429
+ chat_template_kwargs=chat_template_kwargs,
430
+ add_generation_prompt=add_generation_prompt,
431
+ continue_final_message=continue_final_message,
432
+ tools=tools,
433
+ tokenization_kwargs=tokenization_kwargs,
434
+ mm_processor_kwargs=mm_processor_kwargs,
435
+ )
436
+ for conversation in maybe_tqdm(
437
+ seq_convs,
438
+ use_tqdm=use_tqdm,
439
+ desc="Rendering conversations",
440
+ )
441
+ ),
442
+ params=seq_params,
443
+ lora_requests=seq_lora_requests,
444
+ priorities=seq_priority,
445
+ )
446
+
447
+ def _adjust_params_for_parsing(
448
+ self, params: Sequence[SamplingParams | PoolingParams]
449
+ ) -> None:
450
+ """Set ``skip_special_tokens=False`` when the model encodes
451
+ structured output syntax as special tokens.
452
+
453
+ Models like Gemma4 register thinking delimiters
454
+ (``<|channel>``/``<channel|>``) and tool call tokens
455
+ (``<|tool_call>``/``<tool_call|>``/``<|"|>``) as special tokens.
456
+ The default ``skip_special_tokens=True`` strips them from
457
+ ``output.text``, breaking parsing of both reasoning blocks and
458
+ tool calls.
459
+
460
+ This is a no-op for models whose structured tokens are regular
461
+ text tokens (e.g. DeepSeek's ``<think>``/``</think>``).
462
+ """
463
+ # The offline API currently lacks a unified rendering pipeline.
464
+ # Until the planned Renderer refactor is complete, we hardcode
465
+ # this token preservation logic specifically for Gemma4 models
466
+ # to avoid regressions on other models.
467
+ hf_config = getattr(self.model_config, "hf_config", None)
468
+ architectures = getattr(hf_config, "architectures", [])
469
+
470
+ if any("Gemma4" in arch for arch in architectures):
471
+ tokenizer = self.renderer.get_tokenizer()
472
+ vocab = tokenizer.get_vocab()
473
+ special_ids = set(getattr(tokenizer, "all_special_ids", []))
474
+
475
+ # Tokens used for thinking delimiters and tool call syntax
476
+ # that some models (Gemma4) register as special tokens.
477
+ structured_tokens = (
478
+ "<|channel>",
479
+ "<channel|>", # thinking delimiters
480
+ "<|tool_call>",
481
+ "<tool_call|>", # tool call delimiters
482
+ '<|"|>', # string quoting in tool args
483
+ )
484
+ needs_special = any(
485
+ vocab.get(tok) in special_ids
486
+ for tok in structured_tokens
487
+ if tok in vocab
488
+ )
489
+ if needs_special:
490
+ for sp in params:
491
+ if isinstance(sp, SamplingParams) and sp.skip_special_tokens:
492
+ sp.skip_special_tokens = False
493
+
494
+ def _render_and_run_requests(
495
+ self,
496
+ prompts: Iterable[EngineInput],
497
+ params: Sequence[SamplingParams | PoolingParams],
498
+ output_type: type[_O],
499
+ *,
500
+ lora_requests: Sequence[LoRARequest | None] | None = None,
501
+ priorities: Sequence[int] | None = None,
502
+ use_tqdm: bool | Callable[..., tqdm] = True,
503
+ ):
504
+ if isinstance(prompts, (list, tuple)):
505
+ logger.warning_once(
506
+ "Rendering all prompts before adding them to the engine "
507
+ "is less efficient than performing both on the same prompt "
508
+ "before processing the next prompt. You should instead pass "
509
+ "a generator that renders one prompt per iteration, as that allows "
510
+ "engine execution to begin for the first prompt while processing "
511
+ "the next prompt."
512
+ )
513
+
514
+ self._render_and_add_requests(
515
+ prompts=prompts,
516
+ params=params,
517
+ lora_requests=lora_requests,
518
+ priorities=priorities,
519
+ )
520
+
521
+ return self._run_engine(output_type, use_tqdm=use_tqdm)
522
+
523
+ def _render_and_add_requests(
524
+ self,
525
+ prompts: Iterable[EngineInput],
526
+ params: Sequence[SamplingParams | PoolingParams],
527
+ *,
528
+ lora_requests: Sequence[LoRARequest | None] | None = None,
529
+ priorities: Sequence[int] | None = None,
530
+ ) -> list[str]:
531
+ added_request_ids: list[str] = []
532
+
533
+ try:
534
+ for i, prompt in enumerate(prompts):
535
+ request_id = self._add_request(
536
+ prompt,
537
+ params[i],
538
+ lora_request=self._resolve_mm_lora(
539
+ prompt,
540
+ None if lora_requests is None else lora_requests[i],
541
+ ),
542
+ priority=0 if priorities is None else priorities[i],
543
+ )
544
+ added_request_ids.append(request_id)
545
+ except Exception as e:
546
+ if added_request_ids:
547
+ self.llm_engine.abort_request(added_request_ids, internal=True)
548
+ raise e
549
+
550
+ return added_request_ids
551
+
552
+ def _add_request(
553
+ self,
554
+ prompt: EngineInput,
555
+ params: SamplingParams | PoolingParams,
556
+ lora_request: LoRARequest | None = None,
557
+ priority: int = 0,
558
+ ) -> str:
559
+ if isinstance(params, SamplingParams):
560
+ # We only care about the final output
561
+ params.output_kind = RequestOutputKind.FINAL_ONLY
562
+
563
+ request_id = str(next(self.request_counter))
564
+
565
+ return self.llm_engine.add_request(
566
+ request_id,
567
+ prompt,
568
+ params,
569
+ lora_request=lora_request,
570
+ priority=priority,
571
+ )
572
+
573
+ def _run_engine(
574
+ self,
575
+ output_type: type[_O] | tuple[type[_O], ...],
576
+ *,
577
+ use_tqdm: bool | Callable[..., tqdm] = True,
578
+ ) -> list[_O]:
579
+ # Initialize tqdm.
580
+ if use_tqdm:
581
+ num_requests = self.llm_engine.get_num_unfinished_requests()
582
+ tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
583
+ pbar = tqdm_func(
584
+ total=num_requests,
585
+ desc="Processed prompts",
586
+ dynamic_ncols=True,
587
+ postfix=(f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s"),
588
+ )
589
+
590
+ # Run the engine.
591
+ outputs: list[_O] = []
592
+ total_in_toks = 0
593
+ total_out_toks = 0
594
+ while self.llm_engine.has_unfinished_requests():
595
+ step_outputs = self.llm_engine.step()
596
+ for output in step_outputs:
597
+ assert isinstance(output, output_type)
598
+ if output.finished:
599
+ outputs.append(output) # type: ignore[arg-type]
600
+ if use_tqdm:
601
+ if isinstance(output, RequestOutput):
602
+ # Calculate tokens only for RequestOutput
603
+ n = len(output.outputs)
604
+ assert output.prompt_token_ids is not None
605
+ total_in_toks += len(output.prompt_token_ids) * n
606
+ in_spd = total_in_toks / pbar.format_dict["elapsed"]
607
+ total_out_toks += sum(
608
+ len(stp.token_ids) for stp in output.outputs
609
+ )
610
+ out_spd = total_out_toks / pbar.format_dict["elapsed"]
611
+ pbar.postfix = (
612
+ f"est. speed input: {in_spd:.2f} toks/s, "
613
+ f"output: {out_spd:.2f} toks/s"
614
+ )
615
+ pbar.update(n)
616
+ else:
617
+ pbar.update(1)
618
+ if pbar.n == num_requests:
619
+ pbar.refresh()
620
+
621
+ if use_tqdm:
622
+ pbar.close()
623
+ # Sort the outputs by request ID.
624
+ # This is necessary because some requests may be finished earlier than
625
+ # its previous requests.
626
+ return sorted(outputs, key=lambda x: int(x.request_id))
vllm/entrypoints/pooling/offline.py
+4
-68
@@ -1,34 +1,24 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from abc import ABC, abstractmethod
5
-from collections.abc import Callable, Iterable, Sequence
4
+from collections.abc import Callable, Sequence
5
from typing import Any
6
7
from tqdm.auto import tqdm
9
-from typing_extensions import TypeVar
8
11
-from vllm.config import ModelConfig
9
from vllm.entrypoints.chat_utils import ChatTemplateConfig
13
-from vllm.inputs import (
14
- DataPrompt,
15
- EngineInput,
16
- PromptType,
17
-)
10
+from vllm.entrypoints.offline_utils import OfflineInferenceMixin
11
+from vllm.inputs import DataPrompt, PromptType
12
from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
14
from vllm.outputs import (
15
ClassificationRequestOutput,
16
EmbeddingRequestOutput,
17
PoolingRequestOutput,
24
- RequestOutput,
18
ScoringRequestOutput,
19
)
20
from vllm.pooling_params import PoolingParams
28
-from vllm.renderers import BaseRenderer
29
-from vllm.sampling_params import SamplingParams
21
from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
31
-from vllm.v1.engine.llm_engine import LLMEngine
22
23
from .factories import init_pooling_io_processors
24
from .scoring.io_processor import ScoringIOProcessor
@@ -37,20 +27,10 @@ from .typing import OfflineInputsContext, OfflineOutputsContext
27
28
logger = init_logger(__name__)
29
40
-_P = TypeVar("_P", bound=SamplingParams | PoolingParams | None)
41
-_O = TypeVar(
42
- "_O",
43
- bound=RequestOutput | PoolingRequestOutput,
44
- default=RequestOutput | PoolingRequestOutput,
45
-)
46
-
30
48
-class PoolingOfflineMixin(ABC):
31
+class PoolingOfflineMixin(OfflineInferenceMixin):
32
"""Offline inference for pooling models"""
33
51
- renderer: BaseRenderer
52
- llm_engine: "LLMEngine"
53
- model_config: ModelConfig
34
runner_type: str
35
chat_template: str | None
36
supported_tasks: tuple[SupportedTask, ...]
@@ -464,47 +444,3 @@ class PoolingOfflineMixin(ABC):
444
)
445
446
return [ScoringRequestOutput.from_base(item) for item in outputs]
467
-
468
- @abstractmethod
469
- def _params_to_seq(
470
- self,
471
- params: _P | Sequence[_P],
472
- num_requests: int,
473
- ) -> Sequence[_P]:
474
- raise NotImplementedError
475
-
476
- @abstractmethod
477
- def _lora_request_to_seq(
478
- self,
479
- lora_request: LoRARequest | None | Sequence[LoRARequest | None],
480
- num_requests: int,
481
- ) -> Sequence[LoRARequest | None]:
482
- raise NotImplementedError
483
-
484
- @abstractmethod
485
- def _priority_to_seq(
486
- self,
487
- priority: list[int] | None,
488
- num_requests: int,
489
- ) -> Sequence[int]:
490
- raise NotImplementedError
491
-
492
- @abstractmethod
493
- def _render_and_add_requests(
494
- self,
495
- prompts: Iterable[EngineInput],
496
- params: Sequence[SamplingParams | PoolingParams],
497
- *,
498
- lora_requests: Sequence[LoRARequest | None] | None = None,
499
- priorities: Sequence[int] | None = None,
500
- ) -> list[str]:
501
- raise NotImplementedError
502
-
503
- @abstractmethod
504
- def _run_engine(
505
- self,
506
- output_type: type[_O] | tuple[type[_O], ...],
507
- *,
508
- use_tqdm: bool | Callable[..., tqdm] = True,
509
- ) -> list[_O]:
510
- raise NotImplementedError