@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e1a5c2f0a1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Model] Whisper model implementation (#11280)

Co-authored-by: Aurick Qiao <aurick.qiao@snowflake.com>

Aurick Qiao committed Jan 3, 2025 at 03:39 UTC e1a5c2f0a123835558b1b1c9895181161527c55e
16 files changed +1045 -55
.buildkite/test-pipeline.yaml
+2
@@ -363,12 +363,14 @@ steps:
363 - tests/models/decoder_only/audio_language
364 - tests/models/decoder_only/vision_language
365 - tests/models/embedding/vision_language
366 + - tests/models/encoder_decoder/audio_language
367 - tests/models/encoder_decoder/vision_language
368 commands:
369 - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
370 - pytest -v -s models/decoder_only/audio_language -m 'core_model or quant_model'
371 - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'core_model or quant_model'
372 - pytest -v -s models/embedding/vision_language -m core_model
373 + - pytest -v -s models/encoder_decoder/audio_language -m core_model
374 - pytest -v -s models/encoder_decoder/language -m core_model
375 - pytest -v -s models/encoder_decoder/vision_language -m core_model
376
examples/offline_inference_whisper.py new
+59
@@ -0,0 +1,59 @@
1 +import time
2 +
3 +from vllm import LLM, SamplingParams
4 +from vllm.assets.audio import AudioAsset
5 +
6 +# Create a Whisper encoder/decoder model instance
7 +llm = LLM(
8 + model="openai/whisper-large-v3",
9 + max_model_len=448,
10 + max_num_seqs=400,
11 + limit_mm_per_prompt={"audio": 1},
12 + kv_cache_dtype="fp8",
13 +)
14 +
15 +prompts = [
16 + {
17 + "prompt": "<|startoftranscript|>",
18 + "multi_modal_data": {
19 + "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
20 + },
21 + },
22 + { # Test explicit encoder/decoder prompt
23 + "encoder_prompt": {
24 + "prompt": "",
25 + "multi_modal_data": {
26 + "audio": AudioAsset("winning_call").audio_and_sample_rate,
27 + },
28 + },
29 + "decoder_prompt": "<|startoftranscript|>",
30 + }
31 +] * 1024
32 +
33 +# Create a sampling params object.
34 +sampling_params = SamplingParams(
35 + temperature=0,
36 + top_p=1.0,
37 + max_tokens=200,
38 +)
39 +
40 +start = time.time()
41 +
42 +# Generate output tokens from the prompts. The output is a list of
43 +# RequestOutput objects that contain the prompt, generated
44 +# text, and other information.
45 +outputs = llm.generate(prompts, sampling_params)
46 +
47 +# Print the outputs.
48 +for output in outputs:
49 + prompt = output.prompt
50 + encoder_prompt = output.encoder_prompt
51 + generated_text = output.outputs[0].text
52 + print(f"Encoder prompt: {encoder_prompt!r}, "
53 + f"Decoder prompt: {prompt!r}, "
54 + f"Generated text: {generated_text!r}")
55 +
56 +duration = time.time() - start
57 +
58 +print("Duration:", duration)
59 +print("RPS:", len(prompts) / duration)
tests/models/encoder_decoder/audio_language/__init__.py
tests/models/encoder_decoder/audio_language/test_whisper.py new
+136
@@ -0,0 +1,136 @@
1 +"""Compare the outputs of HF and vLLM for Whisper models using greedy sampling.
2 +
3 +Run `pytest tests/models/encoder_decoder/audio/test_whisper.py`.
4 +"""
5 +from typing import Optional
6 +
7 +import pytest
8 +
9 +from vllm import LLM, SamplingParams
10 +from vllm.assets.audio import AudioAsset
11 +
12 +from ....utils import fork_new_process_for_each_test, multi_gpu_test
13 +
14 +PROMPTS = [
15 + {
16 + "prompt":
17 + "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
18 + "multi_modal_data": {
19 + "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
20 + },
21 + },
22 + { # Test explicit encoder/decoder prompt
23 + "encoder_prompt": {
24 + "prompt": "",
25 + "multi_modal_data": {
26 + "audio": AudioAsset("winning_call").audio_and_sample_rate,
27 + },
28 + },
29 + "decoder_prompt":
30 + "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
31 + }
32 +]
33 +
34 +EXPECTED = {
35 + "openai/whisper-tiny": [
36 + " He has birth words I spoke in the original corner of that. And a"
37 + " little piece of black coat poetry. Mary had a little sandwich,"
38 + " sweet, with white and snow. And everyone had it very went the last"
39 + " would sure to go.",
40 + " >> And the old one, fit John the way to Edgar Martinez. >> One more"
41 + " to line down the field line for our base camp. Here comes joy. Here"
42 + " is June and the third base. They're going to wave him in. The throw"
43 + " to the plate will be late. The Mariners are going to play for the"
44 + " American League Championship. I don't believe it. It just continues"
45 + " by all five."
46 + ],
47 + "openai/whisper-small": [
48 + " The first words I spoke in the original pornograph. A little piece"
49 + " of practical poetry. Mary had a little lamb, its fleece was quite a"
50 + " slow, and everywhere that Mary went the lamb was sure to go.",
51 + " And the old one pitch on the way to Edgar Martinez one month. Here"
52 + " comes joy. Here is Junior to third base. They're gonna wave him"
53 + " in. The throw to the plate will be late. The Mariners are going to"
54 + " play for the American League Championship. I don't believe it. It"
55 + " just continues. My, oh my."
56 + ],
57 + "openai/whisper-medium": [
58 + " The first words I spoke in the original phonograph, a little piece"
59 + " of practical poetry. Mary had a little lamb, its fleece was quite as"
60 + " slow, and everywhere that Mary went the lamb was sure to go.",
61 + " And the 0-1 pitch on the way to Edgar Martinez swung on the line"
62 + " down the left field line for Obeyshev. Here comes Joy. Here is"
63 + " Jorgen at third base. They're going to wave him in. The throw to the"
64 + " plate will be late. The Mariners are going to play for the American"
65 + " League Championship. I don't believe it. It just continues. My, oh"
66 + " my."
67 + ],
68 + "openai/whisper-large-v3": [
69 + " The first words I spoke in the original phonograph, a little piece"
70 + " of practical poetry. Mary had a little lamb, its feet were quite as"
71 + " slow, and everywhere that Mary went, the lamb was sure to go.",
72 + " And the 0-1 pitch on the way to Edgar Martinez. Swung on the line."
73 + " Now the left field line for a base hit. Here comes Joy. Here is"
74 + " Junior to third base. They're going to wave him in. The throw to the"
75 + " plate will be late. The Mariners are going to play for the American"
76 + " League Championship. I don't believe it. It just continues. My, oh,"
77 + " my."
78 + ],
79 + "openai/whisper-large-v3-turbo": [
80 + " The first words I spoke in the original phonograph, a little piece"
81 + " of practical poetry. Mary had a little lamb, its streets were quite"
82 + " as slow, and everywhere that Mary went the lamb was sure to go.",
83 + " And the 0-1 pitch on the way to Edgar Martinez. Swung on the line"
84 + " down the left field line for a base hit. Here comes Joy. Here is"
85 + " Junior to third base. They're going to wave him in. The throw to the"
86 + " plate will be late. The Mariners are going to play for the American"
87 + " League Championship. I don't believe it. It just continues. My, oh,"
88 + " my."
89 + ]
90 +}
91 +
92 +
93 +def run_test(
94 + model: str,
95 + *,
96 + tensor_parallel_size: int,
97 + distributed_executor_backend: Optional[str] = None,
98 +) -> None:
99 + prompt_list = PROMPTS * 10
100 + expected_list = EXPECTED[model] * 10
101 +
102 + llm = LLM(
103 + model=model,
104 + tensor_parallel_size=tensor_parallel_size,
105 + distributed_executor_backend=distributed_executor_backend,
106 + )
107 +
108 + sampling_params = SamplingParams(
109 + temperature=0,
110 + top_p=1.0,
111 + max_tokens=200,
112 + )
113 +
114 + outputs = llm.generate(prompt_list, sampling_params)
115 +
116 + for output, expected in zip(outputs, expected_list):
117 + print(output.outputs[0].text)
118 + assert output.outputs[0].text == expected
119 +
120 +
121 +@fork_new_process_for_each_test
122 +@pytest.mark.core_model
123 +@pytest.mark.parametrize(
124 + "model", ["openai/whisper-small", "openai/whisper-large-v3-turbo"])
125 +def test_models(model) -> None:
126 + run_test(model, tensor_parallel_size=1)
127 +
128 +
129 +@multi_gpu_test(num_gpus=2)
130 +@pytest.mark.core_model
131 +@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
132 +@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
133 +def test_models_distributed(model, distributed_executor_backend) -> None:
134 + run_test(model,
135 + tensor_parallel_size=2,
136 + distributed_executor_backend=distributed_executor_backend)
tests/models/registry.py
+1
@@ -204,6 +204,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
204 "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3"),
205 # [Encoder-decoder]
206 "MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
207 + "WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501
208 }
209
210 _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
vllm/config.py
+2
@@ -2312,6 +2312,8 @@ def _get_and_verify_max_len(
2312 "seq_length",
2313 # Command-R
2314 "model_max_length",
2315 + # Whisper
2316 + "max_target_positions",
2317 # Others
2318 "max_sequence_length",
2319 "max_seq_length",
vllm/inputs/preprocess.py
+28 -8
@@ -184,10 +184,16 @@ class InputPreprocessor:
184 corresponding token IDs.
185 """
186 tokenizer = self.get_tokenizer_group()
187 -
187 + add_special_tokens = None
188 + if self.model_config.hf_config.model_type == "whisper":
189 + # For Whisper, special tokens should be provided by the user based
190 + # on the task and language of their request. Also needed to avoid
191 + # appending an EOS token to the prompt which disrupts generation.
192 + add_special_tokens = False
193 return tokenizer.encode(request_id=request_id,
194 prompt=prompt,
190 - lora_request=lora_request)
195 + lora_request=lora_request,
196 + add_special_tokens=add_special_tokens)
197
198 async def _tokenize_prompt_async(
199 self,
@@ -197,10 +203,17 @@ class InputPreprocessor:
203 ) -> List[int]:
204 """Async version of :meth:`_tokenize_prompt`."""
205 tokenizer = self.get_tokenizer_group()
200 -
201 - return await tokenizer.encode_async(request_id=request_id,
202 - prompt=prompt,
203 - lora_request=lora_request)
206 + add_special_tokens = None
207 + if self.model_config.hf_config.model_type == "whisper":
208 + # For Whisper, special tokens should be provided by the user based
209 + # on the task and language of their request. Also needed to avoid
210 + # appending an EOS token to the prompt which disrupts generation.
211 + add_special_tokens = False
212 + return await tokenizer.encode_async(
213 + request_id=request_id,
214 + prompt=prompt,
215 + lora_request=lora_request,
216 + add_special_tokens=add_special_tokens)
217
218 def _can_process_multimodal(self) -> bool:
219 model_config = self.model_config
@@ -439,8 +452,15 @@ class InputPreprocessor:
452 assert_never(encoder_inputs) # type: ignore[arg-type]
453
454 if decoder_inputs is None:
442 - dec_token_ids = self._prepare_decoder_input_ids_for_generation(
443 - None)
455 + if self.model_config.hf_config.model_type == "whisper":
456 + # For Whisper models, the text prompt should go to the decoder.
457 + # If no explicit encoder/decoder inputs, then copy the prompt
458 + # from the encoder to the decoder. The encoder tokens are later
459 + # overridden by the audio features.
460 + dec_token_ids = encoder_inputs["prompt_token_ids"].copy()
461 + else:
462 + dec_token_ids = self._prepare_decoder_input_ids_for_generation(
463 + None)
464 decoder_inputs = token_inputs(dec_token_ids)
465 elif (decoder_inputs["type"] == "token"
466 or decoder_inputs["type"] == "multimodal"):
vllm/model_executor/models/registry.py
+1
@@ -170,6 +170,7 @@ _MULTIMODAL_MODELS = {
170 "UltravoxModel": ("ultravox", "UltravoxModel"),
171 # [Encoder-decoder]
172 "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501
173 + "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"), # noqa: E501
174 }
175
176 _SPECULATIVE_DECODING_MODELS = {
vllm/model_executor/models/whisper.py new
+737
@@ -0,0 +1,737 @@
1 +import math
2 +from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
3 + Union)
4 +
5 +import numpy as np
6 +import torch
7 +from torch import nn
8 +from transformers.models.whisper.modeling_whisper import sinusoids
9 +
10 +from vllm.attention import Attention, AttentionMetadata, AttentionType
11 +from vllm.config import CacheConfig, VllmConfig
12 +from vllm.distributed import get_tensor_model_parallel_world_size
13 +from vllm.inputs import INPUT_REGISTRY, DummyData, InputContext
14 +from vllm.logger import init_logger
15 +from vllm.model_executor.layers.activation import get_act_fn
16 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 + QKVParallelLinear,
18 + RowParallelLinear)
19 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
20 +from vllm.model_executor.layers.quantization.base_config import (
21 + QuantizationConfig)
22 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
24 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
25 +from vllm.model_executor.sampling_metadata import SamplingMetadata
26 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
27 + NestedTensors)
28 +from vllm.multimodal.audio import resample_audio
29 +from vllm.sequence import SequenceData
30 +from vllm.transformers_utils.processor import cached_get_processor
31 +
32 +from .interfaces import SupportsMultiModal
33 +from .utils import AutoWeightsLoader, WeightsMapper, make_layers
34 +
35 +logger = init_logger(__name__)
36 +
37 +
38 +class WhisperAudioInputs(TypedDict):
39 + input_features: NestedTensors
40 + """Shape: `(batch_size, 128, M)`"""
41 +
42 +
43 +class WhisperPositionalEmbedding(nn.Embedding):
44 +
45 + def __init__(self,
46 + num_positions: int,
47 + embedding_dim: int,
48 + padding_idx: Optional[int] = None):
49 + super().__init__(num_positions, embedding_dim)
50 +
51 + def forward(self, position_ids):
52 + return self.weight[position_ids]
53 +
54 +
55 +class WhisperAttention(nn.Module):
56 +
57 + def __init__(
58 + self,
59 + embed_dim: int,
60 + num_heads: int,
61 + bias: bool = True,
62 + attn_type: AttentionType = AttentionType.DECODER,
63 + cache_config: Optional[CacheConfig] = None,
64 + quant_config: Optional[QuantizationConfig] = None,
65 + prefix: str = "",
66 + ):
67 + super().__init__()
68 + self.embed_dim = embed_dim
69 + tp_size = get_tensor_model_parallel_world_size()
70 + self.total_num_heads = num_heads
71 + assert self.total_num_heads % tp_size == 0
72 + self.num_heads = self.total_num_heads // tp_size
73 + if self.total_num_heads >= tp_size:
74 + # Number of heads is greater than TP size, so we partition
75 + # the KV heads across multiple tensor parallel GPUs.
76 + assert self.total_num_heads % tp_size == 0
77 + else:
78 + # Number of heads is less than TP size, so we replicate
79 + # the KV heads across multiple tensor parallel GPUs.
80 + assert tp_size % self.total_num_heads == 0
81 + self.num_kv_heads = max(1, self.total_num_heads // tp_size)
82 + self.head_dim = self.embed_dim // self.total_num_heads
83 + self.q_size = self.num_heads * self.head_dim
84 + self.kv_size = self.num_kv_heads * self.head_dim
85 + self.attn_type = attn_type
86 +
87 + if (self.head_dim * num_heads) != self.embed_dim:
88 + raise ValueError(
89 + f"embed_dim must be divisible by num_heads (got `embed_dim`: "
90 + f"{self.embed_dim} and `num_heads`: {num_heads}).")
91 + self.scaling = self.head_dim**-0.5
92 +
93 + self._init_qkv(embed_dim, bias, quant_config, prefix=prefix)
94 + self.out_proj = RowParallelLinear(
95 + input_size=embed_dim,
96 + output_size=embed_dim,
97 + bias=bias,
98 + quant_config=quant_config,
99 + prefix=f"{prefix}.out_proj",
100 + )
101 + self.attn = Attention(
102 + self.num_heads,
103 + self.head_dim,
104 + self.scaling,
105 + num_kv_heads=self.num_kv_heads,
106 + cache_config=cache_config,
107 + quant_config=quant_config,
108 + prefix=f"{prefix}.attn",
109 + )
110 +
111 + def _init_qkv(
112 + self,
113 + embed_dim: int,
114 + bias: bool = True,
115 + quant_config: Optional[QuantizationConfig] = None,
116 + prefix: str = "",
117 + ) -> None:
118 + self.qkv_proj = QKVParallelLinear(
119 + hidden_size=embed_dim,
120 + head_size=self.head_dim,
121 + total_num_heads=self.total_num_heads,
122 + total_num_kv_heads=self.total_num_heads,
123 + bias=bias,
124 + quant_config=quant_config,
125 + prefix=f"{prefix}.qkv_proj",
126 + )
127 +
128 + def forward(
129 + self,
130 + hidden_states: torch.Tensor,
131 + kv_cache: torch.Tensor,
132 + attn_metadata: AttentionMetadata,
133 + ):
134 + qkv, _ = self.qkv_proj(hidden_states)
135 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
136 +
137 + attn_output = self.attn(q,
138 + k,
139 + v,
140 + kv_cache,
141 + attn_metadata,
142 + attn_type=self.attn_type)
143 +
144 + output, _ = self.out_proj(attn_output)
145 +
146 + return output
147 +
148 +
149 +class WhisperCrossAttention(WhisperAttention):
150 +
151 + def __init__(
152 + self,
153 + embed_dim: int,
154 + num_heads: int,
155 + bias: bool = True,
156 + cache_config: Optional[CacheConfig] = None,
157 + quant_config: Optional[QuantizationConfig] = None,
158 + prefix: str = "",
159 + ):
160 + super().__init__(
161 + embed_dim=embed_dim,
162 + num_heads=num_heads,
163 + bias=bias,
164 + cache_config=cache_config,
165 + quant_config=quant_config,
166 + prefix=prefix,
167 + )
168 +
169 + def _init_qkv(
170 + self,
171 + embed_dim: int,
172 + bias: bool = True,
173 + quant_config: Optional[QuantizationConfig] = None,
174 + prefix: str = "",
175 + ) -> None:
176 + self.q_proj = ColumnParallelLinear(
177 + input_size=embed_dim,
178 + output_size=embed_dim,
179 + bias=bias,
180 + quant_config=quant_config,
181 + prefix=f"{prefix}.q_proj",
182 + )
183 + self.kv_proj = QKVParallelLinear(
184 + hidden_size=embed_dim,
185 + head_size=self.head_dim,
186 + total_num_heads=0,
187 + total_num_kv_heads=self.total_num_heads,
188 + bias=bias,
189 + quant_config=quant_config,
190 + prefix=f"{prefix}.kv_proj",
191 + )
192 +
193 + def forward(
194 + self,
195 + hidden_states: torch.Tensor,
196 + encoder_hidden_states: Optional[torch.Tensor],
197 + kv_cache: torch.Tensor,
198 + attn_metadata: AttentionMetadata,
199 + ):
200 + q, _ = self.q_proj(hidden_states)
201 +
202 + # Encoder hidden states are only computed once during prefill phase.
203 + # Afterwards, the keys and values should be available in the kv-cache.
204 + if encoder_hidden_states is not None:
205 + kv, _ = self.kv_proj(encoder_hidden_states)
206 + k, v = kv.split([self.kv_size, self.kv_size], dim=-1)
207 + else:
208 + k = v = None
209 +
210 + attn_output = self.attn(q,
211 + k,
212 + v,
213 + kv_cache,
214 + attn_metadata,
215 + attn_type=AttentionType.ENCODER_DECODER)
216 +
217 + output, _ = self.out_proj(attn_output)
218 +
219 + return output
220 +
221 +
222 +class WhisperMLP(nn.Module):
223 +
224 + def __init__(
225 + self,
226 + embed_dim: int,
227 + ffn_dim: int,
228 + act_fn: str,
229 + quant_config: Optional[QuantizationConfig] = None,
230 + prefix: str = "",
231 + ):
232 + super().__init__()
233 +
234 + self.activation_fn = get_act_fn(act_fn)
235 + self.fc1 = ColumnParallelLinear(
236 + input_size=embed_dim,
237 + output_size=ffn_dim,
238 + quant_config=quant_config,
239 + prefix=f"{prefix}.fc1",
240 + )
241 + self.fc2 = RowParallelLinear(
242 + input_size=ffn_dim,
243 + output_size=embed_dim,
244 + quant_config=quant_config,
245 + prefix=f"{prefix}.fc2",
246 + )
247 +
248 + def forward(self, hidden_states: torch.Tensor):
249 + hidden_states, _ = self.fc1(hidden_states)
250 + hidden_states = self.activation_fn(hidden_states)
251 + hidden_states, _ = self.fc2(hidden_states)
252 + return hidden_states
253 +
254 +
255 +class WhisperEncoderLayer(nn.Module):
256 +
257 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
258 + super().__init__()
259 + config = vllm_config.model_config.hf_config
260 + cache_config = vllm_config.cache_config
261 + quant_config = vllm_config.quant_config
262 +
263 + self.embed_dim = config.d_model
264 + self.self_attn = WhisperAttention(
265 + embed_dim=self.embed_dim,
266 + num_heads=config.encoder_attention_heads,
267 + attn_type=AttentionType.ENCODER,
268 + cache_config=cache_config,
269 + quant_config=quant_config,
270 + prefix=f"{prefix}.self_attn",
271 + )
272 + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
273 + self.mlp = WhisperMLP(
274 + embed_dim=config.d_model,
275 + ffn_dim=config.encoder_ffn_dim,
276 + act_fn=config.activation_function,
277 + quant_config=quant_config,
278 + prefix=f"{prefix}.mlp",
279 + )
280 + self.final_layer_norm = nn.LayerNorm(self.embed_dim)
281 +
282 + def forward(
283 + self,
284 + hidden_states: torch.Tensor,
285 + kv_cache: torch.Tensor,
286 + attn_metadata: AttentionMetadata,
287 + ):
288 + residual = hidden_states
289 + hidden_states = self.self_attn_layer_norm(hidden_states)
290 + hidden_states = self.self_attn(
291 + hidden_states=hidden_states,
292 + kv_cache=kv_cache,
293 + attn_metadata=attn_metadata,
294 + )
295 + hidden_states = residual + hidden_states
296 + residual = hidden_states
297 + hidden_states = self.final_layer_norm(hidden_states)
298 + hidden_states = self.mlp(hidden_states)
299 + hidden_states = residual + hidden_states
300 +
301 + if hidden_states.isinf().any() or hidden_states.isnan().any():
302 + clamp_value = torch.finfo(hidden_states.dtype).max - 1000
303 + hidden_states = torch.clamp(hidden_states,
304 + min=-clamp_value,
305 + max=clamp_value)
306 +
307 + return hidden_states
308 +
309 +
310 +class WhisperDecoderLayer(nn.Module):
311 +
312 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
313 + super().__init__()
314 + config = vllm_config.model_config.hf_config
315 + cache_config = vllm_config.cache_config
316 + quant_config = vllm_config.quant_config
317 +
318 + self.self_attn = WhisperAttention(
319 + embed_dim=config.d_model,
320 + num_heads=config.decoder_attention_heads,
321 + attn_type=AttentionType.DECODER,
322 + cache_config=cache_config,
323 + quant_config=quant_config,
324 + prefix=f"{prefix}.self_attn",
325 + )
326 + self.self_attn_layer_norm = nn.LayerNorm(config.d_model)
327 + self.encoder_attn = WhisperCrossAttention(
328 + embed_dim=config.d_model,
329 + num_heads=config.decoder_attention_heads,
330 + cache_config=cache_config,
331 + quant_config=quant_config,
332 + prefix=f"{prefix}.encoder_attn",
333 + )
334 + self.encoder_attn_layer_norm = nn.LayerNorm(config.d_model)
335 + self.mlp = WhisperMLP(
336 + embed_dim=config.d_model,
337 + ffn_dim=config.decoder_ffn_dim,
338 + act_fn=config.activation_function,
339 + quant_config=quant_config,
340 + prefix=f"{prefix}.mlp",
341 + )
342 + self.final_layer_norm = nn.LayerNorm(config.d_model)
343 +
344 + def forward(
345 + self,
346 + hidden_states: torch.Tensor,
347 + encoder_hidden_states: Optional[torch.Tensor],
348 + kv_cache: torch.Tensor,
349 + attn_metadata: AttentionMetadata,
350 + ):
351 + residual = hidden_states
352 + hidden_states = self.self_attn_layer_norm(hidden_states)
353 + hidden_states = self.self_attn(hidden_states=hidden_states,
354 + kv_cache=kv_cache,
355 + attn_metadata=attn_metadata)
356 + hidden_states = residual + hidden_states
357 +
358 + residual = hidden_states
359 + hidden_states = self.encoder_attn_layer_norm(hidden_states)
360 + hidden_states = self.encoder_attn(
361 + hidden_states=hidden_states,
362 + encoder_hidden_states=encoder_hidden_states,
363 + kv_cache=kv_cache,
364 + attn_metadata=attn_metadata,
365 + )
366 + hidden_states = residual + hidden_states
367 +
368 + residual = hidden_states
369 + hidden_states = self.final_layer_norm(hidden_states)
370 + hidden_states = self.mlp(hidden_states)
371 + hidden_states = residual + hidden_states
372 +
373 + return hidden_states
374 +
375 +
376 +class WhisperEncoder(nn.Module):
377 +
378 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
379 + super().__init__()
380 + config = vllm_config.model_config.hf_config
381 + embed_dim = config.d_model
382 + self.num_mel_bins = config.num_mel_bins
383 + self.padding_idx = config.pad_token_id
384 + self.max_source_positions = config.max_source_positions
385 + self.embed_scale = (math.sqrt(embed_dim)
386 + if config.scale_embedding else 1.0)
387 +
388 + self.conv1 = nn.Conv1d(self.num_mel_bins,
389 + embed_dim,
390 + kernel_size=3,
391 + padding=1)
392 + self.conv2 = nn.Conv1d(embed_dim,
393 + embed_dim,
394 + kernel_size=3,
395 + stride=2,
396 + padding=1)
397 + self.embed_positions = nn.Embedding(self.max_source_positions,
398 + embed_dim)
399 + self.start_layer, self.end_layer, self.layers = make_layers(
400 + config.encoder_layers,
401 + lambda prefix: WhisperEncoderLayer(vllm_config=vllm_config,
402 + prefix=f"{prefix}.layers"),
403 + prefix=f"{prefix}.layers",
404 + )
405 + self.layer_norm = nn.LayerNorm(config.d_model)
406 +
407 + with torch.no_grad():
408 + self.embed_positions.weight.copy_(
409 + sinusoids(*self.embed_positions.weight.shape))
410 +
411 + def forward(
412 + self,
413 + input_features: Union[torch.Tensor, List[torch.Tensor]],
414 + kv_caches: List[torch.Tensor],
415 + attn_metadata: AttentionMetadata,
416 + ):
417 + hidden_states = []
418 + for features in input_features:
419 + embeds = nn.functional.gelu(self.conv1(features))
420 + embeds = nn.functional.gelu(self.conv2(embeds))
421 + embeds = embeds.permute(1, 0)
422 + embeds = embeds + self.embed_positions.weight[:embeds.size(0), :]
423 + hidden_states.append(embeds)
424 + hidden_states = torch.cat(hidden_states)
425 +
426 + for idx, encoder_layer in enumerate(self.layers):
427 + hidden_states = encoder_layer(
428 + hidden_states,
429 + kv_cache=kv_caches[idx],
430 + attn_metadata=attn_metadata,
431 + )
432 +
433 + hidden_states = self.layer_norm(hidden_states)
434 + return hidden_states
435 +
436 +
437 +class WhisperDecoder(nn.Module):
438 +
439 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
440 + super().__init__()
441 + config = vllm_config.model_config.hf_config
442 + self.layerdrop = config.decoder_layerdrop
443 + self.padding_idx = config.pad_token_id
444 + self.max_target_positions = config.max_target_positions
445 + self.max_source_positions = config.max_source_positions
446 + self.embed_scale = (math.sqrt(config.d_model)
447 + if config.scale_embedding else 1.0)
448 +
449 + self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model,
450 + self.padding_idx)
451 + self.embed_positions = WhisperPositionalEmbedding(
452 + self.max_target_positions, config.d_model)
453 + self.start_layer, self.end_layer, self.layers = make_layers(
454 + config.decoder_layers,
455 + lambda prefix: WhisperDecoderLayer(vllm_config=vllm_config,
456 + prefix=f"{prefix}.layers"),
457 + prefix=f"{prefix}.layers",
458 + )
459 + self.layer_norm = nn.LayerNorm(config.d_model)
460 +
461 + def forward(
462 + self,
463 + input_ids,
464 + positions: torch.Tensor,
465 + encoder_hidden_states: Optional[torch.Tensor],
466 + kv_caches: List[torch.Tensor],
467 + attn_metadata: AttentionMetadata,
468 + ):
469 + inputs_embeds = self.get_input_embeddings(input_ids)
470 + positions = self.embed_positions(positions)
471 + hidden_states = inputs_embeds + positions
472 +
473 + for idx, decoder_layer in enumerate(self.layers):
474 + hidden_states = decoder_layer(
475 + hidden_states,
476 + encoder_hidden_states=encoder_hidden_states,
477 + kv_cache=kv_caches[idx],
478 + attn_metadata=attn_metadata,
479 + )
480 +
481 + hidden_states = self.layer_norm(hidden_states)
482 + return hidden_states
483 +
484 + def get_input_embeddings(
485 + self,
486 + input_ids: torch.Tensor,
487 + ) -> torch.Tensor:
488 + return self.embed_tokens(input_ids)
489 +
490 +
491 +class WhisperModel(nn.Module):
492 +
493 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
494 + super().__init__()
495 + self.encoder = WhisperEncoder(vllm_config=vllm_config,
496 + prefix=f"{prefix}.encoder")
497 + self.decoder = WhisperDecoder(vllm_config=vllm_config,
498 + prefix=f"{prefix}.decoder")
499 +
500 + def forward(
501 + self,
502 + input_features: Optional[Union[torch.Tensor, List[torch.Tensor]]],
503 + input_ids: Optional[torch.Tensor],
504 + positions: torch.Tensor,
505 + kv_caches: List[torch.Tensor],
506 + attn_metadata: AttentionMetadata,
507 + ) -> torch.Tensor:
508 + encoder_outputs = self.get_encoder_outputs(
509 + input_features,
510 + kv_caches=kv_caches,
511 + attn_metadata=attn_metadata,
512 + )
513 + decoder_outputs = self.decoder(
514 + input_ids=input_ids,
515 + positions=positions,
516 + encoder_hidden_states=encoder_outputs,
517 + kv_caches=kv_caches,
518 + attn_metadata=attn_metadata,
519 + )
520 + return decoder_outputs
521 +
522 + def get_encoder_outputs(
523 + self,
524 + input_features: Optional[Union[torch.Tensor, List[torch.Tensor]]],
525 + kv_caches: List[torch.Tensor],
526 + attn_metadata: AttentionMetadata,
527 + ) -> Optional[torch.Tensor]:
528 + if input_features is None:
529 + return None
530 + return self.encoder(
531 + input_features,
532 + kv_caches=kv_caches,
533 + attn_metadata=attn_metadata,
534 + )
535 +
536 + def load_weights(self, weights: Iterable[Tuple[str,
537 + torch.Tensor]]) -> Set[str]:
538 + stacked_params_mapping = [
539 + # (param_name, shard_name, shard_id)
540 + (".self_attn.qkv_proj", ".self_attn.q_proj", "q"),
541 + (".self_attn.qkv_proj", ".self_attn.k_proj", "k"),
542 + (".self_attn.qkv_proj", ".self_attn.v_proj", "v"),
543 + (".encoder_attn.kv_proj", ".encoder_attn.k_proj", "k"),
544 + (".encoder_attn.kv_proj", ".encoder_attn.v_proj", "v"),
545 + ]
546 + params_dict = dict(self.named_parameters())
547 + loaded_params: Set[str] = set()
548 + for name, loaded_weight in weights:
549 + for param_name, weight_name, shard_id in stacked_params_mapping:
550 + if weight_name not in name:
551 + continue
552 + name = name.replace(weight_name, param_name)
553 + # Skip loading extra bias for GPTQ models.
554 + if name.endswith(".bias") and name not in params_dict:
555 + continue
556 +
557 + param = params_dict[name]
558 + weight_loader = param.weight_loader
559 + weight_loader(param, loaded_weight, shard_id)
560 + break
561 + else:
562 + # Skip loading extra bias for GPTQ models.
563 + if name.endswith(".bias") and name not in params_dict:
564 + continue
565 +
566 + param = params_dict[name]
567 + weight_loader = getattr(param, "weight_loader",
568 + default_weight_loader)
569 + weight_loader(param, loaded_weight)
570 + loaded_params.add(name)
571 + return loaded_params
572 +
573 +
574 +def get_max_whisper_audio_tokens(ctx: InputContext) -> int:
575 + return ctx.model_config.hf_config.max_source_positions
576 +
577 +
578 +def dummy_encoder_data_for_whisper(ctx: InputContext, seq_len: int,
579 + mm_counts: Mapping[str, int]):
580 + assert mm_counts["audio"] == 1
581 + num_tokens = get_max_whisper_audio_tokens(ctx)
582 + processor = cached_get_processor(ctx.model_config.model)
583 + chunk_length = processor.feature_extractor.chunk_length
584 + sampling_rate = processor.feature_extractor.sampling_rate
585 + num_samples = chunk_length * sampling_rate
586 + return DummyData(
587 + SequenceData.from_prompt_token_counts((0, num_tokens)),
588 + {"audio": [(np.zeros(num_samples), sampling_rate)]},
589 + )
590 +
591 +
592 +def input_processor_for_whisper(ctx: InputContext, inputs):
593 + multi_modal_data = inputs["encoder"]["multi_modal_data"]
594 + if isinstance(multi_modal_data["audio"], list):
595 + assert len(multi_modal_data["audio"]) == 1
596 + multi_modal_data["audio"] = multi_modal_data["audio"][0]
597 + # Resample and process audio
598 + audio, orig_sr = multi_modal_data["audio"]
599 + processor = cached_get_processor(ctx.model_config.model)
600 + target_sr = processor.feature_extractor.sampling_rate
601 + audio = resample_audio(audio, orig_sr=orig_sr, target_sr=target_sr)
602 + multi_modal_data["audio"] = (audio, target_sr)
603 + # Pre-allocate placeholder tokens in encoder sequence
604 + num_tokens = get_max_whisper_audio_tokens(ctx)
605 + inputs["encoder"]["prompt_token_ids"] = [0] * num_tokens
606 + return inputs
607 +
608 +
609 +def input_mapper_for_whisper(
610 + ctx: InputContext,
611 + multi_modal_data: Union[np.ndarray, List[np.ndarray]],
612 +) -> MultiModalKwargs:
613 + if not isinstance(multi_modal_data, list):
614 + multi_modal_data = [multi_modal_data]
615 +
616 + assert len(multi_modal_data) == 1
617 +
618 + if len(multi_modal_data) == 0:
619 + return MultiModalKwargs()
620 +
621 + processor = cached_get_processor(ctx.model_config.model)
622 + sampling_rate = processor.feature_extractor.sampling_rate
623 +
624 + audios = [audio for audio, _ in multi_modal_data]
625 +
626 + kwargs = processor(audios,
627 + sampling_rate=sampling_rate,
628 + return_tensors="pt")
629 + kwargs["input_features"] = kwargs["input_features"].squeeze(0).to(
630 + ctx.model_config.dtype)
631 +
632 + return MultiModalKwargs(kwargs)
633 +
634 +
635 +@INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_whisper)
636 +@INPUT_REGISTRY.register_input_processor(input_processor_for_whisper)
637 +@MULTIMODAL_REGISTRY.register_input_mapper("audio", input_mapper_for_whisper)
638 +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
639 + "audio", get_max_whisper_audio_tokens)
640 +class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
641 +
642 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
643 + super().__init__()
644 + config = vllm_config.model_config.hf_config
645 + quant_config = vllm_config.quant_config
646 + self.config = config
647 + self.dtype = vllm_config.model_config.dtype
648 +
649 + self.model = WhisperModel(vllm_config=vllm_config, prefix=prefix)
650 + self.unpadded_vocab_size = config.vocab_size
651 + self.proj_out = ParallelLMHead(config.vocab_size,
652 + config.d_model,
653 + quant_config=quant_config)
654 + self.proj_out = self.proj_out.tie_weights(
655 + self.model.decoder.embed_tokens)
656 + logit_scale = getattr(config, "logit_scale", 1.0)
657 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
658 + config.vocab_size, logit_scale)
659 + self.sampler = Sampler()
660 +
661 + def forward(
662 + self,
663 + input_ids: torch.Tensor,
664 + positions: torch.Tensor,
665 + kv_caches: List[torch.Tensor],
666 + attn_metadata: AttentionMetadata,
667 + **kwargs,
668 + ) -> torch.Tensor:
669 + audio_input = self._parse_and_validate_audio_input(**kwargs)
670 + decoder_outputs = self.model(
671 + input_features=audio_input["input_features"],
672 + input_ids=input_ids,
673 + positions=positions,
674 + kv_caches=kv_caches,
675 + attn_metadata=attn_metadata,
676 + )
677 + return decoder_outputs
678 +
679 + def get_multimodal_embeddings(
680 + self,
681 + kv_caches: List[torch.Tensor],
682 + attn_metadata: AttentionMetadata,
683 + **kwargs,
684 + ) -> Optional[NestedTensors]:
685 + # TODO: This method does not obey the interface for SupportsMultiModal.
686 + # Refactor this once encoder/decoder support is implemented in V1.
687 + audio_input = self._parse_and_validate_audio_input(**kwargs)
688 + return self.model.get_encoder_outputs(
689 + audio_input["input_features"],
690 + kv_caches=kv_caches,
691 + attn_metadata=attn_metadata,
692 + )
693 +
694 + def get_input_embeddings(
695 + self,
696 + input_ids: torch.Tensor,
697 + multimodal_embeddings: Optional[NestedTensors] = None,
698 + attn_metadata: Optional[AttentionMetadata] = None,
699 + ) -> torch.Tensor:
700 + # TODO: This method just returns the decoder sequence embeddings since
701 + # Whisper does not have encoder text tokens. Refactor this once
702 + # encoder/decoder support is implemented in V1.
703 + return self.model.decoder.get_input_embeddings(input_ids)
704 +
705 + def _parse_and_validate_audio_input(
706 + self, **kwargs: object) -> WhisperAudioInputs:
707 + input_features = kwargs.pop("input_features", None)
708 +
709 + if input_features is not None:
710 + if not isinstance(input_features, (torch.Tensor, list)):
711 + raise ValueError("Incorrect type of audio features. "
712 + f"Got type: {type(input_features)}")
713 + input_features = [feat.to(self.dtype) for feat in input_features]
714 +
715 + return WhisperAudioInputs(input_features=input_features)
716 +
717 + def compute_logits(self, hidden_states: torch.Tensor,
718 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
719 + logits = self.logits_processor(self.proj_out, hidden_states,
720 + sampling_metadata)
721 + return logits
722 +
723 + def sample(
724 + self,
725 + logits: torch.Tensor,
726 + sampling_metadata: SamplingMetadata,
727 + ) -> Optional[SamplerOutput]:
728 + next_tokens = self.sampler(logits, sampling_metadata)
729 + return next_tokens
730 +
731 + def load_weights(self, weights: Iterable[Tuple[str,
732 + torch.Tensor]]) -> Set[str]:
733 + loader = AutoWeightsLoader(self, skip_prefixes=["proj_out."])
734 + loaded_weights = [(name, loaded_weight)
735 + for name, loaded_weight in weights]
736 + mapper = WeightsMapper({".fc1.": ".mlp.fc1.", ".fc2.": ".mlp.fc2."})
737 + return loader.load_weights(loaded_weights, mapper=mapper)
\ No newline at end of file
vllm/multimodal/processing.py
+7 -21
@@ -16,7 +16,7 @@ from transformers import BatchFeature, ProcessorMixin
16
17 from vllm.inputs import DummyData, InputProcessingContext
18 from vllm.logger import init_logger
19 -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
19 +from vllm.transformers_utils.tokenizer import AnyTokenizer, encode_tokens
20 from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
21
22 from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
@@ -57,24 +57,6 @@ class PromptReplacement:
57 )
58
59
60 -def _encode(
61 - tokenizer: AnyTokenizer,
62 - text: str,
63 - *,
64 - add_special_tokens: bool = False,
65 -) -> list[int]:
66 - """
67 - Backend-agnostic equivalent of HF's
68 - :code:`tokenizer.encode(text, add_special_tokens=...)`.
69 - """
70 - if isinstance(tokenizer, MistralTokenizer):
71 - return tokenizer.tokenizer.encode(text,
72 - bos=add_special_tokens,
73 - eos=add_special_tokens)
74 -
75 - return tokenizer.encode(text, add_special_tokens=add_special_tokens)
76 -
77 -
60 @lru_cache(maxsize=2048)
61 def _cached_encode(
62 tokenizer: AnyTokenizer,
@@ -82,7 +64,9 @@ def _cached_encode(
64 *,
65 add_special_tokens: bool = False,
66 ) -> list[int]:
85 - return _encode(tokenizer, text, add_special_tokens=add_special_tokens)
67 + return encode_tokens(tokenizer,
68 + text,
69 + add_special_tokens=add_special_tokens)
70
71
72 def _decode(
@@ -983,7 +967,9 @@ class BaseMultiModalProcessor(ABC):
967 mm_item_counts,
968 )
969
986 - token_ids = _encode(tokenizer, text)
970 + token_ids = encode_tokens(tokenizer,
971 + text,
972 + add_special_tokens=False)
973 matched_repls = [match.prompt_repl for match in text_matches]
974
975 placeholders = self._find_placeholders(matched_repls, token_ids,
vllm/sequence.py
+15 -3
@@ -710,15 +710,27 @@ class SequenceGroup:
710
711 @property
712 def multi_modal_data(self) -> MultiModalDataDict:
713 - return self.first_seq.multi_modal_data
713 + if self.first_seq.multi_modal_data:
714 + return self.first_seq.multi_modal_data
715 + elif self.encoder_seq is not None:
716 + return self.encoder_seq.multi_modal_data
717 + return {}
718
719 @property
720 def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
717 - return self.first_seq.multi_modal_placeholders
721 + if self.first_seq.multi_modal_data:
722 + return self.first_seq.multi_modal_placeholders
723 + elif self.encoder_seq is not None:
724 + return self.encoder_seq.multi_modal_placeholders
725 + return {}
726
727 @property
728 def mm_processor_kwargs(self) -> Dict[str, Any]:
721 - return self.first_seq.mm_processor_kwargs
729 + if self.first_seq.multi_modal_data:
730 + return self.first_seq.mm_processor_kwargs
731 + elif self.encoder_seq is not None:
732 + return self.encoder_seq.mm_processor_kwargs
733 + return {}
734
735 @property
736 def lora_int_id(self) -> int:
vllm/transformers_utils/tokenizer.py
+19
@@ -21,6 +21,25 @@ AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast,
21 MistralTokenizer]
22
23
24 +def encode_tokens(
25 + tokenizer: AnyTokenizer,
26 + text: str,
27 + *,
28 + add_special_tokens: Optional[bool] = None,
29 +) -> list[int]:
30 + """
31 + Backend-agnostic equivalent of HF's
32 + :code:`tokenizer.encode(text, add_special_tokens=...)`.
33 + """
34 + if isinstance(tokenizer, MistralTokenizer):
35 + return tokenizer.tokenizer.encode(text,
36 + bos=add_special_tokens,
37 + eos=add_special_tokens)
38 + elif add_special_tokens is not None:
39 + return tokenizer.encode(text, add_special_tokens=add_special_tokens)
40 + return tokenizer.encode(text)
41 +
42 +
43 def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
44 """Get tokenizer with cached properties.
45
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+4 -2
@@ -32,7 +32,8 @@ class BaseTokenizerGroup(ABC):
32 def encode(self,
33 prompt: str,
34 request_id: Optional[str] = None,
35 - lora_request: Optional[LoRARequest] = None) -> List[int]:
35 + lora_request: Optional[LoRARequest] = None,
36 + add_special_tokens: Optional[bool] = None) -> List[int]:
37 """Encode a prompt using the tokenizer group."""
38 pass
39
@@ -41,7 +42,8 @@ class BaseTokenizerGroup(ABC):
42 self,
43 prompt: str,
44 request_id: Optional[str] = None,
44 - lora_request: Optional[LoRARequest] = None) -> List[int]:
45 + lora_request: Optional[LoRARequest] = None,
46 + add_special_tokens: Optional[bool] = None) -> List[int]:
47 """Encode a prompt using the tokenizer group."""
48 pass
49
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+18 -10
@@ -112,7 +112,8 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
112 def encode(self,
113 prompt: str,
114 request_id: Optional[str] = None,
115 - lora_request: Optional[LoRARequest] = None) -> List[int]:
115 + lora_request: Optional[LoRARequest] = None,
116 + add_special_tokens: Optional[bool] = None) -> List[int]:
117 """Encode a prompt using the tokenizer group.
118
119 We pick an idle actor and use it to encode the prompt.
@@ -132,7 +133,8 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
133 ret = ray.get(
134 actor.encode.remote(request_id=request_id,
135 prompt=prompt,
135 - lora_request=lora_request))
136 + lora_request=lora_request,
137 + add_special_tokens=add_special_tokens))
138 except ActorDiedError as e:
139 # If the actor is dead, we first try to reinitialize it.
140 logger.warning("%s died with ActorDiedError, reinitializing.",
@@ -143,7 +145,8 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
145 ret = ray.get(
146 actor.encode.remote(request_id=request_id,
147 prompt=prompt,
146 - lora_request=lora_request))
148 + lora_request=lora_request,
149 + add_special_tokens=add_special_tokens))
150 except ActorDiedError as e:
151 logger.error(
152 "%s died for second time in a row, marking "
@@ -160,7 +163,8 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
163 self,
164 prompt: str,
165 request_id: Optional[str] = None,
163 - lora_request: Optional[LoRARequest] = None) -> List[int]:
166 + lora_request: Optional[LoRARequest] = None,
167 + add_special_tokens: Optional[bool] = None) -> List[int]:
168 """Encode a prompt using the tokenizer group.
169
170 We pick an idle actor and use it to encode the prompt.
@@ -177,9 +181,11 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
181 actor_is_alive = True
182 original_actor = actor
183 try:
180 - ret = await actor.encode.remote(request_id=request_id,
181 - prompt=prompt,
182 - lora_request=lora_request)
184 + ret = await actor.encode.remote(
185 + request_id=request_id,
186 + prompt=prompt,
187 + lora_request=lora_request,
188 + add_special_tokens=add_special_tokens)
189 except ActorDiedError as e:
190 # If the actor is dead, we first try to reinitialize it.
191 logger.warning("%s died with ActorDiedError, reinitializing.",
@@ -187,9 +193,11 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
193 exc_info=e)
194 actor = self._init_actor()
195 try:
190 - ret = await actor.encode.remote(request_id=request_id,
191 - prompt=prompt,
192 - lora_request=lora_request)
196 + ret = await actor.encode.remote(
197 + request_id=request_id,
198 + prompt=prompt,
199 + lora_request=lora_request,
200 + add_special_tokens=add_special_tokens)
201 except ActorDiedError as e:
202 logger.error(
203 "%s died for second time in a row, marking "
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+11 -5
@@ -2,7 +2,7 @@ from typing import List, Optional
2
3 from vllm.config import TokenizerPoolConfig
4 from vllm.lora.request import LoRARequest
5 -from vllm.transformers_utils.tokenizer import (AnyTokenizer,
5 +from vllm.transformers_utils.tokenizer import (AnyTokenizer, encode_tokens,
6 get_lora_tokenizer,
7 get_lora_tokenizer_async,
8 get_tokenizer)
@@ -55,9 +55,12 @@ class TokenizerGroup(BaseTokenizerGroup):
55 def encode(self,
56 prompt: str,
57 request_id: Optional[str] = None,
58 - lora_request: Optional[LoRARequest] = None) -> List[int]:
58 + lora_request: Optional[LoRARequest] = None,
59 + add_special_tokens: Optional[bool] = None) -> List[int]:
60 tokenizer = self.get_lora_tokenizer(lora_request)
60 - ret = tokenizer.encode(prompt)
61 + ret = encode_tokens(tokenizer,
62 + prompt,
63 + add_special_tokens=add_special_tokens)
64 self._raise_if_input_too_long(ret, lora_request)
65 return ret
66
@@ -65,9 +68,12 @@ class TokenizerGroup(BaseTokenizerGroup):
68 self,
69 prompt: str,
70 request_id: Optional[str] = None,
68 - lora_request: Optional[LoRARequest] = None) -> List[int]:
71 + lora_request: Optional[LoRARequest] = None,
72 + add_special_tokens: Optional[bool] = None) -> List[int]:
73 tokenizer = await self.get_lora_tokenizer_async(lora_request)
70 - ret = tokenizer.encode(prompt)
74 + ret = encode_tokens(tokenizer,
75 + prompt,
76 + add_special_tokens=add_special_tokens)
77 self._raise_if_input_too_long(ret, lora_request)
78 return ret
79
vllm/worker/enc_dec_model_runner.py
+5 -6
@@ -287,12 +287,11 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
287 seq_len,
288 self.mm_registry,
289 is_encoder_data=False)
290 - encoder_dummy_data \
291 - = self.input_registry.dummy_data_for_profiling(
292 - self.model_config,
293 - seq_len,
294 - self.mm_registry,
295 - is_encoder_data=True)
290 + encoder_dummy_data = self.input_registry \
291 + .dummy_data_for_profiling(self.model_config,
292 + seq_len,
293 + self.mm_registry,
294 + is_encoder_data=True)
295
296 # Having more tokens is over-conservative but otherwise fine
297 assert len(