A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Model] Whisper model implementation (#11280)
Co-authored-by: Aurick Qiao <aurick.qiao@snowflake.com>
Aurick Qiao committed
Jan 3, 2025 at 03:39 UTC
e1a5c2f0a123835558b1b1c9895181161527c55e
16 files changed
+1045
-55
.buildkite/test-pipeline.yaml
+2
@@ -363,12 +363,14 @@ steps:
363
- tests/models/decoder_only/audio_language
364
- tests/models/decoder_only/vision_language
365
- tests/models/embedding/vision_language
366
+ - tests/models/encoder_decoder/audio_language
367
- tests/models/encoder_decoder/vision_language
368
commands:
369
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
370
- pytest -v -s models/decoder_only/audio_language -m 'core_model or quant_model'
371
- pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'core_model or quant_model'
372
- pytest -v -s models/embedding/vision_language -m core_model
373
+ - pytest -v -s models/encoder_decoder/audio_language -m core_model
374
- pytest -v -s models/encoder_decoder/language -m core_model
375
- pytest -v -s models/encoder_decoder/vision_language -m core_model
376
examples/offline_inference_whisper.py
new
+59
@@ -0,0 +1,59 @@
1
+import time
2
+
3
+from vllm import LLM, SamplingParams
4
+from vllm.assets.audio import AudioAsset
5
+
6
+# Create a Whisper encoder/decoder model instance
7
+llm = LLM(
8
+ model="openai/whisper-large-v3",
9
+ max_model_len=448,
10
+ max_num_seqs=400,
11
+ limit_mm_per_prompt={"audio": 1},
12
+ kv_cache_dtype="fp8",
13
+)
14
+
15
+prompts = [
16
+ {
17
+ "prompt": "<|startoftranscript|>",
18
+ "multi_modal_data": {
19
+ "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
20
+ },
21
+ },
22
+ { # Test explicit encoder/decoder prompt
23
+ "encoder_prompt": {
24
+ "prompt": "",
25
+ "multi_modal_data": {
26
+ "audio": AudioAsset("winning_call").audio_and_sample_rate,
27
+ },
28
+ },
29
+ "decoder_prompt": "<|startoftranscript|>",
30
+ }
31
+] * 1024
32
+
33
+# Create a sampling params object.
34
+sampling_params = SamplingParams(
35
+ temperature=0,
36
+ top_p=1.0,
37
+ max_tokens=200,
38
+)
39
+
40
+start = time.time()
41
+
42
+# Generate output tokens from the prompts. The output is a list of
43
+# RequestOutput objects that contain the prompt, generated
44
+# text, and other information.
45
+outputs = llm.generate(prompts, sampling_params)
46
+
47
+# Print the outputs.
48
+for output in outputs:
49
+ prompt = output.prompt
50
+ encoder_prompt = output.encoder_prompt
51
+ generated_text = output.outputs[0].text
52
+ print(f"Encoder prompt: {encoder_prompt!r}, "
53
+ f"Decoder prompt: {prompt!r}, "
54
+ f"Generated text: {generated_text!r}")
55
+
56
+duration = time.time() - start
57
+
58
+print("Duration:", duration)
59
+print("RPS:", len(prompts) / duration)
tests/models/encoder_decoder/audio_language/__init__.py
tests/models/encoder_decoder/audio_language/test_whisper.py
new
+136
@@ -0,0 +1,136 @@
1
+"""Compare the outputs of HF and vLLM for Whisper models using greedy sampling.
2
+
3
+Run `pytest tests/models/encoder_decoder/audio/test_whisper.py`.
4
+"""
5
+from typing import Optional
6
+
7
+import pytest
8
+
9
+from vllm import LLM, SamplingParams
10
+from vllm.assets.audio import AudioAsset
11
+
12
+from ....utils import fork_new_process_for_each_test, multi_gpu_test
13
+
14
+PROMPTS = [
15
+ {
16
+ "prompt":
17
+ "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
18
+ "multi_modal_data": {
19
+ "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
20
+ },
21
+ },
22
+ { # Test explicit encoder/decoder prompt
23
+ "encoder_prompt": {
24
+ "prompt": "",
25
+ "multi_modal_data": {
26
+ "audio": AudioAsset("winning_call").audio_and_sample_rate,
27
+ },
28
+ },
29
+ "decoder_prompt":
30
+ "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
31
+ }
32
+]
33
+
34
+EXPECTED = {
35
+ "openai/whisper-tiny": [
36
+ " He has birth words I spoke in the original corner of that. And a"
37
+ " little piece of black coat poetry. Mary had a little sandwich,"
38
+ " sweet, with white and snow. And everyone had it very went the last"
39
+ " would sure to go.",
40
+ " >> And the old one, fit John the way to Edgar Martinez. >> One more"
41
+ " to line down the field line for our base camp. Here comes joy. Here"
42
+ " is June and the third base. They're going to wave him in. The throw"
43
+ " to the plate will be late. The Mariners are going to play for the"
44
+ " American League Championship. I don't believe it. It just continues"
45
+ " by all five."
46
+ ],
47
+ "openai/whisper-small": [
48
+ " The first words I spoke in the original pornograph. A little piece"
49
+ " of practical poetry. Mary had a little lamb, its fleece was quite a"
50
+ " slow, and everywhere that Mary went the lamb was sure to go.",
51
+ " And the old one pitch on the way to Edgar Martinez one month. Here"
52
+ " comes joy. Here is Junior to third base. They're gonna wave him"
53
+ " in. The throw to the plate will be late. The Mariners are going to"
54
+ " play for the American League Championship. I don't believe it. It"
55
+ " just continues. My, oh my."
56
+ ],
57
+ "openai/whisper-medium": [
58
+ " The first words I spoke in the original phonograph, a little piece"
59
+ " of practical poetry. Mary had a little lamb, its fleece was quite as"
60
+ " slow, and everywhere that Mary went the lamb was sure to go.",
61
+ " And the 0-1 pitch on the way to Edgar Martinez swung on the line"
62
+ " down the left field line for Obeyshev. Here comes Joy. Here is"
63
+ " Jorgen at third base. They're going to wave him in. The throw to the"
64
+ " plate will be late. The Mariners are going to play for the American"
65
+ " League Championship. I don't believe it. It just continues. My, oh"
66
+ " my."
67
+ ],
68
+ "openai/whisper-large-v3": [
69
+ " The first words I spoke in the original phonograph, a little piece"
70
+ " of practical poetry. Mary had a little lamb, its feet were quite as"
71
+ " slow, and everywhere that Mary went, the lamb was sure to go.",
72
+ " And the 0-1 pitch on the way to Edgar Martinez. Swung on the line."
73
+ " Now the left field line for a base hit. Here comes Joy. Here is"
74
+ " Junior to third base. They're going to wave him in. The throw to the"
75
+ " plate will be late. The Mariners are going to play for the American"
76
+ " League Championship. I don't believe it. It just continues. My, oh,"
77
+ " my."
78
+ ],
79
+ "openai/whisper-large-v3-turbo": [
80
+ " The first words I spoke in the original phonograph, a little piece"
81
+ " of practical poetry. Mary had a little lamb, its streets were quite"
82
+ " as slow, and everywhere that Mary went the lamb was sure to go.",
83
+ " And the 0-1 pitch on the way to Edgar Martinez. Swung on the line"
84
+ " down the left field line for a base hit. Here comes Joy. Here is"
85
+ " Junior to third base. They're going to wave him in. The throw to the"
86
+ " plate will be late. The Mariners are going to play for the American"
87
+ " League Championship. I don't believe it. It just continues. My, oh,"
88
+ " my."
89
+ ]
90
+}
91
+
92
+
93
+def run_test(
94
+ model: str,
95
+ *,
96
+ tensor_parallel_size: int,
97
+ distributed_executor_backend: Optional[str] = None,
98
+) -> None:
99
+ prompt_list = PROMPTS * 10
100
+ expected_list = EXPECTED[model] * 10
101
+
102
+ llm = LLM(
103
+ model=model,
104
+ tensor_parallel_size=tensor_parallel_size,
105
+ distributed_executor_backend=distributed_executor_backend,
106
+ )
107
+
108
+ sampling_params = SamplingParams(
109
+ temperature=0,
110
+ top_p=1.0,
111
+ max_tokens=200,
112
+ )
113
+
114
+ outputs = llm.generate(prompt_list, sampling_params)
115
+
116
+ for output, expected in zip(outputs, expected_list):
117
+ print(output.outputs[0].text)
118
+ assert output.outputs[0].text == expected
119
+
120
+
121
+@fork_new_process_for_each_test
122
+@pytest.mark.core_model
123
+@pytest.mark.parametrize(
124
+ "model", ["openai/whisper-small", "openai/whisper-large-v3-turbo"])
125
+def test_models(model) -> None:
126
+ run_test(model, tensor_parallel_size=1)
127
+
128
+
129
+@multi_gpu_test(num_gpus=2)
130
+@pytest.mark.core_model
131
+@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
132
+@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
133
+def test_models_distributed(model, distributed_executor_backend) -> None:
134
+ run_test(model,
135
+ tensor_parallel_size=2,
136
+ distributed_executor_backend=distributed_executor_backend)
tests/models/registry.py
+1
@@ -204,6 +204,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
204
"UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3"),
205
# [Encoder-decoder]
206
"MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
207
+ "WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501
208
}
209
210
_SPECULATIVE_DECODING_EXAMPLE_MODELS = {
vllm/config.py
+2
@@ -2312,6 +2312,8 @@ def _get_and_verify_max_len(
2312
"seq_length",
2313
# Command-R
2314
"model_max_length",
2315
+ # Whisper
2316
+ "max_target_positions",
2317
# Others
2318
"max_sequence_length",
2319
"max_seq_length",
vllm/inputs/preprocess.py
+28
-8
@@ -184,10 +184,16 @@ class InputPreprocessor:
184
corresponding token IDs.
185
"""
186
tokenizer = self.get_tokenizer_group()
187
-
187
+ add_special_tokens = None
188
+ if self.model_config.hf_config.model_type == "whisper":
189
+ # For Whisper, special tokens should be provided by the user based
190
+ # on the task and language of their request. Also needed to avoid
191
+ # appending an EOS token to the prompt which disrupts generation.
192
+ add_special_tokens = False
193
return tokenizer.encode(request_id=request_id,
194
prompt=prompt,
190
- lora_request=lora_request)
195
+ lora_request=lora_request,
196
+ add_special_tokens=add_special_tokens)
197
198
async def _tokenize_prompt_async(
199
self,
@@ -197,10 +203,17 @@ class InputPreprocessor:
203
) -> List[int]:
204
"""Async version of :meth:`_tokenize_prompt`."""
205
tokenizer = self.get_tokenizer_group()
200
-
201
- return await tokenizer.encode_async(request_id=request_id,
202
- prompt=prompt,
203
- lora_request=lora_request)
206
+ add_special_tokens = None
207
+ if self.model_config.hf_config.model_type == "whisper":
208
+ # For Whisper, special tokens should be provided by the user based
209
+ # on the task and language of their request. Also needed to avoid
210
+ # appending an EOS token to the prompt which disrupts generation.
211
+ add_special_tokens = False
212
+ return await tokenizer.encode_async(
213
+ request_id=request_id,
214
+ prompt=prompt,
215
+ lora_request=lora_request,
216
+ add_special_tokens=add_special_tokens)
217
218
def _can_process_multimodal(self) -> bool:
219
model_config = self.model_config
@@ -439,8 +452,15 @@ class InputPreprocessor:
452
assert_never(encoder_inputs) # type: ignore[arg-type]
453
454
if decoder_inputs is None:
442
- dec_token_ids = self._prepare_decoder_input_ids_for_generation(
443
- None)
455
+ if self.model_config.hf_config.model_type == "whisper":
456
+ # For Whisper models, the text prompt should go to the decoder.
457
+ # If no explicit encoder/decoder inputs, then copy the prompt
458
+ # from the encoder to the decoder. The encoder tokens are later
459
+ # overridden by the audio features.
460
+ dec_token_ids = encoder_inputs["prompt_token_ids"].copy()
461
+ else:
462
+ dec_token_ids = self._prepare_decoder_input_ids_for_generation(
463
+ None)
464
decoder_inputs = token_inputs(dec_token_ids)
465
elif (decoder_inputs["type"] == "token"
466
or decoder_inputs["type"] == "multimodal"):
vllm/model_executor/models/registry.py
+1
@@ -170,6 +170,7 @@ _MULTIMODAL_MODELS = {
170
"UltravoxModel": ("ultravox", "UltravoxModel"),
171
# [Encoder-decoder]
172
"MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501
173
+ "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"), # noqa: E501
174
}
175
176
_SPECULATIVE_DECODING_MODELS = {
vllm/model_executor/models/whisper.py
new
+737
@@ -0,0 +1,737 @@
1
+import math
2
+from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
3
+ Union)
4
+
5
+import numpy as np
6
+import torch
7
+from torch import nn
8
+from transformers.models.whisper.modeling_whisper import sinusoids
9
+
10
+from vllm.attention import Attention, AttentionMetadata, AttentionType
11
+from vllm.config import CacheConfig, VllmConfig
12
+from vllm.distributed import get_tensor_model_parallel_world_size
13
+from vllm.inputs import INPUT_REGISTRY, DummyData, InputContext
14
+from vllm.logger import init_logger
15
+from vllm.model_executor.layers.activation import get_act_fn
16
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
+ QKVParallelLinear,
18
+ RowParallelLinear)
19
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
20
+from vllm.model_executor.layers.quantization.base_config import (
21
+ QuantizationConfig)
22
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
23
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
24
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
25
+from vllm.model_executor.sampling_metadata import SamplingMetadata
26
+from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
27
+ NestedTensors)
28
+from vllm.multimodal.audio import resample_audio
29
+from vllm.sequence import SequenceData
30
+from vllm.transformers_utils.processor import cached_get_processor
31
+
32
+from .interfaces import SupportsMultiModal
33
+from .utils import AutoWeightsLoader, WeightsMapper, make_layers
34
+
35
+logger = init_logger(__name__)
36
+
37
+
38
+class WhisperAudioInputs(TypedDict):
39
+ input_features: NestedTensors
40
+ """Shape: `(batch_size, 128, M)`"""
41
+
42
+
43
+class WhisperPositionalEmbedding(nn.Embedding):
44
+
45
+ def __init__(self,
46
+ num_positions: int,
47
+ embedding_dim: int,
48
+ padding_idx: Optional[int] = None):
49
+ super().__init__(num_positions, embedding_dim)
50
+
51
+ def forward(self, position_ids):
52
+ return self.weight[position_ids]
53
+
54
+
55
+class WhisperAttention(nn.Module):
56
+
57
+ def __init__(
58
+ self,
59
+ embed_dim: int,
60
+ num_heads: int,
61
+ bias: bool = True,
62
+ attn_type: AttentionType = AttentionType.DECODER,
63
+ cache_config: Optional[CacheConfig] = None,
64
+ quant_config: Optional[QuantizationConfig] = None,
65
+ prefix: str = "",
66
+ ):
67
+ super().__init__()
68
+ self.embed_dim = embed_dim
69
+ tp_size = get_tensor_model_parallel_world_size()
70
+ self.total_num_heads = num_heads
71
+ assert self.total_num_heads % tp_size == 0
72
+ self.num_heads = self.total_num_heads // tp_size
73
+ if self.total_num_heads >= tp_size:
74
+ # Number of heads is greater than TP size, so we partition
75
+ # the KV heads across multiple tensor parallel GPUs.
76
+ assert self.total_num_heads % tp_size == 0
77
+ else:
78
+ # Number of heads is less than TP size, so we replicate
79
+ # the KV heads across multiple tensor parallel GPUs.
80
+ assert tp_size % self.total_num_heads == 0
81
+ self.num_kv_heads = max(1, self.total_num_heads // tp_size)
82
+ self.head_dim = self.embed_dim // self.total_num_heads
83
+ self.q_size = self.num_heads * self.head_dim
84
+ self.kv_size = self.num_kv_heads * self.head_dim
85
+ self.attn_type = attn_type
86
+
87
+ if (self.head_dim * num_heads) != self.embed_dim:
88
+ raise ValueError(
89
+ f"embed_dim must be divisible by num_heads (got `embed_dim`: "
90
+ f"{self.embed_dim} and `num_heads`: {num_heads}).")
91
+ self.scaling = self.head_dim**-0.5
92
+
93
+ self._init_qkv(embed_dim, bias, quant_config, prefix=prefix)
94
+ self.out_proj = RowParallelLinear(
95
+ input_size=embed_dim,
96
+ output_size=embed_dim,
97
+ bias=bias,
98
+ quant_config=quant_config,
99
+ prefix=f"{prefix}.out_proj",
100
+ )
101
+ self.attn = Attention(
102
+ self.num_heads,
103
+ self.head_dim,
104
+ self.scaling,
105
+ num_kv_heads=self.num_kv_heads,
106
+ cache_config=cache_config,
107
+ quant_config=quant_config,
108
+ prefix=f"{prefix}.attn",
109
+ )
110
+
111
+ def _init_qkv(
112
+ self,
113
+ embed_dim: int,
114
+ bias: bool = True,
115
+ quant_config: Optional[QuantizationConfig] = None,
116
+ prefix: str = "",
117
+ ) -> None:
118
+ self.qkv_proj = QKVParallelLinear(
119
+ hidden_size=embed_dim,
120
+ head_size=self.head_dim,
121
+ total_num_heads=self.total_num_heads,
122
+ total_num_kv_heads=self.total_num_heads,
123
+ bias=bias,
124
+ quant_config=quant_config,
125
+ prefix=f"{prefix}.qkv_proj",
126
+ )
127
+
128
+ def forward(
129
+ self,
130
+ hidden_states: torch.Tensor,
131
+ kv_cache: torch.Tensor,
132
+ attn_metadata: AttentionMetadata,
133
+ ):
134
+ qkv, _ = self.qkv_proj(hidden_states)
135
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
136
+
137
+ attn_output = self.attn(q,
138
+ k,
139
+ v,
140
+ kv_cache,
141
+ attn_metadata,
142
+ attn_type=self.attn_type)
143
+
144
+ output, _ = self.out_proj(attn_output)
145
+
146
+ return output
147
+
148
+
149
+class WhisperCrossAttention(WhisperAttention):
150
+
151
+ def __init__(
152
+ self,
153
+ embed_dim: int,
154
+ num_heads: int,
155
+ bias: bool = True,
156
+ cache_config: Optional[CacheConfig] = None,
157
+ quant_config: Optional[QuantizationConfig] = None,
158
+ prefix: str = "",
159
+ ):
160
+ super().__init__(
161
+ embed_dim=embed_dim,
162
+ num_heads=num_heads,
163
+ bias=bias,
164
+ cache_config=cache_config,
165
+ quant_config=quant_config,
166
+ prefix=prefix,
167
+ )
168
+
169
+ def _init_qkv(
170
+ self,
171
+ embed_dim: int,
172
+ bias: bool = True,
173
+ quant_config: Optional[QuantizationConfig] = None,
174
+ prefix: str = "",
175
+ ) -> None:
176
+ self.q_proj = ColumnParallelLinear(
177
+ input_size=embed_dim,
178
+ output_size=embed_dim,
179
+ bias=bias,
180
+ quant_config=quant_config,
181
+ prefix=f"{prefix}.q_proj",
182
+ )
183
+ self.kv_proj = QKVParallelLinear(
184
+ hidden_size=embed_dim,
185
+ head_size=self.head_dim,
186
+ total_num_heads=0,
187
+ total_num_kv_heads=self.total_num_heads,
188
+ bias=bias,
189
+ quant_config=quant_config,
190
+ prefix=f"{prefix}.kv_proj",
191
+ )
192
+
193
+ def forward(
194
+ self,
195
+ hidden_states: torch.Tensor,
196
+ encoder_hidden_states: Optional[torch.Tensor],
197
+ kv_cache: torch.Tensor,
198
+ attn_metadata: AttentionMetadata,
199
+ ):
200
+ q, _ = self.q_proj(hidden_states)
201
+
202
+ # Encoder hidden states are only computed once during prefill phase.
203
+ # Afterwards, the keys and values should be available in the kv-cache.
204
+ if encoder_hidden_states is not None:
205
+ kv, _ = self.kv_proj(encoder_hidden_states)
206
+ k, v = kv.split([self.kv_size, self.kv_size], dim=-1)
207
+ else:
208
+ k = v = None
209
+
210
+ attn_output = self.attn(q,
211
+ k,
212
+ v,
213
+ kv_cache,
214
+ attn_metadata,
215
+ attn_type=AttentionType.ENCODER_DECODER)
216
+
217
+ output, _ = self.out_proj(attn_output)
218
+
219
+ return output
220
+
221
+
222
+class WhisperMLP(nn.Module):
223
+
224
+ def __init__(
225
+ self,
226
+ embed_dim: int,
227
+ ffn_dim: int,
228
+ act_fn: str,
229
+ quant_config: Optional[QuantizationConfig] = None,
230
+ prefix: str = "",
231
+ ):
232
+ super().__init__()
233
+
234
+ self.activation_fn = get_act_fn(act_fn)
235
+ self.fc1 = ColumnParallelLinear(
236
+ input_size=embed_dim,
237
+ output_size=ffn_dim,
238
+ quant_config=quant_config,
239
+ prefix=f"{prefix}.fc1",
240
+ )
241
+ self.fc2 = RowParallelLinear(
242
+ input_size=ffn_dim,
243
+ output_size=embed_dim,
244
+ quant_config=quant_config,
245
+ prefix=f"{prefix}.fc2",
246
+ )
247
+
248
+ def forward(self, hidden_states: torch.Tensor):
249
+ hidden_states, _ = self.fc1(hidden_states)
250
+ hidden_states = self.activation_fn(hidden_states)
251
+ hidden_states, _ = self.fc2(hidden_states)
252
+ return hidden_states
253
+
254
+
255
+class WhisperEncoderLayer(nn.Module):
256
+
257
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
258
+ super().__init__()
259
+ config = vllm_config.model_config.hf_config
260
+ cache_config = vllm_config.cache_config
261
+ quant_config = vllm_config.quant_config
262
+
263
+ self.embed_dim = config.d_model
264
+ self.self_attn = WhisperAttention(
265
+ embed_dim=self.embed_dim,
266
+ num_heads=config.encoder_attention_heads,
267
+ attn_type=AttentionType.ENCODER,
268
+ cache_config=cache_config,
269
+ quant_config=quant_config,
270
+ prefix=f"{prefix}.self_attn",
271
+ )
272
+ self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
273
+ self.mlp = WhisperMLP(
274
+ embed_dim=config.d_model,
275
+ ffn_dim=config.encoder_ffn_dim,
276
+ act_fn=config.activation_function,
277
+ quant_config=quant_config,
278
+ prefix=f"{prefix}.mlp",
279
+ )
280
+ self.final_layer_norm = nn.LayerNorm(self.embed_dim)
281
+
282
+ def forward(
283
+ self,
284
+ hidden_states: torch.Tensor,
285
+ kv_cache: torch.Tensor,
286
+ attn_metadata: AttentionMetadata,
287
+ ):
288
+ residual = hidden_states
289
+ hidden_states = self.self_attn_layer_norm(hidden_states)
290
+ hidden_states = self.self_attn(
291
+ hidden_states=hidden_states,
292
+ kv_cache=kv_cache,
293
+ attn_metadata=attn_metadata,
294
+ )
295
+ hidden_states = residual + hidden_states
296
+ residual = hidden_states
297
+ hidden_states = self.final_layer_norm(hidden_states)
298
+ hidden_states = self.mlp(hidden_states)
299
+ hidden_states = residual + hidden_states
300
+
301
+ if hidden_states.isinf().any() or hidden_states.isnan().any():
302
+ clamp_value = torch.finfo(hidden_states.dtype).max - 1000
303
+ hidden_states = torch.clamp(hidden_states,
304
+ min=-clamp_value,
305
+ max=clamp_value)
306
+
307
+ return hidden_states
308
+
309
+
310
+class WhisperDecoderLayer(nn.Module):
311
+
312
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
313
+ super().__init__()
314
+ config = vllm_config.model_config.hf_config
315
+ cache_config = vllm_config.cache_config
316
+ quant_config = vllm_config.quant_config
317
+
318
+ self.self_attn = WhisperAttention(
319
+ embed_dim=config.d_model,
320
+ num_heads=config.decoder_attention_heads,
321
+ attn_type=AttentionType.DECODER,
322
+ cache_config=cache_config,
323
+ quant_config=quant_config,
324
+ prefix=f"{prefix}.self_attn",
325
+ )
326
+ self.self_attn_layer_norm = nn.LayerNorm(config.d_model)
327
+ self.encoder_attn = WhisperCrossAttention(
328
+ embed_dim=config.d_model,
329
+ num_heads=config.decoder_attention_heads,
330
+ cache_config=cache_config,
331
+ quant_config=quant_config,
332
+ prefix=f"{prefix}.encoder_attn",
333
+ )
334
+ self.encoder_attn_layer_norm = nn.LayerNorm(config.d_model)
335
+ self.mlp = WhisperMLP(
336
+ embed_dim=config.d_model,
337
+ ffn_dim=config.decoder_ffn_dim,
338
+ act_fn=config.activation_function,
339
+ quant_config=quant_config,
340
+ prefix=f"{prefix}.mlp",
341
+ )
342
+ self.final_layer_norm = nn.LayerNorm(config.d_model)
343
+
344
+ def forward(
345
+ self,
346
+ hidden_states: torch.Tensor,
347
+ encoder_hidden_states: Optional[torch.Tensor],
348
+ kv_cache: torch.Tensor,
349
+ attn_metadata: AttentionMetadata,
350
+ ):
351
+ residual = hidden_states
352
+ hidden_states = self.self_attn_layer_norm(hidden_states)
353
+ hidden_states = self.self_attn(hidden_states=hidden_states,
354
+ kv_cache=kv_cache,
355
+ attn_metadata=attn_metadata)
356
+ hidden_states = residual + hidden_states
357
+
358
+ residual = hidden_states
359
+ hidden_states = self.encoder_attn_layer_norm(hidden_states)
360
+ hidden_states = self.encoder_attn(
361
+ hidden_states=hidden_states,
362
+ encoder_hidden_states=encoder_hidden_states,
363
+ kv_cache=kv_cache,
364
+ attn_metadata=attn_metadata,
365
+ )
366
+ hidden_states = residual + hidden_states
367
+
368
+ residual = hidden_states
369
+ hidden_states = self.final_layer_norm(hidden_states)
370
+ hidden_states = self.mlp(hidden_states)
371
+ hidden_states = residual + hidden_states
372
+
373
+ return hidden_states
374
+
375
+
376
+class WhisperEncoder(nn.Module):
377
+
378
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
379
+ super().__init__()
380
+ config = vllm_config.model_config.hf_config
381
+ embed_dim = config.d_model
382
+ self.num_mel_bins = config.num_mel_bins
383
+ self.padding_idx = config.pad_token_id
384
+ self.max_source_positions = config.max_source_positions
385
+ self.embed_scale = (math.sqrt(embed_dim)
386
+ if config.scale_embedding else 1.0)
387
+
388
+ self.conv1 = nn.Conv1d(self.num_mel_bins,
389
+ embed_dim,
390
+ kernel_size=3,
391
+ padding=1)
392
+ self.conv2 = nn.Conv1d(embed_dim,
393
+ embed_dim,
394
+ kernel_size=3,
395
+ stride=2,
396
+ padding=1)
397
+ self.embed_positions = nn.Embedding(self.max_source_positions,
398
+ embed_dim)
399
+ self.start_layer, self.end_layer, self.layers = make_layers(
400
+ config.encoder_layers,
401
+ lambda prefix: WhisperEncoderLayer(vllm_config=vllm_config,
402
+ prefix=f"{prefix}.layers"),
403
+ prefix=f"{prefix}.layers",
404
+ )
405
+ self.layer_norm = nn.LayerNorm(config.d_model)
406
+
407
+ with torch.no_grad():
408
+ self.embed_positions.weight.copy_(
409
+ sinusoids(*self.embed_positions.weight.shape))
410
+
411
+ def forward(
412
+ self,
413
+ input_features: Union[torch.Tensor, List[torch.Tensor]],
414
+ kv_caches: List[torch.Tensor],
415
+ attn_metadata: AttentionMetadata,
416
+ ):
417
+ hidden_states = []
418
+ for features in input_features:
419
+ embeds = nn.functional.gelu(self.conv1(features))
420
+ embeds = nn.functional.gelu(self.conv2(embeds))
421
+ embeds = embeds.permute(1, 0)
422
+ embeds = embeds + self.embed_positions.weight[:embeds.size(0), :]
423
+ hidden_states.append(embeds)
424
+ hidden_states = torch.cat(hidden_states)
425
+
426
+ for idx, encoder_layer in enumerate(self.layers):
427
+ hidden_states = encoder_layer(
428
+ hidden_states,
429
+ kv_cache=kv_caches[idx],
430
+ attn_metadata=attn_metadata,
431
+ )
432
+
433
+ hidden_states = self.layer_norm(hidden_states)
434
+ return hidden_states
435
+
436
+
437
+class WhisperDecoder(nn.Module):
438
+
439
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
440
+ super().__init__()
441
+ config = vllm_config.model_config.hf_config
442
+ self.layerdrop = config.decoder_layerdrop
443
+ self.padding_idx = config.pad_token_id
444
+ self.max_target_positions = config.max_target_positions
445
+ self.max_source_positions = config.max_source_positions
446
+ self.embed_scale = (math.sqrt(config.d_model)
447
+ if config.scale_embedding else 1.0)
448
+
449
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model,
450
+ self.padding_idx)
451
+ self.embed_positions = WhisperPositionalEmbedding(
452
+ self.max_target_positions, config.d_model)
453
+ self.start_layer, self.end_layer, self.layers = make_layers(
454
+ config.decoder_layers,
455
+ lambda prefix: WhisperDecoderLayer(vllm_config=vllm_config,
456
+ prefix=f"{prefix}.layers"),
457
+ prefix=f"{prefix}.layers",
458
+ )
459
+ self.layer_norm = nn.LayerNorm(config.d_model)
460
+
461
+ def forward(
462
+ self,
463
+ input_ids,
464
+ positions: torch.Tensor,
465
+ encoder_hidden_states: Optional[torch.Tensor],
466
+ kv_caches: List[torch.Tensor],
467
+ attn_metadata: AttentionMetadata,
468
+ ):
469
+ inputs_embeds = self.get_input_embeddings(input_ids)
470
+ positions = self.embed_positions(positions)
471
+ hidden_states = inputs_embeds + positions
472
+
473
+ for idx, decoder_layer in enumerate(self.layers):
474
+ hidden_states = decoder_layer(
475
+ hidden_states,
476
+ encoder_hidden_states=encoder_hidden_states,
477
+ kv_cache=kv_caches[idx],
478
+ attn_metadata=attn_metadata,
479
+ )
480
+
481
+ hidden_states = self.layer_norm(hidden_states)
482
+ return hidden_states
483
+
484
+ def get_input_embeddings(
485
+ self,
486
+ input_ids: torch.Tensor,
487
+ ) -> torch.Tensor:
488
+ return self.embed_tokens(input_ids)
489
+
490
+
491
+class WhisperModel(nn.Module):
492
+
493
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
494
+ super().__init__()
495
+ self.encoder = WhisperEncoder(vllm_config=vllm_config,
496
+ prefix=f"{prefix}.encoder")
497
+ self.decoder = WhisperDecoder(vllm_config=vllm_config,
498
+ prefix=f"{prefix}.decoder")
499
+
500
+ def forward(
501
+ self,
502
+ input_features: Optional[Union[torch.Tensor, List[torch.Tensor]]],
503
+ input_ids: Optional[torch.Tensor],
504
+ positions: torch.Tensor,
505
+ kv_caches: List[torch.Tensor],
506
+ attn_metadata: AttentionMetadata,
507
+ ) -> torch.Tensor:
508
+ encoder_outputs = self.get_encoder_outputs(
509
+ input_features,
510
+ kv_caches=kv_caches,
511
+ attn_metadata=attn_metadata,
512
+ )
513
+ decoder_outputs = self.decoder(
514
+ input_ids=input_ids,
515
+ positions=positions,
516
+ encoder_hidden_states=encoder_outputs,
517
+ kv_caches=kv_caches,
518
+ attn_metadata=attn_metadata,
519
+ )
520
+ return decoder_outputs
521
+
522
+ def get_encoder_outputs(
523
+ self,
524
+ input_features: Optional[Union[torch.Tensor, List[torch.Tensor]]],
525
+ kv_caches: List[torch.Tensor],
526
+ attn_metadata: AttentionMetadata,
527
+ ) -> Optional[torch.Tensor]:
528
+ if input_features is None:
529
+ return None
530
+ return self.encoder(
531
+ input_features,
532
+ kv_caches=kv_caches,
533
+ attn_metadata=attn_metadata,
534
+ )
535
+
536
+ def load_weights(self, weights: Iterable[Tuple[str,
537
+ torch.Tensor]]) -> Set[str]:
538
+ stacked_params_mapping = [
539
+ # (param_name, shard_name, shard_id)
540
+ (".self_attn.qkv_proj", ".self_attn.q_proj", "q"),
541
+ (".self_attn.qkv_proj", ".self_attn.k_proj", "k"),
542
+ (".self_attn.qkv_proj", ".self_attn.v_proj", "v"),
543
+ (".encoder_attn.kv_proj", ".encoder_attn.k_proj", "k"),
544
+ (".encoder_attn.kv_proj", ".encoder_attn.v_proj", "v"),
545
+ ]
546
+ params_dict = dict(self.named_parameters())
547
+ loaded_params: Set[str] = set()
548
+ for name, loaded_weight in weights:
549
+ for param_name, weight_name, shard_id in stacked_params_mapping:
550
+ if weight_name not in name:
551
+ continue
552
+ name = name.replace(weight_name, param_name)
553
+ # Skip loading extra bias for GPTQ models.
554
+ if name.endswith(".bias") and name not in params_dict:
555
+ continue
556
+
557
+ param = params_dict[name]
558
+ weight_loader = param.weight_loader
559
+ weight_loader(param, loaded_weight, shard_id)
560
+ break
561
+ else:
562
+ # Skip loading extra bias for GPTQ models.
563
+ if name.endswith(".bias") and name not in params_dict:
564
+ continue
565
+
566
+ param = params_dict[name]
567
+ weight_loader = getattr(param, "weight_loader",
568
+ default_weight_loader)
569
+ weight_loader(param, loaded_weight)
570
+ loaded_params.add(name)
571
+ return loaded_params
572
+
573
+
574
+def get_max_whisper_audio_tokens(ctx: InputContext) -> int:
575
+ return ctx.model_config.hf_config.max_source_positions
576
+
577
+
578
+def dummy_encoder_data_for_whisper(ctx: InputContext, seq_len: int,
579
+ mm_counts: Mapping[str, int]):
580
+ assert mm_counts["audio"] == 1
581
+ num_tokens = get_max_whisper_audio_tokens(ctx)
582
+ processor = cached_get_processor(ctx.model_config.model)
583
+ chunk_length = processor.feature_extractor.chunk_length
584
+ sampling_rate = processor.feature_extractor.sampling_rate
585
+ num_samples = chunk_length * sampling_rate
586
+ return DummyData(
587
+ SequenceData.from_prompt_token_counts((0, num_tokens)),
588
+ {"audio": [(np.zeros(num_samples), sampling_rate)]},
589
+ )
590
+
591
+
592
+def input_processor_for_whisper(ctx: InputContext, inputs):
593
+ multi_modal_data = inputs["encoder"]["multi_modal_data"]
594
+ if isinstance(multi_modal_data["audio"], list):
595
+ assert len(multi_modal_data["audio"]) == 1
596
+ multi_modal_data["audio"] = multi_modal_data["audio"][0]
597
+ # Resample and process audio
598
+ audio, orig_sr = multi_modal_data["audio"]
599
+ processor = cached_get_processor(ctx.model_config.model)
600
+ target_sr = processor.feature_extractor.sampling_rate
601
+ audio = resample_audio(audio, orig_sr=orig_sr, target_sr=target_sr)
602
+ multi_modal_data["audio"] = (audio, target_sr)
603
+ # Pre-allocate placeholder tokens in encoder sequence
604
+ num_tokens = get_max_whisper_audio_tokens(ctx)
605
+ inputs["encoder"]["prompt_token_ids"] = [0] * num_tokens
606
+ return inputs
607
+
608
+
609
+def input_mapper_for_whisper(
610
+ ctx: InputContext,
611
+ multi_modal_data: Union[np.ndarray, List[np.ndarray]],
612
+) -> MultiModalKwargs:
613
+ if not isinstance(multi_modal_data, list):
614
+ multi_modal_data = [multi_modal_data]
615
+
616
+ assert len(multi_modal_data) == 1
617
+
618
+ if len(multi_modal_data) == 0:
619
+ return MultiModalKwargs()
620
+
621
+ processor = cached_get_processor(ctx.model_config.model)
622
+ sampling_rate = processor.feature_extractor.sampling_rate
623
+
624
+ audios = [audio for audio, _ in multi_modal_data]
625
+
626
+ kwargs = processor(audios,
627
+ sampling_rate=sampling_rate,
628
+ return_tensors="pt")
629
+ kwargs["input_features"] = kwargs["input_features"].squeeze(0).to(
630
+ ctx.model_config.dtype)
631
+
632
+ return MultiModalKwargs(kwargs)
633
+
634
+
635
+@INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_whisper)
636
+@INPUT_REGISTRY.register_input_processor(input_processor_for_whisper)
637
+@MULTIMODAL_REGISTRY.register_input_mapper("audio", input_mapper_for_whisper)
638
+@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
639
+ "audio", get_max_whisper_audio_tokens)
640
+class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
641
+
642
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
643
+ super().__init__()
644
+ config = vllm_config.model_config.hf_config
645
+ quant_config = vllm_config.quant_config
646
+ self.config = config
647
+ self.dtype = vllm_config.model_config.dtype
648
+
649
+ self.model = WhisperModel(vllm_config=vllm_config, prefix=prefix)
650
+ self.unpadded_vocab_size = config.vocab_size
651
+ self.proj_out = ParallelLMHead(config.vocab_size,
652
+ config.d_model,
653
+ quant_config=quant_config)
654
+ self.proj_out = self.proj_out.tie_weights(
655
+ self.model.decoder.embed_tokens)
656
+ logit_scale = getattr(config, "logit_scale", 1.0)
657
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
658
+ config.vocab_size, logit_scale)
659
+ self.sampler = Sampler()
660
+
661
+ def forward(
662
+ self,
663
+ input_ids: torch.Tensor,
664
+ positions: torch.Tensor,
665
+ kv_caches: List[torch.Tensor],
666
+ attn_metadata: AttentionMetadata,
667
+ **kwargs,
668
+ ) -> torch.Tensor:
669
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
670
+ decoder_outputs = self.model(
671
+ input_features=audio_input["input_features"],
672
+ input_ids=input_ids,
673
+ positions=positions,
674
+ kv_caches=kv_caches,
675
+ attn_metadata=attn_metadata,
676
+ )
677
+ return decoder_outputs
678
+
679
+ def get_multimodal_embeddings(
680
+ self,
681
+ kv_caches: List[torch.Tensor],
682
+ attn_metadata: AttentionMetadata,
683
+ **kwargs,
684
+ ) -> Optional[NestedTensors]:
685
+ # TODO: This method does not obey the interface for SupportsMultiModal.
686
+ # Refactor this once encoder/decoder support is implemented in V1.
687
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
688
+ return self.model.get_encoder_outputs(
689
+ audio_input["input_features"],
690
+ kv_caches=kv_caches,
691
+ attn_metadata=attn_metadata,
692
+ )
693
+
694
+ def get_input_embeddings(
695
+ self,
696
+ input_ids: torch.Tensor,
697
+ multimodal_embeddings: Optional[NestedTensors] = None,
698
+ attn_metadata: Optional[AttentionMetadata] = None,
699
+ ) -> torch.Tensor:
700
+ # TODO: This method just returns the decoder sequence embeddings since
701
+ # Whisper does not have encoder text tokens. Refactor this once
702
+ # encoder/decoder support is implemented in V1.
703
+ return self.model.decoder.get_input_embeddings(input_ids)
704
+
705
+ def _parse_and_validate_audio_input(
706
+ self, **kwargs: object) -> WhisperAudioInputs:
707
+ input_features = kwargs.pop("input_features", None)
708
+
709
+ if input_features is not None:
710
+ if not isinstance(input_features, (torch.Tensor, list)):
711
+ raise ValueError("Incorrect type of audio features. "
712
+ f"Got type: {type(input_features)}")
713
+ input_features = [feat.to(self.dtype) for feat in input_features]
714
+
715
+ return WhisperAudioInputs(input_features=input_features)
716
+
717
+ def compute_logits(self, hidden_states: torch.Tensor,
718
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
719
+ logits = self.logits_processor(self.proj_out, hidden_states,
720
+ sampling_metadata)
721
+ return logits
722
+
723
+ def sample(
724
+ self,
725
+ logits: torch.Tensor,
726
+ sampling_metadata: SamplingMetadata,
727
+ ) -> Optional[SamplerOutput]:
728
+ next_tokens = self.sampler(logits, sampling_metadata)
729
+ return next_tokens
730
+
731
+ def load_weights(self, weights: Iterable[Tuple[str,
732
+ torch.Tensor]]) -> Set[str]:
733
+ loader = AutoWeightsLoader(self, skip_prefixes=["proj_out."])
734
+ loaded_weights = [(name, loaded_weight)
735
+ for name, loaded_weight in weights]
736
+ mapper = WeightsMapper({".fc1.": ".mlp.fc1.", ".fc2.": ".mlp.fc2."})
737
+ return loader.load_weights(loaded_weights, mapper=mapper)
\ No newline at end of file
vllm/multimodal/processing.py
+7
-21
@@ -16,7 +16,7 @@ from transformers import BatchFeature, ProcessorMixin
16
17
from vllm.inputs import DummyData, InputProcessingContext
18
from vllm.logger import init_logger
19
-from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
19
+from vllm.transformers_utils.tokenizer import AnyTokenizer, encode_tokens
20
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
21
22
from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
@@ -57,24 +57,6 @@ class PromptReplacement:
57
)
58
59
60
-def _encode(
61
- tokenizer: AnyTokenizer,
62
- text: str,
63
- *,
64
- add_special_tokens: bool = False,
65
-) -> list[int]:
66
- """
67
- Backend-agnostic equivalent of HF's
68
- :code:`tokenizer.encode(text, add_special_tokens=...)`.
69
- """
70
- if isinstance(tokenizer, MistralTokenizer):
71
- return tokenizer.tokenizer.encode(text,
72
- bos=add_special_tokens,
73
- eos=add_special_tokens)
74
-
75
- return tokenizer.encode(text, add_special_tokens=add_special_tokens)
76
-
77
-
60
@lru_cache(maxsize=2048)
61
def _cached_encode(
62
tokenizer: AnyTokenizer,
@@ -82,7 +64,9 @@ def _cached_encode(
64
*,
65
add_special_tokens: bool = False,
66
) -> list[int]:
85
- return _encode(tokenizer, text, add_special_tokens=add_special_tokens)
67
+ return encode_tokens(tokenizer,
68
+ text,
69
+ add_special_tokens=add_special_tokens)
70
71
72
def _decode(
@@ -983,7 +967,9 @@ class BaseMultiModalProcessor(ABC):
967
mm_item_counts,
968
)
969
986
- token_ids = _encode(tokenizer, text)
970
+ token_ids = encode_tokens(tokenizer,
971
+ text,
972
+ add_special_tokens=False)
973
matched_repls = [match.prompt_repl for match in text_matches]
974
975
placeholders = self._find_placeholders(matched_repls, token_ids,
vllm/sequence.py
+15
-3
@@ -710,15 +710,27 @@ class SequenceGroup:
710
711
@property
712
def multi_modal_data(self) -> MultiModalDataDict:
713
- return self.first_seq.multi_modal_data
713
+ if self.first_seq.multi_modal_data:
714
+ return self.first_seq.multi_modal_data
715
+ elif self.encoder_seq is not None:
716
+ return self.encoder_seq.multi_modal_data
717
+ return {}
718
719
@property
720
def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
717
- return self.first_seq.multi_modal_placeholders
721
+ if self.first_seq.multi_modal_data:
722
+ return self.first_seq.multi_modal_placeholders
723
+ elif self.encoder_seq is not None:
724
+ return self.encoder_seq.multi_modal_placeholders
725
+ return {}
726
727
@property
728
def mm_processor_kwargs(self) -> Dict[str, Any]:
721
- return self.first_seq.mm_processor_kwargs
729
+ if self.first_seq.multi_modal_data:
730
+ return self.first_seq.mm_processor_kwargs
731
+ elif self.encoder_seq is not None:
732
+ return self.encoder_seq.mm_processor_kwargs
733
+ return {}
734
735
@property
736
def lora_int_id(self) -> int:
vllm/transformers_utils/tokenizer.py
+19
@@ -21,6 +21,25 @@ AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast,
21
MistralTokenizer]
22
23
24
+def encode_tokens(
25
+ tokenizer: AnyTokenizer,
26
+ text: str,
27
+ *,
28
+ add_special_tokens: Optional[bool] = None,
29
+) -> list[int]:
30
+ """
31
+ Backend-agnostic equivalent of HF's
32
+ :code:`tokenizer.encode(text, add_special_tokens=...)`.
33
+ """
34
+ if isinstance(tokenizer, MistralTokenizer):
35
+ return tokenizer.tokenizer.encode(text,
36
+ bos=add_special_tokens,
37
+ eos=add_special_tokens)
38
+ elif add_special_tokens is not None:
39
+ return tokenizer.encode(text, add_special_tokens=add_special_tokens)
40
+ return tokenizer.encode(text)
41
+
42
+
43
def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
44
"""Get tokenizer with cached properties.
45
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+4
-2
@@ -32,7 +32,8 @@ class BaseTokenizerGroup(ABC):
32
def encode(self,
33
prompt: str,
34
request_id: Optional[str] = None,
35
- lora_request: Optional[LoRARequest] = None) -> List[int]:
35
+ lora_request: Optional[LoRARequest] = None,
36
+ add_special_tokens: Optional[bool] = None) -> List[int]:
37
"""Encode a prompt using the tokenizer group."""
38
pass
39
@@ -41,7 +42,8 @@ class BaseTokenizerGroup(ABC):
42
self,
43
prompt: str,
44
request_id: Optional[str] = None,
44
- lora_request: Optional[LoRARequest] = None) -> List[int]:
45
+ lora_request: Optional[LoRARequest] = None,
46
+ add_special_tokens: Optional[bool] = None) -> List[int]:
47
"""Encode a prompt using the tokenizer group."""
48
pass
49
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+18
-10
@@ -112,7 +112,8 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
112
def encode(self,
113
prompt: str,
114
request_id: Optional[str] = None,
115
- lora_request: Optional[LoRARequest] = None) -> List[int]:
115
+ lora_request: Optional[LoRARequest] = None,
116
+ add_special_tokens: Optional[bool] = None) -> List[int]:
117
"""Encode a prompt using the tokenizer group.
118
119
We pick an idle actor and use it to encode the prompt.
@@ -132,7 +133,8 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
133
ret = ray.get(
134
actor.encode.remote(request_id=request_id,
135
prompt=prompt,
135
- lora_request=lora_request))
136
+ lora_request=lora_request,
137
+ add_special_tokens=add_special_tokens))
138
except ActorDiedError as e:
139
# If the actor is dead, we first try to reinitialize it.
140
logger.warning("%s died with ActorDiedError, reinitializing.",
@@ -143,7 +145,8 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
145
ret = ray.get(
146
actor.encode.remote(request_id=request_id,
147
prompt=prompt,
146
- lora_request=lora_request))
148
+ lora_request=lora_request,
149
+ add_special_tokens=add_special_tokens))
150
except ActorDiedError as e:
151
logger.error(
152
"%s died for second time in a row, marking "
@@ -160,7 +163,8 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
163
self,
164
prompt: str,
165
request_id: Optional[str] = None,
163
- lora_request: Optional[LoRARequest] = None) -> List[int]:
166
+ lora_request: Optional[LoRARequest] = None,
167
+ add_special_tokens: Optional[bool] = None) -> List[int]:
168
"""Encode a prompt using the tokenizer group.
169
170
We pick an idle actor and use it to encode the prompt.
@@ -177,9 +181,11 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
181
actor_is_alive = True
182
original_actor = actor
183
try:
180
- ret = await actor.encode.remote(request_id=request_id,
181
- prompt=prompt,
182
- lora_request=lora_request)
184
+ ret = await actor.encode.remote(
185
+ request_id=request_id,
186
+ prompt=prompt,
187
+ lora_request=lora_request,
188
+ add_special_tokens=add_special_tokens)
189
except ActorDiedError as e:
190
# If the actor is dead, we first try to reinitialize it.
191
logger.warning("%s died with ActorDiedError, reinitializing.",
@@ -187,9 +193,11 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
193
exc_info=e)
194
actor = self._init_actor()
195
try:
190
- ret = await actor.encode.remote(request_id=request_id,
191
- prompt=prompt,
192
- lora_request=lora_request)
196
+ ret = await actor.encode.remote(
197
+ request_id=request_id,
198
+ prompt=prompt,
199
+ lora_request=lora_request,
200
+ add_special_tokens=add_special_tokens)
201
except ActorDiedError as e:
202
logger.error(
203
"%s died for second time in a row, marking "
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+11
-5
@@ -2,7 +2,7 @@ from typing import List, Optional
2
3
from vllm.config import TokenizerPoolConfig
4
from vllm.lora.request import LoRARequest
5
-from vllm.transformers_utils.tokenizer import (AnyTokenizer,
5
+from vllm.transformers_utils.tokenizer import (AnyTokenizer, encode_tokens,
6
get_lora_tokenizer,
7
get_lora_tokenizer_async,
8
get_tokenizer)
@@ -55,9 +55,12 @@ class TokenizerGroup(BaseTokenizerGroup):
55
def encode(self,
56
prompt: str,
57
request_id: Optional[str] = None,
58
- lora_request: Optional[LoRARequest] = None) -> List[int]:
58
+ lora_request: Optional[LoRARequest] = None,
59
+ add_special_tokens: Optional[bool] = None) -> List[int]:
60
tokenizer = self.get_lora_tokenizer(lora_request)
60
- ret = tokenizer.encode(prompt)
61
+ ret = encode_tokens(tokenizer,
62
+ prompt,
63
+ add_special_tokens=add_special_tokens)
64
self._raise_if_input_too_long(ret, lora_request)
65
return ret
66
@@ -65,9 +68,12 @@ class TokenizerGroup(BaseTokenizerGroup):
68
self,
69
prompt: str,
70
request_id: Optional[str] = None,
68
- lora_request: Optional[LoRARequest] = None) -> List[int]:
71
+ lora_request: Optional[LoRARequest] = None,
72
+ add_special_tokens: Optional[bool] = None) -> List[int]:
73
tokenizer = await self.get_lora_tokenizer_async(lora_request)
70
- ret = tokenizer.encode(prompt)
74
+ ret = encode_tokens(tokenizer,
75
+ prompt,
76
+ add_special_tokens=add_special_tokens)
77
self._raise_if_input_too_long(ret, lora_request)
78
return ret
79
vllm/worker/enc_dec_model_runner.py
+5
-6
@@ -287,12 +287,11 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
287
seq_len,
288
self.mm_registry,
289
is_encoder_data=False)
290
- encoder_dummy_data \
291
- = self.input_registry.dummy_data_for_profiling(
292
- self.model_config,
293
- seq_len,
294
- self.mm_registry,
295
- is_encoder_data=True)
290
+ encoder_dummy_data = self.input_registry \
291
+ .dummy_data_for_profiling(self.model_config,
292
+ seq_len,
293
+ self.mm_registry,
294
+ is_encoder_data=True)
295
296
# Having more tokens is over-conservative but otherwise fine
297
assert len(