A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Update attention interface in `Whisper` (#11784)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Jan 6, 2025 at 20:36 UTC
0f3f3c86ec44467fa80b60cb9f971f9ede028f76
1 file changed
+11
-13
vllm/model_executor/models/whisper.py
+11
-13
@@ -106,6 +106,7 @@ class WhisperAttention(nn.Module):
106
cache_config=cache_config,
107
quant_config=quant_config,
108
prefix=f"{prefix}.attn",
109
+ attn_type=self.attn_type,
110
)
111
112
def _init_qkv(
@@ -134,12 +135,7 @@ class WhisperAttention(nn.Module):
135
qkv, _ = self.qkv_proj(hidden_states)
136
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
137
137
- attn_output = self.attn(q,
138
- k,
139
- v,
140
- kv_cache,
141
- attn_metadata,
142
- attn_type=self.attn_type)
138
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
139
140
output, _ = self.out_proj(attn_output)
141
@@ -164,6 +160,7 @@ class WhisperCrossAttention(WhisperAttention):
160
cache_config=cache_config,
161
quant_config=quant_config,
162
prefix=prefix,
163
+ attn_type=AttentionType.ENCODER_DECODER,
164
)
165
166
def _init_qkv(
@@ -207,12 +204,13 @@ class WhisperCrossAttention(WhisperAttention):
204
else:
205
k = v = None
206
210
- attn_output = self.attn(q,
211
- k,
212
- v,
213
- kv_cache,
214
- attn_metadata,
215
- attn_type=AttentionType.ENCODER_DECODER)
207
+ attn_output = self.attn(
208
+ q,
209
+ k,
210
+ v,
211
+ kv_cache,
212
+ attn_metadata,
213
+ )
214
215
output, _ = self.out_proj(attn_output)
216
@@ -734,4 +732,4 @@ class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
732
loaded_weights = [(name, loaded_weight)
733
for name, loaded_weight in weights]
734
mapper = WeightsMapper({".fc1.": ".mlp.fc1.", ".fc2.": ".mlp.fc2."})
737
- return loader.load_weights(loaded_weights, mapper=mapper)
\ No newline at end of file
735
+ return loader.load_weights(loaded_weights, mapper=mapper)