@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f3f3c86ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Update attention interface in `Whisper` (#11784)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Jan 6, 2025 at 20:36 UTC 0f3f3c86ec44467fa80b60cb9f971f9ede028f76
1 file changed +11 -13
vllm/model_executor/models/whisper.py
+11 -13
@@ -106,6 +106,7 @@ class WhisperAttention(nn.Module):
106 cache_config=cache_config,
107 quant_config=quant_config,
108 prefix=f"{prefix}.attn",
109 + attn_type=self.attn_type,
110 )
111
112 def _init_qkv(
@@ -134,12 +135,7 @@ class WhisperAttention(nn.Module):
135 qkv, _ = self.qkv_proj(hidden_states)
136 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
137
137 - attn_output = self.attn(q,
138 - k,
139 - v,
140 - kv_cache,
141 - attn_metadata,
142 - attn_type=self.attn_type)
138 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
139
140 output, _ = self.out_proj(attn_output)
141
@@ -164,6 +160,7 @@ class WhisperCrossAttention(WhisperAttention):
160 cache_config=cache_config,
161 quant_config=quant_config,
162 prefix=prefix,
163 + attn_type=AttentionType.ENCODER_DECODER,
164 )
165
166 def _init_qkv(
@@ -207,12 +204,13 @@ class WhisperCrossAttention(WhisperAttention):
204 else:
205 k = v = None
206
210 - attn_output = self.attn(q,
211 - k,
212 - v,
213 - kv_cache,
214 - attn_metadata,
215 - attn_type=AttentionType.ENCODER_DECODER)
207 + attn_output = self.attn(
208 + q,
209 + k,
210 + v,
211 + kv_cache,
212 + attn_metadata,
213 + )
214
215 output, _ = self.out_proj(attn_output)
216
@@ -734,4 +732,4 @@ class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
732 loaded_weights = [(name, loaded_weight)
733 for name, loaded_weight in weights]
734 mapper = WeightsMapper({".fc1.": ".mlp.fc1.", ".fc2.": ".mlp.fc2."})
737 - return loader.load_weights(loaded_weights, mapper=mapper)
\ No newline at end of file
735 + return loader.load_weights(loaded_weights, mapper=mapper)