A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
fix DbrxFusedNormAttention missing cache_config (#5340)
Co-authored-by: team <calvinn.ng@ahrefs.com>
Calvinn Ng committed
Jun 8, 2024 at 05:10 UTC
767c727a81ae9ec570d30d55b7afc783775d5a05
1 file changed
+2
-1
vllm/model_executor/models/dbrx.py
+2
-1
@@ -247,11 +247,12 @@ class DbrxFusedNormAttention(nn.Module):
247
def __init__(
248
self,
249
config: DbrxConfig,
250
+ cache_config: Optional[CacheConfig] = None,
251
quant_config: Optional[QuantizationConfig] = None,
252
):
253
super().__init__()
254
self.d_model = config.d_model
254
- self.attn = DbrxAttention(config, quant_config)
255
+ self.attn = DbrxAttention(config, cache_config, quant_config)
256
self.norm_1 = nn.LayerNorm(self.d_model)
257
self.norm_2 = nn.LayerNorm(self.d_model)
258