@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 767c727a81
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

fix DbrxFusedNormAttention missing cache_config (#5340)

Co-authored-by: team <calvinn.ng@ahrefs.com>

Calvinn Ng committed Jun 8, 2024 at 05:10 UTC 767c727a81ae9ec570d30d55b7afc783775d5a05
1 file changed +2 -1
vllm/model_executor/models/dbrx.py
+2 -1
@@ -247,11 +247,12 @@ class DbrxFusedNormAttention(nn.Module):
247 def __init__(
248 self,
249 config: DbrxConfig,
250 + cache_config: Optional[CacheConfig] = None,
251 quant_config: Optional[QuantizationConfig] = None,
252 ):
253 super().__init__()
254 self.d_model = config.d_model
254 - self.attn = DbrxAttention(config, quant_config)
255 + self.attn = DbrxAttention(config, cache_config, quant_config)
256 self.norm_1 = nn.LayerNorm(self.d_model)
257 self.norm_2 = nn.LayerNorm(self.d_model)
258