A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Misc][Attention][Quantization] init property earlier (#13733)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
wangxiyuan committed
Feb 25, 2025 at 11:19 UTC
ab1091d5f2fc879ba9e62002f4d9eec013984d4d
1 file changed
+5
-4
vllm/attention/layer.py
+5
-4
@@ -85,6 +85,11 @@ class Attention(nn.Module):
85
self._k_scale_float = 1.0
86
self._v_scale_float = 1.0
87
88
+ self.num_heads = num_heads
89
+ self.head_size = head_size
90
+ self.num_kv_heads = num_kv_heads
91
+ self.sliding_window = sliding_window
92
+
93
quant_method = quant_config.get_quant_method(
94
self, prefix=prefix) if quant_config else None
95
if quant_method is not None:
@@ -116,10 +121,6 @@ class Attention(nn.Module):
121
alibi_slopes, sliding_window, kv_cache_dtype,
122
blocksparse_params, logits_soft_cap, attn_type,
123
**extra_impl_args)
119
- self.num_heads = num_heads
120
- self.head_size = head_size
121
- self.num_kv_heads = num_kv_heads
122
- self.sliding_window = sliding_window
124
self.backend = backend_name_to_enum(attn_backend.get_name())
125
self.dtype = dtype
126