A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 23:00 UTC
[Fix] Use torch.empty for output in attention+quant fusion (#31785)
Signed-off-by: elvischenv <219235043+elvischenv@users.noreply.github.com>
elvischenv committed
Mar 11, 2026 at 12:26 UTC
7d6abdd02241a135e2429de1b583dbfb6f76d6ff
1 file changed
+2
-4
vllm/compilation/passes/fusion/attn_quant_fusion.py
+2
-4
@@ -170,9 +170,8 @@ class AttentionFp8StaticQuantPattern(AttentionQuantPattern):
170
kv_cache_dummy_dep: torch.Tensor,
171
) -> torch.Tensor:
172
# attn output in quant_dtype
173
- output_attn = torch.ops.aten.full.default(
173
+ output_attn = torch.empty(
174
[q.shape[0], self.num_heads, self.head_size],
175
- 0.0,
175
dtype=self.quant_dtype,
176
device=q.device,
177
)
@@ -271,9 +270,8 @@ class AttentionNvfp4QuantPattern(AttentionQuantPattern):
270
kv_cache_dummy_dep: torch.Tensor,
271
) -> tuple[torch.Tensor, torch.Tensor]:
272
# attention output in quant_dtype
274
- output_attn = torch.ops.aten.full.default(
273
+ output_attn = torch.empty(
274
[q.shape[0], self.num_heads, self.head_size // 2],
276
- 0.0,
275
dtype=self.quant_dtype,
276
device=q.device,
277
)