A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 22:15 UTC
Re-enable FlashInfer for Llama4 on Blackwell in e2e fusion tests (#28966)
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: ProExpertProg <11367180+ProExpertProg@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Copilot committed
Nov 19, 2025 at 13:32 UTC
61728cd1dfb03cbbfa03924f2a2cda311cfc13ac
2 files changed
+6
-8
.buildkite/test-pipeline.yaml
+2
@@ -930,6 +930,8 @@ steps:
930
- csrc/quantization/fp4/
931
- vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
932
- vllm/v1/attention/backends/flashinfer.py
933
+ - vllm/v1/worker/
934
+ - vllm/v1/cudagraph_dispatcher.py
935
- vllm/compilation/
936
# can affect pattern matching
937
- vllm/model_executor/layers/layernorm.py
tests/compile/distributed/test_fusions_e2e.py
+4
-8
@@ -47,12 +47,8 @@ if current_platform.is_cuda():
47
ModelBackendTestCase(
48
# Use smaller model for L40s in CI
49
model_name="RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
50
- # TODO while llama4 is broken, use FLASHINFER for llama3 on Blackwell
51
- # so FI attention+fp8_quant is at least tested once
50
model_kwargs=dict(max_model_len=1024, kv_cache_dtype="fp8"),
53
- backend=AttentionBackendEnum.FLASHINFER
54
- if is_blackwell()
55
- else AttentionBackendEnum.TRITON_ATTN,
51
+ backend=AttentionBackendEnum.TRITON_ATTN,
52
matches=Matches(
53
attention_fusion=32,
54
allreduce_fusion=65,
@@ -65,9 +61,9 @@ if current_platform.is_cuda():
61
model_kwargs=dict(max_model_len=1024, kv_cache_dtype="fp8"),
62
# TODO FlashInfer attn broken on Hopper with kvcache=fp8:
63
# https://github.com/vllm-project/vllm/issues/28568
68
- # TODO FlashInfer attn broken on Blackwell for llama4:
69
- # https://github.com/vllm-project/vllm/issues/28604
70
- backend=AttentionBackendEnum.TRITON_ATTN,
64
+ backend=AttentionBackendEnum.FLASHINFER
65
+ if is_blackwell()
66
+ else AttentionBackendEnum.TRITON_ATTN,
67
matches=Matches(
68
attention_fusion=48,
69
allreduce_fusion=96,