A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[gpt-oss] raise error for flashinfer backend without trtllm (#24482)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Sep 10, 2025 at 14:33 UTC
b5e383cd8b62975dec605bed05e22d273c296c7a
1 file changed
+10
-2
vllm/v1/attention/backends/flashinfer.py
+10
-2
@@ -216,7 +216,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
216
self.window_left = self.global_hyperparameters.window_left
217
self.logits_soft_cap = self.global_hyperparameters.logits_soft_cap
218
self.has_sinks = self.global_hyperparameters.has_sinks
219
-
219
+ if self.has_sinks and not supports_trtllm_attention()[0]:
220
+ raise NotImplementedError(
221
+ "FlashInfer backend currently does not support attention "
222
+ "sinks, please use trtllm on blackwell or flash attention on "
223
+ "earlier GPUs.")
224
# Preparing persistent buffers (device-side)
225
self.paged_kv_indptr = torch.zeros(max_num_reqs + 1,
226
dtype=torch.int32,
@@ -408,7 +412,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
412
self.q_data_type,
413
is_prefill=False,
414
has_sinks=self.has_sinks)
411
-
415
+ if self.has_sinks and not (prefill_use_trtllm and decode_use_trtllm):
416
+ raise NotImplementedError(
417
+ "FlashInfer backend currently does not support attention "
418
+ "sinks, please use trtllm on blackwell or flash attention on "
419
+ "earlier GPUs.")
420
attn_metadata = FlashInferMetadata(
421
num_actual_tokens=num_actual_tokens,
422
q_data_type=self.q_data_type,