| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for v1 attention backends without GPUModelRunner dependency.""" |
| 4 | |
| 5 | from dataclasses import replace |
| 6 | from functools import partial |
| 7 | from types import SimpleNamespace |
| 8 | |
| 9 | import pytest |
| 10 | import torch |
| 11 | from torch.nn.attention.flex_attention import create_block_mask, flex_attention |
| 12 | from transformers import LlamaConfig |
| 13 | |
| 14 | from tests.v1.attention.utils import ( |
| 15 | BatchSpec, |
| 16 | create_common_attn_metadata, |
| 17 | create_standard_kv_cache_spec, |
| 18 | create_vllm_config, |
| 19 | try_backend_includes_kv_cache_update, |
| 20 | try_get_attention_backend, |
| 21 | ) |
| 22 | from vllm.config import ( |
| 23 | DiffusionConfig, |
| 24 | ModelConfig, |
| 25 | SpeculativeConfig, |
| 26 | set_current_vllm_config, |
| 27 | ) |
| 28 | from vllm.platforms import current_platform |
| 29 | from vllm.utils.math_utils import cdiv |
| 30 | from vllm.utils.torch_utils import ( |
| 31 | STR_DTYPE_TO_TORCH_DTYPE, |
| 32 | is_quantized_kv_cache, |
| 33 | is_torch_equal_or_newer, |
| 34 | set_random_seed, |
| 35 | ) |
| 36 | from vllm.v1.attention.backend import ( |
| 37 | AttentionCGSupport, |
| 38 | AttentionMetadataBuilder, |
| 39 | AttentionType, |
| 40 | CommonAttentionMetadata, |
| 41 | max_decode_query_len, |
| 42 | ) |
| 43 | from vllm.v1.attention.backends.registry import AttentionBackendEnum |
| 44 | from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout |
| 45 | |
| 46 | BACKENDS_TO_TEST = [ |
| 47 | AttentionBackendEnum.FLASH_ATTN, |
| 48 | AttentionBackendEnum.FLASHINFER, |
| 49 | AttentionBackendEnum.FLEX_ATTENTION, |
| 50 | AttentionBackendEnum.TRITON_ATTN, |
| 51 | "FLEX_ATTENTION_SLOW", |
| 52 | ] |
| 53 | |
| 54 | |
| 55 | def _actual_backend(backend: AttentionBackendEnum | str) -> AttentionBackendEnum: |
| 56 | """Resolve pseudo-backends (FLEX_ATTENTION_SLOW) to their real enum.""" |
| 57 | if isinstance(backend, str): |
| 58 | if backend != "FLEX_ATTENTION_SLOW": |
| 59 | raise ValueError(f"Unknown pseudo-backend: {backend}") |
| 60 | return AttentionBackendEnum.FLEX_ATTENTION |
| 61 | return backend |
| 62 | |
| 63 | |
| 64 | DEVICE_TYPE = current_platform.device_type |
| 65 | |
| 66 | # Use the platform's preferred FP8 type so the stored cache matches what the |
| 67 | # backends reinterpret at runtime. On ROCm gfx94x this is e4m3fnuz, not e4m3fn; |
| 68 | # storing e4m3fn bytes there would be re-read as fnuz and produce NaNs. |
| 69 | FP8_KV_CACHE_DTYPES = { |
| 70 | "fp8": current_platform.fp8_dtype(), |
| 71 | "fp8_e4m3": current_platform.fp8_dtype(), |
| 72 | } |
| 73 | |
| 74 | # Remove flashinfer from the list if it's not available |
| 75 | try: |
| 76 | import flashinfer # noqa: F401 |
| 77 | except ImportError: |
| 78 | BACKENDS_TO_TEST.remove(AttentionBackendEnum.FLASHINFER) |
| 79 | |
| 80 | |
| 81 | def _convert_dtype_to_torch(dtype): |
| 82 | """Convert ModelDType to torch.dtype.""" |
| 83 | if isinstance(dtype, str): |
| 84 | if dtype == "auto": |
| 85 | return torch.float16 # Default dtype for testing |
| 86 | elif dtype in STR_DTYPE_TO_TORCH_DTYPE: |
| 87 | return STR_DTYPE_TO_TORCH_DTYPE[dtype] |
| 88 | else: |
| 89 | raise ValueError(f"Unknown dtype: {dtype}") |
| 90 | elif isinstance(dtype, torch.dtype): |
| 91 | return dtype |
| 92 | else: |
| 93 | raise ValueError(f"Unknown dtype: {dtype}") |
| 94 | |
| 95 | |
| 96 | # Define common batch configurations |
| 97 | BATCH_SPECS = { |
| 98 | "small_decode": BatchSpec(seq_lens=[32, 40], query_lens=[1, 1]), |
| 99 | "small_prefill": BatchSpec(seq_lens=[32, 40], query_lens=[8, 8]), |
| 100 | "mixed_small": BatchSpec(seq_lens=[32, 40, 48, 56], query_lens=[1, 1, 5, 5]), |
| 101 | "medium_decode": BatchSpec( |
| 102 | seq_lens=[128, 256, 512, 1024, 128, 256, 512, 1024], |
| 103 | query_lens=[1, 1, 1, 1, 1, 1, 1, 1], |
| 104 | ), |
| 105 | "medium_prefill": BatchSpec( |
| 106 | seq_lens=[256, 512, 1024, 2048], query_lens=[16, 16, 16, 16] |
| 107 | ), |
| 108 | "mixed_medium": BatchSpec( |
| 109 | seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[1, 1, 1, 7, 7, 7] |
| 110 | ), |
| 111 | "large_decode": BatchSpec(seq_lens=[2048] * 32, query_lens=[1] * 32), |
| 112 | "large_prefill": BatchSpec(seq_lens=[4096] * 8, query_lens=[32] * 8), |
| 113 | "mixed_large": BatchSpec( |
| 114 | seq_lens=[1024, 2048, 4096, 1024, 2048, 4096], query_lens=[1, 1, 1, 32, 32, 32] |
| 115 | ), |
| 116 | "single_decode": BatchSpec(seq_lens=[1024], query_lens=[1]), |
| 117 | "single_prefill": BatchSpec(seq_lens=[1024], query_lens=[64]), |
| 118 | # encoder-only |
| 119 | "small_encoder_prefill": BatchSpec( |
| 120 | seq_lens=[32, 64, 128, 256], query_lens=[32, 64, 128, 256] |
| 121 | ), |
| 122 | "medium_encoder_prefill": BatchSpec( |
| 123 | seq_lens=[256, 512, 1024, 2048], query_lens=[256, 512, 1024, 2048] |
| 124 | ), |
| 125 | } |
| 126 | |
| 127 | |
| 128 | def create_and_prepopulate_kv_cache( |
| 129 | k_contexts: list[torch.Tensor], |
| 130 | v_contexts: list[torch.Tensor], |
| 131 | block_size: int, |
| 132 | num_kv_heads: int, |
| 133 | head_size: int, |
| 134 | dtype: torch.dtype, |
| 135 | device: torch.device, |
| 136 | num_blocks: int, |
| 137 | common_attn_metadata: CommonAttentionMetadata, |
| 138 | layout: KVCacheLayout, |
| 139 | randomize_blocks: bool = True, |
| 140 | kv_cache_dtype: str = "auto", |
| 141 | ) -> torch.Tensor: |
| 142 | """Create and prepopulate a KV cache with context data. |
| 143 | |
| 144 | Args: |
| 145 | k_contexts: List of key context tensors for each sequence |
| 146 | v_contexts: List of value context tensors for each sequence |
| 147 | block_size: Size of each block |
| 148 | num_kv_heads: Number of KV heads |
| 149 | head_size: Size of each head |
| 150 | dtype: Data type for the cache |
| 151 | device: Device to create the cache on |
| 152 | num_blocks: Total number of blocks in the cache |
| 153 | common_attn_metadata: Provides seq lens, block table and slot mapping |
| 154 | layout: Physical layout to allocate in; the cache is returned as the |
| 155 | logical ``[B, H, N, C]`` view (as ``create_kv_cache_views`` does) |
| 156 | randomize_blocks: Whether to randomly permute blocks |
| 157 | or use sequential order |
| 158 | kv_cache_dtype: Cache dtype string; fp8 caches use fp8 storage |
| 159 | |
| 160 | Returns: |
| 161 | A 4D tensor in logical ``(num_blocks, num_kv_heads, block_size, |
| 162 | 2 * head_size)`` order with strides determined by ``layout``. |
| 163 | |
| 164 | """ |
| 165 | batch_size = len(k_contexts) |
| 166 | seq_lens = common_attn_metadata.seq_lens.cpu() |
| 167 | query_lens = ( |
| 168 | common_attn_metadata.query_start_loc_cpu[1:] |
| 169 | - common_attn_metadata.query_start_loc_cpu[:-1] |
| 170 | ) |
| 171 | context_lens = seq_lens - query_lens |
| 172 | block_table = common_attn_metadata.block_table_tensor |
| 173 | slot_mapping = common_attn_metadata.slot_mapping |
| 174 | |
| 175 | # For an fp8 kv cache, store the cache in the fp8 dtype so that assigning |
| 176 | # the higher-precision context tensors quantizes them, mirroring runtime. |
| 177 | fp8_kv_cache = is_quantized_kv_cache(kv_cache_dtype) |
| 178 | storage_dtype = FP8_KV_CACHE_DTYPES[kv_cache_dtype] if fp8_kv_cache else dtype |
| 179 | |
| 180 | # Logical 5D shape is always [L, B, H, N, C]. Cross-layer layouts need |
| 181 | # at least two layers to reproduce the inter-layer gaps in a layer view. |
| 182 | logical_4d = (num_blocks, num_kv_heads, block_size, 2 * head_size) |
| 183 | num_layers = 1 if layout.is_layer_compact else 2 |
| 184 | logical_5d = (num_layers, *logical_4d) |
| 185 | physical_5d = tuple(logical_5d[i] for i in layout.stride_order) |
| 186 | inv_order = [layout.stride_order.index(i) for i in range(5)] |
| 187 | |
| 188 | kv_cache_physical = torch.zeros(physical_5d, dtype=storage_dtype, device=device) |
| 189 | # Permute to logical [L, B, H, N, C], then select a layer. This mirrors |
| 190 | # create_kv_cache_views and retains cross-layer strides in the 4D view. |
| 191 | kv_cache = kv_cache_physical.permute(*inv_order)[0] |
| 192 | |
| 193 | # Write context tokens into the cache via the logical view: |
| 194 | # kv_cache[block, :, token_in_block, :] routes correctly regardless |
| 195 | # of physical layout. |
| 196 | # Start from block_id=1 since block_id=0 is considered the null block |
| 197 | start_block_idx = 1 |
| 198 | for i in range(batch_size): |
| 199 | k_context, v_context = k_contexts[i], v_contexts[i] |
| 200 | t = torch.arange(k_context.shape[0], device=device) |
| 201 | blk = start_block_idx + t // block_size |
| 202 | off = t % block_size |
| 203 | # Advanced indexing on (blk, off) yields [T, H, hs] destinations. |
| 204 | # index_put is dtype-strict; cast like the scalar path would. |
| 205 | kv_cache[blk, :, off, :head_size] = k_context.to(kv_cache.dtype) |
| 206 | kv_cache[blk, :, off, head_size:] = v_context.to(kv_cache.dtype) |
| 207 | # Stay block aligned and allocate enough blocks for the new tokens |
| 208 | start_block_idx += cdiv(int(seq_lens[i]), block_size) |
| 209 | |
| 210 | blocks_end = start_block_idx |
| 211 | |
| 212 | # Permute the context blocks (excluding block 0 which is null) |
| 213 | if randomize_blocks: |
| 214 | # Random permutation starting from block 1 |
| 215 | perm = torch.randperm(blocks_end - 1) + 1 |
| 216 | else: |
| 217 | # Sequential order starting from block 1 |
| 218 | perm = torch.arange(1, blocks_end) |
| 219 | |
| 220 | inv_perm = torch.zeros(blocks_end, dtype=torch.long, device=device) |
| 221 | # Add 1 to account for starting from block 1 |
| 222 | inv_perm[1:] = torch.argsort(perm) + 1 |
| 223 | kv_cache[1:blocks_end, ...] = kv_cache[perm, ...] |
| 224 | |
| 225 | # Construct the right block table |
| 226 | # Start from block_id=1 since block_id=0 is considered the null block |
| 227 | start_block_idx = 1 |
| 228 | for i in range(batch_size): |
| 229 | num_blocks_for_seq = cdiv(int(seq_lens[i]), block_size) |
| 230 | start = start_block_idx |
| 231 | end = start + num_blocks_for_seq |
| 232 | block_table[i, :num_blocks_for_seq] = inv_perm[start:end] |
| 233 | start_block_idx += num_blocks_for_seq |
| 234 | |
| 235 | # Create a realistic slot mapping that corresponds to the block table |
| 236 | for i in range(batch_size): |
| 237 | token_offsets = torch.arange(int(query_lens[i])) + int(context_lens[i]) |
| 238 | block_indices = token_offsets // block_size |
| 239 | token_inter_block_offsets = token_offsets % block_size |
| 240 | start = common_attn_metadata.query_start_loc_cpu[i] |
| 241 | end = common_attn_metadata.query_start_loc_cpu[i + 1] |
| 242 | slot_mapping[start:end] = block_table[ |
| 243 | i, block_indices |
| 244 | ] * block_size + token_inter_block_offsets.to(device) |
| 245 | |
| 246 | if fp8_kv_cache: |
| 247 | kv_cache = kv_cache.view(torch.uint8) |
| 248 | |
| 249 | return kv_cache |
| 250 | |
| 251 | |
| 252 | class MockAttentionLayer: |
| 253 | """A mock attention layer for testing.""" |
| 254 | |
| 255 | def __init__( |
| 256 | self, |
| 257 | device: torch.device, |
| 258 | k_scale: float = 1.0, |
| 259 | v_scale: float = 1.0, |
| 260 | ): |
| 261 | self._q_scale = torch.tensor(1.0, device=device) |
| 262 | self._k_scale = torch.tensor(k_scale, device=device) |
| 263 | self._v_scale = torch.tensor(v_scale, device=device) |
| 264 | # Add float versions for flashinfer |
| 265 | self._q_scale_float = 1.0 |
| 266 | self._k_scale_float = k_scale |
| 267 | self._v_scale_float = v_scale |
| 268 | |
| 269 | |
| 270 | def _clone_kv_cache_in_layout( |
| 271 | kv_cache: torch.Tensor, layout: KVCacheLayout |
| 272 | ) -> torch.Tensor: |
| 273 | """Copy a logical [B, H, N, C] cache into a fresh allocation in `layout`.""" |
| 274 | logical_5d = (1, *kv_cache.shape) |
| 275 | physical = torch.zeros( |
| 276 | tuple(logical_5d[i] for i in layout.stride_order), |
| 277 | dtype=kv_cache.dtype, |
| 278 | device=kv_cache.device, |
| 279 | ) |
| 280 | inv_order = [layout.stride_order.index(i) for i in range(5)] |
| 281 | view = physical.permute(*inv_order)[0] |
| 282 | view.copy_(kv_cache) |
| 283 | return view |
| 284 | |
| 285 | |
| 286 | def run_attention_backend( |
| 287 | backend: AttentionBackendEnum | str, |
| 288 | kv_cache_spec: FullAttentionSpec, |
| 289 | layer_names: list[str], |
| 290 | vllm_config, |
| 291 | device: torch.device, |
| 292 | common_attn_metadata: CommonAttentionMetadata, |
| 293 | query: torch.Tensor, |
| 294 | key: torch.Tensor, |
| 295 | value: torch.Tensor, |
| 296 | kv_cache: torch.Tensor, |
| 297 | attn_type: AttentionType = AttentionType.DECODER, |
| 298 | sliding_window: int | None = None, |
| 299 | kv_cache_dtype: str = "auto", |
| 300 | sinks: torch.Tensor | None = None, |
| 301 | use_cuda_graph: bool = False, |
| 302 | layer_k_scale: float = 1.0, |
| 303 | layer_v_scale: float = 1.0, |
| 304 | ) -> torch.Tensor: |
| 305 | """Run attention computation using the specified backend's AttentionImpl.""" |
| 306 | use_direct_block_mask = is_torch_equal_or_newer("2.9.0.dev0") |
| 307 | if backend == "FLEX_ATTENTION_SLOW": |
| 308 | use_direct_block_mask = False |
| 309 | backend = _actual_backend(backend) |
| 310 | |
| 311 | builder_cls, impl_cls = try_get_attention_backend(backend) |
| 312 | |
| 313 | # Mock flashinfer's get_per_layer_parameters if needed |
| 314 | if backend == AttentionBackendEnum.FLASHINFER: |
| 315 | import unittest.mock |
| 316 | |
| 317 | from vllm.v1.attention.backends.utils import PerLayerParameters |
| 318 | |
| 319 | def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls): |
| 320 | # Return mock parameters for a single layer |
| 321 | head_size = vllm_config.model_config.get_head_size() |
| 322 | return { |
| 323 | layer_name: PerLayerParameters( |
| 324 | window_left=-1, # No sliding window |
| 325 | logits_soft_cap=0.0, # No soft cap |
| 326 | sm_scale=1.0 / (head_size**0.5), # Standard scale |
| 327 | has_sinks=sinks is not None, |
| 328 | ) |
| 329 | for layer_name in layer_names |
| 330 | } |
| 331 | |
| 332 | with unittest.mock.patch( |
| 333 | "vllm.v1.attention.backends.flashinfer.get_per_layer_parameters", |
| 334 | mock_get_per_layer_parameters, |
| 335 | ): |
| 336 | builder = builder_cls(kv_cache_spec, layer_names, vllm_config, device) |
| 337 | attn_metadata = builder.build( |
| 338 | common_prefix_len=0, |
| 339 | common_attn_metadata=common_attn_metadata, |
| 340 | ) |
| 341 | else: |
| 342 | # Build metadata |
| 343 | builder = builder_cls(kv_cache_spec, layer_names, vllm_config, device) |
| 344 | if backend == AttentionBackendEnum.FLEX_ATTENTION: |
| 345 | builder.direct_build = use_direct_block_mask |
| 346 | attn_metadata = builder.build( |
| 347 | common_prefix_len=0, |
| 348 | common_attn_metadata=common_attn_metadata, |
| 349 | ) |
| 350 | |
| 351 | # Instantiate implementation |
| 352 | num_heads = vllm_config.model_config.get_num_attention_heads( |
| 353 | vllm_config.parallel_config |
| 354 | ) |
| 355 | num_kv_heads = vllm_config.model_config.get_num_kv_heads( |
| 356 | vllm_config.parallel_config |
| 357 | ) |
| 358 | head_size = vllm_config.model_config.get_head_size() |
| 359 | scale = 1.0 / (head_size**0.5) |
| 360 | # Impls capture the current vllm config at construction, as in model loading. |
| 361 | with set_current_vllm_config(vllm_config): |
| 362 | impl = impl_cls( |
| 363 | num_heads=num_heads, |
| 364 | head_size=head_size, |
| 365 | scale=scale, |
| 366 | num_kv_heads=num_kv_heads, |
| 367 | alibi_slopes=None, |
| 368 | sliding_window=sliding_window, |
| 369 | attn_type=attn_type, |
| 370 | kv_cache_dtype=kv_cache_dtype, |
| 371 | **({"sinks": sinks} if sinks is not None else {}), |
| 372 | ) |
| 373 | |
| 374 | # Create mock layer and output buffer |
| 375 | mock_layer = MockAttentionLayer(device, layer_k_scale, layer_v_scale) |
| 376 | output = torch.empty_like(query) |
| 377 | |
| 378 | if is_quantized_kv_cache(kv_cache_dtype) and impl.supports_quant_query_input: |
| 379 | query = query.to(current_platform.fp8_dtype()) |
| 380 | |
| 381 | # Run forward pass |
| 382 | # NOTE: The query, key, and value are already shaped correctly |
| 383 | # in the calling test function. |
| 384 | if not try_backend_includes_kv_cache_update(backend): |
| 385 | impl.do_kv_cache_update( |
| 386 | mock_layer, key, value, kv_cache, attn_metadata.slot_mapping |
| 387 | ) |
| 388 | if use_cuda_graph: |
| 389 | impl.forward( |
| 390 | mock_layer, query, key, value, kv_cache, attn_metadata, output=output |
| 391 | ) |
| 392 | graph = torch.cuda.CUDAGraph() |
| 393 | with torch.cuda.graph(graph): |
| 394 | impl.forward( |
| 395 | mock_layer, |
| 396 | query, |
| 397 | key, |
| 398 | value, |
| 399 | kv_cache, |
| 400 | attn_metadata, |
| 401 | output=output, |
| 402 | ) |
| 403 | graph.replay() |
| 404 | else: |
| 405 | backend_output = impl.forward( |
| 406 | mock_layer, query, key, value, kv_cache, attn_metadata, output=output |
| 407 | ) |
| 408 | if backend_output is not None: |
| 409 | output = backend_output |
| 410 | |
| 411 | return output |
| 412 | |
| 413 | |
| 414 | def _test_backend_correctness( |
| 415 | batch_spec: BatchSpec, |
| 416 | model: str, |
| 417 | backend_to_test: list[AttentionBackendEnum | str], |
| 418 | mask_mod, |
| 419 | *, |
| 420 | causal: bool = True, |
| 421 | attn_type: AttentionType = AttentionType.DECODER, |
| 422 | block_size: int = 16, |
| 423 | atol: float = 1e-2, |
| 424 | rtol: float = 1e-2, |
| 425 | tensor_parallel_size: int = 1, |
| 426 | kv_cache_dtype: str = "auto", |
| 427 | use_sinks: bool = False, |
| 428 | layout: KVCacheLayout | None = None, |
| 429 | use_cuda_graph: bool = False, |
| 430 | num_speculative_tokens: int = 0, |
| 431 | model_dtype: torch.dtype | None = None, |
| 432 | max_num_seqs: int | None = None, |
| 433 | max_num_batched_tokens: int | None = None, |
| 434 | layer_k_scale: float = 1.0, |
| 435 | layer_v_scale: float = 1.0, |
| 436 | ): |
| 437 | """Test that all backends produce similar outputs to a reference implementation |
| 438 | using FlexAttention or an explicit attention-sink reference. |
| 439 | |
| 440 | This test works by: |
| 441 | 1. Generating a batch of sequences with specified context and query lengths. |
| 442 | 2. Computing a ground-truth attention output using torch.sdpa on |
| 443 | contiguous Q, K, and V tensors. |
| 444 | 3. Simulating vLLM's paged KV cache: It takes the context portion of the |
| 445 | K/V tensors and manually places them into a paged buffer according to |
| 446 | the test's (randomly generated) block table. |
| 447 | 4. Running each vLLM attention backend with the new queries and the |
| 448 | simulated paged KV cache. |
| 449 | 5. Comparing the vLLM backend's output to the ground-truth SDPA output. |
| 450 | |
| 451 | Note: When tensor_parallel_size > 1, we simulate the head partitioning |
| 452 | by overriding the model config to use fewer heads, without requiring |
| 453 | multiple GPUs. This tests that backends work correctly with different |
| 454 | head counts. |
| 455 | """ |
| 456 | set_random_seed(42) |
| 457 | |
| 458 | hf_config_override = None |
| 459 | if tensor_parallel_size > 1: |
| 460 | from vllm.config import ModelConfig |
| 461 | |
| 462 | temp_config = ModelConfig(model=model, max_model_len=1) |
| 463 | original_num_heads = temp_config.hf_text_config.num_attention_heads |
| 464 | original_num_kv_heads = getattr( |
| 465 | temp_config.hf_text_config, "num_key_value_heads", None |
| 466 | ) |
| 467 | hf_config_override = { |
| 468 | "num_attention_heads": original_num_heads // tensor_parallel_size, |
| 469 | } |
| 470 | if original_num_kv_heads is not None: |
| 471 | hf_config_override["num_key_value_heads"] = max( |
| 472 | 1, original_num_kv_heads // tensor_parallel_size |
| 473 | ) |
| 474 | |
| 475 | vllm_config = create_vllm_config( |
| 476 | model_name=model, |
| 477 | tensor_parallel_size=1, # Always use TP=1 to avoid multi-GPU requirements |
| 478 | max_model_len=max(batch_spec.seq_lens), |
| 479 | dtype=model_dtype or "auto", |
| 480 | block_size=block_size, |
| 481 | num_gpu_blocks=8192, |
| 482 | hf_config_override=hf_config_override, |
| 483 | ) |
| 484 | if max_num_seqs is not None: |
| 485 | vllm_config.scheduler_config.max_num_seqs = max_num_seqs |
| 486 | if max_num_batched_tokens is not None: |
| 487 | vllm_config.scheduler_config.max_num_batched_tokens = max_num_batched_tokens |
| 488 | if num_speculative_tokens > 0: |
| 489 | vllm_config.speculative_config = SimpleNamespace( |
| 490 | num_speculative_tokens=num_speculative_tokens |
| 491 | ) |
| 492 | vllm_config.cache_config.cache_dtype = kv_cache_dtype |
| 493 | device = torch.device(f"{DEVICE_TYPE}:0") |
| 494 | |
| 495 | kv_cache_spec = create_standard_kv_cache_spec(vllm_config, attn_type) |
| 496 | |
| 497 | # 1. Setup |
| 498 | batch_size = batch_spec.batch_size |
| 499 | seq_lens = batch_spec.seq_lens |
| 500 | query_lens = batch_spec.query_lens |
| 501 | num_q_heads = vllm_config.model_config.get_num_attention_heads( |
| 502 | vllm_config.parallel_config |
| 503 | ) |
| 504 | num_kv_heads = vllm_config.model_config.get_num_kv_heads( |
| 505 | vllm_config.parallel_config |
| 506 | ) |
| 507 | sinks = ( |
| 508 | torch.linspace(-1.0, 1.0, num_q_heads, dtype=torch.float32, device=device) |
| 509 | if use_sinks |
| 510 | else None |
| 511 | ) |
| 512 | head_size = vllm_config.model_config.get_head_size() |
| 513 | sliding_window = vllm_config.model_config.get_sliding_window() |
| 514 | dtype = _convert_dtype_to_torch(vllm_config.model_config.dtype) |
| 515 | block_size = vllm_config.cache_config.block_size |
| 516 | scale = 1.0 / (head_size**0.5) |
| 517 | |
| 518 | fp8_kv_cache = is_quantized_kv_cache(kv_cache_dtype) |
| 519 | if fp8_kv_cache: |
| 520 | query_fp8_dtype = current_platform.fp8_dtype() |
| 521 | kv_fp8_dtype = FP8_KV_CACHE_DTYPES[kv_cache_dtype] |
| 522 | atol = max(atol, 6e-2) |
| 523 | rtol = max(rtol, 1e-1) |
| 524 | |
| 525 | # 2. Generate data and compute SDPA reference output |
| 526 | all_q_vllm, all_k_vllm, all_v_vllm = [], [], [] |
| 527 | all_sdpa_outputs = [] |
| 528 | k_contexts, v_contexts = [], [] |
| 529 | |
| 530 | for i in range(batch_size): |
| 531 | s_len = seq_lens[i] |
| 532 | q_len = query_lens[i] |
| 533 | context_len = s_len - q_len |
| 534 | |
| 535 | # Generate Q, K, V for the whole sequence to be used in SDPA |
| 536 | q = torch.randn(q_len, num_q_heads, head_size, dtype=dtype, device=device) |
| 537 | k_full = torch.randn(s_len, num_kv_heads, head_size, dtype=dtype, device=device) |
| 538 | v_full = torch.randn(s_len, num_kv_heads, head_size, dtype=dtype, device=device) |
| 539 | |
| 540 | if fp8_kv_cache: |
| 541 | q_ref = q.to(query_fp8_dtype).to(dtype) |
| 542 | k_ref = k_full.to(kv_fp8_dtype).to(dtype) |
| 543 | v_ref = v_full.to(kv_fp8_dtype).to(dtype) |
| 544 | else: |
| 545 | q_ref, k_ref, v_ref = q, k_full, v_full |
| 546 | |
| 547 | # SDPA expects (N, H, L, D), so unsqueeze batch and permute |
| 548 | q_sdpa_in = q_ref.unsqueeze(0).transpose(1, 2) |
| 549 | k_sdpa_in = k_ref.unsqueeze(0).transpose(1, 2) |
| 550 | v_sdpa_in = v_ref.unsqueeze(0).transpose(1, 2) |
| 551 | |
| 552 | if num_q_heads != num_kv_heads: |
| 553 | assert num_q_heads % num_kv_heads == 0, ( |
| 554 | f"num_q_heads ({num_q_heads}) must be divisible by " |
| 555 | f"num_kv_heads ({num_kv_heads})" |
| 556 | ) |
| 557 | repeats = num_q_heads // num_kv_heads |
| 558 | k_sdpa_in = k_sdpa_in.repeat_interleave(repeats, dim=1) |
| 559 | v_sdpa_in = v_sdpa_in.repeat_interleave(repeats, dim=1) |
| 560 | |
| 561 | # Create causal mask: query token i attends to positions 0 to |
| 562 | # (context_len + i) |
| 563 | kv_len = s_len |
| 564 | |
| 565 | final_mask_mod = partial(mask_mod, context_len=context_len) |
| 566 | if sinks is None: |
| 567 | block_mask = create_block_mask( |
| 568 | final_mask_mod, |
| 569 | B=None, |
| 570 | H=None, |
| 571 | Q_LEN=q_len, |
| 572 | KV_LEN=kv_len, |
| 573 | device=device, |
| 574 | ) |
| 575 | sdpa_out_i = flex_attention( |
| 576 | q_sdpa_in, |
| 577 | k_sdpa_in, |
| 578 | v_sdpa_in, |
| 579 | block_mask=block_mask, |
| 580 | scale=scale, |
| 581 | enable_gqa=True, |
| 582 | ) |
| 583 | else: |
| 584 | scores = ( |
| 585 | torch.matmul(q_sdpa_in.float(), k_sdpa_in.float().transpose(-2, -1)) |
| 586 | * scale |
| 587 | ) |
| 588 | q_idx = torch.arange(q_len, device=device).unsqueeze(1) |
| 589 | kv_idx = torch.arange(kv_len, device=device).unsqueeze(0) |
| 590 | zero = torch.zeros((), dtype=torch.int32, device=device) |
| 591 | valid = final_mask_mod(zero, zero, q_idx, kv_idx) |
| 592 | scores.masked_fill_(~valid.unsqueeze(0).unsqueeze(0), -torch.inf) |
| 593 | sink_logits = sinks.view(1, num_q_heads, 1, 1).expand(1, -1, q_len, -1) |
| 594 | weights = torch.softmax(torch.cat((scores, sink_logits), dim=-1), dim=-1)[ |
| 595 | ..., :kv_len |
| 596 | ] |
| 597 | sdpa_out_i = torch.matmul(weights, v_sdpa_in.float()).to(dtype) |
| 598 | |
| 599 | all_sdpa_outputs.append(sdpa_out_i.transpose(1, 2).squeeze(0)) |
| 600 | |
| 601 | # Inputs for vLLM backends are just the new tokens |
| 602 | all_q_vllm.append(q) |
| 603 | all_k_vllm.append(k_full[context_len:]) |
| 604 | all_v_vllm.append(v_full[context_len:]) |
| 605 | |
| 606 | # Contextual K/V data used to populate the paged cache |
| 607 | k_contexts.append(k_full[:context_len]) |
| 608 | v_contexts.append(v_full[:context_len]) |
| 609 | |
| 610 | query_vllm = torch.cat(all_q_vllm, dim=0) |
| 611 | key_vllm = torch.cat(all_k_vllm, dim=0) |
| 612 | value_vllm = torch.cat(all_v_vllm, dim=0) |
| 613 | sdpa_output = torch.cat(all_sdpa_outputs, dim=0) |
| 614 | |
| 615 | common_attn_metadata = create_common_attn_metadata( |
| 616 | batch_spec, vllm_config.cache_config.block_size, device |
| 617 | ) |
| 618 | common_attn_metadata.causal = causal |
| 619 | |
| 620 | # 3. Simulate Paged KV Cache and a realistic slot_mapping |
| 621 | # Mirror selector-time resolution locally (caller-requested layout, then |
| 622 | # any backend-required layout, then the default chain). |
| 623 | declared = [ |
| 624 | supported |
| 625 | for backend in backend_to_test |
| 626 | if ( |
| 627 | supported := _actual_backend(backend) |
| 628 | .get_class() |
| 629 | .supported_kv_cache_layouts() |
| 630 | ) |
| 631 | is not None |
| 632 | ] |
| 633 | if layout is None: |
| 634 | # Mirror the resolver: the shared cache uses the declared sets' preferred |
| 635 | # layout; backends that don't support it get a per-backend copy below. |
| 636 | layout = min(declared, key=len)[0] if declared else KVCacheLayout.LBNHC |
| 637 | kv_cache = create_and_prepopulate_kv_cache( |
| 638 | k_contexts=k_contexts, |
| 639 | v_contexts=v_contexts, |
| 640 | block_size=block_size, |
| 641 | num_kv_heads=num_kv_heads, |
| 642 | head_size=head_size, |
| 643 | dtype=dtype, |
| 644 | device=device, |
| 645 | num_blocks=vllm_config.cache_config.num_gpu_blocks or 1000, |
| 646 | common_attn_metadata=common_attn_metadata, |
| 647 | layout=layout, |
| 648 | randomize_blocks=True, |
| 649 | kv_cache_dtype=kv_cache_dtype, |
| 650 | ) |
| 651 | |
| 652 | # 4. Run vLLM backends and compare |
| 653 | # Note: flex_attention has known Triton kernel compatibility issues |
| 654 | # with test infrastructures |
| 655 | for backend_name in backend_to_test: |
| 656 | backend_cls = _actual_backend(backend_name).get_class() |
| 657 | |
| 658 | if is_quantized_kv_cache(kv_cache_dtype) and ( |
| 659 | not backend_cls.supports_kv_cache_dtype(kv_cache_dtype) |
| 660 | ): |
| 661 | continue |
| 662 | |
| 663 | kv_cache_for_backend = kv_cache |
| 664 | backend_layout = layout |
| 665 | |
| 666 | backend_supported = backend_cls.supported_kv_cache_layouts() |
| 667 | if backend_supported is not None and backend_layout not in backend_supported: |
| 668 | # Production resolution never pairs this backend with the shared layout |
| 669 | # (e.g. flex is LBNHC-only next to FlashInfer's LBHNC); give it a copy |
| 670 | # of the cache in a layout it supports. |
| 671 | backend_layout = backend_supported[0] |
| 672 | kv_cache_for_backend = _clone_kv_cache_in_layout(kv_cache, backend_layout) |
| 673 | |
| 674 | if backend_name == AttentionBackendEnum.B12X: |
| 675 | cache_dtype = ( |
| 676 | FP8_KV_CACHE_DTYPES[kv_cache_dtype] |
| 677 | if is_quantized_kv_cache(kv_cache_dtype) |
| 678 | else kv_cache.dtype |
| 679 | ) |
| 680 | typed_cache = kv_cache.view(cache_dtype) |
| 681 | key_cache = typed_cache[..., :head_size].permute(0, 2, 1, 3) |
| 682 | value_cache = typed_cache[..., head_size:].permute(0, 2, 1, 3) |
| 683 | packed_cache = torch.stack((key_cache, value_cache), dim=1).flatten(-2) |
| 684 | if is_quantized_kv_cache(kv_cache_dtype): |
| 685 | packed_cache = packed_cache.view(torch.uint8) |
| 686 | kv_cache_for_backend = _clone_kv_cache_in_layout( |
| 687 | packed_cache, backend_layout |
| 688 | ) |
| 689 | |
| 690 | # FlashInfer reads the layout at plan time; set it to match |
| 691 | # the physical order of the test cache. |
| 692 | vllm_config.cache_config.kv_cache_layout = backend_layout.name |
| 693 | |
| 694 | backend_output = run_attention_backend( |
| 695 | backend_name, |
| 696 | kv_cache_spec, |
| 697 | ["placeholder"], |
| 698 | vllm_config, |
| 699 | device, |
| 700 | common_attn_metadata, |
| 701 | query_vllm, |
| 702 | key_vllm, |
| 703 | value_vllm, |
| 704 | kv_cache_for_backend, |
| 705 | sliding_window=sliding_window, |
| 706 | attn_type=attn_type, |
| 707 | kv_cache_dtype=kv_cache_dtype, |
| 708 | sinks=sinks, |
| 709 | use_cuda_graph=use_cuda_graph, |
| 710 | layer_k_scale=layer_k_scale, |
| 711 | layer_v_scale=layer_v_scale, |
| 712 | ) |
| 713 | |
| 714 | # Check shape and dtype consistency |
| 715 | assert backend_output.shape == sdpa_output.shape, ( |
| 716 | f"[{backend_name}] shape {backend_output.shape} != " |
| 717 | f"SDPA shape {sdpa_output.shape}" |
| 718 | ) |
| 719 | assert backend_output.dtype == sdpa_output.dtype, ( |
| 720 | f"[{backend_name}] dtype {backend_output.dtype} != " |
| 721 | f"SDPA dtype {sdpa_output.dtype}" |
| 722 | ) |
| 723 | |
| 724 | assert torch.isfinite(backend_output).all(), ( |
| 725 | f"[{backend_name}] produced non-finite values" |
| 726 | ) |
| 727 | |
| 728 | # Check numerical similarity |
| 729 | def error_msg(msg: str, backend_name: AttentionBackendEnum | str): |
| 730 | return f"[{backend_name}] output differs from SDPA baseline. {msg}" |
| 731 | |
| 732 | torch.testing.assert_close( |
| 733 | backend_output, |
| 734 | sdpa_output, |
| 735 | rtol=rtol, |
| 736 | atol=atol, |
| 737 | msg=partial(error_msg, backend_name=backend_name), |
| 738 | ) |
| 739 | |
| 740 | |
| 741 | @pytest.mark.parametrize("layout", ["BLHNC", "BHLNC"]) |
| 742 | @pytest.mark.parametrize("batch_spec_name", ["small_decode", "small_prefill"]) |
| 743 | @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) |
| 744 | def test_flashinfer_cross_layer_layout( |
| 745 | default_vllm_config, |
| 746 | layout: str, |
| 747 | batch_spec_name: str, |
| 748 | kv_cache_dtype: str, |
| 749 | ): |
| 750 | if AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST: |
| 751 | pytest.skip("FlashInfer is not installed") |
| 752 | |
| 753 | def causal_mask_mod( |
| 754 | b: torch.Tensor, |
| 755 | h: torch.Tensor, |
| 756 | q_idx: torch.Tensor, |
| 757 | kv_idx: torch.Tensor, |
| 758 | *, |
| 759 | context_len: int, |
| 760 | ): |
| 761 | return (q_idx + context_len) >= kv_idx |
| 762 | |
| 763 | _test_backend_correctness( |
| 764 | batch_spec=BATCH_SPECS[batch_spec_name], |
| 765 | model="meta-llama/Meta-Llama-3-8B", |
| 766 | backend_to_test=[AttentionBackendEnum.FLASHINFER], |
| 767 | mask_mod=causal_mask_mod, |
| 768 | kv_cache_dtype=kv_cache_dtype, |
| 769 | layout=KVCacheLayout[layout], |
| 770 | ) |
| 771 | |
| 772 | |
| 773 | @pytest.mark.parametrize( |
| 774 | "batch_spec_name", |
| 775 | [ |
| 776 | "small_decode", |
| 777 | "small_prefill", |
| 778 | "mixed_small", |
| 779 | "medium_decode", |
| 780 | "medium_prefill", |
| 781 | "mixed_medium", |
| 782 | "large_decode", |
| 783 | "large_prefill", |
| 784 | "single_decode", |
| 785 | "single_prefill", |
| 786 | ], |
| 787 | ) |
| 788 | @pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"]) |
| 789 | @pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4]) |
| 790 | @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"]) |
| 791 | def test_causal_backend_correctness( |
| 792 | default_vllm_config, |
| 793 | batch_spec_name: str, |
| 794 | model: str, |
| 795 | tensor_parallel_size: int, |
| 796 | kv_cache_dtype: str, |
| 797 | ): |
| 798 | """Test backend's correctness with causal attention.""" |
| 799 | |
| 800 | def causal_mask_mod( |
| 801 | b: torch.Tensor, |
| 802 | h: torch.Tensor, |
| 803 | q_idx: torch.Tensor, |
| 804 | kv_idx: torch.Tensor, |
| 805 | *, |
| 806 | context_len: int, |
| 807 | ): |
| 808 | return (q_idx + context_len) >= kv_idx |
| 809 | |
| 810 | batch_spec = BATCH_SPECS[batch_spec_name] |
| 811 | LARGE_BLOCK_BACKENDS = ( |
| 812 | [AttentionBackendEnum.FLEX_ATTENTION] |
| 813 | if is_torch_equal_or_newer("2.9.0.dev0") |
| 814 | else [] |
| 815 | ) |
| 816 | |
| 817 | if current_platform.is_rocm(): |
| 818 | SMALL_BLOCK_BACKENDS = [ |
| 819 | x |
| 820 | for x in BACKENDS_TO_TEST |
| 821 | if ( |
| 822 | x not in LARGE_BLOCK_BACKENDS |
| 823 | and x is not AttentionBackendEnum.FLASH_ATTN |
| 824 | ) |
| 825 | ] |
| 826 | else: |
| 827 | SMALL_BLOCK_BACKENDS = [ |
| 828 | x for x in BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS |
| 829 | ] |
| 830 | |
| 831 | _test_backend_correctness( |
| 832 | batch_spec, |
| 833 | model, |
| 834 | SMALL_BLOCK_BACKENDS, |
| 835 | causal_mask_mod, |
| 836 | tensor_parallel_size=tensor_parallel_size, |
| 837 | kv_cache_dtype=kv_cache_dtype, |
| 838 | ) |
| 839 | |
| 840 | # Fast FlexAttention needs to run with block_size=128 |
| 841 | if LARGE_BLOCK_BACKENDS: |
| 842 | _test_backend_correctness( |
| 843 | batch_spec, |
| 844 | model, |
| 845 | LARGE_BLOCK_BACKENDS, |
| 846 | causal_mask_mod, |
| 847 | block_size=128, |
| 848 | tensor_parallel_size=tensor_parallel_size, |
| 849 | kv_cache_dtype=kv_cache_dtype, |
| 850 | ) |
| 851 | |
| 852 | |
| 853 | @pytest.mark.skipif( |
| 854 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 855 | reason="FlashInfer is not available.", |
| 856 | ) |
| 857 | def test_flashinfer_xqa_bmm1_scale_matches_decode_q_dtype(): |
| 858 | """XQA decode should only apply q_scale when decode Q is FP8.""" |
| 859 | from vllm.v1.attention.backends import flashinfer as flashinfer_backend |
| 860 | |
| 861 | class MockLayer: |
| 862 | _q_scale_float = 2.0 |
| 863 | _k_scale_float = 3.0 |
| 864 | |
| 865 | impl = object.__new__(flashinfer_backend.FlashInferImpl) |
| 866 | impl.scale = 0.5 |
| 867 | impl.kv_cache_dtype = "fp8" |
| 868 | |
| 869 | assert impl.get_xqa_bmm1_scale(MockLayer, torch.bfloat16) == 1.5 |
| 870 | assert impl.get_xqa_bmm1_scale(MockLayer, torch.float8_e4m3fn) == 3.0 |
| 871 | |
| 872 | |
| 873 | @pytest.mark.skipif( |
| 874 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 875 | reason="FlashInfer is not available.", |
| 876 | ) |
| 877 | def test_flashinfer_xqa_draft_masks(): |
| 878 | from vllm.v1.attention.backends import flashinfer as flashinfer_backend |
| 879 | |
| 880 | device = torch.device("cpu") |
| 881 | causal = flashinfer_backend._make_xqa_draft_block_mask(3, True, device) |
| 882 | full = flashinfer_backend._make_xqa_draft_block_mask(3, False, device) |
| 883 | ragged = flashinfer_backend._make_xqa_ragged_draft_block_mask( |
| 884 | [2, 3], 3, True, device |
| 885 | ) |
| 886 | |
| 887 | assert torch.equal( |
| 888 | causal.view(torch.int16), |
| 889 | torch.tensor([[1, 0], [3, 0], [7, 0]], dtype=torch.int16), |
| 890 | ) |
| 891 | assert torch.equal( |
| 892 | full.view(torch.int16), torch.tensor([[7, 0]] * 3, dtype=torch.int16) |
| 893 | ) |
| 894 | assert torch.equal( |
| 895 | ragged.view(torch.int16), |
| 896 | torch.tensor( |
| 897 | [[1, 0], [3, 0], [1, 0], [3, 0], [7, 0]], |
| 898 | dtype=torch.int16, |
| 899 | ), |
| 900 | ) |
| 901 | |
| 902 | |
| 903 | @pytest.mark.skipif( |
| 904 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 905 | reason="FlashInfer is not available.", |
| 906 | ) |
| 907 | def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding(): |
| 908 | """CUDA-graph padding stays as zero-length requests in ragged offsets.""" |
| 909 | from vllm.v1.attention.backends import flashinfer as flashinfer_backend |
| 910 | |
| 911 | device = torch.device("cpu") |
| 912 | builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder) |
| 913 | builder.use_xqa = True |
| 914 | qo_indptr = torch.tensor([0, 3, 9, 15, 15], dtype=torch.int32, device=device) |
| 915 | |
| 916 | q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens( |
| 917 | qo_indptr, |
| 918 | qo_indptr, |
| 919 | num_decodes=4, |
| 920 | num_decode_tokens=21, |
| 921 | ) |
| 922 | |
| 923 | assert q_len == 6 |
| 924 | assert q_lens == [3, 6, 6, 0] |
| 925 | assert q_cu_seq_lens is not None |
| 926 | assert q_cu_seq_lens.tolist() == [0, 3, 9, 15, 15] |
| 927 | |
| 928 | |
| 929 | @pytest.mark.skipif( |
| 930 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 931 | reason="FlashInfer is not available.", |
| 932 | ) |
| 933 | def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypatch): |
| 934 | """Non-ragged XQA keeps one query/output row per padded request.""" |
| 935 | import unittest.mock |
| 936 | |
| 937 | from vllm.v1.attention.backends import flashinfer as flashinfer_backend |
| 938 | |
| 939 | impl = object.__new__(flashinfer_backend.FlashInferImpl) |
| 940 | impl.scale = 1.0 |
| 941 | impl.bmm1_scale = None |
| 942 | impl.bmm2_scale = None |
| 943 | impl.kv_cache_dtype = "auto" |
| 944 | impl.is_kvcache_nvfp4 = False |
| 945 | impl.head_size = 16 |
| 946 | impl.dcp_world_size = 1 |
| 947 | impl.o_sf_scale = None |
| 948 | impl.window_left = -1 |
| 949 | impl.sinks = None |
| 950 | |
| 951 | layer = unittest.mock.Mock( |
| 952 | _q_scale=torch.tensor(1.0), |
| 953 | _q_scale_float=1.0, |
| 954 | _k_scale_float=1.0, |
| 955 | _v_scale_float=1.0, |
| 956 | ) |
| 957 | decode = flashinfer_backend.FlashInferTrtllmAPIDecode( |
| 958 | kernel=flashinfer_backend.FlashInferDecodeKernel.XQA, |
| 959 | block_tables=torch.zeros((4, 1), dtype=torch.int32), |
| 960 | seq_lens=torch.tensor([8, 8, 0, 0], dtype=torch.int32), |
| 961 | max_seq_len=8, |
| 962 | q_len_per_req=1, |
| 963 | ) |
| 964 | attn_metadata = flashinfer_backend.FlashInferMetadata( |
| 965 | kv_cache_layout=KVCacheLayout.LBHNC, |
| 966 | num_actual_tokens=2, |
| 967 | slot_mapping=torch.empty(0, dtype=torch.int64), |
| 968 | q_data_type_prefill=torch.bfloat16, |
| 969 | q_data_type_decode=torch.bfloat16, |
| 970 | num_decodes=4, |
| 971 | num_decode_tokens=2, |
| 972 | num_prefills=0, |
| 973 | num_prefill_tokens=0, |
| 974 | causal=True, |
| 975 | prefill=None, |
| 976 | decode=decode, |
| 977 | use_cascade=False, |
| 978 | cascade_wrapper=None, |
| 979 | ) |
| 980 | |
| 981 | seen_shapes = {} |
| 982 | |
| 983 | def mock_xqa(**kwargs): |
| 984 | seen_shapes["query"] = kwargs["query"].shape |
| 985 | seen_shapes["out"] = kwargs["out"].shape |
| 986 | seen_shapes["block_tables"] = kwargs["block_tables"].shape |
| 987 | |
| 988 | monkeypatch.setattr( |
| 989 | flashinfer_backend, |
| 990 | "_get_trtllm_workspace_buffer", |
| 991 | lambda: torch.empty(1, dtype=torch.uint8), |
| 992 | ) |
| 993 | monkeypatch.setattr( |
| 994 | flashinfer_backend, "flashinfer_xqa_batch_decode_with_kv_cache", mock_xqa |
| 995 | ) |
| 996 | |
| 997 | query = torch.zeros((4, 1, 16), dtype=torch.bfloat16) |
| 998 | output = torch.empty_like(query) |
| 999 | result = impl.forward( |
| 1000 | layer, |
| 1001 | query, |
| 1002 | torch.empty_like(query), |
| 1003 | torch.empty_like(query), |
| 1004 | torch.zeros((1, 1, 1, 32), dtype=torch.bfloat16), |
| 1005 | attn_metadata, |
| 1006 | output, |
| 1007 | ) |
| 1008 | |
| 1009 | assert result is output |
| 1010 | assert seen_shapes == { |
| 1011 | "query": torch.Size([4, 1, 16]), |
| 1012 | "out": torch.Size([4, 1, 16]), |
| 1013 | "block_tables": torch.Size([4, 1]), |
| 1014 | } |
| 1015 | |
| 1016 | |
| 1017 | @pytest.mark.skipif( |
| 1018 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 1019 | reason="FlashInfer is not available.", |
| 1020 | ) |
| 1021 | def test_flashinfer_xqa_query_lens_require_exact_uniform_product(): |
| 1022 | from vllm.v1.attention.backends import flashinfer as flashinfer_backend |
| 1023 | |
| 1024 | builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder) |
| 1025 | builder.use_xqa = True |
| 1026 | qo_indptr = torch.tensor([0, 3, 3, 3], dtype=torch.int32) |
| 1027 | |
| 1028 | q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens( |
| 1029 | qo_indptr, |
| 1030 | qo_indptr, |
| 1031 | num_decodes=3, |
| 1032 | num_decode_tokens=6, |
| 1033 | ) |
| 1034 | |
| 1035 | assert q_len == 3 |
| 1036 | assert q_lens == [3, 0, 0] |
| 1037 | assert q_cu_seq_lens is not None |
| 1038 | assert q_cu_seq_lens.tolist() == [0, 3, 3, 3] |
| 1039 | |
| 1040 | |
| 1041 | @pytest.mark.skipif( |
| 1042 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 1043 | reason="FlashInfer is not available.", |
| 1044 | ) |
| 1045 | @pytest.mark.parametrize( |
| 1046 | ("qo_indptr_values", "num_decode_tokens", "expected_q_len"), |
| 1047 | [ |
| 1048 | ([0, 3, 3], 3, 3), |
| 1049 | ([0, 2, 4, 4, 4], 4, 2), |
| 1050 | ], |
| 1051 | ) |
| 1052 | def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets( |
| 1053 | qo_indptr_values: list[int], |
| 1054 | num_decode_tokens: int, |
| 1055 | expected_q_len: int, |
| 1056 | ): |
| 1057 | """Padded speculative decode keeps its actual packed query width.""" |
| 1058 | from vllm.v1.attention.backends import flashinfer as flashinfer_backend |
| 1059 | |
| 1060 | builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder) |
| 1061 | builder.use_xqa = False |
| 1062 | qo_indptr = torch.tensor(qo_indptr_values, dtype=torch.int32) |
| 1063 | |
| 1064 | q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens( |
| 1065 | qo_indptr, |
| 1066 | qo_indptr, |
| 1067 | num_decodes=len(qo_indptr_values) - 1, |
| 1068 | num_decode_tokens=num_decode_tokens, |
| 1069 | ) |
| 1070 | |
| 1071 | assert q_len == expected_q_len |
| 1072 | assert q_lens is None |
| 1073 | assert q_cu_seq_lens is not None |
| 1074 | assert q_cu_seq_lens.tolist() == qo_indptr_values |
| 1075 | |
| 1076 | |
| 1077 | @pytest.mark.skipif( |
| 1078 | not current_platform.is_cuda() |
| 1079 | or AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 1080 | reason="FlashInfer with CUDA is required.", |
| 1081 | ) |
| 1082 | @pytest.mark.parametrize( |
| 1083 | "runner_type,spec_kind,has_upper_bound,expected_copies", |
| 1084 | [ |
| 1085 | ("generate", None, True, 0), |
| 1086 | ("pooling", None, True, 0), |
| 1087 | ("generate", None, False, 1), |
| 1088 | ("generate", "spec_decode", True, 1), |
| 1089 | ("generate", "diffusion", True, 1), |
| 1090 | ], |
| 1091 | ) |
| 1092 | def test_flashinfer_avoids_seq_lens_copy_without_spec_tokens( |
| 1093 | monkeypatch, runner_type, spec_kind, has_upper_bound, expected_copies |
| 1094 | ): |
| 1095 | """Planning reuses the host seq_lens bound only when it is exact. |
| 1096 | |
| 1097 | Without speculative token accounting (no spec-decode drafts, no dLLM |
| 1098 | canvas tokens) scheduled tokens always land in KV, so the CPU upper bound |
| 1099 | is exact regardless of runner type. Any speculative accounting keeps the |
| 1100 | bound optimistic (rejected tokens are in-flight) and forces the copy. |
| 1101 | """ |
| 1102 | import unittest.mock |
| 1103 | |
| 1104 | from vllm.v1.attention.backends import flashinfer as flashinfer_backend |
| 1105 | from vllm.v1.attention.backends.utils import PerLayerParameters |
| 1106 | |
| 1107 | monkeypatch.setattr( |
| 1108 | flashinfer_backend, |
| 1109 | "get_per_layer_parameters", |
| 1110 | lambda *args: { |
| 1111 | "layer.0": PerLayerParameters( |
| 1112 | window_left=-1, logits_soft_cap=0.0, sm_scale=0.125 |
| 1113 | ) |
| 1114 | }, |
| 1115 | ) |
| 1116 | |
| 1117 | config = create_vllm_config( |
| 1118 | model_name="Qwen/Qwen3-0.6B", max_model_len=64, max_num_seqs=3 |
| 1119 | ) |
| 1120 | config.model_config.runner_type = runner_type |
| 1121 | config.attention_config.use_trtllm_attention = False |
| 1122 | if spec_kind == "spec_decode": |
| 1123 | config.speculative_config = SpeculativeConfig( |
| 1124 | method="ngram", num_speculative_tokens=3 |
| 1125 | ) |
| 1126 | elif spec_kind == "diffusion": |
| 1127 | config.diffusion_config = DiffusionConfig(canvas_length=8) |
| 1128 | device = torch.device(f"{DEVICE_TYPE}:0") |
| 1129 | # Include a one-token final chunk and both partial and full KV pages. |
| 1130 | common_attn_metadata = create_common_attn_metadata( |
| 1131 | BatchSpec(seq_lens=[17, 20, 32], query_lens=[1, 4, 16]), 16, device |
| 1132 | ) |
| 1133 | if not has_upper_bound: |
| 1134 | common_attn_metadata.seq_lens_cpu_upper_bound = None |
| 1135 | elif spec_kind is not None: |
| 1136 | # Simulate rejected speculative tokens: CPU says 18, GPU says 17. |
| 1137 | assert common_attn_metadata.seq_lens_cpu_upper_bound is not None |
| 1138 | common_attn_metadata.seq_lens_cpu_upper_bound[0] += 1 |
| 1139 | |
| 1140 | with set_current_vllm_config(config): |
| 1141 | builder = flashinfer_backend.FlashInferMetadataBuilder( |
| 1142 | create_standard_kv_cache_spec(config), ["layer.0"], config, device |
| 1143 | ) |
| 1144 | with unittest.mock.patch.object( |
| 1145 | common_attn_metadata.seq_lens, |
| 1146 | "cpu", |
| 1147 | wraps=common_attn_metadata.seq_lens.cpu, |
| 1148 | ) as copy_to_cpu: |
| 1149 | metadata = builder.build(0, common_attn_metadata) |
| 1150 | |
| 1151 | assert copy_to_cpu.call_count == expected_copies |
| 1152 | assert metadata.num_decodes == 1 |
| 1153 | assert metadata.num_prefills == 2 |
| 1154 | assert builder.paged_kv_indptr.cpu[:4].tolist() == [0, 2, 4, 6] |
| 1155 | assert builder.paged_kv_last_page_len.cpu[:3].tolist() == [1, 4, 16] |
| 1156 | |
| 1157 | |
| 1158 | @pytest.mark.skipif( |
| 1159 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 1160 | reason="FlashInfer is not available.", |
| 1161 | ) |
| 1162 | @pytest.mark.parametrize( |
| 1163 | "adaptive,decode_kernel,dcp_size,expected_bound", |
| 1164 | [ |
| 1165 | (None, "TRTLLM_GEN", 1, None), |
| 1166 | (False, "TRTLLM_GEN", 1, None), |
| 1167 | (True, "TRTLLM_GEN", 1, 8), |
| 1168 | (True, "XQA", 1, None), |
| 1169 | (True, "TRTLLM_GEN", 2, None), |
| 1170 | ], |
| 1171 | ) |
| 1172 | def test_flashinfer_varlen_cudagraph_capability( |
| 1173 | monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound |
| 1174 | ): |
| 1175 | """Only the trtllm-gen path that adaptive verification enables replays |
| 1176 | varlen decode graphs, up to the decode width; the uniform level is fixed.""" |
| 1177 | from vllm.model_executor.layers.attention.chunked_local_attention import ( |
| 1178 | create_chunked_local_attention_backend, |
| 1179 | ) |
| 1180 | from vllm.v1.attention.backends import flashinfer as fi |
| 1181 | |
| 1182 | config = SimpleNamespace( |
| 1183 | attention_config=SimpleNamespace(use_non_causal=False), |
| 1184 | speculative_config=SimpleNamespace( |
| 1185 | enable_adaptive_verification=adaptive, |
| 1186 | num_speculative_tokens=7, |
| 1187 | parallel_drafting=True, |
| 1188 | ) |
| 1189 | if adaptive is not None |
| 1190 | else None, |
| 1191 | parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size), |
| 1192 | model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32), |
| 1193 | use_v2_model_runner=True, |
| 1194 | ) |
| 1195 | monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True) |
| 1196 | monkeypatch.setattr( |
| 1197 | fi.FlashInferMetadataBuilder, |
| 1198 | "_get_flashinfer_trtllm_api_decode_kernel", |
| 1199 | staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]), |
| 1200 | ) |
| 1201 | spec = FullAttentionSpec( |
| 1202 | block_size=16, |
| 1203 | num_kv_heads=4, |
| 1204 | head_size=64, |
| 1205 | dtype=torch.bfloat16, |
| 1206 | ) |
| 1207 | builder_cls = fi.FlashInferMetadataBuilder |
| 1208 | assert builder_cls.get_cudagraph_support(config, spec) == ( |
| 1209 | AttentionCGSupport.UNIFORM_BATCH |
| 1210 | if dcp_size == 1 |
| 1211 | else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE |
| 1212 | ) |
| 1213 | bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec) |
| 1214 | assert bound == expected_bound |
| 1215 | if bound is not None: |
| 1216 | # A wrapper forcing NEVER inherits the override but not the bound. |
| 1217 | chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16) |
| 1218 | wrapped_cls = chunked_local.get_builder_cls() |
| 1219 | assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None |
| 1220 | |
| 1221 | |
| 1222 | @pytest.mark.parametrize( |
| 1223 | "use_v2_model_runner,parallel_drafting,expected", |
| 1224 | [ |
| 1225 | (True, None, 1), |
| 1226 | (True, False, 8), |
| 1227 | (True, True, 8), |
| 1228 | (False, False, 8), |
| 1229 | (False, True, 15), |
| 1230 | ], |
| 1231 | ) |
| 1232 | def test_spec_as_decode_width_follows_model_runner( |
| 1233 | use_v2_model_runner, parallel_drafting, expected |
| 1234 | ): |
| 1235 | """Only model runner V1's parallel drafter widens decode requests.""" |
| 1236 | |
| 1237 | class Builder(AttentionMetadataBuilder): |
| 1238 | def __init__(self, vllm_config): |
| 1239 | self.vllm_config = vllm_config |
| 1240 | |
| 1241 | def build(self, common_prefix_len, common_attn_metadata, fast_build=False): |
| 1242 | raise NotImplementedError |
| 1243 | |
| 1244 | config = SimpleNamespace( |
| 1245 | speculative_config=None |
| 1246 | if parallel_drafting is None |
| 1247 | else SimpleNamespace( |
| 1248 | num_speculative_tokens=7, parallel_drafting=parallel_drafting |
| 1249 | ), |
| 1250 | parallel_config=SimpleNamespace(decode_context_parallel_size=1), |
| 1251 | use_v2_model_runner=use_v2_model_runner, |
| 1252 | ) |
| 1253 | assert max_decode_query_len(config) == expected |
| 1254 | builder = Builder(config) |
| 1255 | builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True) |
| 1256 | assert builder.reorder_batch_threshold == expected |
| 1257 | |
| 1258 | |
| 1259 | @pytest.mark.skipif( |
| 1260 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST |
| 1261 | or not current_platform.is_device_capability_family(100), |
| 1262 | reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs", |
| 1263 | ) |
| 1264 | @pytest.mark.parametrize( |
| 1265 | "head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads", |
| 1266 | [ |
| 1267 | pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"), |
| 1268 | pytest.param( |
| 1269 | 256, |
| 1270 | 1024, |
| 1271 | 40, |
| 1272 | 32, |
| 1273 | 16, |
| 1274 | 8, |
| 1275 | id="gemma4-sliding-gqa", |
| 1276 | ), |
| 1277 | pytest.param( |
| 1278 | 512, |
| 1279 | None, |
| 1280 | 40, |
| 1281 | 32, |
| 1282 | 16, |
| 1283 | 1, |
| 1284 | id="gemma4-global-mqa", |
| 1285 | ), |
| 1286 | ], |
| 1287 | ) |
| 1288 | def test_flashinfer_varlen_decode_graph_replays_changed_layout( |
| 1289 | tmp_path, |
| 1290 | monkeypatch, |
| 1291 | head_size, |
| 1292 | sliding_window, |
| 1293 | num_tokens, |
| 1294 | num_reqs, |
| 1295 | num_heads, |
| 1296 | num_kv_heads, |
| 1297 | ): |
| 1298 | """Replay changes device offsets, active slots, context, and KV writes. |
| 1299 | |
| 1300 | The graph is captured with evenly spread dummy queries and an explicit |
| 1301 | bound of eight. CPU offsets then remain stale while device offsets change. |
| 1302 | Compare real output rows with independent FP32 causal/windowed attention, |
| 1303 | and verify cache writes including preservation of the null padding block. |
| 1304 | """ |
| 1305 | from vllm.v1.attention.backends import flashinfer as fi |
| 1306 | from vllm.v1.attention.backends.utils import PerLayerParameters |
| 1307 | |
| 1308 | set_random_seed(0) |
| 1309 | monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") |
| 1310 | block_size = 16 |
| 1311 | dtype = torch.bfloat16 |
| 1312 | LlamaConfig( |
| 1313 | architectures=["LlamaForCausalLM"], |
| 1314 | hidden_size=num_heads * head_size, |
| 1315 | head_dim=head_size, |
| 1316 | num_attention_heads=num_heads, |
| 1317 | num_key_value_heads=num_kv_heads, |
| 1318 | num_hidden_layers=1, |
| 1319 | max_position_embeddings=4096, |
| 1320 | ).save_pretrained(tmp_path) |
| 1321 | num_blocks = 1 + num_reqs * cdiv(2112, block_size) |
| 1322 | config = create_vllm_config( |
| 1323 | model_name=str(tmp_path), |
| 1324 | max_model_len=4096, |
| 1325 | dtype=dtype, |
| 1326 | num_gpu_blocks=num_blocks, |
| 1327 | block_size=block_size, |
| 1328 | max_num_seqs=32, |
| 1329 | max_num_batched_tokens=256, |
| 1330 | ) |
| 1331 | config.cache_config.kv_cache_layout = "BLHNC" |
| 1332 | config.speculative_config = SimpleNamespace( |
| 1333 | num_speculative_tokens=7, |
| 1334 | enable_adaptive_verification=True, |
| 1335 | parallel_drafting=False, |
| 1336 | ) |
| 1337 | spec = create_standard_kv_cache_spec(config) |
| 1338 | scale = head_size**-0.5 |
| 1339 | monkeypatch.setattr( |
| 1340 | fi, |
| 1341 | "get_per_layer_parameters", |
| 1342 | lambda *_: { |
| 1343 | "layer": PerLayerParameters( |
| 1344 | window_left=-1 if sliding_window is None else sliding_window - 1, |
| 1345 | logits_soft_cap=0.0, |
| 1346 | sm_scale=scale, |
| 1347 | has_sinks=False, |
| 1348 | ), |
| 1349 | }, |
| 1350 | ) |
| 1351 | device = torch.device("cuda") |
| 1352 | layout = KVCacheLayout.BLHNC |
| 1353 | query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device) |
| 1354 | key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device) |
| 1355 | value = torch.empty_like(key) |
| 1356 | output = torch.empty_like(query) |
| 1357 | dummy_lens = [ |
| 1358 | num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs) |
| 1359 | ] |
| 1360 | common = create_common_attn_metadata( |
| 1361 | BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens), |
| 1362 | block_size, |
| 1363 | device, |
| 1364 | ) |
| 1365 | common.max_query_len = 8 |
| 1366 | cache = None |
| 1367 | |
| 1368 | def prepare(query_lens): |
| 1369 | nonlocal cache |
| 1370 | query_lens = query_lens + [0] * (num_reqs - len(query_lens)) |
| 1371 | query.normal_() |
| 1372 | key.normal_() |
| 1373 | value.normal_() |
| 1374 | # Alternating short prefills, nonempty decode contexts, and contexts |
| 1375 | # beyond the sliding window; inactive requests have zero sequence length. |
| 1376 | contexts = [ |
| 1377 | ([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens) |
| 1378 | ] |
| 1379 | ks = [ |
| 1380 | torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device) |
| 1381 | for c in contexts |
| 1382 | ] |
| 1383 | vs = [torch.randn_like(k) for k in ks] |
| 1384 | actual = create_common_attn_metadata( |
| 1385 | BatchSpec( |
| 1386 | seq_lens=[c + q for c, q in zip(contexts, query_lens)], |
| 1387 | query_lens=query_lens, |
| 1388 | ), |
| 1389 | block_size, |
| 1390 | device, |
| 1391 | ) |
| 1392 | fresh_cache = create_and_prepopulate_kv_cache( |
| 1393 | ks, |
| 1394 | vs, |
| 1395 | block_size, |
| 1396 | num_kv_heads, |
| 1397 | head_size, |
| 1398 | dtype, |
| 1399 | device, |
| 1400 | num_blocks, |
| 1401 | actual, |
| 1402 | layout, |
| 1403 | randomize_blocks=True, |
| 1404 | ) |
| 1405 | if cache is None: |
| 1406 | cache = fresh_cache |
| 1407 | else: |
| 1408 | cache.copy_(fresh_cache) |
| 1409 | common.query_start_loc.copy_(actual.query_start_loc) |
| 1410 | common.seq_lens.copy_(actual.seq_lens) |
| 1411 | common.block_table_tensor.zero_() |
| 1412 | common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_( |
| 1413 | actual.block_table_tensor |
| 1414 | ) |
| 1415 | common.slot_mapping.fill_(-1) |
| 1416 | common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping) |
| 1417 | return contexts, ks, vs |
| 1418 | |
| 1419 | prepare(dummy_lens) |
| 1420 | with set_current_vllm_config(config): |
| 1421 | builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device) |
| 1422 | metadata = builder.build(0, common) |
| 1423 | assert metadata.num_prefills == 0 |
| 1424 | assert metadata.decode.q_len_per_req == 8 |
| 1425 | impl = fi.FlashInferImpl( |
| 1426 | num_heads=num_heads, |
| 1427 | num_kv_heads=num_kv_heads, |
| 1428 | head_size=head_size, |
| 1429 | scale=scale, |
| 1430 | alibi_slopes=None, |
| 1431 | sliding_window=sliding_window, |
| 1432 | kv_cache_dtype="auto", |
| 1433 | attn_type=AttentionType.DECODER, |
| 1434 | ) |
| 1435 | layer = MockAttentionLayer(device) |
| 1436 | |
| 1437 | def forward(attn_metadata=metadata): |
| 1438 | impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping) |
| 1439 | impl.forward(layer, query, key, value, cache, attn_metadata, output=output) |
| 1440 | |
| 1441 | for _ in range(3): |
| 1442 | forward() |
| 1443 | torch.accelerator.synchronize() |
| 1444 | graph = torch.cuda.CUDAGraph() |
| 1445 | with torch.cuda.graph(graph): |
| 1446 | forward() |
| 1447 | |
| 1448 | layouts = [[8], [1] * num_reqs] |
| 1449 | if num_tokens >= 16: |
| 1450 | layouts.append([8, 4, 3, 1]) |
| 1451 | if num_tokens >= 40: |
| 1452 | layouts.append([8] * 5) |
| 1453 | |
| 1454 | def check(query_lens, run): |
| 1455 | contexts, ks, vs = prepare(query_lens) |
| 1456 | assert cache is not None |
| 1457 | output.fill_(float("nan")) |
| 1458 | run() |
| 1459 | start = 0 |
| 1460 | for i, q_len in enumerate(query_lens): |
| 1461 | if q_len == 0: |
| 1462 | continue |
| 1463 | end = start + q_len |
| 1464 | full_k = ( |
| 1465 | torch.cat((ks[i], key[start:end])) |
| 1466 | .float() |
| 1467 | .repeat_interleave(num_heads // num_kv_heads, dim=1) |
| 1468 | ) |
| 1469 | full_v = ( |
| 1470 | torch.cat((vs[i], value[start:end])) |
| 1471 | .float() |
| 1472 | .repeat_interleave(num_heads // num_kv_heads, dim=1) |
| 1473 | ) |
| 1474 | scores = ( |
| 1475 | torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale |
| 1476 | ) |
| 1477 | q_pos = torch.arange(q_len, device=device) + contexts[i] |
| 1478 | k_pos = torch.arange(full_k.shape[0], device=device) |
| 1479 | allowed = k_pos[None, :] <= q_pos[:, None] |
| 1480 | if sliding_window is not None: |
| 1481 | allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window |
| 1482 | probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1) |
| 1483 | reference = torch.einsum("hqk,khd->qhd", probs, full_v) |
| 1484 | torch.testing.assert_close( |
| 1485 | output[start:end].float(), |
| 1486 | reference, |
| 1487 | atol=2e-2, |
| 1488 | rtol=2e-2, |
| 1489 | ) |
| 1490 | slots = common.slot_mapping[start:end] |
| 1491 | written = cache[slots // block_size, :, slots % block_size, :] |
| 1492 | torch.testing.assert_close(written[..., :head_size], key[start:end]) |
| 1493 | torch.testing.assert_close(written[..., head_size:], value[start:end]) |
| 1494 | start = end |
| 1495 | assert torch.count_nonzero(cache[0]) == 0 |
| 1496 | |
| 1497 | for query_lens in layouts: |
| 1498 | check(query_lens, graph.replay) |
| 1499 | |
| 1500 | if num_tokens == 40: |
| 1501 | # A longer prefill forces a separate launch in eager/piecewise mode. |
| 1502 | # Its length must not become the decode kernel's query bound, and |
| 1503 | # compacted CPU placeholders still need not match the GPU allocation. |
| 1504 | common.num_reqs = 5 |
| 1505 | common.max_query_len = 24 |
| 1506 | common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40]) |
| 1507 | common.query_start_loc_cpu[6:] = 40 |
| 1508 | |
| 1509 | def mixed_forward(): |
| 1510 | with set_current_vllm_config(config): |
| 1511 | metadata = builder.build( |
| 1512 | 0, |
| 1513 | replace( |
| 1514 | common, |
| 1515 | query_start_loc=common.query_start_loc[:6], |
| 1516 | query_start_loc_cpu=common.query_start_loc_cpu[:6], |
| 1517 | seq_lens=common.seq_lens[:5], |
| 1518 | seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5], |
| 1519 | block_table_tensor=common.block_table_tensor[:5], |
| 1520 | ), |
| 1521 | ) |
| 1522 | assert metadata.num_decodes == 4 |
| 1523 | assert metadata.num_prefills == 1 |
| 1524 | assert metadata.decode.q_len_per_req == 8 |
| 1525 | forward(metadata) |
| 1526 | |
| 1527 | check([8, 4, 3, 1, 24], mixed_forward) |
| 1528 | |
| 1529 | |
| 1530 | @pytest.mark.skipif( |
| 1531 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 1532 | reason="FlashInfer is not available.", |
| 1533 | ) |
| 1534 | @pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) |
| 1535 | def test_flashinfer_attention_sinks_refreshed_after_reload(dtype): |
| 1536 | from vllm.v1.attention.backends import flashinfer as flashinfer_backend |
| 1537 | |
| 1538 | source_sinks = torch.tensor([1.0, 2.0], dtype=dtype) |
| 1539 | impl = object.__new__(flashinfer_backend.FlashInferImpl) |
| 1540 | impl._sinks_source = source_sinks |
| 1541 | impl.sinks = source_sinks |
| 1542 | |
| 1543 | impl.process_weights_after_loading(dtype) |
| 1544 | |
| 1545 | assert impl.sinks is not None |
| 1546 | sinks_ptr = impl.sinks.data_ptr() |
| 1547 | assert impl.sinks.dtype == torch.float32 |
| 1548 | torch.testing.assert_close(impl.sinks, source_sinks.float()) |
| 1549 | |
| 1550 | source_sinks.copy_(torch.tensor([3.0, 4.0], dtype=dtype)) |
| 1551 | impl.process_weights_after_loading(dtype) |
| 1552 | |
| 1553 | assert impl.sinks.data_ptr() == sinks_ptr |
| 1554 | torch.testing.assert_close(impl.sinks, source_sinks.float()) |
| 1555 | |
| 1556 | |
| 1557 | @pytest.mark.skipif( |
| 1558 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 1559 | reason="FlashInfer is not available.", |
| 1560 | ) |
| 1561 | def test_flashinfer_native_prefill_with_sinks(default_vllm_config): |
| 1562 | supported = current_platform.is_cuda() and ( |
| 1563 | current_platform.is_device_capability(90) |
| 1564 | or current_platform.is_device_capability_family(120) |
| 1565 | ) |
| 1566 | if not supported: |
| 1567 | pytest.skip("Native FlashInfer prefill with sinks requires SM90 or SM12x.") |
| 1568 | |
| 1569 | from vllm.v1.attention.backends.flashinfer import FlashInferBackend |
| 1570 | |
| 1571 | if not FlashInferBackend.supports_sink(): |
| 1572 | pytest.skip("FlashInfer attention sinks are not available in this setup.") |
| 1573 | |
| 1574 | def causal_mask_mod( |
| 1575 | b: torch.Tensor, |
| 1576 | h: torch.Tensor, |
| 1577 | q_idx: torch.Tensor, |
| 1578 | kv_idx: torch.Tensor, |
| 1579 | *, |
| 1580 | context_len: int, |
| 1581 | ): |
| 1582 | return (q_idx + context_len) >= kv_idx |
| 1583 | |
| 1584 | _test_backend_correctness( |
| 1585 | BATCH_SPECS["small_prefill"], |
| 1586 | "meta-llama/Meta-Llama-3-8B", |
| 1587 | [AttentionBackendEnum.FLASHINFER], |
| 1588 | causal_mask_mod, |
| 1589 | use_sinks=True, |
| 1590 | ) |
| 1591 | |
| 1592 | |
| 1593 | @pytest.mark.skipif( |
| 1594 | AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, |
| 1595 | reason="FlashInfer is not available.", |
| 1596 | ) |
| 1597 | def test_flashinfer_xqa_decode_correctness(default_vllm_config): |
| 1598 | """FlashInfer should route SM90/SM12x decode through XQA and match SDPA.""" |
| 1599 | supported = current_platform.is_cuda() and ( |
| 1600 | current_platform.is_device_capability(90) |
| 1601 | or current_platform.is_device_capability_family(120) |
| 1602 | ) |
| 1603 | if not supported: |
| 1604 | pytest.skip("FlashInfer XQA decode requires SM90 or SM12x.") |
| 1605 | |
| 1606 | import unittest.mock |
| 1607 | |
| 1608 | from vllm.utils.flashinfer import can_use_trtllm_attention |
| 1609 | from vllm.v1.attention.backends import flashinfer as flashinfer_backend |
| 1610 | from vllm.v1.attention.backends.utils import PerLayerParameters |
| 1611 | |
| 1612 | def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls): |
| 1613 | return { |
| 1614 | "placeholder": PerLayerParameters( |
| 1615 | window_left=-1, |
| 1616 | logits_soft_cap=0.0, |
| 1617 | sm_scale=1.0, |
| 1618 | ) |
| 1619 | } |
| 1620 | |
| 1621 | def causal_mask_mod( |
| 1622 | b: torch.Tensor, |
| 1623 | h: torch.Tensor, |
| 1624 | q_idx: torch.Tensor, |
| 1625 | kv_idx: torch.Tensor, |
| 1626 | *, |
| 1627 | context_len: int, |
| 1628 | ): |
| 1629 | return (q_idx + context_len) >= kv_idx |
| 1630 | |
| 1631 | batch_spec = BATCH_SPECS["small_decode"] |
| 1632 | vllm_config = create_vllm_config( |
| 1633 | model_name="meta-llama/Meta-Llama-3-8B", |
| 1634 | max_model_len=max(batch_spec.seq_lens), |
| 1635 | block_size=16, |
| 1636 | ) |
| 1637 | device = torch.device(f"{DEVICE_TYPE}:0") |
| 1638 | kv_cache_spec = FullAttentionSpec( |
| 1639 | block_size=vllm_config.cache_config.block_size, |
| 1640 | num_kv_heads=vllm_config.model_config.get_num_kv_heads( |
| 1641 | vllm_config.parallel_config |
| 1642 | ), |
| 1643 | head_size=vllm_config.model_config.get_head_size(), |
| 1644 | dtype=vllm_config.model_config.dtype, |
| 1645 | ) |
| 1646 | |
| 1647 | with set_current_vllm_config(vllm_config): |
| 1648 | if not can_use_trtllm_attention( |
| 1649 | vllm_config.model_config.get_num_attention_heads( |
| 1650 | vllm_config.parallel_config |
| 1651 | ), |
| 1652 | kv_cache_spec.num_kv_heads, |
| 1653 | is_prefill=False, |
| 1654 | ): |
| 1655 | pytest.skip("FlashInfer XQA decode is not available in this setup.") |
| 1656 | |
| 1657 | with unittest.mock.patch( |
| 1658 | "vllm.v1.attention.backends.flashinfer.get_per_layer_parameters", |
| 1659 | mock_get_per_layer_parameters, |
| 1660 | ): |
| 1661 | builder = flashinfer_backend.FlashInferMetadataBuilder( |
| 1662 | kv_cache_spec, ["placeholder"], vllm_config, device |
| 1663 | ) |
| 1664 | common_attn_metadata = create_common_attn_metadata( |
| 1665 | batch_spec, vllm_config.cache_config.block_size, device |
| 1666 | ) |
| 1667 | attn_metadata = builder.build(0, common_attn_metadata) |
| 1668 | |
| 1669 | assert ( |
| 1670 | flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support( |
| 1671 | vllm_config, kv_cache_spec |
| 1672 | ) |
| 1673 | == AttentionCGSupport.UNIFORM_BATCH |
| 1674 | ) |
| 1675 | wide_head_kv_cache_spec = FullAttentionSpec( |
| 1676 | block_size=vllm_config.cache_config.block_size, |
| 1677 | num_kv_heads=kv_cache_spec.num_kv_heads, |
| 1678 | head_size=512, |
| 1679 | dtype=vllm_config.model_config.dtype, |
| 1680 | ) |
| 1681 | assert ( |
| 1682 | flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support( |
| 1683 | vllm_config, wide_head_kv_cache_spec |
| 1684 | ) |
| 1685 | == AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE |
| 1686 | ) |
| 1687 | assert isinstance( |
| 1688 | attn_metadata.decode, |
| 1689 | flashinfer_backend.FlashInferTrtllmAPIDecode, |
| 1690 | ) |
| 1691 | assert attn_metadata.decode.kernel == flashinfer_backend.FlashInferDecodeKernel.XQA |
| 1692 | |
| 1693 | _test_backend_correctness( |
| 1694 | batch_spec, |
| 1695 | "meta-llama/Meta-Llama-3-8B", |
| 1696 | [AttentionBackendEnum.FLASHINFER], |
| 1697 | causal_mask_mod, |
| 1698 | ) |
| 1699 | |
| 1700 | |
| 1701 | if current_platform.is_rocm(): |
| 1702 | # FLASH_ATTN is not supported on ROCm |
| 1703 | SLIDING_WINDOW_BACKENDS_TO_TEST = [ |
| 1704 | AttentionBackendEnum.FLEX_ATTENTION, |
| 1705 | AttentionBackendEnum.TRITON_ATTN, |
| 1706 | "FLEX_ATTENTION_SLOW", |
| 1707 | ] |
| 1708 | else: |
| 1709 | SLIDING_WINDOW_BACKENDS_TO_TEST = [ |
| 1710 | AttentionBackendEnum.FLASH_ATTN, |
| 1711 | AttentionBackendEnum.FLEX_ATTENTION, |
| 1712 | AttentionBackendEnum.TRITON_ATTN, |
| 1713 | "FLEX_ATTENTION_SLOW", |
| 1714 | ] |
| 1715 | |
| 1716 | # Encoder-only FlexAttention always uses the slow builder, so the pseudo-backend |
| 1717 | # would run an identical implementation twice. |
| 1718 | SLIDING_WINDOW_ENCODER_BACKENDS_TO_TEST = [ |
| 1719 | backend |
| 1720 | for backend in SLIDING_WINDOW_BACKENDS_TO_TEST |
| 1721 | if backend != "FLEX_ATTENTION_SLOW" |
| 1722 | ] |
| 1723 | |
| 1724 | |
| 1725 | @pytest.mark.parametrize( |
| 1726 | "batch_spec_name", |
| 1727 | [ |
| 1728 | "small_decode", |
| 1729 | "small_prefill", |
| 1730 | "mixed_medium", |
| 1731 | "large_decode", |
| 1732 | "large_prefill", |
| 1733 | "mixed_large", |
| 1734 | ], |
| 1735 | ) |
| 1736 | @pytest.mark.parametrize("model", ["microsoft/Phi-tiny-MoE-instruct"]) |
| 1737 | @pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4]) |
| 1738 | def test_sliding_window_backend_correctness( |
| 1739 | default_vllm_config, |
| 1740 | batch_spec_name: str, |
| 1741 | model: str, |
| 1742 | tensor_parallel_size: int, |
| 1743 | ): |
| 1744 | """Test backend's correctness with sliding window attention.""" |
| 1745 | |
| 1746 | def sliding_window_mask_mod( |
| 1747 | b: torch.Tensor, |
| 1748 | h: torch.Tensor, |
| 1749 | q_idx: torch.Tensor, |
| 1750 | kv_idx: torch.Tensor, |
| 1751 | *, |
| 1752 | context_len: int, |
| 1753 | sliding_window: int, |
| 1754 | ): |
| 1755 | causal_mask = q_idx + context_len >= kv_idx |
| 1756 | window_mask = q_idx + context_len - kv_idx < sliding_window |
| 1757 | return causal_mask & window_mask |
| 1758 | |
| 1759 | batch_spec = BATCH_SPECS[batch_spec_name] |
| 1760 | model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens)) |
| 1761 | sliding_window = model_config.get_sliding_window() |
| 1762 | sliding_window_mask_mod_fn = partial( |
| 1763 | sliding_window_mask_mod, sliding_window=sliding_window |
| 1764 | ) |
| 1765 | |
| 1766 | LARGE_BLOCK_BACKENDS = ( |
| 1767 | [AttentionBackendEnum.FLEX_ATTENTION] |
| 1768 | if is_torch_equal_or_newer("2.9.0.dev0") |
| 1769 | else [] |
| 1770 | ) |
| 1771 | SMALL_BLOCK_BACKENDS = [ |
| 1772 | x for x in SLIDING_WINDOW_BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS |
| 1773 | ] |
| 1774 | _test_backend_correctness( |
| 1775 | batch_spec, |
| 1776 | model, |
| 1777 | SMALL_BLOCK_BACKENDS, |
| 1778 | sliding_window_mask_mod_fn, |
| 1779 | tensor_parallel_size=tensor_parallel_size, |
| 1780 | ) |
| 1781 | |
| 1782 | # Fast FlexAttention needs to run with block_size=128 |
| 1783 | if LARGE_BLOCK_BACKENDS: |
| 1784 | _test_backend_correctness( |
| 1785 | batch_spec, |
| 1786 | model, |
| 1787 | LARGE_BLOCK_BACKENDS, |
| 1788 | sliding_window_mask_mod_fn, |
| 1789 | block_size=128, |
| 1790 | tensor_parallel_size=tensor_parallel_size, |
| 1791 | ) |
| 1792 | |
| 1793 | |
| 1794 | @pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test") |
| 1795 | def test_rocm_aiter_fa_unquantized_cache_ignores_kv_scales( |
| 1796 | default_vllm_config, |
| 1797 | ): |
| 1798 | """Stale scale tensors must not affect an unquantized KV cache.""" |
| 1799 | from vllm._aiter_ops import is_aiter_found_and_supported |
| 1800 | |
| 1801 | if not is_aiter_found_and_supported(): |
| 1802 | pytest.skip("AITER is required") |
| 1803 | |
| 1804 | def sliding_window_mask_mod( |
| 1805 | b: torch.Tensor, |
| 1806 | h: torch.Tensor, |
| 1807 | q_idx: torch.Tensor, |
| 1808 | kv_idx: torch.Tensor, |
| 1809 | *, |
| 1810 | context_len: int, |
| 1811 | sliding_window: int, |
| 1812 | ): |
| 1813 | causal_mask = q_idx + context_len >= kv_idx |
| 1814 | window_mask = q_idx + context_len - kv_idx < sliding_window |
| 1815 | return causal_mask & window_mask |
| 1816 | |
| 1817 | model = "microsoft/Phi-tiny-MoE-instruct" |
| 1818 | batch_spec = BATCH_SPECS["small_decode"] |
| 1819 | model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens)) |
| 1820 | sliding_window = model_config.get_sliding_window() |
| 1821 | assert sliding_window is not None |
| 1822 | |
| 1823 | _test_backend_correctness( |
| 1824 | batch_spec, |
| 1825 | model, |
| 1826 | [AttentionBackendEnum.ROCM_AITER_FA], |
| 1827 | partial(sliding_window_mask_mod, sliding_window=sliding_window), |
| 1828 | layer_k_scale=0.25, |
| 1829 | layer_v_scale=0.5, |
| 1830 | ) |
| 1831 | |
| 1832 | |
| 1833 | @pytest.mark.parametrize( |
| 1834 | "batch_spec_name", |
| 1835 | [ |
| 1836 | "small_encoder_prefill", |
| 1837 | "medium_encoder_prefill", |
| 1838 | ], |
| 1839 | ) |
| 1840 | @pytest.mark.parametrize("model", ["google/embeddinggemma-300m"]) |
| 1841 | @pytest.mark.parametrize("tensor_parallel_size", [1, 2]) |
| 1842 | def test_sliding_window_encoder_backend_correctness( |
| 1843 | default_vllm_config, |
| 1844 | batch_spec_name: str, |
| 1845 | model: str, |
| 1846 | tensor_parallel_size: int, |
| 1847 | ): |
| 1848 | """Test backend's correctness with sliding window attention.""" |
| 1849 | |
| 1850 | def bidi_sliding_window_mask_mod( |
| 1851 | b: torch.Tensor, |
| 1852 | h: torch.Tensor, |
| 1853 | q_idx: torch.Tensor, |
| 1854 | kv_idx: torch.Tensor, |
| 1855 | *, |
| 1856 | context_len: int, |
| 1857 | sliding_window: int, |
| 1858 | ): |
| 1859 | return torch.abs(q_idx + context_len - kv_idx) < sliding_window |
| 1860 | |
| 1861 | batch_spec = BATCH_SPECS[batch_spec_name] |
| 1862 | model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens)) |
| 1863 | sliding_window = model_config.get_sliding_window() |
| 1864 | sliding_window_mask_mod_fn = partial( |
| 1865 | bidi_sliding_window_mask_mod, sliding_window=sliding_window |
| 1866 | ) |
| 1867 | |
| 1868 | _test_backend_correctness( |
| 1869 | batch_spec, |
| 1870 | model, |
| 1871 | SLIDING_WINDOW_ENCODER_BACKENDS_TO_TEST, |
| 1872 | sliding_window_mask_mod_fn, |
| 1873 | causal=False, |
| 1874 | attn_type=AttentionType.ENCODER_ONLY, |
| 1875 | tensor_parallel_size=tensor_parallel_size, |
| 1876 | ) |
| 1877 | |
| 1878 | |
| 1879 | NON_CAUSAL_BACKENDS_TO_TEST = [ |
| 1880 | AttentionBackendEnum.FLASH_ATTN, |
| 1881 | AttentionBackendEnum.FLEX_ATTENTION, |
| 1882 | "FLEX_ATTENTION_SLOW", |
| 1883 | ] |
| 1884 | |
| 1885 | if current_platform.is_rocm(): |
| 1886 | NON_CAUSAL_BACKENDS_TO_TEST = [ |
| 1887 | x |
| 1888 | for x in NON_CAUSAL_BACKENDS_TO_TEST |
| 1889 | if x is not AttentionBackendEnum.FLASH_ATTN |
| 1890 | ] |
| 1891 | |
| 1892 | |
| 1893 | @pytest.mark.parametrize( |
| 1894 | "batch_spec_name", |
| 1895 | [ |
| 1896 | "small_decode", |
| 1897 | "small_prefill", |
| 1898 | "mixed_small", |
| 1899 | ], |
| 1900 | ) |
| 1901 | @pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"]) |
| 1902 | def test_non_causal_backend_correctness( |
| 1903 | default_vllm_config, |
| 1904 | batch_spec_name: str, |
| 1905 | model: str, |
| 1906 | ): |
| 1907 | """Test backend's correctness with non-causal (bidirectional) decoder |
| 1908 | attention, as used by DFlash speculative decoding.""" |
| 1909 | |
| 1910 | def bidirectional_mask_mod( |
| 1911 | b: torch.Tensor, |
| 1912 | h: torch.Tensor, |
| 1913 | q_idx: torch.Tensor, |
| 1914 | kv_idx: torch.Tensor, |
| 1915 | *, |
| 1916 | context_len: int, |
| 1917 | ): |
| 1918 | return q_idx >= 0 # Always True |
| 1919 | |
| 1920 | batch_spec = BATCH_SPECS[batch_spec_name] |
| 1921 | LARGE_BLOCK_BACKENDS = ( |
| 1922 | [AttentionBackendEnum.FLEX_ATTENTION] |
| 1923 | if is_torch_equal_or_newer("2.9.0.dev0") |
| 1924 | else [] |
| 1925 | ) |
| 1926 | |
| 1927 | SMALL_BLOCK_BACKENDS = [ |
| 1928 | x for x in NON_CAUSAL_BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS |
| 1929 | ] |
| 1930 | |
| 1931 | _test_backend_correctness( |
| 1932 | batch_spec, |
| 1933 | model, |
| 1934 | SMALL_BLOCK_BACKENDS, |
| 1935 | bidirectional_mask_mod, |
| 1936 | causal=False, |
| 1937 | ) |
| 1938 | |
| 1939 | if LARGE_BLOCK_BACKENDS: |
| 1940 | _test_backend_correctness( |
| 1941 | batch_spec, |
| 1942 | model, |
| 1943 | LARGE_BLOCK_BACKENDS, |
| 1944 | bidirectional_mask_mod, |
| 1945 | causal=False, |
| 1946 | block_size=128, |
| 1947 | ) |