| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import torch |
| 5 | |
| 6 | import vllm._custom_ops as ops |
| 7 | from vllm.v1.attention.backends.utils import NULL_BLOCK_ID |
| 8 | |
| 9 | |
| 10 | def _mamba_chunk_scan_combined_fwd_cpu( |
| 11 | x, |
| 12 | dt, |
| 13 | A, |
| 14 | B, |
| 15 | C, |
| 16 | chunk_size, |
| 17 | out, |
| 18 | D=None, |
| 19 | z=None, |
| 20 | dt_bias=None, |
| 21 | initial_states=None, |
| 22 | return_intermediate_states=False, |
| 23 | seq_idx=None, |
| 24 | cu_seqlens=None, |
| 25 | cu_chunk_seqlens=None, |
| 26 | last_chunk_indices=None, |
| 27 | dt_softplus=False, |
| 28 | dt_limit=(0.0, float("inf")), |
| 29 | state_dtype=None, |
| 30 | **kwargs, |
| 31 | ): |
| 32 | seqlen, nheads, headdim = x.shape |
| 33 | _, ngroups, dstate = B.shape |
| 34 | |
| 35 | assert cu_seqlens is not None |
| 36 | batch = cu_seqlens.size(0) - 1 |
| 37 | |
| 38 | dt_f = dt.float() |
| 39 | if dt_bias is not None: |
| 40 | dt_f = dt_f + dt_bias.float().unsqueeze(0) |
| 41 | if dt_softplus: |
| 42 | dt_f = torch.nn.functional.softplus(dt_f) |
| 43 | if dt_limit[0] > 0.0 or dt_limit[1] < float("inf"): |
| 44 | dt_f = dt_f.clamp(min=dt_limit[0], max=dt_limit[1]) |
| 45 | |
| 46 | all_states = torch.zeros( |
| 47 | batch, nheads, headdim, dstate, dtype=torch.float32, device=x.device |
| 48 | ) |
| 49 | if initial_states is not None: |
| 50 | all_states.copy_(initial_states.float()) |
| 51 | |
| 52 | assert out.is_contiguous(), ( |
| 53 | "_mamba_chunk_scan_combined_fwd_cpu: `out` must be " |
| 54 | "pre-allocated as a contiguous tensor" |
| 55 | ) |
| 56 | |
| 57 | D_1d = None |
| 58 | if D is not None: |
| 59 | d = D.float() |
| 60 | while d.dim() > 1 and d.stride(-1) == 0: |
| 61 | d = d.squeeze(-1) |
| 62 | D_1d = d.contiguous() |
| 63 | |
| 64 | ops.mamba_chunk_scan_fwd_cpu( |
| 65 | out, |
| 66 | all_states, |
| 67 | x, |
| 68 | dt_f, |
| 69 | A, |
| 70 | B, |
| 71 | C, |
| 72 | D_1d, |
| 73 | z, |
| 74 | cu_seqlens.to(torch.int32), |
| 75 | ) |
| 76 | |
| 77 | out_dtype = state_dtype if state_dtype is not None else x.dtype |
| 78 | all_states = all_states.to(out_dtype) |
| 79 | |
| 80 | return all_states |
| 81 | |
| 82 | |
| 83 | def selective_state_update( |
| 84 | state, |
| 85 | x, |
| 86 | dt, |
| 87 | A, |
| 88 | B, |
| 89 | C, |
| 90 | D=None, |
| 91 | z=None, |
| 92 | dt_bias=None, |
| 93 | dt_softplus=False, |
| 94 | state_batch_indices=None, |
| 95 | dst_state_batch_indices=None, |
| 96 | null_block_id=NULL_BLOCK_ID, |
| 97 | out=None, |
| 98 | num_accepted_tokens=None, |
| 99 | cu_seqlens=None, |
| 100 | is_blackwell=False, |
| 101 | enable_stochastic_rounding=False, |
| 102 | cache_philox_rounds=0, |
| 103 | ): |
| 104 | """CPU implementation for selective_state_update.""" |
| 105 | # Ensure out tensor exists |
| 106 | if out is None: |
| 107 | out = torch.empty_like(x if x.dim() == 2 else x) |
| 108 | |
| 109 | _state = state.unsqueeze(1) if state.dim() == 3 else state |
| 110 | _x = x.unsqueeze(1) if x.dim() == 2 else x |
| 111 | _dt = dt.unsqueeze(1) if dt.dim() == 2 else dt |
| 112 | _A = A.unsqueeze(0) if A.dim() == 2 else A |
| 113 | _B = B.unsqueeze(1) if B.dim() == 2 else B |
| 114 | _C = C.unsqueeze(1) if C.dim() == 2 else C |
| 115 | _D = D.unsqueeze(0) if (D is not None and D.dim() == 1) else D |
| 116 | _z = z.unsqueeze(1) if (z is not None and z.dim() == 2) else z |
| 117 | _dt_bias = ( |
| 118 | dt_bias.unsqueeze(0) |
| 119 | if (dt_bias is not None and dt_bias.dim() == 1) |
| 120 | else dt_bias |
| 121 | ) |
| 122 | _out = out.unsqueeze(1) if out.dim() == 2 else out |
| 123 | |
| 124 | _sbi = state_batch_indices |
| 125 | _dsbi = dst_state_batch_indices |
| 126 | ops.selective_state_update_cpu( |
| 127 | _state, |
| 128 | _x, |
| 129 | _dt, |
| 130 | _A, |
| 131 | _B, |
| 132 | _C, |
| 133 | _D, |
| 134 | _z, |
| 135 | _dt_bias, |
| 136 | dt_softplus, |
| 137 | _sbi, |
| 138 | _dsbi, |
| 139 | null_block_id, |
| 140 | _out, |
| 141 | num_accepted_tokens, |
| 142 | cu_seqlens, |
| 143 | ) |
| 144 | return _out.squeeze(1) if out.dim() == 2 else _out |