| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """LoRA kernels metadata preparation utilities.""" |
| 4 | |
| 5 | import bisect |
| 6 | from dataclasses import dataclass, field |
| 7 | |
| 8 | import torch |
| 9 | |
| 10 | |
| 11 | @dataclass |
| 12 | class LoRAKernelMeta: |
| 13 | token_lora_mapping: torch.Tensor |
| 14 | token_indices_sorted_by_lora_ids: torch.Tensor |
| 15 | active_lora_ids: torch.Tensor |
| 16 | num_tokens_per_lora: torch.Tensor |
| 17 | lora_token_start_loc: torch.Tensor |
| 18 | |
| 19 | # The V1 architecture uses the traced torch.compile graphs to execute |
| 20 | # a forward pass. Things to note about this process, |
| 21 | # 1. The tracing infers all python scalar datatype objects into a constant |
| 22 | # value. |
| 23 | # 2. The tracing cannot handle dynamic control flow. (dynamic control flow |
| 24 | # is an experimental feature in pytorch) |
| 25 | # 3. The internals of torch.ops functions are not traced. |
| 26 | # We disguise the "no_lora" flag as a cpu tensor and leverage point number 3 |
| 27 | # to early exit from inside the lora_expand / lora_shrink torch operation. |
| 28 | no_lora_flag_cpu: torch.Tensor |
| 29 | |
| 30 | # Number of active LoRAs (unique non-(-1) values in token_lora_mapping). |
| 31 | # Stored as a CPU tensor (not a Python int) so that torch.compile treats |
| 32 | # it as a dynamic value rather than baking it as a constant at trace time. |
| 33 | # This follows the same pattern as no_lora_flag_cpu above. |
| 34 | num_active_loras_cpu: torch.Tensor |
| 35 | |
| 36 | # Default num_active_loras value (max_loras + 1) as a CPU tensor, |
| 37 | # used when specialize_active_lora is False to avoid allocating a |
| 38 | # new tensor on every meta_args() call. |
| 39 | default_num_active_loras_cpu: torch.Tensor |
| 40 | |
| 41 | # Captured LoRA counts for cudagraph specialization (sorted list). |
| 42 | # When specialize_active_lora is enabled, num_active_loras is rounded up |
| 43 | # to the nearest value in this list to match cudagraph capture keys. |
| 44 | # Empty list means no specialization (use actual count). |
| 45 | captured_lora_counts: list[int] = field(default_factory=list) |
| 46 | |
| 47 | @staticmethod |
| 48 | def make( |
| 49 | max_loras: int, |
| 50 | max_num_tokens: int, |
| 51 | device: torch.device | str, |
| 52 | captured_lora_counts: list[int] | None = None, |
| 53 | ) -> "LoRAKernelMeta": |
| 54 | token_lora_mapping = torch.empty( |
| 55 | max_num_tokens, dtype=torch.int32, device=device |
| 56 | ) |
| 57 | |
| 58 | token_indices_sorted_by_lora_ids = torch.empty( |
| 59 | max_num_tokens, dtype=torch.int32, device=device |
| 60 | ) |
| 61 | |
| 62 | # +1 because "no-lora" is also a possibility |
| 63 | # example: let max_loras be 3, active_lora_ids of [-1, 0, 2, 1] |
| 64 | # is a possibility. |
| 65 | active_lora_ids = torch.empty(max_loras + 1, dtype=torch.int32, device=device) |
| 66 | |
| 67 | # using running example, [3, 10, 5, 2] is a possibility. |
| 68 | num_tokens_per_lora = torch.zeros( |
| 69 | max_loras + 1, dtype=torch.int32, device=device |
| 70 | ) |
| 71 | |
| 72 | # +2 for this because, the first index is always 0. |
| 73 | # using running example, lora_token_start_loc |
| 74 | # is [0, 3, 13, 18, 20]. |
| 75 | lora_token_start_loc = torch.zeros( |
| 76 | max_loras + 2, dtype=torch.int32, device=device |
| 77 | ) |
| 78 | |
| 79 | no_lora_flag_cpu = torch.tensor([False], dtype=torch.bool, device="cpu") |
| 80 | |
| 81 | num_active_loras_cpu = torch.tensor([0], dtype=torch.int32, device="cpu") |
| 82 | default_num_active_loras_cpu = torch.tensor( |
| 83 | [max_loras + 1], dtype=torch.int32, device="cpu" |
| 84 | ) |
| 85 | |
| 86 | return LoRAKernelMeta( |
| 87 | token_lora_mapping=token_lora_mapping, |
| 88 | token_indices_sorted_by_lora_ids=token_indices_sorted_by_lora_ids, |
| 89 | active_lora_ids=active_lora_ids, |
| 90 | num_tokens_per_lora=num_tokens_per_lora, |
| 91 | lora_token_start_loc=lora_token_start_loc, |
| 92 | no_lora_flag_cpu=no_lora_flag_cpu, |
| 93 | num_active_loras_cpu=num_active_loras_cpu, |
| 94 | default_num_active_loras_cpu=default_num_active_loras_cpu, |
| 95 | captured_lora_counts=sorted(captured_lora_counts) |
| 96 | if captured_lora_counts |
| 97 | else [], |
| 98 | ) |
| 99 | |
| 100 | def _reset(self): |
| 101 | self.active_lora_ids.fill_(-1) |
| 102 | self.num_tokens_per_lora.fill_(0) |
| 103 | self.lora_token_start_loc.fill_(0) |
| 104 | self.no_lora_flag_cpu.fill_(False) |
| 105 | self.num_active_loras_cpu.fill_(0) |
| 106 | |
| 107 | def prepare_tensors(self, token_lora_mapping: torch.Tensor) -> None: |
| 108 | """Prepare kernel metadata tensors for the current forward pass. |
| 109 | |
| 110 | Args: |
| 111 | token_lora_mapping (torch.Tensor): Tensor containing lora indices |
| 112 | for each input token. |
| 113 | |
| 114 | """ |
| 115 | self._reset() |
| 116 | |
| 117 | # Check and record no-lora case. |
| 118 | no_lora = torch.all(token_lora_mapping == -1) |
| 119 | self.no_lora_flag_cpu[0] = no_lora |
| 120 | |
| 121 | if no_lora: |
| 122 | # Early exit. LoRA kernels will not be run. |
| 123 | return |
| 124 | |
| 125 | num_tokens = token_lora_mapping.size(0) |
| 126 | |
| 127 | # copy token lora mapping |
| 128 | self.token_lora_mapping[:num_tokens].copy_( |
| 129 | token_lora_mapping, non_blocking=True |
| 130 | ) |
| 131 | |
| 132 | # token_indices_sorted_by_lora_ids |
| 133 | _, token_indices_sorted_by_lora_ids = torch.sort( |
| 134 | token_lora_mapping, stable=True |
| 135 | ) |
| 136 | # start gpu transfer |
| 137 | self.token_indices_sorted_by_lora_ids[:num_tokens].copy_( |
| 138 | token_indices_sorted_by_lora_ids, non_blocking=True |
| 139 | ) |
| 140 | |
| 141 | # active_lora_ids, num_tokens_per_lora |
| 142 | lora_ids, num_tokens_per_lora = torch.unique( |
| 143 | token_lora_mapping, sorted=True, return_counts=True |
| 144 | ) |
| 145 | self.active_lora_ids[: lora_ids.size(0)].copy_(lora_ids, non_blocking=True) |
| 146 | self.num_tokens_per_lora[: num_tokens_per_lora.size(0)].copy_( |
| 147 | num_tokens_per_lora, non_blocking=True |
| 148 | ) |
| 149 | |
| 150 | num_active_loras = lora_ids.size(0) |
| 151 | |
| 152 | # Round up num_active_loras to match cudagraph capture keys. |
| 153 | # This ensures the kernel grid dimension matches the captured graph. |
| 154 | if self.captured_lora_counts and num_active_loras > 0: |
| 155 | idx = bisect.bisect_left(self.captured_lora_counts, num_active_loras) |
| 156 | if idx < len(self.captured_lora_counts): |
| 157 | num_active_loras = self.captured_lora_counts[idx] |
| 158 | |
| 159 | self.num_active_loras_cpu[0] = num_active_loras |
| 160 | |
| 161 | # lora_token_start_loc |
| 162 | lora_token_start_loc = torch.cumsum(num_tokens_per_lora, dim=0) |
| 163 | self.lora_token_start_loc[1 : 1 + lora_token_start_loc.size(0)].copy_( |
| 164 | lora_token_start_loc, non_blocking=True |
| 165 | ) |
| 166 | |
| 167 | def meta_args( |
| 168 | self, |
| 169 | token_nums: int, |
| 170 | specialize_active_lora: bool, |
| 171 | ) -> tuple[ |
| 172 | torch.Tensor, |
| 173 | torch.Tensor, |
| 174 | torch.Tensor, |
| 175 | torch.Tensor, |
| 176 | torch.Tensor, |
| 177 | torch.Tensor, |
| 178 | torch.Tensor, |
| 179 | ]: |
| 180 | """This function returns the kernel metadata required for the current |
| 181 | forward pass execution of the kernel. The function returns all the |
| 182 | metadata required by the kernel, in order, as a tuple, so it can be |
| 183 | unpacked directly during the lora_shrink/lora_expand function call. |
| 184 | |
| 185 | Args: |
| 186 | token_nums (int): Number of input tokens in the current forward |
| 187 | pass of the kernel. |
| 188 | specialize_active_lora (bool): If True, report the actual active |
| 189 | LoRA count; otherwise report the padded default, which keeps |
| 190 | the kernel launch shape stable. |
| 191 | |
| 192 | """ |
| 193 | if specialize_active_lora: |
| 194 | num_active_loras = self.num_active_loras_cpu |
| 195 | else: |
| 196 | num_active_loras = self.default_num_active_loras_cpu |
| 197 | return ( |
| 198 | self.token_lora_mapping[:token_nums], |
| 199 | self.token_indices_sorted_by_lora_ids[:token_nums], |
| 200 | self.num_tokens_per_lora, |
| 201 | self.lora_token_start_loc, |
| 202 | self.active_lora_ids, |
| 203 | self.no_lora_flag_cpu, |
| 204 | num_active_loras, |
| 205 | ) |