| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from collections.abc import Iterable |
| 4 | |
| 5 | import torch |
| 6 | |
| 7 | from vllm.logger import init_logger |
| 8 | from vllm.pooling_params import PoolingParams |
| 9 | from vllm.v1.outputs import PoolerOutput |
| 10 | from vllm.v1.pool.late_interaction import ( |
| 11 | LATE_INTERACTION_MODE_CACHE_QUERY, |
| 12 | LATE_INTERACTION_MODE_SCORE_DOC, |
| 13 | compute_maxsim_score_batched, |
| 14 | ) |
| 15 | |
| 16 | logger = init_logger(__name__) |
| 17 | |
| 18 | |
| 19 | class LateInteractionRunner: |
| 20 | """Worker-side state and postprocessing for late-interaction scoring.""" |
| 21 | |
| 22 | def __init__(self, enable_flash: bool = True) -> None: |
| 23 | # query_key -> token embeddings for late-interaction scoring. |
| 24 | self._query_cache: dict[str, torch.Tensor] = {} |
| 25 | # query_key -> remaining number of docs that should use this query. |
| 26 | self._query_uses: dict[str, int] = {} |
| 27 | # doc request id -> query key. |
| 28 | self._doc_query_keys: dict[str, str] = {} |
| 29 | # Fused Triton scoring (PoolerConfig.enable_flash_late_interaction |
| 30 | # gates it; a runtime kernel failure disables it for the process). |
| 31 | if enable_flash: |
| 32 | try: |
| 33 | from vllm.v1.pool.flash_maxsim import ( # noqa: F401 |
| 34 | flash_maxsim_rerank_direct, |
| 35 | ) |
| 36 | except ImportError: |
| 37 | enable_flash = False |
| 38 | self._flash_enabled = enable_flash |
| 39 | |
| 40 | def clear(self) -> None: |
| 41 | self._query_cache.clear() |
| 42 | self._query_uses.clear() |
| 43 | self._doc_query_keys.clear() |
| 44 | |
| 45 | def release_queries(self, query_keys: Iterable[str]) -> None: |
| 46 | query_keys = set(query_keys) |
| 47 | for query_key in query_keys: |
| 48 | self._query_cache.pop(query_key, None) |
| 49 | self._query_uses.pop(query_key, None) |
| 50 | |
| 51 | self._doc_query_keys = { |
| 52 | req_id: query_key |
| 53 | for req_id, query_key in self._doc_query_keys.items() |
| 54 | if query_key not in query_keys |
| 55 | } |
| 56 | |
| 57 | def register_request( |
| 58 | self, req_id: str, pooling_params: PoolingParams | None |
| 59 | ) -> None: |
| 60 | mode, query_key, _ = self._parse_late_interaction_meta(pooling_params) |
| 61 | if mode == LATE_INTERACTION_MODE_SCORE_DOC and query_key is not None: |
| 62 | self._doc_query_keys[req_id] = query_key |
| 63 | else: |
| 64 | self._doc_query_keys.pop(req_id, None) |
| 65 | |
| 66 | def on_requests_finished(self, finished_req_ids: Iterable[str]) -> None: |
| 67 | for req_id in finished_req_ids: |
| 68 | query_key = self._doc_query_keys.pop(req_id, None) |
| 69 | if query_key is not None: |
| 70 | self._release_query_use(query_key) |
| 71 | |
| 72 | def postprocess_pooler_output( |
| 73 | self, |
| 74 | raw_pooler_output: PoolerOutput, |
| 75 | pooling_params: list[PoolingParams], |
| 76 | req_ids: list[str], |
| 77 | finished_mask: list[bool], |
| 78 | ) -> PoolerOutput: |
| 79 | if not isinstance(raw_pooler_output, list): |
| 80 | return raw_pooler_output |
| 81 | |
| 82 | num_reqs = len(pooling_params) |
| 83 | if len(raw_pooler_output) != num_reqs: |
| 84 | raise ValueError( |
| 85 | "raw_pooler_output and pooling_params must have the same length." |
| 86 | ) |
| 87 | if len(req_ids) != num_reqs: |
| 88 | raise ValueError("req_ids and pooling_params must have the same length.") |
| 89 | if len(finished_mask) != num_reqs: |
| 90 | raise ValueError( |
| 91 | "finished_mask and pooling_params must have the same length." |
| 92 | ) |
| 93 | |
| 94 | if not any(finished_mask): |
| 95 | return raw_pooler_output |
| 96 | if not any(p.late_interaction_params is not None for p in pooling_params): |
| 97 | return raw_pooler_output |
| 98 | |
| 99 | outputs: list[torch.Tensor | None] = list(raw_pooler_output) |
| 100 | score_indices: list[int] = [] |
| 101 | score_req_ids: list[str] = [] |
| 102 | score_query_keys: list[str] = [] |
| 103 | score_queries: list[torch.Tensor] = [] |
| 104 | score_docs: list[torch.Tensor] = [] |
| 105 | for i, (req_id, output, params, finished) in enumerate( |
| 106 | zip(req_ids, outputs, pooling_params, finished_mask) |
| 107 | ): |
| 108 | if not finished or output is None: |
| 109 | continue |
| 110 | |
| 111 | mode, query_key, query_uses = self._parse_late_interaction_meta(params) |
| 112 | if mode is None: |
| 113 | continue |
| 114 | |
| 115 | assert query_key is not None |
| 116 | if mode == LATE_INTERACTION_MODE_CACHE_QUERY: |
| 117 | assert query_uses is not None |
| 118 | # `output` can be a view into the current step's hidden-states |
| 119 | # buffer, so clone it before storing across scheduling steps. |
| 120 | self._query_cache[query_key] = output.clone() |
| 121 | self._query_uses[query_key] = query_uses |
| 122 | outputs[i] = torch.zeros((), device=output.device, dtype=torch.float32) |
| 123 | continue |
| 124 | |
| 125 | if mode == LATE_INTERACTION_MODE_SCORE_DOC: |
| 126 | query_output = self._query_cache.get(query_key) |
| 127 | if query_output is None: |
| 128 | raise ValueError( |
| 129 | "late-interaction query cache miss for key " |
| 130 | f"{query_key!r}. Ensure query requests are executed " |
| 131 | "before their paired document requests." |
| 132 | ) |
| 133 | |
| 134 | score_indices.append(i) |
| 135 | score_req_ids.append(req_id) |
| 136 | score_query_keys.append(query_key) |
| 137 | score_queries.append(query_output) |
| 138 | score_docs.append(output) |
| 139 | continue |
| 140 | |
| 141 | raise ValueError(f"Unsupported late-interaction mode: {mode!r}") |
| 142 | |
| 143 | if score_indices: |
| 144 | score_values = self._score(score_queries, score_docs) |
| 145 | for i, req_id, query_key, score in zip( |
| 146 | score_indices, score_req_ids, score_query_keys, score_values |
| 147 | ): |
| 148 | outputs[i] = score |
| 149 | self._doc_query_keys.pop(req_id, None) |
| 150 | self._release_query_use(query_key) |
| 151 | |
| 152 | return outputs |
| 153 | |
| 154 | def _score( |
| 155 | self, |
| 156 | queries: list[torch.Tensor], |
| 157 | docs: list[torch.Tensor], |
| 158 | ) -> list[torch.Tensor]: |
| 159 | """Score (query_i, doc_i) pairs: the fused Triton kernel handles the |
| 160 | common rerank pattern (one shared query, CUDA inputs); multiple |
| 161 | distinct queries, CPU tensors, or a kernel failure fall back to the |
| 162 | reference scorer.""" |
| 163 | if self._flash_enabled and docs and queries[0].is_cuda: |
| 164 | first = queries[0] |
| 165 | if all(q is first for q in queries): |
| 166 | try: |
| 167 | return self._score_flash_shared_query(first, docs) |
| 168 | except Exception as exc: |
| 169 | # A persistent compile/launch failure must not take down |
| 170 | # requests: serve through the reference scorer and stop |
| 171 | # trying the kernel for the rest of the process. |
| 172 | self._flash_enabled = False |
| 173 | logger.warning( |
| 174 | "flash-maxsim scoring failed (%s); falling back to " |
| 175 | "the reference MaxSim path for this process.", |
| 176 | exc, |
| 177 | ) |
| 178 | return compute_maxsim_score_batched(queries, docs) |
| 179 | |
| 180 | @staticmethod |
| 181 | def _score_flash_shared_query( |
| 182 | query: torch.Tensor, |
| 183 | docs: list[torch.Tensor], |
| 184 | ) -> list[torch.Tensor]: |
| 185 | """One kernel launch for all docs against a single shared query: |
| 186 | docs pack into one [total_tokens, d] tensor (single cat, no padding) |
| 187 | and the kernel reads each by (offset, length) — the [B, Lq, Ld] |
| 188 | similarity tensor is never materialised.""" |
| 189 | from vllm.v1.pool.flash_maxsim import flash_maxsim_rerank_direct |
| 190 | |
| 191 | device = docs[0].device |
| 192 | lengths = [int(d.shape[0]) for d in docs] |
| 193 | offsets = [0] * len(lengths) |
| 194 | for i in range(1, len(lengths)): |
| 195 | offsets[i] = offsets[i - 1] + lengths[i - 1] |
| 196 | packed = docs[0] if len(docs) == 1 else torch.cat(docs, dim=0) |
| 197 | scores = flash_maxsim_rerank_direct( |
| 198 | query, |
| 199 | packed, |
| 200 | torch.tensor(offsets, device=device, dtype=torch.int32), |
| 201 | torch.tensor(lengths, device=device, dtype=torch.int32), |
| 202 | max(lengths), |
| 203 | ) |
| 204 | return list(scores.unbind(0)) |
| 205 | |
| 206 | def _release_query_use(self, query_key: str) -> None: |
| 207 | remaining = self._query_uses.get(query_key, 1) - 1 |
| 208 | if remaining <= 0: |
| 209 | self._query_uses.pop(query_key, None) |
| 210 | self._query_cache.pop(query_key, None) |
| 211 | else: |
| 212 | self._query_uses[query_key] = remaining |
| 213 | |
| 214 | @staticmethod |
| 215 | def _parse_late_interaction_meta( |
| 216 | pooling_params: PoolingParams | None, |
| 217 | ) -> tuple[str | None, str | None, int | None]: |
| 218 | if pooling_params is None or pooling_params.late_interaction_params is None: |
| 219 | return None, None, None |
| 220 | |
| 221 | late_interaction_params = pooling_params.late_interaction_params |
| 222 | mode = late_interaction_params.mode |
| 223 | |
| 224 | query_key = late_interaction_params.query_key |
| 225 | if not isinstance(query_key, str) or not query_key: |
| 226 | raise ValueError( |
| 227 | "late-interaction request is missing a valid query key in " |
| 228 | "pooling_params.late_interaction_params." |
| 229 | ) |
| 230 | |
| 231 | if mode == LATE_INTERACTION_MODE_CACHE_QUERY: |
| 232 | query_uses_raw = late_interaction_params.query_uses |
| 233 | if query_uses_raw is None: |
| 234 | query_uses_raw = 1 |
| 235 | try: |
| 236 | query_uses = max(1, int(query_uses_raw)) |
| 237 | except (TypeError, ValueError) as exc: |
| 238 | raise ValueError( |
| 239 | "late-interaction query uses must be an integer value." |
| 240 | ) from exc |
| 241 | return mode, query_key, query_uses |
| 242 | |
| 243 | return mode, query_key, None |