| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """GPU-accelerated N-gram proposer using fully async PyTorch tensor operations. |
| 4 | |
| 5 | This version uses a fully vectorized approach with unfold and argmax for |
| 6 | finding the first match across all sequences in parallel. |
| 7 | """ |
| 8 | |
| 9 | import torch |
| 10 | from torch import nn |
| 11 | |
| 12 | from vllm.compilation.decorators import support_torch_compile |
| 13 | from vllm.config import ( |
| 14 | CompilationConfig, |
| 15 | CompilationMode, |
| 16 | CUDAGraphMode, |
| 17 | VllmConfig, |
| 18 | ) |
| 19 | from vllm.forward_context import set_forward_context |
| 20 | from vllm.utils.torch_utils import async_tensor_h2d |
| 21 | from vllm.v1.core.sched.output import SchedulerOutput |
| 22 | from vllm.v1.utils import record_function_or_nullcontext |
| 23 | from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch |
| 24 | |
| 25 | |
| 26 | @support_torch_compile() |
| 27 | class NgramGPUKernel(nn.Module): |
| 28 | """GPU-accelerated N-gram proposer using fully async tensor operations.""" |
| 29 | |
| 30 | def __init__( |
| 31 | self, vllm_config: VllmConfig, prefix: str = "", device: torch.device = "cuda" |
| 32 | ): |
| 33 | super().__init__() |
| 34 | |
| 35 | assert vllm_config.speculative_config is not None |
| 36 | assert vllm_config.speculative_config.prompt_lookup_min is not None |
| 37 | assert vllm_config.speculative_config.prompt_lookup_max is not None |
| 38 | |
| 39 | self.min_n = vllm_config.speculative_config.prompt_lookup_min |
| 40 | self.max_n = vllm_config.speculative_config.prompt_lookup_max |
| 41 | self.k = vllm_config.speculative_config.num_speculative_tokens |
| 42 | self.max_model_len = vllm_config.model_config.max_model_len |
| 43 | self.max_num_seqs = vllm_config.scheduler_config.max_num_seqs |
| 44 | self.device = device |
| 45 | |
| 46 | def _find_first_and_extract_all_n_parallel( |
| 47 | self, |
| 48 | token_ids: torch.Tensor, |
| 49 | seq_lengths: torch.Tensor, |
| 50 | min_ngram_len: int, |
| 51 | max_ngram_len: int, |
| 52 | num_draft_tokens: int, |
| 53 | ) -> torch.Tensor: |
| 54 | """Find suffix n-gram matches and extract following tokens. |
| 55 | Searches for the earliest prior occurrence of the trailing n-gram, |
| 56 | tries multiple lengths, and picks the longest valid match. |
| 57 | |
| 58 | Args: |
| 59 | token_ids: Token IDs for each sequence |
| 60 | seq_lengths: Actual length of each sequence (excluding padding) |
| 61 | min_ngram_len: Minimum n-gram size to search for (e.g., 2) |
| 62 | max_ngram_len: Maximum n-gram size to search for (e.g., 5) |
| 63 | num_draft_tokens: Number of tokens to extract after match (k) |
| 64 | |
| 65 | Returns: |
| 66 | Draft token predictions; -1 means invalid/no match. |
| 67 | |
| 68 | """ |
| 69 | batch_size = token_ids.shape[0] |
| 70 | max_seq_len = token_ids.shape[1] |
| 71 | device = token_ids.device |
| 72 | num_ngram_sizes = max_ngram_len - min_ngram_len + 1 |
| 73 | |
| 74 | # All n-gram sizes to try. |
| 75 | ngram_lengths = torch.arange(min_ngram_len, max_ngram_len + 1, device=device) |
| 76 | batch_indices = torch.arange(batch_size, device=device) |
| 77 | |
| 78 | # Earliest match per (sequence, ngram_len); -1 means no match. |
| 79 | first_match_positions = torch.full( |
| 80 | (batch_size, num_ngram_sizes), -1, dtype=torch.long, device=device |
| 81 | ) |
| 82 | |
| 83 | for i, ngram_len in enumerate(range(min_ngram_len, max_ngram_len + 1)): |
| 84 | # Sliding windows of size ngram_len; unfold is O(1) view. |
| 85 | search_windows = token_ids.unfold(1, ngram_len, 1) |
| 86 | num_windows = search_windows.shape[1] |
| 87 | |
| 88 | # Trailing suffix (last ngram_len tokens) for each sequence. |
| 89 | suffix_starts = seq_lengths - ngram_len |
| 90 | suffix_indices = suffix_starts.unsqueeze(1) + torch.arange( |
| 91 | ngram_len, device=device |
| 92 | ) |
| 93 | suffix_indices.clamp_(min=0) |
| 94 | suffix = torch.gather(token_ids, 1, suffix_indices) |
| 95 | |
| 96 | # Window matches for each sequence. |
| 97 | matches = (search_windows == suffix.unsqueeze(1)).all(dim=-1) |
| 98 | |
| 99 | # Match must leave room for at least one draft token. |
| 100 | max_valid_suffix_start = seq_lengths - ngram_len - 1 |
| 101 | window_positions = torch.arange(num_windows, device=device) |
| 102 | valid_mask = window_positions <= max_valid_suffix_start.unsqueeze(1) |
| 103 | final_matches = matches & valid_mask |
| 104 | |
| 105 | # Find earliest match (argmax=0 when empty; verify with has_match). |
| 106 | first_match_idx = torch.argmax(final_matches.int(), dim=1) |
| 107 | has_match = final_matches[batch_indices, first_match_idx] |
| 108 | |
| 109 | # Store valid match positions (window index = position). |
| 110 | first_match_positions[:, i] = torch.where(has_match, first_match_idx, -1) |
| 111 | |
| 112 | # Select the longest n-gram with a match. |
| 113 | best_ngram_idx = (first_match_positions >= 0).int().flip(dims=[1]).argmax(dim=1) |
| 114 | best_ngram_idx = num_ngram_sizes - 1 - best_ngram_idx # Flip back |
| 115 | |
| 116 | # Match position for the best n-gram. |
| 117 | best_match_pos = first_match_positions[batch_indices, best_ngram_idx] |
| 118 | |
| 119 | # Avoid data-dependent branching. |
| 120 | has_any_match = best_match_pos >= 0 |
| 121 | |
| 122 | # Length of the best matching n-gram. |
| 123 | best_ngram_lengths = ngram_lengths[best_ngram_idx] |
| 124 | |
| 125 | # Start position right after the matched suffix. |
| 126 | draft_start = torch.where( |
| 127 | has_any_match, |
| 128 | best_match_pos + best_ngram_lengths, |
| 129 | torch.zeros_like(best_match_pos), |
| 130 | ) |
| 131 | tokens_available = seq_lengths - draft_start |
| 132 | |
| 133 | # Gather indices for draft tokens. |
| 134 | draft_indices = draft_start.unsqueeze(1) + torch.arange( |
| 135 | num_draft_tokens, device=device |
| 136 | ) |
| 137 | draft_indices.clamp_(min=0, max=max_seq_len - 1) |
| 138 | |
| 139 | # Extract draft tokens; gather always runs. |
| 140 | draft_tokens = torch.gather(token_ids, 1, draft_indices) |
| 141 | |
| 142 | # Mask positions beyond available tokens. |
| 143 | position_indices = torch.arange(num_draft_tokens, device=device).unsqueeze(0) |
| 144 | valid_positions = position_indices < tokens_available.unsqueeze(1) |
| 145 | |
| 146 | draft_tokens = torch.where( |
| 147 | valid_positions, |
| 148 | draft_tokens, |
| 149 | torch.full_like(draft_tokens, -1), |
| 150 | ) |
| 151 | |
| 152 | # If no match, mask all positions. |
| 153 | draft_tokens = torch.where( |
| 154 | has_any_match.unsqueeze(1), |
| 155 | draft_tokens, |
| 156 | torch.full_like(draft_tokens, -1), |
| 157 | ) |
| 158 | |
| 159 | return draft_tokens |
| 160 | |
| 161 | def forward( |
| 162 | self, |
| 163 | num_tokens_no_spec: torch.Tensor, |
| 164 | token_ids_gpu: torch.Tensor, |
| 165 | combined_mask: torch.Tensor, |
| 166 | ) -> tuple[torch.Tensor, torch.Tensor]: |
| 167 | """Forward pass for N-gram proposal using GPU tensor operations. |
| 168 | |
| 169 | Args: |
| 170 | num_tokens_no_spec: Number of tokens for each sequence [batch_size] |
| 171 | token_ids_gpu: Token IDs [batch_size, max_len] |
| 172 | combined_mask: Whether each sequence is valid for spec decode [batch_size] |
| 173 | |
| 174 | Returns: |
| 175 | draft_tokens: [batch_size, k] on GPU |
| 176 | num_valid_draft_tokens: [batch_size] int32 on GPU, count of |
| 177 | leading valid (non -1) tokens per request. |
| 178 | |
| 179 | """ |
| 180 | device = token_ids_gpu.device |
| 181 | |
| 182 | # Infer batch size to preserve dynamic shape. |
| 183 | actual_batch_size = token_ids_gpu.shape[0] |
| 184 | |
| 185 | # Allocate in forward so torch.compile can optimize. |
| 186 | # NOTE(patchy): Do NOT pre-allocate this as a buffer |
| 187 | # it breaks torch.compile |
| 188 | draft_tokens = torch.full( |
| 189 | (actual_batch_size, self.k), -1, dtype=torch.int32, device=device |
| 190 | ) |
| 191 | |
| 192 | results = self._find_first_and_extract_all_n_parallel( |
| 193 | token_ids_gpu, |
| 194 | num_tokens_no_spec, |
| 195 | min_ngram_len=self.min_n, |
| 196 | max_ngram_len=self.max_n, |
| 197 | num_draft_tokens=self.k, |
| 198 | ) |
| 199 | |
| 200 | draft_tokens = torch.where(combined_mask.unsqueeze(1), results, -1) |
| 201 | |
| 202 | # Count leading contiguous valid (non -1) tokens per request. |
| 203 | is_valid = draft_tokens != -1 # [batch, k] |
| 204 | cum_valid = is_valid.int().cumsum(dim=1) # [batch, k] |
| 205 | positions = torch.arange(1, self.k + 1, device=device).unsqueeze(0) |
| 206 | num_valid_draft_tokens = (cum_valid == positions).int().sum(dim=1) |
| 207 | |
| 208 | return draft_tokens, num_valid_draft_tokens |
| 209 | |
| 210 | def load_model(self, *args, **kwargs): |
| 211 | """No model to load for N-gram proposer.""" |
| 212 | pass |
| 213 | |
| 214 | |
| 215 | class NgramProposerGPU: |
| 216 | def __init__(self, vllm_config: VllmConfig, device: torch.device, runner=None): |
| 217 | assert vllm_config.speculative_config is not None |
| 218 | assert vllm_config.speculative_config.prompt_lookup_min is not None |
| 219 | assert vllm_config.speculative_config.prompt_lookup_max is not None |
| 220 | |
| 221 | compilation_config = CompilationConfig( |
| 222 | mode=CompilationMode.VLLM_COMPILE, |
| 223 | custom_ops=["none"], |
| 224 | splitting_ops=[], |
| 225 | compile_sizes=[], |
| 226 | inductor_compile_config={ |
| 227 | "enable_auto_functionalized_v2": False, |
| 228 | "max_autotune": True, |
| 229 | "aggressive_fusion": True, |
| 230 | "triton.autotune_pointwise": True, |
| 231 | "coordinate_descent_tuning": True, |
| 232 | "use_mixed_mm": False, |
| 233 | }, |
| 234 | cudagraph_mode=CUDAGraphMode.NONE, |
| 235 | ) |
| 236 | model_config = vllm_config.model_config |
| 237 | speculative_config = vllm_config.speculative_config |
| 238 | scheduler_config = vllm_config.scheduler_config |
| 239 | |
| 240 | self.vllm_config = VllmConfig( |
| 241 | compilation_config=compilation_config, |
| 242 | model_config=model_config, |
| 243 | speculative_config=speculative_config, |
| 244 | scheduler_config=scheduler_config, |
| 245 | ) |
| 246 | |
| 247 | self.min_n = vllm_config.speculative_config.prompt_lookup_min |
| 248 | self.max_n = vllm_config.speculative_config.prompt_lookup_max |
| 249 | self.k = vllm_config.speculative_config.num_speculative_tokens |
| 250 | self.max_model_len = vllm_config.model_config.max_model_len |
| 251 | self.max_num_seqs = vllm_config.scheduler_config.max_num_seqs |
| 252 | self.device = device |
| 253 | |
| 254 | self.kernel = NgramGPUKernel( |
| 255 | vllm_config=self.vllm_config, prefix="ngram_gpu_kernel", device=device |
| 256 | ) |
| 257 | self.kernel.to(device) |
| 258 | self.kernel.eval() |
| 259 | |
| 260 | self._dummy_run() |
| 261 | |
| 262 | def _dummy_run(self): |
| 263 | token_ids, num_tokens, sampled_flags, valid_mask = self._generate_dummy_data( |
| 264 | batch_size=self.max_num_seqs, |
| 265 | max_seq_len=self.max_model_len, |
| 266 | pattern_len=self.k, |
| 267 | device=self.device, |
| 268 | ) |
| 269 | |
| 270 | combined_mask = sampled_flags & valid_mask & (num_tokens >= self.min_n) |
| 271 | |
| 272 | for _ in range(3): |
| 273 | with set_forward_context(None, self.vllm_config): |
| 274 | _, _ = self.kernel(num_tokens, token_ids, combined_mask) |
| 275 | |
| 276 | def _generate_dummy_data( |
| 277 | self, |
| 278 | batch_size: int, |
| 279 | max_seq_len: int, |
| 280 | pattern_len: int, |
| 281 | device: str = "cuda", |
| 282 | ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: |
| 283 | """Generate random test data with n-gram repetitions. |
| 284 | |
| 285 | Args: |
| 286 | batch_size: Number of sequences in the batch |
| 287 | max_seq_len: Maximum sequence length |
| 288 | pattern_len: Length of patterns to inject for matching |
| 289 | device: Device to place tensors on |
| 290 | |
| 291 | Returns: |
| 292 | token_ids: [batch_size, max_seq_len] tensor |
| 293 | num_tokens: [batch_size] tensor |
| 294 | sampled_flags: [batch_size] bool tensor |
| 295 | valid_mask: [batch_size] bool tensor |
| 296 | |
| 297 | """ |
| 298 | token_ids = torch.zeros( |
| 299 | batch_size, |
| 300 | max_seq_len, |
| 301 | dtype=torch.int32, |
| 302 | device=device, |
| 303 | ) |
| 304 | |
| 305 | num_tokens = torch.randint( |
| 306 | pattern_len, max_seq_len, (batch_size,), dtype=torch.int32, device=device |
| 307 | ) |
| 308 | |
| 309 | sampled_flags = torch.ones(batch_size, dtype=torch.bool, device=device) |
| 310 | valid_mask = torch.ones(batch_size, dtype=torch.bool, device=device) |
| 311 | |
| 312 | return token_ids, num_tokens, sampled_flags, valid_mask |
| 313 | |
| 314 | def propose( |
| 315 | self, |
| 316 | num_speculative_tokens: int, |
| 317 | num_tokens_no_spec: torch.Tensor, # [batch_size] |
| 318 | token_ids_gpu: torch.Tensor, # [batch_size, max_len] |
| 319 | valid_sampled_token_ids_gpu: torch.Tensor, # [batch_size, num_spec_tokens + 1] |
| 320 | valid_sampled_tokens_count: torch.Tensor, # [batch_size] |
| 321 | ) -> tuple[torch.Tensor, torch.Tensor]: |
| 322 | """Propose draft tokens using GPU-accelerated n-gram matching. |
| 323 | |
| 324 | Scatter sampled tokens into `token_ids_gpu`, compute temporary |
| 325 | updated lengths, then run the kernel. |
| 326 | |
| 327 | Args: |
| 328 | num_speculative_tokens: Number of speculative tokens to propose. |
| 329 | num_tokens_no_spec: Number of tokens per sequence (read-only) |
| 330 | token_ids_gpu: Token IDs tensor (modified in-place with new tokens) |
| 331 | valid_sampled_token_ids_gpu: Newly sampled tokens to scatter |
| 332 | valid_sampled_tokens_count: Count of valid tokens per sequence |
| 333 | |
| 334 | Returns: |
| 335 | draft_tokens: Proposed draft token IDs [batch_size, k] |
| 336 | num_valid_draft_tokens: Count of leading valid draft tokens |
| 337 | per request [batch_size] |
| 338 | |
| 339 | """ |
| 340 | assert num_speculative_tokens == self.k |
| 341 | assert token_ids_gpu.device == self.device |
| 342 | assert num_tokens_no_spec.device == self.device |
| 343 | |
| 344 | batch_size = num_tokens_no_spec.shape[0] |
| 345 | max_seq_len = token_ids_gpu.shape[1] |
| 346 | max_new_tokens = valid_sampled_token_ids_gpu.shape[1] # num_spec_tokens + 1 |
| 347 | |
| 348 | # Scatter newly sampled tokens into token_ids_gpu. |
| 349 | offsets = torch.arange(max_new_tokens, device=self.device) |
| 350 | write_positions = num_tokens_no_spec.unsqueeze(1) + offsets.unsqueeze(0) |
| 351 | valid_write_mask = offsets.unsqueeze(0) < valid_sampled_tokens_count.unsqueeze( |
| 352 | 1 |
| 353 | ) |
| 354 | in_bounds = write_positions < max_seq_len |
| 355 | scatter_mask = ( |
| 356 | valid_write_mask & (valid_sampled_token_ids_gpu != -1) & in_bounds |
| 357 | ) |
| 358 | |
| 359 | write_positions.clamp_(max=max_seq_len - 1) |
| 360 | write_positions_long = write_positions.long() |
| 361 | existing_values = token_ids_gpu.gather(1, write_positions_long) |
| 362 | |
| 363 | tokens_cast = valid_sampled_token_ids_gpu.to(token_ids_gpu.dtype) |
| 364 | tokens_to_scatter = torch.where( |
| 365 | scatter_mask, |
| 366 | tokens_cast, |
| 367 | existing_values, |
| 368 | ) |
| 369 | token_ids_gpu.scatter_(1, write_positions_long, tokens_to_scatter) |
| 370 | |
| 371 | num_tokens_tmp = (num_tokens_no_spec + valid_sampled_tokens_count).to( |
| 372 | torch.int32 |
| 373 | ) |
| 374 | |
| 375 | # Compute validity masks. |
| 376 | sampled_flags = valid_sampled_tokens_count > 0 |
| 377 | valid_mask = torch.ones(batch_size, dtype=torch.bool, device=self.device) |
| 378 | |
| 379 | with set_forward_context(None, self.vllm_config): |
| 380 | combined_mask = sampled_flags & valid_mask & (num_tokens_tmp >= self.min_n) |
| 381 | |
| 382 | with record_function_or_nullcontext("ngram_proposer_gpu: kernel"): |
| 383 | draft_tokens, num_valid_draft_tokens = self.kernel( |
| 384 | num_tokens_tmp, |
| 385 | token_ids_gpu, |
| 386 | combined_mask, |
| 387 | ) |
| 388 | |
| 389 | return draft_tokens, num_valid_draft_tokens |
| 390 | |
| 391 | def update_token_ids_ngram( |
| 392 | self, |
| 393 | sampled_token_ids: torch.Tensor | list[list[int]], |
| 394 | gpu_input_batch: InputBatch, |
| 395 | token_ids_gpu: torch.Tensor, |
| 396 | num_tokens_no_spec: torch.Tensor, |
| 397 | discard_request_mask: torch.Tensor, |
| 398 | ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: |
| 399 | """Prepare speculative decoding inputs on device: |
| 400 | compute next token ids and valid counts, honoring discarded requests |
| 401 | and rejected tokens, without CPU-GPU sync. |
| 402 | """ |
| 403 | num_reqs = gpu_input_batch.num_reqs |
| 404 | |
| 405 | if isinstance(sampled_token_ids, list): |
| 406 | # When disable_padded_drafter_batch=True, sampled_token_ids is |
| 407 | # an irregular list[list[int]] where sublists may have different |
| 408 | # lengths (including empty lists for discarded requests). |
| 409 | # Pad all sublists to the same length with -1 before converting |
| 410 | # to tensor. |
| 411 | max_len = max( |
| 412 | (len(sublist) for sublist in sampled_token_ids), |
| 413 | default=0, |
| 414 | ) |
| 415 | # Ensure at least length 1 for tensor creation |
| 416 | max_len = max(max_len, 1) |
| 417 | padded_list = [ |
| 418 | sublist + [-1] * (max_len - len(sublist)) |
| 419 | for sublist in sampled_token_ids |
| 420 | ] |
| 421 | sampled_token_ids = torch.tensor( |
| 422 | padded_list, dtype=torch.int32, device=self.device |
| 423 | ) |
| 424 | assert isinstance(sampled_token_ids, torch.Tensor), ( |
| 425 | "sampled_token_ids should be a torch.Tensor for ngram_gpu" |
| 426 | ) |
| 427 | |
| 428 | # Backup last valid token before speculative tokens. |
| 429 | backup_indices = num_tokens_no_spec[:num_reqs] - 1 |
| 430 | backup_indices.clamp_(min=0) |
| 431 | backup_indices = backup_indices.long() |
| 432 | backup_next_token_ids = torch.gather( |
| 433 | token_ids_gpu[:num_reqs], dim=1, index=backup_indices.unsqueeze(1) |
| 434 | ).squeeze(1) |
| 435 | |
| 436 | valid_sampled_token_ids_gpu = sampled_token_ids.clone() |
| 437 | # Invalidate sampled tokens for discarded requests. |
| 438 | discard_mask_expanded = discard_request_mask[:num_reqs].unsqueeze(1) |
| 439 | valid_sampled_token_ids_gpu.masked_fill_(discard_mask_expanded, -1) |
| 440 | |
| 441 | # Mask valid tokens within each request. |
| 442 | valid_mask = (valid_sampled_token_ids_gpu != -1) & ( |
| 443 | valid_sampled_token_ids_gpu < gpu_input_batch.vocab_size |
| 444 | ) |
| 445 | |
| 446 | # Count valid tokens per request. |
| 447 | valid_sampled_tokens_count = valid_mask.sum(dim=1).to(torch.int32) |
| 448 | |
| 449 | # Rightmost valid index per row. |
| 450 | last_valid_indices = valid_sampled_tokens_count - 1 |
| 451 | has_valid_sample = last_valid_indices >= 0 |
| 452 | last_valid_indices.clamp_(min=0) |
| 453 | |
| 454 | # Last valid token from each row; undefined if none. |
| 455 | selected_tokens = torch.gather( |
| 456 | valid_sampled_token_ids_gpu, 1, last_valid_indices.unsqueeze(1) |
| 457 | ).squeeze(1) |
| 458 | |
| 459 | # Use last token if valid; otherwise fallback to backup. |
| 460 | next_token_ids = torch.where( |
| 461 | has_valid_sample, |
| 462 | selected_tokens, |
| 463 | backup_next_token_ids, |
| 464 | ) |
| 465 | |
| 466 | return next_token_ids, valid_sampled_tokens_count, valid_sampled_token_ids_gpu |
| 467 | |
| 468 | def load_model(self, *args, **kwargs): |
| 469 | self.kernel.load_model(*args, **kwargs) |
| 470 | |
| 471 | |
| 472 | def update_scheduler_for_invalid_drafts( |
| 473 | num_valid_draft_tokens_event: torch.cuda.Event, |
| 474 | num_valid_draft_tokens_cpu: torch.Tensor, |
| 475 | scheduler_output: "SchedulerOutput", |
| 476 | req_id_to_index: dict[str, int], |
| 477 | ) -> None: |
| 478 | """Trim invalid speculative slots using per-request valid draft counts. |
| 479 | |
| 480 | Args: |
| 481 | num_valid_draft_tokens_event: Event for async D2H completion. |
| 482 | num_valid_draft_tokens_cpu: CPU buffer of valid draft counts. |
| 483 | scheduler_output: Scheduler metadata to update in-place. |
| 484 | req_id_to_index: Request-id to batch-index mapping. |
| 485 | |
| 486 | """ |
| 487 | req_data = scheduler_output.scheduled_cached_reqs |
| 488 | num_valid_draft_tokens_event.synchronize() |
| 489 | |
| 490 | for req_id in req_data.req_ids: |
| 491 | req_index = req_id_to_index.get(req_id) |
| 492 | if req_index is None: |
| 493 | continue |
| 494 | |
| 495 | spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(req_id) |
| 496 | if spec_token_ids is None: |
| 497 | continue |
| 498 | |
| 499 | scheduled_k = len(spec_token_ids) |
| 500 | |
| 501 | valid_k = int(num_valid_draft_tokens_cpu[req_index].item()) |
| 502 | valid_k = max(0, min(valid_k, scheduled_k)) |
| 503 | |
| 504 | tokens_to_trim = scheduled_k - valid_k |
| 505 | scheduler_output.total_num_scheduled_tokens -= tokens_to_trim |
| 506 | scheduler_output.num_scheduled_tokens[req_id] -= tokens_to_trim |
| 507 | |
| 508 | if valid_k == 0: |
| 509 | scheduler_output.scheduled_spec_decode_tokens.pop(req_id, None) |
| 510 | else: |
| 511 | scheduler_output.scheduled_spec_decode_tokens[req_id] = spec_token_ids[ |
| 512 | :valid_k |
| 513 | ] |
| 514 | |
| 515 | |
| 516 | def update_ngram_gpu_tensors_incremental( |
| 517 | input_batch: InputBatch, |
| 518 | token_ids_gpu_tensor: torch.Tensor, |
| 519 | num_tokens_no_spec_gpu: torch.Tensor, |
| 520 | new_reqs: list[CachedRequestState], |
| 521 | device: torch.device, |
| 522 | _pinned_idx_buf: torch.Tensor, |
| 523 | _pinned_val_buf: torch.Tensor, |
| 524 | ) -> None: |
| 525 | """Incrementally update token_ids_gpu_tensor and num_tokens_no_spec_gpu |
| 526 | for ngram GPU proposer. |
| 527 | """ |
| 528 | prev_req_id_to_index = input_batch.prev_req_id_to_index |
| 529 | curr_req_id_to_index = input_batch.req_id_to_index |
| 530 | |
| 531 | if not curr_req_id_to_index: |
| 532 | return |
| 533 | |
| 534 | active_indices = list(curr_req_id_to_index.values()) |
| 535 | n_active = len(active_indices) |
| 536 | |
| 537 | # Use resident pinned buffers to avoid per-call allocation. |
| 538 | active_idx_cpu = _pinned_idx_buf[:n_active] |
| 539 | active_idx_cpu.copy_(torch.as_tensor(active_indices, dtype=torch.long)) |
| 540 | |
| 541 | active_idx_gpu = active_idx_cpu.to(device=device, non_blocking=True) |
| 542 | |
| 543 | new_req_ids = {req.req_id for req in new_reqs} |
| 544 | |
| 545 | # First run, no previous state. |
| 546 | if prev_req_id_to_index is None: |
| 547 | for idx in active_indices: |
| 548 | num_tokens = input_batch.num_tokens_no_spec[idx] |
| 549 | if num_tokens > 0: |
| 550 | token_ids_gpu_tensor[idx, :num_tokens].copy_( |
| 551 | input_batch.token_ids_cpu_tensor[idx, :num_tokens].pin_memory(), |
| 552 | non_blocking=True, |
| 553 | ) |
| 554 | |
| 555 | _sync_num_tokens( |
| 556 | input_batch, |
| 557 | num_tokens_no_spec_gpu, |
| 558 | active_idx_cpu, |
| 559 | active_idx_gpu, |
| 560 | n_active, |
| 561 | device, |
| 562 | _pinned_val_buf, |
| 563 | ) |
| 564 | return |
| 565 | |
| 566 | # Detect index changes for reorder. |
| 567 | reorder_src: list[int] = [] |
| 568 | reorder_dst: list[int] = [] |
| 569 | |
| 570 | for req_id, curr_idx in curr_req_id_to_index.items(): |
| 571 | if req_id in new_req_ids: |
| 572 | continue |
| 573 | prev_idx = prev_req_id_to_index.get(req_id) |
| 574 | if prev_idx is not None and prev_idx != curr_idx: |
| 575 | reorder_src.append(prev_idx) |
| 576 | reorder_dst.append(curr_idx) |
| 577 | |
| 578 | if reorder_src: |
| 579 | src_tensor = async_tensor_h2d(reorder_src, dtype=torch.long, device=device) |
| 580 | dst_tensor = async_tensor_h2d(reorder_dst, dtype=torch.long, device=device) |
| 581 | |
| 582 | temp_token_ids = token_ids_gpu_tensor[src_tensor].clone() |
| 583 | temp_num_tokens = num_tokens_no_spec_gpu[src_tensor].clone() |
| 584 | |
| 585 | token_ids_gpu_tensor[dst_tensor] = temp_token_ids |
| 586 | num_tokens_no_spec_gpu[dst_tensor] = temp_num_tokens |
| 587 | |
| 588 | # Full copy for new/resumed requests. |
| 589 | for req_state in new_reqs: |
| 590 | new_req_idx = curr_req_id_to_index.get(req_state.req_id) |
| 591 | if new_req_idx is None: |
| 592 | continue |
| 593 | |
| 594 | num_tokens = input_batch.num_tokens_no_spec[new_req_idx] |
| 595 | if num_tokens > 0: |
| 596 | token_ids_gpu_tensor[new_req_idx, :num_tokens].copy_( |
| 597 | input_batch.token_ids_cpu_tensor[new_req_idx, :num_tokens].pin_memory(), |
| 598 | non_blocking=True, |
| 599 | ) |
| 600 | |
| 601 | # Always batch-sync sequence lengths from CPU for ALL active requests. |
| 602 | _sync_num_tokens( |
| 603 | input_batch, |
| 604 | num_tokens_no_spec_gpu, |
| 605 | active_idx_cpu, |
| 606 | active_idx_gpu, |
| 607 | n_active, |
| 608 | device, |
| 609 | _pinned_val_buf, |
| 610 | ) |
| 611 | |
| 612 | |
| 613 | def _sync_num_tokens( |
| 614 | input_batch: InputBatch, |
| 615 | num_tokens_no_spec_gpu: torch.Tensor, |
| 616 | active_idx_cpu: torch.Tensor, |
| 617 | active_idx_gpu: torch.Tensor, |
| 618 | n_active: int, |
| 619 | device: torch.device, |
| 620 | _pinned_val_buf: torch.Tensor, |
| 621 | ) -> None: |
| 622 | """Batch-sync GPU sequence lengths from CPU source of truth. |
| 623 | |
| 624 | Inputs: |
| 625 | input_batch: Batch container with CPU length tensor. |
| 626 | num_tokens_no_spec_gpu: Destination GPU length tensor. |
| 627 | active_idx_cpu: Active request indices on CPU. |
| 628 | active_idx_gpu: Active request indices on GPU. |
| 629 | n_active: Number of active requests. |
| 630 | device: Target CUDA device. |
| 631 | _pinned_val_buf: Resident pinned int32 staging buffer. |
| 632 | Outputs: |
| 633 | None (updates num_tokens_no_spec_gpu in-place). |
| 634 | """ |
| 635 | src_cpu = input_batch.num_tokens_no_spec_cpu_tensor |
| 636 | vals = _pinned_val_buf[:n_active] |
| 637 | vals.copy_(src_cpu.index_select(0, active_idx_cpu)) |
| 638 | |
| 639 | num_tokens_no_spec_gpu.index_copy_( |
| 640 | 0, |
| 641 | active_idx_gpu, |
| 642 | vals.to(device=device, non_blocking=True), |
| 643 | ) |
| 644 | |
| 645 | |
| 646 | def copy_num_valid_draft_tokens( |
| 647 | num_valid_draft_tokens_cpu: torch.Tensor, |
| 648 | num_valid_draft_tokens_copy_stream: torch.cuda.Stream, |
| 649 | num_valid_draft_tokens_event: torch.cuda.Event, |
| 650 | num_valid_draft_tokens: torch.Tensor | None, |
| 651 | batch_size: int, |
| 652 | ) -> None: |
| 653 | """Async D2H copy of per-request valid draft counts.""" |
| 654 | if num_valid_draft_tokens is None: |
| 655 | return |
| 656 | |
| 657 | num_reqs_to_copy = min(batch_size, num_valid_draft_tokens.shape[0]) |
| 658 | if num_reqs_to_copy <= 0: |
| 659 | return |
| 660 | |
| 661 | default_stream = torch.cuda.current_stream() |
| 662 | with torch.cuda.stream(num_valid_draft_tokens_copy_stream): |
| 663 | num_valid_draft_tokens_copy_stream.wait_stream(default_stream) |
| 664 | num_valid_draft_tokens_cpu[:num_reqs_to_copy].copy_( |
| 665 | num_valid_draft_tokens[:num_reqs_to_copy], non_blocking=True |
| 666 | ) |
| 667 | num_valid_draft_tokens_event.record() |