@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ea48fb35c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1][Minor] Move cascade attn logic outside _prepare_inputs (#12943)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Feb 8, 2025 at 00:39 UTC 4ea48fb35cf67d61a1c3f18e3981c362e1d8e26f
1 file changed +89 -61
vllm/v1/worker/gpu_model_runner.py
+89 -61
@@ -476,67 +476,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
476 self.device, non_blocking=True).long()
477
478 # Prepare for cascade attention if needed.
479 - common_prefix_len = (scheduler_output.num_common_prefix_blocks *
480 - self.block_size)
481 - if common_prefix_len == 0:
482 - # Common case.
483 - use_cascade = False
484 - else:
485 - # NOTE(woosuk): Cascade attention uses two attention kernels: one
486 - # for the common prefix and the other for the rest. For the first
487 - # kernel, we concatenate all the query tokens (possibly from
488 - # different requests) and treat them as if they are from the same
489 - # request. Then, we use bi-directional attention to process the
490 - # common prefix in the KV cache. Importantly, this means that the
491 - # first kernel does not do any masking.
492 -
493 - # Consider the following example:
494 - # Request 1's input query: [D, E, X]
495 - # Request 1's kv cache: [A, B, C, D, E, X]
496 - # Request 1's num_computed_tokens: 3 (i.e., [A, B, C])
497 - # Request 2's input query: [E, Y]
498 - # Request 2's kv cache: [A, B, C, D, E, Y]
499 - # Request 2's num_computed_tokens: 4 (i.e., [A, B, C, D])
500 -
501 - # If we use [A, B, C, D, E] as the common prefix, then the
502 - # first kernel will compute the bi-directional attention between
503 - # input query [D, E, X, E, Y] and common prefix [A, B, C, D, E].
504 - # However, this is wrong because D in Request 1 should not attend to
505 - # E in the common prefix (i.e., we need masking).
506 - # To avoid this, [A, B, C, D] should be the common prefix.
507 - # That is, the common prefix should be capped by the minimum
508 - # num_computed_tokens among the requests, and plus one to include
509 - # the first token of the query.
510 -
511 - # In practice, we use [A, B, C] as the common prefix, instead of
512 - # [A, B, C, D] (i.e., the common prefix is capped by the minimum
513 - # num_computed_tokens, without plus one).
514 - # This is because of an implementation detail: We want to always
515 - # use two kernels for cascade attention. Let's imagine:
516 - # Request 3's input query: [D]
517 - # Request 3's kv cache: [A, B, C, D]
518 - # Request 3's num_computed_tokens: 4 (i.e., [A, B, C, D])
519 - # If we use [A, B, C, D] as the common prefix for Request 1-3,
520 - # then Request 3 will be processed only by the first kernel,
521 - # and the second kernel will get an empty input. While this is not
522 - # a fundamental problem, our current implementation does not support
523 - # this case.
524 - common_prefix_len = min(
525 - common_prefix_len,
526 - self.input_batch.num_computed_tokens_cpu[:num_reqs].min())
527 - # common_prefix_len should be a multiple of the block size.
528 - common_prefix_len = (common_prefix_len // self.block_size *
529 - self.block_size)
530 - use_cascade = FlashAttentionBackend.use_cascade_attention(
531 - common_prefix_len=common_prefix_len,
532 - query_lens=num_scheduled_tokens,
533 - num_query_heads=self.num_query_heads,
534 - num_kv_heads=self.num_kv_heads,
535 - use_alibi=False, # FIXME
536 - use_sliding_window=self.sliding_window is not None,
537 - num_sms=self.num_sms,
538 - )
539 -
479 + common_prefix_len = self._compute_cascade_attn_prefix_len(
480 + num_scheduled_tokens,
481 + scheduler_output.num_common_prefix_blocks,
482 + )
483 + use_cascade = common_prefix_len > 0
484 if use_cascade:
485 # TODO: Optimize.
486 cu_prefix_query_lens = torch.tensor(
@@ -581,6 +525,90 @@ class GPUModelRunner(LoRAModelRunnerMixin):
525 logits_indices = query_start_loc[1:] - 1
526 return attn_metadata, logits_indices
527
528 + def _compute_cascade_attn_prefix_len(
529 + self,
530 + num_scheduled_tokens: np.ndarray,
531 + num_common_prefix_blocks: int,
532 + ) -> int:
533 + """Compute the length of the common prefix for cascade attention.
534 +
535 + NOTE(woosuk): The common prefix length returned by this function
536 + represents the length used specifically for cascade attention, not the
537 + actual number of tokens shared between requests. When cascade attention
538 + is disabled (use_cascade=False), this function returns 0 even if
539 + requests share common tokens. Additionally, the common prefix length is
540 + truncated to a multiple of the block size and may be further truncated
541 + due to implementation details explained below.
542 +
543 + Args:
544 + num_scheduled_tokens: Number of tokens scheduled per request.
545 + num_common_prefix_blocks: Number of shared KV cache blocks.
546 +
547 + Returns:
548 + int: Length of common prefix in tokens.
549 + """
550 + common_prefix_len = num_common_prefix_blocks * self.block_size
551 + if common_prefix_len == 0:
552 + # Common case.
553 + return 0
554 +
555 + # NOTE(woosuk): Cascade attention uses two attention kernels: one
556 + # for the common prefix and the other for the rest. For the first
557 + # kernel, we concatenate all the query tokens (possibly from
558 + # different requests) and treat them as if they are from the same
559 + # request. Then, we use bi-directional attention to process the
560 + # common prefix in the KV cache. Importantly, this means that the
561 + # first kernel does not do any masking.
562 +
563 + # Consider the following example:
564 + # Request 1's input query: [D, E, X]
565 + # Request 1's kv cache: [A, B, C, D, E, X]
566 + # Request 1's num_computed_tokens: 3 (i.e., [A, B, C])
567 + # Request 2's input query: [E, Y]
568 + # Request 2's kv cache: [A, B, C, D, E, Y]
569 + # Request 2's num_computed_tokens: 4 (i.e., [A, B, C, D])
570 +
571 + # If we use [A, B, C, D, E] as the common prefix, then the
572 + # first kernel will compute the bi-directional attention between
573 + # input query [D, E, X, E, Y] and common prefix [A, B, C, D, E].
574 + # However, this is wrong because D in Request 1 should not attend to
575 + # E in the common prefix (i.e., we need masking).
576 + # To avoid this, [A, B, C, D] should be the common prefix.
577 + # That is, the common prefix should be capped by the minimum
578 + # num_computed_tokens among the requests, and plus one to include
579 + # the first token of the query.
580 +
581 + # In practice, we use [A, B, C] as the common prefix, instead of
582 + # [A, B, C, D] (i.e., the common prefix is capped by the minimum
583 + # num_computed_tokens, without plus one).
584 + # This is because of an implementation detail: We want to always
585 + # use two kernels for cascade attention. Let's imagine:
586 + # Request 3's input query: [D]
587 + # Request 3's kv cache: [A, B, C, D]
588 + # Request 3's num_computed_tokens: 4 (i.e., [A, B, C, D])
589 + # If we use [A, B, C, D] as the common prefix for Request 1-3,
590 + # then Request 3 will be processed only by the first kernel,
591 + # and the second kernel will get an empty input. While this is not
592 + # a fundamental problem, our current implementation does not support
593 + # this case.
594 + num_reqs = len(num_scheduled_tokens)
595 + common_prefix_len = min(
596 + common_prefix_len,
597 + self.input_batch.num_computed_tokens_cpu[:num_reqs].min())
598 + # common_prefix_len should be a multiple of the block size.
599 + common_prefix_len = (common_prefix_len // self.block_size *
600 + self.block_size)
601 + use_cascade = FlashAttentionBackend.use_cascade_attention(
602 + common_prefix_len=common_prefix_len,
603 + query_lens=num_scheduled_tokens,
604 + num_query_heads=self.num_query_heads,
605 + num_kv_heads=self.num_kv_heads,
606 + use_alibi=False, # FIXME
607 + use_sliding_window=self.sliding_window is not None,
608 + num_sms=self.num_sms,
609 + )
610 + return common_prefix_len if use_cascade else 0
611 +
612 def _calc_mrope_positions(self, scheduler_output: "SchedulerOutput"):
613 mrope_pos_ptr = 0
614 num_reqs = self.input_batch.num_reqs