A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[CI] fix pre-commit error (#12494)
Signed-off-by: Mengqing Cao <cmq0113@163.com>
Mengqing Cao committed
Jan 28, 2025 at 14:11 UTC
dd66fd2b01e1195b7ccc8ffcd4b5d49ff1946a56
2 files changed
+29
-16
vllm/attention/ops/nki_flash_attn.py
+25
-12
@@ -106,11 +106,12 @@ def _flash_attention_core(
106
assert (continuous_batching_mask
107
is not None), "continuous_batching_mask input is required."
108
if continuous_batching_mask is not None:
109
- assert (logit_bias_tile is
110
- None), "continuous_batching_mask does not support logit_bias!"
109
+ assert (
110
+ logit_bias_tile
111
+ is None), "continuous_batching_mask does not support logit_bias!"
112
113
# mask are used to only apply computation to the lower half of the matrix,
113
- # which reduce the arthimetic intensity by half
114
+ # which reduce the arithmetic intensity by half
115
forward_mask = (q_tile_idx * B_P_SIZE >= local_k_large_tile_idx *
116
LARGE_TILE_SZ if use_causal_mask else None)
117
@@ -468,9 +469,11 @@ def flash_paged_attention(
469
block_in_partition)
470
loaded_v = nl.load(value_cache[block_tables_sbuf[v_i, j], :,
471
head_id, :])
471
- cur_v_tile[partition_idx,
472
- nl.ds(block_in_partition *
473
- block_size, block_size), :, ] = loaded_v
472
+ cur_v_tile[
473
+ partition_idx,
474
+ nl.ds(block_in_partition * block_size, block_size),
475
+ :,
476
+ ] = loaded_v
477
478
cur_mask = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
479
dtype=mask.dtype)
@@ -601,20 +604,30 @@ def flash_paged_attention(
604
)
605
606
nl.store(
604
- o[batch_id, head_id * q_h_per_k_h + i_q_h,
605
- nl.ds(i * B_P_SIZE, B_P_SIZE), :, ],
607
+ o[
608
+ batch_id,
609
+ head_id * q_h_per_k_h + i_q_h,
610
+ nl.ds(i * B_P_SIZE, B_P_SIZE),
611
+ :,
612
+ ],
613
out,
614
)
615
# maximum and summation statistics
616
if return_debug_tensors:
617
nl.store(
611
- hbm_m_buffer[batch_id, head_id * q_h_per_k_h + i_q_h,
612
- nl.ds(i * B_P_SIZE, B_P_SIZE), ],
618
+ hbm_m_buffer[
619
+ batch_id,
620
+ head_id * q_h_per_k_h + i_q_h,
621
+ nl.ds(i * B_P_SIZE, B_P_SIZE),
622
+ ],
623
m_buffer[i, i_q_h, :, :],
624
)
625
nl.store(
616
- hbm_l_buffer[batch_id, head_id * q_h_per_k_h + i_q_h,
617
- nl.ds(i * B_P_SIZE, B_P_SIZE), ],
626
+ hbm_l_buffer[
627
+ batch_id,
628
+ head_id * q_h_per_k_h + i_q_h,
629
+ nl.ds(i * B_P_SIZE, B_P_SIZE),
630
+ ],
631
l_buffer[:, i, i_q_h],
632
)
633
nl.store(
vllm/spec_decode/spec_decode_worker.py
+4
-4
@@ -870,10 +870,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
870
accepted_index = accepted_token_ids + 1 # Convert -1 to 0
871
accepted_index = accepted_index.count_nonzero(dim=1).add_(-1) # b
872
# Drop non-terminal prefill chunks hidden states.
873
- hidden_states = hidden_states[
874
- accepted_index != VLLM_INVALID_TOKEN_ID]
875
- accepted_index = accepted_index[
876
- accepted_index != VLLM_INVALID_TOKEN_ID]
873
+ hidden_states = hidden_states[accepted_index !=
874
+ VLLM_INVALID_TOKEN_ID]
875
+ accepted_index = accepted_index[accepted_index !=
876
+ VLLM_INVALID_TOKEN_ID]
877
assert len(accepted_index) == hidden_states.shape[0] == len(
878
terminal_metadata)
879
index = accepted_index[:, None, None].expand(-1, 1,