@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dd66fd2b01
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[CI] fix pre-commit error (#12494)

Signed-off-by: Mengqing Cao <cmq0113@163.com>

Mengqing Cao committed Jan 28, 2025 at 14:11 UTC dd66fd2b01e1195b7ccc8ffcd4b5d49ff1946a56
2 files changed +29 -16
vllm/attention/ops/nki_flash_attn.py
+25 -12
@@ -106,11 +106,12 @@ def _flash_attention_core(
106 assert (continuous_batching_mask
107 is not None), "continuous_batching_mask input is required."
108 if continuous_batching_mask is not None:
109 - assert (logit_bias_tile is
110 - None), "continuous_batching_mask does not support logit_bias!"
109 + assert (
110 + logit_bias_tile
111 + is None), "continuous_batching_mask does not support logit_bias!"
112
113 # mask are used to only apply computation to the lower half of the matrix,
113 - # which reduce the arthimetic intensity by half
114 + # which reduce the arithmetic intensity by half
115 forward_mask = (q_tile_idx * B_P_SIZE >= local_k_large_tile_idx *
116 LARGE_TILE_SZ if use_causal_mask else None)
117
@@ -468,9 +469,11 @@ def flash_paged_attention(
469 block_in_partition)
470 loaded_v = nl.load(value_cache[block_tables_sbuf[v_i, j], :,
471 head_id, :])
471 - cur_v_tile[partition_idx,
472 - nl.ds(block_in_partition *
473 - block_size, block_size), :, ] = loaded_v
472 + cur_v_tile[
473 + partition_idx,
474 + nl.ds(block_in_partition * block_size, block_size),
475 + :,
476 + ] = loaded_v
477
478 cur_mask = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
479 dtype=mask.dtype)
@@ -601,20 +604,30 @@ def flash_paged_attention(
604 )
605
606 nl.store(
604 - o[batch_id, head_id * q_h_per_k_h + i_q_h,
605 - nl.ds(i * B_P_SIZE, B_P_SIZE), :, ],
607 + o[
608 + batch_id,
609 + head_id * q_h_per_k_h + i_q_h,
610 + nl.ds(i * B_P_SIZE, B_P_SIZE),
611 + :,
612 + ],
613 out,
614 )
615 # maximum and summation statistics
616 if return_debug_tensors:
617 nl.store(
611 - hbm_m_buffer[batch_id, head_id * q_h_per_k_h + i_q_h,
612 - nl.ds(i * B_P_SIZE, B_P_SIZE), ],
618 + hbm_m_buffer[
619 + batch_id,
620 + head_id * q_h_per_k_h + i_q_h,
621 + nl.ds(i * B_P_SIZE, B_P_SIZE),
622 + ],
623 m_buffer[i, i_q_h, :, :],
624 )
625 nl.store(
616 - hbm_l_buffer[batch_id, head_id * q_h_per_k_h + i_q_h,
617 - nl.ds(i * B_P_SIZE, B_P_SIZE), ],
626 + hbm_l_buffer[
627 + batch_id,
628 + head_id * q_h_per_k_h + i_q_h,
629 + nl.ds(i * B_P_SIZE, B_P_SIZE),
630 + ],
631 l_buffer[:, i, i_q_h],
632 )
633 nl.store(
vllm/spec_decode/spec_decode_worker.py
+4 -4
@@ -870,10 +870,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
870 accepted_index = accepted_token_ids + 1 # Convert -1 to 0
871 accepted_index = accepted_index.count_nonzero(dim=1).add_(-1) # b
872 # Drop non-terminal prefill chunks hidden states.
873 - hidden_states = hidden_states[
874 - accepted_index != VLLM_INVALID_TOKEN_ID]
875 - accepted_index = accepted_index[
876 - accepted_index != VLLM_INVALID_TOKEN_ID]
873 + hidden_states = hidden_states[accepted_index !=
874 + VLLM_INVALID_TOKEN_ID]
875 + accepted_index = accepted_index[accepted_index !=
876 + VLLM_INVALID_TOKEN_ID]
877 assert len(accepted_index) == hidden_states.shape[0] == len(
878 terminal_metadata)
879 index = accepted_index[:, None, None].expand(-1, 1,