@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ec2affa8ae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Kernel] Flashinfer correctness fix for v0.1.3 (#7319)

Lily Liu committed Aug 12, 2024 at 00:59 UTC ec2affa8ae2664db88df2ff0248219e355d6629a
3 files changed +20 -24
.buildkite/test-pipeline.yaml
-5
@@ -60,8 +60,6 @@ steps:
60 - vllm/
61 - tests/basic_correctness
62 commands:
63 - # This flashinfer installation will fail on AMD ROCm, so it is set as optional.
64 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl || true
63 - pytest -v -s basic_correctness/test_basic_correctness.py
64 - pytest -v -s basic_correctness/test_cpu_offload.py
65 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
@@ -157,7 +155,6 @@ steps:
155 - vllm/
156 - tests/models
157 commands:
160 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
158 - pytest -v -s models -m \"not vlm\"
159
160 - label: Vision Language Models Test # 42min
@@ -212,7 +209,6 @@ steps:
209 - vllm/attention
210 - tests/kernels
211 commands:
215 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
212 - pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
213 parallelism: 4
214
@@ -331,7 +327,6 @@ steps:
327 # NOTE: don't test llama model here, it seems hf implementation is buggy
328 # see https://github.com/vllm-project/vllm/pull/5689 for details
329 - pytest -v -s distributed/test_custom_all_reduce.py
334 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
330 - TARGET_TEST_SUITE=A100 pytest -v -s distributed/test_basic_distributed_correctness.py
331 - pytest -v -s -x lora/test_mixtral.py
332
Dockerfile
+1 -1
@@ -194,7 +194,7 @@ RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamb
194 python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir
195
196 RUN --mount=type=cache,target=/root/.cache/pip \
197 - python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
197 + python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.3/flashinfer-0.1.3+cu121torch2.4-cp310-cp310-linux_x86_64.whl
198 #################### vLLM installation IMAGE ####################
199
200
vllm/attention/backends/flashinfer.py
+19 -18
@@ -117,6 +117,7 @@ class FlashInferMetadata(AttentionMetadata):
117 # The data type of the paged kv cache
118 data_type: torch.dtype = None
119 device: torch.device = torch.device("cuda")
120 + is_profile_run: bool = False
121
122 def __post_init__(self):
123 # Refer to
@@ -127,7 +128,6 @@ class FlashInferMetadata(AttentionMetadata):
128 raise ValueError(
129 f"Only {supported_head_sizes} are supported for head_dim,",
130 f"received {self.head_dim}.")
130 - self.is_profile_run = is_block_tables_empty(self.block_tables)
131
132 def begin_forward(self):
133 if self.num_prefill_tokens > 0:
@@ -141,23 +141,20 @@ class FlashInferMetadata(AttentionMetadata):
141 assert self.paged_kv_last_page_len is not None
142 batch_size = self.query_start_loc.shape[0] - 1
143 assert batch_size >= 0
144 - # The profile run does not read kv cache.
145 - # Both paged_kv_indices and paged_kv_last_page_len are empty.
146 - # paged_kv_indptr is a zero tensor with size batch_size + 1.
147 - if self.is_profile_run:
148 - self.paged_kv_indptr = torch.zeros(batch_size + 1,
149 - device=self.device)
150 - else:
144 + # We will use flash attention for profiling to
145 + # determine the number of blocks. Therefore,
146 + # we don't need to prepare the input for flashinfer for profile run.
147 + if not self.is_profile_run:
148 self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
152 - self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
153 - self.device)
154 - self.paged_kv_indices = self.paged_kv_indices.to(self.device)
155 - self.prefill_wrapper.end_forward()
156 - self.prefill_wrapper.begin_forward(
157 - self.query_start_loc, self.paged_kv_indptr,
158 - self.paged_kv_indices, self.paged_kv_last_page_len,
159 - self.num_qo_heads, self.num_kv_heads, self.head_dim,
160 - self.page_size)
149 + self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
150 + self.device)
151 + self.paged_kv_indices = self.paged_kv_indices.to(self.device)
152 + self.prefill_wrapper.end_forward()
153 + self.prefill_wrapper.begin_forward(
154 + self.query_start_loc, self.paged_kv_indptr,
155 + self.paged_kv_indices, self.paged_kv_last_page_len,
156 + self.num_qo_heads, self.num_kv_heads, self.head_dim,
157 + self.page_size)
158 else:
159 if not self.use_cuda_graph:
160 assert self.paged_kv_indices is not None
@@ -249,6 +246,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
246 # paged_kv_last_page_len is the length of the last page of each request
247 self.paged_kv_last_page_len: List[int] = []
248
249 + self.is_profile_run: bool = False
250 +
251 def _add_seq_group(
252 self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
253 chunked_prefill_enabled: bool):
@@ -305,6 +304,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
304 # and paged_kv_last_page_len for profile run because we will
305 # create dummy inputs.
306 if is_profile_run:
307 + self.is_profile_run = is_profile_run
308 return
309
310 block_table = block_tables[seq_id]
@@ -435,7 +435,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
435 query_start_loc=query_start_loc,
436 device=device,
437 data_type=kv_cache_dtype,
438 - use_cuda_graph=use_captured_graph)
438 + use_cuda_graph=use_captured_graph,
439 + is_profile_run=self.is_profile_run)
440
441
442 class FlashInferImpl(AttentionImpl):