A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Flashinfer correctness fix for v0.1.3 (#7319)
Lily Liu committed
Aug 12, 2024 at 00:59 UTC
ec2affa8ae2664db88df2ff0248219e355d6629a
3 files changed
+20
-24
.buildkite/test-pipeline.yaml
-5
@@ -60,8 +60,6 @@ steps:
60
- vllm/
61
- tests/basic_correctness
62
commands:
63
- # This flashinfer installation will fail on AMD ROCm, so it is set as optional.
64
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl || true
63
- pytest -v -s basic_correctness/test_basic_correctness.py
64
- pytest -v -s basic_correctness/test_cpu_offload.py
65
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
@@ -157,7 +155,6 @@ steps:
155
- vllm/
156
- tests/models
157
commands:
160
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
158
- pytest -v -s models -m \"not vlm\"
159
160
- label: Vision Language Models Test # 42min
@@ -212,7 +209,6 @@ steps:
209
- vllm/attention
210
- tests/kernels
211
commands:
215
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
212
- pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
213
parallelism: 4
214
@@ -331,7 +327,6 @@ steps:
327
# NOTE: don't test llama model here, it seems hf implementation is buggy
328
# see https://github.com/vllm-project/vllm/pull/5689 for details
329
- pytest -v -s distributed/test_custom_all_reduce.py
334
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
330
- TARGET_TEST_SUITE=A100 pytest -v -s distributed/test_basic_distributed_correctness.py
331
- pytest -v -s -x lora/test_mixtral.py
332
Dockerfile
+1
-1
@@ -194,7 +194,7 @@ RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamb
194
python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir
195
196
RUN --mount=type=cache,target=/root/.cache/pip \
197
- python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
197
+ python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.3/flashinfer-0.1.3+cu121torch2.4-cp310-cp310-linux_x86_64.whl
198
#################### vLLM installation IMAGE ####################
199
200
vllm/attention/backends/flashinfer.py
+19
-18
@@ -117,6 +117,7 @@ class FlashInferMetadata(AttentionMetadata):
117
# The data type of the paged kv cache
118
data_type: torch.dtype = None
119
device: torch.device = torch.device("cuda")
120
+ is_profile_run: bool = False
121
122
def __post_init__(self):
123
# Refer to
@@ -127,7 +128,6 @@ class FlashInferMetadata(AttentionMetadata):
128
raise ValueError(
129
f"Only {supported_head_sizes} are supported for head_dim,",
130
f"received {self.head_dim}.")
130
- self.is_profile_run = is_block_tables_empty(self.block_tables)
131
132
def begin_forward(self):
133
if self.num_prefill_tokens > 0:
@@ -141,23 +141,20 @@ class FlashInferMetadata(AttentionMetadata):
141
assert self.paged_kv_last_page_len is not None
142
batch_size = self.query_start_loc.shape[0] - 1
143
assert batch_size >= 0
144
- # The profile run does not read kv cache.
145
- # Both paged_kv_indices and paged_kv_last_page_len are empty.
146
- # paged_kv_indptr is a zero tensor with size batch_size + 1.
147
- if self.is_profile_run:
148
- self.paged_kv_indptr = torch.zeros(batch_size + 1,
149
- device=self.device)
150
- else:
144
+ # We will use flash attention for profiling to
145
+ # determine the number of blocks. Therefore,
146
+ # we don't need to prepare the input for flashinfer for profile run.
147
+ if not self.is_profile_run:
148
self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
152
- self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
153
- self.device)
154
- self.paged_kv_indices = self.paged_kv_indices.to(self.device)
155
- self.prefill_wrapper.end_forward()
156
- self.prefill_wrapper.begin_forward(
157
- self.query_start_loc, self.paged_kv_indptr,
158
- self.paged_kv_indices, self.paged_kv_last_page_len,
159
- self.num_qo_heads, self.num_kv_heads, self.head_dim,
160
- self.page_size)
149
+ self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
150
+ self.device)
151
+ self.paged_kv_indices = self.paged_kv_indices.to(self.device)
152
+ self.prefill_wrapper.end_forward()
153
+ self.prefill_wrapper.begin_forward(
154
+ self.query_start_loc, self.paged_kv_indptr,
155
+ self.paged_kv_indices, self.paged_kv_last_page_len,
156
+ self.num_qo_heads, self.num_kv_heads, self.head_dim,
157
+ self.page_size)
158
else:
159
if not self.use_cuda_graph:
160
assert self.paged_kv_indices is not None
@@ -249,6 +246,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
246
# paged_kv_last_page_len is the length of the last page of each request
247
self.paged_kv_last_page_len: List[int] = []
248
249
+ self.is_profile_run: bool = False
250
+
251
def _add_seq_group(
252
self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
253
chunked_prefill_enabled: bool):
@@ -305,6 +304,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
304
# and paged_kv_last_page_len for profile run because we will
305
# create dummy inputs.
306
if is_profile_run:
307
+ self.is_profile_run = is_profile_run
308
return
309
310
block_table = block_tables[seq_id]
@@ -435,7 +435,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
435
query_start_loc=query_start_loc,
436
device=device,
437
data_type=kv_cache_dtype,
438
- use_cuda_graph=use_captured_graph)
438
+ use_cuda_graph=use_captured_graph,
439
+ is_profile_run=self.is_profile_run)
440
441
442
class FlashInferImpl(AttentionImpl):