A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Core/Bugfix] Add query dtype as per FlashInfer API requirements. (#8173)
Elfie Guo committed
Sep 4, 2024 at 22:12 UTC
e39ebf5cf5ec8f7449d633b6428333a99a206a1c
2 files changed
+10
-2
tests/kernels/test_flashinfer.py
+2
-1
@@ -445,7 +445,8 @@ def test_flashinfer_decode_with_paged_fp8_kv(
445
head_size,
446
block_size,
447
"NONE",
448
- data_type=dtype)
448
+ data_type=dtype,
449
+ q_data_type=dtype)
450
output = wrapper.forward(query,
451
kv_cache_fp8,
452
logits_soft_cap=soft_cap,
vllm/attention/backends/flashinfer.py
+8
-1
@@ -224,6 +224,7 @@ class FlashInferState(AttentionState):
224
query_start_loc=query_start_loc_host,
225
device=self.runner.device,
226
data_type=kv_cache_dtype,
227
+ q_data_type=self.runner.model_config.dtype,
228
use_cuda_graph=True,
229
decode_wrapper=self._graph_decode_wrapper,
230
prefill_wrapper=None)
@@ -292,6 +293,8 @@ class FlashInferMetadata(AttentionMetadata):
293
page_size: Optional[int] = None
294
# The data type of the paged kv cache
295
data_type: torch.dtype = None
296
+ # The data type of the query
297
+ q_data_type: torch.dtype = None
298
device: torch.device = torch.device("cuda")
299
is_profile_run: bool = False
300
@@ -353,7 +356,10 @@ class FlashInferMetadata(AttentionMetadata):
356
self.page_size,
357
# Disable flashinfer's pos encoding and use vllm's rope.
358
pos_encoding_mode="NONE",
356
- data_type=self.data_type)
359
+ # kv-cache data type.
360
+ data_type=self.data_type,
361
+ # query data type.
362
+ q_data_type=self.q_data_type)
363
364
def asdict_zerocopy(self,
365
skip_fields: Optional[Set[str]] = None
@@ -617,6 +623,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
623
query_start_loc=query_start_loc,
624
device=device,
625
data_type=kv_cache_dtype,
626
+ q_data_type=self.runner.model_config.dtype,
627
use_cuda_graph=use_captured_graph,
628
is_profile_run=self.is_profile_run)
629