@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e39ebf5cf5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Core/Bugfix] Add query dtype as per FlashInfer API requirements. (#8173)

Elfie Guo committed Sep 4, 2024 at 22:12 UTC e39ebf5cf5ec8f7449d633b6428333a99a206a1c
2 files changed +10 -2
tests/kernels/test_flashinfer.py
+2 -1
@@ -445,7 +445,8 @@ def test_flashinfer_decode_with_paged_fp8_kv(
445 head_size,
446 block_size,
447 "NONE",
448 - data_type=dtype)
448 + data_type=dtype,
449 + q_data_type=dtype)
450 output = wrapper.forward(query,
451 kv_cache_fp8,
452 logits_soft_cap=soft_cap,
vllm/attention/backends/flashinfer.py
+8 -1
@@ -224,6 +224,7 @@ class FlashInferState(AttentionState):
224 query_start_loc=query_start_loc_host,
225 device=self.runner.device,
226 data_type=kv_cache_dtype,
227 + q_data_type=self.runner.model_config.dtype,
228 use_cuda_graph=True,
229 decode_wrapper=self._graph_decode_wrapper,
230 prefill_wrapper=None)
@@ -292,6 +293,8 @@ class FlashInferMetadata(AttentionMetadata):
293 page_size: Optional[int] = None
294 # The data type of the paged kv cache
295 data_type: torch.dtype = None
296 + # The data type of the query
297 + q_data_type: torch.dtype = None
298 device: torch.device = torch.device("cuda")
299 is_profile_run: bool = False
300
@@ -353,7 +356,10 @@ class FlashInferMetadata(AttentionMetadata):
356 self.page_size,
357 # Disable flashinfer's pos encoding and use vllm's rope.
358 pos_encoding_mode="NONE",
356 - data_type=self.data_type)
359 + # kv-cache data type.
360 + data_type=self.data_type,
361 + # query data type.
362 + q_data_type=self.q_data_type)
363
364 def asdict_zerocopy(self,
365 skip_fields: Optional[Set[str]] = None
@@ -617,6 +623,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
623 query_start_loc=query_start_loc,
624 device=device,
625 data_type=kv_cache_dtype,
626 + q_data_type=self.runner.model_config.dtype,
627 use_cuda_graph=use_captured_graph,
628 is_profile_run=self.is_profile_run)
629