@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e9630458c7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[SpecDecode] Support FlashInfer in DraftModelRunner (#6926)

Bongwon Jang committed Aug 6, 2024 at 00:05 UTC e9630458c7b11732e147c120817c53420280d471
1 file changed +47
vllm/spec_decode/draft_model_runner.py
+47
@@ -11,6 +11,17 @@ except ModuleNotFoundError:
11 from vllm.attention.backends.rocm_flash_attn import (
12 ROCmFlashAttentionMetadata as FlashAttentionMetadata)
13
14 +try:
15 + from flashinfer import BatchDecodeWithPagedKVCacheWrapper
16 + from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
17 + from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
18 + FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
19 +except ImportError:
20 + BatchDecodeWithPagedKVCacheWrapper = None
21 + CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
22 + BatchPrefillWithPagedKVCacheWrapper = None
23 + FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
24 +
25 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
26 ModelConfig, MultiModalConfig, ParallelConfig,
27 PromptAdapterConfig, SchedulerConfig)
@@ -79,6 +90,11 @@ class TP1DraftModelRunner(ModelRunner):
90 return_hidden_states=return_hidden_states,
91 )
92
93 + self.flashinfer_decode_workspace_buffer = None
94 + self.flashinfer_decode_wrapper = None
95 + self.flashinfer_prefill_workspace_buffer = None
96 + self.flashinfer_prefill_wrapper = None
97 +
98 def _update_flash_attn_metadata(self, attn_metadata, num_seqs,
99 num_queries):
100 assert isinstance(attn_metadata, FlashAttentionMetadata)
@@ -286,6 +302,37 @@ class TP1DraftModelRunner(ModelRunner):
302 model_input.prompt_adapter_requests,
303 model_input.prompt_adapter_mapping)
304
305 + if self.attn_backend.get_name() == "flashinfer":
306 + assert model_input.attn_metadata is not None
307 + assert model_input.input_tokens is not None
308 + if self.flashinfer_decode_workspace_buffer is None:
309 + self.flashinfer_decode_workspace_buffer = torch.empty(
310 + FLASHINFER_WORKSPACE_BUFFER_SIZE,
311 + dtype=torch.uint8,
312 + device=self.device)
313 + self.flashinfer_decode_wrapper = \
314 + BatchDecodeWithPagedKVCacheWrapper(
315 + self.flashinfer_decode_workspace_buffer, "NHD")
316 + self.flashinfer_prefill_workspace_buffer = torch.empty(
317 + FLASHINFER_WORKSPACE_BUFFER_SIZE,
318 + dtype=torch.uint8,
319 + device=self.device)
320 + self.flashinfer_prefill_wrapper = \
321 + BatchPrefillWithPagedKVCacheWrapper(
322 + self.flashinfer_prefill_workspace_buffer, "NHD")
323 +
324 + model_input.attn_metadata.prefill_wrapper = \
325 + self.flashinfer_prefill_wrapper
326 + if model_input.attn_metadata.use_cuda_graph:
327 + batch_size = model_input.input_tokens.shape[0]
328 + model_input.attn_metadata.decode_wrapper = \
329 + self.graph_runners[model_input.
330 + virtual_engine][batch_size].flashinfer_decode_wrapper
331 + else:
332 + model_input.attn_metadata.decode_wrapper = \
333 + self.flashinfer_decode_wrapper
334 + model_input.attn_metadata.begin_forward()
335 +
336 # Detect exec mode
337 assert model_input.attn_metadata is not None
338 use_cuda_graph = False