@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4089985552
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Integrate Piecewise CUDA graphs (#10058)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 5, 2024 at 22:16 UTC 40899855520eb9497606bdb2b1b4e619233e598a
3 files changed +133 -36
vllm/compilation/backends.py
+5 -2
@@ -496,8 +496,11 @@ class PiecewiseBackend:
496 return entry.runnable(*args)
497
498 if self.is_first_graph:
499 - logger.info("Capturing a cudagraph for shape %s",
500 - runtime_shape)
499 + # Since we capture cudagraph for many different shapes and
500 + # capturing is fast, we don't need to log it for every shape.
501 + # We only log it in the debug mode.
502 + logger.debug("Capturing a cudagraph for shape %s",
503 + runtime_shape)
504
505 input_addresses = [
506 x.data_ptr() for x in args if isinstance(x, torch.Tensor)
vllm/v1/attention/backends/flash_attn.py
+21 -14
@@ -51,6 +51,7 @@ class FlashAttentionMetadata:
51 # |-------------------- seq_len ---------------------|
52 # |-- query_len ---|
53
54 + num_actual_tokens: int # Number of tokens excluding padding.
55 max_query_len: int
56 query_start_loc: torch.Tensor
57 max_seq_len: int
@@ -134,7 +135,9 @@ class FlashAttentionImpl(AttentionImpl):
135 assert k_scale == 1.0 and v_scale == 1.0, (
136 "key/v_scale is not supported in FlashAttention.")
137
137 - output = torch.ops.vllm.unified_flash_attention(
138 + output = torch.empty_like(query)
139 + torch.ops.vllm.unified_flash_attention(
140 + output,
141 query,
142 key,
143 value,
@@ -154,6 +157,7 @@ class FlashAttentionImpl(AttentionImpl):
157
158
159 def unified_flash_attention(
160 + output: torch.Tensor,
161 query: torch.Tensor,
162 key: torch.Tensor,
163 value: torch.Tensor,
@@ -168,17 +172,17 @@ def unified_flash_attention(
172 window_size: Optional[List[int]] = None,
173 alibi_slopes: Optional[torch.Tensor] = None,
174 logits_soft_cap: Optional[float] = None,
171 -) -> torch.Tensor:
175 +) -> None:
176 current_metadata = get_forward_context()
177 if current_metadata is None:
178 # Profiling run.
175 - return torch.empty_like(query)
179 + return
180
181 assert current_metadata is not None
182 assert isinstance(current_metadata, FlashAttentionMetadata)
183 attn_metadata: FlashAttentionMetadata = current_metadata
184 + num_actual_tokens = attn_metadata.num_actual_tokens
185
181 - num_tokens, hidden_size = query.shape
186 # Reshape the query, key, and value tensors.
187 query = query.view(-1, num_heads, head_size)
188 key = key.view(-1, num_kv_heads, head_size)
@@ -188,18 +192,18 @@ def unified_flash_attention(
192 key_cache = kv_cache[0]
193 value_cache = kv_cache[1]
194 torch.ops._C_cache_ops.reshape_and_cache_flash(
191 - key,
192 - value,
193 - kv_cache[0],
194 - kv_cache[1],
195 + key[:num_actual_tokens],
196 + value[:num_actual_tokens],
197 + key_cache,
198 + value_cache,
199 attn_metadata.slot_mapping,
200 kv_cache_dtype,
201 k_scale,
202 v_scale,
203 )
204
201 - output = flash_attn_varlen_func(
202 - q=query,
205 + attn_output = flash_attn_varlen_func(
206 + q=query[:num_actual_tokens],
207 k=key_cache,
208 v=value_cache,
209 cu_seqlens_q=attn_metadata.query_start_loc,
@@ -213,10 +217,13 @@ def unified_flash_attention(
217 block_table=attn_metadata.block_table,
218 softcap=logits_soft_cap,
219 )
216 - return output.view(num_tokens, hidden_size)
220 + attn_output = attn_output.view(num_actual_tokens, -1)
221 + # TODO(woosuk): Optimize this.
222 + output[:num_actual_tokens].copy_(attn_output)
223
224
225 def unified_flash_attention_fake(
226 + output: torch.Tensor,
227 query: torch.Tensor,
228 key: torch.Tensor,
229 value: torch.Tensor,
@@ -231,13 +238,13 @@ def unified_flash_attention_fake(
238 window_size: Optional[List[int]] = None,
239 alibi_slopes: Optional[torch.Tensor] = None,
240 logits_soft_cap: Optional[float] = None,
234 -) -> torch.Tensor:
235 - return torch.empty_like(query)
241 +) -> None:
242 + return
243
244
245 direct_register_custom_op(
246 op_name="unified_flash_attention",
247 op_func=unified_flash_attention,
241 - mutates_args=["kv_cache"],
248 + mutates_args=["kv_cache", "output"],
249 fake_impl=unified_flash_attention_fake,
250 )
vllm/v1/worker/gpu_model_runner.py
+107 -20
@@ -1,3 +1,5 @@
1 +import os
2 +import time
3 from dataclasses import dataclass
4 from typing import TYPE_CHECKING, Dict, List, Optional, Set
5 from unittest.mock import patch
@@ -7,11 +9,16 @@ import torch
9 import torch.distributed
10 import torch.nn as nn
11
12 +from vllm import envs
13 +from vllm.compilation.compile_context import set_compile_context
14 +from vllm.compilation.config import CompilationConfig
15 +from vllm.compilation.levels import CompilationLevel
16 from vllm.config import VllmConfig
17 from vllm.forward_context import set_forward_context
18 from vllm.logger import init_logger
19 from vllm.model_executor.model_loader import get_model
20 from vllm.multimodal import MultiModalDataDict
21 +from vllm.plugins import set_compilation_config
22 from vllm.sampling_params import SamplingParams, SamplingType
23 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
24 is_pin_memory_available)
@@ -86,6 +93,18 @@ class GPUModelRunner:
93 pin_memory=self.pin_memory,
94 )
95
96 + self.use_cuda_graph = (envs.VLLM_TORCH_COMPILE_LEVEL
97 + == CompilationLevel.PIECEWISE
98 + and not self.model_config.enforce_eager)
99 + # TODO(woosuk): Provide an option to tune the max cudagraph batch size.
100 + self.cudagraph_batch_sizes = [1, 2, 4] + [i for i in range(8, 513, 8)]
101 + self.input_ids = torch.zeros(self.max_num_tokens,
102 + dtype=torch.int32,
103 + device=self.device)
104 + self.positions = torch.zeros(self.max_num_tokens,
105 + dtype=torch.int64,
106 + device=self.device)
107 +
108 def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
109 # Remove stopped requests from the cached states.
110 # Keep the states of the pre-empted requests.
@@ -268,12 +287,16 @@ class GPUModelRunner:
287 seq_start_loc_np[0] = 0
288 np.cumsum(seq_lens, out=seq_start_loc_np[1:])
289
271 - input_ids = input_ids.to(self.device, non_blocking=True)
272 - positions = positions.to(self.device, non_blocking=True).long()
290 + self.input_ids[:total_num_scheduled_tokens].copy_(input_ids,
291 + non_blocking=True)
292 + self.positions[:total_num_scheduled_tokens].copy_(positions,
293 + non_blocking=True)
294 +
295 query_start_loc = query_start_loc.to(self.device, non_blocking=True)
296 seq_start_loc = seq_start_loc.to(self.device, non_blocking=True)
297 slot_mapping = slot_mapping.to(self.device, non_blocking=True).long()
298 attn_metadata = FlashAttentionMetadata(
299 + num_actual_tokens=total_num_scheduled_tokens,
300 max_query_len=max_num_scheduled_tokens,
301 query_start_loc=query_start_loc,
302 max_seq_len=max_seq_len,
@@ -287,7 +310,7 @@ class GPUModelRunner:
310 # token from the partial request.
311 # TODO: Support prompt logprobs.
312 logits_indices = query_start_loc[1:] - 1
290 - return input_ids, positions, attn_metadata, logits_indices
313 + return attn_metadata, logits_indices
314
315 def _prepare_sampling(
316 self,
@@ -310,16 +333,26 @@ class GPUModelRunner:
333 scheduler_output: "SchedulerOutput",
334 ) -> ModelRunnerOutput:
335 self._update_states(scheduler_output)
313 - inputs = self._prepare_inputs(scheduler_output)
314 - input_ids, positions, attn_metadata, logits_indices = inputs
336 + attn_metadata, logits_indices = self._prepare_inputs(scheduler_output)
337 + num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
338 + if (self.use_cuda_graph
339 + and num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]):
340 + # Use piecewise CUDA graphs.
341 + # Add padding to the batch size.
342 + num_input_tokens = self._get_padded_batch_size(
343 + num_scheduled_tokens)
344 + else:
345 + # Eager mode.
346 + num_input_tokens = num_scheduled_tokens
347
348 with set_forward_context(attn_metadata):
349 hidden_states = self.model(
318 - input_ids=input_ids,
319 - positions=positions,
350 + input_ids=self.input_ids[:num_input_tokens],
351 + positions=self.positions[:num_input_tokens],
352 kv_caches=self.kv_caches,
321 - attn_metadata=attn_metadata,
353 + attn_metadata=None,
354 )
355 + hidden_states = hidden_states[:num_scheduled_tokens]
356 hidden_states = hidden_states[logits_indices]
357 logits = self.model.compute_logits(hidden_states, None)
358
@@ -371,6 +404,18 @@ class GPUModelRunner:
404 return model_runner_output
405
406 def load_model(self) -> None:
407 + if self.use_cuda_graph:
408 + # FIXME(woosuk): Currently, the custom ops are not supported
409 + # in the piecewise compilation mode. We rely on TorchInductor
410 + # to optimize the model.
411 + os.environ["VLLM_CUSTOM_OPS"] = "none"
412 + set_compilation_config(
413 + CompilationConfig(
414 + use_cudagraph=True,
415 + non_cudagraph_ops=["vllm.unified_flash_attention"],
416 + use_inductor=True,
417 + ))
418 +
419 logger.info("Starting to load model %s...", self.model_config.model)
420 with DeviceMemoryProfiler() as m: # noqa: SIM117
421 with patch("vllm.model_executor.layers.sampler.Sampler", Sampler):
@@ -381,26 +426,61 @@ class GPUModelRunner:
426 self.model_memory_usage / float(2**30))
427
428 def _dummy_run(self, model: nn.Module, num_tokens: int) -> None:
384 - input_ids = torch.zeros(num_tokens,
385 - dtype=torch.int32,
386 - device=self.device)
387 - positions = torch.zeros(num_tokens,
388 - dtype=torch.long,
389 - device=self.device)
390 - kv_caches = [None for _ in range(self.num_attn_layers)]
391 - model(input_ids, positions, kv_caches, attn_metadata=None)
392 - return
429 + # use an empty tensor instead of `None`` to force Dynamo to pass
430 + # it by reference, rather by specializing on the value `None`.
431 + # the `dtype` argument does not matter, and we use `float32` as
432 + # a placeholder (it has wide hardware support).
433 + # it is important to create tensors inside the loop, rather than
434 + # multiplying the list, to avoid Dynamo from treating them as
435 + # tensor aliasing.
436 + dummy_kv_caches = [
437 + torch.tensor([], dtype=torch.float32, device=self.device)
438 + for _ in range(self.num_attn_layers)
439 + ]
440 + with set_forward_context(None): # noqa: SIM117
441 + with set_compile_context(self.cudagraph_batch_sizes):
442 + # Trigger compilation for general shape.
443 + model(self.input_ids,
444 + self.positions,
445 + dummy_kv_caches,
446 + attn_metadata=None)
447
448 @torch.inference_mode()
449 def profile_run(self) -> None:
450 self._dummy_run(self.model, self.max_num_tokens)
451 torch.cuda.synchronize()
398 - return
452
453 @torch.inference_mode()
454 def capture_model(self) -> None:
402 - # TODO: Implement CUDA graph support.
403 - return
455 + if not self.use_cuda_graph:
456 + logger.warning(
457 + "Skipping CUDA graph capture. Please set "
458 + "VLLM_TORCH_COMPILE_LEVEL=%d to use CUDA graphs.",
459 + CompilationLevel.PIECEWISE)
460 + return
461 +
462 + start_time = time.perf_counter()
463 + start_free_gpu_memory = torch.cuda.mem_get_info()[0]
464 +
465 + with set_forward_context(None):
466 + # Trigger CUDA graph capture for specific shapes.
467 + # Capture the large shapes first so that the smaller shapes
468 + # can reuse the memory pool allocated for the large shapes.
469 + for num_tokens in reversed(self.cudagraph_batch_sizes):
470 + self.model(
471 + self.input_ids[:num_tokens],
472 + self.positions[:num_tokens],
473 + kv_caches=self.kv_caches,
474 + attn_metadata=None,
475 + )
476 +
477 + end_time = time.perf_counter()
478 + end_free_gpu_memory = torch.cuda.mem_get_info()[0]
479 + elapsed_time = end_time - start_time
480 + cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
481 + # This usually takes 5~20 seconds.
482 + logger.info("Graph capturing finished in %.0f secs, took %.2f GiB",
483 + elapsed_time, cuda_graph_size / (1 << 30))
484
485 def initialize_kv_cache(self, num_blocks: int) -> None:
486 assert len(self.kv_caches) == 0
@@ -412,6 +492,13 @@ class GPUModelRunner:
492 dtype=self.kv_cache_dtype,
493 device=self.device))
494
495 + def _get_padded_batch_size(self, batch_size: int) -> Optional[int]:
496 + # TODO: Optimize this?
497 + for size in self.cudagraph_batch_sizes:
498 + if batch_size <= size:
499 + return size
500 + return None
501 +
502
503 @dataclass
504 class CachedRequestState: