A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] Integrate Piecewise CUDA graphs (#10058)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 5, 2024 at 22:16 UTC
40899855520eb9497606bdb2b1b4e619233e598a
3 files changed
+133
-36
vllm/compilation/backends.py
+5
-2
@@ -496,8 +496,11 @@ class PiecewiseBackend:
496
return entry.runnable(*args)
497
498
if self.is_first_graph:
499
- logger.info("Capturing a cudagraph for shape %s",
500
- runtime_shape)
499
+ # Since we capture cudagraph for many different shapes and
500
+ # capturing is fast, we don't need to log it for every shape.
501
+ # We only log it in the debug mode.
502
+ logger.debug("Capturing a cudagraph for shape %s",
503
+ runtime_shape)
504
505
input_addresses = [
506
x.data_ptr() for x in args if isinstance(x, torch.Tensor)
vllm/v1/attention/backends/flash_attn.py
+21
-14
@@ -51,6 +51,7 @@ class FlashAttentionMetadata:
51
# |-------------------- seq_len ---------------------|
52
# |-- query_len ---|
53
54
+ num_actual_tokens: int # Number of tokens excluding padding.
55
max_query_len: int
56
query_start_loc: torch.Tensor
57
max_seq_len: int
@@ -134,7 +135,9 @@ class FlashAttentionImpl(AttentionImpl):
135
assert k_scale == 1.0 and v_scale == 1.0, (
136
"key/v_scale is not supported in FlashAttention.")
137
137
- output = torch.ops.vllm.unified_flash_attention(
138
+ output = torch.empty_like(query)
139
+ torch.ops.vllm.unified_flash_attention(
140
+ output,
141
query,
142
key,
143
value,
@@ -154,6 +157,7 @@ class FlashAttentionImpl(AttentionImpl):
157
158
159
def unified_flash_attention(
160
+ output: torch.Tensor,
161
query: torch.Tensor,
162
key: torch.Tensor,
163
value: torch.Tensor,
@@ -168,17 +172,17 @@ def unified_flash_attention(
172
window_size: Optional[List[int]] = None,
173
alibi_slopes: Optional[torch.Tensor] = None,
174
logits_soft_cap: Optional[float] = None,
171
-) -> torch.Tensor:
175
+) -> None:
176
current_metadata = get_forward_context()
177
if current_metadata is None:
178
# Profiling run.
175
- return torch.empty_like(query)
179
+ return
180
181
assert current_metadata is not None
182
assert isinstance(current_metadata, FlashAttentionMetadata)
183
attn_metadata: FlashAttentionMetadata = current_metadata
184
+ num_actual_tokens = attn_metadata.num_actual_tokens
185
181
- num_tokens, hidden_size = query.shape
186
# Reshape the query, key, and value tensors.
187
query = query.view(-1, num_heads, head_size)
188
key = key.view(-1, num_kv_heads, head_size)
@@ -188,18 +192,18 @@ def unified_flash_attention(
192
key_cache = kv_cache[0]
193
value_cache = kv_cache[1]
194
torch.ops._C_cache_ops.reshape_and_cache_flash(
191
- key,
192
- value,
193
- kv_cache[0],
194
- kv_cache[1],
195
+ key[:num_actual_tokens],
196
+ value[:num_actual_tokens],
197
+ key_cache,
198
+ value_cache,
199
attn_metadata.slot_mapping,
200
kv_cache_dtype,
201
k_scale,
202
v_scale,
203
)
204
201
- output = flash_attn_varlen_func(
202
- q=query,
205
+ attn_output = flash_attn_varlen_func(
206
+ q=query[:num_actual_tokens],
207
k=key_cache,
208
v=value_cache,
209
cu_seqlens_q=attn_metadata.query_start_loc,
@@ -213,10 +217,13 @@ def unified_flash_attention(
217
block_table=attn_metadata.block_table,
218
softcap=logits_soft_cap,
219
)
216
- return output.view(num_tokens, hidden_size)
220
+ attn_output = attn_output.view(num_actual_tokens, -1)
221
+ # TODO(woosuk): Optimize this.
222
+ output[:num_actual_tokens].copy_(attn_output)
223
224
225
def unified_flash_attention_fake(
226
+ output: torch.Tensor,
227
query: torch.Tensor,
228
key: torch.Tensor,
229
value: torch.Tensor,
@@ -231,13 +238,13 @@ def unified_flash_attention_fake(
238
window_size: Optional[List[int]] = None,
239
alibi_slopes: Optional[torch.Tensor] = None,
240
logits_soft_cap: Optional[float] = None,
234
-) -> torch.Tensor:
235
- return torch.empty_like(query)
241
+) -> None:
242
+ return
243
244
245
direct_register_custom_op(
246
op_name="unified_flash_attention",
247
op_func=unified_flash_attention,
241
- mutates_args=["kv_cache"],
248
+ mutates_args=["kv_cache", "output"],
249
fake_impl=unified_flash_attention_fake,
250
)
vllm/v1/worker/gpu_model_runner.py
+107
-20
@@ -1,3 +1,5 @@
1
+import os
2
+import time
3
from dataclasses import dataclass
4
from typing import TYPE_CHECKING, Dict, List, Optional, Set
5
from unittest.mock import patch
@@ -7,11 +9,16 @@ import torch
9
import torch.distributed
10
import torch.nn as nn
11
12
+from vllm import envs
13
+from vllm.compilation.compile_context import set_compile_context
14
+from vllm.compilation.config import CompilationConfig
15
+from vllm.compilation.levels import CompilationLevel
16
from vllm.config import VllmConfig
17
from vllm.forward_context import set_forward_context
18
from vllm.logger import init_logger
19
from vllm.model_executor.model_loader import get_model
20
from vllm.multimodal import MultiModalDataDict
21
+from vllm.plugins import set_compilation_config
22
from vllm.sampling_params import SamplingParams, SamplingType
23
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
24
is_pin_memory_available)
@@ -86,6 +93,18 @@ class GPUModelRunner:
93
pin_memory=self.pin_memory,
94
)
95
96
+ self.use_cuda_graph = (envs.VLLM_TORCH_COMPILE_LEVEL
97
+ == CompilationLevel.PIECEWISE
98
+ and not self.model_config.enforce_eager)
99
+ # TODO(woosuk): Provide an option to tune the max cudagraph batch size.
100
+ self.cudagraph_batch_sizes = [1, 2, 4] + [i for i in range(8, 513, 8)]
101
+ self.input_ids = torch.zeros(self.max_num_tokens,
102
+ dtype=torch.int32,
103
+ device=self.device)
104
+ self.positions = torch.zeros(self.max_num_tokens,
105
+ dtype=torch.int64,
106
+ device=self.device)
107
+
108
def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
109
# Remove stopped requests from the cached states.
110
# Keep the states of the pre-empted requests.
@@ -268,12 +287,16 @@ class GPUModelRunner:
287
seq_start_loc_np[0] = 0
288
np.cumsum(seq_lens, out=seq_start_loc_np[1:])
289
271
- input_ids = input_ids.to(self.device, non_blocking=True)
272
- positions = positions.to(self.device, non_blocking=True).long()
290
+ self.input_ids[:total_num_scheduled_tokens].copy_(input_ids,
291
+ non_blocking=True)
292
+ self.positions[:total_num_scheduled_tokens].copy_(positions,
293
+ non_blocking=True)
294
+
295
query_start_loc = query_start_loc.to(self.device, non_blocking=True)
296
seq_start_loc = seq_start_loc.to(self.device, non_blocking=True)
297
slot_mapping = slot_mapping.to(self.device, non_blocking=True).long()
298
attn_metadata = FlashAttentionMetadata(
299
+ num_actual_tokens=total_num_scheduled_tokens,
300
max_query_len=max_num_scheduled_tokens,
301
query_start_loc=query_start_loc,
302
max_seq_len=max_seq_len,
@@ -287,7 +310,7 @@ class GPUModelRunner:
310
# token from the partial request.
311
# TODO: Support prompt logprobs.
312
logits_indices = query_start_loc[1:] - 1
290
- return input_ids, positions, attn_metadata, logits_indices
313
+ return attn_metadata, logits_indices
314
315
def _prepare_sampling(
316
self,
@@ -310,16 +333,26 @@ class GPUModelRunner:
333
scheduler_output: "SchedulerOutput",
334
) -> ModelRunnerOutput:
335
self._update_states(scheduler_output)
313
- inputs = self._prepare_inputs(scheduler_output)
314
- input_ids, positions, attn_metadata, logits_indices = inputs
336
+ attn_metadata, logits_indices = self._prepare_inputs(scheduler_output)
337
+ num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
338
+ if (self.use_cuda_graph
339
+ and num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]):
340
+ # Use piecewise CUDA graphs.
341
+ # Add padding to the batch size.
342
+ num_input_tokens = self._get_padded_batch_size(
343
+ num_scheduled_tokens)
344
+ else:
345
+ # Eager mode.
346
+ num_input_tokens = num_scheduled_tokens
347
348
with set_forward_context(attn_metadata):
349
hidden_states = self.model(
318
- input_ids=input_ids,
319
- positions=positions,
350
+ input_ids=self.input_ids[:num_input_tokens],
351
+ positions=self.positions[:num_input_tokens],
352
kv_caches=self.kv_caches,
321
- attn_metadata=attn_metadata,
353
+ attn_metadata=None,
354
)
355
+ hidden_states = hidden_states[:num_scheduled_tokens]
356
hidden_states = hidden_states[logits_indices]
357
logits = self.model.compute_logits(hidden_states, None)
358
@@ -371,6 +404,18 @@ class GPUModelRunner:
404
return model_runner_output
405
406
def load_model(self) -> None:
407
+ if self.use_cuda_graph:
408
+ # FIXME(woosuk): Currently, the custom ops are not supported
409
+ # in the piecewise compilation mode. We rely on TorchInductor
410
+ # to optimize the model.
411
+ os.environ["VLLM_CUSTOM_OPS"] = "none"
412
+ set_compilation_config(
413
+ CompilationConfig(
414
+ use_cudagraph=True,
415
+ non_cudagraph_ops=["vllm.unified_flash_attention"],
416
+ use_inductor=True,
417
+ ))
418
+
419
logger.info("Starting to load model %s...", self.model_config.model)
420
with DeviceMemoryProfiler() as m: # noqa: SIM117
421
with patch("vllm.model_executor.layers.sampler.Sampler", Sampler):
@@ -381,26 +426,61 @@ class GPUModelRunner:
426
self.model_memory_usage / float(2**30))
427
428
def _dummy_run(self, model: nn.Module, num_tokens: int) -> None:
384
- input_ids = torch.zeros(num_tokens,
385
- dtype=torch.int32,
386
- device=self.device)
387
- positions = torch.zeros(num_tokens,
388
- dtype=torch.long,
389
- device=self.device)
390
- kv_caches = [None for _ in range(self.num_attn_layers)]
391
- model(input_ids, positions, kv_caches, attn_metadata=None)
392
- return
429
+ # use an empty tensor instead of `None`` to force Dynamo to pass
430
+ # it by reference, rather by specializing on the value `None`.
431
+ # the `dtype` argument does not matter, and we use `float32` as
432
+ # a placeholder (it has wide hardware support).
433
+ # it is important to create tensors inside the loop, rather than
434
+ # multiplying the list, to avoid Dynamo from treating them as
435
+ # tensor aliasing.
436
+ dummy_kv_caches = [
437
+ torch.tensor([], dtype=torch.float32, device=self.device)
438
+ for _ in range(self.num_attn_layers)
439
+ ]
440
+ with set_forward_context(None): # noqa: SIM117
441
+ with set_compile_context(self.cudagraph_batch_sizes):
442
+ # Trigger compilation for general shape.
443
+ model(self.input_ids,
444
+ self.positions,
445
+ dummy_kv_caches,
446
+ attn_metadata=None)
447
448
@torch.inference_mode()
449
def profile_run(self) -> None:
450
self._dummy_run(self.model, self.max_num_tokens)
451
torch.cuda.synchronize()
398
- return
452
453
@torch.inference_mode()
454
def capture_model(self) -> None:
402
- # TODO: Implement CUDA graph support.
403
- return
455
+ if not self.use_cuda_graph:
456
+ logger.warning(
457
+ "Skipping CUDA graph capture. Please set "
458
+ "VLLM_TORCH_COMPILE_LEVEL=%d to use CUDA graphs.",
459
+ CompilationLevel.PIECEWISE)
460
+ return
461
+
462
+ start_time = time.perf_counter()
463
+ start_free_gpu_memory = torch.cuda.mem_get_info()[0]
464
+
465
+ with set_forward_context(None):
466
+ # Trigger CUDA graph capture for specific shapes.
467
+ # Capture the large shapes first so that the smaller shapes
468
+ # can reuse the memory pool allocated for the large shapes.
469
+ for num_tokens in reversed(self.cudagraph_batch_sizes):
470
+ self.model(
471
+ self.input_ids[:num_tokens],
472
+ self.positions[:num_tokens],
473
+ kv_caches=self.kv_caches,
474
+ attn_metadata=None,
475
+ )
476
+
477
+ end_time = time.perf_counter()
478
+ end_free_gpu_memory = torch.cuda.mem_get_info()[0]
479
+ elapsed_time = end_time - start_time
480
+ cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
481
+ # This usually takes 5~20 seconds.
482
+ logger.info("Graph capturing finished in %.0f secs, took %.2f GiB",
483
+ elapsed_time, cuda_graph_size / (1 << 30))
484
485
def initialize_kv_cache(self, num_blocks: int) -> None:
486
assert len(self.kv_caches) == 0
@@ -412,6 +492,13 @@ class GPUModelRunner:
492
dtype=self.kv_cache_dtype,
493
device=self.device))
494
495
+ def _get_padded_batch_size(self, batch_size: int) -> Optional[int]:
496
+ # TODO: Optimize this?
497
+ for size in self.cudagraph_batch_sizes:
498
+ if batch_size <= size:
499
+ return size
500
+ return None
501
+
502
503
@dataclass
504
class CachedRequestState: