A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[core] cudagraph output with tensor weak reference (#9724)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Oct 27, 2024 at 00:19 UTC
8549c82660cfa59a13cccd622f8afcc29cbd4281
4 files changed
+50
-28
csrc/ops.h
+24
@@ -5,6 +5,30 @@
5
6
#include "core/scalar_type.hpp"
7
8
+#include <vector>
9
+
10
+torch::Tensor weak_ref_tensor(torch::Tensor& tensor) {
11
+ // Ensure tensor is on CUDA
12
+ if (!tensor.is_cuda()) {
13
+ throw std::runtime_error("Tensor must be on CUDA device");
14
+ }
15
+
16
+ // Get the raw data pointer
17
+ void* data_ptr = tensor.data_ptr();
18
+
19
+ // Get tensor sizes and strides
20
+ std::vector<int64_t> sizes = tensor.sizes().vec();
21
+ std::vector<int64_t> strides = tensor.strides().vec();
22
+
23
+ // Get tensor options (dtype, device)
24
+ auto options = tensor.options();
25
+
26
+ // Create a new tensor from the raw data pointer
27
+ auto new_tensor = torch::from_blob(data_ptr, sizes, strides, options);
28
+
29
+ return new_tensor;
30
+}
31
+
32
void paged_attention_v1(
33
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
34
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
csrc/torch_bindings.cpp
+3
@@ -18,6 +18,9 @@
18
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
19
// vLLM custom ops
20
21
+ ops.def("weak_ref_tensor(Tensor input) -> Tensor");
22
+ ops.impl("weak_ref_tensor", torch::kCUDA, &weak_ref_tensor);
23
+
24
// Attention ops
25
// Compute the attention between an input query and the cached
26
// keys/values using PagedAttention.
vllm/utils.py
+9
@@ -1479,3 +1479,12 @@ class LazyDict(Mapping, Generic[T]):
1479
1480
def __len__(self):
1481
return len(self._factory)
1482
+
1483
+
1484
+def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:
1485
+ """
1486
+ Create a weak reference to a tensor.
1487
+ The new tensor will share the same data as the original tensor,
1488
+ but will not keep the original tensor alive.
1489
+ """
1490
+ return torch.ops._C.weak_ref_tensor(tensor)
vllm/worker/model_runner.py
+14
-28
@@ -50,7 +50,7 @@ from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
50
from vllm.transformers_utils.config import uses_mrope
51
from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d,
52
flatten_2d_lists, is_hip, is_pin_memory_available,
53
- supports_dynamo)
53
+ supports_dynamo, weak_ref_tensor)
54
from vllm.worker.model_runner_base import (
55
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
56
_add_attn_metadata_broadcastable_dict,
@@ -1426,12 +1426,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1426
dtype=self.model_config.dtype,
1427
device=self.device)
1428
1429
- # Prepare buffer for outputs. These will be reused for all batch sizes.
1430
- # It will be filled after the first graph capture.
1431
- hidden_or_intermediate_states: List[Optional[torch.Tensor]] = [
1432
- None
1433
- ] * self.parallel_config.pipeline_parallel_size
1434
-
1429
graph_batch_size = self.max_batchsize_to_capture
1430
batch_size_capture_list = [
1431
bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
@@ -1474,12 +1468,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1468
input_tokens[:batch_size],
1469
"positions":
1470
input_positions[..., :batch_size],
1477
- "hidden_or_intermediate_states":
1478
- hidden_or_intermediate_states[
1479
- virtual_engine] # type: ignore
1480
- [:batch_size]
1481
- if hidden_or_intermediate_states[virtual_engine]
1482
- is not None else None,
1471
"intermediate_inputs":
1472
intermediate_inputs[:batch_size]
1473
if intermediate_inputs is not None else None,
@@ -1762,15 +1750,13 @@ class CUDAGraphRunner(nn.Module):
1750
self,
1751
input_ids: torch.Tensor,
1752
positions: torch.Tensor,
1765
- hidden_or_intermediate_states: Optional[Union[IntermediateTensors,
1766
- torch.Tensor]],
1753
intermediate_inputs: Optional[IntermediateTensors],
1754
kv_caches: List[torch.Tensor],
1755
attn_metadata: AttentionMetadata,
1756
memory_pool: Optional[Tuple[int, int]],
1757
stream: torch.cuda.Stream,
1758
**kwargs,
1773
- ) -> Union[torch.Tensor, IntermediateTensors]:
1759
+ ):
1760
assert self._graph is None
1761
# Run the model a few times without capturing the graph.
1762
# This is to make sure that the captured graph does not include the
@@ -1799,20 +1785,21 @@ class CUDAGraphRunner(nn.Module):
1785
intermediate_tensors=intermediate_inputs,
1786
**kwargs,
1787
)
1802
- if hidden_or_intermediate_states is not None:
1803
- if get_pp_group().is_last_rank:
1804
- hidden_or_intermediate_states.copy_(
1805
- output_hidden_or_intermediate_states)
1806
- else:
1807
- for key in hidden_or_intermediate_states.tensors:
1808
- hidden_or_intermediate_states[key].copy_(
1809
- output_hidden_or_intermediate_states[key])
1810
- else:
1811
- hidden_or_intermediate_states = (
1788
+
1789
+ if isinstance(output_hidden_or_intermediate_states, torch.Tensor):
1790
+ hidden_or_intermediate_states = weak_ref_tensor(
1791
output_hidden_or_intermediate_states)
1792
+ elif isinstance(output_hidden_or_intermediate_states,
1793
+ IntermediateTensors):
1794
+ hidden_or_intermediate_states = IntermediateTensors(
1795
+ tensors={
1796
+ key: weak_ref_tensor(value)
1797
+ for key, value in
1798
+ output_hidden_or_intermediate_states.tensors.items()
1799
+ })
1800
1801
del output_hidden_or_intermediate_states
1815
- # make sure `output_hidden_states` is deleted
1802
+ # make sure `output_hidden_or_intermediate_states` is deleted
1803
# in the graph's memory pool
1804
gc.collect()
1805
torch.cuda.synchronize()
@@ -1837,7 +1824,6 @@ class CUDAGraphRunner(nn.Module):
1824
}
1825
else:
1826
self.output_buffers = hidden_or_intermediate_states
1840
- return hidden_or_intermediate_states
1827
1828
def forward(
1829
self,