@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8549c82660
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[core] cudagraph output with tensor weak reference (#9724)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Oct 27, 2024 at 00:19 UTC 8549c82660cfa59a13cccd622f8afcc29cbd4281
4 files changed +50 -28
csrc/ops.h
+24
@@ -5,6 +5,30 @@
5
6 #include "core/scalar_type.hpp"
7
8 +#include <vector>
9 +
10 +torch::Tensor weak_ref_tensor(torch::Tensor& tensor) {
11 + // Ensure tensor is on CUDA
12 + if (!tensor.is_cuda()) {
13 + throw std::runtime_error("Tensor must be on CUDA device");
14 + }
15 +
16 + // Get the raw data pointer
17 + void* data_ptr = tensor.data_ptr();
18 +
19 + // Get tensor sizes and strides
20 + std::vector<int64_t> sizes = tensor.sizes().vec();
21 + std::vector<int64_t> strides = tensor.strides().vec();
22 +
23 + // Get tensor options (dtype, device)
24 + auto options = tensor.options();
25 +
26 + // Create a new tensor from the raw data pointer
27 + auto new_tensor = torch::from_blob(data_ptr, sizes, strides, options);
28 +
29 + return new_tensor;
30 +}
31 +
32 void paged_attention_v1(
33 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
34 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
csrc/torch_bindings.cpp
+3
@@ -18,6 +18,9 @@
18 TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
19 // vLLM custom ops
20
21 + ops.def("weak_ref_tensor(Tensor input) -> Tensor");
22 + ops.impl("weak_ref_tensor", torch::kCUDA, &weak_ref_tensor);
23 +
24 // Attention ops
25 // Compute the attention between an input query and the cached
26 // keys/values using PagedAttention.
vllm/utils.py
+9
@@ -1479,3 +1479,12 @@ class LazyDict(Mapping, Generic[T]):
1479
1480 def __len__(self):
1481 return len(self._factory)
1482 +
1483 +
1484 +def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:
1485 + """
1486 + Create a weak reference to a tensor.
1487 + The new tensor will share the same data as the original tensor,
1488 + but will not keep the original tensor alive.
1489 + """
1490 + return torch.ops._C.weak_ref_tensor(tensor)
vllm/worker/model_runner.py
+14 -28
@@ -50,7 +50,7 @@ from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
50 from vllm.transformers_utils.config import uses_mrope
51 from vllm.utils import (DeviceMemoryProfiler, PyObjectCache, async_tensor_h2d,
52 flatten_2d_lists, is_hip, is_pin_memory_available,
53 - supports_dynamo)
53 + supports_dynamo, weak_ref_tensor)
54 from vllm.worker.model_runner_base import (
55 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
56 _add_attn_metadata_broadcastable_dict,
@@ -1426,12 +1426,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1426 dtype=self.model_config.dtype,
1427 device=self.device)
1428
1429 - # Prepare buffer for outputs. These will be reused for all batch sizes.
1430 - # It will be filled after the first graph capture.
1431 - hidden_or_intermediate_states: List[Optional[torch.Tensor]] = [
1432 - None
1433 - ] * self.parallel_config.pipeline_parallel_size
1434 -
1429 graph_batch_size = self.max_batchsize_to_capture
1430 batch_size_capture_list = [
1431 bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
@@ -1474,12 +1468,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1468 input_tokens[:batch_size],
1469 "positions":
1470 input_positions[..., :batch_size],
1477 - "hidden_or_intermediate_states":
1478 - hidden_or_intermediate_states[
1479 - virtual_engine] # type: ignore
1480 - [:batch_size]
1481 - if hidden_or_intermediate_states[virtual_engine]
1482 - is not None else None,
1471 "intermediate_inputs":
1472 intermediate_inputs[:batch_size]
1473 if intermediate_inputs is not None else None,
@@ -1762,15 +1750,13 @@ class CUDAGraphRunner(nn.Module):
1750 self,
1751 input_ids: torch.Tensor,
1752 positions: torch.Tensor,
1765 - hidden_or_intermediate_states: Optional[Union[IntermediateTensors,
1766 - torch.Tensor]],
1753 intermediate_inputs: Optional[IntermediateTensors],
1754 kv_caches: List[torch.Tensor],
1755 attn_metadata: AttentionMetadata,
1756 memory_pool: Optional[Tuple[int, int]],
1757 stream: torch.cuda.Stream,
1758 **kwargs,
1773 - ) -> Union[torch.Tensor, IntermediateTensors]:
1759 + ):
1760 assert self._graph is None
1761 # Run the model a few times without capturing the graph.
1762 # This is to make sure that the captured graph does not include the
@@ -1799,20 +1785,21 @@ class CUDAGraphRunner(nn.Module):
1785 intermediate_tensors=intermediate_inputs,
1786 **kwargs,
1787 )
1802 - if hidden_or_intermediate_states is not None:
1803 - if get_pp_group().is_last_rank:
1804 - hidden_or_intermediate_states.copy_(
1805 - output_hidden_or_intermediate_states)
1806 - else:
1807 - for key in hidden_or_intermediate_states.tensors:
1808 - hidden_or_intermediate_states[key].copy_(
1809 - output_hidden_or_intermediate_states[key])
1810 - else:
1811 - hidden_or_intermediate_states = (
1788 +
1789 + if isinstance(output_hidden_or_intermediate_states, torch.Tensor):
1790 + hidden_or_intermediate_states = weak_ref_tensor(
1791 output_hidden_or_intermediate_states)
1792 + elif isinstance(output_hidden_or_intermediate_states,
1793 + IntermediateTensors):
1794 + hidden_or_intermediate_states = IntermediateTensors(
1795 + tensors={
1796 + key: weak_ref_tensor(value)
1797 + for key, value in
1798 + output_hidden_or_intermediate_states.tensors.items()
1799 + })
1800
1801 del output_hidden_or_intermediate_states
1815 - # make sure `output_hidden_states` is deleted
1802 + # make sure `output_hidden_or_intermediate_states` is deleted
1803 # in the graph's memory pool
1804 gc.collect()
1805 torch.cuda.synchronize()
@@ -1837,7 +1824,6 @@ class CUDAGraphRunner(nn.Module):
1824 }
1825 else:
1826 self.output_buffers = hidden_or_intermediate_states
1840 - return hidden_or_intermediate_states
1827
1828 def forward(
1829 self,