@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 63575bc2e1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Optimization] change python dict to pytorch tensor (#4607)

youkaichao committed May 6, 2024 at 21:30 UTC 63575bc2e197b85ce1c911421ff30c5459e35e9c
19 files changed +77 -81
csrc/cache.h
+1 -1
@@ -13,7 +13,7 @@ void swap_blocks(
13 void copy_blocks(
14 std::vector<torch::Tensor>& key_caches,
15 std::vector<torch::Tensor>& value_caches,
16 - const std::map<int64_t, std::vector<int64_t>>& block_mapping);
16 + torch::Tensor& block_mapping);
17
18 void reshape_and_cache(
19 torch::Tensor& key,
csrc/cache_kernels.cu
+5 -15
@@ -97,7 +97,7 @@ __global__ void copy_blocks_kernel(
97 void copy_blocks(
98 std::vector<torch::Tensor>& key_caches,
99 std::vector<torch::Tensor>& value_caches,
100 - const std::map<int64_t, std::vector<int64_t>>& block_mapping) {
100 + torch::Tensor& block_mapping) {
101 int num_layers = key_caches.size();
102 TORCH_CHECK(num_layers == value_caches.size());
103 if (num_layers == 0) {
@@ -114,17 +114,9 @@ void copy_blocks(
114 key_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(key_caches[layer_idx].data_ptr());
115 value_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(value_caches[layer_idx].data_ptr());
116 }
117 - // Create block mapping array.
118 - std::vector<int64_t> block_mapping_vec;
119 - for (const auto& pair : block_mapping) {
120 - int64_t src_block_number = pair.first;
121 - for (int64_t dst_block_number : pair.second) {
122 - block_mapping_vec.push_back(src_block_number);
123 - block_mapping_vec.push_back(dst_block_number);
124 - }
125 - }
126 - int64_t* block_mapping_array = block_mapping_vec.data();
127 - int num_pairs = block_mapping_vec.size() / 2;
117 +
118 + // block_mapping is a 2D tensor with shape (num_pairs, 2).
119 + int num_pairs = block_mapping.size(0);
120
121 // Move the data structures to the GPU.
122 // NOTE: This synchronizes the CPU and GPU.
@@ -132,8 +124,6 @@ void copy_blocks(
124 key_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
125 torch::Tensor value_cache_ptrs_tensor = torch::from_blob(
126 value_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
135 - torch::Tensor block_mapping_tensor = torch::from_blob(
136 - block_mapping_array, {2 * num_pairs}, torch::kInt64).to(cache_device);
127
128 // Launch the kernel.
129 const int numel_per_block = key_caches[0][0].numel();
@@ -146,7 +136,7 @@ void copy_blocks(
136 vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
137 key_cache_ptrs_tensor.data_ptr<int64_t>(),
138 value_cache_ptrs_tensor.data_ptr<int64_t>(),
149 - block_mapping_tensor.data_ptr<int64_t>(),
139 + block_mapping.data_ptr<int64_t>(),
140 numel_per_block);
141 }));
142 }
csrc/cpu/cache.cpp
+6 -14
@@ -8,16 +8,16 @@ template <typename scalar_t>
8 void copy_blocks_cpu_impl(
9 std::vector<torch::Tensor> &key_caches,
10 std::vector<torch::Tensor> &value_caches,
11 - const std::vector<std::pair<int64_t, int64_t>> mapping_pairs,
11 + const torch::Tensor& mapping_pairs,
12 const int element_num_per_block, const int layer_num) {
13 - const size_t pair_num = mapping_pairs.size();
13 + const size_t pair_num = mapping_pairs.size(0);
14 const size_t block_bytes = sizeof(scalar_t) * element_num_per_block;
15 #pragma omp parallel for collapse(2)
16 for (int layer = 0; layer < layer_num; ++layer) {
17 for (size_t pair = 0; pair < pair_num; ++pair) {
18 - int64_t source_offset = element_num_per_block * mapping_pairs[pair].first;
18 + int64_t source_offset = element_num_per_block * mapping_pairs[pair][0].item<int64_t>();
19 int64_t target_offset =
20 - element_num_per_block * mapping_pairs[pair].second;
20 + element_num_per_block * mapping_pairs[pair][1].item<int64_t>();
21 scalar_t *key_cache_ptr = key_caches[layer].data_ptr<scalar_t>();
22 scalar_t *source_ptr = key_cache_ptr + source_offset;
23 scalar_t *target_ptr = key_cache_ptr + target_offset;
@@ -83,26 +83,18 @@ void reshape_and_cache_cpu_impl(
83
84 void copy_blocks(std::vector<torch::Tensor> &key_caches,
85 std::vector<torch::Tensor> &value_caches,
86 - const std::map<int64_t, std::vector<int64_t>> &block_mapping) {
86 + torch::Tensor& block_mapping) {
87 int num_layers = key_caches.size();
88 TORCH_CHECK(num_layers == value_caches.size());
89 if (num_layers == 0) {
90 return;
91 }
92
93 - std::vector<std::pair<int64_t, int64_t>> mapping_pairs;
94 - mapping_pairs.reserve(block_mapping.size());
95 - for (const auto &pair : block_mapping) {
96 - for (const auto &dst : pair.second) {
97 - mapping_pairs.emplace_back(pair.first, dst);
98 - }
99 - }
100 -
93 const int element_num_per_block = key_caches[0][0].numel();
94 VLLM_DISPATCH_FLOATING_TYPES(
95 key_caches[0].scalar_type(), "copy_blocks_cpu_impl", [&] {
96 CPU_KERNEL_GUARD_IN(copy_blocks_cpu_impl)
105 - copy_blocks_cpu_impl<scalar_t>(key_caches, value_caches, mapping_pairs,
97 + copy_blocks_cpu_impl<scalar_t>(key_caches, value_caches, block_mapping,
98 element_num_per_block, num_layers);
99 CPU_KERNEL_GUARD_OUT(copy_blocks_cpu_impl)
100 });
tests/core/test_scheduler.py
+4 -4
@@ -568,7 +568,7 @@ def test_decode_schedule_preempted():
568 # Both should be preempted, not swapped.
569 assert output.blocks_to_swap_out == {}
570 # Nothing is copied.
571 - assert output.blocks_to_copy == {}
571 + assert output.blocks_to_copy == []
572
573
574 def test_decode_swap_beam_search():
@@ -618,7 +618,7 @@ def test_decode_swap_beam_search():
618 # Both should be preempted, not swapped.
619 assert output.blocks_to_swap_out == expected_swap_mapping
620 # Nothing is copied.
621 - assert output.blocks_to_copy == {}
621 + assert output.blocks_to_copy == []
622
623
624 def test_schedule_decode_blocks_to_copy_update():
@@ -650,7 +650,7 @@ def test_schedule_decode_blocks_to_copy_update():
650 assert output.blocks_to_swap_out == {}
651 # Since append_slot returns the source -> dist mapping, it should
652 # applied.
653 - assert output.blocks_to_copy == {2: [3]}
653 + assert output.blocks_to_copy == [(2, 3)]
654
655
656 def test_schedule_swapped_simple():
@@ -853,7 +853,7 @@ def test_schedule_swapped_blocks_to_copy():
853 assert len(remaining_swapped) == 0
854 assert len(output.decode_seq_groups) == 1
855 assert len(output.prefill_seq_groups) == 0
856 - assert output.blocks_to_copy == {2: [3]}
856 + assert output.blocks_to_copy == [(2, 3)]
857
858
859 def test_scheduling_budget():
tests/kernels/test_cache.py
+12 -9
@@ -63,12 +63,13 @@ def test_copy_blocks(
63 src_blocks = random.sample(range(num_blocks), num_mappings)
64 remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))
65 dst_blocks = random.sample(remainig_blocks, 2 * num_mappings)
66 - block_mapping = {}
66 + block_mapping = []
67 for i in range(num_mappings):
68 src = src_blocks[i]
69 dst1 = dst_blocks[2 * i]
70 dst2 = dst_blocks[2 * i + 1]
71 - block_mapping[src] = [dst1, dst2]
71 + block_mapping.append((src, dst1))
72 + block_mapping.append((src, dst2))
73
74 # Create the KV caches.
75 key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
@@ -81,15 +82,17 @@ def test_copy_blocks(
82 cloned_value_caches = [value_cache.clone() for value_cache in value_caches]
83
84 # Call the copy blocks kernel.
84 - ops.copy_blocks(key_caches, value_caches, block_mapping)
85 + block_mapping_tensor = torch.tensor(block_mapping,
86 + dtype=torch.int64,
87 + device=device).view(-1, 2)
88 + ops.copy_blocks(key_caches, value_caches, block_mapping_tensor)
89
90 # Run the reference implementation.
87 - for src, dsts in block_mapping.items():
88 - for dst in dsts:
89 - for cloned_key_cache in cloned_key_caches:
90 - cloned_key_cache[dst].copy_(cloned_key_cache[src])
91 - for cloned_value_cache in cloned_value_caches:
92 - cloned_value_cache[dst].copy_(cloned_value_cache[src])
91 + for src, dst in block_mapping:
92 + for cloned_key_cache in cloned_key_caches:
93 + cloned_key_cache[dst].copy_(cloned_key_cache[src])
94 + for cloned_value_cache in cloned_value_caches:
95 + cloned_value_cache[dst].copy_(cloned_value_cache[src])
96
97 # Compare the results.
98 for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
tests/worker/test_swap.py
+1 -1
@@ -59,7 +59,7 @@ def test_swap() -> None:
59 seq_group_metadata_list=[],
60 blocks_to_swap_in={},
61 blocks_to_swap_out=blocks_to_swap_out,
62 - blocks_to_copy={},
62 + blocks_to_copy=[],
63 )
64 worker.execute_model(execute_model_req=execute_model_req)
65
vllm/attention/backends/abstract.py
+1 -1
@@ -42,7 +42,7 @@ class AttentionBackend(ABC):
42 @abstractmethod
43 def copy_blocks(
44 kv_caches: List[torch.Tensor],
45 - src_to_dists: Dict[int, List[int]],
45 + src_to_dists: torch.Tensor,
46 ) -> None:
47 raise NotImplementedError
48
vllm/attention/backends/flash_attn.py
+1 -1
@@ -48,7 +48,7 @@ class FlashAttentionBackend(AttentionBackend):
48 @staticmethod
49 def copy_blocks(
50 kv_caches: List[torch.Tensor],
51 - src_to_dists: Dict[int, List[int]],
51 + src_to_dists: torch.Tensor,
52 ) -> None:
53 PagedAttention.copy_blocks(kv_caches, src_to_dists)
54
vllm/attention/backends/flashinfer.py
+1 -1
@@ -48,7 +48,7 @@ class FlashInferBackend(AttentionBackend):
48 @staticmethod
49 def copy_blocks(
50 kv_caches: List[torch.Tensor],
51 - src_to_dists: Dict[int, List[int]],
51 + src_to_dists: torch.Tensor,
52 ) -> None:
53 raise NotImplementedError
54
vllm/attention/backends/rocm_flash_attn.py
+1 -1
@@ -46,7 +46,7 @@ class ROCmFlashAttentionBackend(AttentionBackend):
46 @staticmethod
47 def copy_blocks(
48 kv_caches: List[torch.Tensor],
49 - src_to_dists: Dict[int, List[int]],
49 + src_to_dists: torch.Tensor,
50 ) -> None:
51 PagedAttention.copy_blocks(kv_caches, src_to_dists)
52
vllm/attention/backends/torch_sdpa.py
+1 -1
@@ -44,7 +44,7 @@ class TorchSDPABackend(AttentionBackend):
44 @staticmethod
45 def copy_blocks(
46 kv_caches: List[torch.Tensor],
47 - src_to_dists: Dict[int, List[int]],
47 + src_to_dists: torch.Tensor,
48 ) -> None:
49 PagedAttention.copy_blocks(kv_caches, src_to_dists)
50
vllm/attention/backends/xformers.py
+1 -1
@@ -49,7 +49,7 @@ class XFormersBackend(AttentionBackend):
49 @staticmethod
50 def copy_blocks(
51 kv_caches: List[torch.Tensor],
52 - src_to_dists: Dict[int, List[int]],
52 + src_to_dists: torch.Tensor,
53 ) -> None:
54 PagedAttention.copy_blocks(kv_caches, src_to_dists)
55
vllm/attention/ops/paged_attn.py
+1 -1
@@ -209,7 +209,7 @@ class PagedAttention:
209 @staticmethod
210 def copy_blocks(
211 kv_caches: List[torch.Tensor],
212 - src_to_dists: Dict[int, List[int]],
212 + src_to_dists: torch.Tensor,
213 ) -> None:
214 key_caches = [kv_cache[0] for kv_cache in kv_caches]
215 value_caches = [kv_cache[1] for kv_cache in kv_caches]
vllm/core/scheduler.py
+20 -21
@@ -13,7 +13,6 @@ from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
14 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
15 SequenceGroupMetadata, SequenceStatus)
16 -from vllm.utils import merge_dicts
16
17 logger = init_logger(__name__)
18
@@ -122,8 +121,8 @@ class SchedulerOutputs:
121 blocks_to_swap_in: Dict[int, int]
122 # Blocks to swap out. Dict of GPU -> CPU block number.
123 blocks_to_swap_out: Dict[int, int]
125 - # Blocks to copy. Source to a list of dest blocks.
126 - blocks_to_copy: Dict[int, List[int]]
124 + # Blocks to copy. Source to dest block.
125 + blocks_to_copy: List[Tuple[int, int]]
126 # Sequence groups that are going to be ignored.
127 ignored_seq_groups: List[SequenceGroup]
128 # The number of slots for lookahead decoding.
@@ -177,7 +176,7 @@ class SchedulerRunningOutputs:
176 # The blocks to swap out.
177 blocks_to_swap_out: Dict[int, int]
178 # The blocks to copy.
180 - blocks_to_copy: Dict[int, List[int]]
179 + blocks_to_copy: List[Tuple[int, int]]
180 # The number of slots for lookahead decoding.
181 num_lookahead_slots: int
182
@@ -189,7 +188,7 @@ class SchedulerRunningOutputs:
188 preempted=[],
189 swapped_out=[],
190 blocks_to_swap_out={},
192 - blocks_to_copy={},
191 + blocks_to_copy=[],
192 num_lookahead_slots=0,
193 )
194
@@ -209,7 +208,7 @@ class SchedulerSwappedInOutputs:
208 # The blocks to swap in.
209 blocks_to_swap_in: Dict[int, int]
210 # The blocks to copy.
212 - blocks_to_copy: Dict[int, List[int]]
211 + blocks_to_copy: List[Tuple[int, int]]
212 # The number of slots for lookahead decoding.
213 num_lookahead_slots: int
214 # Infeasible sequence groups.
@@ -221,7 +220,7 @@ class SchedulerSwappedInOutputs:
220 decode_seq_groups=[],
221 prefill_seq_groups=[],
222 blocks_to_swap_in={},
224 - blocks_to_copy={},
223 + blocks_to_copy=[],
224 num_lookahead_slots=0,
225 infeasible_seq_groups=[],
226 )
@@ -394,7 +393,7 @@ class Scheduler:
393 """
394 # Blocks that need to be swapped or copied before model execution.
395 blocks_to_swap_out: Dict[int, int] = {}
397 - blocks_to_copy: Dict[int, List[int]] = {}
396 + blocks_to_copy: List[Tuple[int, int]] = []
397
398 decode_seq_groups: List[ScheduledSequenceGroup] = []
399 prefill_seq_groups: List[ScheduledSequenceGroup] = []
@@ -511,7 +510,7 @@ class Scheduler:
510 """
511 # Blocks that need to be swapped or copied before model execution.
512 blocks_to_swap_in: Dict[int, int] = {}
514 - blocks_to_copy: Dict[int, List[int]] = {}
513 + blocks_to_copy: List[Tuple[int, int]] = []
514 decode_seq_groups: List[ScheduledSequenceGroup] = []
515 prefill_seq_groups: List[ScheduledSequenceGroup] = []
516 now = time.time()
@@ -794,8 +793,8 @@ class Scheduler:
793 num_batched_tokens=budget.num_batched_tokens,
794 blocks_to_swap_in=swapped_in.blocks_to_swap_in,
795 blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
797 - blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
798 - swapped_in.blocks_to_copy),
796 + blocks_to_copy=running_scheduled.blocks_to_copy +
797 + swapped_in.blocks_to_copy,
798 ignored_seq_groups=prefills.ignored_seq_groups +
799 swapped_in.infeasible_seq_groups,
800 num_lookahead_slots=running_scheduled.num_lookahead_slots,
@@ -882,8 +881,8 @@ class Scheduler:
881 num_batched_tokens=budget.num_batched_tokens,
882 blocks_to_swap_in=swapped_in.blocks_to_swap_in,
883 blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
885 - blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
886 - swapped_in.blocks_to_copy),
884 + blocks_to_copy=running_scheduled.blocks_to_copy +
885 + swapped_in.blocks_to_copy,
886 ignored_seq_groups=prefills.ignored_seq_groups,
887 num_lookahead_slots=running_scheduled.num_lookahead_slots,
888 running_queue_size=len(self.running),
@@ -1011,17 +1010,18 @@ class Scheduler:
1010 def _append_slots(
1011 self,
1012 seq_group: SequenceGroup,
1014 - blocks_to_copy: Dict[int, List[int]],
1013 + blocks_to_copy: List[Tuple[int, int]],
1014 ) -> None:
1015 """Appends new slots to the sequences in the given sequence group.
1016
1017 Args:
1018 seq_group (SequenceGroup): The sequence group containing the
1019 sequences to append slots to.
1021 - blocks_to_copy (Dict[int, List[int]]): A dictionary mapping source
1022 - block indices to lists of destination block indices. This
1023 - dictionary is updated with the new source and destination block
1024 - indices for the appended slots.
1020 + blocks_to_copy (List[Tuple[int, int]]): A list of tuple of two
1021 + ints, the first int is the source block index, and the second
1022 + int is the destination block index. This list is updated with
1023 + the new source and destination block indices for the appended
1024 + slots.
1025 """
1026 num_lookahead_slots = self._get_num_lookahead_slots(is_prefill=False)
1027
@@ -1029,9 +1029,8 @@ class Scheduler:
1029 cows = self.block_manager.append_slots(seq, num_lookahead_slots)
1030
1031 for src, dests in cows.items():
1032 - if src not in blocks_to_copy:
1033 - blocks_to_copy[src] = []
1034 - blocks_to_copy[src].extend(dests)
1032 + for dest in dests:
1033 + blocks_to_copy.append((src, dest))
1034
1035 def _preempt(
1036 self,
vllm/distributed/communication_op.py
+7
@@ -203,6 +203,9 @@ def broadcast_tensor_dict(
203 group=metadata_group)
204 async_handles = []
205 for tensor in tensor_list:
206 + if tensor.numel() == 0:
207 + # Skip broadcasting empty tensors.
208 + continue
209 async_handles.append(
210 torch.distributed.broadcast(tensor,
211 src=src,
@@ -224,6 +227,10 @@ def broadcast_tensor_dict(
227 tensor = torch.empty(value.size,
228 dtype=value.dtype,
229 device="cuda")
230 + if tensor.numel() == 0:
231 + # Skip broadcasting empty tensors.
232 + tensor_dict[key] = tensor
233 + continue
234 async_handle = torch.distributed.broadcast(tensor,
235 src=src,
236 async_op=True,
vllm/sequence.py
+3 -3
@@ -2,7 +2,7 @@
2 import copy
3 import enum
4 from dataclasses import dataclass, field
5 -from typing import TYPE_CHECKING, Dict, List, Optional, Union
5 +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
6
7 from vllm.block import LogicalTokenBlock
8 from vllm.lora.request import LoRARequest
@@ -745,8 +745,8 @@ class ExecuteModelRequest:
745 blocks_to_swap_in: Dict[int, int] = field(default_factory=dict)
746 # Blocks to swap out. Dict of GPU -> CPU block number.
747 blocks_to_swap_out: Dict[int, int] = field(default_factory=dict)
748 - # Blocks to copy. Source to a list of dest blocks.
749 - blocks_to_copy: Dict[int, List[int]] = field(default_factory=dict)
748 + # Blocks to copy. Source to dest block.
749 + blocks_to_copy: List[Tuple[int, int]] = field(default_factory=list)
750 # The number of slots for lookahead decoding.
751 num_lookahead_slots: int = 0
752 # The number of requests in the running queue.
vllm/worker/cache_engine.py
+1 -1
@@ -77,7 +77,7 @@ class CacheEngine:
77 self.attn_backend.swap_blocks(self.gpu_cache[i], self.cpu_cache[i],
78 src_to_dst)
79
80 - def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
80 + def copy(self, src_to_dsts: torch.Tensor) -> None:
81 self.attn_backend.copy_blocks(self.gpu_cache, src_to_dsts)
82
83 @staticmethod
vllm/worker/cpu_worker.py
+5 -2
@@ -248,9 +248,9 @@ class CPUWorker(LoraNotSupportedWorkerBase):
248
249 def cache_copy(
250 self,
251 - blocks_to_copy: Dict[int, List[int]],
251 + blocks_to_copy: torch.Tensor,
252 ) -> None:
253 - if blocks_to_copy:
253 + if blocks_to_copy.numel() > 0:
254 self.cache_engine.copy(blocks_to_copy)
255
256 @torch.inference_mode()
@@ -269,6 +269,9 @@ class CPUWorker(LoraNotSupportedWorkerBase):
269 num_seq_groups: int = len(seq_group_metadata_list)
270 assert execute_model_req is not None
271 blocks_to_copy = execute_model_req.blocks_to_copy
272 + blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
273 + device="cpu",
274 + dtype=torch.int64).view(-1, 2)
275 assert len(execute_model_req.blocks_to_swap_in) == 0
276 assert len(execute_model_req.blocks_to_swap_out) == 0
277 data: Dict[str, Any] = {
vllm/worker/worker.py
+5 -3
@@ -197,7 +197,7 @@ class Worker(WorkerBase):
197 self,
198 blocks_to_swap_in: Dict[int, int],
199 blocks_to_swap_out: Dict[int, int],
200 - blocks_to_copy: Dict[int, List[int]],
200 + blocks_to_copy: torch.Tensor,
201 ) -> None:
202 # Issue cache operations.
203 # TODO(woosuk): Profile swapping overhead and optimize if needed.
@@ -205,7 +205,7 @@ class Worker(WorkerBase):
205 self.cache_engine.swap_in(blocks_to_swap_in)
206 if blocks_to_swap_out:
207 self.cache_engine.swap_out(blocks_to_swap_out)
208 - if blocks_to_copy:
208 + if blocks_to_copy.numel() > 0:
209 self.cache_engine.copy(blocks_to_copy)
210
211 @torch.inference_mode()
@@ -225,7 +225,9 @@ class Worker(WorkerBase):
225 num_seq_groups = len(seq_group_metadata_list)
226 blocks_to_swap_in = execute_model_req.blocks_to_swap_in
227 blocks_to_swap_out = execute_model_req.blocks_to_swap_out
228 - blocks_to_copy = execute_model_req.blocks_to_copy
228 + blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
229 + device=self.device,
230 + dtype=torch.int64).view(-1, 2)
231 data: Dict[str, Any] = {
232 "num_seq_groups": num_seq_groups,
233 "blocks_to_swap_in": blocks_to_swap_in,