@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 20cfcdec99
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Optimization] change python dict to pytorch tensor for blocks to swap (#4659)

youkaichao committed May 8, 2024 at 12:07 UTC 20cfcdec998b39f5dbb0dc89efe4122f95f5cb16
21 files changed +137 -109
csrc/cache.h
+2 -2
@@ -8,12 +8,12 @@
8 void swap_blocks(
9 torch::Tensor& src,
10 torch::Tensor& dst,
11 - const std::map<int64_t, int64_t>& block_mapping);
11 + const torch::Tensor& block_mapping);
12
13 void copy_blocks(
14 std::vector<torch::Tensor>& key_caches,
15 std::vector<torch::Tensor>& value_caches,
16 - torch::Tensor& block_mapping);
16 + const torch::Tensor& block_mapping);
17
18 void reshape_and_cache(
19 torch::Tensor& key,
csrc/cache_kernels.cu
+11 -5
@@ -23,7 +23,7 @@
23 void swap_blocks(
24 torch::Tensor& src,
25 torch::Tensor& dst,
26 - const std::map<int64_t, int64_t>& block_mapping) {
26 + const torch::Tensor& block_mapping) {
27 torch::Device src_device = src.device();
28 torch::Device dst_device = dst.device();
29 cudaMemcpyKind memcpy_type;
@@ -40,6 +40,11 @@ void swap_blocks(
40 TORCH_CHECK(false, "Invalid device combination");
41 }
42
43 + // NOTE(youkaichao): keep in mind that `block_mapping` should be
44 + // a cpu tensor, otherwise every `item` call will require a gpu-cpu
45 + // synchronization.
46 + TORCH_CHECK(block_mapping.device().is_cpu(), "block_mapping must be on CPU");
47 +
48 char *src_ptr = static_cast<char*>(src.data_ptr());
49 char *dst_ptr = static_cast<char*>(dst.data_ptr());
50
@@ -47,9 +52,10 @@ void swap_blocks(
52 const at::cuda::OptionalCUDAGuard device_guard(src_device.is_cuda() ? src_device : dst_device);
53 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
54 // NOTE(woosuk): This can be slow if the number of blocks is large.
50 - for (const auto& pair : block_mapping) {
51 - int64_t src_block_number = pair.first;
52 - int64_t dst_block_number = pair.second;
55 + const int64_t num_blocks = block_mapping.size(0);
56 + for (size_t i = 0; i < num_blocks; i++) {
57 + int64_t src_block_number = block_mapping[i][0].item<int64_t>();
58 + int64_t dst_block_number = block_mapping[i][1].item<int64_t>();
59 int64_t src_offset = src_block_number * block_size_in_bytes;
60 int64_t dst_offset = dst_block_number * block_size_in_bytes;
61 cudaMemcpyAsync(
@@ -97,7 +103,7 @@ __global__ void copy_blocks_kernel(
103 void copy_blocks(
104 std::vector<torch::Tensor>& key_caches,
105 std::vector<torch::Tensor>& value_caches,
100 - torch::Tensor& block_mapping) {
106 + const torch::Tensor& block_mapping) {
107 int num_layers = key_caches.size();
108 TORCH_CHECK(num_layers == value_caches.size());
109 if (num_layers == 0) {
csrc/cpu/cache.cpp
+2 -2
@@ -83,7 +83,7 @@ void reshape_and_cache_cpu_impl(
83
84 void copy_blocks(std::vector<torch::Tensor> &key_caches,
85 std::vector<torch::Tensor> &value_caches,
86 - torch::Tensor& block_mapping) {
86 + const torch::Tensor& block_mapping) {
87 int num_layers = key_caches.size();
88 TORCH_CHECK(num_layers == value_caches.size());
89 if (num_layers == 0) {
@@ -128,6 +128,6 @@ void reshape_and_cache(torch::Tensor &key, torch::Tensor &value,
128 }
129
130 void swap_blocks(torch::Tensor &src, torch::Tensor &dst,
131 - const std::map<int64_t, int64_t> &block_mapping) {
131 + const torch::Tensor&block_mapping) {
132 TORCH_CHECK(false, "swap_blocks is unsupported on CPU.")
133 }
tests/core/test_block_manager.py
+2 -2
@@ -219,7 +219,7 @@ def test_swap():
219 before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
220 before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
221 mapping = block_manager.swap_out(seq_group)
222 - assert list(mapping.keys()) == gpu_blocks
222 + assert [x[0] for x in mapping] == gpu_blocks
223 after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
224 after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
225 assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
@@ -232,7 +232,7 @@ def test_swap():
232 before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
233 before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
234 mapping = block_manager.swap_in(seq_group)
235 - assert list(mapping.keys()) == cpu_blocks
235 + assert [x[0] for x in mapping] == cpu_blocks
236 after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
237 after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
238 assert before_cpu_blocks + len(cpu_blocks) == after_cpu_blocks
tests/core/test_chunked_prefill_scheduler.py
+16 -16
@@ -355,8 +355,8 @@ def test_swap():
355 _, out = schedule_and_update_computed_tokens(scheduler)
356 assert len(out.scheduled_seq_groups) == 0
357 assert out.num_batched_tokens == 0
358 - assert out.blocks_to_swap_out != {}
359 - assert out.blocks_to_swap_in == {}
358 + assert out.blocks_to_swap_out != []
359 + assert out.blocks_to_swap_in == []
360
361 # Add 1 more task. Swap should be prioritized over new prefill.
362 _, seq_group = create_dummy_prompt("2", prompt_length=60)
@@ -365,8 +365,8 @@ def test_swap():
365 assert len(out.scheduled_seq_groups) == 1
366 # 3 decodes. It is swapped in.
367 assert out.num_batched_tokens == 30
368 - assert out.blocks_to_swap_in != {}
369 - assert out.blocks_to_swap_out == {}
368 + assert out.blocks_to_swap_in != []
369 + assert out.blocks_to_swap_out == []
370
371
372 def test_running_prefill_prioritized_over_swap():
@@ -406,8 +406,8 @@ def test_running_prefill_prioritized_over_swap():
406 _, out = schedule_and_update_computed_tokens(scheduler)
407 assert len(out.scheduled_seq_groups) == 0
408 assert out.num_batched_tokens == 0
409 - assert out.blocks_to_swap_out != {}
410 - assert out.blocks_to_swap_in == {}
409 + assert out.blocks_to_swap_out != []
410 + assert out.blocks_to_swap_in == []
411
412 # Add 1 more task. Swap is not possible, so prefill is running.
413 scheduler.block_manager.can_swap_in = MagicMock()
@@ -419,8 +419,8 @@ def test_running_prefill_prioritized_over_swap():
419 assert len(out.scheduled_seq_groups) == 1
420 # 3 decodes. It is swapped in.
421 assert out.num_batched_tokens == 30
422 - assert out.blocks_to_swap_in == {}
423 - assert out.blocks_to_swap_out == {}
422 + assert out.blocks_to_swap_in == []
423 + assert out.blocks_to_swap_out == []
424 assert out.scheduled_seq_groups[0].seq_group == seq_group2
425
426 # Now although swap is possible, running prefill is prioritized.
@@ -429,8 +429,8 @@ def test_running_prefill_prioritized_over_swap():
429 assert len(out.scheduled_seq_groups) == 1
430 # 3 decodes. It is swapped in.
431 assert out.num_batched_tokens == 30
432 - assert out.blocks_to_swap_in == {}
433 - assert out.blocks_to_swap_out == {}
432 + assert out.blocks_to_swap_in == []
433 + assert out.blocks_to_swap_out == []
434 assert not seq_group2.is_prefill()
435 assert out.scheduled_seq_groups[0].seq_group == seq_group2
436 append_new_token(seq_group2, 1)
@@ -440,8 +440,8 @@ def test_running_prefill_prioritized_over_swap():
440 assert len(out.scheduled_seq_groups) == 1
441 # 3 decodes. It is swapped in.
442 assert out.num_batched_tokens == 1
443 - assert out.blocks_to_swap_in == {}
444 - assert out.blocks_to_swap_out == {}
443 + assert out.blocks_to_swap_in == []
444 + assert out.blocks_to_swap_out == []
445 assert not seq_group2.is_prefill()
446 assert out.scheduled_seq_groups[0].seq_group == seq_group2
447 append_new_token(seq_group2, 1)
@@ -451,8 +451,8 @@ def test_running_prefill_prioritized_over_swap():
451 _, out = schedule_and_update_computed_tokens(scheduler)
452 assert len(out.scheduled_seq_groups) == 1
453 assert out.num_batched_tokens == 30
454 - assert out.blocks_to_swap_in != {}
455 - assert out.blocks_to_swap_out == {}
454 + assert out.blocks_to_swap_in != []
455 + assert out.blocks_to_swap_out == []
456
457
458 def test_chunked_prefill_preempt():
@@ -493,8 +493,8 @@ def test_chunked_prefill_preempt():
493 _, out = schedule_and_update_computed_tokens(scheduler)
494 assert len(out.scheduled_seq_groups) == 0
495 assert out.num_batched_tokens == 0
496 - assert out.blocks_to_swap_out == {}
497 - assert out.blocks_to_swap_in == {}
496 + assert out.blocks_to_swap_out == []
497 + assert out.blocks_to_swap_in == []
498
499 # Make sure we can reschedule preempted request.
500 _, out = schedule_and_update_computed_tokens(scheduler)
tests/core/test_scheduler.py
+17 -17
@@ -293,8 +293,8 @@ def test_swapped_out_prioritized():
293 seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
294 assert len(out.scheduled_seq_groups) == 2
295 assert out.num_batched_tokens == 2
296 - assert out.blocks_to_swap_out != {}
297 - assert out.blocks_to_swap_in == {}
296 + assert out.blocks_to_swap_out != []
297 + assert out.blocks_to_swap_in == []
298 append_new_token(out, 1)
299
300 # Add 1 more task. Swap should be prioritized over prefill.
@@ -305,8 +305,8 @@ def test_swapped_out_prioritized():
305 assert len(out.scheduled_seq_groups) == 3
306 # 3 decodes. It is swapped in.
307 assert out.num_batched_tokens == 3
308 - assert out.blocks_to_swap_in != {}
309 - assert out.blocks_to_swap_out == {}
308 + assert out.blocks_to_swap_in != []
309 + assert out.blocks_to_swap_out == []
310
311
312 def initialize_scheduler(*,
@@ -566,7 +566,7 @@ def test_decode_schedule_preempted():
566 # NOTE: When enable_chunk is False, num_seqs budget is not updated.
567 # assert budget.num_curr_seqs == 1
568 # Both should be preempted, not swapped.
569 - assert output.blocks_to_swap_out == {}
569 + assert output.blocks_to_swap_out == []
570 # Nothing is copied.
571 assert output.blocks_to_copy == []
572
@@ -599,7 +599,7 @@ def test_decode_swap_beam_search():
599 scheduler.block_manager.can_append_slots.side_effect = (
600 cannot_append_second_group)
601 scheduler.block_manager.swap_out = MagicMock()
602 - expected_swap_mapping = {"5": "7"}
602 + expected_swap_mapping = [("5", "7")]
603 scheduler.block_manager.swap_out.return_value = expected_swap_mapping
604
605 remainig_running, output = scheduler._schedule_running(
@@ -647,7 +647,7 @@ def test_schedule_decode_blocks_to_copy_update():
647 assert len(output.preempted) == 0
648 assert len(output.swapped_out) == 0
649 # Nothing is preempted.
650 - assert output.blocks_to_swap_out == {}
650 + assert output.blocks_to_swap_out == []
651 # Since append_slot returns the source -> dist mapping, it should
652 # applied.
653 assert output.blocks_to_copy == [(2, 3)]
@@ -658,7 +658,7 @@ def test_schedule_swapped_simple():
658 swapped = deque()
659 policy = PolicyFactory.get_policy(policy_name="fcfs")
660 curr_loras = None
661 - blocks_to_swap_out = {}
661 + blocks_to_swap_out = []
662 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
663 scheduler._allocate_and_set_running(seq_group)
664 append_new_token_seq_group(60, seq_group, 1)
@@ -674,9 +674,9 @@ def test_schedule_swapped_simple():
674 assert len(output.decode_seq_groups) == 1
675 assert len(output.prefill_seq_groups) == 0
676 # swap in is the reverse of swap out
677 - blocks_to_swap_in_reverse = {}
678 - for swapin, swapout in output.blocks_to_swap_in.items():
679 - blocks_to_swap_in_reverse[swapout] = swapin
677 + blocks_to_swap_in_reverse = []
678 + for swapin, swapout in output.blocks_to_swap_in:
679 + blocks_to_swap_in_reverse.append((swapout, swapin))
680 assert blocks_to_swap_out == blocks_to_swap_in_reverse
681
682
@@ -685,7 +685,7 @@ def test_schedule_swapped_max_token_budget():
685 swapped = deque()
686 policy = PolicyFactory.get_policy(policy_name="fcfs")
687 curr_loras = None
688 - blocks_to_swap_out = {}
688 + blocks_to_swap_out = []
689 for _ in range(2):
690 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
691 scheduler._allocate_and_set_running(seq_group)
@@ -719,7 +719,7 @@ def test_schedule_swapped_max_seqs():
719 swapped = deque()
720 policy = PolicyFactory.get_policy(policy_name="fcfs")
721 curr_loras = None
722 - blocks_to_swap_out = {}
722 + blocks_to_swap_out = []
723 for i in range(4):
724 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
725 scheduler._allocate_and_set_running(seq_group)
@@ -752,7 +752,7 @@ def test_schedule_swapped_max_loras():
752 swapped = deque()
753 policy = PolicyFactory.get_policy(policy_name="fcfs")
754 curr_loras = set()
755 - blocks_to_swap_out = {}
755 + blocks_to_swap_out = []
756 for i in range(2):
757 _, seq_group = create_dummy_prompt(str(i),
758 prompt_length=60,
@@ -781,7 +781,7 @@ def test_schedule_swapped_cannot_swap_in():
781 swapped = deque()
782 policy = PolicyFactory.get_policy(policy_name="fcfs")
783 curr_loras = None
784 - blocks_to_swap_out = {}
784 + blocks_to_swap_out = []
785 for _ in range(2):
786 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
787 scheduler._allocate_and_set_running(seq_group)
@@ -808,7 +808,7 @@ def test_infeasible_swap():
808 swapped = deque()
809 policy = PolicyFactory.get_policy(policy_name="fcfs")
810 curr_loras = None
811 - blocks_to_swap_out = {}
811 + blocks_to_swap_out = []
812 for _ in range(2):
813 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
814 scheduler._allocate_and_set_running(seq_group)
@@ -839,7 +839,7 @@ def test_schedule_swapped_blocks_to_copy():
839 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
840 scheduler._allocate_and_set_running(seq_group)
841 append_new_token_seq_group(60, seq_group, 1)
842 - blocks_to_swap_out = {}
842 + blocks_to_swap_out = []
843 scheduler._swap_out(seq_group, blocks_to_swap_out)
844 swapped.append(seq_group)
845
tests/kernels/test_cache.py
+9 -4
@@ -315,7 +315,10 @@ def test_swap_blocks(
315 else:
316 dst_blocks = random.sample(range(num_blocks), num_mappings)
317
318 - block_mapping = dict(zip(src_blocks, dst_blocks))
318 + block_mapping = list(zip(src_blocks, dst_blocks))
319 + block_mapping_tensor = torch.tensor(block_mapping,
320 + dtype=torch.int64,
321 + device="cpu").view(-1, 2)
322
323 # Create the KV caches on the first device.
324 src_key_caches, src_value_caches = kv_cache_factory(
@@ -331,10 +334,12 @@ def test_swap_blocks(
334 src_value_caches_clone = src_value_caches[0].clone()
335
336 # Call the swap_blocks kernel.
334 - ops.swap_blocks(src_key_caches[0], dist_key_caches[0], block_mapping)
335 - ops.swap_blocks(src_value_caches[0], dist_value_caches[0], block_mapping)
337 + ops.swap_blocks(src_key_caches[0], dist_key_caches[0],
338 + block_mapping_tensor)
339 + ops.swap_blocks(src_value_caches[0], dist_value_caches[0],
340 + block_mapping_tensor)
341
337 - for src, dst in block_mapping.items():
342 + for src, dst in block_mapping:
343 assert torch.allclose(src_key_caches_clone[src].cpu(),
344 dist_key_caches[0][dst].cpu())
345 assert torch.allclose(src_value_caches_clone[src].cpu(),
tests/worker/test_swap.py
+12 -12
@@ -54,10 +54,10 @@ def test_swap() -> None:
54 a.cuda(), b.cuda(), rtol=0.0, atol=0.0)
55
56 # Test swap out.
57 - blocks_to_swap_out = {3: 72, 56: 35, 84: 34}
57 + blocks_to_swap_out = [(3, 72), (56, 35), (84, 34)]
58 execute_model_req = ExecuteModelRequest(
59 seq_group_metadata_list=[],
60 - blocks_to_swap_in={},
60 + blocks_to_swap_in=[],
61 blocks_to_swap_out=blocks_to_swap_out,
62 blocks_to_copy=[],
63 )
@@ -66,24 +66,24 @@ def test_swap() -> None:
66 for i in range(num_layers):
67 gpu_key_cache, gpu_value_cache = gpu_cache[i]
68 cpu_key_cache, cpu_value_cache = cpu_cache[i]
69 - for src, dst in blocks_to_swap_out.items():
69 + for src, dst in blocks_to_swap_out:
70 assert allclose(gpu_key_cache[src], cpu_key_cache[dst])
71 assert allclose(gpu_value_cache[src], cpu_value_cache[dst])
72
73 # Test swap in.
74 - execute_model_req.blocks_to_swap_out = {}
75 - execute_model_req.blocks_to_swap_in = {
76 - 19: 45,
77 - 67: 23,
78 - 12: 78,
79 - 40: 99,
80 - 1: 71
81 - }
74 + execute_model_req.blocks_to_swap_out = []
75 + execute_model_req.blocks_to_swap_in = [
76 + (19, 45),
77 + (67, 23),
78 + (12, 78),
79 + (40, 99),
80 + (1, 71),
81 + ]
82 worker.execute_model(execute_model_req=execute_model_req)
83
84 for i in range(num_layers):
85 gpu_key_cache, gpu_value_cache = gpu_cache[i]
86 cpu_key_cache, cpu_value_cache = cpu_cache[i]
87 - for src, dst in execute_model_req.blocks_to_swap_in.items():
87 + for src, dst in execute_model_req.blocks_to_swap_in:
88 assert allclose(gpu_key_cache[dst], cpu_key_cache[src])
89 assert allclose(gpu_value_cache[dst], cpu_value_cache[src])
vllm/attention/backends/abstract.py
+1 -1
@@ -39,7 +39,7 @@ class AttentionBackend(ABC):
39 def swap_blocks(
40 src_kv_cache: torch.Tensor,
41 dst_kv_cache: torch.Tensor,
42 - src_to_dst: Dict[int, int],
42 + src_to_dst: torch.Tensor,
43 ) -> None:
44 raise NotImplementedError
45
vllm/attention/backends/flash_attn.py
+2 -2
@@ -5,7 +5,7 @@ XFormers backend. The duplicated code will be removed once we use flash-attn or
5 flashinfer for all the attention operations.
6 """
7 from dataclasses import dataclass
8 -from typing import Dict, List, Optional, Tuple, Type
8 +from typing import List, Optional, Tuple, Type
9
10 import torch
11 from flash_attn import flash_attn_varlen_func
@@ -45,7 +45,7 @@ class FlashAttentionBackend(AttentionBackend):
45 def swap_blocks(
46 src_kv_cache: torch.Tensor,
47 dst_kv_cache: torch.Tensor,
48 - src_to_dst: Dict[int, int],
48 + src_to_dst: torch.Tensor,
49 ) -> None:
50 PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
51
vllm/attention/backends/flashinfer.py
+1 -1
@@ -39,7 +39,7 @@ class FlashInferBackend(AttentionBackend):
39 def swap_blocks(
40 src_kv_cache: torch.Tensor,
41 dst_kv_cache: torch.Tensor,
42 - src_to_dst: Dict[int, int],
42 + src_to_dst: torch.Tensor,
43 ) -> None:
44 raise NotImplementedError
45
vllm/attention/backends/rocm_flash_attn.py
+2 -2
@@ -1,6 +1,6 @@
1 """Attention layer ROCm GPUs."""
2 from dataclasses import dataclass
3 -from typing import Dict, List, Optional, Tuple, Type
3 +from typing import List, Optional, Tuple, Type
4
5 import torch
6
@@ -43,7 +43,7 @@ class ROCmFlashAttentionBackend(AttentionBackend):
43 def swap_blocks(
44 src_kv_cache: torch.Tensor,
45 dst_kv_cache: torch.Tensor,
46 - src_to_dst: Dict[int, int],
46 + src_to_dst: torch.Tensor,
47 ) -> None:
48 PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
49
vllm/attention/backends/torch_sdpa.py
+2 -2
@@ -1,7 +1,7 @@
1 """ Attention layer with torch scaled_dot_product_attention
2 and PagedAttention."""
3 from dataclasses import dataclass
4 -from typing import Dict, List, Optional, Tuple, Type
4 +from typing import List, Optional, Tuple, Type
5
6 import torch
7 from torch.nn.functional import scaled_dot_product_attention
@@ -41,7 +41,7 @@ class TorchSDPABackend(AttentionBackend):
41 def swap_blocks(
42 src_kv_cache: torch.Tensor,
43 dst_kv_cache: torch.Tensor,
44 - src_to_dst: Dict[int, int],
44 + src_to_dst: torch.Tensor,
45 ) -> None:
46 PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
47
vllm/attention/ops/paged_attn.py
+2 -2
@@ -1,5 +1,5 @@
1 from dataclasses import dataclass
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import List, Optional, Tuple
3
4 import torch
5
@@ -196,7 +196,7 @@ class PagedAttention:
196 def swap_blocks(
197 src_kv_cache: torch.Tensor,
198 dst_kv_cache: torch.Tensor,
199 - src_to_dst: Dict[int, int],
199 + src_to_dst: torch.Tensor,
200 ) -> None:
201 src_key_cache = src_kv_cache[0]
202 dst_key_cache = dst_kv_cache[0]
vllm/core/block_manager_v1.py
+8 -4
@@ -473,11 +473,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
473
474 def swap_in(self,
475 seq_group: SequenceGroup,
476 - num_lookahead_slots: int = 0) -> Dict[int, int]:
476 + num_lookahead_slots: int = 0) -> List[Tuple[int, int]]:
477 assert (num_lookahead_slots == 0
478 ), "BlockSpaceManagerV1 does not support lookahead allocation"
479
480 # CPU block -> GPU block.
481 + # dict is efficient in lookup `if cpu_block in mapping`
482 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
483 for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
484 new_block_table: BlockTable = []
@@ -500,14 +501,16 @@ class BlockSpaceManagerV1(BlockSpaceManager):
501 cpu_block.block_number: gpu_block.block_number
502 for cpu_block, gpu_block in mapping.items()
503 }
503 - return block_number_mapping
504 + # convert to list of tuples once here
505 + return list(block_number_mapping.items())
506
507 def can_swap_out(self, seq_group: SequenceGroup) -> bool:
508 blocks = self._get_physical_blocks(seq_group)
509 return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
510
509 - def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
511 + def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
512 # GPU block -> CPU block.
513 + # dict is efficient in lookup `if gpu_block in mapping`
514 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
515 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
516 new_block_table: BlockTable = []
@@ -530,7 +533,8 @@ class BlockSpaceManagerV1(BlockSpaceManager):
533 gpu_block.block_number: cpu_block.block_number
534 for gpu_block, cpu_block in mapping.items()
535 }
533 - return block_number_mapping
536 + # convert to list of tuples once here
537 + return list(block_number_mapping.items())
538
539 def _free_block_table(self, block_table: BlockTable) -> None:
540 # when using a sliding window, each seq will only use up
vllm/core/block_manager_v2.py
+2 -2
@@ -243,13 +243,13 @@ class BlockSpaceManagerV2(BlockSpaceManager):
243 return AllocStatus.LATER
244
245 def swap_in(self, seq_group: SequenceGroup,
246 - num_lookahead_slots: int) -> Dict[int, int]:
246 + num_lookahead_slots: int) -> List[Tuple[int, int]]:
247 raise NotImplementedError
248
249 def can_swap_out(self, seq_group: SequenceGroup) -> bool:
250 return False
251
252 - def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
252 + def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
253 raise NotImplementedError
254
255 def get_num_free_gpu_blocks(self) -> int:
vllm/core/interfaces.py
+3 -3
@@ -1,6 +1,6 @@
1 import enum
2 from abc import ABC, abstractmethod
3 -from typing import Dict, List
3 +from typing import List
4 from typing import Sequence as GenericSequence
5 from typing import Tuple
6
@@ -69,7 +69,7 @@ class BlockSpaceManager(ABC):
69
70 @abstractmethod
71 def swap_in(self, seq_group: SequenceGroup,
72 - num_lookahead_slots: int) -> Dict[int, int]:
72 + num_lookahead_slots: int) -> List[Tuple[int, int]]:
73 pass
74
75 @abstractmethod
@@ -77,7 +77,7 @@ class BlockSpaceManager(ABC):
77 pass
78
79 @abstractmethod
80 - def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
80 + def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
81 pass
82
83 @abstractmethod
vllm/core/scheduler.py
+16 -16
@@ -117,10 +117,10 @@ class SchedulerOutputs:
117 num_prefill_groups: int
118 # Total number of batched tokens.
119 num_batched_tokens: int
120 - # Blocks to swap in. Dict of CPU -> GPU block number.
121 - blocks_to_swap_in: Dict[int, int]
122 - # Blocks to swap out. Dict of GPU -> CPU block number.
123 - blocks_to_swap_out: Dict[int, int]
120 + # Blocks to swap in. List of CPU -> GPU block number.
121 + blocks_to_swap_in: List[Tuple[int, int]]
122 + # Blocks to swap out. List of GPU -> CPU block number.
123 + blocks_to_swap_out: List[Tuple[int, int]]
124 # Blocks to copy. Source to dest block.
125 blocks_to_copy: List[Tuple[int, int]]
126 # Sequence groups that are going to be ignored.
@@ -174,7 +174,7 @@ class SchedulerRunningOutputs:
174 # Sequences that are swapped out.
175 swapped_out: List[SequenceGroup]
176 # The blocks to swap out.
177 - blocks_to_swap_out: Dict[int, int]
177 + blocks_to_swap_out: List[Tuple[int, int]]
178 # The blocks to copy.
179 blocks_to_copy: List[Tuple[int, int]]
180 # The number of slots for lookahead decoding.
@@ -187,7 +187,7 @@ class SchedulerRunningOutputs:
187 prefill_seq_groups=[],
188 preempted=[],
189 swapped_out=[],
190 - blocks_to_swap_out={},
190 + blocks_to_swap_out=[],
191 blocks_to_copy=[],
192 num_lookahead_slots=0,
193 )
@@ -206,7 +206,7 @@ class SchedulerSwappedInOutputs:
206 # phase. I.e., it means the prefill has been chunked.
207 prefill_seq_groups: List[SequenceGroup]
208 # The blocks to swap in.
209 - blocks_to_swap_in: Dict[int, int]
209 + blocks_to_swap_in: List[Tuple[int, int]]
210 # The blocks to copy.
211 blocks_to_copy: List[Tuple[int, int]]
212 # The number of slots for lookahead decoding.
@@ -219,7 +219,7 @@ class SchedulerSwappedInOutputs:
219 return SchedulerSwappedInOutputs(
220 decode_seq_groups=[],
221 prefill_seq_groups=[],
222 - blocks_to_swap_in={},
222 + blocks_to_swap_in=[],
223 blocks_to_copy=[],
224 num_lookahead_slots=0,
225 infeasible_seq_groups=[],
@@ -392,7 +392,7 @@ class Scheduler:
392 scheduling and SchedulerRunningOutputs.
393 """
394 # Blocks that need to be swapped or copied before model execution.
395 - blocks_to_swap_out: Dict[int, int] = {}
395 + blocks_to_swap_out: List[Tuple[int, int]] = []
396 blocks_to_copy: List[Tuple[int, int]] = []
397
398 decode_seq_groups: List[ScheduledSequenceGroup] = []
@@ -509,7 +509,7 @@ class Scheduler:
509 SchedulerSwappedInOutputs.
510 """
511 # Blocks that need to be swapped or copied before model execution.
512 - blocks_to_swap_in: Dict[int, int] = {}
512 + blocks_to_swap_in: List[Tuple[int, int]] = []
513 blocks_to_copy: List[Tuple[int, int]] = []
514 decode_seq_groups: List[ScheduledSequenceGroup] = []
515 prefill_seq_groups: List[ScheduledSequenceGroup] = []
@@ -1032,7 +1032,7 @@ class Scheduler:
1032 def _preempt(
1033 self,
1034 seq_group: SequenceGroup,
1035 - blocks_to_swap_out: Dict[int, int],
1035 + blocks_to_swap_out: List[Tuple[int, int]],
1036 preemption_mode: Optional[PreemptionMode] = None,
1037 ) -> PreemptionMode:
1038 # If preemption mode is not specified, we determine the mode as follows:
@@ -1073,24 +1073,24 @@ class Scheduler:
1073 def _preempt_by_swap(
1074 self,
1075 seq_group: SequenceGroup,
1076 - blocks_to_swap_out: Dict[int, int],
1076 + blocks_to_swap_out: List[Tuple[int, int]],
1077 ) -> None:
1078 self._swap_out(seq_group, blocks_to_swap_out)
1079
1080 def _swap_in(
1081 self,
1082 seq_group: SequenceGroup,
1083 - blocks_to_swap_in: Dict[int, int],
1083 + blocks_to_swap_in: List[Tuple[int, int]],
1084 ) -> None:
1085 mapping = self.block_manager.swap_in(seq_group)
1086 - blocks_to_swap_in.update(mapping)
1086 + blocks_to_swap_in.extend(mapping)
1087 for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
1088 seq.status = SequenceStatus.RUNNING
1089
1090 def _swap_out(
1091 self,
1092 seq_group: SequenceGroup,
1093 - blocks_to_swap_out: Dict[int, int],
1093 + blocks_to_swap_out: List[Tuple[int, int]],
1094 ) -> None:
1095 if not self.block_manager.can_swap_out(seq_group):
1096 # FIXME(woosuk): Abort the sequence group instead of aborting the
@@ -1099,7 +1099,7 @@ class Scheduler:
1099 "Aborted due to the lack of CPU swap space. Please increase "
1100 "the swap space to avoid this error.")
1101 mapping = self.block_manager.swap_out(seq_group)
1102 - blocks_to_swap_out.update(mapping)
1102 + blocks_to_swap_out.extend(mapping)
1103 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
1104 seq.status = SequenceStatus.SWAPPED
1105
vllm/sequence.py
+4 -4
@@ -741,10 +741,10 @@ class ExecuteModelRequest:
741 """The model execution request."""
742 # The sequence group metadata list.
743 seq_group_metadata_list: List[SequenceGroupMetadata]
744 - # Blocks to swap in. Dict of CPU -> GPU block number.
745 - blocks_to_swap_in: Dict[int, int] = field(default_factory=dict)
746 - # Blocks to swap out. Dict of GPU -> CPU block number.
747 - blocks_to_swap_out: Dict[int, int] = field(default_factory=dict)
744 + # Blocks to swap in. List of CPU -> GPU block number.
745 + blocks_to_swap_in: List[Tuple[int, int]] = field(default_factory=list)
746 + # Blocks to swap out. List of GPU -> CPU block number.
747 + blocks_to_swap_out: List[Tuple[int, int]] = field(default_factory=list)
748 # Blocks to copy. Source to dest block.
749 blocks_to_copy: List[Tuple[int, int]] = field(default_factory=list)
750 # The number of slots for lookahead decoding.
vllm/worker/cache_engine.py
+3 -3
@@ -1,5 +1,5 @@
1 """CacheEngine class for managing the KV cache."""
2 -from typing import Dict, List
2 +from typing import List
3
4 import torch
5
@@ -67,12 +67,12 @@ class CacheEngine:
67 device=device))
68 return kv_cache
69
70 - def swap_in(self, src_to_dst: Dict[int, int]) -> None:
70 + def swap_in(self, src_to_dst: torch.Tensor) -> None:
71 for i in range(self.num_layers):
72 self.attn_backend.swap_blocks(self.cpu_cache[i], self.gpu_cache[i],
73 src_to_dst)
74
75 - def swap_out(self, src_to_dst: Dict[int, int]) -> None:
75 + def swap_out(self, src_to_dst: torch.Tensor) -> None:
76 for i in range(self.num_layers):
77 self.attn_backend.swap_blocks(self.gpu_cache[i], self.cpu_cache[i],
78 src_to_dst)
vllm/worker/worker.py
+20 -7
@@ -195,15 +195,14 @@ class Worker(WorkerBase):
195
196 def cache_swap(
197 self,
198 - blocks_to_swap_in: Dict[int, int],
199 - blocks_to_swap_out: Dict[int, int],
198 + blocks_to_swap_in: torch.Tensor,
199 + blocks_to_swap_out: torch.Tensor,
200 blocks_to_copy: torch.Tensor,
201 ) -> None:
202 # Issue cache operations.
203 - # TODO(woosuk): Profile swapping overhead and optimize if needed.
204 - if blocks_to_swap_in:
203 + if blocks_to_swap_in.numel() > 0:
204 self.cache_engine.swap_in(blocks_to_swap_in)
206 - if blocks_to_swap_out:
205 + if blocks_to_swap_out.numel() > 0:
206 self.cache_engine.swap_out(blocks_to_swap_out)
207 if blocks_to_copy.numel() > 0:
208 self.cache_engine.copy(blocks_to_copy)
@@ -219,12 +218,26 @@ class Worker(WorkerBase):
218 else:
219 seq_group_metadata_list = execute_model_req.seq_group_metadata_list
220
221 + blocks_to_swap_in: torch.Tensor
222 + blocks_to_swap_out: torch.Tensor
223 + blocks_to_copy: torch.Tensor
224 if self.is_driver_worker:
225 assert seq_group_metadata_list is not None
226 assert execute_model_req is not None
227 num_seq_groups = len(seq_group_metadata_list)
226 - blocks_to_swap_in = execute_model_req.blocks_to_swap_in
227 - blocks_to_swap_out = execute_model_req.blocks_to_swap_out
228 + # `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
229 + # they contain parameters to launch cudamemcpyasync.
230 + blocks_to_swap_in = torch.tensor(
231 + execute_model_req.blocks_to_swap_in,
232 + device="cpu",
233 + dtype=torch.int64).view(-1, 2)
234 + blocks_to_swap_out = torch.tensor(
235 + execute_model_req.blocks_to_swap_out,
236 + device="cpu",
237 + dtype=torch.int64).view(-1, 2)
238 + # `blocks_to_copy` is a gpu tensor. The src and tgt of
239 + # blocks to copy are in the same device, and `blocks_to_copy`
240 + # can be used directly within cuda kernels.
241 blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
242 device=self.device,
243 dtype=torch.int64).view(-1, 2)