A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Optimization] change python dict to pytorch tensor for blocks to swap (#4659)
youkaichao committed
May 8, 2024 at 12:07 UTC
20cfcdec998b39f5dbb0dc89efe4122f95f5cb16
21 files changed
+137
-109
csrc/cache.h
+2
-2
@@ -8,12 +8,12 @@
8
void swap_blocks(
9
torch::Tensor& src,
10
torch::Tensor& dst,
11
- const std::map<int64_t, int64_t>& block_mapping);
11
+ const torch::Tensor& block_mapping);
12
13
void copy_blocks(
14
std::vector<torch::Tensor>& key_caches,
15
std::vector<torch::Tensor>& value_caches,
16
- torch::Tensor& block_mapping);
16
+ const torch::Tensor& block_mapping);
17
18
void reshape_and_cache(
19
torch::Tensor& key,
csrc/cache_kernels.cu
+11
-5
@@ -23,7 +23,7 @@
23
void swap_blocks(
24
torch::Tensor& src,
25
torch::Tensor& dst,
26
- const std::map<int64_t, int64_t>& block_mapping) {
26
+ const torch::Tensor& block_mapping) {
27
torch::Device src_device = src.device();
28
torch::Device dst_device = dst.device();
29
cudaMemcpyKind memcpy_type;
@@ -40,6 +40,11 @@ void swap_blocks(
40
TORCH_CHECK(false, "Invalid device combination");
41
}
42
43
+ // NOTE(youkaichao): keep in mind that `block_mapping` should be
44
+ // a cpu tensor, otherwise every `item` call will require a gpu-cpu
45
+ // synchronization.
46
+ TORCH_CHECK(block_mapping.device().is_cpu(), "block_mapping must be on CPU");
47
+
48
char *src_ptr = static_cast<char*>(src.data_ptr());
49
char *dst_ptr = static_cast<char*>(dst.data_ptr());
50
@@ -47,9 +52,10 @@ void swap_blocks(
52
const at::cuda::OptionalCUDAGuard device_guard(src_device.is_cuda() ? src_device : dst_device);
53
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
54
// NOTE(woosuk): This can be slow if the number of blocks is large.
50
- for (const auto& pair : block_mapping) {
51
- int64_t src_block_number = pair.first;
52
- int64_t dst_block_number = pair.second;
55
+ const int64_t num_blocks = block_mapping.size(0);
56
+ for (size_t i = 0; i < num_blocks; i++) {
57
+ int64_t src_block_number = block_mapping[i][0].item<int64_t>();
58
+ int64_t dst_block_number = block_mapping[i][1].item<int64_t>();
59
int64_t src_offset = src_block_number * block_size_in_bytes;
60
int64_t dst_offset = dst_block_number * block_size_in_bytes;
61
cudaMemcpyAsync(
@@ -97,7 +103,7 @@ __global__ void copy_blocks_kernel(
103
void copy_blocks(
104
std::vector<torch::Tensor>& key_caches,
105
std::vector<torch::Tensor>& value_caches,
100
- torch::Tensor& block_mapping) {
106
+ const torch::Tensor& block_mapping) {
107
int num_layers = key_caches.size();
108
TORCH_CHECK(num_layers == value_caches.size());
109
if (num_layers == 0) {
csrc/cpu/cache.cpp
+2
-2
@@ -83,7 +83,7 @@ void reshape_and_cache_cpu_impl(
83
84
void copy_blocks(std::vector<torch::Tensor> &key_caches,
85
std::vector<torch::Tensor> &value_caches,
86
- torch::Tensor& block_mapping) {
86
+ const torch::Tensor& block_mapping) {
87
int num_layers = key_caches.size();
88
TORCH_CHECK(num_layers == value_caches.size());
89
if (num_layers == 0) {
@@ -128,6 +128,6 @@ void reshape_and_cache(torch::Tensor &key, torch::Tensor &value,
128
}
129
130
void swap_blocks(torch::Tensor &src, torch::Tensor &dst,
131
- const std::map<int64_t, int64_t> &block_mapping) {
131
+ const torch::Tensor&block_mapping) {
132
TORCH_CHECK(false, "swap_blocks is unsupported on CPU.")
133
}
tests/core/test_block_manager.py
+2
-2
@@ -219,7 +219,7 @@ def test_swap():
219
before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
220
before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
221
mapping = block_manager.swap_out(seq_group)
222
- assert list(mapping.keys()) == gpu_blocks
222
+ assert [x[0] for x in mapping] == gpu_blocks
223
after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
224
after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
225
assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
@@ -232,7 +232,7 @@ def test_swap():
232
before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
233
before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
234
mapping = block_manager.swap_in(seq_group)
235
- assert list(mapping.keys()) == cpu_blocks
235
+ assert [x[0] for x in mapping] == cpu_blocks
236
after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
237
after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
238
assert before_cpu_blocks + len(cpu_blocks) == after_cpu_blocks
tests/core/test_chunked_prefill_scheduler.py
+16
-16
@@ -355,8 +355,8 @@ def test_swap():
355
_, out = schedule_and_update_computed_tokens(scheduler)
356
assert len(out.scheduled_seq_groups) == 0
357
assert out.num_batched_tokens == 0
358
- assert out.blocks_to_swap_out != {}
359
- assert out.blocks_to_swap_in == {}
358
+ assert out.blocks_to_swap_out != []
359
+ assert out.blocks_to_swap_in == []
360
361
# Add 1 more task. Swap should be prioritized over new prefill.
362
_, seq_group = create_dummy_prompt("2", prompt_length=60)
@@ -365,8 +365,8 @@ def test_swap():
365
assert len(out.scheduled_seq_groups) == 1
366
# 3 decodes. It is swapped in.
367
assert out.num_batched_tokens == 30
368
- assert out.blocks_to_swap_in != {}
369
- assert out.blocks_to_swap_out == {}
368
+ assert out.blocks_to_swap_in != []
369
+ assert out.blocks_to_swap_out == []
370
371
372
def test_running_prefill_prioritized_over_swap():
@@ -406,8 +406,8 @@ def test_running_prefill_prioritized_over_swap():
406
_, out = schedule_and_update_computed_tokens(scheduler)
407
assert len(out.scheduled_seq_groups) == 0
408
assert out.num_batched_tokens == 0
409
- assert out.blocks_to_swap_out != {}
410
- assert out.blocks_to_swap_in == {}
409
+ assert out.blocks_to_swap_out != []
410
+ assert out.blocks_to_swap_in == []
411
412
# Add 1 more task. Swap is not possible, so prefill is running.
413
scheduler.block_manager.can_swap_in = MagicMock()
@@ -419,8 +419,8 @@ def test_running_prefill_prioritized_over_swap():
419
assert len(out.scheduled_seq_groups) == 1
420
# 3 decodes. It is swapped in.
421
assert out.num_batched_tokens == 30
422
- assert out.blocks_to_swap_in == {}
423
- assert out.blocks_to_swap_out == {}
422
+ assert out.blocks_to_swap_in == []
423
+ assert out.blocks_to_swap_out == []
424
assert out.scheduled_seq_groups[0].seq_group == seq_group2
425
426
# Now although swap is possible, running prefill is prioritized.
@@ -429,8 +429,8 @@ def test_running_prefill_prioritized_over_swap():
429
assert len(out.scheduled_seq_groups) == 1
430
# 3 decodes. It is swapped in.
431
assert out.num_batched_tokens == 30
432
- assert out.blocks_to_swap_in == {}
433
- assert out.blocks_to_swap_out == {}
432
+ assert out.blocks_to_swap_in == []
433
+ assert out.blocks_to_swap_out == []
434
assert not seq_group2.is_prefill()
435
assert out.scheduled_seq_groups[0].seq_group == seq_group2
436
append_new_token(seq_group2, 1)
@@ -440,8 +440,8 @@ def test_running_prefill_prioritized_over_swap():
440
assert len(out.scheduled_seq_groups) == 1
441
# 3 decodes. It is swapped in.
442
assert out.num_batched_tokens == 1
443
- assert out.blocks_to_swap_in == {}
444
- assert out.blocks_to_swap_out == {}
443
+ assert out.blocks_to_swap_in == []
444
+ assert out.blocks_to_swap_out == []
445
assert not seq_group2.is_prefill()
446
assert out.scheduled_seq_groups[0].seq_group == seq_group2
447
append_new_token(seq_group2, 1)
@@ -451,8 +451,8 @@ def test_running_prefill_prioritized_over_swap():
451
_, out = schedule_and_update_computed_tokens(scheduler)
452
assert len(out.scheduled_seq_groups) == 1
453
assert out.num_batched_tokens == 30
454
- assert out.blocks_to_swap_in != {}
455
- assert out.blocks_to_swap_out == {}
454
+ assert out.blocks_to_swap_in != []
455
+ assert out.blocks_to_swap_out == []
456
457
458
def test_chunked_prefill_preempt():
@@ -493,8 +493,8 @@ def test_chunked_prefill_preempt():
493
_, out = schedule_and_update_computed_tokens(scheduler)
494
assert len(out.scheduled_seq_groups) == 0
495
assert out.num_batched_tokens == 0
496
- assert out.blocks_to_swap_out == {}
497
- assert out.blocks_to_swap_in == {}
496
+ assert out.blocks_to_swap_out == []
497
+ assert out.blocks_to_swap_in == []
498
499
# Make sure we can reschedule preempted request.
500
_, out = schedule_and_update_computed_tokens(scheduler)
tests/core/test_scheduler.py
+17
-17
@@ -293,8 +293,8 @@ def test_swapped_out_prioritized():
293
seq_group_meta, out = schedule_and_update_computed_tokens(scheduler)
294
assert len(out.scheduled_seq_groups) == 2
295
assert out.num_batched_tokens == 2
296
- assert out.blocks_to_swap_out != {}
297
- assert out.blocks_to_swap_in == {}
296
+ assert out.blocks_to_swap_out != []
297
+ assert out.blocks_to_swap_in == []
298
append_new_token(out, 1)
299
300
# Add 1 more task. Swap should be prioritized over prefill.
@@ -305,8 +305,8 @@ def test_swapped_out_prioritized():
305
assert len(out.scheduled_seq_groups) == 3
306
# 3 decodes. It is swapped in.
307
assert out.num_batched_tokens == 3
308
- assert out.blocks_to_swap_in != {}
309
- assert out.blocks_to_swap_out == {}
308
+ assert out.blocks_to_swap_in != []
309
+ assert out.blocks_to_swap_out == []
310
311
312
def initialize_scheduler(*,
@@ -566,7 +566,7 @@ def test_decode_schedule_preempted():
566
# NOTE: When enable_chunk is False, num_seqs budget is not updated.
567
# assert budget.num_curr_seqs == 1
568
# Both should be preempted, not swapped.
569
- assert output.blocks_to_swap_out == {}
569
+ assert output.blocks_to_swap_out == []
570
# Nothing is copied.
571
assert output.blocks_to_copy == []
572
@@ -599,7 +599,7 @@ def test_decode_swap_beam_search():
599
scheduler.block_manager.can_append_slots.side_effect = (
600
cannot_append_second_group)
601
scheduler.block_manager.swap_out = MagicMock()
602
- expected_swap_mapping = {"5": "7"}
602
+ expected_swap_mapping = [("5", "7")]
603
scheduler.block_manager.swap_out.return_value = expected_swap_mapping
604
605
remainig_running, output = scheduler._schedule_running(
@@ -647,7 +647,7 @@ def test_schedule_decode_blocks_to_copy_update():
647
assert len(output.preempted) == 0
648
assert len(output.swapped_out) == 0
649
# Nothing is preempted.
650
- assert output.blocks_to_swap_out == {}
650
+ assert output.blocks_to_swap_out == []
651
# Since append_slot returns the source -> dist mapping, it should
652
# applied.
653
assert output.blocks_to_copy == [(2, 3)]
@@ -658,7 +658,7 @@ def test_schedule_swapped_simple():
658
swapped = deque()
659
policy = PolicyFactory.get_policy(policy_name="fcfs")
660
curr_loras = None
661
- blocks_to_swap_out = {}
661
+ blocks_to_swap_out = []
662
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
663
scheduler._allocate_and_set_running(seq_group)
664
append_new_token_seq_group(60, seq_group, 1)
@@ -674,9 +674,9 @@ def test_schedule_swapped_simple():
674
assert len(output.decode_seq_groups) == 1
675
assert len(output.prefill_seq_groups) == 0
676
# swap in is the reverse of swap out
677
- blocks_to_swap_in_reverse = {}
678
- for swapin, swapout in output.blocks_to_swap_in.items():
679
- blocks_to_swap_in_reverse[swapout] = swapin
677
+ blocks_to_swap_in_reverse = []
678
+ for swapin, swapout in output.blocks_to_swap_in:
679
+ blocks_to_swap_in_reverse.append((swapout, swapin))
680
assert blocks_to_swap_out == blocks_to_swap_in_reverse
681
682
@@ -685,7 +685,7 @@ def test_schedule_swapped_max_token_budget():
685
swapped = deque()
686
policy = PolicyFactory.get_policy(policy_name="fcfs")
687
curr_loras = None
688
- blocks_to_swap_out = {}
688
+ blocks_to_swap_out = []
689
for _ in range(2):
690
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
691
scheduler._allocate_and_set_running(seq_group)
@@ -719,7 +719,7 @@ def test_schedule_swapped_max_seqs():
719
swapped = deque()
720
policy = PolicyFactory.get_policy(policy_name="fcfs")
721
curr_loras = None
722
- blocks_to_swap_out = {}
722
+ blocks_to_swap_out = []
723
for i in range(4):
724
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
725
scheduler._allocate_and_set_running(seq_group)
@@ -752,7 +752,7 @@ def test_schedule_swapped_max_loras():
752
swapped = deque()
753
policy = PolicyFactory.get_policy(policy_name="fcfs")
754
curr_loras = set()
755
- blocks_to_swap_out = {}
755
+ blocks_to_swap_out = []
756
for i in range(2):
757
_, seq_group = create_dummy_prompt(str(i),
758
prompt_length=60,
@@ -781,7 +781,7 @@ def test_schedule_swapped_cannot_swap_in():
781
swapped = deque()
782
policy = PolicyFactory.get_policy(policy_name="fcfs")
783
curr_loras = None
784
- blocks_to_swap_out = {}
784
+ blocks_to_swap_out = []
785
for _ in range(2):
786
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
787
scheduler._allocate_and_set_running(seq_group)
@@ -808,7 +808,7 @@ def test_infeasible_swap():
808
swapped = deque()
809
policy = PolicyFactory.get_policy(policy_name="fcfs")
810
curr_loras = None
811
- blocks_to_swap_out = {}
811
+ blocks_to_swap_out = []
812
for _ in range(2):
813
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
814
scheduler._allocate_and_set_running(seq_group)
@@ -839,7 +839,7 @@ def test_schedule_swapped_blocks_to_copy():
839
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
840
scheduler._allocate_and_set_running(seq_group)
841
append_new_token_seq_group(60, seq_group, 1)
842
- blocks_to_swap_out = {}
842
+ blocks_to_swap_out = []
843
scheduler._swap_out(seq_group, blocks_to_swap_out)
844
swapped.append(seq_group)
845
tests/kernels/test_cache.py
+9
-4
@@ -315,7 +315,10 @@ def test_swap_blocks(
315
else:
316
dst_blocks = random.sample(range(num_blocks), num_mappings)
317
318
- block_mapping = dict(zip(src_blocks, dst_blocks))
318
+ block_mapping = list(zip(src_blocks, dst_blocks))
319
+ block_mapping_tensor = torch.tensor(block_mapping,
320
+ dtype=torch.int64,
321
+ device="cpu").view(-1, 2)
322
323
# Create the KV caches on the first device.
324
src_key_caches, src_value_caches = kv_cache_factory(
@@ -331,10 +334,12 @@ def test_swap_blocks(
334
src_value_caches_clone = src_value_caches[0].clone()
335
336
# Call the swap_blocks kernel.
334
- ops.swap_blocks(src_key_caches[0], dist_key_caches[0], block_mapping)
335
- ops.swap_blocks(src_value_caches[0], dist_value_caches[0], block_mapping)
337
+ ops.swap_blocks(src_key_caches[0], dist_key_caches[0],
338
+ block_mapping_tensor)
339
+ ops.swap_blocks(src_value_caches[0], dist_value_caches[0],
340
+ block_mapping_tensor)
341
337
- for src, dst in block_mapping.items():
342
+ for src, dst in block_mapping:
343
assert torch.allclose(src_key_caches_clone[src].cpu(),
344
dist_key_caches[0][dst].cpu())
345
assert torch.allclose(src_value_caches_clone[src].cpu(),
tests/worker/test_swap.py
+12
-12
@@ -54,10 +54,10 @@ def test_swap() -> None:
54
a.cuda(), b.cuda(), rtol=0.0, atol=0.0)
55
56
# Test swap out.
57
- blocks_to_swap_out = {3: 72, 56: 35, 84: 34}
57
+ blocks_to_swap_out = [(3, 72), (56, 35), (84, 34)]
58
execute_model_req = ExecuteModelRequest(
59
seq_group_metadata_list=[],
60
- blocks_to_swap_in={},
60
+ blocks_to_swap_in=[],
61
blocks_to_swap_out=blocks_to_swap_out,
62
blocks_to_copy=[],
63
)
@@ -66,24 +66,24 @@ def test_swap() -> None:
66
for i in range(num_layers):
67
gpu_key_cache, gpu_value_cache = gpu_cache[i]
68
cpu_key_cache, cpu_value_cache = cpu_cache[i]
69
- for src, dst in blocks_to_swap_out.items():
69
+ for src, dst in blocks_to_swap_out:
70
assert allclose(gpu_key_cache[src], cpu_key_cache[dst])
71
assert allclose(gpu_value_cache[src], cpu_value_cache[dst])
72
73
# Test swap in.
74
- execute_model_req.blocks_to_swap_out = {}
75
- execute_model_req.blocks_to_swap_in = {
76
- 19: 45,
77
- 67: 23,
78
- 12: 78,
79
- 40: 99,
80
- 1: 71
81
- }
74
+ execute_model_req.blocks_to_swap_out = []
75
+ execute_model_req.blocks_to_swap_in = [
76
+ (19, 45),
77
+ (67, 23),
78
+ (12, 78),
79
+ (40, 99),
80
+ (1, 71),
81
+ ]
82
worker.execute_model(execute_model_req=execute_model_req)
83
84
for i in range(num_layers):
85
gpu_key_cache, gpu_value_cache = gpu_cache[i]
86
cpu_key_cache, cpu_value_cache = cpu_cache[i]
87
- for src, dst in execute_model_req.blocks_to_swap_in.items():
87
+ for src, dst in execute_model_req.blocks_to_swap_in:
88
assert allclose(gpu_key_cache[dst], cpu_key_cache[src])
89
assert allclose(gpu_value_cache[dst], cpu_value_cache[src])
vllm/attention/backends/abstract.py
+1
-1
@@ -39,7 +39,7 @@ class AttentionBackend(ABC):
39
def swap_blocks(
40
src_kv_cache: torch.Tensor,
41
dst_kv_cache: torch.Tensor,
42
- src_to_dst: Dict[int, int],
42
+ src_to_dst: torch.Tensor,
43
) -> None:
44
raise NotImplementedError
45
vllm/attention/backends/flash_attn.py
+2
-2
@@ -5,7 +5,7 @@ XFormers backend. The duplicated code will be removed once we use flash-attn or
5
flashinfer for all the attention operations.
6
"""
7
from dataclasses import dataclass
8
-from typing import Dict, List, Optional, Tuple, Type
8
+from typing import List, Optional, Tuple, Type
9
10
import torch
11
from flash_attn import flash_attn_varlen_func
@@ -45,7 +45,7 @@ class FlashAttentionBackend(AttentionBackend):
45
def swap_blocks(
46
src_kv_cache: torch.Tensor,
47
dst_kv_cache: torch.Tensor,
48
- src_to_dst: Dict[int, int],
48
+ src_to_dst: torch.Tensor,
49
) -> None:
50
PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
51
vllm/attention/backends/flashinfer.py
+1
-1
@@ -39,7 +39,7 @@ class FlashInferBackend(AttentionBackend):
39
def swap_blocks(
40
src_kv_cache: torch.Tensor,
41
dst_kv_cache: torch.Tensor,
42
- src_to_dst: Dict[int, int],
42
+ src_to_dst: torch.Tensor,
43
) -> None:
44
raise NotImplementedError
45
vllm/attention/backends/rocm_flash_attn.py
+2
-2
@@ -1,6 +1,6 @@
1
"""Attention layer ROCm GPUs."""
2
from dataclasses import dataclass
3
-from typing import Dict, List, Optional, Tuple, Type
3
+from typing import List, Optional, Tuple, Type
4
5
import torch
6
@@ -43,7 +43,7 @@ class ROCmFlashAttentionBackend(AttentionBackend):
43
def swap_blocks(
44
src_kv_cache: torch.Tensor,
45
dst_kv_cache: torch.Tensor,
46
- src_to_dst: Dict[int, int],
46
+ src_to_dst: torch.Tensor,
47
) -> None:
48
PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
49
vllm/attention/backends/torch_sdpa.py
+2
-2
@@ -1,7 +1,7 @@
1
""" Attention layer with torch scaled_dot_product_attention
2
and PagedAttention."""
3
from dataclasses import dataclass
4
-from typing import Dict, List, Optional, Tuple, Type
4
+from typing import List, Optional, Tuple, Type
5
6
import torch
7
from torch.nn.functional import scaled_dot_product_attention
@@ -41,7 +41,7 @@ class TorchSDPABackend(AttentionBackend):
41
def swap_blocks(
42
src_kv_cache: torch.Tensor,
43
dst_kv_cache: torch.Tensor,
44
- src_to_dst: Dict[int, int],
44
+ src_to_dst: torch.Tensor,
45
) -> None:
46
PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
47
vllm/attention/ops/paged_attn.py
+2
-2
@@ -1,5 +1,5 @@
1
from dataclasses import dataclass
2
-from typing import Dict, List, Optional, Tuple
2
+from typing import List, Optional, Tuple
3
4
import torch
5
@@ -196,7 +196,7 @@ class PagedAttention:
196
def swap_blocks(
197
src_kv_cache: torch.Tensor,
198
dst_kv_cache: torch.Tensor,
199
- src_to_dst: Dict[int, int],
199
+ src_to_dst: torch.Tensor,
200
) -> None:
201
src_key_cache = src_kv_cache[0]
202
dst_key_cache = dst_kv_cache[0]
vllm/core/block_manager_v1.py
+8
-4
@@ -473,11 +473,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
473
474
def swap_in(self,
475
seq_group: SequenceGroup,
476
- num_lookahead_slots: int = 0) -> Dict[int, int]:
476
+ num_lookahead_slots: int = 0) -> List[Tuple[int, int]]:
477
assert (num_lookahead_slots == 0
478
), "BlockSpaceManagerV1 does not support lookahead allocation"
479
480
# CPU block -> GPU block.
481
+ # dict is efficient in lookup `if cpu_block in mapping`
482
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
483
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
484
new_block_table: BlockTable = []
@@ -500,14 +501,16 @@ class BlockSpaceManagerV1(BlockSpaceManager):
501
cpu_block.block_number: gpu_block.block_number
502
for cpu_block, gpu_block in mapping.items()
503
}
503
- return block_number_mapping
504
+ # convert to list of tuples once here
505
+ return list(block_number_mapping.items())
506
507
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
508
blocks = self._get_physical_blocks(seq_group)
509
return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
510
509
- def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
511
+ def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
512
# GPU block -> CPU block.
513
+ # dict is efficient in lookup `if gpu_block in mapping`
514
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
515
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
516
new_block_table: BlockTable = []
@@ -530,7 +533,8 @@ class BlockSpaceManagerV1(BlockSpaceManager):
533
gpu_block.block_number: cpu_block.block_number
534
for gpu_block, cpu_block in mapping.items()
535
}
533
- return block_number_mapping
536
+ # convert to list of tuples once here
537
+ return list(block_number_mapping.items())
538
539
def _free_block_table(self, block_table: BlockTable) -> None:
540
# when using a sliding window, each seq will only use up
vllm/core/block_manager_v2.py
+2
-2
@@ -243,13 +243,13 @@ class BlockSpaceManagerV2(BlockSpaceManager):
243
return AllocStatus.LATER
244
245
def swap_in(self, seq_group: SequenceGroup,
246
- num_lookahead_slots: int) -> Dict[int, int]:
246
+ num_lookahead_slots: int) -> List[Tuple[int, int]]:
247
raise NotImplementedError
248
249
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
250
return False
251
252
- def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
252
+ def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
253
raise NotImplementedError
254
255
def get_num_free_gpu_blocks(self) -> int:
vllm/core/interfaces.py
+3
-3
@@ -1,6 +1,6 @@
1
import enum
2
from abc import ABC, abstractmethod
3
-from typing import Dict, List
3
+from typing import List
4
from typing import Sequence as GenericSequence
5
from typing import Tuple
6
@@ -69,7 +69,7 @@ class BlockSpaceManager(ABC):
69
70
@abstractmethod
71
def swap_in(self, seq_group: SequenceGroup,
72
- num_lookahead_slots: int) -> Dict[int, int]:
72
+ num_lookahead_slots: int) -> List[Tuple[int, int]]:
73
pass
74
75
@abstractmethod
@@ -77,7 +77,7 @@ class BlockSpaceManager(ABC):
77
pass
78
79
@abstractmethod
80
- def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
80
+ def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
81
pass
82
83
@abstractmethod
vllm/core/scheduler.py
+16
-16
@@ -117,10 +117,10 @@ class SchedulerOutputs:
117
num_prefill_groups: int
118
# Total number of batched tokens.
119
num_batched_tokens: int
120
- # Blocks to swap in. Dict of CPU -> GPU block number.
121
- blocks_to_swap_in: Dict[int, int]
122
- # Blocks to swap out. Dict of GPU -> CPU block number.
123
- blocks_to_swap_out: Dict[int, int]
120
+ # Blocks to swap in. List of CPU -> GPU block number.
121
+ blocks_to_swap_in: List[Tuple[int, int]]
122
+ # Blocks to swap out. List of GPU -> CPU block number.
123
+ blocks_to_swap_out: List[Tuple[int, int]]
124
# Blocks to copy. Source to dest block.
125
blocks_to_copy: List[Tuple[int, int]]
126
# Sequence groups that are going to be ignored.
@@ -174,7 +174,7 @@ class SchedulerRunningOutputs:
174
# Sequences that are swapped out.
175
swapped_out: List[SequenceGroup]
176
# The blocks to swap out.
177
- blocks_to_swap_out: Dict[int, int]
177
+ blocks_to_swap_out: List[Tuple[int, int]]
178
# The blocks to copy.
179
blocks_to_copy: List[Tuple[int, int]]
180
# The number of slots for lookahead decoding.
@@ -187,7 +187,7 @@ class SchedulerRunningOutputs:
187
prefill_seq_groups=[],
188
preempted=[],
189
swapped_out=[],
190
- blocks_to_swap_out={},
190
+ blocks_to_swap_out=[],
191
blocks_to_copy=[],
192
num_lookahead_slots=0,
193
)
@@ -206,7 +206,7 @@ class SchedulerSwappedInOutputs:
206
# phase. I.e., it means the prefill has been chunked.
207
prefill_seq_groups: List[SequenceGroup]
208
# The blocks to swap in.
209
- blocks_to_swap_in: Dict[int, int]
209
+ blocks_to_swap_in: List[Tuple[int, int]]
210
# The blocks to copy.
211
blocks_to_copy: List[Tuple[int, int]]
212
# The number of slots for lookahead decoding.
@@ -219,7 +219,7 @@ class SchedulerSwappedInOutputs:
219
return SchedulerSwappedInOutputs(
220
decode_seq_groups=[],
221
prefill_seq_groups=[],
222
- blocks_to_swap_in={},
222
+ blocks_to_swap_in=[],
223
blocks_to_copy=[],
224
num_lookahead_slots=0,
225
infeasible_seq_groups=[],
@@ -392,7 +392,7 @@ class Scheduler:
392
scheduling and SchedulerRunningOutputs.
393
"""
394
# Blocks that need to be swapped or copied before model execution.
395
- blocks_to_swap_out: Dict[int, int] = {}
395
+ blocks_to_swap_out: List[Tuple[int, int]] = []
396
blocks_to_copy: List[Tuple[int, int]] = []
397
398
decode_seq_groups: List[ScheduledSequenceGroup] = []
@@ -509,7 +509,7 @@ class Scheduler:
509
SchedulerSwappedInOutputs.
510
"""
511
# Blocks that need to be swapped or copied before model execution.
512
- blocks_to_swap_in: Dict[int, int] = {}
512
+ blocks_to_swap_in: List[Tuple[int, int]] = []
513
blocks_to_copy: List[Tuple[int, int]] = []
514
decode_seq_groups: List[ScheduledSequenceGroup] = []
515
prefill_seq_groups: List[ScheduledSequenceGroup] = []
@@ -1032,7 +1032,7 @@ class Scheduler:
1032
def _preempt(
1033
self,
1034
seq_group: SequenceGroup,
1035
- blocks_to_swap_out: Dict[int, int],
1035
+ blocks_to_swap_out: List[Tuple[int, int]],
1036
preemption_mode: Optional[PreemptionMode] = None,
1037
) -> PreemptionMode:
1038
# If preemption mode is not specified, we determine the mode as follows:
@@ -1073,24 +1073,24 @@ class Scheduler:
1073
def _preempt_by_swap(
1074
self,
1075
seq_group: SequenceGroup,
1076
- blocks_to_swap_out: Dict[int, int],
1076
+ blocks_to_swap_out: List[Tuple[int, int]],
1077
) -> None:
1078
self._swap_out(seq_group, blocks_to_swap_out)
1079
1080
def _swap_in(
1081
self,
1082
seq_group: SequenceGroup,
1083
- blocks_to_swap_in: Dict[int, int],
1083
+ blocks_to_swap_in: List[Tuple[int, int]],
1084
) -> None:
1085
mapping = self.block_manager.swap_in(seq_group)
1086
- blocks_to_swap_in.update(mapping)
1086
+ blocks_to_swap_in.extend(mapping)
1087
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
1088
seq.status = SequenceStatus.RUNNING
1089
1090
def _swap_out(
1091
self,
1092
seq_group: SequenceGroup,
1093
- blocks_to_swap_out: Dict[int, int],
1093
+ blocks_to_swap_out: List[Tuple[int, int]],
1094
) -> None:
1095
if not self.block_manager.can_swap_out(seq_group):
1096
# FIXME(woosuk): Abort the sequence group instead of aborting the
@@ -1099,7 +1099,7 @@ class Scheduler:
1099
"Aborted due to the lack of CPU swap space. Please increase "
1100
"the swap space to avoid this error.")
1101
mapping = self.block_manager.swap_out(seq_group)
1102
- blocks_to_swap_out.update(mapping)
1102
+ blocks_to_swap_out.extend(mapping)
1103
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
1104
seq.status = SequenceStatus.SWAPPED
1105
vllm/sequence.py
+4
-4
@@ -741,10 +741,10 @@ class ExecuteModelRequest:
741
"""The model execution request."""
742
# The sequence group metadata list.
743
seq_group_metadata_list: List[SequenceGroupMetadata]
744
- # Blocks to swap in. Dict of CPU -> GPU block number.
745
- blocks_to_swap_in: Dict[int, int] = field(default_factory=dict)
746
- # Blocks to swap out. Dict of GPU -> CPU block number.
747
- blocks_to_swap_out: Dict[int, int] = field(default_factory=dict)
744
+ # Blocks to swap in. List of CPU -> GPU block number.
745
+ blocks_to_swap_in: List[Tuple[int, int]] = field(default_factory=list)
746
+ # Blocks to swap out. List of GPU -> CPU block number.
747
+ blocks_to_swap_out: List[Tuple[int, int]] = field(default_factory=list)
748
# Blocks to copy. Source to dest block.
749
blocks_to_copy: List[Tuple[int, int]] = field(default_factory=list)
750
# The number of slots for lookahead decoding.
vllm/worker/cache_engine.py
+3
-3
@@ -1,5 +1,5 @@
1
"""CacheEngine class for managing the KV cache."""
2
-from typing import Dict, List
2
+from typing import List
3
4
import torch
5
@@ -67,12 +67,12 @@ class CacheEngine:
67
device=device))
68
return kv_cache
69
70
- def swap_in(self, src_to_dst: Dict[int, int]) -> None:
70
+ def swap_in(self, src_to_dst: torch.Tensor) -> None:
71
for i in range(self.num_layers):
72
self.attn_backend.swap_blocks(self.cpu_cache[i], self.gpu_cache[i],
73
src_to_dst)
74
75
- def swap_out(self, src_to_dst: Dict[int, int]) -> None:
75
+ def swap_out(self, src_to_dst: torch.Tensor) -> None:
76
for i in range(self.num_layers):
77
self.attn_backend.swap_blocks(self.gpu_cache[i], self.cpu_cache[i],
78
src_to_dst)
vllm/worker/worker.py
+20
-7
@@ -195,15 +195,14 @@ class Worker(WorkerBase):
195
196
def cache_swap(
197
self,
198
- blocks_to_swap_in: Dict[int, int],
199
- blocks_to_swap_out: Dict[int, int],
198
+ blocks_to_swap_in: torch.Tensor,
199
+ blocks_to_swap_out: torch.Tensor,
200
blocks_to_copy: torch.Tensor,
201
) -> None:
202
# Issue cache operations.
203
- # TODO(woosuk): Profile swapping overhead and optimize if needed.
204
- if blocks_to_swap_in:
203
+ if blocks_to_swap_in.numel() > 0:
204
self.cache_engine.swap_in(blocks_to_swap_in)
206
- if blocks_to_swap_out:
205
+ if blocks_to_swap_out.numel() > 0:
206
self.cache_engine.swap_out(blocks_to_swap_out)
207
if blocks_to_copy.numel() > 0:
208
self.cache_engine.copy(blocks_to_copy)
@@ -219,12 +218,26 @@ class Worker(WorkerBase):
218
else:
219
seq_group_metadata_list = execute_model_req.seq_group_metadata_list
220
221
+ blocks_to_swap_in: torch.Tensor
222
+ blocks_to_swap_out: torch.Tensor
223
+ blocks_to_copy: torch.Tensor
224
if self.is_driver_worker:
225
assert seq_group_metadata_list is not None
226
assert execute_model_req is not None
227
num_seq_groups = len(seq_group_metadata_list)
226
- blocks_to_swap_in = execute_model_req.blocks_to_swap_in
227
- blocks_to_swap_out = execute_model_req.blocks_to_swap_out
228
+ # `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
229
+ # they contain parameters to launch cudamemcpyasync.
230
+ blocks_to_swap_in = torch.tensor(
231
+ execute_model_req.blocks_to_swap_in,
232
+ device="cpu",
233
+ dtype=torch.int64).view(-1, 2)
234
+ blocks_to_swap_out = torch.tensor(
235
+ execute_model_req.blocks_to_swap_out,
236
+ device="cpu",
237
+ dtype=torch.int64).view(-1, 2)
238
+ # `blocks_to_copy` is a gpu tensor. The src and tgt of
239
+ # blocks to copy are in the same device, and `blocks_to_copy`
240
+ # can be used directly within cuda kernels.
241
blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
242
device=self.device,
243
dtype=torch.int64).view(-1, 2)