A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Input Batch Relocation (#10962)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Dec 9, 2024 at 12:33 UTC
25b79d9fd38e2c53ce281be23241d8939ec7320c
2 files changed
+283
-270
vllm/v1/worker/gpu_input_batch.py
new
+280
@@ -0,0 +1,280 @@
1
+# Datastructures defining an input batch
2
+
3
+from dataclasses import dataclass
4
+from typing import TYPE_CHECKING, Dict, List, Optional, Set
5
+
6
+import numpy as np
7
+import torch
8
+
9
+from vllm.multimodal import MultiModalKwargs
10
+from vllm.sampling_params import SamplingParams, SamplingType
11
+from vllm.v1.sample.metadata import SamplingMetadata
12
+
13
+if TYPE_CHECKING:
14
+ from vllm.multimodal.inputs import PlaceholderRange
15
+
16
+
17
+@dataclass
18
+class CachedRequestState:
19
+
20
+ req_id: str
21
+ prompt_token_ids: List[int]
22
+ prompt: Optional[str]
23
+ mm_inputs: List[MultiModalKwargs]
24
+ mm_positions: List["PlaceholderRange"]
25
+ sampling_params: SamplingParams
26
+ generator: Optional[torch.Generator]
27
+
28
+ block_ids: List[int]
29
+ num_computed_tokens: int
30
+ output_token_ids: List[int]
31
+
32
+ @property
33
+ def num_tokens(self) -> int:
34
+ return len(self.prompt_token_ids) + len(self.output_token_ids)
35
+
36
+
37
+class InputBatch:
38
+
39
+ def __init__(
40
+ self,
41
+ max_num_reqs: int,
42
+ max_model_len: int,
43
+ max_num_blocks_per_req: int,
44
+ device: torch.device,
45
+ pin_memory: bool,
46
+ ):
47
+ self.max_num_reqs = max_num_reqs
48
+ self.max_model_len = max_model_len
49
+ self.max_num_blocks_per_req = max_num_blocks_per_req
50
+ self.device = device
51
+ self.pin_memory = pin_memory
52
+
53
+ self.req_ids: List[Optional[str]] = [None] * max_num_reqs
54
+ self.req_id_to_index: Dict[str, int] = {}
55
+
56
+ self.token_ids_cpu = np.empty((max_num_reqs, max_model_len),
57
+ dtype=np.int32)
58
+ self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
59
+
60
+ # Attention-related.
61
+ self.block_table = torch.zeros((max_num_reqs, max_num_blocks_per_req),
62
+ device=self.device,
63
+ dtype=torch.int32)
64
+ self.block_table_cpu_tensor = torch.zeros(
65
+ (max_num_reqs, max_num_blocks_per_req),
66
+ device="cpu",
67
+ dtype=torch.int32,
68
+ pin_memory=pin_memory,
69
+ )
70
+ self.block_table_cpu = self.block_table_cpu_tensor.numpy()
71
+
72
+ # Sampling-related.
73
+ self.temperature = torch.empty((max_num_reqs, ),
74
+ dtype=torch.float32,
75
+ device=device)
76
+ self.temperature_cpu_tensor = torch.empty((max_num_reqs, ),
77
+ dtype=torch.float32,
78
+ device="cpu",
79
+ pin_memory=pin_memory)
80
+ self.temperature_cpu = self.temperature_cpu_tensor.numpy()
81
+ self.greedy_reqs: Set[str] = set()
82
+ self.random_reqs: Set[str] = set()
83
+
84
+ self.top_p = torch.empty((max_num_reqs, ),
85
+ dtype=torch.float32,
86
+ device=device)
87
+ self.top_p_cpu_tensor = torch.empty((max_num_reqs, ),
88
+ dtype=torch.float32,
89
+ device="cpu",
90
+ pin_memory=pin_memory)
91
+ self.top_p_cpu = self.top_p_cpu_tensor.numpy()
92
+ self.top_p_reqs: Set[str] = set()
93
+
94
+ self.top_k = torch.empty((max_num_reqs, ),
95
+ dtype=torch.int32,
96
+ device=device)
97
+ self.top_k_cpu_tensor = torch.empty((max_num_reqs, ),
98
+ dtype=torch.int32,
99
+ device="cpu",
100
+ pin_memory=pin_memory)
101
+ self.top_k_cpu = self.top_k_cpu_tensor.numpy()
102
+ self.top_k_reqs: Set[str] = set()
103
+
104
+ # req_index -> generator
105
+ self.generators: Dict[int, torch.Generator] = {}
106
+
107
+ self.num_logprobs: Dict[str, int] = {}
108
+ self.prompt_logprob_reqs: Set[str] = set()
109
+
110
+ def add_request(
111
+ self,
112
+ request: "CachedRequestState",
113
+ req_index: Optional[int] = None,
114
+ ) -> None:
115
+ if req_index is None:
116
+ req_index = self.num_reqs
117
+ assert req_index < self.max_num_reqs
118
+
119
+ req_id = request.req_id
120
+ self.req_ids[req_index] = req_id
121
+ self.req_id_to_index[req_id] = req_index
122
+
123
+ # Copy the prompt token ids and output token ids.
124
+ num_prompt_tokens = len(request.prompt_token_ids)
125
+ self.token_ids_cpu[
126
+ req_index, :num_prompt_tokens] = request.prompt_token_ids
127
+ start_idx = num_prompt_tokens
128
+ end_idx = start_idx + len(request.output_token_ids)
129
+ self.token_ids_cpu[req_index,
130
+ start_idx:end_idx] = request.output_token_ids
131
+
132
+ self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens
133
+ num_blocks = len(request.block_ids)
134
+ self.block_table_cpu[req_index, :num_blocks] = request.block_ids
135
+
136
+ sampling_params = request.sampling_params
137
+ self.temperature_cpu[req_index] = sampling_params.temperature
138
+ if sampling_params.sampling_type == SamplingType.GREEDY:
139
+ self.greedy_reqs.add(req_id)
140
+ else:
141
+ self.random_reqs.add(req_id)
142
+
143
+ self.top_p_cpu[req_index] = sampling_params.top_p
144
+ if sampling_params.top_p < 1:
145
+ self.top_p_reqs.add(req_id)
146
+ self.top_k_cpu[req_index] = sampling_params.top_k
147
+ if sampling_params.top_k > 0:
148
+ self.top_k_reqs.add(req_id)
149
+
150
+ self.generators[req_index] = request.generator
151
+
152
+ num_logprobs = sampling_params.logprobs
153
+ if num_logprobs is not None and num_logprobs > 0:
154
+ self.num_logprobs[req_id] = num_logprobs
155
+ if sampling_params.prompt_logprobs:
156
+ self.prompt_logprob_reqs.add(req_id)
157
+
158
+ def remove_request(self, req_id: str) -> Optional[int]:
159
+ req_index = self.req_id_to_index.pop(req_id, None)
160
+ if req_index is None:
161
+ return None
162
+ self.req_ids[req_index] = None
163
+
164
+ self.greedy_reqs.discard(req_id)
165
+ self.random_reqs.discard(req_id)
166
+ self.top_p_reqs.discard(req_id)
167
+ self.top_k_reqs.discard(req_id)
168
+ self.generators.pop(req_index, None)
169
+ self.num_logprobs.pop(req_id, None)
170
+ self.prompt_logprob_reqs.discard(req_id)
171
+ return req_index
172
+
173
+ def clear(self) -> None:
174
+ self.req_ids = [None] * self.max_num_reqs
175
+ self.req_id_to_index.clear()
176
+ self.greedy_reqs.clear()
177
+ self.random_reqs.clear()
178
+ self.top_p_reqs.clear()
179
+ self.top_k_reqs.clear()
180
+ self.generators.clear()
181
+ self.num_logprobs.clear()
182
+ self.prompt_logprob_reqs.clear()
183
+
184
+ def condense(self, empty_req_indices: List[int]) -> None:
185
+ if self.num_reqs == 0:
186
+ # The batched states are empty.
187
+ return
188
+
189
+ # NOTE(woosuk): This function assumes that the empty_req_indices
190
+ # is sorted in descending order.
191
+ last_req_index = self.num_reqs + len(empty_req_indices) - 1
192
+ while empty_req_indices:
193
+ # Find the largest non-empty index.
194
+ while last_req_index in empty_req_indices:
195
+ last_req_index -= 1
196
+
197
+ # Find the smallest empty index.
198
+ empty_index = empty_req_indices.pop()
199
+ if empty_index >= last_req_index:
200
+ break
201
+
202
+ # Swap the states.
203
+ req_id = self.req_ids[last_req_index]
204
+ self.req_ids[empty_index] = req_id
205
+ self.req_ids[last_req_index] = None
206
+ self.req_id_to_index[req_id] = empty_index
207
+
208
+ # TODO(woosuk): Optimize the copy of token_ids_cpu and
209
+ # block_table_cpu.
210
+ self.token_ids_cpu[empty_index] = self.token_ids_cpu[
211
+ last_req_index]
212
+ self.num_computed_tokens_cpu[
213
+ empty_index] = self.num_computed_tokens_cpu[last_req_index]
214
+ self.block_table_cpu[empty_index] = self.block_table_cpu[
215
+ last_req_index]
216
+ self.temperature_cpu[empty_index] = self.temperature_cpu[
217
+ last_req_index]
218
+ self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
219
+ self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
220
+ generator = self.generators.pop(last_req_index, None)
221
+ if generator is not None:
222
+ self.generators[empty_index] = generator
223
+
224
+ # Decrement last_req_index since it is now empty.
225
+ last_req_index -= 1
226
+
227
+ def make_sampling_metadata(
228
+ self,
229
+ skip_copy: bool = False,
230
+ ) -> SamplingMetadata:
231
+ if not skip_copy:
232
+ self.temperature[:self.num_reqs].copy_(
233
+ self.temperature_cpu_tensor[:self.num_reqs], non_blocking=True)
234
+ self.top_p[:self.num_reqs].copy_(
235
+ self.top_p_cpu_tensor[:self.num_reqs], non_blocking=True)
236
+ self.top_k[:self.num_reqs].copy_(
237
+ self.top_k_cpu_tensor[:self.num_reqs], non_blocking=True)
238
+ return SamplingMetadata(
239
+ temperature=self.temperature[:self.num_reqs],
240
+ all_greedy=self.all_greedy,
241
+ all_random=self.all_random,
242
+ top_p=self.top_p[:self.num_reqs],
243
+ top_k=self.top_k[:self.num_reqs],
244
+ no_top_p=self.no_top_p,
245
+ no_top_k=self.no_top_k,
246
+ generators=self.generators,
247
+ max_num_logprobs=self.max_num_logprobs,
248
+ )
249
+
250
+ @property
251
+ def num_reqs(self) -> int:
252
+ return len(self.req_id_to_index)
253
+
254
+ @property
255
+ def all_greedy(self) -> bool:
256
+ return len(self.random_reqs) == 0
257
+
258
+ @property
259
+ def all_random(self) -> bool:
260
+ return len(self.greedy_reqs) == 0
261
+
262
+ @property
263
+ def no_top_p(self) -> bool:
264
+ return len(self.top_p_reqs) == 0
265
+
266
+ @property
267
+ def no_top_k(self) -> bool:
268
+ return len(self.top_k_reqs) == 0
269
+
270
+ @property
271
+ def max_num_logprobs(self) -> int:
272
+ return max(self.num_logprobs.values()) if self.num_logprobs else 0
273
+
274
+ @property
275
+ def no_logprob(self) -> bool:
276
+ return len(self.num_logprobs) == 0
277
+
278
+ @property
279
+ def no_prompt_logprob(self) -> bool:
280
+ return len(self.prompt_logprob_reqs) == 0
vllm/v1/worker/gpu_model_runner.py
+3
-270
@@ -1,7 +1,6 @@
1
import gc
2
import time
3
-from dataclasses import dataclass
4
-from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
3
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
4
5
import numpy as np
6
import torch
@@ -15,16 +14,16 @@ from vllm.inputs import INPUT_REGISTRY, InputRegistry
14
from vllm.logger import init_logger
15
from vllm.model_executor.model_loader import get_model
16
from vllm.multimodal import MultiModalKwargs
18
-from vllm.sampling_params import SamplingParams, SamplingType
17
+from vllm.sampling_params import SamplingType
18
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
19
is_pin_memory_available)
20
from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
21
FlashAttentionMetadata)
22
from vllm.v1.outputs import ModelRunnerOutput
23
from vllm.v1.sample.metadata import SamplingMetadata
24
+from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
25
26
if TYPE_CHECKING:
27
- from vllm.multimodal.inputs import PlaceholderRange
27
from vllm.v1.core.scheduler import SchedulerOutput
28
29
logger = init_logger(__name__)
@@ -609,269 +608,3 @@ class GPUModelRunner:
608
if batch_size <= size:
609
return size
610
return None
612
-
613
-
614
-@dataclass
615
-class CachedRequestState:
616
-
617
- req_id: str
618
- prompt_token_ids: List[int]
619
- prompt: Optional[str]
620
- mm_inputs: List[MultiModalKwargs]
621
- mm_positions: List["PlaceholderRange"]
622
- sampling_params: SamplingParams
623
- generator: Optional[torch.Generator]
624
-
625
- block_ids: List[int]
626
- num_computed_tokens: int
627
- output_token_ids: List[int]
628
-
629
- @property
630
- def num_tokens(self) -> int:
631
- return len(self.prompt_token_ids) + len(self.output_token_ids)
632
-
633
-
634
-class InputBatch:
635
-
636
- def __init__(
637
- self,
638
- max_num_reqs: int,
639
- max_model_len: int,
640
- max_num_blocks_per_req: int,
641
- device: torch.device,
642
- pin_memory: bool,
643
- ):
644
- self.max_num_reqs = max_num_reqs
645
- self.max_model_len = max_model_len
646
- self.max_num_blocks_per_req = max_num_blocks_per_req
647
- self.device = device
648
- self.pin_memory = pin_memory
649
-
650
- self.req_ids: List[Optional[str]] = [None] * max_num_reqs
651
- self.req_id_to_index: Dict[str, int] = {}
652
-
653
- self.token_ids_cpu = np.empty((max_num_reqs, max_model_len),
654
- dtype=np.int32)
655
- self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
656
-
657
- # Attention-related.
658
- self.block_table = torch.zeros((max_num_reqs, max_num_blocks_per_req),
659
- device=self.device,
660
- dtype=torch.int32)
661
- self.block_table_cpu_tensor = torch.zeros(
662
- (max_num_reqs, max_num_blocks_per_req),
663
- device="cpu",
664
- dtype=torch.int32,
665
- pin_memory=pin_memory,
666
- )
667
- self.block_table_cpu = self.block_table_cpu_tensor.numpy()
668
-
669
- # Sampling-related.
670
- self.temperature = torch.empty((max_num_reqs, ),
671
- dtype=torch.float32,
672
- device=device)
673
- self.temperature_cpu_tensor = torch.empty((max_num_reqs, ),
674
- dtype=torch.float32,
675
- device="cpu",
676
- pin_memory=pin_memory)
677
- self.temperature_cpu = self.temperature_cpu_tensor.numpy()
678
- self.greedy_reqs: Set[str] = set()
679
- self.random_reqs: Set[str] = set()
680
-
681
- self.top_p = torch.empty((max_num_reqs, ),
682
- dtype=torch.float32,
683
- device=device)
684
- self.top_p_cpu_tensor = torch.empty((max_num_reqs, ),
685
- dtype=torch.float32,
686
- device="cpu",
687
- pin_memory=pin_memory)
688
- self.top_p_cpu = self.top_p_cpu_tensor.numpy()
689
- self.top_p_reqs: Set[str] = set()
690
-
691
- self.top_k = torch.empty((max_num_reqs, ),
692
- dtype=torch.int32,
693
- device=device)
694
- self.top_k_cpu_tensor = torch.empty((max_num_reqs, ),
695
- dtype=torch.int32,
696
- device="cpu",
697
- pin_memory=pin_memory)
698
- self.top_k_cpu = self.top_k_cpu_tensor.numpy()
699
- self.top_k_reqs: Set[str] = set()
700
-
701
- # req_index -> generator
702
- self.generators: Dict[int, torch.Generator] = {}
703
-
704
- self.num_logprobs: Dict[str, int] = {}
705
- self.prompt_logprob_reqs: Set[str] = set()
706
-
707
- def add_request(
708
- self,
709
- request: "CachedRequestState",
710
- req_index: Optional[int] = None,
711
- ) -> None:
712
- if req_index is None:
713
- req_index = self.num_reqs
714
- assert req_index < self.max_num_reqs
715
-
716
- req_id = request.req_id
717
- self.req_ids[req_index] = req_id
718
- self.req_id_to_index[req_id] = req_index
719
-
720
- # Copy the prompt token ids and output token ids.
721
- num_prompt_tokens = len(request.prompt_token_ids)
722
- self.token_ids_cpu[
723
- req_index, :num_prompt_tokens] = request.prompt_token_ids
724
- start_idx = num_prompt_tokens
725
- end_idx = start_idx + len(request.output_token_ids)
726
- self.token_ids_cpu[req_index,
727
- start_idx:end_idx] = request.output_token_ids
728
-
729
- self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens
730
- num_blocks = len(request.block_ids)
731
- self.block_table_cpu[req_index, :num_blocks] = request.block_ids
732
-
733
- sampling_params = request.sampling_params
734
- self.temperature_cpu[req_index] = sampling_params.temperature
735
- if sampling_params.sampling_type == SamplingType.GREEDY:
736
- self.greedy_reqs.add(req_id)
737
- else:
738
- self.random_reqs.add(req_id)
739
-
740
- self.top_p_cpu[req_index] = sampling_params.top_p
741
- if sampling_params.top_p < 1:
742
- self.top_p_reqs.add(req_id)
743
- self.top_k_cpu[req_index] = sampling_params.top_k
744
- if sampling_params.top_k > 0:
745
- self.top_k_reqs.add(req_id)
746
-
747
- self.generators[req_index] = request.generator
748
-
749
- num_logprobs = sampling_params.logprobs
750
- if num_logprobs is not None and num_logprobs > 0:
751
- self.num_logprobs[req_id] = num_logprobs
752
- if sampling_params.prompt_logprobs:
753
- self.prompt_logprob_reqs.add(req_id)
754
-
755
- def remove_request(self, req_id: str) -> Optional[int]:
756
- req_index = self.req_id_to_index.pop(req_id, None)
757
- if req_index is None:
758
- return None
759
- self.req_ids[req_index] = None
760
-
761
- self.greedy_reqs.discard(req_id)
762
- self.random_reqs.discard(req_id)
763
- self.top_p_reqs.discard(req_id)
764
- self.top_k_reqs.discard(req_id)
765
- self.generators.pop(req_index, None)
766
- self.num_logprobs.pop(req_id, None)
767
- self.prompt_logprob_reqs.discard(req_id)
768
- return req_index
769
-
770
- def clear(self) -> None:
771
- self.req_ids = [None] * self.max_num_reqs
772
- self.req_id_to_index.clear()
773
- self.greedy_reqs.clear()
774
- self.random_reqs.clear()
775
- self.top_p_reqs.clear()
776
- self.top_k_reqs.clear()
777
- self.generators.clear()
778
- self.num_logprobs.clear()
779
- self.prompt_logprob_reqs.clear()
780
-
781
- def condense(self, empty_req_indices: List[int]) -> None:
782
- if self.num_reqs == 0:
783
- # The batched states are empty.
784
- return
785
-
786
- # NOTE(woosuk): This function assumes that the empty_req_indices
787
- # is sorted in descending order.
788
- last_req_index = self.num_reqs + len(empty_req_indices) - 1
789
- while empty_req_indices:
790
- # Find the largest non-empty index.
791
- while last_req_index in empty_req_indices:
792
- last_req_index -= 1
793
-
794
- # Find the smallest empty index.
795
- empty_index = empty_req_indices.pop()
796
- if empty_index >= last_req_index:
797
- break
798
-
799
- # Swap the states.
800
- req_id = self.req_ids[last_req_index]
801
- self.req_ids[empty_index] = req_id
802
- self.req_ids[last_req_index] = None
803
- self.req_id_to_index[req_id] = empty_index
804
-
805
- # TODO(woosuk): Optimize the copy of token_ids_cpu and
806
- # block_table_cpu.
807
- self.token_ids_cpu[empty_index] = self.token_ids_cpu[
808
- last_req_index]
809
- self.num_computed_tokens_cpu[
810
- empty_index] = self.num_computed_tokens_cpu[last_req_index]
811
- self.block_table_cpu[empty_index] = self.block_table_cpu[
812
- last_req_index]
813
- self.temperature_cpu[empty_index] = self.temperature_cpu[
814
- last_req_index]
815
- self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
816
- self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
817
- generator = self.generators.pop(last_req_index, None)
818
- if generator is not None:
819
- self.generators[empty_index] = generator
820
-
821
- # Decrement last_req_index since it is now empty.
822
- last_req_index -= 1
823
-
824
- def make_sampling_metadata(
825
- self,
826
- skip_copy: bool = False,
827
- ) -> SamplingMetadata:
828
- if not skip_copy:
829
- self.temperature[:self.num_reqs].copy_(
830
- self.temperature_cpu_tensor[:self.num_reqs], non_blocking=True)
831
- self.top_p[:self.num_reqs].copy_(
832
- self.top_p_cpu_tensor[:self.num_reqs], non_blocking=True)
833
- self.top_k[:self.num_reqs].copy_(
834
- self.top_k_cpu_tensor[:self.num_reqs], non_blocking=True)
835
- return SamplingMetadata(
836
- temperature=self.temperature[:self.num_reqs],
837
- all_greedy=self.all_greedy,
838
- all_random=self.all_random,
839
- top_p=self.top_p[:self.num_reqs],
840
- top_k=self.top_k[:self.num_reqs],
841
- no_top_p=self.no_top_p,
842
- no_top_k=self.no_top_k,
843
- generators=self.generators,
844
- max_num_logprobs=self.max_num_logprobs,
845
- )
846
-
847
- @property
848
- def num_reqs(self) -> int:
849
- return len(self.req_id_to_index)
850
-
851
- @property
852
- def all_greedy(self) -> bool:
853
- return len(self.random_reqs) == 0
854
-
855
- @property
856
- def all_random(self) -> bool:
857
- return len(self.greedy_reqs) == 0
858
-
859
- @property
860
- def no_top_p(self) -> bool:
861
- return len(self.top_p_reqs) == 0
862
-
863
- @property
864
- def no_top_k(self) -> bool:
865
- return len(self.top_k_reqs) == 0
866
-
867
- @property
868
- def max_num_logprobs(self) -> int:
869
- return max(self.num_logprobs.values()) if self.num_logprobs else 0
870
-
871
- @property
872
- def no_logprob(self) -> bool:
873
- return len(self.num_logprobs) == 0
874
-
875
- @property
876
- def no_prompt_logprob(self) -> bool:
877
- return len(self.prompt_logprob_reqs) == 0