@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 25b79d9fd3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] Input Batch Relocation (#10962)

Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Dec 9, 2024 at 12:33 UTC 25b79d9fd38e2c53ce281be23241d8939ec7320c
2 files changed +283 -270
vllm/v1/worker/gpu_input_batch.py new
+280
@@ -0,0 +1,280 @@
1 +# Datastructures defining an input batch
2 +
3 +from dataclasses import dataclass
4 +from typing import TYPE_CHECKING, Dict, List, Optional, Set
5 +
6 +import numpy as np
7 +import torch
8 +
9 +from vllm.multimodal import MultiModalKwargs
10 +from vllm.sampling_params import SamplingParams, SamplingType
11 +from vllm.v1.sample.metadata import SamplingMetadata
12 +
13 +if TYPE_CHECKING:
14 + from vllm.multimodal.inputs import PlaceholderRange
15 +
16 +
17 +@dataclass
18 +class CachedRequestState:
19 +
20 + req_id: str
21 + prompt_token_ids: List[int]
22 + prompt: Optional[str]
23 + mm_inputs: List[MultiModalKwargs]
24 + mm_positions: List["PlaceholderRange"]
25 + sampling_params: SamplingParams
26 + generator: Optional[torch.Generator]
27 +
28 + block_ids: List[int]
29 + num_computed_tokens: int
30 + output_token_ids: List[int]
31 +
32 + @property
33 + def num_tokens(self) -> int:
34 + return len(self.prompt_token_ids) + len(self.output_token_ids)
35 +
36 +
37 +class InputBatch:
38 +
39 + def __init__(
40 + self,
41 + max_num_reqs: int,
42 + max_model_len: int,
43 + max_num_blocks_per_req: int,
44 + device: torch.device,
45 + pin_memory: bool,
46 + ):
47 + self.max_num_reqs = max_num_reqs
48 + self.max_model_len = max_model_len
49 + self.max_num_blocks_per_req = max_num_blocks_per_req
50 + self.device = device
51 + self.pin_memory = pin_memory
52 +
53 + self.req_ids: List[Optional[str]] = [None] * max_num_reqs
54 + self.req_id_to_index: Dict[str, int] = {}
55 +
56 + self.token_ids_cpu = np.empty((max_num_reqs, max_model_len),
57 + dtype=np.int32)
58 + self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
59 +
60 + # Attention-related.
61 + self.block_table = torch.zeros((max_num_reqs, max_num_blocks_per_req),
62 + device=self.device,
63 + dtype=torch.int32)
64 + self.block_table_cpu_tensor = torch.zeros(
65 + (max_num_reqs, max_num_blocks_per_req),
66 + device="cpu",
67 + dtype=torch.int32,
68 + pin_memory=pin_memory,
69 + )
70 + self.block_table_cpu = self.block_table_cpu_tensor.numpy()
71 +
72 + # Sampling-related.
73 + self.temperature = torch.empty((max_num_reqs, ),
74 + dtype=torch.float32,
75 + device=device)
76 + self.temperature_cpu_tensor = torch.empty((max_num_reqs, ),
77 + dtype=torch.float32,
78 + device="cpu",
79 + pin_memory=pin_memory)
80 + self.temperature_cpu = self.temperature_cpu_tensor.numpy()
81 + self.greedy_reqs: Set[str] = set()
82 + self.random_reqs: Set[str] = set()
83 +
84 + self.top_p = torch.empty((max_num_reqs, ),
85 + dtype=torch.float32,
86 + device=device)
87 + self.top_p_cpu_tensor = torch.empty((max_num_reqs, ),
88 + dtype=torch.float32,
89 + device="cpu",
90 + pin_memory=pin_memory)
91 + self.top_p_cpu = self.top_p_cpu_tensor.numpy()
92 + self.top_p_reqs: Set[str] = set()
93 +
94 + self.top_k = torch.empty((max_num_reqs, ),
95 + dtype=torch.int32,
96 + device=device)
97 + self.top_k_cpu_tensor = torch.empty((max_num_reqs, ),
98 + dtype=torch.int32,
99 + device="cpu",
100 + pin_memory=pin_memory)
101 + self.top_k_cpu = self.top_k_cpu_tensor.numpy()
102 + self.top_k_reqs: Set[str] = set()
103 +
104 + # req_index -> generator
105 + self.generators: Dict[int, torch.Generator] = {}
106 +
107 + self.num_logprobs: Dict[str, int] = {}
108 + self.prompt_logprob_reqs: Set[str] = set()
109 +
110 + def add_request(
111 + self,
112 + request: "CachedRequestState",
113 + req_index: Optional[int] = None,
114 + ) -> None:
115 + if req_index is None:
116 + req_index = self.num_reqs
117 + assert req_index < self.max_num_reqs
118 +
119 + req_id = request.req_id
120 + self.req_ids[req_index] = req_id
121 + self.req_id_to_index[req_id] = req_index
122 +
123 + # Copy the prompt token ids and output token ids.
124 + num_prompt_tokens = len(request.prompt_token_ids)
125 + self.token_ids_cpu[
126 + req_index, :num_prompt_tokens] = request.prompt_token_ids
127 + start_idx = num_prompt_tokens
128 + end_idx = start_idx + len(request.output_token_ids)
129 + self.token_ids_cpu[req_index,
130 + start_idx:end_idx] = request.output_token_ids
131 +
132 + self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens
133 + num_blocks = len(request.block_ids)
134 + self.block_table_cpu[req_index, :num_blocks] = request.block_ids
135 +
136 + sampling_params = request.sampling_params
137 + self.temperature_cpu[req_index] = sampling_params.temperature
138 + if sampling_params.sampling_type == SamplingType.GREEDY:
139 + self.greedy_reqs.add(req_id)
140 + else:
141 + self.random_reqs.add(req_id)
142 +
143 + self.top_p_cpu[req_index] = sampling_params.top_p
144 + if sampling_params.top_p < 1:
145 + self.top_p_reqs.add(req_id)
146 + self.top_k_cpu[req_index] = sampling_params.top_k
147 + if sampling_params.top_k > 0:
148 + self.top_k_reqs.add(req_id)
149 +
150 + self.generators[req_index] = request.generator
151 +
152 + num_logprobs = sampling_params.logprobs
153 + if num_logprobs is not None and num_logprobs > 0:
154 + self.num_logprobs[req_id] = num_logprobs
155 + if sampling_params.prompt_logprobs:
156 + self.prompt_logprob_reqs.add(req_id)
157 +
158 + def remove_request(self, req_id: str) -> Optional[int]:
159 + req_index = self.req_id_to_index.pop(req_id, None)
160 + if req_index is None:
161 + return None
162 + self.req_ids[req_index] = None
163 +
164 + self.greedy_reqs.discard(req_id)
165 + self.random_reqs.discard(req_id)
166 + self.top_p_reqs.discard(req_id)
167 + self.top_k_reqs.discard(req_id)
168 + self.generators.pop(req_index, None)
169 + self.num_logprobs.pop(req_id, None)
170 + self.prompt_logprob_reqs.discard(req_id)
171 + return req_index
172 +
173 + def clear(self) -> None:
174 + self.req_ids = [None] * self.max_num_reqs
175 + self.req_id_to_index.clear()
176 + self.greedy_reqs.clear()
177 + self.random_reqs.clear()
178 + self.top_p_reqs.clear()
179 + self.top_k_reqs.clear()
180 + self.generators.clear()
181 + self.num_logprobs.clear()
182 + self.prompt_logprob_reqs.clear()
183 +
184 + def condense(self, empty_req_indices: List[int]) -> None:
185 + if self.num_reqs == 0:
186 + # The batched states are empty.
187 + return
188 +
189 + # NOTE(woosuk): This function assumes that the empty_req_indices
190 + # is sorted in descending order.
191 + last_req_index = self.num_reqs + len(empty_req_indices) - 1
192 + while empty_req_indices:
193 + # Find the largest non-empty index.
194 + while last_req_index in empty_req_indices:
195 + last_req_index -= 1
196 +
197 + # Find the smallest empty index.
198 + empty_index = empty_req_indices.pop()
199 + if empty_index >= last_req_index:
200 + break
201 +
202 + # Swap the states.
203 + req_id = self.req_ids[last_req_index]
204 + self.req_ids[empty_index] = req_id
205 + self.req_ids[last_req_index] = None
206 + self.req_id_to_index[req_id] = empty_index
207 +
208 + # TODO(woosuk): Optimize the copy of token_ids_cpu and
209 + # block_table_cpu.
210 + self.token_ids_cpu[empty_index] = self.token_ids_cpu[
211 + last_req_index]
212 + self.num_computed_tokens_cpu[
213 + empty_index] = self.num_computed_tokens_cpu[last_req_index]
214 + self.block_table_cpu[empty_index] = self.block_table_cpu[
215 + last_req_index]
216 + self.temperature_cpu[empty_index] = self.temperature_cpu[
217 + last_req_index]
218 + self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
219 + self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
220 + generator = self.generators.pop(last_req_index, None)
221 + if generator is not None:
222 + self.generators[empty_index] = generator
223 +
224 + # Decrement last_req_index since it is now empty.
225 + last_req_index -= 1
226 +
227 + def make_sampling_metadata(
228 + self,
229 + skip_copy: bool = False,
230 + ) -> SamplingMetadata:
231 + if not skip_copy:
232 + self.temperature[:self.num_reqs].copy_(
233 + self.temperature_cpu_tensor[:self.num_reqs], non_blocking=True)
234 + self.top_p[:self.num_reqs].copy_(
235 + self.top_p_cpu_tensor[:self.num_reqs], non_blocking=True)
236 + self.top_k[:self.num_reqs].copy_(
237 + self.top_k_cpu_tensor[:self.num_reqs], non_blocking=True)
238 + return SamplingMetadata(
239 + temperature=self.temperature[:self.num_reqs],
240 + all_greedy=self.all_greedy,
241 + all_random=self.all_random,
242 + top_p=self.top_p[:self.num_reqs],
243 + top_k=self.top_k[:self.num_reqs],
244 + no_top_p=self.no_top_p,
245 + no_top_k=self.no_top_k,
246 + generators=self.generators,
247 + max_num_logprobs=self.max_num_logprobs,
248 + )
249 +
250 + @property
251 + def num_reqs(self) -> int:
252 + return len(self.req_id_to_index)
253 +
254 + @property
255 + def all_greedy(self) -> bool:
256 + return len(self.random_reqs) == 0
257 +
258 + @property
259 + def all_random(self) -> bool:
260 + return len(self.greedy_reqs) == 0
261 +
262 + @property
263 + def no_top_p(self) -> bool:
264 + return len(self.top_p_reqs) == 0
265 +
266 + @property
267 + def no_top_k(self) -> bool:
268 + return len(self.top_k_reqs) == 0
269 +
270 + @property
271 + def max_num_logprobs(self) -> int:
272 + return max(self.num_logprobs.values()) if self.num_logprobs else 0
273 +
274 + @property
275 + def no_logprob(self) -> bool:
276 + return len(self.num_logprobs) == 0
277 +
278 + @property
279 + def no_prompt_logprob(self) -> bool:
280 + return len(self.prompt_logprob_reqs) == 0
vllm/v1/worker/gpu_model_runner.py
+3 -270
@@ -1,7 +1,6 @@
1 import gc
2 import time
3 -from dataclasses import dataclass
4 -from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
3 +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
4
5 import numpy as np
6 import torch
@@ -15,16 +14,16 @@ from vllm.inputs import INPUT_REGISTRY, InputRegistry
14 from vllm.logger import init_logger
15 from vllm.model_executor.model_loader import get_model
16 from vllm.multimodal import MultiModalKwargs
18 -from vllm.sampling_params import SamplingParams, SamplingType
17 +from vllm.sampling_params import SamplingType
18 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
19 is_pin_memory_available)
20 from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
21 FlashAttentionMetadata)
22 from vllm.v1.outputs import ModelRunnerOutput
23 from vllm.v1.sample.metadata import SamplingMetadata
24 +from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
25
26 if TYPE_CHECKING:
27 - from vllm.multimodal.inputs import PlaceholderRange
27 from vllm.v1.core.scheduler import SchedulerOutput
28
29 logger = init_logger(__name__)
@@ -609,269 +608,3 @@ class GPUModelRunner:
608 if batch_size <= size:
609 return size
610 return None
612 -
613 -
614 -@dataclass
615 -class CachedRequestState:
616 -
617 - req_id: str
618 - prompt_token_ids: List[int]
619 - prompt: Optional[str]
620 - mm_inputs: List[MultiModalKwargs]
621 - mm_positions: List["PlaceholderRange"]
622 - sampling_params: SamplingParams
623 - generator: Optional[torch.Generator]
624 -
625 - block_ids: List[int]
626 - num_computed_tokens: int
627 - output_token_ids: List[int]
628 -
629 - @property
630 - def num_tokens(self) -> int:
631 - return len(self.prompt_token_ids) + len(self.output_token_ids)
632 -
633 -
634 -class InputBatch:
635 -
636 - def __init__(
637 - self,
638 - max_num_reqs: int,
639 - max_model_len: int,
640 - max_num_blocks_per_req: int,
641 - device: torch.device,
642 - pin_memory: bool,
643 - ):
644 - self.max_num_reqs = max_num_reqs
645 - self.max_model_len = max_model_len
646 - self.max_num_blocks_per_req = max_num_blocks_per_req
647 - self.device = device
648 - self.pin_memory = pin_memory
649 -
650 - self.req_ids: List[Optional[str]] = [None] * max_num_reqs
651 - self.req_id_to_index: Dict[str, int] = {}
652 -
653 - self.token_ids_cpu = np.empty((max_num_reqs, max_model_len),
654 - dtype=np.int32)
655 - self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
656 -
657 - # Attention-related.
658 - self.block_table = torch.zeros((max_num_reqs, max_num_blocks_per_req),
659 - device=self.device,
660 - dtype=torch.int32)
661 - self.block_table_cpu_tensor = torch.zeros(
662 - (max_num_reqs, max_num_blocks_per_req),
663 - device="cpu",
664 - dtype=torch.int32,
665 - pin_memory=pin_memory,
666 - )
667 - self.block_table_cpu = self.block_table_cpu_tensor.numpy()
668 -
669 - # Sampling-related.
670 - self.temperature = torch.empty((max_num_reqs, ),
671 - dtype=torch.float32,
672 - device=device)
673 - self.temperature_cpu_tensor = torch.empty((max_num_reqs, ),
674 - dtype=torch.float32,
675 - device="cpu",
676 - pin_memory=pin_memory)
677 - self.temperature_cpu = self.temperature_cpu_tensor.numpy()
678 - self.greedy_reqs: Set[str] = set()
679 - self.random_reqs: Set[str] = set()
680 -
681 - self.top_p = torch.empty((max_num_reqs, ),
682 - dtype=torch.float32,
683 - device=device)
684 - self.top_p_cpu_tensor = torch.empty((max_num_reqs, ),
685 - dtype=torch.float32,
686 - device="cpu",
687 - pin_memory=pin_memory)
688 - self.top_p_cpu = self.top_p_cpu_tensor.numpy()
689 - self.top_p_reqs: Set[str] = set()
690 -
691 - self.top_k = torch.empty((max_num_reqs, ),
692 - dtype=torch.int32,
693 - device=device)
694 - self.top_k_cpu_tensor = torch.empty((max_num_reqs, ),
695 - dtype=torch.int32,
696 - device="cpu",
697 - pin_memory=pin_memory)
698 - self.top_k_cpu = self.top_k_cpu_tensor.numpy()
699 - self.top_k_reqs: Set[str] = set()
700 -
701 - # req_index -> generator
702 - self.generators: Dict[int, torch.Generator] = {}
703 -
704 - self.num_logprobs: Dict[str, int] = {}
705 - self.prompt_logprob_reqs: Set[str] = set()
706 -
707 - def add_request(
708 - self,
709 - request: "CachedRequestState",
710 - req_index: Optional[int] = None,
711 - ) -> None:
712 - if req_index is None:
713 - req_index = self.num_reqs
714 - assert req_index < self.max_num_reqs
715 -
716 - req_id = request.req_id
717 - self.req_ids[req_index] = req_id
718 - self.req_id_to_index[req_id] = req_index
719 -
720 - # Copy the prompt token ids and output token ids.
721 - num_prompt_tokens = len(request.prompt_token_ids)
722 - self.token_ids_cpu[
723 - req_index, :num_prompt_tokens] = request.prompt_token_ids
724 - start_idx = num_prompt_tokens
725 - end_idx = start_idx + len(request.output_token_ids)
726 - self.token_ids_cpu[req_index,
727 - start_idx:end_idx] = request.output_token_ids
728 -
729 - self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens
730 - num_blocks = len(request.block_ids)
731 - self.block_table_cpu[req_index, :num_blocks] = request.block_ids
732 -
733 - sampling_params = request.sampling_params
734 - self.temperature_cpu[req_index] = sampling_params.temperature
735 - if sampling_params.sampling_type == SamplingType.GREEDY:
736 - self.greedy_reqs.add(req_id)
737 - else:
738 - self.random_reqs.add(req_id)
739 -
740 - self.top_p_cpu[req_index] = sampling_params.top_p
741 - if sampling_params.top_p < 1:
742 - self.top_p_reqs.add(req_id)
743 - self.top_k_cpu[req_index] = sampling_params.top_k
744 - if sampling_params.top_k > 0:
745 - self.top_k_reqs.add(req_id)
746 -
747 - self.generators[req_index] = request.generator
748 -
749 - num_logprobs = sampling_params.logprobs
750 - if num_logprobs is not None and num_logprobs > 0:
751 - self.num_logprobs[req_id] = num_logprobs
752 - if sampling_params.prompt_logprobs:
753 - self.prompt_logprob_reqs.add(req_id)
754 -
755 - def remove_request(self, req_id: str) -> Optional[int]:
756 - req_index = self.req_id_to_index.pop(req_id, None)
757 - if req_index is None:
758 - return None
759 - self.req_ids[req_index] = None
760 -
761 - self.greedy_reqs.discard(req_id)
762 - self.random_reqs.discard(req_id)
763 - self.top_p_reqs.discard(req_id)
764 - self.top_k_reqs.discard(req_id)
765 - self.generators.pop(req_index, None)
766 - self.num_logprobs.pop(req_id, None)
767 - self.prompt_logprob_reqs.discard(req_id)
768 - return req_index
769 -
770 - def clear(self) -> None:
771 - self.req_ids = [None] * self.max_num_reqs
772 - self.req_id_to_index.clear()
773 - self.greedy_reqs.clear()
774 - self.random_reqs.clear()
775 - self.top_p_reqs.clear()
776 - self.top_k_reqs.clear()
777 - self.generators.clear()
778 - self.num_logprobs.clear()
779 - self.prompt_logprob_reqs.clear()
780 -
781 - def condense(self, empty_req_indices: List[int]) -> None:
782 - if self.num_reqs == 0:
783 - # The batched states are empty.
784 - return
785 -
786 - # NOTE(woosuk): This function assumes that the empty_req_indices
787 - # is sorted in descending order.
788 - last_req_index = self.num_reqs + len(empty_req_indices) - 1
789 - while empty_req_indices:
790 - # Find the largest non-empty index.
791 - while last_req_index in empty_req_indices:
792 - last_req_index -= 1
793 -
794 - # Find the smallest empty index.
795 - empty_index = empty_req_indices.pop()
796 - if empty_index >= last_req_index:
797 - break
798 -
799 - # Swap the states.
800 - req_id = self.req_ids[last_req_index]
801 - self.req_ids[empty_index] = req_id
802 - self.req_ids[last_req_index] = None
803 - self.req_id_to_index[req_id] = empty_index
804 -
805 - # TODO(woosuk): Optimize the copy of token_ids_cpu and
806 - # block_table_cpu.
807 - self.token_ids_cpu[empty_index] = self.token_ids_cpu[
808 - last_req_index]
809 - self.num_computed_tokens_cpu[
810 - empty_index] = self.num_computed_tokens_cpu[last_req_index]
811 - self.block_table_cpu[empty_index] = self.block_table_cpu[
812 - last_req_index]
813 - self.temperature_cpu[empty_index] = self.temperature_cpu[
814 - last_req_index]
815 - self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
816 - self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
817 - generator = self.generators.pop(last_req_index, None)
818 - if generator is not None:
819 - self.generators[empty_index] = generator
820 -
821 - # Decrement last_req_index since it is now empty.
822 - last_req_index -= 1
823 -
824 - def make_sampling_metadata(
825 - self,
826 - skip_copy: bool = False,
827 - ) -> SamplingMetadata:
828 - if not skip_copy:
829 - self.temperature[:self.num_reqs].copy_(
830 - self.temperature_cpu_tensor[:self.num_reqs], non_blocking=True)
831 - self.top_p[:self.num_reqs].copy_(
832 - self.top_p_cpu_tensor[:self.num_reqs], non_blocking=True)
833 - self.top_k[:self.num_reqs].copy_(
834 - self.top_k_cpu_tensor[:self.num_reqs], non_blocking=True)
835 - return SamplingMetadata(
836 - temperature=self.temperature[:self.num_reqs],
837 - all_greedy=self.all_greedy,
838 - all_random=self.all_random,
839 - top_p=self.top_p[:self.num_reqs],
840 - top_k=self.top_k[:self.num_reqs],
841 - no_top_p=self.no_top_p,
842 - no_top_k=self.no_top_k,
843 - generators=self.generators,
844 - max_num_logprobs=self.max_num_logprobs,
845 - )
846 -
847 - @property
848 - def num_reqs(self) -> int:
849 - return len(self.req_id_to_index)
850 -
851 - @property
852 - def all_greedy(self) -> bool:
853 - return len(self.random_reqs) == 0
854 -
855 - @property
856 - def all_random(self) -> bool:
857 - return len(self.greedy_reqs) == 0
858 -
859 - @property
860 - def no_top_p(self) -> bool:
861 - return len(self.top_p_reqs) == 0
862 -
863 - @property
864 - def no_top_k(self) -> bool:
865 - return len(self.top_k_reqs) == 0
866 -
867 - @property
868 - def max_num_logprobs(self) -> int:
869 - return max(self.num_logprobs.values()) if self.num_logprobs else 0
870 -
871 - @property
872 - def no_logprob(self) -> bool:
873 - return len(self.num_logprobs) == 0
874 -
875 - @property
876 - def no_prompt_logprob(self) -> bool:
877 - return len(self.prompt_logprob_reqs) == 0