@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8d66a7b6d7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Rename variables and methods (#91)

Woosuk Kwon committed May 10, 2023 at 00:58 UTC 8d66a7b6d747dd28fb29b998473f159ec08be2da
7 files changed +64 -83
cacheflow/block.py
+1 -1
@@ -27,7 +27,7 @@ class LogicalTokenBlock:
27 def is_full(self) -> bool:
28 return self.num_tokens == self.block_size
29
30 - def append(self, token_ids: List[int]) -> None:
30 + def append_tokens(self, token_ids: List[int]) -> None:
31 assert len(token_ids) <= self.get_num_empty_slots()
32 self.token_ids[self.num_tokens:self.num_tokens + len(token_ids)] = token_ids
33 self.num_tokens += len(token_ids)
cacheflow/core/block_manager.py
+4 -4
@@ -97,15 +97,15 @@ class BlockSpaceManager:
97 for seq in seq_group.seqs:
98 self.block_tables[seq.seq_id] = block_table.copy()
99
100 - def can_append(self, seq_group: SequenceGroup) -> bool:
100 + def can_append_slot(self, seq_group: SequenceGroup) -> bool:
101 # Simple heuristic: If there is at least one free block
102 # for each sequence, we can append.
103 num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
104 num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
105 return num_seqs <= num_free_gpu_blocks
106
107 - def append(self, seq: Sequence) -> Optional[Tuple[int, int]]:
108 - """Allocate a physical slot for the new token."""
107 + def append_slot(self, seq: Sequence) -> Optional[Tuple[int, int]]:
108 + """Allocate a physical slot for a new token."""
109 logical_blocks = seq.logical_token_blocks
110 block_table = self.block_tables[seq.seq_id]
111
@@ -156,7 +156,7 @@ class BlockSpaceManager:
156 num_free_blocks = self.gpu_allocator.get_num_free_blocks()
157 # NOTE: Conservatively, we assume that every sequence will allocate
158 # at least one free block right after the swap-in.
159 - # NOTE: This should match the logic in can_append().
159 + # NOTE: This should match the logic in can_append_slot().
160 num_required_blocks = len(blocks) + num_swapped_seqs
161 return num_free_blocks - num_required_blocks >= self.watermark_blocks
162
cacheflow/core/scheduler.py
+12 -12
@@ -9,7 +9,7 @@ from cacheflow.core.policy import PolicyFactory
9 from cacheflow.sampling_params import SamplingParams
10 from cacheflow.sequence import Sequence
11 from cacheflow.sequence import SequenceGroup
12 -from cacheflow.sequence import SequenceGroupInputs
12 +from cacheflow.sequence import SequenceGroupMetadata
13 from cacheflow.sequence import SequenceOutputs
14 from cacheflow.sequence import SequenceStatus
15
@@ -105,7 +105,7 @@ class Scheduler:
105 preempted: List[SequenceGroup] = []
106 while self.running:
107 seq_group = self.running.pop(0)
108 - while not self.block_manager.can_append(seq_group):
108 + while not self.block_manager.can_append_slot(seq_group):
109 if self.running:
110 # Preempt the lowest-priority sequence groups.
111 victim_seq_group = self.running.pop(-1)
@@ -119,7 +119,7 @@ class Scheduler:
119 break
120 else:
121 # Append new slots to the sequence group.
122 - self._append(seq_group, blocks_to_copy)
122 + self._append_slot(seq_group, blocks_to_copy)
123 running.append(seq_group)
124 self.running = running
125
@@ -143,7 +143,7 @@ class Scheduler:
143
144 seq_group = self.swapped.pop(0)
145 self._swap_in(seq_group, blocks_to_swap_in)
146 - self._append(seq_group, blocks_to_copy)
146 + self._append_slot(seq_group, blocks_to_copy)
147 self.running.append(seq_group)
148
149 num_batched_tokens = sum(
@@ -252,7 +252,7 @@ class Scheduler:
252 prompt_group_ids = scheduler_output[3]
253
254 # Create input data structures.
255 - input_seq_groups: List[SequenceGroupInputs] = []
255 + seq_group_metadata_list: List[SequenceGroupMetadata] = []
256 updated_seq_groups: List[SequenceGroup] = self.running.copy()
257
258 for seq_group in self.running:
@@ -274,7 +274,7 @@ class Scheduler:
274 # sequence length
275 seq_len = seq.get_len()
276
277 - input_seq_group = SequenceGroupInputs(
277 + seq_group_metadata = SequenceGroupMetadata(
278 group_id=group_id,
279 is_prompt=is_prompt,
280 input_tokens=input_tokens,
@@ -283,14 +283,14 @@ class Scheduler:
283 sampling_params=self.sampling_params[group_id],
284 block_tables=block_tables,
285 )
286 - input_seq_groups.append(input_seq_group)
286 + seq_group_metadata_list.append(seq_group_metadata)
287
288 # Execute the first stage of the pipeline.
289 - if input_seq_groups or blocks_to_swap_in or blocks_to_swap_out:
289 + if seq_group_metadata_list or blocks_to_swap_in or blocks_to_swap_out:
290 # Swap in and swap out should never happen at the same time.
291 assert not (blocks_to_swap_in and blocks_to_swap_out)
292 self.controllers[0].execute_stage(
293 - input_seq_groups,
293 + seq_group_metadata_list,
294 blocks_to_swap_in=blocks_to_swap_in,
295 blocks_to_swap_out=blocks_to_swap_out,
296 blocks_to_copy=blocks_to_copy,
@@ -330,7 +330,7 @@ class Scheduler:
330
331 # Append a new token to the sequence.
332 output = seq_outputs[seq.seq_id]
333 - seq.append(output.output_token, output.logprobs)
333 + seq.append_token(output.output_token, output.logprobs)
334
335 # Check if the sequence has generated a stop token.
336 if output.output_token in stop_token_ids:
@@ -360,13 +360,13 @@ class Scheduler:
360 if seq_group.group_id not in self.num_steps:
361 self.num_steps[seq_group.group_id] = 0
362
363 - def _append(
363 + def _append_slot(
364 self,
365 seq_group: SequenceGroup,
366 blocks_to_copy: Dict[int, List[int]],
367 ) -> None:
368 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
369 - ret = self.block_manager.append(seq)
369 + ret = self.block_manager.append_slot(seq)
370 if ret is not None:
371 src_block, dst_block = ret
372 if src_block in blocks_to_copy:
cacheflow/sampling_params.py
+2 -10
@@ -1,4 +1,4 @@
1 -from typing import Optional, Set, Dict
1 +from typing import Dict, Set
2
3
4 class SamplingParams:
@@ -12,7 +12,6 @@ class SamplingParams:
12 stop_token_ids: Set[int],
13 max_num_steps: int,
14 num_logprobs: int,
15 - context_window_size: Optional[int],
15 ) -> None:
16 if n < 1:
17 raise ValueError(f'n must be at least 1, got {n}.')
@@ -27,10 +26,6 @@ class SamplingParams:
26 if num_logprobs < 0:
27 raise ValueError(
28 f'num_logprobs must be non-negative, got {num_logprobs}.')
30 - if context_window_size is not None and context_window_size < 0:
31 - raise ValueError(
32 - 'context_window_size must be non-negative, '
33 - f'got {context_window_size}.')
29
30 if use_beam_search:
31 if n == 1:
@@ -58,7 +53,6 @@ class SamplingParams:
53 self.stop_token_ids = stop_token_ids
54 self.max_num_steps = max_num_steps
55 self.num_logprobs = num_logprobs
61 - self.context_window_size = context_window_size
56
57 def __repr__(self) -> str:
58 return (f'SamplingParams(n={self.n}, '
@@ -67,8 +61,7 @@ class SamplingParams:
61 f'use_beam_search={self.use_beam_search}, '
62 f'stop_token_ids={self.stop_token_ids}, '
63 f'max_num_steps={self.max_num_steps}, '
70 - f'num_logprobs={self.num_logprobs}, '
71 - f'context_window_size={self.context_window_size})')
64 + f'num_logprobs={self.num_logprobs}')
65
66 @classmethod
67 def from_dict(cls, d: Dict) -> 'SamplingParams':
@@ -80,5 +73,4 @@ class SamplingParams:
73 stop_token_ids=set(d.get('stop_token_ids', set())),
74 max_num_steps=d.get('max_num_steps', 16),
75 num_logprobs=d.get('num_logprobs', 0),
83 - context_window_size=d.get('context_window_size', None),
76 )
cacheflow/sequence.py
+13 -12
@@ -18,45 +18,46 @@ class Sequence:
18 def __init__(
19 self,
20 seq_id: int,
21 - token_ids: List[int],
21 + prompt_token_ids: List[int],
22 block_size: int,
23 ) -> None:
24 self.seq_id = seq_id
25 self.block_size = block_size
26
27 + self.prompt_len = len(prompt_token_ids)
28 self.logical_token_blocks: List[LogicalTokenBlock] = []
28 - # Initialize the logical token blocks with the given token ids.
29 - self.add(token_ids)
29 + # Initialize the logical token blocks with the prompt token ids.
30 + self._append_tokens(prompt_token_ids)
31
31 - self.prompt_len = len(token_ids)
32 self.status = SequenceStatus.WAITING
33 + # Used for beam search.
34 self.output_logprobs: List[Dict[int, float]] = []
35 self.cumulative_logprobs = 0.0
36
36 - def add_block(self) -> None:
37 + def _append_logical_block(self) -> None:
38 block = LogicalTokenBlock(
39 block_number=len(self.logical_token_blocks),
40 block_size=self.block_size,
41 )
42 self.logical_token_blocks.append(block)
43
43 - def add(self, token_ids: List[int]) -> None:
44 + def _append_tokens(self, token_ids: List[int]) -> None:
45 while token_ids:
46 if not self.logical_token_blocks:
46 - self.add_block()
47 + self._append_logical_block()
48
49 last_block = self.logical_token_blocks[-1]
50 if last_block.is_full():
50 - self.add_block()
51 + self._append_logical_block()
52 last_block = self.logical_token_blocks[-1]
53
54 num_empty_slots = last_block.get_num_empty_slots()
54 - last_block.append(token_ids[:num_empty_slots])
55 + last_block.append_tokens(token_ids[:num_empty_slots])
56 token_ids = token_ids[num_empty_slots:]
57
57 - def append(self, token_id: int, logprobs: Dict[int, float]) -> None:
58 + def append_token(self, token_id: int, logprobs: Dict[int, float]) -> None:
59 assert token_id in logprobs
59 - self.add([token_id])
60 + self._append_tokens([token_id])
61 self.output_logprobs.append(logprobs)
62 self.cumulative_logprobs += logprobs[token_id]
63
@@ -121,7 +122,7 @@ class SequenceGroup:
122 f'num_seqs={len(self.seqs)})')
123
124
124 -class SequenceGroupInputs:
125 +class SequenceGroupMetadata:
126
127 def __init__(
128 self,
cacheflow/worker/controller.py
+3 -15
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Union, Tuple, Optional
1 +from typing import List, Optional, Tuple, Union
2
3 try:
4 import ray
@@ -6,7 +6,6 @@ except ImportError:
6 ray = None
7
8 from cacheflow.core.scheduler import Scheduler
9 -from cacheflow.sequence import SequenceGroupInputs
9 from cacheflow.worker.worker import Worker
10
11
@@ -81,23 +80,12 @@ class Controller:
80 self.next_node = next_node
81 self.is_last_stage = isinstance(next_node, Scheduler)
82
84 - def execute_stage(
85 - self,
86 - input_seq_groups: List[SequenceGroupInputs],
87 - blocks_to_swap_in: Dict[int, int],
88 - blocks_to_swap_out: Dict[int, int],
89 - blocks_to_copy: Dict[int, List[int]],
90 - ) -> None:
83 + def execute_stage(self, *args, **kwargs) -> None:
84 all_outputs = []
85 for worker in self.workers:
86 executor = (worker.execute_stage.remote
87 if self.use_ray else worker.execute_stage)
95 - output = executor(
96 - input_seq_groups,
97 - blocks_to_swap_in,
98 - blocks_to_swap_out,
99 - blocks_to_copy,
100 - )
88 + output = executor(*args, **kwargs)
89 all_outputs.append(output)
90
91 if self.use_ray:
cacheflow/worker/worker.py
+29 -29
@@ -8,10 +8,11 @@ from cacheflow.model_executor.parallel_utils.parallel_state import (
8 initialize_all_reduce_launcher,
9 get_tensor_model_parallel_world_size)
10 from cacheflow.sampling_params import SamplingParams
11 -from cacheflow.sequence import SequenceGroupInputs
11 +from cacheflow.sequence import SequenceGroupMetadata
12 from cacheflow.sequence import SequenceOutputs
13 from cacheflow.worker.cache_engine import CacheEngine
14
15 +
16 class Worker:
17
18 def __init__(
@@ -93,30 +94,29 @@ class Worker:
94
95 def prepare_inputs(
96 self,
96 - input_seq_groups: List[SequenceGroupInputs],
97 + seq_group_metadata_list: List[SequenceGroupMetadata],
98 ) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
99 seq_groups: List[Tuple[List[int], SamplingParams]] = []
100 seq_logprobs: Dict[int, float] = {}
100 - sampling_params: Dict[int, SamplingParams] = {}
101 input_tokens: List[int] = []
102 input_positions: List[int] = []
103 slot_mapping: List[int] = []
104
105 # Add prompt tokens.
106 prompt_lens: List[int] = []
107 - for input_seq_group in input_seq_groups:
108 - if not input_seq_group.is_prompt:
107 + for seq_group_metadata in seq_group_metadata_list:
108 + if not seq_group_metadata.is_prompt:
109 continue
110
111 - seq_ids = list(input_seq_group.input_tokens.keys())
112 - sampling_params = input_seq_group.sampling_params
111 + seq_ids = list(seq_group_metadata.input_tokens.keys())
112 + sampling_params = seq_group_metadata.sampling_params
113 seq_groups.append((seq_ids, sampling_params))
114 - seq_logprobs.update(input_seq_group.seq_logprobs)
114 + seq_logprobs.update(seq_group_metadata.seq_logprobs)
115
116 # Use any sequence in the group.
117 seq_id = seq_ids[0]
118
119 - prompt_tokens = input_seq_group.input_tokens[seq_id]
119 + prompt_tokens = seq_group_metadata.input_tokens[seq_id]
120 prompt_len = len(prompt_tokens)
121 prompt_lens.append(prompt_len)
122
@@ -126,7 +126,7 @@ class Worker:
126 input_positions.extend(range(len(prompt_tokens)))
127
128 # Compute the slot mapping.
129 - block_table = input_seq_group.block_tables[seq_id]
129 + block_table = seq_group_metadata.block_tables[seq_id]
130 for i in range(prompt_len):
131 block_number = block_table[i // self.block_size]
132 block_offset = i % self.block_size
@@ -138,31 +138,31 @@ class Worker:
138 max_num_blocks_per_seq = 0
139 context_lens: List[int] = []
140 generation_block_tables: List[List[int]] = []
141 - for input_seq_group in input_seq_groups:
142 - if input_seq_group.is_prompt:
141 + for seq_group_metadata in seq_group_metadata_list:
142 + if seq_group_metadata.is_prompt:
143 continue
144
145 - seq_ids = list(input_seq_group.input_tokens.keys())
146 - sampling_params = input_seq_group.sampling_params
145 + seq_ids = list(seq_group_metadata.input_tokens.keys())
146 + sampling_params = seq_group_metadata.sampling_params
147 seq_groups.append((seq_ids, sampling_params))
148 - seq_logprobs.update(input_seq_group.seq_logprobs)
148 + seq_logprobs.update(seq_group_metadata.seq_logprobs)
149
150 for seq_id in seq_ids:
151 - assert len(input_seq_group.input_tokens[seq_id]) == 1
152 - generation_token = input_seq_group.input_tokens[seq_id][0]
151 + assert len(seq_group_metadata.input_tokens[seq_id]) == 1
152 + generation_token = seq_group_metadata.input_tokens[seq_id][0]
153 input_tokens.append(generation_token)
154
155 - position = input_seq_group.context_len - 1
155 + position = seq_group_metadata.context_len - 1
156 input_positions.append(position)
157
158 - block_table = input_seq_group.block_tables[seq_id]
158 + block_table = seq_group_metadata.block_tables[seq_id]
159 generation_block_tables.append(block_table)
160
161 max_context_len = max(
162 - max_context_len, input_seq_group.context_len)
162 + max_context_len, seq_group_metadata.context_len)
163 max_num_blocks_per_seq = max(
164 max_num_blocks_per_seq, len(block_table))
165 - context_lens.append(input_seq_group.context_len)
165 + context_lens.append(seq_group_metadata.context_len)
166
167 block_number = block_table[position // self.block_size]
168 block_offset = position % self.block_size
@@ -203,30 +203,30 @@ class Worker:
203 @torch.inference_mode()
204 def execute_stage(
205 self,
206 - input_seq_groups: List[SequenceGroupInputs],
206 + seq_group_metadata_list: List[SequenceGroupMetadata],
207 blocks_to_swap_in: Dict[int, int],
208 blocks_to_swap_out: Dict[int, int],
209 blocks_to_copy: Dict[int, List[int]],
210 ) -> Dict[int, SequenceOutputs]:
211 # Issue cache operations.
212 - command_issued = False
212 + issued_cache_op = False
213 if blocks_to_swap_in:
214 self.cache_engine.swap_in(blocks_to_swap_in)
215 - command_issued = True
215 + issued_cache_op = True
216 if blocks_to_swap_out:
217 self.cache_engine.swap_out(blocks_to_swap_out)
218 - command_issued = True
218 + issued_cache_op = True
219 if blocks_to_copy:
220 self.cache_engine.copy(blocks_to_copy)
221 - command_issued = True
221 + issued_cache_op = True
222
223 - if command_issued:
223 + if issued_cache_op:
224 cache_events = self.cache_events
225 else:
226 cache_events = None
227
228 # If there is no input, we don't need to execute the model.
229 - if not input_seq_groups:
229 + if not seq_group_metadata_list:
230 if cache_events is not None:
231 for event in cache_events:
232 event.wait()
@@ -234,7 +234,7 @@ class Worker:
234
235 # Prepare input tensors.
236 input_tokens, input_positions, input_metadata = self.prepare_inputs(
237 - input_seq_groups)
237 + seq_group_metadata_list)
238
239 # Execute the model.
240 output = self.model(