A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Rename variables and methods (#91)
Woosuk Kwon committed
May 10, 2023 at 00:58 UTC
8d66a7b6d747dd28fb29b998473f159ec08be2da
7 files changed
+64
-83
cacheflow/block.py
+1
-1
@@ -27,7 +27,7 @@ class LogicalTokenBlock:
27
def is_full(self) -> bool:
28
return self.num_tokens == self.block_size
29
30
- def append(self, token_ids: List[int]) -> None:
30
+ def append_tokens(self, token_ids: List[int]) -> None:
31
assert len(token_ids) <= self.get_num_empty_slots()
32
self.token_ids[self.num_tokens:self.num_tokens + len(token_ids)] = token_ids
33
self.num_tokens += len(token_ids)
cacheflow/core/block_manager.py
+4
-4
@@ -97,15 +97,15 @@ class BlockSpaceManager:
97
for seq in seq_group.seqs:
98
self.block_tables[seq.seq_id] = block_table.copy()
99
100
- def can_append(self, seq_group: SequenceGroup) -> bool:
100
+ def can_append_slot(self, seq_group: SequenceGroup) -> bool:
101
# Simple heuristic: If there is at least one free block
102
# for each sequence, we can append.
103
num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
104
num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
105
return num_seqs <= num_free_gpu_blocks
106
107
- def append(self, seq: Sequence) -> Optional[Tuple[int, int]]:
108
- """Allocate a physical slot for the new token."""
107
+ def append_slot(self, seq: Sequence) -> Optional[Tuple[int, int]]:
108
+ """Allocate a physical slot for a new token."""
109
logical_blocks = seq.logical_token_blocks
110
block_table = self.block_tables[seq.seq_id]
111
@@ -156,7 +156,7 @@ class BlockSpaceManager:
156
num_free_blocks = self.gpu_allocator.get_num_free_blocks()
157
# NOTE: Conservatively, we assume that every sequence will allocate
158
# at least one free block right after the swap-in.
159
- # NOTE: This should match the logic in can_append().
159
+ # NOTE: This should match the logic in can_append_slot().
160
num_required_blocks = len(blocks) + num_swapped_seqs
161
return num_free_blocks - num_required_blocks >= self.watermark_blocks
162
cacheflow/core/scheduler.py
+12
-12
@@ -9,7 +9,7 @@ from cacheflow.core.policy import PolicyFactory
9
from cacheflow.sampling_params import SamplingParams
10
from cacheflow.sequence import Sequence
11
from cacheflow.sequence import SequenceGroup
12
-from cacheflow.sequence import SequenceGroupInputs
12
+from cacheflow.sequence import SequenceGroupMetadata
13
from cacheflow.sequence import SequenceOutputs
14
from cacheflow.sequence import SequenceStatus
15
@@ -105,7 +105,7 @@ class Scheduler:
105
preempted: List[SequenceGroup] = []
106
while self.running:
107
seq_group = self.running.pop(0)
108
- while not self.block_manager.can_append(seq_group):
108
+ while not self.block_manager.can_append_slot(seq_group):
109
if self.running:
110
# Preempt the lowest-priority sequence groups.
111
victim_seq_group = self.running.pop(-1)
@@ -119,7 +119,7 @@ class Scheduler:
119
break
120
else:
121
# Append new slots to the sequence group.
122
- self._append(seq_group, blocks_to_copy)
122
+ self._append_slot(seq_group, blocks_to_copy)
123
running.append(seq_group)
124
self.running = running
125
@@ -143,7 +143,7 @@ class Scheduler:
143
144
seq_group = self.swapped.pop(0)
145
self._swap_in(seq_group, blocks_to_swap_in)
146
- self._append(seq_group, blocks_to_copy)
146
+ self._append_slot(seq_group, blocks_to_copy)
147
self.running.append(seq_group)
148
149
num_batched_tokens = sum(
@@ -252,7 +252,7 @@ class Scheduler:
252
prompt_group_ids = scheduler_output[3]
253
254
# Create input data structures.
255
- input_seq_groups: List[SequenceGroupInputs] = []
255
+ seq_group_metadata_list: List[SequenceGroupMetadata] = []
256
updated_seq_groups: List[SequenceGroup] = self.running.copy()
257
258
for seq_group in self.running:
@@ -274,7 +274,7 @@ class Scheduler:
274
# sequence length
275
seq_len = seq.get_len()
276
277
- input_seq_group = SequenceGroupInputs(
277
+ seq_group_metadata = SequenceGroupMetadata(
278
group_id=group_id,
279
is_prompt=is_prompt,
280
input_tokens=input_tokens,
@@ -283,14 +283,14 @@ class Scheduler:
283
sampling_params=self.sampling_params[group_id],
284
block_tables=block_tables,
285
)
286
- input_seq_groups.append(input_seq_group)
286
+ seq_group_metadata_list.append(seq_group_metadata)
287
288
# Execute the first stage of the pipeline.
289
- if input_seq_groups or blocks_to_swap_in or blocks_to_swap_out:
289
+ if seq_group_metadata_list or blocks_to_swap_in or blocks_to_swap_out:
290
# Swap in and swap out should never happen at the same time.
291
assert not (blocks_to_swap_in and blocks_to_swap_out)
292
self.controllers[0].execute_stage(
293
- input_seq_groups,
293
+ seq_group_metadata_list,
294
blocks_to_swap_in=blocks_to_swap_in,
295
blocks_to_swap_out=blocks_to_swap_out,
296
blocks_to_copy=blocks_to_copy,
@@ -330,7 +330,7 @@ class Scheduler:
330
331
# Append a new token to the sequence.
332
output = seq_outputs[seq.seq_id]
333
- seq.append(output.output_token, output.logprobs)
333
+ seq.append_token(output.output_token, output.logprobs)
334
335
# Check if the sequence has generated a stop token.
336
if output.output_token in stop_token_ids:
@@ -360,13 +360,13 @@ class Scheduler:
360
if seq_group.group_id not in self.num_steps:
361
self.num_steps[seq_group.group_id] = 0
362
363
- def _append(
363
+ def _append_slot(
364
self,
365
seq_group: SequenceGroup,
366
blocks_to_copy: Dict[int, List[int]],
367
) -> None:
368
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
369
- ret = self.block_manager.append(seq)
369
+ ret = self.block_manager.append_slot(seq)
370
if ret is not None:
371
src_block, dst_block = ret
372
if src_block in blocks_to_copy:
cacheflow/sampling_params.py
+2
-10
@@ -1,4 +1,4 @@
1
-from typing import Optional, Set, Dict
1
+from typing import Dict, Set
2
3
4
class SamplingParams:
@@ -12,7 +12,6 @@ class SamplingParams:
12
stop_token_ids: Set[int],
13
max_num_steps: int,
14
num_logprobs: int,
15
- context_window_size: Optional[int],
15
) -> None:
16
if n < 1:
17
raise ValueError(f'n must be at least 1, got {n}.')
@@ -27,10 +26,6 @@ class SamplingParams:
26
if num_logprobs < 0:
27
raise ValueError(
28
f'num_logprobs must be non-negative, got {num_logprobs}.')
30
- if context_window_size is not None and context_window_size < 0:
31
- raise ValueError(
32
- 'context_window_size must be non-negative, '
33
- f'got {context_window_size}.')
29
30
if use_beam_search:
31
if n == 1:
@@ -58,7 +53,6 @@ class SamplingParams:
53
self.stop_token_ids = stop_token_ids
54
self.max_num_steps = max_num_steps
55
self.num_logprobs = num_logprobs
61
- self.context_window_size = context_window_size
56
57
def __repr__(self) -> str:
58
return (f'SamplingParams(n={self.n}, '
@@ -67,8 +61,7 @@ class SamplingParams:
61
f'use_beam_search={self.use_beam_search}, '
62
f'stop_token_ids={self.stop_token_ids}, '
63
f'max_num_steps={self.max_num_steps}, '
70
- f'num_logprobs={self.num_logprobs}, '
71
- f'context_window_size={self.context_window_size})')
64
+ f'num_logprobs={self.num_logprobs}')
65
66
@classmethod
67
def from_dict(cls, d: Dict) -> 'SamplingParams':
@@ -80,5 +73,4 @@ class SamplingParams:
73
stop_token_ids=set(d.get('stop_token_ids', set())),
74
max_num_steps=d.get('max_num_steps', 16),
75
num_logprobs=d.get('num_logprobs', 0),
83
- context_window_size=d.get('context_window_size', None),
76
)
cacheflow/sequence.py
+13
-12
@@ -18,45 +18,46 @@ class Sequence:
18
def __init__(
19
self,
20
seq_id: int,
21
- token_ids: List[int],
21
+ prompt_token_ids: List[int],
22
block_size: int,
23
) -> None:
24
self.seq_id = seq_id
25
self.block_size = block_size
26
27
+ self.prompt_len = len(prompt_token_ids)
28
self.logical_token_blocks: List[LogicalTokenBlock] = []
28
- # Initialize the logical token blocks with the given token ids.
29
- self.add(token_ids)
29
+ # Initialize the logical token blocks with the prompt token ids.
30
+ self._append_tokens(prompt_token_ids)
31
31
- self.prompt_len = len(token_ids)
32
self.status = SequenceStatus.WAITING
33
+ # Used for beam search.
34
self.output_logprobs: List[Dict[int, float]] = []
35
self.cumulative_logprobs = 0.0
36
36
- def add_block(self) -> None:
37
+ def _append_logical_block(self) -> None:
38
block = LogicalTokenBlock(
39
block_number=len(self.logical_token_blocks),
40
block_size=self.block_size,
41
)
42
self.logical_token_blocks.append(block)
43
43
- def add(self, token_ids: List[int]) -> None:
44
+ def _append_tokens(self, token_ids: List[int]) -> None:
45
while token_ids:
46
if not self.logical_token_blocks:
46
- self.add_block()
47
+ self._append_logical_block()
48
49
last_block = self.logical_token_blocks[-1]
50
if last_block.is_full():
50
- self.add_block()
51
+ self._append_logical_block()
52
last_block = self.logical_token_blocks[-1]
53
54
num_empty_slots = last_block.get_num_empty_slots()
54
- last_block.append(token_ids[:num_empty_slots])
55
+ last_block.append_tokens(token_ids[:num_empty_slots])
56
token_ids = token_ids[num_empty_slots:]
57
57
- def append(self, token_id: int, logprobs: Dict[int, float]) -> None:
58
+ def append_token(self, token_id: int, logprobs: Dict[int, float]) -> None:
59
assert token_id in logprobs
59
- self.add([token_id])
60
+ self._append_tokens([token_id])
61
self.output_logprobs.append(logprobs)
62
self.cumulative_logprobs += logprobs[token_id]
63
@@ -121,7 +122,7 @@ class SequenceGroup:
122
f'num_seqs={len(self.seqs)})')
123
124
124
-class SequenceGroupInputs:
125
+class SequenceGroupMetadata:
126
127
def __init__(
128
self,
cacheflow/worker/controller.py
+3
-15
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Union, Tuple, Optional
1
+from typing import List, Optional, Tuple, Union
2
3
try:
4
import ray
@@ -6,7 +6,6 @@ except ImportError:
6
ray = None
7
8
from cacheflow.core.scheduler import Scheduler
9
-from cacheflow.sequence import SequenceGroupInputs
9
from cacheflow.worker.worker import Worker
10
11
@@ -81,23 +80,12 @@ class Controller:
80
self.next_node = next_node
81
self.is_last_stage = isinstance(next_node, Scheduler)
82
84
- def execute_stage(
85
- self,
86
- input_seq_groups: List[SequenceGroupInputs],
87
- blocks_to_swap_in: Dict[int, int],
88
- blocks_to_swap_out: Dict[int, int],
89
- blocks_to_copy: Dict[int, List[int]],
90
- ) -> None:
83
+ def execute_stage(self, *args, **kwargs) -> None:
84
all_outputs = []
85
for worker in self.workers:
86
executor = (worker.execute_stage.remote
87
if self.use_ray else worker.execute_stage)
95
- output = executor(
96
- input_seq_groups,
97
- blocks_to_swap_in,
98
- blocks_to_swap_out,
99
- blocks_to_copy,
100
- )
88
+ output = executor(*args, **kwargs)
89
all_outputs.append(output)
90
91
if self.use_ray:
cacheflow/worker/worker.py
+29
-29
@@ -8,10 +8,11 @@ from cacheflow.model_executor.parallel_utils.parallel_state import (
8
initialize_all_reduce_launcher,
9
get_tensor_model_parallel_world_size)
10
from cacheflow.sampling_params import SamplingParams
11
-from cacheflow.sequence import SequenceGroupInputs
11
+from cacheflow.sequence import SequenceGroupMetadata
12
from cacheflow.sequence import SequenceOutputs
13
from cacheflow.worker.cache_engine import CacheEngine
14
15
+
16
class Worker:
17
18
def __init__(
@@ -93,30 +94,29 @@ class Worker:
94
95
def prepare_inputs(
96
self,
96
- input_seq_groups: List[SequenceGroupInputs],
97
+ seq_group_metadata_list: List[SequenceGroupMetadata],
98
) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
99
seq_groups: List[Tuple[List[int], SamplingParams]] = []
100
seq_logprobs: Dict[int, float] = {}
100
- sampling_params: Dict[int, SamplingParams] = {}
101
input_tokens: List[int] = []
102
input_positions: List[int] = []
103
slot_mapping: List[int] = []
104
105
# Add prompt tokens.
106
prompt_lens: List[int] = []
107
- for input_seq_group in input_seq_groups:
108
- if not input_seq_group.is_prompt:
107
+ for seq_group_metadata in seq_group_metadata_list:
108
+ if not seq_group_metadata.is_prompt:
109
continue
110
111
- seq_ids = list(input_seq_group.input_tokens.keys())
112
- sampling_params = input_seq_group.sampling_params
111
+ seq_ids = list(seq_group_metadata.input_tokens.keys())
112
+ sampling_params = seq_group_metadata.sampling_params
113
seq_groups.append((seq_ids, sampling_params))
114
- seq_logprobs.update(input_seq_group.seq_logprobs)
114
+ seq_logprobs.update(seq_group_metadata.seq_logprobs)
115
116
# Use any sequence in the group.
117
seq_id = seq_ids[0]
118
119
- prompt_tokens = input_seq_group.input_tokens[seq_id]
119
+ prompt_tokens = seq_group_metadata.input_tokens[seq_id]
120
prompt_len = len(prompt_tokens)
121
prompt_lens.append(prompt_len)
122
@@ -126,7 +126,7 @@ class Worker:
126
input_positions.extend(range(len(prompt_tokens)))
127
128
# Compute the slot mapping.
129
- block_table = input_seq_group.block_tables[seq_id]
129
+ block_table = seq_group_metadata.block_tables[seq_id]
130
for i in range(prompt_len):
131
block_number = block_table[i // self.block_size]
132
block_offset = i % self.block_size
@@ -138,31 +138,31 @@ class Worker:
138
max_num_blocks_per_seq = 0
139
context_lens: List[int] = []
140
generation_block_tables: List[List[int]] = []
141
- for input_seq_group in input_seq_groups:
142
- if input_seq_group.is_prompt:
141
+ for seq_group_metadata in seq_group_metadata_list:
142
+ if seq_group_metadata.is_prompt:
143
continue
144
145
- seq_ids = list(input_seq_group.input_tokens.keys())
146
- sampling_params = input_seq_group.sampling_params
145
+ seq_ids = list(seq_group_metadata.input_tokens.keys())
146
+ sampling_params = seq_group_metadata.sampling_params
147
seq_groups.append((seq_ids, sampling_params))
148
- seq_logprobs.update(input_seq_group.seq_logprobs)
148
+ seq_logprobs.update(seq_group_metadata.seq_logprobs)
149
150
for seq_id in seq_ids:
151
- assert len(input_seq_group.input_tokens[seq_id]) == 1
152
- generation_token = input_seq_group.input_tokens[seq_id][0]
151
+ assert len(seq_group_metadata.input_tokens[seq_id]) == 1
152
+ generation_token = seq_group_metadata.input_tokens[seq_id][0]
153
input_tokens.append(generation_token)
154
155
- position = input_seq_group.context_len - 1
155
+ position = seq_group_metadata.context_len - 1
156
input_positions.append(position)
157
158
- block_table = input_seq_group.block_tables[seq_id]
158
+ block_table = seq_group_metadata.block_tables[seq_id]
159
generation_block_tables.append(block_table)
160
161
max_context_len = max(
162
- max_context_len, input_seq_group.context_len)
162
+ max_context_len, seq_group_metadata.context_len)
163
max_num_blocks_per_seq = max(
164
max_num_blocks_per_seq, len(block_table))
165
- context_lens.append(input_seq_group.context_len)
165
+ context_lens.append(seq_group_metadata.context_len)
166
167
block_number = block_table[position // self.block_size]
168
block_offset = position % self.block_size
@@ -203,30 +203,30 @@ class Worker:
203
@torch.inference_mode()
204
def execute_stage(
205
self,
206
- input_seq_groups: List[SequenceGroupInputs],
206
+ seq_group_metadata_list: List[SequenceGroupMetadata],
207
blocks_to_swap_in: Dict[int, int],
208
blocks_to_swap_out: Dict[int, int],
209
blocks_to_copy: Dict[int, List[int]],
210
) -> Dict[int, SequenceOutputs]:
211
# Issue cache operations.
212
- command_issued = False
212
+ issued_cache_op = False
213
if blocks_to_swap_in:
214
self.cache_engine.swap_in(blocks_to_swap_in)
215
- command_issued = True
215
+ issued_cache_op = True
216
if blocks_to_swap_out:
217
self.cache_engine.swap_out(blocks_to_swap_out)
218
- command_issued = True
218
+ issued_cache_op = True
219
if blocks_to_copy:
220
self.cache_engine.copy(blocks_to_copy)
221
- command_issued = True
221
+ issued_cache_op = True
222
223
- if command_issued:
223
+ if issued_cache_op:
224
cache_events = self.cache_events
225
else:
226
cache_events = None
227
228
# If there is no input, we don't need to execute the model.
229
- if not input_seq_groups:
229
+ if not seq_group_metadata_list:
230
if cache_events is not None:
231
for event in cache_events:
232
event.wait()
@@ -234,7 +234,7 @@ class Worker:
234
235
# Prepare input tensors.
236
input_tokens, input_positions, input_metadata = self.prepare_inputs(
237
- input_seq_groups)
237
+ seq_group_metadata_list)
238
239
# Execute the model.
240
output = self.model(