A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add Frontend
Woosuk Kwon committed
Feb 24, 2023 at 11:46 UTC
1132fae0ca7f6103b0dd7b96932c75f5ed780288
2 files changed
+80
-17
cacheflow/master/frontend.py
new
+56
@@ -0,0 +1,56 @@
1
+from typing import List, Optional, Tuple
2
+
3
+from transformers import AutoTokenizer
4
+
5
+from cacheflow.sampling_params import SamplingParams
6
+from cacheflow.sequence import Sequence
7
+from cacheflow.sequence import SequenceGroup
8
+from cacheflow.utils import Counter
9
+
10
+
11
+class Frontend:
12
+
13
+ def __init__(
14
+ self,
15
+ model_name: str,
16
+ block_size: int,
17
+ ) -> None:
18
+ self.block_size = block_size
19
+
20
+ self.tokenizer = AutoTokenizer.from_pretrained(model_name)
21
+ self.seq_group_counter = Counter()
22
+ self.seq_counter = Counter()
23
+ self.inputs: List[Tuple[SequenceGroup, SamplingParams]] = []
24
+
25
+ def query(
26
+ self,
27
+ prompt: str,
28
+ sampling_params: Optional[SamplingParams] = None,
29
+ ) -> None:
30
+ if sampling_params is None:
31
+ sampling_params = SamplingParams()
32
+ token_ids: List[int] = self.tokenizer.encode(prompt)
33
+
34
+ seqs: List[Sequence] = []
35
+ for _ in range(sampling_params.n):
36
+ seq_id = next(self.seq_counter)
37
+ seq = Sequence(seq_id, token_ids, block_size=self.block_size)
38
+ seqs.append(seq)
39
+
40
+ group_id = next(self.seq_group_counter)
41
+ seq_group = SequenceGroup(group_id, seqs)
42
+ self.inputs.append((seq_group, sampling_params))
43
+
44
+ def get_inputs(self) -> List[Tuple[SequenceGroup, SamplingParams]]:
45
+ inputs = self.inputs
46
+ self.inputs = []
47
+ return inputs
48
+
49
+ def print_response(
50
+ self,
51
+ seq_group: SequenceGroup,
52
+ ) -> None:
53
+ for seq in seq_group.seqs:
54
+ token_ids = seq.get_token_ids()
55
+ output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
56
+ print(f'Seq {seq.seq_id}: {output}')
cacheflow/master/scheduler.py
+24
-17
@@ -1,6 +1,8 @@
1
from typing import Dict, List, Tuple
2
3
from cacheflow.master.block_manager import BlockSpaceManager
4
+from cacheflow.master.frontend import Frontend
5
+from cacheflow.sampling_params import SamplingParams
6
from cacheflow.sequence import Sequence
7
from cacheflow.sequence import SequenceGroup
8
from cacheflow.sequence import SequenceStatus
@@ -12,11 +14,13 @@ class Scheduler:
14
15
def __init__(
16
self,
17
+ frontend: Frontend,
18
controllers: List,
19
block_size: int,
20
num_gpu_blocks: int,
21
num_cpu_blocks: int,
22
) -> None:
23
+ self.frontend = frontend
24
self.controllers = controllers
25
self.block_size = block_size
26
self.num_gpu_blocks = num_gpu_blocks
@@ -33,16 +37,20 @@ class Scheduler:
37
self.running: List[SequenceGroup] = []
38
# Mapping: group_id -> num_steps.
39
self.num_steps: Dict[int, int] = {}
36
- # Mapping: group_id -> max_num_steps.
37
- self.max_num_steps: Dict[int, int] = {}
38
- # Mapping: group_id -> stop_token_ids.
39
- self.stop_token_ids: Dict[int, List[int]] = {}
40
+ # Mapping: group_id -> sampling params.
41
+ self.sampling_params: Dict[int, SamplingParams] = {}
42
43
# Swapped sequence groups (LIFO).
44
self.swapped: List[SequenceGroup] = []
45
# Pending sequence groups (FIFO).
46
self.pending: List[SequenceGroup] = []
47
48
+ def _fetch_inputs(self) -> None:
49
+ inputs = self.frontend.get_inputs()
50
+ for seq_group, sampling_params in inputs:
51
+ self.pending.append(seq_group)
52
+ self.sampling_params[seq_group.group_id] = sampling_params
53
+
54
def _free_seq(self, seq: Sequence) -> None:
55
seq.status = SequenceStatus.FINISHED
56
self.block_manager.free(seq)
@@ -145,6 +153,7 @@ class Scheduler:
153
# TODO(woosuk): Add a batching policy to control the batch size.
154
if not self.swapped:
155
# FIXME(woosuk): Acquire a lock to protect pending.
156
+ self._fetch_inputs()
157
for i, seq_group in enumerate(self.pending):
158
num_prompt_tokens = seq_group.seqs[0].get_len()
159
if self.block_manager.can_allocate(seq_group):
@@ -205,7 +214,7 @@ class Scheduler:
214
for seq_group in self.running:
215
group_id = seq_group.group_id
216
self.num_steps[group_id] += 1
208
- stop_token_ids = self.stop_token_ids[group_id]
217
+ stop_token_ids = self.sampling_params[group_id].stop_token_ids
218
219
for seq in seq_group.seqs:
220
if seq.status == SequenceStatus.FINISHED:
@@ -230,24 +239,22 @@ class Scheduler:
239
continue
240
241
# Check if the sequence has reached the maximum number of steps.
233
- if self.num_steps[group_id] == self.max_num_steps[group_id]:
242
+ max_num_steps = self.sampling_params[group_id].max_num_steps
243
+ if self.num_steps[group_id] == max_num_steps:
244
self._free_seq(seq)
245
continue
246
247
# Update the running sequences.
248
running: List[SequenceGroup] = []
249
for seq_group in self.running:
240
- if all(seq.status == SequenceStatus.FINISHED for seq in seq_group.seqs):
241
- del self.num_steps[seq_group.group_id]
242
- del self.max_num_steps[seq_group.group_id]
243
- del self.stop_token_ids[seq_group.group_id]
244
- # TODO: Return the seq_group to the client.
245
- from transformers import AutoTokenizer
246
- tokenizer = AutoTokenizer.from_pretrained('facebook/opt-125m')
247
- for seq in seq_group.seqs:
248
- token_ids = seq.get_token_ids()
249
- output = tokenizer.decode(token_ids, skip_special_tokens=True)
250
- print(f'Seq {seq.seq_id}: {output}')
250
+ if seq_group.is_finished():
251
+ self._return(seq_group)
252
else:
253
running.append(seq_group)
254
self.running = running
255
+
256
+ def _return(self, seq_group: SequenceGroup) -> None:
257
+ group_id = seq_group.group_id
258
+ del self.num_steps[group_id]
259
+ del self.sampling_params[group_id]
260
+ self.frontend.print_response(seq_group)