@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1132fae0ca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add Frontend

Woosuk Kwon committed Feb 24, 2023 at 11:46 UTC 1132fae0ca7f6103b0dd7b96932c75f5ed780288
2 files changed +80 -17
cacheflow/master/frontend.py new
+56
@@ -0,0 +1,56 @@
1 +from typing import List, Optional, Tuple
2 +
3 +from transformers import AutoTokenizer
4 +
5 +from cacheflow.sampling_params import SamplingParams
6 +from cacheflow.sequence import Sequence
7 +from cacheflow.sequence import SequenceGroup
8 +from cacheflow.utils import Counter
9 +
10 +
11 +class Frontend:
12 +
13 + def __init__(
14 + self,
15 + model_name: str,
16 + block_size: int,
17 + ) -> None:
18 + self.block_size = block_size
19 +
20 + self.tokenizer = AutoTokenizer.from_pretrained(model_name)
21 + self.seq_group_counter = Counter()
22 + self.seq_counter = Counter()
23 + self.inputs: List[Tuple[SequenceGroup, SamplingParams]] = []
24 +
25 + def query(
26 + self,
27 + prompt: str,
28 + sampling_params: Optional[SamplingParams] = None,
29 + ) -> None:
30 + if sampling_params is None:
31 + sampling_params = SamplingParams()
32 + token_ids: List[int] = self.tokenizer.encode(prompt)
33 +
34 + seqs: List[Sequence] = []
35 + for _ in range(sampling_params.n):
36 + seq_id = next(self.seq_counter)
37 + seq = Sequence(seq_id, token_ids, block_size=self.block_size)
38 + seqs.append(seq)
39 +
40 + group_id = next(self.seq_group_counter)
41 + seq_group = SequenceGroup(group_id, seqs)
42 + self.inputs.append((seq_group, sampling_params))
43 +
44 + def get_inputs(self) -> List[Tuple[SequenceGroup, SamplingParams]]:
45 + inputs = self.inputs
46 + self.inputs = []
47 + return inputs
48 +
49 + def print_response(
50 + self,
51 + seq_group: SequenceGroup,
52 + ) -> None:
53 + for seq in seq_group.seqs:
54 + token_ids = seq.get_token_ids()
55 + output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
56 + print(f'Seq {seq.seq_id}: {output}')
cacheflow/master/scheduler.py
+24 -17
@@ -1,6 +1,8 @@
1 from typing import Dict, List, Tuple
2
3 from cacheflow.master.block_manager import BlockSpaceManager
4 +from cacheflow.master.frontend import Frontend
5 +from cacheflow.sampling_params import SamplingParams
6 from cacheflow.sequence import Sequence
7 from cacheflow.sequence import SequenceGroup
8 from cacheflow.sequence import SequenceStatus
@@ -12,11 +14,13 @@ class Scheduler:
14
15 def __init__(
16 self,
17 + frontend: Frontend,
18 controllers: List,
19 block_size: int,
20 num_gpu_blocks: int,
21 num_cpu_blocks: int,
22 ) -> None:
23 + self.frontend = frontend
24 self.controllers = controllers
25 self.block_size = block_size
26 self.num_gpu_blocks = num_gpu_blocks
@@ -33,16 +37,20 @@ class Scheduler:
37 self.running: List[SequenceGroup] = []
38 # Mapping: group_id -> num_steps.
39 self.num_steps: Dict[int, int] = {}
36 - # Mapping: group_id -> max_num_steps.
37 - self.max_num_steps: Dict[int, int] = {}
38 - # Mapping: group_id -> stop_token_ids.
39 - self.stop_token_ids: Dict[int, List[int]] = {}
40 + # Mapping: group_id -> sampling params.
41 + self.sampling_params: Dict[int, SamplingParams] = {}
42
43 # Swapped sequence groups (LIFO).
44 self.swapped: List[SequenceGroup] = []
45 # Pending sequence groups (FIFO).
46 self.pending: List[SequenceGroup] = []
47
48 + def _fetch_inputs(self) -> None:
49 + inputs = self.frontend.get_inputs()
50 + for seq_group, sampling_params in inputs:
51 + self.pending.append(seq_group)
52 + self.sampling_params[seq_group.group_id] = sampling_params
53 +
54 def _free_seq(self, seq: Sequence) -> None:
55 seq.status = SequenceStatus.FINISHED
56 self.block_manager.free(seq)
@@ -145,6 +153,7 @@ class Scheduler:
153 # TODO(woosuk): Add a batching policy to control the batch size.
154 if not self.swapped:
155 # FIXME(woosuk): Acquire a lock to protect pending.
156 + self._fetch_inputs()
157 for i, seq_group in enumerate(self.pending):
158 num_prompt_tokens = seq_group.seqs[0].get_len()
159 if self.block_manager.can_allocate(seq_group):
@@ -205,7 +214,7 @@ class Scheduler:
214 for seq_group in self.running:
215 group_id = seq_group.group_id
216 self.num_steps[group_id] += 1
208 - stop_token_ids = self.stop_token_ids[group_id]
217 + stop_token_ids = self.sampling_params[group_id].stop_token_ids
218
219 for seq in seq_group.seqs:
220 if seq.status == SequenceStatus.FINISHED:
@@ -230,24 +239,22 @@ class Scheduler:
239 continue
240
241 # Check if the sequence has reached the maximum number of steps.
233 - if self.num_steps[group_id] == self.max_num_steps[group_id]:
242 + max_num_steps = self.sampling_params[group_id].max_num_steps
243 + if self.num_steps[group_id] == max_num_steps:
244 self._free_seq(seq)
245 continue
246
247 # Update the running sequences.
248 running: List[SequenceGroup] = []
249 for seq_group in self.running:
240 - if all(seq.status == SequenceStatus.FINISHED for seq in seq_group.seqs):
241 - del self.num_steps[seq_group.group_id]
242 - del self.max_num_steps[seq_group.group_id]
243 - del self.stop_token_ids[seq_group.group_id]
244 - # TODO: Return the seq_group to the client.
245 - from transformers import AutoTokenizer
246 - tokenizer = AutoTokenizer.from_pretrained('facebook/opt-125m')
247 - for seq in seq_group.seqs:
248 - token_ids = seq.get_token_ids()
249 - output = tokenizer.decode(token_ids, skip_special_tokens=True)
250 - print(f'Seq {seq.seq_id}: {output}')
250 + if seq_group.is_finished():
251 + self._return(seq_group)
252 else:
253 running.append(seq_group)
254 self.running = running
255 +
256 + def _return(self, seq_group: SequenceGroup) -> None:
257 + group_id = seq_group.group_id
258 + del self.num_steps[group_id]
259 + del self.sampling_params[group_id]
260 + self.frontend.print_response(seq_group)