@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1a7eb7da61
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Support beam search & parallel generation (#7)

Woosuk Kwon committed Mar 10, 2023 at 09:58 UTC 1a7eb7da6157541ed7867c9aff94231695f2cee9
16 files changed +662 -163
cacheflow/block.py
+4
@@ -35,6 +35,10 @@ class LogicalTokenBlock:
35 def get_token_ids(self) -> List[int]:
36 return self.token_ids[:self.num_tokens]
37
38 + def get_last_token_id(self) -> int:
39 + assert self.num_tokens > 0
40 + return self.token_ids[self.num_tokens - 1]
41 +
42
43 class PhysicalTokenBlock:
44
cacheflow/master/frontend.py
+28 -5
@@ -1,4 +1,4 @@
1 -from typing import List, Optional, Tuple
1 +from typing import List, Optional, Set, Tuple
2
3 from transformers import AutoTokenizer
4
@@ -25,12 +25,35 @@ class Frontend:
25 def query(
26 self,
27 prompt: str,
28 - sampling_params: Optional[SamplingParams] = None,
28 + n: int = 1,
29 + temperature: float = 1.0,
30 + top_p: float = 1.0,
31 + use_beam_search: bool = False,
32 + stop_token_ids: Set[int] = set(),
33 + max_num_steps: int = 16, # From OpenAI API.
34 + num_logprobs: int = 0,
35 + context_window_size: Optional[int] = None,
36 ) -> None:
30 - if sampling_params is None:
31 - sampling_params = SamplingParams()
32 - token_ids: List[int] = self.tokenizer.encode(prompt)
37 + # Stop when we see an EOS token.
38 + stop_token_ids.add(self.tokenizer.eos_token_id)
39 + sampling_params = SamplingParams(
40 + n=n,
41 + temperature=temperature,
42 + top_p=top_p,
43 + use_beam_search=use_beam_search,
44 + stop_token_ids=stop_token_ids,
45 + max_num_steps=max_num_steps,
46 + num_logprobs=num_logprobs,
47 + context_window_size=context_window_size,
48 + )
49 + token_ids = self.tokenizer.encode(prompt)
50 + self._add_query(token_ids, sampling_params)
51
52 + def _add_query(
53 + self,
54 + token_ids: List[int],
55 + sampling_params: SamplingParams,
56 + ) -> None:
57 seqs: List[Sequence] = []
58 for _ in range(sampling_params.n):
59 seq_id = next(self.seq_counter)
cacheflow/master/scheduler.py
+55 -36
@@ -1,10 +1,12 @@
1 -from typing import Dict, List, Tuple
1 +from typing import Dict, List
2
3 from cacheflow.master.block_manager import BlockSpaceManager
4 from cacheflow.master.frontend import Frontend
5 from cacheflow.sampling_params import SamplingParams
6 from cacheflow.sequence import Sequence
7 from cacheflow.sequence import SequenceGroup
8 +from cacheflow.sequence import SequenceGroupInputs
9 +from cacheflow.sequence import SequenceOutputs
10 from cacheflow.sequence import SequenceStatus
11
12 _MAX_NUM_BATCHED_TOKENS = 2048
@@ -66,7 +68,7 @@ class Scheduler:
68 def _append(
69 self,
70 seq_group: SequenceGroup,
69 - blocks_to_copy: Dict[int, int],
71 + blocks_to_copy: Dict[int, List[int]],
72 ) -> None:
73 for seq in seq_group.seqs:
74 if seq.status == SequenceStatus.FINISHED:
@@ -74,7 +76,10 @@ class Scheduler:
76 ret = self.block_manager.append(seq)
77 if ret is not None:
78 src_block, dst_block = ret
77 - blocks_to_copy[src_block] = dst_block
79 + if src_block in blocks_to_copy:
80 + blocks_to_copy[src_block].append(dst_block)
81 + else:
82 + blocks_to_copy[src_block] = [dst_block]
83
84 def _swap_in(
85 self,
@@ -83,9 +88,8 @@ class Scheduler:
88 ) -> None:
89 mapping = self.block_manager.swap_in(seq_group)
90 blocks_to_swap_in.update(mapping)
86 - for seq in seq_group.seqs:
87 - if seq.status == SequenceStatus.SWAPPED:
88 - seq.status = SequenceStatus.RUNNING
91 + for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
92 + seq.status = SequenceStatus.RUNNING
93 self.running.append(seq_group)
94
95 def _swap_out(
@@ -96,16 +100,15 @@ class Scheduler:
100 assert self.block_manager.can_swap_out(seq_group)
101 mapping = self.block_manager.swap_out(seq_group)
102 blocks_to_swap_out.update(mapping)
99 - for seq in seq_group.seqs:
100 - if seq.status == SequenceStatus.RUNNING:
101 - seq.status = SequenceStatus.SWAPPED
103 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
104 + seq.status = SequenceStatus.SWAPPED
105 self.swapped.append(seq_group)
106
107 def step(self) -> None:
108 # Blocks that need to be swaped or copied before model execution.
109 blocks_to_swap_in: Dict[int, int] = {}
110 blocks_to_swap_out: Dict[int, int] = {}
108 - blocks_to_copy: Dict[int, int] = {}
111 + blocks_to_copy: Dict[int, List[int]] = {}
112
113 # 1. Reserve new slots for the running sequences.
114 # NOTE: Here we implicitly assume FCFS scheduling.
@@ -143,6 +146,10 @@ class Scheduler:
146 # All swapped sequences are swapped in.
147 self.swapped.clear()
148
149 + # Ensure that swap-in and swap-out never happen at the same timestep.
150 + if blocks_to_swap_in:
151 + assert not blocks_to_swap_out
152 +
153 num_batched_tokens = sum(
154 seq_group.num_seqs(status=SequenceStatus.RUNNING)
155 for seq_group in self.running
@@ -152,7 +159,6 @@ class Scheduler:
159 # NOTE: Here we implicitly assume FCFS scheduling.
160 # TODO(woosuk): Add a batching policy to control the batch size.
161 if not self.swapped:
155 - # FIXME(woosuk): Acquire a lock to protect pending.
162 self._fetch_inputs()
163 for i, seq_group in enumerate(self.pending):
164 num_prompt_tokens = seq_group.seqs[0].get_len()
@@ -168,39 +174,45 @@ class Scheduler:
174 else:
175 self.pending.clear()
176
171 - # Ensure that swap-in and swap-out never happen at the same timestep.
172 - if blocks_to_swap_in:
173 - assert not blocks_to_swap_out
174 -
177 # 4. Create input data structures.
176 - prompt_tokens: Dict[int, List[int]] = {}
177 - generation_tokens: Dict[int, int] = {}
178 - context_lens: Dict[int, int] = {}
179 - block_tables: Dict[int, List[int]] = {}
178 + input_seq_groups: List[SequenceGroupInputs] = []
179 for seq_group in self.running:
180 group_id = seq_group.group_id
181 num_steps = self.num_steps[group_id]
182 +
183 # NOTE(woosuk): We assume that the number of steps is 0
184 # for the prompt sequences.
185 is_prompt = num_steps == 0
186 - for seq in seq_group.seqs:
187 - if seq.status != SequenceStatus.RUNNING:
188 - continue
186
187 + input_tokens: Dict[int, List[int]] = {}
188 + seq_logprobs: Dict[int, float] = {}
189 + block_tables: Dict[int, List[int]] = {}
190 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
191 seq_id = seq.seq_id
192 block_tables[seq_id] = self.block_manager.get_block_table(seq)
193 if is_prompt:
193 - prompt_tokens[seq_id] = seq.get_token_ids()
194 + input_tokens[seq_id] = seq.get_token_ids()
195 else:
195 - generation_tokens[seq_id] = seq.get_token_ids()[-1]
196 - context_lens[seq_id] = seq.get_len()
196 + input_tokens[seq_id] = [seq.get_last_token_id()]
197 + seq_logprobs[seq_id] = seq.cumulative_logprobs
198 + # NOTE(woosuk): Sequences in the same group have the same
199 + # sequence length
200 + seq_len = seq.get_len()
201 +
202 + input_seq_group = SequenceGroupInputs(
203 + group_id=group_id,
204 + is_prompt=is_prompt,
205 + input_tokens=input_tokens,
206 + context_len=seq_len,
207 + seq_logprobs=seq_logprobs,
208 + sampling_params=self.sampling_params[group_id],
209 + block_tables=block_tables,
210 + )
211 + input_seq_groups.append(input_seq_group)
212
213 # 5. Execute the first stage of the pipeline.
214 self.controllers[0].execute_stage(
200 - prompt_tokens,
201 - generation_tokens,
202 - context_lens,
203 - block_tables,
215 + input_seq_groups,
216 blocks_to_swap_in,
217 blocks_to_swap_out,
218 blocks_to_copy,
@@ -208,7 +220,7 @@ class Scheduler:
220
221 def post_step(
222 self,
211 - next_tokens: Dict[int, Tuple[int, int]],
223 + seq_outputs: Dict[int, SequenceOutputs],
224 ) -> None:
225 # Update the running sequences and free blocks.
226 for seq_group in self.running:
@@ -216,25 +228,32 @@ class Scheduler:
228 self.num_steps[group_id] += 1
229 stop_token_ids = self.sampling_params[group_id].stop_token_ids
230
231 + # Process beam search results before processing the next tokens.
232 for seq in seq_group.seqs:
233 if seq.status == SequenceStatus.FINISHED:
234 continue
235
223 - parent_seq_id, next_token = next_tokens[seq.seq_id]
224 - if seq.seq_id != parent_seq_id:
236 + output = seq_outputs[seq.seq_id]
237 + if seq.seq_id != output.parent_seq_id:
238 # The sequence is a fork of the parent sequence (beam search).
239 # Free the current sequence.
240 self.block_manager.free(seq)
241 # Fork the parent sequence.
229 - parent_seq = seq_group.find(parent_seq_id)
230 - seq.logical_token_blocks = parent_seq.logical_token_blocks.copy()
242 + parent_seq = seq_group.find(output.parent_seq_id)
243 + parent_seq.fork(seq)
244 self.block_manager.fork(parent_seq, seq)
245
246 + # Process the next tokens.
247 + for seq in seq_group.seqs:
248 + if seq.status == SequenceStatus.FINISHED:
249 + continue
250 +
251 # Append a new token to the sequence.
234 - seq.append([next_token])
252 + output = seq_outputs[seq.seq_id]
253 + seq.append(output.output_token, output.logprobs)
254
255 # Check if the sequence has generated a stop token.
237 - if next_token in stop_token_ids:
256 + if output.output_token in stop_token_ids:
257 self._free_seq(seq)
258 continue
259
cacheflow/models/__init__.py
+3 -1
@@ -1,8 +1,10 @@
1 from cacheflow.models.input_metadata import InputMetadata
2 from cacheflow.models.model_utils import get_model
3 +from cacheflow.models.model_utils import set_seed
4
5
6 __all__ = [
6 - 'get_model',
7 'InputMetadata',
8 + 'get_model',
9 + 'set_seed'
10 ]
cacheflow/models/input_metadata.py
+11 -7
@@ -1,21 +1,24 @@
1 -from typing import List
1 +from typing import List, Dict, Tuple
2
3 import torch
4
5 +from cacheflow.sampling_params import SamplingParams
6 +
7
8 class InputMetadata:
9
10 def __init__(
11 self,
10 - seq_ids: List[int],
12 + seq_groups: List[Tuple[List[int], SamplingParams]],
13 + seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
14 prompt_lens: List[int],
15 slot_mapping: torch.Tensor,
16 context_lens: torch.Tensor,
14 - # FIXME: Rename
17 max_context_len: int,
18 block_tables: torch.Tensor,
19 ) -> None:
18 - self.seq_ids = seq_ids
20 + self.seq_groups = seq_groups
21 + self.seq_logprobs = seq_logprobs
22 self.prompt_lens = prompt_lens
23 self.slot_mapping = slot_mapping
24 self.context_lens = context_lens
@@ -23,19 +26,20 @@ class InputMetadata:
26 self.block_tables = block_tables
27
28 self.num_prompts = len(prompt_lens)
29 + self.num_prompt_tokens = sum(prompt_lens)
30 self.num_generation_tokens = context_lens.shape[0]
31 self.num_valid_tokens = slot_mapping.shape[0]
32 if block_tables.numel() > 0:
33 self.max_num_blocks_per_seq = block_tables.shape[1]
34 else:
35 self.max_num_blocks_per_seq = 0
32 - assert self.num_generation_tokens == block_tables.shape[0]
33 - assert self.num_prompts + self.num_generation_tokens == len(seq_ids)
36 + assert block_tables.shape[0] == self.num_generation_tokens
37 + assert context_lens.shape[0] == self.num_generation_tokens
38
39 def __repr__(self) -> str:
40 return (f'InputMetadata('
37 - f'seq_ids={self.seq_ids}, '
41 f'num_prompts={self.num_prompts}, '
42 + f'num_prompt_tokens={self.num_prompt_tokens}, '
43 f'num_generation_tokens={self.num_generation_tokens}, '
44 f'num_valid_tokens={self.num_valid_tokens}, '
45 f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
cacheflow/models/model_utils.py
+10
@@ -1,5 +1,7 @@
1 +import random
2 from typing import Union
3
4 +import numpy as np
5 import torch
6 import torch.nn as nn
7
@@ -30,3 +32,11 @@ def get_model(
32 model = hf_model.from_pretrained(model_name, torch_dtype=torch_dtype)
33 return model.eval()
34 raise ValueError(f'Invalid model name: {model_name}')
35 +
36 +
37 +def set_seed(seed: int) -> None:
38 + random.seed(seed)
39 + np.random.seed(seed)
40 + torch.manual_seed(seed)
41 + if torch.cuda.is_available():
42 + torch.cuda.manual_seed_all(seed)
cacheflow/models/opt.py
+2 -1
@@ -9,6 +9,7 @@ from transformers import PreTrainedModel
9 from cacheflow.models import InputMetadata
10 from cacheflow.models.attention import OPTCacheFlowAttention
11 from cacheflow.models.sample import Sampler
12 +from cacheflow.sequence import SequenceOutputs
13
14 KVCache = Tuple[torch.Tensor, torch.Tensor]
15
@@ -261,7 +262,7 @@ class OPTForCausalLM(OPTPreTrainedModel):
262 kv_caches: List[KVCache],
263 input_metadata: InputMetadata,
264 cache_events: Optional[List[torch.cuda.Event]],
264 - ) -> Dict[int, Tuple[int, int]]:
265 + ) -> Dict[int, SequenceOutputs]:
266 hidden_states = self.model(
267 input_ids, positions, kv_caches, input_metadata, cache_events)
268 next_tokens = self.sampler(
cacheflow/models/sample.py
+258 -17
@@ -4,6 +4,8 @@ import torch
4 import torch.nn as nn
5
6 from cacheflow.models import InputMetadata
7 +from cacheflow.sampling_params import SamplingParams
8 +from cacheflow.sequence import SequenceOutputs
9
10
11 class Sampler(nn.Module):
@@ -16,27 +18,266 @@ class Sampler(nn.Module):
18 embedding: torch.Tensor,
19 hidden_states: torch.Tensor,
20 input_metadata: InputMetadata,
19 - ) -> Dict[int, Tuple[int, int]]:
20 - # Get the hidden states of the last tokens.
21 - start_idx = 0
22 - last_token_indicies: List[int] = []
23 - for prompt_len in input_metadata.prompt_lens:
24 - last_token_indicies.append(start_idx + prompt_len - 1)
25 - start_idx += prompt_len
26 - last_token_indicies.extend(
27 - range(start_idx, start_idx + input_metadata.num_generation_tokens))
28 - hidden_states = hidden_states[last_token_indicies]
21 + ) -> Dict[int, SequenceOutputs]:
22 + # Get the hidden states that we use for sampling.
23 + hidden_states = _prune_hidden_states(hidden_states, input_metadata)
24
25 # Get the logits for the next tokens.
26 logits = torch.matmul(hidden_states, embedding.t())
27
28 + # Apply temperature scaling.
29 + temperatures = _get_temperatures(input_metadata)
30 + assert len(temperatures) == logits.shape[0]
31 + if any(t != 1.0 for t in temperatures):
32 + t = torch.tensor(
33 + temperatures, dtype=logits.dtype, device=logits.device)
34 + # Use in-place division to avoid creating a new tensor.
35 + logits.div_(t.unsqueeze(dim=1))
36 +
37 + # Compute the probabilities.
38 + probs = torch.softmax(logits, dim=-1, dtype=torch.float)
39 + # Compute the log probabilities (before applying top-p).
40 + logprobs = torch.log(probs)
41 +
42 + # Apply top-p truncation.
43 + top_ps = _get_top_ps(input_metadata)
44 + assert len(top_ps) == probs.shape[0]
45 + if any(p < 1.0 for p in top_ps):
46 + p = torch.tensor(top_ps, dtype=probs.dtype, device=probs.device)
47 + probs = _apply_top_p(probs, p)
48 +
49 # Sample the next tokens.
34 - # TODO(woosuk): Implement other sampling methods.
35 - next_token_ids = torch.argmax(logits, dim=-1)
50 + return _sample(probs, logprobs, input_metadata)
51 +
52 +
53 +def _prune_hidden_states(
54 + hidden_states: torch.Tensor,
55 + input_metadata: InputMetadata,
56 +) -> torch.Tensor:
57 + start_idx = 0
58 + last_token_indicies: List[int] = []
59 + for prompt_len in input_metadata.prompt_lens:
60 + last_token_indicies.append(start_idx + prompt_len - 1)
61 + start_idx += prompt_len
62 + last_token_indicies.extend(
63 + range(start_idx, start_idx + input_metadata.num_generation_tokens))
64 + return hidden_states[last_token_indicies]
65 +
66 +
67 +def _get_temperatures(
68 + input_metadata: InputMetadata,
69 +) -> List[float]:
70 + # Collect the temperatures for the logits.
71 + temperatures: List[float] = []
72 + for i, seq_group in enumerate(input_metadata.seq_groups):
73 + seq_ids, sampling_params = seq_group
74 + temperature = sampling_params.temperature
75 + if temperature == 0.0:
76 + # NOTE: Zero temperature means deterministic sampling
77 + # (i.e., greedy sampling or beam search).
78 + # Set the temperature to 1 to avoid division by zero.
79 + temperature = 1.0
80 +
81 + if i < input_metadata.num_prompts:
82 + # A prompt input.
83 + temperatures.append(temperature)
84 + else:
85 + # A generation token.
86 + temperatures += [temperature] * len(seq_ids)
87 + return temperatures
88 +
89 +
90 +def _get_top_ps(
91 + input_metadata: InputMetadata,
92 +) -> List[float]:
93 + top_ps: List[float] = []
94 + for i, seq_group in enumerate(input_metadata.seq_groups):
95 + seq_ids, sampling_params = seq_group
96 + if i < input_metadata.num_prompts:
97 + # A prompt input.
98 + top_ps.append(sampling_params.top_p)
99 + else:
100 + # A generation token.
101 + top_ps += [sampling_params.top_p] * len(seq_ids)
102 + return top_ps
103 +
104 +
105 +def _apply_top_p(
106 + probs: torch.Tensor,
107 + p: torch.Tensor,
108 +) -> torch.Tensor:
109 + # TODO(woosuk): Optimize.
110 + probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
111 + probs_sum = torch.cumsum(probs_sort, dim=-1)
112 + mask = (probs_sum - probs_sort) > p.unsqueeze(dim=1)
113 + probs_sort[mask] = 0.0
114 + probs_sort.div_(probs_sort.sum(dim=-1, keepdim=True))
115 + probs = torch.gather(
116 + probs_sort, dim=-1, index=torch.argsort(probs_idx, dim=-1))
117 + return probs
118 +
119 +
120 +def _get_topk_logprobs(
121 + logprobs: torch.Tensor,
122 + num_logprobs: int,
123 +) -> Dict[int, float]:
124 + if num_logprobs == 0:
125 + return {}
126 +
127 + topk_logprobs, topk_ids = torch.topk(logprobs, num_logprobs)
128 + if num_logprobs == 1:
129 + topk_logprobs = [topk_logprobs.item()]
130 + topk_ids = [topk_ids.item()]
131 + else:
132 + topk_logprobs = topk_logprobs.tolist()
133 + topk_ids = topk_ids.tolist()
134 +
135 + token_to_logprob: Dict[int, float] = {}
136 + for token_id, logprob in zip(topk_ids, topk_logprobs):
137 + token_to_logprob[token_id] = logprob
138 + return token_to_logprob
139 +
140 +
141 +def _sample_from_prompt(
142 + prob: torch.Tensor,
143 + sampling_params: SamplingParams,
144 +) -> List[int]:
145 + if sampling_params.use_beam_search:
146 + # Beam search.
147 + beam_width = sampling_params.n
148 + _, next_token_ids = torch.topk(prob, beam_width)
149 + next_token_ids = next_token_ids.tolist()
150 + elif sampling_params.temperature == 0.0:
151 + # Greedy sampling.
152 + assert sampling_params.n == 1
153 + next_token_id = torch.argmax(prob)
154 + next_token_ids = [next_token_id.item()]
155 + else:
156 + # Neucleus sampling.
157 + # Sample n tokens for the prompt.
158 + n = sampling_params.n
159 + next_token_ids = torch.multinomial(
160 + prob, num_samples=n, replacement=True)
161 next_token_ids = next_token_ids.tolist()
162 + return next_token_ids
163 +
164 +
165 +def _sample_from_generation_tokens(
166 + seq_ids: List[int],
167 + probs: torch.Tensor,
168 + logprobs: torch.Tensor,
169 + seq_logprobs: List[float],
170 + sampling_params: SamplingParams,
171 +) -> Tuple[List[int], List[int]]:
172 + # NOTE(woosuk): sampling_params.n can be greater than
173 + # len(seq_ids) because some sequences in the group might have
174 + # been already terminated.
175 + if sampling_params.use_beam_search:
176 + # Beam search.
177 + # Add cumulative logprobs for the sequences in the group.
178 + seq_logprobs = torch.tensor(
179 + seq_logprobs, dtype=torch.float, device=logprobs.device)
180 + logprobs = logprobs + seq_logprobs.unsqueeze(dim=1)
181 +
182 + vocab_size = logprobs.size(-1)
183 + beam_width = len(seq_ids)
184 + _, topk_ids = torch.topk(logprobs.flatten(), beam_width)
185 + seq_idx = torch.div(topk_ids, vocab_size, rounding_mode='floor').tolist()
186 + beam_seq_ids = [seq_ids[i] for i in seq_idx]
187 + token_ids = (topk_ids % vocab_size).tolist()
188 +
189 + beam_outputs: Dict[int, Tuple[int, int]] = {}
190 + outstanding_beams: List[Tuple[int, int]] = []
191 + # If a beam survives, continue with it.
192 + for seq_id, token_id in zip(beam_seq_ids, token_ids):
193 + if seq_id not in beam_outputs:
194 + beam_outputs[seq_id] = (seq_id, token_id)
195 + else:
196 + outstanding_beams.append((seq_id, token_id))
197 +
198 + # If a beam is discarded, fork another beam.
199 + for seq_id in seq_ids:
200 + if seq_id not in beam_outputs:
201 + beam_outputs[seq_id] = outstanding_beams.pop()
202 + assert not outstanding_beams
203 +
204 + parent_seq_ids = [beam_outputs[seq_id][0] for seq_id in seq_ids]
205 + next_token_ids = [beam_outputs[seq_id][1] for seq_id in seq_ids]
206 + elif sampling_params.temperature == 0.0:
207 + # Greedy sampling.
208 + assert len(seq_ids) == 1
209 + next_token_id = torch.argmax(probs, dim=-1)
210 + next_token_ids = [next_token_id.item()]
211 + parent_seq_ids = seq_ids
212 + else:
213 + # Neucleus sampling.
214 + # Sample 1 token for each sequence in the group.
215 + next_token_ids = torch.multinomial(
216 + probs, num_samples=1, replacement=True)
217 + next_token_ids = next_token_ids.squeeze(dim=-1).tolist()
218 + parent_seq_ids = seq_ids
219 + return parent_seq_ids, next_token_ids
220 +
221 +
222 +def _sample(
223 + probs: torch.Tensor,
224 + logprobs: torch.Tensor,
225 + input_metadata: InputMetadata,
226 +) -> Dict[int, SequenceOutputs]:
227 + seq_outputs: Dict[int, SequenceOutputs] = {}
228 +
229 + # TODO(woosuk): Optimize.
230 + idx = 0
231 + for i, seq_group in enumerate(input_metadata.seq_groups):
232 + seq_ids, sampling_params = seq_group
233 + if i < input_metadata.num_prompts:
234 + # Generate the next tokens for a prompt input.
235 + assert len(seq_ids) == sampling_params.n
236 + prob = probs[idx]
237 + logprob = logprobs[idx]
238 + idx += 1
239 +
240 + # Sample the next tokens.
241 + next_token_ids = _sample_from_prompt(prob, sampling_params)
242 + # Get top-k log probabilities for the next tokens.
243 + next_logprobs = _get_topk_logprobs(
244 + logprob, sampling_params.num_logprobs)
245 +
246 + # Build the output.
247 + for seq_id, next_token_id in zip(seq_ids, next_token_ids):
248 + output_logprobs = next_logprobs.copy()
249 + output_logprobs[next_token_id] = logprob[next_token_id].item()
250 + seq_outputs[seq_id] = SequenceOutputs(
251 + seq_id, seq_id, next_token_id, output_logprobs)
252 + else:
253 + # Generate the next tokens for generation tokens.
254 + prob = probs[idx:idx + len(seq_ids)]
255 + logprob = logprobs[idx:idx + len(seq_ids)]
256 + idx += len(seq_ids)
257 +
258 + # Sample the next tokens.
259 + seq_logprobs = [
260 + input_metadata.seq_logprobs[seq_id] for seq_id in seq_ids]
261 + parent_seq_ids, next_token_ids = _sample_from_generation_tokens(
262 + seq_ids, prob, logprob, seq_logprobs, sampling_params)
263 +
264 + # Get top-k log probabilities for the next tokens.
265 + next_logprobs: Dict[int, Dict[int, float]] = {}
266 + for i, seq_id in enumerate(seq_ids):
267 + next_logprobs[seq_id] = _get_topk_logprobs(
268 + logprob[i], sampling_params.num_logprobs)
269 +
270 + # Build the output.
271 + for seq_id, parent_seq_id, next_token_id in zip(
272 + seq_ids, parent_seq_ids, next_token_ids):
273 + i = seq_ids.index(parent_seq_id)
274 + output_logprobs = next_logprobs[parent_seq_id].copy()
275 + output_logprobs[next_token_id] = logprob[i, next_token_id].item()
276 + seq_outputs[seq_id] = SequenceOutputs(
277 + seq_id,
278 + parent_seq_id,
279 + next_token_id,
280 + output_logprobs,
281 + )
282
38 - # Return the next tokens.
39 - next_tokens: Dict[int, Tuple[int, int]] = {}
40 - for seq_id, token_id in zip(input_metadata.seq_ids, next_token_ids):
41 - next_tokens[seq_id] = (seq_id, token_id)
42 - return next_tokens
283 + return seq_outputs
cacheflow/sampling_params.py
+54 -19
@@ -5,27 +5,51 @@ class SamplingParams:
5
6 def __init__(
7 self,
8 - n: int = 1,
9 - temperature: float = 1.0,
10 - top_p: float = 1.0,
11 - use_beam_search: bool = False,
12 - stop_token_ids: Set[int] = [],
13 - max_num_steps: int = 16, # From OpenAI API.
14 - max_context_len: Optional[int] = None,
8 + n: int,
9 + temperature: float,
10 + top_p: float,
11 + use_beam_search: bool,
12 + stop_token_ids: Set[int],
13 + max_num_steps: int,
14 + num_logprobs: int,
15 + context_window_size: Optional[int],
16 ) -> None:
16 - assert n >= 1
17 - assert temperature >= 0.0
18 - assert 0.0 < top_p <= 1.0
17 + if n < 1:
18 + raise ValueError(f'n must be at least 1, got {n}.')
19 + if temperature < 0.0:
20 + raise ValueError(
21 + f'temperature must be non-negative, got {temperature}.')
22 + if not 0.0 < top_p <= 1.0:
23 + raise ValueError(f'top_p must be in (0, 1], got {top_p}.')
24 + if max_num_steps < 1:
25 + raise ValueError(
26 + f'max_num_steps must be at least 1, got {max_num_steps}.')
27 + if num_logprobs < 0:
28 + raise ValueError(
29 + f'num_logprobs must be non-negative, got {num_logprobs}.')
30 + if context_window_size is not None and context_window_size < 0:
31 + raise ValueError(
32 + 'context_window_size must be non-negative, '
33 + f'got {context_window_size}.')
34 +
35 if use_beam_search:
20 - assert n > 1
21 - assert temperature > 0.0
22 - assert top_p == 1.0
36 + if n == 1:
37 + raise ValueError(
38 + 'n must be greater than 1 when using beam search.')
39 + if temperature > 0.0:
40 + raise ValueError(
41 + 'temperature must be 0 when using beam search.')
42 + if top_p < 1.0:
43 + raise ValueError(
44 + 'top_p must be 1 when using beam search.')
45 elif temperature == 0.0:
24 - # Zero temperature means greedy decoding.
25 - assert n == 1
26 - assert top_p == 1.0
27 - assert max_num_steps >= 1
28 - assert max_context_len is None or max_context_len >= 0
46 + # Zero temperature means greedy sampling.
47 + if n > 1:
48 + raise ValueError(
49 + 'n must be 1 when using greedy sampling.')
50 + if top_p < 1.0:
51 + raise ValueError(
52 + 'top_p must be 1 when using greedy sampling.')
53
54 self.n = n
55 self.temperature = temperature
@@ -33,4 +57,15 @@ class SamplingParams:
57 self.use_beam_search = use_beam_search
58 self.stop_token_ids = stop_token_ids
59 self.max_num_steps = max_num_steps
36 - self.max_context_len = max_context_len
60 + self.num_logprobs = num_logprobs
61 + self.context_window_size = context_window_size
62 +
63 + def __repr__(self) -> str:
64 + return (f'SamplingParams(n={self.n}, '
65 + f'temperature={self.temperature}, '
66 + f'top_p={self.top_p}, '
67 + f'use_beam_search={self.use_beam_search}, '
68 + f'stop_token_ids={self.stop_token_ids}, '
69 + f'max_num_steps={self.max_num_steps}, '
70 + f'num_logprobs={self.num_logprobs}, '
71 + f'context_window_size={self.context_window_size})')
cacheflow/sequence.py
+72 -6
@@ -1,7 +1,9 @@
1 +import copy
2 import enum
2 -from typing import List, Optional
3 +from typing import Dict, List, Optional
4
5 from cacheflow.block import LogicalTokenBlock
6 +from cacheflow.sampling_params import SamplingParams
7
8
9 class SequenceStatus(enum.Enum):
@@ -24,9 +26,11 @@ class Sequence:
26
27 self.logical_token_blocks: List[LogicalTokenBlock] = []
28 # Initialize the logical token blocks with the given token ids.
27 - self.append(token_ids)
29 + self.add(token_ids)
30
31 self.status = SequenceStatus.PENDING
32 + self.output_logprobs: List[Dict[int, float]] = []
33 + self.cumulative_logprobs = 1.0
34
35 def add_block(self) -> None:
36 block = LogicalTokenBlock(
@@ -35,7 +39,7 @@ class Sequence:
39 )
40 self.logical_token_blocks.append(block)
41
38 - def append(self, token_ids: List[int]) -> None:
42 + def add(self, token_ids: List[int]) -> None:
43 while token_ids:
44 if not self.logical_token_blocks:
45 self.add_block()
@@ -49,6 +53,12 @@ class Sequence:
53 last_block.append(token_ids[:num_empty_slots])
54 token_ids = token_ids[num_empty_slots:]
55
56 + def append(self, token_id: int, logprobs: Dict[int, float]) -> None:
57 + assert token_id in logprobs
58 + self.add([token_id])
59 + self.output_logprobs.append(logprobs)
60 + self.cumulative_logprobs += logprobs[token_id]
61 +
62 def get_len(self) -> int:
63 return sum(block.num_tokens for block in self.logical_token_blocks)
64
@@ -58,6 +68,14 @@ class Sequence:
68 token_ids.extend(block.get_token_ids())
69 return token_ids
70
71 + def get_last_token_id(self) -> int:
72 + return self.logical_token_blocks[-1].get_last_token_id()
73 +
74 + def fork(self, child_seq: 'Sequence') -> 'Sequence':
75 + child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
76 + child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
77 + child_seq.cumulative_logprobs = self.cumulative_logprobs
78 +
79 def __repr__(self) -> str:
80 return (f'Sequence(seq_id={self.seq_id}, '
81 f'status={self.status.name}, '
@@ -74,11 +92,17 @@ class SequenceGroup:
92 self.group_id = group_id
93 self.seqs = seqs
94
77 - def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
95 + def get_seqs(
96 + self,
97 + status: Optional[SequenceStatus] = None,
98 + ) -> List[Sequence]:
99 if status is None:
79 - return len(self.seqs)
100 + return self.seqs
101 else:
81 - return len([seq for seq in self.seqs if seq.status == status])
102 + return [seq for seq in self.seqs if seq.status == status]
103 +
104 + def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
105 + return len(self.get_seqs(status))
106
107 def find(self, seq_id: int) -> Sequence:
108 for seq in self.seqs:
@@ -92,3 +116,45 @@ class SequenceGroup:
116 def __repr__(self) -> str:
117 return (f'SequenceGroup(group_id={self.group_id}, '
118 f'num_seqs={len(self.seqs)})')
119 +
120 +
121 +class SequenceGroupInputs:
122 +
123 + def __init__(
124 + self,
125 + group_id: int,
126 + is_prompt: bool,
127 + input_tokens: Dict[int, List[int]], # Seq id -> token ids.
128 + context_len: int,
129 + seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
130 + sampling_params: SamplingParams,
131 + block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
132 + ) -> None:
133 + self.group_id = group_id
134 + self.is_prompt = is_prompt
135 + self.input_tokens = input_tokens
136 + self.context_len = context_len
137 + self.seq_logprobs = seq_logprobs
138 + self.sampling_params = sampling_params
139 + self.block_tables = block_tables
140 +
141 +
142 +class SequenceOutputs:
143 +
144 + def __init__(
145 + self,
146 + seq_id: int,
147 + parent_seq_id: int,
148 + output_token: int,
149 + logprobs: Dict[int, float], # Token id -> logP(x_i+1 | x_0, ..., x_i).
150 + ) -> None:
151 + self.seq_id = seq_id
152 + self.parent_seq_id = parent_seq_id
153 + self.output_token = output_token
154 + self.logprobs = logprobs
155 +
156 + def __repr__(self) -> str:
157 + return (f'SequenceOutputs(seq_id={self.seq_id}, '
158 + f'parent_seq_id={self.parent_seq_id}, '
159 + f'output_token={self.output_token}), '
160 + f'logprobs={self.logprobs}')
cacheflow/worker/cache_engine.py
+27 -8
@@ -97,7 +97,7 @@ class CacheEngine:
97 cpu_cache.append((key_blocks, value_blocks))
98 return cpu_cache
99
100 - def _copy_blocks(
100 + def _swap(
101 self,
102 src: List[KVCache],
103 dst: List[KVCache],
@@ -108,19 +108,38 @@ class CacheEngine:
108 src_key_cache, src_value_cache = src[i]
109 dst_key_cache, dst_value_cache = dst[i]
110 # Copy the key blocks.
111 - cache_ops.copy_cache_blocks(
111 + cache_ops.swap_blocks(
112 src_key_cache, dst_key_cache, src_to_dst)
113 # Copy the value blocks.
114 - cache_ops.copy_cache_blocks(
114 + cache_ops.swap_blocks(
115 src_value_cache, dst_value_cache, src_to_dst)
116 event = self.events[i]
117 event.record(stream=self.cache_stream)
118
119 - def copy(self, src_to_dst: Dict[int, int]) -> None:
120 - self._copy_blocks(self.gpu_cache, self.gpu_cache, src_to_dst)
121 -
119 def swap_in(self, src_to_dst: Dict[int, int]) -> None:
123 - self._copy_blocks(self.cpu_cache, self.gpu_cache, src_to_dst)
120 + self._swap(self.cpu_cache, self.gpu_cache, src_to_dst)
121
122 def swap_out(self, src_to_dst: Dict[int, int]) -> None:
126 - self._copy_blocks(self.gpu_cache, self.cpu_cache, src_to_dst)
123 + self._swap(self.gpu_cache, self.cpu_cache, src_to_dst)
124 +
125 + def _copy(
126 + self,
127 + src: List[KVCache],
128 + dst: List[KVCache],
129 + src_to_dsts: Dict[int, List[int]],
130 + ) -> None:
131 + with torch.cuda.stream(self.cache_stream):
132 + for i in range(self.num_layers):
133 + src_key_cache, src_value_cache = src[i]
134 + dst_key_cache, dst_value_cache = dst[i]
135 + # Copy the key blocks.
136 + cache_ops.copy_blocks(
137 + src_key_cache, dst_key_cache, src_to_dsts)
138 + # Copy the value blocks.
139 + cache_ops.copy_blocks(
140 + src_value_cache, dst_value_cache, src_to_dsts)
141 + event = self.events[i]
142 + event.record(stream=self.cache_stream)
143 +
144 + def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
145 + self._copy(self.gpu_cache, self.gpu_cache, src_to_dsts)
cacheflow/worker/controller.py
+7 -10
@@ -1,6 +1,7 @@
1 from typing import Dict, List, Union
2
3 from cacheflow.master.scheduler import Scheduler
4 +from cacheflow.sequence import SequenceGroupInputs
5 from cacheflow.worker.worker import Worker
6
7
@@ -14,7 +15,8 @@ class Controller:
15 block_size: int,
16 num_gpu_blocks: int,
17 num_cpu_blocks: int,
17 - dtype: str = 'half',
18 + dtype: str,
19 + seed: int,
20 ) -> None:
21 self.node_id = node_id
22 self.num_workers = num_workers
@@ -37,6 +39,7 @@ class Controller:
39 num_gpu_blocks=num_gpu_blocks,
40 num_cpu_blocks=num_cpu_blocks,
41 dtype=dtype,
42 + seed=seed,
43 )
44 self.workers.append(worker)
45
@@ -49,22 +52,16 @@ class Controller:
52
53 def execute_stage(
54 self,
52 - prompt_tokens: Dict[int, List[int]],
53 - generation_tokens: Dict[int, int],
54 - context_lens: Dict[int, int],
55 - block_tables: Dict[int, List[int]],
55 + input_seq_groups: List[SequenceGroupInputs],
56 blocks_to_swap_in: Dict[int, int],
57 blocks_to_swap_out: Dict[int, int],
58 - blocks_to_copy: Dict[int, int],
58 + blocks_to_copy: Dict[int, List[int]],
59 ) -> None:
60 # FIXME: Support tensor parallelism.
61 assert len(self.workers) == 1
62 worker = self.workers[0]
63 output = worker.execute_stage(
64 - prompt_tokens,
65 - generation_tokens,
66 - context_lens,
67 - block_tables,
64 + input_seq_groups,
65 blocks_to_swap_in,
66 blocks_to_swap_out,
67 blocks_to_copy,
cacheflow/worker/worker.py
+76 -43
@@ -1,9 +1,13 @@
1 -from typing import Dict, List, Tuple, Union
1 +from typing import Dict, List, Tuple
2
3 import torch
4
5 from cacheflow.models import get_model
6 +from cacheflow.models import set_seed
7 from cacheflow.models import InputMetadata
8 +from cacheflow.sampling_params import SamplingParams
9 +from cacheflow.sequence import SequenceGroupInputs
10 +from cacheflow.sequence import SequenceOutputs
11 from cacheflow.worker.cache_engine import CacheEngine
12
13
@@ -18,6 +22,7 @@ class Worker:
22 num_gpu_blocks: int,
23 num_cpu_blocks: int,
24 dtype: str,
25 + seed: int,
26 ) -> None:
27 self.worker_id = worker_id
28 self.gpu_id = gpu_id
@@ -33,6 +38,11 @@ class Worker:
38 self.head_size = self.model.config.hidden_size // self.num_heads
39 self.dtype = self.model.dtype
40
41 + # Set the seed.
42 + # We set the seed after initializing the model to ensure that
43 + # the random state is not affected by the model initialization.
44 + set_seed(seed)
45 +
46 self.cache_engine = CacheEngine(
47 worker_id=worker_id,
48 gpu_id=gpu_id,
@@ -49,55 +59,81 @@ class Worker:
59
60 def prepare_inputs(
61 self,
52 - prompt_tokens: Dict[int, List[int]], # Seq id -> List of input token ids.
53 - generation_tokens: Dict[int, int], # Seq id -> Input token id.
54 - context_lens: Dict[int, int], # Seq id -> Number of tokens participating in attention.
55 - block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
62 + input_seq_groups: List[SequenceGroupInputs],
63 ) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
57 - # TODO(woosuk): Support interactive generation.
58 - # Add the prompt tokens.
59 - prompt_lens: List[int] = []
64 + seq_groups: List[Tuple[List[int], SamplingParams]] = []
65 + seq_logprobs: Dict[int, float] = {}
66 + sampling_params: Dict[int, SamplingParams] = {}
67 input_tokens: List[int] = []
68 input_positions: List[int] = []
69 slot_mapping: List[int] = []
70
64 - prompt_seq_ids = sorted(prompt_tokens.keys())
65 - for seq_id in prompt_seq_ids:
66 - prompt_len = len(prompt_tokens[seq_id])
71 + # Add prompt tokens.
72 + prompt_lens: List[int] = []
73 + for input_seq_group in input_seq_groups:
74 + if not input_seq_group.is_prompt:
75 + continue
76 +
77 + seq_ids = list(input_seq_group.input_tokens.keys())
78 + sampling_params = input_seq_group.sampling_params
79 + seq_groups.append((seq_ids, sampling_params))
80 + seq_logprobs.update(input_seq_group.seq_logprobs)
81 +
82 + # Use any sequence in the group.
83 + seq_id = seq_ids[0]
84 +
85 + prompt_tokens = input_seq_group.input_tokens[seq_id]
86 + prompt_len = len(prompt_tokens)
87 prompt_lens.append(prompt_len)
88
69 - input_tokens.extend(prompt_tokens[seq_id])
70 - input_positions.extend(range(len(prompt_tokens[seq_id])))
89 + input_tokens.extend(prompt_tokens)
90 + # NOTE(woosuk): Here we assume that the first token in the prompt
91 + # is always the first token in the sequence.
92 + input_positions.extend(range(len(prompt_tokens)))
93
72 - block_table = block_tables[seq_id]
94 + # Compute the slot mapping.
95 + block_table = input_seq_group.block_tables[seq_id]
96 for i in range(prompt_len):
97 block_number = block_table[i // self.block_size]
98 block_offset = i % self.block_size
99 slot = block_number * self.block_size + block_offset
100 slot_mapping.append(slot)
101
79 - # Add the generation tokens.
102 + # Add generation tokens.
103 max_context_len = 0
104 max_num_blocks_per_seq = 0
105 + context_lens: List[int] = []
106 generation_block_tables: List[List[int]] = []
83 -
84 - generation_seq_ids = sorted(generation_tokens.keys())
85 - for seq_id in generation_seq_ids:
86 - input_tokens.append(generation_tokens[seq_id])
87 - position_id = context_lens[seq_id] - 1
88 - input_positions.append(position_id)
89 -
90 - block_table = block_tables[seq_id]
91 - generation_block_tables.append(block_table)
92 -
93 - max_context_len = max(max_context_len, context_lens[seq_id])
94 - max_num_blocks_per_seq = max(
95 - max_num_blocks_per_seq, len(block_table))
96 -
97 - block_number = block_table[position_id // self.block_size]
98 - block_offset = position_id % self.block_size
99 - slot = block_number * self.block_size + block_offset
100 - slot_mapping.append(slot)
107 + for input_seq_group in input_seq_groups:
108 + if input_seq_group.is_prompt:
109 + continue
110 +
111 + seq_ids = list(input_seq_group.input_tokens.keys())
112 + sampling_params = input_seq_group.sampling_params
113 + seq_groups.append((seq_ids, sampling_params))
114 + seq_logprobs.update(input_seq_group.seq_logprobs)
115 +
116 + for seq_id in seq_ids:
117 + assert len(input_seq_group.input_tokens[seq_id]) == 1
118 + generation_token = input_seq_group.input_tokens[seq_id][0]
119 + input_tokens.append(generation_token)
120 +
121 + position = input_seq_group.context_len - 1
122 + input_positions.append(position)
123 +
124 + block_table = input_seq_group.block_tables[seq_id]
125 + generation_block_tables.append(block_table)
126 +
127 + max_context_len = max(
128 + max_context_len, input_seq_group.context_len)
129 + max_num_blocks_per_seq = max(
130 + max_num_blocks_per_seq, len(block_table))
131 + context_lens.append(input_seq_group.context_len)
132 +
133 + block_number = block_table[position // self.block_size]
134 + block_offset = position % self.block_size
135 + slot = block_number * self.block_size + block_offset
136 + slot_mapping.append(slot)
137
138 # Optimization: Pad the input length to be a multiple of 8.
139 # This is required for utilizing the Tensor Cores in NVIDIA GPUs.
@@ -112,8 +148,7 @@ class Worker:
148 slot_mapping_tensor = torch.tensor(
149 slot_mapping, dtype=torch.int, device=self.device)
150 context_lens_tensor = torch.tensor(
115 - [context_lens[seq_id] for seq_id in generation_seq_ids],
116 - dtype=torch.int, device=self.device)
151 + context_lens, dtype=torch.int, device=self.device)
152 padded_block_tables = [
153 _pad_to_max(block_table, max_num_blocks_per_seq)
154 for block_table in generation_block_tables]
@@ -121,7 +156,8 @@ class Worker:
156 padded_block_tables, dtype=torch.int, device=self.device)
157
158 input_metadata = InputMetadata(
124 - seq_ids=prompt_seq_ids + generation_seq_ids,
159 + seq_groups=seq_groups,
160 + seq_logprobs=seq_logprobs,
161 prompt_lens=prompt_lens,
162 slot_mapping=slot_mapping_tensor,
163 context_lens=context_lens_tensor,
@@ -133,14 +169,11 @@ class Worker:
169 @torch.inference_mode()
170 def execute_stage(
171 self,
136 - prompt_tokens: Dict[int, List[int]], # Seq id -> List of input token ids.
137 - generation_tokens: Dict[int, int], # Seq id -> Input token id.
138 - context_lens: Dict[int, int], # Seq id -> Number of tokens participating in attention.
139 - block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
172 + input_seq_groups: List[SequenceGroupInputs],
173 blocks_to_swap_in: Dict[int, int],
174 blocks_to_swap_out: Dict[int, int],
142 - blocks_to_copy: Dict[int, int],
143 - ) -> Union[torch.Tensor, Dict[int, Tuple[int, int]]]:
175 + blocks_to_copy: Dict[int, List[int]],
176 + ) -> Dict[int, SequenceOutputs]:
177 # Issue cache operations.
178 command_issued = False
179 if blocks_to_swap_in:
@@ -160,7 +193,7 @@ class Worker:
193
194 # Prepare input tensors.
195 input_tokens, input_positions, input_metadata = self.prepare_inputs(
163 - prompt_tokens, generation_tokens, context_lens, block_tables)
196 + input_seq_groups)
197
198 # Execute the model.
199 output = self.model(
csrc/cache.cpp
+14 -2
@@ -1,10 +1,18 @@
1 #include <torch/extension.h>
2
3 -void copy_blocks(
3 +#include <map>
4 +#include <vector>
5 +
6 +void swap_blocks(
7 torch::Tensor& src,
8 torch::Tensor& dst,
9 const std::map<int64_t, int64_t>& block_mapping);
10
11 +void copy_blocks(
12 + torch::Tensor& src,
13 + torch::Tensor& dst,
14 + const std::map<int64_t, std::vector<int64_t>>& block_mapping);
15 +
16 void reshape_and_cache(
17 torch::Tensor& key,
18 torch::Tensor& value,
@@ -14,7 +22,11 @@ void reshape_and_cache(
22
23 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
24 m.def(
17 - "copy_cache_blocks",
25 + "swap_blocks",
26 + &swap_blocks,
27 + "Swap in (out) the cache blocks from src to dst");
28 + m.def(
29 + "copy_blocks",
30 &copy_blocks,
31 "Copy the cache blocks from src to dst");
32 m.def(
csrc/cache_kernels.cu
+31 -1
@@ -5,8 +5,9 @@
5 #include <algorithm>
6 #include <cassert>
7 #include <map>
8 +#include <vector>
9
9 -void copy_blocks(
10 +void swap_blocks(
11 torch::Tensor& src,
12 torch::Tensor& dst,
13 const std::map<int64_t, int64_t>& block_mapping) {
@@ -43,6 +44,35 @@ void copy_blocks(
44 }
45 }
46
47 +void copy_blocks(
48 + torch::Tensor& src,
49 + torch::Tensor& dst,
50 + const std::map<int64_t, std::vector<int64_t>>& block_mapping) {
51 + torch::Device src_device = src.device();
52 + torch::Device dst_device = dst.device();
53 + assert(src_device.is_cuda() && dst_device.is_cuda());
54 + cudaMemcpyKind memcpy_type = cudaMemcpyDeviceToDevice;
55 +
56 + void *src_ptr = src.data_ptr();
57 + void *dst_ptr = dst.data_ptr();
58 +
59 + const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
60 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
61 + for (const auto& pair : block_mapping) {
62 + int64_t src_block_number = pair.first;
63 + for (int64_t dst_block_number : pair.second) {
64 + int64_t src_offset = src_block_number * block_size_in_bytes;
65 + int64_t dst_offset = dst_block_number * block_size_in_bytes;
66 + cudaMemcpyAsync(
67 + dst_ptr + dst_offset,
68 + src_ptr + src_offset,
69 + block_size_in_bytes,
70 + memcpy_type,
71 + stream);
72 + }
73 + }
74 +}
75 +
76 template<typename scalar_t>
77 __global__ void reshape_and_cache_kernel(
78 const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
server.py
+10 -7
@@ -15,6 +15,8 @@ parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of
15 parser.add_argument('--num-cpu-blocks', type=int, default=32, help='number of CPU blocks (per GPU)')
16 # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
17 parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
18 +# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
19 +parser.add_argument('--seed', type=int, default=0, help='random seed')
20 args = parser.parse_args()
21
22
@@ -30,6 +32,7 @@ def main():
32 num_gpu_blocks=args.num_gpu_blocks,
33 num_cpu_blocks=args.num_cpu_blocks,
34 dtype=args.dtype,
35 + seed=args.seed,
36 )
37 controllers.append(controller)
38
@@ -52,18 +55,18 @@ def main():
55 controllers[i].set_next(controllers[i + 1])
56 controllers[-1].set_next(scheduler)
57
58 + # Test the following inputs.
59 test_inputs = [
56 - 'Ion Stoica is a',
57 - 'UC Berkeley is',
58 - 'The future of cloud computing is',
60 + ('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),
61 + ('UC Berkeley is', {'n': 3, 'temperature': 0.8, 'top_p': 0.99}),
62 + ('The future of cloud computing is', {}), # Use default parameters.
63 ]
60 -
61 - # FIXME
64 while True:
65 if test_inputs:
64 - frontend.query(test_inputs.pop())
66 + text, sampling_params = test_inputs.pop(0)
67 + frontend.query(text, **sampling_params)
68 scheduler.step()
66 - if not scheduler.pending and not scheduler.running:
69 + if not (scheduler.pending or scheduler.running or test_inputs):
70 break
71
72