A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Support beam search & parallel generation (#7)
Woosuk Kwon committed
Mar 10, 2023 at 09:58 UTC
1a7eb7da6157541ed7867c9aff94231695f2cee9
16 files changed
+662
-163
cacheflow/block.py
+4
@@ -35,6 +35,10 @@ class LogicalTokenBlock:
35
def get_token_ids(self) -> List[int]:
36
return self.token_ids[:self.num_tokens]
37
38
+ def get_last_token_id(self) -> int:
39
+ assert self.num_tokens > 0
40
+ return self.token_ids[self.num_tokens - 1]
41
+
42
43
class PhysicalTokenBlock:
44
cacheflow/master/frontend.py
+28
-5
@@ -1,4 +1,4 @@
1
-from typing import List, Optional, Tuple
1
+from typing import List, Optional, Set, Tuple
2
3
from transformers import AutoTokenizer
4
@@ -25,12 +25,35 @@ class Frontend:
25
def query(
26
self,
27
prompt: str,
28
- sampling_params: Optional[SamplingParams] = None,
28
+ n: int = 1,
29
+ temperature: float = 1.0,
30
+ top_p: float = 1.0,
31
+ use_beam_search: bool = False,
32
+ stop_token_ids: Set[int] = set(),
33
+ max_num_steps: int = 16, # From OpenAI API.
34
+ num_logprobs: int = 0,
35
+ context_window_size: Optional[int] = None,
36
) -> None:
30
- if sampling_params is None:
31
- sampling_params = SamplingParams()
32
- token_ids: List[int] = self.tokenizer.encode(prompt)
37
+ # Stop when we see an EOS token.
38
+ stop_token_ids.add(self.tokenizer.eos_token_id)
39
+ sampling_params = SamplingParams(
40
+ n=n,
41
+ temperature=temperature,
42
+ top_p=top_p,
43
+ use_beam_search=use_beam_search,
44
+ stop_token_ids=stop_token_ids,
45
+ max_num_steps=max_num_steps,
46
+ num_logprobs=num_logprobs,
47
+ context_window_size=context_window_size,
48
+ )
49
+ token_ids = self.tokenizer.encode(prompt)
50
+ self._add_query(token_ids, sampling_params)
51
52
+ def _add_query(
53
+ self,
54
+ token_ids: List[int],
55
+ sampling_params: SamplingParams,
56
+ ) -> None:
57
seqs: List[Sequence] = []
58
for _ in range(sampling_params.n):
59
seq_id = next(self.seq_counter)
cacheflow/master/scheduler.py
+55
-36
@@ -1,10 +1,12 @@
1
-from typing import Dict, List, Tuple
1
+from typing import Dict, List
2
3
from cacheflow.master.block_manager import BlockSpaceManager
4
from cacheflow.master.frontend import Frontend
5
from cacheflow.sampling_params import SamplingParams
6
from cacheflow.sequence import Sequence
7
from cacheflow.sequence import SequenceGroup
8
+from cacheflow.sequence import SequenceGroupInputs
9
+from cacheflow.sequence import SequenceOutputs
10
from cacheflow.sequence import SequenceStatus
11
12
_MAX_NUM_BATCHED_TOKENS = 2048
@@ -66,7 +68,7 @@ class Scheduler:
68
def _append(
69
self,
70
seq_group: SequenceGroup,
69
- blocks_to_copy: Dict[int, int],
71
+ blocks_to_copy: Dict[int, List[int]],
72
) -> None:
73
for seq in seq_group.seqs:
74
if seq.status == SequenceStatus.FINISHED:
@@ -74,7 +76,10 @@ class Scheduler:
76
ret = self.block_manager.append(seq)
77
if ret is not None:
78
src_block, dst_block = ret
77
- blocks_to_copy[src_block] = dst_block
79
+ if src_block in blocks_to_copy:
80
+ blocks_to_copy[src_block].append(dst_block)
81
+ else:
82
+ blocks_to_copy[src_block] = [dst_block]
83
84
def _swap_in(
85
self,
@@ -83,9 +88,8 @@ class Scheduler:
88
) -> None:
89
mapping = self.block_manager.swap_in(seq_group)
90
blocks_to_swap_in.update(mapping)
86
- for seq in seq_group.seqs:
87
- if seq.status == SequenceStatus.SWAPPED:
88
- seq.status = SequenceStatus.RUNNING
91
+ for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
92
+ seq.status = SequenceStatus.RUNNING
93
self.running.append(seq_group)
94
95
def _swap_out(
@@ -96,16 +100,15 @@ class Scheduler:
100
assert self.block_manager.can_swap_out(seq_group)
101
mapping = self.block_manager.swap_out(seq_group)
102
blocks_to_swap_out.update(mapping)
99
- for seq in seq_group.seqs:
100
- if seq.status == SequenceStatus.RUNNING:
101
- seq.status = SequenceStatus.SWAPPED
103
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
104
+ seq.status = SequenceStatus.SWAPPED
105
self.swapped.append(seq_group)
106
107
def step(self) -> None:
108
# Blocks that need to be swaped or copied before model execution.
109
blocks_to_swap_in: Dict[int, int] = {}
110
blocks_to_swap_out: Dict[int, int] = {}
108
- blocks_to_copy: Dict[int, int] = {}
111
+ blocks_to_copy: Dict[int, List[int]] = {}
112
113
# 1. Reserve new slots for the running sequences.
114
# NOTE: Here we implicitly assume FCFS scheduling.
@@ -143,6 +146,10 @@ class Scheduler:
146
# All swapped sequences are swapped in.
147
self.swapped.clear()
148
149
+ # Ensure that swap-in and swap-out never happen at the same timestep.
150
+ if blocks_to_swap_in:
151
+ assert not blocks_to_swap_out
152
+
153
num_batched_tokens = sum(
154
seq_group.num_seqs(status=SequenceStatus.RUNNING)
155
for seq_group in self.running
@@ -152,7 +159,6 @@ class Scheduler:
159
# NOTE: Here we implicitly assume FCFS scheduling.
160
# TODO(woosuk): Add a batching policy to control the batch size.
161
if not self.swapped:
155
- # FIXME(woosuk): Acquire a lock to protect pending.
162
self._fetch_inputs()
163
for i, seq_group in enumerate(self.pending):
164
num_prompt_tokens = seq_group.seqs[0].get_len()
@@ -168,39 +174,45 @@ class Scheduler:
174
else:
175
self.pending.clear()
176
171
- # Ensure that swap-in and swap-out never happen at the same timestep.
172
- if blocks_to_swap_in:
173
- assert not blocks_to_swap_out
174
-
177
# 4. Create input data structures.
176
- prompt_tokens: Dict[int, List[int]] = {}
177
- generation_tokens: Dict[int, int] = {}
178
- context_lens: Dict[int, int] = {}
179
- block_tables: Dict[int, List[int]] = {}
178
+ input_seq_groups: List[SequenceGroupInputs] = []
179
for seq_group in self.running:
180
group_id = seq_group.group_id
181
num_steps = self.num_steps[group_id]
182
+
183
# NOTE(woosuk): We assume that the number of steps is 0
184
# for the prompt sequences.
185
is_prompt = num_steps == 0
186
- for seq in seq_group.seqs:
187
- if seq.status != SequenceStatus.RUNNING:
188
- continue
186
187
+ input_tokens: Dict[int, List[int]] = {}
188
+ seq_logprobs: Dict[int, float] = {}
189
+ block_tables: Dict[int, List[int]] = {}
190
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
191
seq_id = seq.seq_id
192
block_tables[seq_id] = self.block_manager.get_block_table(seq)
193
if is_prompt:
193
- prompt_tokens[seq_id] = seq.get_token_ids()
194
+ input_tokens[seq_id] = seq.get_token_ids()
195
else:
195
- generation_tokens[seq_id] = seq.get_token_ids()[-1]
196
- context_lens[seq_id] = seq.get_len()
196
+ input_tokens[seq_id] = [seq.get_last_token_id()]
197
+ seq_logprobs[seq_id] = seq.cumulative_logprobs
198
+ # NOTE(woosuk): Sequences in the same group have the same
199
+ # sequence length
200
+ seq_len = seq.get_len()
201
+
202
+ input_seq_group = SequenceGroupInputs(
203
+ group_id=group_id,
204
+ is_prompt=is_prompt,
205
+ input_tokens=input_tokens,
206
+ context_len=seq_len,
207
+ seq_logprobs=seq_logprobs,
208
+ sampling_params=self.sampling_params[group_id],
209
+ block_tables=block_tables,
210
+ )
211
+ input_seq_groups.append(input_seq_group)
212
213
# 5. Execute the first stage of the pipeline.
214
self.controllers[0].execute_stage(
200
- prompt_tokens,
201
- generation_tokens,
202
- context_lens,
203
- block_tables,
215
+ input_seq_groups,
216
blocks_to_swap_in,
217
blocks_to_swap_out,
218
blocks_to_copy,
@@ -208,7 +220,7 @@ class Scheduler:
220
221
def post_step(
222
self,
211
- next_tokens: Dict[int, Tuple[int, int]],
223
+ seq_outputs: Dict[int, SequenceOutputs],
224
) -> None:
225
# Update the running sequences and free blocks.
226
for seq_group in self.running:
@@ -216,25 +228,32 @@ class Scheduler:
228
self.num_steps[group_id] += 1
229
stop_token_ids = self.sampling_params[group_id].stop_token_ids
230
231
+ # Process beam search results before processing the next tokens.
232
for seq in seq_group.seqs:
233
if seq.status == SequenceStatus.FINISHED:
234
continue
235
223
- parent_seq_id, next_token = next_tokens[seq.seq_id]
224
- if seq.seq_id != parent_seq_id:
236
+ output = seq_outputs[seq.seq_id]
237
+ if seq.seq_id != output.parent_seq_id:
238
# The sequence is a fork of the parent sequence (beam search).
239
# Free the current sequence.
240
self.block_manager.free(seq)
241
# Fork the parent sequence.
229
- parent_seq = seq_group.find(parent_seq_id)
230
- seq.logical_token_blocks = parent_seq.logical_token_blocks.copy()
242
+ parent_seq = seq_group.find(output.parent_seq_id)
243
+ parent_seq.fork(seq)
244
self.block_manager.fork(parent_seq, seq)
245
246
+ # Process the next tokens.
247
+ for seq in seq_group.seqs:
248
+ if seq.status == SequenceStatus.FINISHED:
249
+ continue
250
+
251
# Append a new token to the sequence.
234
- seq.append([next_token])
252
+ output = seq_outputs[seq.seq_id]
253
+ seq.append(output.output_token, output.logprobs)
254
255
# Check if the sequence has generated a stop token.
237
- if next_token in stop_token_ids:
256
+ if output.output_token in stop_token_ids:
257
self._free_seq(seq)
258
continue
259
cacheflow/models/__init__.py
+3
-1
@@ -1,8 +1,10 @@
1
from cacheflow.models.input_metadata import InputMetadata
2
from cacheflow.models.model_utils import get_model
3
+from cacheflow.models.model_utils import set_seed
4
5
6
__all__ = [
6
- 'get_model',
7
'InputMetadata',
8
+ 'get_model',
9
+ 'set_seed'
10
]
cacheflow/models/input_metadata.py
+11
-7
@@ -1,21 +1,24 @@
1
-from typing import List
1
+from typing import List, Dict, Tuple
2
3
import torch
4
5
+from cacheflow.sampling_params import SamplingParams
6
+
7
8
class InputMetadata:
9
10
def __init__(
11
self,
10
- seq_ids: List[int],
12
+ seq_groups: List[Tuple[List[int], SamplingParams]],
13
+ seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
14
prompt_lens: List[int],
15
slot_mapping: torch.Tensor,
16
context_lens: torch.Tensor,
14
- # FIXME: Rename
17
max_context_len: int,
18
block_tables: torch.Tensor,
19
) -> None:
18
- self.seq_ids = seq_ids
20
+ self.seq_groups = seq_groups
21
+ self.seq_logprobs = seq_logprobs
22
self.prompt_lens = prompt_lens
23
self.slot_mapping = slot_mapping
24
self.context_lens = context_lens
@@ -23,19 +26,20 @@ class InputMetadata:
26
self.block_tables = block_tables
27
28
self.num_prompts = len(prompt_lens)
29
+ self.num_prompt_tokens = sum(prompt_lens)
30
self.num_generation_tokens = context_lens.shape[0]
31
self.num_valid_tokens = slot_mapping.shape[0]
32
if block_tables.numel() > 0:
33
self.max_num_blocks_per_seq = block_tables.shape[1]
34
else:
35
self.max_num_blocks_per_seq = 0
32
- assert self.num_generation_tokens == block_tables.shape[0]
33
- assert self.num_prompts + self.num_generation_tokens == len(seq_ids)
36
+ assert block_tables.shape[0] == self.num_generation_tokens
37
+ assert context_lens.shape[0] == self.num_generation_tokens
38
39
def __repr__(self) -> str:
40
return (f'InputMetadata('
37
- f'seq_ids={self.seq_ids}, '
41
f'num_prompts={self.num_prompts}, '
42
+ f'num_prompt_tokens={self.num_prompt_tokens}, '
43
f'num_generation_tokens={self.num_generation_tokens}, '
44
f'num_valid_tokens={self.num_valid_tokens}, '
45
f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
cacheflow/models/model_utils.py
+10
@@ -1,5 +1,7 @@
1
+import random
2
from typing import Union
3
4
+import numpy as np
5
import torch
6
import torch.nn as nn
7
@@ -30,3 +32,11 @@ def get_model(
32
model = hf_model.from_pretrained(model_name, torch_dtype=torch_dtype)
33
return model.eval()
34
raise ValueError(f'Invalid model name: {model_name}')
35
+
36
+
37
+def set_seed(seed: int) -> None:
38
+ random.seed(seed)
39
+ np.random.seed(seed)
40
+ torch.manual_seed(seed)
41
+ if torch.cuda.is_available():
42
+ torch.cuda.manual_seed_all(seed)
cacheflow/models/opt.py
+2
-1
@@ -9,6 +9,7 @@ from transformers import PreTrainedModel
9
from cacheflow.models import InputMetadata
10
from cacheflow.models.attention import OPTCacheFlowAttention
11
from cacheflow.models.sample import Sampler
12
+from cacheflow.sequence import SequenceOutputs
13
14
KVCache = Tuple[torch.Tensor, torch.Tensor]
15
@@ -261,7 +262,7 @@ class OPTForCausalLM(OPTPreTrainedModel):
262
kv_caches: List[KVCache],
263
input_metadata: InputMetadata,
264
cache_events: Optional[List[torch.cuda.Event]],
264
- ) -> Dict[int, Tuple[int, int]]:
265
+ ) -> Dict[int, SequenceOutputs]:
266
hidden_states = self.model(
267
input_ids, positions, kv_caches, input_metadata, cache_events)
268
next_tokens = self.sampler(
cacheflow/models/sample.py
+258
-17
@@ -4,6 +4,8 @@ import torch
4
import torch.nn as nn
5
6
from cacheflow.models import InputMetadata
7
+from cacheflow.sampling_params import SamplingParams
8
+from cacheflow.sequence import SequenceOutputs
9
10
11
class Sampler(nn.Module):
@@ -16,27 +18,266 @@ class Sampler(nn.Module):
18
embedding: torch.Tensor,
19
hidden_states: torch.Tensor,
20
input_metadata: InputMetadata,
19
- ) -> Dict[int, Tuple[int, int]]:
20
- # Get the hidden states of the last tokens.
21
- start_idx = 0
22
- last_token_indicies: List[int] = []
23
- for prompt_len in input_metadata.prompt_lens:
24
- last_token_indicies.append(start_idx + prompt_len - 1)
25
- start_idx += prompt_len
26
- last_token_indicies.extend(
27
- range(start_idx, start_idx + input_metadata.num_generation_tokens))
28
- hidden_states = hidden_states[last_token_indicies]
21
+ ) -> Dict[int, SequenceOutputs]:
22
+ # Get the hidden states that we use for sampling.
23
+ hidden_states = _prune_hidden_states(hidden_states, input_metadata)
24
25
# Get the logits for the next tokens.
26
logits = torch.matmul(hidden_states, embedding.t())
27
28
+ # Apply temperature scaling.
29
+ temperatures = _get_temperatures(input_metadata)
30
+ assert len(temperatures) == logits.shape[0]
31
+ if any(t != 1.0 for t in temperatures):
32
+ t = torch.tensor(
33
+ temperatures, dtype=logits.dtype, device=logits.device)
34
+ # Use in-place division to avoid creating a new tensor.
35
+ logits.div_(t.unsqueeze(dim=1))
36
+
37
+ # Compute the probabilities.
38
+ probs = torch.softmax(logits, dim=-1, dtype=torch.float)
39
+ # Compute the log probabilities (before applying top-p).
40
+ logprobs = torch.log(probs)
41
+
42
+ # Apply top-p truncation.
43
+ top_ps = _get_top_ps(input_metadata)
44
+ assert len(top_ps) == probs.shape[0]
45
+ if any(p < 1.0 for p in top_ps):
46
+ p = torch.tensor(top_ps, dtype=probs.dtype, device=probs.device)
47
+ probs = _apply_top_p(probs, p)
48
+
49
# Sample the next tokens.
34
- # TODO(woosuk): Implement other sampling methods.
35
- next_token_ids = torch.argmax(logits, dim=-1)
50
+ return _sample(probs, logprobs, input_metadata)
51
+
52
+
53
+def _prune_hidden_states(
54
+ hidden_states: torch.Tensor,
55
+ input_metadata: InputMetadata,
56
+) -> torch.Tensor:
57
+ start_idx = 0
58
+ last_token_indicies: List[int] = []
59
+ for prompt_len in input_metadata.prompt_lens:
60
+ last_token_indicies.append(start_idx + prompt_len - 1)
61
+ start_idx += prompt_len
62
+ last_token_indicies.extend(
63
+ range(start_idx, start_idx + input_metadata.num_generation_tokens))
64
+ return hidden_states[last_token_indicies]
65
+
66
+
67
+def _get_temperatures(
68
+ input_metadata: InputMetadata,
69
+) -> List[float]:
70
+ # Collect the temperatures for the logits.
71
+ temperatures: List[float] = []
72
+ for i, seq_group in enumerate(input_metadata.seq_groups):
73
+ seq_ids, sampling_params = seq_group
74
+ temperature = sampling_params.temperature
75
+ if temperature == 0.0:
76
+ # NOTE: Zero temperature means deterministic sampling
77
+ # (i.e., greedy sampling or beam search).
78
+ # Set the temperature to 1 to avoid division by zero.
79
+ temperature = 1.0
80
+
81
+ if i < input_metadata.num_prompts:
82
+ # A prompt input.
83
+ temperatures.append(temperature)
84
+ else:
85
+ # A generation token.
86
+ temperatures += [temperature] * len(seq_ids)
87
+ return temperatures
88
+
89
+
90
+def _get_top_ps(
91
+ input_metadata: InputMetadata,
92
+) -> List[float]:
93
+ top_ps: List[float] = []
94
+ for i, seq_group in enumerate(input_metadata.seq_groups):
95
+ seq_ids, sampling_params = seq_group
96
+ if i < input_metadata.num_prompts:
97
+ # A prompt input.
98
+ top_ps.append(sampling_params.top_p)
99
+ else:
100
+ # A generation token.
101
+ top_ps += [sampling_params.top_p] * len(seq_ids)
102
+ return top_ps
103
+
104
+
105
+def _apply_top_p(
106
+ probs: torch.Tensor,
107
+ p: torch.Tensor,
108
+) -> torch.Tensor:
109
+ # TODO(woosuk): Optimize.
110
+ probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
111
+ probs_sum = torch.cumsum(probs_sort, dim=-1)
112
+ mask = (probs_sum - probs_sort) > p.unsqueeze(dim=1)
113
+ probs_sort[mask] = 0.0
114
+ probs_sort.div_(probs_sort.sum(dim=-1, keepdim=True))
115
+ probs = torch.gather(
116
+ probs_sort, dim=-1, index=torch.argsort(probs_idx, dim=-1))
117
+ return probs
118
+
119
+
120
+def _get_topk_logprobs(
121
+ logprobs: torch.Tensor,
122
+ num_logprobs: int,
123
+) -> Dict[int, float]:
124
+ if num_logprobs == 0:
125
+ return {}
126
+
127
+ topk_logprobs, topk_ids = torch.topk(logprobs, num_logprobs)
128
+ if num_logprobs == 1:
129
+ topk_logprobs = [topk_logprobs.item()]
130
+ topk_ids = [topk_ids.item()]
131
+ else:
132
+ topk_logprobs = topk_logprobs.tolist()
133
+ topk_ids = topk_ids.tolist()
134
+
135
+ token_to_logprob: Dict[int, float] = {}
136
+ for token_id, logprob in zip(topk_ids, topk_logprobs):
137
+ token_to_logprob[token_id] = logprob
138
+ return token_to_logprob
139
+
140
+
141
+def _sample_from_prompt(
142
+ prob: torch.Tensor,
143
+ sampling_params: SamplingParams,
144
+) -> List[int]:
145
+ if sampling_params.use_beam_search:
146
+ # Beam search.
147
+ beam_width = sampling_params.n
148
+ _, next_token_ids = torch.topk(prob, beam_width)
149
+ next_token_ids = next_token_ids.tolist()
150
+ elif sampling_params.temperature == 0.0:
151
+ # Greedy sampling.
152
+ assert sampling_params.n == 1
153
+ next_token_id = torch.argmax(prob)
154
+ next_token_ids = [next_token_id.item()]
155
+ else:
156
+ # Neucleus sampling.
157
+ # Sample n tokens for the prompt.
158
+ n = sampling_params.n
159
+ next_token_ids = torch.multinomial(
160
+ prob, num_samples=n, replacement=True)
161
next_token_ids = next_token_ids.tolist()
162
+ return next_token_ids
163
+
164
+
165
+def _sample_from_generation_tokens(
166
+ seq_ids: List[int],
167
+ probs: torch.Tensor,
168
+ logprobs: torch.Tensor,
169
+ seq_logprobs: List[float],
170
+ sampling_params: SamplingParams,
171
+) -> Tuple[List[int], List[int]]:
172
+ # NOTE(woosuk): sampling_params.n can be greater than
173
+ # len(seq_ids) because some sequences in the group might have
174
+ # been already terminated.
175
+ if sampling_params.use_beam_search:
176
+ # Beam search.
177
+ # Add cumulative logprobs for the sequences in the group.
178
+ seq_logprobs = torch.tensor(
179
+ seq_logprobs, dtype=torch.float, device=logprobs.device)
180
+ logprobs = logprobs + seq_logprobs.unsqueeze(dim=1)
181
+
182
+ vocab_size = logprobs.size(-1)
183
+ beam_width = len(seq_ids)
184
+ _, topk_ids = torch.topk(logprobs.flatten(), beam_width)
185
+ seq_idx = torch.div(topk_ids, vocab_size, rounding_mode='floor').tolist()
186
+ beam_seq_ids = [seq_ids[i] for i in seq_idx]
187
+ token_ids = (topk_ids % vocab_size).tolist()
188
+
189
+ beam_outputs: Dict[int, Tuple[int, int]] = {}
190
+ outstanding_beams: List[Tuple[int, int]] = []
191
+ # If a beam survives, continue with it.
192
+ for seq_id, token_id in zip(beam_seq_ids, token_ids):
193
+ if seq_id not in beam_outputs:
194
+ beam_outputs[seq_id] = (seq_id, token_id)
195
+ else:
196
+ outstanding_beams.append((seq_id, token_id))
197
+
198
+ # If a beam is discarded, fork another beam.
199
+ for seq_id in seq_ids:
200
+ if seq_id not in beam_outputs:
201
+ beam_outputs[seq_id] = outstanding_beams.pop()
202
+ assert not outstanding_beams
203
+
204
+ parent_seq_ids = [beam_outputs[seq_id][0] for seq_id in seq_ids]
205
+ next_token_ids = [beam_outputs[seq_id][1] for seq_id in seq_ids]
206
+ elif sampling_params.temperature == 0.0:
207
+ # Greedy sampling.
208
+ assert len(seq_ids) == 1
209
+ next_token_id = torch.argmax(probs, dim=-1)
210
+ next_token_ids = [next_token_id.item()]
211
+ parent_seq_ids = seq_ids
212
+ else:
213
+ # Neucleus sampling.
214
+ # Sample 1 token for each sequence in the group.
215
+ next_token_ids = torch.multinomial(
216
+ probs, num_samples=1, replacement=True)
217
+ next_token_ids = next_token_ids.squeeze(dim=-1).tolist()
218
+ parent_seq_ids = seq_ids
219
+ return parent_seq_ids, next_token_ids
220
+
221
+
222
+def _sample(
223
+ probs: torch.Tensor,
224
+ logprobs: torch.Tensor,
225
+ input_metadata: InputMetadata,
226
+) -> Dict[int, SequenceOutputs]:
227
+ seq_outputs: Dict[int, SequenceOutputs] = {}
228
+
229
+ # TODO(woosuk): Optimize.
230
+ idx = 0
231
+ for i, seq_group in enumerate(input_metadata.seq_groups):
232
+ seq_ids, sampling_params = seq_group
233
+ if i < input_metadata.num_prompts:
234
+ # Generate the next tokens for a prompt input.
235
+ assert len(seq_ids) == sampling_params.n
236
+ prob = probs[idx]
237
+ logprob = logprobs[idx]
238
+ idx += 1
239
+
240
+ # Sample the next tokens.
241
+ next_token_ids = _sample_from_prompt(prob, sampling_params)
242
+ # Get top-k log probabilities for the next tokens.
243
+ next_logprobs = _get_topk_logprobs(
244
+ logprob, sampling_params.num_logprobs)
245
+
246
+ # Build the output.
247
+ for seq_id, next_token_id in zip(seq_ids, next_token_ids):
248
+ output_logprobs = next_logprobs.copy()
249
+ output_logprobs[next_token_id] = logprob[next_token_id].item()
250
+ seq_outputs[seq_id] = SequenceOutputs(
251
+ seq_id, seq_id, next_token_id, output_logprobs)
252
+ else:
253
+ # Generate the next tokens for generation tokens.
254
+ prob = probs[idx:idx + len(seq_ids)]
255
+ logprob = logprobs[idx:idx + len(seq_ids)]
256
+ idx += len(seq_ids)
257
+
258
+ # Sample the next tokens.
259
+ seq_logprobs = [
260
+ input_metadata.seq_logprobs[seq_id] for seq_id in seq_ids]
261
+ parent_seq_ids, next_token_ids = _sample_from_generation_tokens(
262
+ seq_ids, prob, logprob, seq_logprobs, sampling_params)
263
+
264
+ # Get top-k log probabilities for the next tokens.
265
+ next_logprobs: Dict[int, Dict[int, float]] = {}
266
+ for i, seq_id in enumerate(seq_ids):
267
+ next_logprobs[seq_id] = _get_topk_logprobs(
268
+ logprob[i], sampling_params.num_logprobs)
269
+
270
+ # Build the output.
271
+ for seq_id, parent_seq_id, next_token_id in zip(
272
+ seq_ids, parent_seq_ids, next_token_ids):
273
+ i = seq_ids.index(parent_seq_id)
274
+ output_logprobs = next_logprobs[parent_seq_id].copy()
275
+ output_logprobs[next_token_id] = logprob[i, next_token_id].item()
276
+ seq_outputs[seq_id] = SequenceOutputs(
277
+ seq_id,
278
+ parent_seq_id,
279
+ next_token_id,
280
+ output_logprobs,
281
+ )
282
38
- # Return the next tokens.
39
- next_tokens: Dict[int, Tuple[int, int]] = {}
40
- for seq_id, token_id in zip(input_metadata.seq_ids, next_token_ids):
41
- next_tokens[seq_id] = (seq_id, token_id)
42
- return next_tokens
283
+ return seq_outputs
cacheflow/sampling_params.py
+54
-19
@@ -5,27 +5,51 @@ class SamplingParams:
5
6
def __init__(
7
self,
8
- n: int = 1,
9
- temperature: float = 1.0,
10
- top_p: float = 1.0,
11
- use_beam_search: bool = False,
12
- stop_token_ids: Set[int] = [],
13
- max_num_steps: int = 16, # From OpenAI API.
14
- max_context_len: Optional[int] = None,
8
+ n: int,
9
+ temperature: float,
10
+ top_p: float,
11
+ use_beam_search: bool,
12
+ stop_token_ids: Set[int],
13
+ max_num_steps: int,
14
+ num_logprobs: int,
15
+ context_window_size: Optional[int],
16
) -> None:
16
- assert n >= 1
17
- assert temperature >= 0.0
18
- assert 0.0 < top_p <= 1.0
17
+ if n < 1:
18
+ raise ValueError(f'n must be at least 1, got {n}.')
19
+ if temperature < 0.0:
20
+ raise ValueError(
21
+ f'temperature must be non-negative, got {temperature}.')
22
+ if not 0.0 < top_p <= 1.0:
23
+ raise ValueError(f'top_p must be in (0, 1], got {top_p}.')
24
+ if max_num_steps < 1:
25
+ raise ValueError(
26
+ f'max_num_steps must be at least 1, got {max_num_steps}.')
27
+ if num_logprobs < 0:
28
+ raise ValueError(
29
+ f'num_logprobs must be non-negative, got {num_logprobs}.')
30
+ if context_window_size is not None and context_window_size < 0:
31
+ raise ValueError(
32
+ 'context_window_size must be non-negative, '
33
+ f'got {context_window_size}.')
34
+
35
if use_beam_search:
20
- assert n > 1
21
- assert temperature > 0.0
22
- assert top_p == 1.0
36
+ if n == 1:
37
+ raise ValueError(
38
+ 'n must be greater than 1 when using beam search.')
39
+ if temperature > 0.0:
40
+ raise ValueError(
41
+ 'temperature must be 0 when using beam search.')
42
+ if top_p < 1.0:
43
+ raise ValueError(
44
+ 'top_p must be 1 when using beam search.')
45
elif temperature == 0.0:
24
- # Zero temperature means greedy decoding.
25
- assert n == 1
26
- assert top_p == 1.0
27
- assert max_num_steps >= 1
28
- assert max_context_len is None or max_context_len >= 0
46
+ # Zero temperature means greedy sampling.
47
+ if n > 1:
48
+ raise ValueError(
49
+ 'n must be 1 when using greedy sampling.')
50
+ if top_p < 1.0:
51
+ raise ValueError(
52
+ 'top_p must be 1 when using greedy sampling.')
53
54
self.n = n
55
self.temperature = temperature
@@ -33,4 +57,15 @@ class SamplingParams:
57
self.use_beam_search = use_beam_search
58
self.stop_token_ids = stop_token_ids
59
self.max_num_steps = max_num_steps
36
- self.max_context_len = max_context_len
60
+ self.num_logprobs = num_logprobs
61
+ self.context_window_size = context_window_size
62
+
63
+ def __repr__(self) -> str:
64
+ return (f'SamplingParams(n={self.n}, '
65
+ f'temperature={self.temperature}, '
66
+ f'top_p={self.top_p}, '
67
+ f'use_beam_search={self.use_beam_search}, '
68
+ f'stop_token_ids={self.stop_token_ids}, '
69
+ f'max_num_steps={self.max_num_steps}, '
70
+ f'num_logprobs={self.num_logprobs}, '
71
+ f'context_window_size={self.context_window_size})')
cacheflow/sequence.py
+72
-6
@@ -1,7 +1,9 @@
1
+import copy
2
import enum
2
-from typing import List, Optional
3
+from typing import Dict, List, Optional
4
5
from cacheflow.block import LogicalTokenBlock
6
+from cacheflow.sampling_params import SamplingParams
7
8
9
class SequenceStatus(enum.Enum):
@@ -24,9 +26,11 @@ class Sequence:
26
27
self.logical_token_blocks: List[LogicalTokenBlock] = []
28
# Initialize the logical token blocks with the given token ids.
27
- self.append(token_ids)
29
+ self.add(token_ids)
30
31
self.status = SequenceStatus.PENDING
32
+ self.output_logprobs: List[Dict[int, float]] = []
33
+ self.cumulative_logprobs = 1.0
34
35
def add_block(self) -> None:
36
block = LogicalTokenBlock(
@@ -35,7 +39,7 @@ class Sequence:
39
)
40
self.logical_token_blocks.append(block)
41
38
- def append(self, token_ids: List[int]) -> None:
42
+ def add(self, token_ids: List[int]) -> None:
43
while token_ids:
44
if not self.logical_token_blocks:
45
self.add_block()
@@ -49,6 +53,12 @@ class Sequence:
53
last_block.append(token_ids[:num_empty_slots])
54
token_ids = token_ids[num_empty_slots:]
55
56
+ def append(self, token_id: int, logprobs: Dict[int, float]) -> None:
57
+ assert token_id in logprobs
58
+ self.add([token_id])
59
+ self.output_logprobs.append(logprobs)
60
+ self.cumulative_logprobs += logprobs[token_id]
61
+
62
def get_len(self) -> int:
63
return sum(block.num_tokens for block in self.logical_token_blocks)
64
@@ -58,6 +68,14 @@ class Sequence:
68
token_ids.extend(block.get_token_ids())
69
return token_ids
70
71
+ def get_last_token_id(self) -> int:
72
+ return self.logical_token_blocks[-1].get_last_token_id()
73
+
74
+ def fork(self, child_seq: 'Sequence') -> 'Sequence':
75
+ child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
76
+ child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
77
+ child_seq.cumulative_logprobs = self.cumulative_logprobs
78
+
79
def __repr__(self) -> str:
80
return (f'Sequence(seq_id={self.seq_id}, '
81
f'status={self.status.name}, '
@@ -74,11 +92,17 @@ class SequenceGroup:
92
self.group_id = group_id
93
self.seqs = seqs
94
77
- def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
95
+ def get_seqs(
96
+ self,
97
+ status: Optional[SequenceStatus] = None,
98
+ ) -> List[Sequence]:
99
if status is None:
79
- return len(self.seqs)
100
+ return self.seqs
101
else:
81
- return len([seq for seq in self.seqs if seq.status == status])
102
+ return [seq for seq in self.seqs if seq.status == status]
103
+
104
+ def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
105
+ return len(self.get_seqs(status))
106
107
def find(self, seq_id: int) -> Sequence:
108
for seq in self.seqs:
@@ -92,3 +116,45 @@ class SequenceGroup:
116
def __repr__(self) -> str:
117
return (f'SequenceGroup(group_id={self.group_id}, '
118
f'num_seqs={len(self.seqs)})')
119
+
120
+
121
+class SequenceGroupInputs:
122
+
123
+ def __init__(
124
+ self,
125
+ group_id: int,
126
+ is_prompt: bool,
127
+ input_tokens: Dict[int, List[int]], # Seq id -> token ids.
128
+ context_len: int,
129
+ seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
130
+ sampling_params: SamplingParams,
131
+ block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
132
+ ) -> None:
133
+ self.group_id = group_id
134
+ self.is_prompt = is_prompt
135
+ self.input_tokens = input_tokens
136
+ self.context_len = context_len
137
+ self.seq_logprobs = seq_logprobs
138
+ self.sampling_params = sampling_params
139
+ self.block_tables = block_tables
140
+
141
+
142
+class SequenceOutputs:
143
+
144
+ def __init__(
145
+ self,
146
+ seq_id: int,
147
+ parent_seq_id: int,
148
+ output_token: int,
149
+ logprobs: Dict[int, float], # Token id -> logP(x_i+1 | x_0, ..., x_i).
150
+ ) -> None:
151
+ self.seq_id = seq_id
152
+ self.parent_seq_id = parent_seq_id
153
+ self.output_token = output_token
154
+ self.logprobs = logprobs
155
+
156
+ def __repr__(self) -> str:
157
+ return (f'SequenceOutputs(seq_id={self.seq_id}, '
158
+ f'parent_seq_id={self.parent_seq_id}, '
159
+ f'output_token={self.output_token}), '
160
+ f'logprobs={self.logprobs}')
cacheflow/worker/cache_engine.py
+27
-8
@@ -97,7 +97,7 @@ class CacheEngine:
97
cpu_cache.append((key_blocks, value_blocks))
98
return cpu_cache
99
100
- def _copy_blocks(
100
+ def _swap(
101
self,
102
src: List[KVCache],
103
dst: List[KVCache],
@@ -108,19 +108,38 @@ class CacheEngine:
108
src_key_cache, src_value_cache = src[i]
109
dst_key_cache, dst_value_cache = dst[i]
110
# Copy the key blocks.
111
- cache_ops.copy_cache_blocks(
111
+ cache_ops.swap_blocks(
112
src_key_cache, dst_key_cache, src_to_dst)
113
# Copy the value blocks.
114
- cache_ops.copy_cache_blocks(
114
+ cache_ops.swap_blocks(
115
src_value_cache, dst_value_cache, src_to_dst)
116
event = self.events[i]
117
event.record(stream=self.cache_stream)
118
119
- def copy(self, src_to_dst: Dict[int, int]) -> None:
120
- self._copy_blocks(self.gpu_cache, self.gpu_cache, src_to_dst)
121
-
119
def swap_in(self, src_to_dst: Dict[int, int]) -> None:
123
- self._copy_blocks(self.cpu_cache, self.gpu_cache, src_to_dst)
120
+ self._swap(self.cpu_cache, self.gpu_cache, src_to_dst)
121
122
def swap_out(self, src_to_dst: Dict[int, int]) -> None:
126
- self._copy_blocks(self.gpu_cache, self.cpu_cache, src_to_dst)
123
+ self._swap(self.gpu_cache, self.cpu_cache, src_to_dst)
124
+
125
+ def _copy(
126
+ self,
127
+ src: List[KVCache],
128
+ dst: List[KVCache],
129
+ src_to_dsts: Dict[int, List[int]],
130
+ ) -> None:
131
+ with torch.cuda.stream(self.cache_stream):
132
+ for i in range(self.num_layers):
133
+ src_key_cache, src_value_cache = src[i]
134
+ dst_key_cache, dst_value_cache = dst[i]
135
+ # Copy the key blocks.
136
+ cache_ops.copy_blocks(
137
+ src_key_cache, dst_key_cache, src_to_dsts)
138
+ # Copy the value blocks.
139
+ cache_ops.copy_blocks(
140
+ src_value_cache, dst_value_cache, src_to_dsts)
141
+ event = self.events[i]
142
+ event.record(stream=self.cache_stream)
143
+
144
+ def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
145
+ self._copy(self.gpu_cache, self.gpu_cache, src_to_dsts)
cacheflow/worker/controller.py
+7
-10
@@ -1,6 +1,7 @@
1
from typing import Dict, List, Union
2
3
from cacheflow.master.scheduler import Scheduler
4
+from cacheflow.sequence import SequenceGroupInputs
5
from cacheflow.worker.worker import Worker
6
7
@@ -14,7 +15,8 @@ class Controller:
15
block_size: int,
16
num_gpu_blocks: int,
17
num_cpu_blocks: int,
17
- dtype: str = 'half',
18
+ dtype: str,
19
+ seed: int,
20
) -> None:
21
self.node_id = node_id
22
self.num_workers = num_workers
@@ -37,6 +39,7 @@ class Controller:
39
num_gpu_blocks=num_gpu_blocks,
40
num_cpu_blocks=num_cpu_blocks,
41
dtype=dtype,
42
+ seed=seed,
43
)
44
self.workers.append(worker)
45
@@ -49,22 +52,16 @@ class Controller:
52
53
def execute_stage(
54
self,
52
- prompt_tokens: Dict[int, List[int]],
53
- generation_tokens: Dict[int, int],
54
- context_lens: Dict[int, int],
55
- block_tables: Dict[int, List[int]],
55
+ input_seq_groups: List[SequenceGroupInputs],
56
blocks_to_swap_in: Dict[int, int],
57
blocks_to_swap_out: Dict[int, int],
58
- blocks_to_copy: Dict[int, int],
58
+ blocks_to_copy: Dict[int, List[int]],
59
) -> None:
60
# FIXME: Support tensor parallelism.
61
assert len(self.workers) == 1
62
worker = self.workers[0]
63
output = worker.execute_stage(
64
- prompt_tokens,
65
- generation_tokens,
66
- context_lens,
67
- block_tables,
64
+ input_seq_groups,
65
blocks_to_swap_in,
66
blocks_to_swap_out,
67
blocks_to_copy,
cacheflow/worker/worker.py
+76
-43
@@ -1,9 +1,13 @@
1
-from typing import Dict, List, Tuple, Union
1
+from typing import Dict, List, Tuple
2
3
import torch
4
5
from cacheflow.models import get_model
6
+from cacheflow.models import set_seed
7
from cacheflow.models import InputMetadata
8
+from cacheflow.sampling_params import SamplingParams
9
+from cacheflow.sequence import SequenceGroupInputs
10
+from cacheflow.sequence import SequenceOutputs
11
from cacheflow.worker.cache_engine import CacheEngine
12
13
@@ -18,6 +22,7 @@ class Worker:
22
num_gpu_blocks: int,
23
num_cpu_blocks: int,
24
dtype: str,
25
+ seed: int,
26
) -> None:
27
self.worker_id = worker_id
28
self.gpu_id = gpu_id
@@ -33,6 +38,11 @@ class Worker:
38
self.head_size = self.model.config.hidden_size // self.num_heads
39
self.dtype = self.model.dtype
40
41
+ # Set the seed.
42
+ # We set the seed after initializing the model to ensure that
43
+ # the random state is not affected by the model initialization.
44
+ set_seed(seed)
45
+
46
self.cache_engine = CacheEngine(
47
worker_id=worker_id,
48
gpu_id=gpu_id,
@@ -49,55 +59,81 @@ class Worker:
59
60
def prepare_inputs(
61
self,
52
- prompt_tokens: Dict[int, List[int]], # Seq id -> List of input token ids.
53
- generation_tokens: Dict[int, int], # Seq id -> Input token id.
54
- context_lens: Dict[int, int], # Seq id -> Number of tokens participating in attention.
55
- block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
62
+ input_seq_groups: List[SequenceGroupInputs],
63
) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
57
- # TODO(woosuk): Support interactive generation.
58
- # Add the prompt tokens.
59
- prompt_lens: List[int] = []
64
+ seq_groups: List[Tuple[List[int], SamplingParams]] = []
65
+ seq_logprobs: Dict[int, float] = {}
66
+ sampling_params: Dict[int, SamplingParams] = {}
67
input_tokens: List[int] = []
68
input_positions: List[int] = []
69
slot_mapping: List[int] = []
70
64
- prompt_seq_ids = sorted(prompt_tokens.keys())
65
- for seq_id in prompt_seq_ids:
66
- prompt_len = len(prompt_tokens[seq_id])
71
+ # Add prompt tokens.
72
+ prompt_lens: List[int] = []
73
+ for input_seq_group in input_seq_groups:
74
+ if not input_seq_group.is_prompt:
75
+ continue
76
+
77
+ seq_ids = list(input_seq_group.input_tokens.keys())
78
+ sampling_params = input_seq_group.sampling_params
79
+ seq_groups.append((seq_ids, sampling_params))
80
+ seq_logprobs.update(input_seq_group.seq_logprobs)
81
+
82
+ # Use any sequence in the group.
83
+ seq_id = seq_ids[0]
84
+
85
+ prompt_tokens = input_seq_group.input_tokens[seq_id]
86
+ prompt_len = len(prompt_tokens)
87
prompt_lens.append(prompt_len)
88
69
- input_tokens.extend(prompt_tokens[seq_id])
70
- input_positions.extend(range(len(prompt_tokens[seq_id])))
89
+ input_tokens.extend(prompt_tokens)
90
+ # NOTE(woosuk): Here we assume that the first token in the prompt
91
+ # is always the first token in the sequence.
92
+ input_positions.extend(range(len(prompt_tokens)))
93
72
- block_table = block_tables[seq_id]
94
+ # Compute the slot mapping.
95
+ block_table = input_seq_group.block_tables[seq_id]
96
for i in range(prompt_len):
97
block_number = block_table[i // self.block_size]
98
block_offset = i % self.block_size
99
slot = block_number * self.block_size + block_offset
100
slot_mapping.append(slot)
101
79
- # Add the generation tokens.
102
+ # Add generation tokens.
103
max_context_len = 0
104
max_num_blocks_per_seq = 0
105
+ context_lens: List[int] = []
106
generation_block_tables: List[List[int]] = []
83
-
84
- generation_seq_ids = sorted(generation_tokens.keys())
85
- for seq_id in generation_seq_ids:
86
- input_tokens.append(generation_tokens[seq_id])
87
- position_id = context_lens[seq_id] - 1
88
- input_positions.append(position_id)
89
-
90
- block_table = block_tables[seq_id]
91
- generation_block_tables.append(block_table)
92
-
93
- max_context_len = max(max_context_len, context_lens[seq_id])
94
- max_num_blocks_per_seq = max(
95
- max_num_blocks_per_seq, len(block_table))
96
-
97
- block_number = block_table[position_id // self.block_size]
98
- block_offset = position_id % self.block_size
99
- slot = block_number * self.block_size + block_offset
100
- slot_mapping.append(slot)
107
+ for input_seq_group in input_seq_groups:
108
+ if input_seq_group.is_prompt:
109
+ continue
110
+
111
+ seq_ids = list(input_seq_group.input_tokens.keys())
112
+ sampling_params = input_seq_group.sampling_params
113
+ seq_groups.append((seq_ids, sampling_params))
114
+ seq_logprobs.update(input_seq_group.seq_logprobs)
115
+
116
+ for seq_id in seq_ids:
117
+ assert len(input_seq_group.input_tokens[seq_id]) == 1
118
+ generation_token = input_seq_group.input_tokens[seq_id][0]
119
+ input_tokens.append(generation_token)
120
+
121
+ position = input_seq_group.context_len - 1
122
+ input_positions.append(position)
123
+
124
+ block_table = input_seq_group.block_tables[seq_id]
125
+ generation_block_tables.append(block_table)
126
+
127
+ max_context_len = max(
128
+ max_context_len, input_seq_group.context_len)
129
+ max_num_blocks_per_seq = max(
130
+ max_num_blocks_per_seq, len(block_table))
131
+ context_lens.append(input_seq_group.context_len)
132
+
133
+ block_number = block_table[position // self.block_size]
134
+ block_offset = position % self.block_size
135
+ slot = block_number * self.block_size + block_offset
136
+ slot_mapping.append(slot)
137
138
# Optimization: Pad the input length to be a multiple of 8.
139
# This is required for utilizing the Tensor Cores in NVIDIA GPUs.
@@ -112,8 +148,7 @@ class Worker:
148
slot_mapping_tensor = torch.tensor(
149
slot_mapping, dtype=torch.int, device=self.device)
150
context_lens_tensor = torch.tensor(
115
- [context_lens[seq_id] for seq_id in generation_seq_ids],
116
- dtype=torch.int, device=self.device)
151
+ context_lens, dtype=torch.int, device=self.device)
152
padded_block_tables = [
153
_pad_to_max(block_table, max_num_blocks_per_seq)
154
for block_table in generation_block_tables]
@@ -121,7 +156,8 @@ class Worker:
156
padded_block_tables, dtype=torch.int, device=self.device)
157
158
input_metadata = InputMetadata(
124
- seq_ids=prompt_seq_ids + generation_seq_ids,
159
+ seq_groups=seq_groups,
160
+ seq_logprobs=seq_logprobs,
161
prompt_lens=prompt_lens,
162
slot_mapping=slot_mapping_tensor,
163
context_lens=context_lens_tensor,
@@ -133,14 +169,11 @@ class Worker:
169
@torch.inference_mode()
170
def execute_stage(
171
self,
136
- prompt_tokens: Dict[int, List[int]], # Seq id -> List of input token ids.
137
- generation_tokens: Dict[int, int], # Seq id -> Input token id.
138
- context_lens: Dict[int, int], # Seq id -> Number of tokens participating in attention.
139
- block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
172
+ input_seq_groups: List[SequenceGroupInputs],
173
blocks_to_swap_in: Dict[int, int],
174
blocks_to_swap_out: Dict[int, int],
142
- blocks_to_copy: Dict[int, int],
143
- ) -> Union[torch.Tensor, Dict[int, Tuple[int, int]]]:
175
+ blocks_to_copy: Dict[int, List[int]],
176
+ ) -> Dict[int, SequenceOutputs]:
177
# Issue cache operations.
178
command_issued = False
179
if blocks_to_swap_in:
@@ -160,7 +193,7 @@ class Worker:
193
194
# Prepare input tensors.
195
input_tokens, input_positions, input_metadata = self.prepare_inputs(
163
- prompt_tokens, generation_tokens, context_lens, block_tables)
196
+ input_seq_groups)
197
198
# Execute the model.
199
output = self.model(
csrc/cache.cpp
+14
-2
@@ -1,10 +1,18 @@
1
#include <torch/extension.h>
2
3
-void copy_blocks(
3
+#include <map>
4
+#include <vector>
5
+
6
+void swap_blocks(
7
torch::Tensor& src,
8
torch::Tensor& dst,
9
const std::map<int64_t, int64_t>& block_mapping);
10
11
+void copy_blocks(
12
+ torch::Tensor& src,
13
+ torch::Tensor& dst,
14
+ const std::map<int64_t, std::vector<int64_t>>& block_mapping);
15
+
16
void reshape_and_cache(
17
torch::Tensor& key,
18
torch::Tensor& value,
@@ -14,7 +22,11 @@ void reshape_and_cache(
22
23
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
24
m.def(
17
- "copy_cache_blocks",
25
+ "swap_blocks",
26
+ &swap_blocks,
27
+ "Swap in (out) the cache blocks from src to dst");
28
+ m.def(
29
+ "copy_blocks",
30
©_blocks,
31
"Copy the cache blocks from src to dst");
32
m.def(
csrc/cache_kernels.cu
+31
-1
@@ -5,8 +5,9 @@
5
#include <algorithm>
6
#include <cassert>
7
#include <map>
8
+#include <vector>
9
9
-void copy_blocks(
10
+void swap_blocks(
11
torch::Tensor& src,
12
torch::Tensor& dst,
13
const std::map<int64_t, int64_t>& block_mapping) {
@@ -43,6 +44,35 @@ void copy_blocks(
44
}
45
}
46
47
+void copy_blocks(
48
+ torch::Tensor& src,
49
+ torch::Tensor& dst,
50
+ const std::map<int64_t, std::vector<int64_t>>& block_mapping) {
51
+ torch::Device src_device = src.device();
52
+ torch::Device dst_device = dst.device();
53
+ assert(src_device.is_cuda() && dst_device.is_cuda());
54
+ cudaMemcpyKind memcpy_type = cudaMemcpyDeviceToDevice;
55
+
56
+ void *src_ptr = src.data_ptr();
57
+ void *dst_ptr = dst.data_ptr();
58
+
59
+ const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
60
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
61
+ for (const auto& pair : block_mapping) {
62
+ int64_t src_block_number = pair.first;
63
+ for (int64_t dst_block_number : pair.second) {
64
+ int64_t src_offset = src_block_number * block_size_in_bytes;
65
+ int64_t dst_offset = dst_block_number * block_size_in_bytes;
66
+ cudaMemcpyAsync(
67
+ dst_ptr + dst_offset,
68
+ src_ptr + src_offset,
69
+ block_size_in_bytes,
70
+ memcpy_type,
71
+ stream);
72
+ }
73
+ }
74
+}
75
+
76
template<typename scalar_t>
77
__global__ void reshape_and_cache_kernel(
78
const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
server.py
+10
-7
@@ -15,6 +15,8 @@ parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of
15
parser.add_argument('--num-cpu-blocks', type=int, default=32, help='number of CPU blocks (per GPU)')
16
# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
17
parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
18
+# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
19
+parser.add_argument('--seed', type=int, default=0, help='random seed')
20
args = parser.parse_args()
21
22
@@ -30,6 +32,7 @@ def main():
32
num_gpu_blocks=args.num_gpu_blocks,
33
num_cpu_blocks=args.num_cpu_blocks,
34
dtype=args.dtype,
35
+ seed=args.seed,
36
)
37
controllers.append(controller)
38
@@ -52,18 +55,18 @@ def main():
55
controllers[i].set_next(controllers[i + 1])
56
controllers[-1].set_next(scheduler)
57
58
+ # Test the following inputs.
59
test_inputs = [
56
- 'Ion Stoica is a',
57
- 'UC Berkeley is',
58
- 'The future of cloud computing is',
60
+ ('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),
61
+ ('UC Berkeley is', {'n': 3, 'temperature': 0.8, 'top_p': 0.99}),
62
+ ('The future of cloud computing is', {}), # Use default parameters.
63
]
60
-
61
- # FIXME
64
while True:
65
if test_inputs:
64
- frontend.query(test_inputs.pop())
66
+ text, sampling_params = test_inputs.pop(0)
67
+ frontend.query(text, **sampling_params)
68
scheduler.step()
66
- if not scheduler.pending and not scheduler.running:
69
+ if not (scheduler.pending or scheduler.running or test_inputs):
70
break
71
72