A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Implement presence and frequency penalties (#95)
Woosuk Kwon committed
May 10, 2023 at 23:39 UTC
55f8b0a5def22ed6b85d3b91b726a7573d54313b
9 files changed
+215
-82
cacheflow/core/scheduler.py
+7
-16
@@ -3,11 +3,12 @@ import time
3
from typing import Dict, List, Optional, Tuple
4
5
from cacheflow.core.block_manager import BlockSpaceManager
6
-from cacheflow.logger import init_logger
6
from cacheflow.core.policy import PolicyFactory
7
+from cacheflow.logger import init_logger
8
from cacheflow.sampling_params import SamplingParams
9
-from cacheflow.sequence import (Sequence, SequenceGroup, SequenceGroupMetadata,
10
- SequenceOutputs, SequenceStatus)
9
+from cacheflow.sequence import (Sequence, SequenceData, SequenceGroup,
10
+ SequenceGroupMetadata, SequenceOutputs,
11
+ SequenceStatus)
12
13
logger = init_logger(__name__)
14
@@ -246,27 +247,17 @@ class Scheduler:
247
group_id = seq_group.group_id
248
is_prompt = group_id in prompt_group_ids
249
249
- input_tokens: Dict[int, List[int]] = {}
250
- seq_logprobs: Dict[int, float] = {}
250
+ seq_data: Dict[int, List[SequenceData]] = {}
251
block_tables: Dict[int, List[int]] = {}
252
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
253
seq_id = seq.seq_id
254
+ seq_data[seq_id] = seq.data
255
block_tables[seq_id] = self.block_manager.get_block_table(seq)
255
- if is_prompt:
256
- input_tokens[seq_id] = seq.get_token_ids()
257
- else:
258
- input_tokens[seq_id] = [seq.get_last_token_id()]
259
- seq_logprobs[seq_id] = seq.cumulative_logprobs
260
- # NOTE(woosuk): Sequences in the same group have the same
261
- # sequence length
262
- seq_len = seq.get_len()
256
257
seq_group_metadata = SequenceGroupMetadata(
258
group_id=group_id,
259
is_prompt=is_prompt,
267
- input_tokens=input_tokens,
268
- context_len=seq_len,
269
- seq_logprobs=seq_logprobs,
260
+ seq_data=seq_data,
261
sampling_params=self.sampling_params[group_id],
262
block_tables=block_tables,
263
)
cacheflow/frontend/fastapi_frontend.py
+1
-1
@@ -96,7 +96,7 @@ class FastAPIServer:
96
seqs: List[Sequence] = []
97
for _ in range(sampling_params.n):
98
seq_id = next(self.seq_counter)
99
- seq = Sequence(seq_id, token_ids, block_size=self.block_size)
99
+ seq = Sequence(seq_id, prompt, token_ids, block_size=self.block_size)
100
seqs.append(seq)
101
102
arrival_time = time.time()
cacheflow/frontend/simple_frontend.py
+3
-2
@@ -35,10 +35,11 @@ class SimpleFrontend:
35
sampling_params: SamplingParams,
36
) -> None:
37
token_ids = self.tokenizer.encode(prompt)
38
- self._add_query(token_ids, sampling_params)
38
+ self._add_query(prompt, token_ids, sampling_params)
39
40
def _add_query(
41
self,
42
+ prompt: str,
43
token_ids: List[int],
44
sampling_params: SamplingParams,
45
arrival_time: Optional[float] = None,
@@ -48,7 +49,7 @@ class SimpleFrontend:
49
seqs: List[Sequence] = []
50
for _ in range(sampling_params.n):
51
seq_id = next(self.seq_counter)
51
- seq = Sequence(seq_id, token_ids, block_size=self.block_size)
52
+ seq = Sequence(seq_id, prompt, token_ids, block_size=self.block_size)
53
seqs.append(seq)
54
55
group_id = next(self.seq_group_counter)
cacheflow/model_executor/input_metadata.py
+6
-4
@@ -1,17 +1,18 @@
1
-from typing import List, Dict, Tuple
1
+from typing import Dict, List, Tuple
2
3
import torch
4
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
5
6
from cacheflow.sampling_params import SamplingParams
7
+from cacheflow.sequence import SequenceData
8
9
10
class InputMetadata:
11
12
def __init__(
13
self,
13
- seq_groups: List[Tuple[List[int], SamplingParams]],
14
- seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
14
+ seq_groups: List[Tuple[List[int], SamplingParams]], # List of (seq_ids, sampling_params).
15
+ seq_data: Dict[int, SequenceData], # Seq_id -> SequenceData.
16
prompt_lens: List[int],
17
slot_mapping: torch.Tensor,
18
context_lens: torch.Tensor,
@@ -19,7 +20,7 @@ class InputMetadata:
20
block_tables: torch.Tensor,
21
) -> None:
22
self.seq_groups = seq_groups
22
- self.seq_logprobs = seq_logprobs
23
+ self.seq_data = seq_data
24
self.prompt_lens = prompt_lens
25
self.slot_mapping = slot_mapping
26
self.context_lens = context_lens
@@ -39,6 +40,7 @@ class InputMetadata:
40
assert context_lens.shape[0] == self.num_generation_tokens
41
42
def __repr__(self) -> str:
43
+ # Print only useful metadata.
44
return (f'InputMetadata('
45
f'num_valid_tokens={self.num_valid_tokens}, '
46
f'num_prompt_tokens={self.num_prompt_tokens}, '
cacheflow/model_executor/layers/sampler.py
+106
-8
@@ -1,5 +1,6 @@
1
from typing import Dict, List, Tuple
2
3
+import numpy as np
4
import torch
5
import torch.nn as nn
6
@@ -31,6 +32,16 @@ class Sampler(nn.Module):
32
# Remove paddings in vocab (if any).
33
logits = logits[:, :self.vocab_size]
34
35
+ # Apply presence and frequency penalties.
36
+ output_tokens = _get_output_tokens(input_metadata)
37
+ assert len(output_tokens) == logits.shape[0]
38
+ presence_penalties, frequency_penalties = _get_penalties(input_metadata)
39
+ assert len(presence_penalties) == logits.shape[0]
40
+ assert len(frequency_penalties) == logits.shape[0]
41
+ logits = _apply_penalties(
42
+ logits, output_tokens, presence_penalties, frequency_penalties,
43
+ self.vocab_size)
44
+
45
# Apply temperature scaling.
46
temperatures = _get_temperatures(input_metadata)
47
assert len(temperatures) == logits.shape[0]
@@ -43,16 +54,14 @@ class Sampler(nn.Module):
54
# We use float32 for probabilities and log probabilities.
55
# Compute the probabilities.
56
probs = torch.softmax(logits, dim=-1, dtype=torch.float)
46
- # Compute the log probabilities (before applying top-p).
57
+ # Compute the log probabilities (before applying top-p and top-k).
58
logprobs = torch.log(probs)
59
60
# Apply top-p and top-k truncation.
61
top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
62
assert len(top_ps) == len(top_ks) == probs.shape[0]
63
if any(p < 1.0 for p in top_ps) or any(k != -1 for k in top_ks):
53
- p = torch.tensor(top_ps, dtype=probs.dtype, device=probs.device)
54
- k = torch.tensor(top_ks, dtype=torch.int, device=probs.device)
55
- probs = _apply_top_p_top_k(probs, p, k)
64
+ probs = _apply_top_p_top_k(probs, top_ps, top_ks)
65
66
# Sample the next tokens.
67
return _sample(probs, logprobs, input_metadata)
@@ -72,6 +81,93 @@ def _prune_hidden_states(
81
return hidden_states[last_token_indicies]
82
83
84
+def _get_penalties(
85
+ input_metadata: InputMetadata,
86
+) -> Tuple[List[float], List[float]]:
87
+ # Collect the presence and frequency penalties.
88
+ presence_penalties: List[float] = []
89
+ frequency_penalties: List[float] = []
90
+ for i, seq_group in enumerate(input_metadata.seq_groups):
91
+ seq_ids, sampling_params = seq_group
92
+ p = sampling_params.presence_penalty
93
+ f = sampling_params.frequency_penalty
94
+ if i < input_metadata.num_prompts:
95
+ # A prompt input.
96
+ presence_penalties.append(p)
97
+ frequency_penalties.append(f)
98
+ else:
99
+ # A generation token.
100
+ presence_penalties += [p] * len(seq_ids)
101
+ frequency_penalties += [f] * len(seq_ids)
102
+ return presence_penalties, frequency_penalties
103
+
104
+
105
+def _get_output_tokens(
106
+ input_metadata: InputMetadata,
107
+) -> List[List[int]]:
108
+ output_tokens: List[List[int]] = []
109
+ for i, seq_group in enumerate(input_metadata.seq_groups):
110
+ seq_ids, _ = seq_group
111
+ if i < input_metadata.num_prompts:
112
+ # A prompt input.
113
+ # NOTE: While the prompt input usually has no output tokens,
114
+ # it may have output tokens in the case of recomputation.
115
+ seq_id = seq_ids[0]
116
+ seq_data = input_metadata.seq_data[seq_id]
117
+ output_tokens.append(seq_data.output_token_ids)
118
+ else:
119
+ # A generation token.
120
+ for seq_id in seq_ids:
121
+ seq_data = input_metadata.seq_data[seq_id]
122
+ output_tokens.append(seq_data.output_token_ids)
123
+ return output_tokens
124
+
125
+
126
+def _apply_penalties(
127
+ logits: torch.Tensor,
128
+ output_tokens: List[List[int]],
129
+ presence_penalties: List[float],
130
+ frequency_penalties: List[float],
131
+ vocab_size: int,
132
+) -> torch.Tensor:
133
+ num_seqs = logits.shape[0]
134
+ # Collect the indices of sequences that have non-zero penalties.
135
+ indices = []
136
+ for i in range(num_seqs):
137
+ if not output_tokens[i]:
138
+ continue
139
+ p = presence_penalties[i]
140
+ f = frequency_penalties[i]
141
+ if p == 0.0 and f == 0.0:
142
+ continue
143
+ indices.append(i)
144
+
145
+ # Return early if all sequences have zero penalties.
146
+ if not indices:
147
+ return logits
148
+
149
+ bin_counts = []
150
+ for i in indices:
151
+ bin_counts.append(np.bincount(output_tokens[i], minlength=vocab_size))
152
+ bin_counts = np.stack(bin_counts, axis=0)
153
+ bin_counts = torch.from_numpy(bin_counts).to(dtype=logits.dtype,
154
+ device=logits.device)
155
+
156
+ frequency_penalties = [frequency_penalties[i] for i in indices]
157
+ frequency_penalties = torch.tensor(
158
+ frequency_penalties, dtype=logits.dtype, device=logits.device)
159
+ presence_penalties = [presence_penalties[i] for i in indices]
160
+ presence_penalties = torch.tensor(
161
+ presence_penalties, dtype=logits.dtype, device=logits.device)
162
+
163
+ # We follow the definition in OpenAI API.
164
+ # Refer to https://platform.openai.com/docs/api-reference/parameter-details
165
+ logits[indices] -= frequency_penalties.unsqueeze(dim=1) * bin_counts
166
+ presence_mask = (bin_counts > 0.0).to(dtype=logits.dtype)
167
+ logits[indices] -= presence_penalties.unsqueeze(dim=1) * presence_mask
168
+ return logits
169
+
170
+
171
def _get_temperatures(
172
input_metadata: InputMetadata,
173
) -> List[float]:
@@ -121,10 +217,11 @@ def _get_top_p_top_k(
217
218
def _apply_top_p_top_k(
219
probs: torch.Tensor,
124
- p: torch.Tensor,
125
- k: torch.Tensor,
220
+ top_ps: List[float],
221
+ top_ks: List[int],
222
) -> torch.Tensor:
127
- # TODO(woosuk): Optimize.
223
+ p = torch.tensor(top_ps, dtype=probs.dtype, device=probs.device)
224
+ k = torch.tensor(top_ks, dtype=torch.int, device=probs.device)
225
probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
226
227
# Apply top-p.
@@ -286,7 +383,8 @@ def _sample(
383
384
# Sample the next tokens.
385
seq_logprobs = [
289
- input_metadata.seq_logprobs[seq_id] for seq_id in seq_ids]
386
+ input_metadata.seq_data[seq_id].cumulative_logprobs
387
+ for seq_id in seq_ids]
388
parent_seq_ids, next_token_ids = _sample_from_generation_tokens(
389
seq_ids, prob, logprob, seq_logprobs, sampling_params)
390
cacheflow/sampling_params.py
+26
-9
@@ -6,6 +6,8 @@ class SamplingParams:
6
def __init__(
7
self,
8
n: int,
9
+ presence_penalty: float,
10
+ frequency_penalty: float,
11
temperature: float,
12
top_p: float,
13
top_k: int,
@@ -16,6 +18,12 @@ class SamplingParams:
18
) -> None:
19
if n < 1:
20
raise ValueError(f"n must be at least 1, got {n}.")
21
+ if not -2.0 <= presence_penalty <= 2.0:
22
+ raise ValueError(
23
+ f"presence_penalty must be in [-2, 2], got {presence_penalty}.")
24
+ if not -2.0 <= frequency_penalty <= 2.0:
25
+ raise ValueError(
26
+ f"frequency_penalty must be in [-2, 2], got {frequency_penalty}.")
27
if temperature < 0.0:
28
raise ValueError(
29
f"temperature must be non-negative, got {temperature}.")
@@ -57,6 +65,8 @@ class SamplingParams:
65
"top_k must be -1 when using greedy sampling.")
66
67
self.n = n
68
+ self.presence_penalty = presence_penalty
69
+ self.frequency_penalty = frequency_penalty
70
self.temperature = temperature
71
self.top_p = top_p
72
self.top_k = top_k
@@ -67,6 +77,8 @@ class SamplingParams:
77
78
def __repr__(self) -> str:
79
return (f"SamplingParams(n={self.n}, "
80
+ f"presence_penalty={self.presence_penalty}, "
81
+ f"frequency_penalty={self.frequency_penalty}, "
82
f"temperature={self.temperature}, "
83
f"top_p={self.top_p}, "
84
f"top_k={self.top_k},"
@@ -77,13 +89,18 @@ class SamplingParams:
89
90
@classmethod
91
def from_dict(cls, d: Dict) -> "SamplingParams":
80
- return cls(
81
- n=d.get("n", 1),
82
- temperature=d.get("temperature", 1.0),
83
- top_p=d.get("top_p", 1.0),
84
- top_k=d.get("top_k", -1),
85
- use_beam_search=d.get("use_beam_search", False),
86
- stop_token_ids=set(d.get("stop_token_ids", set())),
87
- max_num_steps=d.get("max_num_steps", 16),
88
- num_logprobs=d.get("num_logprobs", 0),
92
+ sampling_params = cls(
93
+ n=d.pop("n", 1),
94
+ presence_penalty=d.pop("presence_penalty", 0.0),
95
+ frequency_penalty=d.pop("frequency_penalty", 0.0),
96
+ temperature=d.pop("temperature", 1.0),
97
+ top_p=d.pop("top_p", 1.0),
98
+ top_k=d.pop("top_k", -1),
99
+ use_beam_search=d.pop("use_beam_search", False),
100
+ stop_token_ids=set(d.pop("stop_token_ids", set())),
101
+ max_num_steps=d.pop("max_num_steps", 16),
102
+ num_logprobs=d.pop("num_logprobs", 0),
103
)
104
+ if d:
105
+ raise ValueError(f"Unrecognized keys in dict: {d.keys()}")
106
+ return sampling_params
cacheflow/sequence.py
+46
-23
@@ -13,26 +13,55 @@ class SequenceStatus(enum.Enum):
13
FINISHED = enum.auto()
14
15
16
+class SequenceData:
17
+
18
+ def __init__(
19
+ self,
20
+ prompt_token_ids: List[int],
21
+ ) -> None:
22
+ self.prompt_token_ids = prompt_token_ids
23
+
24
+ self.output_token_ids: List[int] = []
25
+ self.cumulative_logprobs = 0.0
26
+
27
+ def get_len(self) -> int:
28
+ return len(self.output_token_ids) + len(self.prompt_token_ids)
29
+
30
+ def get_token_ids(self) -> List[int]:
31
+ return self.prompt_token_ids + self.output_token_ids
32
+
33
+ def get_last_token_id(self) -> int:
34
+ if not self.output_token_ids:
35
+ return self.prompt_token_ids[-1]
36
+ return self.output_token_ids[-1]
37
+
38
+ def __repr__(self) -> str:
39
+ return (f"SequenceData("
40
+ f"prompt={self.prompt}, "
41
+ f"prompt_token_ids={self.prompt_token_ids}, "
42
+ f"output_token_ids={self.output_token_ids})")
43
+
44
+
45
class Sequence:
46
47
def __init__(
48
self,
49
seq_id: int,
50
+ prompt: str,
51
prompt_token_ids: List[int],
52
block_size: int,
53
) -> None:
54
self.seq_id = seq_id
55
+ self.prompt = prompt
56
self.block_size = block_size
57
27
- self.prompt_len = len(prompt_token_ids)
58
+ self.data = SequenceData(prompt_token_ids)
59
+ self.output_logprobs: List[Dict[int, float]] = []
60
+
61
self.logical_token_blocks: List[LogicalTokenBlock] = []
62
# Initialize the logical token blocks with the prompt token ids.
30
- self._append_tokens(prompt_token_ids)
31
-
63
+ self._append_tokens_to_blocks(prompt_token_ids)
64
self.status = SequenceStatus.WAITING
33
- # Used for beam search.
34
- self.output_logprobs: List[Dict[int, float]] = []
35
- self.cumulative_logprobs = 0.0
65
66
def _append_logical_block(self) -> None:
67
block = LogicalTokenBlock(
@@ -41,7 +70,7 @@ class Sequence:
70
)
71
self.logical_token_blocks.append(block)
72
44
- def _append_tokens(self, token_ids: List[int]) -> None:
73
+ def _append_tokens_to_blocks(self, token_ids: List[int]) -> None:
74
while token_ids:
75
if not self.logical_token_blocks:
76
self._append_logical_block()
@@ -57,26 +86,24 @@ class Sequence:
86
87
def append_token(self, token_id: int, logprobs: Dict[int, float]) -> None:
88
assert token_id in logprobs
60
- self._append_tokens([token_id])
89
+ self._append_tokens_to_blocks([token_id])
90
self.output_logprobs.append(logprobs)
62
- self.cumulative_logprobs += logprobs[token_id]
91
+ self.data.output_token_ids.append(token_id)
92
+ self.data.cumulative_logprobs += logprobs[token_id]
93
94
def get_len(self) -> int:
65
- return sum(block.num_tokens for block in self.logical_token_blocks)
95
+ return self.data.get_len()
96
97
def get_token_ids(self) -> List[int]:
68
- token_ids: List[int] = []
69
- for block in self.logical_token_blocks:
70
- token_ids.extend(block.get_token_ids())
71
- return token_ids
98
+ return self.data.get_token_ids()
99
100
def get_last_token_id(self) -> int:
74
- return self.logical_token_blocks[-1].get_last_token_id()
101
+ return self.data.get_last_token_id()
102
103
def fork(self, child_seq: 'Sequence') -> 'Sequence':
104
child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
105
child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
79
- child_seq.cumulative_logprobs = self.cumulative_logprobs
106
+ child_seq.data = copy.deepcopy(self.data)
107
108
def __repr__(self) -> str:
109
return (f'Sequence(seq_id={self.seq_id}, '
@@ -128,17 +155,13 @@ class SequenceGroupMetadata:
155
self,
156
group_id: int,
157
is_prompt: bool,
131
- input_tokens: Dict[int, List[int]], # Seq id -> token ids.
132
- context_len: int,
133
- seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
158
+ seq_data: Dict[int, SequenceData], # Seq id -> sequence data.
159
sampling_params: SamplingParams,
135
- block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
160
+ block_tables: Dict[int, List[int]], # Seq id -> list of physical block numbers.
161
) -> None:
162
self.group_id = group_id
163
self.is_prompt = is_prompt
139
- self.input_tokens = input_tokens
140
- self.context_len = context_len
141
- self.seq_logprobs = seq_logprobs
164
+ self.seq_data = seq_data
165
self.sampling_params = sampling_params
166
self.block_tables = block_tables
167
cacheflow/worker/worker.py
+18
-17
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Tuple, Optional
1
+from typing import Dict, List, Optional, Tuple
2
3
import torch
4
@@ -8,8 +8,8 @@ from cacheflow.model_executor.parallel_utils.parallel_state import (
8
initialize_all_reduce_launcher,
9
get_tensor_model_parallel_world_size)
10
from cacheflow.sampling_params import SamplingParams
11
-from cacheflow.sequence import SequenceGroupMetadata
12
-from cacheflow.sequence import SequenceOutputs
11
+from cacheflow.sequence import (SequenceData, SequenceGroupMetadata,
12
+ SequenceOutputs)
13
from cacheflow.worker.cache_engine import CacheEngine
14
15
@@ -72,7 +72,6 @@ class Worker:
72
self.cache_events = self.cache_engine.events
73
self.gpu_cache = self.cache_engine.gpu_cache
74
75
-
75
def init_distributed_environment(self,
76
distributed_init_method: str,
77
rank: int,
@@ -96,7 +95,6 @@ class Worker:
95
seq_group_metadata_list: List[SequenceGroupMetadata],
96
) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
97
seq_groups: List[Tuple[List[int], SamplingParams]] = []
99
- seq_logprobs: Dict[int, float] = {}
98
input_tokens: List[int] = []
99
input_positions: List[int] = []
100
slot_mapping: List[int] = []
@@ -107,15 +105,15 @@ class Worker:
105
if not seq_group_metadata.is_prompt:
106
continue
107
110
- seq_ids = list(seq_group_metadata.input_tokens.keys())
108
+ seq_ids = list(seq_group_metadata.seq_data.keys())
109
sampling_params = seq_group_metadata.sampling_params
110
seq_groups.append((seq_ids, sampling_params))
113
- seq_logprobs.update(seq_group_metadata.seq_logprobs)
111
112
# Use any sequence in the group.
113
seq_id = seq_ids[0]
114
118
- prompt_tokens = seq_group_metadata.input_tokens[seq_id]
115
+ seq_data = seq_group_metadata.seq_data[seq_id]
116
+ prompt_tokens = seq_data.get_token_ids()
117
prompt_len = len(prompt_tokens)
118
prompt_lens.append(prompt_len)
119
@@ -141,27 +139,26 @@ class Worker:
139
if seq_group_metadata.is_prompt:
140
continue
141
144
- seq_ids = list(seq_group_metadata.input_tokens.keys())
142
+ seq_ids = list(seq_group_metadata.seq_data.keys())
143
sampling_params = seq_group_metadata.sampling_params
144
seq_groups.append((seq_ids, sampling_params))
147
- seq_logprobs.update(seq_group_metadata.seq_logprobs)
145
146
for seq_id in seq_ids:
150
- assert len(seq_group_metadata.input_tokens[seq_id]) == 1
151
- generation_token = seq_group_metadata.input_tokens[seq_id][0]
147
+ seq_data = seq_group_metadata.seq_data[seq_id]
148
+ generation_token = seq_data.get_last_token_id()
149
input_tokens.append(generation_token)
150
154
- position = seq_group_metadata.context_len - 1
151
+ context_len = seq_data.get_len()
152
+ position = context_len - 1
153
input_positions.append(position)
154
155
block_table = seq_group_metadata.block_tables[seq_id]
156
generation_block_tables.append(block_table)
157
160
- max_context_len = max(
161
- max_context_len, seq_group_metadata.context_len)
158
+ max_context_len = max(max_context_len, context_len)
159
max_num_blocks_per_seq = max(
160
max_num_blocks_per_seq, len(block_table))
164
- context_lens.append(seq_group_metadata.context_len)
161
+ context_lens.append(context_len)
162
163
block_number = block_table[position // self.block_size]
164
block_offset = position % self.block_size
@@ -188,9 +185,13 @@ class Worker:
185
block_tables_tensor = torch.tensor(
186
padded_block_tables, dtype=torch.int, device='cuda')
187
188
+ seq_data: Dict[int, SequenceData] = {}
189
+ for seq_group_metadata in seq_group_metadata_list:
190
+ seq_data.update(seq_group_metadata.seq_data)
191
+
192
input_metadata = InputMetadata(
193
seq_groups=seq_groups,
193
- seq_logprobs=seq_logprobs,
194
+ seq_data=seq_data,
195
prompt_lens=prompt_lens,
196
slot_mapping=slot_mapping_tensor,
197
context_lens=context_lens_tensor,
simple_server.py
+2
-2
@@ -11,8 +11,8 @@ def main(args: argparse.Namespace):
11
# Test the following inputs.
12
test_inputs = [
13
("A robot may not injure a human being", {}), # Use default parameters.
14
- ("To be or not to be,", {"temperature": 0.8, "top_k": 5}),
15
- ("What is the meaning of life?", {"n": 2, "temperature": 0.8, "top_p": 0.95}),
14
+ ("To be or not to be,", {"temperature": 0.8, "top_k": 5, "presence_penalty": 0.2}),
15
+ ("What is the meaning of life?", {"n": 2, "temperature": 0.8, "top_p": 0.95, "frequency_penalty": 0.1}),
16
("It is only with the heart that one can see rightly", {"n": 3, "use_beam_search": True, "temperature": 0.0}),
17
]
18
while True: