@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 55f8b0a5de
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Implement presence and frequency penalties (#95)

Woosuk Kwon committed May 10, 2023 at 23:39 UTC 55f8b0a5def22ed6b85d3b91b726a7573d54313b
9 files changed +215 -82
cacheflow/core/scheduler.py
+7 -16
@@ -3,11 +3,12 @@ import time
3 from typing import Dict, List, Optional, Tuple
4
5 from cacheflow.core.block_manager import BlockSpaceManager
6 -from cacheflow.logger import init_logger
6 from cacheflow.core.policy import PolicyFactory
7 +from cacheflow.logger import init_logger
8 from cacheflow.sampling_params import SamplingParams
9 -from cacheflow.sequence import (Sequence, SequenceGroup, SequenceGroupMetadata,
10 - SequenceOutputs, SequenceStatus)
9 +from cacheflow.sequence import (Sequence, SequenceData, SequenceGroup,
10 + SequenceGroupMetadata, SequenceOutputs,
11 + SequenceStatus)
12
13 logger = init_logger(__name__)
14
@@ -246,27 +247,17 @@ class Scheduler:
247 group_id = seq_group.group_id
248 is_prompt = group_id in prompt_group_ids
249
249 - input_tokens: Dict[int, List[int]] = {}
250 - seq_logprobs: Dict[int, float] = {}
250 + seq_data: Dict[int, List[SequenceData]] = {}
251 block_tables: Dict[int, List[int]] = {}
252 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
253 seq_id = seq.seq_id
254 + seq_data[seq_id] = seq.data
255 block_tables[seq_id] = self.block_manager.get_block_table(seq)
255 - if is_prompt:
256 - input_tokens[seq_id] = seq.get_token_ids()
257 - else:
258 - input_tokens[seq_id] = [seq.get_last_token_id()]
259 - seq_logprobs[seq_id] = seq.cumulative_logprobs
260 - # NOTE(woosuk): Sequences in the same group have the same
261 - # sequence length
262 - seq_len = seq.get_len()
256
257 seq_group_metadata = SequenceGroupMetadata(
258 group_id=group_id,
259 is_prompt=is_prompt,
267 - input_tokens=input_tokens,
268 - context_len=seq_len,
269 - seq_logprobs=seq_logprobs,
260 + seq_data=seq_data,
261 sampling_params=self.sampling_params[group_id],
262 block_tables=block_tables,
263 )
cacheflow/frontend/fastapi_frontend.py
+1 -1
@@ -96,7 +96,7 @@ class FastAPIServer:
96 seqs: List[Sequence] = []
97 for _ in range(sampling_params.n):
98 seq_id = next(self.seq_counter)
99 - seq = Sequence(seq_id, token_ids, block_size=self.block_size)
99 + seq = Sequence(seq_id, prompt, token_ids, block_size=self.block_size)
100 seqs.append(seq)
101
102 arrival_time = time.time()
cacheflow/frontend/simple_frontend.py
+3 -2
@@ -35,10 +35,11 @@ class SimpleFrontend:
35 sampling_params: SamplingParams,
36 ) -> None:
37 token_ids = self.tokenizer.encode(prompt)
38 - self._add_query(token_ids, sampling_params)
38 + self._add_query(prompt, token_ids, sampling_params)
39
40 def _add_query(
41 self,
42 + prompt: str,
43 token_ids: List[int],
44 sampling_params: SamplingParams,
45 arrival_time: Optional[float] = None,
@@ -48,7 +49,7 @@ class SimpleFrontend:
49 seqs: List[Sequence] = []
50 for _ in range(sampling_params.n):
51 seq_id = next(self.seq_counter)
51 - seq = Sequence(seq_id, token_ids, block_size=self.block_size)
52 + seq = Sequence(seq_id, prompt, token_ids, block_size=self.block_size)
53 seqs.append(seq)
54
55 group_id = next(self.seq_group_counter)
cacheflow/model_executor/input_metadata.py
+6 -4
@@ -1,17 +1,18 @@
1 -from typing import List, Dict, Tuple
1 +from typing import Dict, List, Tuple
2
3 import torch
4 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
5
6 from cacheflow.sampling_params import SamplingParams
7 +from cacheflow.sequence import SequenceData
8
9
10 class InputMetadata:
11
12 def __init__(
13 self,
13 - seq_groups: List[Tuple[List[int], SamplingParams]],
14 - seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
14 + seq_groups: List[Tuple[List[int], SamplingParams]], # List of (seq_ids, sampling_params).
15 + seq_data: Dict[int, SequenceData], # Seq_id -> SequenceData.
16 prompt_lens: List[int],
17 slot_mapping: torch.Tensor,
18 context_lens: torch.Tensor,
@@ -19,7 +20,7 @@ class InputMetadata:
20 block_tables: torch.Tensor,
21 ) -> None:
22 self.seq_groups = seq_groups
22 - self.seq_logprobs = seq_logprobs
23 + self.seq_data = seq_data
24 self.prompt_lens = prompt_lens
25 self.slot_mapping = slot_mapping
26 self.context_lens = context_lens
@@ -39,6 +40,7 @@ class InputMetadata:
40 assert context_lens.shape[0] == self.num_generation_tokens
41
42 def __repr__(self) -> str:
43 + # Print only useful metadata.
44 return (f'InputMetadata('
45 f'num_valid_tokens={self.num_valid_tokens}, '
46 f'num_prompt_tokens={self.num_prompt_tokens}, '
cacheflow/model_executor/layers/sampler.py
+106 -8
@@ -1,5 +1,6 @@
1 from typing import Dict, List, Tuple
2
3 +import numpy as np
4 import torch
5 import torch.nn as nn
6
@@ -31,6 +32,16 @@ class Sampler(nn.Module):
32 # Remove paddings in vocab (if any).
33 logits = logits[:, :self.vocab_size]
34
35 + # Apply presence and frequency penalties.
36 + output_tokens = _get_output_tokens(input_metadata)
37 + assert len(output_tokens) == logits.shape[0]
38 + presence_penalties, frequency_penalties = _get_penalties(input_metadata)
39 + assert len(presence_penalties) == logits.shape[0]
40 + assert len(frequency_penalties) == logits.shape[0]
41 + logits = _apply_penalties(
42 + logits, output_tokens, presence_penalties, frequency_penalties,
43 + self.vocab_size)
44 +
45 # Apply temperature scaling.
46 temperatures = _get_temperatures(input_metadata)
47 assert len(temperatures) == logits.shape[0]
@@ -43,16 +54,14 @@ class Sampler(nn.Module):
54 # We use float32 for probabilities and log probabilities.
55 # Compute the probabilities.
56 probs = torch.softmax(logits, dim=-1, dtype=torch.float)
46 - # Compute the log probabilities (before applying top-p).
57 + # Compute the log probabilities (before applying top-p and top-k).
58 logprobs = torch.log(probs)
59
60 # Apply top-p and top-k truncation.
61 top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
62 assert len(top_ps) == len(top_ks) == probs.shape[0]
63 if any(p < 1.0 for p in top_ps) or any(k != -1 for k in top_ks):
53 - p = torch.tensor(top_ps, dtype=probs.dtype, device=probs.device)
54 - k = torch.tensor(top_ks, dtype=torch.int, device=probs.device)
55 - probs = _apply_top_p_top_k(probs, p, k)
64 + probs = _apply_top_p_top_k(probs, top_ps, top_ks)
65
66 # Sample the next tokens.
67 return _sample(probs, logprobs, input_metadata)
@@ -72,6 +81,93 @@ def _prune_hidden_states(
81 return hidden_states[last_token_indicies]
82
83
84 +def _get_penalties(
85 + input_metadata: InputMetadata,
86 +) -> Tuple[List[float], List[float]]:
87 + # Collect the presence and frequency penalties.
88 + presence_penalties: List[float] = []
89 + frequency_penalties: List[float] = []
90 + for i, seq_group in enumerate(input_metadata.seq_groups):
91 + seq_ids, sampling_params = seq_group
92 + p = sampling_params.presence_penalty
93 + f = sampling_params.frequency_penalty
94 + if i < input_metadata.num_prompts:
95 + # A prompt input.
96 + presence_penalties.append(p)
97 + frequency_penalties.append(f)
98 + else:
99 + # A generation token.
100 + presence_penalties += [p] * len(seq_ids)
101 + frequency_penalties += [f] * len(seq_ids)
102 + return presence_penalties, frequency_penalties
103 +
104 +
105 +def _get_output_tokens(
106 + input_metadata: InputMetadata,
107 +) -> List[List[int]]:
108 + output_tokens: List[List[int]] = []
109 + for i, seq_group in enumerate(input_metadata.seq_groups):
110 + seq_ids, _ = seq_group
111 + if i < input_metadata.num_prompts:
112 + # A prompt input.
113 + # NOTE: While the prompt input usually has no output tokens,
114 + # it may have output tokens in the case of recomputation.
115 + seq_id = seq_ids[0]
116 + seq_data = input_metadata.seq_data[seq_id]
117 + output_tokens.append(seq_data.output_token_ids)
118 + else:
119 + # A generation token.
120 + for seq_id in seq_ids:
121 + seq_data = input_metadata.seq_data[seq_id]
122 + output_tokens.append(seq_data.output_token_ids)
123 + return output_tokens
124 +
125 +
126 +def _apply_penalties(
127 + logits: torch.Tensor,
128 + output_tokens: List[List[int]],
129 + presence_penalties: List[float],
130 + frequency_penalties: List[float],
131 + vocab_size: int,
132 +) -> torch.Tensor:
133 + num_seqs = logits.shape[0]
134 + # Collect the indices of sequences that have non-zero penalties.
135 + indices = []
136 + for i in range(num_seqs):
137 + if not output_tokens[i]:
138 + continue
139 + p = presence_penalties[i]
140 + f = frequency_penalties[i]
141 + if p == 0.0 and f == 0.0:
142 + continue
143 + indices.append(i)
144 +
145 + # Return early if all sequences have zero penalties.
146 + if not indices:
147 + return logits
148 +
149 + bin_counts = []
150 + for i in indices:
151 + bin_counts.append(np.bincount(output_tokens[i], minlength=vocab_size))
152 + bin_counts = np.stack(bin_counts, axis=0)
153 + bin_counts = torch.from_numpy(bin_counts).to(dtype=logits.dtype,
154 + device=logits.device)
155 +
156 + frequency_penalties = [frequency_penalties[i] for i in indices]
157 + frequency_penalties = torch.tensor(
158 + frequency_penalties, dtype=logits.dtype, device=logits.device)
159 + presence_penalties = [presence_penalties[i] for i in indices]
160 + presence_penalties = torch.tensor(
161 + presence_penalties, dtype=logits.dtype, device=logits.device)
162 +
163 + # We follow the definition in OpenAI API.
164 + # Refer to https://platform.openai.com/docs/api-reference/parameter-details
165 + logits[indices] -= frequency_penalties.unsqueeze(dim=1) * bin_counts
166 + presence_mask = (bin_counts > 0.0).to(dtype=logits.dtype)
167 + logits[indices] -= presence_penalties.unsqueeze(dim=1) * presence_mask
168 + return logits
169 +
170 +
171 def _get_temperatures(
172 input_metadata: InputMetadata,
173 ) -> List[float]:
@@ -121,10 +217,11 @@ def _get_top_p_top_k(
217
218 def _apply_top_p_top_k(
219 probs: torch.Tensor,
124 - p: torch.Tensor,
125 - k: torch.Tensor,
220 + top_ps: List[float],
221 + top_ks: List[int],
222 ) -> torch.Tensor:
127 - # TODO(woosuk): Optimize.
223 + p = torch.tensor(top_ps, dtype=probs.dtype, device=probs.device)
224 + k = torch.tensor(top_ks, dtype=torch.int, device=probs.device)
225 probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
226
227 # Apply top-p.
@@ -286,7 +383,8 @@ def _sample(
383
384 # Sample the next tokens.
385 seq_logprobs = [
289 - input_metadata.seq_logprobs[seq_id] for seq_id in seq_ids]
386 + input_metadata.seq_data[seq_id].cumulative_logprobs
387 + for seq_id in seq_ids]
388 parent_seq_ids, next_token_ids = _sample_from_generation_tokens(
389 seq_ids, prob, logprob, seq_logprobs, sampling_params)
390
cacheflow/sampling_params.py
+26 -9
@@ -6,6 +6,8 @@ class SamplingParams:
6 def __init__(
7 self,
8 n: int,
9 + presence_penalty: float,
10 + frequency_penalty: float,
11 temperature: float,
12 top_p: float,
13 top_k: int,
@@ -16,6 +18,12 @@ class SamplingParams:
18 ) -> None:
19 if n < 1:
20 raise ValueError(f"n must be at least 1, got {n}.")
21 + if not -2.0 <= presence_penalty <= 2.0:
22 + raise ValueError(
23 + f"presence_penalty must be in [-2, 2], got {presence_penalty}.")
24 + if not -2.0 <= frequency_penalty <= 2.0:
25 + raise ValueError(
26 + f"frequency_penalty must be in [-2, 2], got {frequency_penalty}.")
27 if temperature < 0.0:
28 raise ValueError(
29 f"temperature must be non-negative, got {temperature}.")
@@ -57,6 +65,8 @@ class SamplingParams:
65 "top_k must be -1 when using greedy sampling.")
66
67 self.n = n
68 + self.presence_penalty = presence_penalty
69 + self.frequency_penalty = frequency_penalty
70 self.temperature = temperature
71 self.top_p = top_p
72 self.top_k = top_k
@@ -67,6 +77,8 @@ class SamplingParams:
77
78 def __repr__(self) -> str:
79 return (f"SamplingParams(n={self.n}, "
80 + f"presence_penalty={self.presence_penalty}, "
81 + f"frequency_penalty={self.frequency_penalty}, "
82 f"temperature={self.temperature}, "
83 f"top_p={self.top_p}, "
84 f"top_k={self.top_k},"
@@ -77,13 +89,18 @@ class SamplingParams:
89
90 @classmethod
91 def from_dict(cls, d: Dict) -> "SamplingParams":
80 - return cls(
81 - n=d.get("n", 1),
82 - temperature=d.get("temperature", 1.0),
83 - top_p=d.get("top_p", 1.0),
84 - top_k=d.get("top_k", -1),
85 - use_beam_search=d.get("use_beam_search", False),
86 - stop_token_ids=set(d.get("stop_token_ids", set())),
87 - max_num_steps=d.get("max_num_steps", 16),
88 - num_logprobs=d.get("num_logprobs", 0),
92 + sampling_params = cls(
93 + n=d.pop("n", 1),
94 + presence_penalty=d.pop("presence_penalty", 0.0),
95 + frequency_penalty=d.pop("frequency_penalty", 0.0),
96 + temperature=d.pop("temperature", 1.0),
97 + top_p=d.pop("top_p", 1.0),
98 + top_k=d.pop("top_k", -1),
99 + use_beam_search=d.pop("use_beam_search", False),
100 + stop_token_ids=set(d.pop("stop_token_ids", set())),
101 + max_num_steps=d.pop("max_num_steps", 16),
102 + num_logprobs=d.pop("num_logprobs", 0),
103 )
104 + if d:
105 + raise ValueError(f"Unrecognized keys in dict: {d.keys()}")
106 + return sampling_params
cacheflow/sequence.py
+46 -23
@@ -13,26 +13,55 @@ class SequenceStatus(enum.Enum):
13 FINISHED = enum.auto()
14
15
16 +class SequenceData:
17 +
18 + def __init__(
19 + self,
20 + prompt_token_ids: List[int],
21 + ) -> None:
22 + self.prompt_token_ids = prompt_token_ids
23 +
24 + self.output_token_ids: List[int] = []
25 + self.cumulative_logprobs = 0.0
26 +
27 + def get_len(self) -> int:
28 + return len(self.output_token_ids) + len(self.prompt_token_ids)
29 +
30 + def get_token_ids(self) -> List[int]:
31 + return self.prompt_token_ids + self.output_token_ids
32 +
33 + def get_last_token_id(self) -> int:
34 + if not self.output_token_ids:
35 + return self.prompt_token_ids[-1]
36 + return self.output_token_ids[-1]
37 +
38 + def __repr__(self) -> str:
39 + return (f"SequenceData("
40 + f"prompt={self.prompt}, "
41 + f"prompt_token_ids={self.prompt_token_ids}, "
42 + f"output_token_ids={self.output_token_ids})")
43 +
44 +
45 class Sequence:
46
47 def __init__(
48 self,
49 seq_id: int,
50 + prompt: str,
51 prompt_token_ids: List[int],
52 block_size: int,
53 ) -> None:
54 self.seq_id = seq_id
55 + self.prompt = prompt
56 self.block_size = block_size
57
27 - self.prompt_len = len(prompt_token_ids)
58 + self.data = SequenceData(prompt_token_ids)
59 + self.output_logprobs: List[Dict[int, float]] = []
60 +
61 self.logical_token_blocks: List[LogicalTokenBlock] = []
62 # Initialize the logical token blocks with the prompt token ids.
30 - self._append_tokens(prompt_token_ids)
31 -
63 + self._append_tokens_to_blocks(prompt_token_ids)
64 self.status = SequenceStatus.WAITING
33 - # Used for beam search.
34 - self.output_logprobs: List[Dict[int, float]] = []
35 - self.cumulative_logprobs = 0.0
65
66 def _append_logical_block(self) -> None:
67 block = LogicalTokenBlock(
@@ -41,7 +70,7 @@ class Sequence:
70 )
71 self.logical_token_blocks.append(block)
72
44 - def _append_tokens(self, token_ids: List[int]) -> None:
73 + def _append_tokens_to_blocks(self, token_ids: List[int]) -> None:
74 while token_ids:
75 if not self.logical_token_blocks:
76 self._append_logical_block()
@@ -57,26 +86,24 @@ class Sequence:
86
87 def append_token(self, token_id: int, logprobs: Dict[int, float]) -> None:
88 assert token_id in logprobs
60 - self._append_tokens([token_id])
89 + self._append_tokens_to_blocks([token_id])
90 self.output_logprobs.append(logprobs)
62 - self.cumulative_logprobs += logprobs[token_id]
91 + self.data.output_token_ids.append(token_id)
92 + self.data.cumulative_logprobs += logprobs[token_id]
93
94 def get_len(self) -> int:
65 - return sum(block.num_tokens for block in self.logical_token_blocks)
95 + return self.data.get_len()
96
97 def get_token_ids(self) -> List[int]:
68 - token_ids: List[int] = []
69 - for block in self.logical_token_blocks:
70 - token_ids.extend(block.get_token_ids())
71 - return token_ids
98 + return self.data.get_token_ids()
99
100 def get_last_token_id(self) -> int:
74 - return self.logical_token_blocks[-1].get_last_token_id()
101 + return self.data.get_last_token_id()
102
103 def fork(self, child_seq: 'Sequence') -> 'Sequence':
104 child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
105 child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
79 - child_seq.cumulative_logprobs = self.cumulative_logprobs
106 + child_seq.data = copy.deepcopy(self.data)
107
108 def __repr__(self) -> str:
109 return (f'Sequence(seq_id={self.seq_id}, '
@@ -128,17 +155,13 @@ class SequenceGroupMetadata:
155 self,
156 group_id: int,
157 is_prompt: bool,
131 - input_tokens: Dict[int, List[int]], # Seq id -> token ids.
132 - context_len: int,
133 - seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
158 + seq_data: Dict[int, SequenceData], # Seq id -> sequence data.
159 sampling_params: SamplingParams,
135 - block_tables: Dict[int, List[int]], # Seq id -> List of physical block numbers.
160 + block_tables: Dict[int, List[int]], # Seq id -> list of physical block numbers.
161 ) -> None:
162 self.group_id = group_id
163 self.is_prompt = is_prompt
139 - self.input_tokens = input_tokens
140 - self.context_len = context_len
141 - self.seq_logprobs = seq_logprobs
164 + self.seq_data = seq_data
165 self.sampling_params = sampling_params
166 self.block_tables = block_tables
167
cacheflow/worker/worker.py
+18 -17
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Tuple, Optional
1 +from typing import Dict, List, Optional, Tuple
2
3 import torch
4
@@ -8,8 +8,8 @@ from cacheflow.model_executor.parallel_utils.parallel_state import (
8 initialize_all_reduce_launcher,
9 get_tensor_model_parallel_world_size)
10 from cacheflow.sampling_params import SamplingParams
11 -from cacheflow.sequence import SequenceGroupMetadata
12 -from cacheflow.sequence import SequenceOutputs
11 +from cacheflow.sequence import (SequenceData, SequenceGroupMetadata,
12 + SequenceOutputs)
13 from cacheflow.worker.cache_engine import CacheEngine
14
15
@@ -72,7 +72,6 @@ class Worker:
72 self.cache_events = self.cache_engine.events
73 self.gpu_cache = self.cache_engine.gpu_cache
74
75 -
75 def init_distributed_environment(self,
76 distributed_init_method: str,
77 rank: int,
@@ -96,7 +95,6 @@ class Worker:
95 seq_group_metadata_list: List[SequenceGroupMetadata],
96 ) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
97 seq_groups: List[Tuple[List[int], SamplingParams]] = []
99 - seq_logprobs: Dict[int, float] = {}
98 input_tokens: List[int] = []
99 input_positions: List[int] = []
100 slot_mapping: List[int] = []
@@ -107,15 +105,15 @@ class Worker:
105 if not seq_group_metadata.is_prompt:
106 continue
107
110 - seq_ids = list(seq_group_metadata.input_tokens.keys())
108 + seq_ids = list(seq_group_metadata.seq_data.keys())
109 sampling_params = seq_group_metadata.sampling_params
110 seq_groups.append((seq_ids, sampling_params))
113 - seq_logprobs.update(seq_group_metadata.seq_logprobs)
111
112 # Use any sequence in the group.
113 seq_id = seq_ids[0]
114
118 - prompt_tokens = seq_group_metadata.input_tokens[seq_id]
115 + seq_data = seq_group_metadata.seq_data[seq_id]
116 + prompt_tokens = seq_data.get_token_ids()
117 prompt_len = len(prompt_tokens)
118 prompt_lens.append(prompt_len)
119
@@ -141,27 +139,26 @@ class Worker:
139 if seq_group_metadata.is_prompt:
140 continue
141
144 - seq_ids = list(seq_group_metadata.input_tokens.keys())
142 + seq_ids = list(seq_group_metadata.seq_data.keys())
143 sampling_params = seq_group_metadata.sampling_params
144 seq_groups.append((seq_ids, sampling_params))
147 - seq_logprobs.update(seq_group_metadata.seq_logprobs)
145
146 for seq_id in seq_ids:
150 - assert len(seq_group_metadata.input_tokens[seq_id]) == 1
151 - generation_token = seq_group_metadata.input_tokens[seq_id][0]
147 + seq_data = seq_group_metadata.seq_data[seq_id]
148 + generation_token = seq_data.get_last_token_id()
149 input_tokens.append(generation_token)
150
154 - position = seq_group_metadata.context_len - 1
151 + context_len = seq_data.get_len()
152 + position = context_len - 1
153 input_positions.append(position)
154
155 block_table = seq_group_metadata.block_tables[seq_id]
156 generation_block_tables.append(block_table)
157
160 - max_context_len = max(
161 - max_context_len, seq_group_metadata.context_len)
158 + max_context_len = max(max_context_len, context_len)
159 max_num_blocks_per_seq = max(
160 max_num_blocks_per_seq, len(block_table))
164 - context_lens.append(seq_group_metadata.context_len)
161 + context_lens.append(context_len)
162
163 block_number = block_table[position // self.block_size]
164 block_offset = position % self.block_size
@@ -188,9 +185,13 @@ class Worker:
185 block_tables_tensor = torch.tensor(
186 padded_block_tables, dtype=torch.int, device='cuda')
187
188 + seq_data: Dict[int, SequenceData] = {}
189 + for seq_group_metadata in seq_group_metadata_list:
190 + seq_data.update(seq_group_metadata.seq_data)
191 +
192 input_metadata = InputMetadata(
193 seq_groups=seq_groups,
193 - seq_logprobs=seq_logprobs,
194 + seq_data=seq_data,
195 prompt_lens=prompt_lens,
196 slot_mapping=slot_mapping_tensor,
197 context_lens=context_lens_tensor,
simple_server.py
+2 -2
@@ -11,8 +11,8 @@ def main(args: argparse.Namespace):
11 # Test the following inputs.
12 test_inputs = [
13 ("A robot may not injure a human being", {}), # Use default parameters.
14 - ("To be or not to be,", {"temperature": 0.8, "top_k": 5}),
15 - ("What is the meaning of life?", {"n": 2, "temperature": 0.8, "top_p": 0.95}),
14 + ("To be or not to be,", {"temperature": 0.8, "top_k": 5, "presence_penalty": 0.2}),
15 + ("What is the meaning of life?", {"n": 2, "temperature": 0.8, "top_p": 0.95, "frequency_penalty": 0.1}),
16 ("It is only with the heart that one can see rightly", {"n": 3, "use_beam_search": True, "temperature": 0.0}),
17 ]
18 while True: