A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Implement stop strings and best_of (#114)
Woosuk Kwon committed
May 21, 2023 at 11:18 UTC
f746ced08d224113110adfc5526d952e51972515
9 files changed
+162
-116
cacheflow/core/block_manager.py
+6
-6
@@ -80,7 +80,7 @@ class BlockSpaceManager:
80
def can_allocate(self, seq_group: SequenceGroup) -> bool:
81
# FIXME(woosuk): Here we assume that all sequences in the group share
82
# the same prompt. This may not be true for preempted sequences.
83
- seq = seq_group.seqs[0]
83
+ seq = seq_group.get_seqs()[0]
84
num_required_blocks = len(seq.logical_token_blocks)
85
num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
86
# Use watermark to avoid frequent cache eviction.
@@ -88,7 +88,7 @@ class BlockSpaceManager:
88
89
def allocate(self, seq_group: SequenceGroup) -> None:
90
# NOTE: Here we assume that all sequences in the group have the same prompt.
91
- seq = seq_group.seqs[0]
91
+ seq = seq_group.get_seqs()[0]
92
93
# Allocate new physical token blocks that will store the prompt tokens.
94
block_table: BlockTable = []
@@ -99,7 +99,7 @@ class BlockSpaceManager:
99
block_table.append(block)
100
101
# Assign the block table for each sequence.
102
- for seq in seq_group.seqs:
102
+ for seq in seq_group.get_seqs():
103
self.block_tables[seq.seq_id] = block_table.copy()
104
105
def can_append_slot(self, seq_group: SequenceGroup) -> bool:
@@ -147,7 +147,7 @@ class BlockSpaceManager:
147
# NOTE: Here, we assume that the physical blocks are only shared by
148
# the sequences in the same group.
149
blocks: Set[PhysicalTokenBlock] = set()
150
- for seq in seq_group.seqs:
150
+ for seq in seq_group.get_seqs():
151
if seq.status == SequenceStatus.FINISHED:
152
continue
153
block_table = self.block_tables[seq.seq_id]
@@ -168,7 +168,7 @@ class BlockSpaceManager:
168
def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
169
# CPU block -> GPU block.
170
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
171
- for seq in seq_group.seqs:
171
+ for seq in seq_group.get_seqs():
172
if seq.status == SequenceStatus.FINISHED:
173
continue
174
new_block_table: BlockTable = []
@@ -199,7 +199,7 @@ class BlockSpaceManager:
199
def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
200
# GPU block -> CPU block.
201
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
202
- for seq in seq_group.seqs:
202
+ for seq in seq_group.get_seqs():
203
if seq.status == SequenceStatus.FINISHED:
204
continue
205
new_block_table: BlockTable = []
cacheflow/core/scheduler.py
+15
-48
@@ -73,8 +73,6 @@ class Scheduler:
73
self.waiting: List[SequenceGroup] = []
74
# Sequence groups in the RUNNING state.
75
self.running: List[SequenceGroup] = []
76
- # Mapping: request_id -> num_steps.
77
- self.num_steps: Dict[str, int] = {}
76
# Sequence groups in the SWAPPED state.
77
self.swapped: List[SequenceGroup] = []
78
@@ -84,7 +82,6 @@ class Scheduler:
82
83
def add_seq_group(self, seq_group: SequenceGroup) -> None:
84
# Add sequence groups to the waiting queue.
87
- assert seq_group.request_id not in self.num_steps
85
self.waiting.append(seq_group)
86
87
def has_unfinished_seqs(self) -> bool:
@@ -178,7 +175,7 @@ class Scheduler:
175
break
176
177
# If the number of batched tokens exceeds the limit, stop.
181
- num_prompt_tokens = seq_group.seqs[0].get_len()
178
+ num_prompt_tokens = seq_group.get_seqs()[0].get_len()
179
if (num_batched_tokens + num_prompt_tokens
180
> self.scheduler_config.max_num_batched_tokens):
181
break
@@ -278,15 +275,8 @@ class Scheduler:
275
) -> List[SequenceGroup]:
276
# Update the running sequences and free blocks.
277
for seq_group in self.running:
281
- request_id = seq_group.request_id
282
- self.num_steps[request_id] += 1
283
- stop_token_ids = seq_group.sampling_params.stop_token_ids
284
-
285
- # Process beam search results before processing the next tokens.
286
- for seq in seq_group.seqs:
287
- if seq.status == SequenceStatus.FINISHED:
288
- continue
289
-
278
+ # Process beam search results before processing the new tokens.
279
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
280
output = seq_outputs[seq.seq_id]
281
if seq.seq_id != output.parent_seq_id:
282
# The sequence is a fork of the parent sequence (beam search).
@@ -297,43 +287,27 @@ class Scheduler:
287
parent_seq.fork(seq)
288
self.block_manager.fork(parent_seq, seq)
289
300
- # Process the next tokens.
301
- for seq in seq_group.seqs:
302
- if seq.status == SequenceStatus.FINISHED:
303
- continue
304
-
290
+ # Process the new tokens.
291
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
292
# Append a new token to the sequence.
293
output = seq_outputs[seq.seq_id]
294
seq.append_token(output.output_token, output.logprobs)
295
+ return self.running.copy()
296
309
- # Check if the sequence has generated a stop token.
310
- if output.output_token in stop_token_ids:
311
- self._free_seq(seq)
312
- continue
297
+ def free_seq(self, seq: Sequence) -> None:
298
+ seq.status = SequenceStatus.FINISHED
299
+ self.block_manager.free(seq)
300
314
- # Check if the sequence has reached the maximum number of steps.
315
- max_num_steps = seq_group.sampling_params.max_tokens
316
- if self.num_steps[request_id] == max_num_steps:
317
- self._free_seq(seq)
318
- continue
319
-
320
- # Update the running sequences.
321
- updated = self.running.copy()
322
- running: List[SequenceGroup] = []
323
- for seq_group in self.running:
324
- if seq_group.is_finished():
325
- self._free_seq_group(seq_group)
326
- else:
327
- running.append(seq_group)
328
- self.running = running
329
- return updated
301
+ def free_finished_seq_groups(self) -> None:
302
+ self.running = [
303
+ seq_group for seq_group in self.running
304
+ if not seq_group.is_finished()
305
+ ]
306
307
def _allocate(self, seq_group: SequenceGroup) -> None:
308
self.block_manager.allocate(seq_group)
333
- for seq in seq_group.seqs:
309
+ for seq in seq_group.get_seqs():
310
seq.status = SequenceStatus.RUNNING
335
- if seq_group.request_id not in self.num_steps:
336
- self.num_steps[seq_group.request_id] = 0
311
312
def _append_slot(
313
self,
@@ -403,13 +377,6 @@ class Scheduler:
377
self._swap_out(seq_group, blocks_to_swap_out)
378
self.swapped.append(seq_group)
379
406
- def _free_seq(self, seq: Sequence) -> None:
407
- seq.status = SequenceStatus.FINISHED
408
- self.block_manager.free(seq)
409
-
410
- def _free_seq_group(self, seq_group: SequenceGroup) -> None:
411
- del self.num_steps[seq_group.request_id]
412
-
380
def _swap_in(
381
self,
382
seq_group: SequenceGroup,
cacheflow/entrypoints/fastapi_server.py
+3
-2
@@ -123,6 +123,7 @@ if __name__ == "__main__":
123
parallel_config = server_configs[2]
124
distributed_init_method, stage_devices = initialize_cluster(parallel_config)
125
126
- server = FastAPIServer(
127
- args.use_ray, *server_configs, distributed_init_method, stage_devices)
126
+ server = FastAPIServer(args.use_ray, *server_configs,
127
+ distributed_init_method, stage_devices,
128
+ log_stats=not args.disable_log_stats)
129
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/model_executor/layers/sampler.py
+8
-8
@@ -283,20 +283,20 @@ def _sample_from_prompt(
283
) -> List[int]:
284
if sampling_params.use_beam_search:
285
# Beam search.
286
- beam_width = sampling_params.n
286
+ beam_width = sampling_params.best_of
287
_, next_token_ids = torch.topk(prob, beam_width)
288
next_token_ids = next_token_ids.tolist()
289
elif sampling_params.temperature == 0.0:
290
# Greedy sampling.
291
- assert sampling_params.n == 1
291
+ assert sampling_params.best_of == 1
292
next_token_id = torch.argmax(prob)
293
next_token_ids = [next_token_id.item()]
294
else:
295
# Random sampling.
296
- # Sample n tokens for the prompt.
297
- n = sampling_params.n
296
+ # Sample `best_of` tokens for the prompt.
297
+ num_seqs = sampling_params.best_of
298
next_token_ids = torch.multinomial(
299
- prob, num_samples=n, replacement=True)
299
+ prob, num_samples=num_seqs, replacement=True)
300
next_token_ids = next_token_ids.tolist()
301
return next_token_ids
302
@@ -308,7 +308,7 @@ def _sample_from_generation_tokens(
308
seq_logprobs: List[float],
309
sampling_params: SamplingParams,
310
) -> Tuple[List[int], List[int]]:
311
- # NOTE(woosuk): sampling_params.n can be greater than
311
+ # NOTE(woosuk): sampling_params.best_of can be greater than
312
# len(seq_ids) because some sequences in the group might have
313
# been already terminated.
314
if sampling_params.use_beam_search:
@@ -372,7 +372,7 @@ def _sample(
372
seq_ids, sampling_params = seq_group
373
if i < input_metadata.num_prompts:
374
# Generate the next tokens for a prompt input.
375
- assert len(seq_ids) == sampling_params.n
375
+ assert len(seq_ids) == sampling_params.best_of
376
prob = probs[idx]
377
logprob = logprobs[idx]
378
idx += 1
@@ -397,7 +397,7 @@ def _sample(
397
398
# Sample the next tokens.
399
seq_logprobs = [
400
- input_metadata.seq_data[seq_id].cumulative_logprobs
400
+ input_metadata.seq_data[seq_id].cumulative_logprob
401
for seq_id in seq_ids]
402
parent_seq_ids, next_token_ids = _sample_from_generation_tokens(
403
seq_ids, prob, logprob, seq_logprobs, sampling_params)
cacheflow/outputs.py
+23
-21
@@ -1,6 +1,4 @@
1
-from typing import Dict, List, Union
2
-
3
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
1
+from typing import Dict, List
2
3
from cacheflow.sequence import SequenceGroup
4
@@ -9,20 +7,23 @@ class CompletionOutput:
7
8
def __init__(
9
self,
10
+ index: int,
11
text: str,
12
token_ids: List[int],
14
- cumulative_logprobs: float,
13
+ cumulative_logprob: float,
14
logprobs: List[Dict[int, float]],
15
) -> None:
16
+ self.index = index
17
self.text = text
18
self.token_ids = token_ids
19
- self.cumulative_logprobs = cumulative_logprobs
19
+ self.cumulative_logprob = cumulative_logprob
20
self.logprobs = logprobs
21
22
def __repr__(self) -> str:
23
- return (f"CompletionOutput(output={self.text!r}, "
23
+ return (f"CompletionOutput(index={self.index}, "
24
+ f"text={self.text!r}, "
25
f"token_ids={self.token_ids}, "
25
- f"cumulative_logprobs={self.cumulative_logprobs}, "
26
+ f"cumulative_logprob={self.cumulative_logprob}, "
27
f"logprobs={self.logprobs})")
28
29
@@ -43,31 +44,32 @@ class RequestOutput:
44
self.done = done
45
46
@staticmethod
46
- def from_seq_group(
47
- seq_group: SequenceGroup,
48
- tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
49
- ) -> "RequestOutput":
50
- outputs: List[CompletionOutput] = []
47
+ def from_seq_group(seq_group: SequenceGroup) -> "RequestOutput":
48
+ # Get the top-n sequences.
49
+ n = seq_group.sampling_params.n
50
seqs = seq_group.get_seqs()
52
- for seq in seqs:
53
- output_token_ids = seq.data.output_token_ids
54
- output_str = tokenizer.decode(output_token_ids,
55
- skip_special_tokens=True)
56
- seq_logprobs = seq.data.cumulative_logprobs
51
+ assert n <= len(seqs)
52
+ sorted_seqs = sorted(
53
+ seqs, key=lambda seq: seq.get_cumulative_logprob(), reverse=True)
54
+ top_n_seqs = sorted_seqs[:n]
55
56
+ # Create the outputs.
57
+ outputs: List[CompletionOutput] = []
58
+ for seq in top_n_seqs:
59
logprobs = seq.output_logprobs
60
if seq_group.sampling_params.logprobs == 0:
61
# NOTE: We need to take care of this case because the sequence
62
# always has the logprobs of the sampled tokens even if the
63
# logprobs are not requested.
64
logprobs = {}
64
- output = CompletionOutput(output_str, output_token_ids,
65
- seq_logprobs, logprobs)
65
+ output = CompletionOutput(seqs.index(seq), seq.output_text,
66
+ seq.get_output_token_ids(),
67
+ seq.get_cumulative_logprob(), logprobs)
68
outputs.append(output)
69
70
# Every sequence in the sequence group should have the same prompt.
69
- prompt = seqs[0].prompt
70
- prompt_token_ids = seqs[0].data.prompt_token_ids
71
+ prompt = top_n_seqs[0].prompt
72
+ prompt_token_ids = top_n_seqs[0].data.prompt_token_ids
73
return RequestOutput(seq_group.request_id, prompt, prompt_token_ids,
74
outputs, seq_group.is_finished())
75
cacheflow/sampling_params.py
+29
-11
@@ -1,5 +1,5 @@
1
"""Sampling parameters for text generation."""
2
-from typing import Set
2
+from typing import List, Optional, Union
3
4
5
class SamplingParams:
@@ -10,8 +10,12 @@ class SamplingParams:
10
In addition, we support beam search, which is not supported by OpenAI.
11
12
Args:
13
- n: Number of output sequences to generate from the given prompt. This is
14
- regarded as the beam width when using beam search.
13
+ n: Number of output sequences to return for the given prompt.
14
+ best_of: Number of output sequences that are generated from the prompt.
15
+ From these `best_of` sequences, the top `n` sequences are returned.
16
+ `best_of` must be greater than or equal to `n`. This is treated as
17
+ the beam width when `use_beam_search` is True. By default, `best_of`
18
+ is set to `n`.
19
presence_penalty: Float that penalizes new tokens based on whether they
20
appear in the generated text so far. Values > 0 encourage the model
21
to use new tokens, while values < 0 encourage the model to repeat
@@ -28,7 +32,10 @@ class SamplingParams:
32
top_k: Integer that controls the number of top tokens to consider. Set
33
to -1 to consider all tokens.
34
use_beam_search: Whether to use beam search instead of sampling.
31
- stop_token_ids: Set of token IDs that indicate the end of a sequence.
35
+ stop: List of strings that stop the generation when they are generated.
36
+ The returned output will not contain the stop strings.
37
+ ignore_eos: Whether to ignore the EOS token and continue generating
38
+ tokens after the EOS token is generated.
39
max_tokens: Maximum number of tokens to generate per output sequence.
40
logprobs: Number of log probabilities to return per output token.
41
"""
@@ -36,24 +43,28 @@ class SamplingParams:
43
def __init__(
44
self,
45
n: int = 1,
46
+ best_of: Optional[int] = None,
47
presence_penalty: float = 0.0,
48
frequency_penalty: float = 0.0,
49
temperature: float = 1.0,
50
top_p: float = 1.0,
51
top_k: int = -1,
52
use_beam_search: bool = False,
45
- stop_token_ids: Set[int] = set(),
53
+ stop: Union[str, List[str]] = [],
54
+ ignore_eos: bool = False,
55
max_tokens: int = 16,
56
logprobs: int = 0,
57
) -> None:
58
self.n = n
59
+ self.best_of = best_of if best_of is not None else n
60
self.presence_penalty = presence_penalty
61
self.frequency_penalty = frequency_penalty
62
self.temperature = temperature
63
self.top_p = top_p
64
self.top_k = top_k
65
self.use_beam_search = use_beam_search
56
- self.stop_token_ids = stop_token_ids
66
+ self.stop = [stop] if isinstance(stop, str) else list(stop)
67
+ self.ignore_eos = ignore_eos
68
self.max_tokens = max_tokens
69
self.logprobs = logprobs
70
@@ -67,6 +78,9 @@ class SamplingParams:
78
def _verify_args(self) -> None:
79
if self.n < 1:
80
raise ValueError(f"n must be at least 1, got {self.n}.")
81
+ if self.best_of < self.n:
82
+ raise ValueError(f"best_of must be greater than or equal to n, "
83
+ f"got n={self.n} and best_of={self.best_of}.")
84
if not -2.0 <= self.presence_penalty <= 2.0:
85
raise ValueError("presence_penalty must be in [-2, 2], got "
86
f"{self.presence_penalty}.")
@@ -89,8 +103,9 @@ class SamplingParams:
103
f"logprobs must be non-negative, got {self.logprobs}.")
104
105
def _verity_beam_search(self) -> None:
92
- if self.n == 1:
93
- raise ValueError("n must be greater than 1 when using beam search.")
106
+ if self.best_of == 1:
107
+ raise ValueError("best_of must be greater than 1 when using beam "
108
+ f"search. Got {self.best_of}.")
109
if self.temperature > 0.0:
110
raise ValueError("temperature must be 0 when using beam search.")
111
if self.top_p < 1.0:
@@ -99,8 +114,9 @@ class SamplingParams:
114
raise ValueError("top_k must be -1 when using beam search.")
115
116
def _verify_greedy_sampling(self) -> None:
102
- if self.n > 1:
103
- raise ValueError("n must be 1 when using greedy sampling.")
117
+ if self.best_of > 1:
118
+ raise ValueError("best_of must be 1 when using greedy sampling."
119
+ f"Got {self.best_of}.")
120
if self.top_p < 1.0:
121
raise ValueError("top_p must be 1 when using greedy sampling.")
122
if self.top_k != -1:
@@ -108,12 +124,14 @@ class SamplingParams:
124
125
def __repr__(self) -> str:
126
return (f"SamplingParams(n={self.n}, "
127
+ f"best_of={self.best_of}, "
128
f"presence_penalty={self.presence_penalty}, "
129
f"frequency_penalty={self.frequency_penalty}, "
130
f"temperature={self.temperature}, "
131
f"top_p={self.top_p}, "
132
f"top_k={self.top_k},"
133
f"use_beam_search={self.use_beam_search}, "
117
- f"stop_token_ids={self.stop_token_ids}, "
134
+ f"stop={self.stop}, "
135
+ f"ignore_eos={self.ignore_eos}, "
136
f"max_tokens={self.max_tokens}, "
137
f"logprobs={self.logprobs})")
cacheflow/sequence.py
+21
-5
@@ -22,11 +22,18 @@ class SequenceData:
22
self.prompt_token_ids = prompt_token_ids
23
24
self.output_token_ids: List[int] = []
25
- self.cumulative_logprobs = 0.0
25
+ self.cumulative_logprob = 0.0
26
+
27
+ def append_token(self, token_id: int, logprob: float) -> None:
28
+ self.output_token_ids.append(token_id)
29
+ self.cumulative_logprob += logprob
30
31
def get_len(self) -> int:
32
return len(self.output_token_ids) + len(self.prompt_token_ids)
33
34
+ def get_output_len(self) -> int:
35
+ return len(self.output_token_ids)
36
+
37
def get_token_ids(self) -> List[int]:
38
return self.prompt_token_ids + self.output_token_ids
39
@@ -37,9 +44,9 @@ class SequenceData:
44
45
def __repr__(self) -> str:
46
return (f"SequenceData("
40
- f"prompt={self.prompt}, "
47
f"prompt_token_ids={self.prompt_token_ids}, "
42
- f"output_token_ids={self.output_token_ids})")
48
+ f"output_token_ids={self.output_token_ids}, "
49
+ f"cumulative_logprob={self.cumulative_logprob})")
50
51
52
class Sequence:
@@ -57,6 +64,7 @@ class Sequence:
64
65
self.data = SequenceData(prompt_token_ids)
66
self.output_logprobs: List[Dict[int, float]] = []
67
+ self.output_text = ""
68
69
self.logical_token_blocks: List[LogicalTokenBlock] = []
70
# Initialize the logical token blocks with the prompt token ids.
@@ -88,18 +96,26 @@ class Sequence:
96
assert token_id in logprobs
97
self._append_tokens_to_blocks([token_id])
98
self.output_logprobs.append(logprobs)
91
- self.data.output_token_ids.append(token_id)
92
- self.data.cumulative_logprobs += logprobs[token_id]
99
+ self.data.append_token(token_id, logprobs[token_id])
100
101
def get_len(self) -> int:
102
return self.data.get_len()
103
104
+ def get_output_len(self) -> int:
105
+ return self.data.get_output_len()
106
+
107
def get_token_ids(self) -> List[int]:
108
return self.data.get_token_ids()
109
110
def get_last_token_id(self) -> int:
111
return self.data.get_last_token_id()
112
113
+ def get_output_token_ids(self) -> List[int]:
114
+ return self.data.output_token_ids
115
+
116
+ def get_cumulative_logprob(self) -> float:
117
+ return self.data.cumulative_logprob
118
+
119
def fork(self, child_seq: 'Sequence') -> 'Sequence':
120
child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
121
child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
cacheflow/server/llm_server.py
+55
-13
@@ -13,7 +13,7 @@ from cacheflow.logger import init_logger
13
from cacheflow.outputs import RequestOutput
14
from cacheflow.sampling_params import SamplingParams
15
from cacheflow.server.tokenizer_utils import get_tokenizer
16
-from cacheflow.sequence import Sequence, SequenceGroup
16
+from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
17
from cacheflow.utils import Counter
18
from cacheflow.worker.worker import Worker
19
@@ -49,7 +49,6 @@ class LLMServer:
49
self.parallel_config = parallel_config
50
self.scheduler_config = scheduler_config
51
self.log_stats = log_stats
52
-
52
self._verify_args()
53
54
self.tokenizer = get_tokenizer(model_config.model)
@@ -124,15 +123,11 @@ class LLMServer:
123
# Create the sequences.
124
block_size = self.cache_config.block_size
125
seqs: List[Sequence] = []
127
- for _ in range(sampling_params.n):
126
+ for _ in range(sampling_params.best_of):
127
seq_id = next(self.seq_counter)
128
seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
129
seqs.append(seq)
130
132
- # FIXME(woosuk)
133
- # Add the EOS token to the stop token list.
134
- sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
135
-
131
# Create the sequence group.
132
seq_group = SequenceGroup(request_id, seqs, sampling_params,
133
arrival_time)
@@ -157,18 +152,65 @@ class LLMServer:
152
blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
153
blocks_to_copy=scheduler_outputs.blocks_to_copy,
154
)
160
- # Update the scheduler.
161
- updated_seq_groups = self.scheduler.update(output)
155
+ # Update the scheduler with the model outputs.
156
+ seq_groups = self.scheduler.update(output)
157
+
158
+ # Decode the sequences.
159
+ self._decode_sequences(seq_groups)
160
+ # Stop the sequences that meet the stopping criteria.
161
+ self._stop_sequences(seq_groups)
162
+ # Free the finished sequence groups.
163
+ self.scheduler.free_finished_seq_groups()
164
165
# Create the outputs.
166
request_outputs: List[RequestOutput] = []
165
- for seq_group in updated_seq_groups:
166
- # TODO(woosuk): Batch-decode the outputs for speedup.
167
- request_output = RequestOutput.from_seq_group(seq_group,
168
- self.tokenizer)
167
+ for seq_group in seq_groups:
168
+ request_output = RequestOutput.from_seq_group(seq_group)
169
request_outputs.append(request_output)
170
return request_outputs
171
172
+ def _decode_sequences(self, seq_groups: List[SequenceGroup]) -> None:
173
+ # Batch-decode the sequence outputs.
174
+ seqs: List[Sequence] = []
175
+ for seq_group in seq_groups:
176
+ seqs.extend(seq_group.get_seqs(status=SequenceStatus.RUNNING))
177
+ output_tokens_per_seq = []
178
+ for seq in seqs:
179
+ output_tokens_per_seq.append(seq.get_output_token_ids())
180
+ output_texts = self.tokenizer.batch_decode(output_tokens_per_seq,
181
+ skip_special_tokens=True)
182
+ # Update the sequences with the output texts.
183
+ for seq, output_text in zip(seqs, output_texts):
184
+ seq.output_text = output_text
185
+
186
+ def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
187
+ # Stop the sequences.
188
+ for seq_group in seq_groups:
189
+ sampling_params = seq_group.sampling_params
190
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
191
+ # Check if the sequence has generated a stop string.
192
+ stopped = False
193
+ for stop_str in sampling_params.stop:
194
+ if seq.output_text.endswith(stop_str):
195
+ # Truncate the output text so that the stop string is
196
+ # not included in the output.
197
+ seq.output_text = seq.output_text[:-len(stop_str)]
198
+ self.scheduler.free_seq(seq)
199
+ stopped = True
200
+ break
201
+ if stopped:
202
+ continue
203
+
204
+ # Check if the sequence has reached max_tokens.
205
+ if seq.get_output_len() == sampling_params.max_tokens:
206
+ self.scheduler.free_seq(seq)
207
+ continue
208
+ # Check if the sequence has generated the EOS token.
209
+ if not sampling_params.ignore_eos:
210
+ if seq.get_last_token_id() == self.tokenizer.eos_token_id:
211
+ self.scheduler.free_seq(seq)
212
+ continue
213
+
214
def _run_workers(
215
self,
216
method: str,
examples/simple_server.py
+2
-2
@@ -15,9 +15,9 @@ def main(args: argparse.Namespace):
15
("To be or not to be,",
16
SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
17
("What is the meaning of life?",
18
- SamplingParams(n=2, temperature=0.8, top_p=0.95, frequency_penalty=0.1)),
18
+ SamplingParams(n=2, best_of=5, temperature=0.8, top_p=0.95, frequency_penalty=0.1)),
19
("It is only with the heart that one can see rightly",
20
- SamplingParams(n=3, use_beam_search=True, temperature=0.0)),
20
+ SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
21
]
22
23
# Run the server.