@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f746ced08d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Implement stop strings and best_of (#114)

Woosuk Kwon committed May 21, 2023 at 11:18 UTC f746ced08d224113110adfc5526d952e51972515
9 files changed +162 -116
cacheflow/core/block_manager.py
+6 -6
@@ -80,7 +80,7 @@ class BlockSpaceManager:
80 def can_allocate(self, seq_group: SequenceGroup) -> bool:
81 # FIXME(woosuk): Here we assume that all sequences in the group share
82 # the same prompt. This may not be true for preempted sequences.
83 - seq = seq_group.seqs[0]
83 + seq = seq_group.get_seqs()[0]
84 num_required_blocks = len(seq.logical_token_blocks)
85 num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
86 # Use watermark to avoid frequent cache eviction.
@@ -88,7 +88,7 @@ class BlockSpaceManager:
88
89 def allocate(self, seq_group: SequenceGroup) -> None:
90 # NOTE: Here we assume that all sequences in the group have the same prompt.
91 - seq = seq_group.seqs[0]
91 + seq = seq_group.get_seqs()[0]
92
93 # Allocate new physical token blocks that will store the prompt tokens.
94 block_table: BlockTable = []
@@ -99,7 +99,7 @@ class BlockSpaceManager:
99 block_table.append(block)
100
101 # Assign the block table for each sequence.
102 - for seq in seq_group.seqs:
102 + for seq in seq_group.get_seqs():
103 self.block_tables[seq.seq_id] = block_table.copy()
104
105 def can_append_slot(self, seq_group: SequenceGroup) -> bool:
@@ -147,7 +147,7 @@ class BlockSpaceManager:
147 # NOTE: Here, we assume that the physical blocks are only shared by
148 # the sequences in the same group.
149 blocks: Set[PhysicalTokenBlock] = set()
150 - for seq in seq_group.seqs:
150 + for seq in seq_group.get_seqs():
151 if seq.status == SequenceStatus.FINISHED:
152 continue
153 block_table = self.block_tables[seq.seq_id]
@@ -168,7 +168,7 @@ class BlockSpaceManager:
168 def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
169 # CPU block -> GPU block.
170 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
171 - for seq in seq_group.seqs:
171 + for seq in seq_group.get_seqs():
172 if seq.status == SequenceStatus.FINISHED:
173 continue
174 new_block_table: BlockTable = []
@@ -199,7 +199,7 @@ class BlockSpaceManager:
199 def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
200 # GPU block -> CPU block.
201 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
202 - for seq in seq_group.seqs:
202 + for seq in seq_group.get_seqs():
203 if seq.status == SequenceStatus.FINISHED:
204 continue
205 new_block_table: BlockTable = []
cacheflow/core/scheduler.py
+15 -48
@@ -73,8 +73,6 @@ class Scheduler:
73 self.waiting: List[SequenceGroup] = []
74 # Sequence groups in the RUNNING state.
75 self.running: List[SequenceGroup] = []
76 - # Mapping: request_id -> num_steps.
77 - self.num_steps: Dict[str, int] = {}
76 # Sequence groups in the SWAPPED state.
77 self.swapped: List[SequenceGroup] = []
78
@@ -84,7 +82,6 @@ class Scheduler:
82
83 def add_seq_group(self, seq_group: SequenceGroup) -> None:
84 # Add sequence groups to the waiting queue.
87 - assert seq_group.request_id not in self.num_steps
85 self.waiting.append(seq_group)
86
87 def has_unfinished_seqs(self) -> bool:
@@ -178,7 +175,7 @@ class Scheduler:
175 break
176
177 # If the number of batched tokens exceeds the limit, stop.
181 - num_prompt_tokens = seq_group.seqs[0].get_len()
178 + num_prompt_tokens = seq_group.get_seqs()[0].get_len()
179 if (num_batched_tokens + num_prompt_tokens
180 > self.scheduler_config.max_num_batched_tokens):
181 break
@@ -278,15 +275,8 @@ class Scheduler:
275 ) -> List[SequenceGroup]:
276 # Update the running sequences and free blocks.
277 for seq_group in self.running:
281 - request_id = seq_group.request_id
282 - self.num_steps[request_id] += 1
283 - stop_token_ids = seq_group.sampling_params.stop_token_ids
284 -
285 - # Process beam search results before processing the next tokens.
286 - for seq in seq_group.seqs:
287 - if seq.status == SequenceStatus.FINISHED:
288 - continue
289 -
278 + # Process beam search results before processing the new tokens.
279 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
280 output = seq_outputs[seq.seq_id]
281 if seq.seq_id != output.parent_seq_id:
282 # The sequence is a fork of the parent sequence (beam search).
@@ -297,43 +287,27 @@ class Scheduler:
287 parent_seq.fork(seq)
288 self.block_manager.fork(parent_seq, seq)
289
300 - # Process the next tokens.
301 - for seq in seq_group.seqs:
302 - if seq.status == SequenceStatus.FINISHED:
303 - continue
304 -
290 + # Process the new tokens.
291 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
292 # Append a new token to the sequence.
293 output = seq_outputs[seq.seq_id]
294 seq.append_token(output.output_token, output.logprobs)
295 + return self.running.copy()
296
309 - # Check if the sequence has generated a stop token.
310 - if output.output_token in stop_token_ids:
311 - self._free_seq(seq)
312 - continue
297 + def free_seq(self, seq: Sequence) -> None:
298 + seq.status = SequenceStatus.FINISHED
299 + self.block_manager.free(seq)
300
314 - # Check if the sequence has reached the maximum number of steps.
315 - max_num_steps = seq_group.sampling_params.max_tokens
316 - if self.num_steps[request_id] == max_num_steps:
317 - self._free_seq(seq)
318 - continue
319 -
320 - # Update the running sequences.
321 - updated = self.running.copy()
322 - running: List[SequenceGroup] = []
323 - for seq_group in self.running:
324 - if seq_group.is_finished():
325 - self._free_seq_group(seq_group)
326 - else:
327 - running.append(seq_group)
328 - self.running = running
329 - return updated
301 + def free_finished_seq_groups(self) -> None:
302 + self.running = [
303 + seq_group for seq_group in self.running
304 + if not seq_group.is_finished()
305 + ]
306
307 def _allocate(self, seq_group: SequenceGroup) -> None:
308 self.block_manager.allocate(seq_group)
333 - for seq in seq_group.seqs:
309 + for seq in seq_group.get_seqs():
310 seq.status = SequenceStatus.RUNNING
335 - if seq_group.request_id not in self.num_steps:
336 - self.num_steps[seq_group.request_id] = 0
311
312 def _append_slot(
313 self,
@@ -403,13 +377,6 @@ class Scheduler:
377 self._swap_out(seq_group, blocks_to_swap_out)
378 self.swapped.append(seq_group)
379
406 - def _free_seq(self, seq: Sequence) -> None:
407 - seq.status = SequenceStatus.FINISHED
408 - self.block_manager.free(seq)
409 -
410 - def _free_seq_group(self, seq_group: SequenceGroup) -> None:
411 - del self.num_steps[seq_group.request_id]
412 -
380 def _swap_in(
381 self,
382 seq_group: SequenceGroup,
cacheflow/entrypoints/fastapi_server.py
+3 -2
@@ -123,6 +123,7 @@ if __name__ == "__main__":
123 parallel_config = server_configs[2]
124 distributed_init_method, stage_devices = initialize_cluster(parallel_config)
125
126 - server = FastAPIServer(
127 - args.use_ray, *server_configs, distributed_init_method, stage_devices)
126 + server = FastAPIServer(args.use_ray, *server_configs,
127 + distributed_init_method, stage_devices,
128 + log_stats=not args.disable_log_stats)
129 uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/model_executor/layers/sampler.py
+8 -8
@@ -283,20 +283,20 @@ def _sample_from_prompt(
283 ) -> List[int]:
284 if sampling_params.use_beam_search:
285 # Beam search.
286 - beam_width = sampling_params.n
286 + beam_width = sampling_params.best_of
287 _, next_token_ids = torch.topk(prob, beam_width)
288 next_token_ids = next_token_ids.tolist()
289 elif sampling_params.temperature == 0.0:
290 # Greedy sampling.
291 - assert sampling_params.n == 1
291 + assert sampling_params.best_of == 1
292 next_token_id = torch.argmax(prob)
293 next_token_ids = [next_token_id.item()]
294 else:
295 # Random sampling.
296 - # Sample n tokens for the prompt.
297 - n = sampling_params.n
296 + # Sample `best_of` tokens for the prompt.
297 + num_seqs = sampling_params.best_of
298 next_token_ids = torch.multinomial(
299 - prob, num_samples=n, replacement=True)
299 + prob, num_samples=num_seqs, replacement=True)
300 next_token_ids = next_token_ids.tolist()
301 return next_token_ids
302
@@ -308,7 +308,7 @@ def _sample_from_generation_tokens(
308 seq_logprobs: List[float],
309 sampling_params: SamplingParams,
310 ) -> Tuple[List[int], List[int]]:
311 - # NOTE(woosuk): sampling_params.n can be greater than
311 + # NOTE(woosuk): sampling_params.best_of can be greater than
312 # len(seq_ids) because some sequences in the group might have
313 # been already terminated.
314 if sampling_params.use_beam_search:
@@ -372,7 +372,7 @@ def _sample(
372 seq_ids, sampling_params = seq_group
373 if i < input_metadata.num_prompts:
374 # Generate the next tokens for a prompt input.
375 - assert len(seq_ids) == sampling_params.n
375 + assert len(seq_ids) == sampling_params.best_of
376 prob = probs[idx]
377 logprob = logprobs[idx]
378 idx += 1
@@ -397,7 +397,7 @@ def _sample(
397
398 # Sample the next tokens.
399 seq_logprobs = [
400 - input_metadata.seq_data[seq_id].cumulative_logprobs
400 + input_metadata.seq_data[seq_id].cumulative_logprob
401 for seq_id in seq_ids]
402 parent_seq_ids, next_token_ids = _sample_from_generation_tokens(
403 seq_ids, prob, logprob, seq_logprobs, sampling_params)
cacheflow/outputs.py
+23 -21
@@ -1,6 +1,4 @@
1 -from typing import Dict, List, Union
2 -
3 -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
1 +from typing import Dict, List
2
3 from cacheflow.sequence import SequenceGroup
4
@@ -9,20 +7,23 @@ class CompletionOutput:
7
8 def __init__(
9 self,
10 + index: int,
11 text: str,
12 token_ids: List[int],
14 - cumulative_logprobs: float,
13 + cumulative_logprob: float,
14 logprobs: List[Dict[int, float]],
15 ) -> None:
16 + self.index = index
17 self.text = text
18 self.token_ids = token_ids
19 - self.cumulative_logprobs = cumulative_logprobs
19 + self.cumulative_logprob = cumulative_logprob
20 self.logprobs = logprobs
21
22 def __repr__(self) -> str:
23 - return (f"CompletionOutput(output={self.text!r}, "
23 + return (f"CompletionOutput(index={self.index}, "
24 + f"text={self.text!r}, "
25 f"token_ids={self.token_ids}, "
25 - f"cumulative_logprobs={self.cumulative_logprobs}, "
26 + f"cumulative_logprob={self.cumulative_logprob}, "
27 f"logprobs={self.logprobs})")
28
29
@@ -43,31 +44,32 @@ class RequestOutput:
44 self.done = done
45
46 @staticmethod
46 - def from_seq_group(
47 - seq_group: SequenceGroup,
48 - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
49 - ) -> "RequestOutput":
50 - outputs: List[CompletionOutput] = []
47 + def from_seq_group(seq_group: SequenceGroup) -> "RequestOutput":
48 + # Get the top-n sequences.
49 + n = seq_group.sampling_params.n
50 seqs = seq_group.get_seqs()
52 - for seq in seqs:
53 - output_token_ids = seq.data.output_token_ids
54 - output_str = tokenizer.decode(output_token_ids,
55 - skip_special_tokens=True)
56 - seq_logprobs = seq.data.cumulative_logprobs
51 + assert n <= len(seqs)
52 + sorted_seqs = sorted(
53 + seqs, key=lambda seq: seq.get_cumulative_logprob(), reverse=True)
54 + top_n_seqs = sorted_seqs[:n]
55
56 + # Create the outputs.
57 + outputs: List[CompletionOutput] = []
58 + for seq in top_n_seqs:
59 logprobs = seq.output_logprobs
60 if seq_group.sampling_params.logprobs == 0:
61 # NOTE: We need to take care of this case because the sequence
62 # always has the logprobs of the sampled tokens even if the
63 # logprobs are not requested.
64 logprobs = {}
64 - output = CompletionOutput(output_str, output_token_ids,
65 - seq_logprobs, logprobs)
65 + output = CompletionOutput(seqs.index(seq), seq.output_text,
66 + seq.get_output_token_ids(),
67 + seq.get_cumulative_logprob(), logprobs)
68 outputs.append(output)
69
70 # Every sequence in the sequence group should have the same prompt.
69 - prompt = seqs[0].prompt
70 - prompt_token_ids = seqs[0].data.prompt_token_ids
71 + prompt = top_n_seqs[0].prompt
72 + prompt_token_ids = top_n_seqs[0].data.prompt_token_ids
73 return RequestOutput(seq_group.request_id, prompt, prompt_token_ids,
74 outputs, seq_group.is_finished())
75
cacheflow/sampling_params.py
+29 -11
@@ -1,5 +1,5 @@
1 """Sampling parameters for text generation."""
2 -from typing import Set
2 +from typing import List, Optional, Union
3
4
5 class SamplingParams:
@@ -10,8 +10,12 @@ class SamplingParams:
10 In addition, we support beam search, which is not supported by OpenAI.
11
12 Args:
13 - n: Number of output sequences to generate from the given prompt. This is
14 - regarded as the beam width when using beam search.
13 + n: Number of output sequences to return for the given prompt.
14 + best_of: Number of output sequences that are generated from the prompt.
15 + From these `best_of` sequences, the top `n` sequences are returned.
16 + `best_of` must be greater than or equal to `n`. This is treated as
17 + the beam width when `use_beam_search` is True. By default, `best_of`
18 + is set to `n`.
19 presence_penalty: Float that penalizes new tokens based on whether they
20 appear in the generated text so far. Values > 0 encourage the model
21 to use new tokens, while values < 0 encourage the model to repeat
@@ -28,7 +32,10 @@ class SamplingParams:
32 top_k: Integer that controls the number of top tokens to consider. Set
33 to -1 to consider all tokens.
34 use_beam_search: Whether to use beam search instead of sampling.
31 - stop_token_ids: Set of token IDs that indicate the end of a sequence.
35 + stop: List of strings that stop the generation when they are generated.
36 + The returned output will not contain the stop strings.
37 + ignore_eos: Whether to ignore the EOS token and continue generating
38 + tokens after the EOS token is generated.
39 max_tokens: Maximum number of tokens to generate per output sequence.
40 logprobs: Number of log probabilities to return per output token.
41 """
@@ -36,24 +43,28 @@ class SamplingParams:
43 def __init__(
44 self,
45 n: int = 1,
46 + best_of: Optional[int] = None,
47 presence_penalty: float = 0.0,
48 frequency_penalty: float = 0.0,
49 temperature: float = 1.0,
50 top_p: float = 1.0,
51 top_k: int = -1,
52 use_beam_search: bool = False,
45 - stop_token_ids: Set[int] = set(),
53 + stop: Union[str, List[str]] = [],
54 + ignore_eos: bool = False,
55 max_tokens: int = 16,
56 logprobs: int = 0,
57 ) -> None:
58 self.n = n
59 + self.best_of = best_of if best_of is not None else n
60 self.presence_penalty = presence_penalty
61 self.frequency_penalty = frequency_penalty
62 self.temperature = temperature
63 self.top_p = top_p
64 self.top_k = top_k
65 self.use_beam_search = use_beam_search
56 - self.stop_token_ids = stop_token_ids
66 + self.stop = [stop] if isinstance(stop, str) else list(stop)
67 + self.ignore_eos = ignore_eos
68 self.max_tokens = max_tokens
69 self.logprobs = logprobs
70
@@ -67,6 +78,9 @@ class SamplingParams:
78 def _verify_args(self) -> None:
79 if self.n < 1:
80 raise ValueError(f"n must be at least 1, got {self.n}.")
81 + if self.best_of < self.n:
82 + raise ValueError(f"best_of must be greater than or equal to n, "
83 + f"got n={self.n} and best_of={self.best_of}.")
84 if not -2.0 <= self.presence_penalty <= 2.0:
85 raise ValueError("presence_penalty must be in [-2, 2], got "
86 f"{self.presence_penalty}.")
@@ -89,8 +103,9 @@ class SamplingParams:
103 f"logprobs must be non-negative, got {self.logprobs}.")
104
105 def _verity_beam_search(self) -> None:
92 - if self.n == 1:
93 - raise ValueError("n must be greater than 1 when using beam search.")
106 + if self.best_of == 1:
107 + raise ValueError("best_of must be greater than 1 when using beam "
108 + f"search. Got {self.best_of}.")
109 if self.temperature > 0.0:
110 raise ValueError("temperature must be 0 when using beam search.")
111 if self.top_p < 1.0:
@@ -99,8 +114,9 @@ class SamplingParams:
114 raise ValueError("top_k must be -1 when using beam search.")
115
116 def _verify_greedy_sampling(self) -> None:
102 - if self.n > 1:
103 - raise ValueError("n must be 1 when using greedy sampling.")
117 + if self.best_of > 1:
118 + raise ValueError("best_of must be 1 when using greedy sampling."
119 + f"Got {self.best_of}.")
120 if self.top_p < 1.0:
121 raise ValueError("top_p must be 1 when using greedy sampling.")
122 if self.top_k != -1:
@@ -108,12 +124,14 @@ class SamplingParams:
124
125 def __repr__(self) -> str:
126 return (f"SamplingParams(n={self.n}, "
127 + f"best_of={self.best_of}, "
128 f"presence_penalty={self.presence_penalty}, "
129 f"frequency_penalty={self.frequency_penalty}, "
130 f"temperature={self.temperature}, "
131 f"top_p={self.top_p}, "
132 f"top_k={self.top_k},"
133 f"use_beam_search={self.use_beam_search}, "
117 - f"stop_token_ids={self.stop_token_ids}, "
134 + f"stop={self.stop}, "
135 + f"ignore_eos={self.ignore_eos}, "
136 f"max_tokens={self.max_tokens}, "
137 f"logprobs={self.logprobs})")
cacheflow/sequence.py
+21 -5
@@ -22,11 +22,18 @@ class SequenceData:
22 self.prompt_token_ids = prompt_token_ids
23
24 self.output_token_ids: List[int] = []
25 - self.cumulative_logprobs = 0.0
25 + self.cumulative_logprob = 0.0
26 +
27 + def append_token(self, token_id: int, logprob: float) -> None:
28 + self.output_token_ids.append(token_id)
29 + self.cumulative_logprob += logprob
30
31 def get_len(self) -> int:
32 return len(self.output_token_ids) + len(self.prompt_token_ids)
33
34 + def get_output_len(self) -> int:
35 + return len(self.output_token_ids)
36 +
37 def get_token_ids(self) -> List[int]:
38 return self.prompt_token_ids + self.output_token_ids
39
@@ -37,9 +44,9 @@ class SequenceData:
44
45 def __repr__(self) -> str:
46 return (f"SequenceData("
40 - f"prompt={self.prompt}, "
47 f"prompt_token_ids={self.prompt_token_ids}, "
42 - f"output_token_ids={self.output_token_ids})")
48 + f"output_token_ids={self.output_token_ids}, "
49 + f"cumulative_logprob={self.cumulative_logprob})")
50
51
52 class Sequence:
@@ -57,6 +64,7 @@ class Sequence:
64
65 self.data = SequenceData(prompt_token_ids)
66 self.output_logprobs: List[Dict[int, float]] = []
67 + self.output_text = ""
68
69 self.logical_token_blocks: List[LogicalTokenBlock] = []
70 # Initialize the logical token blocks with the prompt token ids.
@@ -88,18 +96,26 @@ class Sequence:
96 assert token_id in logprobs
97 self._append_tokens_to_blocks([token_id])
98 self.output_logprobs.append(logprobs)
91 - self.data.output_token_ids.append(token_id)
92 - self.data.cumulative_logprobs += logprobs[token_id]
99 + self.data.append_token(token_id, logprobs[token_id])
100
101 def get_len(self) -> int:
102 return self.data.get_len()
103
104 + def get_output_len(self) -> int:
105 + return self.data.get_output_len()
106 +
107 def get_token_ids(self) -> List[int]:
108 return self.data.get_token_ids()
109
110 def get_last_token_id(self) -> int:
111 return self.data.get_last_token_id()
112
113 + def get_output_token_ids(self) -> List[int]:
114 + return self.data.output_token_ids
115 +
116 + def get_cumulative_logprob(self) -> float:
117 + return self.data.cumulative_logprob
118 +
119 def fork(self, child_seq: 'Sequence') -> 'Sequence':
120 child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
121 child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
cacheflow/server/llm_server.py
+55 -13
@@ -13,7 +13,7 @@ from cacheflow.logger import init_logger
13 from cacheflow.outputs import RequestOutput
14 from cacheflow.sampling_params import SamplingParams
15 from cacheflow.server.tokenizer_utils import get_tokenizer
16 -from cacheflow.sequence import Sequence, SequenceGroup
16 +from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
17 from cacheflow.utils import Counter
18 from cacheflow.worker.worker import Worker
19
@@ -49,7 +49,6 @@ class LLMServer:
49 self.parallel_config = parallel_config
50 self.scheduler_config = scheduler_config
51 self.log_stats = log_stats
52 -
52 self._verify_args()
53
54 self.tokenizer = get_tokenizer(model_config.model)
@@ -124,15 +123,11 @@ class LLMServer:
123 # Create the sequences.
124 block_size = self.cache_config.block_size
125 seqs: List[Sequence] = []
127 - for _ in range(sampling_params.n):
126 + for _ in range(sampling_params.best_of):
127 seq_id = next(self.seq_counter)
128 seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
129 seqs.append(seq)
130
132 - # FIXME(woosuk)
133 - # Add the EOS token to the stop token list.
134 - sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
135 -
131 # Create the sequence group.
132 seq_group = SequenceGroup(request_id, seqs, sampling_params,
133 arrival_time)
@@ -157,18 +152,65 @@ class LLMServer:
152 blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
153 blocks_to_copy=scheduler_outputs.blocks_to_copy,
154 )
160 - # Update the scheduler.
161 - updated_seq_groups = self.scheduler.update(output)
155 + # Update the scheduler with the model outputs.
156 + seq_groups = self.scheduler.update(output)
157 +
158 + # Decode the sequences.
159 + self._decode_sequences(seq_groups)
160 + # Stop the sequences that meet the stopping criteria.
161 + self._stop_sequences(seq_groups)
162 + # Free the finished sequence groups.
163 + self.scheduler.free_finished_seq_groups()
164
165 # Create the outputs.
166 request_outputs: List[RequestOutput] = []
165 - for seq_group in updated_seq_groups:
166 - # TODO(woosuk): Batch-decode the outputs for speedup.
167 - request_output = RequestOutput.from_seq_group(seq_group,
168 - self.tokenizer)
167 + for seq_group in seq_groups:
168 + request_output = RequestOutput.from_seq_group(seq_group)
169 request_outputs.append(request_output)
170 return request_outputs
171
172 + def _decode_sequences(self, seq_groups: List[SequenceGroup]) -> None:
173 + # Batch-decode the sequence outputs.
174 + seqs: List[Sequence] = []
175 + for seq_group in seq_groups:
176 + seqs.extend(seq_group.get_seqs(status=SequenceStatus.RUNNING))
177 + output_tokens_per_seq = []
178 + for seq in seqs:
179 + output_tokens_per_seq.append(seq.get_output_token_ids())
180 + output_texts = self.tokenizer.batch_decode(output_tokens_per_seq,
181 + skip_special_tokens=True)
182 + # Update the sequences with the output texts.
183 + for seq, output_text in zip(seqs, output_texts):
184 + seq.output_text = output_text
185 +
186 + def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
187 + # Stop the sequences.
188 + for seq_group in seq_groups:
189 + sampling_params = seq_group.sampling_params
190 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
191 + # Check if the sequence has generated a stop string.
192 + stopped = False
193 + for stop_str in sampling_params.stop:
194 + if seq.output_text.endswith(stop_str):
195 + # Truncate the output text so that the stop string is
196 + # not included in the output.
197 + seq.output_text = seq.output_text[:-len(stop_str)]
198 + self.scheduler.free_seq(seq)
199 + stopped = True
200 + break
201 + if stopped:
202 + continue
203 +
204 + # Check if the sequence has reached max_tokens.
205 + if seq.get_output_len() == sampling_params.max_tokens:
206 + self.scheduler.free_seq(seq)
207 + continue
208 + # Check if the sequence has generated the EOS token.
209 + if not sampling_params.ignore_eos:
210 + if seq.get_last_token_id() == self.tokenizer.eos_token_id:
211 + self.scheduler.free_seq(seq)
212 + continue
213 +
214 def _run_workers(
215 self,
216 method: str,
examples/simple_server.py
+2 -2
@@ -15,9 +15,9 @@ def main(args: argparse.Namespace):
15 ("To be or not to be,",
16 SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
17 ("What is the meaning of life?",
18 - SamplingParams(n=2, temperature=0.8, top_p=0.95, frequency_penalty=0.1)),
18 + SamplingParams(n=2, best_of=5, temperature=0.8, top_p=0.95, frequency_penalty=0.1)),
19 ("It is only with the heart that one can see rightly",
20 - SamplingParams(n=3, use_beam_search=True, temperature=0.0)),
20 + SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
21 ]
22
23 # Run the server.