@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 002800f081
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Align vLLM's beam search implementation with HF generate (#857)

Zhuohan Li committed Sep 4, 2023 at 17:29 UTC 002800f08142574aeafed7c33982503d28d53cdc
24 files changed +599 -263
docs/source/models/adding_model.rst
+1 -1
@@ -59,7 +59,7 @@ Next, you need to rewrite the :code:`forward` methods of your model by following
59 + kv_caches: List[KVCache],
60 + input_metadata: InputMetadata,
61 + cache_events: Optional[List[torch.cuda.Event]],
62 - +) -> Dict[int, SequenceOutputs]:
62 + +) -> SamplerOutput:
63
64 3. Update the code by considering that :code:`input_ids` and :code:`positions` are now flattened tensors.
65 4. Replace the attention operation with either :code:`GPTPagedAttention` or :code:`GPTNeoXPagedAttention`, depending on the model's architecture.
tests/conftest.py
+56 -10
@@ -67,8 +67,8 @@ class HfRunner:
67 output_ids,
68 skip_special_tokens=True,
69 clean_up_tokenization_spaces=False,
70 - )[0]
71 - output_ids = output_ids[0].cpu().tolist()
70 + )
71 + output_ids = output_ids.cpu().tolist()
72 outputs.append((output_ids, output_str))
73 return outputs
74
@@ -77,8 +77,34 @@ class HfRunner:
77 prompts: List[str],
78 max_tokens: int,
79 ) -> List[Tuple[List[int], str]]:
80 - return self.generate(prompts, do_sample=False,
81 - max_new_tokens=max_tokens)
80 + outputs = self.generate(prompts,
81 + do_sample=False,
82 + max_new_tokens=max_tokens)
83 + for i in range(len(outputs)):
84 + output_ids, output_str = outputs[i]
85 + outputs[i] = (output_ids[0], output_str[0])
86 + return outputs
87 +
88 + def generate_beam_search(
89 + self,
90 + prompts: List[str],
91 + beam_width: int,
92 + max_tokens: int,
93 + ) -> List[Tuple[List[int], str]]:
94 + outputs = self.generate(prompts,
95 + do_sample=False,
96 + max_new_tokens=max_tokens,
97 + num_beams=beam_width,
98 + num_return_sequences=beam_width)
99 + for i in range(len(outputs)):
100 + output_ids, output_str = outputs[i]
101 + for j in range(len(output_ids)):
102 + output_ids[j] = [
103 + x for x in output_ids[j]
104 + if x != self.tokenizer.pad_token_id
105 + ]
106 + outputs[i] = (output_ids, output_str)
107 + return outputs
108
109
110 @pytest.fixture
@@ -107,15 +133,20 @@ class VllmRunner:
133 prompts: List[str],
134 sampling_params: SamplingParams,
135 ) -> List[Tuple[List[int], str]]:
110 - req_outputs = self.model.generate(
111 - prompts, sampling_params=sampling_params)
136 + req_outputs = self.model.generate(prompts,
137 + sampling_params=sampling_params)
138 outputs = []
139 for req_output in req_outputs:
140 prompt_str = req_output.prompt
141 prompt_ids = req_output.prompt_token_ids
116 - output_str = req_output.outputs[0].text
117 - output_ids = req_output.outputs[0].token_ids
118 - outputs.append((prompt_ids + output_ids, prompt_str + output_str))
142 + req_sample_output_ids = []
143 + req_sample_output_strs = []
144 + for sample in req_output.outputs:
145 + output_str = sample.text
146 + output_ids = sample.token_ids
147 + req_sample_output_ids.append(prompt_ids + output_ids)
148 + req_sample_output_strs.append(prompt_str + output_str)
149 + outputs.append((req_sample_output_ids, req_sample_output_strs))
150 return outputs
151
152 def generate_greedy(
@@ -124,7 +155,22 @@ class VllmRunner:
155 max_tokens: int,
156 ) -> List[Tuple[List[int], str]]:
157 greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
127 - return self.generate(prompts, greedy_params)
158 + outputs = self.generate(prompts, greedy_params)
159 + return [(output_ids[0], output_str[0]) for output_ids, output_str in
160 + outputs]
161 +
162 + def generate_beam_search(
163 + self,
164 + prompts: List[str],
165 + beam_width: int,
166 + max_tokens: int,
167 + ) -> List[Tuple[List[int], str]]:
168 + beam_search_params = SamplingParams(n=beam_width,
169 + use_beam_search=True,
170 + temperature=0.0,
171 + max_tokens=max_tokens)
172 + outputs = self.generate(prompts, beam_search_params)
173 + return outputs
174
175
176 @pytest.fixture
tests/samplers/test_beam_search.py new
+46
@@ -0,0 +1,46 @@
1 +"""Compare the outputs of HF and vLLM when using beam search.
2 +
3 +Run `pytest tests/samplers/test_beam_search.py --forked`.
4 +"""
5 +import pytest
6 +
7 +# FIXME(zhuohan): The test can not pass if we:
8 +# 1. Increase max_tokens to 256.
9 +# 2. Increase beam_width to 8.
10 +# 3. Use the model "huggyllama/llama-7b".
11 +MAX_TOKENS = [128]
12 +BEAM_WIDTHS = [4]
13 +MODELS = ["facebook/opt-125m"]
14 +
15 +
16 +@pytest.mark.parametrize("model", MODELS)
17 +@pytest.mark.parametrize("dtype", ["half"])
18 +@pytest.mark.parametrize("max_tokens", MAX_TOKENS)
19 +@pytest.mark.parametrize("beam_width", BEAM_WIDTHS)
20 +def test_beam_search_single_input(
21 + hf_runner,
22 + vllm_runner,
23 + example_prompts,
24 + model: str,
25 + dtype: str,
26 + max_tokens: int,
27 + beam_width: int,
28 +) -> None:
29 + hf_model = hf_runner(model, dtype=dtype)
30 + hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
31 + max_tokens)
32 + del hf_model
33 +
34 + vllm_model = vllm_runner(model, dtype=dtype)
35 + vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
36 + max_tokens)
37 + del vllm_model
38 +
39 + for i in range(len(example_prompts)):
40 + hf_output_ids, _ = hf_outputs[i]
41 + vllm_output_ids, _ = vllm_outputs[i]
42 + assert len(hf_output_ids) == len(vllm_output_ids)
43 + for j in range(len(hf_output_ids)):
44 + assert hf_output_ids[j] == vllm_output_ids[j], (
45 + f"Test{i} output{j}:\nHF: {hf_output_ids}\n"
46 + f"vLLM: {vllm_output_ids}")
vllm/core/block_manager.py
+2 -6
@@ -172,9 +172,7 @@ class BlockSpaceManager:
172 def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
173 # CPU block -> GPU block.
174 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
175 - for seq in seq_group.get_seqs():
176 - if seq.is_finished():
177 - continue
175 + for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
176 new_block_table: BlockTable = []
177 block_table = self.block_tables[seq.seq_id]
178
@@ -203,9 +201,7 @@ class BlockSpaceManager:
201 def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
202 # GPU block -> CPU block.
203 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
206 - for seq in seq_group.get_seqs():
207 - if seq.is_finished():
208 - continue
204 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
205 new_block_table: BlockTable = []
206 block_table = self.block_tables[seq.seq_id]
207
vllm/core/scheduler.py
+45 -69
@@ -7,8 +7,7 @@ from vllm.core.block_manager import BlockSpaceManager
7 from vllm.core.policy import PolicyFactory
8 from vllm.logger import init_logger
9 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
10 - SequenceGroupMetadata, SequenceOutputs,
11 - SequenceStatus)
10 + SequenceGroupMetadata, SequenceStatus)
11
12 logger = init_logger(__name__)
13
@@ -76,6 +75,7 @@ class Scheduler:
75 num_cpu_blocks=self.cache_config.num_cpu_blocks,
76 )
77
78 + # TODO(zhuohan): Use deque instead of list for better performance.
79 # Sequence groups in the WAITING state.
80 self.waiting: List[SequenceGroup] = []
81 # Sequence groups in the RUNNING state.
@@ -96,10 +96,11 @@ class Scheduler:
96 if seq_group.request_id in request_ids:
97 # Remove the sequence group from the state queue.
98 state_queue.remove(seq_group)
99 - for seq in seq_group.seqs:
99 + for seq in seq_group.get_seqs():
100 if seq.is_finished():
101 continue
102 - self.free_seq(seq, SequenceStatus.FINISHED_ABORTED)
102 + seq.status = SequenceStatus.FINISHED_ABORTED
103 + self.free_seq(seq)
104 request_ids.remove(seq_group.request_id)
105 if not request_ids:
106 return
@@ -123,6 +124,10 @@ class Scheduler:
124 if not self.swapped:
125 ignored_seq_groups: List[SequenceGroup] = []
126 scheduled: List[SequenceGroup] = []
127 + # The total number of sequences on the fly, including the
128 + # requests in the generation phase.
129 + num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
130 + for seq_group in self.running)
131 num_batched_tokens = 0
132 # Optimization: We do not sort the waiting queue since the preempted
133 # sequence groups are added to the front and the new sequence groups
@@ -130,6 +135,9 @@ class Scheduler:
135 while self.waiting:
136 seq_group = self.waiting[0]
137
138 + assert seq_group.num_seqs() == 1, (
139 + "Waiting sequence group should have only one prompt "
140 + "sequence.")
141 num_prompt_tokens = seq_group.get_seqs()[0].get_len()
142 if num_prompt_tokens > self.prompt_limit:
143 logger.warning(
@@ -152,11 +160,7 @@ class Scheduler:
160
161 # The total number of sequences in the RUNNING state should not
162 # exceed the maximum number of sequences.
155 - num_new_seqs = seq_group.num_seqs(
156 - status=SequenceStatus.WAITING)
157 - num_curr_seqs = sum(
158 - seq_group.num_seqs(status=SequenceStatus.RUNNING)
159 - for seq_group in self.running)
163 + num_new_seqs = seq_group.get_max_num_running_seqs()
164 if (num_curr_seqs + num_new_seqs >
165 self.scheduler_config.max_num_seqs):
166 break
@@ -165,6 +169,7 @@ class Scheduler:
169 self._allocate(seq_group)
170 self.running.append(seq_group)
171 num_batched_tokens += num_prompt_tokens
172 + num_curr_seqs += num_new_seqs
173 scheduled.append(seq_group)
174
175 if scheduled:
@@ -210,30 +215,32 @@ class Scheduler:
215
216 # Swap in the sequence groups in the SWAPPED state if possible.
217 self.swapped = self.policy.sort_by_priority(now, self.swapped)
213 - while self.swapped and not blocks_to_swap_out:
214 - seq_group = self.swapped[0]
215 - # If the sequence group has been preempted in this step, stop.
216 - if seq_group in preempted:
217 - break
218 - # If the sequence group cannot be swapped in, stop.
219 - if not self.block_manager.can_swap_in(seq_group):
220 - break
221 -
222 - # The total number of sequences in the RUNNING state should not
223 - # exceed the maximum number of sequences.
224 - num_new_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
225 - num_curr_seqs = sum(
226 - seq_group.num_seqs(status=SequenceStatus.RUNNING)
227 - for seq_group in self.running)
228 - if (num_curr_seqs + num_new_seqs >
229 - self.scheduler_config.max_num_seqs):
230 - break
231 -
232 - seq_group = self.swapped.pop(0)
233 - self._swap_in(seq_group, blocks_to_swap_in)
234 - self._append_slot(seq_group, blocks_to_copy)
235 - self.running.append(seq_group)
218 + if not preempted:
219 + num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
220 + for seq_group in self.running)
221 +
222 + while self.swapped:
223 + seq_group = self.swapped[0]
224 + # If the sequence group cannot be swapped in, stop.
225 + if not self.block_manager.can_swap_in(seq_group):
226 + break
227
228 + # The total number of sequences in the RUNNING state should not
229 + # exceed the maximum number of sequences.
230 + num_new_seqs = seq_group.get_max_num_running_seqs()
231 + if (num_curr_seqs + num_new_seqs >
232 + self.scheduler_config.max_num_seqs):
233 + break
234 +
235 + seq_group = self.swapped.pop(0)
236 + self._swap_in(seq_group, blocks_to_swap_in)
237 + self._append_slot(seq_group, blocks_to_copy)
238 + num_curr_seqs += num_new_seqs
239 + self.running.append(seq_group)
240 +
241 + # Each sequence in the generation phase only takes one token slot.
242 + # Therefore, the number of batched tokens is equal to the number of
243 + # sequences in the RUNNING state.
244 num_batched_tokens = sum(
245 seq_group.num_seqs(status=SequenceStatus.RUNNING)
246 for seq_group in self.running)
@@ -275,40 +282,10 @@ class Scheduler:
282 seq_group_metadata_list.append(seq_group_metadata)
283 return seq_group_metadata_list, scheduler_outputs
284
278 - def update(
279 - self,
280 - seq_outputs: Dict[int, SequenceOutputs],
281 - ) -> List[SequenceGroup]:
282 - scheduled: List[SequenceGroup] = []
283 - for seq_group in self.running:
284 - for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
285 - if seq.seq_id in seq_outputs:
286 - scheduled.append(seq_group)
287 - break
288 -
289 - # Update the scheduled sequences and free blocks.
290 - for seq_group in scheduled:
291 - # Process beam search results before processing the new tokens.
292 - for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
293 - output = seq_outputs[seq.seq_id]
294 - if seq.seq_id != output.parent_seq_id:
295 - # The sequence is a fork of the parent sequence (beam
296 - # search). Free the current sequence.
297 - self.block_manager.free(seq)
298 - # Fork the parent sequence.
299 - parent_seq = seq_group.find(output.parent_seq_id)
300 - parent_seq.fork(seq)
301 - self.block_manager.fork(parent_seq, seq)
302 -
303 - # Process the new tokens.
304 - for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
305 - # Append a new token to the sequence.
306 - output = seq_outputs[seq.seq_id]
307 - seq.append_token_id(output.output_token, output.logprobs)
308 - return scheduled
285 + def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None:
286 + self.block_manager.fork(parent_seq, child_seq)
287
310 - def free_seq(self, seq: Sequence, finish_status: SequenceStatus) -> None:
311 - seq.status = finish_status
288 + def free_seq(self, seq: Sequence) -> None:
289 self.block_manager.free(seq)
290
291 def free_finished_seq_groups(self) -> None:
@@ -345,8 +322,8 @@ class Scheduler:
322 # If preemption mode is not specified, we determine the mode as follows:
323 # We use recomputation by default since it incurs lower overhead than
324 # swapping. However, when the sequence group has multiple sequences
348 - # (e.g., beam search), recomputation is not supported. In such a case,
349 - # we use swapping instead.
325 + # (e.g., beam search), recomputation is not currently supported. In
326 + # such a case, we use swapping instead.
327 # FIXME(woosuk): This makes our scheduling policy a bit bizarre.
328 # As swapped sequences are prioritized over waiting sequences,
329 # sequence groups with multiple sequences are implicitly prioritized
@@ -354,8 +331,7 @@ class Scheduler:
331 # TODO(woosuk): Support recomputation for sequence groups with multiple
332 # sequences. This may require a more sophisticated CUDA kernel.
333 if preemption_mode is None:
357 - seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
358 - if len(seqs) == 1:
334 + if seq_group.get_max_num_running_seqs() == 1:
335 preemption_mode = PreemptionMode.RECOMPUTE
336 else:
337 preemption_mode = PreemptionMode.SWAP
vllm/engine/llm_engine.py
+260 -65
@@ -11,7 +11,8 @@ from vllm.engine.ray_utils import RayWorker, initialize_cluster, ray
11 from vllm.logger import init_logger
12 from vllm.outputs import RequestOutput
13 from vllm.sampling_params import SamplingParams
14 -from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupMetadata,
14 +from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
15 + SequenceGroupMetadata, SequenceOutputs,
16 SequenceStatus)
17 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
18 get_tokenizer)
@@ -258,14 +259,11 @@ class LLMEngine:
259
260 # Create the sequences.
261 block_size = self.cache_config.block_size
261 - seqs: List[Sequence] = []
262 - for _ in range(sampling_params.best_of):
263 - seq_id = next(self.seq_counter)
264 - seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
265 - seqs.append(seq)
262 + seq_id = next(self.seq_counter)
263 + seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
264
265 # Create the sequence group.
268 - seq_group = SequenceGroup(request_id, seqs, sampling_params,
266 + seq_group = SequenceGroup(request_id, [seq], sampling_params,
267 arrival_time)
268
269 # Add the sequence group to the scheduler.
@@ -303,22 +301,230 @@ class LLMEngine:
301 ]
302 return seq_group_metadata_list, scheduler_outputs, None
303
306 - def _process_worker_outputs(
307 - self, output,
304 + def _check_beam_search_early_stopping(
305 + self,
306 + early_stopping: Union[bool, str],
307 + sampling_params: SamplingParams,
308 + best_running_seq: Sequence,
309 + current_worst_seq: Sequence,
310 + ) -> bool:
311 + assert sampling_params.use_beam_search
312 + length_penalty = sampling_params.length_penalty
313 + if early_stopping is True:
314 + return True
315 +
316 + current_worst_score = (current_worst_seq.get_beam_search_score(
317 + length_penalty=length_penalty,
318 + eos_token_id=self.tokenizer.eos_token_id))
319 + if early_stopping is False:
320 + highest_attainable_score = (best_running_seq.get_beam_search_score(
321 + length_penalty=length_penalty,
322 + eos_token_id=self.tokenizer.eos_token_id))
323 + else:
324 + assert early_stopping == "never"
325 + if length_penalty > 0.0:
326 + # If length_penalty > 0.0, beam search will prefer longer
327 + # sequences. The highest attainable score calculation is
328 + # based on the longest possible sequence length in this case.
329 + max_possible_length = max(
330 + best_running_seq.get_prompt_len() +
331 + sampling_params.max_tokens,
332 + self.scheduler_config.max_model_len)
333 + highest_attainable_score = (
334 + best_running_seq.get_beam_search_score(
335 + length_penalty=length_penalty,
336 + eos_token_id=self.tokenizer.eos_token_id,
337 + seq_len=max_possible_length))
338 + else:
339 + # Otherwise, beam search will prefer shorter sequences. The
340 + # highest attainable score calculation is based on the current
341 + # sequence length.
342 + highest_attainable_score = (
343 + best_running_seq.get_beam_search_score(
344 + length_penalty=length_penalty,
345 + eos_token_id=self.tokenizer.eos_token_id))
346 + return current_worst_score >= highest_attainable_score
347 +
348 + def _process_sequence_group_samples(
349 + self, seq_group: SequenceGroup,
350 + samples: List[SequenceOutputs]) -> None:
351 + parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
352 + existing_finished_seqs = seq_group.get_finished_seqs()
353 + parent_child_dict = {
354 + parent_seq.seq_id: []
355 + for parent_seq in parent_seqs
356 + }
357 + for sample in samples:
358 + parent_child_dict[sample.parent_seq_id].append(sample)
359 + # List of (child, parent)
360 + child_seqs: List[Tuple[Sequence, Sequence]] = []
361 +
362 + # Process the child samples for each parent sequence
363 + for parent in parent_seqs:
364 + child_samples: List[SequenceOutputs] = parent_child_dict[
365 + parent.seq_id]
366 + if len(child_samples) == 0:
367 + # This parent sequence has no children samples. Remove
368 + # the parent sequence from the sequence group since it will
369 + # not be used in the future iterations.
370 + parent.status = SequenceStatus.FINISHED_ABORTED
371 + seq_group.remove(parent.seq_id)
372 + self.scheduler.free_seq(parent)
373 + continue
374 + # Fork the parent sequence if there are multiple child samples.
375 + for child_sample in child_samples[:-1]:
376 + new_child_seq_id = next(self.seq_counter)
377 + child = parent.fork(new_child_seq_id)
378 + child.append_token_id(child_sample.output_token,
379 + child_sample.logprobs)
380 + child_seqs.append((child, parent))
381 + # Continue the parent sequence for the last child sample.
382 + # We reuse the parent sequence here to reduce redundant memory
383 + # copies, especially when using non-beam search sampling methods.
384 + last_child_sample = child_samples[-1]
385 + parent.append_token_id(last_child_sample.output_token,
386 + last_child_sample.logprobs)
387 + child_seqs.append((parent, parent))
388 +
389 + for seq, _ in child_seqs:
390 + self._decode_sequence(seq)
391 + self._check_stop(seq, seq_group.sampling_params)
392 +
393 + # Non-beam search case
394 + if not seq_group.sampling_params.use_beam_search:
395 + # For newly created child sequences, add them to the sequence group
396 + # and fork them in block manager if they are not finished.
397 + for seq, parent in child_seqs:
398 + if seq is not parent:
399 + seq_group.add(seq)
400 + if not seq.is_finished():
401 + self.scheduler.fork_seq(parent, seq)
402 +
403 + # Free the finished and selected parent sequences' memory in block
404 + # manager. Keep them in the sequence group as candidate output.
405 + # NOTE: we need to fork the new sequences before freeing the
406 + # old sequences.
407 + for seq, parent in child_seqs:
408 + if seq is parent and seq.is_finished():
409 + self.scheduler.free_seq(seq)
410 + return
411 +
412 + # Beam search case
413 + # Select the child sequences to keep in the sequence group.
414 + selected_child_seqs = []
415 + unselected_child_seqs = []
416 + beam_width = seq_group.sampling_params.best_of
417 + length_penalty = seq_group.sampling_params.length_penalty
418 +
419 + # Select the newly finished sequences with the highest scores
420 + # to replace existing finished sequences.
421 + # Tuple of (seq, parent, is_new)
422 + existing_finished_seqs = [(seq, None, False)
423 + for seq in existing_finished_seqs]
424 + new_finished_seqs = [(seq, parent, True) for seq, parent in child_seqs
425 + if seq.is_finished()]
426 + all_finished_seqs = existing_finished_seqs + new_finished_seqs
427 + # Sort the finished sequences by their scores.
428 + all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
429 + length_penalty=length_penalty,
430 + eos_token_id=self.tokenizer.eos_token_id),
431 + reverse=True)
432 + for seq, parent, is_new in all_finished_seqs[:beam_width]:
433 + if is_new:
434 + # A newly generated child sequence finishes and has a high
435 + # score, so we will add it into the sequence group.
436 + selected_child_seqs.append((seq, parent))
437 + for seq, parent, is_new in all_finished_seqs[beam_width:]:
438 + if is_new:
439 + # A newly generated child sequence finishes but has a low
440 + # score, so we will not add it into the sequence group.
441 + # Additionally, if this sequence is a continuation of a
442 + # parent sequence, we will need remove the parent sequence
443 + # from the sequence group.
444 + unselected_child_seqs.append((seq, parent))
445 + else:
446 + # An existing finished sequence has a low score, so we will
447 + # remove it from the sequence group.
448 + seq_group.remove(seq.seq_id)
449 +
450 + # select the top beam_width sequences from the running
451 + # sequences for the next iteration to continue the beam
452 + # search.
453 + running_child_seqs = [(seq, parent) for seq, parent in child_seqs
454 + if not seq.is_finished()]
455 + # Sort the running sequences by their scores.
456 + running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
457 + length_penalty=length_penalty,
458 + eos_token_id=self.tokenizer.eos_token_id),
459 + reverse=True)
460 +
461 + # Check if we can stop the beam search.
462 + if len(running_child_seqs) == 0:
463 + # No running sequences, stop the beam search.
464 + stop_beam_search = True
465 + elif len(all_finished_seqs) < beam_width:
466 + # Not enough finished sequences, continue the beam search.
467 + stop_beam_search = False
468 + else:
469 + # Check the early stopping criteria
470 + best_running_seq = running_child_seqs[0][0]
471 + current_worst_seq = all_finished_seqs[beam_width - 1][0]
472 + stop_beam_search = self._check_beam_search_early_stopping(
473 + seq_group.sampling_params.early_stopping,
474 + seq_group.sampling_params, best_running_seq, current_worst_seq)
475 +
476 + if stop_beam_search:
477 + # Stop the beam search and remove all the running sequences from
478 + # the sequence group.
479 + unselected_child_seqs.extend(running_child_seqs)
480 + else:
481 + # Continue the beam search and select the top beam_width sequences
482 + # to continue the beam search.
483 + selected_child_seqs.extend(running_child_seqs[:beam_width])
484 + # The remaining running sequences will not be used in the next
485 + # iteration. Again, if these sequences are continuations of
486 + # parent sequences, we will need to remove the parent sequences
487 + # from the sequence group.
488 + unselected_child_seqs.extend(running_child_seqs[beam_width:])
489 +
490 + # For newly created child sequences, add them to the sequence group
491 + # and fork them in block manager if they are not finished.
492 + for seq, parent in selected_child_seqs:
493 + if seq is not parent:
494 + seq_group.add(seq)
495 + if not seq.is_finished():
496 + self.scheduler.fork_seq(parent, seq)
497 +
498 + # Free the finished and selected parent sequences' memory in block
499 + # manager. Keep them in the sequence group as candidate output.
500 + for seq, parent in selected_child_seqs:
501 + if seq is parent and seq.is_finished():
502 + self.scheduler.free_seq(seq)
503 +
504 + # Remove the unselected parent sequences from the sequence group and
505 + # free their memory in block manager.
506 + for seq, parent in unselected_child_seqs:
507 + if seq is parent:
508 + # Remove the parent sequence if it is not selected for next
509 + # iteration
510 + seq_group.remove(seq.seq_id)
511 + self.scheduler.free_seq(seq)
512 +
513 + def _process_model_outputs(
514 + self, output: SamplerOutput,
515 scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]:
309 - # Update the scheduler with the model outputs.
310 - seq_groups = self.scheduler.update(output)
516 + # Update the scheduled sequence groups with the model outputs.
517 + scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
518 + for seq_group, samples in zip(scheduled_seq_groups, output):
519 + self._process_sequence_group_samples(seq_group, samples)
520
312 - # Decode the sequences.
313 - self._decode_sequences(seq_groups)
314 - # Stop the sequences that meet the stopping criteria.
315 - self._stop_sequences(seq_groups)
521 # Free the finished sequence groups.
522 self.scheduler.free_finished_seq_groups()
523
524 # Create the outputs.
525 request_outputs: List[RequestOutput] = []
321 - for seq_group in seq_groups + scheduler_outputs.ignored_seq_groups:
526 + for seq_group in (scheduled_seq_groups +
527 + scheduler_outputs.ignored_seq_groups):
528 request_output = RequestOutput.from_seq_group(seq_group)
529 request_outputs.append(request_output)
530
@@ -351,7 +557,7 @@ class LLMEngine:
557 blocks_to_copy=scheduler_outputs.blocks_to_copy,
558 )
559
354 - return self._process_worker_outputs(output, scheduler_outputs)
560 + return self._process_model_outputs(output, scheduler_outputs)
561
562 def _log_system_stats(
563 self,
@@ -416,55 +622,44 @@ class LLMEngine:
622 f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
623 self.last_logging_time = now
624
419 - def _decode_sequences(self, seq_groups: List[SequenceGroup]) -> None:
420 - """Decodes the sequence outputs."""
421 - for seq_group in seq_groups:
422 - for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
423 - new_token, new_output_text = detokenize_incrementally(
424 - self.tokenizer,
425 - seq.output_tokens,
426 - seq.get_last_token_id(),
427 - skip_special_tokens=True,
428 - )
429 - if new_token is not None:
430 - seq.output_tokens.append(new_token)
431 - seq.output_text = new_output_text
432 -
433 - def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
625 + def _decode_sequence(self, seq: Sequence) -> None:
626 + """Decodes the new token for a sequence."""
627 + new_token, new_output_text = detokenize_incrementally(
628 + self.tokenizer,
629 + seq.output_tokens,
630 + seq.get_last_token_id(),
631 + skip_special_tokens=True,
632 + )
633 + if new_token is not None:
634 + seq.output_tokens.append(new_token)
635 + seq.output_text = new_output_text
636 +
637 + def _check_stop(self, seq: Sequence,
638 + sampling_params: SamplingParams) -> None:
639 """Stop the finished sequences."""
435 - for seq_group in seq_groups:
436 - sampling_params = seq_group.sampling_params
437 - for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
438 - # Check if the sequence has generated a stop string.
439 - stopped = False
440 - for stop_str in sampling_params.stop:
441 - if seq.output_text.endswith(stop_str):
442 - # Truncate the output text so that the stop string is
443 - # not included in the output.
444 - seq.output_text = seq.output_text[:-len(stop_str)]
445 - self.scheduler.free_seq(
446 - seq, SequenceStatus.FINISHED_STOPPED)
447 - stopped = True
448 - break
449 - if stopped:
450 - continue
451 -
452 - # Check if the sequence has reached max_model_len.
453 - if seq.get_len() > self.scheduler_config.max_model_len:
454 - self.scheduler.free_seq(
455 - seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
456 - continue
457 - # Check if the sequence has reached max_tokens.
458 - if seq.get_output_len() == sampling_params.max_tokens:
459 - self.scheduler.free_seq(
460 - seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
461 - continue
462 - # Check if the sequence has generated the EOS token.
463 - if not sampling_params.ignore_eos:
464 - if seq.get_last_token_id() == self.tokenizer.eos_token_id:
465 - self.scheduler.free_seq(
466 - seq, SequenceStatus.FINISHED_STOPPED)
467 - continue
640 + for stop_str in sampling_params.stop:
641 + if seq.output_text.endswith(stop_str):
642 + # Truncate the output text so that the stop string is
643 + # not included in the output.
644 + seq.output_text = seq.output_text[:-len(stop_str)]
645 + seq.status = SequenceStatus.FINISHED_STOPPED
646 + return
647 +
648 + # Check if the sequence has reached max_model_len.
649 + if seq.get_len() > self.scheduler_config.max_model_len:
650 + seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
651 + return
652 +
653 + # Check if the sequence has reached max_tokens.
654 + if seq.get_output_len() == sampling_params.max_tokens:
655 + seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
656 + return
657 +
658 + # Check if the sequence has generated the EOS token.
659 + if ((not sampling_params.ignore_eos)
660 + and seq.get_last_token_id() == self.tokenizer.eos_token_id):
661 + seq.status = SequenceStatus.FINISHED_STOPPED
662 + return
663
664 def _run_workers(
665 self,
vllm/model_executor/layers/sampler.py
+31 -42
@@ -9,7 +9,7 @@ from vllm.model_executor.input_metadata import InputMetadata
9 from vllm.model_executor.parallel_utils.tensor_parallel import (
10 gather_from_tensor_model_parallel_region)
11 from vllm.sampling_params import SamplingParams
12 -from vllm.sequence import SequenceOutputs
12 +from vllm.sequence import SamplerOutput, SequenceOutputs
13
14 _SAMPLING_EPS = 1e-5
15
@@ -39,7 +39,7 @@ class Sampler(nn.Module):
39 hidden_states: torch.Tensor,
40 input_metadata: InputMetadata,
41 embedding_bias: Optional[torch.Tensor] = None,
42 - ) -> Dict[int, SequenceOutputs]:
42 + ) -> SamplerOutput:
43 # Get the hidden states that we use for sampling.
44 hidden_states = _prune_hidden_states(hidden_states, input_metadata)
45
@@ -292,7 +292,13 @@ def _sample_from_prompt(
292 if sampling_params.use_beam_search:
293 # Beam search.
294 beam_width = sampling_params.best_of
295 - _, next_token_ids = torch.topk(prob, beam_width)
295 + # Sample 2 * beam_width candidates to make sure that with high
296 + # probability we can get `beam_width` candidates in addition to
297 + # the finished sequences for the next iteration. See
298 + # https://github.com/tensorflow/tensor2tensor/blob/bafdc1b67730430d38d6ab802cbd51f9d053ba2e/tensor2tensor/utils/beam_search.py#L557-L563
299 + # for details. See also HF reference:
300 + # https://github.com/huggingface/transformers/blob/a4dd53d88e4852f023332d284ff07a01afcd5681/src/transformers/generation/utils.py#L3063-L3065
301 + _, next_token_ids = torch.topk(prob, 2 * beam_width)
302 next_token_ids = next_token_ids.tolist()
303 elif sampling_params.temperature < _SAMPLING_EPS:
304 # Greedy sampling.
@@ -330,29 +336,11 @@ def _sample_from_generation_tokens(
336
337 vocab_size = logprobs.size(-1)
338 beam_width = len(seq_ids)
333 - _, topk_ids = torch.topk(logprobs.flatten(), beam_width)
339 + _, topk_ids = torch.topk(logprobs.flatten(), 2 * beam_width)
340 topk_ids = topk_ids.tolist()
341 seq_idx = [i // vocab_size for i in topk_ids]
336 - beam_seq_ids = [seq_ids[i] for i in seq_idx]
337 - token_ids = [i % vocab_size for i in topk_ids]
338 -
339 - beam_outputs: Dict[int, Tuple[int, int]] = {}
340 - outstanding_beams: List[Tuple[int, int]] = []
341 - # If a beam survives, continue with it.
342 - for seq_id, token_id in zip(beam_seq_ids, token_ids):
343 - if seq_id not in beam_outputs:
344 - beam_outputs[seq_id] = (seq_id, token_id)
345 - else:
346 - outstanding_beams.append((seq_id, token_id))
347 -
348 - # If a beam is discarded, fork another beam.
349 - for seq_id in seq_ids:
350 - if seq_id not in beam_outputs:
351 - beam_outputs[seq_id] = outstanding_beams.pop()
352 - assert not outstanding_beams
353 -
354 - parent_seq_ids = [beam_outputs[seq_id][0] for seq_id in seq_ids]
355 - next_token_ids = [beam_outputs[seq_id][1] for seq_id in seq_ids]
342 + parent_seq_ids = [seq_ids[i] for i in seq_idx]
343 + next_token_ids = [i % vocab_size for i in topk_ids]
344 elif sampling_params.temperature < _SAMPLING_EPS:
345 # Greedy sampling.
346 assert len(seq_ids) == 1
@@ -374,16 +362,18 @@ def _sample(
362 probs: torch.Tensor,
363 logprobs: torch.Tensor,
364 input_metadata: InputMetadata,
377 -) -> Dict[int, SequenceOutputs]:
378 - seq_outputs: Dict[int, SequenceOutputs] = {}
365 +) -> SamplerOutput:
366 + seq_outputs: SamplerOutput = []
367
368 # TODO(woosuk): Optimize.
369 idx = 0
370 for i, seq_group in enumerate(input_metadata.seq_groups):
371 + seq_group_outputs: List[SequenceOutputs] = []
372 seq_ids, sampling_params = seq_group
373 if i < input_metadata.num_prompts:
374 # Generate the next tokens for a prompt input.
386 - assert len(seq_ids) == sampling_params.best_of
375 + assert len(seq_ids) == 1, "Prompt input should have only one seq."
376 + parent_seq_id = seq_ids[0]
377 prob = probs[idx]
378 logprob = logprobs[idx]
379 idx += 1
@@ -395,17 +385,18 @@ def _sample(
385 sampling_params.logprobs)
386
387 # Build the output.
398 - for seq_id, next_token_id in zip(seq_ids, next_token_ids):
388 + for next_token_id in next_token_ids:
389 output_logprobs = next_logprobs.copy()
390 output_logprobs[next_token_id] = logprob[next_token_id].item()
401 - seq_outputs[seq_id] = SequenceOutputs(seq_id, seq_id,
402 - next_token_id,
403 - output_logprobs)
391 + seq_group_outputs.append(
392 + SequenceOutputs(parent_seq_id, next_token_id,
393 + output_logprobs))
394 else:
395 # Generate the next tokens for generation tokens.
406 - prob = probs[idx:idx + len(seq_ids)]
407 - logprob = logprobs[idx:idx + len(seq_ids)]
408 - idx += len(seq_ids)
396 + num_parent_seqs = len(seq_ids)
397 + prob = probs[idx:idx + num_parent_seqs]
398 + logprob = logprobs[idx:idx + num_parent_seqs]
399 + idx += num_parent_seqs
400
401 # Sample the next tokens.
402 seq_logprobs = [
@@ -422,17 +413,15 @@ def _sample(
413 logprob[j], sampling_params.logprobs)
414
415 # Build the output.
425 - for seq_id, parent_seq_id, next_token_id in zip(
426 - seq_ids, parent_seq_ids, next_token_ids):
416 + for parent_seq_id, next_token_id in zip(parent_seq_ids,
417 + next_token_ids):
418 j = seq_ids.index(parent_seq_id)
419 output_logprobs = next_logprobs[parent_seq_id].copy()
420 output_logprobs[next_token_id] = logprob[j,
421 next_token_id].item()
431 - seq_outputs[seq_id] = SequenceOutputs(
432 - seq_id,
433 - parent_seq_id,
434 - next_token_id,
435 - output_logprobs,
436 - )
422 + seq_group_outputs.append(
423 + SequenceOutputs(parent_seq_id, next_token_id,
424 + output_logprobs))
425 + seq_outputs.append(seq_group_outputs)
426
427 return seq_outputs
vllm/model_executor/models/aquila.py
+3 -3
@@ -25,7 +25,7 @@
25 The input of the model is flattened to a 1D tensor of tokens. The model uses
26 InputMetadata to extract the original 2D shape of the input.
27 """
28 -from typing import Dict, List, Optional, Tuple
28 +from typing import List, Optional, Tuple
29
30 import torch
31 from torch import nn
@@ -41,7 +41,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 from vllm.model_executor.parallel_utils.tensor_parallel import (
43 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
44 -from vllm.sequence import SequenceOutputs
44 +from vllm.sequence import SamplerOutput
45 from vllm.transformers_utils.configs.aquila import AquilaConfig
46
47 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -273,7 +273,7 @@ class AquilaForCausalLM(nn.Module):
273 kv_caches: List[KVCache],
274 input_metadata: InputMetadata,
275 cache_events: Optional[List[torch.cuda.Event]],
276 - ) -> Dict[int, SequenceOutputs]:
276 + ) -> SamplerOutput:
277 hidden_states = self.model(input_ids, positions, kv_caches,
278 input_metadata, cache_events)
279 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/model_executor/models/baichuan.py
+3 -3
@@ -23,12 +23,11 @@ The input of the model is flattened to a 1D tensor of tokens. The model uses
23 InputMetadata to extract the original 2D shape of the input.
24 """
25 import math
26 -from typing import Dict, List, Optional, Tuple
26 +from typing import List, Optional, Tuple
27
28 import torch
29 from torch import nn
30
31 -from vllm.sequence import SequenceOutputs
31 from vllm.model_executor.input_metadata import InputMetadata
32 from vllm.model_executor.layers.activation import SiluAndMul
33 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -42,6 +41,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 from vllm.model_executor.parallel_utils.tensor_parallel import (
43 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
44 +from vllm.sequence import SamplerOutput
45 from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
46
47 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -290,7 +290,7 @@ class BaiChuanBaseForCausalLM(nn.Module):
290 kv_caches: List[KVCache],
291 input_metadata: InputMetadata,
292 cache_events: Optional[List[torch.cuda.Event]],
293 - ) -> Dict[int, SequenceOutputs]:
293 + ) -> SamplerOutput:
294 hidden_states = self.model(input_ids, positions, kv_caches,
295 input_metadata, cache_events)
296 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/model_executor/models/bloom.py
+3 -3
@@ -21,7 +21,7 @@ The input of the model is flattened to a 1D tensor of tokens. The model uses
21 InputMetadata to extract the original 2D shape of the input.
22 """
23 import math
24 -from typing import Dict, List, Optional, Tuple
24 +from typing import List, Optional, Tuple
25
26 import torch
27 from torch import nn
@@ -37,7 +37,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
37 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38 from vllm.model_executor.parallel_utils.tensor_parallel import (
39 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
40 -from vllm.sequence import SequenceOutputs
40 +from vllm.sequence import SamplerOutput
41
42 KVCache = Tuple[torch.Tensor, torch.Tensor]
43
@@ -264,7 +264,7 @@ class BloomForCausalLM(nn.Module):
264 kv_caches: List[KVCache],
265 input_metadata: InputMetadata,
266 cache_events: Optional[List[torch.cuda.Event]],
267 - ) -> Dict[int, SequenceOutputs]:
267 + ) -> SamplerOutput:
268 hidden_states = self.transformer(input_ids, positions, kv_caches,
269 input_metadata, cache_events)
270 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
vllm/model_executor/models/falcon.py
+3 -3
@@ -19,7 +19,7 @@
19 """PyTorch Falcon model."""
20
21 import math
22 -from typing import Dict, List, Optional, Tuple, Union
22 +from typing import List, Optional, Tuple, Union
23
24 import torch
25 from torch import nn
@@ -38,7 +38,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
38 from vllm.model_executor.parallel_utils.tensor_parallel import (
39 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear,
40 reduce_from_tensor_model_parallel_region)
41 -from vllm.sequence import SequenceOutputs
41 +from vllm.sequence import SamplerOutput
42 from vllm.transformers_utils.configs import RWConfig
43
44 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -397,7 +397,7 @@ class FalconForCausalLM(nn.Module):
397 kv_caches: List[KVCache],
398 input_metadata: InputMetadata,
399 cache_events: Optional[List[torch.cuda.Event]],
400 - ) -> Dict[int, SequenceOutputs]:
400 + ) -> SamplerOutput:
401 hidden_states = self.transformer(
402 input_ids,
403 positions,
vllm/model_executor/models/gpt2.py
+3 -3
@@ -21,7 +21,7 @@
21 The input of the model is flattened to a 1D tensor of tokens. The model uses
22 InputMetadata to extract the original 2D shape of the input.
23 """
24 -from typing import Dict, List, Optional, Tuple
24 +from typing import List, Optional, Tuple
25
26 import torch
27 from torch import nn
@@ -38,7 +38,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
38 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39 from vllm.model_executor.parallel_utils.tensor_parallel import (
40 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
41 -from vllm.sequence import SequenceOutputs
41 +from vllm.sequence import SamplerOutput
42
43 KVCache = Tuple[torch.Tensor, torch.Tensor]
44
@@ -218,7 +218,7 @@ class GPT2LMHeadModel(nn.Module):
218 kv_caches: List[KVCache],
219 input_metadata: InputMetadata,
220 cache_events: Optional[List[torch.cuda.Event]],
221 - ) -> Dict[int, SequenceOutputs]:
221 + ) -> SamplerOutput:
222 hidden_states = self.transformer(input_ids, positions, kv_caches,
223 input_metadata, cache_events)
224 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
vllm/model_executor/models/gpt_bigcode.py
+3 -3
@@ -22,7 +22,7 @@
22 The input of the model is flattened to a 1D tensor of tokens. The model uses
23 InputMetadata to extract the original 2D shape of the input.
24 """
25 -from typing import Dict, List, Optional, Tuple
25 +from typing import List, Optional, Tuple
26
27 import torch
28 from torch import nn
@@ -39,7 +39,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
39 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40 from vllm.model_executor.parallel_utils.tensor_parallel import (
41 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
42 -from vllm.sequence import SequenceOutputs
42 +from vllm.sequence import SamplerOutput
43
44 KVCache = Tuple[torch.Tensor, torch.Tensor]
45
@@ -246,7 +246,7 @@ class GPTBigCodeForCausalLM(nn.Module):
246 kv_caches: List[KVCache],
247 input_metadata: InputMetadata,
248 cache_events: Optional[List[torch.cuda.Event]],
249 - ) -> Dict[int, SequenceOutputs]:
249 + ) -> SamplerOutput:
250 hidden_states = self.transformer(input_ids, positions, kv_caches,
251 input_metadata, cache_events)
252 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
vllm/model_executor/models/gpt_j.py
+3 -3
@@ -20,7 +20,7 @@
20 The input of the model is flattened to a 1D tensor of tokens. The model uses
21 InputMetadata to extract the original 2D shape of the input.
22 """
23 -from typing import Dict, List, Optional, Tuple
23 +from typing import List, Optional, Tuple
24
25 import torch
26 from torch import nn
@@ -36,7 +36,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
36 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37 from vllm.model_executor.parallel_utils.tensor_parallel import (
38 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
39 -from vllm.sequence import SequenceOutputs
39 +from vllm.sequence import SamplerOutput
40
41 KVCache = Tuple[torch.Tensor, torch.Tensor]
42
@@ -203,7 +203,7 @@ class GPTJForCausalLM(nn.Module):
203 kv_caches: List[KVCache],
204 input_metadata: InputMetadata,
205 cache_events: Optional[List[torch.cuda.Event]],
206 - ) -> Dict[int, SequenceOutputs]:
206 + ) -> SamplerOutput:
207 hidden_states = self.transformer(input_ids, positions, kv_caches,
208 input_metadata, cache_events)
209 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/model_executor/models/gpt_neox.py
+3 -3
@@ -20,7 +20,7 @@
20 The input of the model is flattened to a 1D tensor of tokens. The model uses
21 InputMetadata to extract the original 2D shape of the input.
22 """
23 -from typing import Dict, List, Optional, Tuple
23 +from typing import List, Optional, Tuple
24
25 import torch
26 from torch import nn
@@ -36,7 +36,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
36 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37 from vllm.model_executor.parallel_utils.tensor_parallel import (
38 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
39 -from vllm.sequence import SequenceOutputs
39 +from vllm.sequence import SamplerOutput
40
41 KVCache = Tuple[torch.Tensor, torch.Tensor]
42
@@ -215,7 +215,7 @@ class GPTNeoXForCausalLM(nn.Module):
215 kv_caches: List[KVCache],
216 input_metadata: InputMetadata,
217 cache_events: Optional[List[torch.cuda.Event]],
218 - ) -> Dict[int, SequenceOutputs]:
218 + ) -> SamplerOutput:
219 hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
220 input_metadata, cache_events)
221 next_tokens = self.sampler(self.embed_out.weight, hidden_states,
vllm/model_executor/models/internlm.py
+3 -3
@@ -1,5 +1,5 @@
1 # -*- coding: utf-8 -*-
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import List, Optional, Tuple
3
4 import torch
5 from torch import nn
@@ -17,7 +17,7 @@ from vllm.model_executor.parallel_utils.tensor_parallel import (
17 from vllm.model_executor.weight_utils import (
18 hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
19 load_tensor_parallel_weights)
20 -from vllm.sequence import SequenceOutputs
20 +from vllm.sequence import SamplerOutput
21
22 KVCache = Tuple[torch.Tensor, torch.Tensor]
23
@@ -218,7 +218,7 @@ class InternLMForCausalLM(nn.Module):
218 kv_caches: List[KVCache],
219 input_metadata: InputMetadata,
220 cache_events: Optional[List[torch.cuda.Event]],
221 - ) -> Dict[int, SequenceOutputs]:
221 + ) -> SamplerOutput:
222 hidden_states = self.model(input_ids, positions, kv_caches,
223 input_metadata, cache_events)
224 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/model_executor/models/llama.py
+3 -3
@@ -25,7 +25,7 @@
25 The input of the model is flattened to a 1D tensor of tokens. The model uses
26 InputMetadata to extract the original 2D shape of the input.
27 """
28 -from typing import Dict, List, Optional, Tuple
28 +from typing import List, Optional, Tuple
29
30 import torch
31 from torch import nn
@@ -43,7 +43,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
43 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44 from vllm.model_executor.parallel_utils.tensor_parallel import (
45 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
46 -from vllm.sequence import SequenceOutputs
46 +from vllm.sequence import SamplerOutput
47
48 KVCache = Tuple[torch.Tensor, torch.Tensor]
49
@@ -256,7 +256,7 @@ class LlamaForCausalLM(nn.Module):
256 kv_caches: List[KVCache],
257 input_metadata: InputMetadata,
258 cache_events: Optional[List[torch.cuda.Event]],
259 - ) -> Dict[int, SequenceOutputs]:
259 + ) -> SamplerOutput:
260 hidden_states = self.model(input_ids, positions, kv_caches,
261 input_metadata, cache_events)
262 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/model_executor/models/mpt.py
+3 -3
@@ -1,7 +1,7 @@
1 # coding=utf-8
2 # Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
3 import math
4 -from typing import Dict, List, Optional, Tuple
4 +from typing import List, Optional, Tuple
5
6 import torch
7 import torch.nn as nn
@@ -16,7 +16,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
16 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
17 from vllm.model_executor.parallel_utils.tensor_parallel import (
18 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
19 -from vllm.sequence import SequenceOutputs
19 +from vllm.sequence import SamplerOutput
20 from vllm.transformers_utils.configs.mpt import MPTConfig
21
22 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -230,7 +230,7 @@ class MPTForCausalLM(nn.Module):
230 kv_caches: List[KVCache],
231 input_metadata: InputMetadata,
232 cache_events: Optional[List[torch.cuda.Event]],
233 - ) -> Dict[int, SequenceOutputs]:
233 + ) -> SamplerOutput:
234 hidden_states = self.transformer(input_ids, positions, kv_caches,
235 input_metadata, cache_events)
236 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
vllm/model_executor/models/opt.py
+3 -3
@@ -21,7 +21,7 @@
21 The input of the model is flattened to a 1D tensor of tokens. The model uses
22 InputMetadata to extract the original 2D shape of the input.
23 """
24 -from typing import Dict, List, Optional, Tuple
24 +from typing import List, Optional, Tuple
25
26 import torch
27 from torch import nn
@@ -37,7 +37,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
37 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38 from vllm.model_executor.parallel_utils.tensor_parallel import (
39 VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
40 -from vllm.sequence import SequenceOutputs
40 +from vllm.sequence import SamplerOutput
41
42 KVCache = Tuple[torch.Tensor, torch.Tensor]
43
@@ -282,7 +282,7 @@ class OPTForCausalLM(nn.Module):
282 kv_caches: List[KVCache],
283 input_metadata: InputMetadata,
284 cache_events: Optional[List[torch.cuda.Event]],
285 - ) -> Dict[int, SequenceOutputs]:
285 + ) -> SamplerOutput:
286 hidden_states = self.model(input_ids, positions, kv_caches,
287 input_metadata, cache_events)
288 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
vllm/model_executor/models/qwen.py
+3 -3
@@ -8,7 +8,7 @@
8 The input of the model is flattened to a 1D tensor of tokens. The model uses
9 InputMetadata to extract the original 2D shape of the input.
10 """
11 -from typing import Dict, List, Optional, Tuple
11 +from typing import List, Optional, Tuple
12
13 import torch
14 from torch import nn
@@ -32,7 +32,7 @@ from vllm.model_executor.parallel_utils.tensor_parallel import (
32 ColumnParallelLinear,
33 RowParallelLinear,
34 )
35 -from vllm.sequence import SequenceOutputs
35 +from vllm.sequence import SamplerOutput
36 from vllm.transformers_utils.configs.qwen import QWenConfig
37
38 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -235,7 +235,7 @@ class QWenLMHeadModel(nn.Module):
235 kv_caches: List[KVCache],
236 input_metadata: InputMetadata,
237 cache_events: Optional[List[torch.cuda.Event]],
238 - ) -> Dict[int, SequenceOutputs]:
238 + ) -> SamplerOutput:
239 hidden_states = self.transformer(input_ids, positions, kv_caches,
240 input_metadata, cache_events)
241 next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/outputs.py
+6 -4
@@ -75,10 +75,12 @@ class RequestOutput:
75 # Get the top-n sequences.
76 n = seq_group.sampling_params.n
77 seqs = seq_group.get_seqs()
78 - assert n <= len(seqs)
79 - sorted_seqs = sorted(seqs,
80 - key=lambda seq: seq.get_cumulative_logprob(),
81 - reverse=True)
78 + if seq_group.sampling_params.use_beam_search:
79 + sorting_key = lambda seq: seq.get_beam_search_score(
80 + seq_group.sampling_params.length_penalty)
81 + else:
82 + sorting_key = lambda seq: seq.get_cumulative_logprob()
83 + sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
84 top_n_seqs = sorted_seqs[:n]
85
86 # Create the outputs.
vllm/sampling_params.py
+34 -3
@@ -34,6 +34,15 @@ class SamplingParams:
34 top_k: Integer that controls the number of top tokens to consider. Set
35 to -1 to consider all tokens.
36 use_beam_search: Whether to use beam search instead of sampling.
37 + length_penalty: Float that penalizes sequences based on their length.
38 + Used in beam search.
39 + early_stopping: Controls the stopping condition for beam search. It
40 + accepts the following values: `True`, where the generation stops as
41 + soon as there are `best_of` complete candidates; `False`, where an
42 + heuristic is applied and the generation stops when is it very
43 + unlikely to find better candidates; `"never"`, where the beam search
44 + procedure only stops when there cannot be better candidates
45 + (canonical beam search algorithm).
46 stop: List of strings that stop the generation when they are generated.
47 The returned output will not contain the stop strings.
48 ignore_eos: Whether to ignore the EOS token and continue generating
@@ -52,6 +61,8 @@ class SamplingParams:
61 top_p: float = 1.0,
62 top_k: int = -1,
63 use_beam_search: bool = False,
64 + length_penalty: float = 1.0,
65 + early_stopping: Union[bool, str] = False,
66 stop: Union[None, str, List[str]] = None,
67 ignore_eos: bool = False,
68 max_tokens: int = 16,
@@ -65,6 +76,8 @@ class SamplingParams:
76 self.top_p = top_p
77 self.top_k = top_k
78 self.use_beam_search = use_beam_search
79 + self.length_penalty = length_penalty
80 + self.early_stopping = early_stopping
81 if stop is None:
82 self.stop = []
83 elif isinstance(stop, str):
@@ -78,9 +91,11 @@ class SamplingParams:
91 self._verify_args()
92 if self.use_beam_search:
93 self._verify_beam_search()
81 - elif self.temperature < _SAMPLING_EPS:
82 - # Zero temperature means greedy sampling.
83 - self._verify_greedy_sampling()
94 + else:
95 + self._verify_non_beam_search()
96 + if self.temperature < _SAMPLING_EPS:
97 + # Zero temperature means greedy sampling.
98 + self._verify_greedy_sampling()
99
100 def _verify_args(self) -> None:
101 if self.n < 1:
@@ -119,6 +134,20 @@ class SamplingParams:
134 raise ValueError("top_p must be 1 when using beam search.")
135 if self.top_k != -1:
136 raise ValueError("top_k must be -1 when using beam search.")
137 + if self.early_stopping not in [True, False, "never"]:
138 + raise ValueError(
139 + f"early_stopping must be True, False, or 'never', "
140 + f"got {self.early_stopping}.")
141 +
142 + def _verify_non_beam_search(self) -> None:
143 + if self.early_stopping is not False:
144 + raise ValueError("early_stopping is not effective and must be "
145 + "False when not using beam search.")
146 + if (self.length_penalty < 1.0 - _SAMPLING_EPS
147 + or self.length_penalty > 1.0 + _SAMPLING_EPS):
148 + raise ValueError(
149 + "length_penalty is not effective and must be the "
150 + "default value of 1.0 when not using beam search.")
151
152 def _verify_greedy_sampling(self) -> None:
153 if self.best_of > 1:
@@ -138,6 +167,8 @@ class SamplingParams:
167 f"top_p={self.top_p}, "
168 f"top_k={self.top_k}, "
169 f"use_beam_search={self.use_beam_search}, "
170 + f"length_penalty={self.length_penalty}, "
171 + f"early_stopping={self.early_stopping}, "
172 f"stop={self.stop}, "
173 f"ignore_eos={self.ignore_eos}, "
174 f"max_tokens={self.max_tokens}, "
vllm/sequence.py
+77 -22
@@ -69,6 +69,9 @@ class SequenceData:
69 def get_len(self) -> int:
70 return len(self.output_token_ids) + len(self.prompt_token_ids)
71
72 + def get_prompt_len(self) -> int:
73 + return len(self.prompt_token_ids)
74 +
75 def get_output_len(self) -> int:
76 return len(self.output_token_ids)
77
@@ -155,6 +158,9 @@ class Sequence:
158 def get_len(self) -> int:
159 return self.data.get_len()
160
161 + def get_prompt_len(self) -> int:
162 + return self.data.get_prompt_len()
163 +
164 def get_output_len(self) -> int:
165 return self.data.get_output_len()
166
@@ -170,14 +176,32 @@ class Sequence:
176 def get_cumulative_logprob(self) -> float:
177 return self.data.cumulative_logprob
178
179 + def get_beam_search_score(self,
180 + length_penalty: float = 0.0,
181 + seq_len: Optional[int] = None,
182 + eos_token_id: Optional[int] = None) -> float:
183 + """Calculate the beam search score with length penalty.
184 +
185 + Adapted from
186 +
187 + https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
188 + """
189 + if seq_len is None:
190 + seq_len = self.get_len()
191 + # Note: HF implementation does not count the EOS token
192 + # towards the length, we align with that here for testing.
193 + if (eos_token_id is not None
194 + and self.get_last_token_id() == eos_token_id):
195 + seq_len -= 1
196 + return self.get_cumulative_logprob() / (seq_len**length_penalty)
197 +
198 def is_finished(self) -> bool:
199 return SequenceStatus.is_finished(self.status)
200
176 - def fork(self, child_seq: "Sequence") -> None:
177 - child_seq.logical_token_blocks = copy.deepcopy(
178 - self.logical_token_blocks)
179 - child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
180 - child_seq.data = copy.deepcopy(self.data)
201 + def fork(self, new_seq_id: int) -> "Sequence":
202 + new_seq = copy.deepcopy(self)
203 + new_seq.seq_id = new_seq_id
204 + return new_seq
205
206 def __repr__(self) -> str:
207 return (f"Sequence(seq_id={self.seq_id}, "
@@ -203,35 +227,66 @@ class SequenceGroup:
227 arrival_time: float,
228 ) -> None:
229 self.request_id = request_id
206 - self.seqs = seqs
230 + self.seqs_dict = {seq.seq_id: seq for seq in seqs}
231 self.sampling_params = sampling_params
232 self.arrival_time = arrival_time
233
234 + def get_max_num_running_seqs(self) -> int:
235 + """The maximum number of sequences running in parallel in the remaining
236 + lifetime of the request."""
237 + if self.sampling_params.use_beam_search:
238 + # For beam search, maximally there will always be `best_of` beam
239 + # candidates running in the future.
240 + return self.sampling_params.best_of
241 + else:
242 + if self.sampling_params.best_of > self.num_seqs():
243 + # At prompt stage, the sequence group is not yet filled up
244 + # and only have one sequence running. However, in the
245 + # generation stage, we will have `best_of` sequences running.
246 + return self.sampling_params.best_of
247 + # At sampling stages, return the number of actual sequences
248 + # running.
249 + return self.num_seqs(status=SequenceStatus.RUNNING)
250 +
251 def get_seqs(
252 self,
253 status: Optional[SequenceStatus] = None,
254 ) -> List[Sequence]:
255 if status is None:
215 - return self.seqs
256 + return list(self.seqs_dict.values())
257 else:
217 - return [seq for seq in self.seqs if seq.status == status]
258 + return [
259 + seq for seq in self.seqs_dict.values() if seq.status == status
260 + ]
261 +
262 + def get_finished_seqs(self) -> List[Sequence]:
263 + return [seq for seq in self.seqs_dict.values() if seq.is_finished()]
264
265 def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
266 return len(self.get_seqs(status))
267
268 def find(self, seq_id: int) -> Sequence:
223 - for seq in self.seqs:
224 - if seq.seq_id == seq_id:
225 - return seq
226 - raise ValueError(f"Sequence {seq_id} not found.")
269 + if seq_id not in self.seqs_dict:
270 + raise ValueError(f"Sequence {seq_id} not found.")
271 + return self.seqs_dict[seq_id]
272 +
273 + def add(self, seq: Sequence) -> None:
274 + if seq.seq_id in self.seqs_dict:
275 + raise ValueError(f"Sequence {seq.seq_id} already exists.")
276 + self.seqs_dict[seq.seq_id] = seq
277 +
278 + def remove(self, seq_id: int) -> None:
279 + if seq_id not in self.seqs_dict:
280 + raise ValueError(f"Sequence {seq_id} not found.")
281 + del self.seqs_dict[seq_id]
282
283 def is_finished(self) -> bool:
229 - return all(seq.is_finished() for seq in self.seqs)
284 + return all(seq.is_finished() for seq in self.get_seqs())
285
286 def __repr__(self) -> str:
287 return (f"SequenceGroup(request_id={self.request_id}, "
288 f"sampling_params={self.sampling_params}, "
234 - f"num_seqs={len(self.seqs)})")
289 + f"num_seqs={len(self.seqs_dict)})")
290
291
292 class SequenceGroupMetadata:
@@ -266,7 +321,6 @@ class SequenceOutputs:
321 """The model output associated with a sequence.
322
323 Args:
269 - seq_id: The ID of the sequence.
324 parent_seq_id: The ID of the parent sequence (for forking in beam
325 search).
326 output_token: The output token ID.
@@ -276,26 +330,27 @@ class SequenceOutputs:
330
331 def __init__(
332 self,
279 - seq_id: int,
333 parent_seq_id: int,
334 output_token: int,
335 logprobs: Dict[int, float],
336 ) -> None:
284 - self.seq_id = seq_id
337 self.parent_seq_id = parent_seq_id
338 self.output_token = output_token
339 self.logprobs = logprobs
340
341 def __repr__(self) -> str:
290 - return (f"SequenceOutputs(seq_id={self.seq_id}, "
291 - f"parent_seq_id={self.parent_seq_id}, "
342 + return (f"SequenceOutputs(parent_seq_id={self.parent_seq_id}, "
343 f"output_token={self.output_token}), "
344 f"logprobs={self.logprobs}")
345
346 def __eq__(self, other: object) -> bool:
347 if not isinstance(other, SequenceOutputs):
297 - return NotImplemented
298 - return (self.seq_id == other.seq_id
299 - and self.parent_seq_id == other.parent_seq_id
348 + return NotImplementedError()
349 + return (self.parent_seq_id == other.parent_seq_id
350 and self.output_token == other.output_token
351 and self.logprobs == other.logprobs)
352 +
353 +
354 +# For each sequence group, we generate a list of SequenceOutputs object,
355 +# each of which contains one possible candidate for the next token.
356 +SamplerOutput = List[List[SequenceOutputs]]
vllm/worker/worker.py
+2 -2
@@ -11,7 +11,7 @@ from vllm.model_executor import get_model, InputMetadata, set_random_seed
11 from vllm.model_executor.parallel_utils.parallel_state import (
12 initialize_model_parallel)
13 from vllm.sampling_params import SamplingParams
14 -from vllm.sequence import SequenceData, SequenceGroupMetadata, SequenceOutputs
14 +from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15 from vllm.worker.cache_engine import CacheEngine
16 from vllm.utils import get_gpu_memory
17
@@ -260,7 +260,7 @@ class Worker:
260 blocks_to_swap_in: Dict[int, int],
261 blocks_to_swap_out: Dict[int, int],
262 blocks_to_copy: Dict[int, List[int]],
263 - ) -> Dict[int, SequenceOutputs]:
263 + ) -> SamplerOutput:
264 # Issue cache operations.
265 issued_cache_op = False
266 if blocks_to_swap_in: