A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Align vLLM's beam search implementation with HF generate (#857)
Zhuohan Li committed
Sep 4, 2023 at 17:29 UTC
002800f08142574aeafed7c33982503d28d53cdc
24 files changed
+599
-263
docs/source/models/adding_model.rst
+1
-1
@@ -59,7 +59,7 @@ Next, you need to rewrite the :code:`forward` methods of your model by following
59
+ kv_caches: List[KVCache],
60
+ input_metadata: InputMetadata,
61
+ cache_events: Optional[List[torch.cuda.Event]],
62
- +) -> Dict[int, SequenceOutputs]:
62
+ +) -> SamplerOutput:
63
64
3. Update the code by considering that :code:`input_ids` and :code:`positions` are now flattened tensors.
65
4. Replace the attention operation with either :code:`GPTPagedAttention` or :code:`GPTNeoXPagedAttention`, depending on the model's architecture.
tests/conftest.py
+56
-10
@@ -67,8 +67,8 @@ class HfRunner:
67
output_ids,
68
skip_special_tokens=True,
69
clean_up_tokenization_spaces=False,
70
- )[0]
71
- output_ids = output_ids[0].cpu().tolist()
70
+ )
71
+ output_ids = output_ids.cpu().tolist()
72
outputs.append((output_ids, output_str))
73
return outputs
74
@@ -77,8 +77,34 @@ class HfRunner:
77
prompts: List[str],
78
max_tokens: int,
79
) -> List[Tuple[List[int], str]]:
80
- return self.generate(prompts, do_sample=False,
81
- max_new_tokens=max_tokens)
80
+ outputs = self.generate(prompts,
81
+ do_sample=False,
82
+ max_new_tokens=max_tokens)
83
+ for i in range(len(outputs)):
84
+ output_ids, output_str = outputs[i]
85
+ outputs[i] = (output_ids[0], output_str[0])
86
+ return outputs
87
+
88
+ def generate_beam_search(
89
+ self,
90
+ prompts: List[str],
91
+ beam_width: int,
92
+ max_tokens: int,
93
+ ) -> List[Tuple[List[int], str]]:
94
+ outputs = self.generate(prompts,
95
+ do_sample=False,
96
+ max_new_tokens=max_tokens,
97
+ num_beams=beam_width,
98
+ num_return_sequences=beam_width)
99
+ for i in range(len(outputs)):
100
+ output_ids, output_str = outputs[i]
101
+ for j in range(len(output_ids)):
102
+ output_ids[j] = [
103
+ x for x in output_ids[j]
104
+ if x != self.tokenizer.pad_token_id
105
+ ]
106
+ outputs[i] = (output_ids, output_str)
107
+ return outputs
108
109
110
@pytest.fixture
@@ -107,15 +133,20 @@ class VllmRunner:
133
prompts: List[str],
134
sampling_params: SamplingParams,
135
) -> List[Tuple[List[int], str]]:
110
- req_outputs = self.model.generate(
111
- prompts, sampling_params=sampling_params)
136
+ req_outputs = self.model.generate(prompts,
137
+ sampling_params=sampling_params)
138
outputs = []
139
for req_output in req_outputs:
140
prompt_str = req_output.prompt
141
prompt_ids = req_output.prompt_token_ids
116
- output_str = req_output.outputs[0].text
117
- output_ids = req_output.outputs[0].token_ids
118
- outputs.append((prompt_ids + output_ids, prompt_str + output_str))
142
+ req_sample_output_ids = []
143
+ req_sample_output_strs = []
144
+ for sample in req_output.outputs:
145
+ output_str = sample.text
146
+ output_ids = sample.token_ids
147
+ req_sample_output_ids.append(prompt_ids + output_ids)
148
+ req_sample_output_strs.append(prompt_str + output_str)
149
+ outputs.append((req_sample_output_ids, req_sample_output_strs))
150
return outputs
151
152
def generate_greedy(
@@ -124,7 +155,22 @@ class VllmRunner:
155
max_tokens: int,
156
) -> List[Tuple[List[int], str]]:
157
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
127
- return self.generate(prompts, greedy_params)
158
+ outputs = self.generate(prompts, greedy_params)
159
+ return [(output_ids[0], output_str[0]) for output_ids, output_str in
160
+ outputs]
161
+
162
+ def generate_beam_search(
163
+ self,
164
+ prompts: List[str],
165
+ beam_width: int,
166
+ max_tokens: int,
167
+ ) -> List[Tuple[List[int], str]]:
168
+ beam_search_params = SamplingParams(n=beam_width,
169
+ use_beam_search=True,
170
+ temperature=0.0,
171
+ max_tokens=max_tokens)
172
+ outputs = self.generate(prompts, beam_search_params)
173
+ return outputs
174
175
176
@pytest.fixture
tests/samplers/test_beam_search.py
new
+46
@@ -0,0 +1,46 @@
1
+"""Compare the outputs of HF and vLLM when using beam search.
2
+
3
+Run `pytest tests/samplers/test_beam_search.py --forked`.
4
+"""
5
+import pytest
6
+
7
+# FIXME(zhuohan): The test can not pass if we:
8
+# 1. Increase max_tokens to 256.
9
+# 2. Increase beam_width to 8.
10
+# 3. Use the model "huggyllama/llama-7b".
11
+MAX_TOKENS = [128]
12
+BEAM_WIDTHS = [4]
13
+MODELS = ["facebook/opt-125m"]
14
+
15
+
16
+@pytest.mark.parametrize("model", MODELS)
17
+@pytest.mark.parametrize("dtype", ["half"])
18
+@pytest.mark.parametrize("max_tokens", MAX_TOKENS)
19
+@pytest.mark.parametrize("beam_width", BEAM_WIDTHS)
20
+def test_beam_search_single_input(
21
+ hf_runner,
22
+ vllm_runner,
23
+ example_prompts,
24
+ model: str,
25
+ dtype: str,
26
+ max_tokens: int,
27
+ beam_width: int,
28
+) -> None:
29
+ hf_model = hf_runner(model, dtype=dtype)
30
+ hf_outputs = hf_model.generate_beam_search(example_prompts, beam_width,
31
+ max_tokens)
32
+ del hf_model
33
+
34
+ vllm_model = vllm_runner(model, dtype=dtype)
35
+ vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
36
+ max_tokens)
37
+ del vllm_model
38
+
39
+ for i in range(len(example_prompts)):
40
+ hf_output_ids, _ = hf_outputs[i]
41
+ vllm_output_ids, _ = vllm_outputs[i]
42
+ assert len(hf_output_ids) == len(vllm_output_ids)
43
+ for j in range(len(hf_output_ids)):
44
+ assert hf_output_ids[j] == vllm_output_ids[j], (
45
+ f"Test{i} output{j}:\nHF: {hf_output_ids}\n"
46
+ f"vLLM: {vllm_output_ids}")
vllm/core/block_manager.py
+2
-6
@@ -172,9 +172,7 @@ class BlockSpaceManager:
172
def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
173
# CPU block -> GPU block.
174
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
175
- for seq in seq_group.get_seqs():
176
- if seq.is_finished():
177
- continue
175
+ for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
176
new_block_table: BlockTable = []
177
block_table = self.block_tables[seq.seq_id]
178
@@ -203,9 +201,7 @@ class BlockSpaceManager:
201
def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
202
# GPU block -> CPU block.
203
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
206
- for seq in seq_group.get_seqs():
207
- if seq.is_finished():
208
- continue
204
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
205
new_block_table: BlockTable = []
206
block_table = self.block_tables[seq.seq_id]
207
vllm/core/scheduler.py
+45
-69
@@ -7,8 +7,7 @@ from vllm.core.block_manager import BlockSpaceManager
7
from vllm.core.policy import PolicyFactory
8
from vllm.logger import init_logger
9
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
10
- SequenceGroupMetadata, SequenceOutputs,
11
- SequenceStatus)
10
+ SequenceGroupMetadata, SequenceStatus)
11
12
logger = init_logger(__name__)
13
@@ -76,6 +75,7 @@ class Scheduler:
75
num_cpu_blocks=self.cache_config.num_cpu_blocks,
76
)
77
78
+ # TODO(zhuohan): Use deque instead of list for better performance.
79
# Sequence groups in the WAITING state.
80
self.waiting: List[SequenceGroup] = []
81
# Sequence groups in the RUNNING state.
@@ -96,10 +96,11 @@ class Scheduler:
96
if seq_group.request_id in request_ids:
97
# Remove the sequence group from the state queue.
98
state_queue.remove(seq_group)
99
- for seq in seq_group.seqs:
99
+ for seq in seq_group.get_seqs():
100
if seq.is_finished():
101
continue
102
- self.free_seq(seq, SequenceStatus.FINISHED_ABORTED)
102
+ seq.status = SequenceStatus.FINISHED_ABORTED
103
+ self.free_seq(seq)
104
request_ids.remove(seq_group.request_id)
105
if not request_ids:
106
return
@@ -123,6 +124,10 @@ class Scheduler:
124
if not self.swapped:
125
ignored_seq_groups: List[SequenceGroup] = []
126
scheduled: List[SequenceGroup] = []
127
+ # The total number of sequences on the fly, including the
128
+ # requests in the generation phase.
129
+ num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
130
+ for seq_group in self.running)
131
num_batched_tokens = 0
132
# Optimization: We do not sort the waiting queue since the preempted
133
# sequence groups are added to the front and the new sequence groups
@@ -130,6 +135,9 @@ class Scheduler:
135
while self.waiting:
136
seq_group = self.waiting[0]
137
138
+ assert seq_group.num_seqs() == 1, (
139
+ "Waiting sequence group should have only one prompt "
140
+ "sequence.")
141
num_prompt_tokens = seq_group.get_seqs()[0].get_len()
142
if num_prompt_tokens > self.prompt_limit:
143
logger.warning(
@@ -152,11 +160,7 @@ class Scheduler:
160
161
# The total number of sequences in the RUNNING state should not
162
# exceed the maximum number of sequences.
155
- num_new_seqs = seq_group.num_seqs(
156
- status=SequenceStatus.WAITING)
157
- num_curr_seqs = sum(
158
- seq_group.num_seqs(status=SequenceStatus.RUNNING)
159
- for seq_group in self.running)
163
+ num_new_seqs = seq_group.get_max_num_running_seqs()
164
if (num_curr_seqs + num_new_seqs >
165
self.scheduler_config.max_num_seqs):
166
break
@@ -165,6 +169,7 @@ class Scheduler:
169
self._allocate(seq_group)
170
self.running.append(seq_group)
171
num_batched_tokens += num_prompt_tokens
172
+ num_curr_seqs += num_new_seqs
173
scheduled.append(seq_group)
174
175
if scheduled:
@@ -210,30 +215,32 @@ class Scheduler:
215
216
# Swap in the sequence groups in the SWAPPED state if possible.
217
self.swapped = self.policy.sort_by_priority(now, self.swapped)
213
- while self.swapped and not blocks_to_swap_out:
214
- seq_group = self.swapped[0]
215
- # If the sequence group has been preempted in this step, stop.
216
- if seq_group in preempted:
217
- break
218
- # If the sequence group cannot be swapped in, stop.
219
- if not self.block_manager.can_swap_in(seq_group):
220
- break
221
-
222
- # The total number of sequences in the RUNNING state should not
223
- # exceed the maximum number of sequences.
224
- num_new_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
225
- num_curr_seqs = sum(
226
- seq_group.num_seqs(status=SequenceStatus.RUNNING)
227
- for seq_group in self.running)
228
- if (num_curr_seqs + num_new_seqs >
229
- self.scheduler_config.max_num_seqs):
230
- break
231
-
232
- seq_group = self.swapped.pop(0)
233
- self._swap_in(seq_group, blocks_to_swap_in)
234
- self._append_slot(seq_group, blocks_to_copy)
235
- self.running.append(seq_group)
218
+ if not preempted:
219
+ num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
220
+ for seq_group in self.running)
221
+
222
+ while self.swapped:
223
+ seq_group = self.swapped[0]
224
+ # If the sequence group cannot be swapped in, stop.
225
+ if not self.block_manager.can_swap_in(seq_group):
226
+ break
227
228
+ # The total number of sequences in the RUNNING state should not
229
+ # exceed the maximum number of sequences.
230
+ num_new_seqs = seq_group.get_max_num_running_seqs()
231
+ if (num_curr_seqs + num_new_seqs >
232
+ self.scheduler_config.max_num_seqs):
233
+ break
234
+
235
+ seq_group = self.swapped.pop(0)
236
+ self._swap_in(seq_group, blocks_to_swap_in)
237
+ self._append_slot(seq_group, blocks_to_copy)
238
+ num_curr_seqs += num_new_seqs
239
+ self.running.append(seq_group)
240
+
241
+ # Each sequence in the generation phase only takes one token slot.
242
+ # Therefore, the number of batched tokens is equal to the number of
243
+ # sequences in the RUNNING state.
244
num_batched_tokens = sum(
245
seq_group.num_seqs(status=SequenceStatus.RUNNING)
246
for seq_group in self.running)
@@ -275,40 +282,10 @@ class Scheduler:
282
seq_group_metadata_list.append(seq_group_metadata)
283
return seq_group_metadata_list, scheduler_outputs
284
278
- def update(
279
- self,
280
- seq_outputs: Dict[int, SequenceOutputs],
281
- ) -> List[SequenceGroup]:
282
- scheduled: List[SequenceGroup] = []
283
- for seq_group in self.running:
284
- for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
285
- if seq.seq_id in seq_outputs:
286
- scheduled.append(seq_group)
287
- break
288
-
289
- # Update the scheduled sequences and free blocks.
290
- for seq_group in scheduled:
291
- # Process beam search results before processing the new tokens.
292
- for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
293
- output = seq_outputs[seq.seq_id]
294
- if seq.seq_id != output.parent_seq_id:
295
- # The sequence is a fork of the parent sequence (beam
296
- # search). Free the current sequence.
297
- self.block_manager.free(seq)
298
- # Fork the parent sequence.
299
- parent_seq = seq_group.find(output.parent_seq_id)
300
- parent_seq.fork(seq)
301
- self.block_manager.fork(parent_seq, seq)
302
-
303
- # Process the new tokens.
304
- for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
305
- # Append a new token to the sequence.
306
- output = seq_outputs[seq.seq_id]
307
- seq.append_token_id(output.output_token, output.logprobs)
308
- return scheduled
285
+ def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None:
286
+ self.block_manager.fork(parent_seq, child_seq)
287
310
- def free_seq(self, seq: Sequence, finish_status: SequenceStatus) -> None:
311
- seq.status = finish_status
288
+ def free_seq(self, seq: Sequence) -> None:
289
self.block_manager.free(seq)
290
291
def free_finished_seq_groups(self) -> None:
@@ -345,8 +322,8 @@ class Scheduler:
322
# If preemption mode is not specified, we determine the mode as follows:
323
# We use recomputation by default since it incurs lower overhead than
324
# swapping. However, when the sequence group has multiple sequences
348
- # (e.g., beam search), recomputation is not supported. In such a case,
349
- # we use swapping instead.
325
+ # (e.g., beam search), recomputation is not currently supported. In
326
+ # such a case, we use swapping instead.
327
# FIXME(woosuk): This makes our scheduling policy a bit bizarre.
328
# As swapped sequences are prioritized over waiting sequences,
329
# sequence groups with multiple sequences are implicitly prioritized
@@ -354,8 +331,7 @@ class Scheduler:
331
# TODO(woosuk): Support recomputation for sequence groups with multiple
332
# sequences. This may require a more sophisticated CUDA kernel.
333
if preemption_mode is None:
357
- seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
358
- if len(seqs) == 1:
334
+ if seq_group.get_max_num_running_seqs() == 1:
335
preemption_mode = PreemptionMode.RECOMPUTE
336
else:
337
preemption_mode = PreemptionMode.SWAP
vllm/engine/llm_engine.py
+260
-65
@@ -11,7 +11,8 @@ from vllm.engine.ray_utils import RayWorker, initialize_cluster, ray
11
from vllm.logger import init_logger
12
from vllm.outputs import RequestOutput
13
from vllm.sampling_params import SamplingParams
14
-from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupMetadata,
14
+from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
15
+ SequenceGroupMetadata, SequenceOutputs,
16
SequenceStatus)
17
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
18
get_tokenizer)
@@ -258,14 +259,11 @@ class LLMEngine:
259
260
# Create the sequences.
261
block_size = self.cache_config.block_size
261
- seqs: List[Sequence] = []
262
- for _ in range(sampling_params.best_of):
263
- seq_id = next(self.seq_counter)
264
- seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
265
- seqs.append(seq)
262
+ seq_id = next(self.seq_counter)
263
+ seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
264
265
# Create the sequence group.
268
- seq_group = SequenceGroup(request_id, seqs, sampling_params,
266
+ seq_group = SequenceGroup(request_id, [seq], sampling_params,
267
arrival_time)
268
269
# Add the sequence group to the scheduler.
@@ -303,22 +301,230 @@ class LLMEngine:
301
]
302
return seq_group_metadata_list, scheduler_outputs, None
303
306
- def _process_worker_outputs(
307
- self, output,
304
+ def _check_beam_search_early_stopping(
305
+ self,
306
+ early_stopping: Union[bool, str],
307
+ sampling_params: SamplingParams,
308
+ best_running_seq: Sequence,
309
+ current_worst_seq: Sequence,
310
+ ) -> bool:
311
+ assert sampling_params.use_beam_search
312
+ length_penalty = sampling_params.length_penalty
313
+ if early_stopping is True:
314
+ return True
315
+
316
+ current_worst_score = (current_worst_seq.get_beam_search_score(
317
+ length_penalty=length_penalty,
318
+ eos_token_id=self.tokenizer.eos_token_id))
319
+ if early_stopping is False:
320
+ highest_attainable_score = (best_running_seq.get_beam_search_score(
321
+ length_penalty=length_penalty,
322
+ eos_token_id=self.tokenizer.eos_token_id))
323
+ else:
324
+ assert early_stopping == "never"
325
+ if length_penalty > 0.0:
326
+ # If length_penalty > 0.0, beam search will prefer longer
327
+ # sequences. The highest attainable score calculation is
328
+ # based on the longest possible sequence length in this case.
329
+ max_possible_length = max(
330
+ best_running_seq.get_prompt_len() +
331
+ sampling_params.max_tokens,
332
+ self.scheduler_config.max_model_len)
333
+ highest_attainable_score = (
334
+ best_running_seq.get_beam_search_score(
335
+ length_penalty=length_penalty,
336
+ eos_token_id=self.tokenizer.eos_token_id,
337
+ seq_len=max_possible_length))
338
+ else:
339
+ # Otherwise, beam search will prefer shorter sequences. The
340
+ # highest attainable score calculation is based on the current
341
+ # sequence length.
342
+ highest_attainable_score = (
343
+ best_running_seq.get_beam_search_score(
344
+ length_penalty=length_penalty,
345
+ eos_token_id=self.tokenizer.eos_token_id))
346
+ return current_worst_score >= highest_attainable_score
347
+
348
+ def _process_sequence_group_samples(
349
+ self, seq_group: SequenceGroup,
350
+ samples: List[SequenceOutputs]) -> None:
351
+ parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
352
+ existing_finished_seqs = seq_group.get_finished_seqs()
353
+ parent_child_dict = {
354
+ parent_seq.seq_id: []
355
+ for parent_seq in parent_seqs
356
+ }
357
+ for sample in samples:
358
+ parent_child_dict[sample.parent_seq_id].append(sample)
359
+ # List of (child, parent)
360
+ child_seqs: List[Tuple[Sequence, Sequence]] = []
361
+
362
+ # Process the child samples for each parent sequence
363
+ for parent in parent_seqs:
364
+ child_samples: List[SequenceOutputs] = parent_child_dict[
365
+ parent.seq_id]
366
+ if len(child_samples) == 0:
367
+ # This parent sequence has no children samples. Remove
368
+ # the parent sequence from the sequence group since it will
369
+ # not be used in the future iterations.
370
+ parent.status = SequenceStatus.FINISHED_ABORTED
371
+ seq_group.remove(parent.seq_id)
372
+ self.scheduler.free_seq(parent)
373
+ continue
374
+ # Fork the parent sequence if there are multiple child samples.
375
+ for child_sample in child_samples[:-1]:
376
+ new_child_seq_id = next(self.seq_counter)
377
+ child = parent.fork(new_child_seq_id)
378
+ child.append_token_id(child_sample.output_token,
379
+ child_sample.logprobs)
380
+ child_seqs.append((child, parent))
381
+ # Continue the parent sequence for the last child sample.
382
+ # We reuse the parent sequence here to reduce redundant memory
383
+ # copies, especially when using non-beam search sampling methods.
384
+ last_child_sample = child_samples[-1]
385
+ parent.append_token_id(last_child_sample.output_token,
386
+ last_child_sample.logprobs)
387
+ child_seqs.append((parent, parent))
388
+
389
+ for seq, _ in child_seqs:
390
+ self._decode_sequence(seq)
391
+ self._check_stop(seq, seq_group.sampling_params)
392
+
393
+ # Non-beam search case
394
+ if not seq_group.sampling_params.use_beam_search:
395
+ # For newly created child sequences, add them to the sequence group
396
+ # and fork them in block manager if they are not finished.
397
+ for seq, parent in child_seqs:
398
+ if seq is not parent:
399
+ seq_group.add(seq)
400
+ if not seq.is_finished():
401
+ self.scheduler.fork_seq(parent, seq)
402
+
403
+ # Free the finished and selected parent sequences' memory in block
404
+ # manager. Keep them in the sequence group as candidate output.
405
+ # NOTE: we need to fork the new sequences before freeing the
406
+ # old sequences.
407
+ for seq, parent in child_seqs:
408
+ if seq is parent and seq.is_finished():
409
+ self.scheduler.free_seq(seq)
410
+ return
411
+
412
+ # Beam search case
413
+ # Select the child sequences to keep in the sequence group.
414
+ selected_child_seqs = []
415
+ unselected_child_seqs = []
416
+ beam_width = seq_group.sampling_params.best_of
417
+ length_penalty = seq_group.sampling_params.length_penalty
418
+
419
+ # Select the newly finished sequences with the highest scores
420
+ # to replace existing finished sequences.
421
+ # Tuple of (seq, parent, is_new)
422
+ existing_finished_seqs = [(seq, None, False)
423
+ for seq in existing_finished_seqs]
424
+ new_finished_seqs = [(seq, parent, True) for seq, parent in child_seqs
425
+ if seq.is_finished()]
426
+ all_finished_seqs = existing_finished_seqs + new_finished_seqs
427
+ # Sort the finished sequences by their scores.
428
+ all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
429
+ length_penalty=length_penalty,
430
+ eos_token_id=self.tokenizer.eos_token_id),
431
+ reverse=True)
432
+ for seq, parent, is_new in all_finished_seqs[:beam_width]:
433
+ if is_new:
434
+ # A newly generated child sequence finishes and has a high
435
+ # score, so we will add it into the sequence group.
436
+ selected_child_seqs.append((seq, parent))
437
+ for seq, parent, is_new in all_finished_seqs[beam_width:]:
438
+ if is_new:
439
+ # A newly generated child sequence finishes but has a low
440
+ # score, so we will not add it into the sequence group.
441
+ # Additionally, if this sequence is a continuation of a
442
+ # parent sequence, we will need remove the parent sequence
443
+ # from the sequence group.
444
+ unselected_child_seqs.append((seq, parent))
445
+ else:
446
+ # An existing finished sequence has a low score, so we will
447
+ # remove it from the sequence group.
448
+ seq_group.remove(seq.seq_id)
449
+
450
+ # select the top beam_width sequences from the running
451
+ # sequences for the next iteration to continue the beam
452
+ # search.
453
+ running_child_seqs = [(seq, parent) for seq, parent in child_seqs
454
+ if not seq.is_finished()]
455
+ # Sort the running sequences by their scores.
456
+ running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
457
+ length_penalty=length_penalty,
458
+ eos_token_id=self.tokenizer.eos_token_id),
459
+ reverse=True)
460
+
461
+ # Check if we can stop the beam search.
462
+ if len(running_child_seqs) == 0:
463
+ # No running sequences, stop the beam search.
464
+ stop_beam_search = True
465
+ elif len(all_finished_seqs) < beam_width:
466
+ # Not enough finished sequences, continue the beam search.
467
+ stop_beam_search = False
468
+ else:
469
+ # Check the early stopping criteria
470
+ best_running_seq = running_child_seqs[0][0]
471
+ current_worst_seq = all_finished_seqs[beam_width - 1][0]
472
+ stop_beam_search = self._check_beam_search_early_stopping(
473
+ seq_group.sampling_params.early_stopping,
474
+ seq_group.sampling_params, best_running_seq, current_worst_seq)
475
+
476
+ if stop_beam_search:
477
+ # Stop the beam search and remove all the running sequences from
478
+ # the sequence group.
479
+ unselected_child_seqs.extend(running_child_seqs)
480
+ else:
481
+ # Continue the beam search and select the top beam_width sequences
482
+ # to continue the beam search.
483
+ selected_child_seqs.extend(running_child_seqs[:beam_width])
484
+ # The remaining running sequences will not be used in the next
485
+ # iteration. Again, if these sequences are continuations of
486
+ # parent sequences, we will need to remove the parent sequences
487
+ # from the sequence group.
488
+ unselected_child_seqs.extend(running_child_seqs[beam_width:])
489
+
490
+ # For newly created child sequences, add them to the sequence group
491
+ # and fork them in block manager if they are not finished.
492
+ for seq, parent in selected_child_seqs:
493
+ if seq is not parent:
494
+ seq_group.add(seq)
495
+ if not seq.is_finished():
496
+ self.scheduler.fork_seq(parent, seq)
497
+
498
+ # Free the finished and selected parent sequences' memory in block
499
+ # manager. Keep them in the sequence group as candidate output.
500
+ for seq, parent in selected_child_seqs:
501
+ if seq is parent and seq.is_finished():
502
+ self.scheduler.free_seq(seq)
503
+
504
+ # Remove the unselected parent sequences from the sequence group and
505
+ # free their memory in block manager.
506
+ for seq, parent in unselected_child_seqs:
507
+ if seq is parent:
508
+ # Remove the parent sequence if it is not selected for next
509
+ # iteration
510
+ seq_group.remove(seq.seq_id)
511
+ self.scheduler.free_seq(seq)
512
+
513
+ def _process_model_outputs(
514
+ self, output: SamplerOutput,
515
scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]:
309
- # Update the scheduler with the model outputs.
310
- seq_groups = self.scheduler.update(output)
516
+ # Update the scheduled sequence groups with the model outputs.
517
+ scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
518
+ for seq_group, samples in zip(scheduled_seq_groups, output):
519
+ self._process_sequence_group_samples(seq_group, samples)
520
312
- # Decode the sequences.
313
- self._decode_sequences(seq_groups)
314
- # Stop the sequences that meet the stopping criteria.
315
- self._stop_sequences(seq_groups)
521
# Free the finished sequence groups.
522
self.scheduler.free_finished_seq_groups()
523
524
# Create the outputs.
525
request_outputs: List[RequestOutput] = []
321
- for seq_group in seq_groups + scheduler_outputs.ignored_seq_groups:
526
+ for seq_group in (scheduled_seq_groups +
527
+ scheduler_outputs.ignored_seq_groups):
528
request_output = RequestOutput.from_seq_group(seq_group)
529
request_outputs.append(request_output)
530
@@ -351,7 +557,7 @@ class LLMEngine:
557
blocks_to_copy=scheduler_outputs.blocks_to_copy,
558
)
559
354
- return self._process_worker_outputs(output, scheduler_outputs)
560
+ return self._process_model_outputs(output, scheduler_outputs)
561
562
def _log_system_stats(
563
self,
@@ -416,55 +622,44 @@ class LLMEngine:
622
f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
623
self.last_logging_time = now
624
419
- def _decode_sequences(self, seq_groups: List[SequenceGroup]) -> None:
420
- """Decodes the sequence outputs."""
421
- for seq_group in seq_groups:
422
- for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
423
- new_token, new_output_text = detokenize_incrementally(
424
- self.tokenizer,
425
- seq.output_tokens,
426
- seq.get_last_token_id(),
427
- skip_special_tokens=True,
428
- )
429
- if new_token is not None:
430
- seq.output_tokens.append(new_token)
431
- seq.output_text = new_output_text
432
-
433
- def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
625
+ def _decode_sequence(self, seq: Sequence) -> None:
626
+ """Decodes the new token for a sequence."""
627
+ new_token, new_output_text = detokenize_incrementally(
628
+ self.tokenizer,
629
+ seq.output_tokens,
630
+ seq.get_last_token_id(),
631
+ skip_special_tokens=True,
632
+ )
633
+ if new_token is not None:
634
+ seq.output_tokens.append(new_token)
635
+ seq.output_text = new_output_text
636
+
637
+ def _check_stop(self, seq: Sequence,
638
+ sampling_params: SamplingParams) -> None:
639
"""Stop the finished sequences."""
435
- for seq_group in seq_groups:
436
- sampling_params = seq_group.sampling_params
437
- for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
438
- # Check if the sequence has generated a stop string.
439
- stopped = False
440
- for stop_str in sampling_params.stop:
441
- if seq.output_text.endswith(stop_str):
442
- # Truncate the output text so that the stop string is
443
- # not included in the output.
444
- seq.output_text = seq.output_text[:-len(stop_str)]
445
- self.scheduler.free_seq(
446
- seq, SequenceStatus.FINISHED_STOPPED)
447
- stopped = True
448
- break
449
- if stopped:
450
- continue
451
-
452
- # Check if the sequence has reached max_model_len.
453
- if seq.get_len() > self.scheduler_config.max_model_len:
454
- self.scheduler.free_seq(
455
- seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
456
- continue
457
- # Check if the sequence has reached max_tokens.
458
- if seq.get_output_len() == sampling_params.max_tokens:
459
- self.scheduler.free_seq(
460
- seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
461
- continue
462
- # Check if the sequence has generated the EOS token.
463
- if not sampling_params.ignore_eos:
464
- if seq.get_last_token_id() == self.tokenizer.eos_token_id:
465
- self.scheduler.free_seq(
466
- seq, SequenceStatus.FINISHED_STOPPED)
467
- continue
640
+ for stop_str in sampling_params.stop:
641
+ if seq.output_text.endswith(stop_str):
642
+ # Truncate the output text so that the stop string is
643
+ # not included in the output.
644
+ seq.output_text = seq.output_text[:-len(stop_str)]
645
+ seq.status = SequenceStatus.FINISHED_STOPPED
646
+ return
647
+
648
+ # Check if the sequence has reached max_model_len.
649
+ if seq.get_len() > self.scheduler_config.max_model_len:
650
+ seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
651
+ return
652
+
653
+ # Check if the sequence has reached max_tokens.
654
+ if seq.get_output_len() == sampling_params.max_tokens:
655
+ seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
656
+ return
657
+
658
+ # Check if the sequence has generated the EOS token.
659
+ if ((not sampling_params.ignore_eos)
660
+ and seq.get_last_token_id() == self.tokenizer.eos_token_id):
661
+ seq.status = SequenceStatus.FINISHED_STOPPED
662
+ return
663
664
def _run_workers(
665
self,
vllm/model_executor/layers/sampler.py
+31
-42
@@ -9,7 +9,7 @@ from vllm.model_executor.input_metadata import InputMetadata
9
from vllm.model_executor.parallel_utils.tensor_parallel import (
10
gather_from_tensor_model_parallel_region)
11
from vllm.sampling_params import SamplingParams
12
-from vllm.sequence import SequenceOutputs
12
+from vllm.sequence import SamplerOutput, SequenceOutputs
13
14
_SAMPLING_EPS = 1e-5
15
@@ -39,7 +39,7 @@ class Sampler(nn.Module):
39
hidden_states: torch.Tensor,
40
input_metadata: InputMetadata,
41
embedding_bias: Optional[torch.Tensor] = None,
42
- ) -> Dict[int, SequenceOutputs]:
42
+ ) -> SamplerOutput:
43
# Get the hidden states that we use for sampling.
44
hidden_states = _prune_hidden_states(hidden_states, input_metadata)
45
@@ -292,7 +292,13 @@ def _sample_from_prompt(
292
if sampling_params.use_beam_search:
293
# Beam search.
294
beam_width = sampling_params.best_of
295
- _, next_token_ids = torch.topk(prob, beam_width)
295
+ # Sample 2 * beam_width candidates to make sure that with high
296
+ # probability we can get `beam_width` candidates in addition to
297
+ # the finished sequences for the next iteration. See
298
+ # https://github.com/tensorflow/tensor2tensor/blob/bafdc1b67730430d38d6ab802cbd51f9d053ba2e/tensor2tensor/utils/beam_search.py#L557-L563
299
+ # for details. See also HF reference:
300
+ # https://github.com/huggingface/transformers/blob/a4dd53d88e4852f023332d284ff07a01afcd5681/src/transformers/generation/utils.py#L3063-L3065
301
+ _, next_token_ids = torch.topk(prob, 2 * beam_width)
302
next_token_ids = next_token_ids.tolist()
303
elif sampling_params.temperature < _SAMPLING_EPS:
304
# Greedy sampling.
@@ -330,29 +336,11 @@ def _sample_from_generation_tokens(
336
337
vocab_size = logprobs.size(-1)
338
beam_width = len(seq_ids)
333
- _, topk_ids = torch.topk(logprobs.flatten(), beam_width)
339
+ _, topk_ids = torch.topk(logprobs.flatten(), 2 * beam_width)
340
topk_ids = topk_ids.tolist()
341
seq_idx = [i // vocab_size for i in topk_ids]
336
- beam_seq_ids = [seq_ids[i] for i in seq_idx]
337
- token_ids = [i % vocab_size for i in topk_ids]
338
-
339
- beam_outputs: Dict[int, Tuple[int, int]] = {}
340
- outstanding_beams: List[Tuple[int, int]] = []
341
- # If a beam survives, continue with it.
342
- for seq_id, token_id in zip(beam_seq_ids, token_ids):
343
- if seq_id not in beam_outputs:
344
- beam_outputs[seq_id] = (seq_id, token_id)
345
- else:
346
- outstanding_beams.append((seq_id, token_id))
347
-
348
- # If a beam is discarded, fork another beam.
349
- for seq_id in seq_ids:
350
- if seq_id not in beam_outputs:
351
- beam_outputs[seq_id] = outstanding_beams.pop()
352
- assert not outstanding_beams
353
-
354
- parent_seq_ids = [beam_outputs[seq_id][0] for seq_id in seq_ids]
355
- next_token_ids = [beam_outputs[seq_id][1] for seq_id in seq_ids]
342
+ parent_seq_ids = [seq_ids[i] for i in seq_idx]
343
+ next_token_ids = [i % vocab_size for i in topk_ids]
344
elif sampling_params.temperature < _SAMPLING_EPS:
345
# Greedy sampling.
346
assert len(seq_ids) == 1
@@ -374,16 +362,18 @@ def _sample(
362
probs: torch.Tensor,
363
logprobs: torch.Tensor,
364
input_metadata: InputMetadata,
377
-) -> Dict[int, SequenceOutputs]:
378
- seq_outputs: Dict[int, SequenceOutputs] = {}
365
+) -> SamplerOutput:
366
+ seq_outputs: SamplerOutput = []
367
368
# TODO(woosuk): Optimize.
369
idx = 0
370
for i, seq_group in enumerate(input_metadata.seq_groups):
371
+ seq_group_outputs: List[SequenceOutputs] = []
372
seq_ids, sampling_params = seq_group
373
if i < input_metadata.num_prompts:
374
# Generate the next tokens for a prompt input.
386
- assert len(seq_ids) == sampling_params.best_of
375
+ assert len(seq_ids) == 1, "Prompt input should have only one seq."
376
+ parent_seq_id = seq_ids[0]
377
prob = probs[idx]
378
logprob = logprobs[idx]
379
idx += 1
@@ -395,17 +385,18 @@ def _sample(
385
sampling_params.logprobs)
386
387
# Build the output.
398
- for seq_id, next_token_id in zip(seq_ids, next_token_ids):
388
+ for next_token_id in next_token_ids:
389
output_logprobs = next_logprobs.copy()
390
output_logprobs[next_token_id] = logprob[next_token_id].item()
401
- seq_outputs[seq_id] = SequenceOutputs(seq_id, seq_id,
402
- next_token_id,
403
- output_logprobs)
391
+ seq_group_outputs.append(
392
+ SequenceOutputs(parent_seq_id, next_token_id,
393
+ output_logprobs))
394
else:
395
# Generate the next tokens for generation tokens.
406
- prob = probs[idx:idx + len(seq_ids)]
407
- logprob = logprobs[idx:idx + len(seq_ids)]
408
- idx += len(seq_ids)
396
+ num_parent_seqs = len(seq_ids)
397
+ prob = probs[idx:idx + num_parent_seqs]
398
+ logprob = logprobs[idx:idx + num_parent_seqs]
399
+ idx += num_parent_seqs
400
401
# Sample the next tokens.
402
seq_logprobs = [
@@ -422,17 +413,15 @@ def _sample(
413
logprob[j], sampling_params.logprobs)
414
415
# Build the output.
425
- for seq_id, parent_seq_id, next_token_id in zip(
426
- seq_ids, parent_seq_ids, next_token_ids):
416
+ for parent_seq_id, next_token_id in zip(parent_seq_ids,
417
+ next_token_ids):
418
j = seq_ids.index(parent_seq_id)
419
output_logprobs = next_logprobs[parent_seq_id].copy()
420
output_logprobs[next_token_id] = logprob[j,
421
next_token_id].item()
431
- seq_outputs[seq_id] = SequenceOutputs(
432
- seq_id,
433
- parent_seq_id,
434
- next_token_id,
435
- output_logprobs,
436
- )
422
+ seq_group_outputs.append(
423
+ SequenceOutputs(parent_seq_id, next_token_id,
424
+ output_logprobs))
425
+ seq_outputs.append(seq_group_outputs)
426
427
return seq_outputs
vllm/model_executor/models/aquila.py
+3
-3
@@ -25,7 +25,7 @@
25
The input of the model is flattened to a 1D tensor of tokens. The model uses
26
InputMetadata to extract the original 2D shape of the input.
27
"""
28
-from typing import Dict, List, Optional, Tuple
28
+from typing import List, Optional, Tuple
29
30
import torch
31
from torch import nn
@@ -41,7 +41,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
from vllm.model_executor.parallel_utils.tensor_parallel import (
43
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
44
-from vllm.sequence import SequenceOutputs
44
+from vllm.sequence import SamplerOutput
45
from vllm.transformers_utils.configs.aquila import AquilaConfig
46
47
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -273,7 +273,7 @@ class AquilaForCausalLM(nn.Module):
273
kv_caches: List[KVCache],
274
input_metadata: InputMetadata,
275
cache_events: Optional[List[torch.cuda.Event]],
276
- ) -> Dict[int, SequenceOutputs]:
276
+ ) -> SamplerOutput:
277
hidden_states = self.model(input_ids, positions, kv_caches,
278
input_metadata, cache_events)
279
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/model_executor/models/baichuan.py
+3
-3
@@ -23,12 +23,11 @@ The input of the model is flattened to a 1D tensor of tokens. The model uses
23
InputMetadata to extract the original 2D shape of the input.
24
"""
25
import math
26
-from typing import Dict, List, Optional, Tuple
26
+from typing import List, Optional, Tuple
27
28
import torch
29
from torch import nn
30
31
-from vllm.sequence import SequenceOutputs
31
from vllm.model_executor.input_metadata import InputMetadata
32
from vllm.model_executor.layers.activation import SiluAndMul
33
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -42,6 +41,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
from vllm.model_executor.parallel_utils.tensor_parallel import (
43
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
44
+from vllm.sequence import SamplerOutput
45
from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
46
47
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -290,7 +290,7 @@ class BaiChuanBaseForCausalLM(nn.Module):
290
kv_caches: List[KVCache],
291
input_metadata: InputMetadata,
292
cache_events: Optional[List[torch.cuda.Event]],
293
- ) -> Dict[int, SequenceOutputs]:
293
+ ) -> SamplerOutput:
294
hidden_states = self.model(input_ids, positions, kv_caches,
295
input_metadata, cache_events)
296
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/model_executor/models/bloom.py
+3
-3
@@ -21,7 +21,7 @@ The input of the model is flattened to a 1D tensor of tokens. The model uses
21
InputMetadata to extract the original 2D shape of the input.
22
"""
23
import math
24
-from typing import Dict, List, Optional, Tuple
24
+from typing import List, Optional, Tuple
25
26
import torch
27
from torch import nn
@@ -37,7 +37,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
37
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38
from vllm.model_executor.parallel_utils.tensor_parallel import (
39
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
40
-from vllm.sequence import SequenceOutputs
40
+from vllm.sequence import SamplerOutput
41
42
KVCache = Tuple[torch.Tensor, torch.Tensor]
43
@@ -264,7 +264,7 @@ class BloomForCausalLM(nn.Module):
264
kv_caches: List[KVCache],
265
input_metadata: InputMetadata,
266
cache_events: Optional[List[torch.cuda.Event]],
267
- ) -> Dict[int, SequenceOutputs]:
267
+ ) -> SamplerOutput:
268
hidden_states = self.transformer(input_ids, positions, kv_caches,
269
input_metadata, cache_events)
270
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
vllm/model_executor/models/falcon.py
+3
-3
@@ -19,7 +19,7 @@
19
"""PyTorch Falcon model."""
20
21
import math
22
-from typing import Dict, List, Optional, Tuple, Union
22
+from typing import List, Optional, Tuple, Union
23
24
import torch
25
from torch import nn
@@ -38,7 +38,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
38
from vllm.model_executor.parallel_utils.tensor_parallel import (
39
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear,
40
reduce_from_tensor_model_parallel_region)
41
-from vllm.sequence import SequenceOutputs
41
+from vllm.sequence import SamplerOutput
42
from vllm.transformers_utils.configs import RWConfig
43
44
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -397,7 +397,7 @@ class FalconForCausalLM(nn.Module):
397
kv_caches: List[KVCache],
398
input_metadata: InputMetadata,
399
cache_events: Optional[List[torch.cuda.Event]],
400
- ) -> Dict[int, SequenceOutputs]:
400
+ ) -> SamplerOutput:
401
hidden_states = self.transformer(
402
input_ids,
403
positions,
vllm/model_executor/models/gpt2.py
+3
-3
@@ -21,7 +21,7 @@
21
The input of the model is flattened to a 1D tensor of tokens. The model uses
22
InputMetadata to extract the original 2D shape of the input.
23
"""
24
-from typing import Dict, List, Optional, Tuple
24
+from typing import List, Optional, Tuple
25
26
import torch
27
from torch import nn
@@ -38,7 +38,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
38
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39
from vllm.model_executor.parallel_utils.tensor_parallel import (
40
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
41
-from vllm.sequence import SequenceOutputs
41
+from vllm.sequence import SamplerOutput
42
43
KVCache = Tuple[torch.Tensor, torch.Tensor]
44
@@ -218,7 +218,7 @@ class GPT2LMHeadModel(nn.Module):
218
kv_caches: List[KVCache],
219
input_metadata: InputMetadata,
220
cache_events: Optional[List[torch.cuda.Event]],
221
- ) -> Dict[int, SequenceOutputs]:
221
+ ) -> SamplerOutput:
222
hidden_states = self.transformer(input_ids, positions, kv_caches,
223
input_metadata, cache_events)
224
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
vllm/model_executor/models/gpt_bigcode.py
+3
-3
@@ -22,7 +22,7 @@
22
The input of the model is flattened to a 1D tensor of tokens. The model uses
23
InputMetadata to extract the original 2D shape of the input.
24
"""
25
-from typing import Dict, List, Optional, Tuple
25
+from typing import List, Optional, Tuple
26
27
import torch
28
from torch import nn
@@ -39,7 +39,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
39
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40
from vllm.model_executor.parallel_utils.tensor_parallel import (
41
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
42
-from vllm.sequence import SequenceOutputs
42
+from vllm.sequence import SamplerOutput
43
44
KVCache = Tuple[torch.Tensor, torch.Tensor]
45
@@ -246,7 +246,7 @@ class GPTBigCodeForCausalLM(nn.Module):
246
kv_caches: List[KVCache],
247
input_metadata: InputMetadata,
248
cache_events: Optional[List[torch.cuda.Event]],
249
- ) -> Dict[int, SequenceOutputs]:
249
+ ) -> SamplerOutput:
250
hidden_states = self.transformer(input_ids, positions, kv_caches,
251
input_metadata, cache_events)
252
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
vllm/model_executor/models/gpt_j.py
+3
-3
@@ -20,7 +20,7 @@
20
The input of the model is flattened to a 1D tensor of tokens. The model uses
21
InputMetadata to extract the original 2D shape of the input.
22
"""
23
-from typing import Dict, List, Optional, Tuple
23
+from typing import List, Optional, Tuple
24
25
import torch
26
from torch import nn
@@ -36,7 +36,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
36
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37
from vllm.model_executor.parallel_utils.tensor_parallel import (
38
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
39
-from vllm.sequence import SequenceOutputs
39
+from vllm.sequence import SamplerOutput
40
41
KVCache = Tuple[torch.Tensor, torch.Tensor]
42
@@ -203,7 +203,7 @@ class GPTJForCausalLM(nn.Module):
203
kv_caches: List[KVCache],
204
input_metadata: InputMetadata,
205
cache_events: Optional[List[torch.cuda.Event]],
206
- ) -> Dict[int, SequenceOutputs]:
206
+ ) -> SamplerOutput:
207
hidden_states = self.transformer(input_ids, positions, kv_caches,
208
input_metadata, cache_events)
209
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/model_executor/models/gpt_neox.py
+3
-3
@@ -20,7 +20,7 @@
20
The input of the model is flattened to a 1D tensor of tokens. The model uses
21
InputMetadata to extract the original 2D shape of the input.
22
"""
23
-from typing import Dict, List, Optional, Tuple
23
+from typing import List, Optional, Tuple
24
25
import torch
26
from torch import nn
@@ -36,7 +36,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
36
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37
from vllm.model_executor.parallel_utils.tensor_parallel import (
38
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
39
-from vllm.sequence import SequenceOutputs
39
+from vllm.sequence import SamplerOutput
40
41
KVCache = Tuple[torch.Tensor, torch.Tensor]
42
@@ -215,7 +215,7 @@ class GPTNeoXForCausalLM(nn.Module):
215
kv_caches: List[KVCache],
216
input_metadata: InputMetadata,
217
cache_events: Optional[List[torch.cuda.Event]],
218
- ) -> Dict[int, SequenceOutputs]:
218
+ ) -> SamplerOutput:
219
hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
220
input_metadata, cache_events)
221
next_tokens = self.sampler(self.embed_out.weight, hidden_states,
vllm/model_executor/models/internlm.py
+3
-3
@@ -1,5 +1,5 @@
1
# -*- coding: utf-8 -*-
2
-from typing import Dict, List, Optional, Tuple
2
+from typing import List, Optional, Tuple
3
4
import torch
5
from torch import nn
@@ -17,7 +17,7 @@ from vllm.model_executor.parallel_utils.tensor_parallel import (
17
from vllm.model_executor.weight_utils import (
18
hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
19
load_tensor_parallel_weights)
20
-from vllm.sequence import SequenceOutputs
20
+from vllm.sequence import SamplerOutput
21
22
KVCache = Tuple[torch.Tensor, torch.Tensor]
23
@@ -218,7 +218,7 @@ class InternLMForCausalLM(nn.Module):
218
kv_caches: List[KVCache],
219
input_metadata: InputMetadata,
220
cache_events: Optional[List[torch.cuda.Event]],
221
- ) -> Dict[int, SequenceOutputs]:
221
+ ) -> SamplerOutput:
222
hidden_states = self.model(input_ids, positions, kv_caches,
223
input_metadata, cache_events)
224
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/model_executor/models/llama.py
+3
-3
@@ -25,7 +25,7 @@
25
The input of the model is flattened to a 1D tensor of tokens. The model uses
26
InputMetadata to extract the original 2D shape of the input.
27
"""
28
-from typing import Dict, List, Optional, Tuple
28
+from typing import List, Optional, Tuple
29
30
import torch
31
from torch import nn
@@ -43,7 +43,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
43
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44
from vllm.model_executor.parallel_utils.tensor_parallel import (
45
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
46
-from vllm.sequence import SequenceOutputs
46
+from vllm.sequence import SamplerOutput
47
48
KVCache = Tuple[torch.Tensor, torch.Tensor]
49
@@ -256,7 +256,7 @@ class LlamaForCausalLM(nn.Module):
256
kv_caches: List[KVCache],
257
input_metadata: InputMetadata,
258
cache_events: Optional[List[torch.cuda.Event]],
259
- ) -> Dict[int, SequenceOutputs]:
259
+ ) -> SamplerOutput:
260
hidden_states = self.model(input_ids, positions, kv_caches,
261
input_metadata, cache_events)
262
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/model_executor/models/mpt.py
+3
-3
@@ -1,7 +1,7 @@
1
# coding=utf-8
2
# Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
3
import math
4
-from typing import Dict, List, Optional, Tuple
4
+from typing import List, Optional, Tuple
5
6
import torch
7
import torch.nn as nn
@@ -16,7 +16,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
16
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
17
from vllm.model_executor.parallel_utils.tensor_parallel import (
18
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
19
-from vllm.sequence import SequenceOutputs
19
+from vllm.sequence import SamplerOutput
20
from vllm.transformers_utils.configs.mpt import MPTConfig
21
22
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -230,7 +230,7 @@ class MPTForCausalLM(nn.Module):
230
kv_caches: List[KVCache],
231
input_metadata: InputMetadata,
232
cache_events: Optional[List[torch.cuda.Event]],
233
- ) -> Dict[int, SequenceOutputs]:
233
+ ) -> SamplerOutput:
234
hidden_states = self.transformer(input_ids, positions, kv_caches,
235
input_metadata, cache_events)
236
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
vllm/model_executor/models/opt.py
+3
-3
@@ -21,7 +21,7 @@
21
The input of the model is flattened to a 1D tensor of tokens. The model uses
22
InputMetadata to extract the original 2D shape of the input.
23
"""
24
-from typing import Dict, List, Optional, Tuple
24
+from typing import List, Optional, Tuple
25
26
import torch
27
from torch import nn
@@ -37,7 +37,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
37
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38
from vllm.model_executor.parallel_utils.tensor_parallel import (
39
VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
40
-from vllm.sequence import SequenceOutputs
40
+from vllm.sequence import SamplerOutput
41
42
KVCache = Tuple[torch.Tensor, torch.Tensor]
43
@@ -282,7 +282,7 @@ class OPTForCausalLM(nn.Module):
282
kv_caches: List[KVCache],
283
input_metadata: InputMetadata,
284
cache_events: Optional[List[torch.cuda.Event]],
285
- ) -> Dict[int, SequenceOutputs]:
285
+ ) -> SamplerOutput:
286
hidden_states = self.model(input_ids, positions, kv_caches,
287
input_metadata, cache_events)
288
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
vllm/model_executor/models/qwen.py
+3
-3
@@ -8,7 +8,7 @@
8
The input of the model is flattened to a 1D tensor of tokens. The model uses
9
InputMetadata to extract the original 2D shape of the input.
10
"""
11
-from typing import Dict, List, Optional, Tuple
11
+from typing import List, Optional, Tuple
12
13
import torch
14
from torch import nn
@@ -32,7 +32,7 @@ from vllm.model_executor.parallel_utils.tensor_parallel import (
32
ColumnParallelLinear,
33
RowParallelLinear,
34
)
35
-from vllm.sequence import SequenceOutputs
35
+from vllm.sequence import SamplerOutput
36
from vllm.transformers_utils.configs.qwen import QWenConfig
37
38
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -235,7 +235,7 @@ class QWenLMHeadModel(nn.Module):
235
kv_caches: List[KVCache],
236
input_metadata: InputMetadata,
237
cache_events: Optional[List[torch.cuda.Event]],
238
- ) -> Dict[int, SequenceOutputs]:
238
+ ) -> SamplerOutput:
239
hidden_states = self.transformer(input_ids, positions, kv_caches,
240
input_metadata, cache_events)
241
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
vllm/outputs.py
+6
-4
@@ -75,10 +75,12 @@ class RequestOutput:
75
# Get the top-n sequences.
76
n = seq_group.sampling_params.n
77
seqs = seq_group.get_seqs()
78
- assert n <= len(seqs)
79
- sorted_seqs = sorted(seqs,
80
- key=lambda seq: seq.get_cumulative_logprob(),
81
- reverse=True)
78
+ if seq_group.sampling_params.use_beam_search:
79
+ sorting_key = lambda seq: seq.get_beam_search_score(
80
+ seq_group.sampling_params.length_penalty)
81
+ else:
82
+ sorting_key = lambda seq: seq.get_cumulative_logprob()
83
+ sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
84
top_n_seqs = sorted_seqs[:n]
85
86
# Create the outputs.
vllm/sampling_params.py
+34
-3
@@ -34,6 +34,15 @@ class SamplingParams:
34
top_k: Integer that controls the number of top tokens to consider. Set
35
to -1 to consider all tokens.
36
use_beam_search: Whether to use beam search instead of sampling.
37
+ length_penalty: Float that penalizes sequences based on their length.
38
+ Used in beam search.
39
+ early_stopping: Controls the stopping condition for beam search. It
40
+ accepts the following values: `True`, where the generation stops as
41
+ soon as there are `best_of` complete candidates; `False`, where an
42
+ heuristic is applied and the generation stops when is it very
43
+ unlikely to find better candidates; `"never"`, where the beam search
44
+ procedure only stops when there cannot be better candidates
45
+ (canonical beam search algorithm).
46
stop: List of strings that stop the generation when they are generated.
47
The returned output will not contain the stop strings.
48
ignore_eos: Whether to ignore the EOS token and continue generating
@@ -52,6 +61,8 @@ class SamplingParams:
61
top_p: float = 1.0,
62
top_k: int = -1,
63
use_beam_search: bool = False,
64
+ length_penalty: float = 1.0,
65
+ early_stopping: Union[bool, str] = False,
66
stop: Union[None, str, List[str]] = None,
67
ignore_eos: bool = False,
68
max_tokens: int = 16,
@@ -65,6 +76,8 @@ class SamplingParams:
76
self.top_p = top_p
77
self.top_k = top_k
78
self.use_beam_search = use_beam_search
79
+ self.length_penalty = length_penalty
80
+ self.early_stopping = early_stopping
81
if stop is None:
82
self.stop = []
83
elif isinstance(stop, str):
@@ -78,9 +91,11 @@ class SamplingParams:
91
self._verify_args()
92
if self.use_beam_search:
93
self._verify_beam_search()
81
- elif self.temperature < _SAMPLING_EPS:
82
- # Zero temperature means greedy sampling.
83
- self._verify_greedy_sampling()
94
+ else:
95
+ self._verify_non_beam_search()
96
+ if self.temperature < _SAMPLING_EPS:
97
+ # Zero temperature means greedy sampling.
98
+ self._verify_greedy_sampling()
99
100
def _verify_args(self) -> None:
101
if self.n < 1:
@@ -119,6 +134,20 @@ class SamplingParams:
134
raise ValueError("top_p must be 1 when using beam search.")
135
if self.top_k != -1:
136
raise ValueError("top_k must be -1 when using beam search.")
137
+ if self.early_stopping not in [True, False, "never"]:
138
+ raise ValueError(
139
+ f"early_stopping must be True, False, or 'never', "
140
+ f"got {self.early_stopping}.")
141
+
142
+ def _verify_non_beam_search(self) -> None:
143
+ if self.early_stopping is not False:
144
+ raise ValueError("early_stopping is not effective and must be "
145
+ "False when not using beam search.")
146
+ if (self.length_penalty < 1.0 - _SAMPLING_EPS
147
+ or self.length_penalty > 1.0 + _SAMPLING_EPS):
148
+ raise ValueError(
149
+ "length_penalty is not effective and must be the "
150
+ "default value of 1.0 when not using beam search.")
151
152
def _verify_greedy_sampling(self) -> None:
153
if self.best_of > 1:
@@ -138,6 +167,8 @@ class SamplingParams:
167
f"top_p={self.top_p}, "
168
f"top_k={self.top_k}, "
169
f"use_beam_search={self.use_beam_search}, "
170
+ f"length_penalty={self.length_penalty}, "
171
+ f"early_stopping={self.early_stopping}, "
172
f"stop={self.stop}, "
173
f"ignore_eos={self.ignore_eos}, "
174
f"max_tokens={self.max_tokens}, "
vllm/sequence.py
+77
-22
@@ -69,6 +69,9 @@ class SequenceData:
69
def get_len(self) -> int:
70
return len(self.output_token_ids) + len(self.prompt_token_ids)
71
72
+ def get_prompt_len(self) -> int:
73
+ return len(self.prompt_token_ids)
74
+
75
def get_output_len(self) -> int:
76
return len(self.output_token_ids)
77
@@ -155,6 +158,9 @@ class Sequence:
158
def get_len(self) -> int:
159
return self.data.get_len()
160
161
+ def get_prompt_len(self) -> int:
162
+ return self.data.get_prompt_len()
163
+
164
def get_output_len(self) -> int:
165
return self.data.get_output_len()
166
@@ -170,14 +176,32 @@ class Sequence:
176
def get_cumulative_logprob(self) -> float:
177
return self.data.cumulative_logprob
178
179
+ def get_beam_search_score(self,
180
+ length_penalty: float = 0.0,
181
+ seq_len: Optional[int] = None,
182
+ eos_token_id: Optional[int] = None) -> float:
183
+ """Calculate the beam search score with length penalty.
184
+
185
+ Adapted from
186
+
187
+ https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
188
+ """
189
+ if seq_len is None:
190
+ seq_len = self.get_len()
191
+ # Note: HF implementation does not count the EOS token
192
+ # towards the length, we align with that here for testing.
193
+ if (eos_token_id is not None
194
+ and self.get_last_token_id() == eos_token_id):
195
+ seq_len -= 1
196
+ return self.get_cumulative_logprob() / (seq_len**length_penalty)
197
+
198
def is_finished(self) -> bool:
199
return SequenceStatus.is_finished(self.status)
200
176
- def fork(self, child_seq: "Sequence") -> None:
177
- child_seq.logical_token_blocks = copy.deepcopy(
178
- self.logical_token_blocks)
179
- child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
180
- child_seq.data = copy.deepcopy(self.data)
201
+ def fork(self, new_seq_id: int) -> "Sequence":
202
+ new_seq = copy.deepcopy(self)
203
+ new_seq.seq_id = new_seq_id
204
+ return new_seq
205
206
def __repr__(self) -> str:
207
return (f"Sequence(seq_id={self.seq_id}, "
@@ -203,35 +227,66 @@ class SequenceGroup:
227
arrival_time: float,
228
) -> None:
229
self.request_id = request_id
206
- self.seqs = seqs
230
+ self.seqs_dict = {seq.seq_id: seq for seq in seqs}
231
self.sampling_params = sampling_params
232
self.arrival_time = arrival_time
233
234
+ def get_max_num_running_seqs(self) -> int:
235
+ """The maximum number of sequences running in parallel in the remaining
236
+ lifetime of the request."""
237
+ if self.sampling_params.use_beam_search:
238
+ # For beam search, maximally there will always be `best_of` beam
239
+ # candidates running in the future.
240
+ return self.sampling_params.best_of
241
+ else:
242
+ if self.sampling_params.best_of > self.num_seqs():
243
+ # At prompt stage, the sequence group is not yet filled up
244
+ # and only have one sequence running. However, in the
245
+ # generation stage, we will have `best_of` sequences running.
246
+ return self.sampling_params.best_of
247
+ # At sampling stages, return the number of actual sequences
248
+ # running.
249
+ return self.num_seqs(status=SequenceStatus.RUNNING)
250
+
251
def get_seqs(
252
self,
253
status: Optional[SequenceStatus] = None,
254
) -> List[Sequence]:
255
if status is None:
215
- return self.seqs
256
+ return list(self.seqs_dict.values())
257
else:
217
- return [seq for seq in self.seqs if seq.status == status]
258
+ return [
259
+ seq for seq in self.seqs_dict.values() if seq.status == status
260
+ ]
261
+
262
+ def get_finished_seqs(self) -> List[Sequence]:
263
+ return [seq for seq in self.seqs_dict.values() if seq.is_finished()]
264
265
def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
266
return len(self.get_seqs(status))
267
268
def find(self, seq_id: int) -> Sequence:
223
- for seq in self.seqs:
224
- if seq.seq_id == seq_id:
225
- return seq
226
- raise ValueError(f"Sequence {seq_id} not found.")
269
+ if seq_id not in self.seqs_dict:
270
+ raise ValueError(f"Sequence {seq_id} not found.")
271
+ return self.seqs_dict[seq_id]
272
+
273
+ def add(self, seq: Sequence) -> None:
274
+ if seq.seq_id in self.seqs_dict:
275
+ raise ValueError(f"Sequence {seq.seq_id} already exists.")
276
+ self.seqs_dict[seq.seq_id] = seq
277
+
278
+ def remove(self, seq_id: int) -> None:
279
+ if seq_id not in self.seqs_dict:
280
+ raise ValueError(f"Sequence {seq_id} not found.")
281
+ del self.seqs_dict[seq_id]
282
283
def is_finished(self) -> bool:
229
- return all(seq.is_finished() for seq in self.seqs)
284
+ return all(seq.is_finished() for seq in self.get_seqs())
285
286
def __repr__(self) -> str:
287
return (f"SequenceGroup(request_id={self.request_id}, "
288
f"sampling_params={self.sampling_params}, "
234
- f"num_seqs={len(self.seqs)})")
289
+ f"num_seqs={len(self.seqs_dict)})")
290
291
292
class SequenceGroupMetadata:
@@ -266,7 +321,6 @@ class SequenceOutputs:
321
"""The model output associated with a sequence.
322
323
Args:
269
- seq_id: The ID of the sequence.
324
parent_seq_id: The ID of the parent sequence (for forking in beam
325
search).
326
output_token: The output token ID.
@@ -276,26 +330,27 @@ class SequenceOutputs:
330
331
def __init__(
332
self,
279
- seq_id: int,
333
parent_seq_id: int,
334
output_token: int,
335
logprobs: Dict[int, float],
336
) -> None:
284
- self.seq_id = seq_id
337
self.parent_seq_id = parent_seq_id
338
self.output_token = output_token
339
self.logprobs = logprobs
340
341
def __repr__(self) -> str:
290
- return (f"SequenceOutputs(seq_id={self.seq_id}, "
291
- f"parent_seq_id={self.parent_seq_id}, "
342
+ return (f"SequenceOutputs(parent_seq_id={self.parent_seq_id}, "
343
f"output_token={self.output_token}), "
344
f"logprobs={self.logprobs}")
345
346
def __eq__(self, other: object) -> bool:
347
if not isinstance(other, SequenceOutputs):
297
- return NotImplemented
298
- return (self.seq_id == other.seq_id
299
- and self.parent_seq_id == other.parent_seq_id
348
+ return NotImplementedError()
349
+ return (self.parent_seq_id == other.parent_seq_id
350
and self.output_token == other.output_token
351
and self.logprobs == other.logprobs)
352
+
353
+
354
+# For each sequence group, we generate a list of SequenceOutputs object,
355
+# each of which contains one possible candidate for the next token.
356
+SamplerOutput = List[List[SequenceOutputs]]
vllm/worker/worker.py
+2
-2
@@ -11,7 +11,7 @@ from vllm.model_executor import get_model, InputMetadata, set_random_seed
11
from vllm.model_executor.parallel_utils.parallel_state import (
12
initialize_model_parallel)
13
from vllm.sampling_params import SamplingParams
14
-from vllm.sequence import SequenceData, SequenceGroupMetadata, SequenceOutputs
14
+from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15
from vllm.worker.cache_engine import CacheEngine
16
from vllm.utils import get_gpu_memory
17
@@ -260,7 +260,7 @@ class Worker:
260
blocks_to_swap_in: Dict[int, int],
261
blocks_to_swap_out: Dict[int, int],
262
blocks_to_copy: Dict[int, List[int]],
263
- ) -> Dict[int, SequenceOutputs]:
263
+ ) -> SamplerOutput:
264
# Issue cache operations.
265
issued_cache_op = False
266
if blocks_to_swap_in: