A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Store `eos_token_id` in `Sequence` for easy access (#3166)
Nick Hill committed
Mar 5, 2024 at 15:35 UTC
8999ec3c1632c91c194ab27df6bf274f5bcb0b5f
6 files changed
+44
-49
tests/test_cache_block_hashing.py
+2
-1
@@ -54,7 +54,8 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int):
54
for prompt in prompts:
55
hashes[-1].append([])
56
prompt_token_ids = tokenizer.encode(prompt)
57
- seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
57
+ seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
58
+ tokenizer.tokenizer.eos_token_id)
59
60
num_blocks = len(prompt_token_ids) // block_size
61
for idx in range(num_blocks):
vllm/core/scheduler.py
+3
-4
@@ -59,10 +59,9 @@ class SchedulerOutputs:
59
and not self.blocks_to_swap_out and not self.blocks_to_copy)
60
61
def _sort_by_lora_ids(self) -> bool:
62
- self.scheduled_seq_groups = sorted(
63
- self.scheduled_seq_groups,
64
- key=lambda g: (g.lora_request.lora_int_id
65
- if g.lora_request else 0, g.request_id))
62
+ self.scheduled_seq_groups = sorted(self.scheduled_seq_groups,
63
+ key=lambda g:
64
+ (g.lora_int_id, g.request_id))
65
66
@property
67
def lora_requests(self) -> Set[LoRARequest]:
vllm/engine/llm_engine.py
+13
-17
@@ -491,8 +491,10 @@ class LLMEngine:
491
# Create the sequences.
492
block_size = self.cache_config.block_size
493
seq_id = next(self.seq_counter)
494
+ eos_token_id = self.tokenizer.get_lora_tokenizer(
495
+ lora_request).eos_token_id
496
seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
495
- lora_request)
497
+ eos_token_id, lora_request)
498
499
# Defensive copy of SamplingParams, which are used by the sampler,
500
# this doesn't deep-copy LogitsProcessor objects
@@ -548,15 +550,13 @@ class LLMEngine:
550
if early_stopping is True:
551
return True
552
551
- current_worst_score = (current_worst_seq.get_beam_search_score(
553
+ current_worst_score = current_worst_seq.get_beam_search_score(
554
length_penalty=length_penalty,
553
- eos_token_id=self.get_tokenizer_for_seq(
554
- current_worst_seq).eos_token_id))
555
+ eos_token_id=current_worst_seq.eos_token_id)
556
if early_stopping is False:
556
- highest_attainable_score = (best_running_seq.get_beam_search_score(
557
+ highest_attainable_score = best_running_seq.get_beam_search_score(
558
length_penalty=length_penalty,
558
- eos_token_id=self.get_tokenizer_for_seq(
559
- best_running_seq).eos_token_id))
559
+ eos_token_id=best_running_seq.eos_token_id)
560
else:
561
assert early_stopping == "never"
562
if length_penalty > 0.0:
@@ -570,8 +570,7 @@ class LLMEngine:
570
highest_attainable_score = (
571
best_running_seq.get_beam_search_score(
572
length_penalty=length_penalty,
573
- eos_token_id=self.get_tokenizer_for_seq(
574
- best_running_seq).eos_token_id,
573
+ eos_token_id=best_running_seq.eos_token_id,
574
seq_len=max_possible_length))
575
else:
576
# Otherwise, beam search will prefer shorter sequences. The
@@ -580,8 +579,7 @@ class LLMEngine:
579
highest_attainable_score = (
580
best_running_seq.get_beam_search_score(
581
length_penalty=length_penalty,
583
- eos_token_id=self.get_tokenizer_for_seq(
584
- best_running_seq).eos_token_id))
582
+ eos_token_id=best_running_seq.eos_token_id))
583
return current_worst_score >= highest_attainable_score
584
585
def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
@@ -679,8 +677,7 @@ class LLMEngine:
677
all_finished_seqs = existing_finished_seqs + new_finished_seqs
678
# Sort the finished sequences by their scores.
679
all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
682
- length_penalty=length_penalty,
683
- eos_token_id=self.get_tokenizer_for_seq(x[0]).eos_token_id),
680
+ length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
681
reverse=True)
682
for seq, parent, is_new in all_finished_seqs[:beam_width]:
683
if is_new:
@@ -707,8 +704,7 @@ class LLMEngine:
704
if not seq.is_finished()]
705
# Sort the running sequences by their scores.
706
running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
710
- length_penalty=length_penalty,
711
- eos_token_id=self.get_tokenizer_for_seq(x[0]).eos_token_id),
707
+ length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
708
reverse=True)
709
710
# Check if we can stop the beam search.
@@ -1014,8 +1010,8 @@ class LLMEngine:
1010
return
1011
1012
# Check if the sequence has generated the EOS token.
1017
- if ((not sampling_params.ignore_eos) and seq.get_last_token_id()
1018
- == self.get_tokenizer_for_seq(seq).eos_token_id):
1013
+ if ((not sampling_params.ignore_eos)
1014
+ and seq.get_last_token_id() == seq.eos_token_id):
1015
seq.status = SequenceStatus.FINISHED_STOPPED
1016
return
1017
vllm/model_executor/layers/sampler.py
-1
@@ -516,7 +516,6 @@ def _get_logprobs(
516
if (i < sampling_metadata.num_prompts
517
and sampling_params.prompt_logprobs is not None):
518
num_logprobs = sampling_params.prompt_logprobs
519
- prompt_len = sampling_metadata.prompt_lens[i]
519
prompt_tokens = sampling_metadata.seq_data[
520
seq_ids[0]].prompt_token_ids
521
group_prompt_logprobs: PromptLogprobs = [None]
vllm/outputs.py
+21
-20
@@ -90,29 +90,30 @@ class RequestOutput:
90
# Get the top-n sequences.
91
n = seq_group.sampling_params.n
92
seqs = seq_group.get_seqs()
93
- if seq_group.sampling_params.use_beam_search:
94
- sorting_key = lambda seq: seq.get_beam_search_score(
95
- seq_group.sampling_params.length_penalty)
93
+ if n == 1:
94
+ top_n_seqs = seqs
95
else:
97
- sorting_key = lambda seq: seq.get_cumulative_logprob()
98
- sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
99
- top_n_seqs = sorted_seqs[:n]
96
+ if seq_group.sampling_params.use_beam_search:
97
+ sorting_key = lambda seq: seq.get_beam_search_score(
98
+ seq_group.sampling_params.length_penalty)
99
+ else:
100
+ sorting_key = lambda seq: seq.get_cumulative_logprob()
101
+ sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
102
+ top_n_seqs = sorted_seqs[:n]
103
104
# Create the outputs.
102
- outputs: List[CompletionOutput] = []
103
- for seq in top_n_seqs:
104
- logprobs = seq.output_logprobs
105
- if seq_group.sampling_params.logprobs is None:
106
- # NOTE: We need to take care of this case because the sequence
107
- # always has the logprobs of the sampled tokens even if the
108
- # logprobs are not requested.
109
- logprobs = None
110
- finshed_reason = SequenceStatus.get_finished_reason(seq.status)
111
- output = CompletionOutput(seqs.index(seq), seq.output_text,
112
- seq.get_output_token_ids(),
113
- seq.get_cumulative_logprob(), logprobs,
114
- finshed_reason)
115
- outputs.append(output)
105
+ # NOTE: We need omit logprobs here explicitly because the sequence
106
+ # always has the logprobs of the sampled tokens even if the
107
+ # logprobs are not requested.
108
+ include_logprobs = seq_group.sampling_params.logprobs
109
+ outputs = [
110
+ CompletionOutput(seqs.index(seq), seq.output_text,
111
+ seq.get_output_token_ids(),
112
+ seq.get_cumulative_logprob(),
113
+ seq.output_logprobs if include_logprobs else None,
114
+ SequenceStatus.get_finished_reason(seq.status))
115
+ for seq in top_n_seqs
116
+ ]
117
118
# Every sequence in the sequence group should have the same prompt.
119
prompt = seq_group.prompt
vllm/sequence.py
+5
-6
@@ -142,11 +142,13 @@ class Sequence:
142
prompt: str,
143
prompt_token_ids: List[int],
144
block_size: int,
145
+ eos_token_id: int,
146
lora_request: Optional[LoRARequest] = None,
147
) -> None:
148
self.seq_id = seq_id
149
self.prompt = prompt
150
self.block_size = block_size
151
+ self.eos_token_id = eos_token_id
152
self.lora_request = lora_request
153
154
self.data = SequenceData(prompt_token_ids)
@@ -362,12 +364,9 @@ class SequenceGroup:
364
self,
365
status: Optional[SequenceStatus] = None,
366
) -> List[Sequence]:
365
- if status is None:
366
- return list(self.seqs_dict.values())
367
- else:
368
- return [
369
- seq for seq in self.seqs_dict.values() if seq.status == status
370
- ]
367
+ return list(self.seqs_dict.values()) if status is None else [
368
+ seq for seq in self.seqs_dict.values() if seq.status == status
369
+ ]
370
371
def get_unfinished_seqs(self) -> List[Sequence]:
372
return [