@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8999ec3c16
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Store `eos_token_id` in `Sequence` for easy access (#3166)

Nick Hill committed Mar 5, 2024 at 15:35 UTC 8999ec3c1632c91c194ab27df6bf274f5bcb0b5f
6 files changed +44 -49
tests/test_cache_block_hashing.py
+2 -1
@@ -54,7 +54,8 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int):
54 for prompt in prompts:
55 hashes[-1].append([])
56 prompt_token_ids = tokenizer.encode(prompt)
57 - seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
57 + seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
58 + tokenizer.tokenizer.eos_token_id)
59
60 num_blocks = len(prompt_token_ids) // block_size
61 for idx in range(num_blocks):
vllm/core/scheduler.py
+3 -4
@@ -59,10 +59,9 @@ class SchedulerOutputs:
59 and not self.blocks_to_swap_out and not self.blocks_to_copy)
60
61 def _sort_by_lora_ids(self) -> bool:
62 - self.scheduled_seq_groups = sorted(
63 - self.scheduled_seq_groups,
64 - key=lambda g: (g.lora_request.lora_int_id
65 - if g.lora_request else 0, g.request_id))
62 + self.scheduled_seq_groups = sorted(self.scheduled_seq_groups,
63 + key=lambda g:
64 + (g.lora_int_id, g.request_id))
65
66 @property
67 def lora_requests(self) -> Set[LoRARequest]:
vllm/engine/llm_engine.py
+13 -17
@@ -491,8 +491,10 @@ class LLMEngine:
491 # Create the sequences.
492 block_size = self.cache_config.block_size
493 seq_id = next(self.seq_counter)
494 + eos_token_id = self.tokenizer.get_lora_tokenizer(
495 + lora_request).eos_token_id
496 seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
495 - lora_request)
497 + eos_token_id, lora_request)
498
499 # Defensive copy of SamplingParams, which are used by the sampler,
500 # this doesn't deep-copy LogitsProcessor objects
@@ -548,15 +550,13 @@ class LLMEngine:
550 if early_stopping is True:
551 return True
552
551 - current_worst_score = (current_worst_seq.get_beam_search_score(
553 + current_worst_score = current_worst_seq.get_beam_search_score(
554 length_penalty=length_penalty,
553 - eos_token_id=self.get_tokenizer_for_seq(
554 - current_worst_seq).eos_token_id))
555 + eos_token_id=current_worst_seq.eos_token_id)
556 if early_stopping is False:
556 - highest_attainable_score = (best_running_seq.get_beam_search_score(
557 + highest_attainable_score = best_running_seq.get_beam_search_score(
558 length_penalty=length_penalty,
558 - eos_token_id=self.get_tokenizer_for_seq(
559 - best_running_seq).eos_token_id))
559 + eos_token_id=best_running_seq.eos_token_id)
560 else:
561 assert early_stopping == "never"
562 if length_penalty > 0.0:
@@ -570,8 +570,7 @@ class LLMEngine:
570 highest_attainable_score = (
571 best_running_seq.get_beam_search_score(
572 length_penalty=length_penalty,
573 - eos_token_id=self.get_tokenizer_for_seq(
574 - best_running_seq).eos_token_id,
573 + eos_token_id=best_running_seq.eos_token_id,
574 seq_len=max_possible_length))
575 else:
576 # Otherwise, beam search will prefer shorter sequences. The
@@ -580,8 +579,7 @@ class LLMEngine:
579 highest_attainable_score = (
580 best_running_seq.get_beam_search_score(
581 length_penalty=length_penalty,
583 - eos_token_id=self.get_tokenizer_for_seq(
584 - best_running_seq).eos_token_id))
582 + eos_token_id=best_running_seq.eos_token_id))
583 return current_worst_score >= highest_attainable_score
584
585 def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
@@ -679,8 +677,7 @@ class LLMEngine:
677 all_finished_seqs = existing_finished_seqs + new_finished_seqs
678 # Sort the finished sequences by their scores.
679 all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
682 - length_penalty=length_penalty,
683 - eos_token_id=self.get_tokenizer_for_seq(x[0]).eos_token_id),
680 + length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
681 reverse=True)
682 for seq, parent, is_new in all_finished_seqs[:beam_width]:
683 if is_new:
@@ -707,8 +704,7 @@ class LLMEngine:
704 if not seq.is_finished()]
705 # Sort the running sequences by their scores.
706 running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
710 - length_penalty=length_penalty,
711 - eos_token_id=self.get_tokenizer_for_seq(x[0]).eos_token_id),
707 + length_penalty=length_penalty, eos_token_id=x[0].eos_token_id),
708 reverse=True)
709
710 # Check if we can stop the beam search.
@@ -1014,8 +1010,8 @@ class LLMEngine:
1010 return
1011
1012 # Check if the sequence has generated the EOS token.
1017 - if ((not sampling_params.ignore_eos) and seq.get_last_token_id()
1018 - == self.get_tokenizer_for_seq(seq).eos_token_id):
1013 + if ((not sampling_params.ignore_eos)
1014 + and seq.get_last_token_id() == seq.eos_token_id):
1015 seq.status = SequenceStatus.FINISHED_STOPPED
1016 return
1017
vllm/model_executor/layers/sampler.py
-1
@@ -516,7 +516,6 @@ def _get_logprobs(
516 if (i < sampling_metadata.num_prompts
517 and sampling_params.prompt_logprobs is not None):
518 num_logprobs = sampling_params.prompt_logprobs
519 - prompt_len = sampling_metadata.prompt_lens[i]
519 prompt_tokens = sampling_metadata.seq_data[
520 seq_ids[0]].prompt_token_ids
521 group_prompt_logprobs: PromptLogprobs = [None]
vllm/outputs.py
+21 -20
@@ -90,29 +90,30 @@ class RequestOutput:
90 # Get the top-n sequences.
91 n = seq_group.sampling_params.n
92 seqs = seq_group.get_seqs()
93 - if seq_group.sampling_params.use_beam_search:
94 - sorting_key = lambda seq: seq.get_beam_search_score(
95 - seq_group.sampling_params.length_penalty)
93 + if n == 1:
94 + top_n_seqs = seqs
95 else:
97 - sorting_key = lambda seq: seq.get_cumulative_logprob()
98 - sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
99 - top_n_seqs = sorted_seqs[:n]
96 + if seq_group.sampling_params.use_beam_search:
97 + sorting_key = lambda seq: seq.get_beam_search_score(
98 + seq_group.sampling_params.length_penalty)
99 + else:
100 + sorting_key = lambda seq: seq.get_cumulative_logprob()
101 + sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
102 + top_n_seqs = sorted_seqs[:n]
103
104 # Create the outputs.
102 - outputs: List[CompletionOutput] = []
103 - for seq in top_n_seqs:
104 - logprobs = seq.output_logprobs
105 - if seq_group.sampling_params.logprobs is None:
106 - # NOTE: We need to take care of this case because the sequence
107 - # always has the logprobs of the sampled tokens even if the
108 - # logprobs are not requested.
109 - logprobs = None
110 - finshed_reason = SequenceStatus.get_finished_reason(seq.status)
111 - output = CompletionOutput(seqs.index(seq), seq.output_text,
112 - seq.get_output_token_ids(),
113 - seq.get_cumulative_logprob(), logprobs,
114 - finshed_reason)
115 - outputs.append(output)
105 + # NOTE: We need omit logprobs here explicitly because the sequence
106 + # always has the logprobs of the sampled tokens even if the
107 + # logprobs are not requested.
108 + include_logprobs = seq_group.sampling_params.logprobs
109 + outputs = [
110 + CompletionOutput(seqs.index(seq), seq.output_text,
111 + seq.get_output_token_ids(),
112 + seq.get_cumulative_logprob(),
113 + seq.output_logprobs if include_logprobs else None,
114 + SequenceStatus.get_finished_reason(seq.status))
115 + for seq in top_n_seqs
116 + ]
117
118 # Every sequence in the sequence group should have the same prompt.
119 prompt = seq_group.prompt
vllm/sequence.py
+5 -6
@@ -142,11 +142,13 @@ class Sequence:
142 prompt: str,
143 prompt_token_ids: List[int],
144 block_size: int,
145 + eos_token_id: int,
146 lora_request: Optional[LoRARequest] = None,
147 ) -> None:
148 self.seq_id = seq_id
149 self.prompt = prompt
150 self.block_size = block_size
151 + self.eos_token_id = eos_token_id
152 self.lora_request = lora_request
153
154 self.data = SequenceData(prompt_token_ids)
@@ -362,12 +364,9 @@ class SequenceGroup:
364 self,
365 status: Optional[SequenceStatus] = None,
366 ) -> List[Sequence]:
365 - if status is None:
366 - return list(self.seqs_dict.values())
367 - else:
368 - return [
369 - seq for seq in self.seqs_dict.values() if seq.status == status
370 - ]
367 + return list(self.seqs_dict.values()) if status is None else [
368 + seq for seq in self.seqs_dict.values() if seq.status == status
369 + ]
370
371 def get_unfinished_seqs(self) -> List[Sequence]:
372 return [