@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1ffc8a7362
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[BugFix] Typing fixes to RequestOutput.prompt and beam search (#9473)

Nick Hill committed Oct 18, 2024 at 08:19 UTC 1ffc8a73628ee8e3f6ad5aab54782d64050d17ea
4 files changed +26 -14
vllm/beam_search.py
+5 -2
@@ -1,5 +1,7 @@
1 from dataclasses import dataclass
2 -from typing import List, Optional
2 +from typing import Dict, List, Optional
3 +
4 +from vllm.sequence import Logprob
5
6
7 @dataclass
@@ -11,6 +13,7 @@ class BeamSearchSequence:
13 """
14 # The tokens includes the prompt.
15 tokens: List[int]
16 + logprobs: List[Dict[int, Logprob]]
17 cum_logprob: float = 0.0
18 text: Optional[str] = None
19
@@ -28,7 +31,7 @@ class BeamSearchInstance:
31
32 def __init__(self, prompt_tokens: List[int]):
33 self.beams: List[BeamSearchSequence] = [
31 - BeamSearchSequence(tokens=prompt_tokens)
34 + BeamSearchSequence(tokens=prompt_tokens, logprobs=[])
35 ]
36 self.completed: List[BeamSearchSequence] = []
37
vllm/engine/protocol.py
+19 -10
@@ -59,7 +59,7 @@ class EngineClient(ABC):
59
60 async def beam_search(
61 self,
62 - prompt: Union[PromptType, List[int]],
62 + prompt: Union[str, List[int]],
63 request_id: str,
64 params: BeamSearchParams,
65 ) -> AsyncGenerator[RequestOutput, None]:
@@ -71,9 +71,13 @@ class EngineClient(ABC):
71 length_penalty = params.length_penalty
72
73 tokenizer = await self.get_tokenizer(lora_request=None)
74 - tokenizedPrompt = prompt if isinstance(
75 - prompt, list) else tokenizer.encode(prompt)
76 - tokenizedLength = len(tokenizedPrompt)
74 + if isinstance(prompt, str):
75 + tokenized_prompt = tokenizer.encode(prompt)
76 + prompt_text = prompt
77 + else:
78 + tokenized_prompt = prompt
79 + prompt_text = None
80 + tokenized_length = len(tokenized_prompt)
81
82 sort_beams_key = create_sort_beams_key_function(
83 tokenizer.eos_token_id, length_penalty)
@@ -81,7 +85,11 @@ class EngineClient(ABC):
85 beam_search_params = SamplingParams(logprobs=2 * beam_width,
86 max_tokens=1,
87 temperature=temperature)
84 - all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
88 + all_beams = [
89 + BeamSearchSequence(tokens=tokenized_prompt,
90 + logprobs=[],
91 + cum_logprob=0)
92 + ]
93 completed = []
94
95 for _ in range(max_tokens):
@@ -114,6 +122,7 @@ class EngineClient(ABC):
122 for token_id, logprob_obj in logprobs.items():
123 new_beam = BeamSearchSequence(
124 tokens=current_beam.tokens + [token_id],
125 + logprobs=current_beam.logprobs + [logprobs],
126 cum_logprob=current_beam.cum_logprob +
127 logprob_obj.logprob)
128
@@ -131,22 +140,22 @@ class EngineClient(ABC):
140 best_beams = sorted_completed[:beam_width]
141
142 for beam in best_beams:
134 - beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
143 + beam.text = tokenizer.decode(beam.tokens[tokenized_length:])
144
145 beam_search_output = RequestOutput(
146 request_id=request_id,
138 - prompt=prompt,
147 + prompt=prompt_text,
148 outputs=[
149 CompletionOutput(
150 text=beam.text,
151 cumulative_logprob=beam.cum_logprob,
143 - token_ids=beam.tokens,
152 + token_ids=beam.tokens[tokenized_length:],
153 index=i,
145 - logprobs=beam.cum_logprob,
154 + logprobs=beam.logprobs,
155 ) for (i, beam) in enumerate(best_beams)
156 ],
157 finished=True,
149 - prompt_token_ids=tokenizedPrompt,
158 + prompt_token_ids=tokenized_prompt,
159 prompt_logprobs=None)
160
161 yield beam_search_output
vllm/entrypoints/llm.py
+1
@@ -433,6 +433,7 @@ class LLM:
433 for token_id, logprob_obj in logprobs.items():
434 new_beam = BeamSearchSequence(
435 tokens=current_beam.tokens + [token_id],
436 + logprobs=current_beam.logprobs + [logprobs],
437 cum_logprob=current_beam.cum_logprob +
438 logprob_obj.logprob)
439
vllm/outputs.py
+1 -2
@@ -4,7 +4,6 @@ from typing import List, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Union
6
7 -from vllm.inputs import PromptType
7 from vllm.lora.request import LoRARequest
8 from vllm.sampling_params import RequestOutputKind
9 from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
@@ -93,7 +92,7 @@ class RequestOutput:
92 def __init__(
93 self,
94 request_id: str,
96 - prompt: Optional[PromptType],
95 + prompt: Optional[str],
96 prompt_token_ids: Optional[List[int]],
97 prompt_logprobs: Optional[PromptLogprobs],
98 outputs: List[CompletionOutput],