A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[BugFix] Typing fixes to RequestOutput.prompt and beam search (#9473)
Nick Hill committed
Oct 18, 2024 at 08:19 UTC
1ffc8a73628ee8e3f6ad5aab54782d64050d17ea
4 files changed
+26
-14
vllm/beam_search.py
+5
-2
@@ -1,5 +1,7 @@
1
from dataclasses import dataclass
2
-from typing import List, Optional
2
+from typing import Dict, List, Optional
3
+
4
+from vllm.sequence import Logprob
5
6
7
@dataclass
@@ -11,6 +13,7 @@ class BeamSearchSequence:
13
"""
14
# The tokens includes the prompt.
15
tokens: List[int]
16
+ logprobs: List[Dict[int, Logprob]]
17
cum_logprob: float = 0.0
18
text: Optional[str] = None
19
@@ -28,7 +31,7 @@ class BeamSearchInstance:
31
32
def __init__(self, prompt_tokens: List[int]):
33
self.beams: List[BeamSearchSequence] = [
31
- BeamSearchSequence(tokens=prompt_tokens)
34
+ BeamSearchSequence(tokens=prompt_tokens, logprobs=[])
35
]
36
self.completed: List[BeamSearchSequence] = []
37
vllm/engine/protocol.py
+19
-10
@@ -59,7 +59,7 @@ class EngineClient(ABC):
59
60
async def beam_search(
61
self,
62
- prompt: Union[PromptType, List[int]],
62
+ prompt: Union[str, List[int]],
63
request_id: str,
64
params: BeamSearchParams,
65
) -> AsyncGenerator[RequestOutput, None]:
@@ -71,9 +71,13 @@ class EngineClient(ABC):
71
length_penalty = params.length_penalty
72
73
tokenizer = await self.get_tokenizer(lora_request=None)
74
- tokenizedPrompt = prompt if isinstance(
75
- prompt, list) else tokenizer.encode(prompt)
76
- tokenizedLength = len(tokenizedPrompt)
74
+ if isinstance(prompt, str):
75
+ tokenized_prompt = tokenizer.encode(prompt)
76
+ prompt_text = prompt
77
+ else:
78
+ tokenized_prompt = prompt
79
+ prompt_text = None
80
+ tokenized_length = len(tokenized_prompt)
81
82
sort_beams_key = create_sort_beams_key_function(
83
tokenizer.eos_token_id, length_penalty)
@@ -81,7 +85,11 @@ class EngineClient(ABC):
85
beam_search_params = SamplingParams(logprobs=2 * beam_width,
86
max_tokens=1,
87
temperature=temperature)
84
- all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
88
+ all_beams = [
89
+ BeamSearchSequence(tokens=tokenized_prompt,
90
+ logprobs=[],
91
+ cum_logprob=0)
92
+ ]
93
completed = []
94
95
for _ in range(max_tokens):
@@ -114,6 +122,7 @@ class EngineClient(ABC):
122
for token_id, logprob_obj in logprobs.items():
123
new_beam = BeamSearchSequence(
124
tokens=current_beam.tokens + [token_id],
125
+ logprobs=current_beam.logprobs + [logprobs],
126
cum_logprob=current_beam.cum_logprob +
127
logprob_obj.logprob)
128
@@ -131,22 +140,22 @@ class EngineClient(ABC):
140
best_beams = sorted_completed[:beam_width]
141
142
for beam in best_beams:
134
- beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
143
+ beam.text = tokenizer.decode(beam.tokens[tokenized_length:])
144
145
beam_search_output = RequestOutput(
146
request_id=request_id,
138
- prompt=prompt,
147
+ prompt=prompt_text,
148
outputs=[
149
CompletionOutput(
150
text=beam.text,
151
cumulative_logprob=beam.cum_logprob,
143
- token_ids=beam.tokens,
152
+ token_ids=beam.tokens[tokenized_length:],
153
index=i,
145
- logprobs=beam.cum_logprob,
154
+ logprobs=beam.logprobs,
155
) for (i, beam) in enumerate(best_beams)
156
],
157
finished=True,
149
- prompt_token_ids=tokenizedPrompt,
158
+ prompt_token_ids=tokenized_prompt,
159
prompt_logprobs=None)
160
161
yield beam_search_output
vllm/entrypoints/llm.py
+1
@@ -433,6 +433,7 @@ class LLM:
433
for token_id, logprob_obj in logprobs.items():
434
new_beam = BeamSearchSequence(
435
tokens=current_beam.tokens + [token_id],
436
+ logprobs=current_beam.logprobs + [logprobs],
437
cum_logprob=current_beam.cum_logprob +
438
logprob_obj.logprob)
439
vllm/outputs.py
+1
-2
@@ -4,7 +4,6 @@ from typing import List, Optional
4
from typing import Sequence as GenericSequence
5
from typing import Union
6
7
-from vllm.inputs import PromptType
7
from vllm.lora.request import LoRARequest
8
from vllm.sampling_params import RequestOutputKind
9
from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
@@ -93,7 +92,7 @@ class RequestOutput:
92
def __init__(
93
self,
94
request_id: str,
96
- prompt: Optional[PromptType],
95
+ prompt: Optional[str],
96
prompt_token_ids: Optional[List[int]],
97
prompt_logprobs: Optional[PromptLogprobs],
98
outputs: List[CompletionOutput],