A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Engine][Chore] use local variable and remove output var assignment (#24554)
Signed-off-by: Guy Stone <guys@spotify.com>
Guy Stone committed
Sep 11, 2025 at 02:05 UTC
8a894084d21a1ae3bd22743e285c79bda735f910
1 file changed
+6
-7
vllm/engine/protocol.py
+6
-7
@@ -78,6 +78,7 @@ class EngineClient(ABC):
78
preprocessor = await self.get_input_preprocessor()
79
tokenizer_group = preprocessor.get_tokenizer_group()
80
tokenizer = await tokenizer_group.get_lora_tokenizer_async()
81
+ eos_token_id = tokenizer.eos_token_id
82
83
if is_explicit_encoder_decoder_prompt(prompt):
84
raise NotImplementedError
@@ -104,7 +105,7 @@ class EngineClient(ABC):
105
tokenized_length = len(prompt_token_ids)
106
107
sort_beams_key = create_sort_beams_key_function(
107
- tokenizer.eos_token_id, length_penalty)
108
+ eos_token_id, length_penalty)
109
110
beam_search_params = SamplingParams(
111
logprobs=2 * beam_width,
@@ -154,7 +155,7 @@ class EngineClient(ABC):
155
if result.outputs[0].logprobs is not None:
156
logprobs = result.outputs[0].logprobs[0]
157
for token_id, logprob_obj in logprobs.items():
157
- if token_id == tokenizer.eos_token_id and \
158
+ if token_id == eos_token_id and \
159
not ignore_eos:
160
completed.append(
161
BeamSearchSequence(
@@ -166,7 +167,7 @@ class EngineClient(ABC):
167
cum_logprob=current_beam.cum_logprob +
168
logprob_obj.logprob,
169
finish_reason="stop",
169
- stop_reason=tokenizer.eos_token_id))
170
+ stop_reason=eos_token_id))
171
else:
172
new_beams.append(
173
BeamSearchSequence(
@@ -189,14 +190,14 @@ class EngineClient(ABC):
190
best_beams = sorted_completed[:beam_width]
191
192
for beam in best_beams:
192
- if (beam.tokens[-1] == tokenizer.eos_token_id and not ignore_eos):
193
+ if (beam.tokens[-1] == eos_token_id and not ignore_eos):
194
# Skip the eos token in the text.
195
tokens = beam.tokens[tokenized_length:-1]
196
else:
197
tokens = beam.tokens[tokenized_length:]
198
beam.text = tokenizer.decode(tokens)
199
199
- beam_search_output = RequestOutput(
200
+ yield RequestOutput(
201
request_id=request_id,
202
prompt=prompt_text,
203
outputs=[
@@ -214,8 +215,6 @@ class EngineClient(ABC):
215
prompt_token_ids=prompt_token_ids,
216
prompt_logprobs=None)
217
217
- yield beam_search_output
218
-
218
@abstractmethod
219
def encode(
220
self,