A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Bugfix] Fix a bug in RequestOutput.finished (#202)
Woosuk Kwon committed
Jun 22, 2023 at 00:17 UTC
14f0b39cda15b948e0d5e7c87c4cef09e8db240a
4 files changed
+10
-8
examples/llm_engine_example.py
+1
-1
@@ -30,7 +30,7 @@ def main(args: argparse.Namespace):
30
31
request_outputs = engine.step()
32
for request_output in request_outputs:
33
- if request_output.finished():
33
+ if request_output.finished:
34
print(request_output)
35
36
if not (engine.has_unfinished_requests() or test_prompts):
vllm/engine/async_llm_engine.py
+1
-1
@@ -154,7 +154,7 @@ class AsyncLLMEngine:
154
yield request_output
155
156
# Once finished, release the resources of the sequence group.
157
- if request_output.finished():
157
+ if request_output.finished:
158
if self.log_requests:
159
logger.info(f"Finished request {request_id}.")
160
vllm/entrypoints/llm.py
+1
-1
@@ -133,7 +133,7 @@ class LLM:
133
while self.llm_engine.has_unfinished_requests():
134
step_outputs = self.llm_engine.step()
135
for output in step_outputs:
136
- if output.finished():
136
+ if output.finished:
137
outputs.append(output)
138
if use_tqdm:
139
pbar.update(1)
vllm/outputs.py
+7
-5
@@ -60,11 +60,13 @@ class RequestOutput:
60
prompt: str,
61
prompt_token_ids: List[int],
62
outputs: List[CompletionOutput],
63
+ finished: bool,
64
) -> None:
65
self.request_id = request_id
66
self.prompt = prompt
67
self.prompt_token_ids = prompt_token_ids
68
self.outputs = outputs
69
+ self.finished = finished
70
71
@classmethod
72
def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
@@ -95,13 +97,13 @@ class RequestOutput:
97
# Every sequence in the sequence group should have the same prompt.
98
prompt = top_n_seqs[0].prompt
99
prompt_token_ids = top_n_seqs[0].data.prompt_token_ids
98
- return cls(seq_group.request_id, prompt, prompt_token_ids, outputs)
100
+ finished = seq_group.is_finished()
101
+ return cls(seq_group.request_id, prompt, prompt_token_ids, outputs,
102
+ finished)
103
104
def __repr__(self) -> str:
105
return (f"RequestOutput(request_id={self.request_id}, "
106
f"prompt={self.prompt!r}, "
107
f"prompt_token_ids={self.prompt_token_ids}, "
104
- f"outputs={self.outputs})")
105
-
106
- def finished(self) -> bool:
107
- return all(output.finished() for output in self.outputs)
108
+ f"outputs={self.outputs}, "
109
+ f"finished={self.finished})")