@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 14f0b39cda
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Bugfix] Fix a bug in RequestOutput.finished (#202)

Woosuk Kwon committed Jun 22, 2023 at 00:17 UTC 14f0b39cda15b948e0d5e7c87c4cef09e8db240a
4 files changed +10 -8
examples/llm_engine_example.py
+1 -1
@@ -30,7 +30,7 @@ def main(args: argparse.Namespace):
30
31 request_outputs = engine.step()
32 for request_output in request_outputs:
33 - if request_output.finished():
33 + if request_output.finished:
34 print(request_output)
35
36 if not (engine.has_unfinished_requests() or test_prompts):
vllm/engine/async_llm_engine.py
+1 -1
@@ -154,7 +154,7 @@ class AsyncLLMEngine:
154 yield request_output
155
156 # Once finished, release the resources of the sequence group.
157 - if request_output.finished():
157 + if request_output.finished:
158 if self.log_requests:
159 logger.info(f"Finished request {request_id}.")
160
vllm/entrypoints/llm.py
+1 -1
@@ -133,7 +133,7 @@ class LLM:
133 while self.llm_engine.has_unfinished_requests():
134 step_outputs = self.llm_engine.step()
135 for output in step_outputs:
136 - if output.finished():
136 + if output.finished:
137 outputs.append(output)
138 if use_tqdm:
139 pbar.update(1)
vllm/outputs.py
+7 -5
@@ -60,11 +60,13 @@ class RequestOutput:
60 prompt: str,
61 prompt_token_ids: List[int],
62 outputs: List[CompletionOutput],
63 + finished: bool,
64 ) -> None:
65 self.request_id = request_id
66 self.prompt = prompt
67 self.prompt_token_ids = prompt_token_ids
68 self.outputs = outputs
69 + self.finished = finished
70
71 @classmethod
72 def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
@@ -95,13 +97,13 @@ class RequestOutput:
97 # Every sequence in the sequence group should have the same prompt.
98 prompt = top_n_seqs[0].prompt
99 prompt_token_ids = top_n_seqs[0].data.prompt_token_ids
98 - return cls(seq_group.request_id, prompt, prompt_token_ids, outputs)
100 + finished = seq_group.is_finished()
101 + return cls(seq_group.request_id, prompt, prompt_token_ids, outputs,
102 + finished)
103
104 def __repr__(self) -> str:
105 return (f"RequestOutput(request_id={self.request_id}, "
106 f"prompt={self.prompt!r}, "
107 f"prompt_token_ids={self.prompt_token_ids}, "
104 - f"outputs={self.outputs})")
105 -
106 - def finished(self) -> bool:
107 - return all(output.finished() for output in self.outputs)
108 + f"outputs={self.outputs}, "
109 + f"finished={self.finished})")