@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ad388d25a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

Type-fix: make execute_model output type optional (#12020)

Keyun Tong committed Jan 15, 2025 at 01:44 UTC ad388d25a8e668545ef91c3634b67a241155e2ea
2 files changed +2 -1
vllm/v1/executor/uniproc_executor.py
+1
@@ -70,6 +70,7 @@ class UniprocExecutor(Executor):
70 scheduler_output,
71 ) -> ModelRunnerOutput:
72 output = self.worker.execute_model(scheduler_output)
73 + assert output is not None
74 return output
75
76 def profile(self, is_start: bool = True):
vllm/v1/worker/gpu_worker.py
+1 -1
@@ -200,7 +200,7 @@ class Worker:
200 def execute_model(
201 self,
202 scheduler_output: "SchedulerOutput",
203 - ) -> ModelRunnerOutput:
203 + ) -> Optional[ModelRunnerOutput]:
204 output = self.model_runner.execute_model(scheduler_output)
205 return output if self.rank == 0 else None
206