A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
Type-fix: make execute_model output type optional (#12020)
Keyun Tong committed
Jan 15, 2025 at 01:44 UTC
ad388d25a8e668545ef91c3634b67a241155e2ea
2 files changed
+2
-1
vllm/v1/executor/uniproc_executor.py
+1
@@ -70,6 +70,7 @@ class UniprocExecutor(Executor):
70
scheduler_output,
71
) -> ModelRunnerOutput:
72
output = self.worker.execute_model(scheduler_output)
73
+ assert output is not None
74
return output
75
76
def profile(self, is_start: bool = True):
vllm/v1/worker/gpu_worker.py
+1
-1
@@ -200,7 +200,7 @@ class Worker:
200
def execute_model(
201
self,
202
scheduler_output: "SchedulerOutput",
203
- ) -> ModelRunnerOutput:
203
+ ) -> Optional[ModelRunnerOutput]:
204
output = self.model_runner.execute_model(scheduler_output)
205
return output if self.rank == 0 else None
206