@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b7955ef17b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix timeout error in the FastAPI frontend (#34)

Zhuohan Li committed May 19, 2023 at 14:00 UTC b7955ef17b8d899327b25564f20665ec3ffa71cb
1 file changed +5 -2
cacheflow/frontend/fastapi_frontend.py
+5 -2
@@ -83,7 +83,7 @@ class FastAPIServer:
83 self.is_server_running = True
84 updated_seq_groups = await self.server.step.remote()
85 self.is_server_running = False
86 - # Notify the waiting coroutines that there new outputs ready.
86 + # Notify the waiting coroutines that there are new outputs ready.
87 for seq_group in updated_seq_groups:
88 group_id = seq_group.group_id
89 self.running_seq_groups[group_id] = seq_group
@@ -121,7 +121,10 @@ class FastAPIServer:
121 # Wait for new output. The group_event will be set in server_step
122 # when there is new output available for the sequence group.
123 # Added a timeout to prevent deadlock.
124 - await asyncio.wait_for(group_event.wait(), timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
124 + try:
125 + await asyncio.wait_for(group_event.wait(), timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
126 + except asyncio.TimeoutError:
127 + continue
128 # Reset the event to wait for the next output.
129 group_event.clear()
130 # Decode and return new outputs