A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix timeout error in the FastAPI frontend (#34)
Zhuohan Li committed
May 19, 2023 at 14:00 UTC
b7955ef17b8d899327b25564f20665ec3ffa71cb
1 file changed
+5
-2
cacheflow/frontend/fastapi_frontend.py
+5
-2
@@ -83,7 +83,7 @@ class FastAPIServer:
83
self.is_server_running = True
84
updated_seq_groups = await self.server.step.remote()
85
self.is_server_running = False
86
- # Notify the waiting coroutines that there new outputs ready.
86
+ # Notify the waiting coroutines that there are new outputs ready.
87
for seq_group in updated_seq_groups:
88
group_id = seq_group.group_id
89
self.running_seq_groups[group_id] = seq_group
@@ -121,7 +121,10 @@ class FastAPIServer:
121
# Wait for new output. The group_event will be set in server_step
122
# when there is new output available for the sequence group.
123
# Added a timeout to prevent deadlock.
124
- await asyncio.wait_for(group_event.wait(), timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
124
+ try:
125
+ await asyncio.wait_for(group_event.wait(), timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
126
+ except asyncio.TimeoutError:
127
+ continue
128
# Reset the event to wait for the next output.
129
group_event.clear()
130
# Decode and return new outputs