A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Small async_llm_engine refactor (#2618)
Murali Andoorveedu committed
Jan 27, 2024 at 23:28 UTC
89be30fa7d51035cee96d1573ffbe8b8ba6db878
1 file changed
+2
-4
vllm/engine/async_llm_engine.py
+2
-4
@@ -53,7 +53,7 @@ class AsyncStream:
53
self._queue.put_nowait(item)
54
55
def finish(self) -> None:
56
- self._queue.put_nowait(StopIteration)
56
+ self._queue.put_nowait(StopAsyncIteration())
57
self._finished = True
58
59
@property
@@ -65,9 +65,7 @@ class AsyncStream:
65
66
async def __anext__(self) -> RequestOutput:
67
result = await self._queue.get()
68
- if result is StopIteration:
69
- raise StopAsyncIteration
70
- elif isinstance(result, Exception):
68
+ if isinstance(result, Exception):
69
raise result
70
return result
71