@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 89be30fa7d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Small async_llm_engine refactor (#2618)

Murali Andoorveedu committed Jan 27, 2024 at 23:28 UTC 89be30fa7d51035cee96d1573ffbe8b8ba6db878
1 file changed +2 -4
vllm/engine/async_llm_engine.py
+2 -4
@@ -53,7 +53,7 @@ class AsyncStream:
53 self._queue.put_nowait(item)
54
55 def finish(self) -> None:
56 - self._queue.put_nowait(StopIteration)
56 + self._queue.put_nowait(StopAsyncIteration())
57 self._finished = True
58
59 @property
@@ -65,9 +65,7 @@ class AsyncStream:
65
66 async def __anext__(self) -> RequestOutput:
67 result = await self._queue.get()
68 - if result is StopIteration:
69 - raise StopAsyncIteration
70 - elif isinstance(result, Exception):
68 + if isinstance(result, Exception):
69 raise result
70 return result
71