A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Use queue for finished requests (#957)
Antoni Baum committed
Sep 5, 2023 at 19:27 UTC
c9927c1a6aca1b74d9868397e7b711ea38242e99
2 files changed
+13
-9
tests/conftest.py
+2
-2
@@ -156,8 +156,8 @@ class VllmRunner:
156
) -> List[Tuple[List[int], str]]:
157
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
158
outputs = self.generate(prompts, greedy_params)
159
- return [(output_ids[0], output_str[0]) for output_ids, output_str in
160
- outputs]
159
+ return [(output_ids[0], output_str[0])
160
+ for output_ids, output_str in outputs]
161
162
def generate_beam_search(
163
self,
vllm/engine/async_llm_engine.py
+11
-7
@@ -1,7 +1,7 @@
1
import asyncio
2
import time
3
from functools import partial
4
-from typing import Any, Dict, Iterable, List, Optional, Set, Type, Union
4
+from typing import Any, Dict, Iterable, List, Optional, Type, Union
5
6
from vllm.config import ModelConfig
7
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -152,7 +152,7 @@ class AsyncLLMEngine:
152
153
# Request id -> stream.
154
self.request_streams: Dict[str, AsyncStream] = {}
155
- self.finished_requests: Set[str] = set()
155
+ self.finished_requests: asyncio.Queue[str] = asyncio.Queue()
156
self.background_loop = None
157
if start_engine_loop:
158
self.start_background_loop()
@@ -194,12 +194,14 @@ class AsyncLLMEngine:
194
if self.log_requests:
195
logger.info(f"Finished request {request_id}.")
196
self.request_streams[request_id].finish()
197
- self.finished_requests.add(request_id)
197
+ self.finished_requests.put_nowait(request_id)
198
199
- await self._engine_abort(self.finished_requests)
200
- for request_id in self.finished_requests:
199
+ finished_request = set()
200
+ while not self.finished_requests.empty():
201
+ finished_request.add(self.finished_requests.get_nowait())
202
+ await self._engine_abort(finished_request)
203
+ for request_id in finished_request:
204
del self.request_streams[request_id]
202
- self.finished_requests.clear()
205
206
async def _engine_abort(self, request_ids: Iterable[str]):
207
if self.engine_use_ray:
@@ -226,6 +228,8 @@ class AsyncLLMEngine:
228
f"sampling params: {sampling_params}, "
229
f"prompt token ids: {prompt_token_ids}.")
230
231
+ if request_id in self.request_streams:
232
+ raise KeyError(f"Request {request_id} already exists.")
233
stream = AsyncStream(request_id)
234
self.request_streams[request_id] = stream
235
@@ -316,7 +320,7 @@ class AsyncLLMEngine:
320
logger.info(f"Aborted request {request_id}.")
321
322
self.request_streams[request_id].finish()
319
- self.finished_requests.add(request_id)
323
+ self.finished_requests.put_nowait(request_id)
324
325
async def get_model_config(self) -> ModelConfig:
326
"""Get the model configuration of the vLLM engine."""