@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c9927c1a6a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Use queue for finished requests (#957)

Antoni Baum committed Sep 5, 2023 at 19:27 UTC c9927c1a6aca1b74d9868397e7b711ea38242e99
2 files changed +13 -9
tests/conftest.py
+2 -2
@@ -156,8 +156,8 @@ class VllmRunner:
156 ) -> List[Tuple[List[int], str]]:
157 greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
158 outputs = self.generate(prompts, greedy_params)
159 - return [(output_ids[0], output_str[0]) for output_ids, output_str in
160 - outputs]
159 + return [(output_ids[0], output_str[0])
160 + for output_ids, output_str in outputs]
161
162 def generate_beam_search(
163 self,
vllm/engine/async_llm_engine.py
+11 -7
@@ -1,7 +1,7 @@
1 import asyncio
2 import time
3 from functools import partial
4 -from typing import Any, Dict, Iterable, List, Optional, Set, Type, Union
4 +from typing import Any, Dict, Iterable, List, Optional, Type, Union
5
6 from vllm.config import ModelConfig
7 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -152,7 +152,7 @@ class AsyncLLMEngine:
152
153 # Request id -> stream.
154 self.request_streams: Dict[str, AsyncStream] = {}
155 - self.finished_requests: Set[str] = set()
155 + self.finished_requests: asyncio.Queue[str] = asyncio.Queue()
156 self.background_loop = None
157 if start_engine_loop:
158 self.start_background_loop()
@@ -194,12 +194,14 @@ class AsyncLLMEngine:
194 if self.log_requests:
195 logger.info(f"Finished request {request_id}.")
196 self.request_streams[request_id].finish()
197 - self.finished_requests.add(request_id)
197 + self.finished_requests.put_nowait(request_id)
198
199 - await self._engine_abort(self.finished_requests)
200 - for request_id in self.finished_requests:
199 + finished_request = set()
200 + while not self.finished_requests.empty():
201 + finished_request.add(self.finished_requests.get_nowait())
202 + await self._engine_abort(finished_request)
203 + for request_id in finished_request:
204 del self.request_streams[request_id]
202 - self.finished_requests.clear()
205
206 async def _engine_abort(self, request_ids: Iterable[str]):
207 if self.engine_use_ray:
@@ -226,6 +228,8 @@ class AsyncLLMEngine:
228 f"sampling params: {sampling_params}, "
229 f"prompt token ids: {prompt_token_ids}.")
230
231 + if request_id in self.request_streams:
232 + raise KeyError(f"Request {request_id} already exists.")
233 stream = AsyncStream(request_id)
234 self.request_streams[request_id] = stream
235
@@ -316,7 +320,7 @@ class AsyncLLMEngine:
320 logger.info(f"Aborted request {request_id}.")
321
322 self.request_streams[request_id].finish()
319 - self.finished_requests.add(request_id)
323 + self.finished_requests.put_nowait(request_id)
324
325 async def get_model_config(self) -> ModelConfig:
326 """Get the model configuration of the vLLM engine."""