A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Frontend][Core] Move `merge_async_iterators` to utils (#4026)
Cyrus Leung committed
Apr 12, 2024 at 13:30 UTC
7fd3949a0b1c6cd0dcd7066aca48d9d589f2f68e
2 files changed
+39
-39
vllm/entrypoints/openai/serving_completion.py
+1
-37
@@ -1,4 +1,3 @@
1
-import asyncio
1
import time
2
from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
3
Optional, Tuple)
@@ -17,7 +16,7 @@ from vllm.logger import init_logger
16
from vllm.model_executor.guided_decoding import (
17
get_guided_decoding_logits_processor)
18
from vllm.outputs import RequestOutput
20
-from vllm.utils import random_uuid
19
+from vllm.utils import merge_async_iterators, random_uuid
20
21
logger = init_logger(__name__)
22
@@ -50,41 +49,6 @@ def parse_prompt_format(prompt) -> Tuple[bool, list]:
49
return prompt_is_tokens, prompts
50
51
53
-def merge_async_iterators(*iterators):
54
- """Merge multiple asynchronous iterators into a single iterator.
55
-
56
- This method handle the case where some iterators finish before others.
57
- When it yields, it yields a tuple (i, item) where i is the index of the
58
- iterator that yields the item.
59
- """
60
- queue = asyncio.Queue()
61
-
62
- finished = [False] * len(iterators)
63
-
64
- async def producer(i, iterator):
65
- try:
66
- async for item in iterator:
67
- await queue.put((i, item))
68
- except Exception as e:
69
- await queue.put(e)
70
- finished[i] = True
71
-
72
- _tasks = [
73
- asyncio.create_task(producer(i, iterator))
74
- for i, iterator in enumerate(iterators)
75
- ]
76
-
77
- async def consumer():
78
- while not all(finished) or not queue.empty():
79
- item = await queue.get()
80
- if isinstance(item, Exception):
81
- raise item
82
- yield item
83
- await asyncio.gather(*_tasks)
84
-
85
- return consumer()
86
-
87
-
52
class OpenAIServingCompletion(OpenAIServing):
53
54
def __init__(self,
vllm/utils.py
+38
-2
@@ -9,8 +9,8 @@ import warnings
9
from collections import OrderedDict, defaultdict
10
from functools import lru_cache, partial
11
from platform import uname
12
-from typing import (Any, Awaitable, Callable, Dict, Generic, Hashable, List,
13
- Optional, Tuple, TypeVar, Union)
12
+from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
13
+ Hashable, List, Optional, Tuple, TypeVar, Union)
14
15
import psutil
16
import torch
@@ -181,6 +181,42 @@ def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
181
return _async_wrapper
182
183
184
+def merge_async_iterators(
185
+ *iterators: AsyncIterator[T]) -> AsyncIterator[Tuple[int, T]]:
186
+ """Merge multiple asynchronous iterators into a single iterator.
187
+
188
+ This method handle the case where some iterators finish before others.
189
+ When it yields, it yields a tuple (i, item) where i is the index of the
190
+ iterator that yields the item.
191
+ """
192
+ queue: asyncio.Queue[Union[Tuple[int, T], Exception]] = asyncio.Queue()
193
+
194
+ finished = [False] * len(iterators)
195
+
196
+ async def producer(i: int, iterator: AsyncIterator[T]):
197
+ try:
198
+ async for item in iterator:
199
+ await queue.put((i, item))
200
+ except Exception as e:
201
+ await queue.put(e)
202
+ finished[i] = True
203
+
204
+ _tasks = [
205
+ asyncio.create_task(producer(i, iterator))
206
+ for i, iterator in enumerate(iterators)
207
+ ]
208
+
209
+ async def consumer():
210
+ while not all(finished) or not queue.empty():
211
+ item = await queue.get()
212
+ if isinstance(item, Exception):
213
+ raise item
214
+ yield item
215
+ await asyncio.gather(*_tasks)
216
+
217
+ return consumer()
218
+
219
+
220
def get_ip() -> str:
221
host_ip = os.environ.get("HOST_IP")
222
if host_ip: