@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7fd3949a0b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Frontend][Core] Move `merge_async_iterators` to utils (#4026)

Cyrus Leung committed Apr 12, 2024 at 13:30 UTC 7fd3949a0b1c6cd0dcd7066aca48d9d589f2f68e
2 files changed +39 -39
vllm/entrypoints/openai/serving_completion.py
+1 -37
@@ -1,4 +1,3 @@
1 -import asyncio
1 import time
2 from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
3 Optional, Tuple)
@@ -17,7 +16,7 @@ from vllm.logger import init_logger
16 from vllm.model_executor.guided_decoding import (
17 get_guided_decoding_logits_processor)
18 from vllm.outputs import RequestOutput
20 -from vllm.utils import random_uuid
19 +from vllm.utils import merge_async_iterators, random_uuid
20
21 logger = init_logger(__name__)
22
@@ -50,41 +49,6 @@ def parse_prompt_format(prompt) -> Tuple[bool, list]:
49 return prompt_is_tokens, prompts
50
51
53 -def merge_async_iterators(*iterators):
54 - """Merge multiple asynchronous iterators into a single iterator.
55 -
56 - This method handle the case where some iterators finish before others.
57 - When it yields, it yields a tuple (i, item) where i is the index of the
58 - iterator that yields the item.
59 - """
60 - queue = asyncio.Queue()
61 -
62 - finished = [False] * len(iterators)
63 -
64 - async def producer(i, iterator):
65 - try:
66 - async for item in iterator:
67 - await queue.put((i, item))
68 - except Exception as e:
69 - await queue.put(e)
70 - finished[i] = True
71 -
72 - _tasks = [
73 - asyncio.create_task(producer(i, iterator))
74 - for i, iterator in enumerate(iterators)
75 - ]
76 -
77 - async def consumer():
78 - while not all(finished) or not queue.empty():
79 - item = await queue.get()
80 - if isinstance(item, Exception):
81 - raise item
82 - yield item
83 - await asyncio.gather(*_tasks)
84 -
85 - return consumer()
86 -
87 -
52 class OpenAIServingCompletion(OpenAIServing):
53
54 def __init__(self,
vllm/utils.py
+38 -2
@@ -9,8 +9,8 @@ import warnings
9 from collections import OrderedDict, defaultdict
10 from functools import lru_cache, partial
11 from platform import uname
12 -from typing import (Any, Awaitable, Callable, Dict, Generic, Hashable, List,
13 - Optional, Tuple, TypeVar, Union)
12 +from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
13 + Hashable, List, Optional, Tuple, TypeVar, Union)
14
15 import psutil
16 import torch
@@ -181,6 +181,42 @@ def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
181 return _async_wrapper
182
183
184 +def merge_async_iterators(
185 + *iterators: AsyncIterator[T]) -> AsyncIterator[Tuple[int, T]]:
186 + """Merge multiple asynchronous iterators into a single iterator.
187 +
188 + This method handle the case where some iterators finish before others.
189 + When it yields, it yields a tuple (i, item) where i is the index of the
190 + iterator that yields the item.
191 + """
192 + queue: asyncio.Queue[Union[Tuple[int, T], Exception]] = asyncio.Queue()
193 +
194 + finished = [False] * len(iterators)
195 +
196 + async def producer(i: int, iterator: AsyncIterator[T]):
197 + try:
198 + async for item in iterator:
199 + await queue.put((i, item))
200 + except Exception as e:
201 + await queue.put(e)
202 + finished[i] = True
203 +
204 + _tasks = [
205 + asyncio.create_task(producer(i, iterator))
206 + for i, iterator in enumerate(iterators)
207 + ]
208 +
209 + async def consumer():
210 + while not all(finished) or not queue.empty():
211 + item = await queue.get()
212 + if isinstance(item, Exception):
213 + raise item
214 + yield item
215 + await asyncio.gather(*_tasks)
216 +
217 + return consumer()
218 +
219 +
220 def get_ip() -> str:
221 host_ip = os.environ.get("HOST_IP")
222 if host_ip: