A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Misc] Manage HTTP connections in one place (#6600)
Cyrus Leung committed
Jul 23, 2024 at 12:32 UTC
97234be0ec67f48ed5e65bc0290f329dfb33798e
7 files changed
+215
-85
tests/conftest.py
+8
@@ -16,6 +16,7 @@ from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
16
from vllm import LLM, SamplingParams
17
from vllm.assets.image import ImageAsset
18
from vllm.config import TokenizerPoolConfig
19
+from vllm.connections import global_http_connection
20
from vllm.distributed import (destroy_distributed_environment,
21
destroy_model_parallel)
22
from vllm.inputs import TextPrompt
@@ -74,6 +75,13 @@ IMAGE_ASSETS = _ImageAssets()
75
"""Singleton instance of :class:`_ImageAssets`."""
76
77
78
+@pytest.fixture(autouse=True)
79
+def init_test_http_connection():
80
+ # pytest_asyncio may use a different event loop per test
81
+ # so we need to make sure the async client is created anew
82
+ global_http_connection.reuse_client = False
83
+
84
+
85
def cleanup():
86
destroy_model_parallel()
87
destroy_distributed_environment()
tests/entrypoints/openai/test_vision.py
+4
-6
@@ -2,9 +2,8 @@ from typing import Dict, List
2
3
import openai
4
import pytest
5
-import pytest_asyncio
5
7
-from vllm.multimodal.utils import ImageFetchAiohttp, encode_image_base64
6
+from vllm.multimodal.utils import encode_image_base64, fetch_image
7
8
from ...utils import VLLM_PATH, RemoteOpenAIServer
9
@@ -42,11 +41,10 @@ def client(server):
41
return server.get_async_client()
42
43
45
-@pytest_asyncio.fixture(scope="session")
46
-async def base64_encoded_image() -> Dict[str, str]:
44
+@pytest.fixture(scope="session")
45
+def base64_encoded_image() -> Dict[str, str]:
46
return {
48
- image_url:
49
- encode_image_base64(await ImageFetchAiohttp.fetch_image(image_url))
47
+ image_url: encode_image_base64(fetch_image(image_url))
48
for image_url in TEST_IMAGE_URLS
49
}
50
tests/multimodal/test_utils.py
+5
-5
@@ -7,7 +7,7 @@ import numpy as np
7
import pytest
8
from PIL import Image
9
10
-from vllm.multimodal.utils import ImageFetchAiohttp, fetch_image
10
+from vllm.multimodal.utils import async_fetch_image, fetch_image
11
12
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
13
TEST_IMAGE_URLS = [
@@ -37,15 +37,15 @@ def _image_equals(a: Image.Image, b: Image.Image) -> bool:
37
return (np.asarray(a) == np.asarray(b.convert(a.mode))).all()
38
39
40
-@pytest.mark.asyncio(scope="module")
40
+@pytest.mark.asyncio
41
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
42
async def test_fetch_image_http(image_url: str):
43
image_sync = fetch_image(image_url)
44
- image_async = await ImageFetchAiohttp.fetch_image(image_url)
44
+ image_async = await async_fetch_image(image_url)
45
assert _image_equals(image_sync, image_async)
46
47
48
-@pytest.mark.asyncio(scope="module")
48
+@pytest.mark.asyncio
49
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
50
@pytest.mark.parametrize("suffix", get_supported_suffixes())
51
async def test_fetch_image_base64(url_images: Dict[str, Image.Image],
@@ -78,5 +78,5 @@ async def test_fetch_image_base64(url_images: Dict[str, Image.Image],
78
else:
79
pass # Lossy format; only check that image can be opened
80
81
- data_image_async = await ImageFetchAiohttp.fetch_image(data_url)
81
+ data_image_async = await async_fetch_image(data_url)
82
assert _image_equals(data_image_sync, data_image_async)
vllm/assets/image.py
+6
-7
@@ -1,11 +1,12 @@
1
-import shutil
1
from dataclasses import dataclass
2
from functools import lru_cache
3
from typing import Literal
4
6
-import requests
5
from PIL import Image
6
7
+from vllm.connections import global_http_connection
8
+from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
9
+
10
from .base import get_cache_dir
11
12
@@ -22,11 +23,9 @@ def get_air_example_data_2_asset(filename: str) -> Image.Image:
23
if not image_path.exists():
24
base_url = "https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava"
25
25
- with requests.get(f"{base_url}/{filename}", stream=True) as response:
26
- response.raise_for_status()
27
-
28
- with image_path.open("wb") as f:
29
- shutil.copyfileobj(response.raw, f)
26
+ global_http_connection.download_file(f"{base_url}/{filename}",
27
+ image_path,
28
+ timeout=VLLM_IMAGE_FETCH_TIMEOUT)
29
30
return Image.open(image_path)
31
vllm/connections.py
new
+167
@@ -0,0 +1,167 @@
1
+from pathlib import Path
2
+from typing import Mapping, Optional
3
+from urllib.parse import urlparse
4
+
5
+import aiohttp
6
+import requests
7
+
8
+from vllm.version import __version__ as VLLM_VERSION
9
+
10
+
11
+class HTTPConnection:
12
+ """Helper class to send HTTP requests."""
13
+
14
+ def __init__(self, *, reuse_client: bool = True) -> None:
15
+ super().__init__()
16
+
17
+ self.reuse_client = reuse_client
18
+
19
+ self._sync_client: Optional[requests.Session] = None
20
+ self._async_client: Optional[aiohttp.ClientSession] = None
21
+
22
+ def get_sync_client(self) -> requests.Session:
23
+ if self._sync_client is None or not self.reuse_client:
24
+ self._sync_client = requests.Session()
25
+
26
+ return self._sync_client
27
+
28
+ # NOTE: We intentionally use an async function even though it is not
29
+ # required, so that the client is only accessible inside async event loop
30
+ async def get_async_client(self) -> aiohttp.ClientSession:
31
+ if self._async_client is None or not self.reuse_client:
32
+ self._async_client = aiohttp.ClientSession()
33
+
34
+ return self._async_client
35
+
36
+ def _validate_http_url(self, url: str):
37
+ parsed_url = urlparse(url)
38
+
39
+ if parsed_url.scheme not in ("http", "https"):
40
+ raise ValueError("Invalid HTTP URL: A valid HTTP URL "
41
+ "must have scheme 'http' or 'https'.")
42
+
43
+ def _headers(self, **extras: str) -> Mapping[str, str]:
44
+ return {"User-Agent": f"vLLM/{VLLM_VERSION}", **extras}
45
+
46
+ def get_response(
47
+ self,
48
+ url: str,
49
+ *,
50
+ stream: bool = False,
51
+ timeout: Optional[float] = None,
52
+ extra_headers: Optional[Mapping[str, str]] = None,
53
+ ):
54
+ self._validate_http_url(url)
55
+
56
+ client = self.get_sync_client()
57
+ extra_headers = extra_headers or {}
58
+
59
+ return client.get(url,
60
+ headers=self._headers(**extra_headers),
61
+ stream=stream,
62
+ timeout=timeout)
63
+
64
+ async def get_async_response(
65
+ self,
66
+ url: str,
67
+ *,
68
+ timeout: Optional[float] = None,
69
+ extra_headers: Optional[Mapping[str, str]] = None,
70
+ ):
71
+ self._validate_http_url(url)
72
+
73
+ client = await self.get_async_client()
74
+ extra_headers = extra_headers or {}
75
+
76
+ return client.get(url,
77
+ headers=self._headers(**extra_headers),
78
+ timeout=timeout)
79
+
80
+ def get_bytes(self, url: str, *, timeout: Optional[float] = None) -> bytes:
81
+ with self.get_response(url, timeout=timeout) as r:
82
+ r.raise_for_status()
83
+
84
+ return r.content
85
+
86
+ async def async_get_bytes(
87
+ self,
88
+ url: str,
89
+ *,
90
+ timeout: Optional[float] = None,
91
+ ) -> bytes:
92
+ async with await self.get_async_response(url, timeout=timeout) as r:
93
+ r.raise_for_status()
94
+
95
+ return await r.read()
96
+
97
+ def get_text(self, url: str, *, timeout: Optional[float] = None) -> str:
98
+ with self.get_response(url, timeout=timeout) as r:
99
+ r.raise_for_status()
100
+
101
+ return r.text
102
+
103
+ async def async_get_text(
104
+ self,
105
+ url: str,
106
+ *,
107
+ timeout: Optional[float] = None,
108
+ ) -> str:
109
+ async with await self.get_async_response(url, timeout=timeout) as r:
110
+ r.raise_for_status()
111
+
112
+ return await r.text()
113
+
114
+ def get_json(self, url: str, *, timeout: Optional[float] = None) -> str:
115
+ with self.get_response(url, timeout=timeout) as r:
116
+ r.raise_for_status()
117
+
118
+ return r.json()
119
+
120
+ async def async_get_json(
121
+ self,
122
+ url: str,
123
+ *,
124
+ timeout: Optional[float] = None,
125
+ ) -> str:
126
+ async with await self.get_async_response(url, timeout=timeout) as r:
127
+ r.raise_for_status()
128
+
129
+ return await r.json()
130
+
131
+ def download_file(
132
+ self,
133
+ url: str,
134
+ save_path: Path,
135
+ *,
136
+ timeout: Optional[float] = None,
137
+ chunk_size: int = 128,
138
+ ) -> Path:
139
+ with self.get_response(url, timeout=timeout) as r:
140
+ r.raise_for_status()
141
+
142
+ with save_path.open("wb") as f:
143
+ for chunk in r.iter_content(chunk_size):
144
+ f.write(chunk)
145
+
146
+ return save_path
147
+
148
+ async def async_download_file(
149
+ self,
150
+ url: str,
151
+ save_path: Path,
152
+ *,
153
+ timeout: Optional[float] = None,
154
+ chunk_size: int = 128,
155
+ ) -> Path:
156
+ async with await self.get_async_response(url, timeout=timeout) as r:
157
+ r.raise_for_status()
158
+
159
+ with save_path.open("wb") as f:
160
+ async for chunk in r.content.iter_chunked(chunk_size):
161
+ f.write(chunk)
162
+
163
+ return save_path
164
+
165
+
166
+global_http_connection = HTTPConnection()
167
+"""The global :class:`HTTPConnection` instance used by vLLM."""
vllm/multimodal/utils.py
+22
-66
@@ -1,26 +1,12 @@
1
import base64
2
from io import BytesIO
3
-from typing import Optional, Union
4
-from urllib.parse import urlparse
3
+from typing import Union
4
6
-import aiohttp
7
-import requests
5
from PIL import Image
6
7
+from vllm.connections import global_http_connection
8
from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
9
from vllm.multimodal.base import MultiModalDataDict
12
-from vllm.version import __version__ as VLLM_VERSION
13
-
14
-
15
-def _validate_remote_url(url: str, *, name: str):
16
- parsed_url = urlparse(url)
17
- if parsed_url.scheme not in ["http", "https"]:
18
- raise ValueError(f"Invalid '{name}': A valid '{name}' "
19
- "must have scheme 'http' or 'https'.")
20
-
21
-
22
-def _get_request_headers():
23
- return {"User-Agent": f"vLLM/{VLLM_VERSION}"}
10
11
12
def _load_image_from_bytes(b: bytes):
@@ -42,13 +28,8 @@ def fetch_image(image_url: str, *, image_mode: str = "RGB") -> Image.Image:
28
By default, the image is converted into RGB format.
29
"""
30
if image_url.startswith('http'):
45
- _validate_remote_url(image_url, name="image_url")
46
-
47
- headers = _get_request_headers()
48
-
49
- with requests.get(url=image_url, headers=headers) as response:
50
- response.raise_for_status()
51
- image_raw = response.content
31
+ image_raw = global_http_connection.get_bytes(
32
+ image_url, timeout=VLLM_IMAGE_FETCH_TIMEOUT)
33
image = _load_image_from_bytes(image_raw)
34
35
elif image_url.startswith('data:image'):
@@ -60,55 +41,30 @@ def fetch_image(image_url: str, *, image_mode: str = "RGB") -> Image.Image:
41
return image.convert(image_mode)
42
43
63
-class ImageFetchAiohttp:
64
- aiohttp_client: Optional[aiohttp.ClientSession] = None
65
-
66
- @classmethod
67
- def get_aiohttp_client(cls) -> aiohttp.ClientSession:
68
- if cls.aiohttp_client is None:
69
- timeout = aiohttp.ClientTimeout(total=VLLM_IMAGE_FETCH_TIMEOUT)
70
- connector = aiohttp.TCPConnector()
71
- cls.aiohttp_client = aiohttp.ClientSession(timeout=timeout,
72
- connector=connector)
73
-
74
- return cls.aiohttp_client
75
-
76
- @classmethod
77
- async def fetch_image(
78
- cls,
79
- image_url: str,
80
- *,
81
- image_mode: str = "RGB",
82
- ) -> Image.Image:
83
- """
84
- Asynchronously load a PIL image from a HTTP or base64 data URL.
85
-
86
- By default, the image is converted into RGB format.
87
- """
88
-
89
- if image_url.startswith('http'):
90
- _validate_remote_url(image_url, name="image_url")
91
-
92
- client = cls.get_aiohttp_client()
93
- headers = _get_request_headers()
44
+async def async_fetch_image(image_url: str,
45
+ *,
46
+ image_mode: str = "RGB") -> Image.Image:
47
+ """
48
+ Asynchronously load a PIL image from a HTTP or base64 data URL.
49
95
- async with client.get(url=image_url, headers=headers) as response:
96
- response.raise_for_status()
97
- image_raw = await response.read()
98
- image = _load_image_from_bytes(image_raw)
50
+ By default, the image is converted into RGB format.
51
+ """
52
+ if image_url.startswith('http'):
53
+ image_raw = await global_http_connection.async_get_bytes(
54
+ image_url, timeout=VLLM_IMAGE_FETCH_TIMEOUT)
55
+ image = _load_image_from_bytes(image_raw)
56
100
- elif image_url.startswith('data:image'):
101
- image = _load_image_from_data_url(image_url)
102
- else:
103
- raise ValueError(
104
- "Invalid 'image_url': A valid 'image_url' must start "
105
- "with either 'data:image' or 'http'.")
57
+ elif image_url.startswith('data:image'):
58
+ image = _load_image_from_data_url(image_url)
59
+ else:
60
+ raise ValueError("Invalid 'image_url': A valid 'image_url' must start "
61
+ "with either 'data:image' or 'http'.")
62
107
- return image.convert(image_mode)
63
+ return image.convert(image_mode)
64
65
66
async def async_get_and_parse_image(image_url: str) -> MultiModalDataDict:
111
- image = await ImageFetchAiohttp.fetch_image(image_url)
67
+ image = await async_fetch_image(image_url)
68
return {"image": image}
69
70
vllm/usage/usage_lib.py
+3
-1
@@ -16,6 +16,7 @@ import requests
16
import torch
17
18
import vllm.envs as envs
19
+from vllm.connections import global_http_connection
20
from vllm.version import __version__ as VLLM_VERSION
21
22
_config_home = envs.VLLM_CONFIG_ROOT
@@ -204,7 +205,8 @@ class UsageMessage:
205
206
def _send_to_server(self, data):
207
try:
207
- requests.post(_USAGE_STATS_SERVER, json=data)
208
+ global_http_client = global_http_connection.get_sync_client()
209
+ global_http_client.post(_USAGE_STATS_SERVER, json=data)
210
except requests.exceptions.RequestException:
211
# silently ignore unless we are using debug log
212
logging.debug("Failed to send usage data to server")