A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Tests] Disable retries and use context manager for openai client (#7565)
Nick Hill committed
Aug 26, 2024 at 21:33 UTC
39178c7fbc6ab47b5448db101476c36d0ed38d7a
15 files changed
+130
-93
tests/async_engine/test_openapi_server_ray.py
+5
-3
@@ -1,5 +1,6 @@
1
import openai # use the official client for correctness check
2
import pytest
3
+import pytest_asyncio
4
5
from ..utils import VLLM_PATH, RemoteOpenAIServer
6
@@ -31,9 +32,10 @@ def server():
32
yield remote_server
33
34
34
-@pytest.fixture(scope="module")
35
-def client(server):
36
- return server.get_async_client()
35
+@pytest_asyncio.fixture
36
+async def client(server):
37
+ async with server.get_async_client() as async_client:
38
+ yield async_client
39
40
41
@pytest.mark.asyncio
tests/entrypoints/openai/test_audio.py
+5
-3
@@ -2,6 +2,7 @@ from typing import Dict, List
2
3
import openai
4
import pytest
5
+import pytest_asyncio
6
7
from vllm.assets.audio import AudioAsset
8
from vllm.multimodal.utils import encode_audio_base64, fetch_audio
@@ -28,9 +29,10 @@ def server():
29
yield remote_server
30
31
31
-@pytest.fixture(scope="module")
32
-def client(server):
33
- return server.get_async_client()
32
+@pytest_asyncio.fixture
33
+async def client(server):
34
+ async with server.get_async_client() as async_client:
35
+ yield async_client
36
37
38
@pytest.fixture(scope="session")
tests/entrypoints/openai/test_basic.py
+5
-3
@@ -2,6 +2,7 @@ from http import HTTPStatus
2
3
import openai
4
import pytest
5
+import pytest_asyncio
6
import requests
7
8
from vllm.version import __version__ as VLLM_VERSION
@@ -28,9 +29,10 @@ def server():
29
yield remote_server
30
31
31
-@pytest.fixture(scope="module")
32
-def client(server):
33
- return server.get_async_client()
32
+@pytest_asyncio.fixture
33
+async def client(server):
34
+ async with server.get_async_client() as async_client:
35
+ yield async_client
36
37
38
@pytest.mark.asyncio
tests/entrypoints/openai/test_chat.py
+5
-3
@@ -6,6 +6,7 @@ from typing import Dict, List, Optional
6
import jsonschema
7
import openai # use the official client for correctness check
8
import pytest
9
+import pytest_asyncio
10
import torch
11
from openai import BadRequestError
12
@@ -46,9 +47,10 @@ def server(zephyr_lora_files, zephyr_lora_added_tokens_files): # noqa: F811
47
yield remote_server
48
49
49
-@pytest.fixture(scope="module")
50
-def client(server):
51
- return server.get_async_client()
50
+@pytest_asyncio.fixture
51
+async def client(server):
52
+ async with server.get_async_client() as async_client:
53
+ yield async_client
54
55
56
@pytest.mark.asyncio
tests/entrypoints/openai/test_completion.py
+9
-2
@@ -8,6 +8,7 @@ from typing import Dict, List, Optional
8
import jsonschema
9
import openai # use the official client for correctness check
10
import pytest
11
+import pytest_asyncio
12
# downloading lora to test lora requests
13
from huggingface_hub import snapshot_download
14
from openai import BadRequestError
@@ -89,11 +90,17 @@ def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files,
90
91
@pytest.fixture(scope="module",
92
params=["", "--disable-frontend-multiprocessing"])
92
-def client(default_server_args, request):
93
+def server(default_server_args, request):
94
if request.param:
95
default_server_args.append(request.param)
96
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
96
- yield remote_server.get_async_client()
97
+ yield remote_server
98
+
99
+
100
+@pytest_asyncio.fixture
101
+async def client(server):
102
+ async with server.get_async_client() as async_client:
103
+ yield async_client
104
105
106
@pytest.mark.asyncio
tests/entrypoints/openai/test_embedding.py
+5
-4
@@ -3,6 +3,7 @@ import base64
3
import numpy as np
4
import openai
5
import pytest
6
+import pytest_asyncio
7
8
from ...utils import RemoteOpenAIServer
9
@@ -24,10 +25,10 @@ def embedding_server():
25
yield remote_server
26
27
27
-@pytest.mark.asyncio
28
-@pytest.fixture(scope="module")
29
-def embedding_client(embedding_server):
30
- return embedding_server.get_async_client()
28
+@pytest_asyncio.fixture
29
+async def embedding_client(embedding_server):
30
+ async with embedding_server.get_async_client() as async_client:
31
+ yield async_client
32
33
34
@pytest.mark.asyncio
tests/entrypoints/openai/test_encoder_decoder.py
+5
-3
@@ -1,5 +1,6 @@
1
import openai
2
import pytest
3
+import pytest_asyncio
4
5
from ...utils import RemoteOpenAIServer
6
@@ -18,9 +19,10 @@ def server():
19
yield remote_server
20
21
21
-@pytest.fixture(scope="module")
22
-def client(server):
23
- return server.get_async_client()
22
+@pytest_asyncio.fixture
23
+async def client(server):
24
+ async with server.get_async_client() as async_client:
25
+ yield async_client
26
27
28
@pytest.mark.asyncio
tests/entrypoints/openai/test_metrics.py
+9
-2
@@ -6,6 +6,7 @@ from http import HTTPStatus
6
7
import openai
8
import pytest
9
+import pytest_asyncio
10
import requests
11
from prometheus_client.parser import text_string_to_metric_families
12
from transformers import AutoTokenizer
@@ -35,11 +36,17 @@ def default_server_args():
36
"--enable-chunked-prefill",
37
"--disable-frontend-multiprocessing",
38
])
38
-def client(default_server_args, request):
39
+def server(default_server_args, request):
40
if request.param:
41
default_server_args.append(request.param)
42
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
42
- yield remote_server.get_async_client()
43
+ yield remote_server
44
+
45
+
46
+@pytest_asyncio.fixture
47
+async def client(server):
48
+ async with server.get_async_client() as cl:
49
+ yield cl
50
51
52
_PROMPT = "Hello my name is Robert and I love magic"
tests/entrypoints/openai/test_models.py
+5
-3
@@ -1,5 +1,6 @@
1
import openai # use the official client for correctness check
2
import pytest
3
+import pytest_asyncio
4
# downloading lora to test lora requests
5
from huggingface_hub import snapshot_download
6
@@ -43,9 +44,10 @@ def server(zephyr_lora_files):
44
yield remote_server
45
46
46
-@pytest.fixture(scope="module")
47
-def client(server):
48
- return server.get_async_client()
47
+@pytest_asyncio.fixture
48
+async def client(server):
49
+ async with server.get_async_client() as async_client:
50
+ yield async_client
51
52
53
@pytest.mark.asyncio
tests/entrypoints/openai/test_return_tokens_as_ids.py
+51
-47
@@ -25,59 +25,63 @@ def server_with_return_tokens_as_token_ids_flag(
25
@pytest.mark.asyncio
26
async def test_completion_return_tokens_as_token_ids_completion(
27
server_with_return_tokens_as_token_ids_flag):
28
- client = server_with_return_tokens_as_token_ids_flag.get_async_client()
28
+ async with server_with_return_tokens_as_token_ids_flag.get_async_client(
29
+ ) as client:
30
30
- completion = await client.completions.create(
31
- model=MODEL_NAME,
32
- # Include Unicode characters to test for dividing a single
33
- # character across multiple tokens: 🎉 is [28705, 31862] for the
34
- # Zephyr tokenizer
35
- prompt="Say 'Hello, world! 🎉'",
36
- echo=True,
37
- temperature=0,
38
- max_tokens=10,
39
- logprobs=1)
31
+ completion = await client.completions.create(
32
+ model=MODEL_NAME,
33
+ # Include Unicode characters to test for dividing a single
34
+ # character across multiple tokens: 🎉 is [28705, 31862] for the
35
+ # Zephyr tokenizer
36
+ prompt="Say 'Hello, world! 🎉'",
37
+ echo=True,
38
+ temperature=0,
39
+ max_tokens=10,
40
+ logprobs=1)
41
41
- text = completion.choices[0].text
42
- token_strs = completion.choices[0].logprobs.tokens
43
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
44
- # Check that the token representations are consistent between raw tokens
45
- # and top_logprobs
46
- # Slice off the first one, because there's no scoring associated with BOS
47
- top_logprobs = completion.choices[0].logprobs.top_logprobs[1:]
48
- top_logprob_keys = [
49
- next(iter(logprob_by_tokens)) for logprob_by_tokens in top_logprobs
50
- ]
51
- assert token_strs[1:] == top_logprob_keys
42
+ text = completion.choices[0].text
43
+ token_strs = completion.choices[0].logprobs.tokens
44
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
45
+ # Check that the token representations are consistent between raw
46
+ # tokens and top_logprobs
47
+ # Slice off the first one, because there's no scoring associated
48
+ # with BOS
49
+ top_logprobs = completion.choices[0].logprobs.top_logprobs[1:]
50
+ top_logprob_keys = [
51
+ next(iter(logprob_by_tokens)) for logprob_by_tokens in top_logprobs
52
+ ]
53
+ assert token_strs[1:] == top_logprob_keys
54
53
- # Check that decoding the tokens gives the expected text
54
- tokens = [int(token.removeprefix("token_id:")) for token in token_strs]
55
- assert text == tokenizer.decode(tokens, skip_special_tokens=True)
55
+ # Check that decoding the tokens gives the expected text
56
+ tokens = [int(token.removeprefix("token_id:")) for token in token_strs]
57
+ assert text == tokenizer.decode(tokens, skip_special_tokens=True)
58
59
60
@pytest.mark.asyncio
61
async def test_chat_return_tokens_as_token_ids_completion(
62
server_with_return_tokens_as_token_ids_flag):
61
- client = server_with_return_tokens_as_token_ids_flag.get_async_client()
62
- response = await client.chat.completions.create(
63
- model=MODEL_NAME,
64
- # Include Unicode characters to test for dividing a single
65
- # character across multiple tokens: 🎉 is [28705, 31862] for the
66
- # Zephyr tokenizer
67
- messages=[{
68
- "role": "system",
69
- "content": "You like to respond in only emojis, like 🎉"
70
- }, {
71
- "role": "user",
72
- "content": "Please write some emojis: 🐱🐶🎉"
73
- }],
74
- temperature=0,
75
- max_tokens=8,
76
- logprobs=True)
63
+ async with server_with_return_tokens_as_token_ids_flag.get_async_client(
64
+ ) as client:
65
+ response = await client.chat.completions.create(
66
+ model=MODEL_NAME,
67
+ # Include Unicode characters to test for dividing a single
68
+ # character across multiple tokens: 🎉 is [28705, 31862] for the
69
+ # Zephyr tokenizer
70
+ messages=[{
71
+ "role": "system",
72
+ "content": "You like to respond in only emojis, like 🎉"
73
+ }, {
74
+ "role": "user",
75
+ "content": "Please write some emojis: 🐱🐶🎉"
76
+ }],
77
+ temperature=0,
78
+ max_tokens=8,
79
+ logprobs=True)
80
78
- text = response.choices[0].message.content
79
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
80
- token_ids = []
81
- for logprob_content in response.choices[0].logprobs.content:
82
- token_ids.append(int(logprob_content.token.removeprefix("token_id:")))
83
- assert tokenizer.decode(token_ids, skip_special_tokens=True) == text
81
+ text = response.choices[0].message.content
82
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
83
+ token_ids = []
84
+ for logprob_content in response.choices[0].logprobs.content:
85
+ token_ids.append(
86
+ int(logprob_content.token.removeprefix("token_id:")))
87
+ assert tokenizer.decode(token_ids, skip_special_tokens=True) == text
tests/entrypoints/openai/test_shutdown.py
+9
-8
@@ -35,13 +35,14 @@ async def test_shutdown_on_engine_failure(tmp_path):
35
]
36
37
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
38
- client = remote_server.get_async_client()
38
+ async with remote_server.get_async_client() as client:
39
40
- with pytest.raises(openai.APIConnectionError):
41
- # This crashes the engine
42
- await client.completions.create(model="bad-adapter",
43
- prompt="Hello, my name is")
40
+ with pytest.raises(
41
+ (openai.APIConnectionError, openai.InternalServerError)):
42
+ # This crashes the engine
43
+ await client.completions.create(model="bad-adapter",
44
+ prompt="Hello, my name is")
45
45
- # Now the server should shut down
46
- return_code = remote_server.proc.wait(timeout=1)
47
- assert return_code is not None
46
+ # Now the server should shut down
47
+ return_code = remote_server.proc.wait(timeout=3)
48
+ assert return_code is not None
tests/entrypoints/openai/test_tokenization.py
+5
-3
@@ -1,5 +1,6 @@
1
import openai # use the official client for correctness check
2
import pytest
3
+import pytest_asyncio
4
import requests
5
6
from vllm.transformers_utils.tokenizer import get_tokenizer
@@ -42,9 +43,10 @@ def tokenizer_name(model_name: str,
43
model_name == "zephyr-lora2") else model_name
44
45
45
-@pytest.fixture(scope="module")
46
-def client(server):
47
- return server.get_async_client()
46
+@pytest_asyncio.fixture
47
+async def client(server):
48
+ async with server.get_async_client() as async_client:
49
+ yield async_client
50
51
52
@pytest.mark.asyncio
tests/entrypoints/openai/test_vision.py
+5
-3
@@ -2,6 +2,7 @@ from typing import Dict, List
2
3
import openai
4
import pytest
5
+import pytest_asyncio
6
7
from vllm.multimodal.utils import encode_image_base64, fetch_image
8
@@ -36,9 +37,10 @@ def server():
37
yield remote_server
38
39
39
-@pytest.fixture(scope="module")
40
-def client(server):
41
- return server.get_async_client()
40
+@pytest_asyncio.fixture
41
+async def client(server):
42
+ async with server.get_async_client() as async_client:
43
+ yield async_client
44
45
46
@pytest.fixture(scope="session")
tests/multi_step/test_correctness_async_llm.py
+6
-6
@@ -28,12 +28,12 @@ async def completions_with_server_args(prompts: List[str], model_name: str,
28
29
outputs = None
30
with RemoteOpenAIServer(model_name, server_cli_args) as server:
31
- client = server.get_async_client()
32
- outputs = await client.completions.create(model=model_name,
33
- prompt=prompts,
34
- temperature=0,
35
- stream=False,
36
- max_tokens=5)
31
+ async with server.get_async_client() as client:
32
+ outputs = await client.completions.create(model=model_name,
33
+ prompt=prompts,
34
+ temperature=0,
35
+ stream=False,
36
+ max_tokens=5)
37
assert outputs is not None
38
39
return outputs
tests/utils.py
+1
@@ -154,6 +154,7 @@ class RemoteOpenAIServer:
154
return openai.AsyncOpenAI(
155
base_url=self.url_for("v1"),
156
api_key=self.DUMMY_API_KEY,
157
+ max_retries=0,
158
)
159
160