@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 39178c7fbc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Tests] Disable retries and use context manager for openai client (#7565)

Nick Hill committed Aug 26, 2024 at 21:33 UTC 39178c7fbc6ab47b5448db101476c36d0ed38d7a
15 files changed +130 -93
tests/async_engine/test_openapi_server_ray.py
+5 -3
@@ -1,5 +1,6 @@
1 import openai # use the official client for correctness check
2 import pytest
3 +import pytest_asyncio
4
5 from ..utils import VLLM_PATH, RemoteOpenAIServer
6
@@ -31,9 +32,10 @@ def server():
32 yield remote_server
33
34
34 -@pytest.fixture(scope="module")
35 -def client(server):
36 - return server.get_async_client()
35 +@pytest_asyncio.fixture
36 +async def client(server):
37 + async with server.get_async_client() as async_client:
38 + yield async_client
39
40
41 @pytest.mark.asyncio
tests/entrypoints/openai/test_audio.py
+5 -3
@@ -2,6 +2,7 @@ from typing import Dict, List
2
3 import openai
4 import pytest
5 +import pytest_asyncio
6
7 from vllm.assets.audio import AudioAsset
8 from vllm.multimodal.utils import encode_audio_base64, fetch_audio
@@ -28,9 +29,10 @@ def server():
29 yield remote_server
30
31
31 -@pytest.fixture(scope="module")
32 -def client(server):
33 - return server.get_async_client()
32 +@pytest_asyncio.fixture
33 +async def client(server):
34 + async with server.get_async_client() as async_client:
35 + yield async_client
36
37
38 @pytest.fixture(scope="session")
tests/entrypoints/openai/test_basic.py
+5 -3
@@ -2,6 +2,7 @@ from http import HTTPStatus
2
3 import openai
4 import pytest
5 +import pytest_asyncio
6 import requests
7
8 from vllm.version import __version__ as VLLM_VERSION
@@ -28,9 +29,10 @@ def server():
29 yield remote_server
30
31
31 -@pytest.fixture(scope="module")
32 -def client(server):
33 - return server.get_async_client()
32 +@pytest_asyncio.fixture
33 +async def client(server):
34 + async with server.get_async_client() as async_client:
35 + yield async_client
36
37
38 @pytest.mark.asyncio
tests/entrypoints/openai/test_chat.py
+5 -3
@@ -6,6 +6,7 @@ from typing import Dict, List, Optional
6 import jsonschema
7 import openai # use the official client for correctness check
8 import pytest
9 +import pytest_asyncio
10 import torch
11 from openai import BadRequestError
12
@@ -46,9 +47,10 @@ def server(zephyr_lora_files, zephyr_lora_added_tokens_files): # noqa: F811
47 yield remote_server
48
49
49 -@pytest.fixture(scope="module")
50 -def client(server):
51 - return server.get_async_client()
50 +@pytest_asyncio.fixture
51 +async def client(server):
52 + async with server.get_async_client() as async_client:
53 + yield async_client
54
55
56 @pytest.mark.asyncio
tests/entrypoints/openai/test_completion.py
+9 -2
@@ -8,6 +8,7 @@ from typing import Dict, List, Optional
8 import jsonschema
9 import openai # use the official client for correctness check
10 import pytest
11 +import pytest_asyncio
12 # downloading lora to test lora requests
13 from huggingface_hub import snapshot_download
14 from openai import BadRequestError
@@ -89,11 +90,17 @@ def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files,
90
91 @pytest.fixture(scope="module",
92 params=["", "--disable-frontend-multiprocessing"])
92 -def client(default_server_args, request):
93 +def server(default_server_args, request):
94 if request.param:
95 default_server_args.append(request.param)
96 with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
96 - yield remote_server.get_async_client()
97 + yield remote_server
98 +
99 +
100 +@pytest_asyncio.fixture
101 +async def client(server):
102 + async with server.get_async_client() as async_client:
103 + yield async_client
104
105
106 @pytest.mark.asyncio
tests/entrypoints/openai/test_embedding.py
+5 -4
@@ -3,6 +3,7 @@ import base64
3 import numpy as np
4 import openai
5 import pytest
6 +import pytest_asyncio
7
8 from ...utils import RemoteOpenAIServer
9
@@ -24,10 +25,10 @@ def embedding_server():
25 yield remote_server
26
27
27 -@pytest.mark.asyncio
28 -@pytest.fixture(scope="module")
29 -def embedding_client(embedding_server):
30 - return embedding_server.get_async_client()
28 +@pytest_asyncio.fixture
29 +async def embedding_client(embedding_server):
30 + async with embedding_server.get_async_client() as async_client:
31 + yield async_client
32
33
34 @pytest.mark.asyncio
tests/entrypoints/openai/test_encoder_decoder.py
+5 -3
@@ -1,5 +1,6 @@
1 import openai
2 import pytest
3 +import pytest_asyncio
4
5 from ...utils import RemoteOpenAIServer
6
@@ -18,9 +19,10 @@ def server():
19 yield remote_server
20
21
21 -@pytest.fixture(scope="module")
22 -def client(server):
23 - return server.get_async_client()
22 +@pytest_asyncio.fixture
23 +async def client(server):
24 + async with server.get_async_client() as async_client:
25 + yield async_client
26
27
28 @pytest.mark.asyncio
tests/entrypoints/openai/test_metrics.py
+9 -2
@@ -6,6 +6,7 @@ from http import HTTPStatus
6
7 import openai
8 import pytest
9 +import pytest_asyncio
10 import requests
11 from prometheus_client.parser import text_string_to_metric_families
12 from transformers import AutoTokenizer
@@ -35,11 +36,17 @@ def default_server_args():
36 "--enable-chunked-prefill",
37 "--disable-frontend-multiprocessing",
38 ])
38 -def client(default_server_args, request):
39 +def server(default_server_args, request):
40 if request.param:
41 default_server_args.append(request.param)
42 with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
42 - yield remote_server.get_async_client()
43 + yield remote_server
44 +
45 +
46 +@pytest_asyncio.fixture
47 +async def client(server):
48 + async with server.get_async_client() as cl:
49 + yield cl
50
51
52 _PROMPT = "Hello my name is Robert and I love magic"
tests/entrypoints/openai/test_models.py
+5 -3
@@ -1,5 +1,6 @@
1 import openai # use the official client for correctness check
2 import pytest
3 +import pytest_asyncio
4 # downloading lora to test lora requests
5 from huggingface_hub import snapshot_download
6
@@ -43,9 +44,10 @@ def server(zephyr_lora_files):
44 yield remote_server
45
46
46 -@pytest.fixture(scope="module")
47 -def client(server):
48 - return server.get_async_client()
47 +@pytest_asyncio.fixture
48 +async def client(server):
49 + async with server.get_async_client() as async_client:
50 + yield async_client
51
52
53 @pytest.mark.asyncio
tests/entrypoints/openai/test_return_tokens_as_ids.py
+51 -47
@@ -25,59 +25,63 @@ def server_with_return_tokens_as_token_ids_flag(
25 @pytest.mark.asyncio
26 async def test_completion_return_tokens_as_token_ids_completion(
27 server_with_return_tokens_as_token_ids_flag):
28 - client = server_with_return_tokens_as_token_ids_flag.get_async_client()
28 + async with server_with_return_tokens_as_token_ids_flag.get_async_client(
29 + ) as client:
30
30 - completion = await client.completions.create(
31 - model=MODEL_NAME,
32 - # Include Unicode characters to test for dividing a single
33 - # character across multiple tokens: 🎉 is [28705, 31862] for the
34 - # Zephyr tokenizer
35 - prompt="Say 'Hello, world! 🎉'",
36 - echo=True,
37 - temperature=0,
38 - max_tokens=10,
39 - logprobs=1)
31 + completion = await client.completions.create(
32 + model=MODEL_NAME,
33 + # Include Unicode characters to test for dividing a single
34 + # character across multiple tokens: 🎉 is [28705, 31862] for the
35 + # Zephyr tokenizer
36 + prompt="Say 'Hello, world! 🎉'",
37 + echo=True,
38 + temperature=0,
39 + max_tokens=10,
40 + logprobs=1)
41
41 - text = completion.choices[0].text
42 - token_strs = completion.choices[0].logprobs.tokens
43 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
44 - # Check that the token representations are consistent between raw tokens
45 - # and top_logprobs
46 - # Slice off the first one, because there's no scoring associated with BOS
47 - top_logprobs = completion.choices[0].logprobs.top_logprobs[1:]
48 - top_logprob_keys = [
49 - next(iter(logprob_by_tokens)) for logprob_by_tokens in top_logprobs
50 - ]
51 - assert token_strs[1:] == top_logprob_keys
42 + text = completion.choices[0].text
43 + token_strs = completion.choices[0].logprobs.tokens
44 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
45 + # Check that the token representations are consistent between raw
46 + # tokens and top_logprobs
47 + # Slice off the first one, because there's no scoring associated
48 + # with BOS
49 + top_logprobs = completion.choices[0].logprobs.top_logprobs[1:]
50 + top_logprob_keys = [
51 + next(iter(logprob_by_tokens)) for logprob_by_tokens in top_logprobs
52 + ]
53 + assert token_strs[1:] == top_logprob_keys
54
53 - # Check that decoding the tokens gives the expected text
54 - tokens = [int(token.removeprefix("token_id:")) for token in token_strs]
55 - assert text == tokenizer.decode(tokens, skip_special_tokens=True)
55 + # Check that decoding the tokens gives the expected text
56 + tokens = [int(token.removeprefix("token_id:")) for token in token_strs]
57 + assert text == tokenizer.decode(tokens, skip_special_tokens=True)
58
59
60 @pytest.mark.asyncio
61 async def test_chat_return_tokens_as_token_ids_completion(
62 server_with_return_tokens_as_token_ids_flag):
61 - client = server_with_return_tokens_as_token_ids_flag.get_async_client()
62 - response = await client.chat.completions.create(
63 - model=MODEL_NAME,
64 - # Include Unicode characters to test for dividing a single
65 - # character across multiple tokens: 🎉 is [28705, 31862] for the
66 - # Zephyr tokenizer
67 - messages=[{
68 - "role": "system",
69 - "content": "You like to respond in only emojis, like 🎉"
70 - }, {
71 - "role": "user",
72 - "content": "Please write some emojis: 🐱🐶🎉"
73 - }],
74 - temperature=0,
75 - max_tokens=8,
76 - logprobs=True)
63 + async with server_with_return_tokens_as_token_ids_flag.get_async_client(
64 + ) as client:
65 + response = await client.chat.completions.create(
66 + model=MODEL_NAME,
67 + # Include Unicode characters to test for dividing a single
68 + # character across multiple tokens: 🎉 is [28705, 31862] for the
69 + # Zephyr tokenizer
70 + messages=[{
71 + "role": "system",
72 + "content": "You like to respond in only emojis, like 🎉"
73 + }, {
74 + "role": "user",
75 + "content": "Please write some emojis: 🐱🐶🎉"
76 + }],
77 + temperature=0,
78 + max_tokens=8,
79 + logprobs=True)
80
78 - text = response.choices[0].message.content
79 - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
80 - token_ids = []
81 - for logprob_content in response.choices[0].logprobs.content:
82 - token_ids.append(int(logprob_content.token.removeprefix("token_id:")))
83 - assert tokenizer.decode(token_ids, skip_special_tokens=True) == text
81 + text = response.choices[0].message.content
82 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
83 + token_ids = []
84 + for logprob_content in response.choices[0].logprobs.content:
85 + token_ids.append(
86 + int(logprob_content.token.removeprefix("token_id:")))
87 + assert tokenizer.decode(token_ids, skip_special_tokens=True) == text
tests/entrypoints/openai/test_shutdown.py
+9 -8
@@ -35,13 +35,14 @@ async def test_shutdown_on_engine_failure(tmp_path):
35 ]
36
37 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
38 - client = remote_server.get_async_client()
38 + async with remote_server.get_async_client() as client:
39
40 - with pytest.raises(openai.APIConnectionError):
41 - # This crashes the engine
42 - await client.completions.create(model="bad-adapter",
43 - prompt="Hello, my name is")
40 + with pytest.raises(
41 + (openai.APIConnectionError, openai.InternalServerError)):
42 + # This crashes the engine
43 + await client.completions.create(model="bad-adapter",
44 + prompt="Hello, my name is")
45
45 - # Now the server should shut down
46 - return_code = remote_server.proc.wait(timeout=1)
47 - assert return_code is not None
46 + # Now the server should shut down
47 + return_code = remote_server.proc.wait(timeout=3)
48 + assert return_code is not None
tests/entrypoints/openai/test_tokenization.py
+5 -3
@@ -1,5 +1,6 @@
1 import openai # use the official client for correctness check
2 import pytest
3 +import pytest_asyncio
4 import requests
5
6 from vllm.transformers_utils.tokenizer import get_tokenizer
@@ -42,9 +43,10 @@ def tokenizer_name(model_name: str,
43 model_name == "zephyr-lora2") else model_name
44
45
45 -@pytest.fixture(scope="module")
46 -def client(server):
47 - return server.get_async_client()
46 +@pytest_asyncio.fixture
47 +async def client(server):
48 + async with server.get_async_client() as async_client:
49 + yield async_client
50
51
52 @pytest.mark.asyncio
tests/entrypoints/openai/test_vision.py
+5 -3
@@ -2,6 +2,7 @@ from typing import Dict, List
2
3 import openai
4 import pytest
5 +import pytest_asyncio
6
7 from vllm.multimodal.utils import encode_image_base64, fetch_image
8
@@ -36,9 +37,10 @@ def server():
37 yield remote_server
38
39
39 -@pytest.fixture(scope="module")
40 -def client(server):
41 - return server.get_async_client()
40 +@pytest_asyncio.fixture
41 +async def client(server):
42 + async with server.get_async_client() as async_client:
43 + yield async_client
44
45
46 @pytest.fixture(scope="session")
tests/multi_step/test_correctness_async_llm.py
+6 -6
@@ -28,12 +28,12 @@ async def completions_with_server_args(prompts: List[str], model_name: str,
28
29 outputs = None
30 with RemoteOpenAIServer(model_name, server_cli_args) as server:
31 - client = server.get_async_client()
32 - outputs = await client.completions.create(model=model_name,
33 - prompt=prompts,
34 - temperature=0,
35 - stream=False,
36 - max_tokens=5)
31 + async with server.get_async_client() as client:
32 + outputs = await client.completions.create(model=model_name,
33 + prompt=prompts,
34 + temperature=0,
35 + stream=False,
36 + max_tokens=5)
37 assert outputs is not None
38
39 return outputs
tests/utils.py
+1
@@ -154,6 +154,7 @@ class RemoteOpenAIServer:
154 return openai.AsyncOpenAI(
155 base_url=self.url_for("v1"),
156 api_key=self.DUMMY_API_KEY,
157 + max_retries=0,
158 )
159
160