A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[CI/Build] Simplify OpenAI server setup in tests (#5100)
Cyrus Leung committed
Jun 14, 2024 at 02:21 UTC
39873476f8a1cf97bdf5651b4535ae60358ff15b
6 files changed
+285
-238
tests/async_engine/test_openapi_server_ray.py
+15
-16
@@ -4,16 +4,22 @@ import pytest
4
# and debugging.
5
import ray
6
7
-from ..utils import ServerRunner
7
+from ..utils import VLLM_PATH, RemoteOpenAIServer
8
9
# any model with a chat template should work here
10
MODEL_NAME = "facebook/opt-125m"
11
12
13
@pytest.fixture(scope="module")
14
-def server():
15
- ray.init()
16
- server_runner = ServerRunner.remote([
14
+def ray_ctx():
15
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
16
+ yield
17
+ ray.shutdown()
18
+
19
+
20
+@pytest.fixture(scope="module")
21
+def server(ray_ctx):
22
+ return RemoteOpenAIServer([
23
"--model",
24
MODEL_NAME,
25
# use half precision for speed and memory savings in CI environment
@@ -24,22 +30,15 @@ def server():
30
"--enforce-eager",
31
"--engine-use-ray"
32
])
27
- ray.get(server_runner.ready.remote())
28
- yield server_runner
29
- ray.shutdown()
33
34
35
@pytest.fixture(scope="module")
33
-def client():
34
- client = openai.AsyncOpenAI(
35
- base_url="http://localhost:8000/v1",
36
- api_key="token-abc123",
37
- )
38
- yield client
36
+def client(server):
37
+ return server.get_async_client()
38
39
40
@pytest.mark.asyncio
42
-async def test_check_models(server, client: openai.AsyncOpenAI):
41
+async def test_check_models(client: openai.AsyncOpenAI):
42
models = await client.models.list()
43
models = models.data
44
served_model = models[0]
@@ -48,7 +47,7 @@ async def test_check_models(server, client: openai.AsyncOpenAI):
47
48
49
@pytest.mark.asyncio
51
-async def test_single_completion(server, client: openai.AsyncOpenAI):
50
+async def test_single_completion(client: openai.AsyncOpenAI):
51
completion = await client.completions.create(model=MODEL_NAME,
52
prompt="Hello, my name is",
53
max_tokens=5,
@@ -72,7 +71,7 @@ async def test_single_completion(server, client: openai.AsyncOpenAI):
71
72
73
@pytest.mark.asyncio
75
-async def test_single_chat_session(server, client: openai.AsyncOpenAI):
74
+async def test_single_chat_session(client: openai.AsyncOpenAI):
75
messages = [{
76
"role": "system",
77
"content": "you are a helpful assistant"
tests/entrypoints/test_openai_embedding.py
new
+113
@@ -0,0 +1,113 @@
1
+import openai
2
+import pytest
3
+import ray
4
+
5
+from ..utils import VLLM_PATH, RemoteOpenAIServer
6
+
7
+EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
8
+
9
+pytestmark = pytest.mark.openai
10
+
11
+
12
+@pytest.fixture(scope="module")
13
+def ray_ctx():
14
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
15
+ yield
16
+ ray.shutdown()
17
+
18
+
19
+@pytest.fixture(scope="module")
20
+def embedding_server(ray_ctx):
21
+ return RemoteOpenAIServer([
22
+ "--model",
23
+ EMBEDDING_MODEL_NAME,
24
+ # use half precision for speed and memory savings in CI environment
25
+ "--dtype",
26
+ "bfloat16",
27
+ "--enforce-eager",
28
+ "--max-model-len",
29
+ "8192",
30
+ "--enforce-eager",
31
+ ])
32
+
33
+
34
+@pytest.mark.asyncio
35
+@pytest.fixture(scope="module")
36
+def embedding_client(embedding_server):
37
+ return embedding_server.get_async_client()
38
+
39
+
40
+@pytest.mark.asyncio
41
+@pytest.mark.parametrize(
42
+ "model_name",
43
+ [EMBEDDING_MODEL_NAME],
44
+)
45
+async def test_single_embedding(embedding_client: openai.AsyncOpenAI,
46
+ model_name: str):
47
+ input_texts = [
48
+ "The chef prepared a delicious meal.",
49
+ ]
50
+
51
+ # test single embedding
52
+ embeddings = await embedding_client.embeddings.create(
53
+ model=model_name,
54
+ input=input_texts,
55
+ encoding_format="float",
56
+ )
57
+ assert embeddings.id is not None
58
+ assert len(embeddings.data) == 1
59
+ assert len(embeddings.data[0].embedding) == 4096
60
+ assert embeddings.usage.completion_tokens == 0
61
+ assert embeddings.usage.prompt_tokens == 9
62
+ assert embeddings.usage.total_tokens == 9
63
+
64
+ # test using token IDs
65
+ input_tokens = [1, 1, 1, 1, 1]
66
+ embeddings = await embedding_client.embeddings.create(
67
+ model=model_name,
68
+ input=input_tokens,
69
+ encoding_format="float",
70
+ )
71
+ assert embeddings.id is not None
72
+ assert len(embeddings.data) == 1
73
+ assert len(embeddings.data[0].embedding) == 4096
74
+ assert embeddings.usage.completion_tokens == 0
75
+ assert embeddings.usage.prompt_tokens == 5
76
+ assert embeddings.usage.total_tokens == 5
77
+
78
+
79
+@pytest.mark.asyncio
80
+@pytest.mark.parametrize(
81
+ "model_name",
82
+ [EMBEDDING_MODEL_NAME],
83
+)
84
+async def test_batch_embedding(embedding_client: openai.AsyncOpenAI,
85
+ model_name: str):
86
+ # test List[str]
87
+ input_texts = [
88
+ "The cat sat on the mat.", "A feline was resting on a rug.",
89
+ "Stars twinkle brightly in the night sky."
90
+ ]
91
+ embeddings = await embedding_client.embeddings.create(
92
+ model=model_name,
93
+ input=input_texts,
94
+ encoding_format="float",
95
+ )
96
+ assert embeddings.id is not None
97
+ assert len(embeddings.data) == 3
98
+ assert len(embeddings.data[0].embedding) == 4096
99
+
100
+ # test List[List[int]]
101
+ input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
102
+ [25, 32, 64, 77]]
103
+ embeddings = await embedding_client.embeddings.create(
104
+ model=model_name,
105
+ input=input_tokens,
106
+ encoding_format="float",
107
+ )
108
+ assert embeddings.id is not None
109
+ assert len(embeddings.data) == 4
110
+ assert len(embeddings.data[0].embedding) == 4096
111
+ assert embeddings.usage.completion_tokens == 0
112
+ assert embeddings.usage.prompt_tokens == 17
113
+ assert embeddings.usage.total_tokens == 17
tests/entrypoints/test_openai_server.py
+47
-159
@@ -15,11 +15,10 @@ from openai import BadRequestError
15
16
from vllm.transformers_utils.tokenizer import get_tokenizer
17
18
-from ..utils import ServerRunner
18
+from ..utils import VLLM_PATH, RemoteOpenAIServer
19
20
# any model with a chat template should work here
21
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
22
-EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
22
# technically this needs Mistral-7B-v0.1 as base, but we're not testing
23
# generation quality here
24
LORA_NAME = "typeof/zephyr-7b-beta-lora"
@@ -80,9 +79,15 @@ def zephyr_lora_files():
79
80
81
@pytest.fixture(scope="module")
83
-def server(zephyr_lora_files):
84
- ray.init()
85
- server_runner = ServerRunner.remote([
82
+def ray_ctx():
83
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
84
+ yield
85
+ ray.shutdown()
86
+
87
+
88
+@pytest.fixture(scope="module")
89
+def server(zephyr_lora_files, ray_ctx):
90
+ return RemoteOpenAIServer([
91
"--model",
92
MODEL_NAME,
93
# use half precision for speed and memory savings in CI environment
@@ -91,8 +96,6 @@ def server(zephyr_lora_files):
96
"--max-model-len",
97
"8192",
98
"--enforce-eager",
94
- "--gpu-memory-utilization",
95
- "0.75",
99
# lora config below
100
"--enable-lora",
101
"--lora-modules",
@@ -105,43 +108,14 @@ def server(zephyr_lora_files):
108
"--max-num-seqs",
109
"128",
110
])
108
- ray.get(server_runner.ready.remote())
109
- yield server_runner
110
- ray.shutdown()
111
112
113
@pytest.fixture(scope="module")
114
-def embedding_server(zephyr_lora_files):
115
- ray.shutdown()
116
- ray.init()
117
- server_runner = ServerRunner.remote([
118
- "--model",
119
- EMBEDDING_MODEL_NAME,
120
- # use half precision for speed and memory savings in CI environment
121
- "--dtype",
122
- "bfloat16",
123
- "--enforce-eager",
124
- "--gpu-memory-utilization",
125
- "0.75",
126
- "--max-model-len",
127
- "8192",
128
- ])
129
- ray.get(server_runner.ready.remote())
130
- yield server_runner
131
- ray.shutdown()
132
-
133
-
134
-@pytest.fixture(scope="module")
135
-def client():
136
- client = openai.AsyncOpenAI(
137
- base_url="http://localhost:8000/v1",
138
- api_key="token-abc123",
139
- )
140
- yield client
114
+def client(server):
115
+ return server.get_async_client()
116
117
143
-@pytest.mark.asyncio
144
-async def test_check_models(server, client: openai.AsyncOpenAI):
118
+async def test_check_models(client: openai.AsyncOpenAI):
119
models = await client.models.list()
120
models = models.data
121
served_model = models[0]
@@ -158,8 +132,7 @@ async def test_check_models(server, client: openai.AsyncOpenAI):
132
"model_name",
133
[MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
134
)
161
-async def test_single_completion(server, client: openai.AsyncOpenAI,
162
- model_name: str):
135
+async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
136
completion = await client.completions.create(model=model_name,
137
prompt="Hello, my name is",
138
max_tokens=5,
@@ -190,8 +163,7 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
163
"model_name",
164
[MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
165
)
193
-async def test_no_logprobs(server, client: openai.AsyncOpenAI,
194
- model_name: str):
166
+async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
167
# test using token IDs
168
completion = await client.completions.create(
169
model=MODEL_NAME,
@@ -210,8 +182,7 @@ async def test_no_logprobs(server, client: openai.AsyncOpenAI,
182
"model_name",
183
[MODEL_NAME, "zephyr-lora"],
184
)
213
-async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
214
- model_name: str):
185
+async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
186
# test using token IDs
187
completion = await client.completions.create(
188
model=MODEL_NAME,
@@ -232,8 +203,7 @@ async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
203
"model_name",
204
[MODEL_NAME, "zephyr-lora"],
205
)
235
-async def test_some_logprobs(server, client: openai.AsyncOpenAI,
236
- model_name: str):
206
+async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
207
# test using token IDs
208
completion = await client.completions.create(
209
model=MODEL_NAME,
@@ -254,7 +224,7 @@ async def test_some_logprobs(server, client: openai.AsyncOpenAI,
224
"model_name",
225
[MODEL_NAME, "zephyr-lora"],
226
)
257
-async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
227
+async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
228
model_name: str):
229
230
with pytest.raises(
@@ -300,8 +270,7 @@ async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
270
"model_name",
271
[MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
272
)
303
-async def test_no_logprobs_chat(server, client: openai.AsyncOpenAI,
304
- model_name: str):
273
+async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
274
messages = [{
275
"role": "system",
276
"content": "you are a helpful assistant"
@@ -326,8 +295,7 @@ async def test_no_logprobs_chat(server, client: openai.AsyncOpenAI,
295
"model_name",
296
[MODEL_NAME, "zephyr-lora"],
297
)
329
-async def test_zero_logprobs_chat(server, client: openai.AsyncOpenAI,
330
- model_name: str):
298
+async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
299
messages = [{
300
"role": "system",
301
"content": "you are a helpful assistant"
@@ -354,8 +322,7 @@ async def test_zero_logprobs_chat(server, client: openai.AsyncOpenAI,
322
"model_name",
323
[MODEL_NAME, "zephyr-lora"],
324
)
357
-async def test_some_logprobs_chat(server, client: openai.AsyncOpenAI,
358
- model_name: str):
325
+async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
326
messages = [{
327
"role": "system",
328
"content": "you are a helpful assistant"
@@ -382,7 +349,7 @@ async def test_some_logprobs_chat(server, client: openai.AsyncOpenAI,
349
"model_name",
350
[MODEL_NAME, "zephyr-lora"],
351
)
385
-async def test_too_many_chat_logprobs(server, client: openai.AsyncOpenAI,
352
+async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
353
model_name: str):
354
messages = [{
355
"role": "system",
@@ -425,7 +392,7 @@ async def test_too_many_chat_logprobs(server, client: openai.AsyncOpenAI,
392
"model_name",
393
[MODEL_NAME, "zephyr-lora"],
394
)
428
-async def test_single_chat_session(server, client: openai.AsyncOpenAI,
395
+async def test_single_chat_session(client: openai.AsyncOpenAI,
396
model_name: str):
397
messages = [{
398
"role": "system",
@@ -470,7 +437,7 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
437
"model_name",
438
[MODEL_NAME, "zephyr-lora"],
439
)
473
-async def test_completion_streaming(server, client: openai.AsyncOpenAI,
440
+async def test_completion_streaming(client: openai.AsyncOpenAI,
441
model_name: str):
442
prompt = "What is an LLM?"
443
@@ -505,8 +472,7 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI,
472
"model_name",
473
[MODEL_NAME, "zephyr-lora"],
474
)
508
-async def test_chat_streaming(server, client: openai.AsyncOpenAI,
509
- model_name: str):
475
+async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
476
messages = [{
477
"role": "system",
478
"content": "you are a helpful assistant"
@@ -555,8 +521,7 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI,
521
"model_name",
522
["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
523
)
558
-async def test_chat_completion_stream_options(server,
559
- client: openai.AsyncOpenAI,
524
+async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
525
model_name: str):
526
messages = [{
527
"role": "system",
@@ -626,7 +591,7 @@ async def test_chat_completion_stream_options(server,
591
"model_name",
592
["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
593
)
629
-async def test_completion_stream_options(server, client: openai.AsyncOpenAI,
594
+async def test_completion_stream_options(client: openai.AsyncOpenAI,
595
model_name: str):
596
prompt = "What is the capital of France?"
597
@@ -688,8 +653,7 @@ async def test_completion_stream_options(server, client: openai.AsyncOpenAI,
653
"model_name",
654
[MODEL_NAME, "zephyr-lora"],
655
)
691
-async def test_batch_completions(server, client: openai.AsyncOpenAI,
692
- model_name: str):
656
+async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
657
# test simple list
658
batch = await client.completions.create(
659
model=model_name,
@@ -737,7 +701,7 @@ async def test_batch_completions(server, client: openai.AsyncOpenAI,
701
702
703
@pytest.mark.asyncio
740
-async def test_logits_bias(server, client: openai.AsyncOpenAI):
704
+async def test_logits_bias(client: openai.AsyncOpenAI):
705
prompt = "Hello, my name is"
706
max_tokens = 5
707
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
@@ -786,7 +750,7 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
750
@pytest.mark.asyncio
751
@pytest.mark.parametrize("guided_decoding_backend",
752
["outlines", "lm-format-enforcer"])
789
-async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
753
+async def test_guided_json_completion(client: openai.AsyncOpenAI,
754
guided_decoding_backend: str):
755
completion = await client.completions.create(
756
model=MODEL_NAME,
@@ -808,7 +772,7 @@ async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
772
@pytest.mark.asyncio
773
@pytest.mark.parametrize("guided_decoding_backend",
774
["outlines", "lm-format-enforcer"])
811
-async def test_guided_json_chat(server, client: openai.AsyncOpenAI,
775
+async def test_guided_json_chat(client: openai.AsyncOpenAI,
776
guided_decoding_backend: str):
777
messages = [{
778
"role": "system",
@@ -855,7 +819,7 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI,
819
@pytest.mark.asyncio
820
@pytest.mark.parametrize("guided_decoding_backend",
821
["outlines", "lm-format-enforcer"])
858
-async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
822
+async def test_guided_regex_completion(client: openai.AsyncOpenAI,
823
guided_decoding_backend: str):
824
completion = await client.completions.create(
825
model=MODEL_NAME,
@@ -875,7 +839,7 @@ async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
839
@pytest.mark.asyncio
840
@pytest.mark.parametrize("guided_decoding_backend",
841
["outlines", "lm-format-enforcer"])
878
-async def test_guided_regex_chat(server, client: openai.AsyncOpenAI,
842
+async def test_guided_regex_chat(client: openai.AsyncOpenAI,
843
guided_decoding_backend: str):
844
messages = [{
845
"role": "system",
@@ -913,7 +877,7 @@ async def test_guided_regex_chat(server, client: openai.AsyncOpenAI,
877
@pytest.mark.asyncio
878
@pytest.mark.parametrize("guided_decoding_backend",
879
["outlines", "lm-format-enforcer"])
916
-async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
880
+async def test_guided_choice_completion(client: openai.AsyncOpenAI,
881
guided_decoding_backend: str):
882
completion = await client.completions.create(
883
model=MODEL_NAME,
@@ -933,7 +897,7 @@ async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
897
@pytest.mark.asyncio
898
@pytest.mark.parametrize("guided_decoding_backend",
899
["outlines", "lm-format-enforcer"])
936
-async def test_guided_choice_chat(server, client: openai.AsyncOpenAI,
900
+async def test_guided_choice_chat(client: openai.AsyncOpenAI,
901
guided_decoding_backend: str):
902
messages = [{
903
"role": "system",
@@ -972,7 +936,7 @@ async def test_guided_choice_chat(server, client: openai.AsyncOpenAI,
936
@pytest.mark.asyncio
937
@pytest.mark.parametrize("guided_decoding_backend",
938
["outlines", "lm-format-enforcer"])
975
-async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
939
+async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
940
guided_decoding_backend: str):
941
with pytest.raises(openai.BadRequestError):
942
_ = await client.completions.create(
@@ -1008,7 +972,7 @@ async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
972
@pytest.mark.asyncio
973
@pytest.mark.parametrize("guided_decoding_backend",
974
["outlines", "lm-format-enforcer"])
1011
-async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
975
+async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
976
guided_decoding_backend: str):
977
messages = [{
978
"role": "system",
@@ -1040,7 +1004,7 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
1004
@pytest.mark.asyncio
1005
@pytest.mark.parametrize("guided_decoding_backend",
1006
["outlines", "lm-format-enforcer"])
1043
-async def test_named_tool_use(server, client: openai.AsyncOpenAI,
1007
+async def test_named_tool_use(client: openai.AsyncOpenAI,
1008
guided_decoding_backend: str):
1009
messages = [{
1010
"role": "system",
@@ -1131,7 +1095,7 @@ async def test_named_tool_use(server, client: openai.AsyncOpenAI,
1095
@pytest.mark.asyncio
1096
@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1097
async def test_required_tool_use_not_yet_supported(
1134
- server, client: openai.AsyncOpenAI, guided_decoding_backend: str):
1098
+ client: openai.AsyncOpenAI, guided_decoding_backend: str):
1099
messages = [{
1100
"role": "system",
1101
"content": "you are a helpful assistant"
@@ -1177,7 +1141,7 @@ async def test_required_tool_use_not_yet_supported(
1141
@pytest.mark.asyncio
1142
@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1143
async def test_inconsistent_tool_choice_and_tools(
1180
- server, client: openai.AsyncOpenAI, guided_decoding_backend: str):
1144
+ client: openai.AsyncOpenAI, guided_decoding_backend: str):
1145
messages = [{
1146
"role": "system",
1147
"content": "you are a helpful assistant"
@@ -1223,7 +1187,7 @@ async def test_inconsistent_tool_choice_and_tools(
1187
1188
1189
@pytest.mark.asyncio
1226
-async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
1190
+async def test_response_format_json_object(client: openai.AsyncOpenAI):
1191
for _ in range(2):
1192
resp = await client.chat.completions.create(
1193
model=MODEL_NAME,
@@ -1243,7 +1207,7 @@ async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
1207
1208
1209
@pytest.mark.asyncio
1246
-async def test_extra_fields(server, client: openai.AsyncOpenAI):
1210
+async def test_extra_fields(client: openai.AsyncOpenAI):
1211
with pytest.raises(BadRequestError) as exc_info:
1212
await client.chat.completions.create(
1213
model=MODEL_NAME,
@@ -1259,7 +1223,7 @@ async def test_extra_fields(server, client: openai.AsyncOpenAI):
1223
1224
1225
@pytest.mark.asyncio
1262
-async def test_complex_message_content(server, client: openai.AsyncOpenAI):
1226
+async def test_complex_message_content(client: openai.AsyncOpenAI):
1227
resp = await client.chat.completions.create(
1228
model=MODEL_NAME,
1229
messages=[{
@@ -1279,7 +1243,7 @@ async def test_complex_message_content(server, client: openai.AsyncOpenAI):
1243
1244
1245
@pytest.mark.asyncio
1282
-async def test_custom_role(server, client: openai.AsyncOpenAI):
1246
+async def test_custom_role(client: openai.AsyncOpenAI):
1247
# Not sure how the model handles custom roles so we just check that
1248
# both string and complex message content are handled in the same way
1249
@@ -1310,7 +1274,7 @@ async def test_custom_role(server, client: openai.AsyncOpenAI):
1274
1275
1276
@pytest.mark.asyncio
1313
-async def test_guided_grammar(server, client: openai.AsyncOpenAI):
1277
+async def test_guided_grammar(client: openai.AsyncOpenAI):
1278
simple_sql_grammar = """
1279
start: select_statement
1280
@@ -1351,7 +1315,7 @@ number: "1" | "2"
1315
[MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
1316
)
1317
@pytest.mark.parametrize("logprobs_arg", [1, 0])
1354
-async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1318
+async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
1319
model_name: str, logprobs_arg: int):
1320
tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
1321
# test using text and token IDs
@@ -1380,7 +1344,7 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1344
1345
1346
@pytest.mark.asyncio
1383
-async def test_long_seed(server, client: openai.AsyncOpenAI):
1347
+async def test_long_seed(client: openai.AsyncOpenAI):
1348
for seed in [
1349
torch.iinfo(torch.long).min - 1,
1350
torch.iinfo(torch.long).max + 1
@@ -1399,81 +1363,5 @@ async def test_long_seed(server, client: openai.AsyncOpenAI):
1363
or "less_than_equal" in exc_info.value.message)
1364
1365
1402
-@pytest.mark.asyncio
1403
-@pytest.mark.parametrize(
1404
- "model_name",
1405
- [EMBEDDING_MODEL_NAME],
1406
-)
1407
-async def test_single_embedding(embedding_server, client: openai.AsyncOpenAI,
1408
- model_name: str):
1409
- input_texts = [
1410
- "The chef prepared a delicious meal.",
1411
- ]
1412
-
1413
- # test single embedding
1414
- embeddings = await client.embeddings.create(
1415
- model=model_name,
1416
- input=input_texts,
1417
- encoding_format="float",
1418
- )
1419
- assert embeddings.id is not None
1420
- assert len(embeddings.data) == 1
1421
- assert len(embeddings.data[0].embedding) == 4096
1422
- assert embeddings.usage.completion_tokens == 0
1423
- assert embeddings.usage.prompt_tokens == 9
1424
- assert embeddings.usage.total_tokens == 9
1425
-
1426
- # test using token IDs
1427
- input_tokens = [1, 1, 1, 1, 1]
1428
- embeddings = await client.embeddings.create(
1429
- model=model_name,
1430
- input=input_tokens,
1431
- encoding_format="float",
1432
- )
1433
- assert embeddings.id is not None
1434
- assert len(embeddings.data) == 1
1435
- assert len(embeddings.data[0].embedding) == 4096
1436
- assert embeddings.usage.completion_tokens == 0
1437
- assert embeddings.usage.prompt_tokens == 5
1438
- assert embeddings.usage.total_tokens == 5
1439
-
1440
-
1441
-@pytest.mark.asyncio
1442
-@pytest.mark.parametrize(
1443
- "model_name",
1444
- [EMBEDDING_MODEL_NAME],
1445
-)
1446
-async def test_batch_embedding(embedding_server, client: openai.AsyncOpenAI,
1447
- model_name: str):
1448
- # test List[str]
1449
- input_texts = [
1450
- "The cat sat on the mat.", "A feline was resting on a rug.",
1451
- "Stars twinkle brightly in the night sky."
1452
- ]
1453
- embeddings = await client.embeddings.create(
1454
- model=model_name,
1455
- input=input_texts,
1456
- encoding_format="float",
1457
- )
1458
- assert embeddings.id is not None
1459
- assert len(embeddings.data) == 3
1460
- assert len(embeddings.data[0].embedding) == 4096
1461
-
1462
- # test List[List[int]]
1463
- input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
1464
- [25, 32, 64, 77]]
1465
- embeddings = await client.embeddings.create(
1466
- model=model_name,
1467
- input=input_tokens,
1468
- encoding_format="float",
1469
- )
1470
- assert embeddings.id is not None
1471
- assert len(embeddings.data) == 4
1472
- assert len(embeddings.data[0].embedding) == 4096
1473
- assert embeddings.usage.completion_tokens == 0
1474
- assert embeddings.usage.prompt_tokens == 17
1475
- assert embeddings.usage.total_tokens == 17
1476
-
1477
-
1366
if __name__ == "__main__":
1367
pytest.main([__file__])
tests/entrypoints/test_openai_vision.py
+17
-18
@@ -8,7 +8,7 @@ import ray
8
9
from vllm.multimodal.utils import ImageFetchAiohttp, encode_image_base64
10
11
-from ..utils import ServerRunner
11
+from ..utils import VLLM_PATH, RemoteOpenAIServer
12
13
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
14
LLAVA_CHAT_TEMPLATE = (Path(__file__).parent.parent.parent /
@@ -25,10 +25,16 @@ TEST_IMAGE_URLS = [
25
pytestmark = pytest.mark.openai
26
27
28
+@pytest.fixture(scope="module")
29
+def ray_ctx():
30
+ ray.init(runtime_env={"working_dir": VLLM_PATH})
31
+ yield
32
+ ray.shutdown()
33
+
34
+
35
@pytest.fixture(scope="module")
36
def server():
30
- ray.init()
31
- server_runner = ServerRunner.remote([
37
+ return RemoteOpenAIServer([
38
"--model",
39
MODEL_NAME,
40
"--dtype",
@@ -47,18 +53,11 @@ def server():
53
"--chat-template",
54
str(LLAVA_CHAT_TEMPLATE),
55
])
50
- ray.get(server_runner.ready.remote())
51
- yield server_runner
52
- ray.shutdown()
56
57
55
-@pytest.fixture(scope="session")
56
-def client():
57
- client = openai.AsyncOpenAI(
58
- base_url="http://localhost:8000/v1",
59
- api_key="token-abc123",
60
- )
61
- yield client
58
+@pytest.fixture(scope="module")
59
+def client(server):
60
+ return server.get_async_client()
61
62
63
@pytest_asyncio.fixture(scope="session")
@@ -73,7 +72,7 @@ async def base64_encoded_image() -> Dict[str, str]:
72
@pytest.mark.asyncio
73
@pytest.mark.parametrize("model_name", [MODEL_NAME])
74
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
76
-async def test_single_chat_session_image(server, client: openai.AsyncOpenAI,
75
+async def test_single_chat_session_image(client: openai.AsyncOpenAI,
76
model_name: str, image_url: str):
77
messages = [{
78
"role":
@@ -126,7 +125,7 @@ async def test_single_chat_session_image(server, client: openai.AsyncOpenAI,
125
@pytest.mark.parametrize("model_name", [MODEL_NAME])
126
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
127
async def test_single_chat_session_image_base64encoded(
129
- server, client: openai.AsyncOpenAI, model_name: str, image_url: str,
128
+ client: openai.AsyncOpenAI, model_name: str, image_url: str,
129
base64_encoded_image: Dict[str, str]):
130
131
messages = [{
@@ -180,7 +179,7 @@ async def test_single_chat_session_image_base64encoded(
179
@pytest.mark.asyncio
180
@pytest.mark.parametrize("model_name", [MODEL_NAME])
181
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
183
-async def test_chat_streaming_image(server, client: openai.AsyncOpenAI,
182
+async def test_chat_streaming_image(client: openai.AsyncOpenAI,
183
model_name: str, image_url: str):
184
messages = [{
185
"role":
@@ -237,8 +236,8 @@ async def test_chat_streaming_image(server, client: openai.AsyncOpenAI,
236
@pytest.mark.asyncio
237
@pytest.mark.parametrize("model_name", [MODEL_NAME])
238
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
240
-async def test_multi_image_input(server, client: openai.AsyncOpenAI,
241
- model_name: str, image_url: str):
239
+async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
240
+ image_url: str):
241
242
messages = [{
243
"role":
tests/tensorizer_loader/test_tensorizer.py
+5
-9
@@ -22,11 +22,12 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
22
tensorize_vllm_model)
23
24
from ..conftest import VllmRunner, cleanup
25
-from ..utils import ServerRunner
25
+from ..utils import RemoteOpenAIServer
26
27
# yapf conflicts with isort for this docstring
28
29
30
+
31
prompts = [
32
"Hello, my name is",
33
"The president of the United States is",
@@ -216,18 +217,13 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
217
openai_args = [
218
"--model", model_ref, "--dtype", "float16", "--load-format",
219
"tensorizer", "--model-loader-extra-config",
219
- json.dumps(model_loader_extra_config), "--port", "8000"
220
+ json.dumps(model_loader_extra_config),
221
]
222
222
- server = ServerRunner.remote(openai_args)
223
-
224
- assert ray.get(server.ready.remote())
223
+ server = RemoteOpenAIServer(openai_args)
224
print("Server ready.")
225
227
- client = openai.OpenAI(
228
- base_url="http://localhost:8000/v1",
229
- api_key="token-abc123",
230
- )
226
+ client = server.get_client()
227
completion = client.completions.create(model=model_ref,
228
prompt="Hello, my name is",
229
max_tokens=5,
tests/utils.py
+88
-36
@@ -4,57 +4,109 @@ import sys
4
import time
5
import warnings
6
from contextlib import contextmanager
7
+from typing import List
8
9
+import openai
10
import ray
11
import requests
12
13
from vllm.distributed import (ensure_model_parallel_initialized,
14
init_distributed_environment)
15
+from vllm.entrypoints.openai.cli_args import make_arg_parser
16
from vllm.utils import get_open_port
17
18
# Path to root of repository so that utilities can be imported by ray workers
19
VLLM_PATH = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))
20
21
19
-@ray.remote(num_gpus=1)
20
-class ServerRunner:
22
+class RemoteOpenAIServer:
23
+ DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
24
MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
25
23
- def __init__(self, args):
24
- env = os.environ.copy()
25
- env["PYTHONUNBUFFERED"] = "1"
26
- self.proc = subprocess.Popen(
27
- [sys.executable, "-m", "vllm.entrypoints.openai.api_server"] +
28
- args,
29
- env=env,
30
- stdout=sys.stdout,
31
- stderr=sys.stderr,
26
+ @ray.remote(num_gpus=1)
27
+ class _RemoteRunner:
28
+
29
+ def __init__(self, cli_args: List[str], *, wait_url: str,
30
+ wait_timeout: float) -> None:
31
+ env = os.environ.copy()
32
+ env["PYTHONUNBUFFERED"] = "1"
33
+ self.proc = subprocess.Popen(
34
+ [
35
+ sys.executable, "-m", "vllm.entrypoints.openai.api_server",
36
+ *cli_args
37
+ ],
38
+ env=env,
39
+ stdout=sys.stdout,
40
+ stderr=sys.stderr,
41
+ )
42
+
43
+ self._wait_for_server(url=wait_url, timeout=wait_timeout)
44
+
45
+ def ready(self):
46
+ return True
47
+
48
+ def _wait_for_server(self, *, url: str, timeout: float):
49
+ # run health check
50
+ start = time.time()
51
+ while True:
52
+ try:
53
+ if requests.get(url).status_code == 200:
54
+ break
55
+ except Exception as err:
56
+ if self.proc.poll() is not None:
57
+ raise RuntimeError(
58
+ "Server exited unexpectedly.") from err
59
+
60
+ time.sleep(0.5)
61
+ if time.time() - start > timeout:
62
+ raise RuntimeError(
63
+ "Server failed to start in time.") from err
64
+
65
+ def __del__(self):
66
+ if hasattr(self, "proc"):
67
+ self.proc.terminate()
68
+
69
+ def __init__(self, cli_args: List[str], *, auto_port: bool = True) -> None:
70
+ if auto_port:
71
+ if "-p" in cli_args or "--port" in cli_args:
72
+ raise ValueError("You have manually specified the port"
73
+ "when `auto_port=True`.")
74
+
75
+ cli_args = cli_args + ["--port", str(get_open_port())]
76
+
77
+ parser = make_arg_parser()
78
+ args = parser.parse_args(cli_args)
79
+ self.host = str(args.host or 'localhost')
80
+ self.port = int(args.port)
81
+
82
+ self._runner = self._RemoteRunner.remote(
83
+ cli_args,
84
+ wait_url=self.url_for("health"),
85
+ wait_timeout=self.MAX_SERVER_START_WAIT_S)
86
+
87
+ self._wait_until_ready()
88
+
89
+ @property
90
+ def url_root(self) -> str:
91
+ return f"http://{self.host}:{self.port}"
92
+
93
+ def url_for(self, *parts: str) -> str:
94
+ return self.url_root + "/" + "/".join(parts)
95
+
96
+ def _wait_until_ready(self) -> None:
97
+ ray.get(self._runner.ready.remote())
98
+
99
+ def get_client(self):
100
+ return openai.OpenAI(
101
+ base_url=self.url_for("v1"),
102
+ api_key=self.DUMMY_API_KEY,
103
+ )
104
+
105
+ def get_async_client(self):
106
+ return openai.AsyncOpenAI(
107
+ base_url=self.url_for("v1"),
108
+ api_key=self.DUMMY_API_KEY,
109
)
33
- self._wait_for_server()
34
-
35
- def ready(self):
36
- return True
37
-
38
- def _wait_for_server(self):
39
- # run health check
40
- start = time.time()
41
- while True:
42
- try:
43
- if requests.get(
44
- "http://localhost:8000/health").status_code == 200:
45
- break
46
- except Exception as err:
47
- if self.proc.poll() is not None:
48
- raise RuntimeError("Server exited unexpectedly.") from err
49
-
50
- time.sleep(0.5)
51
- if time.time() - start > self.MAX_SERVER_START_WAIT_S:
52
- raise RuntimeError(
53
- "Server failed to start in time.") from err
54
-
55
- def __del__(self):
56
- if hasattr(self, "proc"):
57
- self.proc.terminate()
110
111
112
def init_test_distributed_environment(