@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 39873476f8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[CI/Build] Simplify OpenAI server setup in tests (#5100)

Cyrus Leung committed Jun 14, 2024 at 02:21 UTC 39873476f8a1cf97bdf5651b4535ae60358ff15b
6 files changed +285 -238
tests/async_engine/test_openapi_server_ray.py
+15 -16
@@ -4,16 +4,22 @@ import pytest
4 # and debugging.
5 import ray
6
7 -from ..utils import ServerRunner
7 +from ..utils import VLLM_PATH, RemoteOpenAIServer
8
9 # any model with a chat template should work here
10 MODEL_NAME = "facebook/opt-125m"
11
12
13 @pytest.fixture(scope="module")
14 -def server():
15 - ray.init()
16 - server_runner = ServerRunner.remote([
14 +def ray_ctx():
15 + ray.init(runtime_env={"working_dir": VLLM_PATH})
16 + yield
17 + ray.shutdown()
18 +
19 +
20 +@pytest.fixture(scope="module")
21 +def server(ray_ctx):
22 + return RemoteOpenAIServer([
23 "--model",
24 MODEL_NAME,
25 # use half precision for speed and memory savings in CI environment
@@ -24,22 +30,15 @@ def server():
30 "--enforce-eager",
31 "--engine-use-ray"
32 ])
27 - ray.get(server_runner.ready.remote())
28 - yield server_runner
29 - ray.shutdown()
33
34
35 @pytest.fixture(scope="module")
33 -def client():
34 - client = openai.AsyncOpenAI(
35 - base_url="http://localhost:8000/v1",
36 - api_key="token-abc123",
37 - )
38 - yield client
36 +def client(server):
37 + return server.get_async_client()
38
39
40 @pytest.mark.asyncio
42 -async def test_check_models(server, client: openai.AsyncOpenAI):
41 +async def test_check_models(client: openai.AsyncOpenAI):
42 models = await client.models.list()
43 models = models.data
44 served_model = models[0]
@@ -48,7 +47,7 @@ async def test_check_models(server, client: openai.AsyncOpenAI):
47
48
49 @pytest.mark.asyncio
51 -async def test_single_completion(server, client: openai.AsyncOpenAI):
50 +async def test_single_completion(client: openai.AsyncOpenAI):
51 completion = await client.completions.create(model=MODEL_NAME,
52 prompt="Hello, my name is",
53 max_tokens=5,
@@ -72,7 +71,7 @@ async def test_single_completion(server, client: openai.AsyncOpenAI):
71
72
73 @pytest.mark.asyncio
75 -async def test_single_chat_session(server, client: openai.AsyncOpenAI):
74 +async def test_single_chat_session(client: openai.AsyncOpenAI):
75 messages = [{
76 "role": "system",
77 "content": "you are a helpful assistant"
tests/entrypoints/test_openai_embedding.py new
+113
@@ -0,0 +1,113 @@
1 +import openai
2 +import pytest
3 +import ray
4 +
5 +from ..utils import VLLM_PATH, RemoteOpenAIServer
6 +
7 +EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
8 +
9 +pytestmark = pytest.mark.openai
10 +
11 +
12 +@pytest.fixture(scope="module")
13 +def ray_ctx():
14 + ray.init(runtime_env={"working_dir": VLLM_PATH})
15 + yield
16 + ray.shutdown()
17 +
18 +
19 +@pytest.fixture(scope="module")
20 +def embedding_server(ray_ctx):
21 + return RemoteOpenAIServer([
22 + "--model",
23 + EMBEDDING_MODEL_NAME,
24 + # use half precision for speed and memory savings in CI environment
25 + "--dtype",
26 + "bfloat16",
27 + "--enforce-eager",
28 + "--max-model-len",
29 + "8192",
30 + "--enforce-eager",
31 + ])
32 +
33 +
34 +@pytest.mark.asyncio
35 +@pytest.fixture(scope="module")
36 +def embedding_client(embedding_server):
37 + return embedding_server.get_async_client()
38 +
39 +
40 +@pytest.mark.asyncio
41 +@pytest.mark.parametrize(
42 + "model_name",
43 + [EMBEDDING_MODEL_NAME],
44 +)
45 +async def test_single_embedding(embedding_client: openai.AsyncOpenAI,
46 + model_name: str):
47 + input_texts = [
48 + "The chef prepared a delicious meal.",
49 + ]
50 +
51 + # test single embedding
52 + embeddings = await embedding_client.embeddings.create(
53 + model=model_name,
54 + input=input_texts,
55 + encoding_format="float",
56 + )
57 + assert embeddings.id is not None
58 + assert len(embeddings.data) == 1
59 + assert len(embeddings.data[0].embedding) == 4096
60 + assert embeddings.usage.completion_tokens == 0
61 + assert embeddings.usage.prompt_tokens == 9
62 + assert embeddings.usage.total_tokens == 9
63 +
64 + # test using token IDs
65 + input_tokens = [1, 1, 1, 1, 1]
66 + embeddings = await embedding_client.embeddings.create(
67 + model=model_name,
68 + input=input_tokens,
69 + encoding_format="float",
70 + )
71 + assert embeddings.id is not None
72 + assert len(embeddings.data) == 1
73 + assert len(embeddings.data[0].embedding) == 4096
74 + assert embeddings.usage.completion_tokens == 0
75 + assert embeddings.usage.prompt_tokens == 5
76 + assert embeddings.usage.total_tokens == 5
77 +
78 +
79 +@pytest.mark.asyncio
80 +@pytest.mark.parametrize(
81 + "model_name",
82 + [EMBEDDING_MODEL_NAME],
83 +)
84 +async def test_batch_embedding(embedding_client: openai.AsyncOpenAI,
85 + model_name: str):
86 + # test List[str]
87 + input_texts = [
88 + "The cat sat on the mat.", "A feline was resting on a rug.",
89 + "Stars twinkle brightly in the night sky."
90 + ]
91 + embeddings = await embedding_client.embeddings.create(
92 + model=model_name,
93 + input=input_texts,
94 + encoding_format="float",
95 + )
96 + assert embeddings.id is not None
97 + assert len(embeddings.data) == 3
98 + assert len(embeddings.data[0].embedding) == 4096
99 +
100 + # test List[List[int]]
101 + input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
102 + [25, 32, 64, 77]]
103 + embeddings = await embedding_client.embeddings.create(
104 + model=model_name,
105 + input=input_tokens,
106 + encoding_format="float",
107 + )
108 + assert embeddings.id is not None
109 + assert len(embeddings.data) == 4
110 + assert len(embeddings.data[0].embedding) == 4096
111 + assert embeddings.usage.completion_tokens == 0
112 + assert embeddings.usage.prompt_tokens == 17
113 + assert embeddings.usage.total_tokens == 17
tests/entrypoints/test_openai_server.py
+47 -159
@@ -15,11 +15,10 @@ from openai import BadRequestError
15
16 from vllm.transformers_utils.tokenizer import get_tokenizer
17
18 -from ..utils import ServerRunner
18 +from ..utils import VLLM_PATH, RemoteOpenAIServer
19
20 # any model with a chat template should work here
21 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
22 -EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
22 # technically this needs Mistral-7B-v0.1 as base, but we're not testing
23 # generation quality here
24 LORA_NAME = "typeof/zephyr-7b-beta-lora"
@@ -80,9 +79,15 @@ def zephyr_lora_files():
79
80
81 @pytest.fixture(scope="module")
83 -def server(zephyr_lora_files):
84 - ray.init()
85 - server_runner = ServerRunner.remote([
82 +def ray_ctx():
83 + ray.init(runtime_env={"working_dir": VLLM_PATH})
84 + yield
85 + ray.shutdown()
86 +
87 +
88 +@pytest.fixture(scope="module")
89 +def server(zephyr_lora_files, ray_ctx):
90 + return RemoteOpenAIServer([
91 "--model",
92 MODEL_NAME,
93 # use half precision for speed and memory savings in CI environment
@@ -91,8 +96,6 @@ def server(zephyr_lora_files):
96 "--max-model-len",
97 "8192",
98 "--enforce-eager",
94 - "--gpu-memory-utilization",
95 - "0.75",
99 # lora config below
100 "--enable-lora",
101 "--lora-modules",
@@ -105,43 +108,14 @@ def server(zephyr_lora_files):
108 "--max-num-seqs",
109 "128",
110 ])
108 - ray.get(server_runner.ready.remote())
109 - yield server_runner
110 - ray.shutdown()
111
112
113 @pytest.fixture(scope="module")
114 -def embedding_server(zephyr_lora_files):
115 - ray.shutdown()
116 - ray.init()
117 - server_runner = ServerRunner.remote([
118 - "--model",
119 - EMBEDDING_MODEL_NAME,
120 - # use half precision for speed and memory savings in CI environment
121 - "--dtype",
122 - "bfloat16",
123 - "--enforce-eager",
124 - "--gpu-memory-utilization",
125 - "0.75",
126 - "--max-model-len",
127 - "8192",
128 - ])
129 - ray.get(server_runner.ready.remote())
130 - yield server_runner
131 - ray.shutdown()
132 -
133 -
134 -@pytest.fixture(scope="module")
135 -def client():
136 - client = openai.AsyncOpenAI(
137 - base_url="http://localhost:8000/v1",
138 - api_key="token-abc123",
139 - )
140 - yield client
114 +def client(server):
115 + return server.get_async_client()
116
117
143 -@pytest.mark.asyncio
144 -async def test_check_models(server, client: openai.AsyncOpenAI):
118 +async def test_check_models(client: openai.AsyncOpenAI):
119 models = await client.models.list()
120 models = models.data
121 served_model = models[0]
@@ -158,8 +132,7 @@ async def test_check_models(server, client: openai.AsyncOpenAI):
132 "model_name",
133 [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
134 )
161 -async def test_single_completion(server, client: openai.AsyncOpenAI,
162 - model_name: str):
135 +async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
136 completion = await client.completions.create(model=model_name,
137 prompt="Hello, my name is",
138 max_tokens=5,
@@ -190,8 +163,7 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
163 "model_name",
164 [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
165 )
193 -async def test_no_logprobs(server, client: openai.AsyncOpenAI,
194 - model_name: str):
166 +async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
167 # test using token IDs
168 completion = await client.completions.create(
169 model=MODEL_NAME,
@@ -210,8 +182,7 @@ async def test_no_logprobs(server, client: openai.AsyncOpenAI,
182 "model_name",
183 [MODEL_NAME, "zephyr-lora"],
184 )
213 -async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
214 - model_name: str):
185 +async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
186 # test using token IDs
187 completion = await client.completions.create(
188 model=MODEL_NAME,
@@ -232,8 +203,7 @@ async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
203 "model_name",
204 [MODEL_NAME, "zephyr-lora"],
205 )
235 -async def test_some_logprobs(server, client: openai.AsyncOpenAI,
236 - model_name: str):
206 +async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
207 # test using token IDs
208 completion = await client.completions.create(
209 model=MODEL_NAME,
@@ -254,7 +224,7 @@ async def test_some_logprobs(server, client: openai.AsyncOpenAI,
224 "model_name",
225 [MODEL_NAME, "zephyr-lora"],
226 )
257 -async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
227 +async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
228 model_name: str):
229
230 with pytest.raises(
@@ -300,8 +270,7 @@ async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
270 "model_name",
271 [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
272 )
303 -async def test_no_logprobs_chat(server, client: openai.AsyncOpenAI,
304 - model_name: str):
273 +async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
274 messages = [{
275 "role": "system",
276 "content": "you are a helpful assistant"
@@ -326,8 +295,7 @@ async def test_no_logprobs_chat(server, client: openai.AsyncOpenAI,
295 "model_name",
296 [MODEL_NAME, "zephyr-lora"],
297 )
329 -async def test_zero_logprobs_chat(server, client: openai.AsyncOpenAI,
330 - model_name: str):
298 +async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
299 messages = [{
300 "role": "system",
301 "content": "you are a helpful assistant"
@@ -354,8 +322,7 @@ async def test_zero_logprobs_chat(server, client: openai.AsyncOpenAI,
322 "model_name",
323 [MODEL_NAME, "zephyr-lora"],
324 )
357 -async def test_some_logprobs_chat(server, client: openai.AsyncOpenAI,
358 - model_name: str):
325 +async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
326 messages = [{
327 "role": "system",
328 "content": "you are a helpful assistant"
@@ -382,7 +349,7 @@ async def test_some_logprobs_chat(server, client: openai.AsyncOpenAI,
349 "model_name",
350 [MODEL_NAME, "zephyr-lora"],
351 )
385 -async def test_too_many_chat_logprobs(server, client: openai.AsyncOpenAI,
352 +async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
353 model_name: str):
354 messages = [{
355 "role": "system",
@@ -425,7 +392,7 @@ async def test_too_many_chat_logprobs(server, client: openai.AsyncOpenAI,
392 "model_name",
393 [MODEL_NAME, "zephyr-lora"],
394 )
428 -async def test_single_chat_session(server, client: openai.AsyncOpenAI,
395 +async def test_single_chat_session(client: openai.AsyncOpenAI,
396 model_name: str):
397 messages = [{
398 "role": "system",
@@ -470,7 +437,7 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
437 "model_name",
438 [MODEL_NAME, "zephyr-lora"],
439 )
473 -async def test_completion_streaming(server, client: openai.AsyncOpenAI,
440 +async def test_completion_streaming(client: openai.AsyncOpenAI,
441 model_name: str):
442 prompt = "What is an LLM?"
443
@@ -505,8 +472,7 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI,
472 "model_name",
473 [MODEL_NAME, "zephyr-lora"],
474 )
508 -async def test_chat_streaming(server, client: openai.AsyncOpenAI,
509 - model_name: str):
475 +async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
476 messages = [{
477 "role": "system",
478 "content": "you are a helpful assistant"
@@ -555,8 +521,7 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI,
521 "model_name",
522 ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
523 )
558 -async def test_chat_completion_stream_options(server,
559 - client: openai.AsyncOpenAI,
524 +async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
525 model_name: str):
526 messages = [{
527 "role": "system",
@@ -626,7 +591,7 @@ async def test_chat_completion_stream_options(server,
591 "model_name",
592 ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
593 )
629 -async def test_completion_stream_options(server, client: openai.AsyncOpenAI,
594 +async def test_completion_stream_options(client: openai.AsyncOpenAI,
595 model_name: str):
596 prompt = "What is the capital of France?"
597
@@ -688,8 +653,7 @@ async def test_completion_stream_options(server, client: openai.AsyncOpenAI,
653 "model_name",
654 [MODEL_NAME, "zephyr-lora"],
655 )
691 -async def test_batch_completions(server, client: openai.AsyncOpenAI,
692 - model_name: str):
656 +async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
657 # test simple list
658 batch = await client.completions.create(
659 model=model_name,
@@ -737,7 +701,7 @@ async def test_batch_completions(server, client: openai.AsyncOpenAI,
701
702
703 @pytest.mark.asyncio
740 -async def test_logits_bias(server, client: openai.AsyncOpenAI):
704 +async def test_logits_bias(client: openai.AsyncOpenAI):
705 prompt = "Hello, my name is"
706 max_tokens = 5
707 tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
@@ -786,7 +750,7 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
750 @pytest.mark.asyncio
751 @pytest.mark.parametrize("guided_decoding_backend",
752 ["outlines", "lm-format-enforcer"])
789 -async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
753 +async def test_guided_json_completion(client: openai.AsyncOpenAI,
754 guided_decoding_backend: str):
755 completion = await client.completions.create(
756 model=MODEL_NAME,
@@ -808,7 +772,7 @@ async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
772 @pytest.mark.asyncio
773 @pytest.mark.parametrize("guided_decoding_backend",
774 ["outlines", "lm-format-enforcer"])
811 -async def test_guided_json_chat(server, client: openai.AsyncOpenAI,
775 +async def test_guided_json_chat(client: openai.AsyncOpenAI,
776 guided_decoding_backend: str):
777 messages = [{
778 "role": "system",
@@ -855,7 +819,7 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI,
819 @pytest.mark.asyncio
820 @pytest.mark.parametrize("guided_decoding_backend",
821 ["outlines", "lm-format-enforcer"])
858 -async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
822 +async def test_guided_regex_completion(client: openai.AsyncOpenAI,
823 guided_decoding_backend: str):
824 completion = await client.completions.create(
825 model=MODEL_NAME,
@@ -875,7 +839,7 @@ async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
839 @pytest.mark.asyncio
840 @pytest.mark.parametrize("guided_decoding_backend",
841 ["outlines", "lm-format-enforcer"])
878 -async def test_guided_regex_chat(server, client: openai.AsyncOpenAI,
842 +async def test_guided_regex_chat(client: openai.AsyncOpenAI,
843 guided_decoding_backend: str):
844 messages = [{
845 "role": "system",
@@ -913,7 +877,7 @@ async def test_guided_regex_chat(server, client: openai.AsyncOpenAI,
877 @pytest.mark.asyncio
878 @pytest.mark.parametrize("guided_decoding_backend",
879 ["outlines", "lm-format-enforcer"])
916 -async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
880 +async def test_guided_choice_completion(client: openai.AsyncOpenAI,
881 guided_decoding_backend: str):
882 completion = await client.completions.create(
883 model=MODEL_NAME,
@@ -933,7 +897,7 @@ async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
897 @pytest.mark.asyncio
898 @pytest.mark.parametrize("guided_decoding_backend",
899 ["outlines", "lm-format-enforcer"])
936 -async def test_guided_choice_chat(server, client: openai.AsyncOpenAI,
900 +async def test_guided_choice_chat(client: openai.AsyncOpenAI,
901 guided_decoding_backend: str):
902 messages = [{
903 "role": "system",
@@ -972,7 +936,7 @@ async def test_guided_choice_chat(server, client: openai.AsyncOpenAI,
936 @pytest.mark.asyncio
937 @pytest.mark.parametrize("guided_decoding_backend",
938 ["outlines", "lm-format-enforcer"])
975 -async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
939 +async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
940 guided_decoding_backend: str):
941 with pytest.raises(openai.BadRequestError):
942 _ = await client.completions.create(
@@ -1008,7 +972,7 @@ async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
972 @pytest.mark.asyncio
973 @pytest.mark.parametrize("guided_decoding_backend",
974 ["outlines", "lm-format-enforcer"])
1011 -async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
975 +async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
976 guided_decoding_backend: str):
977 messages = [{
978 "role": "system",
@@ -1040,7 +1004,7 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
1004 @pytest.mark.asyncio
1005 @pytest.mark.parametrize("guided_decoding_backend",
1006 ["outlines", "lm-format-enforcer"])
1043 -async def test_named_tool_use(server, client: openai.AsyncOpenAI,
1007 +async def test_named_tool_use(client: openai.AsyncOpenAI,
1008 guided_decoding_backend: str):
1009 messages = [{
1010 "role": "system",
@@ -1131,7 +1095,7 @@ async def test_named_tool_use(server, client: openai.AsyncOpenAI,
1095 @pytest.mark.asyncio
1096 @pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1097 async def test_required_tool_use_not_yet_supported(
1134 - server, client: openai.AsyncOpenAI, guided_decoding_backend: str):
1098 + client: openai.AsyncOpenAI, guided_decoding_backend: str):
1099 messages = [{
1100 "role": "system",
1101 "content": "you are a helpful assistant"
@@ -1177,7 +1141,7 @@ async def test_required_tool_use_not_yet_supported(
1141 @pytest.mark.asyncio
1142 @pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1143 async def test_inconsistent_tool_choice_and_tools(
1180 - server, client: openai.AsyncOpenAI, guided_decoding_backend: str):
1144 + client: openai.AsyncOpenAI, guided_decoding_backend: str):
1145 messages = [{
1146 "role": "system",
1147 "content": "you are a helpful assistant"
@@ -1223,7 +1187,7 @@ async def test_inconsistent_tool_choice_and_tools(
1187
1188
1189 @pytest.mark.asyncio
1226 -async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
1190 +async def test_response_format_json_object(client: openai.AsyncOpenAI):
1191 for _ in range(2):
1192 resp = await client.chat.completions.create(
1193 model=MODEL_NAME,
@@ -1243,7 +1207,7 @@ async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
1207
1208
1209 @pytest.mark.asyncio
1246 -async def test_extra_fields(server, client: openai.AsyncOpenAI):
1210 +async def test_extra_fields(client: openai.AsyncOpenAI):
1211 with pytest.raises(BadRequestError) as exc_info:
1212 await client.chat.completions.create(
1213 model=MODEL_NAME,
@@ -1259,7 +1223,7 @@ async def test_extra_fields(server, client: openai.AsyncOpenAI):
1223
1224
1225 @pytest.mark.asyncio
1262 -async def test_complex_message_content(server, client: openai.AsyncOpenAI):
1226 +async def test_complex_message_content(client: openai.AsyncOpenAI):
1227 resp = await client.chat.completions.create(
1228 model=MODEL_NAME,
1229 messages=[{
@@ -1279,7 +1243,7 @@ async def test_complex_message_content(server, client: openai.AsyncOpenAI):
1243
1244
1245 @pytest.mark.asyncio
1282 -async def test_custom_role(server, client: openai.AsyncOpenAI):
1246 +async def test_custom_role(client: openai.AsyncOpenAI):
1247 # Not sure how the model handles custom roles so we just check that
1248 # both string and complex message content are handled in the same way
1249
@@ -1310,7 +1274,7 @@ async def test_custom_role(server, client: openai.AsyncOpenAI):
1274
1275
1276 @pytest.mark.asyncio
1313 -async def test_guided_grammar(server, client: openai.AsyncOpenAI):
1277 +async def test_guided_grammar(client: openai.AsyncOpenAI):
1278 simple_sql_grammar = """
1279 start: select_statement
1280
@@ -1351,7 +1315,7 @@ number: "1" | "2"
1315 [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
1316 )
1317 @pytest.mark.parametrize("logprobs_arg", [1, 0])
1354 -async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1318 +async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
1319 model_name: str, logprobs_arg: int):
1320 tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
1321 # test using text and token IDs
@@ -1380,7 +1344,7 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1344
1345
1346 @pytest.mark.asyncio
1383 -async def test_long_seed(server, client: openai.AsyncOpenAI):
1347 +async def test_long_seed(client: openai.AsyncOpenAI):
1348 for seed in [
1349 torch.iinfo(torch.long).min - 1,
1350 torch.iinfo(torch.long).max + 1
@@ -1399,81 +1363,5 @@ async def test_long_seed(server, client: openai.AsyncOpenAI):
1363 or "less_than_equal" in exc_info.value.message)
1364
1365
1402 -@pytest.mark.asyncio
1403 -@pytest.mark.parametrize(
1404 - "model_name",
1405 - [EMBEDDING_MODEL_NAME],
1406 -)
1407 -async def test_single_embedding(embedding_server, client: openai.AsyncOpenAI,
1408 - model_name: str):
1409 - input_texts = [
1410 - "The chef prepared a delicious meal.",
1411 - ]
1412 -
1413 - # test single embedding
1414 - embeddings = await client.embeddings.create(
1415 - model=model_name,
1416 - input=input_texts,
1417 - encoding_format="float",
1418 - )
1419 - assert embeddings.id is not None
1420 - assert len(embeddings.data) == 1
1421 - assert len(embeddings.data[0].embedding) == 4096
1422 - assert embeddings.usage.completion_tokens == 0
1423 - assert embeddings.usage.prompt_tokens == 9
1424 - assert embeddings.usage.total_tokens == 9
1425 -
1426 - # test using token IDs
1427 - input_tokens = [1, 1, 1, 1, 1]
1428 - embeddings = await client.embeddings.create(
1429 - model=model_name,
1430 - input=input_tokens,
1431 - encoding_format="float",
1432 - )
1433 - assert embeddings.id is not None
1434 - assert len(embeddings.data) == 1
1435 - assert len(embeddings.data[0].embedding) == 4096
1436 - assert embeddings.usage.completion_tokens == 0
1437 - assert embeddings.usage.prompt_tokens == 5
1438 - assert embeddings.usage.total_tokens == 5
1439 -
1440 -
1441 -@pytest.mark.asyncio
1442 -@pytest.mark.parametrize(
1443 - "model_name",
1444 - [EMBEDDING_MODEL_NAME],
1445 -)
1446 -async def test_batch_embedding(embedding_server, client: openai.AsyncOpenAI,
1447 - model_name: str):
1448 - # test List[str]
1449 - input_texts = [
1450 - "The cat sat on the mat.", "A feline was resting on a rug.",
1451 - "Stars twinkle brightly in the night sky."
1452 - ]
1453 - embeddings = await client.embeddings.create(
1454 - model=model_name,
1455 - input=input_texts,
1456 - encoding_format="float",
1457 - )
1458 - assert embeddings.id is not None
1459 - assert len(embeddings.data) == 3
1460 - assert len(embeddings.data[0].embedding) == 4096
1461 -
1462 - # test List[List[int]]
1463 - input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
1464 - [25, 32, 64, 77]]
1465 - embeddings = await client.embeddings.create(
1466 - model=model_name,
1467 - input=input_tokens,
1468 - encoding_format="float",
1469 - )
1470 - assert embeddings.id is not None
1471 - assert len(embeddings.data) == 4
1472 - assert len(embeddings.data[0].embedding) == 4096
1473 - assert embeddings.usage.completion_tokens == 0
1474 - assert embeddings.usage.prompt_tokens == 17
1475 - assert embeddings.usage.total_tokens == 17
1476 -
1477 -
1366 if __name__ == "__main__":
1367 pytest.main([__file__])
tests/entrypoints/test_openai_vision.py
+17 -18
@@ -8,7 +8,7 @@ import ray
8
9 from vllm.multimodal.utils import ImageFetchAiohttp, encode_image_base64
10
11 -from ..utils import ServerRunner
11 +from ..utils import VLLM_PATH, RemoteOpenAIServer
12
13 MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
14 LLAVA_CHAT_TEMPLATE = (Path(__file__).parent.parent.parent /
@@ -25,10 +25,16 @@ TEST_IMAGE_URLS = [
25 pytestmark = pytest.mark.openai
26
27
28 +@pytest.fixture(scope="module")
29 +def ray_ctx():
30 + ray.init(runtime_env={"working_dir": VLLM_PATH})
31 + yield
32 + ray.shutdown()
33 +
34 +
35 @pytest.fixture(scope="module")
36 def server():
30 - ray.init()
31 - server_runner = ServerRunner.remote([
37 + return RemoteOpenAIServer([
38 "--model",
39 MODEL_NAME,
40 "--dtype",
@@ -47,18 +53,11 @@ def server():
53 "--chat-template",
54 str(LLAVA_CHAT_TEMPLATE),
55 ])
50 - ray.get(server_runner.ready.remote())
51 - yield server_runner
52 - ray.shutdown()
56
57
55 -@pytest.fixture(scope="session")
56 -def client():
57 - client = openai.AsyncOpenAI(
58 - base_url="http://localhost:8000/v1",
59 - api_key="token-abc123",
60 - )
61 - yield client
58 +@pytest.fixture(scope="module")
59 +def client(server):
60 + return server.get_async_client()
61
62
63 @pytest_asyncio.fixture(scope="session")
@@ -73,7 +72,7 @@ async def base64_encoded_image() -> Dict[str, str]:
72 @pytest.mark.asyncio
73 @pytest.mark.parametrize("model_name", [MODEL_NAME])
74 @pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
76 -async def test_single_chat_session_image(server, client: openai.AsyncOpenAI,
75 +async def test_single_chat_session_image(client: openai.AsyncOpenAI,
76 model_name: str, image_url: str):
77 messages = [{
78 "role":
@@ -126,7 +125,7 @@ async def test_single_chat_session_image(server, client: openai.AsyncOpenAI,
125 @pytest.mark.parametrize("model_name", [MODEL_NAME])
126 @pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
127 async def test_single_chat_session_image_base64encoded(
129 - server, client: openai.AsyncOpenAI, model_name: str, image_url: str,
128 + client: openai.AsyncOpenAI, model_name: str, image_url: str,
129 base64_encoded_image: Dict[str, str]):
130
131 messages = [{
@@ -180,7 +179,7 @@ async def test_single_chat_session_image_base64encoded(
179 @pytest.mark.asyncio
180 @pytest.mark.parametrize("model_name", [MODEL_NAME])
181 @pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
183 -async def test_chat_streaming_image(server, client: openai.AsyncOpenAI,
182 +async def test_chat_streaming_image(client: openai.AsyncOpenAI,
183 model_name: str, image_url: str):
184 messages = [{
185 "role":
@@ -237,8 +236,8 @@ async def test_chat_streaming_image(server, client: openai.AsyncOpenAI,
236 @pytest.mark.asyncio
237 @pytest.mark.parametrize("model_name", [MODEL_NAME])
238 @pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
240 -async def test_multi_image_input(server, client: openai.AsyncOpenAI,
241 - model_name: str, image_url: str):
239 +async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
240 + image_url: str):
241
242 messages = [{
243 "role":
tests/tensorizer_loader/test_tensorizer.py
+5 -9
@@ -22,11 +22,12 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
22 tensorize_vllm_model)
23
24 from ..conftest import VllmRunner, cleanup
25 -from ..utils import ServerRunner
25 +from ..utils import RemoteOpenAIServer
26
27 # yapf conflicts with isort for this docstring
28
29
30 +
31 prompts = [
32 "Hello, my name is",
33 "The president of the United States is",
@@ -216,18 +217,13 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
217 openai_args = [
218 "--model", model_ref, "--dtype", "float16", "--load-format",
219 "tensorizer", "--model-loader-extra-config",
219 - json.dumps(model_loader_extra_config), "--port", "8000"
220 + json.dumps(model_loader_extra_config),
221 ]
222
222 - server = ServerRunner.remote(openai_args)
223 -
224 - assert ray.get(server.ready.remote())
223 + server = RemoteOpenAIServer(openai_args)
224 print("Server ready.")
225
227 - client = openai.OpenAI(
228 - base_url="http://localhost:8000/v1",
229 - api_key="token-abc123",
230 - )
226 + client = server.get_client()
227 completion = client.completions.create(model=model_ref,
228 prompt="Hello, my name is",
229 max_tokens=5,
tests/utils.py
+88 -36
@@ -4,57 +4,109 @@ import sys
4 import time
5 import warnings
6 from contextlib import contextmanager
7 +from typing import List
8
9 +import openai
10 import ray
11 import requests
12
13 from vllm.distributed import (ensure_model_parallel_initialized,
14 init_distributed_environment)
15 +from vllm.entrypoints.openai.cli_args import make_arg_parser
16 from vllm.utils import get_open_port
17
18 # Path to root of repository so that utilities can be imported by ray workers
19 VLLM_PATH = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))
20
21
19 -@ray.remote(num_gpus=1)
20 -class ServerRunner:
22 +class RemoteOpenAIServer:
23 + DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
24 MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
25
23 - def __init__(self, args):
24 - env = os.environ.copy()
25 - env["PYTHONUNBUFFERED"] = "1"
26 - self.proc = subprocess.Popen(
27 - [sys.executable, "-m", "vllm.entrypoints.openai.api_server"] +
28 - args,
29 - env=env,
30 - stdout=sys.stdout,
31 - stderr=sys.stderr,
26 + @ray.remote(num_gpus=1)
27 + class _RemoteRunner:
28 +
29 + def __init__(self, cli_args: List[str], *, wait_url: str,
30 + wait_timeout: float) -> None:
31 + env = os.environ.copy()
32 + env["PYTHONUNBUFFERED"] = "1"
33 + self.proc = subprocess.Popen(
34 + [
35 + sys.executable, "-m", "vllm.entrypoints.openai.api_server",
36 + *cli_args
37 + ],
38 + env=env,
39 + stdout=sys.stdout,
40 + stderr=sys.stderr,
41 + )
42 +
43 + self._wait_for_server(url=wait_url, timeout=wait_timeout)
44 +
45 + def ready(self):
46 + return True
47 +
48 + def _wait_for_server(self, *, url: str, timeout: float):
49 + # run health check
50 + start = time.time()
51 + while True:
52 + try:
53 + if requests.get(url).status_code == 200:
54 + break
55 + except Exception as err:
56 + if self.proc.poll() is not None:
57 + raise RuntimeError(
58 + "Server exited unexpectedly.") from err
59 +
60 + time.sleep(0.5)
61 + if time.time() - start > timeout:
62 + raise RuntimeError(
63 + "Server failed to start in time.") from err
64 +
65 + def __del__(self):
66 + if hasattr(self, "proc"):
67 + self.proc.terminate()
68 +
69 + def __init__(self, cli_args: List[str], *, auto_port: bool = True) -> None:
70 + if auto_port:
71 + if "-p" in cli_args or "--port" in cli_args:
72 + raise ValueError("You have manually specified the port"
73 + "when `auto_port=True`.")
74 +
75 + cli_args = cli_args + ["--port", str(get_open_port())]
76 +
77 + parser = make_arg_parser()
78 + args = parser.parse_args(cli_args)
79 + self.host = str(args.host or 'localhost')
80 + self.port = int(args.port)
81 +
82 + self._runner = self._RemoteRunner.remote(
83 + cli_args,
84 + wait_url=self.url_for("health"),
85 + wait_timeout=self.MAX_SERVER_START_WAIT_S)
86 +
87 + self._wait_until_ready()
88 +
89 + @property
90 + def url_root(self) -> str:
91 + return f"http://{self.host}:{self.port}"
92 +
93 + def url_for(self, *parts: str) -> str:
94 + return self.url_root + "/" + "/".join(parts)
95 +
96 + def _wait_until_ready(self) -> None:
97 + ray.get(self._runner.ready.remote())
98 +
99 + def get_client(self):
100 + return openai.OpenAI(
101 + base_url=self.url_for("v1"),
102 + api_key=self.DUMMY_API_KEY,
103 + )
104 +
105 + def get_async_client(self):
106 + return openai.AsyncOpenAI(
107 + base_url=self.url_for("v1"),
108 + api_key=self.DUMMY_API_KEY,
109 )
33 - self._wait_for_server()
34 -
35 - def ready(self):
36 - return True
37 -
38 - def _wait_for_server(self):
39 - # run health check
40 - start = time.time()
41 - while True:
42 - try:
43 - if requests.get(
44 - "http://localhost:8000/health").status_code == 200:
45 - break
46 - except Exception as err:
47 - if self.proc.poll() is not None:
48 - raise RuntimeError("Server exited unexpectedly.") from err
49 -
50 - time.sleep(0.5)
51 - if time.time() - start > self.MAX_SERVER_START_WAIT_S:
52 - raise RuntimeError(
53 - "Server failed to start in time.") from err
54 -
55 - def __del__(self):
56 - if hasattr(self, "proc"):
57 - self.proc.terminate()
110
111
112 def init_test_distributed_environment(