A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix][CI/Build] Test prompt adapters in openai entrypoint tests (#6419)
Joe committed
Jul 15, 2024 at 18:54 UTC
d92b3c5cdef59533347ac714a70274f186943019
2 files changed
+54
-31
tests/entrypoints/openai/test_completion.py
+51
-29
@@ -17,9 +17,13 @@ from ...utils import RemoteOpenAIServer
17
18
# any model with a chat template should work here
19
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
20
-# technically this needs Mistral-7B-v0.1 as base, but we're not testing
21
-# generation quality here
20
+# technically these adapters use a different base model,
21
+# but we're not testing generation quality here
22
LORA_NAME = "typeof/zephyr-7b-beta-lora"
23
+PA_NAME = "swapnilbp/llama_tweet_ptune"
24
+# if PA_NAME changes, PA_NUM_VIRTUAL_TOKENS might also
25
+# need to change to match the prompt adapter
26
+PA_NUM_VIRTUAL_TOKENS = 8
27
28
29
@pytest.fixture(scope="module")
@@ -28,7 +32,12 @@ def zephyr_lora_files():
32
33
34
@pytest.fixture(scope="module")
31
-def server(zephyr_lora_files):
35
+def zephyr_pa_files():
36
+ return snapshot_download(repo_id=PA_NAME)
37
+
38
+
39
+@pytest.fixture(scope="module")
40
+def server(zephyr_lora_files, zephyr_pa_files):
41
with RemoteOpenAIServer([
42
"--model",
43
MODEL_NAME,
@@ -37,8 +46,10 @@ def server(zephyr_lora_files):
46
"bfloat16",
47
"--max-model-len",
48
"8192",
49
+ "--max-num-seqs",
50
+ "128",
51
"--enforce-eager",
41
- # lora config below
52
+ # lora config
53
"--enable-lora",
54
"--lora-modules",
55
f"zephyr-lora={zephyr_lora_files}",
@@ -47,7 +58,14 @@ def server(zephyr_lora_files):
58
"64",
59
"--max-cpu-loras",
60
"2",
50
- "--max-num-seqs",
61
+ # pa config
62
+ "--enable-prompt-adapter",
63
+ "--prompt-adapters",
64
+ f"zephyr-pa={zephyr_pa_files}",
65
+ f"zephyr-pa2={zephyr_pa_files}",
66
+ "--max-prompt-adapters",
67
+ "2",
68
+ "--max-prompt-adapter-token",
69
"128",
70
]) as remote_server:
71
yield remote_server
@@ -60,11 +78,14 @@ def client(server):
78
79
@pytest.mark.asyncio
80
@pytest.mark.parametrize(
63
- # first test base model, then test loras
64
- "model_name",
65
- [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
81
+ # first test base model, then test loras, then test prompt adapters
82
+ "model_name,num_virtual_tokens",
83
+ [(MODEL_NAME, 0), ("zephyr-lora", 0), ("zephyr-lora2", 0),
84
+ ("zephyr-pa", PA_NUM_VIRTUAL_TOKENS),
85
+ ("zephyr-pa2", PA_NUM_VIRTUAL_TOKENS)],
86
)
67
-async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
87
+async def test_single_completion(client: openai.AsyncOpenAI, model_name: str,
88
+ num_virtual_tokens: int):
89
completion = await client.completions.create(model=model_name,
90
prompt="Hello, my name is",
91
max_tokens=5,
@@ -77,28 +98,30 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
98
assert len(choice.text) >= 5
99
assert choice.finish_reason == "length"
100
assert completion.usage == openai.types.CompletionUsage(
80
- completion_tokens=5, prompt_tokens=6, total_tokens=11)
101
+ completion_tokens=5,
102
+ prompt_tokens=6 + num_virtual_tokens,
103
+ total_tokens=11 + num_virtual_tokens)
104
105
# test using token IDs
106
completion = await client.completions.create(
84
- model=MODEL_NAME,
107
+ model=model_name,
108
prompt=[0, 0, 0, 0, 0],
109
max_tokens=5,
110
temperature=0.0,
111
)
89
- assert len(completion.choices[0].text) >= 5
112
+ assert len(completion.choices[0].text) >= 1
113
114
115
@pytest.mark.asyncio
116
@pytest.mark.parametrize(
94
- # first test base model, then test loras
117
+ # first test base model, then test loras, then test prompt adapters
118
"model_name",
96
- [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
119
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2", "zephyr-pa", "zephyr-pa2"],
120
)
121
async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
122
# test using token IDs
123
completion = await client.completions.create(
101
- model=MODEL_NAME,
124
+ model=model_name,
125
prompt=[0, 0, 0, 0, 0],
126
max_tokens=5,
127
temperature=0.0,
@@ -110,14 +133,14 @@ async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
133
134
@pytest.mark.asyncio
135
@pytest.mark.parametrize(
113
- # just test 1 lora hereafter
136
+ # just test 1 lora and 1 pa hereafter
137
"model_name",
115
- [MODEL_NAME, "zephyr-lora"],
138
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
139
)
140
async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
141
# test using token IDs
142
completion = await client.completions.create(
120
- model=MODEL_NAME,
143
+ model=model_name,
144
prompt=[0, 0, 0, 0, 0],
145
max_tokens=5,
146
temperature=0.0,
@@ -133,12 +156,12 @@ async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
156
@pytest.mark.asyncio
157
@pytest.mark.parametrize(
158
"model_name",
136
- [MODEL_NAME, "zephyr-lora"],
159
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
160
)
161
async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
162
# test using token IDs
163
completion = await client.completions.create(
141
- model=MODEL_NAME,
164
+ model=model_name,
165
prompt=[0, 0, 0, 0, 0],
166
max_tokens=5,
167
temperature=0.0,
@@ -154,7 +177,7 @@ async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
177
@pytest.mark.asyncio
178
@pytest.mark.parametrize(
179
"model_name",
157
- [MODEL_NAME, "zephyr-lora"],
180
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
181
)
182
async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
183
model_name: str):
@@ -162,7 +185,7 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
185
with pytest.raises(
186
(openai.BadRequestError, openai.APIError)): # test using token IDs
187
await client.completions.create(
165
- model=MODEL_NAME,
188
+ model=model_name,
189
prompt=[0, 0, 0, 0, 0],
190
max_tokens=5,
191
temperature=0.0,
@@ -174,7 +197,7 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
197
with pytest.raises(
198
(openai.BadRequestError, openai.APIError)): # test using token IDs
199
stream = await client.completions.create(
177
- model=MODEL_NAME,
200
+ model=model_name,
201
prompt=[0, 0, 0, 0, 0],
202
max_tokens=5,
203
temperature=0.0,
@@ -199,7 +222,7 @@ async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
222
@pytest.mark.asyncio
223
@pytest.mark.parametrize(
224
"model_name",
202
- [MODEL_NAME, "zephyr-lora"],
225
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
226
)
227
async def test_completion_streaming(client: openai.AsyncOpenAI,
228
model_name: str):
@@ -233,7 +256,7 @@ async def test_completion_streaming(client: openai.AsyncOpenAI,
256
@pytest.mark.asyncio
257
@pytest.mark.parametrize(
258
"model_name",
236
- ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
259
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
260
)
261
async def test_completion_stream_options(client: openai.AsyncOpenAI,
262
model_name: str):
@@ -369,9 +392,8 @@ async def test_completion_stream_options(client: openai.AsyncOpenAI,
392
393
@pytest.mark.asyncio
394
@pytest.mark.parametrize(
372
- # just test 1 lora hereafter
395
"model_name",
374
- [MODEL_NAME, "zephyr-lora"],
396
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
397
)
398
async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
399
# test both text and token IDs
@@ -623,7 +645,7 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
645
)
646
async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
647
base_url = str(client.base_url)[:-3].strip("/")
626
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
648
+ tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
649
650
for add_special in [False, True]:
651
prompt = "This is a test prompt."
@@ -650,7 +672,7 @@ async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
672
)
673
async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
674
base_url = str(client.base_url)[:-3]
653
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
675
+ tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
676
677
prompt = "This is a test prompt."
678
tokens = tokenizer.encode(prompt, add_special_tokens=False)
vllm/entrypoints/openai/serving_engine.py
+3
-2
@@ -1,4 +1,5 @@
1
import json
2
+import pathlib
3
from dataclasses import dataclass
4
from http import HTTPStatus
5
from typing import Any, Dict, List, Optional, Tuple, Union
@@ -74,8 +75,8 @@ class OpenAIServing:
75
self.prompt_adapter_requests = []
76
if prompt_adapters is not None:
77
for i, prompt_adapter in enumerate(prompt_adapters, start=1):
77
- with open(f"./{prompt_adapter.local_path}"
78
- f"/adapter_config.json") as f:
78
+ with pathlib.Path(prompt_adapter.local_path,
79
+ "adapter_config.json").open() as f:
80
adapter_config = json.load(f)
81
num_virtual_tokens = adapter_config["num_virtual_tokens"]
82
self.prompt_adapter_requests.append(