7
import requests
8
import ray # using Ray for overall ease of process management, parallel requests, and debugging.
9
import openai # use the official client for correctness check
10
+from huggingface_hub import snapshot_download # downloading lora to test lora requests
11
12
MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
13
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta" # any model with a chat template should work here
14
+LORA_NAME = "typeof/zephyr-7b-beta-lora" # technically this needs Mistral-7B-v0.1 as base, but we're not testing generation quality here
15
16
pytestmark = pytest.mark.asyncio
17
56
57
58
@pytest.fixture(scope="session")
57
-def server():
59
+def zephyr_lora_files():
60
+ return snapshot_download(repo_id=LORA_NAME)
61
+
62
+
63
+@pytest.fixture(scope="session")
64
+def server(zephyr_lora_files):
65
ray.init()
66
server_runner = ServerRunner.remote([
67
"--model",
71
"--max-model-len",
72
"8192",
73
"--enforce-eager",
74
+ # lora config below
75
+ "--enable-lora",
76
+ "--lora-modules",
77
+ f"zephyr-lora={zephyr_lora_files}",
78
+ f"zephyr-lora2={zephyr_lora_files}",
79
+ "--max-lora-rank",
80
+ "64",
81
+ "--max-cpu-loras",
82
+ "2",
83
+ "--max-num-seqs",
84
+ "128"
85
])
86
ray.get(server_runner.ready.remote())
87
yield server_runner
97
yield client
98
99
82
-async def test_single_completion(server, client: openai.AsyncOpenAI):
83
- completion = await client.completions.create(model=MODEL_NAME,
100
+async def test_check_models(server, client: openai.AsyncOpenAI):
101
+ models = await client.models.list()
102
+ models = models.data
103
+ served_model = models[0]
104
+ lora_models = models[1:]
105
+ assert served_model.id == MODEL_NAME
106
+ assert all(model.root == MODEL_NAME for model in models)
107
+ assert lora_models[0].id == "zephyr-lora"
108
+ assert lora_models[1].id == "zephyr-lora2"
109
+
110
+
111
+@pytest.mark.parametrize(
112
+ # first test base model, then test loras
113
+ "model_name",
114
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
115
+)
116
+async def test_single_completion(server, client: openai.AsyncOpenAI,
117
+ model_name: str):
118
+ completion = await client.completions.create(model=model_name,
119
prompt="Hello, my name is",
120
max_tokens=5,
121
temperature=0.0)
139
completion.choices[0].text) >= 5
140
141
107
-async def test_single_chat_session(server, client: openai.AsyncOpenAI):
142
+@pytest.mark.parametrize(
143
+ # just test 1 lora hereafter
144
+ "model_name",
145
+ [MODEL_NAME, "zephyr-lora"],
146
+)
147
+async def test_single_chat_session(server, client: openai.AsyncOpenAI,
148
+ model_name: str):
149
messages = [{
150
"role": "system",
151
"content": "you are a helpful assistant"
156
157
# test single completion
158
chat_completion = await client.chat.completions.create(
118
- model=MODEL_NAME,
159
+ model=model_name,
160
messages=messages,
161
max_tokens=10,
162
)
180
assert message.content is not None and len(message.content) >= 0
181
182
142
-async def test_completion_streaming(server, client: openai.AsyncOpenAI):
183
+@pytest.mark.parametrize(
184
+ # just test 1 lora hereafter
185
+ "model_name",
186
+ [MODEL_NAME, "zephyr-lora"],
187
+)
188
+async def test_completion_streaming(server, client: openai.AsyncOpenAI,
189
+ model_name: str):
190
prompt = "What is an LLM?"
191
192
single_completion = await client.completions.create(
146
- model=MODEL_NAME,
193
+ model=model_name,
194
prompt=prompt,
195
max_tokens=5,
196
temperature=0.0,
199
single_usage = single_completion.usage
200
201
stream = await client.completions.create(
155
- model=MODEL_NAME,
202
+ model=model_name,
203
prompt=prompt,
204
max_tokens=5,
205
temperature=0.0,
213
assert "".join(chunks) == single_output
214
215
169
-async def test_chat_streaming(server, client: openai.AsyncOpenAI):
216
+@pytest.mark.parametrize(
217
+ # just test 1 lora hereafter
218
+ "model_name",
219
+ [MODEL_NAME, "zephyr-lora"],
220
+)
221
+async def test_chat_streaming(server, client: openai.AsyncOpenAI,
222
+ model_name: str):
223
messages = [{
224
"role": "system",
225
"content": "you are a helpful assistant"
230
231
# test single completion
232
chat_completion = await client.chat.completions.create(
180
- model=MODEL_NAME,
233
+ model=model_name,
234
messages=messages,
235
max_tokens=10,
236
temperature=0.0,
240
241
# test streaming
242
stream = await client.chat.completions.create(
190
- model=MODEL_NAME,
243
+ model=model_name,
244
messages=messages,
245
max_tokens=10,
246
temperature=0.0,
257
assert "".join(chunks) == output
258
259
207
-async def test_batch_completions(server, client: openai.AsyncOpenAI):
260
+@pytest.mark.parametrize(
261
+ # just test 1 lora hereafter
262
+ "model_name",
263
+ [MODEL_NAME, "zephyr-lora"],
264
+)
265
+async def test_batch_completions(server, client: openai.AsyncOpenAI,
266
+ model_name: str):
267
# test simple list
268
batch = await client.completions.create(
210
- model=MODEL_NAME,
269
+ model=model_name,
270
prompt=["Hello, my name is", "Hello, my name is"],
271
max_tokens=5,
272
temperature=0.0,
276
277
# test n = 2
278
batch = await client.completions.create(
220
- model=MODEL_NAME,
279
+ model=model_name,
280
prompt=["Hello, my name is", "Hello, my name is"],
281
n=2,
282
max_tokens=5,
295
296
# test streaming
297
batch = await client.completions.create(
239
- model=MODEL_NAME,
298
+ model=model_name,
299
prompt=["Hello, my name is", "Hello, my name is"],
300
max_tokens=5,
301
temperature=0.0,