A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[CI/Build] [2/3] Reorganize entrypoints tests (#5904)
Cyrus Leung committed
Jun 28, 2024 at 22:59 UTC
3b752a6555a67bb4863ea821fc30fdda38b27355
4 files changed
+653
-598
tests/entrypoints/test_llm_generate_multiple_loras.py
+1
-1
@@ -44,7 +44,7 @@ def llm():
44
cleanup()
45
46
47
-@pytest.fixture(scope="session")
47
+@pytest.fixture(scope="module")
48
def zephyr_lora_files():
49
return snapshot_download(repo_id=LORA_NAME)
50
tests/entrypoints/test_openai_chat.py
+1
-1
@@ -72,7 +72,7 @@ TEST_CHOICE = [
72
pytestmark = pytest.mark.openai
73
74
75
-@pytest.fixture(scope="session")
75
+@pytest.fixture(scope="module")
76
def zephyr_lora_files():
77
return snapshot_download(repo_id=LORA_NAME)
78
tests/entrypoints/test_openai_completion.py
new
+650
@@ -0,0 +1,650 @@
1
+# imports for guided decoding tests
2
+import json
3
+import re
4
+from typing import List
5
+
6
+import jsonschema
7
+import openai # use the official client for correctness check
8
+import pytest
9
+# using Ray for overall ease of process management, parallel requests,
10
+# and debugging.
11
+import ray
12
+import requests
13
+# downloading lora to test lora requests
14
+from huggingface_hub import snapshot_download
15
+from openai import BadRequestError
16
+
17
+from vllm.transformers_utils.tokenizer import get_tokenizer
18
+
19
+from ..utils import RemoteOpenAIServer
20
+
21
+# any model with a chat template should work here
22
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
23
+# technically this needs Mistral-7B-v0.1 as base, but we're not testing
24
+# generation quality here
25
+LORA_NAME = "typeof/zephyr-7b-beta-lora"
26
+
27
+TEST_SCHEMA = {
28
+ "type": "object",
29
+ "properties": {
30
+ "name": {
31
+ "type": "string"
32
+ },
33
+ "age": {
34
+ "type": "integer"
35
+ },
36
+ "skills": {
37
+ "type": "array",
38
+ "items": {
39
+ "type": "string",
40
+ "maxLength": 10
41
+ },
42
+ "minItems": 3
43
+ },
44
+ "work history": {
45
+ "type": "array",
46
+ "items": {
47
+ "type": "object",
48
+ "properties": {
49
+ "company": {
50
+ "type": "string"
51
+ },
52
+ "duration": {
53
+ "type": "string"
54
+ },
55
+ "position": {
56
+ "type": "string"
57
+ }
58
+ },
59
+ "required": ["company", "position"]
60
+ }
61
+ }
62
+ },
63
+ "required": ["name", "age", "skills", "work history"]
64
+}
65
+
66
+TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
67
+ r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
68
+
69
+TEST_CHOICE = [
70
+ "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
71
+ "Swift", "Kotlin"
72
+]
73
+
74
+pytestmark = pytest.mark.openai
75
+
76
+
77
+@pytest.fixture(scope="module")
78
+def zephyr_lora_files():
79
+ return snapshot_download(repo_id=LORA_NAME)
80
+
81
+
82
+@pytest.fixture(scope="module")
83
+def ray_ctx():
84
+ ray.init()
85
+ yield
86
+ ray.shutdown()
87
+
88
+
89
+@pytest.fixture(scope="module")
90
+def server(zephyr_lora_files, ray_ctx):
91
+ return RemoteOpenAIServer([
92
+ "--model",
93
+ MODEL_NAME,
94
+ # use half precision for speed and memory savings in CI environment
95
+ "--dtype",
96
+ "bfloat16",
97
+ "--max-model-len",
98
+ "8192",
99
+ "--enforce-eager",
100
+ # lora config below
101
+ "--enable-lora",
102
+ "--lora-modules",
103
+ f"zephyr-lora={zephyr_lora_files}",
104
+ f"zephyr-lora2={zephyr_lora_files}",
105
+ "--max-lora-rank",
106
+ "64",
107
+ "--max-cpu-loras",
108
+ "2",
109
+ "--max-num-seqs",
110
+ "128",
111
+ ])
112
+
113
+
114
+@pytest.fixture(scope="module")
115
+def client(server):
116
+ return server.get_async_client()
117
+
118
+
119
+@pytest.mark.asyncio
120
+@pytest.mark.parametrize(
121
+ # first test base model, then test loras
122
+ "model_name",
123
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
124
+)
125
+async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
126
+ completion = await client.completions.create(model=model_name,
127
+ prompt="Hello, my name is",
128
+ max_tokens=5,
129
+ temperature=0.0)
130
+
131
+ assert completion.id is not None
132
+ assert completion.choices is not None and len(completion.choices) == 1
133
+
134
+ choice = completion.choices[0]
135
+ assert len(choice.text) >= 5
136
+ assert choice.finish_reason == "length"
137
+ assert completion.usage == openai.types.CompletionUsage(
138
+ completion_tokens=5, prompt_tokens=6, total_tokens=11)
139
+
140
+ # test using token IDs
141
+ completion = await client.completions.create(
142
+ model=MODEL_NAME,
143
+ prompt=[0, 0, 0, 0, 0],
144
+ max_tokens=5,
145
+ temperature=0.0,
146
+ )
147
+ assert len(completion.choices[0].text) >= 5
148
+
149
+
150
+@pytest.mark.asyncio
151
+@pytest.mark.parametrize(
152
+ # first test base model, then test loras
153
+ "model_name",
154
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
155
+)
156
+async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
157
+ # test using token IDs
158
+ completion = await client.completions.create(
159
+ model=MODEL_NAME,
160
+ prompt=[0, 0, 0, 0, 0],
161
+ max_tokens=5,
162
+ temperature=0.0,
163
+ logprobs=None,
164
+ )
165
+ choice = completion.choices[0]
166
+ assert choice.logprobs is None
167
+
168
+
169
+@pytest.mark.asyncio
170
+@pytest.mark.parametrize(
171
+ # just test 1 lora hereafter
172
+ "model_name",
173
+ [MODEL_NAME, "zephyr-lora"],
174
+)
175
+async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
176
+ # test using token IDs
177
+ completion = await client.completions.create(
178
+ model=MODEL_NAME,
179
+ prompt=[0, 0, 0, 0, 0],
180
+ max_tokens=5,
181
+ temperature=0.0,
182
+ logprobs=0,
183
+ )
184
+ choice = completion.choices[0]
185
+ assert choice.logprobs is not None
186
+ assert choice.logprobs.token_logprobs is not None
187
+ assert choice.logprobs.top_logprobs is not None
188
+ assert len(choice.logprobs.top_logprobs[0]) == 1
189
+
190
+
191
+@pytest.mark.asyncio
192
+@pytest.mark.parametrize(
193
+ "model_name",
194
+ [MODEL_NAME, "zephyr-lora"],
195
+)
196
+async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
197
+ # test using token IDs
198
+ completion = await client.completions.create(
199
+ model=MODEL_NAME,
200
+ prompt=[0, 0, 0, 0, 0],
201
+ max_tokens=5,
202
+ temperature=0.0,
203
+ logprobs=5,
204
+ )
205
+ choice = completion.choices[0]
206
+ assert choice.logprobs is not None
207
+ assert choice.logprobs.token_logprobs is not None
208
+ assert choice.logprobs.top_logprobs is not None
209
+ assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
210
+
211
+
212
+@pytest.mark.asyncio
213
+@pytest.mark.parametrize(
214
+ "model_name",
215
+ [MODEL_NAME, "zephyr-lora"],
216
+)
217
+async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
218
+ model_name: str):
219
+
220
+ with pytest.raises(
221
+ (openai.BadRequestError, openai.APIError)): # test using token IDs
222
+ await client.completions.create(
223
+ model=MODEL_NAME,
224
+ prompt=[0, 0, 0, 0, 0],
225
+ max_tokens=5,
226
+ temperature=0.0,
227
+ # vLLM has higher default max_logprobs (20 instead of 5) to support
228
+ # both Completion API and Chat Completion API
229
+ logprobs=21,
230
+ )
231
+ ...
232
+ with pytest.raises(
233
+ (openai.BadRequestError, openai.APIError)): # test using token IDs
234
+ stream = await client.completions.create(
235
+ model=MODEL_NAME,
236
+ prompt=[0, 0, 0, 0, 0],
237
+ max_tokens=5,
238
+ temperature=0.0,
239
+ # vLLM has higher default max_logprobs (20 instead of 5) to support
240
+ # both Completion API and Chat Completion API
241
+ logprobs=30,
242
+ stream=True,
243
+ )
244
+ async for chunk in stream:
245
+ ...
246
+
247
+ # the server should still work afterwards
248
+ completion = await client.completions.create(
249
+ model=model_name,
250
+ prompt=[0, 0, 0, 0, 0],
251
+ max_tokens=5,
252
+ temperature=0.0,
253
+ )
254
+ assert len(completion.choices[0].text) >= 0
255
+
256
+
257
+@pytest.mark.asyncio
258
+@pytest.mark.parametrize(
259
+ "model_name",
260
+ [MODEL_NAME, "zephyr-lora"],
261
+)
262
+async def test_completion_streaming(client: openai.AsyncOpenAI,
263
+ model_name: str):
264
+ prompt = "What is an LLM?"
265
+
266
+ single_completion = await client.completions.create(
267
+ model=model_name,
268
+ prompt=prompt,
269
+ max_tokens=5,
270
+ temperature=0.0,
271
+ )
272
+ single_output = single_completion.choices[0].text
273
+ stream = await client.completions.create(model=model_name,
274
+ prompt=prompt,
275
+ max_tokens=5,
276
+ temperature=0.0,
277
+ stream=True)
278
+ chunks: List[str] = []
279
+ finish_reason_count = 0
280
+ async for chunk in stream:
281
+ chunks.append(chunk.choices[0].text)
282
+ if chunk.choices[0].finish_reason is not None:
283
+ finish_reason_count += 1
284
+ # finish reason should only return in last block
285
+ assert finish_reason_count == 1
286
+ assert chunk.choices[0].finish_reason == "length"
287
+ assert chunk.choices[0].text
288
+ assert "".join(chunks) == single_output
289
+
290
+
291
+@pytest.mark.asyncio
292
+@pytest.mark.parametrize(
293
+ "model_name",
294
+ ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
295
+)
296
+async def test_completion_stream_options(client: openai.AsyncOpenAI,
297
+ model_name: str):
298
+ prompt = "What is the capital of France?"
299
+
300
+ # Test stream=True, stream_options={"include_usage": False}
301
+ stream = await client.completions.create(
302
+ model=model_name,
303
+ prompt=prompt,
304
+ max_tokens=5,
305
+ temperature=0.0,
306
+ stream=True,
307
+ stream_options={"include_usage": False})
308
+ async for chunk in stream:
309
+ assert chunk.usage is None
310
+
311
+ # Test stream=True, stream_options={"include_usage": True}
312
+ stream = await client.completions.create(
313
+ model=model_name,
314
+ prompt=prompt,
315
+ max_tokens=5,
316
+ temperature=0.0,
317
+ stream=True,
318
+ stream_options={"include_usage": True})
319
+ async for chunk in stream:
320
+ if chunk.choices[0].finish_reason is None:
321
+ assert chunk.usage is None
322
+ else:
323
+ assert chunk.usage is None
324
+ final_chunk = await stream.__anext__()
325
+ assert final_chunk.usage is not None
326
+ assert final_chunk.usage.prompt_tokens > 0
327
+ assert final_chunk.usage.completion_tokens > 0
328
+ assert final_chunk.usage.total_tokens == (
329
+ final_chunk.usage.prompt_tokens +
330
+ final_chunk.usage.completion_tokens)
331
+ assert final_chunk.choices == []
332
+
333
+ # Test stream=False, stream_options={"include_usage": None}
334
+ with pytest.raises(BadRequestError):
335
+ await client.completions.create(model=model_name,
336
+ prompt=prompt,
337
+ max_tokens=5,
338
+ temperature=0.0,
339
+ stream=False,
340
+ stream_options={"include_usage": None})
341
+
342
+ # Test stream=False, stream_options={"include_usage": True}
343
+ with pytest.raises(BadRequestError):
344
+ await client.completions.create(model=model_name,
345
+ prompt=prompt,
346
+ max_tokens=5,
347
+ temperature=0.0,
348
+ stream=False,
349
+ stream_options={"include_usage": True})
350
+
351
+
352
+@pytest.mark.asyncio
353
+@pytest.mark.parametrize(
354
+ # just test 1 lora hereafter
355
+ "model_name",
356
+ [MODEL_NAME, "zephyr-lora"],
357
+)
358
+async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
359
+ # test both text and token IDs
360
+ for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
361
+ # test simple list
362
+ batch = await client.completions.create(
363
+ model=model_name,
364
+ prompt=prompts,
365
+ max_tokens=5,
366
+ temperature=0.0,
367
+ )
368
+ assert len(batch.choices) == 2
369
+ assert batch.choices[0].text == batch.choices[1].text
370
+
371
+ # test n = 2
372
+ batch = await client.completions.create(
373
+ model=model_name,
374
+ prompt=prompts,
375
+ n=2,
376
+ max_tokens=5,
377
+ temperature=0.0,
378
+ extra_body=dict(
379
+ # NOTE: this has to be true for n > 1 in vLLM, but not necessary
380
+ # for official client.
381
+ use_beam_search=True),
382
+ )
383
+ assert len(batch.choices) == 4
384
+ assert batch.choices[0].text != batch.choices[
385
+ 1].text, "beam search should be different"
386
+ assert batch.choices[0].text == batch.choices[
387
+ 2].text, "two copies of the same prompt should be the same"
388
+ assert batch.choices[1].text == batch.choices[
389
+ 3].text, "two copies of the same prompt should be the same"
390
+
391
+ # test streaming
392
+ batch = await client.completions.create(
393
+ model=model_name,
394
+ prompt=prompts,
395
+ max_tokens=5,
396
+ temperature=0.0,
397
+ stream=True,
398
+ )
399
+ texts = [""] * 2
400
+ async for chunk in batch:
401
+ assert len(chunk.choices) == 1
402
+ choice = chunk.choices[0]
403
+ texts[choice.index] += choice.text
404
+ assert texts[0] == texts[1]
405
+
406
+
407
+@pytest.mark.asyncio
408
+async def test_logits_bias(client: openai.AsyncOpenAI):
409
+ prompt = "Hello, my name is"
410
+ max_tokens = 5
411
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
412
+
413
+ # Test exclusive selection
414
+ token_id = 1000
415
+ completion = await client.completions.create(
416
+ model=MODEL_NAME,
417
+ prompt=prompt,
418
+ max_tokens=max_tokens,
419
+ temperature=0.0,
420
+ logit_bias={str(token_id): 100},
421
+ seed=42,
422
+ )
423
+ assert len(completion.choices[0].text) >= 5
424
+ response_tokens = tokenizer(completion.choices[0].text,
425
+ add_special_tokens=False)["input_ids"]
426
+ expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
427
+ add_special_tokens=False)["input_ids"]
428
+ assert all([
429
+ response == expected
430
+ for response, expected in zip(response_tokens, expected_tokens)
431
+ ])
432
+
433
+ # Test ban
434
+ completion = await client.completions.create(
435
+ model=MODEL_NAME,
436
+ prompt=prompt,
437
+ max_tokens=max_tokens,
438
+ temperature=0.0,
439
+ )
440
+ response_tokens = tokenizer(completion.choices[0].text,
441
+ add_special_tokens=False)["input_ids"]
442
+ first_response = completion.choices[0].text
443
+ completion = await client.completions.create(
444
+ model=MODEL_NAME,
445
+ prompt=prompt,
446
+ max_tokens=max_tokens,
447
+ temperature=0.0,
448
+ logit_bias={str(token): -100
449
+ for token in response_tokens},
450
+ )
451
+ assert first_response != completion.choices[0].text
452
+
453
+
454
+@pytest.mark.asyncio
455
+@pytest.mark.parametrize("guided_decoding_backend",
456
+ ["outlines", "lm-format-enforcer"])
457
+async def test_guided_json_completion(client: openai.AsyncOpenAI,
458
+ guided_decoding_backend: str):
459
+ completion = await client.completions.create(
460
+ model=MODEL_NAME,
461
+ prompt=f"Give an example JSON for an employee profile "
462
+ f"that fits this schema: {TEST_SCHEMA}",
463
+ n=3,
464
+ temperature=1.0,
465
+ max_tokens=500,
466
+ extra_body=dict(guided_json=TEST_SCHEMA,
467
+ guided_decoding_backend=guided_decoding_backend))
468
+
469
+ assert completion.id is not None
470
+ assert len(completion.choices) == 3
471
+ for i in range(3):
472
+ output_json = json.loads(completion.choices[i].text)
473
+ jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
474
+
475
+
476
+@pytest.mark.asyncio
477
+@pytest.mark.parametrize("guided_decoding_backend",
478
+ ["outlines", "lm-format-enforcer"])
479
+async def test_guided_regex_completion(client: openai.AsyncOpenAI,
480
+ guided_decoding_backend: str):
481
+ completion = await client.completions.create(
482
+ model=MODEL_NAME,
483
+ prompt=f"Give an example IPv4 address with this regex: {TEST_REGEX}",
484
+ n=3,
485
+ temperature=1.0,
486
+ max_tokens=20,
487
+ extra_body=dict(guided_regex=TEST_REGEX,
488
+ guided_decoding_backend=guided_decoding_backend))
489
+
490
+ assert completion.id is not None
491
+ assert len(completion.choices) == 3
492
+ for i in range(3):
493
+ assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
494
+
495
+
496
+@pytest.mark.asyncio
497
+@pytest.mark.parametrize("guided_decoding_backend",
498
+ ["outlines", "lm-format-enforcer"])
499
+async def test_guided_choice_completion(client: openai.AsyncOpenAI,
500
+ guided_decoding_backend: str):
501
+ completion = await client.completions.create(
502
+ model=MODEL_NAME,
503
+ prompt="The best language for type-safe systems programming is ",
504
+ n=2,
505
+ temperature=1.0,
506
+ max_tokens=10,
507
+ extra_body=dict(guided_choice=TEST_CHOICE,
508
+ guided_decoding_backend=guided_decoding_backend))
509
+
510
+ assert completion.id is not None
511
+ assert len(completion.choices) == 2
512
+ for i in range(2):
513
+ assert completion.choices[i].text in TEST_CHOICE
514
+
515
+
516
+@pytest.mark.asyncio
517
+async def test_guided_grammar(client: openai.AsyncOpenAI):
518
+ simple_sql_grammar = """
519
+start: select_statement
520
+
521
+select_statement: "SELECT" column "from" table "where" condition
522
+
523
+column: "col_1" | "col_2"
524
+table: "table_1" | "table_2"
525
+condition: column "=" number
526
+
527
+number: "1" | "2"
528
+"""
529
+
530
+ completion = await client.completions.create(
531
+ model=MODEL_NAME,
532
+ prompt=("Generate a sql state that select col_1 from "
533
+ "table_1 where it is equals to 1"),
534
+ temperature=1.0,
535
+ max_tokens=500,
536
+ extra_body=dict(guided_grammar=simple_sql_grammar))
537
+
538
+ content = completion.choices[0].text
539
+
540
+ # use Lark to parse the output, and make sure it's a valid parse tree
541
+ from lark import Lark
542
+ parser = Lark(simple_sql_grammar)
543
+ parser.parse(content)
544
+
545
+ # remove spaces for comparison b/c we removed them in the grammar
546
+ ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(" ", "")
547
+
548
+ assert content.strip() == ground_truth
549
+
550
+
551
+@pytest.mark.asyncio
552
+@pytest.mark.parametrize(
553
+ # first test base model, then test loras
554
+ "model_name",
555
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
556
+)
557
+@pytest.mark.parametrize("logprobs_arg", [1, 0])
558
+async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
559
+ model_name: str, logprobs_arg: int):
560
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
561
+ # test using text and token IDs
562
+ for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
563
+ completion = await client.completions.create(model=model_name,
564
+ prompt=prompt,
565
+ max_tokens=5,
566
+ temperature=0.0,
567
+ echo=True,
568
+ logprobs=logprobs_arg)
569
+
570
+ prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
571
+ list) else prompt
572
+ assert re.search(r"^" + prompt_text, completion.choices[0].text)
573
+ logprobs = completion.choices[0].logprobs
574
+ assert logprobs is not None
575
+ assert len(logprobs.text_offset) > 5
576
+ assert (len(logprobs.token_logprobs) > 5
577
+ and logprobs.token_logprobs[0] is None)
578
+ assert (len(logprobs.top_logprobs) > 5
579
+ and logprobs.top_logprobs[0] is None)
580
+ for top_logprobs in logprobs.top_logprobs[1:]:
581
+ assert max(logprobs_arg,
582
+ 1) <= len(top_logprobs) <= logprobs_arg + 1
583
+ assert len(logprobs.tokens) > 5
584
+
585
+
586
+@pytest.mark.asyncio
587
+@pytest.mark.parametrize("guided_decoding_backend",
588
+ ["outlines", "lm-format-enforcer"])
589
+async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
590
+ guided_decoding_backend: str):
591
+ with pytest.raises(openai.BadRequestError):
592
+ _ = await client.completions.create(
593
+ model=MODEL_NAME,
594
+ prompt="Give an example JSON that fits this schema: 42",
595
+ extra_body=dict(guided_json=42,
596
+ guided_decoding_backend=guided_decoding_backend))
597
+
598
+ with pytest.raises(openai.BadRequestError):
599
+ _ = await client.completions.create(
600
+ model=MODEL_NAME,
601
+ prompt="Give an example string that fits this regex",
602
+ extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
603
+
604
+
605
+@pytest.mark.asyncio
606
+@pytest.mark.parametrize(
607
+ "model_name",
608
+ [MODEL_NAME],
609
+)
610
+async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
611
+ base_url = str(client.base_url)[:-3]
612
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
613
+
614
+ for add_special in [False, True]:
615
+ prompt = "This is a test prompt."
616
+ tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
617
+
618
+ response = requests.post(base_url + "/tokenize",
619
+ json={
620
+ "add_special_tokens": add_special,
621
+ "model": model_name,
622
+ "prompt": prompt
623
+ })
624
+ response.raise_for_status()
625
+ assert response.json() == {
626
+ "tokens": tokens,
627
+ "count": len(tokens),
628
+ "max_model_len": 8192
629
+ }
630
+
631
+
632
+@pytest.mark.asyncio
633
+@pytest.mark.parametrize(
634
+ "model_name",
635
+ [MODEL_NAME],
636
+)
637
+async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
638
+ base_url = str(client.base_url)[:-3]
639
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
640
+
641
+ prompt = "This is a test prompt."
642
+ tokens = tokenizer.encode(prompt, add_special_tokens=False)
643
+
644
+ response = requests.post(base_url + "detokenize",
645
+ json={
646
+ "model": model_name,
647
+ "tokens": tokens
648
+ })
649
+ response.raise_for_status()
650
+ assert response.json() == {"prompt": prompt}
tests/entrypoints/test_openai_server.py
+1
-596
@@ -1,20 +1,10 @@
1
-# imports for guided decoding tests
2
-import json
3
-import re
4
-from typing import List
5
-
6
-import jsonschema
1
import openai # use the official client for correctness check
2
import pytest
3
# using Ray for overall ease of process management, parallel requests,
4
# and debugging.
5
import ray
12
-import requests
6
# downloading lora to test lora requests
7
from huggingface_hub import snapshot_download
15
-from openai import BadRequestError
16
-
17
-from vllm.transformers_utils.tokenizer import get_tokenizer
8
9
from ..utils import RemoteOpenAIServer
10
@@ -24,57 +14,10 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
14
# generation quality here
15
LORA_NAME = "typeof/zephyr-7b-beta-lora"
16
27
-TEST_SCHEMA = {
28
- "type": "object",
29
- "properties": {
30
- "name": {
31
- "type": "string"
32
- },
33
- "age": {
34
- "type": "integer"
35
- },
36
- "skills": {
37
- "type": "array",
38
- "items": {
39
- "type": "string",
40
- "maxLength": 10
41
- },
42
- "minItems": 3
43
- },
44
- "work history": {
45
- "type": "array",
46
- "items": {
47
- "type": "object",
48
- "properties": {
49
- "company": {
50
- "type": "string"
51
- },
52
- "duration": {
53
- "type": "string"
54
- },
55
- "position": {
56
- "type": "string"
57
- }
58
- },
59
- "required": ["company", "position"]
60
- }
61
- }
62
- },
63
- "required": ["name", "age", "skills", "work history"]
64
-}
65
-
66
-TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
67
- r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
68
-
69
-TEST_CHOICE = [
70
- "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
71
- "Swift", "Kotlin"
72
-]
73
-
17
pytestmark = pytest.mark.openai
18
19
77
-@pytest.fixture(scope="session")
20
+@pytest.fixture(scope="module")
21
def zephyr_lora_files():
22
return snapshot_download(repo_id=LORA_NAME)
23
@@ -126,541 +69,3 @@ async def test_check_models(client: openai.AsyncOpenAI):
69
assert all(model.root == MODEL_NAME for model in models)
70
assert lora_models[0].id == "zephyr-lora"
71
assert lora_models[1].id == "zephyr-lora2"
129
-
130
-
131
-@pytest.mark.asyncio
132
-@pytest.mark.parametrize(
133
- # first test base model, then test loras
134
- "model_name",
135
- [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
136
-)
137
-async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
138
- completion = await client.completions.create(model=model_name,
139
- prompt="Hello, my name is",
140
- max_tokens=5,
141
- temperature=0.0)
142
-
143
- assert completion.id is not None
144
- assert completion.choices is not None and len(completion.choices) == 1
145
-
146
- choice = completion.choices[0]
147
- assert len(choice.text) >= 5
148
- assert choice.finish_reason == "length"
149
- assert completion.usage == openai.types.CompletionUsage(
150
- completion_tokens=5, prompt_tokens=6, total_tokens=11)
151
-
152
- # test using token IDs
153
- completion = await client.completions.create(
154
- model=MODEL_NAME,
155
- prompt=[0, 0, 0, 0, 0],
156
- max_tokens=5,
157
- temperature=0.0,
158
- )
159
- assert len(completion.choices[0].text) >= 5
160
-
161
-
162
-@pytest.mark.asyncio
163
-@pytest.mark.parametrize(
164
- # first test base model, then test loras
165
- "model_name",
166
- [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
167
-)
168
-async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
169
- # test using token IDs
170
- completion = await client.completions.create(
171
- model=MODEL_NAME,
172
- prompt=[0, 0, 0, 0, 0],
173
- max_tokens=5,
174
- temperature=0.0,
175
- logprobs=None,
176
- )
177
- choice = completion.choices[0]
178
- assert choice.logprobs is None
179
-
180
-
181
-@pytest.mark.asyncio
182
-@pytest.mark.parametrize(
183
- # just test 1 lora hereafter
184
- "model_name",
185
- [MODEL_NAME, "zephyr-lora"],
186
-)
187
-async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
188
- # test using token IDs
189
- completion = await client.completions.create(
190
- model=MODEL_NAME,
191
- prompt=[0, 0, 0, 0, 0],
192
- max_tokens=5,
193
- temperature=0.0,
194
- logprobs=0,
195
- )
196
- choice = completion.choices[0]
197
- assert choice.logprobs is not None
198
- assert choice.logprobs.token_logprobs is not None
199
- assert choice.logprobs.top_logprobs is not None
200
- assert len(choice.logprobs.top_logprobs[0]) == 1
201
-
202
-
203
-@pytest.mark.asyncio
204
-@pytest.mark.parametrize(
205
- "model_name",
206
- [MODEL_NAME, "zephyr-lora"],
207
-)
208
-async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
209
- # test using token IDs
210
- completion = await client.completions.create(
211
- model=MODEL_NAME,
212
- prompt=[0, 0, 0, 0, 0],
213
- max_tokens=5,
214
- temperature=0.0,
215
- logprobs=5,
216
- )
217
- choice = completion.choices[0]
218
- assert choice.logprobs is not None
219
- assert choice.logprobs.token_logprobs is not None
220
- assert choice.logprobs.top_logprobs is not None
221
- assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
222
-
223
-
224
-@pytest.mark.asyncio
225
-@pytest.mark.parametrize(
226
- "model_name",
227
- [MODEL_NAME, "zephyr-lora"],
228
-)
229
-async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
230
- model_name: str):
231
-
232
- with pytest.raises(
233
- (openai.BadRequestError, openai.APIError)): # test using token IDs
234
- await client.completions.create(
235
- model=MODEL_NAME,
236
- prompt=[0, 0, 0, 0, 0],
237
- max_tokens=5,
238
- temperature=0.0,
239
- # vLLM has higher default max_logprobs (20 instead of 5) to support
240
- # both Completion API and Chat Completion API
241
- logprobs=21,
242
- )
243
- ...
244
- with pytest.raises(
245
- (openai.BadRequestError, openai.APIError)): # test using token IDs
246
- stream = await client.completions.create(
247
- model=MODEL_NAME,
248
- prompt=[0, 0, 0, 0, 0],
249
- max_tokens=5,
250
- temperature=0.0,
251
- # vLLM has higher default max_logprobs (20 instead of 5) to support
252
- # both Completion API and Chat Completion API
253
- logprobs=30,
254
- stream=True,
255
- )
256
- async for chunk in stream:
257
- ...
258
-
259
- # the server should still work afterwards
260
- completion = await client.completions.create(
261
- model=model_name,
262
- prompt=[0, 0, 0, 0, 0],
263
- max_tokens=5,
264
- temperature=0.0,
265
- )
266
- assert len(completion.choices[0].text) >= 0
267
-
268
-
269
-@pytest.mark.asyncio
270
-@pytest.mark.parametrize(
271
- "model_name",
272
- [MODEL_NAME, "zephyr-lora"],
273
-)
274
-async def test_completion_streaming(client: openai.AsyncOpenAI,
275
- model_name: str):
276
- prompt = "What is an LLM?"
277
-
278
- single_completion = await client.completions.create(
279
- model=model_name,
280
- prompt=prompt,
281
- max_tokens=5,
282
- temperature=0.0,
283
- )
284
- single_output = single_completion.choices[0].text
285
- stream = await client.completions.create(model=model_name,
286
- prompt=prompt,
287
- max_tokens=5,
288
- temperature=0.0,
289
- stream=True)
290
- chunks: List[str] = []
291
- finish_reason_count = 0
292
- async for chunk in stream:
293
- chunks.append(chunk.choices[0].text)
294
- if chunk.choices[0].finish_reason is not None:
295
- finish_reason_count += 1
296
- # finish reason should only return in last block
297
- assert finish_reason_count == 1
298
- assert chunk.choices[0].finish_reason == "length"
299
- assert chunk.choices[0].text
300
- assert "".join(chunks) == single_output
301
-
302
-
303
-@pytest.mark.asyncio
304
-@pytest.mark.parametrize(
305
- "model_name",
306
- ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
307
-)
308
-async def test_completion_stream_options(client: openai.AsyncOpenAI,
309
- model_name: str):
310
- prompt = "What is the capital of France?"
311
-
312
- # Test stream=True, stream_options={"include_usage": False}
313
- stream = await client.completions.create(
314
- model=model_name,
315
- prompt=prompt,
316
- max_tokens=5,
317
- temperature=0.0,
318
- stream=True,
319
- stream_options={"include_usage": False})
320
- async for chunk in stream:
321
- assert chunk.usage is None
322
-
323
- # Test stream=True, stream_options={"include_usage": True}
324
- stream = await client.completions.create(
325
- model=model_name,
326
- prompt=prompt,
327
- max_tokens=5,
328
- temperature=0.0,
329
- stream=True,
330
- stream_options={"include_usage": True})
331
- async for chunk in stream:
332
- if chunk.choices[0].finish_reason is None:
333
- assert chunk.usage is None
334
- else:
335
- assert chunk.usage is None
336
- final_chunk = await stream.__anext__()
337
- assert final_chunk.usage is not None
338
- assert final_chunk.usage.prompt_tokens > 0
339
- assert final_chunk.usage.completion_tokens > 0
340
- assert final_chunk.usage.total_tokens == (
341
- final_chunk.usage.prompt_tokens +
342
- final_chunk.usage.completion_tokens)
343
- assert final_chunk.choices == []
344
-
345
- # Test stream=False, stream_options={"include_usage": None}
346
- with pytest.raises(BadRequestError):
347
- await client.completions.create(model=model_name,
348
- prompt=prompt,
349
- max_tokens=5,
350
- temperature=0.0,
351
- stream=False,
352
- stream_options={"include_usage": None})
353
-
354
- # Test stream=False, stream_options={"include_usage": True}
355
- with pytest.raises(BadRequestError):
356
- await client.completions.create(model=model_name,
357
- prompt=prompt,
358
- max_tokens=5,
359
- temperature=0.0,
360
- stream=False,
361
- stream_options={"include_usage": True})
362
-
363
-
364
-@pytest.mark.asyncio
365
-@pytest.mark.parametrize(
366
- # just test 1 lora hereafter
367
- "model_name",
368
- [MODEL_NAME, "zephyr-lora"],
369
-)
370
-async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
371
- # test both text and token IDs
372
- for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
373
- # test simple list
374
- batch = await client.completions.create(
375
- model=model_name,
376
- prompt=prompts,
377
- max_tokens=5,
378
- temperature=0.0,
379
- )
380
- assert len(batch.choices) == 2
381
- assert batch.choices[0].text == batch.choices[1].text
382
-
383
- # test n = 2
384
- batch = await client.completions.create(
385
- model=model_name,
386
- prompt=prompts,
387
- n=2,
388
- max_tokens=5,
389
- temperature=0.0,
390
- extra_body=dict(
391
- # NOTE: this has to be true for n > 1 in vLLM, but not necessary
392
- # for official client.
393
- use_beam_search=True),
394
- )
395
- assert len(batch.choices) == 4
396
- assert batch.choices[0].text != batch.choices[
397
- 1].text, "beam search should be different"
398
- assert batch.choices[0].text == batch.choices[
399
- 2].text, "two copies of the same prompt should be the same"
400
- assert batch.choices[1].text == batch.choices[
401
- 3].text, "two copies of the same prompt should be the same"
402
-
403
- # test streaming
404
- batch = await client.completions.create(
405
- model=model_name,
406
- prompt=prompts,
407
- max_tokens=5,
408
- temperature=0.0,
409
- stream=True,
410
- )
411
- texts = [""] * 2
412
- async for chunk in batch:
413
- assert len(chunk.choices) == 1
414
- choice = chunk.choices[0]
415
- texts[choice.index] += choice.text
416
- assert texts[0] == texts[1]
417
-
418
-
419
-@pytest.mark.asyncio
420
-async def test_logits_bias(client: openai.AsyncOpenAI):
421
- prompt = "Hello, my name is"
422
- max_tokens = 5
423
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
424
-
425
- # Test exclusive selection
426
- token_id = 1000
427
- completion = await client.completions.create(
428
- model=MODEL_NAME,
429
- prompt=prompt,
430
- max_tokens=max_tokens,
431
- temperature=0.0,
432
- logit_bias={str(token_id): 100},
433
- seed=42,
434
- )
435
- assert len(completion.choices[0].text) >= 5
436
- response_tokens = tokenizer(completion.choices[0].text,
437
- add_special_tokens=False)["input_ids"]
438
- expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
439
- add_special_tokens=False)["input_ids"]
440
- assert all([
441
- response == expected
442
- for response, expected in zip(response_tokens, expected_tokens)
443
- ])
444
-
445
- # Test ban
446
- completion = await client.completions.create(
447
- model=MODEL_NAME,
448
- prompt=prompt,
449
- max_tokens=max_tokens,
450
- temperature=0.0,
451
- )
452
- response_tokens = tokenizer(completion.choices[0].text,
453
- add_special_tokens=False)["input_ids"]
454
- first_response = completion.choices[0].text
455
- completion = await client.completions.create(
456
- model=MODEL_NAME,
457
- prompt=prompt,
458
- max_tokens=max_tokens,
459
- temperature=0.0,
460
- logit_bias={str(token): -100
461
- for token in response_tokens},
462
- )
463
- assert first_response != completion.choices[0].text
464
-
465
-
466
-@pytest.mark.asyncio
467
-@pytest.mark.parametrize("guided_decoding_backend",
468
- ["outlines", "lm-format-enforcer"])
469
-async def test_guided_json_completion(client: openai.AsyncOpenAI,
470
- guided_decoding_backend: str):
471
- completion = await client.completions.create(
472
- model=MODEL_NAME,
473
- prompt=f"Give an example JSON for an employee profile "
474
- f"that fits this schema: {TEST_SCHEMA}",
475
- n=3,
476
- temperature=1.0,
477
- max_tokens=500,
478
- extra_body=dict(guided_json=TEST_SCHEMA,
479
- guided_decoding_backend=guided_decoding_backend))
480
-
481
- assert completion.id is not None
482
- assert len(completion.choices) == 3
483
- for i in range(3):
484
- output_json = json.loads(completion.choices[i].text)
485
- jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
486
-
487
-
488
-@pytest.mark.asyncio
489
-@pytest.mark.parametrize("guided_decoding_backend",
490
- ["outlines", "lm-format-enforcer"])
491
-async def test_guided_regex_completion(client: openai.AsyncOpenAI,
492
- guided_decoding_backend: str):
493
- completion = await client.completions.create(
494
- model=MODEL_NAME,
495
- prompt=f"Give an example IPv4 address with this regex: {TEST_REGEX}",
496
- n=3,
497
- temperature=1.0,
498
- max_tokens=20,
499
- extra_body=dict(guided_regex=TEST_REGEX,
500
- guided_decoding_backend=guided_decoding_backend))
501
-
502
- assert completion.id is not None
503
- assert len(completion.choices) == 3
504
- for i in range(3):
505
- assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
506
-
507
-
508
-@pytest.mark.asyncio
509
-@pytest.mark.parametrize("guided_decoding_backend",
510
- ["outlines", "lm-format-enforcer"])
511
-async def test_guided_choice_completion(client: openai.AsyncOpenAI,
512
- guided_decoding_backend: str):
513
- completion = await client.completions.create(
514
- model=MODEL_NAME,
515
- prompt="The best language for type-safe systems programming is ",
516
- n=2,
517
- temperature=1.0,
518
- max_tokens=10,
519
- extra_body=dict(guided_choice=TEST_CHOICE,
520
- guided_decoding_backend=guided_decoding_backend))
521
-
522
- assert completion.id is not None
523
- assert len(completion.choices) == 2
524
- for i in range(2):
525
- assert completion.choices[i].text in TEST_CHOICE
526
-
527
-
528
-@pytest.mark.asyncio
529
-async def test_guided_grammar(client: openai.AsyncOpenAI):
530
- simple_sql_grammar = """
531
-start: select_statement
532
-
533
-select_statement: "SELECT" column "from" table "where" condition
534
-
535
-column: "col_1" | "col_2"
536
-table: "table_1" | "table_2"
537
-condition: column "=" number
538
-
539
-number: "1" | "2"
540
-"""
541
-
542
- completion = await client.completions.create(
543
- model=MODEL_NAME,
544
- prompt=("Generate a sql state that select col_1 from "
545
- "table_1 where it is equals to 1"),
546
- temperature=1.0,
547
- max_tokens=500,
548
- extra_body=dict(guided_grammar=simple_sql_grammar))
549
-
550
- content = completion.choices[0].text
551
-
552
- # use Lark to parse the output, and make sure it's a valid parse tree
553
- from lark import Lark
554
- parser = Lark(simple_sql_grammar)
555
- parser.parse(content)
556
-
557
- # remove spaces for comparison b/c we removed them in the grammar
558
- ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(" ", "")
559
-
560
- assert content.strip() == ground_truth
561
-
562
-
563
-@pytest.mark.asyncio
564
-@pytest.mark.parametrize(
565
- # first test base model, then test loras
566
- "model_name",
567
- [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
568
-)
569
-@pytest.mark.parametrize("logprobs_arg", [1, 0])
570
-async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
571
- model_name: str, logprobs_arg: int):
572
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
573
- # test using text and token IDs
574
- for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
575
- completion = await client.completions.create(model=model_name,
576
- prompt=prompt,
577
- max_tokens=5,
578
- temperature=0.0,
579
- echo=True,
580
- logprobs=logprobs_arg)
581
-
582
- prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
583
- list) else prompt
584
- assert re.search(r"^" + prompt_text, completion.choices[0].text)
585
- logprobs = completion.choices[0].logprobs
586
- assert logprobs is not None
587
- assert len(logprobs.text_offset) > 5
588
- assert (len(logprobs.token_logprobs) > 5
589
- and logprobs.token_logprobs[0] is None)
590
- assert (len(logprobs.top_logprobs) > 5
591
- and logprobs.top_logprobs[0] is None)
592
- for top_logprobs in logprobs.top_logprobs[1:]:
593
- assert max(logprobs_arg,
594
- 1) <= len(top_logprobs) <= logprobs_arg + 1
595
- assert len(logprobs.tokens) > 5
596
-
597
-
598
-@pytest.mark.asyncio
599
-@pytest.mark.parametrize("guided_decoding_backend",
600
- ["outlines", "lm-format-enforcer"])
601
-async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
602
- guided_decoding_backend: str):
603
- with pytest.raises(openai.BadRequestError):
604
- _ = await client.completions.create(
605
- model=MODEL_NAME,
606
- prompt="Give an example JSON that fits this schema: 42",
607
- extra_body=dict(guided_json=42,
608
- guided_decoding_backend=guided_decoding_backend))
609
-
610
- with pytest.raises(openai.BadRequestError):
611
- _ = await client.completions.create(
612
- model=MODEL_NAME,
613
- prompt="Give an example string that fits this regex",
614
- extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
615
-
616
-
617
-@pytest.mark.asyncio
618
-@pytest.mark.parametrize(
619
- "model_name",
620
- [MODEL_NAME],
621
-)
622
-async def test_tokenize(client: openai.AsyncOpenAI, model_name: str):
623
- base_url = str(client.base_url)[:-3]
624
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
625
-
626
- for add_special in [False, True]:
627
- prompt = "This is a test prompt."
628
- tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
629
-
630
- response = requests.post(base_url + "/tokenize",
631
- json={
632
- "add_special_tokens": add_special,
633
- "model": model_name,
634
- "prompt": prompt
635
- })
636
- response.raise_for_status()
637
- assert response.json() == {
638
- "tokens": tokens,
639
- "count": len(tokens),
640
- "max_model_len": 8192
641
- }
642
-
643
-
644
-@pytest.mark.asyncio
645
-@pytest.mark.parametrize(
646
- "model_name",
647
- [MODEL_NAME],
648
-)
649
-async def test_detokenize(client: openai.AsyncOpenAI, model_name: str):
650
- base_url = str(client.base_url)[:-3]
651
- tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME, tokenizer_mode="fast")
652
-
653
- prompt = "This is a test prompt."
654
- tokens = tokenizer.encode(prompt, add_special_tokens=False)
655
-
656
- response = requests.post(base_url + "detokenize",
657
- json={
658
- "model": model_name,
659
- "tokens": tokens
660
- })
661
- response.raise_for_status()
662
- assert response.json() == {"prompt": prompt}
663
-
664
-
665
-if __name__ == "__main__":
666
- pytest.main([__file__])