A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Misc] add fixture to guided processor tests (#6341)
Yihuan Bu committed
Jul 12, 2024 at 12:55 UTC
b039cbbce37c1e6a1fdb284a41bd27cfef1b3783
4 files changed
+144
-206
tests/entrypoints/openai/conftest.py
new
+69
@@ -0,0 +1,69 @@
1
+import pytest
2
+
3
+
4
+@pytest.fixture
5
+def sample_regex():
6
+ return (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
7
+ r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
8
+
9
+
10
+@pytest.fixture
11
+def sample_json_schema():
12
+ return {
13
+ "type": "object",
14
+ "properties": {
15
+ "name": {
16
+ "type": "string"
17
+ },
18
+ "age": {
19
+ "type": "integer"
20
+ },
21
+ "skills": {
22
+ "type": "array",
23
+ "items": {
24
+ "type": "string",
25
+ "maxLength": 10
26
+ },
27
+ "minItems": 3
28
+ },
29
+ "work_history": {
30
+ "type": "array",
31
+ "items": {
32
+ "type": "object",
33
+ "properties": {
34
+ "company": {
35
+ "type": "string"
36
+ },
37
+ "duration": {
38
+ "type": "number"
39
+ },
40
+ "position": {
41
+ "type": "string"
42
+ }
43
+ },
44
+ "required": ["company", "position"]
45
+ }
46
+ }
47
+ },
48
+ "required": ["name", "age", "skills", "work_history"]
49
+ }
50
+
51
+
52
+@pytest.fixture
53
+def sample_guided_choice():
54
+ return [
55
+ "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript",
56
+ "Ruby", "Swift", "Kotlin"
57
+ ]
58
+
59
+
60
+@pytest.fixture
61
+def sample_sql_statements():
62
+ return ("""
63
+start: select_statement
64
+select_statement: "SELECT" column "from" table "where" condition
65
+column: "col_1" | "col_2"
66
+table: "table_1" | "table_2"
67
+condition: column "=" number
68
+number: "1" | "2"
69
+""")
\ No newline at end of file
tests/entrypoints/openai/test_chat.py
+39
-80
@@ -22,53 +22,6 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
22
# generation quality here
23
LORA_NAME = "typeof/zephyr-7b-beta-lora"
24
25
-TEST_SCHEMA = {
26
- "type": "object",
27
- "properties": {
28
- "name": {
29
- "type": "string"
30
- },
31
- "age": {
32
- "type": "integer"
33
- },
34
- "skills": {
35
- "type": "array",
36
- "items": {
37
- "type": "string",
38
- "maxLength": 10
39
- },
40
- "minItems": 3
41
- },
42
- "work history": {
43
- "type": "array",
44
- "items": {
45
- "type": "object",
46
- "properties": {
47
- "company": {
48
- "type": "string"
49
- },
50
- "duration": {
51
- "type": "string"
52
- },
53
- "position": {
54
- "type": "string"
55
- }
56
- },
57
- "required": ["company", "position"]
58
- }
59
- }
60
- },
61
- "required": ["name", "age", "skills", "work history"]
62
-}
63
-
64
-TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
65
- r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
66
-
67
-TEST_CHOICE = [
68
- "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
69
- "Swift", "Kotlin"
70
-]
71
-
25
26
@pytest.fixture(scope="module")
27
def zephyr_lora_files():
@@ -408,7 +361,8 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
361
@pytest.mark.parametrize("guided_decoding_backend",
362
["outlines", "lm-format-enforcer"])
363
async def test_guided_choice_chat(client: openai.AsyncOpenAI,
411
- guided_decoding_backend: str):
364
+ guided_decoding_backend: str,
365
+ sample_guided_choice):
366
messages = [{
367
"role": "system",
368
"content": "you are a helpful assistant"
@@ -422,10 +376,10 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
376
model=MODEL_NAME,
377
messages=messages,
378
max_tokens=10,
425
- extra_body=dict(guided_choice=TEST_CHOICE,
379
+ extra_body=dict(guided_choice=sample_guided_choice,
380
guided_decoding_backend=guided_decoding_backend))
381
choice1 = chat_completion.choices[0].message.content
428
- assert choice1 in TEST_CHOICE
382
+ assert choice1 in sample_guided_choice
383
384
messages.append({"role": "assistant", "content": choice1})
385
messages.append({
@@ -436,10 +390,10 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
390
model=MODEL_NAME,
391
messages=messages,
392
max_tokens=10,
439
- extra_body=dict(guided_choice=TEST_CHOICE,
393
+ extra_body=dict(guided_choice=sample_guided_choice,
394
guided_decoding_backend=guided_decoding_backend))
395
choice2 = chat_completion.choices[0].message.content
442
- assert choice2 in TEST_CHOICE
396
+ assert choice2 in sample_guided_choice
397
assert choice1 != choice2
398
399
@@ -447,7 +401,8 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
401
@pytest.mark.parametrize("guided_decoding_backend",
402
["outlines", "lm-format-enforcer"])
403
async def test_guided_json_chat(client: openai.AsyncOpenAI,
450
- guided_decoding_backend: str):
404
+ guided_decoding_backend: str,
405
+ sample_json_schema):
406
messages = [{
407
"role": "system",
408
"content": "you are a helpful assistant"
@@ -456,18 +411,18 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
411
"user",
412
"content":
413
f"Give an example JSON for an employee profile that "
459
- f"fits this schema: {TEST_SCHEMA}"
414
+ f"fits this schema: {sample_json_schema}"
415
}]
416
chat_completion = await client.chat.completions.create(
417
model=MODEL_NAME,
418
messages=messages,
419
max_tokens=1000,
465
- extra_body=dict(guided_json=TEST_SCHEMA,
420
+ extra_body=dict(guided_json=sample_json_schema,
421
guided_decoding_backend=guided_decoding_backend))
422
message = chat_completion.choices[0].message
423
assert message.content is not None
424
json1 = json.loads(message.content)
470
- jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
425
+ jsonschema.validate(instance=json1, schema=sample_json_schema)
426
427
messages.append({"role": "assistant", "content": message.content})
428
messages.append({
@@ -480,12 +435,12 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
435
model=MODEL_NAME,
436
messages=messages,
437
max_tokens=1000,
483
- extra_body=dict(guided_json=TEST_SCHEMA,
438
+ extra_body=dict(guided_json=sample_json_schema,
439
guided_decoding_backend=guided_decoding_backend))
440
message = chat_completion.choices[0].message
441
assert message.content is not None
442
json2 = json.loads(message.content)
488
- jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
443
+ jsonschema.validate(instance=json2, schema=sample_json_schema)
444
assert json1["name"] != json2["name"]
445
assert json1["age"] != json2["age"]
446
@@ -494,7 +449,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
449
@pytest.mark.parametrize("guided_decoding_backend",
450
["outlines", "lm-format-enforcer"])
451
async def test_guided_regex_chat(client: openai.AsyncOpenAI,
497
- guided_decoding_backend: str):
452
+ guided_decoding_backend: str, sample_regex):
453
messages = [{
454
"role": "system",
455
"content": "you are a helpful assistant"
@@ -502,17 +457,17 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI,
457
"role":
458
"user",
459
"content":
505
- f"Give an example IP address with this regex: {TEST_REGEX}"
460
+ f"Give an example IP address with this regex: {sample_regex}"
461
}]
462
chat_completion = await client.chat.completions.create(
463
model=MODEL_NAME,
464
messages=messages,
465
max_tokens=20,
511
- extra_body=dict(guided_regex=TEST_REGEX,
466
+ extra_body=dict(guided_regex=sample_regex,
467
guided_decoding_backend=guided_decoding_backend))
468
ip1 = chat_completion.choices[0].message.content
469
assert ip1 is not None
515
- assert re.fullmatch(TEST_REGEX, ip1) is not None
470
+ assert re.fullmatch(sample_regex, ip1) is not None
471
472
messages.append({"role": "assistant", "content": ip1})
473
messages.append({"role": "user", "content": "Give me a different one"})
@@ -520,11 +475,11 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI,
475
model=MODEL_NAME,
476
messages=messages,
477
max_tokens=20,
523
- extra_body=dict(guided_regex=TEST_REGEX,
478
+ extra_body=dict(guided_regex=sample_regex,
479
guided_decoding_backend=guided_decoding_backend))
480
ip2 = chat_completion.choices[0].message.content
481
assert ip2 is not None
527
- assert re.fullmatch(TEST_REGEX, ip2) is not None
482
+ assert re.fullmatch(sample_regex, ip2) is not None
483
assert ip1 != ip2
484
485
@@ -553,7 +508,8 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):
508
@pytest.mark.parametrize("guided_decoding_backend",
509
["outlines", "lm-format-enforcer"])
510
async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
556
- guided_decoding_backend: str):
511
+ guided_decoding_backend: str,
512
+ sample_guided_choice):
513
messages = [{
514
"role": "system",
515
"content": "you are a helpful assistant"
@@ -569,7 +525,7 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
525
max_tokens=10,
526
logprobs=True,
527
top_logprobs=5,
572
- extra_body=dict(guided_choice=TEST_CHOICE,
528
+ extra_body=dict(guided_choice=sample_guided_choice,
529
guided_decoding_backend=guided_decoding_backend))
530
531
assert chat_completion.choices[0].logprobs is not None
@@ -585,7 +541,8 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
541
@pytest.mark.parametrize("guided_decoding_backend",
542
["outlines", "lm-format-enforcer"])
543
async def test_named_tool_use(client: openai.AsyncOpenAI,
588
- guided_decoding_backend: str):
544
+ guided_decoding_backend: str,
545
+ sample_json_schema):
546
messages = [{
547
"role": "system",
548
"content": "you are a helpful assistant"
@@ -594,7 +551,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
551
"user",
552
"content":
553
f"Give an example JSON for an employee profile that "
597
- f"fits this schema: {TEST_SCHEMA}"
554
+ f"fits this schema: {sample_json_schema}"
555
}]
556
557
# non-streaming
@@ -608,7 +565,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
565
"function": {
566
"name": "dummy_function_name",
567
"description": "This is a dummy function",
611
- "parameters": TEST_SCHEMA
568
+ "parameters": sample_json_schema
569
}
570
}],
571
tool_choice={
@@ -621,7 +578,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
578
assert len(message.content) == 0
579
json_string = message.tool_calls[0].function.arguments
580
json1 = json.loads(json_string)
624
- jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
581
+ jsonschema.validate(instance=json1, schema=sample_json_schema)
582
583
messages.append({"role": "assistant", "content": json_string})
584
messages.append({
@@ -642,7 +599,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
599
"function": {
600
"name": "dummy_function_name",
601
"description": "This is a dummy function",
645
- "parameters": TEST_SCHEMA
602
+ "parameters": sample_json_schema
603
}
604
}],
605
tool_choice={
@@ -667,7 +624,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
624
# finish reason should only return in last block
625
assert finish_reason_count == 1
626
json2 = json.loads("".join(output))
670
- jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
627
+ jsonschema.validate(instance=json2, schema=sample_json_schema)
628
assert json1["name"] != json2["name"]
629
assert json1["age"] != json2["age"]
630
@@ -675,7 +632,8 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
632
@pytest.mark.asyncio
633
@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
634
async def test_required_tool_use_not_yet_supported(
678
- client: openai.AsyncOpenAI, guided_decoding_backend: str):
635
+ client: openai.AsyncOpenAI, guided_decoding_backend: str,
636
+ sample_json_schema):
637
messages = [{
638
"role": "system",
639
"content": "you are a helpful assistant"
@@ -684,7 +642,7 @@ async def test_required_tool_use_not_yet_supported(
642
"user",
643
"content":
644
f"Give an example JSON for an employee profile that "
687
- f"fits this schema: {TEST_SCHEMA}"
645
+ f"fits this schema: {sample_json_schema}"
646
}]
647
648
with pytest.raises(openai.BadRequestError):
@@ -697,7 +655,7 @@ async def test_required_tool_use_not_yet_supported(
655
"function": {
656
"name": "dummy_function_name",
657
"description": "This is a dummy function",
700
- "parameters": TEST_SCHEMA
658
+ "parameters": sample_json_schema
659
}
660
}],
661
tool_choice="required")
@@ -712,7 +670,7 @@ async def test_required_tool_use_not_yet_supported(
670
"function": {
671
"name": "dummy_function_name",
672
"description": "This is a dummy function",
715
- "parameters": TEST_SCHEMA
673
+ "parameters": sample_json_schema
674
}
675
}],
676
tool_choice="auto")
@@ -720,8 +678,9 @@ async def test_required_tool_use_not_yet_supported(
678
679
@pytest.mark.asyncio
680
@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
723
-async def test_inconsistent_tool_choice_and_tools(
724
- client: openai.AsyncOpenAI, guided_decoding_backend: str):
681
+async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
682
+ guided_decoding_backend: str,
683
+ sample_json_schema):
684
messages = [{
685
"role": "system",
686
"content": "you are a helpful assistant"
@@ -730,7 +689,7 @@ async def test_inconsistent_tool_choice_and_tools(
689
"user",
690
"content":
691
f"Give an example JSON for an employee profile that "
733
- f"fits this schema: {TEST_SCHEMA}"
692
+ f"fits this schema: {sample_json_schema}"
693
}]
694
695
with pytest.raises(openai.BadRequestError):
@@ -755,7 +714,7 @@ async def test_inconsistent_tool_choice_and_tools(
714
"function": {
715
"name": "dummy_function_name",
716
"description": "This is a dummy function",
758
- "parameters": TEST_SCHEMA
717
+ "parameters": sample_json_schema
718
}
719
}],
720
tool_choice={
tests/entrypoints/openai/test_completion.py
+23
-74
@@ -24,53 +24,6 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
24
# generation quality here
25
LORA_NAME = "typeof/zephyr-7b-beta-lora"
26
27
-TEST_SCHEMA = {
28
- "type": "object",
29
- "properties": {
30
- "name": {
31
- "type": "string"
32
- },
33
- "age": {
34
- "type": "integer"
35
- },
36
- "skills": {
37
- "type": "array",
38
- "items": {
39
- "type": "string",
40
- "maxLength": 10
41
- },
42
- "minItems": 3
43
- },
44
- "work history": {
45
- "type": "array",
46
- "items": {
47
- "type": "object",
48
- "properties": {
49
- "company": {
50
- "type": "string"
51
- },
52
- "duration": {
53
- "type": "string"
54
- },
55
- "position": {
56
- "type": "string"
57
- }
58
- },
59
- "required": ["company", "position"]
60
- }
61
- }
62
- },
63
- "required": ["name", "age", "skills", "work history"]
64
-}
65
-
66
-TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
67
- r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
68
-
69
-TEST_CHOICE = [
70
- "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
71
- "Swift", "Kotlin"
72
-]
73
-
27
28
@pytest.fixture(scope="module")
29
def zephyr_lora_files():
@@ -529,77 +482,71 @@ async def test_logits_bias(client: openai.AsyncOpenAI):
482
@pytest.mark.parametrize("guided_decoding_backend",
483
["outlines", "lm-format-enforcer"])
484
async def test_guided_json_completion(client: openai.AsyncOpenAI,
532
- guided_decoding_backend: str):
485
+ guided_decoding_backend: str,
486
+ sample_json_schema):
487
completion = await client.completions.create(
488
model=MODEL_NAME,
489
prompt=f"Give an example JSON for an employee profile "
536
- f"that fits this schema: {TEST_SCHEMA}",
490
+ f"that fits this schema: {sample_json_schema}",
491
n=3,
492
temperature=1.0,
493
max_tokens=500,
540
- extra_body=dict(guided_json=TEST_SCHEMA,
494
+ extra_body=dict(guided_json=sample_json_schema,
495
guided_decoding_backend=guided_decoding_backend))
496
497
assert completion.id is not None
498
assert len(completion.choices) == 3
499
for i in range(3):
500
output_json = json.loads(completion.choices[i].text)
547
- jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
501
+ jsonschema.validate(instance=output_json, schema=sample_json_schema)
502
503
504
@pytest.mark.asyncio
505
@pytest.mark.parametrize("guided_decoding_backend",
506
["outlines", "lm-format-enforcer"])
507
async def test_guided_regex_completion(client: openai.AsyncOpenAI,
554
- guided_decoding_backend: str):
508
+ guided_decoding_backend: str,
509
+ sample_regex):
510
completion = await client.completions.create(
511
model=MODEL_NAME,
557
- prompt=f"Give an example IPv4 address with this regex: {TEST_REGEX}",
512
+ prompt=f"Give an example IPv4 address with this regex: {sample_regex}",
513
n=3,
514
temperature=1.0,
515
max_tokens=20,
561
- extra_body=dict(guided_regex=TEST_REGEX,
516
+ extra_body=dict(guided_regex=sample_regex,
517
guided_decoding_backend=guided_decoding_backend))
518
519
assert completion.id is not None
520
assert len(completion.choices) == 3
521
for i in range(3):
567
- assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
522
+ assert re.fullmatch(sample_regex,
523
+ completion.choices[i].text) is not None
524
525
526
@pytest.mark.asyncio
527
@pytest.mark.parametrize("guided_decoding_backend",
528
["outlines", "lm-format-enforcer"])
529
async def test_guided_choice_completion(client: openai.AsyncOpenAI,
574
- guided_decoding_backend: str):
530
+ guided_decoding_backend: str,
531
+ sample_guided_choice):
532
completion = await client.completions.create(
533
model=MODEL_NAME,
534
prompt="The best language for type-safe systems programming is ",
535
n=2,
536
temperature=1.0,
537
max_tokens=10,
581
- extra_body=dict(guided_choice=TEST_CHOICE,
538
+ extra_body=dict(guided_choice=sample_guided_choice,
539
guided_decoding_backend=guided_decoding_backend))
540
541
assert completion.id is not None
542
assert len(completion.choices) == 2
543
for i in range(2):
587
- assert completion.choices[i].text in TEST_CHOICE
544
+ assert completion.choices[i].text in sample_guided_choice
545
546
547
@pytest.mark.asyncio
591
-async def test_guided_grammar(client: openai.AsyncOpenAI):
592
- simple_sql_grammar = """
593
-start: select_statement
594
-
595
-select_statement: "SELECT" column "from" table "where" condition
596
-
597
-column: "col_1" | "col_2"
598
-table: "table_1" | "table_2"
599
-condition: column "=" number
600
-
601
-number: "1" | "2"
602
-"""
548
+async def test_guided_grammar(client: openai.AsyncOpenAI,
549
+ sample_sql_statements):
550
551
completion = await client.completions.create(
552
model=MODEL_NAME,
@@ -607,13 +554,13 @@ number: "1" | "2"
554
"table_1 where it is equals to 1"),
555
temperature=1.0,
556
max_tokens=500,
610
- extra_body=dict(guided_grammar=simple_sql_grammar))
557
+ extra_body=dict(guided_grammar=sample_sql_statements))
558
559
content = completion.choices[0].text
560
561
# use Lark to parse the output, and make sure it's a valid parse tree
562
from lark import Lark
616
- parser = Lark(simple_sql_grammar)
563
+ parser = Lark(sample_sql_statements)
564
parser.parse(content)
565
566
# remove spaces for comparison b/c we removed them in the grammar
@@ -661,7 +608,8 @@ async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
608
@pytest.mark.parametrize("guided_decoding_backend",
609
["outlines", "lm-format-enforcer"])
610
async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
664
- guided_decoding_backend: str):
611
+ guided_decoding_backend: str,
612
+ sample_json_schema, sample_regex):
613
with pytest.raises(openai.BadRequestError):
614
_ = await client.completions.create(
615
model=MODEL_NAME,
@@ -673,7 +621,8 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
621
_ = await client.completions.create(
622
model=MODEL_NAME,
623
prompt="Give an example string that fits this regex",
676
- extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
624
+ extra_body=dict(guided_regex=sample_regex,
625
+ guided_json=sample_json_schema))
626
627
628
@pytest.mark.asyncio
tests/entrypoints/openai/test_guided_processors.py
+13
-52
@@ -10,59 +10,17 @@ from vllm.model_executor.guided_decoding import (
10
from vllm.model_executor.guided_decoding.outlines_logits_processors import (
11
JSONLogitsProcessor, RegexLogitsProcessor)
12
13
-TEST_SCHEMA = {
14
- "type": "object",
15
- "properties": {
16
- "name": {
17
- "type": "string"
18
- },
19
- "age": {
20
- "type": "integer"
21
- },
22
- "skills": {
23
- "type": "array",
24
- "items": {
25
- "type": "string",
26
- "maxLength": 10
27
- },
28
- "minItems": 3
29
- },
30
- "work history": {
31
- "type": "array",
32
- "items": {
33
- "type": "object",
34
- "properties": {
35
- "company": {
36
- "type": "string"
37
- },
38
- "duration": {
39
- "type": "string"
40
- },
41
- "position": {
42
- "type": "string"
43
- }
44
- },
45
- "required": ["company", "position"]
46
- }
47
- }
48
- },
49
- "required": ["name", "age", "skills", "work history"]
50
-}
13
52
-TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
53
- r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
54
-
55
-
56
-def test_guided_logits_processors():
14
+def test_guided_logits_processors(sample_regex, sample_json_schema):
15
"""Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
16
tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
59
- regex_LP = RegexLogitsProcessor(TEST_REGEX, tokenizer)
60
- json_LP = JSONLogitsProcessor(TEST_SCHEMA,
17
+ regex_LP = RegexLogitsProcessor(sample_regex, tokenizer)
18
+ json_LP = JSONLogitsProcessor(sample_json_schema,
19
tokenizer,
20
whitespace_pattern=None)
21
22
token_ids = tokenizer.encode(
65
- f"Give an example IPv4 address with this regex: {TEST_REGEX}")
23
+ f"Give an example IPv4 address with this regex: {sample_regex}")
24
tensor = torch.rand(32000)
25
original_tensor = torch.clone(tensor)
26
regex_LP(token_ids, tensor)
@@ -70,7 +28,8 @@ def test_guided_logits_processors():
28
assert not torch.allclose(tensor, original_tensor)
29
30
token_ids = tokenizer.encode(
73
- f"Give an employee profile that fits this schema: {TEST_SCHEMA}")
31
+ f"Give an employee profile that fits this schema: {sample_json_schema}"
32
+ )
33
tensor = torch.rand(32000)
34
original_tensor = torch.clone(tensor)
35
json_LP(token_ids, tensor)
@@ -80,13 +39,14 @@ def test_guided_logits_processors():
39
40
@pytest.mark.asyncio
41
@pytest.mark.parametrize("backend", ["outlines", "lm-format-enforcer"])
83
-async def test_guided_logits_processor_black_box(backend: str):
42
+async def test_guided_logits_processor_black_box(backend: str, sample_regex,
43
+ sample_json_schema):
44
tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
45
token_ids = tokenizer.encode(
86
- f"Give an example IPv4 address with this regex: {TEST_REGEX}")
46
+ f"Give an example IPv4 address with this regex: {sample_regex}")
47
regex_request = CompletionRequest(model='test',
48
prompt=token_ids,
89
- guided_regex=TEST_REGEX)
49
+ guided_regex=sample_regex)
50
regex_lp = await get_guided_decoding_logits_processor(
51
backend, regex_request, tokenizer)
52
assert regex_lp is not None
@@ -97,10 +57,11 @@ async def test_guided_logits_processor_black_box(backend: str):
57
assert not torch.allclose(tensor, original_tensor)
58
59
token_ids = tokenizer.encode(
100
- f"Give an employee profile that fits this schema: {TEST_SCHEMA}")
60
+ f"Give an employee profile that fits this schema: {sample_json_schema}"
61
+ )
62
json_request = CompletionRequest(model='test',
63
prompt=token_ids,
103
- guided_json=TEST_SCHEMA)
64
+ guided_json=sample_json_schema)
65
json_lp = await get_guided_decoding_logits_processor(
66
backend, json_request, tokenizer)
67
assert json_lp is not None