@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b039cbbce3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Misc] add fixture to guided processor tests (#6341)

Yihuan Bu committed Jul 12, 2024 at 12:55 UTC b039cbbce37c1e6a1fdb284a41bd27cfef1b3783
4 files changed +144 -206
tests/entrypoints/openai/conftest.py new
+69
@@ -0,0 +1,69 @@
1 +import pytest
2 +
3 +
4 +@pytest.fixture
5 +def sample_regex():
6 + return (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
7 + r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
8 +
9 +
10 +@pytest.fixture
11 +def sample_json_schema():
12 + return {
13 + "type": "object",
14 + "properties": {
15 + "name": {
16 + "type": "string"
17 + },
18 + "age": {
19 + "type": "integer"
20 + },
21 + "skills": {
22 + "type": "array",
23 + "items": {
24 + "type": "string",
25 + "maxLength": 10
26 + },
27 + "minItems": 3
28 + },
29 + "work_history": {
30 + "type": "array",
31 + "items": {
32 + "type": "object",
33 + "properties": {
34 + "company": {
35 + "type": "string"
36 + },
37 + "duration": {
38 + "type": "number"
39 + },
40 + "position": {
41 + "type": "string"
42 + }
43 + },
44 + "required": ["company", "position"]
45 + }
46 + }
47 + },
48 + "required": ["name", "age", "skills", "work_history"]
49 + }
50 +
51 +
52 +@pytest.fixture
53 +def sample_guided_choice():
54 + return [
55 + "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript",
56 + "Ruby", "Swift", "Kotlin"
57 + ]
58 +
59 +
60 +@pytest.fixture
61 +def sample_sql_statements():
62 + return ("""
63 +start: select_statement
64 +select_statement: "SELECT" column "from" table "where" condition
65 +column: "col_1" | "col_2"
66 +table: "table_1" | "table_2"
67 +condition: column "=" number
68 +number: "1" | "2"
69 +""")
\ No newline at end of file
tests/entrypoints/openai/test_chat.py
+39 -80
@@ -22,53 +22,6 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
22 # generation quality here
23 LORA_NAME = "typeof/zephyr-7b-beta-lora"
24
25 -TEST_SCHEMA = {
26 - "type": "object",
27 - "properties": {
28 - "name": {
29 - "type": "string"
30 - },
31 - "age": {
32 - "type": "integer"
33 - },
34 - "skills": {
35 - "type": "array",
36 - "items": {
37 - "type": "string",
38 - "maxLength": 10
39 - },
40 - "minItems": 3
41 - },
42 - "work history": {
43 - "type": "array",
44 - "items": {
45 - "type": "object",
46 - "properties": {
47 - "company": {
48 - "type": "string"
49 - },
50 - "duration": {
51 - "type": "string"
52 - },
53 - "position": {
54 - "type": "string"
55 - }
56 - },
57 - "required": ["company", "position"]
58 - }
59 - }
60 - },
61 - "required": ["name", "age", "skills", "work history"]
62 -}
63 -
64 -TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
65 - r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
66 -
67 -TEST_CHOICE = [
68 - "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
69 - "Swift", "Kotlin"
70 -]
71 -
25
26 @pytest.fixture(scope="module")
27 def zephyr_lora_files():
@@ -408,7 +361,8 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
361 @pytest.mark.parametrize("guided_decoding_backend",
362 ["outlines", "lm-format-enforcer"])
363 async def test_guided_choice_chat(client: openai.AsyncOpenAI,
411 - guided_decoding_backend: str):
364 + guided_decoding_backend: str,
365 + sample_guided_choice):
366 messages = [{
367 "role": "system",
368 "content": "you are a helpful assistant"
@@ -422,10 +376,10 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
376 model=MODEL_NAME,
377 messages=messages,
378 max_tokens=10,
425 - extra_body=dict(guided_choice=TEST_CHOICE,
379 + extra_body=dict(guided_choice=sample_guided_choice,
380 guided_decoding_backend=guided_decoding_backend))
381 choice1 = chat_completion.choices[0].message.content
428 - assert choice1 in TEST_CHOICE
382 + assert choice1 in sample_guided_choice
383
384 messages.append({"role": "assistant", "content": choice1})
385 messages.append({
@@ -436,10 +390,10 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
390 model=MODEL_NAME,
391 messages=messages,
392 max_tokens=10,
439 - extra_body=dict(guided_choice=TEST_CHOICE,
393 + extra_body=dict(guided_choice=sample_guided_choice,
394 guided_decoding_backend=guided_decoding_backend))
395 choice2 = chat_completion.choices[0].message.content
442 - assert choice2 in TEST_CHOICE
396 + assert choice2 in sample_guided_choice
397 assert choice1 != choice2
398
399
@@ -447,7 +401,8 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
401 @pytest.mark.parametrize("guided_decoding_backend",
402 ["outlines", "lm-format-enforcer"])
403 async def test_guided_json_chat(client: openai.AsyncOpenAI,
450 - guided_decoding_backend: str):
404 + guided_decoding_backend: str,
405 + sample_json_schema):
406 messages = [{
407 "role": "system",
408 "content": "you are a helpful assistant"
@@ -456,18 +411,18 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
411 "user",
412 "content":
413 f"Give an example JSON for an employee profile that "
459 - f"fits this schema: {TEST_SCHEMA}"
414 + f"fits this schema: {sample_json_schema}"
415 }]
416 chat_completion = await client.chat.completions.create(
417 model=MODEL_NAME,
418 messages=messages,
419 max_tokens=1000,
465 - extra_body=dict(guided_json=TEST_SCHEMA,
420 + extra_body=dict(guided_json=sample_json_schema,
421 guided_decoding_backend=guided_decoding_backend))
422 message = chat_completion.choices[0].message
423 assert message.content is not None
424 json1 = json.loads(message.content)
470 - jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
425 + jsonschema.validate(instance=json1, schema=sample_json_schema)
426
427 messages.append({"role": "assistant", "content": message.content})
428 messages.append({
@@ -480,12 +435,12 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
435 model=MODEL_NAME,
436 messages=messages,
437 max_tokens=1000,
483 - extra_body=dict(guided_json=TEST_SCHEMA,
438 + extra_body=dict(guided_json=sample_json_schema,
439 guided_decoding_backend=guided_decoding_backend))
440 message = chat_completion.choices[0].message
441 assert message.content is not None
442 json2 = json.loads(message.content)
488 - jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
443 + jsonschema.validate(instance=json2, schema=sample_json_schema)
444 assert json1["name"] != json2["name"]
445 assert json1["age"] != json2["age"]
446
@@ -494,7 +449,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
449 @pytest.mark.parametrize("guided_decoding_backend",
450 ["outlines", "lm-format-enforcer"])
451 async def test_guided_regex_chat(client: openai.AsyncOpenAI,
497 - guided_decoding_backend: str):
452 + guided_decoding_backend: str, sample_regex):
453 messages = [{
454 "role": "system",
455 "content": "you are a helpful assistant"
@@ -502,17 +457,17 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI,
457 "role":
458 "user",
459 "content":
505 - f"Give an example IP address with this regex: {TEST_REGEX}"
460 + f"Give an example IP address with this regex: {sample_regex}"
461 }]
462 chat_completion = await client.chat.completions.create(
463 model=MODEL_NAME,
464 messages=messages,
465 max_tokens=20,
511 - extra_body=dict(guided_regex=TEST_REGEX,
466 + extra_body=dict(guided_regex=sample_regex,
467 guided_decoding_backend=guided_decoding_backend))
468 ip1 = chat_completion.choices[0].message.content
469 assert ip1 is not None
515 - assert re.fullmatch(TEST_REGEX, ip1) is not None
470 + assert re.fullmatch(sample_regex, ip1) is not None
471
472 messages.append({"role": "assistant", "content": ip1})
473 messages.append({"role": "user", "content": "Give me a different one"})
@@ -520,11 +475,11 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI,
475 model=MODEL_NAME,
476 messages=messages,
477 max_tokens=20,
523 - extra_body=dict(guided_regex=TEST_REGEX,
478 + extra_body=dict(guided_regex=sample_regex,
479 guided_decoding_backend=guided_decoding_backend))
480 ip2 = chat_completion.choices[0].message.content
481 assert ip2 is not None
527 - assert re.fullmatch(TEST_REGEX, ip2) is not None
482 + assert re.fullmatch(sample_regex, ip2) is not None
483 assert ip1 != ip2
484
485
@@ -553,7 +508,8 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):
508 @pytest.mark.parametrize("guided_decoding_backend",
509 ["outlines", "lm-format-enforcer"])
510 async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
556 - guided_decoding_backend: str):
511 + guided_decoding_backend: str,
512 + sample_guided_choice):
513 messages = [{
514 "role": "system",
515 "content": "you are a helpful assistant"
@@ -569,7 +525,7 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
525 max_tokens=10,
526 logprobs=True,
527 top_logprobs=5,
572 - extra_body=dict(guided_choice=TEST_CHOICE,
528 + extra_body=dict(guided_choice=sample_guided_choice,
529 guided_decoding_backend=guided_decoding_backend))
530
531 assert chat_completion.choices[0].logprobs is not None
@@ -585,7 +541,8 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
541 @pytest.mark.parametrize("guided_decoding_backend",
542 ["outlines", "lm-format-enforcer"])
543 async def test_named_tool_use(client: openai.AsyncOpenAI,
588 - guided_decoding_backend: str):
544 + guided_decoding_backend: str,
545 + sample_json_schema):
546 messages = [{
547 "role": "system",
548 "content": "you are a helpful assistant"
@@ -594,7 +551,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
551 "user",
552 "content":
553 f"Give an example JSON for an employee profile that "
597 - f"fits this schema: {TEST_SCHEMA}"
554 + f"fits this schema: {sample_json_schema}"
555 }]
556
557 # non-streaming
@@ -608,7 +565,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
565 "function": {
566 "name": "dummy_function_name",
567 "description": "This is a dummy function",
611 - "parameters": TEST_SCHEMA
568 + "parameters": sample_json_schema
569 }
570 }],
571 tool_choice={
@@ -621,7 +578,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
578 assert len(message.content) == 0
579 json_string = message.tool_calls[0].function.arguments
580 json1 = json.loads(json_string)
624 - jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
581 + jsonschema.validate(instance=json1, schema=sample_json_schema)
582
583 messages.append({"role": "assistant", "content": json_string})
584 messages.append({
@@ -642,7 +599,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
599 "function": {
600 "name": "dummy_function_name",
601 "description": "This is a dummy function",
645 - "parameters": TEST_SCHEMA
602 + "parameters": sample_json_schema
603 }
604 }],
605 tool_choice={
@@ -667,7 +624,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
624 # finish reason should only return in last block
625 assert finish_reason_count == 1
626 json2 = json.loads("".join(output))
670 - jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
627 + jsonschema.validate(instance=json2, schema=sample_json_schema)
628 assert json1["name"] != json2["name"]
629 assert json1["age"] != json2["age"]
630
@@ -675,7 +632,8 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
632 @pytest.mark.asyncio
633 @pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
634 async def test_required_tool_use_not_yet_supported(
678 - client: openai.AsyncOpenAI, guided_decoding_backend: str):
635 + client: openai.AsyncOpenAI, guided_decoding_backend: str,
636 + sample_json_schema):
637 messages = [{
638 "role": "system",
639 "content": "you are a helpful assistant"
@@ -684,7 +642,7 @@ async def test_required_tool_use_not_yet_supported(
642 "user",
643 "content":
644 f"Give an example JSON for an employee profile that "
687 - f"fits this schema: {TEST_SCHEMA}"
645 + f"fits this schema: {sample_json_schema}"
646 }]
647
648 with pytest.raises(openai.BadRequestError):
@@ -697,7 +655,7 @@ async def test_required_tool_use_not_yet_supported(
655 "function": {
656 "name": "dummy_function_name",
657 "description": "This is a dummy function",
700 - "parameters": TEST_SCHEMA
658 + "parameters": sample_json_schema
659 }
660 }],
661 tool_choice="required")
@@ -712,7 +670,7 @@ async def test_required_tool_use_not_yet_supported(
670 "function": {
671 "name": "dummy_function_name",
672 "description": "This is a dummy function",
715 - "parameters": TEST_SCHEMA
673 + "parameters": sample_json_schema
674 }
675 }],
676 tool_choice="auto")
@@ -720,8 +678,9 @@ async def test_required_tool_use_not_yet_supported(
678
679 @pytest.mark.asyncio
680 @pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
723 -async def test_inconsistent_tool_choice_and_tools(
724 - client: openai.AsyncOpenAI, guided_decoding_backend: str):
681 +async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
682 + guided_decoding_backend: str,
683 + sample_json_schema):
684 messages = [{
685 "role": "system",
686 "content": "you are a helpful assistant"
@@ -730,7 +689,7 @@ async def test_inconsistent_tool_choice_and_tools(
689 "user",
690 "content":
691 f"Give an example JSON for an employee profile that "
733 - f"fits this schema: {TEST_SCHEMA}"
692 + f"fits this schema: {sample_json_schema}"
693 }]
694
695 with pytest.raises(openai.BadRequestError):
@@ -755,7 +714,7 @@ async def test_inconsistent_tool_choice_and_tools(
714 "function": {
715 "name": "dummy_function_name",
716 "description": "This is a dummy function",
758 - "parameters": TEST_SCHEMA
717 + "parameters": sample_json_schema
718 }
719 }],
720 tool_choice={
tests/entrypoints/openai/test_completion.py
+23 -74
@@ -24,53 +24,6 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
24 # generation quality here
25 LORA_NAME = "typeof/zephyr-7b-beta-lora"
26
27 -TEST_SCHEMA = {
28 - "type": "object",
29 - "properties": {
30 - "name": {
31 - "type": "string"
32 - },
33 - "age": {
34 - "type": "integer"
35 - },
36 - "skills": {
37 - "type": "array",
38 - "items": {
39 - "type": "string",
40 - "maxLength": 10
41 - },
42 - "minItems": 3
43 - },
44 - "work history": {
45 - "type": "array",
46 - "items": {
47 - "type": "object",
48 - "properties": {
49 - "company": {
50 - "type": "string"
51 - },
52 - "duration": {
53 - "type": "string"
54 - },
55 - "position": {
56 - "type": "string"
57 - }
58 - },
59 - "required": ["company", "position"]
60 - }
61 - }
62 - },
63 - "required": ["name", "age", "skills", "work history"]
64 -}
65 -
66 -TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
67 - r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
68 -
69 -TEST_CHOICE = [
70 - "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
71 - "Swift", "Kotlin"
72 -]
73 -
27
28 @pytest.fixture(scope="module")
29 def zephyr_lora_files():
@@ -529,77 +482,71 @@ async def test_logits_bias(client: openai.AsyncOpenAI):
482 @pytest.mark.parametrize("guided_decoding_backend",
483 ["outlines", "lm-format-enforcer"])
484 async def test_guided_json_completion(client: openai.AsyncOpenAI,
532 - guided_decoding_backend: str):
485 + guided_decoding_backend: str,
486 + sample_json_schema):
487 completion = await client.completions.create(
488 model=MODEL_NAME,
489 prompt=f"Give an example JSON for an employee profile "
536 - f"that fits this schema: {TEST_SCHEMA}",
490 + f"that fits this schema: {sample_json_schema}",
491 n=3,
492 temperature=1.0,
493 max_tokens=500,
540 - extra_body=dict(guided_json=TEST_SCHEMA,
494 + extra_body=dict(guided_json=sample_json_schema,
495 guided_decoding_backend=guided_decoding_backend))
496
497 assert completion.id is not None
498 assert len(completion.choices) == 3
499 for i in range(3):
500 output_json = json.loads(completion.choices[i].text)
547 - jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
501 + jsonschema.validate(instance=output_json, schema=sample_json_schema)
502
503
504 @pytest.mark.asyncio
505 @pytest.mark.parametrize("guided_decoding_backend",
506 ["outlines", "lm-format-enforcer"])
507 async def test_guided_regex_completion(client: openai.AsyncOpenAI,
554 - guided_decoding_backend: str):
508 + guided_decoding_backend: str,
509 + sample_regex):
510 completion = await client.completions.create(
511 model=MODEL_NAME,
557 - prompt=f"Give an example IPv4 address with this regex: {TEST_REGEX}",
512 + prompt=f"Give an example IPv4 address with this regex: {sample_regex}",
513 n=3,
514 temperature=1.0,
515 max_tokens=20,
561 - extra_body=dict(guided_regex=TEST_REGEX,
516 + extra_body=dict(guided_regex=sample_regex,
517 guided_decoding_backend=guided_decoding_backend))
518
519 assert completion.id is not None
520 assert len(completion.choices) == 3
521 for i in range(3):
567 - assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
522 + assert re.fullmatch(sample_regex,
523 + completion.choices[i].text) is not None
524
525
526 @pytest.mark.asyncio
527 @pytest.mark.parametrize("guided_decoding_backend",
528 ["outlines", "lm-format-enforcer"])
529 async def test_guided_choice_completion(client: openai.AsyncOpenAI,
574 - guided_decoding_backend: str):
530 + guided_decoding_backend: str,
531 + sample_guided_choice):
532 completion = await client.completions.create(
533 model=MODEL_NAME,
534 prompt="The best language for type-safe systems programming is ",
535 n=2,
536 temperature=1.0,
537 max_tokens=10,
581 - extra_body=dict(guided_choice=TEST_CHOICE,
538 + extra_body=dict(guided_choice=sample_guided_choice,
539 guided_decoding_backend=guided_decoding_backend))
540
541 assert completion.id is not None
542 assert len(completion.choices) == 2
543 for i in range(2):
587 - assert completion.choices[i].text in TEST_CHOICE
544 + assert completion.choices[i].text in sample_guided_choice
545
546
547 @pytest.mark.asyncio
591 -async def test_guided_grammar(client: openai.AsyncOpenAI):
592 - simple_sql_grammar = """
593 -start: select_statement
594 -
595 -select_statement: "SELECT" column "from" table "where" condition
596 -
597 -column: "col_1" | "col_2"
598 -table: "table_1" | "table_2"
599 -condition: column "=" number
600 -
601 -number: "1" | "2"
602 -"""
548 +async def test_guided_grammar(client: openai.AsyncOpenAI,
549 + sample_sql_statements):
550
551 completion = await client.completions.create(
552 model=MODEL_NAME,
@@ -607,13 +554,13 @@ number: "1" | "2"
554 "table_1 where it is equals to 1"),
555 temperature=1.0,
556 max_tokens=500,
610 - extra_body=dict(guided_grammar=simple_sql_grammar))
557 + extra_body=dict(guided_grammar=sample_sql_statements))
558
559 content = completion.choices[0].text
560
561 # use Lark to parse the output, and make sure it's a valid parse tree
562 from lark import Lark
616 - parser = Lark(simple_sql_grammar)
563 + parser = Lark(sample_sql_statements)
564 parser.parse(content)
565
566 # remove spaces for comparison b/c we removed them in the grammar
@@ -661,7 +608,8 @@ async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
608 @pytest.mark.parametrize("guided_decoding_backend",
609 ["outlines", "lm-format-enforcer"])
610 async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
664 - guided_decoding_backend: str):
611 + guided_decoding_backend: str,
612 + sample_json_schema, sample_regex):
613 with pytest.raises(openai.BadRequestError):
614 _ = await client.completions.create(
615 model=MODEL_NAME,
@@ -673,7 +621,8 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
621 _ = await client.completions.create(
622 model=MODEL_NAME,
623 prompt="Give an example string that fits this regex",
676 - extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
624 + extra_body=dict(guided_regex=sample_regex,
625 + guided_json=sample_json_schema))
626
627
628 @pytest.mark.asyncio
tests/entrypoints/openai/test_guided_processors.py
+13 -52
@@ -10,59 +10,17 @@ from vllm.model_executor.guided_decoding import (
10 from vllm.model_executor.guided_decoding.outlines_logits_processors import (
11 JSONLogitsProcessor, RegexLogitsProcessor)
12
13 -TEST_SCHEMA = {
14 - "type": "object",
15 - "properties": {
16 - "name": {
17 - "type": "string"
18 - },
19 - "age": {
20 - "type": "integer"
21 - },
22 - "skills": {
23 - "type": "array",
24 - "items": {
25 - "type": "string",
26 - "maxLength": 10
27 - },
28 - "minItems": 3
29 - },
30 - "work history": {
31 - "type": "array",
32 - "items": {
33 - "type": "object",
34 - "properties": {
35 - "company": {
36 - "type": "string"
37 - },
38 - "duration": {
39 - "type": "string"
40 - },
41 - "position": {
42 - "type": "string"
43 - }
44 - },
45 - "required": ["company", "position"]
46 - }
47 - }
48 - },
49 - "required": ["name", "age", "skills", "work history"]
50 -}
13
52 -TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
53 - r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
54 -
55 -
56 -def test_guided_logits_processors():
14 +def test_guided_logits_processors(sample_regex, sample_json_schema):
15 """Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
16 tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
59 - regex_LP = RegexLogitsProcessor(TEST_REGEX, tokenizer)
60 - json_LP = JSONLogitsProcessor(TEST_SCHEMA,
17 + regex_LP = RegexLogitsProcessor(sample_regex, tokenizer)
18 + json_LP = JSONLogitsProcessor(sample_json_schema,
19 tokenizer,
20 whitespace_pattern=None)
21
22 token_ids = tokenizer.encode(
65 - f"Give an example IPv4 address with this regex: {TEST_REGEX}")
23 + f"Give an example IPv4 address with this regex: {sample_regex}")
24 tensor = torch.rand(32000)
25 original_tensor = torch.clone(tensor)
26 regex_LP(token_ids, tensor)
@@ -70,7 +28,8 @@ def test_guided_logits_processors():
28 assert not torch.allclose(tensor, original_tensor)
29
30 token_ids = tokenizer.encode(
73 - f"Give an employee profile that fits this schema: {TEST_SCHEMA}")
31 + f"Give an employee profile that fits this schema: {sample_json_schema}"
32 + )
33 tensor = torch.rand(32000)
34 original_tensor = torch.clone(tensor)
35 json_LP(token_ids, tensor)
@@ -80,13 +39,14 @@ def test_guided_logits_processors():
39
40 @pytest.mark.asyncio
41 @pytest.mark.parametrize("backend", ["outlines", "lm-format-enforcer"])
83 -async def test_guided_logits_processor_black_box(backend: str):
42 +async def test_guided_logits_processor_black_box(backend: str, sample_regex,
43 + sample_json_schema):
44 tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
45 token_ids = tokenizer.encode(
86 - f"Give an example IPv4 address with this regex: {TEST_REGEX}")
46 + f"Give an example IPv4 address with this regex: {sample_regex}")
47 regex_request = CompletionRequest(model='test',
48 prompt=token_ids,
89 - guided_regex=TEST_REGEX)
49 + guided_regex=sample_regex)
50 regex_lp = await get_guided_decoding_logits_processor(
51 backend, regex_request, tokenizer)
52 assert regex_lp is not None
@@ -97,10 +57,11 @@ async def test_guided_logits_processor_black_box(backend: str):
57 assert not torch.allclose(tensor, original_tensor)
58
59 token_ids = tokenizer.encode(
100 - f"Give an employee profile that fits this schema: {TEST_SCHEMA}")
60 + f"Give an employee profile that fits this schema: {sample_json_schema}"
61 + )
62 json_request = CompletionRequest(model='test',
63 prompt=token_ids,
103 - guided_json=TEST_SCHEMA)
64 + guided_json=sample_json_schema)
65 json_lp = await get_guided_decoding_logits_processor(
66 backend, json_request, tokenizer)
67 assert json_lp is not None