A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add guided decoding for OpenAI API server (#2819)
Co-authored-by: br3no <breno@veltefaria.de> Co-authored-by: simon-mo <simon.mo@hey.com>
felixzhu555 committed
Feb 29, 2024 at 14:13 UTC
703e42ee4b3efed3c71e7ae7d15f0f96e05722d4
9 files changed
+597
-1
requirements.txt
+1
@@ -12,4 +12,5 @@ pydantic >= 2.0 # Required for OpenAI server.
12
prometheus_client >= 0.18.0
13
pynvml == 11.5.0
14
triton >= 2.1.0
15
+outlines >= 0.0.27
16
cupy-cuda12x == 12.1.0 # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead.
tests/entrypoints/test_guided_processors.py
new
+75
@@ -0,0 +1,75 @@
1
+# This unit test should be moved to a new
2
+# tests/test_guided_decoding directory.
3
+
4
+from transformers import AutoTokenizer
5
+import torch
6
+
7
+from vllm.model_executor.guided_logits_processors import (RegexLogitsProcessor,
8
+ JSONLogitsProcessor)
9
+
10
+TEST_SCHEMA = {
11
+ "type": "object",
12
+ "properties": {
13
+ "name": {
14
+ "type": "string"
15
+ },
16
+ "age": {
17
+ "type": "integer"
18
+ },
19
+ "skills": {
20
+ "type": "array",
21
+ "items": {
22
+ "type": "string",
23
+ "maxLength": 10
24
+ },
25
+ "minItems": 3
26
+ },
27
+ "work history": {
28
+ "type": "array",
29
+ "items": {
30
+ "type": "object",
31
+ "properties": {
32
+ "company": {
33
+ "type": "string"
34
+ },
35
+ "duration": {
36
+ "type": "string"
37
+ },
38
+ "position": {
39
+ "type": "string"
40
+ }
41
+ },
42
+ "required": ["company", "position"]
43
+ }
44
+ }
45
+ },
46
+ "required": ["name", "age", "skills", "work history"]
47
+}
48
+
49
+TEST_REGEX = r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}" + \
50
+ r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
51
+
52
+
53
+def test_guided_logits_processors():
54
+ """Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
55
+ tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
56
+ regex_LP = RegexLogitsProcessor(TEST_REGEX, tokenizer)
57
+ json_LP = JSONLogitsProcessor(TEST_SCHEMA, tokenizer)
58
+
59
+ regex_LP.init_state()
60
+ token_ids = tokenizer.encode(
61
+ f"Give an example IPv4 address with this regex: {TEST_REGEX}")
62
+ tensor = torch.rand(32000)
63
+ original_tensor = torch.clone(tensor)
64
+ regex_LP(token_ids, tensor)
65
+ assert tensor.shape == original_tensor.shape
66
+ assert not torch.allclose(tensor, original_tensor)
67
+
68
+ json_LP.init_state()
69
+ token_ids = tokenizer.encode(
70
+ f"Give an employee profile that fits this schema: {TEST_SCHEMA}")
71
+ tensor = torch.rand(32000)
72
+ original_tensor = torch.clone(tensor)
73
+ json_LP(token_ids, tensor)
74
+ assert tensor.shape == original_tensor.shape
75
+ assert not torch.allclose(tensor, original_tensor)
tests/entrypoints/test_openai_server.py
+237
@@ -9,12 +9,64 @@ import ray # using Ray for overall ease of process management, parallel request
9
import openai # use the official client for correctness check
10
from huggingface_hub import snapshot_download # downloading lora to test lora requests
11
12
+# imports for guided decoding tests
13
+import json
14
+import jsonschema
15
+import re
16
+
17
from vllm.transformers_utils.tokenizer import get_tokenizer
18
19
MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
20
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta" # any model with a chat template should work here
21
LORA_NAME = "typeof/zephyr-7b-beta-lora" # technically this needs Mistral-7B-v0.1 as base, but we're not testing generation quality here
22
23
+TEST_SCHEMA = {
24
+ "type": "object",
25
+ "properties": {
26
+ "name": {
27
+ "type": "string"
28
+ },
29
+ "age": {
30
+ "type": "integer"
31
+ },
32
+ "skills": {
33
+ "type": "array",
34
+ "items": {
35
+ "type": "string",
36
+ "maxLength": 10
37
+ },
38
+ "minItems": 3
39
+ },
40
+ "work history": {
41
+ "type": "array",
42
+ "items": {
43
+ "type": "object",
44
+ "properties": {
45
+ "company": {
46
+ "type": "string"
47
+ },
48
+ "duration": {
49
+ "type": "string"
50
+ },
51
+ "position": {
52
+ "type": "string"
53
+ }
54
+ },
55
+ "required": ["company", "position"]
56
+ }
57
+ }
58
+ },
59
+ "required": ["name", "age", "skills", "work history"]
60
+}
61
+
62
+TEST_REGEX = r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}" + \
63
+ r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
64
+
65
+TEST_CHOICE = [
66
+ "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
67
+ "Swift", "Kotlin"
68
+]
69
+
70
pytestmark = pytest.mark.asyncio
71
72
@@ -325,6 +377,7 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
377
max_tokens=max_tokens,
378
temperature=0.0,
379
logit_bias={str(token_id): 100},
380
+ seed=42,
381
)
382
assert completion.choices[0].text is not None and len(
383
completion.choices[0].text) >= 5
@@ -358,5 +411,189 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
411
assert first_response != completion.choices[0].text
412
413
414
+async def test_guided_json_completion(server, client: openai.AsyncOpenAI):
415
+ completion = await client.completions.create(
416
+ model=MODEL_NAME,
417
+ prompt=
418
+ f"Give an example JSON for an employee profile that fits this schema: {TEST_SCHEMA}",
419
+ n=3,
420
+ temperature=1.0,
421
+ max_tokens=500,
422
+ extra_body=dict(guided_json=TEST_SCHEMA))
423
+
424
+ assert completion.id is not None
425
+ assert completion.choices is not None and len(completion.choices) == 3
426
+ for i in range(3):
427
+ assert completion.choices[i].text is not None
428
+ output_json = json.loads(completion.choices[i].text)
429
+ jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
430
+
431
+
432
+async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
433
+ messages = [{
434
+ "role": "system",
435
+ "content": "you are a helpful assistant"
436
+ }, {
437
+ "role": "user",
438
+ "content": "Give an example JSON for an employee profile that " + \
439
+ f"fits this schema: {TEST_SCHEMA}"
440
+ }]
441
+ chat_completion = await client.chat.completions.create(
442
+ model=MODEL_NAME,
443
+ messages=messages,
444
+ max_tokens=500,
445
+ extra_body=dict(guided_json=TEST_SCHEMA))
446
+ message = chat_completion.choices[0].message
447
+ assert message.content is not None
448
+ json1 = json.loads(message.content)
449
+ jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
450
+
451
+ messages.append({"role": "assistant", "content": message.content})
452
+ messages.append({
453
+ "role":
454
+ "user",
455
+ "content":
456
+ "Give me another one with a different name and age"
457
+ })
458
+ chat_completion = await client.chat.completions.create(
459
+ model=MODEL_NAME,
460
+ messages=messages,
461
+ max_tokens=500,
462
+ extra_body=dict(guided_json=TEST_SCHEMA))
463
+ message = chat_completion.choices[0].message
464
+ assert message.content is not None
465
+ json2 = json.loads(message.content)
466
+ jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
467
+ assert json1["name"] != json2["name"]
468
+ assert json1["age"] != json2["age"]
469
+
470
+
471
+async def test_guided_regex_completion(server, client: openai.AsyncOpenAI):
472
+ completion = await client.completions.create(
473
+ model=MODEL_NAME,
474
+ prompt=f"Give an example IPv4 address with this regex: {TEST_REGEX}",
475
+ n=3,
476
+ temperature=1.0,
477
+ max_tokens=20,
478
+ extra_body=dict(guided_regex=TEST_REGEX))
479
+
480
+ assert completion.id is not None
481
+ assert completion.choices is not None and len(completion.choices) == 3
482
+ for i in range(3):
483
+ assert completion.choices[i].text is not None
484
+ assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
485
+
486
+
487
+async def test_guided_regex_chat(server, client: openai.AsyncOpenAI):
488
+ messages = [{
489
+ "role": "system",
490
+ "content": "you are a helpful assistant"
491
+ }, {
492
+ "role":
493
+ "user",
494
+ "content":
495
+ f"Give an example IP address with this regex: {TEST_REGEX}"
496
+ }]
497
+ chat_completion = await client.chat.completions.create(
498
+ model=MODEL_NAME,
499
+ messages=messages,
500
+ max_tokens=20,
501
+ extra_body=dict(guided_regex=TEST_REGEX))
502
+ ip1 = chat_completion.choices[0].message.content
503
+ assert ip1 is not None
504
+ assert re.fullmatch(TEST_REGEX, ip1) is not None
505
+
506
+ messages.append({"role": "assistant", "content": ip1})
507
+ messages.append({"role": "user", "content": "Give me a different one"})
508
+ chat_completion = await client.chat.completions.create(
509
+ model=MODEL_NAME,
510
+ messages=messages,
511
+ max_tokens=20,
512
+ extra_body=dict(guided_regex=TEST_REGEX))
513
+ ip2 = chat_completion.choices[0].message.content
514
+ assert ip2 is not None
515
+ assert re.fullmatch(TEST_REGEX, ip2) is not None
516
+ assert ip1 != ip2
517
+
518
+
519
+async def test_guided_choice_completion(server, client: openai.AsyncOpenAI):
520
+ completion = await client.completions.create(
521
+ model=MODEL_NAME,
522
+ prompt="The best language for type-safe systems programming is ",
523
+ n=2,
524
+ temperature=1.0,
525
+ max_tokens=10,
526
+ extra_body=dict(guided_choice=TEST_CHOICE))
527
+
528
+ assert completion.id is not None
529
+ assert completion.choices is not None and len(completion.choices) == 2
530
+ for i in range(2):
531
+ assert completion.choices[i].text in TEST_CHOICE
532
+
533
+
534
+async def test_guided_choice_chat(server, client: openai.AsyncOpenAI):
535
+ messages = [{
536
+ "role": "system",
537
+ "content": "you are a helpful assistant"
538
+ }, {
539
+ "role":
540
+ "user",
541
+ "content":
542
+ "The best language for type-safe systems programming is "
543
+ }]
544
+ chat_completion = await client.chat.completions.create(
545
+ model=MODEL_NAME,
546
+ messages=messages,
547
+ max_tokens=10,
548
+ extra_body=dict(guided_choice=TEST_CHOICE))
549
+ choice1 = chat_completion.choices[0].message.content
550
+ assert choice1 in TEST_CHOICE
551
+
552
+ messages.append({"role": "assistant", "content": choice1})
553
+ messages.append({
554
+ "role": "user",
555
+ "content": "I disagree, pick another one"
556
+ })
557
+ chat_completion = await client.chat.completions.create(
558
+ model=MODEL_NAME,
559
+ messages=messages,
560
+ max_tokens=10,
561
+ extra_body=dict(guided_choice=TEST_CHOICE))
562
+ choice2 = chat_completion.choices[0].message.content
563
+ assert choice2 in TEST_CHOICE
564
+ assert choice1 != choice2
565
+
566
+
567
+async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI):
568
+ with pytest.raises(openai.BadRequestError):
569
+ _ = await client.completions.create(
570
+ model=MODEL_NAME,
571
+ prompt="Give an example JSON that fits this schema: 42",
572
+ extra_body=dict(guided_json=42))
573
+
574
+ messages = [{
575
+ "role": "system",
576
+ "content": "you are a helpful assistant"
577
+ }, {
578
+ "role":
579
+ "user",
580
+ "content":
581
+ "The best language for type-safe systems programming is "
582
+ }]
583
+ with pytest.raises(openai.BadRequestError):
584
+ _ = await client.chat.completions.create(model=MODEL_NAME,
585
+ messages=messages,
586
+ extra_body=dict(guided_regex={
587
+ 1: "Python",
588
+ 2: "C++"
589
+ }))
590
+
591
+ with pytest.raises(openai.BadRequestError):
592
+ _ = await client.completions.create(
593
+ model=MODEL_NAME,
594
+ prompt="Give an example string that fits this regex",
595
+ extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
596
+
597
+
598
if __name__ == "__main__":
599
pytest.main([__file__])
vllm/engine/async_llm_engine.py
+3
@@ -333,6 +333,9 @@ class AsyncLLMEngine:
333
return (self.background_loop is not None
334
and not self.background_loop.done())
335
336
+ def get_tokenizer(self):
337
+ return self.engine.tokenizer.tokenizer
338
+
339
def start_background_loop(self) -> None:
340
"""Start the background loop."""
341
if self.is_running:
vllm/entrypoints/openai/protocol.py
+35
-1
@@ -3,7 +3,7 @@
3
import time
4
from typing import Dict, List, Literal, Optional, Union
5
6
-from pydantic import BaseModel, Field
6
+from pydantic import BaseModel, Field, model_validator
7
8
from vllm.utils import random_uuid
9
from vllm.sampling_params import SamplingParams
@@ -86,6 +86,9 @@ class ChatCompletionRequest(BaseModel):
86
min_p: Optional[float] = 0.0
87
include_stop_str_in_output: Optional[bool] = False
88
length_penalty: Optional[float] = 1.0
89
+ guided_json: Optional[Union[str, dict, BaseModel]] = None
90
+ guided_regex: Optional[str] = None
91
+ guided_choice: Optional[List[str]] = None
92
93
def to_sampling_params(self) -> SamplingParams:
94
if self.logprobs and not self.top_logprobs:
@@ -131,6 +134,20 @@ class ChatCompletionRequest(BaseModel):
134
logits_processors=logits_processors,
135
)
136
137
+ @model_validator(mode="before")
138
+ @classmethod
139
+ def check_guided_decoding_count(cls, data):
140
+ guide_count = sum([
141
+ "guided_json" in data and data["guided_json"] is not None,
142
+ "guided_regex" in data and data["guided_regex"] is not None,
143
+ "guided_choice" in data and data["guided_choice"] is not None
144
+ ])
145
+ if guide_count > 1:
146
+ raise ValueError(
147
+ "You can only use one kind of guided decoding "
148
+ "('guided_json', 'guided_regex' or 'guided_choice').")
149
+ return data
150
+
151
152
class CompletionRequest(BaseModel):
153
model: str
@@ -163,6 +180,9 @@ class CompletionRequest(BaseModel):
180
min_p: Optional[float] = 0.0
181
include_stop_str_in_output: Optional[bool] = False
182
length_penalty: Optional[float] = 1.0
183
+ guided_json: Optional[Union[str, dict, BaseModel]] = None
184
+ guided_regex: Optional[str] = None
185
+ guided_choice: Optional[List[str]] = None
186
187
def to_sampling_params(self):
188
echo_without_generation = self.echo and self.max_tokens == 0
@@ -207,6 +227,20 @@ class CompletionRequest(BaseModel):
227
logits_processors=logits_processors,
228
)
229
230
+ @model_validator(mode="before")
231
+ @classmethod
232
+ def check_guided_decoding_count(cls, data):
233
+ guide_count = sum([
234
+ "guided_json" in data and data["guided_json"] is not None,
235
+ "guided_regex" in data and data["guided_regex"] is not None,
236
+ "guided_choice" in data and data["guided_choice"] is not None
237
+ ])
238
+ if guide_count > 1:
239
+ raise ValueError(
240
+ "You can only use one kind of guided decoding "
241
+ "('guided_json', 'guided_regex' or 'guided_choice').")
242
+ return data
243
+
244
245
class LogProbs(BaseModel):
246
text_offset: List[int] = Field(default_factory=list)
vllm/entrypoints/openai/serving_chat.py
+9
@@ -12,6 +12,7 @@ from vllm.entrypoints.openai.protocol import (
12
UsageInfo)
13
from vllm.outputs import RequestOutput
14
from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
15
+from vllm.model_executor.guided_decoding import get_guided_decoding_logits_processor
16
17
logger = init_logger(__name__)
18
@@ -62,6 +63,14 @@ class OpenAIServingChat(OpenAIServing):
63
prompt=prompt)
64
sampling_params = request.to_sampling_params()
65
lora_request = self._maybe_get_lora(request)
66
+ guided_decode_logits_processor = (
67
+ await get_guided_decoding_logits_processor(
68
+ request, self.engine.get_tokenizer()))
69
+ if guided_decode_logits_processor:
70
+ if sampling_params.logits_processors is None:
71
+ sampling_params.logits_processors = []
72
+ sampling_params.logits_processors.append(
73
+ guided_decode_logits_processor)
74
except ValueError as e:
75
return self.create_error_response(str(e))
76
vllm/entrypoints/openai/serving_completion.py
+9
@@ -16,6 +16,7 @@ from .protocol import (
16
)
17
from vllm.outputs import RequestOutput
18
from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
19
+from vllm.model_executor.guided_decoding import get_guided_decoding_logits_processor
20
21
logger = init_logger(__name__)
22
@@ -286,6 +287,14 @@ class OpenAIServingCompletion(OpenAIServing):
287
try:
288
sampling_params = request.to_sampling_params()
289
lora_request = self._maybe_get_lora(request)
290
+ guided_decode_logit_processor = (
291
+ await get_guided_decoding_logits_processor(
292
+ request, self.engine.get_tokenizer()))
293
+ if guided_decode_logit_processor is not None:
294
+ if sampling_params.logits_processors is None:
295
+ sampling_params.logits_processors = []
296
+ sampling_params.logits_processors.append(
297
+ guided_decode_logit_processor)
298
prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
299
300
for i, prompt in enumerate(prompts):
vllm/model_executor/guided_decoding.py
new
+99
@@ -0,0 +1,99 @@
1
+import asyncio
2
+import concurrent.futures
3
+from copy import copy
4
+from enum import Enum
5
+from functools import lru_cache
6
+from json import dumps as json_dumps
7
+from re import escape as regex_escape
8
+from typing import Union, Tuple
9
+from pydantic import BaseModel
10
+
11
+from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest
12
+from vllm.model_executor.guided_logits_processors import JSONLogitsProcessor, RegexLogitsProcessor
13
+
14
+
15
+class GuidedDecodingMode(Enum):
16
+ JSON = "json"
17
+ REGEX = "regex"
18
+ CHOICE = "choice"
19
+
20
+
21
+global_thread_pool = None # used for generating logits processor fsm
22
+
23
+
24
+async def get_guided_decoding_logits_processor(
25
+ request: Union[CompletionRequest, ChatCompletionRequest],
26
+ tokenizer) -> Union[JSONLogitsProcessor, RegexLogitsProcessor]:
27
+ """
28
+ Given an OpenAI-compatible request, check for guided decoding parameters
29
+ and get the necessary logits processor for the given guide.
30
+ We cache logit processors by (guide, tokenizer), and on cache hit
31
+ we make a shallow copy to reuse the same underlying FSM.
32
+ """
33
+ global global_thread_pool
34
+ guide, mode = _get_guide_and_mode(request)
35
+ if not guide:
36
+ return None
37
+
38
+ if global_thread_pool is None:
39
+ global_thread_pool = concurrent.futures.ThreadPoolExecutor(
40
+ max_workers=2)
41
+ loop = asyncio.get_running_loop()
42
+
43
+ result = await loop.run_in_executor(global_thread_pool,
44
+ _get_cached_logits_processor, guide,
45
+ tokenizer, mode)
46
+
47
+ logits_processor = copy(result)
48
+ # reset logits processor's internal state
49
+ logits_processor.init_state()
50
+ return logits_processor
51
+
52
+
53
+def _get_guide_and_mode(
54
+ request: Union[CompletionRequest, ChatCompletionRequest]
55
+) -> Tuple[str, GuidedDecodingMode]:
56
+
57
+ if request.guided_json:
58
+ if not isinstance(request.guided_json, (str, dict, BaseModel)):
59
+ raise TypeError("JSON schema must be str, dict, or BaseModel")
60
+
61
+ json = request.guided_json
62
+ if isinstance(json, dict):
63
+ # turn dict into hashable string
64
+ json = json_dumps(json, sort_keys=True)
65
+ elif isinstance(json, BaseModel):
66
+ # use pydantic signature so that different model classes
67
+ # with the same fields will get hashed the same
68
+ json = str(json.__signature__)
69
+ return json, GuidedDecodingMode.JSON
70
+
71
+ elif request.guided_regex:
72
+ if not isinstance(request.guided_regex, str):
73
+ raise TypeError("Regex must be string")
74
+ return request.guided_regex, GuidedDecodingMode.REGEX
75
+
76
+ elif request.guided_choice:
77
+ if not isinstance(request.guided_choice, list):
78
+ raise TypeError("Choices must be a list")
79
+
80
+ # choice just uses regex
81
+ choices = [
82
+ regex_escape(str(choice)) for choice in request.guided_choice
83
+ ]
84
+ choices_regex = "(" + "|".join(choices) + ")"
85
+ return choices_regex, GuidedDecodingMode.CHOICE
86
+
87
+ else:
88
+ return None, None
89
+
90
+
91
+@lru_cache(maxsize=32)
92
+def _get_cached_logits_processor(guide: str, tokenizer,
93
+ mode: GuidedDecodingMode):
94
+ if mode == GuidedDecodingMode.JSON:
95
+ return JSONLogitsProcessor(guide, tokenizer)
96
+ elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE:
97
+ return RegexLogitsProcessor(guide, tokenizer)
98
+ else:
99
+ raise ValueError(f"Unknown guided decoding mode {mode}")
vllm/model_executor/guided_logits_processors.py
new
+129
@@ -0,0 +1,129 @@
1
+# Copyright 2024- the Outlines developers
2
+# This file is adapted from
3
+# https://github.com/outlines-dev/outlines/blob/main/outlines/serve/vllm.py
4
+#
5
+# Licensed under the Apache License, Version 2.0 (the "License");
6
+# you may not use this file except in compliance with the License.
7
+# You may obtain a copy of the License at
8
+
9
+# http://www.apache.org/licenses/LICENSE-2.0
10
+
11
+# Unless required by applicable law or agreed to in writing, software
12
+# distributed under the License is distributed on an "AS IS" BASIS,
13
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+# See the License for the specific language governing permissions and
15
+# limitations under the License.
16
+import json
17
+import math
18
+from collections import defaultdict
19
+from typing import Union, DefaultDict, Dict, List, Optional
20
+
21
+import torch
22
+from pydantic import BaseModel
23
+from outlines.fsm.fsm import RegexFSM
24
+from outlines.fsm.json_schema import build_regex_from_schema
25
+
26
+
27
+class RegexLogitsProcessor:
28
+
29
+ def __init__(self, regex_string: str, tokenizer):
30
+ """Compile the FSM that drives the regex-structured generation.
31
+
32
+ Parameters
33
+ ----------
34
+ regex_string
35
+ A string that represents a regular expression
36
+ tokenizer
37
+ The model's tokenizer
38
+
39
+ """
40
+ tokenizer = self.adapt_tokenizer(tokenizer)
41
+ fsm = RegexFSM(regex_string, tokenizer)
42
+ self.fsm = fsm
43
+
44
+ def init_state(self):
45
+ """Initialize the FSM states."""
46
+ self.fsm_state: DefaultDict[int, int] = defaultdict(int)
47
+
48
+ def __call__(self, input_ids: List[int],
49
+ scores: torch.Tensor) -> torch.Tensor:
50
+ """Use the FSM to bias the logits before sampling the next token."""
51
+
52
+ seq_id = hash(tuple(input_ids))
53
+
54
+ if len(input_ids) == 0:
55
+ self.init_state()
56
+ else:
57
+ last_token = input_ids[-1]
58
+ last_seq_id = hash(tuple(input_ids[:-1]))
59
+ self.fsm_state[seq_id] = self.fsm.next_state(
60
+ self.fsm_state[last_seq_id], last_token)
61
+
62
+ allowed_tokens = self.fsm.allowed_token_ids(self.fsm_state[seq_id])
63
+
64
+ mask = torch.full((scores.shape[-1], ),
65
+ -math.inf,
66
+ device=scores.device)
67
+ mask[allowed_tokens] = 0
68
+ scores.add_(mask)
69
+
70
+ return scores
71
+
72
+ def adapt_tokenizer(self, tokenizer):
73
+ """Adapt vLLM's tokenizer to use to compile the FSM.
74
+
75
+ The API of Outlines tokenizers is slightly different to that of
76
+ `transformers`. In addition we need to handle the missing spaces to
77
+ Llama's tokenizer to be able to compile FSMs for this model.
78
+
79
+ """
80
+ tokenizer.vocabulary = tokenizer.get_vocab()
81
+ tokenizer.special_tokens = set(tokenizer.all_special_tokens)
82
+
83
+ def convert_token_to_string(token: str) -> str:
84
+ from transformers.file_utils import SPIECE_UNDERLINE
85
+
86
+ string = tokenizer.convert_tokens_to_string([token])
87
+
88
+ # A hack to handle missing spaces to HF's Llama tokenizers
89
+ if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>":
90
+ return " " + string
91
+
92
+ return string
93
+
94
+ tokenizer.convert_token_to_string = convert_token_to_string
95
+
96
+ return tokenizer
97
+
98
+
99
+class JSONLogitsProcessor(RegexLogitsProcessor):
100
+
101
+ def __init__(self,
102
+ schema: Union[str, Dict, BaseModel],
103
+ tokenizer,
104
+ whitespace_pattern: Optional[str] = None):
105
+ """Compile the FSM that drives the JSON-guided generation.
106
+
107
+ Parameters
108
+ ----------
109
+ schema
110
+ A JSON schema that encodes the structure we want the model to generate
111
+ tokenizer
112
+ The model's tokenizer
113
+ whitespace_pattern
114
+ Pattern to use for JSON syntactic whitespace (doesn't impact string literals)
115
+ Example: allow only a single space or newline with `whitespace_pattern=r"[\n ]?"`
116
+ """
117
+ if isinstance(schema, type(BaseModel)):
118
+ schema_str = json.dumps(schema.model_json_schema())
119
+ elif isinstance(schema, Dict):
120
+ schema_str = json.dumps(schema)
121
+ elif isinstance(schema, str):
122
+ schema_str = schema
123
+ else:
124
+ raise ValueError(
125
+ f"Cannot parse schema {schema}. The schema must be either " +
126
+ "a Pydantic object, a dictionary or a string that contains the JSON "
127
+ + "Schema specification")
128
+ regex_string = build_regex_from_schema(schema_str, whitespace_pattern)
129
+ super().__init__(regex_string, tokenizer)