@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 703e42ee4b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add guided decoding for OpenAI API server (#2819)

Co-authored-by: br3no <breno@veltefaria.de> Co-authored-by: simon-mo <simon.mo@hey.com>

felixzhu555 committed Feb 29, 2024 at 14:13 UTC 703e42ee4b3efed3c71e7ae7d15f0f96e05722d4
9 files changed +597 -1
requirements.txt
+1
@@ -12,4 +12,5 @@ pydantic >= 2.0 # Required for OpenAI server.
12 prometheus_client >= 0.18.0
13 pynvml == 11.5.0
14 triton >= 2.1.0
15 +outlines >= 0.0.27
16 cupy-cuda12x == 12.1.0 # Required for CUDA graphs. CUDA 11.8 users should install cupy-cuda11x instead.
tests/entrypoints/test_guided_processors.py new
+75
@@ -0,0 +1,75 @@
1 +# This unit test should be moved to a new
2 +# tests/test_guided_decoding directory.
3 +
4 +from transformers import AutoTokenizer
5 +import torch
6 +
7 +from vllm.model_executor.guided_logits_processors import (RegexLogitsProcessor,
8 + JSONLogitsProcessor)
9 +
10 +TEST_SCHEMA = {
11 + "type": "object",
12 + "properties": {
13 + "name": {
14 + "type": "string"
15 + },
16 + "age": {
17 + "type": "integer"
18 + },
19 + "skills": {
20 + "type": "array",
21 + "items": {
22 + "type": "string",
23 + "maxLength": 10
24 + },
25 + "minItems": 3
26 + },
27 + "work history": {
28 + "type": "array",
29 + "items": {
30 + "type": "object",
31 + "properties": {
32 + "company": {
33 + "type": "string"
34 + },
35 + "duration": {
36 + "type": "string"
37 + },
38 + "position": {
39 + "type": "string"
40 + }
41 + },
42 + "required": ["company", "position"]
43 + }
44 + }
45 + },
46 + "required": ["name", "age", "skills", "work history"]
47 +}
48 +
49 +TEST_REGEX = r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}" + \
50 + r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
51 +
52 +
53 +def test_guided_logits_processors():
54 + """Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
55 + tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
56 + regex_LP = RegexLogitsProcessor(TEST_REGEX, tokenizer)
57 + json_LP = JSONLogitsProcessor(TEST_SCHEMA, tokenizer)
58 +
59 + regex_LP.init_state()
60 + token_ids = tokenizer.encode(
61 + f"Give an example IPv4 address with this regex: {TEST_REGEX}")
62 + tensor = torch.rand(32000)
63 + original_tensor = torch.clone(tensor)
64 + regex_LP(token_ids, tensor)
65 + assert tensor.shape == original_tensor.shape
66 + assert not torch.allclose(tensor, original_tensor)
67 +
68 + json_LP.init_state()
69 + token_ids = tokenizer.encode(
70 + f"Give an employee profile that fits this schema: {TEST_SCHEMA}")
71 + tensor = torch.rand(32000)
72 + original_tensor = torch.clone(tensor)
73 + json_LP(token_ids, tensor)
74 + assert tensor.shape == original_tensor.shape
75 + assert not torch.allclose(tensor, original_tensor)
tests/entrypoints/test_openai_server.py
+237
@@ -9,12 +9,64 @@ import ray # using Ray for overall ease of process management, parallel request
9 import openai # use the official client for correctness check
10 from huggingface_hub import snapshot_download # downloading lora to test lora requests
11
12 +# imports for guided decoding tests
13 +import json
14 +import jsonschema
15 +import re
16 +
17 from vllm.transformers_utils.tokenizer import get_tokenizer
18
19 MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
20 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta" # any model with a chat template should work here
21 LORA_NAME = "typeof/zephyr-7b-beta-lora" # technically this needs Mistral-7B-v0.1 as base, but we're not testing generation quality here
22
23 +TEST_SCHEMA = {
24 + "type": "object",
25 + "properties": {
26 + "name": {
27 + "type": "string"
28 + },
29 + "age": {
30 + "type": "integer"
31 + },
32 + "skills": {
33 + "type": "array",
34 + "items": {
35 + "type": "string",
36 + "maxLength": 10
37 + },
38 + "minItems": 3
39 + },
40 + "work history": {
41 + "type": "array",
42 + "items": {
43 + "type": "object",
44 + "properties": {
45 + "company": {
46 + "type": "string"
47 + },
48 + "duration": {
49 + "type": "string"
50 + },
51 + "position": {
52 + "type": "string"
53 + }
54 + },
55 + "required": ["company", "position"]
56 + }
57 + }
58 + },
59 + "required": ["name", "age", "skills", "work history"]
60 +}
61 +
62 +TEST_REGEX = r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}" + \
63 + r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
64 +
65 +TEST_CHOICE = [
66 + "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
67 + "Swift", "Kotlin"
68 +]
69 +
70 pytestmark = pytest.mark.asyncio
71
72
@@ -325,6 +377,7 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
377 max_tokens=max_tokens,
378 temperature=0.0,
379 logit_bias={str(token_id): 100},
380 + seed=42,
381 )
382 assert completion.choices[0].text is not None and len(
383 completion.choices[0].text) >= 5
@@ -358,5 +411,189 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
411 assert first_response != completion.choices[0].text
412
413
414 +async def test_guided_json_completion(server, client: openai.AsyncOpenAI):
415 + completion = await client.completions.create(
416 + model=MODEL_NAME,
417 + prompt=
418 + f"Give an example JSON for an employee profile that fits this schema: {TEST_SCHEMA}",
419 + n=3,
420 + temperature=1.0,
421 + max_tokens=500,
422 + extra_body=dict(guided_json=TEST_SCHEMA))
423 +
424 + assert completion.id is not None
425 + assert completion.choices is not None and len(completion.choices) == 3
426 + for i in range(3):
427 + assert completion.choices[i].text is not None
428 + output_json = json.loads(completion.choices[i].text)
429 + jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
430 +
431 +
432 +async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
433 + messages = [{
434 + "role": "system",
435 + "content": "you are a helpful assistant"
436 + }, {
437 + "role": "user",
438 + "content": "Give an example JSON for an employee profile that " + \
439 + f"fits this schema: {TEST_SCHEMA}"
440 + }]
441 + chat_completion = await client.chat.completions.create(
442 + model=MODEL_NAME,
443 + messages=messages,
444 + max_tokens=500,
445 + extra_body=dict(guided_json=TEST_SCHEMA))
446 + message = chat_completion.choices[0].message
447 + assert message.content is not None
448 + json1 = json.loads(message.content)
449 + jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
450 +
451 + messages.append({"role": "assistant", "content": message.content})
452 + messages.append({
453 + "role":
454 + "user",
455 + "content":
456 + "Give me another one with a different name and age"
457 + })
458 + chat_completion = await client.chat.completions.create(
459 + model=MODEL_NAME,
460 + messages=messages,
461 + max_tokens=500,
462 + extra_body=dict(guided_json=TEST_SCHEMA))
463 + message = chat_completion.choices[0].message
464 + assert message.content is not None
465 + json2 = json.loads(message.content)
466 + jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
467 + assert json1["name"] != json2["name"]
468 + assert json1["age"] != json2["age"]
469 +
470 +
471 +async def test_guided_regex_completion(server, client: openai.AsyncOpenAI):
472 + completion = await client.completions.create(
473 + model=MODEL_NAME,
474 + prompt=f"Give an example IPv4 address with this regex: {TEST_REGEX}",
475 + n=3,
476 + temperature=1.0,
477 + max_tokens=20,
478 + extra_body=dict(guided_regex=TEST_REGEX))
479 +
480 + assert completion.id is not None
481 + assert completion.choices is not None and len(completion.choices) == 3
482 + for i in range(3):
483 + assert completion.choices[i].text is not None
484 + assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
485 +
486 +
487 +async def test_guided_regex_chat(server, client: openai.AsyncOpenAI):
488 + messages = [{
489 + "role": "system",
490 + "content": "you are a helpful assistant"
491 + }, {
492 + "role":
493 + "user",
494 + "content":
495 + f"Give an example IP address with this regex: {TEST_REGEX}"
496 + }]
497 + chat_completion = await client.chat.completions.create(
498 + model=MODEL_NAME,
499 + messages=messages,
500 + max_tokens=20,
501 + extra_body=dict(guided_regex=TEST_REGEX))
502 + ip1 = chat_completion.choices[0].message.content
503 + assert ip1 is not None
504 + assert re.fullmatch(TEST_REGEX, ip1) is not None
505 +
506 + messages.append({"role": "assistant", "content": ip1})
507 + messages.append({"role": "user", "content": "Give me a different one"})
508 + chat_completion = await client.chat.completions.create(
509 + model=MODEL_NAME,
510 + messages=messages,
511 + max_tokens=20,
512 + extra_body=dict(guided_regex=TEST_REGEX))
513 + ip2 = chat_completion.choices[0].message.content
514 + assert ip2 is not None
515 + assert re.fullmatch(TEST_REGEX, ip2) is not None
516 + assert ip1 != ip2
517 +
518 +
519 +async def test_guided_choice_completion(server, client: openai.AsyncOpenAI):
520 + completion = await client.completions.create(
521 + model=MODEL_NAME,
522 + prompt="The best language for type-safe systems programming is ",
523 + n=2,
524 + temperature=1.0,
525 + max_tokens=10,
526 + extra_body=dict(guided_choice=TEST_CHOICE))
527 +
528 + assert completion.id is not None
529 + assert completion.choices is not None and len(completion.choices) == 2
530 + for i in range(2):
531 + assert completion.choices[i].text in TEST_CHOICE
532 +
533 +
534 +async def test_guided_choice_chat(server, client: openai.AsyncOpenAI):
535 + messages = [{
536 + "role": "system",
537 + "content": "you are a helpful assistant"
538 + }, {
539 + "role":
540 + "user",
541 + "content":
542 + "The best language for type-safe systems programming is "
543 + }]
544 + chat_completion = await client.chat.completions.create(
545 + model=MODEL_NAME,
546 + messages=messages,
547 + max_tokens=10,
548 + extra_body=dict(guided_choice=TEST_CHOICE))
549 + choice1 = chat_completion.choices[0].message.content
550 + assert choice1 in TEST_CHOICE
551 +
552 + messages.append({"role": "assistant", "content": choice1})
553 + messages.append({
554 + "role": "user",
555 + "content": "I disagree, pick another one"
556 + })
557 + chat_completion = await client.chat.completions.create(
558 + model=MODEL_NAME,
559 + messages=messages,
560 + max_tokens=10,
561 + extra_body=dict(guided_choice=TEST_CHOICE))
562 + choice2 = chat_completion.choices[0].message.content
563 + assert choice2 in TEST_CHOICE
564 + assert choice1 != choice2
565 +
566 +
567 +async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI):
568 + with pytest.raises(openai.BadRequestError):
569 + _ = await client.completions.create(
570 + model=MODEL_NAME,
571 + prompt="Give an example JSON that fits this schema: 42",
572 + extra_body=dict(guided_json=42))
573 +
574 + messages = [{
575 + "role": "system",
576 + "content": "you are a helpful assistant"
577 + }, {
578 + "role":
579 + "user",
580 + "content":
581 + "The best language for type-safe systems programming is "
582 + }]
583 + with pytest.raises(openai.BadRequestError):
584 + _ = await client.chat.completions.create(model=MODEL_NAME,
585 + messages=messages,
586 + extra_body=dict(guided_regex={
587 + 1: "Python",
588 + 2: "C++"
589 + }))
590 +
591 + with pytest.raises(openai.BadRequestError):
592 + _ = await client.completions.create(
593 + model=MODEL_NAME,
594 + prompt="Give an example string that fits this regex",
595 + extra_body=dict(guided_regex=TEST_REGEX, guided_json=TEST_SCHEMA))
596 +
597 +
598 if __name__ == "__main__":
599 pytest.main([__file__])
vllm/engine/async_llm_engine.py
+3
@@ -333,6 +333,9 @@ class AsyncLLMEngine:
333 return (self.background_loop is not None
334 and not self.background_loop.done())
335
336 + def get_tokenizer(self):
337 + return self.engine.tokenizer.tokenizer
338 +
339 def start_background_loop(self) -> None:
340 """Start the background loop."""
341 if self.is_running:
vllm/entrypoints/openai/protocol.py
+35 -1
@@ -3,7 +3,7 @@
3 import time
4 from typing import Dict, List, Literal, Optional, Union
5
6 -from pydantic import BaseModel, Field
6 +from pydantic import BaseModel, Field, model_validator
7
8 from vllm.utils import random_uuid
9 from vllm.sampling_params import SamplingParams
@@ -86,6 +86,9 @@ class ChatCompletionRequest(BaseModel):
86 min_p: Optional[float] = 0.0
87 include_stop_str_in_output: Optional[bool] = False
88 length_penalty: Optional[float] = 1.0
89 + guided_json: Optional[Union[str, dict, BaseModel]] = None
90 + guided_regex: Optional[str] = None
91 + guided_choice: Optional[List[str]] = None
92
93 def to_sampling_params(self) -> SamplingParams:
94 if self.logprobs and not self.top_logprobs:
@@ -131,6 +134,20 @@ class ChatCompletionRequest(BaseModel):
134 logits_processors=logits_processors,
135 )
136
137 + @model_validator(mode="before")
138 + @classmethod
139 + def check_guided_decoding_count(cls, data):
140 + guide_count = sum([
141 + "guided_json" in data and data["guided_json"] is not None,
142 + "guided_regex" in data and data["guided_regex"] is not None,
143 + "guided_choice" in data and data["guided_choice"] is not None
144 + ])
145 + if guide_count > 1:
146 + raise ValueError(
147 + "You can only use one kind of guided decoding "
148 + "('guided_json', 'guided_regex' or 'guided_choice').")
149 + return data
150 +
151
152 class CompletionRequest(BaseModel):
153 model: str
@@ -163,6 +180,9 @@ class CompletionRequest(BaseModel):
180 min_p: Optional[float] = 0.0
181 include_stop_str_in_output: Optional[bool] = False
182 length_penalty: Optional[float] = 1.0
183 + guided_json: Optional[Union[str, dict, BaseModel]] = None
184 + guided_regex: Optional[str] = None
185 + guided_choice: Optional[List[str]] = None
186
187 def to_sampling_params(self):
188 echo_without_generation = self.echo and self.max_tokens == 0
@@ -207,6 +227,20 @@ class CompletionRequest(BaseModel):
227 logits_processors=logits_processors,
228 )
229
230 + @model_validator(mode="before")
231 + @classmethod
232 + def check_guided_decoding_count(cls, data):
233 + guide_count = sum([
234 + "guided_json" in data and data["guided_json"] is not None,
235 + "guided_regex" in data and data["guided_regex"] is not None,
236 + "guided_choice" in data and data["guided_choice"] is not None
237 + ])
238 + if guide_count > 1:
239 + raise ValueError(
240 + "You can only use one kind of guided decoding "
241 + "('guided_json', 'guided_regex' or 'guided_choice').")
242 + return data
243 +
244
245 class LogProbs(BaseModel):
246 text_offset: List[int] = Field(default_factory=list)
vllm/entrypoints/openai/serving_chat.py
+9
@@ -12,6 +12,7 @@ from vllm.entrypoints.openai.protocol import (
12 UsageInfo)
13 from vllm.outputs import RequestOutput
14 from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
15 +from vllm.model_executor.guided_decoding import get_guided_decoding_logits_processor
16
17 logger = init_logger(__name__)
18
@@ -62,6 +63,14 @@ class OpenAIServingChat(OpenAIServing):
63 prompt=prompt)
64 sampling_params = request.to_sampling_params()
65 lora_request = self._maybe_get_lora(request)
66 + guided_decode_logits_processor = (
67 + await get_guided_decoding_logits_processor(
68 + request, self.engine.get_tokenizer()))
69 + if guided_decode_logits_processor:
70 + if sampling_params.logits_processors is None:
71 + sampling_params.logits_processors = []
72 + sampling_params.logits_processors.append(
73 + guided_decode_logits_processor)
74 except ValueError as e:
75 return self.create_error_response(str(e))
76
vllm/entrypoints/openai/serving_completion.py
+9
@@ -16,6 +16,7 @@ from .protocol import (
16 )
17 from vllm.outputs import RequestOutput
18 from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
19 +from vllm.model_executor.guided_decoding import get_guided_decoding_logits_processor
20
21 logger = init_logger(__name__)
22
@@ -286,6 +287,14 @@ class OpenAIServingCompletion(OpenAIServing):
287 try:
288 sampling_params = request.to_sampling_params()
289 lora_request = self._maybe_get_lora(request)
290 + guided_decode_logit_processor = (
291 + await get_guided_decoding_logits_processor(
292 + request, self.engine.get_tokenizer()))
293 + if guided_decode_logit_processor is not None:
294 + if sampling_params.logits_processors is None:
295 + sampling_params.logits_processors = []
296 + sampling_params.logits_processors.append(
297 + guided_decode_logit_processor)
298 prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
299
300 for i, prompt in enumerate(prompts):
vllm/model_executor/guided_decoding.py new
+99
@@ -0,0 +1,99 @@
1 +import asyncio
2 +import concurrent.futures
3 +from copy import copy
4 +from enum import Enum
5 +from functools import lru_cache
6 +from json import dumps as json_dumps
7 +from re import escape as regex_escape
8 +from typing import Union, Tuple
9 +from pydantic import BaseModel
10 +
11 +from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest
12 +from vllm.model_executor.guided_logits_processors import JSONLogitsProcessor, RegexLogitsProcessor
13 +
14 +
15 +class GuidedDecodingMode(Enum):
16 + JSON = "json"
17 + REGEX = "regex"
18 + CHOICE = "choice"
19 +
20 +
21 +global_thread_pool = None # used for generating logits processor fsm
22 +
23 +
24 +async def get_guided_decoding_logits_processor(
25 + request: Union[CompletionRequest, ChatCompletionRequest],
26 + tokenizer) -> Union[JSONLogitsProcessor, RegexLogitsProcessor]:
27 + """
28 + Given an OpenAI-compatible request, check for guided decoding parameters
29 + and get the necessary logits processor for the given guide.
30 + We cache logit processors by (guide, tokenizer), and on cache hit
31 + we make a shallow copy to reuse the same underlying FSM.
32 + """
33 + global global_thread_pool
34 + guide, mode = _get_guide_and_mode(request)
35 + if not guide:
36 + return None
37 +
38 + if global_thread_pool is None:
39 + global_thread_pool = concurrent.futures.ThreadPoolExecutor(
40 + max_workers=2)
41 + loop = asyncio.get_running_loop()
42 +
43 + result = await loop.run_in_executor(global_thread_pool,
44 + _get_cached_logits_processor, guide,
45 + tokenizer, mode)
46 +
47 + logits_processor = copy(result)
48 + # reset logits processor's internal state
49 + logits_processor.init_state()
50 + return logits_processor
51 +
52 +
53 +def _get_guide_and_mode(
54 + request: Union[CompletionRequest, ChatCompletionRequest]
55 +) -> Tuple[str, GuidedDecodingMode]:
56 +
57 + if request.guided_json:
58 + if not isinstance(request.guided_json, (str, dict, BaseModel)):
59 + raise TypeError("JSON schema must be str, dict, or BaseModel")
60 +
61 + json = request.guided_json
62 + if isinstance(json, dict):
63 + # turn dict into hashable string
64 + json = json_dumps(json, sort_keys=True)
65 + elif isinstance(json, BaseModel):
66 + # use pydantic signature so that different model classes
67 + # with the same fields will get hashed the same
68 + json = str(json.__signature__)
69 + return json, GuidedDecodingMode.JSON
70 +
71 + elif request.guided_regex:
72 + if not isinstance(request.guided_regex, str):
73 + raise TypeError("Regex must be string")
74 + return request.guided_regex, GuidedDecodingMode.REGEX
75 +
76 + elif request.guided_choice:
77 + if not isinstance(request.guided_choice, list):
78 + raise TypeError("Choices must be a list")
79 +
80 + # choice just uses regex
81 + choices = [
82 + regex_escape(str(choice)) for choice in request.guided_choice
83 + ]
84 + choices_regex = "(" + "|".join(choices) + ")"
85 + return choices_regex, GuidedDecodingMode.CHOICE
86 +
87 + else:
88 + return None, None
89 +
90 +
91 +@lru_cache(maxsize=32)
92 +def _get_cached_logits_processor(guide: str, tokenizer,
93 + mode: GuidedDecodingMode):
94 + if mode == GuidedDecodingMode.JSON:
95 + return JSONLogitsProcessor(guide, tokenizer)
96 + elif mode == GuidedDecodingMode.REGEX or mode == GuidedDecodingMode.CHOICE:
97 + return RegexLogitsProcessor(guide, tokenizer)
98 + else:
99 + raise ValueError(f"Unknown guided decoding mode {mode}")
vllm/model_executor/guided_logits_processors.py new
+129
@@ -0,0 +1,129 @@
1 +# Copyright 2024- the Outlines developers
2 +# This file is adapted from
3 +# https://github.com/outlines-dev/outlines/blob/main/outlines/serve/vllm.py
4 +#
5 +# Licensed under the Apache License, Version 2.0 (the "License");
6 +# you may not use this file except in compliance with the License.
7 +# You may obtain a copy of the License at
8 +
9 +# http://www.apache.org/licenses/LICENSE-2.0
10 +
11 +# Unless required by applicable law or agreed to in writing, software
12 +# distributed under the License is distributed on an "AS IS" BASIS,
13 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 +# See the License for the specific language governing permissions and
15 +# limitations under the License.
16 +import json
17 +import math
18 +from collections import defaultdict
19 +from typing import Union, DefaultDict, Dict, List, Optional
20 +
21 +import torch
22 +from pydantic import BaseModel
23 +from outlines.fsm.fsm import RegexFSM
24 +from outlines.fsm.json_schema import build_regex_from_schema
25 +
26 +
27 +class RegexLogitsProcessor:
28 +
29 + def __init__(self, regex_string: str, tokenizer):
30 + """Compile the FSM that drives the regex-structured generation.
31 +
32 + Parameters
33 + ----------
34 + regex_string
35 + A string that represents a regular expression
36 + tokenizer
37 + The model's tokenizer
38 +
39 + """
40 + tokenizer = self.adapt_tokenizer(tokenizer)
41 + fsm = RegexFSM(regex_string, tokenizer)
42 + self.fsm = fsm
43 +
44 + def init_state(self):
45 + """Initialize the FSM states."""
46 + self.fsm_state: DefaultDict[int, int] = defaultdict(int)
47 +
48 + def __call__(self, input_ids: List[int],
49 + scores: torch.Tensor) -> torch.Tensor:
50 + """Use the FSM to bias the logits before sampling the next token."""
51 +
52 + seq_id = hash(tuple(input_ids))
53 +
54 + if len(input_ids) == 0:
55 + self.init_state()
56 + else:
57 + last_token = input_ids[-1]
58 + last_seq_id = hash(tuple(input_ids[:-1]))
59 + self.fsm_state[seq_id] = self.fsm.next_state(
60 + self.fsm_state[last_seq_id], last_token)
61 +
62 + allowed_tokens = self.fsm.allowed_token_ids(self.fsm_state[seq_id])
63 +
64 + mask = torch.full((scores.shape[-1], ),
65 + -math.inf,
66 + device=scores.device)
67 + mask[allowed_tokens] = 0
68 + scores.add_(mask)
69 +
70 + return scores
71 +
72 + def adapt_tokenizer(self, tokenizer):
73 + """Adapt vLLM's tokenizer to use to compile the FSM.
74 +
75 + The API of Outlines tokenizers is slightly different to that of
76 + `transformers`. In addition we need to handle the missing spaces to
77 + Llama's tokenizer to be able to compile FSMs for this model.
78 +
79 + """
80 + tokenizer.vocabulary = tokenizer.get_vocab()
81 + tokenizer.special_tokens = set(tokenizer.all_special_tokens)
82 +
83 + def convert_token_to_string(token: str) -> str:
84 + from transformers.file_utils import SPIECE_UNDERLINE
85 +
86 + string = tokenizer.convert_tokens_to_string([token])
87 +
88 + # A hack to handle missing spaces to HF's Llama tokenizers
89 + if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>":
90 + return " " + string
91 +
92 + return string
93 +
94 + tokenizer.convert_token_to_string = convert_token_to_string
95 +
96 + return tokenizer
97 +
98 +
99 +class JSONLogitsProcessor(RegexLogitsProcessor):
100 +
101 + def __init__(self,
102 + schema: Union[str, Dict, BaseModel],
103 + tokenizer,
104 + whitespace_pattern: Optional[str] = None):
105 + """Compile the FSM that drives the JSON-guided generation.
106 +
107 + Parameters
108 + ----------
109 + schema
110 + A JSON schema that encodes the structure we want the model to generate
111 + tokenizer
112 + The model's tokenizer
113 + whitespace_pattern
114 + Pattern to use for JSON syntactic whitespace (doesn't impact string literals)
115 + Example: allow only a single space or newline with `whitespace_pattern=r"[\n ]?"`
116 + """
117 + if isinstance(schema, type(BaseModel)):
118 + schema_str = json.dumps(schema.model_json_schema())
119 + elif isinstance(schema, Dict):
120 + schema_str = json.dumps(schema)
121 + elif isinstance(schema, str):
122 + schema_str = schema
123 + else:
124 + raise ValueError(
125 + f"Cannot parse schema {schema}. The schema must be either " +
126 + "a Pydantic object, a dictionary or a string that contains the JSON "
127 + + "Schema specification")
128 + regex_string = build_regex_from_schema(schema_str, whitespace_pattern)
129 + super().__init__(regex_string, tokenizer)