@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5bfb30a529
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix CFGGuide and use outlines for grammars that can't convert to GBNF (#11389)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Dec 23, 2024 at 10:06 UTC 5bfb30a529283f7d9a8baa6715ab3bfef204cddd
5 files changed +103 -86
tests/entrypoints/llm/test_guided_generate.py
-5
@@ -174,11 +174,6 @@ def test_guided_choice_completion(sample_guided_choice, llm,
174 @pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
175 def test_guided_grammar(sample_sql_statements, llm,
176 guided_decoding_backend: str):
177 - if guided_decoding_backend == "outlines":
178 - pytest.skip("Outlines backend fails in this test case with:\n"
179 - "AttributeError: Error in model execution: 'ParserConf' "
180 - "object has no attribute 'deterministic'")
181 -
177 sampling_params = SamplingParams(temperature=0.8,
178 top_p=0.95,
179 max_tokens=1000,
vllm/model_executor/guided_decoding/__init__.py
+17 -70
@@ -3,6 +3,9 @@ from __future__ import annotations
3 from typing import TYPE_CHECKING
4
5 from vllm.logger import init_logger
6 +from vllm.model_executor.guided_decoding.utils import (
7 + convert_lark_to_gbnf, grammar_is_likely_lark,
8 + has_lmf_unsupported_json_features, has_xgrammar_unsupported_json_features)
9 from vllm.platforms import CpuArchEnum, current_platform
10
11 if TYPE_CHECKING:
@@ -15,76 +18,6 @@ if TYPE_CHECKING:
18 logger = init_logger(__name__)
19
20
18 -def has_xgrammar_unsupported_json_features(schema: dict) -> bool:
19 - """Check if JSON schema contains features unsupported by xgrammar."""
20 -
21 - def check_object(obj: dict) -> bool:
22 - if not isinstance(obj, dict):
23 - return False
24 -
25 - # Check for pattern restrictions
26 - if "pattern" in obj:
27 - return True
28 -
29 - # Check for numeric ranges
30 - if obj.get("type") in ("integer", "number") and any(
31 - key in obj for key in [
32 - "minimum", "maximum", "exclusiveMinimum",
33 - "exclusiveMaximum", "multipleOf"
34 - ]):
35 - return True
36 -
37 - # Recursively check all nested objects and arrays
38 - for value in obj.values():
39 - if isinstance(value, dict):
40 - if check_object(value):
41 - return True
42 - elif isinstance(value, list):
43 - for item in value:
44 - if isinstance(item, dict) and check_object(item):
45 - return True
46 -
47 - return False
48 -
49 - return check_object(schema)
50 -
51 -
52 -def has_lmf_unsupported_json_features(schema: dict) -> bool:
53 - """
54 - Check if JSON schema contains features unsupported
55 - by lm_format_enforcer.
56 -
57 - Known issues:
58 - - Regex patterns:
59 - "grade": {
60 - "type": "string",
61 - "pattern": "^[A-D]$" # Regex pattern
62 - },
63 - """
64 -
65 - def check_object(obj: dict) -> bool:
66 - if not isinstance(obj, dict):
67 - return False
68 -
69 - # Check for pattern restrictions
70 - if "pattern" in obj:
71 - return True
72 -
73 - # Recursively check all nested objects and arrays
74 - for value in obj.values():
75 - if isinstance(value, dict):
76 - if check_object(value):
77 - return True
78 - elif isinstance(value, list):
79 - for item in value:
80 - if isinstance(item, dict) and check_object(item):
81 - return True
82 -
83 - return False
84 -
85 - return check_object(schema)
86 -
87 -
21 def maybe_backend_fallback(
22 guided_params: GuidedDecodingParams) -> GuidedDecodingParams:
23 # lm-format-enforce doesn't support grammar, fallback to xgrammar
@@ -127,6 +60,20 @@ def maybe_backend_fallback(
60 "Falling back to use outlines instead.")
61 guided_params.backend = "outlines"
62
63 + # xgrammar only supports GBNF grammars, so we must convert Lark.
64 + # We must check if the grammar is likely Lark and if that
65 + # grammar is convertible to GBNF
66 + elif (guided_params.grammar is not None
67 + and grammar_is_likely_lark(guided_params.grammar)):
68 + try:
69 + convert_lark_to_gbnf(guided_params.grammar)
70 + except Exception:
71 + logger.warning(
72 + "xgrammar does not support Lark grammars and the "
73 + "grammar failed to convert to GBNF. "
74 + "Falling back to use outlines instead.")
75 + guided_params.backend = "outlines"
76 +
77 if (guided_params.backend == "outlines"
78 and guided_params.json_object is not None):
79 # outlines doesn't support json_object, fallback to xgrammar
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+14 -9
@@ -21,10 +21,11 @@ from typing import Callable, DefaultDict, Dict, List, Union
21
22 import numpy as np
23 import torch
24 -from lark import Lark
24 from outlines import grammars
25 from outlines.caching import cache
27 -from outlines.fsm.guide import CFGGuide, Generate, Guide, RegexGuide, Write
26 +from outlines.fsm.guide import (CFGGuide, CFGState, Generate, Guide,
27 + RegexGuide, Write)
28 +from outlines.fsm.parsing import PartialLark
29 from outlines_core.fsm.json_schema import build_regex_from_schema
30 from pydantic import BaseModel
31 from transformers import PreTrainedTokenizerBase
@@ -34,7 +35,9 @@ class BaseLogitsProcessor:
35
36 def __init__(self, guide: Guide):
37 self._guide: Guide = guide
37 - self._fsm_state: DefaultDict[int, int] = defaultdict(int)
38 + # CFGState is used for the FSM state for CFGGuide
39 + self._fsm_state: DefaultDict[int, Union[int,
40 + CFGState]] = defaultdict(int)
41
42 def __call__(self, input_ids: List[int],
43 scores: torch.Tensor) -> torch.Tensor:
@@ -54,15 +57,13 @@ class BaseLogitsProcessor:
57 # On the first time this is called, we simply re-create
58 # the Lark object.
59 if isinstance(self._guide, CFGGuide):
57 - self._guide.parser = Lark(
60 + self._guide.parser = PartialLark(
61 self._guide.cfg_string,
62 parser="lalr",
60 - lexer="contextual",
61 - propagate_positions=False,
62 - maybe_placeholders=False,
63 - regex=True,
63 import_paths=[grammars.GRAMMAR_PATH],
64 )
65 + self._fsm_state[seq_id] = CFGState(
66 + parser_state=self._guide.parser.parse(""), prev_token=None)
67
68 instruction = self._guide.get_next_instruction(
69 state=self._fsm_state[seq_id])
@@ -200,7 +201,8 @@ def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):
201 string = tokenizer.convert_tokens_to_string([token])
202
203 # A hack to handle missing spaces to HF's Llama tokenizers
203 - if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>":
204 + if (type(token) is str and token.startswith(SPIECE_UNDERLINE)
205 + or token == "<0x20>"):
206 return " " + string
207
208 return string
@@ -211,6 +213,9 @@ def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):
213 """Sync vLLM's decoder with the outlines by returning list."""
214
215 def new_decoder(inp_tokens: List[int]) -> List[str]:
216 + if (isinstance(inp_tokens, list) and len(inp_tokens) == 1
217 + and isinstance(inp_tokens[0], list)):
218 + inp_tokens = inp_tokens[0]
219 return [decoder(inp_tokens)]
220
221 return new_decoder
vllm/model_executor/guided_decoding/utils.py renamed
+70
@@ -1,6 +1,76 @@
1 import re
2
3
4 +def has_xgrammar_unsupported_json_features(schema: dict) -> bool:
5 + """Check if JSON schema contains features unsupported by xgrammar."""
6 +
7 + def check_object(obj: dict) -> bool:
8 + if not isinstance(obj, dict):
9 + return False
10 +
11 + # Check for pattern restrictions
12 + if "pattern" in obj:
13 + return True
14 +
15 + # Check for numeric ranges
16 + if obj.get("type") in ("integer", "number") and any(
17 + key in obj for key in [
18 + "minimum", "maximum", "exclusiveMinimum",
19 + "exclusiveMaximum", "multipleOf"
20 + ]):
21 + return True
22 +
23 + # Recursively check all nested objects and arrays
24 + for value in obj.values():
25 + if isinstance(value, dict):
26 + if check_object(value):
27 + return True
28 + elif isinstance(value, list):
29 + for item in value:
30 + if isinstance(item, dict) and check_object(item):
31 + return True
32 +
33 + return False
34 +
35 + return check_object(schema)
36 +
37 +
38 +def has_lmf_unsupported_json_features(schema: dict) -> bool:
39 + """
40 + Check if JSON schema contains features unsupported
41 + by lm_format_enforcer.
42 +
43 + Known issues:
44 + - Regex patterns:
45 + "grade": {
46 + "type": "string",
47 + "pattern": "^[A-D]$" # Regex pattern
48 + },
49 + """
50 +
51 + def check_object(obj: dict) -> bool:
52 + if not isinstance(obj, dict):
53 + return False
54 +
55 + # Check for pattern restrictions
56 + if "pattern" in obj:
57 + return True
58 +
59 + # Recursively check all nested objects and arrays
60 + for value in obj.values():
61 + if isinstance(value, dict):
62 + if check_object(value):
63 + return True
64 + elif isinstance(value, list):
65 + for item in value:
66 + if isinstance(item, dict) and check_object(item):
67 + return True
68 +
69 + return False
70 +
71 + return check_object(schema)
72 +
73 +
74 def grammar_is_likely_lark(grammar_str: str) -> bool:
75 """
76 Check if grammar appears to use Lark syntax.
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+2 -2
@@ -14,8 +14,8 @@ try:
14 except ImportError:
15 pass
16
17 -from vllm.model_executor.guided_decoding.xgrammar_utils import (
18 - convert_lark_to_gbnf, grammar_is_likely_lark)
17 +from vllm.model_executor.guided_decoding.utils import (convert_lark_to_gbnf,
18 + grammar_is_likely_lark)
19 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
20
21 if TYPE_CHECKING: