@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9323a3153b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Core][Performance] Add XGrammar support for guided decoding and set it as default (#10785)

Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Signed-off-by: mgoin <michael@neuralmagic.com> Co-authored-by: mgoin <michael@neuralmagic.com>

Aaron Pham committed Dec 3, 2024 at 02:17 UTC 9323a3153b20d4a2ca7ac04a2784609d6ce656e0
11 files changed +385 -33
docs/source/conf.py
+1
@@ -178,6 +178,7 @@ autodoc_mock_imports = [
178 "tensorizer",
179 "pynvml",
180 "outlines",
181 + "xgrammar,"
182 "librosa",
183 "soundfile",
184 "gguf",
requirements-common.txt
+1
@@ -19,6 +19,7 @@ prometheus-fastapi-instrumentator >= 7.0.0
19 tiktoken >= 0.6.0 # Required for DBRX tokenizer
20 lm-format-enforcer >= 0.10.9, < 0.11
21 outlines >= 0.0.43, < 0.1
22 +xgrammar
23 typing_extensions >= 4.10
24 filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
25 partial-json-parser # used for parsing partial JSON outputs
tests/entrypoints/llm/test_guided_generate.py
+27
@@ -159,3 +159,30 @@ def test_validation_against_both_guided_decoding_options(sample_regex, llm):
159 sampling_params=sampling_params,
160 use_tqdm=True,
161 guided_options_request=dict(guided_regex=sample_regex))
162 +
163 +
164 +@pytest.mark.skip_global_cleanup
165 +def test_guided_json_object(llm):
166 + sampling_params = SamplingParams(
167 + temperature=1.0,
168 + max_tokens=100,
169 + guided_decoding=GuidedDecodingParams(json_object=True))
170 +
171 + outputs = llm.generate(
172 + prompts=("Generate a JSON object describing a person with name "
173 + "and age for John Smith who is 31 years old."),
174 + sampling_params=sampling_params,
175 + use_tqdm=True)
176 +
177 + assert outputs is not None
178 + for output in outputs:
179 + assert output is not None
180 + assert isinstance(output, RequestOutput)
181 +
182 + generated_text = output.outputs[0].text
183 + print(generated_text)
184 + assert generated_text is not None
185 +
186 + # Parse to verify it is valid JSON
187 + parsed_json = json.loads(generated_text)
188 + assert isinstance(parsed_json, dict)
tests/model_executor/test_guided_processors.py
+2 -1
@@ -36,7 +36,8 @@ def test_guided_logits_processors(sample_regex, sample_json_schema):
36
37
38 @pytest.mark.asyncio
39 -@pytest.mark.parametrize("backend", ["outlines", "lm-format-enforcer"])
39 +@pytest.mark.parametrize("backend",
40 + ["outlines", "lm-format-enforcer", "xgrammar"])
41 async def test_guided_logits_processor_black_box(backend: str, sample_regex,
42 sample_json_schema):
43 tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
vllm/config.py
+8 -7
@@ -1789,15 +1789,15 @@ class PoolerConfig:
1789
1790 step_tag_id: Optional[int] = None
1791 """
1792 - If set, only the score corresponding to the ``step_tag_id`` in the
1792 + If set, only the score corresponding to the ``step_tag_id`` in the
1793 generated sentence should be returned. Otherwise, the scores for all tokens
1794 are returned.
1795 """
1796
1797 returned_token_ids: Optional[List[int]] = None
1798 """
1799 - A list of indices for the vocabulary dimensions to be extracted,
1800 - such as the token IDs of ``good_token`` and ``bad_token`` in the
1799 + A list of indices for the vocabulary dimensions to be extracted,
1800 + such as the token IDs of ``good_token`` and ``bad_token`` in the
1801 ``math-shepherd-mistral-7b-prm`` model.
1802 """
1803
@@ -2031,11 +2031,12 @@ def get_served_model_name(model: str,
2031 class DecodingConfig:
2032 """Dataclass which contains the decoding strategy of the engine"""
2033
2034 - # Which guided decoding algo to use. 'outlines' / 'lm-format-enforcer'
2035 - guided_decoding_backend: str = 'outlines'
2034 + # Which guided decoding algo to use.
2035 + # 'outlines' / 'lm-format-enforcer' / 'xgrammar'
2036 + guided_decoding_backend: str = 'xgrammar'
2037
2038 def __post_init__(self):
2038 - valid_guided_backends = ['outlines', 'lm-format-enforcer']
2039 + valid_guided_backends = ['outlines', 'lm-format-enforcer', 'xgrammar']
2040 backend = self.guided_decoding_backend
2041 if backend not in valid_guided_backends:
2042 raise ValueError(f"Invalid guided_decoding_backend '{backend},"
@@ -2222,7 +2223,7 @@ class CompilationConfig(BaseModel):
2223 from Python, functions can also be passed directly via Python object
2224 constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
2225 - custom inductor passes: see PassConfig for more details
2225 -
2226 +
2227 Why we have different sizes for cudagraph and inductor:
2228 - cudagraph: a cudagraph captured for a specific size can only be used
2229 for the same size. We need to capture all the sizes we want to use.
vllm/engine/arg_utils.py
+5 -4
@@ -168,7 +168,7 @@ class EngineArgs:
168 scheduler_delay_factor: float = 0.0
169 enable_chunked_prefill: Optional[bool] = None
170
171 - guided_decoding_backend: str = 'outlines'
171 + guided_decoding_backend: str = 'xgrammar'
172 # Speculative decoding configuration.
173 speculative_model: Optional[str] = None
174 speculative_model_quantization: Optional[str] = None
@@ -364,11 +364,12 @@ class EngineArgs:
364 parser.add_argument(
365 '--guided-decoding-backend',
366 type=str,
367 - default='outlines',
368 - choices=['outlines', 'lm-format-enforcer'],
367 + default='xgrammar',
368 + choices=['outlines', 'lm-format-enforcer', 'xgrammar'],
369 help='Which engine will be used for guided decoding'
370 ' (JSON schema / regex etc) by default. Currently support '
371 - 'https://github.com/outlines-dev/outlines and '
371 + 'https://github.com/outlines-dev/outlines,'
372 + 'https://github.com/mlc-ai/xgrammar, and '
373 'https://github.com/noamgat/lm-format-enforcer.'
374 ' Can be overridden per request via guided_decoding_backend'
375 ' parameter.')
vllm/engine/async_llm_engine.py
+14 -4
@@ -1,4 +1,5 @@
1 import asyncio
2 +import copy
3 import time
4 import weakref
5 from functools import partial
@@ -507,7 +508,8 @@ class _AsyncLLMEngine(LLMEngine):
508 sampling_params=params,
509 tokenizer=await self.get_tokenizer_async(lora_request),
510 default_guided_backend=self.decoding_config.
510 - guided_decoding_backend)
511 + guided_decoding_backend,
512 + model_config=self.model_config)
513
514 self._add_processed_request(
515 request_id=request_id,
@@ -528,22 +530,30 @@ class _AsyncLLMEngine(LLMEngine):
530
531 async def build_guided_decoding_logits_processor_async(
532 sampling_params: SamplingParams, tokenizer: AnyTokenizer,
531 - default_guided_backend: str) -> SamplingParams:
533 + default_guided_backend: str,
534 + model_config: ModelConfig) -> SamplingParams:
535 """Constructs logits processors based on the guided_decoding,
536 logits_bias, and allowed_token_ids fields in sampling_params. Deletes
537 those fields and adds the constructed logits processors to the
538 logits_processors field. Modifies sampling params in-place and returns
539 the modified sampling params."""
537 - if (guided_decoding := sampling_params.guided_decoding) is None:
540 + if sampling_params.guided_decoding is None:
541 return sampling_params
542
543 + # Defensively copy sampling params since guided decoding logits
544 + # processors can have different state for each request
545 + sampling_params = copy.copy(sampling_params)
546 + guided_decoding = sampling_params.guided_decoding
547 +
548 logger.debug("Building guided decoding logits processor. "
549 "Params: %s", guided_decoding)
550
551 guided_decoding.backend = guided_decoding.backend or default_guided_backend
552
553 processor = await get_guided_decoding_logits_processor(
546 - guided_params=guided_decoding, tokenizer=tokenizer)
554 + guided_params=guided_decoding,
555 + tokenizer=tokenizer,
556 + model_config=model_config)
557
558 if processor:
559 if sampling_params.logits_processors is None:
vllm/engine/llm_engine.py
+11 -4
@@ -1,3 +1,4 @@
1 +import copy
2 import time
3 from collections import Counter as collectionsCounter
4 from collections import deque
@@ -1024,9 +1025,9 @@ class LLMEngine:
1025 This function updates num_computed_tokens for prompt sequences
1026 when Multi-Step is enabled.
1027
1027 - seq_group: SequenceGroup to update the num_computed_tokens for.
1028 + seq_group: SequenceGroup to update the num_computed_tokens for.
1029 seq_group_meta: Metadata of the given SequenceGroup.
1029 - is_first_step_output: Optional[bool] -
1030 + is_first_step_output: Optional[bool] -
1031 When available, is_first_step_output indicates if the appended
1032 output token is the output of the first-step in multi-step.
1033 A value of None indicates that outputs from all steps in
@@ -2036,7 +2037,11 @@ class LLMEngine:
2037
2038 logits_processors = []
2039
2039 - if (guided_decoding := sampling_params.guided_decoding) is not None:
2040 + if sampling_params.guided_decoding is not None:
2041 + # Defensively copy sampling params since guided decoding logits
2042 + # processors can have different state for each request
2043 + sampling_params = copy.copy(sampling_params)
2044 + guided_decoding = sampling_params.guided_decoding
2045
2046 logger.debug(
2047 "Building guided decoding logits processor in "
@@ -2047,7 +2052,9 @@ class LLMEngine:
2052 self.decoding_config.guided_decoding_backend
2053
2054 processor = get_local_guided_decoding_logits_processor(
2050 - guided_params=guided_decoding, tokenizer=tokenizer)
2055 + guided_params=guided_decoding,
2056 + tokenizer=tokenizer,
2057 + model_config=self.model_config)
2058 if processor:
2059 logits_processors.append(processor)
2060
vllm/engine/multiprocessing/client.py
+3 -2
@@ -474,8 +474,8 @@ class MQLLMEngineClient(EngineClient):
474 trace_headers: OpenTelemetry trace headers.
475 prompt_adapter_request: Prompt Adapter request to use
476 for generation, if any.
477 - priority: Priority of the request (lower means earlier handling).
478 - Any priority other than 0 will lead to an error if the
477 + priority: Priority of the request (lower means earlier handling).
478 + Any priority other than 0 will lead to an error if the
479 scheduling policy is not "priority".
480 """
481 if inputs is not None:
@@ -589,6 +589,7 @@ class MQLLMEngineClient(EngineClient):
589 default_guided_backend=(self.decoding_config.guided_decoding_backend
590 if self.decoding_config
591 else DecodingConfig.guided_decoding_backend),
592 + model_config=self.model_config
593 )
594
595 # 1) Create output queue for this requests.
vllm/model_executor/guided_decoding/__init__.py
+62 -11
@@ -1,14 +1,54 @@
1 -from typing import Optional
1 +from __future__ import annotations
2
3 -from vllm.logits_process import LogitsProcessor
4 -from vllm.sampling_params import GuidedDecodingParams
3 +from typing import TYPE_CHECKING
4 +
5 +from vllm.logger import init_logger
6 +
7 +if TYPE_CHECKING:
8 + from transformers import PreTrainedTokenizer
9 +
10 + from vllm.config import ModelConfig
11 + from vllm.logits_process import LogitsProcessor
12 + from vllm.sampling_params import GuidedDecodingParams
13 +
14 +logger = init_logger(__name__)
15 +
16 +
17 +def maybe_backend_fallback(
18 + guided_params: GuidedDecodingParams) -> GuidedDecodingParams:
19 + # lm-format-enforce doesn't support grammar, fallback to xgrammar
20 + if (guided_params.backend == "lm-format-enforcer"
21 + and guided_params.grammar is not None):
22 + logger.warning(
23 + "lm-format-enforcer does not support grammar guided decoding. "
24 + "Falling back to use xgrammar instead.")
25 + guided_params.backend = "xgrammar"
26 +
27 + if guided_params.backend == "xgrammar":
28 + # xgrammar doesn't support regex or choice, fallback to outlines
29 + if guided_params.regex is not None or guided_params.choice is not None:
30 + logger.warning(
31 + "xgrammar only supports json or grammar guided decoding. "
32 + "Falling back to use outlines instead.")
33 + guided_params.backend = "outlines"
34 +
35 + # xgrammar only supports EBNF grammars and uses the GBNF format
36 + # https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md
37 + elif (guided_params.grammar is not None
38 + and "::=" not in guided_params.grammar):
39 + logger.warning("xgrammar only supports EBNF grammars. "
40 + "Falling back to use outlines instead.")
41 + guided_params.backend = "outlines"
42 +
43 + return guided_params
44
45
46 async def get_guided_decoding_logits_processor(
8 - guided_params: GuidedDecodingParams,
9 - tokenizer) -> Optional[LogitsProcessor]:
47 + guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizer,
48 + model_config: ModelConfig) -> LogitsProcessor | None:
49 + guided_params = maybe_backend_fallback(guided_params)
50 # CFG grammar not supported by LMFE, so we use outlines instead
11 - if guided_params.backend == 'outlines' or guided_params.grammar:
51 + if guided_params.backend == 'outlines':
52 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
53 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
54 get_outlines_guided_decoding_logits_processor)
@@ -19,17 +59,23 @@ async def get_guided_decoding_logits_processor(
59 get_local_lm_format_enforcer_guided_decoding_logits_processor)
60 return get_local_lm_format_enforcer_guided_decoding_logits_processor(
61 guided_params, tokenizer)
62 + if guided_params.backend == 'xgrammar':
63 + from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
64 + get_local_xgrammar_guided_decoding_logits_processor)
65 + return get_local_xgrammar_guided_decoding_logits_processor(
66 + guided_params, tokenizer, model_config)
67
68 raise ValueError(
69 f"Unknown guided decoding backend '{guided_params.backend}'. "
25 - "Must be one of 'outlines, 'lm-format-enforcer'")
70 + "Must be one of 'outlines, 'lm-format-enforcer', 'xgrammar'")
71
72
73 def get_local_guided_decoding_logits_processor(
29 - guided_params: GuidedDecodingParams,
30 - tokenizer) -> Optional[LogitsProcessor]:
74 + guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizer,
75 + model_config: ModelConfig) -> LogitsProcessor | None:
76 + guided_params = maybe_backend_fallback(guided_params)
77 # CFG grammar not supported by LMFE, so we use outlines instead
32 - if guided_params.backend == 'outlines' or guided_params.grammar:
78 + if guided_params.backend == 'outlines':
79 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
80 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
81 get_local_outlines_guided_decoding_logits_processor)
@@ -40,7 +86,12 @@ def get_local_guided_decoding_logits_processor(
86 get_local_lm_format_enforcer_guided_decoding_logits_processor)
87 return get_local_lm_format_enforcer_guided_decoding_logits_processor(
88 guided_params, tokenizer)
89 + if guided_params.backend == 'xgrammar':
90 + from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
91 + get_local_xgrammar_guided_decoding_logits_processor)
92 + return get_local_xgrammar_guided_decoding_logits_processor(
93 + guided_params, tokenizer, model_config)
94
95 raise ValueError(
96 f"Unknown guided decoding backend '{guided_params.backend}'. "
46 - "Must be one of 'outlines, 'lm-format-enforcer'")
97 + "Must be one of 'outlines, 'lm-format-enforcer', 'xgrammar'")
vllm/model_executor/guided_decoding/xgrammar_decoding.py new
+251
@@ -0,0 +1,251 @@
1 +# noqa: UP007
2 +from __future__ import annotations
3 +
4 +import json
5 +from dataclasses import dataclass, field
6 +from typing import TYPE_CHECKING, Any, NamedTuple
7 +
8 +import torch
9 +from transformers import PreTrainedTokenizerFast
10 +
11 +try:
12 + import xgrammar as xgr
13 + from xgrammar.base import _core as xgr_core
14 +except ImportError:
15 + pass
16 +
17 +if TYPE_CHECKING:
18 + from transformers import PreTrainedTokenizer
19 +
20 + from vllm.config import ModelConfig
21 + from vllm.sampling_params import GuidedDecodingParams
22 +
23 +
24 +# TODO: passing batch size to max threads here
25 +def get_local_xgrammar_guided_decoding_logits_processor(
26 + guided_params: GuidedDecodingParams,
27 + tokenizer: PreTrainedTokenizer,
28 + model_config: ModelConfig,
29 + max_threads: int = 8):
30 + config = GrammarConfig.from_guided_params(guided_params=guided_params,
31 + model_config=model_config,
32 + tokenizer=tokenizer,
33 + max_threads=max_threads)
34 + return XGrammarLogitsProcessor(config)
35 +
36 +
37 +class TokenizerData(NamedTuple):
38 + """Immutable container for cached tokenizer data."""
39 + encoded_vocab: list[str]
40 + stop_token_ids: list[int] | None
41 + backend_str: str
42 +
43 +
44 +class TokenizerDataCache:
45 + """Cache manager for tokenizer data to avoid repeated processing."""
46 + _cache: dict[int, TokenizerData] = {}
47 +
48 + @classmethod
49 + def get_tokenizer_data(cls,
50 + tokenizer: PreTrainedTokenizer) -> TokenizerData:
51 + tokenizer_hash = hash(tokenizer)
52 +
53 + if tokenizer_hash not in cls._cache:
54 + # Vendored from xgrammar logic since we cannot pickle the tokenizer
55 + # https://github.com/mlc-ai/xgrammar/blob/d77c0a0173ef14779c918e3be7966ba852f7910f/python/xgrammar/tokenizer_info.py#L98 # noqa: E501
56 + try:
57 + encoded_vocab = [
58 + token for token, _ in sorted(tokenizer.get_vocab().items(),
59 + key=lambda x: x[1])
60 + ]
61 + except AttributeError as e:
62 + raise ValueError(
63 + f"Cannot get the vocabulary of the tokenizer "
64 + f"{type(tokenizer)}. The tokenizer should have a "
65 + "get_vocab method.") from e
66 +
67 + stop_token_ids = None
68 + backend_str = xgr.VocabType.RAW
69 + if isinstance(tokenizer, PreTrainedTokenizerFast):
70 + backend_str = tokenizer.backend_tokenizer.to_str()
71 + if stop_token_ids is None and hasattr(
72 + tokenizer,
73 + "eos_token_id") and tokenizer.eos_token_id is not None:
74 + stop_token_ids = [tokenizer.eos_token_id]
75 +
76 + cls._cache[tokenizer_hash] = TokenizerData(
77 + encoded_vocab=encoded_vocab,
78 + stop_token_ids=stop_token_ids,
79 + backend_str=backend_str)
80 +
81 + return cls._cache[tokenizer_hash]
82 +
83 +
84 +class GrammarCompilerCache:
85 + """
86 + Cache for GrammarCompiler instances based on tokenizer.
87 +
88 + This cache reduces the overhead of creating new compiler instances when
89 + using the same tokenizer configuration.
90 + """
91 + _cache: dict[str, xgr.GrammarCompiler] = {}
92 +
93 + @classmethod
94 + def get_compiler(cls, config: GrammarConfig) -> xgr.GrammarCompiler:
95 + cache_key = str(config.tokenizer_hash)
96 +
97 + if cache_key not in cls._cache:
98 + assert config.encoded_vocab is not None
99 + tokenizer_info = xgr.TokenizerInfo._create_from_handle(
100 + xgr_core.TokenizerInfo.from_huggingface(
101 + config.encoded_vocab, config.backend_str,
102 + config.vocab_size, config.stop_token_ids))
103 + cls._cache[cache_key] = xgr.GrammarCompiler(
104 + tokenizer_info, max_threads=config.max_threads)
105 +
106 + return cls._cache[cache_key]
107 +
108 +
109 +@dataclass
110 +class GrammarConfig:
111 + """Serializable configuration for grammar compilation"""
112 + tokenizer_hash: int
113 + vocab_size: int
114 + json_str: str | None = None
115 + grammar_str: str | None = None
116 + json_object: bool | None = None
117 + max_threads: int = 8
118 + # Only populated if tokenizer_hash not in cache
119 + encoded_vocab: list[str] | None = None
120 + stop_token_ids: list[int] | None = None
121 + backend_str: str | None = None
122 +
123 + @classmethod
124 + def from_guided_params(cls,
125 + guided_params: GuidedDecodingParams,
126 + model_config: ModelConfig,
127 + tokenizer: PreTrainedTokenizer,
128 + max_threads: int = 8) -> GrammarConfig:
129 +
130 + tokenizer_hash = hash(tokenizer)
131 + # Only get tokenizer data if not already cached
132 + if tokenizer_hash in TokenizerDataCache._cache:
133 + encoded_vocab = None
134 + stop_token_ids = None
135 + backend_str = None
136 + else:
137 + tokenizer_data = TokenizerDataCache.get_tokenizer_data(tokenizer)
138 + encoded_vocab = tokenizer_data.encoded_vocab
139 + stop_token_ids = tokenizer_data.stop_token_ids
140 + backend_str = tokenizer_data.backend_str
141 +
142 + if guided_params.json:
143 + if not isinstance(guided_params.json, str):
144 + json_str = json.dumps(guided_params.json)
145 + else:
146 + json_str = guided_params.json
147 + return cls(json_str=json_str,
148 + vocab_size=model_config.hf_config.vocab_size,
149 + encoded_vocab=encoded_vocab,
150 + stop_token_ids=stop_token_ids,
151 + backend_str=backend_str,
152 + tokenizer_hash=tokenizer_hash,
153 + max_threads=max_threads)
154 + elif guided_params.grammar:
155 + return cls(grammar_str=guided_params.grammar,
156 + vocab_size=model_config.hf_config.vocab_size,
157 + encoded_vocab=encoded_vocab,
158 + stop_token_ids=stop_token_ids,
159 + backend_str=backend_str,
160 + tokenizer_hash=tokenizer_hash,
161 + max_threads=max_threads)
162 + elif guided_params.json_object:
163 + return cls(json_object=True,
164 + vocab_size=model_config.hf_config.vocab_size,
165 + encoded_vocab=encoded_vocab,
166 + stop_token_ids=stop_token_ids,
167 + backend_str=backend_str,
168 + tokenizer_hash=tokenizer_hash,
169 + max_threads=max_threads)
170 + else:
171 + raise ValueError(
172 + "Currently only support JSON and EBNF grammar mode for xgrammar"
173 + )
174 +
175 +
176 +@dataclass
177 +class XGrammarLogitsProcessor:
178 + """Wrapper class to support pickle protocol"""
179 + config: GrammarConfig
180 +
181 + ctx: xgr.CompiledGrammar | None = None
182 + token_bitmask: torch.Tensor = None # type: ignore[assignment]
183 + matchers: list[xgr.GrammarMatcher] = field(default_factory=list)
184 + batch_size: int = field(default=1)
185 + prefilled: bool = field(default=False)
186 +
187 + def __getstate__(self) -> dict[str, Any]:
188 + return {'config': self.config}
189 +
190 + def __setstate__(self, state: dict[str, Any]):
191 + self.config = state['config']
192 +
193 + self.ctx = None
194 + self.matchers = []
195 + self.batch_size = 1
196 + self.token_bitmask = None # type: ignore[assignment]
197 + self.prefilled = False
198 +
199 + def _ensure_ctx(self):
200 + """Lazily initialize the processor in the worker process"""
201 + if self.ctx is None:
202 + compiler = GrammarCompilerCache.get_compiler(self.config)
203 + if self.config.json_str is not None:
204 + self.ctx = compiler.compile_json_schema(self.config.json_str)
205 + elif self.config.grammar_str is not None:
206 + self.ctx = compiler.compile_grammar(self.config.grammar_str)
207 + elif self.config.json_object:
208 + self.ctx = compiler.compile_builtin_json_grammar()
209 + else:
210 + raise ValueError(
211 + "Invalid configuration for xgrammar logits processor")
212 +
213 + def __call__(self, input_ids: list[int],
214 + scores: torch.Tensor) -> torch.Tensor:
215 + if self.ctx is None:
216 + self._ensure_ctx()
217 +
218 + if len(self.matchers) == 0:
219 + self.matchers = [
220 + xgr.GrammarMatcher(self.ctx) for _ in range(self.batch_size)
221 + ]
222 + self.token_bitmask = xgr.allocate_token_bitmask(
223 + self.batch_size, self.config.vocab_size)
224 +
225 + if not self.prefilled:
226 + # Have not sampled a token yet
227 + self.prefilled = True
228 + else:
229 + for i, matcher in enumerate(self.matchers):
230 + if not matcher.is_terminated():
231 + sampled_token = input_ids[-1]
232 + assert self.matchers[i].accept_token(sampled_token)
233 +
234 + for i, matcher in enumerate(self.matchers):
235 + if not matcher.is_terminated():
236 + # @ubospica: ideally, fill_next_token_bitmask should be
237 + # parallelized with model decoding
238 + # See https://github.com/vllm-project/vllm/pull/10785/files#r1864278303
239 + matcher.fill_next_token_bitmask(self.token_bitmask, i)
240 +
241 + # token_bitmask is a CPU tensor for use with accept_token and
242 + # fill_next_token_bitmask so we move it to the device of scores
243 + device_type = scores.device.type
244 + if device_type != "cuda":
245 + scores = scores.to("cpu")
246 + xgr.apply_token_bitmask_inplace(scores,
247 + self.token_bitmask.to(scores.device))
248 + if device_type != "cuda":
249 + scores = scores.to(device_type)
250 +
251 + return scores