A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Core][Performance] Add XGrammar support for guided decoding and set it as default (#10785)
Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Signed-off-by: mgoin <michael@neuralmagic.com> Co-authored-by: mgoin <michael@neuralmagic.com>
Aaron Pham committed
Dec 3, 2024 at 02:17 UTC
9323a3153b20d4a2ca7ac04a2784609d6ce656e0
11 files changed
+385
-33
docs/source/conf.py
+1
@@ -178,6 +178,7 @@ autodoc_mock_imports = [
178
"tensorizer",
179
"pynvml",
180
"outlines",
181
+ "xgrammar,"
182
"librosa",
183
"soundfile",
184
"gguf",
requirements-common.txt
+1
@@ -19,6 +19,7 @@ prometheus-fastapi-instrumentator >= 7.0.0
19
tiktoken >= 0.6.0 # Required for DBRX tokenizer
20
lm-format-enforcer >= 0.10.9, < 0.11
21
outlines >= 0.0.43, < 0.1
22
+xgrammar
23
typing_extensions >= 4.10
24
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
25
partial-json-parser # used for parsing partial JSON outputs
tests/entrypoints/llm/test_guided_generate.py
+27
@@ -159,3 +159,30 @@ def test_validation_against_both_guided_decoding_options(sample_regex, llm):
159
sampling_params=sampling_params,
160
use_tqdm=True,
161
guided_options_request=dict(guided_regex=sample_regex))
162
+
163
+
164
+@pytest.mark.skip_global_cleanup
165
+def test_guided_json_object(llm):
166
+ sampling_params = SamplingParams(
167
+ temperature=1.0,
168
+ max_tokens=100,
169
+ guided_decoding=GuidedDecodingParams(json_object=True))
170
+
171
+ outputs = llm.generate(
172
+ prompts=("Generate a JSON object describing a person with name "
173
+ "and age for John Smith who is 31 years old."),
174
+ sampling_params=sampling_params,
175
+ use_tqdm=True)
176
+
177
+ assert outputs is not None
178
+ for output in outputs:
179
+ assert output is not None
180
+ assert isinstance(output, RequestOutput)
181
+
182
+ generated_text = output.outputs[0].text
183
+ print(generated_text)
184
+ assert generated_text is not None
185
+
186
+ # Parse to verify it is valid JSON
187
+ parsed_json = json.loads(generated_text)
188
+ assert isinstance(parsed_json, dict)
tests/model_executor/test_guided_processors.py
+2
-1
@@ -36,7 +36,8 @@ def test_guided_logits_processors(sample_regex, sample_json_schema):
36
37
38
@pytest.mark.asyncio
39
-@pytest.mark.parametrize("backend", ["outlines", "lm-format-enforcer"])
39
+@pytest.mark.parametrize("backend",
40
+ ["outlines", "lm-format-enforcer", "xgrammar"])
41
async def test_guided_logits_processor_black_box(backend: str, sample_regex,
42
sample_json_schema):
43
tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
vllm/config.py
+8
-7
@@ -1789,15 +1789,15 @@ class PoolerConfig:
1789
1790
step_tag_id: Optional[int] = None
1791
"""
1792
- If set, only the score corresponding to the ``step_tag_id`` in the
1792
+ If set, only the score corresponding to the ``step_tag_id`` in the
1793
generated sentence should be returned. Otherwise, the scores for all tokens
1794
are returned.
1795
"""
1796
1797
returned_token_ids: Optional[List[int]] = None
1798
"""
1799
- A list of indices for the vocabulary dimensions to be extracted,
1800
- such as the token IDs of ``good_token`` and ``bad_token`` in the
1799
+ A list of indices for the vocabulary dimensions to be extracted,
1800
+ such as the token IDs of ``good_token`` and ``bad_token`` in the
1801
``math-shepherd-mistral-7b-prm`` model.
1802
"""
1803
@@ -2031,11 +2031,12 @@ def get_served_model_name(model: str,
2031
class DecodingConfig:
2032
"""Dataclass which contains the decoding strategy of the engine"""
2033
2034
- # Which guided decoding algo to use. 'outlines' / 'lm-format-enforcer'
2035
- guided_decoding_backend: str = 'outlines'
2034
+ # Which guided decoding algo to use.
2035
+ # 'outlines' / 'lm-format-enforcer' / 'xgrammar'
2036
+ guided_decoding_backend: str = 'xgrammar'
2037
2038
def __post_init__(self):
2038
- valid_guided_backends = ['outlines', 'lm-format-enforcer']
2039
+ valid_guided_backends = ['outlines', 'lm-format-enforcer', 'xgrammar']
2040
backend = self.guided_decoding_backend
2041
if backend not in valid_guided_backends:
2042
raise ValueError(f"Invalid guided_decoding_backend '{backend},"
@@ -2222,7 +2223,7 @@ class CompilationConfig(BaseModel):
2223
from Python, functions can also be passed directly via Python object
2224
constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
2225
- custom inductor passes: see PassConfig for more details
2225
-
2226
+
2227
Why we have different sizes for cudagraph and inductor:
2228
- cudagraph: a cudagraph captured for a specific size can only be used
2229
for the same size. We need to capture all the sizes we want to use.
vllm/engine/arg_utils.py
+5
-4
@@ -168,7 +168,7 @@ class EngineArgs:
168
scheduler_delay_factor: float = 0.0
169
enable_chunked_prefill: Optional[bool] = None
170
171
- guided_decoding_backend: str = 'outlines'
171
+ guided_decoding_backend: str = 'xgrammar'
172
# Speculative decoding configuration.
173
speculative_model: Optional[str] = None
174
speculative_model_quantization: Optional[str] = None
@@ -364,11 +364,12 @@ class EngineArgs:
364
parser.add_argument(
365
'--guided-decoding-backend',
366
type=str,
367
- default='outlines',
368
- choices=['outlines', 'lm-format-enforcer'],
367
+ default='xgrammar',
368
+ choices=['outlines', 'lm-format-enforcer', 'xgrammar'],
369
help='Which engine will be used for guided decoding'
370
' (JSON schema / regex etc) by default. Currently support '
371
- 'https://github.com/outlines-dev/outlines and '
371
+ 'https://github.com/outlines-dev/outlines,'
372
+ 'https://github.com/mlc-ai/xgrammar, and '
373
'https://github.com/noamgat/lm-format-enforcer.'
374
' Can be overridden per request via guided_decoding_backend'
375
' parameter.')
vllm/engine/async_llm_engine.py
+14
-4
@@ -1,4 +1,5 @@
1
import asyncio
2
+import copy
3
import time
4
import weakref
5
from functools import partial
@@ -507,7 +508,8 @@ class _AsyncLLMEngine(LLMEngine):
508
sampling_params=params,
509
tokenizer=await self.get_tokenizer_async(lora_request),
510
default_guided_backend=self.decoding_config.
510
- guided_decoding_backend)
511
+ guided_decoding_backend,
512
+ model_config=self.model_config)
513
514
self._add_processed_request(
515
request_id=request_id,
@@ -528,22 +530,30 @@ class _AsyncLLMEngine(LLMEngine):
530
531
async def build_guided_decoding_logits_processor_async(
532
sampling_params: SamplingParams, tokenizer: AnyTokenizer,
531
- default_guided_backend: str) -> SamplingParams:
533
+ default_guided_backend: str,
534
+ model_config: ModelConfig) -> SamplingParams:
535
"""Constructs logits processors based on the guided_decoding,
536
logits_bias, and allowed_token_ids fields in sampling_params. Deletes
537
those fields and adds the constructed logits processors to the
538
logits_processors field. Modifies sampling params in-place and returns
539
the modified sampling params."""
537
- if (guided_decoding := sampling_params.guided_decoding) is None:
540
+ if sampling_params.guided_decoding is None:
541
return sampling_params
542
543
+ # Defensively copy sampling params since guided decoding logits
544
+ # processors can have different state for each request
545
+ sampling_params = copy.copy(sampling_params)
546
+ guided_decoding = sampling_params.guided_decoding
547
+
548
logger.debug("Building guided decoding logits processor. "
549
"Params: %s", guided_decoding)
550
551
guided_decoding.backend = guided_decoding.backend or default_guided_backend
552
553
processor = await get_guided_decoding_logits_processor(
546
- guided_params=guided_decoding, tokenizer=tokenizer)
554
+ guided_params=guided_decoding,
555
+ tokenizer=tokenizer,
556
+ model_config=model_config)
557
558
if processor:
559
if sampling_params.logits_processors is None:
vllm/engine/llm_engine.py
+11
-4
@@ -1,3 +1,4 @@
1
+import copy
2
import time
3
from collections import Counter as collectionsCounter
4
from collections import deque
@@ -1024,9 +1025,9 @@ class LLMEngine:
1025
This function updates num_computed_tokens for prompt sequences
1026
when Multi-Step is enabled.
1027
1027
- seq_group: SequenceGroup to update the num_computed_tokens for.
1028
+ seq_group: SequenceGroup to update the num_computed_tokens for.
1029
seq_group_meta: Metadata of the given SequenceGroup.
1029
- is_first_step_output: Optional[bool] -
1030
+ is_first_step_output: Optional[bool] -
1031
When available, is_first_step_output indicates if the appended
1032
output token is the output of the first-step in multi-step.
1033
A value of None indicates that outputs from all steps in
@@ -2036,7 +2037,11 @@ class LLMEngine:
2037
2038
logits_processors = []
2039
2039
- if (guided_decoding := sampling_params.guided_decoding) is not None:
2040
+ if sampling_params.guided_decoding is not None:
2041
+ # Defensively copy sampling params since guided decoding logits
2042
+ # processors can have different state for each request
2043
+ sampling_params = copy.copy(sampling_params)
2044
+ guided_decoding = sampling_params.guided_decoding
2045
2046
logger.debug(
2047
"Building guided decoding logits processor in "
@@ -2047,7 +2052,9 @@ class LLMEngine:
2052
self.decoding_config.guided_decoding_backend
2053
2054
processor = get_local_guided_decoding_logits_processor(
2050
- guided_params=guided_decoding, tokenizer=tokenizer)
2055
+ guided_params=guided_decoding,
2056
+ tokenizer=tokenizer,
2057
+ model_config=self.model_config)
2058
if processor:
2059
logits_processors.append(processor)
2060
vllm/engine/multiprocessing/client.py
+3
-2
@@ -474,8 +474,8 @@ class MQLLMEngineClient(EngineClient):
474
trace_headers: OpenTelemetry trace headers.
475
prompt_adapter_request: Prompt Adapter request to use
476
for generation, if any.
477
- priority: Priority of the request (lower means earlier handling).
478
- Any priority other than 0 will lead to an error if the
477
+ priority: Priority of the request (lower means earlier handling).
478
+ Any priority other than 0 will lead to an error if the
479
scheduling policy is not "priority".
480
"""
481
if inputs is not None:
@@ -589,6 +589,7 @@ class MQLLMEngineClient(EngineClient):
589
default_guided_backend=(self.decoding_config.guided_decoding_backend
590
if self.decoding_config
591
else DecodingConfig.guided_decoding_backend),
592
+ model_config=self.model_config
593
)
594
595
# 1) Create output queue for this requests.
vllm/model_executor/guided_decoding/__init__.py
+62
-11
@@ -1,14 +1,54 @@
1
-from typing import Optional
1
+from __future__ import annotations
2
3
-from vllm.logits_process import LogitsProcessor
4
-from vllm.sampling_params import GuidedDecodingParams
3
+from typing import TYPE_CHECKING
4
+
5
+from vllm.logger import init_logger
6
+
7
+if TYPE_CHECKING:
8
+ from transformers import PreTrainedTokenizer
9
+
10
+ from vllm.config import ModelConfig
11
+ from vllm.logits_process import LogitsProcessor
12
+ from vllm.sampling_params import GuidedDecodingParams
13
+
14
+logger = init_logger(__name__)
15
+
16
+
17
+def maybe_backend_fallback(
18
+ guided_params: GuidedDecodingParams) -> GuidedDecodingParams:
19
+ # lm-format-enforce doesn't support grammar, fallback to xgrammar
20
+ if (guided_params.backend == "lm-format-enforcer"
21
+ and guided_params.grammar is not None):
22
+ logger.warning(
23
+ "lm-format-enforcer does not support grammar guided decoding. "
24
+ "Falling back to use xgrammar instead.")
25
+ guided_params.backend = "xgrammar"
26
+
27
+ if guided_params.backend == "xgrammar":
28
+ # xgrammar doesn't support regex or choice, fallback to outlines
29
+ if guided_params.regex is not None or guided_params.choice is not None:
30
+ logger.warning(
31
+ "xgrammar only supports json or grammar guided decoding. "
32
+ "Falling back to use outlines instead.")
33
+ guided_params.backend = "outlines"
34
+
35
+ # xgrammar only supports EBNF grammars and uses the GBNF format
36
+ # https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md
37
+ elif (guided_params.grammar is not None
38
+ and "::=" not in guided_params.grammar):
39
+ logger.warning("xgrammar only supports EBNF grammars. "
40
+ "Falling back to use outlines instead.")
41
+ guided_params.backend = "outlines"
42
+
43
+ return guided_params
44
45
46
async def get_guided_decoding_logits_processor(
8
- guided_params: GuidedDecodingParams,
9
- tokenizer) -> Optional[LogitsProcessor]:
47
+ guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizer,
48
+ model_config: ModelConfig) -> LogitsProcessor | None:
49
+ guided_params = maybe_backend_fallback(guided_params)
50
# CFG grammar not supported by LMFE, so we use outlines instead
11
- if guided_params.backend == 'outlines' or guided_params.grammar:
51
+ if guided_params.backend == 'outlines':
52
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
53
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
54
get_outlines_guided_decoding_logits_processor)
@@ -19,17 +59,23 @@ async def get_guided_decoding_logits_processor(
59
get_local_lm_format_enforcer_guided_decoding_logits_processor)
60
return get_local_lm_format_enforcer_guided_decoding_logits_processor(
61
guided_params, tokenizer)
62
+ if guided_params.backend == 'xgrammar':
63
+ from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
64
+ get_local_xgrammar_guided_decoding_logits_processor)
65
+ return get_local_xgrammar_guided_decoding_logits_processor(
66
+ guided_params, tokenizer, model_config)
67
68
raise ValueError(
69
f"Unknown guided decoding backend '{guided_params.backend}'. "
25
- "Must be one of 'outlines, 'lm-format-enforcer'")
70
+ "Must be one of 'outlines, 'lm-format-enforcer', 'xgrammar'")
71
72
73
def get_local_guided_decoding_logits_processor(
29
- guided_params: GuidedDecodingParams,
30
- tokenizer) -> Optional[LogitsProcessor]:
74
+ guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizer,
75
+ model_config: ModelConfig) -> LogitsProcessor | None:
76
+ guided_params = maybe_backend_fallback(guided_params)
77
# CFG grammar not supported by LMFE, so we use outlines instead
32
- if guided_params.backend == 'outlines' or guided_params.grammar:
78
+ if guided_params.backend == 'outlines':
79
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
80
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
81
get_local_outlines_guided_decoding_logits_processor)
@@ -40,7 +86,12 @@ def get_local_guided_decoding_logits_processor(
86
get_local_lm_format_enforcer_guided_decoding_logits_processor)
87
return get_local_lm_format_enforcer_guided_decoding_logits_processor(
88
guided_params, tokenizer)
89
+ if guided_params.backend == 'xgrammar':
90
+ from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
91
+ get_local_xgrammar_guided_decoding_logits_processor)
92
+ return get_local_xgrammar_guided_decoding_logits_processor(
93
+ guided_params, tokenizer, model_config)
94
95
raise ValueError(
96
f"Unknown guided decoding backend '{guided_params.backend}'. "
46
- "Must be one of 'outlines, 'lm-format-enforcer'")
97
+ "Must be one of 'outlines, 'lm-format-enforcer', 'xgrammar'")
vllm/model_executor/guided_decoding/xgrammar_decoding.py
new
+251
@@ -0,0 +1,251 @@
1
+# noqa: UP007
2
+from __future__ import annotations
3
+
4
+import json
5
+from dataclasses import dataclass, field
6
+from typing import TYPE_CHECKING, Any, NamedTuple
7
+
8
+import torch
9
+from transformers import PreTrainedTokenizerFast
10
+
11
+try:
12
+ import xgrammar as xgr
13
+ from xgrammar.base import _core as xgr_core
14
+except ImportError:
15
+ pass
16
+
17
+if TYPE_CHECKING:
18
+ from transformers import PreTrainedTokenizer
19
+
20
+ from vllm.config import ModelConfig
21
+ from vllm.sampling_params import GuidedDecodingParams
22
+
23
+
24
+# TODO: passing batch size to max threads here
25
+def get_local_xgrammar_guided_decoding_logits_processor(
26
+ guided_params: GuidedDecodingParams,
27
+ tokenizer: PreTrainedTokenizer,
28
+ model_config: ModelConfig,
29
+ max_threads: int = 8):
30
+ config = GrammarConfig.from_guided_params(guided_params=guided_params,
31
+ model_config=model_config,
32
+ tokenizer=tokenizer,
33
+ max_threads=max_threads)
34
+ return XGrammarLogitsProcessor(config)
35
+
36
+
37
+class TokenizerData(NamedTuple):
38
+ """Immutable container for cached tokenizer data."""
39
+ encoded_vocab: list[str]
40
+ stop_token_ids: list[int] | None
41
+ backend_str: str
42
+
43
+
44
+class TokenizerDataCache:
45
+ """Cache manager for tokenizer data to avoid repeated processing."""
46
+ _cache: dict[int, TokenizerData] = {}
47
+
48
+ @classmethod
49
+ def get_tokenizer_data(cls,
50
+ tokenizer: PreTrainedTokenizer) -> TokenizerData:
51
+ tokenizer_hash = hash(tokenizer)
52
+
53
+ if tokenizer_hash not in cls._cache:
54
+ # Vendored from xgrammar logic since we cannot pickle the tokenizer
55
+ # https://github.com/mlc-ai/xgrammar/blob/d77c0a0173ef14779c918e3be7966ba852f7910f/python/xgrammar/tokenizer_info.py#L98 # noqa: E501
56
+ try:
57
+ encoded_vocab = [
58
+ token for token, _ in sorted(tokenizer.get_vocab().items(),
59
+ key=lambda x: x[1])
60
+ ]
61
+ except AttributeError as e:
62
+ raise ValueError(
63
+ f"Cannot get the vocabulary of the tokenizer "
64
+ f"{type(tokenizer)}. The tokenizer should have a "
65
+ "get_vocab method.") from e
66
+
67
+ stop_token_ids = None
68
+ backend_str = xgr.VocabType.RAW
69
+ if isinstance(tokenizer, PreTrainedTokenizerFast):
70
+ backend_str = tokenizer.backend_tokenizer.to_str()
71
+ if stop_token_ids is None and hasattr(
72
+ tokenizer,
73
+ "eos_token_id") and tokenizer.eos_token_id is not None:
74
+ stop_token_ids = [tokenizer.eos_token_id]
75
+
76
+ cls._cache[tokenizer_hash] = TokenizerData(
77
+ encoded_vocab=encoded_vocab,
78
+ stop_token_ids=stop_token_ids,
79
+ backend_str=backend_str)
80
+
81
+ return cls._cache[tokenizer_hash]
82
+
83
+
84
+class GrammarCompilerCache:
85
+ """
86
+ Cache for GrammarCompiler instances based on tokenizer.
87
+
88
+ This cache reduces the overhead of creating new compiler instances when
89
+ using the same tokenizer configuration.
90
+ """
91
+ _cache: dict[str, xgr.GrammarCompiler] = {}
92
+
93
+ @classmethod
94
+ def get_compiler(cls, config: GrammarConfig) -> xgr.GrammarCompiler:
95
+ cache_key = str(config.tokenizer_hash)
96
+
97
+ if cache_key not in cls._cache:
98
+ assert config.encoded_vocab is not None
99
+ tokenizer_info = xgr.TokenizerInfo._create_from_handle(
100
+ xgr_core.TokenizerInfo.from_huggingface(
101
+ config.encoded_vocab, config.backend_str,
102
+ config.vocab_size, config.stop_token_ids))
103
+ cls._cache[cache_key] = xgr.GrammarCompiler(
104
+ tokenizer_info, max_threads=config.max_threads)
105
+
106
+ return cls._cache[cache_key]
107
+
108
+
109
+@dataclass
110
+class GrammarConfig:
111
+ """Serializable configuration for grammar compilation"""
112
+ tokenizer_hash: int
113
+ vocab_size: int
114
+ json_str: str | None = None
115
+ grammar_str: str | None = None
116
+ json_object: bool | None = None
117
+ max_threads: int = 8
118
+ # Only populated if tokenizer_hash not in cache
119
+ encoded_vocab: list[str] | None = None
120
+ stop_token_ids: list[int] | None = None
121
+ backend_str: str | None = None
122
+
123
+ @classmethod
124
+ def from_guided_params(cls,
125
+ guided_params: GuidedDecodingParams,
126
+ model_config: ModelConfig,
127
+ tokenizer: PreTrainedTokenizer,
128
+ max_threads: int = 8) -> GrammarConfig:
129
+
130
+ tokenizer_hash = hash(tokenizer)
131
+ # Only get tokenizer data if not already cached
132
+ if tokenizer_hash in TokenizerDataCache._cache:
133
+ encoded_vocab = None
134
+ stop_token_ids = None
135
+ backend_str = None
136
+ else:
137
+ tokenizer_data = TokenizerDataCache.get_tokenizer_data(tokenizer)
138
+ encoded_vocab = tokenizer_data.encoded_vocab
139
+ stop_token_ids = tokenizer_data.stop_token_ids
140
+ backend_str = tokenizer_data.backend_str
141
+
142
+ if guided_params.json:
143
+ if not isinstance(guided_params.json, str):
144
+ json_str = json.dumps(guided_params.json)
145
+ else:
146
+ json_str = guided_params.json
147
+ return cls(json_str=json_str,
148
+ vocab_size=model_config.hf_config.vocab_size,
149
+ encoded_vocab=encoded_vocab,
150
+ stop_token_ids=stop_token_ids,
151
+ backend_str=backend_str,
152
+ tokenizer_hash=tokenizer_hash,
153
+ max_threads=max_threads)
154
+ elif guided_params.grammar:
155
+ return cls(grammar_str=guided_params.grammar,
156
+ vocab_size=model_config.hf_config.vocab_size,
157
+ encoded_vocab=encoded_vocab,
158
+ stop_token_ids=stop_token_ids,
159
+ backend_str=backend_str,
160
+ tokenizer_hash=tokenizer_hash,
161
+ max_threads=max_threads)
162
+ elif guided_params.json_object:
163
+ return cls(json_object=True,
164
+ vocab_size=model_config.hf_config.vocab_size,
165
+ encoded_vocab=encoded_vocab,
166
+ stop_token_ids=stop_token_ids,
167
+ backend_str=backend_str,
168
+ tokenizer_hash=tokenizer_hash,
169
+ max_threads=max_threads)
170
+ else:
171
+ raise ValueError(
172
+ "Currently only support JSON and EBNF grammar mode for xgrammar"
173
+ )
174
+
175
+
176
+@dataclass
177
+class XGrammarLogitsProcessor:
178
+ """Wrapper class to support pickle protocol"""
179
+ config: GrammarConfig
180
+
181
+ ctx: xgr.CompiledGrammar | None = None
182
+ token_bitmask: torch.Tensor = None # type: ignore[assignment]
183
+ matchers: list[xgr.GrammarMatcher] = field(default_factory=list)
184
+ batch_size: int = field(default=1)
185
+ prefilled: bool = field(default=False)
186
+
187
+ def __getstate__(self) -> dict[str, Any]:
188
+ return {'config': self.config}
189
+
190
+ def __setstate__(self, state: dict[str, Any]):
191
+ self.config = state['config']
192
+
193
+ self.ctx = None
194
+ self.matchers = []
195
+ self.batch_size = 1
196
+ self.token_bitmask = None # type: ignore[assignment]
197
+ self.prefilled = False
198
+
199
+ def _ensure_ctx(self):
200
+ """Lazily initialize the processor in the worker process"""
201
+ if self.ctx is None:
202
+ compiler = GrammarCompilerCache.get_compiler(self.config)
203
+ if self.config.json_str is not None:
204
+ self.ctx = compiler.compile_json_schema(self.config.json_str)
205
+ elif self.config.grammar_str is not None:
206
+ self.ctx = compiler.compile_grammar(self.config.grammar_str)
207
+ elif self.config.json_object:
208
+ self.ctx = compiler.compile_builtin_json_grammar()
209
+ else:
210
+ raise ValueError(
211
+ "Invalid configuration for xgrammar logits processor")
212
+
213
+ def __call__(self, input_ids: list[int],
214
+ scores: torch.Tensor) -> torch.Tensor:
215
+ if self.ctx is None:
216
+ self._ensure_ctx()
217
+
218
+ if len(self.matchers) == 0:
219
+ self.matchers = [
220
+ xgr.GrammarMatcher(self.ctx) for _ in range(self.batch_size)
221
+ ]
222
+ self.token_bitmask = xgr.allocate_token_bitmask(
223
+ self.batch_size, self.config.vocab_size)
224
+
225
+ if not self.prefilled:
226
+ # Have not sampled a token yet
227
+ self.prefilled = True
228
+ else:
229
+ for i, matcher in enumerate(self.matchers):
230
+ if not matcher.is_terminated():
231
+ sampled_token = input_ids[-1]
232
+ assert self.matchers[i].accept_token(sampled_token)
233
+
234
+ for i, matcher in enumerate(self.matchers):
235
+ if not matcher.is_terminated():
236
+ # @ubospica: ideally, fill_next_token_bitmask should be
237
+ # parallelized with model decoding
238
+ # See https://github.com/vllm-project/vllm/pull/10785/files#r1864278303
239
+ matcher.fill_next_token_bitmask(self.token_bitmask, i)
240
+
241
+ # token_bitmask is a CPU tensor for use with accept_token and
242
+ # fill_next_token_bitmask so we move it to the device of scores
243
+ device_type = scores.device.type
244
+ if device_type != "cuda":
245
+ scores = scores.to("cpu")
246
+ xgr.apply_token_bitmask_inplace(scores,
247
+ self.token_bitmask.to(scores.device))
248
+ if device_type != "cuda":
249
+ scores = scores.to(device_type)
250
+
251
+ return scores