A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 23:45 UTC
[BugFix] Adding env variable to disable async grammar compilation (#29996)
Signed-off-by: Alec Solder <alecs@fb.com> Signed-off-by: Alec S <10566873+alecsolder@users.noreply.github.com> Co-authored-by: Alec Solder <alecs@fb.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: 22quinn <33176974+22quinn@users.noreply.github.com>
Alec S committed
Dec 5, 2025 at 03:49 UTC
65ee97288a5b3b622de8d7a460fa965514ff543a
2 files changed
+89
-2
tests/v1/structured_output/test_backend_guidance.py
+74
@@ -1,9 +1,14 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import time
4
+from concurrent.futures import Future
5
+
6
+import pytest
7
from transformers import AutoTokenizer
8
9
from vllm.config import StructuredOutputsConfig, VllmConfig
10
from vllm.config.model import ModelConfig
11
+from vllm.config.parallel import ParallelConfig
12
from vllm.config.speculative import SpeculativeConfig
13
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
14
from vllm.v1.request import Request
@@ -116,3 +121,72 @@ def test_grammar_bitmask_with_specdec():
121
) # EOS not the final token
122
grammar_bitmask(request, prompt[i:]) # EOS not present
123
grammar_bitmask(request, prompt[i:] + [tokenizer.eos_token_id])
124
+
125
+
126
+@pytest.mark.parametrize("async_grammar", [True, False])
127
+def test_grammar_init_async_and_sync(async_grammar):
128
+ """Test grammar initialization works correctly in both async and sync modes.
129
+
130
+ This test validates that the distributed_executor_backend config option
131
+ correctly controls whether grammar compilation happens asynchronously
132
+ (via executor.submit) or synchronously. When set to "external_launcher",
133
+ grammar compilation is synchronous to avoid deadlocks.
134
+ """
135
+ tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
136
+ prompt = tokenizer.encode('{"a": "b"}')
137
+
138
+ # Use "external_launcher" for sync mode, None for async mode
139
+ executor_backend = None if async_grammar else "external_launcher"
140
+ vllm_config = VllmConfig(
141
+ model_config=ModelConfig(tokenizer=TOKENIZER),
142
+ structured_outputs_config=StructuredOutputsConfig(backend="guidance"),
143
+ parallel_config=ParallelConfig(distributed_executor_backend=executor_backend),
144
+ )
145
+ structured_output_manager = StructuredOutputManager(vllm_config)
146
+
147
+ sampling_params = SamplingParams(
148
+ structured_outputs=StructuredOutputsParams(
149
+ json='{"type": "object"}',
150
+ ),
151
+ )
152
+ sampling_params.structured_outputs._backend = "guidance"
153
+
154
+ request = Request(
155
+ "test_request",
156
+ prompt_token_ids=prompt,
157
+ sampling_params=sampling_params,
158
+ pooling_params=None,
159
+ eos_token_id=tokenizer.eos_token_id,
160
+ )
161
+
162
+ structured_output_manager.grammar_init(request)
163
+
164
+ # Check the internal _grammar type immediately after init
165
+ # Before _check_grammar_completion is called, async mode should have a Future
166
+ raw_grammar = request.structured_output_request._grammar
167
+ if async_grammar:
168
+ assert isinstance(raw_grammar, Future), (
169
+ "Async mode should store a Future before completion"
170
+ )
171
+ else:
172
+ assert not isinstance(raw_grammar, Future), (
173
+ "Sync mode should store the grammar directly, not a Future"
174
+ )
175
+
176
+ # Wait for grammar to be ready (handles both async and sync cases)
177
+ start_time = time.time()
178
+ while not request.structured_output_request._check_grammar_completion():
179
+ if time.time() - start_time > 5: # 5-second timeout
180
+ pytest.fail("Grammar compilation timed out")
181
+ time.sleep(0.01)
182
+
183
+ # After completion, _grammar should no longer be a Future
184
+ assert not isinstance(request.structured_output_request._grammar, Future)
185
+
186
+ # Verify grammar is properly initialized and functional
187
+ grammar = request.structured_output_request.grammar
188
+ assert grammar is not None
189
+ assert not grammar.is_terminated()
190
+
191
+ # Verify the grammar can accept valid tokens
192
+ assert grammar.accept_tokens(request.request_id, prompt)
vllm/v1/structured_output/__init__.py
+15
-2
@@ -40,6 +40,16 @@ class StructuredOutputManager:
40
self.reasoner: ReasoningParser | None = None
41
self.vllm_config = vllm_config
42
43
+ # When in external_launcher mode, async grammar compilation causes deadlocks
44
+ # due to external_launcher mode having a scheduler for each TP rank.
45
+ # Async grammar compilation causes the WAITING_FOR_FSM → WAITING transition to
46
+ # happen at different times on different TP ranks,
47
+ # breaking the determinism assumption that external_launcher relies on.
48
+ self._use_async_grammar_compilation = (
49
+ vllm_config.parallel_config.distributed_executor_backend
50
+ != "external_launcher"
51
+ )
52
+
53
self._grammar_bitmask: torch.Tensor | None = None
54
self._full_mask = torch.tensor(-1, dtype=torch.int32)
55
@@ -138,10 +148,13 @@ class StructuredOutputManager:
148
else:
149
raise ValueError(f"Unsupported structured output backend: {backend}")
150
141
- grammar = self.executor.submit(self._async_create_grammar, request)
151
+ if self._use_async_grammar_compilation:
152
+ grammar = self.executor.submit(self._create_grammar, request)
153
+ else:
154
+ grammar = self._create_grammar(request) # type: ignore[assignment]
155
request.structured_output_request.grammar = grammar # type: ignore[assignment]
156
144
- def _async_create_grammar(
157
+ def _create_grammar(
158
self,
159
request: Request,
160
) -> StructuredOutputGrammar: