@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a09ad90a72

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 17:15 UTC

[V1] guidance backend for structured output + `auto` fallback mode (#14779)

Signed-off-by: Russell Bryant <rbryant@redhat.com> Co-authored-by: Loc Huynh <jc1da.3011@gmail.com> Co-authored-by: Michal Moskal <michal@moskal.me>

Russell Bryant committed Mar 25, 2025 at 00:02 UTC a09ad90a72e47060007aa7011969fefb7dedf8f5

9 files changed +345 -111

requirements/common.txt

+1 -1

@@ -18,7 +18,7 @@ pillow # Required for image processing

18 prometheus-fastapi-instrumentator >= 7.0.0

19 tiktoken >= 0.6.0 # Required for DBRX tokenizer

20 lm-format-enforcer >= 0.10.11, < 0.11

21 -llguidance >= 0.7.2, < 0.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64"

21 +llguidance >= 0.7.9, < 0.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64"

22 outlines == 0.1.11

23 lark == 1.2.2

24 xgrammar == 0.1.16; platform_machine == "x86_64" or platform_machine == "aarch64"

tests/v1/entrypoints/llm/test_struct_output_generate.py

+102 -67

@@ -13,7 +13,7 @@ from vllm.entrypoints.llm import LLM

13 from vllm.outputs import RequestOutput

14 from vllm.sampling_params import GuidedDecodingParams, SamplingParams

16 -GUIDED_DECODING_BACKENDS_V1 = ["xgrammar"]

16 +GUIDED_DECODING_BACKENDS_V1 = ["xgrammar", "guidance"]

17 MODELS_TO_TEST = [

18 "Qwen/Qwen2.5-1.5B-Instruct", "mistralai/Ministral-8B-Instruct-2410"

19 ]

@@ -30,12 +30,13 @@ def test_guided_json_completion(

30 model_name: str,

31 ):

32 monkeypatch.setenv("VLLM_USE_V1", "1")

33 - llm = LLM(model=model_name, max_model_len=1024)

34 - sampling_params = SamplingParams(temperature=1.0,

35 - max_tokens=1000,

36 - guided_decoding=GuidedDecodingParams(

37 - json=sample_json_schema,

38 - backend=guided_decoding_backend))

33 + llm = LLM(model=model_name,

34 + max_model_len=1024,

35 + guided_decoding_backend=guided_decoding_backend)

36 + sampling_params = SamplingParams(

37 + temperature=1.0,

38 + max_tokens=1000,

39 + guided_decoding=GuidedDecodingParams(json=sample_json_schema))

40 outputs = llm.generate(prompts=[

41 f"Give an example JSON for an employee profile "

42 f"that fits this schema: {sample_json_schema}"

@@ -111,13 +112,14 @@ def test_guided_json_object(

112 model_name: str,

113 ):

114 monkeypatch.setenv("VLLM_USE_V1", "1")

114 - llm = LLM(model=model_name, max_model_len=1024)

115 - sampling_params = SamplingParams(temperature=1.0,

116 - max_tokens=100,

117 - n=2,

118 - guided_decoding=GuidedDecodingParams(

119 - json_object=True,

120 - backend=guided_decoding_backend))

115 + llm = LLM(model=model_name,

116 + max_model_len=1024,

117 + guided_decoding_backend=guided_decoding_backend)

118 + sampling_params = SamplingParams(

119 + temperature=1.0,

120 + max_tokens=100,

121 + n=2,

122 + guided_decoding=GuidedDecodingParams(json_object=True))

123

124 outputs = llm.generate(

125 prompts=("Generate a JSON object with curly braces for a person with "

@@ -137,12 +139,20 @@ def test_guided_json_object(

139

140 # Parse to verify it is valid JSON

141 parsed_json = json.loads(generated_text)

140 - assert isinstance(parsed_json, dict)

142 + allowed_types: tuple[type, ...] = (dict, )

143 + if guided_decoding_backend == "xgrammar":

144 + # TODO - we are currently too permissive with xgrammar and

145 + # allow # any valid json (typically comes back as a list or

146 + # object). We can fix this by specifying a jsonschema of

147 + # {"type": "object"}, # but we need this fix in a release

148 + # first: https://github.com/mlc-ai/xgrammar/pull/264

149 + allowed_types = (dict, list)

150 + assert isinstance(parsed_json, allowed_types)

151

152

153 @pytest.mark.skip_global_cleanup

154 @pytest.mark.parametrize("guided_decoding_backend",

145 - GUIDED_DECODING_BACKENDS_V1)

155 + GUIDED_DECODING_BACKENDS_V1 + ["auto"])

156 @pytest.mark.parametrize("model_name", MODELS_TO_TEST)

157 def test_guided_json_unsupported_schema(

158 monkeypatch: pytest.MonkeyPatch,

@@ -151,21 +161,43 @@ def test_guided_json_unsupported_schema(

161 model_name: str,

162 ):

163 monkeypatch.setenv("VLLM_USE_V1", "1")

154 - llm = LLM(model=model_name, max_model_len=1024)

155 - sampling_params = SamplingParams(temperature=1.0,

156 - max_tokens=1000,

157 - guided_decoding=GuidedDecodingParams(

158 - json=unsupported_json_schema,

159 - backend=guided_decoding_backend))

160 - with pytest.raises(ValueError,

161 - match="The provided JSON schema contains features "

162 - "not supported by xgrammar."):

163 - llm.generate(prompts=[

164 - f"Give an example JSON for an employee profile "

165 - f"that fits this schema: {unsupported_json_schema}"

166 - ] * 2,

167 - sampling_params=sampling_params,

168 - use_tqdm=True)

164 + llm = LLM(model=model_name,

165 + max_model_len=1024,

166 + guided_decoding_backend=guided_decoding_backend)

167 + sampling_params = SamplingParams(

168 + temperature=1.0,

169 + max_tokens=1000,

170 + guided_decoding=GuidedDecodingParams(json=unsupported_json_schema))

171 + if guided_decoding_backend == "xgrammar":

172 + with pytest.raises(ValueError,

173 + match="The provided JSON schema contains features "

174 + "not supported by xgrammar."):

175 + llm.generate(prompts=[

176 + f"Give an example JSON for an employee profile "

177 + f"that fits this schema: {unsupported_json_schema}"

178 + ] * 2,

179 + sampling_params=sampling_params,

180 + use_tqdm=True)

181 + else:

182 + # This should work for both "guidance" and "auto".

183 +

184 + outputs = llm.generate(

185 + prompts=("Give an example JSON object for a grade "

186 + "that fits this schema: "

187 + f"{unsupported_json_schema}"),

188 + sampling_params=sampling_params,

189 + use_tqdm=True)

190 + assert outputs is not None

191 + for output in outputs:

192 + assert output is not None

193 + assert isinstance(output, RequestOutput)

194 + generated_text = output.outputs[0].text

195 + assert generated_text is not None

196 + print(generated_text)

197 +

198 + # Parse to verify it is valid JSON

199 + parsed_json = json.loads(generated_text)

200 + assert isinstance(parsed_json, dict)

201

202

203 @pytest.mark.skip_global_cleanup

@@ -179,13 +211,14 @@ def test_guided_grammar_ebnf(

211 model_name: str,

212 ):

213 monkeypatch.setenv("VLLM_USE_V1", "1")

182 - llm = LLM(model=model_name, max_model_len=1024)

183 - sampling_params = SamplingParams(temperature=0.8,

184 - top_p=0.95,

185 - max_tokens=1000,

186 - guided_decoding=GuidedDecodingParams(

187 - grammar=sample_sql_ebnf,

188 - backend=guided_decoding_backend))

214 + llm = LLM(model=model_name,

215 + max_model_len=1024,

216 + guided_decoding_backend=guided_decoding_backend)

217 + sampling_params = SamplingParams(

218 + temperature=0.8,

219 + top_p=0.95,

220 + max_tokens=1000,

221 + guided_decoding=GuidedDecodingParams(grammar=sample_sql_ebnf))

222 outputs = llm.generate(

223 prompts=("Generate a sql statement that selects col_1 from "

224 "table_1 where it is equal to 1"),

@@ -222,13 +255,14 @@ def test_guided_grammar_lark(

255 model_name: str,

256 ):

257 monkeypatch.setenv("VLLM_USE_V1", "1")

225 - llm = LLM(model=model_name, max_model_len=1024)

226 - sampling_params = SamplingParams(temperature=0.8,

227 - top_p=0.95,

228 - max_tokens=1000,

229 - guided_decoding=GuidedDecodingParams(

230 - grammar=sample_sql_lark,

231 - backend=guided_decoding_backend))

258 + llm = LLM(model=model_name,

259 + max_model_len=1024,

260 + guided_decoding_backend=guided_decoding_backend)

261 + sampling_params = SamplingParams(

262 + temperature=0.8,

263 + top_p=0.95,

264 + max_tokens=1000,

265 + guided_decoding=GuidedDecodingParams(grammar=sample_sql_lark))

266 outputs = llm.generate(

267 prompts=("Generate a sql statement that selects col_1 from "

268 "table_1 where it is equal to 1"),

@@ -269,16 +303,15 @@ def test_guided_grammar_ebnf_invalid(

303 model_name: str,

304 ):

305 monkeypatch.setenv("VLLM_USE_V1", "1")

272 - llm = LLM(model=model_name, max_model_len=1024)

273 - sampling_params = SamplingParams(temperature=0.8,

274 - top_p=0.95,

275 - max_tokens=1000,

276 - guided_decoding=GuidedDecodingParams(

277 - grammar="not a grammar",

278 - backend=guided_decoding_backend))

279 - with pytest.raises(ValueError,

280 - match="Failed to convert the grammar "

281 - "from Lark to EBNF."):

306 + llm = LLM(model=model_name,

307 + max_model_len=1024,

308 + guided_decoding_backend=guided_decoding_backend)

309 + sampling_params = SamplingParams(

310 + temperature=0.8,

311 + top_p=0.95,

312 + max_tokens=1000,

313 + guided_decoding=GuidedDecodingParams(grammar="not a grammar"))

314 + with pytest.raises(ValueError, match="Failed to convert the grammar "):

315 llm.generate(

316 prompts=("Generate a sql statement that selects col_1 from "

317 "table_1 where it is equal to 1"),

@@ -298,12 +331,13 @@ def test_guided_regex(

331 model_name: str,

332 ):

333 monkeypatch.setenv("VLLM_USE_V1", "1")

301 - llm = LLM(model=model_name, max_model_len=1024)

302 - sampling_params = SamplingParams(temperature=0.8,

303 - top_p=0.95,

304 - guided_decoding=GuidedDecodingParams(

305 - regex=sample_regex,

306 - backend=guided_decoding_backend))

334 + llm = LLM(model=model_name,

335 + max_model_len=1024,

336 + guided_decoding_backend=guided_decoding_backend)

337 + sampling_params = SamplingParams(

338 + temperature=0.8,

339 + top_p=0.95,

340 + guided_decoding=GuidedDecodingParams(regex=sample_regex))

341 outputs = llm.generate(

342 prompts=[

343 f"Give an example IPv4 address with this regex: {sample_regex}"

@@ -335,12 +369,13 @@ def test_guided_choice_completion(

369 model_name: str,

370 ):

371 monkeypatch.setenv("VLLM_USE_V1", "1")

338 - llm = LLM(model=model_name, max_model_len=1024)

339 - sampling_params = SamplingParams(temperature=0.8,

340 - top_p=0.95,

341 - guided_decoding=GuidedDecodingParams(

342 - choice=sample_guided_choice,

343 - backend=guided_decoding_backend))

372 + llm = LLM(model=model_name,

373 + max_model_len=1024,

374 + guided_decoding_backend=guided_decoding_backend)

375 + sampling_params = SamplingParams(

376 + temperature=0.8,

377 + top_p=0.95,

378 + guided_decoding=GuidedDecodingParams(choice=sample_guided_choice))

379 outputs = llm.generate(

380 prompts="The best language for type-safe systems programming is ",

381 sampling_params=sampling_params,

vllm/config.py

+7 -2

@@ -2800,12 +2800,17 @@ class DecodingConfig:

2800 return hash_str

2801

2802 def __post_init__(self):

2803 - valid_guided_backends = [

2804 - 'outlines', 'lm-format-enforcer', 'xgrammar', 'guidance'

2803 + v0_valid_guided_backends = [

2804 + 'outlines', 'lm-format-enforcer', 'xgrammar'

2805 ]

2806 + v1_valid_guided_backends = ['xgrammar', 'guidance', 'auto']

2807

2808 backend = GuidedDecodingParams(

2809 backend=self.guided_decoding_backend).backend_name

2810 + if envs.VLLM_USE_V1:

2811 + valid_guided_backends = v1_valid_guided_backends

2812 + else:

2813 + valid_guided_backends = v0_valid_guided_backends

2814 if backend not in valid_guided_backends:

2815 raise ValueError(f"Invalid guided_decoding_backend '{backend}',"

2816 f" must be one of {valid_guided_backends}")

vllm/engine/arg_utils.py

+9 -12

@@ -391,16 +391,13 @@ class EngineArgs:

391 default='xgrammar',

392 help='Which engine will be used for guided decoding'

393 ' (JSON schema / regex etc) by default. Currently support '

394 - 'https://github.com/outlines-dev/outlines, '

395 - 'https://github.com/mlc-ai/xgrammar, and '

396 - 'https://github.com/noamgat/lm-format-enforcer.'

397 - ' Can be overridden per request via guided_decoding_backend'

398 - ' parameter.\n'

399 - 'Backend-specific options can be supplied in a comma-separated '

400 - 'list following a colon after the backend name. Valid backends and '

401 - 'all available options are: [xgrammar:no-fallback, '

402 - 'xgrammar:disable-any-whitespace, '

403 - 'outlines:no-fallback, lm-format-enforcer:no-fallback]')

394 + 'https://github.com/mlc-ai/xgrammar and '

395 + 'https://github.com/guidance-ai/llguidance.'

396 + 'Valid backend values are "xgrammar", "guidance", and "auto". '

397 + 'With "auto", we will make opinionated choices based on request'

398 + 'contents and what the backend libraries currently support, so '

399 + 'the behavior is subject to change in each release. '

400 + 'The default is xgrammar.')

401 parser.add_argument(

402 '--logits-processor-pattern',

403 type=nullable_str,

@@ -1539,9 +1536,9 @@ class EngineArgs:

1536 recommend_to_remove=False)

1537 return False

1538

1542 - # Only support Xgrammar for guided decoding so far.

1539 + # Xgrammar and Guidance are supported.

1540 SUPPORTED_GUIDED_DECODING = [

1544 - "xgrammar", "xgrammar:disable-any-whitespace"

1541 + "xgrammar", "xgrammar:disable-any-whitespace", "guidance", "auto"

1542 ]

1543 if self.guided_decoding_backend not in SUPPORTED_GUIDED_DECODING:

1544 _raise_or_fallback(feature_name="--guided-decoding-backend",

vllm/v1/engine/processor.py

+32 -7

@@ -4,7 +4,6 @@ import time

4 from collections.abc import Mapping

5 from typing import Optional, Union

7 -import vllm.platforms

7 from vllm.config import VllmConfig

8 from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,

9 PromptType, SingletonInputsAdapter)

@@ -20,7 +19,10 @@ from vllm.prompt_adapter.request import PromptAdapterRequest

19 from vllm.sampling_params import SamplingParams

20 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup

21 from vllm.v1.engine import EngineCoreRequest

23 -from vllm.v1.structured_output.utils import validate_structured_output_request

22 +from vllm.v1.structured_output.backend_guidance import (

23 + validate_guidance_grammar)

24 +from vllm.v1.structured_output.utils import (

25 + validate_structured_output_request_xgrammar)

28 class Processor:

@@ -120,7 +122,9 @@ class Processor:

122 if not params.guided_decoding or not self.decoding_config:

123 return

124

123 - supported_backends = ["xgrammar", "xgrammar:disable-any-whitespace"]

125 + supported_backends = [

126 + "xgrammar", "xgrammar:disable-any-whitespace", "guidance", "auto"

127 + ]

128 engine_level_backend = self.decoding_config.guided_decoding_backend

129 if engine_level_backend not in supported_backends:

130 raise ValueError(f"Only {supported_backends} structured output is "

@@ -134,10 +138,31 @@ class Processor:

138 else:

139 params.guided_decoding.backend = engine_level_backend

140

137 - if vllm.platforms.current_platform.is_tpu():

138 - raise ValueError("Structured output is not supported on TPU.")

139 -

140 - validate_structured_output_request(params)

141 + # Request content validation

142 +

143 + if engine_level_backend == "xgrammar":

144 + # xgrammar with no fallback

145 + validate_structured_output_request_xgrammar(params)

146 + params.guided_decoding.backend = "xgrammar"

147 + elif engine_level_backend == "auto":

148 + # "auto" is an opt-in to opinionated behavior where we try to

149 + # choose a backend based on request contents. This is not the

150 + # default as it is less predictable and subject to change

151 + # between releases as feature support changes.

152 + try:

153 + validate_structured_output_request_xgrammar(params)

154 + params.guided_decoding.backend = "xgrammar"

155 + except ValueError:

156 + # The request includes some jsonschema feature(s) that

157 + # are not supported in xgrammar. Fall back to guidance.

158 + params.guided_decoding.backend = "guidance"

159 +

160 + if params.guided_decoding.backend == "guidance":

161 + # TODO ideally we would have the LLTokenizer here as Lark syntax

162 + # allows <|special_token|> and similar, see

163 + # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens

164 + # Without tokenizer these are disallowed in grammars.

165 + validate_guidance_grammar(params, tokenizer=None)

166

167 def process_inputs(

168 self,

vllm/v1/structured_output/__init__.py

@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Optional

8 from vllm.config import VllmConfig

9 from vllm.logger import init_logger

10 +from vllm.v1.structured_output.backend_guidance import GuidanceBackend

11 from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,

12 StructuredOutputGrammar)

@@ -50,6 +51,8 @@ class StructuredOutputManager:

51 XgrammarBackend)

53 self.backend = XgrammarBackend(self.vllm_config)

54 + elif backend_name == "guidance":

55 + self.backend = GuidanceBackend(self.vllm_config)

56 else:

57 raise ValueError(

58 f"Unsupported structured output backend: {backend_name}")

vllm/v1/structured_output/backend_guidance.py new

+164

@@ -0,0 +1,164 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +

3 +import os

4 +from dataclasses import dataclass

5 +from typing import TYPE_CHECKING, Optional

6 +

7 +import torch

8 +

9 +from vllm.config import VllmConfig

10 +from vllm.logger import init_logger

11 +from vllm.sampling_params import SamplingParams

12 +from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs

13 +from vllm.utils import LazyLoader

14 +from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,

15 + StructuredOutputGrammar,

16 + StructuredOutputOptions)

17 +from vllm.v1.structured_output.request import get_structured_output_key

18 +

19 +if TYPE_CHECKING:

20 + import llguidance

21 + import llguidance.hf as llguidance_hf

22 + import llguidance.torch as llguidance_torch

23 +else:

24 + llguidance = LazyLoader("llguidance", globals(), "llguidance")

25 + llguidance_hf = LazyLoader("llguidance.hf", globals(), "llguidance.hf")

26 + llguidance_torch = LazyLoader("llguidance.torch", globals(),

27 + "llguidance.torch")

28 +

29 +logger = init_logger(__name__)

30 +

31 +

32 +class GuidanceBackend(StructuredOutputBackend):

33 +

34 + def __init__(self, vllm_config: VllmConfig):

35 + self.vllm_config = vllm_config

36 + tokenizer_group = init_tokenizer_from_configs(

37 + model_config=vllm_config.model_config,

38 + scheduler_config=vllm_config.scheduler_config,

39 + parallel_config=vllm_config.parallel_config,

40 + lora_config=vllm_config.lora_config) # type: ignore[arg-type]

41 + tokenizer_group.ping()

42 + self.vllm_config = vllm_config

43 + self.vocab_size = vllm_config.model_config.get_vocab_size()

44 +

45 + tokenizer = tokenizer_group.get_lora_tokenizer(None)

46 + self.ll_tokenizer = llguidance_hf.from_tokenizer(tokenizer, None)

47 +

48 + def compile_grammar(self, request_type: StructuredOutputOptions,

49 + grammar_spec: str) -> StructuredOutputGrammar:

50 + self.serialized_grammar = serialize_guidance_grammar(

51 + request_type, grammar_spec)

52 +

53 + ll_matcher = llguidance.LLMatcher(

54 + self.ll_tokenizer,

55 + self.serialized_grammar,

56 + log_level=int(os.environ.get("LLGUIDANCE_LOG_LEVEL", "1")),

57 + )

58 +

59 + r = GuidanceGrammar(

60 + ll_matcher=ll_matcher,

61 + ll_tokenizer=self.ll_tokenizer,

62 + vocab_size=self.vocab_size,

63 + )

64 +

65 + r.check_error()

66 + return r

67 +

68 + def allocate_token_bitmask(self, max_num_seqs: int):

69 + return llguidance_torch.allocate_token_bitmask(

70 + max_num_seqs, self.ll_tokenizer.vocab_size)

71 +

72 +

73 +@dataclass

74 +class GuidanceGrammar(StructuredOutputGrammar):

75 + ll_matcher: llguidance.LLMatcher

76 + ll_tokenizer: llguidance.LLTokenizer

77 + vocab_size: int

78 + printed_error: bool = False

79 + terminated: bool = False

80 +

81 + def check_error(self):

82 + if not self.printed_error:

83 + err = self.ll_matcher.get_error()

84 + if err:

85 + self.printed_error = True

86 + logger.warning("LLMatcher error: %s", err)

87 +

88 + def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:

89 + """Accepts a list of tokens and advances the parser.

90 +

91 + Returns True if the parser was advanced successfully.

92 + Returns False if the parser failed to advance.

93 + """

94 +

95 + if self.ll_tokenizer.eos_token in tokens:

96 + self.terminated = True

97 +

98 + if self.ll_matcher.is_stopped():

99 + return True

100 +

101 + # TODO - Add jump decoding support in the future:

102 + # self.ll_matcher.compute_ff_bytes() - this should always work

103 + # self.ll_matcher.compute_ff_tokens() - this only works for

104 + # "canonical" tokenizers

105 + # For conversion between the two, see

106 + # https://github.com/guidance-ai/llguidance/blob/main/docs/fast_forward.md

107 +

108 + r = self.ll_matcher.consume_tokens(tokens)

109 +

110 + self.check_error()

111 +

112 + return r

113 +

114 + def fill_bitmask(self, bitmask: torch.Tensor, idx: int) -> None:

115 + # this will automatically return [EOS] mask if the matcher is stopped

116 + # or otherwise in an error state

117 + llguidance_torch.fill_next_token_bitmask(self.ll_matcher, bitmask, idx)

118 + self.check_error()

119 +

120 + def is_terminated(self) -> bool:

121 + return self.terminated

122 +

123 + def reset(self):

124 + # This method may be not needed anymore? TODO

125 + self.ll_matcher.reset()

126 +

127 +

128 +def serialize_guidance_grammar(request_type: StructuredOutputOptions,

129 + grammar_spec: str) -> str:

130 + if request_type == StructuredOutputOptions.JSON:

131 + # TODO: make whitespace_flexible configurable

132 + return llguidance.LLMatcher.grammar_from_json_schema(

133 + grammar_spec, defaults={

134 + "whitespace_flexible": True,

135 + })

136 + elif request_type == StructuredOutputOptions.JSON_OBJECT:

137 + return llguidance.LLMatcher.grammar_from_json_schema(

138 + '{"type": "object"}', defaults={

139 + "whitespace_flexible": True,

140 + })

141 + else:

142 + if request_type == StructuredOutputOptions.REGEX:

143 + tp = "regex"

144 + elif request_type == StructuredOutputOptions.GRAMMAR:

145 + tp = "grammar"

146 + elif request_type == StructuredOutputOptions.CHOICE:

147 + tp = "choice"

148 + else:

149 + logger.error("Validation should have already occurred. "

150 + "Please file an issue.")

151 + raise ValueError("grammar is not of valid supported types. "

152 + f"({request_type!s})")

153 + return llguidance.grammar_from(tp, grammar_spec)

154 +

155 +

156 +def validate_guidance_grammar(

157 + sampling_params: SamplingParams,

158 + tokenizer: Optional[llguidance.LLTokenizer] = None) -> None:

159 + tp, grm = get_structured_output_key(sampling_params)

160 + guidance_grm = serialize_guidance_grammar(tp, grm)

161 + err = llguidance.LLMatcher.validate_grammar(guidance_grm,

162 + tokenizer=tokenizer)

163 + if err:

164 + raise ValueError(f"Grammar error: {err}")

vllm/v1/structured_output/request.py

+26 -21

@@ -53,25 +53,30 @@ class StructuredOutputRequest:

54 @functools.cached_property

55 def structured_output_key(self) -> StructuredOutputKey:

56 - params = self.sampling_params.guided_decoding

57 - assert params is not None, "params can't be None."

58 - if params.json is not None:

59 - if not isinstance(params.json, str):

60 - json_str = json.dumps(params.json)

61 - else:

62 - json_str = params.json

63 - return (StructuredOutputOptions.JSON, json_str)

64 - elif params.json_object:

65 - return (StructuredOutputOptions.JSON_OBJECT, "")

66 - elif params.regex is not None:

67 - return (StructuredOutputOptions.REGEX, params.regex)

68 - elif params.choice is not None:

69 - if not isinstance(params.choice, str):

70 - json_str = json.dumps(params.choice)

71 - else:

72 - json_str = params.choice

73 - return (StructuredOutputOptions.CHOICE, json_str)

74 - elif params.grammar is not None:

75 - return (StructuredOutputOptions.GRAMMAR, params.grammar)

56 + return get_structured_output_key(self.sampling_params)

57 +

58 +

59 +def get_structured_output_key(

60 + sampling_params: SamplingParams) -> StructuredOutputKey:

61 + params = sampling_params.guided_decoding

62 + assert params is not None, "params can't be None."

63 + if params.json is not None:

64 + if not isinstance(params.json, str):

65 + json_str = json.dumps(params.json)

66 + else:

67 + json_str = params.json

68 + return (StructuredOutputOptions.JSON, json_str)

69 + elif params.json_object:

70 + return (StructuredOutputOptions.JSON_OBJECT, "")

71 + elif params.regex is not None:

72 + return (StructuredOutputOptions.REGEX, params.regex)

73 + elif params.choice is not None:

74 + if not isinstance(params.choice, str):

75 + json_str = json.dumps(params.choice)

76 else:

77 - raise ValueError("No valid structured output parameter found")

77 + json_str = params.choice

78 + return (StructuredOutputOptions.CHOICE, json_str)

79 + elif params.grammar is not None:

80 + return (StructuredOutputOptions.GRAMMAR, params.grammar)

81 + else:

82 + raise ValueError("No valid structured output parameter found")

vllm/v1/structured_output/utils.py

+1 -1

@@ -239,7 +239,7 @@ def choice_as_grammar(choice: list[str]) -> str:

239 return grammar

240

241

242 -def validate_structured_output_request(

242 +def validate_structured_output_request_xgrammar(

243 sampling_params: SamplingParams) -> None:

244 """Validate that the request is supported by structured output.

245