A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Bugfix] Fix the issue where llm.generate cannot be called repeatedly after setting GuidedDecodingParams (#16767)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Signed-off-by: Russell Bryant <rbryant@redhat.com> Co-authored-by: Russell Bryant <rbryant@redhat.com>
Chauncey committed
Apr 22, 2025 at 14:02 UTC
acba33a0f1c113f400667b9d821a227e88b305db
3 files changed
+32
-4
tests/v1/entrypoints/llm/test_struct_output_generate.py
+11
-3
@@ -386,13 +386,21 @@ def test_structured_output_auto_mode(
386
max_tokens=1000,
387
guided_decoding=GuidedDecodingParams(json=unsupported_json_schema))
388
389
+ prompts = ("Give an example JSON object for a grade "
390
+ "that fits this schema: "
391
+ f"{unsupported_json_schema}")
392
# This would fail with the default of "xgrammar", but in "auto"
393
# we will handle fallback automatically.
391
- outputs = llm.generate(prompts=("Give an example JSON object for a grade "
392
- "that fits this schema: "
393
- f"{unsupported_json_schema}"),
394
+ outputs = llm.generate(prompts=prompts,
395
sampling_params=sampling_params,
396
use_tqdm=True)
397
+ # Make sure `auto` backend handling doesn't mess up sampling_params
398
+ # and that we can reuse it without error.
399
+ outputs.extend(
400
+ llm.generate(prompts=prompts,
401
+ sampling_params=sampling_params,
402
+ use_tqdm=True))
403
+
404
assert outputs is not None
405
for output in outputs:
406
assert output is not None
vllm/sampling_params.py
+11
@@ -79,6 +79,17 @@ class GuidedDecodingParams:
79
return []
80
return self.backend.split(":")[1].split(",")
81
82
+ def add_option(self, opt_name: str) -> None:
83
+ """Adds an option to the backend options."""
84
+ if not self.backend:
85
+ self.backend = f":{opt_name}"
86
+ elif ":" not in self.backend:
87
+ self.backend += f":{opt_name}"
88
+ else:
89
+ options = set(self.backend_options())
90
+ options.add(opt_name)
91
+ self.backend = f"{self.backend_name}:{','.join(sorted(options))}"
92
+
93
def no_fallback(self) -> bool:
94
"""Returns True if the "no-fallback" option is supplied for the guided
95
decoding backend"""
vllm/v1/engine/processor.py
+10
-1
@@ -155,7 +155,14 @@ class Processor:
155
raise ValueError(f"Only {supported_backends} structured output is "
156
"supported in V1.")
157
if params.guided_decoding.backend:
158
- if params.guided_decoding.backend != engine_level_backend:
158
+ # Request-level backend selection is not supported in V1.
159
+ # The values may differ if `params` is reused and was set
160
+ # to a specific backend based on `auto` behavior in a previous
161
+ # request. We remember that it was set as a result of `auto`
162
+ # using the `_auto` option set on the backend in the params.
163
+ if (params.guided_decoding.backend != engine_level_backend
164
+ and not (engine_level_backend == "auto" and "_auto"
165
+ in params.guided_decoding.backend_options())):
166
raise ValueError(
167
"Request-level structured output backend selection is no "
168
"longer supported. The request specified "
@@ -190,6 +197,8 @@ class Processor:
197
# The request includes some jsonschema feature(s) that
198
# are not supported in xgrammar. Fall back to guidance.
199
params.guided_decoding.backend = "guidance"
200
+ # Remember that this backend was set automatically
201
+ params.guided_decoding.add_option("_auto")
202
203
def process_inputs(
204
self,