A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Bugfix] Fix Mistral guided generation using xgrammar (#15704)
Signed-off-by: Julien Denize <julien.denize@mistral.ai>
Julien Denize committed
Mar 30, 2025 at 05:20 UTC
6909a762012ce665931ff6d482dce17cf927108a
2 files changed
+34
-17
tests/v1/entrypoints/llm/test_struct_output_generate.py
+23
-10
@@ -15,11 +15,20 @@ from vllm.entrypoints.llm import LLM
15
from vllm.outputs import RequestOutput
16
from vllm.sampling_params import GuidedDecodingParams, SamplingParams
17
18
-GUIDED_DECODING_BACKENDS_V1 = [
19
- "xgrammar:disable-any-whitespace", "guidance:disable-any-whitespace"
18
+PARAMS_MODELS_BACKENDS_TOKENIZER_MODE = [
19
+ ("mistralai/Ministral-8B-Instruct-2410", "xgrammar:disable-any-whitespace",
20
+ "auto"),
21
+ ("mistralai/Ministral-8B-Instruct-2410", "guidance:disable-any-whitespace",
22
+ "auto"),
23
+ ("mistralai/Ministral-8B-Instruct-2410", "xgrammar:disable-any-whitespace",
24
+ "mistral"),
25
+ ("Qwen/Qwen2.5-1.5B-Instruct", "xgrammar:disable-any-whitespace", "auto"),
26
+ ("Qwen/Qwen2.5-1.5B-Instruct", "guidance:disable-any-whitespace", "auto"),
27
]
21
-MODELS_TO_TEST = [
22
- "Qwen/Qwen2.5-1.5B-Instruct", "mistralai/Ministral-8B-Instruct-2410"
28
+
29
+PARAMS_MODELS_TOKENIZER_MODE = [
30
+ ("mistralai/Ministral-8B-Instruct-2410", "auto"),
31
+ ("Qwen/Qwen2.5-1.5B-Instruct", "auto"),
32
]
33
34
@@ -37,9 +46,8 @@ class CarDescription(BaseModel):
46
47
48
@pytest.mark.skip_global_cleanup
40
-@pytest.mark.parametrize("guided_decoding_backend",
41
- GUIDED_DECODING_BACKENDS_V1)
42
-@pytest.mark.parametrize("model_name", MODELS_TO_TEST)
49
+@pytest.mark.parametrize("model_name, guided_decoding_backend, tokenizer_mode",
50
+ PARAMS_MODELS_BACKENDS_TOKENIZER_MODE)
51
def test_structured_output(
52
monkeypatch: pytest.MonkeyPatch,
53
sample_json_schema: dict[str, Any],
@@ -49,6 +57,7 @@ def test_structured_output(
57
sample_regex: str,
58
sample_guided_choice: str,
59
guided_decoding_backend: str,
60
+ tokenizer_mode: str,
61
model_name: str,
62
):
63
monkeypatch.setenv("VLLM_USE_V1", "1")
@@ -58,7 +67,8 @@ def test_structured_output(
67
llm = LLM(model=model_name,
68
enforce_eager=True,
69
max_model_len=1024,
61
- guided_decoding_backend=guided_decoding_backend)
70
+ guided_decoding_backend=guided_decoding_backend,
71
+ tokenizer_mode=tokenizer_mode)
72
73
#
74
# Test 1: Generate JSON output based on a provided schema
@@ -324,17 +334,20 @@ def test_structured_output(
334
335
336
@pytest.mark.skip_global_cleanup
327
-@pytest.mark.parametrize("model_name", MODELS_TO_TEST)
337
+@pytest.mark.parametrize("model_name, tokenizer_mode",
338
+ PARAMS_MODELS_TOKENIZER_MODE)
339
def test_structured_output_auto_mode(
340
monkeypatch: pytest.MonkeyPatch,
341
unsupported_json_schema: dict[str, Any],
342
model_name: str,
343
+ tokenizer_mode: str,
344
):
345
monkeypatch.setenv("VLLM_USE_V1", "1")
346
347
llm = LLM(model=model_name,
348
max_model_len=1024,
337
- guided_decoding_backend="auto")
349
+ guided_decoding_backend="auto",
350
+ tokenizer_mode=tokenizer_mode)
351
352
sampling_params = SamplingParams(
353
temperature=1.0,
vllm/v1/structured_output/backend_xgrammar.py
+11
-7
@@ -42,12 +42,15 @@ class XgrammarBackend(StructuredOutputBackend):
42
# NOTE: ideally, xgrammar should handle this accordingly.
43
# refer to https://github.com/mlc-ai/xgrammar/blob/d77c0a0173ef14779c918e3be7966ba852f7910f/python/xgrammar/tokenizer_info.py#L98
44
try:
45
- encoded_vocab = [
46
- token for token, _ in sorted(
47
- tokenizer.get_vocab().items(),
48
- key=lambda x: x[1],
49
- )
50
- ]
45
+ if tokenizer.is_tekken:
46
+ encoded_vocab = tokenizer._vocab
47
+ else:
48
+ encoded_vocab = [
49
+ token for token, _ in sorted(
50
+ tokenizer.get_vocab().items(),
51
+ key=lambda x: x[1],
52
+ )
53
+ ]
54
stop_token_ids = None
55
if hasattr(
56
tokenizer,
@@ -62,7 +65,8 @@ class XgrammarBackend(StructuredOutputBackend):
65
tokenizer_info = xgr.TokenizerInfo( # type: ignore
66
encoded_vocab=encoded_vocab,
67
# NOTE: https://github.com/mlc-ai/xgrammar/blob/5e141f6ff1ca02bc31f9e512e68b61f2a8ae88e5/tests/python/test_tokenizer_info.py#L43 # noqa: E501
65
- vocab_type=xgr.VocabType.BYTE_FALLBACK,
68
+ vocab_type=xgr.VocabType.RAW
69
+ if tokenizer.is_tekken else xgr.VocabType.BYTE_FALLBACK,
70
vocab_size=self.vocab_size,
71
stop_token_ids=stop_token_ids,
72
add_prefix_space=True,