A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix guided decoding with tokenizer mode mistral (#11046)
Wallas Henrique committed
Dec 18, 2024 at 03:34 UTC
8b79f9e107fd4214187bf65485b3ea1bb3191a46
7 files changed
+217
-52
.buildkite/test-pipeline.yaml
+5
-1
@@ -224,8 +224,12 @@ steps:
224
mirror_hardwares: [amd]
225
source_file_dependencies:
226
- vllm/model_executor/layers
227
+ - vllm/model_executor/guided_decoding
228
- tests/test_logits_processor
228
- command: pytest -v -s test_logits_processor.py
229
+ - tests/model_executor/test_guided_processors
230
+ commands:
231
+ - pytest -v -s test_logits_processor.py
232
+ - pytest -v -s model_executor/test_guided_processors.py
233
234
- label: Speculative decoding tests # 30min
235
source_file_dependencies:
requirements-common.txt
+2
-1
@@ -14,12 +14,13 @@ aiohttp
14
openai >= 1.45.0 # Ensure modern openai package (ensure types module present and max_completion_tokens field support)
15
uvicorn[standard]
16
pydantic >= 2.9 # Required for fastapi >= 0.113.0
17
-pillow # Required for image processing
17
prometheus_client >= 0.18.0
18
+pillow # Required for image processing
19
prometheus-fastapi-instrumentator >= 7.0.0
20
tiktoken >= 0.6.0 # Required for DBRX tokenizer
21
lm-format-enforcer >= 0.10.9, < 0.11
22
outlines == 0.1.11
23
+lark == 1.2.2
24
xgrammar >= 0.1.6; platform_machine == "x86_64"
25
typing_extensions >= 4.10
26
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
tests/model_executor/test_guided_processors.py
+48
-6
@@ -1,13 +1,19 @@
1
+import pickle
2
+
3
import pytest
4
import torch
5
from transformers import AutoTokenizer
6
7
+from vllm.config import ModelConfig
8
from vllm.model_executor.guided_decoding import (
6
- get_guided_decoding_logits_processor)
9
+ get_guided_decoding_logits_processor,
10
+ get_local_guided_decoding_logits_processor)
11
from vllm.model_executor.guided_decoding.outlines_logits_processors import (
12
JSONLogitsProcessor, RegexLogitsProcessor)
13
from vllm.sampling_params import GuidedDecodingParams
14
15
+MODEL_NAME = 'HuggingFaceH4/zephyr-7b-beta'
16
+
17
18
def test_guided_logits_processors(sample_regex, sample_json_schema):
19
"""Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
@@ -38,14 +44,29 @@ def test_guided_logits_processors(sample_regex, sample_json_schema):
44
@pytest.mark.asyncio
45
@pytest.mark.parametrize("backend",
46
["outlines", "lm-format-enforcer", "xgrammar"])
41
-async def test_guided_logits_processor_black_box(backend: str, sample_regex,
47
+@pytest.mark.parametrize("is_local", [True, False])
48
+async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
49
+ sample_regex,
50
sample_json_schema):
43
- tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
51
+
52
+ config = ModelConfig(
53
+ MODEL_NAME,
54
+ task="generate",
55
+ tokenizer=MODEL_NAME,
56
+ tokenizer_mode="auto",
57
+ trust_remote_code=False,
58
+ seed=0,
59
+ dtype="bfloat16",
60
+ )
61
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
62
token_ids = tokenizer.encode(
63
f"Give an example IPv4 address with this regex: {sample_regex}")
64
regex_request = GuidedDecodingParams(regex=sample_regex, backend=backend)
47
- regex_lp = await get_guided_decoding_logits_processor(
48
- regex_request, tokenizer)
65
+
66
+ regex_lp = get_local_guided_decoding_logits_processor(
67
+ regex_request, tokenizer, config) if is_local else \
68
+ await get_guided_decoding_logits_processor(
69
+ regex_request, tokenizer, config)
70
assert regex_lp is not None
71
tensor = torch.rand(32000)
72
original_tensor = torch.clone(tensor)
@@ -59,7 +80,7 @@ async def test_guided_logits_processor_black_box(backend: str, sample_regex,
80
json_request = GuidedDecodingParams(json=sample_json_schema,
81
backend=backend)
82
json_lp = await get_guided_decoding_logits_processor(
62
- json_request, tokenizer)
83
+ json_request, tokenizer, config)
84
assert json_lp is not None
85
tensor = torch.rand(32000)
86
original_tensor = torch.clone(tensor)
@@ -84,3 +105,24 @@ def test_multiple_guided_options_not_allowed(sample_json_schema, sample_regex):
105
with pytest.raises(ValueError,
106
match="You can only use one kind of guided"):
107
GuidedDecodingParams(json=sample_json_schema, grammar="test grammar")
108
+
109
+
110
+def test_pickle_xgrammar_tokenizer_data():
111
+
112
+ # TODO: move to another test file for xgrammar
113
+ try:
114
+ import xgrammar as xgr
115
+ except ImportError:
116
+ pytest.skip("Could not import xgrammar to run test")
117
+
118
+ from vllm.model_executor.guided_decoding.xgrammar_decoding import (
119
+ TokenizerData)
120
+ tokenizer_data = TokenizerData(vocab_type=xgr.VocabType.RAW)
121
+ pickled = pickle.dumps(tokenizer_data)
122
+
123
+ assert pickled is not None
124
+
125
+ depickled: TokenizerData = pickle.loads(pickled)
126
+
127
+ assert depickled is not None
128
+ assert depickled.vocab_type == xgr.VocabType.RAW
tests/models/decoder_only/language/test_mistral.py
+84
-2
@@ -3,17 +3,20 @@
3
Run `pytest tests/models/test_mistral.py`.
4
"""
5
import copy
6
+import json
7
8
+import jsonschema
9
+import jsonschema.exceptions
10
import pytest
11
9
-from vllm import SamplingParams
12
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import ( # noqa
13
MistralToolParser)
14
+from vllm.sampling_params import GuidedDecodingParams, SamplingParams
15
16
from ...utils import check_logprobs_close
17
18
MODELS = [
16
- "mistralai/Mistral-7B-Instruct-v0.1",
19
+ "mistralai/Mistral-7B-Instruct-v0.3",
20
]
21
22
MISTRAL_FORMAT_MODELS = [
@@ -126,6 +129,45 @@ MSGS = [
129
}
130
]
131
132
+SAMPLE_JSON_SCHEMA = {
133
+ "type": "object",
134
+ "properties": {
135
+ "name": {
136
+ "type": "string"
137
+ },
138
+ "age": {
139
+ "type": "integer"
140
+ },
141
+ "skills": {
142
+ "type": "array",
143
+ "items": {
144
+ "type": "string",
145
+ "maxLength": 10
146
+ },
147
+ "minItems": 3
148
+ },
149
+ "work_history": {
150
+ "type": "array",
151
+ "items": {
152
+ "type": "object",
153
+ "properties": {
154
+ "company": {
155
+ "type": "string"
156
+ },
157
+ "duration": {
158
+ "type": "number"
159
+ },
160
+ "position": {
161
+ "type": "string"
162
+ }
163
+ },
164
+ "required": ["company", "position"]
165
+ }
166
+ }
167
+ },
168
+ "required": ["name", "age", "skills", "work_history"]
169
+}
170
+
171
172
@pytest.mark.parametrize("model", MODELS)
173
@pytest.mark.parametrize("dtype", ["bfloat16"])
@@ -251,3 +293,43 @@ def test_mistral_function_calling(
293
assert parsed_message.tool_calls[
294
0].function.arguments == '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}' # noqa
295
assert parsed_message.content is None
296
+
297
+
298
+@pytest.mark.parametrize("model", MODELS)
299
+@pytest.mark.parametrize("guided_backend",
300
+ ["outlines", "lm-format-enforcer", "xgrammar"])
301
+def test_mistral_guided_decoding(
302
+ vllm_runner,
303
+ model: str,
304
+ guided_backend: str,
305
+) -> None:
306
+ with vllm_runner(model, dtype='bfloat16',
307
+ tokenizer_mode="mistral") as vllm_model:
308
+
309
+ guided_decoding = GuidedDecodingParams(json=SAMPLE_JSON_SCHEMA,
310
+ backend=guided_backend)
311
+ params = SamplingParams(max_tokens=512,
312
+ temperature=0.7,
313
+ guided_decoding=guided_decoding)
314
+
315
+ messages = [{
316
+ "role": "system",
317
+ "content": "you are a helpful assistant"
318
+ }, {
319
+ "role":
320
+ "user",
321
+ "content":
322
+ f"Give an example JSON for an employee profile that "
323
+ f"fits this schema: {SAMPLE_JSON_SCHEMA}"
324
+ }]
325
+ outputs = vllm_model.model.chat(messages, sampling_params=params)
326
+
327
+ generated_text = outputs[0].outputs[0].text
328
+ json_response = json.loads(generated_text)
329
+ assert outputs is not None
330
+
331
+ try:
332
+ jsonschema.validate(instance=json_response,
333
+ schema=SAMPLE_JSON_SCHEMA)
334
+ except jsonschema.exceptions.ValidationError:
335
+ pytest.fail("Generated response is not valid with JSON schema")
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+73
-40
@@ -3,7 +3,7 @@ from __future__ import annotations
3
4
import json
5
from dataclasses import dataclass, field
6
-from typing import TYPE_CHECKING, Any, NamedTuple
6
+from typing import TYPE_CHECKING, Any
7
8
import torch
9
from transformers import PreTrainedTokenizerFast
@@ -16,6 +16,7 @@ except ImportError:
16
17
from vllm.model_executor.guided_decoding.xgrammar_utils import (
18
convert_lark_to_gbnf, grammar_is_likely_lark)
19
+from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
20
21
if TYPE_CHECKING:
22
from transformers import PreTrainedTokenizer
@@ -37,11 +38,21 @@ def get_local_xgrammar_guided_decoding_logits_processor(
38
return XGrammarLogitsProcessor(config)
39
40
40
-class TokenizerData(NamedTuple):
41
+@dataclass(frozen=True)
42
+class TokenizerData:
43
"""Immutable container for cached tokenizer data."""
42
- encoded_vocab: list[str]
43
- stop_token_ids: list[int] | None
44
- backend_str: str
44
+ encoded_vocab: list[str] = field(default_factory=list)
45
+ stop_token_ids: list[int] | None = None
46
+ # These fields are mutually exclusive: `backend_str` is used to create a
47
+ # TokenizeInfo with `TokenizerInfo.from_huggingface` while `vocab_type` is
48
+ # used within the constructor of TokenizeInfo
49
+ backend_str: str | None = None
50
+ vocab_type: xgr.VocabType | None = None
51
+
52
+ def __post_init__(self):
53
+ # Check for mutual exclusive
54
+ assert not (self.backend_str and self.vocab_type), \
55
+ "backend_str and vocab_type are mutual exclusive"
56
57
58
class TokenizerDataCache:
@@ -68,18 +79,27 @@ class TokenizerDataCache:
79
"get_vocab method.") from e
80
81
stop_token_ids = None
71
- backend_str = xgr.VocabType.RAW
82
+ backend_str = ""
83
+ vocab_type = xgr.VocabType.RAW
84
+
85
+ if stop_token_ids is None and hasattr(
86
+ tokenizer,
87
+ "eos_token_id") and tokenizer.eos_token_id is not None:
88
+ stop_token_ids = [tokenizer.eos_token_id]
89
+
90
if isinstance(tokenizer, PreTrainedTokenizerFast):
91
backend_str = tokenizer.backend_tokenizer.to_str()
74
- if stop_token_ids is None and hasattr(
75
- tokenizer,
76
- "eos_token_id") and tokenizer.eos_token_id is not None:
77
- stop_token_ids = [tokenizer.eos_token_id]
92
+ vocab_type = None
93
+
94
+ elif isinstance(tokenizer, MistralTokenizer):
95
+ # REF: https://github.com/mlc-ai/xgrammar/blob/5e141f6ff1ca02bc31f9e512e68b61f2a8ae88e5/tests/python/test_tokenizer_info.py#L43 # noqa: E501
96
+ vocab_type = xgr.VocabType.BYTE_FALLBACK
97
98
cls._cache[tokenizer_hash] = TokenizerData(
99
encoded_vocab=encoded_vocab,
100
stop_token_ids=stop_token_ids,
82
- backend_str=backend_str)
101
+ backend_str=backend_str,
102
+ vocab_type=vocab_type)
103
104
return cls._cache[tokenizer_hash]
105
@@ -98,11 +118,30 @@ class GrammarCompilerCache:
118
cache_key = str(config.tokenizer_hash)
119
120
if cache_key not in cls._cache:
101
- assert config.encoded_vocab is not None
102
- tokenizer_info = xgr.TokenizerInfo._create_from_handle(
103
- xgr_core.TokenizerInfo.from_huggingface(
104
- config.encoded_vocab, config.backend_str,
105
- config.vocab_size, config.stop_token_ids))
121
+ assert config.tokenizer_data is not None
122
+ assert config.tokenizer_data.encoded_vocab is not None
123
+
124
+ config_data = config.tokenizer_data
125
+
126
+ # In TokenizerDataCache.get_tokenizer_data, a serializable
127
+ # tokenizer_data is created and cached. This data is used to build
128
+ # a tokenizer_info and create an xgrammar compiler.
129
+ # - If tokenizer_data has backend_str set, use
130
+ # xgr_core.TokenizerInfo.from_huggingface (a C++ bind).
131
+ # - Otherwise, use the default constructor with vocab_type.
132
+ # - xgr_core.TokenizerInfo.from_huggingface !=
133
+ # xgr.TokenizerInfo.from_huggingface.
134
+ if config_data.backend_str:
135
+ tokenizer_info = xgr.TokenizerInfo._create_from_handle(
136
+ xgr_core.TokenizerInfo.from_huggingface(
137
+ config_data.encoded_vocab, config_data.backend_str,
138
+ config.vocab_size, config_data.stop_token_ids))
139
+ else:
140
+ tokenizer_info = xgr.TokenizerInfo(
141
+ config_data.encoded_vocab,
142
+ config_data.vocab_type,
143
+ vocab_size=config.vocab_size,
144
+ stop_token_ids=config_data.stop_token_ids)
145
cls._cache[cache_key] = xgr.GrammarCompiler(
146
tokenizer_info, max_threads=config.max_threads)
147
@@ -118,10 +157,7 @@ class GrammarConfig:
157
grammar_str: str | None = None
158
json_object: bool | None = None
159
max_threads: int = 8
121
- # Only populated if tokenizer_hash not in cache
122
- encoded_vocab: list[str] | None = None
123
- stop_token_ids: list[int] | None = None
124
- backend_str: str | None = None
160
+ tokenizer_data: TokenizerData | None = None
161
162
@classmethod
163
def from_guided_params(cls,
@@ -132,9 +168,6 @@ class GrammarConfig:
168
169
tokenizer_hash = hash(tokenizer)
170
tokenizer_data = TokenizerDataCache.get_tokenizer_data(tokenizer)
135
- encoded_vocab = tokenizer_data.encoded_vocab
136
- stop_token_ids = tokenizer_data.stop_token_ids
137
- backend_str = tokenizer_data.backend_str
171
172
if guided_params.json:
173
if not isinstance(guided_params.json, str):
@@ -152,11 +185,9 @@ class GrammarConfig:
185
186
return cls(json_str=json_str,
187
vocab_size=model_config.hf_text_config.vocab_size,
155
- encoded_vocab=encoded_vocab,
156
- stop_token_ids=stop_token_ids,
157
- backend_str=backend_str,
188
tokenizer_hash=tokenizer_hash,
159
- max_threads=max_threads)
189
+ max_threads=max_threads,
190
+ tokenizer_data=tokenizer_data)
191
elif guided_params.grammar:
192
# XGrammar only supports GBNF grammars, so we must convert Lark
193
if grammar_is_likely_lark(guided_params.grammar):
@@ -181,19 +212,17 @@ class GrammarConfig:
212
213
return cls(grammar_str=grammar_str,
214
vocab_size=model_config.hf_text_config.vocab_size,
184
- encoded_vocab=encoded_vocab,
185
- stop_token_ids=stop_token_ids,
186
- backend_str=backend_str,
215
tokenizer_hash=tokenizer_hash,
188
- max_threads=max_threads)
216
+ max_threads=max_threads,
217
+ tokenizer_data=tokenizer_data)
218
elif guided_params.json_object:
190
- return cls(json_object=True,
191
- vocab_size=model_config.hf_text_config.vocab_size,
192
- encoded_vocab=encoded_vocab,
193
- stop_token_ids=stop_token_ids,
194
- backend_str=backend_str,
195
- tokenizer_hash=tokenizer_hash,
196
- max_threads=max_threads)
219
+ return cls(
220
+ json_object=True,
221
+ vocab_size=model_config.hf_text_config.vocab_size,
222
+ tokenizer_hash=tokenizer_hash,
223
+ max_threads=max_threads,
224
+ tokenizer_data=tokenizer_data,
225
+ )
226
else:
227
raise ValueError(
228
"Currently only support JSON and EBNF grammar mode for xgrammar"
@@ -269,10 +298,14 @@ class XGrammarLogitsProcessor:
298
# fill_next_token_bitmask so we move it to the device of scores
299
device_type = scores.device.type
300
if device_type != "cuda":
272
- scores = scores.to("cpu")
301
+ scores = scores.to("cpu").unsqueeze(0)
302
+
303
+ # Note: In this method, if the tensors have different dimensions
304
+ # on CPU device fails, but on GPU it runs without error. Hence the
305
+ # unsqueeze above for scores, to match the token bitmask shape
306
xgr.apply_token_bitmask_inplace(scores,
307
self.token_bitmask.to(scores.device))
308
if device_type != "cuda":
276
- scores = scores.to(device_type)
309
+ scores = scores.to(device_type).squeeze()
310
311
return scores
vllm/transformers_utils/tokenizer.py
+1
-1
@@ -132,7 +132,7 @@ def get_tokenizer(
132
if is_from_mistral_org and tokenizer_mode != "mistral":
133
warnings.warn(
134
'It is strongly recommended to run mistral models with '
135
- '`--tokenizer_mode "mistral"` to ensure correct '
135
+ '`--tokenizer-mode "mistral"` to ensure correct '
136
'encoding and decoding.',
137
FutureWarning,
138
stacklevel=2)
vllm/transformers_utils/tokenizers/mistral.py
+4
-1
@@ -314,12 +314,15 @@ class MistralTokenizer:
314
315
if regular_tokens:
316
decoded_list.append(
317
- self.decode(regular_tokens)) # type: ignore
317
+ self.tokenizer.decode(regular_tokens)) # type: ignore
318
319
decoded = ''.join(decoded_list)
320
321
return decoded
322
323
+ # WARN: Outlines logits processors can overwrite this method.
324
+ # See: guided_decoding/outlines_logits_processors.py::_adapt_tokenizer
325
+ # for more.
326
def decode(self,
327
ids: Union[List[int], int],
328
skip_special_tokens: bool = True) -> str: