@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8b79f9e107
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix guided decoding with tokenizer mode mistral (#11046)

Wallas Henrique committed Dec 18, 2024 at 03:34 UTC 8b79f9e107fd4214187bf65485b3ea1bb3191a46
7 files changed +217 -52
.buildkite/test-pipeline.yaml
+5 -1
@@ -224,8 +224,12 @@ steps:
224 mirror_hardwares: [amd]
225 source_file_dependencies:
226 - vllm/model_executor/layers
227 + - vllm/model_executor/guided_decoding
228 - tests/test_logits_processor
228 - command: pytest -v -s test_logits_processor.py
229 + - tests/model_executor/test_guided_processors
230 + commands:
231 + - pytest -v -s test_logits_processor.py
232 + - pytest -v -s model_executor/test_guided_processors.py
233
234 - label: Speculative decoding tests # 30min
235 source_file_dependencies:
requirements-common.txt
+2 -1
@@ -14,12 +14,13 @@ aiohttp
14 openai >= 1.45.0 # Ensure modern openai package (ensure types module present and max_completion_tokens field support)
15 uvicorn[standard]
16 pydantic >= 2.9 # Required for fastapi >= 0.113.0
17 -pillow # Required for image processing
17 prometheus_client >= 0.18.0
18 +pillow # Required for image processing
19 prometheus-fastapi-instrumentator >= 7.0.0
20 tiktoken >= 0.6.0 # Required for DBRX tokenizer
21 lm-format-enforcer >= 0.10.9, < 0.11
22 outlines == 0.1.11
23 +lark == 1.2.2
24 xgrammar >= 0.1.6; platform_machine == "x86_64"
25 typing_extensions >= 4.10
26 filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
tests/model_executor/test_guided_processors.py
+48 -6
@@ -1,13 +1,19 @@
1 +import pickle
2 +
3 import pytest
4 import torch
5 from transformers import AutoTokenizer
6
7 +from vllm.config import ModelConfig
8 from vllm.model_executor.guided_decoding import (
6 - get_guided_decoding_logits_processor)
9 + get_guided_decoding_logits_processor,
10 + get_local_guided_decoding_logits_processor)
11 from vllm.model_executor.guided_decoding.outlines_logits_processors import (
12 JSONLogitsProcessor, RegexLogitsProcessor)
13 from vllm.sampling_params import GuidedDecodingParams
14
15 +MODEL_NAME = 'HuggingFaceH4/zephyr-7b-beta'
16 +
17
18 def test_guided_logits_processors(sample_regex, sample_json_schema):
19 """Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
@@ -38,14 +44,29 @@ def test_guided_logits_processors(sample_regex, sample_json_schema):
44 @pytest.mark.asyncio
45 @pytest.mark.parametrize("backend",
46 ["outlines", "lm-format-enforcer", "xgrammar"])
41 -async def test_guided_logits_processor_black_box(backend: str, sample_regex,
47 +@pytest.mark.parametrize("is_local", [True, False])
48 +async def test_guided_logits_processor_black_box(backend: str, is_local: bool,
49 + sample_regex,
50 sample_json_schema):
43 - tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
51 +
52 + config = ModelConfig(
53 + MODEL_NAME,
54 + task="generate",
55 + tokenizer=MODEL_NAME,
56 + tokenizer_mode="auto",
57 + trust_remote_code=False,
58 + seed=0,
59 + dtype="bfloat16",
60 + )
61 + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
62 token_ids = tokenizer.encode(
63 f"Give an example IPv4 address with this regex: {sample_regex}")
64 regex_request = GuidedDecodingParams(regex=sample_regex, backend=backend)
47 - regex_lp = await get_guided_decoding_logits_processor(
48 - regex_request, tokenizer)
65 +
66 + regex_lp = get_local_guided_decoding_logits_processor(
67 + regex_request, tokenizer, config) if is_local else \
68 + await get_guided_decoding_logits_processor(
69 + regex_request, tokenizer, config)
70 assert regex_lp is not None
71 tensor = torch.rand(32000)
72 original_tensor = torch.clone(tensor)
@@ -59,7 +80,7 @@ async def test_guided_logits_processor_black_box(backend: str, sample_regex,
80 json_request = GuidedDecodingParams(json=sample_json_schema,
81 backend=backend)
82 json_lp = await get_guided_decoding_logits_processor(
62 - json_request, tokenizer)
83 + json_request, tokenizer, config)
84 assert json_lp is not None
85 tensor = torch.rand(32000)
86 original_tensor = torch.clone(tensor)
@@ -84,3 +105,24 @@ def test_multiple_guided_options_not_allowed(sample_json_schema, sample_regex):
105 with pytest.raises(ValueError,
106 match="You can only use one kind of guided"):
107 GuidedDecodingParams(json=sample_json_schema, grammar="test grammar")
108 +
109 +
110 +def test_pickle_xgrammar_tokenizer_data():
111 +
112 + # TODO: move to another test file for xgrammar
113 + try:
114 + import xgrammar as xgr
115 + except ImportError:
116 + pytest.skip("Could not import xgrammar to run test")
117 +
118 + from vllm.model_executor.guided_decoding.xgrammar_decoding import (
119 + TokenizerData)
120 + tokenizer_data = TokenizerData(vocab_type=xgr.VocabType.RAW)
121 + pickled = pickle.dumps(tokenizer_data)
122 +
123 + assert pickled is not None
124 +
125 + depickled: TokenizerData = pickle.loads(pickled)
126 +
127 + assert depickled is not None
128 + assert depickled.vocab_type == xgr.VocabType.RAW
tests/models/decoder_only/language/test_mistral.py
+84 -2
@@ -3,17 +3,20 @@
3 Run `pytest tests/models/test_mistral.py`.
4 """
5 import copy
6 +import json
7
8 +import jsonschema
9 +import jsonschema.exceptions
10 import pytest
11
9 -from vllm import SamplingParams
12 from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import ( # noqa
13 MistralToolParser)
14 +from vllm.sampling_params import GuidedDecodingParams, SamplingParams
15
16 from ...utils import check_logprobs_close
17
18 MODELS = [
16 - "mistralai/Mistral-7B-Instruct-v0.1",
19 + "mistralai/Mistral-7B-Instruct-v0.3",
20 ]
21
22 MISTRAL_FORMAT_MODELS = [
@@ -126,6 +129,45 @@ MSGS = [
129 }
130 ]
131
132 +SAMPLE_JSON_SCHEMA = {
133 + "type": "object",
134 + "properties": {
135 + "name": {
136 + "type": "string"
137 + },
138 + "age": {
139 + "type": "integer"
140 + },
141 + "skills": {
142 + "type": "array",
143 + "items": {
144 + "type": "string",
145 + "maxLength": 10
146 + },
147 + "minItems": 3
148 + },
149 + "work_history": {
150 + "type": "array",
151 + "items": {
152 + "type": "object",
153 + "properties": {
154 + "company": {
155 + "type": "string"
156 + },
157 + "duration": {
158 + "type": "number"
159 + },
160 + "position": {
161 + "type": "string"
162 + }
163 + },
164 + "required": ["company", "position"]
165 + }
166 + }
167 + },
168 + "required": ["name", "age", "skills", "work_history"]
169 +}
170 +
171
172 @pytest.mark.parametrize("model", MODELS)
173 @pytest.mark.parametrize("dtype", ["bfloat16"])
@@ -251,3 +293,43 @@ def test_mistral_function_calling(
293 assert parsed_message.tool_calls[
294 0].function.arguments == '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}' # noqa
295 assert parsed_message.content is None
296 +
297 +
298 +@pytest.mark.parametrize("model", MODELS)
299 +@pytest.mark.parametrize("guided_backend",
300 + ["outlines", "lm-format-enforcer", "xgrammar"])
301 +def test_mistral_guided_decoding(
302 + vllm_runner,
303 + model: str,
304 + guided_backend: str,
305 +) -> None:
306 + with vllm_runner(model, dtype='bfloat16',
307 + tokenizer_mode="mistral") as vllm_model:
308 +
309 + guided_decoding = GuidedDecodingParams(json=SAMPLE_JSON_SCHEMA,
310 + backend=guided_backend)
311 + params = SamplingParams(max_tokens=512,
312 + temperature=0.7,
313 + guided_decoding=guided_decoding)
314 +
315 + messages = [{
316 + "role": "system",
317 + "content": "you are a helpful assistant"
318 + }, {
319 + "role":
320 + "user",
321 + "content":
322 + f"Give an example JSON for an employee profile that "
323 + f"fits this schema: {SAMPLE_JSON_SCHEMA}"
324 + }]
325 + outputs = vllm_model.model.chat(messages, sampling_params=params)
326 +
327 + generated_text = outputs[0].outputs[0].text
328 + json_response = json.loads(generated_text)
329 + assert outputs is not None
330 +
331 + try:
332 + jsonschema.validate(instance=json_response,
333 + schema=SAMPLE_JSON_SCHEMA)
334 + except jsonschema.exceptions.ValidationError:
335 + pytest.fail("Generated response is not valid with JSON schema")
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+73 -40
@@ -3,7 +3,7 @@ from __future__ import annotations
3
4 import json
5 from dataclasses import dataclass, field
6 -from typing import TYPE_CHECKING, Any, NamedTuple
6 +from typing import TYPE_CHECKING, Any
7
8 import torch
9 from transformers import PreTrainedTokenizerFast
@@ -16,6 +16,7 @@ except ImportError:
16
17 from vllm.model_executor.guided_decoding.xgrammar_utils import (
18 convert_lark_to_gbnf, grammar_is_likely_lark)
19 +from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
20
21 if TYPE_CHECKING:
22 from transformers import PreTrainedTokenizer
@@ -37,11 +38,21 @@ def get_local_xgrammar_guided_decoding_logits_processor(
38 return XGrammarLogitsProcessor(config)
39
40
40 -class TokenizerData(NamedTuple):
41 +@dataclass(frozen=True)
42 +class TokenizerData:
43 """Immutable container for cached tokenizer data."""
42 - encoded_vocab: list[str]
43 - stop_token_ids: list[int] | None
44 - backend_str: str
44 + encoded_vocab: list[str] = field(default_factory=list)
45 + stop_token_ids: list[int] | None = None
46 + # These fields are mutually exclusive: `backend_str` is used to create a
47 + # TokenizeInfo with `TokenizerInfo.from_huggingface` while `vocab_type` is
48 + # used within the constructor of TokenizeInfo
49 + backend_str: str | None = None
50 + vocab_type: xgr.VocabType | None = None
51 +
52 + def __post_init__(self):
53 + # Check for mutual exclusive
54 + assert not (self.backend_str and self.vocab_type), \
55 + "backend_str and vocab_type are mutual exclusive"
56
57
58 class TokenizerDataCache:
@@ -68,18 +79,27 @@ class TokenizerDataCache:
79 "get_vocab method.") from e
80
81 stop_token_ids = None
71 - backend_str = xgr.VocabType.RAW
82 + backend_str = ""
83 + vocab_type = xgr.VocabType.RAW
84 +
85 + if stop_token_ids is None and hasattr(
86 + tokenizer,
87 + "eos_token_id") and tokenizer.eos_token_id is not None:
88 + stop_token_ids = [tokenizer.eos_token_id]
89 +
90 if isinstance(tokenizer, PreTrainedTokenizerFast):
91 backend_str = tokenizer.backend_tokenizer.to_str()
74 - if stop_token_ids is None and hasattr(
75 - tokenizer,
76 - "eos_token_id") and tokenizer.eos_token_id is not None:
77 - stop_token_ids = [tokenizer.eos_token_id]
92 + vocab_type = None
93 +
94 + elif isinstance(tokenizer, MistralTokenizer):
95 + # REF: https://github.com/mlc-ai/xgrammar/blob/5e141f6ff1ca02bc31f9e512e68b61f2a8ae88e5/tests/python/test_tokenizer_info.py#L43 # noqa: E501
96 + vocab_type = xgr.VocabType.BYTE_FALLBACK
97
98 cls._cache[tokenizer_hash] = TokenizerData(
99 encoded_vocab=encoded_vocab,
100 stop_token_ids=stop_token_ids,
82 - backend_str=backend_str)
101 + backend_str=backend_str,
102 + vocab_type=vocab_type)
103
104 return cls._cache[tokenizer_hash]
105
@@ -98,11 +118,30 @@ class GrammarCompilerCache:
118 cache_key = str(config.tokenizer_hash)
119
120 if cache_key not in cls._cache:
101 - assert config.encoded_vocab is not None
102 - tokenizer_info = xgr.TokenizerInfo._create_from_handle(
103 - xgr_core.TokenizerInfo.from_huggingface(
104 - config.encoded_vocab, config.backend_str,
105 - config.vocab_size, config.stop_token_ids))
121 + assert config.tokenizer_data is not None
122 + assert config.tokenizer_data.encoded_vocab is not None
123 +
124 + config_data = config.tokenizer_data
125 +
126 + # In TokenizerDataCache.get_tokenizer_data, a serializable
127 + # tokenizer_data is created and cached. This data is used to build
128 + # a tokenizer_info and create an xgrammar compiler.
129 + # - If tokenizer_data has backend_str set, use
130 + # xgr_core.TokenizerInfo.from_huggingface (a C++ bind).
131 + # - Otherwise, use the default constructor with vocab_type.
132 + # - xgr_core.TokenizerInfo.from_huggingface !=
133 + # xgr.TokenizerInfo.from_huggingface.
134 + if config_data.backend_str:
135 + tokenizer_info = xgr.TokenizerInfo._create_from_handle(
136 + xgr_core.TokenizerInfo.from_huggingface(
137 + config_data.encoded_vocab, config_data.backend_str,
138 + config.vocab_size, config_data.stop_token_ids))
139 + else:
140 + tokenizer_info = xgr.TokenizerInfo(
141 + config_data.encoded_vocab,
142 + config_data.vocab_type,
143 + vocab_size=config.vocab_size,
144 + stop_token_ids=config_data.stop_token_ids)
145 cls._cache[cache_key] = xgr.GrammarCompiler(
146 tokenizer_info, max_threads=config.max_threads)
147
@@ -118,10 +157,7 @@ class GrammarConfig:
157 grammar_str: str | None = None
158 json_object: bool | None = None
159 max_threads: int = 8
121 - # Only populated if tokenizer_hash not in cache
122 - encoded_vocab: list[str] | None = None
123 - stop_token_ids: list[int] | None = None
124 - backend_str: str | None = None
160 + tokenizer_data: TokenizerData | None = None
161
162 @classmethod
163 def from_guided_params(cls,
@@ -132,9 +168,6 @@ class GrammarConfig:
168
169 tokenizer_hash = hash(tokenizer)
170 tokenizer_data = TokenizerDataCache.get_tokenizer_data(tokenizer)
135 - encoded_vocab = tokenizer_data.encoded_vocab
136 - stop_token_ids = tokenizer_data.stop_token_ids
137 - backend_str = tokenizer_data.backend_str
171
172 if guided_params.json:
173 if not isinstance(guided_params.json, str):
@@ -152,11 +185,9 @@ class GrammarConfig:
185
186 return cls(json_str=json_str,
187 vocab_size=model_config.hf_text_config.vocab_size,
155 - encoded_vocab=encoded_vocab,
156 - stop_token_ids=stop_token_ids,
157 - backend_str=backend_str,
188 tokenizer_hash=tokenizer_hash,
159 - max_threads=max_threads)
189 + max_threads=max_threads,
190 + tokenizer_data=tokenizer_data)
191 elif guided_params.grammar:
192 # XGrammar only supports GBNF grammars, so we must convert Lark
193 if grammar_is_likely_lark(guided_params.grammar):
@@ -181,19 +212,17 @@ class GrammarConfig:
212
213 return cls(grammar_str=grammar_str,
214 vocab_size=model_config.hf_text_config.vocab_size,
184 - encoded_vocab=encoded_vocab,
185 - stop_token_ids=stop_token_ids,
186 - backend_str=backend_str,
215 tokenizer_hash=tokenizer_hash,
188 - max_threads=max_threads)
216 + max_threads=max_threads,
217 + tokenizer_data=tokenizer_data)
218 elif guided_params.json_object:
190 - return cls(json_object=True,
191 - vocab_size=model_config.hf_text_config.vocab_size,
192 - encoded_vocab=encoded_vocab,
193 - stop_token_ids=stop_token_ids,
194 - backend_str=backend_str,
195 - tokenizer_hash=tokenizer_hash,
196 - max_threads=max_threads)
219 + return cls(
220 + json_object=True,
221 + vocab_size=model_config.hf_text_config.vocab_size,
222 + tokenizer_hash=tokenizer_hash,
223 + max_threads=max_threads,
224 + tokenizer_data=tokenizer_data,
225 + )
226 else:
227 raise ValueError(
228 "Currently only support JSON and EBNF grammar mode for xgrammar"
@@ -269,10 +298,14 @@ class XGrammarLogitsProcessor:
298 # fill_next_token_bitmask so we move it to the device of scores
299 device_type = scores.device.type
300 if device_type != "cuda":
272 - scores = scores.to("cpu")
301 + scores = scores.to("cpu").unsqueeze(0)
302 +
303 + # Note: In this method, if the tensors have different dimensions
304 + # on CPU device fails, but on GPU it runs without error. Hence the
305 + # unsqueeze above for scores, to match the token bitmask shape
306 xgr.apply_token_bitmask_inplace(scores,
307 self.token_bitmask.to(scores.device))
308 if device_type != "cuda":
276 - scores = scores.to(device_type)
309 + scores = scores.to(device_type).squeeze()
310
311 return scores
vllm/transformers_utils/tokenizer.py
+1 -1
@@ -132,7 +132,7 @@ def get_tokenizer(
132 if is_from_mistral_org and tokenizer_mode != "mistral":
133 warnings.warn(
134 'It is strongly recommended to run mistral models with '
135 - '`--tokenizer_mode "mistral"` to ensure correct '
135 + '`--tokenizer-mode "mistral"` to ensure correct '
136 'encoding and decoding.',
137 FutureWarning,
138 stacklevel=2)
vllm/transformers_utils/tokenizers/mistral.py
+4 -1
@@ -314,12 +314,15 @@ class MistralTokenizer:
314
315 if regular_tokens:
316 decoded_list.append(
317 - self.decode(regular_tokens)) # type: ignore
317 + self.tokenizer.decode(regular_tokens)) # type: ignore
318
319 decoded = ''.join(decoded_list)
320
321 return decoded
322
323 + # WARN: Outlines logits processors can overwrite this method.
324 + # See: guided_decoding/outlines_logits_processors.py::_adapt_tokenizer
325 + # for more.
326 def decode(self,
327 ids: Union[List[int], int],
328 skip_special_tokens: bool = True) -> str: