A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Misc][Bugfix] Disable guided decoding for mistral tokenizer (#8521)
Roger Wang committed
Sep 16, 2024 at 22:22 UTC
ee2bceaaa67bd2f420f62a924da5834a7c1c862b
1 file changed
+23
vllm/model_executor/guided_decoding/__init__.py
+23
@@ -6,6 +6,7 @@ from vllm.entrypoints.openai.protocol import (
6
from vllm.model_executor.guided_decoding.guided_fields import (
7
GuidedDecodingRequest)
8
from vllm.sampling_params import LogitsProcessor
9
+from vllm.transformers_utils.tokenizer import MistralTokenizer
10
11
12
async def get_guided_decoding_logits_processor(
@@ -15,12 +16,23 @@ async def get_guided_decoding_logits_processor(
16
request = _adapt_request_for_tool_use(request)
17
18
if guided_decoding_backend == 'outlines':
19
+ if isinstance(tokenizer, MistralTokenizer):
20
+ raise NotImplementedError(
21
+ "Guided decoding with 'outlines' is currently not supported "
22
+ "for Mistral tokenizer. Please consider contributing to the "
23
+ "'outlines' project if you are interested in this feature.")
24
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
25
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
26
get_outlines_guided_decoding_logits_processor)
27
return await get_outlines_guided_decoding_logits_processor(
28
request, tokenizer)
29
if guided_decoding_backend == 'lm-format-enforcer':
30
+ if isinstance(tokenizer, MistralTokenizer):
31
+ raise NotImplementedError(
32
+ "Guided decoding with 'lm-format-enforcer' is currently not "
33
+ "supported for Mistral tokenizer. Please consider contributing "
34
+ "to the 'lm-format-enforcer' project if you are interested "
35
+ "in this feature.")
36
from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
37
get_lm_format_enforcer_guided_decoding_logits_processor)
38
return await get_lm_format_enforcer_guided_decoding_logits_processor(
@@ -37,12 +49,23 @@ def get_local_guided_decoding_logits_processor(
49
# request = _adapt_request_for_tool_use(request)
50
51
if guided_decoding_backend == 'outlines':
52
+ if isinstance(tokenizer, MistralTokenizer):
53
+ raise NotImplementedError(
54
+ "Guided decoding with 'outlines' is currently not supported "
55
+ "for Mistral tokenizer. Please consider contributing to the "
56
+ "'outlines' project if you are interested in this feature.")
57
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
58
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
59
get_local_outlines_guided_decoding_logits_processor)
60
return get_local_outlines_guided_decoding_logits_processor(
61
guided_options, tokenizer)
62
if guided_decoding_backend == 'lm-format-enforcer':
63
+ if isinstance(tokenizer, MistralTokenizer):
64
+ raise NotImplementedError(
65
+ "Guided decoding with 'lm-format-enforcer' is currently not "
66
+ "supported for Mistral tokenizer. Please consider contributing "
67
+ "to the 'lm-format-enforcer' project if you are interested "
68
+ "in this feature.")
69
from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
70
get_local_lm_format_enforcer_guided_decoding_logits_processor)
71
return get_local_lm_format_enforcer_guided_decoding_logits_processor(