@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ee2bceaaa6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Misc][Bugfix] Disable guided decoding for mistral tokenizer (#8521)

Roger Wang committed Sep 16, 2024 at 22:22 UTC ee2bceaaa67bd2f420f62a924da5834a7c1c862b
1 file changed +23
vllm/model_executor/guided_decoding/__init__.py
+23
@@ -6,6 +6,7 @@ from vllm.entrypoints.openai.protocol import (
6 from vllm.model_executor.guided_decoding.guided_fields import (
7 GuidedDecodingRequest)
8 from vllm.sampling_params import LogitsProcessor
9 +from vllm.transformers_utils.tokenizer import MistralTokenizer
10
11
12 async def get_guided_decoding_logits_processor(
@@ -15,12 +16,23 @@ async def get_guided_decoding_logits_processor(
16 request = _adapt_request_for_tool_use(request)
17
18 if guided_decoding_backend == 'outlines':
19 + if isinstance(tokenizer, MistralTokenizer):
20 + raise NotImplementedError(
21 + "Guided decoding with 'outlines' is currently not supported "
22 + "for Mistral tokenizer. Please consider contributing to the "
23 + "'outlines' project if you are interested in this feature.")
24 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
25 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
26 get_outlines_guided_decoding_logits_processor)
27 return await get_outlines_guided_decoding_logits_processor(
28 request, tokenizer)
29 if guided_decoding_backend == 'lm-format-enforcer':
30 + if isinstance(tokenizer, MistralTokenizer):
31 + raise NotImplementedError(
32 + "Guided decoding with 'lm-format-enforcer' is currently not "
33 + "supported for Mistral tokenizer. Please consider contributing "
34 + "to the 'lm-format-enforcer' project if you are interested "
35 + "in this feature.")
36 from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
37 get_lm_format_enforcer_guided_decoding_logits_processor)
38 return await get_lm_format_enforcer_guided_decoding_logits_processor(
@@ -37,12 +49,23 @@ def get_local_guided_decoding_logits_processor(
49 # request = _adapt_request_for_tool_use(request)
50
51 if guided_decoding_backend == 'outlines':
52 + if isinstance(tokenizer, MistralTokenizer):
53 + raise NotImplementedError(
54 + "Guided decoding with 'outlines' is currently not supported "
55 + "for Mistral tokenizer. Please consider contributing to the "
56 + "'outlines' project if you are interested in this feature.")
57 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
58 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
59 get_local_outlines_guided_decoding_logits_processor)
60 return get_local_outlines_guided_decoding_logits_processor(
61 guided_options, tokenizer)
62 if guided_decoding_backend == 'lm-format-enforcer':
63 + if isinstance(tokenizer, MistralTokenizer):
64 + raise NotImplementedError(
65 + "Guided decoding with 'lm-format-enforcer' is currently not "
66 + "supported for Mistral tokenizer. Please consider contributing "
67 + "to the 'lm-format-enforcer' project if you are interested "
68 + "in this feature.")
69 from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
70 get_local_lm_format_enforcer_guided_decoding_logits_processor)
71 return get_local_lm_format_enforcer_guided_decoding_logits_processor(