@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 05434764cd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

LM Format Enforcer Guided Decoding Support (#3868)

Co-authored-by: Simon Mo <simon.mo@hey.com>

Noam Gat committed Apr 16, 2024 at 08:54 UTC 05434764cd99990035779cf9a4ed86623b528825
13 files changed +304 -87
requirements-common.txt
+1
@@ -11,6 +11,7 @@ uvicorn[standard]
11 pydantic >= 2.0 # Required for OpenAI server.
12 prometheus_client >= 0.18.0
13 tiktoken == 0.6.0 # Required for DBRX tokenizer
14 +lm-format-enforcer == 0.9.3
15 outlines == 0.0.34 # Requires torch >= 2.1.0
16 typing_extensions
17 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
tests/entrypoints/test_guided_processors.py
+39 -3
@@ -1,11 +1,14 @@
1 # This unit test should be moved to a new
2 # tests/test_guided_decoding directory.
3 -
3 +import pytest
4 import torch
5 from transformers import AutoTokenizer
6
7 -from vllm.model_executor.guided_logits_processors import (JSONLogitsProcessor,
8 - RegexLogitsProcessor)
7 +from vllm.entrypoints.openai.protocol import CompletionRequest
8 +from vllm.model_executor.guided_decoding import (
9 + get_guided_decoding_logits_processor)
10 +from vllm.model_executor.guided_decoding.outlines_logits_processors import (
11 + JSONLogitsProcessor, RegexLogitsProcessor)
12
13 TEST_SCHEMA = {
14 "type": "object",
@@ -73,3 +76,36 @@ def test_guided_logits_processors():
76 json_LP(token_ids, tensor)
77 assert tensor.shape == original_tensor.shape
78 assert not torch.allclose(tensor, original_tensor)
79 +
80 +
81 +@pytest.mark.asyncio
82 +@pytest.mark.parametrize("backend", ["outlines", "lm-format-enforcer"])
83 +async def test_guided_logits_processor_black_box(backend: str):
84 + tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
85 + token_ids = tokenizer.encode(
86 + f"Give an example IPv4 address with this regex: {TEST_REGEX}")
87 + regex_request = CompletionRequest(model='test',
88 + prompt=token_ids,
89 + guided_regex=TEST_REGEX)
90 + regex_lp = await get_guided_decoding_logits_processor(
91 + backend, regex_request, tokenizer)
92 + assert regex_lp is not None
93 + tensor = torch.rand(32000)
94 + original_tensor = torch.clone(tensor)
95 + tensor = regex_lp(token_ids, tensor)
96 + assert tensor.shape == original_tensor.shape
97 + assert not torch.allclose(tensor, original_tensor)
98 +
99 + token_ids = tokenizer.encode(
100 + f"Give an employee profile that fits this schema: {TEST_SCHEMA}")
101 + json_request = CompletionRequest(model='test',
102 + prompt=token_ids,
103 + guided_json=TEST_SCHEMA)
104 + json_lp = await get_guided_decoding_logits_processor(
105 + backend, json_request, tokenizer)
106 + assert json_lp is not None
107 + tensor = torch.rand(32000)
108 + original_tensor = torch.clone(tensor)
109 + tensor = json_lp(token_ids, tensor)
110 + assert tensor.shape == original_tensor.shape
111 + assert not torch.allclose(tensor, original_tensor)
tests/entrypoints/test_openai_server.py
+50 -19
@@ -506,7 +506,10 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
506 assert first_response != completion.choices[0].text
507
508
509 -async def test_guided_json_completion(server, client: openai.AsyncOpenAI):
509 +@pytest.mark.parametrize("guided_decoding_backend",
510 + ["outlines", "lm-format-enforcer"])
511 +async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
512 + guided_decoding_backend: str):
513 completion = await client.completions.create(
514 model=MODEL_NAME,
515 prompt=f"Give an example JSON for an employee profile "
@@ -514,7 +517,8 @@ async def test_guided_json_completion(server, client: openai.AsyncOpenAI):
517 n=3,
518 temperature=1.0,
519 max_tokens=500,
517 - extra_body=dict(guided_json=TEST_SCHEMA))
520 + extra_body=dict(guided_json=TEST_SCHEMA,
521 + guided_decoding_backend=guided_decoding_backend))
522
523 assert completion.id is not None
524 assert completion.choices is not None and len(completion.choices) == 3
@@ -524,7 +528,10 @@ async def test_guided_json_completion(server, client: openai.AsyncOpenAI):
528 jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
529
530
527 -async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
531 +@pytest.mark.parametrize("guided_decoding_backend",
532 + ["outlines", "lm-format-enforcer"])
533 +async def test_guided_json_chat(server, client: openai.AsyncOpenAI,
534 + guided_decoding_backend: str):
535 messages = [{
536 "role": "system",
537 "content": "you are a helpful assistant"
@@ -538,8 +545,9 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
545 chat_completion = await client.chat.completions.create(
546 model=MODEL_NAME,
547 messages=messages,
541 - max_tokens=500,
542 - extra_body=dict(guided_json=TEST_SCHEMA))
548 + max_tokens=1000,
549 + extra_body=dict(guided_json=TEST_SCHEMA,
550 + guided_decoding_backend=guided_decoding_backend))
551 message = chat_completion.choices[0].message
552 assert message.content is not None
553 json1 = json.loads(message.content)
@@ -555,8 +563,9 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
563 chat_completion = await client.chat.completions.create(
564 model=MODEL_NAME,
565 messages=messages,
558 - max_tokens=500,
559 - extra_body=dict(guided_json=TEST_SCHEMA))
566 + max_tokens=1000,
567 + extra_body=dict(guided_json=TEST_SCHEMA,
568 + guided_decoding_backend=guided_decoding_backend))
569 message = chat_completion.choices[0].message
570 assert message.content is not None
571 json2 = json.loads(message.content)
@@ -565,14 +574,18 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
574 assert json1["age"] != json2["age"]
575
576
568 -async def test_guided_regex_completion(server, client: openai.AsyncOpenAI):
577 +@pytest.mark.parametrize("guided_decoding_backend",
578 + ["outlines", "lm-format-enforcer"])
579 +async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
580 + guided_decoding_backend: str):
581 completion = await client.completions.create(
582 model=MODEL_NAME,
583 prompt=f"Give an example IPv4 address with this regex: {TEST_REGEX}",
584 n=3,
585 temperature=1.0,
586 max_tokens=20,
575 - extra_body=dict(guided_regex=TEST_REGEX))
587 + extra_body=dict(guided_regex=TEST_REGEX,
588 + guided_decoding_backend=guided_decoding_backend))
589
590 assert completion.id is not None
591 assert completion.choices is not None and len(completion.choices) == 3
@@ -581,7 +594,10 @@ async def test_guided_regex_completion(server, client: openai.AsyncOpenAI):
594 assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
595
596
584 -async def test_guided_regex_chat(server, client: openai.AsyncOpenAI):
597 +@pytest.mark.parametrize("guided_decoding_backend",
598 + ["outlines", "lm-format-enforcer"])
599 +async def test_guided_regex_chat(server, client: openai.AsyncOpenAI,
600 + guided_decoding_backend: str):
601 messages = [{
602 "role": "system",
603 "content": "you are a helpful assistant"
@@ -595,7 +611,8 @@ async def test_guided_regex_chat(server, client: openai.AsyncOpenAI):
611 model=MODEL_NAME,
612 messages=messages,
613 max_tokens=20,
598 - extra_body=dict(guided_regex=TEST_REGEX))
614 + extra_body=dict(guided_regex=TEST_REGEX,
615 + guided_decoding_backend=guided_decoding_backend))
616 ip1 = chat_completion.choices[0].message.content
617 assert ip1 is not None
618 assert re.fullmatch(TEST_REGEX, ip1) is not None
@@ -606,21 +623,26 @@ async def test_guided_regex_chat(server, client: openai.AsyncOpenAI):
623 model=MODEL_NAME,
624 messages=messages,
625 max_tokens=20,
609 - extra_body=dict(guided_regex=TEST_REGEX))
626 + extra_body=dict(guided_regex=TEST_REGEX,
627 + guided_decoding_backend=guided_decoding_backend))
628 ip2 = chat_completion.choices[0].message.content
629 assert ip2 is not None
630 assert re.fullmatch(TEST_REGEX, ip2) is not None
631 assert ip1 != ip2
632
633
616 -async def test_guided_choice_completion(server, client: openai.AsyncOpenAI):
634 +@pytest.mark.parametrize("guided_decoding_backend",
635 + ["outlines", "lm-format-enforcer"])
636 +async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
637 + guided_decoding_backend: str):
638 completion = await client.completions.create(
639 model=MODEL_NAME,
640 prompt="The best language for type-safe systems programming is ",
641 n=2,
642 temperature=1.0,
643 max_tokens=10,
623 - extra_body=dict(guided_choice=TEST_CHOICE))
644 + extra_body=dict(guided_choice=TEST_CHOICE,
645 + guided_decoding_backend=guided_decoding_backend))
646
647 assert completion.id is not None
648 assert completion.choices is not None and len(completion.choices) == 2
@@ -628,7 +650,10 @@ async def test_guided_choice_completion(server, client: openai.AsyncOpenAI):
650 assert completion.choices[i].text in TEST_CHOICE
651
652
631 -async def test_guided_choice_chat(server, client: openai.AsyncOpenAI):
653 +@pytest.mark.parametrize("guided_decoding_backend",
654 + ["outlines", "lm-format-enforcer"])
655 +async def test_guided_choice_chat(server, client: openai.AsyncOpenAI,
656 + guided_decoding_backend: str):
657 messages = [{
658 "role": "system",
659 "content": "you are a helpful assistant"
@@ -642,7 +667,8 @@ async def test_guided_choice_chat(server, client: openai.AsyncOpenAI):
667 model=MODEL_NAME,
668 messages=messages,
669 max_tokens=10,
645 - extra_body=dict(guided_choice=TEST_CHOICE))
670 + extra_body=dict(guided_choice=TEST_CHOICE,
671 + guided_decoding_backend=guided_decoding_backend))
672 choice1 = chat_completion.choices[0].message.content
673 assert choice1 in TEST_CHOICE
674
@@ -655,18 +681,23 @@ async def test_guided_choice_chat(server, client: openai.AsyncOpenAI):
681 model=MODEL_NAME,
682 messages=messages,
683 max_tokens=10,
658 - extra_body=dict(guided_choice=TEST_CHOICE))
684 + extra_body=dict(guided_choice=TEST_CHOICE,
685 + guided_decoding_backend=guided_decoding_backend))
686 choice2 = chat_completion.choices[0].message.content
687 assert choice2 in TEST_CHOICE
688 assert choice1 != choice2
689
690
664 -async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI):
691 +@pytest.mark.parametrize("guided_decoding_backend",
692 + ["outlines", "lm-format-enforcer"])
693 +async def test_guided_decoding_type_error(server, client: openai.AsyncOpenAI,
694 + guided_decoding_backend: str):
695 with pytest.raises(openai.BadRequestError):
696 _ = await client.completions.create(
697 model=MODEL_NAME,
698 prompt="Give an example JSON that fits this schema: 42",
669 - extra_body=dict(guided_json=42))
699 + extra_body=dict(guided_json=42,
700 + guided_decoding_backend=guided_decoding_backend))
701
702 messages = [{
703 "role": "system",
vllm/config.py
+21 -5
@@ -66,8 +66,8 @@ class ModelConfig:
66 weights. If None, we assume the model weights are not quantized.
67 quantization_param_path: Path to JSON file containing scaling factors.
68 Used to load KV cache scaling factors into the model when KV cache
69 - type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also
70 - be used to load activation and weight scaling factors when the
69 + type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also
70 + be used to load activation and weight scaling factors when the
71 model dtype is FP8_E4M3 on ROCm.
72 enforce_eager: Whether to enforce eager execution. If True, we will
73 disable CUDA graph and always execute the model in eager mode.
@@ -422,7 +422,7 @@ class CacheConfig:
422 @dataclass
423 class TokenizerPoolConfig:
424 """Configuration for the tokenizer pool.
425 -
425 +
426 Args:
427 pool_size: Number of tokenizer workers in the pool.
428 pool_type: Type of the pool.
@@ -446,9 +446,9 @@ class TokenizerPoolConfig:
446 tokenizer_pool_extra_config: Optional[Union[str, dict]]
447 ) -> Optional["TokenizerPoolConfig"]:
448 """Create a TokenizerPoolConfig from the given parameters.
449 -
449 +
450 If tokenizer_pool_size is 0, return None.
451 -
451 +
452 Args:
453 tokenizer_pool_size: Number of tokenizer workers in the pool.
454 tokenizer_pool_type: Type of the pool.
@@ -1079,6 +1079,21 @@ def _get_and_verify_max_len(
1079 return int(max_model_len)
1080
1081
1082 +@dataclass
1083 +class DecodingConfig:
1084 + """Dataclass which contains the decoding strategy of the engine"""
1085 +
1086 + # Which guided decoding algo to use. 'outlines' / 'lm-format-enforcer'
1087 + guided_decoding_backend: str = 'outlines'
1088 +
1089 + def __post_init__(self):
1090 + valid_guided_backends = ['outlines', 'lm-format-enforcer']
1091 + backend = self.guided_decoding_backend
1092 + if backend not in valid_guided_backends:
1093 + raise ValueError(f"Invalid guided_decoding_backend '{backend},"
1094 + f"must be one of {valid_guided_backends}")
1095 +
1096 +
1097 @dataclass(frozen=True)
1098 class EngineConfig:
1099 """Dataclass which contains all engine-related configuration. This
@@ -1093,6 +1108,7 @@ class EngineConfig:
1108 lora_config: Optional[LoRAConfig]
1109 vision_language_config: Optional[VisionLanguageConfig]
1110 speculative_config: Optional[SpeculativeConfig]
1111 + decoding_config: Optional[DecodingConfig]
1112 tensorizer_config: Optional[TensorizerConfig]
1113
1114 def __post_init__(self):
vllm/engine/arg_utils.py
+15 -3
@@ -5,9 +5,9 @@ import os
5 from dataclasses import dataclass
6 from typing import BinaryIO, Optional, Union
7
8 -from vllm.config import (CacheConfig, DeviceConfig, EngineConfig, LoRAConfig,
9 - ModelConfig, ParallelConfig, SchedulerConfig,
10 - SpeculativeConfig, TensorizerConfig,
8 +from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
9 + EngineConfig, LoRAConfig, ModelConfig, ParallelConfig,
10 + SchedulerConfig, SpeculativeConfig, TensorizerConfig,
11 TokenizerPoolConfig, VisionLanguageConfig)
12 from vllm.model_executor.tensorizer_loader import TensorizerArgs
13 from vllm.utils import str_to_int_tuple
@@ -80,6 +80,7 @@ class EngineArgs:
80 scheduler_delay_factor: float = 0.0
81 enable_chunked_prefill: bool = False
82
83 + guided_decoding_backend: str = 'outlines'
84 # Speculative decoding configuration.
85 speculative_model: Optional[str] = None
86 num_speculative_tokens: Optional[int] = None
@@ -200,6 +201,13 @@ class EngineArgs:
201 default=EngineArgs.max_model_len,
202 help='model context length. If unspecified, '
203 'will be automatically derived from the model.')
204 + parser.add_argument(
205 + '--guided-decoding-backend',
206 + type=str,
207 + default='outlines',
208 + choices=['outlines', 'lm-format-enforcer'],
209 + help='Which engine will be used for guided decoding'
210 + ' (JSON schema / regex etc)')
211 # Parallel arguments
212 parser.add_argument('--worker-use-ray',
213 action='store_true',
@@ -511,6 +519,9 @@ class EngineArgs:
519 else:
520 vision_language_config = None
521
522 + decoding_config = DecodingConfig(
523 + guided_decoding_backend=self.guided_decoding_backend)
524 +
525 return EngineConfig(model_config=model_config,
526 cache_config=cache_config,
527 parallel_config=parallel_config,
@@ -519,6 +530,7 @@ class EngineArgs:
530 lora_config=lora_config,
531 vision_language_config=vision_language_config,
532 speculative_config=speculative_config,
533 + decoding_config=decoding_config,
534 tensorizer_config=tensorizer_config)
535
536
vllm/engine/llm_engine.py
+7 -3
@@ -4,9 +4,10 @@ from typing import Iterable, List, Optional, Tuple, Type, Union
4 from transformers import PreTrainedTokenizer
5
6 import vllm
7 -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
8 - ParallelConfig, SchedulerConfig, SpeculativeConfig,
9 - TensorizerConfig, VisionLanguageConfig)
7 +from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoRAConfig,
8 + ModelConfig, ParallelConfig, SchedulerConfig,
9 + SpeculativeConfig, TensorizerConfig,
10 + VisionLanguageConfig)
11 from vllm.core.scheduler import Scheduler, SchedulerOutputs
12 from vllm.engine.arg_utils import EngineArgs
13 from vllm.engine.metrics import StatLogger, Stats
@@ -74,6 +75,7 @@ class LLMEngine:
75 lora_config: Optional[LoRAConfig],
76 vision_language_config: Optional[VisionLanguageConfig],
77 speculative_config: Optional[SpeculativeConfig],
78 + decoding_config: Optional[DecodingConfig],
79 tensorizer_config: Optional[TensorizerConfig],
80 executor_class: Type[ExecutorBase],
81 log_stats: bool,
@@ -100,6 +102,7 @@ class LLMEngine:
102 f"kv_cache_dtype={cache_config.cache_dtype}, "
103 f"quantization_param_path={model_config.quantization_param_path}, "
104 f"device_config={device_config.device}, "
105 + f"decoding_config={decoding_config!r}, "
106 f"seed={model_config.seed})")
107 # TODO(woosuk): Print more configs in debug mode.
108
@@ -111,6 +114,7 @@ class LLMEngine:
114 self.scheduler_config = scheduler_config
115 self.device_config = device_config
116 self.speculative_config = speculative_config
117 + self.decoding_config = decoding_config or DecodingConfig()
118 self.tensorizer_config = tensorizer_config
119 self.log_stats = log_stats
120
vllm/entrypoints/openai/protocol.py
+12
@@ -133,6 +133,12 @@ class ChatCompletionRequest(BaseModel):
133 description=(
134 "If specified, the output will follow the context free grammar."),
135 )
136 + guided_decoding_backend: Optional[str] = Field(
137 + default=None,
138 + description=(
139 + "If specified, will override the default guided decoding backend "
140 + "of the server for this specific request. If set, must be either "
141 + "'outlines' / 'lm-format-enforcer'"))
142
143 # doc: end-chat-completion-extra-params
144
@@ -265,6 +271,12 @@ class CompletionRequest(BaseModel):
271 description=(
272 "If specified, the output will follow the context free grammar."),
273 )
274 + guided_decoding_backend: Optional[str] = Field(
275 + default=None,
276 + description=(
277 + "If specified, will override the default guided decoding backend "
278 + "of the server for this specific request. If set, must be one of "
279 + "'outlines' / 'lm-format-enforcer'"))
280
281 # doc: end-completion-extra-params
282
vllm/entrypoints/openai/serving_chat.py
+5 -1
@@ -68,9 +68,13 @@ class OpenAIServingChat(OpenAIServing):
68 request, prompt=prompt)
69 sampling_params = request.to_sampling_params()
70 lora_request = self._maybe_get_lora(request)
71 + decoding_config = self.engine.engine.decoding_config
72 + guided_decoding_backend = request.guided_decoding_backend \
73 + or decoding_config.guided_decoding_backend
74 guided_decode_logits_processor = (
75 await get_guided_decoding_logits_processor(
73 - request, await self.engine.get_tokenizer()))
76 + guided_decoding_backend, request, await
77 + self.engine.get_tokenizer()))
78 if guided_decode_logits_processor:
79 if sampling_params.logits_processors is None:
80 sampling_params.logits_processors = []
vllm/entrypoints/openai/serving_completion.py
+5 -1
@@ -88,9 +88,13 @@ class OpenAIServingCompletion(OpenAIServing):
88 try:
89 sampling_params = request.to_sampling_params()
90 lora_request = self._maybe_get_lora(request)
91 + decoding_config = self.engine.engine.decoding_config
92 + guided_decoding_backend = request.guided_decoding_backend \
93 + or decoding_config.guided_decoding_backend
94 guided_decode_logit_processor = (
95 await get_guided_decoding_logits_processor(
93 - request, await self.engine.get_tokenizer()))
96 + guided_decoding_backend, request, await
97 + self.engine.get_tokenizer()))
98 if guided_decode_logit_processor is not None:
99 if sampling_params.logits_processors is None:
100 sampling_params.logits_processors = []
vllm/model_executor/guided_decoding/__init__.py new
+25
@@ -0,0 +1,25 @@
1 +from typing import Optional, Union
2 +
3 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
4 + CompletionRequest)
5 +from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import (
6 + get_lm_format_enforcer_guided_decoding_logits_processor)
7 +from vllm.model_executor.guided_decoding.outlines_decoding import (
8 + get_outlines_guided_decoding_logits_processor)
9 +from vllm.sampling_params import LogitsProcessor
10 +
11 +
12 +async def get_guided_decoding_logits_processor(
13 + guided_decoding_backend: str, request: Union[CompletionRequest,
14 + ChatCompletionRequest],
15 + tokenizer) -> Optional[LogitsProcessor]:
16 + if guided_decoding_backend == 'outlines':
17 + return await get_outlines_guided_decoding_logits_processor(
18 + request, tokenizer)
19 + if guided_decoding_backend == 'lm-format-enforcer':
20 + return await get_lm_format_enforcer_guided_decoding_logits_processor(
21 + request, tokenizer)
22 +
23 + raise ValueError(
24 + f"Unknown guided decoding backend '{guided_decoding_backend}'. "
25 + "Must be one of 'outlines, 'lm-format-enforcer'")
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py new
+69
@@ -0,0 +1,69 @@
1 +from functools import lru_cache
2 +from json import loads as json_loads
3 +from typing import Optional, Union
4 +
5 +from lmformatenforcer import (CharacterLevelParser, JsonSchemaParser,
6 + RegexParser, StringParser,
7 + TokenEnforcerTokenizerData, UnionParser)
8 +from lmformatenforcer.integrations.vllm import (
9 + build_vllm_logits_processor, build_vllm_token_enforcer_tokenizer_data)
10 +from pydantic import BaseModel
11 +from transformers import PreTrainedTokenizerBase
12 +
13 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14 + CompletionRequest)
15 +from vllm.model_executor.guided_decoding.outlines_decoding import (
16 + get_outlines_guided_decoding_logits_processor)
17 +from vllm.sampling_params import LogitsProcessor
18 +
19 +
20 +async def get_lm_format_enforcer_guided_decoding_logits_processor(
21 + request: Union[CompletionRequest, ChatCompletionRequest],
22 + tokenizer) -> Optional[LogitsProcessor]:
23 + """
24 + Given an OpenAI-compatible request, check for guided decoding parameters
25 + and get the necessary logits processor for the given guide.
26 + We cache logit processors by (guide, tokenizer), and on cache hit
27 + we make a shallow copy to reuse the same underlying FSM.
28 + """
29 +
30 + tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
31 + tokenizer)
32 + character_level_parser: CharacterLevelParser
33 + if request.guided_json:
34 + schema = _normalize_json_schema_object(request.guided_json)
35 + character_level_parser = JsonSchemaParser(schema)
36 + elif request.guided_choice:
37 + character_level_parser = UnionParser(
38 + [StringParser(choice) for choice in request.guided_choice])
39 + elif request.guided_regex:
40 + character_level_parser = RegexParser(request.guided_regex)
41 + elif request.guided_grammar:
42 + # CFG grammar not supported by LMFE, revert to outlines
43 + return await get_outlines_guided_decoding_logits_processor(
44 + request, tokenizer)
45 + elif (request.response_format is not None
46 + and request.response_format.type == "json_object"):
47 + character_level_parser = JsonSchemaParser(
48 + None) # None means any json object
49 + else:
50 + return None
51 +
52 + logits_processor = build_vllm_logits_processor(tokenizer_data,
53 + character_level_parser)
54 + return logits_processor
55 +
56 +
57 +def _normalize_json_schema_object(schema: Union[str, dict, BaseModel]) -> dict:
58 + if isinstance(schema, str):
59 + return json_loads(schema)
60 + if isinstance(schema, dict):
61 + return schema
62 + if isinstance(schema, BaseModel):
63 + return schema.model_json_schema()
64 +
65 +
66 +@lru_cache
67 +def _cached_build_vllm_token_enforcer_tokenizer_data(
68 + tokenizer: PreTrainedTokenizerBase) -> TokenEnforcerTokenizerData:
69 + return build_vllm_token_enforcer_tokenizer_data(tokenizer)
vllm/model_executor/guided_decoding/outlines_decoding.py renamed
+3 -4
@@ -12,9 +12,8 @@ from transformers import PreTrainedTokenizerBase
12
13 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14 CompletionRequest)
15 -from vllm.model_executor.guided_logits_processors import (CFGLogitsProcessor,
16 - JSONLogitsProcessor,
17 - RegexLogitsProcessor)
15 +from vllm.model_executor.guided_decoding.outlines_logits_processors import (
16 + CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor)
17
18
19 class GuidedDecodingMode(Enum):
@@ -54,7 +53,7 @@ pair : UNESCAPED_STRING ":" value
53 global_thread_pool = None # used for generating logits processor fsm
54
55
57 -async def get_guided_decoding_logits_processor(
56 +async def get_outlines_guided_decoding_logits_processor(
57 request: Union[CompletionRequest, ChatCompletionRequest],
58 tokenizer) -> Union[JSONLogitsProcessor, RegexLogitsProcessor]:
59 """
vllm/model_executor/guided_decoding/outlines_logits_processors.py renamed
+52 -48
@@ -13,9 +13,11 @@
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 +import copy
17 import json
18 import math
19 from collections import defaultdict
20 +from functools import lru_cache
21 from typing import Callable, DefaultDict, Dict, List, Optional, Union
22
23 import torch
@@ -27,50 +29,6 @@ from transformers import PreTrainedTokenizerBase
29
30 class BaseLogitsProcessor:
31
30 - def adapt_tokenizer(self, tokenizer: PreTrainedTokenizerBase):
31 - """Adapt vLLM's tokenizer to use to compile the FSM.
32 -
33 - The API of Outlines tokenizers is slightly different to that of
34 - `transformers`. The decoder of outlines, returns a list whereas
35 - the decode of vLLM returns an str. To sync the vLLM decoder with
36 - outlines internal api, the decoder should be adapted. In addition
37 - we need to handle the missing spaces to Llama's tokenizer to be
38 - able to compile FSMs for this model.
39 -
40 - """
41 - if getattr(tokenizer, "_outlines_adapted", False):
42 - return tokenizer
43 -
44 - tokenizer.vocabulary = tokenizer.get_vocab()
45 - tokenizer.special_tokens = set(tokenizer.all_special_tokens)
46 -
47 - def convert_token_to_string(token: str) -> str:
48 - from transformers.file_utils import SPIECE_UNDERLINE
49 -
50 - string = tokenizer.convert_tokens_to_string([token])
51 -
52 - # A hack to handle missing spaces to HF's Llama tokenizers
53 - if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>":
54 - return " " + string
55 -
56 - return string
57 -
58 - def change_decoder(
59 - decoder: Callable[[List[int]], str]
60 - ) -> Callable[[List[int]], List[str]]:
61 - """Sync vLLM's decoder with the outlines by returning list."""
62 -
63 - def new_decoder(inp_tokens: List[int]) -> List[str]:
64 - return [decoder(inp_tokens)]
65 -
66 - return new_decoder
67 -
68 - tokenizer.convert_token_to_string = convert_token_to_string
69 - tokenizer.decode = change_decoder(tokenizer.decode)
70 - setattr(tokenizer, "_outlines_adapted", True) # noqa: B010
71 -
72 - return tokenizer
73 -
32 def init_state(self):
33 """Initialize the FSM states."""
34 self.fsm_state: DefaultDict[int, int] = defaultdict(int)
@@ -78,7 +36,6 @@ class BaseLogitsProcessor:
36 def __call__(self, input_ids: List[int],
37 scores: torch.Tensor) -> torch.Tensor:
38 """Use the FSM to bias the logits before sampling the next token."""
81 -
39 seq_id = hash(tuple(input_ids))
40
41 if len(input_ids) == 0:
@@ -96,7 +53,6 @@ class BaseLogitsProcessor:
53 device=scores.device)
54 mask[allowed_tokens] = 0
55 scores.add_(mask)
99 -
56 return scores
57
58
@@ -113,7 +69,7 @@ class RegexLogitsProcessor(BaseLogitsProcessor):
69 The model's tokenizer
70
71 """
116 - tokenizer = self.adapt_tokenizer(tokenizer)
72 + tokenizer = _adapt_tokenizer(tokenizer)
73 fsm = RegexFSM(regex_string, tokenizer)
74 self.fsm = fsm
75
@@ -167,6 +123,54 @@ class CFGLogitsProcessor(BaseLogitsProcessor):
123 The model's tokenizer
124
125 """
170 - tokenizer = self.adapt_tokenizer(tokenizer)
126 + tokenizer = _adapt_tokenizer(tokenizer)
127 fsm = CFGFSM(cfg, tokenizer)
128 self.fsm = fsm
129 +
130 +
131 +@lru_cache
132 +def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):
133 + """Adapt vLLM's tokenizer to use to compile the FSM.
134 +
135 + The API of Outlines tokenizers is slightly different to that of
136 + `transformers`. The decoder of outlines, returns a list whereas
137 + the decode of vLLM returns an str. To sync the vLLM decoder with
138 + outlines internal api, the decoder should be adapted. In addition
139 + we need to handle the missing spaces to Llama's tokenizer to be
140 + able to compile FSMs for this model.
141 +
142 + """
143 + if getattr(tokenizer, "_outlines_adapted", False):
144 + return tokenizer
145 +
146 + tokenizer = copy.deepcopy(tokenizer)
147 +
148 + tokenizer.vocabulary = tokenizer.get_vocab()
149 + tokenizer.special_tokens = set(tokenizer.all_special_tokens)
150 +
151 + def convert_token_to_string(token: str) -> str:
152 + from transformers.file_utils import SPIECE_UNDERLINE
153 +
154 + string = tokenizer.convert_tokens_to_string([token])
155 +
156 + # A hack to handle missing spaces to HF's Llama tokenizers
157 + if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>":
158 + return " " + string
159 +
160 + return string
161 +
162 + def change_decoder(
163 + decoder: Callable[[List[int]],
164 + str]) -> Callable[[List[int]], List[str]]:
165 + """Sync vLLM's decoder with the outlines by returning list."""
166 +
167 + def new_decoder(inp_tokens: List[int]) -> List[str]:
168 + return [decoder(inp_tokens)]
169 +
170 + return new_decoder
171 +
172 + tokenizer.convert_token_to_string = convert_token_to_string
173 + tokenizer.decode = change_decoder(tokenizer.decode)
174 + setattr(tokenizer, "_outlines_adapted", True) # noqa: B010
175 +
176 + return tokenizer