A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
Support for guided decoding for offline LLM (#6878)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Yihuan Bu committed
Aug 3, 2024 at 23:12 UTC
654bc5ca49bde0969bc95e4b1dbe7fabbb8f631c
9 files changed
+352
-12
docs/source/conf.py
+1
@@ -111,6 +111,7 @@ autodoc_mock_imports = [
111
"tqdm",
112
"tensorizer",
113
"pynvml",
114
+ "outlines",
115
]
116
117
for mock_target in autodoc_mock_imports:
tests/entrypoints/conftest.py
renamed
+21
-1
@@ -1,6 +1,26 @@
1
import pytest
2
3
4
+@pytest.fixture
5
+def sample_prompts():
6
+ return [
7
+ "Hello, my name is",
8
+ "The president of the United States is",
9
+ "The capital of France is",
10
+ "The future of AI is",
11
+ ]
12
+
13
+
14
+@pytest.fixture
15
+def sample_token_ids():
16
+ return [
17
+ [0],
18
+ [0, 1],
19
+ [0, 2, 1],
20
+ [0, 3, 1, 2],
21
+ ]
22
+
23
+
24
@pytest.fixture
25
def sample_regex():
26
return (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
@@ -66,4 +86,4 @@ column: "col_1" | "col_2"
86
table: "table_1" | "table_2"
87
condition: column "=" number
88
number: "1" | "2"
69
-""")
\ No newline at end of file
89
+""")
tests/entrypoints/llm/test_guided_generate.py
new
+142
@@ -0,0 +1,142 @@
1
+import json
2
+import re
3
+import weakref
4
+
5
+import jsonschema
6
+import pytest
7
+
8
+from vllm.entrypoints.llm import LLM
9
+from vllm.outputs import RequestOutput
10
+from vllm.sampling_params import SamplingParams
11
+
12
+from ...conftest import cleanup
13
+
14
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
15
+
16
+
17
+@pytest.fixture(scope="module")
18
+def llm():
19
+ # pytest caches the fixture so we use weakref.proxy to
20
+ # enable garbage collection
21
+ llm = LLM(model=MODEL_NAME, max_model_len=1024)
22
+
23
+ with llm.deprecate_legacy_api():
24
+ yield weakref.proxy(llm)
25
+ del llm
26
+ cleanup()
27
+
28
+
29
+@pytest.mark.skip_global_cleanup
30
+def test_guided_regex(sample_regex, llm):
31
+ sampling_params = SamplingParams(
32
+ temperature=0.8,
33
+ top_p=0.95,
34
+ )
35
+ outputs = llm.generate(
36
+ prompts=[
37
+ f"Give an example IPv4 address with this regex: {sample_regex}"
38
+ ] * 2,
39
+ sampling_params=sampling_params,
40
+ use_tqdm=True,
41
+ guided_options_request=dict(guided_regex=sample_regex))
42
+
43
+ assert outputs is not None
44
+ for output in outputs:
45
+ assert output is not None
46
+ assert isinstance(output, RequestOutput)
47
+ prompt = output.prompt
48
+ generated_text = output.outputs[0].text
49
+ print(generated_text)
50
+ assert generated_text is not None
51
+ assert re.fullmatch(sample_regex, generated_text) is not None
52
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
53
+
54
+
55
+@pytest.mark.skip_global_cleanup
56
+def test_guided_json_completion(sample_json_schema, llm):
57
+ sampling_params = SamplingParams(
58
+ temperature=1.0,
59
+ max_tokens=1000,
60
+ )
61
+ outputs = llm.generate(
62
+ prompts=[
63
+ f"Give an example JSON for an employee profile "
64
+ f"that fits this schema: {sample_json_schema}"
65
+ ] * 2,
66
+ sampling_params=sampling_params,
67
+ use_tqdm=True,
68
+ guided_options_request=dict(guided_json=sample_json_schema))
69
+
70
+ assert outputs is not None
71
+
72
+ for output in outputs:
73
+ assert output is not None
74
+ assert isinstance(output, RequestOutput)
75
+ prompt = output.prompt
76
+
77
+ generated_text = output.outputs[0].text
78
+ assert generated_text is not None
79
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
80
+ output_json = json.loads(generated_text)
81
+ jsonschema.validate(instance=output_json, schema=sample_json_schema)
82
+
83
+
84
+@pytest.mark.skip_global_cleanup
85
+def test_guided_choice_completion(sample_guided_choice, llm):
86
+ sampling_params = SamplingParams(
87
+ temperature=0.8,
88
+ top_p=0.95,
89
+ )
90
+ outputs = llm.generate(
91
+ prompts="The best language for type-safe systems programming is ",
92
+ sampling_params=sampling_params,
93
+ use_tqdm=True,
94
+ guided_options_request=dict(guided_choice=sample_guided_choice))
95
+
96
+ assert outputs is not None
97
+ for output in outputs:
98
+ assert output is not None
99
+ assert isinstance(output, RequestOutput)
100
+ prompt = output.prompt
101
+ generated_text = output.outputs[0].text
102
+ print(generated_text)
103
+ assert generated_text is not None
104
+ assert generated_text in sample_guided_choice
105
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
106
+
107
+
108
+@pytest.mark.skip_global_cleanup
109
+def test_guided_grammar(sample_sql_statements, llm):
110
+
111
+ sampling_params = SamplingParams(
112
+ temperature=0.8,
113
+ top_p=0.95,
114
+ max_tokens=1000,
115
+ )
116
+ outputs = llm.generate(
117
+ prompts=("Generate a sql state that select col_1 from "
118
+ "table_1 where it is equals to 1"),
119
+ sampling_params=sampling_params,
120
+ use_tqdm=True,
121
+ guided_options_request=dict(guided_grammar=sample_sql_statements))
122
+
123
+ assert outputs is not None
124
+ for output in outputs:
125
+ assert output is not None
126
+ assert isinstance(output, RequestOutput)
127
+ prompt = output.prompt
128
+
129
+ generated_text = output.outputs[0].text
130
+ assert generated_text is not None
131
+ # use Lark to parse the output, and make sure it's a valid parse tree
132
+ from lark import Lark
133
+ parser = Lark(sample_sql_statements)
134
+ parser.parse(generated_text)
135
+
136
+ # remove spaces for comparison b/c we removed them in the grammar
137
+ ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(
138
+ " ", "")
139
+
140
+ assert generated_text.strip() == ground_truth
141
+
142
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
vllm/entrypoints/llm.py
+43
-1
@@ -10,6 +10,9 @@ from vllm.inputs import (PromptInputs, TextPrompt, TokensPrompt,
10
parse_and_batch_prompt)
11
from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
13
+from vllm.model_executor.guided_decoding import (
14
+ GuidedDecodingRequest, get_local_guided_decoding_logits_processor)
15
+from vllm.model_executor.guided_decoding.guided_fields import LLMGuidedOptions
16
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
17
from vllm.pooling_params import PoolingParams
18
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -262,6 +265,8 @@ class LLM:
265
use_tqdm: bool = True,
266
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
267
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
268
+ guided_options_request: Optional[Union[LLMGuidedOptions,
269
+ GuidedDecodingRequest]] = None
270
) -> List[RequestOutput]:
271
"""Generates the completions for the input prompts.
272
@@ -303,6 +308,14 @@ class LLM:
308
else:
309
inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
310
311
+ if isinstance(guided_options_request, dict):
312
+ if len(guided_options_request) > 1:
313
+ raise ValueError(
314
+ "You can only use one guided decoding but multiple is "
315
+ f"specified: {guided_options_request}")
316
+ guided_options_request = GuidedDecodingRequest(
317
+ **guided_options_request)
318
+
319
if sampling_params is None:
320
# Use default sampling params.
321
sampling_params = SamplingParams()
@@ -311,7 +324,8 @@ class LLM:
324
inputs=inputs,
325
params=sampling_params,
326
lora_request=lora_request,
314
- prompt_adapter_request=prompt_adapter_request)
327
+ prompt_adapter_request=prompt_adapter_request,
328
+ guided_options=guided_options_request)
329
330
outputs = self._run_engine(use_tqdm=use_tqdm)
331
return LLMEngine.validate_outputs(outputs, RequestOutput)
@@ -508,6 +522,7 @@ class LLM:
522
Sequence[PoolingParams]],
523
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
524
prompt_adapter_request: Optional[PromptAdapterRequest],
525
+ guided_options: Optional[GuidedDecodingRequest] = None,
526
) -> None:
527
if isinstance(inputs, (str, dict)):
528
# Convert a single prompt to a list.
@@ -523,6 +538,15 @@ class LLM:
538
raise ValueError("The lengths of prompts and lora_request "
539
"must be the same.")
540
541
+ if isinstance(params, list):
542
+ params = [
543
+ self._add_guided_processor(param, guided_options)
544
+ if isinstance(param, SamplingParams) else param
545
+ for param in params
546
+ ]
547
+ elif isinstance(params, SamplingParams):
548
+ params = self._add_guided_processor(params, guided_options)
549
+
550
# Add requests to the engine.
551
for i, request_inputs in enumerate(inputs):
552
self._add_request(
@@ -548,6 +572,24 @@ class LLM:
572
lora_request=lora_request,
573
prompt_adapter_request=prompt_adapter_request)
574
575
+ def _add_guided_processor(
576
+ self,
577
+ params: SamplingParams,
578
+ guided_options: Optional[GuidedDecodingRequest] = None):
579
+ if guided_options:
580
+ if guided_options.guided_decoding_backend is None:
581
+ decoding_config = self.llm_engine.get_decoding_config()
582
+ guided_options.guided_decoding_backend = (
583
+ decoding_config.guided_decoding_backend)
584
+ guided_logits_processor = get_local_guided_decoding_logits_processor( #noqa
585
+ guided_options.guided_decoding_backend, guided_options,
586
+ self.get_tokenizer())
587
+ if guided_logits_processor:
588
+ if params.logits_processors is None:
589
+ params.logits_processors = []
590
+ params.logits_processors.append(guided_logits_processor)
591
+ return params
592
+
593
def _run_engine(
594
self, *, use_tqdm: bool
595
) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
vllm/entrypoints/openai/protocol.py
+20
-6
@@ -1,6 +1,7 @@
1
# Adapted from
2
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
3
import time
4
+from argparse import Namespace
5
from typing import Any, Dict, List, Literal, Optional, Union
6
7
import torch
@@ -14,6 +15,23 @@ from vllm.pooling_params import PoolingParams
15
from vllm.sampling_params import LogitsProcessor, SamplingParams
16
from vllm.utils import random_uuid
17
18
+# torch is mocked during docs generation,
19
+# so we have to provide the values as literals
20
+_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
21
+
22
+try:
23
+ from sphinx.ext.autodoc.mock import _MockModule
24
+
25
+ if isinstance(torch, _MockModule):
26
+ _LONG_INFO = _MOCK_LONG_INFO
27
+ else:
28
+ _LONG_INFO = torch.iinfo(torch.long)
29
+except ModuleNotFoundError:
30
+ _LONG_INFO = torch.iinfo(torch.long)
31
+
32
+assert _LONG_INFO.min == _MOCK_LONG_INFO.min
33
+assert _LONG_INFO.max == _MOCK_LONG_INFO.max
34
+
35
36
class OpenAIBaseModel(BaseModel):
37
# OpenAI API does not allow extra fields
@@ -108,9 +126,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
126
n: Optional[int] = 1
127
presence_penalty: Optional[float] = 0.0
128
response_format: Optional[ResponseFormat] = None
111
- seed: Optional[int] = Field(None,
112
- ge=torch.iinfo(torch.long).min,
113
- le=torch.iinfo(torch.long).max)
129
+ seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
130
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
131
stream: Optional[bool] = False
132
stream_options: Optional[StreamOptions] = None
@@ -327,9 +343,7 @@ class CompletionRequest(OpenAIBaseModel):
343
max_tokens: Optional[int] = 16
344
n: int = 1
345
presence_penalty: Optional[float] = 0.0
330
- seed: Optional[int] = Field(None,
331
- ge=torch.iinfo(torch.long).min,
332
- le=torch.iinfo(torch.long).max)
346
+ seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
347
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
348
stream: Optional[bool] = False
349
stream_options: Optional[StreamOptions] = None
vllm/model_executor/guided_decoding/__init__.py
+24
-2
@@ -3,9 +3,10 @@ from typing import Optional, Union
3
from vllm.entrypoints.openai.protocol import (
4
ChatCompletionNamedToolChoiceParam, ChatCompletionRequest,
5
CompletionRequest)
6
-from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import (
7
- get_lm_format_enforcer_guided_decoding_logits_processor)
6
+from vllm.model_executor.guided_decoding.guided_fields import (
7
+ GuidedDecodingRequest)
8
from vllm.model_executor.guided_decoding.outlines_decoding import (
9
+ get_local_outlines_guided_decoding_logits_processor,
10
get_outlines_guided_decoding_logits_processor)
11
from vllm.sampling_params import LogitsProcessor
12
@@ -20,6 +21,8 @@ async def get_guided_decoding_logits_processor(
21
return await get_outlines_guided_decoding_logits_processor(
22
request, tokenizer)
23
if guided_decoding_backend == 'lm-format-enforcer':
24
+ from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
25
+ get_lm_format_enforcer_guided_decoding_logits_processor)
26
return await get_lm_format_enforcer_guided_decoding_logits_processor(
27
request, tokenizer)
28
@@ -28,6 +31,25 @@ async def get_guided_decoding_logits_processor(
31
"Must be one of 'outlines, 'lm-format-enforcer'")
32
33
34
+def get_local_guided_decoding_logits_processor(
35
+ guided_decoding_backend: str, guided_options: GuidedDecodingRequest,
36
+ tokenizer) -> Optional[LogitsProcessor]:
37
+ # request = _adapt_request_for_tool_use(request)
38
+
39
+ if guided_decoding_backend == 'outlines':
40
+ return get_local_outlines_guided_decoding_logits_processor(
41
+ guided_options, tokenizer)
42
+ if guided_decoding_backend == 'lm-format-enforcer':
43
+ from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
44
+ get_local_lm_format_enforcer_guided_decoding_logits_processor)
45
+ return get_local_lm_format_enforcer_guided_decoding_logits_processor(
46
+ guided_options, tokenizer)
47
+
48
+ raise ValueError(
49
+ f"Unknown guided decoding backend '{guided_decoding_backend}'. "
50
+ "Must be one of 'outlines, 'lm-format-enforcer'")
51
+
52
+
53
def _adapt_request_for_tool_use(request: Union[CompletionRequest,
54
ChatCompletionRequest]):
55
# the legacy completion API does not support tool use
vllm/model_executor/guided_decoding/guided_fields.py
new
+38
@@ -0,0 +1,38 @@
1
+from dataclasses import dataclass
2
+from typing import Dict, List, Optional, TypedDict, Union
3
+
4
+from pydantic import BaseModel
5
+
6
+
7
+class LLMGuidedOptions(TypedDict, total=False):
8
+ guided_json: Union[Dict, BaseModel, str]
9
+ guided_regex: str
10
+ guided_choice: List[str]
11
+ guided_grammar: str
12
+ guided_decoding_backend: str
13
+ guided_whitespace_pattern: str
14
+ guided_json_object: bool
15
+
16
+
17
+@dataclass
18
+class GuidedDecodingRequest:
19
+ """One of the fields will be used to retrieve the logit processor."""
20
+ guided_json: Optional[Union[Dict, BaseModel, str]] = None
21
+ guided_regex: Optional[str] = None
22
+ guided_choice: Optional[List[str]] = None
23
+ guided_grammar: Optional[str] = None
24
+ guided_decoding_backend: Optional[str] = None
25
+ guided_whitespace_pattern: Optional[str] = None
26
+ guided_json_object: Optional[bool] = None
27
+
28
+ def __post_init__(self):
29
+ """Validate that some fields are mutually exclusive."""
30
+ guide_count = sum([
31
+ self.guided_json is not None, self.guided_regex is not None,
32
+ self.guided_choice is not None, self.guided_grammar is not None,
33
+ self.guided_json_object is not None
34
+ ])
35
+ if guide_count > 1:
36
+ raise ValueError(
37
+ "You can only use one kind of guided decoding but multiple are "
38
+ f"specified: {self.__dict__}")
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+39
@@ -12,7 +12,10 @@ from transformers import PreTrainedTokenizerBase
12
13
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14
CompletionRequest)
15
+from vllm.model_executor.guided_decoding.guided_fields import (
16
+ GuidedDecodingRequest)
17
from vllm.model_executor.guided_decoding.outlines_decoding import (
18
+ get_local_outlines_guided_decoding_logits_processor,
19
get_outlines_guided_decoding_logits_processor)
20
from vllm.sampling_params import LogitsProcessor
21
@@ -54,6 +57,42 @@ async def get_lm_format_enforcer_guided_decoding_logits_processor(
57
return logits_processor
58
59
60
+def get_local_lm_format_enforcer_guided_decoding_logits_processor(
61
+ guided_options: GuidedDecodingRequest,
62
+ tokenizer) -> Optional[LogitsProcessor]:
63
+ """
64
+ Given an OpenAI-compatible request, check for guided decoding parameters
65
+ and get the necessary logits processor for the given guide.
66
+ We cache logit processors by (guide, tokenizer), and on cache hit
67
+ we make a shallow copy to reuse the same underlying FSM.
68
+ """
69
+
70
+ tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
71
+ tokenizer)
72
+ character_level_parser: CharacterLevelParser
73
+ if guided_options.guided_json:
74
+ schema = _normalize_json_schema_object(guided_options.guided_json)
75
+ character_level_parser = JsonSchemaParser(schema)
76
+ elif guided_options.guided_choice:
77
+ character_level_parser = UnionParser(
78
+ [StringParser(choice) for choice in guided_options.guided_choice])
79
+ elif guided_options.guided_regex:
80
+ character_level_parser = RegexParser(guided_options.guided_regex)
81
+ elif guided_options.guided_grammar:
82
+ # CFG grammar not supported by LMFE, revert to outlines
83
+ return get_local_outlines_guided_decoding_logits_processor(
84
+ guided_options, tokenizer)
85
+ elif guided_options.guided_json_object:
86
+ # None means any json object
87
+ character_level_parser = JsonSchemaParser(None)
88
+ else:
89
+ return None
90
+
91
+ logits_processor = build_vllm_logits_processor(tokenizer_data,
92
+ character_level_parser)
93
+ return logits_processor
94
+
95
+
96
def _normalize_json_schema_object(schema: Union[str, dict, BaseModel]) -> dict:
97
if isinstance(schema, str):
98
return json_loads(schema)
vllm/model_executor/guided_decoding/outlines_decoding.py
+24
-2
@@ -10,6 +10,8 @@ from transformers import PreTrainedTokenizerBase
10
11
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
CompletionRequest)
13
+from vllm.model_executor.guided_decoding.guided_fields import (
14
+ GuidedDecodingRequest)
15
from vllm.model_executor.guided_decoding.outlines_logits_processors import (
16
CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor)
17
@@ -77,8 +79,27 @@ async def get_outlines_guided_decoding_logits_processor(
79
mode, request.guided_whitespace_pattern)
80
81
82
+def get_local_outlines_guided_decoding_logits_processor(
83
+ guided_options: GuidedDecodingRequest, tokenizer: PreTrainedTokenizerBase
84
+) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
85
+ None]:
86
+ """
87
+ Given an OpenAI-compatible request, check for guided decoding parameters
88
+ and get the necessary logits processor for the given guide.
89
+ We cache logit processors by (guide, tokenizer), and on cache hit
90
+ we make a shallow copy to reuse the same underlying FSM.
91
+ """
92
+ guide, mode = _get_guide_and_mode(guided_options)
93
+ if not guide or not mode:
94
+ return None
95
+
96
+ return _get_logits_processor(guide, tokenizer, mode,
97
+ guided_options.guided_whitespace_pattern)
98
+
99
+
100
def _get_guide_and_mode(
81
- request: Union[CompletionRequest, ChatCompletionRequest]
101
+ request: Union[CompletionRequest, ChatCompletionRequest,
102
+ GuidedDecodingRequest]
103
) -> Union[Tuple[str, GuidedDecodingMode], Tuple[None, None]]:
104
105
if request.guided_json:
@@ -102,7 +123,8 @@ def _get_guide_and_mode(
123
return choices_regex, GuidedDecodingMode.CHOICE
124
elif request.guided_grammar:
125
return request.guided_grammar, GuidedDecodingMode.GRAMMAR
105
- elif (request.response_format is not None
126
+ elif (not isinstance(request, GuidedDecodingRequest)
127
+ and request.response_format is not None
128
and request.response_format.type == "json_object"):
129
return JSON_GRAMMAR, GuidedDecodingMode.GRAMMAR
130
else: