@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 654bc5ca49
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

Support for guided decoding for offline LLM (#6878)

Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Yihuan Bu committed Aug 3, 2024 at 23:12 UTC 654bc5ca49bde0969bc95e4b1dbe7fabbb8f631c
9 files changed +352 -12
docs/source/conf.py
+1
@@ -111,6 +111,7 @@ autodoc_mock_imports = [
111 "tqdm",
112 "tensorizer",
113 "pynvml",
114 + "outlines",
115 ]
116
117 for mock_target in autodoc_mock_imports:
tests/entrypoints/conftest.py renamed
+21 -1
@@ -1,6 +1,26 @@
1 import pytest
2
3
4 +@pytest.fixture
5 +def sample_prompts():
6 + return [
7 + "Hello, my name is",
8 + "The president of the United States is",
9 + "The capital of France is",
10 + "The future of AI is",
11 + ]
12 +
13 +
14 +@pytest.fixture
15 +def sample_token_ids():
16 + return [
17 + [0],
18 + [0, 1],
19 + [0, 2, 1],
20 + [0, 3, 1, 2],
21 + ]
22 +
23 +
24 @pytest.fixture
25 def sample_regex():
26 return (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
@@ -66,4 +86,4 @@ column: "col_1" | "col_2"
86 table: "table_1" | "table_2"
87 condition: column "=" number
88 number: "1" | "2"
69 -""")
\ No newline at end of file
89 +""")
tests/entrypoints/llm/test_guided_generate.py new
+142
@@ -0,0 +1,142 @@
1 +import json
2 +import re
3 +import weakref
4 +
5 +import jsonschema
6 +import pytest
7 +
8 +from vllm.entrypoints.llm import LLM
9 +from vllm.outputs import RequestOutput
10 +from vllm.sampling_params import SamplingParams
11 +
12 +from ...conftest import cleanup
13 +
14 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
15 +
16 +
17 +@pytest.fixture(scope="module")
18 +def llm():
19 + # pytest caches the fixture so we use weakref.proxy to
20 + # enable garbage collection
21 + llm = LLM(model=MODEL_NAME, max_model_len=1024)
22 +
23 + with llm.deprecate_legacy_api():
24 + yield weakref.proxy(llm)
25 + del llm
26 + cleanup()
27 +
28 +
29 +@pytest.mark.skip_global_cleanup
30 +def test_guided_regex(sample_regex, llm):
31 + sampling_params = SamplingParams(
32 + temperature=0.8,
33 + top_p=0.95,
34 + )
35 + outputs = llm.generate(
36 + prompts=[
37 + f"Give an example IPv4 address with this regex: {sample_regex}"
38 + ] * 2,
39 + sampling_params=sampling_params,
40 + use_tqdm=True,
41 + guided_options_request=dict(guided_regex=sample_regex))
42 +
43 + assert outputs is not None
44 + for output in outputs:
45 + assert output is not None
46 + assert isinstance(output, RequestOutput)
47 + prompt = output.prompt
48 + generated_text = output.outputs[0].text
49 + print(generated_text)
50 + assert generated_text is not None
51 + assert re.fullmatch(sample_regex, generated_text) is not None
52 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
53 +
54 +
55 +@pytest.mark.skip_global_cleanup
56 +def test_guided_json_completion(sample_json_schema, llm):
57 + sampling_params = SamplingParams(
58 + temperature=1.0,
59 + max_tokens=1000,
60 + )
61 + outputs = llm.generate(
62 + prompts=[
63 + f"Give an example JSON for an employee profile "
64 + f"that fits this schema: {sample_json_schema}"
65 + ] * 2,
66 + sampling_params=sampling_params,
67 + use_tqdm=True,
68 + guided_options_request=dict(guided_json=sample_json_schema))
69 +
70 + assert outputs is not None
71 +
72 + for output in outputs:
73 + assert output is not None
74 + assert isinstance(output, RequestOutput)
75 + prompt = output.prompt
76 +
77 + generated_text = output.outputs[0].text
78 + assert generated_text is not None
79 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
80 + output_json = json.loads(generated_text)
81 + jsonschema.validate(instance=output_json, schema=sample_json_schema)
82 +
83 +
84 +@pytest.mark.skip_global_cleanup
85 +def test_guided_choice_completion(sample_guided_choice, llm):
86 + sampling_params = SamplingParams(
87 + temperature=0.8,
88 + top_p=0.95,
89 + )
90 + outputs = llm.generate(
91 + prompts="The best language for type-safe systems programming is ",
92 + sampling_params=sampling_params,
93 + use_tqdm=True,
94 + guided_options_request=dict(guided_choice=sample_guided_choice))
95 +
96 + assert outputs is not None
97 + for output in outputs:
98 + assert output is not None
99 + assert isinstance(output, RequestOutput)
100 + prompt = output.prompt
101 + generated_text = output.outputs[0].text
102 + print(generated_text)
103 + assert generated_text is not None
104 + assert generated_text in sample_guided_choice
105 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
106 +
107 +
108 +@pytest.mark.skip_global_cleanup
109 +def test_guided_grammar(sample_sql_statements, llm):
110 +
111 + sampling_params = SamplingParams(
112 + temperature=0.8,
113 + top_p=0.95,
114 + max_tokens=1000,
115 + )
116 + outputs = llm.generate(
117 + prompts=("Generate a sql state that select col_1 from "
118 + "table_1 where it is equals to 1"),
119 + sampling_params=sampling_params,
120 + use_tqdm=True,
121 + guided_options_request=dict(guided_grammar=sample_sql_statements))
122 +
123 + assert outputs is not None
124 + for output in outputs:
125 + assert output is not None
126 + assert isinstance(output, RequestOutput)
127 + prompt = output.prompt
128 +
129 + generated_text = output.outputs[0].text
130 + assert generated_text is not None
131 + # use Lark to parse the output, and make sure it's a valid parse tree
132 + from lark import Lark
133 + parser = Lark(sample_sql_statements)
134 + parser.parse(generated_text)
135 +
136 + # remove spaces for comparison b/c we removed them in the grammar
137 + ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(
138 + " ", "")
139 +
140 + assert generated_text.strip() == ground_truth
141 +
142 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
vllm/entrypoints/llm.py
+43 -1
@@ -10,6 +10,9 @@ from vllm.inputs import (PromptInputs, TextPrompt, TokensPrompt,
10 parse_and_batch_prompt)
11 from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
13 +from vllm.model_executor.guided_decoding import (
14 + GuidedDecodingRequest, get_local_guided_decoding_logits_processor)
15 +from vllm.model_executor.guided_decoding.guided_fields import LLMGuidedOptions
16 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
17 from vllm.pooling_params import PoolingParams
18 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -262,6 +265,8 @@ class LLM:
265 use_tqdm: bool = True,
266 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
267 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
268 + guided_options_request: Optional[Union[LLMGuidedOptions,
269 + GuidedDecodingRequest]] = None
270 ) -> List[RequestOutput]:
271 """Generates the completions for the input prompts.
272
@@ -303,6 +308,14 @@ class LLM:
308 else:
309 inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
310
311 + if isinstance(guided_options_request, dict):
312 + if len(guided_options_request) > 1:
313 + raise ValueError(
314 + "You can only use one guided decoding but multiple is "
315 + f"specified: {guided_options_request}")
316 + guided_options_request = GuidedDecodingRequest(
317 + **guided_options_request)
318 +
319 if sampling_params is None:
320 # Use default sampling params.
321 sampling_params = SamplingParams()
@@ -311,7 +324,8 @@ class LLM:
324 inputs=inputs,
325 params=sampling_params,
326 lora_request=lora_request,
314 - prompt_adapter_request=prompt_adapter_request)
327 + prompt_adapter_request=prompt_adapter_request,
328 + guided_options=guided_options_request)
329
330 outputs = self._run_engine(use_tqdm=use_tqdm)
331 return LLMEngine.validate_outputs(outputs, RequestOutput)
@@ -508,6 +522,7 @@ class LLM:
522 Sequence[PoolingParams]],
523 lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
524 prompt_adapter_request: Optional[PromptAdapterRequest],
525 + guided_options: Optional[GuidedDecodingRequest] = None,
526 ) -> None:
527 if isinstance(inputs, (str, dict)):
528 # Convert a single prompt to a list.
@@ -523,6 +538,15 @@ class LLM:
538 raise ValueError("The lengths of prompts and lora_request "
539 "must be the same.")
540
541 + if isinstance(params, list):
542 + params = [
543 + self._add_guided_processor(param, guided_options)
544 + if isinstance(param, SamplingParams) else param
545 + for param in params
546 + ]
547 + elif isinstance(params, SamplingParams):
548 + params = self._add_guided_processor(params, guided_options)
549 +
550 # Add requests to the engine.
551 for i, request_inputs in enumerate(inputs):
552 self._add_request(
@@ -548,6 +572,24 @@ class LLM:
572 lora_request=lora_request,
573 prompt_adapter_request=prompt_adapter_request)
574
575 + def _add_guided_processor(
576 + self,
577 + params: SamplingParams,
578 + guided_options: Optional[GuidedDecodingRequest] = None):
579 + if guided_options:
580 + if guided_options.guided_decoding_backend is None:
581 + decoding_config = self.llm_engine.get_decoding_config()
582 + guided_options.guided_decoding_backend = (
583 + decoding_config.guided_decoding_backend)
584 + guided_logits_processor = get_local_guided_decoding_logits_processor( #noqa
585 + guided_options.guided_decoding_backend, guided_options,
586 + self.get_tokenizer())
587 + if guided_logits_processor:
588 + if params.logits_processors is None:
589 + params.logits_processors = []
590 + params.logits_processors.append(guided_logits_processor)
591 + return params
592 +
593 def _run_engine(
594 self, *, use_tqdm: bool
595 ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
vllm/entrypoints/openai/protocol.py
+20 -6
@@ -1,6 +1,7 @@
1 # Adapted from
2 # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
3 import time
4 +from argparse import Namespace
5 from typing import Any, Dict, List, Literal, Optional, Union
6
7 import torch
@@ -14,6 +15,23 @@ from vllm.pooling_params import PoolingParams
15 from vllm.sampling_params import LogitsProcessor, SamplingParams
16 from vllm.utils import random_uuid
17
18 +# torch is mocked during docs generation,
19 +# so we have to provide the values as literals
20 +_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
21 +
22 +try:
23 + from sphinx.ext.autodoc.mock import _MockModule
24 +
25 + if isinstance(torch, _MockModule):
26 + _LONG_INFO = _MOCK_LONG_INFO
27 + else:
28 + _LONG_INFO = torch.iinfo(torch.long)
29 +except ModuleNotFoundError:
30 + _LONG_INFO = torch.iinfo(torch.long)
31 +
32 +assert _LONG_INFO.min == _MOCK_LONG_INFO.min
33 +assert _LONG_INFO.max == _MOCK_LONG_INFO.max
34 +
35
36 class OpenAIBaseModel(BaseModel):
37 # OpenAI API does not allow extra fields
@@ -108,9 +126,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
126 n: Optional[int] = 1
127 presence_penalty: Optional[float] = 0.0
128 response_format: Optional[ResponseFormat] = None
111 - seed: Optional[int] = Field(None,
112 - ge=torch.iinfo(torch.long).min,
113 - le=torch.iinfo(torch.long).max)
129 + seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
130 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
131 stream: Optional[bool] = False
132 stream_options: Optional[StreamOptions] = None
@@ -327,9 +343,7 @@ class CompletionRequest(OpenAIBaseModel):
343 max_tokens: Optional[int] = 16
344 n: int = 1
345 presence_penalty: Optional[float] = 0.0
330 - seed: Optional[int] = Field(None,
331 - ge=torch.iinfo(torch.long).min,
332 - le=torch.iinfo(torch.long).max)
346 + seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
347 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
348 stream: Optional[bool] = False
349 stream_options: Optional[StreamOptions] = None
vllm/model_executor/guided_decoding/__init__.py
+24 -2
@@ -3,9 +3,10 @@ from typing import Optional, Union
3 from vllm.entrypoints.openai.protocol import (
4 ChatCompletionNamedToolChoiceParam, ChatCompletionRequest,
5 CompletionRequest)
6 -from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import (
7 - get_lm_format_enforcer_guided_decoding_logits_processor)
6 +from vllm.model_executor.guided_decoding.guided_fields import (
7 + GuidedDecodingRequest)
8 from vllm.model_executor.guided_decoding.outlines_decoding import (
9 + get_local_outlines_guided_decoding_logits_processor,
10 get_outlines_guided_decoding_logits_processor)
11 from vllm.sampling_params import LogitsProcessor
12
@@ -20,6 +21,8 @@ async def get_guided_decoding_logits_processor(
21 return await get_outlines_guided_decoding_logits_processor(
22 request, tokenizer)
23 if guided_decoding_backend == 'lm-format-enforcer':
24 + from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
25 + get_lm_format_enforcer_guided_decoding_logits_processor)
26 return await get_lm_format_enforcer_guided_decoding_logits_processor(
27 request, tokenizer)
28
@@ -28,6 +31,25 @@ async def get_guided_decoding_logits_processor(
31 "Must be one of 'outlines, 'lm-format-enforcer'")
32
33
34 +def get_local_guided_decoding_logits_processor(
35 + guided_decoding_backend: str, guided_options: GuidedDecodingRequest,
36 + tokenizer) -> Optional[LogitsProcessor]:
37 + # request = _adapt_request_for_tool_use(request)
38 +
39 + if guided_decoding_backend == 'outlines':
40 + return get_local_outlines_guided_decoding_logits_processor(
41 + guided_options, tokenizer)
42 + if guided_decoding_backend == 'lm-format-enforcer':
43 + from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
44 + get_local_lm_format_enforcer_guided_decoding_logits_processor)
45 + return get_local_lm_format_enforcer_guided_decoding_logits_processor(
46 + guided_options, tokenizer)
47 +
48 + raise ValueError(
49 + f"Unknown guided decoding backend '{guided_decoding_backend}'. "
50 + "Must be one of 'outlines, 'lm-format-enforcer'")
51 +
52 +
53 def _adapt_request_for_tool_use(request: Union[CompletionRequest,
54 ChatCompletionRequest]):
55 # the legacy completion API does not support tool use
vllm/model_executor/guided_decoding/guided_fields.py new
+38
@@ -0,0 +1,38 @@
1 +from dataclasses import dataclass
2 +from typing import Dict, List, Optional, TypedDict, Union
3 +
4 +from pydantic import BaseModel
5 +
6 +
7 +class LLMGuidedOptions(TypedDict, total=False):
8 + guided_json: Union[Dict, BaseModel, str]
9 + guided_regex: str
10 + guided_choice: List[str]
11 + guided_grammar: str
12 + guided_decoding_backend: str
13 + guided_whitespace_pattern: str
14 + guided_json_object: bool
15 +
16 +
17 +@dataclass
18 +class GuidedDecodingRequest:
19 + """One of the fields will be used to retrieve the logit processor."""
20 + guided_json: Optional[Union[Dict, BaseModel, str]] = None
21 + guided_regex: Optional[str] = None
22 + guided_choice: Optional[List[str]] = None
23 + guided_grammar: Optional[str] = None
24 + guided_decoding_backend: Optional[str] = None
25 + guided_whitespace_pattern: Optional[str] = None
26 + guided_json_object: Optional[bool] = None
27 +
28 + def __post_init__(self):
29 + """Validate that some fields are mutually exclusive."""
30 + guide_count = sum([
31 + self.guided_json is not None, self.guided_regex is not None,
32 + self.guided_choice is not None, self.guided_grammar is not None,
33 + self.guided_json_object is not None
34 + ])
35 + if guide_count > 1:
36 + raise ValueError(
37 + "You can only use one kind of guided decoding but multiple are "
38 + f"specified: {self.__dict__}")
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+39
@@ -12,7 +12,10 @@ from transformers import PreTrainedTokenizerBase
12
13 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14 CompletionRequest)
15 +from vllm.model_executor.guided_decoding.guided_fields import (
16 + GuidedDecodingRequest)
17 from vllm.model_executor.guided_decoding.outlines_decoding import (
18 + get_local_outlines_guided_decoding_logits_processor,
19 get_outlines_guided_decoding_logits_processor)
20 from vllm.sampling_params import LogitsProcessor
21
@@ -54,6 +57,42 @@ async def get_lm_format_enforcer_guided_decoding_logits_processor(
57 return logits_processor
58
59
60 +def get_local_lm_format_enforcer_guided_decoding_logits_processor(
61 + guided_options: GuidedDecodingRequest,
62 + tokenizer) -> Optional[LogitsProcessor]:
63 + """
64 + Given an OpenAI-compatible request, check for guided decoding parameters
65 + and get the necessary logits processor for the given guide.
66 + We cache logit processors by (guide, tokenizer), and on cache hit
67 + we make a shallow copy to reuse the same underlying FSM.
68 + """
69 +
70 + tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
71 + tokenizer)
72 + character_level_parser: CharacterLevelParser
73 + if guided_options.guided_json:
74 + schema = _normalize_json_schema_object(guided_options.guided_json)
75 + character_level_parser = JsonSchemaParser(schema)
76 + elif guided_options.guided_choice:
77 + character_level_parser = UnionParser(
78 + [StringParser(choice) for choice in guided_options.guided_choice])
79 + elif guided_options.guided_regex:
80 + character_level_parser = RegexParser(guided_options.guided_regex)
81 + elif guided_options.guided_grammar:
82 + # CFG grammar not supported by LMFE, revert to outlines
83 + return get_local_outlines_guided_decoding_logits_processor(
84 + guided_options, tokenizer)
85 + elif guided_options.guided_json_object:
86 + # None means any json object
87 + character_level_parser = JsonSchemaParser(None)
88 + else:
89 + return None
90 +
91 + logits_processor = build_vllm_logits_processor(tokenizer_data,
92 + character_level_parser)
93 + return logits_processor
94 +
95 +
96 def _normalize_json_schema_object(schema: Union[str, dict, BaseModel]) -> dict:
97 if isinstance(schema, str):
98 return json_loads(schema)
vllm/model_executor/guided_decoding/outlines_decoding.py
+24 -2
@@ -10,6 +10,8 @@ from transformers import PreTrainedTokenizerBase
10
11 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12 CompletionRequest)
13 +from vllm.model_executor.guided_decoding.guided_fields import (
14 + GuidedDecodingRequest)
15 from vllm.model_executor.guided_decoding.outlines_logits_processors import (
16 CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor)
17
@@ -77,8 +79,27 @@ async def get_outlines_guided_decoding_logits_processor(
79 mode, request.guided_whitespace_pattern)
80
81
82 +def get_local_outlines_guided_decoding_logits_processor(
83 + guided_options: GuidedDecodingRequest, tokenizer: PreTrainedTokenizerBase
84 +) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
85 + None]:
86 + """
87 + Given an OpenAI-compatible request, check for guided decoding parameters
88 + and get the necessary logits processor for the given guide.
89 + We cache logit processors by (guide, tokenizer), and on cache hit
90 + we make a shallow copy to reuse the same underlying FSM.
91 + """
92 + guide, mode = _get_guide_and_mode(guided_options)
93 + if not guide or not mode:
94 + return None
95 +
96 + return _get_logits_processor(guide, tokenizer, mode,
97 + guided_options.guided_whitespace_pattern)
98 +
99 +
100 def _get_guide_and_mode(
81 - request: Union[CompletionRequest, ChatCompletionRequest]
101 + request: Union[CompletionRequest, ChatCompletionRequest,
102 + GuidedDecodingRequest]
103 ) -> Union[Tuple[str, GuidedDecodingMode], Tuple[None, None]]:
104
105 if request.guided_json:
@@ -102,7 +123,8 @@ def _get_guide_and_mode(
123 return choices_regex, GuidedDecodingMode.CHOICE
124 elif request.guided_grammar:
125 return request.guided_grammar, GuidedDecodingMode.GRAMMAR
105 - elif (request.response_format is not None
126 + elif (not isinstance(request, GuidedDecodingRequest)
127 + and request.response_format is not None
128 and request.response_format.type == "json_object"):
129 return JSON_GRAMMAR, GuidedDecodingMode.GRAMMAR
130 else: