A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Frontend][Core] Move guided decoding params into sampling params (#8252)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Joe Runde committed
Sep 30, 2024 at 19:34 UTC
062c89e7c9c6fa9fd7fb2d28fd50321c6f78f389
16 files changed
+441
-281
tests/entrypoints/llm/test_guided_generate.py
+43
-23
@@ -7,7 +7,7 @@ import pytest
7
8
from vllm.entrypoints.llm import LLM
9
from vllm.outputs import RequestOutput
10
-from vllm.sampling_params import SamplingParams
10
+from vllm.sampling_params import GuidedDecodingParams, SamplingParams
11
12
from ...conftest import cleanup
13
@@ -31,14 +31,12 @@ def test_guided_regex(sample_regex, llm):
31
sampling_params = SamplingParams(
32
temperature=0.8,
33
top_p=0.95,
34
- )
35
- outputs = llm.generate(
36
- prompts=[
37
- f"Give an example IPv4 address with this regex: {sample_regex}"
38
- ] * 2,
39
- sampling_params=sampling_params,
40
- use_tqdm=True,
41
- guided_options_request=dict(guided_regex=sample_regex))
34
+ guided_decoding=GuidedDecodingParams(regex=sample_regex))
35
+ outputs = llm.generate(prompts=[
36
+ f"Give an example IPv4 address with this regex: {sample_regex}"
37
+ ] * 2,
38
+ sampling_params=sampling_params,
39
+ use_tqdm=True)
40
41
assert outputs is not None
42
for output in outputs:
@@ -57,15 +55,13 @@ def test_guided_json_completion(sample_json_schema, llm):
55
sampling_params = SamplingParams(
56
temperature=1.0,
57
max_tokens=1000,
60
- )
61
- outputs = llm.generate(
62
- prompts=[
63
- f"Give an example JSON for an employee profile "
64
- f"that fits this schema: {sample_json_schema}"
65
- ] * 2,
66
- sampling_params=sampling_params,
67
- use_tqdm=True,
68
- guided_options_request=dict(guided_json=sample_json_schema))
58
+ guided_decoding=GuidedDecodingParams(json=sample_json_schema))
59
+ outputs = llm.generate(prompts=[
60
+ f"Give an example JSON for an employee profile "
61
+ f"that fits this schema: {sample_json_schema}"
62
+ ] * 2,
63
+ sampling_params=sampling_params,
64
+ use_tqdm=True)
65
66
assert outputs is not None
67
@@ -86,12 +82,11 @@ def test_guided_choice_completion(sample_guided_choice, llm):
82
sampling_params = SamplingParams(
83
temperature=0.8,
84
top_p=0.95,
89
- )
85
+ guided_decoding=GuidedDecodingParams(choice=sample_guided_choice))
86
outputs = llm.generate(
87
prompts="The best language for type-safe systems programming is ",
88
sampling_params=sampling_params,
93
- use_tqdm=True,
94
- guided_options_request=dict(guided_choice=sample_guided_choice))
89
+ use_tqdm=True)
90
91
assert outputs is not None
92
for output in outputs:
@@ -112,13 +107,13 @@ def test_guided_grammar(sample_sql_statements, llm):
107
temperature=0.8,
108
top_p=0.95,
109
max_tokens=1000,
115
- )
110
+ guided_decoding=GuidedDecodingParams(grammar=sample_sql_statements))
111
outputs = llm.generate(
112
prompts=("Generate a sql state that select col_1 from "
113
"table_1 where it is equals to 1"),
114
sampling_params=sampling_params,
115
use_tqdm=True,
121
- guided_options_request=dict(guided_grammar=sample_sql_statements))
116
+ )
117
118
assert outputs is not None
119
for output in outputs:
@@ -140,3 +135,28 @@ def test_guided_grammar(sample_sql_statements, llm):
135
assert generated_text.strip() == ground_truth
136
137
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
138
+
139
+
140
+@pytest.mark.skip_global_cleanup
141
+def test_guided_options_request_deprecation_warning(sample_regex, llm):
142
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
143
+
144
+ with pytest.warns(DeprecationWarning, match="guided_options_request"):
145
+ llm.generate(prompts="This should fail",
146
+ sampling_params=sampling_params,
147
+ use_tqdm=True,
148
+ guided_options_request=dict(guided_regex=sample_regex))
149
+
150
+
151
+@pytest.mark.skip_global_cleanup
152
+def test_validation_against_both_guided_decoding_options(sample_regex, llm):
153
+ sampling_params = SamplingParams(
154
+ temperature=0.8,
155
+ top_p=0.95,
156
+ guided_decoding=GuidedDecodingParams(regex=sample_regex))
157
+
158
+ with pytest.raises(ValueError, match="Cannot set both"):
159
+ llm.generate(prompts="This should fail",
160
+ sampling_params=sampling_params,
161
+ use_tqdm=True,
162
+ guided_options_request=dict(guided_regex=sample_regex))
tests/model_executor/conftest.py
new
+49
@@ -0,0 +1,49 @@
1
+import pytest
2
+
3
+
4
+@pytest.fixture
5
+def sample_regex():
6
+ return (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
7
+ r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
8
+
9
+
10
+@pytest.fixture
11
+def sample_json_schema():
12
+ return {
13
+ "type": "object",
14
+ "properties": {
15
+ "name": {
16
+ "type": "string"
17
+ },
18
+ "age": {
19
+ "type": "integer"
20
+ },
21
+ "skills": {
22
+ "type": "array",
23
+ "items": {
24
+ "type": "string",
25
+ "maxLength": 10
26
+ },
27
+ "minItems": 3
28
+ },
29
+ "work_history": {
30
+ "type": "array",
31
+ "items": {
32
+ "type": "object",
33
+ "properties": {
34
+ "company": {
35
+ "type": "string"
36
+ },
37
+ "duration": {
38
+ "type": "number"
39
+ },
40
+ "position": {
41
+ "type": "string"
42
+ }
43
+ },
44
+ "required": ["company", "position"]
45
+ }
46
+ }
47
+ },
48
+ "required": ["name", "age", "skills", "work_history"]
49
+ }
tests/model_executor/test_guided_processors.py
renamed
+24
-11
@@ -1,14 +1,12 @@
1
-# This unit test should be moved to a new
2
-# tests/test_guided_decoding directory.
1
import pytest
2
import torch
3
from transformers import AutoTokenizer
4
7
-from vllm.entrypoints.openai.protocol import CompletionRequest
5
from vllm.model_executor.guided_decoding import (
6
get_guided_decoding_logits_processor)
7
from vllm.model_executor.guided_decoding.outlines_logits_processors import (
8
JSONLogitsProcessor, RegexLogitsProcessor)
9
+from vllm.sampling_params import GuidedDecodingParams
10
11
12
def test_guided_logits_processors(sample_regex, sample_json_schema):
@@ -44,11 +42,9 @@ async def test_guided_logits_processor_black_box(backend: str, sample_regex,
42
tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
43
token_ids = tokenizer.encode(
44
f"Give an example IPv4 address with this regex: {sample_regex}")
47
- regex_request = CompletionRequest(model='test',
48
- prompt=token_ids,
49
- guided_regex=sample_regex)
45
+ regex_request = GuidedDecodingParams(regex=sample_regex, backend=backend)
46
regex_lp = await get_guided_decoding_logits_processor(
51
- backend, regex_request, tokenizer)
47
+ regex_request, tokenizer)
48
assert regex_lp is not None
49
tensor = torch.rand(32000)
50
original_tensor = torch.clone(tensor)
@@ -59,14 +55,31 @@ async def test_guided_logits_processor_black_box(backend: str, sample_regex,
55
token_ids = tokenizer.encode(
56
f"Give an employee profile that fits this schema: {sample_json_schema}"
57
)
62
- json_request = CompletionRequest(model='test',
63
- prompt=token_ids,
64
- guided_json=sample_json_schema)
58
+ json_request = GuidedDecodingParams(json=sample_json_schema,
59
+ backend=backend)
60
json_lp = await get_guided_decoding_logits_processor(
66
- backend, json_request, tokenizer)
61
+ json_request, tokenizer)
62
assert json_lp is not None
63
tensor = torch.rand(32000)
64
original_tensor = torch.clone(tensor)
65
tensor = json_lp(token_ids, tensor)
66
assert tensor.shape == original_tensor.shape
67
assert not torch.allclose(tensor, original_tensor)
68
+
69
+
70
+def test_multiple_guided_options_not_allowed(sample_json_schema, sample_regex):
71
+ with pytest.raises(ValueError,
72
+ match="You can only use one kind of guided"):
73
+ GuidedDecodingParams(json=sample_json_schema, regex=sample_regex)
74
+
75
+ with pytest.raises(ValueError,
76
+ match="You can only use one kind of guided"):
77
+ GuidedDecodingParams(json=sample_json_schema, json_object=True)
78
+
79
+ with pytest.raises(ValueError,
80
+ match="You can only use one kind of guided"):
81
+ GuidedDecodingParams(json=sample_json_schema, choice=["a", "b"])
82
+
83
+ with pytest.raises(ValueError,
84
+ match="You can only use one kind of guided"):
85
+ GuidedDecodingParams(json=sample_json_schema, grammar="test grammar")
vllm/engine/async_llm_engine.py
+44
@@ -20,6 +20,8 @@ from vllm.executor.ray_utils import initialize_ray_cluster
20
from vllm.inputs import PromptType
21
from vllm.logger import init_logger
22
from vllm.lora.request import LoRARequest
23
+from vllm.model_executor.guided_decoding import (
24
+ get_guided_decoding_logits_processor)
25
from vllm.model_executor.layers.sampler import SamplerOutput
26
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
27
from vllm.pooling_params import PoolingParams
@@ -477,6 +479,18 @@ class _AsyncLLMEngine(LLMEngine):
479
)
480
processed_inputs = self.input_processor(preprocessed_inputs)
481
482
+ if isinstance(params, SamplingParams) and \
483
+ params.guided_decoding is not None:
484
+ # Guided decoding has an async implementation for building logits
485
+ # processors in a separate threadpool.
486
+ # We want to invoke that here instead of using the blocking
487
+ # implementation in the LLMEngine
488
+ params = await build_guided_decoding_logits_processor_async(
489
+ sampling_params=params,
490
+ tokenizer=self.get_tokenizer(lora_request),
491
+ default_guided_backend=self.decoding_config.
492
+ guided_decoding_backend)
493
+
494
self._add_processed_request(
495
request_id=request_id,
496
processed_inputs=processed_inputs,
@@ -494,6 +508,36 @@ class _AsyncLLMEngine(LLMEngine):
508
self.model_executor.check_health()
509
510
511
+async def build_guided_decoding_logits_processor_async(
512
+ sampling_params: SamplingParams, tokenizer: AnyTokenizer,
513
+ default_guided_backend: str) -> SamplingParams:
514
+ """Constructs logits processors based on the guided_decoding,
515
+ logits_bias, and allowed_token_ids fields in sampling_params. Deletes
516
+ those fields and adds the constructed logits processors to the
517
+ logits_processors field. Modifies sampling params in-place and returns
518
+ the modified sampling params."""
519
+ if (guided_decoding := sampling_params.guided_decoding) is None:
520
+ return sampling_params
521
+
522
+ logger.debug("Building guided decoding logits processor. "
523
+ "Params: %s", guided_decoding)
524
+
525
+ guided_decoding.backend = guided_decoding.backend or default_guided_backend
526
+
527
+ processor = await get_guided_decoding_logits_processor(
528
+ guided_params=guided_decoding, tokenizer=tokenizer)
529
+
530
+ if processor:
531
+ if sampling_params.logits_processors is None:
532
+ sampling_params.logits_processors = []
533
+ sampling_params.logits_processors.append(processor)
534
+
535
+ # Unset guided decoding params after constructing the lp from them
536
+ sampling_params.guided_decoding = None
537
+
538
+ return sampling_params
539
+
540
+
541
class AsyncLLMEngine:
542
"""An asynchronous wrapper for :class:`LLMEngine`.
543
vllm/engine/llm_engine.py
+54
@@ -25,6 +25,7 @@ from vllm.engine.output_processor.interfaces import (
25
SequenceGroupOutputProcessor)
26
from vllm.engine.output_processor.stop_checker import StopChecker
27
from vllm.engine.output_processor.util import create_output_by_sequence_group
28
+from vllm.entrypoints.openai.logits_processors import get_logits_processors
29
from vllm.executor.executor_base import ExecutorBase
30
from vllm.executor.gpu_executor import GPUExecutor
31
from vllm.executor.ray_utils import initialize_ray_cluster
@@ -33,6 +34,8 @@ from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
34
from vllm.inputs.preprocess import InputPreprocessor
35
from vllm.logger import init_logger
36
from vllm.lora.request import LoRARequest
37
+from vllm.model_executor.guided_decoding import (
38
+ get_local_guided_decoding_logits_processor)
39
from vllm.model_executor.layers.sampler import SamplerOutput
40
from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
41
RequestOutputFactory)
@@ -843,6 +846,9 @@ class LLMEngine:
846
raise ValueError(f"Cannot request more than "
847
f"{max_logprobs} logprobs.")
848
849
+ sampling_params = self._build_logits_processors(
850
+ sampling_params, lora_request)
851
+
852
# Defensive copy of SamplingParams, which are used by the sampler,
853
# this doesn't deep-copy LogitsProcessor objects
854
sampling_params = sampling_params.clone()
@@ -1895,3 +1901,51 @@ class LLMEngine:
1901
# TODO: Find out how many placeholder tokens are there so we can
1902
# check that chunked prefill does not truncate them
1903
# max_batch_len = self.scheduler_config.max_num_batched_tokens
1904
+
1905
+ def _build_logits_processors(
1906
+ self, sampling_params: SamplingParams,
1907
+ lora_request: Optional[LoRARequest]) -> SamplingParams:
1908
+ """Constructs logits processors based on the guided_decoding,
1909
+ logits_bias, and allowed_token_ids fields in sampling_params. Deletes
1910
+ those fields and adds the constructed logits processors to the
1911
+ logits_processors field. Returns the modified sampling params."""
1912
+
1913
+ logits_processors = []
1914
+ if (guided_decoding := sampling_params.guided_decoding) is not None:
1915
+
1916
+ logger.debug(
1917
+ "Building guided decoding logits processor in "
1918
+ "LLMEngine. Params: %s", guided_decoding)
1919
+
1920
+ tokenizer = self.get_tokenizer(lora_request=lora_request)
1921
+ guided_decoding.backend = guided_decoding.backend or \
1922
+ self.decoding_config.guided_decoding_backend
1923
+
1924
+ processor = get_local_guided_decoding_logits_processor(
1925
+ guided_params=guided_decoding, tokenizer=tokenizer)
1926
+ if processor:
1927
+ logits_processors.append(processor)
1928
+
1929
+ # Unset so this doesn't get passed down to the model
1930
+ sampling_params.guided_decoding = None
1931
+
1932
+ if (sampling_params.logit_bias or sampling_params.allowed_token_ids):
1933
+ tokenizer = self.get_tokenizer(lora_request=lora_request)
1934
+
1935
+ processors = get_logits_processors(
1936
+ logit_bias=sampling_params.logit_bias,
1937
+ allowed_token_ids=sampling_params.allowed_token_ids,
1938
+ tokenizer=tokenizer)
1939
+ logits_processors.extend(processors)
1940
+
1941
+ # Unset so these don't get passed down to the model
1942
+ sampling_params.logit_bias = None
1943
+ sampling_params.allowed_token_ids = None
1944
+
1945
+ if logits_processors:
1946
+ if sampling_params.logits_processors is None:
1947
+ sampling_params.logits_processors = logits_processors
1948
+ else:
1949
+ sampling_params.logits_processors.extend(logits_processors)
1950
+
1951
+ return sampling_params
vllm/engine/multiprocessing/client.py
+14
@@ -16,6 +16,8 @@ from vllm.config import DecodingConfig, EngineConfig, ModelConfig
16
from vllm.engine.arg_utils import AsyncEngineArgs
17
# yapf conflicts with isort for this block
18
# yapf: disable
19
+from vllm.engine.async_llm_engine import (
20
+ build_guided_decoding_logits_processor_async)
21
from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
22
IPC_HEALTH_EXT, IPC_INPUT_EXT,
23
IPC_OUTPUT_EXT, RPC_REQUEST_T,
@@ -512,6 +514,18 @@ class MQLLMEngineClient:
514
if self._errored_with is not None:
515
raise ENGINE_DEAD_ERROR(self._errored_with)
516
517
+ # Constructing guided decoding logits processors is expensive, so we do
518
+ # it here to avoid contending with cpu resources and the GIL on the
519
+ # backend process.
520
+ if isinstance(params, SamplingParams) and \
521
+ params.guided_decoding is not None:
522
+ params = await \
523
+ build_guided_decoding_logits_processor_async(
524
+ sampling_params=params,
525
+ tokenizer=await self.get_tokenizer(lora_request),
526
+ default_guided_backend=self.decoding_config.guided_decoding_backend
527
+ )
528
+
529
# 1) Create output queue for this requests.
530
queue: asyncio.Queue[Union[RequestOutput,
531
BaseException]] = asyncio.Queue()
vllm/entrypoints/llm.py
+30
-18
@@ -1,4 +1,5 @@
1
import itertools
2
+import warnings
3
from contextlib import contextmanager
4
from dataclasses import dataclass
5
from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple,
@@ -16,13 +17,13 @@ from vllm.inputs import PromptType, TextPrompt, TokensPrompt
17
from vllm.inputs.parse import parse_and_batch_prompt
18
from vllm.logger import init_logger
19
from vllm.lora.request import LoRARequest
19
-from vllm.model_executor.guided_decoding import (
20
- GuidedDecodingRequest, get_local_guided_decoding_logits_processor)
21
-from vllm.model_executor.guided_decoding.guided_fields import LLMGuidedOptions
20
+from vllm.model_executor.guided_decoding.guided_fields import (
21
+ GuidedDecodingRequest, LLMGuidedOptions)
22
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
23
from vllm.pooling_params import PoolingParams
24
from vllm.prompt_adapter.request import PromptAdapterRequest
25
-from vllm.sampling_params import RequestOutputKind, SamplingParams
25
+from vllm.sampling_params import (GuidedDecodingParams, RequestOutputKind,
26
+ SamplingParams)
27
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
28
get_cached_tokenizer)
29
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
@@ -798,6 +799,14 @@ class LLM:
799
guided_options: Optional[GuidedDecodingRequest] = None,
800
priority: Optional[List[int]] = None,
801
) -> None:
802
+ if guided_options is not None:
803
+ warnings.warn(
804
+ "guided_options_request is deprecated, use "
805
+ "SamplingParams.guided_decoding instead",
806
+ DeprecationWarning,
807
+ stacklevel=2,
808
+ )
809
+
810
if isinstance(prompts, (str, dict)):
811
# Convert a single prompt to a list.
812
prompts = [prompts]
@@ -813,7 +822,7 @@ class LLM:
822
823
for sp in params if isinstance(params, list) else (params, ):
824
if isinstance(sp, SamplingParams):
816
- self._add_guided_processor(sp, guided_options)
825
+ self._add_guided_params(sp, guided_options)
826
827
# We only care about the final output
828
sp.output_kind = RequestOutputKind.FINAL_ONLY
@@ -847,22 +856,25 @@ class LLM:
856
priority=priority,
857
)
858
850
- def _add_guided_processor(
859
+ def _add_guided_params(
860
self,
861
params: SamplingParams,
862
guided_options: Optional[GuidedDecodingRequest] = None):
854
- if guided_options:
855
- if guided_options.guided_decoding_backend is None:
856
- decoding_config = self.llm_engine.get_decoding_config()
857
- guided_options.guided_decoding_backend = (
858
- decoding_config.guided_decoding_backend)
859
- guided_logits_processor = get_local_guided_decoding_logits_processor( #noqa
860
- guided_options.guided_decoding_backend, guided_options,
861
- self.get_tokenizer())
862
- if guided_logits_processor:
863
- if params.logits_processors is None:
864
- params.logits_processors = []
865
- params.logits_processors.append(guided_logits_processor)
863
+ if guided_options is None:
864
+ return params
865
+
866
+ if params.guided_decoding is not None:
867
+ raise ValueError("Cannot set both guided_options_request and"
868
+ "params.guided_decoding.")
869
+
870
+ params.guided_decoding = GuidedDecodingParams(
871
+ json=guided_options.guided_json,
872
+ regex=guided_options.guided_regex,
873
+ choice=guided_options.guided_choice,
874
+ grammar=guided_options.guided_grammar,
875
+ json_object=guided_options.guided_json_object,
876
+ backend=guided_options.guided_decoding_backend,
877
+ whitespace_pattern=guided_options.guided_whitespace_pattern)
878
return params
879
880
def _run_engine(
vllm/entrypoints/openai/protocol.py
+52
-30
@@ -10,12 +10,10 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
10
from typing_extensions import Annotated, Required, TypedDict
11
12
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13
-from vllm.entrypoints.openai.logits_processors import get_logits_processors
13
from vllm.pooling_params import PoolingParams
15
-from vllm.sampling_params import (LogitsProcessor, RequestOutputKind,
14
+from vllm.sampling_params import (GuidedDecodingParams, RequestOutputKind,
15
SamplingParams)
16
from vllm.sequence import Logprob
18
-from vllm.transformers_utils.tokenizer import AnyTokenizer
17
from vllm.utils import random_uuid
18
19
# torch is mocked during docs generation,
@@ -284,10 +282,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
282
283
# doc: end-chat-completion-extra-params
284
287
- def to_sampling_params(
288
- self, tokenizer: AnyTokenizer,
289
- guided_decode_logits_processor: Optional[LogitsProcessor],
290
- default_max_tokens: int) -> SamplingParams:
285
+ def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
286
max_tokens = self.max_tokens
287
if max_tokens is None:
288
max_tokens = default_max_tokens
@@ -296,14 +291,19 @@ class ChatCompletionRequest(OpenAIBaseModel):
291
if prompt_logprobs is None and self.echo:
292
prompt_logprobs = self.top_logprobs
293
299
- # We now allow logprobs being true without top_logrobs.
300
- logits_processors = get_logits_processors(
301
- logit_bias=self.logit_bias,
302
- allowed_token_ids=None,
303
- tokenizer=tokenizer,
304
- )
305
- if guided_decode_logits_processor:
306
- logits_processors.append(guided_decode_logits_processor)
294
+ guided_json_object = None
295
+ if (self.response_format is not None
296
+ and self.response_format.type == "json_object"):
297
+ guided_json_object = True
298
+
299
+ guided_decoding = GuidedDecodingParams.from_optional(
300
+ json=self._get_guided_json_from_tool() or self.guided_json,
301
+ regex=self.guided_regex,
302
+ choice=self.guided_choice,
303
+ grammar=self.guided_grammar,
304
+ json_object=guided_json_object,
305
+ backend=self.guided_decoding_backend,
306
+ whitespace_pattern=self.guided_whitespace_pattern)
307
308
return SamplingParams.from_optional(
309
n=self.n,
@@ -329,11 +329,29 @@ class ChatCompletionRequest(OpenAIBaseModel):
329
spaces_between_special_tokens=self.spaces_between_special_tokens,
330
include_stop_str_in_output=self.include_stop_str_in_output,
331
length_penalty=self.length_penalty,
332
- logits_processors=logits_processors,
332
truncate_prompt_tokens=self.truncate_prompt_tokens,
333
output_kind=RequestOutputKind.DELTA if self.stream \
334
else RequestOutputKind.FINAL_ONLY,
336
- )
335
+ guided_decoding=guided_decoding,
336
+ logit_bias=self.logit_bias)
337
+
338
+ def _get_guided_json_from_tool(
339
+ self) -> Optional[Union[str, dict, BaseModel]]:
340
+ # user has chosen to not use any tool
341
+ if self.tool_choice == "none" or self.tools is None:
342
+ return None
343
+
344
+ # user has chosen to use a named tool
345
+ if type(self.tool_choice) is ChatCompletionNamedToolChoiceParam:
346
+ tool_name = self.tool_choice.function.name
347
+ tools = {tool.function.name: tool.function for tool in self.tools}
348
+ if tool_name not in tools:
349
+ raise ValueError(
350
+ f"Tool '{tool_name}' has not been passed in `tools`.")
351
+ tool = tools[tool_name]
352
+ return tool.parameters
353
+
354
+ return None
355
356
@model_validator(mode="before")
357
@classmethod
@@ -537,10 +555,7 @@ class CompletionRequest(OpenAIBaseModel):
555
556
# doc: end-completion-extra-params
557
540
- def to_sampling_params(
541
- self, tokenizer: AnyTokenizer,
542
- guided_decode_logits_processor: Optional[LogitsProcessor],
543
- default_max_tokens: int) -> SamplingParams:
558
+ def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
559
max_tokens = self.max_tokens
560
if max_tokens is None:
561
max_tokens = default_max_tokens
@@ -551,13 +566,19 @@ class CompletionRequest(OpenAIBaseModel):
566
567
echo_without_generation = self.echo and self.max_tokens == 0
568
554
- logits_processors = get_logits_processors(
555
- logit_bias=self.logit_bias,
556
- allowed_token_ids=self.allowed_token_ids,
557
- tokenizer=tokenizer,
558
- )
559
- if guided_decode_logits_processor:
560
- logits_processors.append(guided_decode_logits_processor)
569
+ guided_json_object = None
570
+ if (self.response_format is not None
571
+ and self.response_format.type == "json_object"):
572
+ guided_json_object = True
573
+
574
+ guided_decoding = GuidedDecodingParams.from_optional(
575
+ json=self.guided_json,
576
+ regex=self.guided_regex,
577
+ choice=self.guided_choice,
578
+ grammar=self.guided_grammar,
579
+ json_object=guided_json_object,
580
+ backend=self.guided_decoding_backend,
581
+ whitespace_pattern=self.guided_whitespace_pattern)
582
583
return SamplingParams.from_optional(
584
n=self.n,
@@ -583,11 +604,12 @@ class CompletionRequest(OpenAIBaseModel):
604
spaces_between_special_tokens=self.spaces_between_special_tokens,
605
include_stop_str_in_output=self.include_stop_str_in_output,
606
length_penalty=self.length_penalty,
586
- logits_processors=logits_processors,
607
truncate_prompt_tokens=self.truncate_prompt_tokens,
608
output_kind=RequestOutputKind.DELTA if self.stream \
609
else RequestOutputKind.FINAL_ONLY,
590
- )
610
+ guided_decoding=guided_decoding,
611
+ logit_bias=self.logit_bias,
612
+ allowed_token_ids=self.allowed_token_ids)
613
614
@model_validator(mode="before")
615
@classmethod
vllm/entrypoints/openai/serving_chat.py
-5
@@ -187,9 +187,6 @@ class OpenAIServingChat(OpenAIServing):
187
raw_request.state.request_metadata = request_metadata
188
189
try:
190
- guided_decode_logits_processor = (
191
- await self._guided_decode_logits_processor(request, tokenizer))
192
-
190
if isinstance(prompt, str):
191
prompt_inputs = self._tokenize_prompt_input(
192
request,
@@ -208,8 +205,6 @@ class OpenAIServingChat(OpenAIServing):
205
assert prompt_inputs is not None
206
207
sampling_params = request.to_sampling_params(
211
- tokenizer,
212
- guided_decode_logits_processor,
208
default_max_tokens=self.max_model_len -
209
len(prompt_inputs["prompt_token_ids"]))
210
vllm/entrypoints/openai/serving_completion.py
-4
@@ -110,8 +110,6 @@ class OpenAIServingCompletion(OpenAIServing):
110
111
tokenizer = await self.engine_client.get_tokenizer(lora_request)
112
113
- guided_decode_logits_processor = (
114
- await self._guided_decode_logits_processor(request, tokenizer))
113
prompts = list(
114
self._tokenize_prompt_input_or_inputs(
115
request,
@@ -123,8 +121,6 @@ class OpenAIServingCompletion(OpenAIServing):
121
122
for i, prompt_inputs in enumerate(prompts):
123
sampling_params = request.to_sampling_params(
126
- tokenizer,
127
- guided_decode_logits_processor,
124
default_max_tokens=self.max_model_len -
125
len(prompt_inputs["prompt_token_ids"]))
126
vllm/entrypoints/openai/serving_engine.py
+1
-12
@@ -27,11 +27,9 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
27
from vllm.inputs.parse import parse_and_batch_prompt
28
from vllm.logger import init_logger
29
from vllm.lora.request import LoRARequest
30
-from vllm.model_executor.guided_decoding import (
31
- get_guided_decoding_logits_processor)
30
from vllm.pooling_params import PoolingParams
31
from vllm.prompt_adapter.request import PromptAdapterRequest
34
-from vllm.sampling_params import LogitsProcessor, SamplingParams
32
+from vllm.sampling_params import SamplingParams
33
from vllm.sequence import Logprob
34
from vllm.transformers_utils.tokenizer import AnyTokenizer
35
from vllm.utils import AtomicCounter
@@ -168,15 +166,6 @@ class OpenAIServing:
166
})
167
return json_str
168
171
- async def _guided_decode_logits_processor(
172
- self, request: Union[ChatCompletionRequest, CompletionRequest],
173
- tokenizer: AnyTokenizer) -> Optional[LogitsProcessor]:
174
- decoding_config = await self.engine_client.get_decoding_config()
175
- guided_decoding_backend = request.guided_decoding_backend \
176
- or decoding_config.guided_decoding_backend
177
- return await get_guided_decoding_logits_processor(
178
- guided_decoding_backend, request, tokenizer)
179
-
169
async def _check_model(
170
self,
171
request: AnyRequest,
vllm/model_executor/guided_decoding/__init__.py
+18
-50
@@ -1,77 +1,45 @@
1
-from typing import Optional, Union
1
+from typing import Optional
2
3
-from vllm.entrypoints.openai.protocol import (
4
- ChatCompletionNamedToolChoiceParam, ChatCompletionRequest,
5
- CompletionRequest)
6
-from vllm.model_executor.guided_decoding.guided_fields import (
7
- GuidedDecodingRequest)
8
-from vllm.sampling_params import LogitsProcessor
3
+from vllm.sampling_params import GuidedDecodingParams, LogitsProcessor
4
5
6
async def get_guided_decoding_logits_processor(
12
- guided_decoding_backend: str, request: Union[CompletionRequest,
13
- ChatCompletionRequest],
7
+ guided_params: GuidedDecodingParams,
8
tokenizer) -> Optional[LogitsProcessor]:
15
- request = _adapt_request_for_tool_use(request)
16
-
17
- if guided_decoding_backend == 'outlines':
9
+ # CFG grammar not supported by LMFE, so we use outlines instead
10
+ if guided_params.backend == 'outlines' or guided_params.grammar:
11
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
12
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
13
get_outlines_guided_decoding_logits_processor)
14
return await get_outlines_guided_decoding_logits_processor(
22
- request, tokenizer)
23
- if guided_decoding_backend == 'lm-format-enforcer':
15
+ guided_params, tokenizer)
16
+ if guided_params.backend == 'lm-format-enforcer':
17
from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
25
- get_lm_format_enforcer_guided_decoding_logits_processor)
26
- return await get_lm_format_enforcer_guided_decoding_logits_processor(
27
- request, tokenizer)
18
+ get_local_lm_format_enforcer_guided_decoding_logits_processor)
19
+ return get_local_lm_format_enforcer_guided_decoding_logits_processor(
20
+ guided_params, tokenizer)
21
22
raise ValueError(
30
- f"Unknown guided decoding backend '{guided_decoding_backend}'. "
23
+ f"Unknown guided decoding backend '{guided_params.backend}'. "
24
"Must be one of 'outlines, 'lm-format-enforcer'")
25
26
27
def get_local_guided_decoding_logits_processor(
35
- guided_decoding_backend: str, guided_options: GuidedDecodingRequest,
28
+ guided_params: GuidedDecodingParams,
29
tokenizer) -> Optional[LogitsProcessor]:
37
- # request = _adapt_request_for_tool_use(request)
38
-
39
- if guided_decoding_backend == 'outlines':
30
+ # CFG grammar not supported by LMFE, so we use outlines instead
31
+ if guided_params.backend == 'outlines' or guided_params.grammar:
32
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
33
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
34
get_local_outlines_guided_decoding_logits_processor)
35
return get_local_outlines_guided_decoding_logits_processor(
44
- guided_options, tokenizer)
45
- if guided_decoding_backend == 'lm-format-enforcer':
36
+ guided_params, tokenizer)
37
+ if guided_params.backend == 'lm-format-enforcer':
38
from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
39
get_local_lm_format_enforcer_guided_decoding_logits_processor)
40
return get_local_lm_format_enforcer_guided_decoding_logits_processor(
49
- guided_options, tokenizer)
41
+ guided_params, tokenizer)
42
43
raise ValueError(
52
- f"Unknown guided decoding backend '{guided_decoding_backend}'. "
44
+ f"Unknown guided decoding backend '{guided_params.backend}'. "
45
"Must be one of 'outlines, 'lm-format-enforcer'")
54
-
55
-
56
-def _adapt_request_for_tool_use(request: Union[CompletionRequest,
57
- ChatCompletionRequest]):
58
- # the legacy completion API does not support tool use
59
- if type(request) is CompletionRequest:
60
- return request
61
-
62
- # user has chosen to not use any tool,
63
- # OR is allowing the model to choose a tool.
64
- if request.tool_choice == "none" or request.tool_choice == "auto":
65
- return request
66
-
67
- # user has chosen to use a named tool
68
- if type(request.tool_choice) is ChatCompletionNamedToolChoiceParam:
69
- tool_name = request.tool_choice.function.name
70
- tools = {tool.function.name: tool.function for tool in request.tools}
71
- if tool_name not in tools:
72
- raise ValueError(
73
- f"Tool '{tool_name}' has not been passed in `tools`.")
74
- tool = tools[tool_name]
75
- request.guided_json = tool.parameters
76
-
77
- return request
vllm/model_executor/guided_decoding/guided_fields.py
+1
@@ -4,6 +4,7 @@ from typing import Dict, List, Optional, TypedDict, Union
4
from pydantic import BaseModel
5
6
7
+# These classes are deprecated, see SamplingParams
8
class LLMGuidedOptions(TypedDict, total=False):
9
guided_json: Union[Dict, BaseModel, str]
10
guided_regex: str
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+16
-74
@@ -7,66 +7,13 @@ from lmformatenforcer import (CharacterLevelParser, JsonSchemaParser,
7
TokenEnforcerTokenizerData, UnionParser)
8
from lmformatenforcer.integrations.vllm import (
9
build_vllm_logits_processor, build_vllm_token_enforcer_tokenizer_data)
10
-from pydantic import BaseModel
10
from transformers import PreTrainedTokenizerBase
11
13
-from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14
- CompletionRequest)
15
-from vllm.model_executor.guided_decoding.guided_fields import (
16
- GuidedDecodingRequest)
17
-from vllm.sampling_params import LogitsProcessor
18
-
19
-
20
-async def get_lm_format_enforcer_guided_decoding_logits_processor(
21
- request: Union[CompletionRequest, ChatCompletionRequest],
22
- tokenizer) -> Optional[LogitsProcessor]:
23
- """
24
- Given an OpenAI-compatible request, check for guided decoding parameters
25
- and get the necessary logits processor for the given guide.
26
- We cache logit processors by (guide, tokenizer), and on cache hit
27
- we make a shallow copy to reuse the same underlying FSM.
28
- """
29
-
30
- tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
31
- tokenizer)
32
- character_level_parser: CharacterLevelParser
33
- if request.guided_json:
34
- schema = _normalize_json_schema_object(request.guided_json)
35
- character_level_parser = JsonSchemaParser(schema)
36
- elif request.guided_choice:
37
- character_level_parser = UnionParser(
38
- [StringParser(choice) for choice in request.guided_choice])
39
- elif request.guided_regex:
40
- character_level_parser = RegexParser(request.guided_regex)
41
- elif request.guided_grammar:
42
- # CFG grammar not supported by LMFE, revert to outlines
43
-
44
- # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
45
- from vllm.model_executor.guided_decoding.outlines_decoding import (
46
- get_outlines_guided_decoding_logits_processor)
47
- return await get_outlines_guided_decoding_logits_processor(
48
- request, tokenizer)
49
- elif (request.response_format is not None
50
- and request.response_format.type == "json_object"):
51
- character_level_parser = JsonSchemaParser(
52
- None) # None means any json object
53
- elif (request.response_format is not None
54
- and request.response_format.type == "json_schema"
55
- and request.response_format.json_schema is not None
56
- and request.response_format.json_schema.json_schema is not None):
57
- schema = _normalize_json_schema_object(
58
- request.response_format.json_schema.json_schema)
59
- character_level_parser = JsonSchemaParser(schema)
60
- else:
61
- return None
62
-
63
- logits_processor = build_vllm_logits_processor(tokenizer_data,
64
- character_level_parser)
65
- return logits_processor
12
+from vllm.sampling_params import GuidedDecodingParams, LogitsProcessor
13
14
15
def get_local_lm_format_enforcer_guided_decoding_logits_processor(
69
- guided_options: GuidedDecodingRequest,
16
+ guided_params: GuidedDecodingParams,
17
tokenizer) -> Optional[LogitsProcessor]:
18
"""
19
Given an OpenAI-compatible request, check for guided decoding parameters
@@ -78,23 +25,20 @@ def get_local_lm_format_enforcer_guided_decoding_logits_processor(
25
tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
26
tokenizer)
27
character_level_parser: CharacterLevelParser
81
- if guided_options.guided_json:
82
- schema = _normalize_json_schema_object(guided_options.guided_json)
83
- character_level_parser = JsonSchemaParser(schema)
84
- elif guided_options.guided_choice:
28
+ if guided_params.json:
29
+ schema_dict = _normalize_json_schema_object(guided_params.json)
30
+ character_level_parser = JsonSchemaParser(schema_dict)
31
+ elif guided_params.choice:
32
character_level_parser = UnionParser(
86
- [StringParser(choice) for choice in guided_options.guided_choice])
87
- elif guided_options.guided_regex:
88
- character_level_parser = RegexParser(guided_options.guided_regex)
89
- elif guided_options.guided_grammar:
90
- # CFG grammar not supported by LMFE, revert to outlines
91
-
92
- # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
93
- from vllm.model_executor.guided_decoding.outlines_decoding import (
94
- get_local_outlines_guided_decoding_logits_processor)
95
- return get_local_outlines_guided_decoding_logits_processor(
96
- guided_options, tokenizer)
97
- elif guided_options.guided_json_object:
33
+ [StringParser(choice) for choice in guided_params.choice])
34
+ elif guided_params.regex:
35
+ character_level_parser = RegexParser(guided_params.regex)
36
+ elif guided_params.grammar:
37
+ # CFG grammar not supported by LMFE
38
+ raise ValueError("Cannot construct a guided decoding logits processor"
39
+ " using the grammar option with the"
40
+ " lm_format_enforcer backend.")
41
+ elif guided_params.json_object:
42
# None means any json object
43
character_level_parser = JsonSchemaParser(None)
44
else:
@@ -105,13 +49,11 @@ def get_local_lm_format_enforcer_guided_decoding_logits_processor(
49
return logits_processor
50
51
108
-def _normalize_json_schema_object(schema: Union[str, dict, BaseModel]) -> dict:
52
+def _normalize_json_schema_object(schema: Union[str, dict]) -> dict:
53
if isinstance(schema, str):
54
return json_loads(schema)
55
if isinstance(schema, dict):
56
return schema
113
- if isinstance(schema, BaseModel):
114
- return schema.model_json_schema()
57
raise AssertionError(f"Unsupported schema type {schema}")
58
59
vllm/model_executor/guided_decoding/outlines_decoding.py
+19
-53
@@ -5,16 +5,11 @@ from json import dumps as json_dumps
5
from re import escape as regex_escape
6
from typing import Tuple, Union
7
8
-from pydantic import BaseModel
8
from transformers import PreTrainedTokenizerBase
9
11
-from vllm.entrypoints.openai.protocol import (
12
- ChatCompletionNamedToolChoiceParam, ChatCompletionRequest,
13
- CompletionRequest)
14
-from vllm.model_executor.guided_decoding.guided_fields import (
15
- GuidedDecodingRequest)
10
from vllm.model_executor.guided_decoding.outlines_logits_processors import (
11
CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor)
12
+from vllm.sampling_params import GuidedDecodingParams
13
14
15
class GuidedDecodingMode(Enum):
@@ -55,8 +50,7 @@ global_thread_pool = None # used for generating logits processor fsm
50
51
52
async def get_outlines_guided_decoding_logits_processor(
58
- request: Union[CompletionRequest,
59
- ChatCompletionRequest], tokenizer: PreTrainedTokenizerBase
53
+ guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase
54
) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
55
None]:
56
"""
@@ -66,7 +60,7 @@ async def get_outlines_guided_decoding_logits_processor(
60
we make a shallow copy to reuse the same underlying FSM.
61
"""
62
global global_thread_pool
69
- guide, mode = _get_guide_and_mode(request)
63
+ guide, mode = _get_guide_and_mode(guided_params)
64
if not guide or not mode:
65
return None
66
@@ -77,11 +71,11 @@ async def get_outlines_guided_decoding_logits_processor(
71
72
return await loop.run_in_executor(global_thread_pool,
73
_get_logits_processor, guide, tokenizer,
80
- mode, request.guided_whitespace_pattern)
74
+ mode, guided_params.whitespace_pattern)
75
76
77
def get_local_outlines_guided_decoding_logits_processor(
84
- guided_options: GuidedDecodingRequest, tokenizer: PreTrainedTokenizerBase
78
+ guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase
79
) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
80
None]:
81
"""
@@ -90,65 +84,37 @@ def get_local_outlines_guided_decoding_logits_processor(
84
We cache logit processors by (guide, tokenizer), and on cache hit
85
we make a shallow copy to reuse the same underlying FSM.
86
"""
93
- guide, mode = _get_guide_and_mode(guided_options)
87
+ guide, mode = _get_guide_and_mode(guided_params)
88
if not guide or not mode:
89
return None
90
91
return _get_logits_processor(guide, tokenizer, mode,
98
- guided_options.guided_whitespace_pattern)
92
+ guided_params.whitespace_pattern)
93
94
95
def _get_guide_and_mode(
102
- request: Union[CompletionRequest, ChatCompletionRequest,
103
- GuidedDecodingRequest]
96
+ guided_params: GuidedDecodingParams
97
) -> Union[Tuple[str, GuidedDecodingMode], Tuple[None, None]]:
105
- # if the request is a chat completion request, AND the tool choice is a
106
- # named tool choice, do guided decoding
107
- # using that tool as the JSON schema
108
- if isinstance(request, ChatCompletionRequest) and isinstance(
109
- request.tool_choice, ChatCompletionNamedToolChoiceParam):
110
- # Guided generation for tools/functions parameters
111
- if request.tool_choice.type == "function":
112
- for tool in request.tools:
113
- if (tool.type == "function" and tool.function.name
114
- == request.tool_choice.function.name):
115
- json = json_dumps(tool.function.parameters, sort_keys=True)
116
- return json, GuidedDecodingMode.JSON
117
- return None, None
118
-
119
- elif request.guided_json:
120
- if isinstance(request.guided_json, dict):
98
+ if guided_params.json:
99
+ if isinstance(guided_params.json, dict):
100
# turn dict into hashable string
122
- json = json_dumps(request.guided_json)
123
- elif isinstance(request.guided_json, BaseModel):
124
- # use pydantic signature so that different model classes
125
- # with the same fields will get hashed the same
126
- json = str(request.guided_json.__signature__)
101
+ json = json_dumps(guided_params.json)
102
else:
128
- json = request.guided_json
103
+ json = guided_params.json
104
return json, GuidedDecodingMode.JSON
130
- elif request.guided_regex:
131
- return request.guided_regex, GuidedDecodingMode.REGEX
132
- elif request.guided_choice:
105
+ elif guided_params.regex:
106
+ return guided_params.regex, GuidedDecodingMode.REGEX
107
+ elif guided_params.choice:
108
# choice just uses regex
109
choices = [
135
- regex_escape(str(choice)) for choice in request.guided_choice
110
+ regex_escape(str(choice)) for choice in guided_params.choice
111
]
112
choices_regex = "(" + "|".join(choices) + ")"
113
return choices_regex, GuidedDecodingMode.CHOICE
139
- elif request.guided_grammar:
140
- return request.guided_grammar, GuidedDecodingMode.GRAMMAR
141
- elif (not isinstance(request, GuidedDecodingRequest)
142
- and request.response_format is not None
143
- and request.response_format.type == "json_object"):
114
+ elif guided_params.grammar:
115
+ return guided_params.grammar, GuidedDecodingMode.GRAMMAR
116
+ elif guided_params.json_object:
117
return JSON_GRAMMAR, GuidedDecodingMode.GRAMMAR
145
- elif (not isinstance(request, GuidedDecodingRequest)
146
- and request.response_format is not None
147
- and request.response_format.type == "json_schema"
148
- and request.response_format.json_schema is not None
149
- and request.response_format.json_schema.json_schema is not None):
150
- json = json_dumps(request.response_format.json_schema.json_schema)
151
- return json, GuidedDecodingMode.JSON
118
else:
119
return None, None
120
vllm/sampling_params.py
+76
-1
@@ -1,11 +1,13 @@
1
"""Sampling parameters for text generation."""
2
import copy
3
+from dataclasses import dataclass
4
from enum import Enum, IntEnum
5
from functools import cached_property
6
from typing import Any, Callable, Dict, List, Optional, Set, Union
7
8
import msgspec
9
import torch
10
+from pydantic import BaseModel
11
from typing_extensions import Annotated
12
13
import vllm.envs as envs
@@ -34,6 +36,54 @@ first argument, and returns a modified tensor of logits
36
to sample from."""
37
38
39
+# maybe make msgspec?
40
+@dataclass
41
+class GuidedDecodingParams:
42
+ """One of these fields will be used to build a logit processor."""
43
+ json: Optional[Union[str, Dict]] = None
44
+ regex: Optional[str] = None
45
+ choice: Optional[List[str]] = None
46
+ grammar: Optional[str] = None
47
+ json_object: Optional[bool] = None
48
+ """These are other options that can be set"""
49
+ backend: Optional[str] = None
50
+ whitespace_pattern: Optional[str] = None
51
+
52
+ @staticmethod
53
+ def from_optional(
54
+ json: Optional[Union[Dict, BaseModel, str]],
55
+ regex: Optional[str] = None,
56
+ choice: Optional[List[str]] = None,
57
+ grammar: Optional[str] = None,
58
+ json_object: Optional[bool] = None,
59
+ backend: Optional[str] = None,
60
+ whitespace_pattern: Optional[str] = None,
61
+ ) -> "GuidedDecodingParams":
62
+ # Extract json schemas from pydantic models
63
+ if isinstance(json, (BaseModel, type(BaseModel))):
64
+ json = json.model_json_schema()
65
+ return GuidedDecodingParams(
66
+ json=json,
67
+ regex=regex,
68
+ choice=choice,
69
+ grammar=grammar,
70
+ json_object=json_object,
71
+ backend=backend,
72
+ whitespace_pattern=whitespace_pattern,
73
+ )
74
+
75
+ def __post_init__(self):
76
+ """Validate that some fields are mutually exclusive."""
77
+ guide_count = sum([
78
+ self.json is not None, self.regex is not None, self.choice
79
+ is not None, self.grammar is not None, self.json_object is not None
80
+ ])
81
+ if guide_count > 1:
82
+ raise ValueError(
83
+ "You can only use one kind of guided decoding but multiple are "
84
+ f"specified: {self.__dict__}")
85
+
86
+
87
class RequestOutputKind(Enum):
88
# Return entire output so far in every RequestOutput
89
CUMULATIVE = 0
@@ -124,6 +174,13 @@ class SamplingParams(
174
truncate_prompt_tokens: If set to an integer k, will use only the last k
175
tokens from the prompt (i.e., left truncation). Defaults to None
176
(i.e., no truncation).
177
+ guided_decoding: If provided, the engine will construct a guided
178
+ decoding logits processor from these parameters. Defaults to None.
179
+ logit_bias: If provided, the engine will construct a logits processor
180
+ that applies these logit biases. Defaults to None.
181
+ allowed_token_ids: If provided, the engine will construct a logits
182
+ processor which only retains scores for the given token ids.
183
+ Defaults to None.
184
"""
185
186
n: int = 1
@@ -164,6 +221,11 @@ class SamplingParams(
221
output_text_buffer_length: int = 0
222
_all_stop_token_ids: Set[int] = msgspec.field(default_factory=set)
223
224
+ # Fields used to construct logits processors
225
+ guided_decoding: Optional[GuidedDecodingParams] = None
226
+ logit_bias: Optional[Dict[int, float]] = None
227
+ allowed_token_ids: Optional[List[int]] = None
228
+
229
@staticmethod
230
def from_optional(
231
n: Optional[int] = 1,
@@ -194,7 +256,16 @@ class SamplingParams(
256
truncate_prompt_tokens: Optional[Annotated[int,
257
msgspec.Meta(ge=1)]] = None,
258
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
259
+ guided_decoding: Optional[GuidedDecodingParams] = None,
260
+ logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]] = None,
261
+ allowed_token_ids: Optional[List[int]] = None,
262
) -> "SamplingParams":
263
+ if logit_bias is not None:
264
+ logit_bias = {
265
+ int(token): bias
266
+ for token, bias in logit_bias.items()
267
+ }
268
+
269
return SamplingParams(
270
n=1 if n is None else n,
271
best_of=best_of,
@@ -226,6 +297,9 @@ class SamplingParams(
297
logits_processors=logits_processors,
298
truncate_prompt_tokens=truncate_prompt_tokens,
299
output_kind=output_kind,
300
+ guided_decoding=guided_decoding,
301
+ logit_bias=logit_bias,
302
+ allowed_token_ids=allowed_token_ids,
303
)
304
305
def __post_init__(self) -> None:
@@ -454,4 +528,5 @@ class SamplingParams(
528
f"skip_special_tokens={self.skip_special_tokens}, "
529
"spaces_between_special_tokens="
530
f"{self.spaces_between_special_tokens}, "
457
- f"truncate_prompt_tokens={self.truncate_prompt_tokens})")
531
+ f"truncate_prompt_tokens={self.truncate_prompt_tokens}), "
532
+ f"guided_decoding={self.guided_decoding}")