@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 062c89e7c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Frontend][Core] Move guided decoding params into sampling params (#8252)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Joe Runde committed Sep 30, 2024 at 19:34 UTC 062c89e7c9c6fa9fd7fb2d28fd50321c6f78f389
16 files changed +441 -281
tests/entrypoints/llm/test_guided_generate.py
+43 -23
@@ -7,7 +7,7 @@ import pytest
7
8 from vllm.entrypoints.llm import LLM
9 from vllm.outputs import RequestOutput
10 -from vllm.sampling_params import SamplingParams
10 +from vllm.sampling_params import GuidedDecodingParams, SamplingParams
11
12 from ...conftest import cleanup
13
@@ -31,14 +31,12 @@ def test_guided_regex(sample_regex, llm):
31 sampling_params = SamplingParams(
32 temperature=0.8,
33 top_p=0.95,
34 - )
35 - outputs = llm.generate(
36 - prompts=[
37 - f"Give an example IPv4 address with this regex: {sample_regex}"
38 - ] * 2,
39 - sampling_params=sampling_params,
40 - use_tqdm=True,
41 - guided_options_request=dict(guided_regex=sample_regex))
34 + guided_decoding=GuidedDecodingParams(regex=sample_regex))
35 + outputs = llm.generate(prompts=[
36 + f"Give an example IPv4 address with this regex: {sample_regex}"
37 + ] * 2,
38 + sampling_params=sampling_params,
39 + use_tqdm=True)
40
41 assert outputs is not None
42 for output in outputs:
@@ -57,15 +55,13 @@ def test_guided_json_completion(sample_json_schema, llm):
55 sampling_params = SamplingParams(
56 temperature=1.0,
57 max_tokens=1000,
60 - )
61 - outputs = llm.generate(
62 - prompts=[
63 - f"Give an example JSON for an employee profile "
64 - f"that fits this schema: {sample_json_schema}"
65 - ] * 2,
66 - sampling_params=sampling_params,
67 - use_tqdm=True,
68 - guided_options_request=dict(guided_json=sample_json_schema))
58 + guided_decoding=GuidedDecodingParams(json=sample_json_schema))
59 + outputs = llm.generate(prompts=[
60 + f"Give an example JSON for an employee profile "
61 + f"that fits this schema: {sample_json_schema}"
62 + ] * 2,
63 + sampling_params=sampling_params,
64 + use_tqdm=True)
65
66 assert outputs is not None
67
@@ -86,12 +82,11 @@ def test_guided_choice_completion(sample_guided_choice, llm):
82 sampling_params = SamplingParams(
83 temperature=0.8,
84 top_p=0.95,
89 - )
85 + guided_decoding=GuidedDecodingParams(choice=sample_guided_choice))
86 outputs = llm.generate(
87 prompts="The best language for type-safe systems programming is ",
88 sampling_params=sampling_params,
93 - use_tqdm=True,
94 - guided_options_request=dict(guided_choice=sample_guided_choice))
89 + use_tqdm=True)
90
91 assert outputs is not None
92 for output in outputs:
@@ -112,13 +107,13 @@ def test_guided_grammar(sample_sql_statements, llm):
107 temperature=0.8,
108 top_p=0.95,
109 max_tokens=1000,
115 - )
110 + guided_decoding=GuidedDecodingParams(grammar=sample_sql_statements))
111 outputs = llm.generate(
112 prompts=("Generate a sql state that select col_1 from "
113 "table_1 where it is equals to 1"),
114 sampling_params=sampling_params,
115 use_tqdm=True,
121 - guided_options_request=dict(guided_grammar=sample_sql_statements))
116 + )
117
118 assert outputs is not None
119 for output in outputs:
@@ -140,3 +135,28 @@ def test_guided_grammar(sample_sql_statements, llm):
135 assert generated_text.strip() == ground_truth
136
137 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
138 +
139 +
140 +@pytest.mark.skip_global_cleanup
141 +def test_guided_options_request_deprecation_warning(sample_regex, llm):
142 + sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
143 +
144 + with pytest.warns(DeprecationWarning, match="guided_options_request"):
145 + llm.generate(prompts="This should fail",
146 + sampling_params=sampling_params,
147 + use_tqdm=True,
148 + guided_options_request=dict(guided_regex=sample_regex))
149 +
150 +
151 +@pytest.mark.skip_global_cleanup
152 +def test_validation_against_both_guided_decoding_options(sample_regex, llm):
153 + sampling_params = SamplingParams(
154 + temperature=0.8,
155 + top_p=0.95,
156 + guided_decoding=GuidedDecodingParams(regex=sample_regex))
157 +
158 + with pytest.raises(ValueError, match="Cannot set both"):
159 + llm.generate(prompts="This should fail",
160 + sampling_params=sampling_params,
161 + use_tqdm=True,
162 + guided_options_request=dict(guided_regex=sample_regex))
tests/model_executor/conftest.py new
+49
@@ -0,0 +1,49 @@
1 +import pytest
2 +
3 +
4 +@pytest.fixture
5 +def sample_regex():
6 + return (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
7 + r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
8 +
9 +
10 +@pytest.fixture
11 +def sample_json_schema():
12 + return {
13 + "type": "object",
14 + "properties": {
15 + "name": {
16 + "type": "string"
17 + },
18 + "age": {
19 + "type": "integer"
20 + },
21 + "skills": {
22 + "type": "array",
23 + "items": {
24 + "type": "string",
25 + "maxLength": 10
26 + },
27 + "minItems": 3
28 + },
29 + "work_history": {
30 + "type": "array",
31 + "items": {
32 + "type": "object",
33 + "properties": {
34 + "company": {
35 + "type": "string"
36 + },
37 + "duration": {
38 + "type": "number"
39 + },
40 + "position": {
41 + "type": "string"
42 + }
43 + },
44 + "required": ["company", "position"]
45 + }
46 + }
47 + },
48 + "required": ["name", "age", "skills", "work_history"]
49 + }
tests/model_executor/test_guided_processors.py renamed
+24 -11
@@ -1,14 +1,12 @@
1 -# This unit test should be moved to a new
2 -# tests/test_guided_decoding directory.
1 import pytest
2 import torch
3 from transformers import AutoTokenizer
4
7 -from vllm.entrypoints.openai.protocol import CompletionRequest
5 from vllm.model_executor.guided_decoding import (
6 get_guided_decoding_logits_processor)
7 from vllm.model_executor.guided_decoding.outlines_logits_processors import (
8 JSONLogitsProcessor, RegexLogitsProcessor)
9 +from vllm.sampling_params import GuidedDecodingParams
10
11
12 def test_guided_logits_processors(sample_regex, sample_json_schema):
@@ -44,11 +42,9 @@ async def test_guided_logits_processor_black_box(backend: str, sample_regex,
42 tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
43 token_ids = tokenizer.encode(
44 f"Give an example IPv4 address with this regex: {sample_regex}")
47 - regex_request = CompletionRequest(model='test',
48 - prompt=token_ids,
49 - guided_regex=sample_regex)
45 + regex_request = GuidedDecodingParams(regex=sample_regex, backend=backend)
46 regex_lp = await get_guided_decoding_logits_processor(
51 - backend, regex_request, tokenizer)
47 + regex_request, tokenizer)
48 assert regex_lp is not None
49 tensor = torch.rand(32000)
50 original_tensor = torch.clone(tensor)
@@ -59,14 +55,31 @@ async def test_guided_logits_processor_black_box(backend: str, sample_regex,
55 token_ids = tokenizer.encode(
56 f"Give an employee profile that fits this schema: {sample_json_schema}"
57 )
62 - json_request = CompletionRequest(model='test',
63 - prompt=token_ids,
64 - guided_json=sample_json_schema)
58 + json_request = GuidedDecodingParams(json=sample_json_schema,
59 + backend=backend)
60 json_lp = await get_guided_decoding_logits_processor(
66 - backend, json_request, tokenizer)
61 + json_request, tokenizer)
62 assert json_lp is not None
63 tensor = torch.rand(32000)
64 original_tensor = torch.clone(tensor)
65 tensor = json_lp(token_ids, tensor)
66 assert tensor.shape == original_tensor.shape
67 assert not torch.allclose(tensor, original_tensor)
68 +
69 +
70 +def test_multiple_guided_options_not_allowed(sample_json_schema, sample_regex):
71 + with pytest.raises(ValueError,
72 + match="You can only use one kind of guided"):
73 + GuidedDecodingParams(json=sample_json_schema, regex=sample_regex)
74 +
75 + with pytest.raises(ValueError,
76 + match="You can only use one kind of guided"):
77 + GuidedDecodingParams(json=sample_json_schema, json_object=True)
78 +
79 + with pytest.raises(ValueError,
80 + match="You can only use one kind of guided"):
81 + GuidedDecodingParams(json=sample_json_schema, choice=["a", "b"])
82 +
83 + with pytest.raises(ValueError,
84 + match="You can only use one kind of guided"):
85 + GuidedDecodingParams(json=sample_json_schema, grammar="test grammar")
vllm/engine/async_llm_engine.py
+44
@@ -20,6 +20,8 @@ from vllm.executor.ray_utils import initialize_ray_cluster
20 from vllm.inputs import PromptType
21 from vllm.logger import init_logger
22 from vllm.lora.request import LoRARequest
23 +from vllm.model_executor.guided_decoding import (
24 + get_guided_decoding_logits_processor)
25 from vllm.model_executor.layers.sampler import SamplerOutput
26 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
27 from vllm.pooling_params import PoolingParams
@@ -477,6 +479,18 @@ class _AsyncLLMEngine(LLMEngine):
479 )
480 processed_inputs = self.input_processor(preprocessed_inputs)
481
482 + if isinstance(params, SamplingParams) and \
483 + params.guided_decoding is not None:
484 + # Guided decoding has an async implementation for building logits
485 + # processors in a separate threadpool.
486 + # We want to invoke that here instead of using the blocking
487 + # implementation in the LLMEngine
488 + params = await build_guided_decoding_logits_processor_async(
489 + sampling_params=params,
490 + tokenizer=self.get_tokenizer(lora_request),
491 + default_guided_backend=self.decoding_config.
492 + guided_decoding_backend)
493 +
494 self._add_processed_request(
495 request_id=request_id,
496 processed_inputs=processed_inputs,
@@ -494,6 +508,36 @@ class _AsyncLLMEngine(LLMEngine):
508 self.model_executor.check_health()
509
510
511 +async def build_guided_decoding_logits_processor_async(
512 + sampling_params: SamplingParams, tokenizer: AnyTokenizer,
513 + default_guided_backend: str) -> SamplingParams:
514 + """Constructs logits processors based on the guided_decoding,
515 + logits_bias, and allowed_token_ids fields in sampling_params. Deletes
516 + those fields and adds the constructed logits processors to the
517 + logits_processors field. Modifies sampling params in-place and returns
518 + the modified sampling params."""
519 + if (guided_decoding := sampling_params.guided_decoding) is None:
520 + return sampling_params
521 +
522 + logger.debug("Building guided decoding logits processor. "
523 + "Params: %s", guided_decoding)
524 +
525 + guided_decoding.backend = guided_decoding.backend or default_guided_backend
526 +
527 + processor = await get_guided_decoding_logits_processor(
528 + guided_params=guided_decoding, tokenizer=tokenizer)
529 +
530 + if processor:
531 + if sampling_params.logits_processors is None:
532 + sampling_params.logits_processors = []
533 + sampling_params.logits_processors.append(processor)
534 +
535 + # Unset guided decoding params after constructing the lp from them
536 + sampling_params.guided_decoding = None
537 +
538 + return sampling_params
539 +
540 +
541 class AsyncLLMEngine:
542 """An asynchronous wrapper for :class:`LLMEngine`.
543
vllm/engine/llm_engine.py
+54
@@ -25,6 +25,7 @@ from vllm.engine.output_processor.interfaces import (
25 SequenceGroupOutputProcessor)
26 from vllm.engine.output_processor.stop_checker import StopChecker
27 from vllm.engine.output_processor.util import create_output_by_sequence_group
28 +from vllm.entrypoints.openai.logits_processors import get_logits_processors
29 from vllm.executor.executor_base import ExecutorBase
30 from vllm.executor.gpu_executor import GPUExecutor
31 from vllm.executor.ray_utils import initialize_ray_cluster
@@ -33,6 +34,8 @@ from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
34 from vllm.inputs.preprocess import InputPreprocessor
35 from vllm.logger import init_logger
36 from vllm.lora.request import LoRARequest
37 +from vllm.model_executor.guided_decoding import (
38 + get_local_guided_decoding_logits_processor)
39 from vllm.model_executor.layers.sampler import SamplerOutput
40 from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
41 RequestOutputFactory)
@@ -843,6 +846,9 @@ class LLMEngine:
846 raise ValueError(f"Cannot request more than "
847 f"{max_logprobs} logprobs.")
848
849 + sampling_params = self._build_logits_processors(
850 + sampling_params, lora_request)
851 +
852 # Defensive copy of SamplingParams, which are used by the sampler,
853 # this doesn't deep-copy LogitsProcessor objects
854 sampling_params = sampling_params.clone()
@@ -1895,3 +1901,51 @@ class LLMEngine:
1901 # TODO: Find out how many placeholder tokens are there so we can
1902 # check that chunked prefill does not truncate them
1903 # max_batch_len = self.scheduler_config.max_num_batched_tokens
1904 +
1905 + def _build_logits_processors(
1906 + self, sampling_params: SamplingParams,
1907 + lora_request: Optional[LoRARequest]) -> SamplingParams:
1908 + """Constructs logits processors based on the guided_decoding,
1909 + logits_bias, and allowed_token_ids fields in sampling_params. Deletes
1910 + those fields and adds the constructed logits processors to the
1911 + logits_processors field. Returns the modified sampling params."""
1912 +
1913 + logits_processors = []
1914 + if (guided_decoding := sampling_params.guided_decoding) is not None:
1915 +
1916 + logger.debug(
1917 + "Building guided decoding logits processor in "
1918 + "LLMEngine. Params: %s", guided_decoding)
1919 +
1920 + tokenizer = self.get_tokenizer(lora_request=lora_request)
1921 + guided_decoding.backend = guided_decoding.backend or \
1922 + self.decoding_config.guided_decoding_backend
1923 +
1924 + processor = get_local_guided_decoding_logits_processor(
1925 + guided_params=guided_decoding, tokenizer=tokenizer)
1926 + if processor:
1927 + logits_processors.append(processor)
1928 +
1929 + # Unset so this doesn't get passed down to the model
1930 + sampling_params.guided_decoding = None
1931 +
1932 + if (sampling_params.logit_bias or sampling_params.allowed_token_ids):
1933 + tokenizer = self.get_tokenizer(lora_request=lora_request)
1934 +
1935 + processors = get_logits_processors(
1936 + logit_bias=sampling_params.logit_bias,
1937 + allowed_token_ids=sampling_params.allowed_token_ids,
1938 + tokenizer=tokenizer)
1939 + logits_processors.extend(processors)
1940 +
1941 + # Unset so these don't get passed down to the model
1942 + sampling_params.logit_bias = None
1943 + sampling_params.allowed_token_ids = None
1944 +
1945 + if logits_processors:
1946 + if sampling_params.logits_processors is None:
1947 + sampling_params.logits_processors = logits_processors
1948 + else:
1949 + sampling_params.logits_processors.extend(logits_processors)
1950 +
1951 + return sampling_params
vllm/engine/multiprocessing/client.py
+14
@@ -16,6 +16,8 @@ from vllm.config import DecodingConfig, EngineConfig, ModelConfig
16 from vllm.engine.arg_utils import AsyncEngineArgs
17 # yapf conflicts with isort for this block
18 # yapf: disable
19 +from vllm.engine.async_llm_engine import (
20 + build_guided_decoding_logits_processor_async)
21 from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
22 IPC_HEALTH_EXT, IPC_INPUT_EXT,
23 IPC_OUTPUT_EXT, RPC_REQUEST_T,
@@ -512,6 +514,18 @@ class MQLLMEngineClient:
514 if self._errored_with is not None:
515 raise ENGINE_DEAD_ERROR(self._errored_with)
516
517 + # Constructing guided decoding logits processors is expensive, so we do
518 + # it here to avoid contending with cpu resources and the GIL on the
519 + # backend process.
520 + if isinstance(params, SamplingParams) and \
521 + params.guided_decoding is not None:
522 + params = await \
523 + build_guided_decoding_logits_processor_async(
524 + sampling_params=params,
525 + tokenizer=await self.get_tokenizer(lora_request),
526 + default_guided_backend=self.decoding_config.guided_decoding_backend
527 + )
528 +
529 # 1) Create output queue for this requests.
530 queue: asyncio.Queue[Union[RequestOutput,
531 BaseException]] = asyncio.Queue()
vllm/entrypoints/llm.py
+30 -18
@@ -1,4 +1,5 @@
1 import itertools
2 +import warnings
3 from contextlib import contextmanager
4 from dataclasses import dataclass
5 from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple,
@@ -16,13 +17,13 @@ from vllm.inputs import PromptType, TextPrompt, TokensPrompt
17 from vllm.inputs.parse import parse_and_batch_prompt
18 from vllm.logger import init_logger
19 from vllm.lora.request import LoRARequest
19 -from vllm.model_executor.guided_decoding import (
20 - GuidedDecodingRequest, get_local_guided_decoding_logits_processor)
21 -from vllm.model_executor.guided_decoding.guided_fields import LLMGuidedOptions
20 +from vllm.model_executor.guided_decoding.guided_fields import (
21 + GuidedDecodingRequest, LLMGuidedOptions)
22 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
23 from vllm.pooling_params import PoolingParams
24 from vllm.prompt_adapter.request import PromptAdapterRequest
25 -from vllm.sampling_params import RequestOutputKind, SamplingParams
25 +from vllm.sampling_params import (GuidedDecodingParams, RequestOutputKind,
26 + SamplingParams)
27 from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
28 get_cached_tokenizer)
29 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
@@ -798,6 +799,14 @@ class LLM:
799 guided_options: Optional[GuidedDecodingRequest] = None,
800 priority: Optional[List[int]] = None,
801 ) -> None:
802 + if guided_options is not None:
803 + warnings.warn(
804 + "guided_options_request is deprecated, use "
805 + "SamplingParams.guided_decoding instead",
806 + DeprecationWarning,
807 + stacklevel=2,
808 + )
809 +
810 if isinstance(prompts, (str, dict)):
811 # Convert a single prompt to a list.
812 prompts = [prompts]
@@ -813,7 +822,7 @@ class LLM:
822
823 for sp in params if isinstance(params, list) else (params, ):
824 if isinstance(sp, SamplingParams):
816 - self._add_guided_processor(sp, guided_options)
825 + self._add_guided_params(sp, guided_options)
826
827 # We only care about the final output
828 sp.output_kind = RequestOutputKind.FINAL_ONLY
@@ -847,22 +856,25 @@ class LLM:
856 priority=priority,
857 )
858
850 - def _add_guided_processor(
859 + def _add_guided_params(
860 self,
861 params: SamplingParams,
862 guided_options: Optional[GuidedDecodingRequest] = None):
854 - if guided_options:
855 - if guided_options.guided_decoding_backend is None:
856 - decoding_config = self.llm_engine.get_decoding_config()
857 - guided_options.guided_decoding_backend = (
858 - decoding_config.guided_decoding_backend)
859 - guided_logits_processor = get_local_guided_decoding_logits_processor( #noqa
860 - guided_options.guided_decoding_backend, guided_options,
861 - self.get_tokenizer())
862 - if guided_logits_processor:
863 - if params.logits_processors is None:
864 - params.logits_processors = []
865 - params.logits_processors.append(guided_logits_processor)
863 + if guided_options is None:
864 + return params
865 +
866 + if params.guided_decoding is not None:
867 + raise ValueError("Cannot set both guided_options_request and"
868 + "params.guided_decoding.")
869 +
870 + params.guided_decoding = GuidedDecodingParams(
871 + json=guided_options.guided_json,
872 + regex=guided_options.guided_regex,
873 + choice=guided_options.guided_choice,
874 + grammar=guided_options.guided_grammar,
875 + json_object=guided_options.guided_json_object,
876 + backend=guided_options.guided_decoding_backend,
877 + whitespace_pattern=guided_options.guided_whitespace_pattern)
878 return params
879
880 def _run_engine(
vllm/entrypoints/openai/protocol.py
+52 -30
@@ -10,12 +10,10 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
10 from typing_extensions import Annotated, Required, TypedDict
11
12 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13 -from vllm.entrypoints.openai.logits_processors import get_logits_processors
13 from vllm.pooling_params import PoolingParams
15 -from vllm.sampling_params import (LogitsProcessor, RequestOutputKind,
14 +from vllm.sampling_params import (GuidedDecodingParams, RequestOutputKind,
15 SamplingParams)
16 from vllm.sequence import Logprob
18 -from vllm.transformers_utils.tokenizer import AnyTokenizer
17 from vllm.utils import random_uuid
18
19 # torch is mocked during docs generation,
@@ -284,10 +282,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
282
283 # doc: end-chat-completion-extra-params
284
287 - def to_sampling_params(
288 - self, tokenizer: AnyTokenizer,
289 - guided_decode_logits_processor: Optional[LogitsProcessor],
290 - default_max_tokens: int) -> SamplingParams:
285 + def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
286 max_tokens = self.max_tokens
287 if max_tokens is None:
288 max_tokens = default_max_tokens
@@ -296,14 +291,19 @@ class ChatCompletionRequest(OpenAIBaseModel):
291 if prompt_logprobs is None and self.echo:
292 prompt_logprobs = self.top_logprobs
293
299 - # We now allow logprobs being true without top_logrobs.
300 - logits_processors = get_logits_processors(
301 - logit_bias=self.logit_bias,
302 - allowed_token_ids=None,
303 - tokenizer=tokenizer,
304 - )
305 - if guided_decode_logits_processor:
306 - logits_processors.append(guided_decode_logits_processor)
294 + guided_json_object = None
295 + if (self.response_format is not None
296 + and self.response_format.type == "json_object"):
297 + guided_json_object = True
298 +
299 + guided_decoding = GuidedDecodingParams.from_optional(
300 + json=self._get_guided_json_from_tool() or self.guided_json,
301 + regex=self.guided_regex,
302 + choice=self.guided_choice,
303 + grammar=self.guided_grammar,
304 + json_object=guided_json_object,
305 + backend=self.guided_decoding_backend,
306 + whitespace_pattern=self.guided_whitespace_pattern)
307
308 return SamplingParams.from_optional(
309 n=self.n,
@@ -329,11 +329,29 @@ class ChatCompletionRequest(OpenAIBaseModel):
329 spaces_between_special_tokens=self.spaces_between_special_tokens,
330 include_stop_str_in_output=self.include_stop_str_in_output,
331 length_penalty=self.length_penalty,
332 - logits_processors=logits_processors,
332 truncate_prompt_tokens=self.truncate_prompt_tokens,
333 output_kind=RequestOutputKind.DELTA if self.stream \
334 else RequestOutputKind.FINAL_ONLY,
336 - )
335 + guided_decoding=guided_decoding,
336 + logit_bias=self.logit_bias)
337 +
338 + def _get_guided_json_from_tool(
339 + self) -> Optional[Union[str, dict, BaseModel]]:
340 + # user has chosen to not use any tool
341 + if self.tool_choice == "none" or self.tools is None:
342 + return None
343 +
344 + # user has chosen to use a named tool
345 + if type(self.tool_choice) is ChatCompletionNamedToolChoiceParam:
346 + tool_name = self.tool_choice.function.name
347 + tools = {tool.function.name: tool.function for tool in self.tools}
348 + if tool_name not in tools:
349 + raise ValueError(
350 + f"Tool '{tool_name}' has not been passed in `tools`.")
351 + tool = tools[tool_name]
352 + return tool.parameters
353 +
354 + return None
355
356 @model_validator(mode="before")
357 @classmethod
@@ -537,10 +555,7 @@ class CompletionRequest(OpenAIBaseModel):
555
556 # doc: end-completion-extra-params
557
540 - def to_sampling_params(
541 - self, tokenizer: AnyTokenizer,
542 - guided_decode_logits_processor: Optional[LogitsProcessor],
543 - default_max_tokens: int) -> SamplingParams:
558 + def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
559 max_tokens = self.max_tokens
560 if max_tokens is None:
561 max_tokens = default_max_tokens
@@ -551,13 +566,19 @@ class CompletionRequest(OpenAIBaseModel):
566
567 echo_without_generation = self.echo and self.max_tokens == 0
568
554 - logits_processors = get_logits_processors(
555 - logit_bias=self.logit_bias,
556 - allowed_token_ids=self.allowed_token_ids,
557 - tokenizer=tokenizer,
558 - )
559 - if guided_decode_logits_processor:
560 - logits_processors.append(guided_decode_logits_processor)
569 + guided_json_object = None
570 + if (self.response_format is not None
571 + and self.response_format.type == "json_object"):
572 + guided_json_object = True
573 +
574 + guided_decoding = GuidedDecodingParams.from_optional(
575 + json=self.guided_json,
576 + regex=self.guided_regex,
577 + choice=self.guided_choice,
578 + grammar=self.guided_grammar,
579 + json_object=guided_json_object,
580 + backend=self.guided_decoding_backend,
581 + whitespace_pattern=self.guided_whitespace_pattern)
582
583 return SamplingParams.from_optional(
584 n=self.n,
@@ -583,11 +604,12 @@ class CompletionRequest(OpenAIBaseModel):
604 spaces_between_special_tokens=self.spaces_between_special_tokens,
605 include_stop_str_in_output=self.include_stop_str_in_output,
606 length_penalty=self.length_penalty,
586 - logits_processors=logits_processors,
607 truncate_prompt_tokens=self.truncate_prompt_tokens,
608 output_kind=RequestOutputKind.DELTA if self.stream \
609 else RequestOutputKind.FINAL_ONLY,
590 - )
610 + guided_decoding=guided_decoding,
611 + logit_bias=self.logit_bias,
612 + allowed_token_ids=self.allowed_token_ids)
613
614 @model_validator(mode="before")
615 @classmethod
vllm/entrypoints/openai/serving_chat.py
-5
@@ -187,9 +187,6 @@ class OpenAIServingChat(OpenAIServing):
187 raw_request.state.request_metadata = request_metadata
188
189 try:
190 - guided_decode_logits_processor = (
191 - await self._guided_decode_logits_processor(request, tokenizer))
192 -
190 if isinstance(prompt, str):
191 prompt_inputs = self._tokenize_prompt_input(
192 request,
@@ -208,8 +205,6 @@ class OpenAIServingChat(OpenAIServing):
205 assert prompt_inputs is not None
206
207 sampling_params = request.to_sampling_params(
211 - tokenizer,
212 - guided_decode_logits_processor,
208 default_max_tokens=self.max_model_len -
209 len(prompt_inputs["prompt_token_ids"]))
210
vllm/entrypoints/openai/serving_completion.py
-4
@@ -110,8 +110,6 @@ class OpenAIServingCompletion(OpenAIServing):
110
111 tokenizer = await self.engine_client.get_tokenizer(lora_request)
112
113 - guided_decode_logits_processor = (
114 - await self._guided_decode_logits_processor(request, tokenizer))
113 prompts = list(
114 self._tokenize_prompt_input_or_inputs(
115 request,
@@ -123,8 +121,6 @@ class OpenAIServingCompletion(OpenAIServing):
121
122 for i, prompt_inputs in enumerate(prompts):
123 sampling_params = request.to_sampling_params(
126 - tokenizer,
127 - guided_decode_logits_processor,
124 default_max_tokens=self.max_model_len -
125 len(prompt_inputs["prompt_token_ids"]))
126
vllm/entrypoints/openai/serving_engine.py
+1 -12
@@ -27,11 +27,9 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
27 from vllm.inputs.parse import parse_and_batch_prompt
28 from vllm.logger import init_logger
29 from vllm.lora.request import LoRARequest
30 -from vllm.model_executor.guided_decoding import (
31 - get_guided_decoding_logits_processor)
30 from vllm.pooling_params import PoolingParams
31 from vllm.prompt_adapter.request import PromptAdapterRequest
34 -from vllm.sampling_params import LogitsProcessor, SamplingParams
32 +from vllm.sampling_params import SamplingParams
33 from vllm.sequence import Logprob
34 from vllm.transformers_utils.tokenizer import AnyTokenizer
35 from vllm.utils import AtomicCounter
@@ -168,15 +166,6 @@ class OpenAIServing:
166 })
167 return json_str
168
171 - async def _guided_decode_logits_processor(
172 - self, request: Union[ChatCompletionRequest, CompletionRequest],
173 - tokenizer: AnyTokenizer) -> Optional[LogitsProcessor]:
174 - decoding_config = await self.engine_client.get_decoding_config()
175 - guided_decoding_backend = request.guided_decoding_backend \
176 - or decoding_config.guided_decoding_backend
177 - return await get_guided_decoding_logits_processor(
178 - guided_decoding_backend, request, tokenizer)
179 -
169 async def _check_model(
170 self,
171 request: AnyRequest,
vllm/model_executor/guided_decoding/__init__.py
+18 -50
@@ -1,77 +1,45 @@
1 -from typing import Optional, Union
1 +from typing import Optional
2
3 -from vllm.entrypoints.openai.protocol import (
4 - ChatCompletionNamedToolChoiceParam, ChatCompletionRequest,
5 - CompletionRequest)
6 -from vllm.model_executor.guided_decoding.guided_fields import (
7 - GuidedDecodingRequest)
8 -from vllm.sampling_params import LogitsProcessor
3 +from vllm.sampling_params import GuidedDecodingParams, LogitsProcessor
4
5
6 async def get_guided_decoding_logits_processor(
12 - guided_decoding_backend: str, request: Union[CompletionRequest,
13 - ChatCompletionRequest],
7 + guided_params: GuidedDecodingParams,
8 tokenizer) -> Optional[LogitsProcessor]:
15 - request = _adapt_request_for_tool_use(request)
16 -
17 - if guided_decoding_backend == 'outlines':
9 + # CFG grammar not supported by LMFE, so we use outlines instead
10 + if guided_params.backend == 'outlines' or guided_params.grammar:
11 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
12 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
13 get_outlines_guided_decoding_logits_processor)
14 return await get_outlines_guided_decoding_logits_processor(
22 - request, tokenizer)
23 - if guided_decoding_backend == 'lm-format-enforcer':
15 + guided_params, tokenizer)
16 + if guided_params.backend == 'lm-format-enforcer':
17 from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
25 - get_lm_format_enforcer_guided_decoding_logits_processor)
26 - return await get_lm_format_enforcer_guided_decoding_logits_processor(
27 - request, tokenizer)
18 + get_local_lm_format_enforcer_guided_decoding_logits_processor)
19 + return get_local_lm_format_enforcer_guided_decoding_logits_processor(
20 + guided_params, tokenizer)
21
22 raise ValueError(
30 - f"Unknown guided decoding backend '{guided_decoding_backend}'. "
23 + f"Unknown guided decoding backend '{guided_params.backend}'. "
24 "Must be one of 'outlines, 'lm-format-enforcer'")
25
26
27 def get_local_guided_decoding_logits_processor(
35 - guided_decoding_backend: str, guided_options: GuidedDecodingRequest,
28 + guided_params: GuidedDecodingParams,
29 tokenizer) -> Optional[LogitsProcessor]:
37 - # request = _adapt_request_for_tool_use(request)
38 -
39 - if guided_decoding_backend == 'outlines':
30 + # CFG grammar not supported by LMFE, so we use outlines instead
31 + if guided_params.backend == 'outlines' or guided_params.grammar:
32 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
33 from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
34 get_local_outlines_guided_decoding_logits_processor)
35 return get_local_outlines_guided_decoding_logits_processor(
44 - guided_options, tokenizer)
45 - if guided_decoding_backend == 'lm-format-enforcer':
36 + guided_params, tokenizer)
37 + if guided_params.backend == 'lm-format-enforcer':
38 from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
39 get_local_lm_format_enforcer_guided_decoding_logits_processor)
40 return get_local_lm_format_enforcer_guided_decoding_logits_processor(
49 - guided_options, tokenizer)
41 + guided_params, tokenizer)
42
43 raise ValueError(
52 - f"Unknown guided decoding backend '{guided_decoding_backend}'. "
44 + f"Unknown guided decoding backend '{guided_params.backend}'. "
45 "Must be one of 'outlines, 'lm-format-enforcer'")
54 -
55 -
56 -def _adapt_request_for_tool_use(request: Union[CompletionRequest,
57 - ChatCompletionRequest]):
58 - # the legacy completion API does not support tool use
59 - if type(request) is CompletionRequest:
60 - return request
61 -
62 - # user has chosen to not use any tool,
63 - # OR is allowing the model to choose a tool.
64 - if request.tool_choice == "none" or request.tool_choice == "auto":
65 - return request
66 -
67 - # user has chosen to use a named tool
68 - if type(request.tool_choice) is ChatCompletionNamedToolChoiceParam:
69 - tool_name = request.tool_choice.function.name
70 - tools = {tool.function.name: tool.function for tool in request.tools}
71 - if tool_name not in tools:
72 - raise ValueError(
73 - f"Tool '{tool_name}' has not been passed in `tools`.")
74 - tool = tools[tool_name]
75 - request.guided_json = tool.parameters
76 -
77 - return request
vllm/model_executor/guided_decoding/guided_fields.py
+1
@@ -4,6 +4,7 @@ from typing import Dict, List, Optional, TypedDict, Union
4 from pydantic import BaseModel
5
6
7 +# These classes are deprecated, see SamplingParams
8 class LLMGuidedOptions(TypedDict, total=False):
9 guided_json: Union[Dict, BaseModel, str]
10 guided_regex: str
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+16 -74
@@ -7,66 +7,13 @@ from lmformatenforcer import (CharacterLevelParser, JsonSchemaParser,
7 TokenEnforcerTokenizerData, UnionParser)
8 from lmformatenforcer.integrations.vllm import (
9 build_vllm_logits_processor, build_vllm_token_enforcer_tokenizer_data)
10 -from pydantic import BaseModel
10 from transformers import PreTrainedTokenizerBase
11
13 -from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14 - CompletionRequest)
15 -from vllm.model_executor.guided_decoding.guided_fields import (
16 - GuidedDecodingRequest)
17 -from vllm.sampling_params import LogitsProcessor
18 -
19 -
20 -async def get_lm_format_enforcer_guided_decoding_logits_processor(
21 - request: Union[CompletionRequest, ChatCompletionRequest],
22 - tokenizer) -> Optional[LogitsProcessor]:
23 - """
24 - Given an OpenAI-compatible request, check for guided decoding parameters
25 - and get the necessary logits processor for the given guide.
26 - We cache logit processors by (guide, tokenizer), and on cache hit
27 - we make a shallow copy to reuse the same underlying FSM.
28 - """
29 -
30 - tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
31 - tokenizer)
32 - character_level_parser: CharacterLevelParser
33 - if request.guided_json:
34 - schema = _normalize_json_schema_object(request.guided_json)
35 - character_level_parser = JsonSchemaParser(schema)
36 - elif request.guided_choice:
37 - character_level_parser = UnionParser(
38 - [StringParser(choice) for choice in request.guided_choice])
39 - elif request.guided_regex:
40 - character_level_parser = RegexParser(request.guided_regex)
41 - elif request.guided_grammar:
42 - # CFG grammar not supported by LMFE, revert to outlines
43 -
44 - # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
45 - from vllm.model_executor.guided_decoding.outlines_decoding import (
46 - get_outlines_guided_decoding_logits_processor)
47 - return await get_outlines_guided_decoding_logits_processor(
48 - request, tokenizer)
49 - elif (request.response_format is not None
50 - and request.response_format.type == "json_object"):
51 - character_level_parser = JsonSchemaParser(
52 - None) # None means any json object
53 - elif (request.response_format is not None
54 - and request.response_format.type == "json_schema"
55 - and request.response_format.json_schema is not None
56 - and request.response_format.json_schema.json_schema is not None):
57 - schema = _normalize_json_schema_object(
58 - request.response_format.json_schema.json_schema)
59 - character_level_parser = JsonSchemaParser(schema)
60 - else:
61 - return None
62 -
63 - logits_processor = build_vllm_logits_processor(tokenizer_data,
64 - character_level_parser)
65 - return logits_processor
12 +from vllm.sampling_params import GuidedDecodingParams, LogitsProcessor
13
14
15 def get_local_lm_format_enforcer_guided_decoding_logits_processor(
69 - guided_options: GuidedDecodingRequest,
16 + guided_params: GuidedDecodingParams,
17 tokenizer) -> Optional[LogitsProcessor]:
18 """
19 Given an OpenAI-compatible request, check for guided decoding parameters
@@ -78,23 +25,20 @@ def get_local_lm_format_enforcer_guided_decoding_logits_processor(
25 tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
26 tokenizer)
27 character_level_parser: CharacterLevelParser
81 - if guided_options.guided_json:
82 - schema = _normalize_json_schema_object(guided_options.guided_json)
83 - character_level_parser = JsonSchemaParser(schema)
84 - elif guided_options.guided_choice:
28 + if guided_params.json:
29 + schema_dict = _normalize_json_schema_object(guided_params.json)
30 + character_level_parser = JsonSchemaParser(schema_dict)
31 + elif guided_params.choice:
32 character_level_parser = UnionParser(
86 - [StringParser(choice) for choice in guided_options.guided_choice])
87 - elif guided_options.guided_regex:
88 - character_level_parser = RegexParser(guided_options.guided_regex)
89 - elif guided_options.guided_grammar:
90 - # CFG grammar not supported by LMFE, revert to outlines
91 -
92 - # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
93 - from vllm.model_executor.guided_decoding.outlines_decoding import (
94 - get_local_outlines_guided_decoding_logits_processor)
95 - return get_local_outlines_guided_decoding_logits_processor(
96 - guided_options, tokenizer)
97 - elif guided_options.guided_json_object:
33 + [StringParser(choice) for choice in guided_params.choice])
34 + elif guided_params.regex:
35 + character_level_parser = RegexParser(guided_params.regex)
36 + elif guided_params.grammar:
37 + # CFG grammar not supported by LMFE
38 + raise ValueError("Cannot construct a guided decoding logits processor"
39 + " using the grammar option with the"
40 + " lm_format_enforcer backend.")
41 + elif guided_params.json_object:
42 # None means any json object
43 character_level_parser = JsonSchemaParser(None)
44 else:
@@ -105,13 +49,11 @@ def get_local_lm_format_enforcer_guided_decoding_logits_processor(
49 return logits_processor
50
51
108 -def _normalize_json_schema_object(schema: Union[str, dict, BaseModel]) -> dict:
52 +def _normalize_json_schema_object(schema: Union[str, dict]) -> dict:
53 if isinstance(schema, str):
54 return json_loads(schema)
55 if isinstance(schema, dict):
56 return schema
113 - if isinstance(schema, BaseModel):
114 - return schema.model_json_schema()
57 raise AssertionError(f"Unsupported schema type {schema}")
58
59
vllm/model_executor/guided_decoding/outlines_decoding.py
+19 -53
@@ -5,16 +5,11 @@ from json import dumps as json_dumps
5 from re import escape as regex_escape
6 from typing import Tuple, Union
7
8 -from pydantic import BaseModel
8 from transformers import PreTrainedTokenizerBase
9
11 -from vllm.entrypoints.openai.protocol import (
12 - ChatCompletionNamedToolChoiceParam, ChatCompletionRequest,
13 - CompletionRequest)
14 -from vllm.model_executor.guided_decoding.guided_fields import (
15 - GuidedDecodingRequest)
10 from vllm.model_executor.guided_decoding.outlines_logits_processors import (
11 CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor)
12 +from vllm.sampling_params import GuidedDecodingParams
13
14
15 class GuidedDecodingMode(Enum):
@@ -55,8 +50,7 @@ global_thread_pool = None # used for generating logits processor fsm
50
51
52 async def get_outlines_guided_decoding_logits_processor(
58 - request: Union[CompletionRequest,
59 - ChatCompletionRequest], tokenizer: PreTrainedTokenizerBase
53 + guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase
54 ) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
55 None]:
56 """
@@ -66,7 +60,7 @@ async def get_outlines_guided_decoding_logits_processor(
60 we make a shallow copy to reuse the same underlying FSM.
61 """
62 global global_thread_pool
69 - guide, mode = _get_guide_and_mode(request)
63 + guide, mode = _get_guide_and_mode(guided_params)
64 if not guide or not mode:
65 return None
66
@@ -77,11 +71,11 @@ async def get_outlines_guided_decoding_logits_processor(
71
72 return await loop.run_in_executor(global_thread_pool,
73 _get_logits_processor, guide, tokenizer,
80 - mode, request.guided_whitespace_pattern)
74 + mode, guided_params.whitespace_pattern)
75
76
77 def get_local_outlines_guided_decoding_logits_processor(
84 - guided_options: GuidedDecodingRequest, tokenizer: PreTrainedTokenizerBase
78 + guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase
79 ) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
80 None]:
81 """
@@ -90,65 +84,37 @@ def get_local_outlines_guided_decoding_logits_processor(
84 We cache logit processors by (guide, tokenizer), and on cache hit
85 we make a shallow copy to reuse the same underlying FSM.
86 """
93 - guide, mode = _get_guide_and_mode(guided_options)
87 + guide, mode = _get_guide_and_mode(guided_params)
88 if not guide or not mode:
89 return None
90
91 return _get_logits_processor(guide, tokenizer, mode,
98 - guided_options.guided_whitespace_pattern)
92 + guided_params.whitespace_pattern)
93
94
95 def _get_guide_and_mode(
102 - request: Union[CompletionRequest, ChatCompletionRequest,
103 - GuidedDecodingRequest]
96 + guided_params: GuidedDecodingParams
97 ) -> Union[Tuple[str, GuidedDecodingMode], Tuple[None, None]]:
105 - # if the request is a chat completion request, AND the tool choice is a
106 - # named tool choice, do guided decoding
107 - # using that tool as the JSON schema
108 - if isinstance(request, ChatCompletionRequest) and isinstance(
109 - request.tool_choice, ChatCompletionNamedToolChoiceParam):
110 - # Guided generation for tools/functions parameters
111 - if request.tool_choice.type == "function":
112 - for tool in request.tools:
113 - if (tool.type == "function" and tool.function.name
114 - == request.tool_choice.function.name):
115 - json = json_dumps(tool.function.parameters, sort_keys=True)
116 - return json, GuidedDecodingMode.JSON
117 - return None, None
118 -
119 - elif request.guided_json:
120 - if isinstance(request.guided_json, dict):
98 + if guided_params.json:
99 + if isinstance(guided_params.json, dict):
100 # turn dict into hashable string
122 - json = json_dumps(request.guided_json)
123 - elif isinstance(request.guided_json, BaseModel):
124 - # use pydantic signature so that different model classes
125 - # with the same fields will get hashed the same
126 - json = str(request.guided_json.__signature__)
101 + json = json_dumps(guided_params.json)
102 else:
128 - json = request.guided_json
103 + json = guided_params.json
104 return json, GuidedDecodingMode.JSON
130 - elif request.guided_regex:
131 - return request.guided_regex, GuidedDecodingMode.REGEX
132 - elif request.guided_choice:
105 + elif guided_params.regex:
106 + return guided_params.regex, GuidedDecodingMode.REGEX
107 + elif guided_params.choice:
108 # choice just uses regex
109 choices = [
135 - regex_escape(str(choice)) for choice in request.guided_choice
110 + regex_escape(str(choice)) for choice in guided_params.choice
111 ]
112 choices_regex = "(" + "|".join(choices) + ")"
113 return choices_regex, GuidedDecodingMode.CHOICE
139 - elif request.guided_grammar:
140 - return request.guided_grammar, GuidedDecodingMode.GRAMMAR
141 - elif (not isinstance(request, GuidedDecodingRequest)
142 - and request.response_format is not None
143 - and request.response_format.type == "json_object"):
114 + elif guided_params.grammar:
115 + return guided_params.grammar, GuidedDecodingMode.GRAMMAR
116 + elif guided_params.json_object:
117 return JSON_GRAMMAR, GuidedDecodingMode.GRAMMAR
145 - elif (not isinstance(request, GuidedDecodingRequest)
146 - and request.response_format is not None
147 - and request.response_format.type == "json_schema"
148 - and request.response_format.json_schema is not None
149 - and request.response_format.json_schema.json_schema is not None):
150 - json = json_dumps(request.response_format.json_schema.json_schema)
151 - return json, GuidedDecodingMode.JSON
118 else:
119 return None, None
120
vllm/sampling_params.py
+76 -1
@@ -1,11 +1,13 @@
1 """Sampling parameters for text generation."""
2 import copy
3 +from dataclasses import dataclass
4 from enum import Enum, IntEnum
5 from functools import cached_property
6 from typing import Any, Callable, Dict, List, Optional, Set, Union
7
8 import msgspec
9 import torch
10 +from pydantic import BaseModel
11 from typing_extensions import Annotated
12
13 import vllm.envs as envs
@@ -34,6 +36,54 @@ first argument, and returns a modified tensor of logits
36 to sample from."""
37
38
39 +# maybe make msgspec?
40 +@dataclass
41 +class GuidedDecodingParams:
42 + """One of these fields will be used to build a logit processor."""
43 + json: Optional[Union[str, Dict]] = None
44 + regex: Optional[str] = None
45 + choice: Optional[List[str]] = None
46 + grammar: Optional[str] = None
47 + json_object: Optional[bool] = None
48 + """These are other options that can be set"""
49 + backend: Optional[str] = None
50 + whitespace_pattern: Optional[str] = None
51 +
52 + @staticmethod
53 + def from_optional(
54 + json: Optional[Union[Dict, BaseModel, str]],
55 + regex: Optional[str] = None,
56 + choice: Optional[List[str]] = None,
57 + grammar: Optional[str] = None,
58 + json_object: Optional[bool] = None,
59 + backend: Optional[str] = None,
60 + whitespace_pattern: Optional[str] = None,
61 + ) -> "GuidedDecodingParams":
62 + # Extract json schemas from pydantic models
63 + if isinstance(json, (BaseModel, type(BaseModel))):
64 + json = json.model_json_schema()
65 + return GuidedDecodingParams(
66 + json=json,
67 + regex=regex,
68 + choice=choice,
69 + grammar=grammar,
70 + json_object=json_object,
71 + backend=backend,
72 + whitespace_pattern=whitespace_pattern,
73 + )
74 +
75 + def __post_init__(self):
76 + """Validate that some fields are mutually exclusive."""
77 + guide_count = sum([
78 + self.json is not None, self.regex is not None, self.choice
79 + is not None, self.grammar is not None, self.json_object is not None
80 + ])
81 + if guide_count > 1:
82 + raise ValueError(
83 + "You can only use one kind of guided decoding but multiple are "
84 + f"specified: {self.__dict__}")
85 +
86 +
87 class RequestOutputKind(Enum):
88 # Return entire output so far in every RequestOutput
89 CUMULATIVE = 0
@@ -124,6 +174,13 @@ class SamplingParams(
174 truncate_prompt_tokens: If set to an integer k, will use only the last k
175 tokens from the prompt (i.e., left truncation). Defaults to None
176 (i.e., no truncation).
177 + guided_decoding: If provided, the engine will construct a guided
178 + decoding logits processor from these parameters. Defaults to None.
179 + logit_bias: If provided, the engine will construct a logits processor
180 + that applies these logit biases. Defaults to None.
181 + allowed_token_ids: If provided, the engine will construct a logits
182 + processor which only retains scores for the given token ids.
183 + Defaults to None.
184 """
185
186 n: int = 1
@@ -164,6 +221,11 @@ class SamplingParams(
221 output_text_buffer_length: int = 0
222 _all_stop_token_ids: Set[int] = msgspec.field(default_factory=set)
223
224 + # Fields used to construct logits processors
225 + guided_decoding: Optional[GuidedDecodingParams] = None
226 + logit_bias: Optional[Dict[int, float]] = None
227 + allowed_token_ids: Optional[List[int]] = None
228 +
229 @staticmethod
230 def from_optional(
231 n: Optional[int] = 1,
@@ -194,7 +256,16 @@ class SamplingParams(
256 truncate_prompt_tokens: Optional[Annotated[int,
257 msgspec.Meta(ge=1)]] = None,
258 output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
259 + guided_decoding: Optional[GuidedDecodingParams] = None,
260 + logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]] = None,
261 + allowed_token_ids: Optional[List[int]] = None,
262 ) -> "SamplingParams":
263 + if logit_bias is not None:
264 + logit_bias = {
265 + int(token): bias
266 + for token, bias in logit_bias.items()
267 + }
268 +
269 return SamplingParams(
270 n=1 if n is None else n,
271 best_of=best_of,
@@ -226,6 +297,9 @@ class SamplingParams(
297 logits_processors=logits_processors,
298 truncate_prompt_tokens=truncate_prompt_tokens,
299 output_kind=output_kind,
300 + guided_decoding=guided_decoding,
301 + logit_bias=logit_bias,
302 + allowed_token_ids=allowed_token_ids,
303 )
304
305 def __post_init__(self) -> None:
@@ -454,4 +528,5 @@ class SamplingParams(
528 f"skip_special_tokens={self.skip_special_tokens}, "
529 "spaces_between_special_tokens="
530 f"{self.spaces_between_special_tokens}, "
457 - f"truncate_prompt_tokens={self.truncate_prompt_tokens})")
531 + f"truncate_prompt_tokens={self.truncate_prompt_tokens}), "
532 + f"guided_decoding={self.guided_decoding}")