A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Frontend] Add `logits_processors` as an extra completion argument (#11150)
Signed-off-by: Brad Hilton <brad.hilton.nw@gmail.com>
Brad Hilton committed
Dec 14, 2024 at 09:46 UTC
9c3dadd1c97df2b37388c6898a0725457391f647
6 files changed
+127
-39
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -30,6 +30,7 @@ class MockModelConfig:
30
tokenizer_revision = None
31
multimodal_config = MultiModalConfig()
32
hf_config = MockHFConfig()
33
+ logits_processor_pattern = None
34
35
36
@dataclass
vllm/config.py
+38
-33
@@ -156,41 +156,45 @@ class ModelConfig:
156
can not be gathered from the vllm arguments.
157
override_pooler_config: Initialize non default pooling config or
158
override default pooling config for the pooling model.
159
+ logits_processor_pattern: Optional regex pattern specifying valid
160
+ logits processor qualified names that can be passed with the
161
+ `logits_processors` extra completion argument. Defaults to None,
162
+ which allows no processors.
163
"""
164
161
- def __init__(
162
- self,
163
- model: str,
164
- task: Union[TaskOption, Literal["draft"]],
165
- tokenizer: str,
166
- tokenizer_mode: str,
167
- trust_remote_code: bool,
168
- dtype: Union[str, torch.dtype],
169
- seed: int,
170
- allowed_local_media_path: str = "",
171
- revision: Optional[str] = None,
172
- code_revision: Optional[str] = None,
173
- rope_scaling: Optional[Dict[str, Any]] = None,
174
- rope_theta: Optional[float] = None,
175
- tokenizer_revision: Optional[str] = None,
176
- max_model_len: Optional[int] = None,
177
- spec_target_max_model_len: Optional[int] = None,
178
- quantization: Optional[str] = None,
179
- quantization_param_path: Optional[str] = None,
180
- enforce_eager: Optional[bool] = None,
181
- max_seq_len_to_capture: Optional[int] = None,
182
- max_logprobs: int = 20,
183
- disable_sliding_window: bool = False,
184
- skip_tokenizer_init: bool = False,
185
- served_model_name: Optional[Union[str, List[str]]] = None,
186
- limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
187
- use_async_output_proc: bool = True,
188
- config_format: ConfigFormat = ConfigFormat.AUTO,
189
- hf_overrides: Optional[HfOverrides] = None,
190
- mm_processor_kwargs: Optional[Dict[str, Any]] = None,
191
- mm_cache_preprocessor: bool = False,
192
- override_neuron_config: Optional[Dict[str, Any]] = None,
193
- override_pooler_config: Optional["PoolerConfig"] = None) -> None:
165
+ def __init__(self,
166
+ model: str,
167
+ task: Union[TaskOption, Literal["draft"]],
168
+ tokenizer: str,
169
+ tokenizer_mode: str,
170
+ trust_remote_code: bool,
171
+ dtype: Union[str, torch.dtype],
172
+ seed: int,
173
+ allowed_local_media_path: str = "",
174
+ revision: Optional[str] = None,
175
+ code_revision: Optional[str] = None,
176
+ rope_scaling: Optional[Dict[str, Any]] = None,
177
+ rope_theta: Optional[float] = None,
178
+ tokenizer_revision: Optional[str] = None,
179
+ max_model_len: Optional[int] = None,
180
+ spec_target_max_model_len: Optional[int] = None,
181
+ quantization: Optional[str] = None,
182
+ quantization_param_path: Optional[str] = None,
183
+ enforce_eager: Optional[bool] = None,
184
+ max_seq_len_to_capture: Optional[int] = None,
185
+ max_logprobs: int = 20,
186
+ disable_sliding_window: bool = False,
187
+ skip_tokenizer_init: bool = False,
188
+ served_model_name: Optional[Union[str, List[str]]] = None,
189
+ limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
190
+ use_async_output_proc: bool = True,
191
+ config_format: ConfigFormat = ConfigFormat.AUTO,
192
+ hf_overrides: Optional[HfOverrides] = None,
193
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None,
194
+ mm_cache_preprocessor: bool = False,
195
+ override_neuron_config: Optional[Dict[str, Any]] = None,
196
+ override_pooler_config: Optional["PoolerConfig"] = None,
197
+ logits_processor_pattern: Optional[str] = None) -> None:
198
self.model = model
199
self.tokenizer = tokenizer
200
self.tokenizer_mode = tokenizer_mode
@@ -316,6 +320,7 @@ class ModelConfig:
320
self.task: Final = task
321
322
self.pooler_config = self._init_pooler_config(override_pooler_config)
323
+ self.logits_processor_pattern = logits_processor_pattern
324
325
self._verify_quantization()
326
self._verify_cuda_graph()
vllm/engine/arg_utils.py
+10
-1
@@ -170,6 +170,7 @@ class EngineArgs:
170
enable_chunked_prefill: Optional[bool] = None
171
172
guided_decoding_backend: str = 'xgrammar'
173
+ logits_processor_pattern: Optional[str] = None
174
# Speculative decoding configuration.
175
speculative_model: Optional[str] = None
176
speculative_model_quantization: Optional[str] = None
@@ -374,6 +375,14 @@ class EngineArgs:
375
'https://github.com/noamgat/lm-format-enforcer.'
376
' Can be overridden per request via guided_decoding_backend'
377
' parameter.')
378
+ parser.add_argument(
379
+ '--logits-processor-pattern',
380
+ type=nullable_str,
381
+ default=None,
382
+ help='Optional regex pattern specifying valid logits processor '
383
+ 'qualified names that can be passed with the `logits_processors` '
384
+ 'extra completion argument. Defaults to None, which allows no '
385
+ 'processors.')
386
# Parallel arguments
387
parser.add_argument(
388
'--distributed-executor-backend',
@@ -975,7 +984,7 @@ class EngineArgs:
984
mm_cache_preprocessor=self.mm_cache_preprocessor,
985
override_neuron_config=self.override_neuron_config,
986
override_pooler_config=self.override_pooler_config,
978
- )
987
+ logits_processor_pattern=self.logits_processor_pattern)
988
989
def create_load_config(self) -> LoadConfig:
990
return LoadConfig(
vllm/entrypoints/openai/protocol.py
+74
-3
@@ -1,5 +1,6 @@
1
# Adapted from
2
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
3
+import re
4
import time
5
from argparse import Namespace
6
from typing import Any, Dict, List, Literal, Optional, Union
@@ -14,7 +15,7 @@ from vllm.pooling_params import PoolingParams
15
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
16
RequestOutputKind, SamplingParams)
17
from vllm.sequence import Logprob
17
-from vllm.utils import random_uuid
18
+from vllm.utils import random_uuid, resolve_obj_by_qualname
19
20
logger = init_logger(__name__)
21
@@ -148,6 +149,46 @@ class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
149
type: Literal["function"] = "function"
150
151
152
+class LogitsProcessorConstructor(BaseModel):
153
+ qualname: str
154
+ args: Optional[List[Any]] = None
155
+ kwargs: Optional[Dict[str, Any]] = None
156
+
157
+
158
+LogitsProcessors = List[Union[str, LogitsProcessorConstructor]]
159
+
160
+
161
+def get_logits_processors(processors: Optional[LogitsProcessors],
162
+ pattern: Optional[str]) -> Optional[List[Any]]:
163
+ if processors and pattern:
164
+ logits_processors = []
165
+ for processor in processors:
166
+ qualname = processor if isinstance(processor,
167
+ str) else processor.qualname
168
+ if not re.match(pattern, qualname):
169
+ raise ValueError(
170
+ f"Logits processor '{qualname}' is not allowed by this "
171
+ "server. See --logits-processor-pattern engine argument "
172
+ "for more information.")
173
+ try:
174
+ logits_processor = resolve_obj_by_qualname(qualname)
175
+ except Exception as e:
176
+ raise ValueError(
177
+ f"Logits processor '{qualname}' could not be resolved: {e}"
178
+ ) from e
179
+ if isinstance(processor, LogitsProcessorConstructor):
180
+ logits_processor = logits_processor(*processor.args or [],
181
+ **processor.kwargs or {})
182
+ logits_processors.append(logits_processor)
183
+ return logits_processors
184
+ elif processors:
185
+ raise ValueError(
186
+ "The `logits_processors` argument is not supported by this "
187
+ "server. See --logits-processor-pattern engine argugment "
188
+ "for more information.")
189
+ return None
190
+
191
+
192
class ChatCompletionRequest(OpenAIBaseModel):
193
# Ordered by official OpenAI API documentation
194
# https://platform.openai.com/docs/api-reference/chat/create
@@ -293,6 +334,17 @@ class ChatCompletionRequest(OpenAIBaseModel):
334
"The request_id related to this request. If the caller does "
335
"not set it, a random_uuid will be generated. This id is used "
336
"through out the inference process and return in response."))
337
+ logits_processors: Optional[LogitsProcessors] = Field(
338
+ default=None,
339
+ description=(
340
+ "A list of either qualified names of logits processors, or "
341
+ "constructor objects, to apply when sampling. A constructor is "
342
+ "a JSON object with a required 'qualname' field specifying the "
343
+ "qualified name of the processor class/factory, and optional "
344
+ "'args' and 'kwargs' fields containing positional and keyword "
345
+ "arguments. For example: {'qualname': "
346
+ "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
347
+ "{'param': 'value'}}."))
348
349
# doc: end-chat-completion-extra-params
350
@@ -314,7 +366,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
366
length_penalty=self.length_penalty,
367
include_stop_str_in_output=self.include_stop_str_in_output)
368
317
- def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
369
+ def to_sampling_params(
370
+ self, default_max_tokens: int,
371
+ logits_processor_pattern: Optional[str]) -> SamplingParams:
372
# TODO(#9845): remove max_tokens when field is removed from OpenAI API
373
max_tokens = self.max_completion_tokens or self.max_tokens
374
if max_tokens is None:
@@ -364,6 +418,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
418
min_tokens=self.min_tokens,
419
skip_special_tokens=self.skip_special_tokens,
420
spaces_between_special_tokens=self.spaces_between_special_tokens,
421
+ logits_processors=get_logits_processors(self.logits_processors,
422
+ logits_processor_pattern),
423
include_stop_str_in_output=self.include_stop_str_in_output,
424
truncate_prompt_tokens=self.truncate_prompt_tokens,
425
output_kind=RequestOutputKind.DELTA if self.stream \
@@ -599,6 +655,17 @@ class CompletionRequest(OpenAIBaseModel):
655
"The priority of the request (lower means earlier handling; "
656
"default: 0). Any priority other than 0 will raise an error "
657
"if the served model does not use priority scheduling."))
658
+ logits_processors: Optional[LogitsProcessors] = Field(
659
+ default=None,
660
+ description=(
661
+ "A list of either qualified names of logits processors, or "
662
+ "constructor objects, to apply when sampling. A constructor is "
663
+ "a JSON object with a required 'qualname' field specifying the "
664
+ "qualified name of the processor class/factory, and optional "
665
+ "'args' and 'kwargs' fields containing positional and keyword "
666
+ "arguments. For example: {'qualname': "
667
+ "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
668
+ "{'param': 'value'}}."))
669
670
# doc: end-completion-extra-params
671
@@ -619,7 +686,9 @@ class CompletionRequest(OpenAIBaseModel):
686
length_penalty=self.length_penalty,
687
include_stop_str_in_output=self.include_stop_str_in_output)
688
622
- def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
689
+ def to_sampling_params(
690
+ self, default_max_tokens: int,
691
+ logits_processor_pattern: Optional[str]) -> SamplingParams:
692
max_tokens = self.max_tokens
693
if max_tokens is None:
694
max_tokens = default_max_tokens
@@ -665,6 +734,8 @@ class CompletionRequest(OpenAIBaseModel):
734
skip_special_tokens=self.skip_special_tokens,
735
spaces_between_special_tokens=self.spaces_between_special_tokens,
736
include_stop_str_in_output=self.include_stop_str_in_output,
737
+ logits_processors=get_logits_processors(self.logits_processors,
738
+ logits_processor_pattern),
739
truncate_prompt_tokens=self.truncate_prompt_tokens,
740
output_kind=RequestOutputKind.DELTA if self.stream \
741
else RequestOutputKind.FINAL_ONLY,
vllm/entrypoints/openai/serving_chat.py
+2
-1
@@ -197,7 +197,8 @@ class OpenAIServingChat(OpenAIServing):
197
default_max_tokens)
198
else:
199
sampling_params = request.to_sampling_params(
200
- default_max_tokens)
200
+ default_max_tokens,
201
+ self.model_config.logits_processor_pattern)
202
203
self._log_inputs(request_id,
204
request_prompts[i],
vllm/entrypoints/openai/serving_completion.py
+2
-1
@@ -123,7 +123,8 @@ class OpenAIServingCompletion(OpenAIServing):
123
default_max_tokens)
124
else:
125
sampling_params = request.to_sampling_params(
126
- default_max_tokens)
126
+ default_max_tokens,
127
+ self.model_config.logits_processor_pattern)
128
129
request_id_item = f"{request_id}-{i}"
130