@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9c3dadd1c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Frontend] Add `logits_processors` as an extra completion argument (#11150)

Signed-off-by: Brad Hilton <brad.hilton.nw@gmail.com>

Brad Hilton committed Dec 14, 2024 at 09:46 UTC 9c3dadd1c97df2b37388c6898a0725457391f647
6 files changed +127 -39
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -30,6 +30,7 @@ class MockModelConfig:
30 tokenizer_revision = None
31 multimodal_config = MultiModalConfig()
32 hf_config = MockHFConfig()
33 + logits_processor_pattern = None
34
35
36 @dataclass
vllm/config.py
+38 -33
@@ -156,41 +156,45 @@ class ModelConfig:
156 can not be gathered from the vllm arguments.
157 override_pooler_config: Initialize non default pooling config or
158 override default pooling config for the pooling model.
159 + logits_processor_pattern: Optional regex pattern specifying valid
160 + logits processor qualified names that can be passed with the
161 + `logits_processors` extra completion argument. Defaults to None,
162 + which allows no processors.
163 """
164
161 - def __init__(
162 - self,
163 - model: str,
164 - task: Union[TaskOption, Literal["draft"]],
165 - tokenizer: str,
166 - tokenizer_mode: str,
167 - trust_remote_code: bool,
168 - dtype: Union[str, torch.dtype],
169 - seed: int,
170 - allowed_local_media_path: str = "",
171 - revision: Optional[str] = None,
172 - code_revision: Optional[str] = None,
173 - rope_scaling: Optional[Dict[str, Any]] = None,
174 - rope_theta: Optional[float] = None,
175 - tokenizer_revision: Optional[str] = None,
176 - max_model_len: Optional[int] = None,
177 - spec_target_max_model_len: Optional[int] = None,
178 - quantization: Optional[str] = None,
179 - quantization_param_path: Optional[str] = None,
180 - enforce_eager: Optional[bool] = None,
181 - max_seq_len_to_capture: Optional[int] = None,
182 - max_logprobs: int = 20,
183 - disable_sliding_window: bool = False,
184 - skip_tokenizer_init: bool = False,
185 - served_model_name: Optional[Union[str, List[str]]] = None,
186 - limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
187 - use_async_output_proc: bool = True,
188 - config_format: ConfigFormat = ConfigFormat.AUTO,
189 - hf_overrides: Optional[HfOverrides] = None,
190 - mm_processor_kwargs: Optional[Dict[str, Any]] = None,
191 - mm_cache_preprocessor: bool = False,
192 - override_neuron_config: Optional[Dict[str, Any]] = None,
193 - override_pooler_config: Optional["PoolerConfig"] = None) -> None:
165 + def __init__(self,
166 + model: str,
167 + task: Union[TaskOption, Literal["draft"]],
168 + tokenizer: str,
169 + tokenizer_mode: str,
170 + trust_remote_code: bool,
171 + dtype: Union[str, torch.dtype],
172 + seed: int,
173 + allowed_local_media_path: str = "",
174 + revision: Optional[str] = None,
175 + code_revision: Optional[str] = None,
176 + rope_scaling: Optional[Dict[str, Any]] = None,
177 + rope_theta: Optional[float] = None,
178 + tokenizer_revision: Optional[str] = None,
179 + max_model_len: Optional[int] = None,
180 + spec_target_max_model_len: Optional[int] = None,
181 + quantization: Optional[str] = None,
182 + quantization_param_path: Optional[str] = None,
183 + enforce_eager: Optional[bool] = None,
184 + max_seq_len_to_capture: Optional[int] = None,
185 + max_logprobs: int = 20,
186 + disable_sliding_window: bool = False,
187 + skip_tokenizer_init: bool = False,
188 + served_model_name: Optional[Union[str, List[str]]] = None,
189 + limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
190 + use_async_output_proc: bool = True,
191 + config_format: ConfigFormat = ConfigFormat.AUTO,
192 + hf_overrides: Optional[HfOverrides] = None,
193 + mm_processor_kwargs: Optional[Dict[str, Any]] = None,
194 + mm_cache_preprocessor: bool = False,
195 + override_neuron_config: Optional[Dict[str, Any]] = None,
196 + override_pooler_config: Optional["PoolerConfig"] = None,
197 + logits_processor_pattern: Optional[str] = None) -> None:
198 self.model = model
199 self.tokenizer = tokenizer
200 self.tokenizer_mode = tokenizer_mode
@@ -316,6 +320,7 @@ class ModelConfig:
320 self.task: Final = task
321
322 self.pooler_config = self._init_pooler_config(override_pooler_config)
323 + self.logits_processor_pattern = logits_processor_pattern
324
325 self._verify_quantization()
326 self._verify_cuda_graph()
vllm/engine/arg_utils.py
+10 -1
@@ -170,6 +170,7 @@ class EngineArgs:
170 enable_chunked_prefill: Optional[bool] = None
171
172 guided_decoding_backend: str = 'xgrammar'
173 + logits_processor_pattern: Optional[str] = None
174 # Speculative decoding configuration.
175 speculative_model: Optional[str] = None
176 speculative_model_quantization: Optional[str] = None
@@ -374,6 +375,14 @@ class EngineArgs:
375 'https://github.com/noamgat/lm-format-enforcer.'
376 ' Can be overridden per request via guided_decoding_backend'
377 ' parameter.')
378 + parser.add_argument(
379 + '--logits-processor-pattern',
380 + type=nullable_str,
381 + default=None,
382 + help='Optional regex pattern specifying valid logits processor '
383 + 'qualified names that can be passed with the `logits_processors` '
384 + 'extra completion argument. Defaults to None, which allows no '
385 + 'processors.')
386 # Parallel arguments
387 parser.add_argument(
388 '--distributed-executor-backend',
@@ -975,7 +984,7 @@ class EngineArgs:
984 mm_cache_preprocessor=self.mm_cache_preprocessor,
985 override_neuron_config=self.override_neuron_config,
986 override_pooler_config=self.override_pooler_config,
978 - )
987 + logits_processor_pattern=self.logits_processor_pattern)
988
989 def create_load_config(self) -> LoadConfig:
990 return LoadConfig(
vllm/entrypoints/openai/protocol.py
+74 -3
@@ -1,5 +1,6 @@
1 # Adapted from
2 # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
3 +import re
4 import time
5 from argparse import Namespace
6 from typing import Any, Dict, List, Literal, Optional, Union
@@ -14,7 +15,7 @@ from vllm.pooling_params import PoolingParams
15 from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
16 RequestOutputKind, SamplingParams)
17 from vllm.sequence import Logprob
17 -from vllm.utils import random_uuid
18 +from vllm.utils import random_uuid, resolve_obj_by_qualname
19
20 logger = init_logger(__name__)
21
@@ -148,6 +149,46 @@ class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
149 type: Literal["function"] = "function"
150
151
152 +class LogitsProcessorConstructor(BaseModel):
153 + qualname: str
154 + args: Optional[List[Any]] = None
155 + kwargs: Optional[Dict[str, Any]] = None
156 +
157 +
158 +LogitsProcessors = List[Union[str, LogitsProcessorConstructor]]
159 +
160 +
161 +def get_logits_processors(processors: Optional[LogitsProcessors],
162 + pattern: Optional[str]) -> Optional[List[Any]]:
163 + if processors and pattern:
164 + logits_processors = []
165 + for processor in processors:
166 + qualname = processor if isinstance(processor,
167 + str) else processor.qualname
168 + if not re.match(pattern, qualname):
169 + raise ValueError(
170 + f"Logits processor '{qualname}' is not allowed by this "
171 + "server. See --logits-processor-pattern engine argument "
172 + "for more information.")
173 + try:
174 + logits_processor = resolve_obj_by_qualname(qualname)
175 + except Exception as e:
176 + raise ValueError(
177 + f"Logits processor '{qualname}' could not be resolved: {e}"
178 + ) from e
179 + if isinstance(processor, LogitsProcessorConstructor):
180 + logits_processor = logits_processor(*processor.args or [],
181 + **processor.kwargs or {})
182 + logits_processors.append(logits_processor)
183 + return logits_processors
184 + elif processors:
185 + raise ValueError(
186 + "The `logits_processors` argument is not supported by this "
187 + "server. See --logits-processor-pattern engine argugment "
188 + "for more information.")
189 + return None
190 +
191 +
192 class ChatCompletionRequest(OpenAIBaseModel):
193 # Ordered by official OpenAI API documentation
194 # https://platform.openai.com/docs/api-reference/chat/create
@@ -293,6 +334,17 @@ class ChatCompletionRequest(OpenAIBaseModel):
334 "The request_id related to this request. If the caller does "
335 "not set it, a random_uuid will be generated. This id is used "
336 "through out the inference process and return in response."))
337 + logits_processors: Optional[LogitsProcessors] = Field(
338 + default=None,
339 + description=(
340 + "A list of either qualified names of logits processors, or "
341 + "constructor objects, to apply when sampling. A constructor is "
342 + "a JSON object with a required 'qualname' field specifying the "
343 + "qualified name of the processor class/factory, and optional "
344 + "'args' and 'kwargs' fields containing positional and keyword "
345 + "arguments. For example: {'qualname': "
346 + "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
347 + "{'param': 'value'}}."))
348
349 # doc: end-chat-completion-extra-params
350
@@ -314,7 +366,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
366 length_penalty=self.length_penalty,
367 include_stop_str_in_output=self.include_stop_str_in_output)
368
317 - def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
369 + def to_sampling_params(
370 + self, default_max_tokens: int,
371 + logits_processor_pattern: Optional[str]) -> SamplingParams:
372 # TODO(#9845): remove max_tokens when field is removed from OpenAI API
373 max_tokens = self.max_completion_tokens or self.max_tokens
374 if max_tokens is None:
@@ -364,6 +418,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
418 min_tokens=self.min_tokens,
419 skip_special_tokens=self.skip_special_tokens,
420 spaces_between_special_tokens=self.spaces_between_special_tokens,
421 + logits_processors=get_logits_processors(self.logits_processors,
422 + logits_processor_pattern),
423 include_stop_str_in_output=self.include_stop_str_in_output,
424 truncate_prompt_tokens=self.truncate_prompt_tokens,
425 output_kind=RequestOutputKind.DELTA if self.stream \
@@ -599,6 +655,17 @@ class CompletionRequest(OpenAIBaseModel):
655 "The priority of the request (lower means earlier handling; "
656 "default: 0). Any priority other than 0 will raise an error "
657 "if the served model does not use priority scheduling."))
658 + logits_processors: Optional[LogitsProcessors] = Field(
659 + default=None,
660 + description=(
661 + "A list of either qualified names of logits processors, or "
662 + "constructor objects, to apply when sampling. A constructor is "
663 + "a JSON object with a required 'qualname' field specifying the "
664 + "qualified name of the processor class/factory, and optional "
665 + "'args' and 'kwargs' fields containing positional and keyword "
666 + "arguments. For example: {'qualname': "
667 + "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
668 + "{'param': 'value'}}."))
669
670 # doc: end-completion-extra-params
671
@@ -619,7 +686,9 @@ class CompletionRequest(OpenAIBaseModel):
686 length_penalty=self.length_penalty,
687 include_stop_str_in_output=self.include_stop_str_in_output)
688
622 - def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
689 + def to_sampling_params(
690 + self, default_max_tokens: int,
691 + logits_processor_pattern: Optional[str]) -> SamplingParams:
692 max_tokens = self.max_tokens
693 if max_tokens is None:
694 max_tokens = default_max_tokens
@@ -665,6 +734,8 @@ class CompletionRequest(OpenAIBaseModel):
734 skip_special_tokens=self.skip_special_tokens,
735 spaces_between_special_tokens=self.spaces_between_special_tokens,
736 include_stop_str_in_output=self.include_stop_str_in_output,
737 + logits_processors=get_logits_processors(self.logits_processors,
738 + logits_processor_pattern),
739 truncate_prompt_tokens=self.truncate_prompt_tokens,
740 output_kind=RequestOutputKind.DELTA if self.stream \
741 else RequestOutputKind.FINAL_ONLY,
vllm/entrypoints/openai/serving_chat.py
+2 -1
@@ -197,7 +197,8 @@ class OpenAIServingChat(OpenAIServing):
197 default_max_tokens)
198 else:
199 sampling_params = request.to_sampling_params(
200 - default_max_tokens)
200 + default_max_tokens,
201 + self.model_config.logits_processor_pattern)
202
203 self._log_inputs(request_id,
204 request_prompts[i],
vllm/entrypoints/openai/serving_completion.py
+2 -1
@@ -123,7 +123,8 @@ class OpenAIServingCompletion(OpenAIServing):
123 default_max_tokens)
124 else:
125 sampling_params = request.to_sampling_params(
126 - default_max_tokens)
126 + default_max_tokens,
127 + self.model_config.logits_processor_pattern)
128
129 request_id_item = f"{request_id}-{i}"
130