@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5aef49806d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Feature] Add load generation config from model (#11164)

Signed-off-by: liuyanyi <wolfsonliu@163.com> Signed-off-by: Yanyi Liu <wolfsonliu@163.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Yanyi Liu committed Dec 19, 2024 at 18:50 UTC 5aef49806da2e6cc8a92c948d44e8a722469135f
10 files changed +307 -74
examples/offline_inference_with_default_generation_config.py new
+30
@@ -0,0 +1,30 @@
1 +from vllm import LLM
2 +
3 +# Sample prompts.
4 +prompts = [
5 + "Hello, my name is",
6 + "The president of the United States is",
7 + "The capital of France is",
8 + "The future of AI is",
9 +]
10 +
11 +# Create an LLM with built-in default generation config.
12 +# The generation config is set to None by default to keep
13 +# the behavior consistent with the previous version.
14 +# If you want to use the default generation config from the model,
15 +# you should set the generation_config to "auto".
16 +llm = LLM(model="Qwen/Qwen2.5-0.5B-Instruct", generation_config="auto")
17 +
18 +# Load the default sampling parameters from the model.
19 +sampling_params = llm.get_default_sampling_params()
20 +# Modify the sampling parameters if needed.
21 +sampling_params.temperature = 0.5
22 +
23 +# Generate texts from the prompts. The output is a list of RequestOutput objects
24 +# that contain the prompt, generated text, and other information.
25 +outputs = llm.generate(prompts, sampling_params)
26 +# Print the outputs.
27 +for output in outputs:
28 + prompt = output.prompt
29 + generated_text = output.outputs[0].text
30 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
tests/entrypoints/openai/test_serving_chat.py
+61
@@ -1,6 +1,7 @@
1 import asyncio
2 from contextlib import suppress
3 from dataclasses import dataclass
4 +from typing import Optional
5 from unittest.mock import MagicMock
6
7 from vllm.config import MultiModalConfig
@@ -31,6 +32,10 @@ class MockModelConfig:
32 multimodal_config = MultiModalConfig()
33 hf_config = MockHFConfig()
34 logits_processor_pattern = None
35 + diff_sampling_param: Optional[dict] = None
36 +
37 + def get_diff_sampling_param(self):
38 + return self.diff_sampling_param or {}
39
40
41 @dataclass
@@ -94,3 +99,59 @@ def test_serving_chat_should_set_correct_max_tokens():
99 asyncio.run(serving_chat.create_chat_completion(req))
100
101 assert mock_engine.generate.call_args.args[1].max_tokens == 10
102 +
103 +
104 +def test_serving_chat_could_load_correct_generation_config():
105 +
106 + mock_model_config = MockModelConfig()
107 + mock_model_config.diff_sampling_param = {
108 + "temperature": 0.5,
109 + "repetition_penalty": 1.05
110 + }
111 +
112 + mock_engine = MagicMock(spec=MQLLMEngineClient)
113 + mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
114 + mock_engine.errored = False
115 +
116 + # Initialize the serving chat
117 + serving_chat = OpenAIServingChat(mock_engine,
118 + mock_model_config,
119 + BASE_MODEL_PATHS,
120 + response_role="assistant",
121 + chat_template=CHAT_TEMPLATE,
122 + chat_template_content_format="auto",
123 + lora_modules=None,
124 + prompt_adapters=None,
125 + request_logger=None)
126 + req = ChatCompletionRequest(
127 + model=MODEL_NAME,
128 + messages=[{
129 + "role": "user",
130 + "content": "what is 1+1?"
131 + }],
132 + guided_decoding_backend="outlines",
133 + )
134 +
135 + with suppress(Exception):
136 + asyncio.run(serving_chat.create_chat_completion(req))
137 +
138 + assert mock_engine.generate.call_args.args[1].temperature == 0.5
139 + assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
140 +
141 + # Test the param when user set it
142 + req.temperature = 0.1
143 +
144 + with suppress(Exception):
145 + asyncio.run(serving_chat.create_chat_completion(req))
146 +
147 + assert mock_engine.generate.call_args.args[1].temperature == 0.1
148 + assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
149 +
150 + # Test When temperature==0.0
151 + req.temperature = 0.0
152 +
153 + with suppress(Exception):
154 + asyncio.run(serving_chat.create_chat_completion(req))
155 +
156 + assert mock_engine.generate.call_args.args[1].temperature == 0.0
157 + assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
vllm/config.py
+57 -2
@@ -27,7 +27,8 @@ from vllm.tracing import is_otel_available, otel_import_error_traceback
27 from vllm.transformers_utils.config import (
28 ConfigFormat, get_config, get_hf_image_processor_config,
29 get_hf_text_config, get_pooling_config,
30 - get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
30 + get_sentence_transformer_tokenizer_config, is_encoder_decoder,
31 + try_get_generation_config, uses_mrope)
32 from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless,
33 get_cpu_memory, print_warning_once, random_uuid,
34 resolve_obj_by_qualname)
@@ -160,6 +161,7 @@ class ModelConfig:
161 logits processor qualified names that can be passed with the
162 `logits_processors` extra completion argument. Defaults to None,
163 which allows no processors.
164 + generation_config: Configuration parameter file for generation.
165 """
166
167 def compute_hash(self) -> str:
@@ -218,7 +220,8 @@ class ModelConfig:
220 disable_mm_preprocessor_cache: bool = False,
221 override_neuron_config: Optional[Dict[str, Any]] = None,
222 override_pooler_config: Optional["PoolerConfig"] = None,
221 - logits_processor_pattern: Optional[str] = None) -> None:
223 + logits_processor_pattern: Optional[str] = None,
224 + generation_config: Optional[str] = None) -> None:
225 self.model = model
226 self.tokenizer = tokenizer
227 self.tokenizer_mode = tokenizer_mode
@@ -348,6 +351,8 @@ class ModelConfig:
351 self.pooler_config = self._init_pooler_config(override_pooler_config)
352 self.logits_processor_pattern = logits_processor_pattern
353
354 + self.generation_config = generation_config
355 +
356 self._verify_quantization()
357 self._verify_cuda_graph()
358 self._verify_bnb_config()
@@ -813,6 +818,56 @@ class ModelConfig:
818
819 return self.multimodal_config
820
821 + def try_get_generation_config(self) -> Dict[str, Any]:
822 + if self.generation_config is None or self.generation_config == "auto":
823 + config = try_get_generation_config(
824 + self.model,
825 + trust_remote_code=self.trust_remote_code,
826 + revision=self.revision,
827 + )
828 + else:
829 + config = try_get_generation_config(
830 + self.generation_config,
831 + trust_remote_code=self.trust_remote_code,
832 + )
833 +
834 + if config is None:
835 + return {}
836 +
837 + return config.to_diff_dict()
838 +
839 + def get_diff_sampling_param(self) -> Dict[str, Any]:
840 + """
841 + This method returns a dictionary containing the parameters
842 + that differ from the default sampling parameters, but only
843 + if `generation_config` is set. If `generation_config` is not
844 + set, an empty dictionary is returned.
845 +
846 + Returns:
847 + Dict[str, Any]: A dictionary with the differing sampling
848 + parameters if `generation_config` is set, otherwise an
849 + empty dictionary.
850 + """
851 + if self.generation_config is None:
852 + # When generation_config is not set
853 + return {}
854 + config = self.try_get_generation_config()
855 + available_params = [
856 + "repetition_penalty",
857 + "temperature",
858 + "top_k",
859 + "top_p",
860 + "min_p",
861 + ]
862 + if any(p in config for p in available_params):
863 + diff_sampling_param = {
864 + p: config.get(p)
865 + for p in available_params if config.get(p) is not None
866 + }
867 + else:
868 + diff_sampling_param = {}
869 + return diff_sampling_param
870 +
871 @property
872 def is_encoder_decoder(self) -> bool:
873 """Extract the HF encoder/decoder model flag."""
vllm/engine/arg_utils.py
+14 -1
@@ -197,6 +197,8 @@ class EngineArgs:
197
198 kv_transfer_config: Optional[KVTransferConfig] = None
199
200 + generation_config: Optional[str] = None
201 +
202 def __post_init__(self):
203 if not self.tokenizer:
204 self.tokenizer = self.model
@@ -942,6 +944,16 @@ class EngineArgs:
944 default="auto",
945 help='The worker class to use for distributed execution.')
946
947 + parser.add_argument(
948 + "--generation-config",
949 + type=nullable_str,
950 + default=None,
951 + help="The folder path to the generation config. "
952 + "Defaults to None, will use the default generation config in vLLM. "
953 + "If set to 'auto', the generation config will be automatically "
954 + "loaded from model. If set to a folder path, the generation config "
955 + "will be loaded from the specified folder path.")
956 +
957 return parser
958
959 @classmethod
@@ -985,7 +997,8 @@ class EngineArgs:
997 disable_mm_preprocessor_cache=self.disable_mm_preprocessor_cache,
998 override_neuron_config=self.override_neuron_config,
999 override_pooler_config=self.override_pooler_config,
988 - logits_processor_pattern=self.logits_processor_pattern)
1000 + logits_processor_pattern=self.logits_processor_pattern,
1001 + generation_config=self.generation_config)
1002
1003 def create_load_config(self) -> LoadConfig:
1004 return LoadConfig(
vllm/engine/llm_engine.py
+4 -19
@@ -5,8 +5,8 @@ from collections import deque
5 from contextlib import contextmanager
6 from dataclasses import dataclass
7 from functools import partial
8 -from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
9 - Iterable, List, Mapping, NamedTuple, Optional)
8 +from typing import (TYPE_CHECKING, Callable, ClassVar, Deque, Dict, Iterable,
9 + List, Mapping, NamedTuple, Optional)
10 from typing import Sequence as GenericSequence
11 from typing import Set, Type, Union, cast, overload
12
@@ -52,7 +52,6 @@ from vllm.sequence import (ExecuteModelRequest, ParallelSampleSequenceGroup,
52 SequenceGroupOutput, SequenceStatus)
53 from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
54 init_tracer)
55 -from vllm.transformers_utils.config import try_get_generation_config
55 from vllm.transformers_utils.detokenizer import Detokenizer
56 from vllm.transformers_utils.tokenizer import AnyTokenizer
57 from vllm.transformers_utils.tokenizer_group import (
@@ -65,20 +64,6 @@ from vllm.version import __version__ as VLLM_VERSION
64 logger = init_logger(__name__)
65 _LOCAL_LOGGING_INTERVAL_SEC = 5
66
68 -
69 -def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
70 - config = try_get_generation_config(
71 - model_config.model,
72 - trust_remote_code=model_config.trust_remote_code,
73 - revision=model_config.revision,
74 - )
75 -
76 - if config is None:
77 - return {}
78 -
79 - return config.to_diff_dict()
80 -
81 -
67 _G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
68 _O = TypeVar("_O", RequestOutput, PoolingRequestOutput)
69
@@ -274,8 +259,8 @@ class LLMEngine:
259 return tokenizer_group.get_lora_tokenizer(sequence.lora_request)
260
261 self.seq_counter = Counter()
277 - self.generation_config_fields = _load_generation_config_dict(
278 - self.model_config)
262 + self.generation_config_fields = (
263 + self.model_config.try_get_generation_config())
264
265 self.input_preprocessor = InputPreprocessor(self.model_config,
266 self.tokenizer,
vllm/entrypoints/llm.py
+8 -1
@@ -258,6 +258,13 @@ class LLM:
258 else:
259 tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
260
261 + def get_default_sampling_params(self) -> SamplingParams:
262 + diff_sampling_param = (
263 + self.llm_engine.model_config.get_diff_sampling_param())
264 + if diff_sampling_param:
265 + return SamplingParams.from_optional(**diff_sampling_param)
266 + return SamplingParams()
267 +
268 @overload
269 def generate(
270 self,
@@ -441,7 +448,7 @@ class LLM:
448
449 if sampling_params is None:
450 # Use default sampling params.
444 - sampling_params = SamplingParams()
451 + sampling_params = self.get_default_sampling_params()
452
453 self._validate_and_add_requests(
454 prompts=parsed_prompts,
vllm/entrypoints/openai/protocol.py
+109 -30
@@ -211,8 +211,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
211 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
212 stream: Optional[bool] = False
213 stream_options: Optional[StreamOptions] = None
214 - temperature: Optional[float] = 1.0
215 - top_p: Optional[float] = 1.0
214 + temperature: Optional[float] = None
215 + top_p: Optional[float] = None
216 tools: Optional[List[ChatCompletionToolsParam]] = None
217 tool_choice: Optional[Union[Literal["none"], Literal["auto"],
218 ChatCompletionNamedToolChoiceParam]] = "none"
@@ -224,9 +224,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
224 # doc: begin-chat-completion-sampling-params
225 best_of: Optional[int] = None
226 use_beam_search: bool = False
227 - top_k: int = -1
228 - min_p: float = 0.0
229 - repetition_penalty: float = 1.0
227 + top_k: Optional[int] = None
228 + min_p: Optional[float] = None
229 + repetition_penalty: Optional[float] = None
230 length_penalty: float = 1.0
231 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
232 include_stop_str_in_output: bool = False
@@ -348,15 +348,32 @@ class ChatCompletionRequest(OpenAIBaseModel):
348
349 # doc: end-chat-completion-extra-params
350
351 - def to_beam_search_params(self,
352 - default_max_tokens: int) -> BeamSearchParams:
351 + # Default sampling parameters for chat completion requests
352 + _DEFAULT_SAMPLING_PARAMS: dict = {
353 + "repetition_penalty": 1.0,
354 + "temperature": 1.0,
355 + "top_p": 1.0,
356 + "top_k": -1,
357 + "min_p": 0.0,
358 + }
359 +
360 + def to_beam_search_params(
361 + self,
362 + default_max_tokens: int,
363 + default_sampling_params: Optional[dict] = None
364 + ) -> BeamSearchParams:
365 # TODO(#9845): remove max_tokens when field is removed from OpenAI API
366 max_tokens = self.max_completion_tokens or self.max_tokens
367 if max_tokens is None:
368 max_tokens = default_max_tokens
369
370 + if default_sampling_params is None:
371 + default_sampling_params = {}
372 n = self.n if self.n is not None else 1
359 - temperature = self.temperature if self.temperature is not None else 0.0
373 +
374 + if (temperature := self.temperature) is None:
375 + temperature = default_sampling_params.get(
376 + "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
377
378 return BeamSearchParams(
379 beam_width=n,
@@ -367,13 +384,36 @@ class ChatCompletionRequest(OpenAIBaseModel):
384 include_stop_str_in_output=self.include_stop_str_in_output)
385
386 def to_sampling_params(
370 - self, default_max_tokens: int,
371 - logits_processor_pattern: Optional[str]) -> SamplingParams:
387 + self,
388 + default_max_tokens: int,
389 + logits_processor_pattern: Optional[str],
390 + default_sampling_params: Optional[dict] = None) -> SamplingParams:
391 # TODO(#9845): remove max_tokens when field is removed from OpenAI API
392 max_tokens = self.max_completion_tokens or self.max_tokens
393 if max_tokens is None:
394 max_tokens = default_max_tokens
395
396 + if default_sampling_params is None:
397 + default_sampling_params = {}
398 + # Default parameters
399 + if (repetition_penalty := self.repetition_penalty) is None:
400 + repetition_penalty = default_sampling_params.get(
401 + "repetition_penalty",
402 + self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
403 + )
404 + if (temperature := self.temperature) is None:
405 + temperature = default_sampling_params.get(
406 + "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
407 + if (top_p := self.top_p) is None:
408 + top_p = default_sampling_params.get(
409 + "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"])
410 + if (top_k := self.top_k) is None:
411 + top_k = default_sampling_params.get(
412 + "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"])
413 + if (min_p := self.min_p) is None:
414 + min_p = default_sampling_params.get(
415 + "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"])
416 +
417 prompt_logprobs = self.prompt_logprobs
418 if prompt_logprobs is None and self.echo:
419 prompt_logprobs = self.top_logprobs
@@ -403,11 +443,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
443 best_of=self.best_of,
444 presence_penalty=self.presence_penalty,
445 frequency_penalty=self.frequency_penalty,
406 - repetition_penalty=self.repetition_penalty,
407 - temperature=self.temperature,
408 - top_p=self.top_p,
409 - top_k=self.top_k,
410 - min_p=self.min_p,
446 + repetition_penalty=repetition_penalty,
447 + temperature=temperature,
448 + top_p=top_p,
449 + top_k=top_k,
450 + min_p=min_p,
451 seed=self.seed,
452 stop=self.stop,
453 stop_token_ids=self.stop_token_ids,
@@ -584,15 +624,15 @@ class CompletionRequest(OpenAIBaseModel):
624 stream: Optional[bool] = False
625 stream_options: Optional[StreamOptions] = None
626 suffix: Optional[str] = None
587 - temperature: Optional[float] = 1.0
588 - top_p: Optional[float] = 1.0
627 + temperature: Optional[float] = None
628 + top_p: Optional[float] = None
629 user: Optional[str] = None
630
631 # doc: begin-completion-sampling-params
632 use_beam_search: bool = False
593 - top_k: int = -1
594 - min_p: float = 0.0
595 - repetition_penalty: float = 1.0
633 + top_k: Optional[int] = None
634 + min_p: Optional[float] = None
635 + repetition_penalty: Optional[float] = None
636 length_penalty: float = 1.0
637 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
638 include_stop_str_in_output: bool = False
@@ -669,14 +709,30 @@ class CompletionRequest(OpenAIBaseModel):
709
710 # doc: end-completion-extra-params
711
672 - def to_beam_search_params(self,
673 - default_max_tokens: int) -> BeamSearchParams:
712 + # Default sampling parameters for completion requests
713 + _DEFAULT_SAMPLING_PARAMS: dict = {
714 + "repetition_penalty": 1.0,
715 + "temperature": 1.0,
716 + "top_p": 1.0,
717 + "top_k": -1,
718 + "min_p": 0.0,
719 + }
720 +
721 + def to_beam_search_params(
722 + self,
723 + default_max_tokens: int,
724 + default_sampling_params: Optional[dict] = None
725 + ) -> BeamSearchParams:
726 max_tokens = self.max_tokens
727 if max_tokens is None:
728 max_tokens = default_max_tokens
729
730 + if default_sampling_params is None:
731 + default_sampling_params = {}
732 n = self.n if self.n is not None else 1
679 - temperature = self.temperature if self.temperature is not None else 0.0
733 +
734 + if (temperature := self.temperature) is None:
735 + temperature = default_sampling_params.get("temperature", 1.0)
736
737 return BeamSearchParams(
738 beam_width=n,
@@ -687,12 +743,35 @@ class CompletionRequest(OpenAIBaseModel):
743 include_stop_str_in_output=self.include_stop_str_in_output)
744
745 def to_sampling_params(
690 - self, default_max_tokens: int,
691 - logits_processor_pattern: Optional[str]) -> SamplingParams:
746 + self,
747 + default_max_tokens: int,
748 + logits_processor_pattern: Optional[str],
749 + default_sampling_params: Optional[dict] = None) -> SamplingParams:
750 max_tokens = self.max_tokens
751 if max_tokens is None:
752 max_tokens = default_max_tokens
753
754 + if default_sampling_params is None:
755 + default_sampling_params = {}
756 + # Default parameters
757 + if (repetition_penalty := self.repetition_penalty) is None:
758 + repetition_penalty = default_sampling_params.get(
759 + "repetition_penalty",
760 + self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
761 + )
762 + if (temperature := self.temperature) is None:
763 + temperature = default_sampling_params.get(
764 + "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
765 + if (top_p := self.top_p) is None:
766 + top_p = default_sampling_params.get(
767 + "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"])
768 + if (top_k := self.top_k) is None:
769 + top_k = default_sampling_params.get(
770 + "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"])
771 + if (min_p := self.min_p) is None:
772 + min_p = default_sampling_params.get(
773 + "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"])
774 +
775 prompt_logprobs = self.prompt_logprobs
776 if prompt_logprobs is None and self.echo:
777 prompt_logprobs = self.logprobs
@@ -718,11 +797,11 @@ class CompletionRequest(OpenAIBaseModel):
797 best_of=self.best_of,
798 presence_penalty=self.presence_penalty,
799 frequency_penalty=self.frequency_penalty,
721 - repetition_penalty=self.repetition_penalty,
722 - temperature=self.temperature,
723 - top_p=self.top_p,
724 - top_k=self.top_k,
725 - min_p=self.min_p,
800 + repetition_penalty=repetition_penalty,
801 + temperature=temperature,
802 + top_p=top_p,
803 + top_k=top_k,
804 + min_p=min_p,
805 seed=self.seed,
806 stop=self.stop,
807 stop_token_ids=self.stop_token_ids,
vllm/entrypoints/openai/serving_chat.py
+10 -2
@@ -91,6 +91,10 @@ class OpenAIServingChat(OpenAIServing):
91 "been registered") from e
92
93 self.enable_prompt_tokens_details = enable_prompt_tokens_details
94 + diff_sampling_param = self.model_config.get_diff_sampling_param()
95 + if diff_sampling_param:
96 + logger.info("Overwriting default chat sampling param with: %s",
97 + diff_sampling_param)
98
99 async def create_chat_completion(
100 self,
@@ -191,13 +195,17 @@ class OpenAIServingChat(OpenAIServing):
195 sampling_params: Union[SamplingParams, BeamSearchParams]
196 default_max_tokens = self.max_model_len - len(
197 engine_prompt["prompt_token_ids"])
198 + # Build default sampling params
199 + default_sampling_params = (
200 + self.model_config.get_diff_sampling_param())
201 if request.use_beam_search:
202 sampling_params = request.to_beam_search_params(
196 - default_max_tokens)
203 + default_max_tokens, default_sampling_params)
204 else:
205 sampling_params = request.to_sampling_params(
206 default_max_tokens,
200 - self.model_config.logits_processor_pattern)
207 + self.model_config.logits_processor_pattern,
208 + default_sampling_params)
209
210 self._log_inputs(request_id,
211 request_prompts[i],
vllm/entrypoints/openai/serving_completion.py
+11 -2
@@ -55,6 +55,11 @@ class OpenAIServingCompletion(OpenAIServing):
55 prompt_adapters=prompt_adapters,
56 request_logger=request_logger,
57 return_tokens_as_token_ids=return_tokens_as_token_ids)
58 + diff_sampling_param = self.model_config.get_diff_sampling_param()
59 + if diff_sampling_param:
60 + logger.info(
61 + "Overwriting default completion sampling param with: %s",
62 + diff_sampling_param)
63
64 async def create_completion(
65 self,
@@ -118,13 +123,17 @@ class OpenAIServingCompletion(OpenAIServing):
123 sampling_params: Union[SamplingParams, BeamSearchParams]
124 default_max_tokens = self.max_model_len - len(
125 engine_prompt["prompt_token_ids"])
126 + # Build default sampling params
127 + default_sampling_params = (
128 + self.model_config.get_diff_sampling_param())
129 if request.use_beam_search:
130 sampling_params = request.to_beam_search_params(
123 - default_max_tokens)
131 + default_max_tokens, default_sampling_params)
132 else:
133 sampling_params = request.to_sampling_params(
134 default_max_tokens,
127 - self.model_config.logits_processor_pattern)
135 + self.model_config.logits_processor_pattern,
136 + default_sampling_params)
137
138 request_id_item = f"{request_id}-{i}"
139
vllm/v1/engine/processor.py
+3 -17
@@ -1,5 +1,5 @@
1 import time
2 -from typing import Any, Dict, Mapping, Optional, Tuple, Union
2 +from typing import Mapping, Optional, Tuple, Union
3
4 from vllm.config import CacheConfig, LoRAConfig, ModelConfig
5 from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
@@ -12,7 +12,6 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
12 from vllm.pooling_params import PoolingParams
13 from vllm.prompt_adapter.request import PromptAdapterRequest
14 from vllm.sampling_params import SamplingParams
15 -from vllm.transformers_utils.config import try_get_generation_config
15 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16 from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
17 from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
@@ -34,8 +33,8 @@ class Processor:
33 self.lora_config = lora_config
34 self.tokenizer = tokenizer
35
37 - self.generation_config_fields = _load_generation_config_dict(
38 - model_config)
36 + self.generation_config_fields = model_config.try_get_generation_config(
37 + )
38 self.input_preprocessor = InputPreprocessor(model_config,
39 self.tokenizer,
40 mm_registry)
@@ -181,16 +180,3 @@ class Processor:
180 # TODO: Find out how many placeholder tokens are there so we can
181 # check that chunked prefill does not truncate them
182 # max_batch_len = self.scheduler_config.max_num_batched_tokens
184 -
185 -
186 -def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
187 - config = try_get_generation_config(
188 - model_config.model,
189 - trust_remote_code=model_config.trust_remote_code,
190 - revision=model_config.revision,
191 - )
192 -
193 - if config is None:
194 - return {}
195 -
196 - return config.to_diff_dict()