A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Feature] Add load generation config from model (#11164)
Signed-off-by: liuyanyi <wolfsonliu@163.com> Signed-off-by: Yanyi Liu <wolfsonliu@163.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Yanyi Liu committed
Dec 19, 2024 at 18:50 UTC
5aef49806da2e6cc8a92c948d44e8a722469135f
10 files changed
+307
-74
examples/offline_inference_with_default_generation_config.py
new
+30
@@ -0,0 +1,30 @@
1
+from vllm import LLM
2
+
3
+# Sample prompts.
4
+prompts = [
5
+ "Hello, my name is",
6
+ "The president of the United States is",
7
+ "The capital of France is",
8
+ "The future of AI is",
9
+]
10
+
11
+# Create an LLM with built-in default generation config.
12
+# The generation config is set to None by default to keep
13
+# the behavior consistent with the previous version.
14
+# If you want to use the default generation config from the model,
15
+# you should set the generation_config to "auto".
16
+llm = LLM(model="Qwen/Qwen2.5-0.5B-Instruct", generation_config="auto")
17
+
18
+# Load the default sampling parameters from the model.
19
+sampling_params = llm.get_default_sampling_params()
20
+# Modify the sampling parameters if needed.
21
+sampling_params.temperature = 0.5
22
+
23
+# Generate texts from the prompts. The output is a list of RequestOutput objects
24
+# that contain the prompt, generated text, and other information.
25
+outputs = llm.generate(prompts, sampling_params)
26
+# Print the outputs.
27
+for output in outputs:
28
+ prompt = output.prompt
29
+ generated_text = output.outputs[0].text
30
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
tests/entrypoints/openai/test_serving_chat.py
+61
@@ -1,6 +1,7 @@
1
import asyncio
2
from contextlib import suppress
3
from dataclasses import dataclass
4
+from typing import Optional
5
from unittest.mock import MagicMock
6
7
from vllm.config import MultiModalConfig
@@ -31,6 +32,10 @@ class MockModelConfig:
32
multimodal_config = MultiModalConfig()
33
hf_config = MockHFConfig()
34
logits_processor_pattern = None
35
+ diff_sampling_param: Optional[dict] = None
36
+
37
+ def get_diff_sampling_param(self):
38
+ return self.diff_sampling_param or {}
39
40
41
@dataclass
@@ -94,3 +99,59 @@ def test_serving_chat_should_set_correct_max_tokens():
99
asyncio.run(serving_chat.create_chat_completion(req))
100
101
assert mock_engine.generate.call_args.args[1].max_tokens == 10
102
+
103
+
104
+def test_serving_chat_could_load_correct_generation_config():
105
+
106
+ mock_model_config = MockModelConfig()
107
+ mock_model_config.diff_sampling_param = {
108
+ "temperature": 0.5,
109
+ "repetition_penalty": 1.05
110
+ }
111
+
112
+ mock_engine = MagicMock(spec=MQLLMEngineClient)
113
+ mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
114
+ mock_engine.errored = False
115
+
116
+ # Initialize the serving chat
117
+ serving_chat = OpenAIServingChat(mock_engine,
118
+ mock_model_config,
119
+ BASE_MODEL_PATHS,
120
+ response_role="assistant",
121
+ chat_template=CHAT_TEMPLATE,
122
+ chat_template_content_format="auto",
123
+ lora_modules=None,
124
+ prompt_adapters=None,
125
+ request_logger=None)
126
+ req = ChatCompletionRequest(
127
+ model=MODEL_NAME,
128
+ messages=[{
129
+ "role": "user",
130
+ "content": "what is 1+1?"
131
+ }],
132
+ guided_decoding_backend="outlines",
133
+ )
134
+
135
+ with suppress(Exception):
136
+ asyncio.run(serving_chat.create_chat_completion(req))
137
+
138
+ assert mock_engine.generate.call_args.args[1].temperature == 0.5
139
+ assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
140
+
141
+ # Test the param when user set it
142
+ req.temperature = 0.1
143
+
144
+ with suppress(Exception):
145
+ asyncio.run(serving_chat.create_chat_completion(req))
146
+
147
+ assert mock_engine.generate.call_args.args[1].temperature == 0.1
148
+ assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
149
+
150
+ # Test When temperature==0.0
151
+ req.temperature = 0.0
152
+
153
+ with suppress(Exception):
154
+ asyncio.run(serving_chat.create_chat_completion(req))
155
+
156
+ assert mock_engine.generate.call_args.args[1].temperature == 0.0
157
+ assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
vllm/config.py
+57
-2
@@ -27,7 +27,8 @@ from vllm.tracing import is_otel_available, otel_import_error_traceback
27
from vllm.transformers_utils.config import (
28
ConfigFormat, get_config, get_hf_image_processor_config,
29
get_hf_text_config, get_pooling_config,
30
- get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
30
+ get_sentence_transformer_tokenizer_config, is_encoder_decoder,
31
+ try_get_generation_config, uses_mrope)
32
from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless,
33
get_cpu_memory, print_warning_once, random_uuid,
34
resolve_obj_by_qualname)
@@ -160,6 +161,7 @@ class ModelConfig:
161
logits processor qualified names that can be passed with the
162
`logits_processors` extra completion argument. Defaults to None,
163
which allows no processors.
164
+ generation_config: Configuration parameter file for generation.
165
"""
166
167
def compute_hash(self) -> str:
@@ -218,7 +220,8 @@ class ModelConfig:
220
disable_mm_preprocessor_cache: bool = False,
221
override_neuron_config: Optional[Dict[str, Any]] = None,
222
override_pooler_config: Optional["PoolerConfig"] = None,
221
- logits_processor_pattern: Optional[str] = None) -> None:
223
+ logits_processor_pattern: Optional[str] = None,
224
+ generation_config: Optional[str] = None) -> None:
225
self.model = model
226
self.tokenizer = tokenizer
227
self.tokenizer_mode = tokenizer_mode
@@ -348,6 +351,8 @@ class ModelConfig:
351
self.pooler_config = self._init_pooler_config(override_pooler_config)
352
self.logits_processor_pattern = logits_processor_pattern
353
354
+ self.generation_config = generation_config
355
+
356
self._verify_quantization()
357
self._verify_cuda_graph()
358
self._verify_bnb_config()
@@ -813,6 +818,56 @@ class ModelConfig:
818
819
return self.multimodal_config
820
821
+ def try_get_generation_config(self) -> Dict[str, Any]:
822
+ if self.generation_config is None or self.generation_config == "auto":
823
+ config = try_get_generation_config(
824
+ self.model,
825
+ trust_remote_code=self.trust_remote_code,
826
+ revision=self.revision,
827
+ )
828
+ else:
829
+ config = try_get_generation_config(
830
+ self.generation_config,
831
+ trust_remote_code=self.trust_remote_code,
832
+ )
833
+
834
+ if config is None:
835
+ return {}
836
+
837
+ return config.to_diff_dict()
838
+
839
+ def get_diff_sampling_param(self) -> Dict[str, Any]:
840
+ """
841
+ This method returns a dictionary containing the parameters
842
+ that differ from the default sampling parameters, but only
843
+ if `generation_config` is set. If `generation_config` is not
844
+ set, an empty dictionary is returned.
845
+
846
+ Returns:
847
+ Dict[str, Any]: A dictionary with the differing sampling
848
+ parameters if `generation_config` is set, otherwise an
849
+ empty dictionary.
850
+ """
851
+ if self.generation_config is None:
852
+ # When generation_config is not set
853
+ return {}
854
+ config = self.try_get_generation_config()
855
+ available_params = [
856
+ "repetition_penalty",
857
+ "temperature",
858
+ "top_k",
859
+ "top_p",
860
+ "min_p",
861
+ ]
862
+ if any(p in config for p in available_params):
863
+ diff_sampling_param = {
864
+ p: config.get(p)
865
+ for p in available_params if config.get(p) is not None
866
+ }
867
+ else:
868
+ diff_sampling_param = {}
869
+ return diff_sampling_param
870
+
871
@property
872
def is_encoder_decoder(self) -> bool:
873
"""Extract the HF encoder/decoder model flag."""
vllm/engine/arg_utils.py
+14
-1
@@ -197,6 +197,8 @@ class EngineArgs:
197
198
kv_transfer_config: Optional[KVTransferConfig] = None
199
200
+ generation_config: Optional[str] = None
201
+
202
def __post_init__(self):
203
if not self.tokenizer:
204
self.tokenizer = self.model
@@ -942,6 +944,16 @@ class EngineArgs:
944
default="auto",
945
help='The worker class to use for distributed execution.')
946
947
+ parser.add_argument(
948
+ "--generation-config",
949
+ type=nullable_str,
950
+ default=None,
951
+ help="The folder path to the generation config. "
952
+ "Defaults to None, will use the default generation config in vLLM. "
953
+ "If set to 'auto', the generation config will be automatically "
954
+ "loaded from model. If set to a folder path, the generation config "
955
+ "will be loaded from the specified folder path.")
956
+
957
return parser
958
959
@classmethod
@@ -985,7 +997,8 @@ class EngineArgs:
997
disable_mm_preprocessor_cache=self.disable_mm_preprocessor_cache,
998
override_neuron_config=self.override_neuron_config,
999
override_pooler_config=self.override_pooler_config,
988
- logits_processor_pattern=self.logits_processor_pattern)
1000
+ logits_processor_pattern=self.logits_processor_pattern,
1001
+ generation_config=self.generation_config)
1002
1003
def create_load_config(self) -> LoadConfig:
1004
return LoadConfig(
vllm/engine/llm_engine.py
+4
-19
@@ -5,8 +5,8 @@ from collections import deque
5
from contextlib import contextmanager
6
from dataclasses import dataclass
7
from functools import partial
8
-from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
9
- Iterable, List, Mapping, NamedTuple, Optional)
8
+from typing import (TYPE_CHECKING, Callable, ClassVar, Deque, Dict, Iterable,
9
+ List, Mapping, NamedTuple, Optional)
10
from typing import Sequence as GenericSequence
11
from typing import Set, Type, Union, cast, overload
12
@@ -52,7 +52,6 @@ from vllm.sequence import (ExecuteModelRequest, ParallelSampleSequenceGroup,
52
SequenceGroupOutput, SequenceStatus)
53
from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
54
init_tracer)
55
-from vllm.transformers_utils.config import try_get_generation_config
55
from vllm.transformers_utils.detokenizer import Detokenizer
56
from vllm.transformers_utils.tokenizer import AnyTokenizer
57
from vllm.transformers_utils.tokenizer_group import (
@@ -65,20 +64,6 @@ from vllm.version import __version__ as VLLM_VERSION
64
logger = init_logger(__name__)
65
_LOCAL_LOGGING_INTERVAL_SEC = 5
66
68
-
69
-def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
70
- config = try_get_generation_config(
71
- model_config.model,
72
- trust_remote_code=model_config.trust_remote_code,
73
- revision=model_config.revision,
74
- )
75
-
76
- if config is None:
77
- return {}
78
-
79
- return config.to_diff_dict()
80
-
81
-
67
_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
68
_O = TypeVar("_O", RequestOutput, PoolingRequestOutput)
69
@@ -274,8 +259,8 @@ class LLMEngine:
259
return tokenizer_group.get_lora_tokenizer(sequence.lora_request)
260
261
self.seq_counter = Counter()
277
- self.generation_config_fields = _load_generation_config_dict(
278
- self.model_config)
262
+ self.generation_config_fields = (
263
+ self.model_config.try_get_generation_config())
264
265
self.input_preprocessor = InputPreprocessor(self.model_config,
266
self.tokenizer,
vllm/entrypoints/llm.py
+8
-1
@@ -258,6 +258,13 @@ class LLM:
258
else:
259
tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
260
261
+ def get_default_sampling_params(self) -> SamplingParams:
262
+ diff_sampling_param = (
263
+ self.llm_engine.model_config.get_diff_sampling_param())
264
+ if diff_sampling_param:
265
+ return SamplingParams.from_optional(**diff_sampling_param)
266
+ return SamplingParams()
267
+
268
@overload
269
def generate(
270
self,
@@ -441,7 +448,7 @@ class LLM:
448
449
if sampling_params is None:
450
# Use default sampling params.
444
- sampling_params = SamplingParams()
451
+ sampling_params = self.get_default_sampling_params()
452
453
self._validate_and_add_requests(
454
prompts=parsed_prompts,
vllm/entrypoints/openai/protocol.py
+109
-30
@@ -211,8 +211,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
211
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
212
stream: Optional[bool] = False
213
stream_options: Optional[StreamOptions] = None
214
- temperature: Optional[float] = 1.0
215
- top_p: Optional[float] = 1.0
214
+ temperature: Optional[float] = None
215
+ top_p: Optional[float] = None
216
tools: Optional[List[ChatCompletionToolsParam]] = None
217
tool_choice: Optional[Union[Literal["none"], Literal["auto"],
218
ChatCompletionNamedToolChoiceParam]] = "none"
@@ -224,9 +224,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
224
# doc: begin-chat-completion-sampling-params
225
best_of: Optional[int] = None
226
use_beam_search: bool = False
227
- top_k: int = -1
228
- min_p: float = 0.0
229
- repetition_penalty: float = 1.0
227
+ top_k: Optional[int] = None
228
+ min_p: Optional[float] = None
229
+ repetition_penalty: Optional[float] = None
230
length_penalty: float = 1.0
231
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
232
include_stop_str_in_output: bool = False
@@ -348,15 +348,32 @@ class ChatCompletionRequest(OpenAIBaseModel):
348
349
# doc: end-chat-completion-extra-params
350
351
- def to_beam_search_params(self,
352
- default_max_tokens: int) -> BeamSearchParams:
351
+ # Default sampling parameters for chat completion requests
352
+ _DEFAULT_SAMPLING_PARAMS: dict = {
353
+ "repetition_penalty": 1.0,
354
+ "temperature": 1.0,
355
+ "top_p": 1.0,
356
+ "top_k": -1,
357
+ "min_p": 0.0,
358
+ }
359
+
360
+ def to_beam_search_params(
361
+ self,
362
+ default_max_tokens: int,
363
+ default_sampling_params: Optional[dict] = None
364
+ ) -> BeamSearchParams:
365
# TODO(#9845): remove max_tokens when field is removed from OpenAI API
366
max_tokens = self.max_completion_tokens or self.max_tokens
367
if max_tokens is None:
368
max_tokens = default_max_tokens
369
370
+ if default_sampling_params is None:
371
+ default_sampling_params = {}
372
n = self.n if self.n is not None else 1
359
- temperature = self.temperature if self.temperature is not None else 0.0
373
+
374
+ if (temperature := self.temperature) is None:
375
+ temperature = default_sampling_params.get(
376
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
377
378
return BeamSearchParams(
379
beam_width=n,
@@ -367,13 +384,36 @@ class ChatCompletionRequest(OpenAIBaseModel):
384
include_stop_str_in_output=self.include_stop_str_in_output)
385
386
def to_sampling_params(
370
- self, default_max_tokens: int,
371
- logits_processor_pattern: Optional[str]) -> SamplingParams:
387
+ self,
388
+ default_max_tokens: int,
389
+ logits_processor_pattern: Optional[str],
390
+ default_sampling_params: Optional[dict] = None) -> SamplingParams:
391
# TODO(#9845): remove max_tokens when field is removed from OpenAI API
392
max_tokens = self.max_completion_tokens or self.max_tokens
393
if max_tokens is None:
394
max_tokens = default_max_tokens
395
396
+ if default_sampling_params is None:
397
+ default_sampling_params = {}
398
+ # Default parameters
399
+ if (repetition_penalty := self.repetition_penalty) is None:
400
+ repetition_penalty = default_sampling_params.get(
401
+ "repetition_penalty",
402
+ self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
403
+ )
404
+ if (temperature := self.temperature) is None:
405
+ temperature = default_sampling_params.get(
406
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
407
+ if (top_p := self.top_p) is None:
408
+ top_p = default_sampling_params.get(
409
+ "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"])
410
+ if (top_k := self.top_k) is None:
411
+ top_k = default_sampling_params.get(
412
+ "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"])
413
+ if (min_p := self.min_p) is None:
414
+ min_p = default_sampling_params.get(
415
+ "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"])
416
+
417
prompt_logprobs = self.prompt_logprobs
418
if prompt_logprobs is None and self.echo:
419
prompt_logprobs = self.top_logprobs
@@ -403,11 +443,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
443
best_of=self.best_of,
444
presence_penalty=self.presence_penalty,
445
frequency_penalty=self.frequency_penalty,
406
- repetition_penalty=self.repetition_penalty,
407
- temperature=self.temperature,
408
- top_p=self.top_p,
409
- top_k=self.top_k,
410
- min_p=self.min_p,
446
+ repetition_penalty=repetition_penalty,
447
+ temperature=temperature,
448
+ top_p=top_p,
449
+ top_k=top_k,
450
+ min_p=min_p,
451
seed=self.seed,
452
stop=self.stop,
453
stop_token_ids=self.stop_token_ids,
@@ -584,15 +624,15 @@ class CompletionRequest(OpenAIBaseModel):
624
stream: Optional[bool] = False
625
stream_options: Optional[StreamOptions] = None
626
suffix: Optional[str] = None
587
- temperature: Optional[float] = 1.0
588
- top_p: Optional[float] = 1.0
627
+ temperature: Optional[float] = None
628
+ top_p: Optional[float] = None
629
user: Optional[str] = None
630
631
# doc: begin-completion-sampling-params
632
use_beam_search: bool = False
593
- top_k: int = -1
594
- min_p: float = 0.0
595
- repetition_penalty: float = 1.0
633
+ top_k: Optional[int] = None
634
+ min_p: Optional[float] = None
635
+ repetition_penalty: Optional[float] = None
636
length_penalty: float = 1.0
637
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
638
include_stop_str_in_output: bool = False
@@ -669,14 +709,30 @@ class CompletionRequest(OpenAIBaseModel):
709
710
# doc: end-completion-extra-params
711
672
- def to_beam_search_params(self,
673
- default_max_tokens: int) -> BeamSearchParams:
712
+ # Default sampling parameters for completion requests
713
+ _DEFAULT_SAMPLING_PARAMS: dict = {
714
+ "repetition_penalty": 1.0,
715
+ "temperature": 1.0,
716
+ "top_p": 1.0,
717
+ "top_k": -1,
718
+ "min_p": 0.0,
719
+ }
720
+
721
+ def to_beam_search_params(
722
+ self,
723
+ default_max_tokens: int,
724
+ default_sampling_params: Optional[dict] = None
725
+ ) -> BeamSearchParams:
726
max_tokens = self.max_tokens
727
if max_tokens is None:
728
max_tokens = default_max_tokens
729
730
+ if default_sampling_params is None:
731
+ default_sampling_params = {}
732
n = self.n if self.n is not None else 1
679
- temperature = self.temperature if self.temperature is not None else 0.0
733
+
734
+ if (temperature := self.temperature) is None:
735
+ temperature = default_sampling_params.get("temperature", 1.0)
736
737
return BeamSearchParams(
738
beam_width=n,
@@ -687,12 +743,35 @@ class CompletionRequest(OpenAIBaseModel):
743
include_stop_str_in_output=self.include_stop_str_in_output)
744
745
def to_sampling_params(
690
- self, default_max_tokens: int,
691
- logits_processor_pattern: Optional[str]) -> SamplingParams:
746
+ self,
747
+ default_max_tokens: int,
748
+ logits_processor_pattern: Optional[str],
749
+ default_sampling_params: Optional[dict] = None) -> SamplingParams:
750
max_tokens = self.max_tokens
751
if max_tokens is None:
752
max_tokens = default_max_tokens
753
754
+ if default_sampling_params is None:
755
+ default_sampling_params = {}
756
+ # Default parameters
757
+ if (repetition_penalty := self.repetition_penalty) is None:
758
+ repetition_penalty = default_sampling_params.get(
759
+ "repetition_penalty",
760
+ self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
761
+ )
762
+ if (temperature := self.temperature) is None:
763
+ temperature = default_sampling_params.get(
764
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
765
+ if (top_p := self.top_p) is None:
766
+ top_p = default_sampling_params.get(
767
+ "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"])
768
+ if (top_k := self.top_k) is None:
769
+ top_k = default_sampling_params.get(
770
+ "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"])
771
+ if (min_p := self.min_p) is None:
772
+ min_p = default_sampling_params.get(
773
+ "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"])
774
+
775
prompt_logprobs = self.prompt_logprobs
776
if prompt_logprobs is None and self.echo:
777
prompt_logprobs = self.logprobs
@@ -718,11 +797,11 @@ class CompletionRequest(OpenAIBaseModel):
797
best_of=self.best_of,
798
presence_penalty=self.presence_penalty,
799
frequency_penalty=self.frequency_penalty,
721
- repetition_penalty=self.repetition_penalty,
722
- temperature=self.temperature,
723
- top_p=self.top_p,
724
- top_k=self.top_k,
725
- min_p=self.min_p,
800
+ repetition_penalty=repetition_penalty,
801
+ temperature=temperature,
802
+ top_p=top_p,
803
+ top_k=top_k,
804
+ min_p=min_p,
805
seed=self.seed,
806
stop=self.stop,
807
stop_token_ids=self.stop_token_ids,
vllm/entrypoints/openai/serving_chat.py
+10
-2
@@ -91,6 +91,10 @@ class OpenAIServingChat(OpenAIServing):
91
"been registered") from e
92
93
self.enable_prompt_tokens_details = enable_prompt_tokens_details
94
+ diff_sampling_param = self.model_config.get_diff_sampling_param()
95
+ if diff_sampling_param:
96
+ logger.info("Overwriting default chat sampling param with: %s",
97
+ diff_sampling_param)
98
99
async def create_chat_completion(
100
self,
@@ -191,13 +195,17 @@ class OpenAIServingChat(OpenAIServing):
195
sampling_params: Union[SamplingParams, BeamSearchParams]
196
default_max_tokens = self.max_model_len - len(
197
engine_prompt["prompt_token_ids"])
198
+ # Build default sampling params
199
+ default_sampling_params = (
200
+ self.model_config.get_diff_sampling_param())
201
if request.use_beam_search:
202
sampling_params = request.to_beam_search_params(
196
- default_max_tokens)
203
+ default_max_tokens, default_sampling_params)
204
else:
205
sampling_params = request.to_sampling_params(
206
default_max_tokens,
200
- self.model_config.logits_processor_pattern)
207
+ self.model_config.logits_processor_pattern,
208
+ default_sampling_params)
209
210
self._log_inputs(request_id,
211
request_prompts[i],
vllm/entrypoints/openai/serving_completion.py
+11
-2
@@ -55,6 +55,11 @@ class OpenAIServingCompletion(OpenAIServing):
55
prompt_adapters=prompt_adapters,
56
request_logger=request_logger,
57
return_tokens_as_token_ids=return_tokens_as_token_ids)
58
+ diff_sampling_param = self.model_config.get_diff_sampling_param()
59
+ if diff_sampling_param:
60
+ logger.info(
61
+ "Overwriting default completion sampling param with: %s",
62
+ diff_sampling_param)
63
64
async def create_completion(
65
self,
@@ -118,13 +123,17 @@ class OpenAIServingCompletion(OpenAIServing):
123
sampling_params: Union[SamplingParams, BeamSearchParams]
124
default_max_tokens = self.max_model_len - len(
125
engine_prompt["prompt_token_ids"])
126
+ # Build default sampling params
127
+ default_sampling_params = (
128
+ self.model_config.get_diff_sampling_param())
129
if request.use_beam_search:
130
sampling_params = request.to_beam_search_params(
123
- default_max_tokens)
131
+ default_max_tokens, default_sampling_params)
132
else:
133
sampling_params = request.to_sampling_params(
134
default_max_tokens,
127
- self.model_config.logits_processor_pattern)
135
+ self.model_config.logits_processor_pattern,
136
+ default_sampling_params)
137
138
request_id_item = f"{request_id}-{i}"
139
vllm/v1/engine/processor.py
+3
-17
@@ -1,5 +1,5 @@
1
import time
2
-from typing import Any, Dict, Mapping, Optional, Tuple, Union
2
+from typing import Mapping, Optional, Tuple, Union
3
4
from vllm.config import CacheConfig, LoRAConfig, ModelConfig
5
from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
@@ -12,7 +12,6 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
12
from vllm.pooling_params import PoolingParams
13
from vllm.prompt_adapter.request import PromptAdapterRequest
14
from vllm.sampling_params import SamplingParams
15
-from vllm.transformers_utils.config import try_get_generation_config
15
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16
from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
17
from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
@@ -34,8 +33,8 @@ class Processor:
33
self.lora_config = lora_config
34
self.tokenizer = tokenizer
35
37
- self.generation_config_fields = _load_generation_config_dict(
38
- model_config)
36
+ self.generation_config_fields = model_config.try_get_generation_config(
37
+ )
38
self.input_preprocessor = InputPreprocessor(model_config,
39
self.tokenizer,
40
mm_registry)
@@ -181,16 +180,3 @@ class Processor:
180
# TODO: Find out how many placeholder tokens are there so we can
181
# check that chunked prefill does not truncate them
182
# max_batch_len = self.scheduler_config.max_num_batched_tokens
184
-
185
-
186
-def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
187
- config = try_get_generation_config(
188
- model_config.model,
189
- trust_remote_code=model_config.trust_remote_code,
190
- revision=model_config.revision,
191
- )
192
-
193
- if config is None:
194
- return {}
195
-
196
- return config.to_diff_dict()