A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
Fix performance when `--generation-config` is not `None` (#14223)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 4, 2025 at 20:59 UTC
9badee53decb3d432dc805336abfb0eb81dfb48f
4 files changed
+23
-25
vllm/entrypoints/llm.py
+6
-4
@@ -244,6 +244,7 @@ class LLM:
244
engine_args, usage_context=UsageContext.LLM_CLASS)
245
246
self.request_counter = Counter()
247
+ self.default_sampling_params: Union[dict[str, Any], None] = None
248
249
@staticmethod
250
def get_engine_class() -> type[LLMEngine]:
@@ -268,10 +269,11 @@ class LLM:
269
tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)
270
271
def get_default_sampling_params(self) -> SamplingParams:
271
- diff_sampling_param = (
272
- self.llm_engine.model_config.get_diff_sampling_param())
273
- if diff_sampling_param:
274
- return SamplingParams.from_optional(**diff_sampling_param)
272
+ if self.default_sampling_params is None:
273
+ self.default_sampling_params = (
274
+ self.llm_engine.model_config.get_diff_sampling_param())
275
+ if self.default_sampling_params:
276
+ return SamplingParams.from_optional(**self.default_sampling_params)
277
return SamplingParams()
278
279
@overload
vllm/entrypoints/openai/serving_chat.py
+6
-8
@@ -105,10 +105,11 @@ class OpenAIServingChat(OpenAIServing):
105
"been registered") from e
106
107
self.enable_prompt_tokens_details = enable_prompt_tokens_details
108
- diff_sampling_param = self.model_config.get_diff_sampling_param()
109
- if diff_sampling_param:
108
+ self.default_sampling_params = (
109
+ self.model_config.get_diff_sampling_param())
110
+ if self.default_sampling_params:
111
logger.info("Overwriting default chat sampling param with: %s",
111
- diff_sampling_param)
112
+ self.default_sampling_params)
113
114
async def create_chat_completion(
115
self,
@@ -210,17 +211,14 @@ class OpenAIServingChat(OpenAIServing):
211
sampling_params: Union[SamplingParams, BeamSearchParams]
212
default_max_tokens = self.max_model_len - len(
213
engine_prompt["prompt_token_ids"])
213
- # Build default sampling params
214
- default_sampling_params = (
215
- self.model_config.get_diff_sampling_param())
214
if request.use_beam_search:
215
sampling_params = request.to_beam_search_params(
218
- default_max_tokens, default_sampling_params)
216
+ default_max_tokens, self.default_sampling_params)
217
else:
218
sampling_params = request.to_sampling_params(
219
default_max_tokens,
220
self.model_config.logits_processor_pattern,
223
- default_sampling_params)
221
+ self.default_sampling_params)
222
223
self._log_inputs(request_id,
224
request_prompts[i],
vllm/entrypoints/openai/serving_completion.py
+6
-8
@@ -51,11 +51,12 @@ class OpenAIServingCompletion(OpenAIServing):
51
models=models,
52
request_logger=request_logger,
53
return_tokens_as_token_ids=return_tokens_as_token_ids)
54
- diff_sampling_param = self.model_config.get_diff_sampling_param()
55
- if diff_sampling_param:
54
+ self.default_sampling_params = (
55
+ self.model_config.get_diff_sampling_param())
56
+ if self.default_sampling_params:
57
logger.info(
58
"Overwriting default completion sampling param with: %s",
58
- diff_sampling_param)
59
+ self.default_sampling_params)
60
61
async def create_completion(
62
self,
@@ -119,17 +120,14 @@ class OpenAIServingCompletion(OpenAIServing):
120
sampling_params: Union[SamplingParams, BeamSearchParams]
121
default_max_tokens = self.max_model_len - len(
122
engine_prompt["prompt_token_ids"])
122
- # Build default sampling params
123
- default_sampling_params = (
124
- self.model_config.get_diff_sampling_param())
123
if request.use_beam_search:
124
sampling_params = request.to_beam_search_params(
127
- default_max_tokens, default_sampling_params)
125
+ default_max_tokens, self.default_sampling_params)
126
else:
127
sampling_params = request.to_sampling_params(
128
default_max_tokens,
129
self.model_config.logits_processor_pattern,
132
- default_sampling_params)
130
+ self.default_sampling_params)
131
132
request_id_item = f"{request_id}-{i}"
133
vllm/entrypoints/openai/serving_transcription.py
+5
-5
@@ -161,11 +161,12 @@ class OpenAIServingTranscription(OpenAIServing):
161
request_logger=request_logger,
162
return_tokens_as_token_ids=return_tokens_as_token_ids)
163
164
- diff_sampling_param = self.model_config.get_diff_sampling_param()
165
- if diff_sampling_param:
164
+ self.default_sampling_params = (
165
+ self.model_config.get_diff_sampling_param())
166
+ if self.default_sampling_params:
167
logger.info(
168
"Overwriting default completion sampling param with: %s",
168
- diff_sampling_param)
169
+ self.default_sampling_params)
170
171
async def _preprocess_transcription(
172
self,
@@ -273,9 +274,8 @@ class OpenAIServingTranscription(OpenAIServing):
274
try:
275
# TODO(rob): subtract len of tokenized prompt.
276
default_max_tokens = self.model_config.max_model_len
276
- default_params = self.model_config.get_diff_sampling_param()
277
sampling_params = request.to_sampling_params(
278
- default_max_tokens, default_params)
278
+ default_max_tokens, self.default_sampling_params)
279
280
self._log_inputs(
281
request_id,