@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d30aa7e9e6

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 18:30 UTC

[Bugfix] Limit profiling run sequence length by max_model_len (#14785)

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>

Kyle Sayers committed Mar 16, 2025 at 10:44 UTC d30aa7e9e6afd6147865c8c9fae8cd21f5ddce3d

5 files changed +9

vllm/inputs/registry.py

@@ -330,6 +330,11 @@ class InputRegistry:

330 from vllm.multimodal import MultiModalKwargs

331 from vllm.multimodal.profiling import MultiModalProfiler

332

333 + if seq_len > model_config.max_model_len:

334 + raise AssertionError(

335 + f"Profiling attempted with sequence length ({seq_len}) "

336 + f"greater than model length ({model_config.max_model_len})")

337 +

338 if mm_registry.has_processor(model_config):

339 tokenizer = cached_tokenizer_from_config(model_config)

340 processor = mm_registry.create_processor(model_config,

vllm/worker/enc_dec_model_runner.py

@@ -281,6 +281,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):

281 for group_id in range(max_num_seqs):

282 seq_len = (max_num_batched_tokens // max_num_seqs +

283 (group_id < max_num_batched_tokens % max_num_seqs))

284 + seq_len = min(seq_len, self.model_config.max_model_len)

285 batch_size += seq_len

286

287 decoder_dummy_data = self.input_registry \

vllm/worker/model_runner.py

@@ -1302,6 +1302,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):

1302 for group_id in range(max_num_seqs):

1303 seq_len = (max_num_batched_tokens // max_num_seqs +

1304 (group_id < max_num_batched_tokens % max_num_seqs))

1305 + seq_len = min(seq_len, self.model_config.max_model_len)

1306 batch_size += seq_len

1307

1308 dummy_data = self.input_registry \

vllm/worker/openvino_model_runner.py

@@ -148,6 +148,7 @@ class OpenVINOModelRunner(ModelRunnerBase):

148 seq_len = min(

149 seq_data.get_len(),

150 computed_len + seq_group_metadata.token_chunk_size,

151 + self.model_config.max_model_len,

152 )

153 if is_prompt:

154 tokens = seq_data.get_token_ids()[computed_len:seq_len]

vllm/worker/xpu_model_runner.py

@@ -466,6 +466,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):

466 for group_id in range(max_num_seqs):

467 seq_len = (max_num_batched_tokens // max_num_seqs +

468 (group_id < max_num_batched_tokens % max_num_seqs))

469 + seq_len = min(seq_len, self.model_config.max_model_len)

470 batch_size += seq_len

471

472 dummy_data = self.input_registry \