A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[VLM] Clean up models (#16873)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Apr 19, 2025 at 20:13 UTC
205d84aaa974d6c7752705bb2835bac6809794cc
4 files changed
+2
-42
examples/offline_inference/mistral-small.py
+1
@@ -62,6 +62,7 @@ def run_simple_demo(args: argparse.Namespace):
62
tokenizer_mode="mistral" if args.format == "mistral" else "auto",
63
config_format="mistral" if args.format == "mistral" else "auto",
64
load_format="mistral" if args.format == "mistral" else "auto",
65
+ limit_mm_per_prompt={"image": 1},
66
max_model_len=4096,
67
max_num_seqs=2,
68
tensor_parallel_size=2,
examples/offline_inference/vision_language.py
+1
-1
@@ -957,7 +957,7 @@ def run_qwen2_5_omni(questions: list[str], modality: str):
957
"max_pixels": 1280 * 28 * 28,
958
"fps": [1],
959
},
960
- disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
960
+ limit_mm_per_prompt={"image": 1},
961
)
962
963
if modality == "image":
vllm/model_executor/models/phi4mm.py
-23
@@ -503,26 +503,6 @@ class Phi4MMProcessingInfo(BaseProcessingInfo):
503
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
504
return {"audio": None, "image": None}
505
506
- def get_mm_max_tokens_per_item(
507
- self,
508
- seq_len: int,
509
- mm_counts: Mapping[str, int],
510
- ) -> Mapping[str, int]:
511
- return {
512
- "image": self.get_max_image_tokens(),
513
- "audio": self.get_max_audio_tokens(),
514
- }
515
-
516
- def get_max_audio_tokens(self) -> int:
517
- sr = self.get_feature_extractor().sampling_rate
518
- num_frames = self.get_audio_num_frames(_AUDIO_MAX_SOUNDFILE_SIZE, sr)
519
- return self._compute_audio_embed_size(num_frames)
520
-
521
- def get_max_image_tokens(self) -> int:
522
- target_width, target_height = self.get_image_size_with_most_features()
523
- return self.get_num_image_tokens(image_width=target_width,
524
- image_height=target_height)
525
-
506
def _find_target_aspect_ratio(
507
self,
508
orig_width: int,
@@ -764,9 +744,6 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
744
num_audios = mm_counts.get("audio", 0)
745
num_images = mm_counts.get("image", 0)
746
767
- target_width, target_height = \
768
- self.info.get_image_size_with_most_features()
769
-
747
target_width, target_height = \
748
self.info.get_image_size_with_most_features()
749
vllm/model_executor/models/qwen2_5_omni_thinker.py
-18
@@ -172,26 +172,9 @@ class Qwen2_5OmniThinkerProcessingInfo(Qwen2AudioProcessingInfo,
172
assert isinstance(feature_extractor, WhisperFeatureExtractor)
173
return feature_extractor
174
175
- def get_max_audio_tokens(self) -> int:
176
- hf_config = self.get_hf_config()
177
- max_source_position = hf_config.audio_config.max_source_positions
178
- output_lengths = (max_source_position - 2) // 2 + 1
179
- return output_lengths
180
-
175
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
176
return {"audio": None, "image": None, "video": None}
177
184
- def get_mm_max_tokens_per_item(
185
- self,
186
- seq_len: int,
187
- mm_counts: Mapping[str, int],
188
- ) -> Mapping[str, int]:
189
- return {
190
- "audio": self.get_max_audio_tokens(),
191
- "image": self.get_max_image_tokens(),
192
- "video": self.get_max_video_tokens(seq_len, mm_counts),
193
- }
194
-
178
179
class Qwen2_5OmniThinkerDummyInputsBuilder(
180
BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo]):
@@ -210,7 +193,6 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
193
return (audio_token * num_audios + image_token * num_images +
194
video_token * num_videos)
195
213
- # TODO: @abstractmethod after transition
196
def get_dummy_mm_data(
197
self,
198
seq_len: int,