A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 12:00 UTC
[Model] Remove unnecessary processor definition for Nemotron Parse (#37456)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 19, 2026 at 02:25 UTC
7476d148db996e6c9c942d5760e94e59cc10787d
3 files changed
-259
vllm/model_executor/models/nemotron_parse.py
-12
@@ -55,7 +55,6 @@ from vllm.multimodal.processing import (
55
)
56
from vllm.renderers import TokenizeParams
57
from vllm.transformers_utils.configs.radio import RadioConfig
58
-from vllm.transformers_utils.processors.nemotron_parse import NemotronParseProcessor
58
from vllm.utils.tensor_schema import TensorSchema, TensorShape
59
from vllm.v1.attention.backend import AttentionType
60
@@ -367,17 +366,6 @@ class NemotronParsePixelInputs(TensorSchema):
366
367
368
class NemotronParseProcessingInfo(BaseProcessingInfo):
370
- def get_hf_config(self):
371
- return self.ctx.get_hf_config()
372
-
373
- def get_hf_processor(self, **kwargs) -> NemotronParseProcessor:
374
- return self.ctx.init_processor(
375
- NemotronParseProcessor,
376
- config=self.get_hf_config(),
377
- tokenizer=self.get_tokenizer(),
378
- **kwargs,
379
- )
380
-
369
def get_default_tok_params(self) -> TokenizeParams:
370
return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
371
vllm/transformers_utils/processors/__init__.py
-2
@@ -25,7 +25,6 @@ __all__ = [
25
"MistralCommonPixtralProcessor",
26
"MistralCommonVoxtralProcessor",
27
"NanoNemotronVLProcessor",
28
- "NemotronParseProcessor",
28
"NemotronVLProcessor",
29
"LlamaNemotronVLEmbedProcessor",
30
"NVLMProcessor",
@@ -50,7 +49,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
49
"MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
50
"MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
51
"NanoNemotronVLProcessor": "vllm.transformers_utils.processors.nano_nemotron_vl",
53
- "NemotronParseProcessor": "vllm.transformers_utils.processors.nemotron_parse",
52
"NemotronVLProcessor": "vllm.transformers_utils.processors.nemotron_vl",
53
"LlamaNemotronVLEmbedProcessor": "vllm.transformers_utils.processors.nemotron_vl",
54
"NVLMProcessor": "vllm.transformers_utils.processors.nvlm_d",
vllm/transformers_utils/processors/nemotron_parse.py
deleted
-245
@@ -1,245 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-#
4
-# Adapted from https://github.com/amalad/vllm/blob/nemotron_parse/vllm/model_executor/models/nemotron_parse.py
5
-# that's based on https://huggingface.co/nvidia/NVIDIA-Nemotron-Parse-v1.1/blob/main/hf_nemotron_parse_modeling.py
6
-from typing import TypeVar
7
-
8
-import numpy as np
9
-import torch
10
-from PIL import Image
11
-from timm.data.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD
12
-from torchvision import transforms as T
13
-from transformers import BatchFeature, PretrainedConfig, TensorType
14
-
15
-from vllm.tokenizers import TokenizerLike
16
-
17
-_T = TypeVar("_T")
18
-
19
-DEFAULT_FINAL_IMAGE_SIZE = (2048, 1648)
20
-
21
-
22
-class NemotronParseImageProcessor:
23
- """
24
- NemotronParse Image Processor
25
- """
26
-
27
- def __init__(
28
- self,
29
- final_size: tuple = DEFAULT_FINAL_IMAGE_SIZE,
30
- **kwargs,
31
- ):
32
- # Ensure final_size is properly formatted
33
- if isinstance(final_size, (list, tuple)) and len(final_size) >= 2:
34
- self.final_size = (int(final_size[0]), int(final_size[1]))
35
- elif isinstance(final_size, (int, float)):
36
- self.final_size = (int(final_size), int(final_size))
37
- else:
38
- self.final_size = DEFAULT_FINAL_IMAGE_SIZE # Default fallback
39
-
40
- self.norm_mean = torch.Tensor(OPENAI_CLIP_MEAN).reshape(1, 3, 1, 1)
41
- self.norm_std = torch.Tensor(OPENAI_CLIP_STD).reshape(1, 3, 1, 1)
42
-
43
- # Create transforms
44
- self._create_transforms()
45
-
46
- def _create_transforms(self):
47
- """Create transform objects."""
48
- try:
49
- import albumentations as A
50
- except ImportError as err:
51
- raise ImportError(
52
- "The package `albumentations` is required to use "
53
- "NemotronParse model. Please install it with `pip install "
54
- "albumentations`."
55
- ) from err
56
-
57
- # Ensure final_size is a tuple of integers
58
- if isinstance(self.final_size, (list, tuple)):
59
- self.target_height, self.target_width = (
60
- int(self.final_size[0]),
61
- int(self.final_size[1]),
62
- )
63
- else:
64
- self.target_height = self.target_width = int(self.final_size)
65
-
66
- import cv2
67
-
68
- self.transform = A.Compose(
69
- [
70
- A.PadIfNeeded(
71
- min_height=self.target_height,
72
- min_width=self.target_width,
73
- border_mode=cv2.BORDER_CONSTANT,
74
- fill=[255, 255, 255],
75
- p=1.0,
76
- ),
77
- ]
78
- )
79
-
80
- self.torch_transform = T.Compose(
81
- [
82
- T.ToTensor(),
83
- ]
84
- )
85
-
86
- def _resize_with_aspect_ratio(self, image: np.ndarray) -> np.ndarray:
87
- """Resize image maintaining aspect ratio (exact replica of original
88
- LongestMaxSizeHW)."""
89
- height, width = image.shape[:2]
90
- max_size_height = self.target_height
91
- max_size_width = self.target_width
92
-
93
- # Original LongestMaxSizeHW algorithm from custom_augmentations.py
94
- aspect_ratio = width / height
95
- new_height = height
96
- new_width = width
97
-
98
- # If height too big then scale image down
99
- if height > max_size_height:
100
- new_height = max_size_height
101
- new_width = int(new_height * aspect_ratio)
102
-
103
- # If width too big, scale image down further
104
- if new_width > max_size_width:
105
- new_width = max_size_width
106
- new_height = int(new_width / aspect_ratio)
107
-
108
- # Use cv2.INTER_LINEAR like the original
109
- import cv2
110
-
111
- return cv2.resize(
112
- image, (new_width, new_height), interpolation=cv2.INTER_LINEAR
113
- )
114
-
115
- def _pad_to_size(self, image: np.ndarray) -> np.ndarray:
116
- """Pad image to target size with white padding (matches A.PadIfNeeded
117
- behavior)."""
118
- h, w = image.shape[:2]
119
- min_height, min_width = self.target_height, self.target_width
120
-
121
- # Only pad if image is smaller than target (matches A.PadIfNeeded logic)
122
- pad_h = max(0, min_height - h)
123
- pad_w = max(0, min_width - w)
124
-
125
- if pad_h == 0 and pad_w == 0:
126
- return image
127
-
128
- # A.PadIfNeeded pads to bottom-right with constant value
129
- if len(image.shape) == 3:
130
- # Color image - pad bottom and right with white (255, 255, 255)
131
- padded = np.pad(
132
- image,
133
- ((0, pad_h), (0, pad_w), (0, 0)),
134
- mode="constant",
135
- constant_values=255,
136
- )
137
- else:
138
- # Grayscale image - pad with white (255)
139
- padded = np.pad(
140
- image, ((0, pad_h), (0, pad_w)), mode="constant", constant_values=255
141
- )
142
-
143
- return padded
144
-
145
- def preprocess(
146
- self,
147
- images: Image.Image | list[Image.Image],
148
- **kwargs,
149
- ) -> dict[str, torch.Tensor]:
150
- """
151
- Preprocess an image or batch of images for the NemotronParse model.
152
-
153
- Args:
154
- images: Input image(s)
155
- """
156
- # Ensure images is a list
157
- if not isinstance(images, list):
158
- images = [images]
159
-
160
- # Convert PIL images to numpy arrays if needed
161
- processed_images = []
162
- for image in images:
163
- if isinstance(image, Image.Image):
164
- image = np.asarray(image)
165
- processed_images.append(image)
166
-
167
- # Apply NemotronParse-specific transforms
168
- pixel_values = []
169
- for image in processed_images:
170
- # Manual resize with aspect ratio preservation
171
- # (replaces LongestMaxSizeHW)
172
- processed_image = self._resize_with_aspect_ratio(image)
173
-
174
- # Apply remaining albumentations transforms if available
175
- if self.transform is not None:
176
- transformed = self.transform(image=processed_image)
177
- processed_image = transformed["image"]
178
- else:
179
- # Fallback: just pad to target size
180
- processed_image = self._pad_to_size(processed_image)
181
-
182
- # Convert to tensor
183
- pixel_values_tensor = self.torch_transform(processed_image)
184
-
185
- # Handle grayscale images
186
- if pixel_values_tensor.shape[0] == 1:
187
- pixel_values_tensor = pixel_values_tensor.expand(3, -1, -1)
188
-
189
- pixel_values.append(pixel_values_tensor)
190
-
191
- # Stack into batch
192
- pixel_values = torch.stack(pixel_values)
193
-
194
- # Normalize pixel values
195
- normalized_values = (pixel_values - self.norm_mean) / self.norm_std
196
- return {"pixel_values": normalized_values}
197
-
198
- def __call__(
199
- self, images: Image.Image | list[Image.Image], **kwargs
200
- ) -> dict[str, torch.Tensor]:
201
- return self.preprocess(images, **kwargs)
202
-
203
-
204
-class NemotronParseProcessor:
205
- """
206
- NemotronParse Processor
207
- """
208
-
209
- def __init__(
210
- self,
211
- config: PretrainedConfig,
212
- tokenizer: TokenizerLike,
213
- **kwargs,
214
- ) -> None:
215
- super().__init__()
216
-
217
- self.config = config
218
- self.tokenizer = tokenizer
219
-
220
- self.image_processor = NemotronParseImageProcessor(final_size=config.image_size)
221
-
222
- def _make_batch_input(self, input_item: _T | list[_T] | None = None) -> list[_T]:
223
- if input_item is None:
224
- input_item = []
225
- if not isinstance(input_item, list):
226
- input_item = [input_item]
227
- return input_item
228
-
229
- def __call__(
230
- self,
231
- text: str | list[str] | None = None,
232
- images: Image.Image | list[Image.Image] | None = None,
233
- return_tensors: str | TensorType | None = None,
234
- **kwargs,
235
- ) -> BatchFeature:
236
- text = self._make_batch_input(text)
237
- images = self._make_batch_input(images)
238
- image_inputs = {} if len(images) == 0 else self.image_processor(images)
239
-
240
- text_inputs = self.tokenizer(text, add_special_tokens=False, **kwargs)
241
- combined_outputs = BatchFeature(
242
- data={**text_inputs, **image_inputs},
243
- tensor_type=return_tensors,
244
- )
245
- return combined_outputs