A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 20:15 UTC
[Models] Consolidate Deepseek-OCR2 processor (#33909)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Feb 6, 2026 at 02:29 UTC
87d0d17ab583740bce777f334a6281edf9822e78
5 files changed
+52
-336
vllm/model_executor/models/deepencoder2.py
+1
-1
@@ -31,7 +31,7 @@ class CustomQwen2Decoder(nn.Module):
31
num_key_value_heads: int = 2,
32
intermediate_size: int = 4864,
33
vocab_size: int = 151936,
34
- attn_implementation: str = "sdpa", # ⭐
34
+ attn_implementation: str = "sdpa",
35
rms_norm_eps: float = 1e-06,
36
rope_theta: float = 1000000.0,
37
attention_dropout: float = 0.0,
vllm/model_executor/models/deepseek_ocr.py
+10
-2
@@ -52,7 +52,6 @@ from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
52
from vllm.transformers_utils.processors.deepseek_ocr import (
53
BASE_SIZE,
54
CROP_MODE,
55
- IMAGE_SIZE,
55
DeepseekOCRProcessor,
56
count_tiles,
57
)
@@ -66,6 +65,7 @@ from .deepencoder import DeepCLIPVisionTransformer, build_sam_vit_b
65
from .deepseek_vl2 import MlpProjector
66
67
# The image token id may be various
68
+IMAGE_SIZE = 640
69
_IMAGE_TOKEN = "<image>"
70
71
@@ -190,7 +190,15 @@ class DeepseekOCRProcessingInfo(BaseProcessingInfo):
190
return self.ctx.get_hf_config(DeepseekVLV2Config)
191
192
def get_hf_processor(self, **kwargs: object):
193
- return self.ctx.get_hf_processor(DeepseekOCRProcessor, **kwargs)
193
+ v1_processor_config = dict(
194
+ image_size=IMAGE_SIZE,
195
+ base_size=BASE_SIZE,
196
+ crop_mode=CROP_MODE,
197
+ strategy="v1",
198
+ )
199
+ return self.ctx.get_hf_processor(
200
+ DeepseekOCRProcessor, **{**kwargs, **v1_processor_config}
201
+ )
202
203
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
204
return {"image": None}
vllm/model_executor/models/deepseek_ocr2.py
+12
-4
@@ -48,11 +48,10 @@ from vllm.multimodal.processing import (
48
from vllm.sequence import IntermediateTensors
49
from vllm.tokenizers import cached_tokenizer_from_config
50
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
51
-from vllm.transformers_utils.processors.deepseek_ocr2 import (
51
+from vllm.transformers_utils.processors.deepseek_ocr import (
52
BASE_SIZE,
53
CROP_MODE,
54
- IMAGE_SIZE,
55
- DeepseekOCR2Processor,
54
+ DeepseekOCRProcessor,
55
)
56
57
from ...transformers_utils.processors.deepseek_ocr import count_tiles
@@ -62,6 +61,7 @@ from .deepseek_ocr import DeepseekOCRImagePixelInputs
61
from .deepseek_vl2 import MlpProjector
62
63
# The image token id may be various
64
+IMAGE_SIZE = 768 # different from deepseek-ocr
65
_IMAGE_TOKEN = "<image>"
66
67
@@ -70,7 +70,15 @@ class DeepseekOCR2ProcessingInfo(BaseProcessingInfo):
70
return self.ctx.get_hf_config(DeepseekVLV2Config)
71
72
def get_hf_processor(self, **kwargs: object):
73
- return self.ctx.get_hf_processor(DeepseekOCR2Processor, **kwargs)
73
+ v2_processor_config = dict(
74
+ image_size=IMAGE_SIZE,
75
+ base_size=BASE_SIZE,
76
+ crop_mode=CROP_MODE,
77
+ strategy="v2",
78
+ )
79
+ return self.ctx.get_hf_processor(
80
+ DeepseekOCRProcessor, **{**kwargs, **v2_processor_config}
81
+ )
82
83
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
84
return {"image": None}
vllm/transformers_utils/processors/deepseek_ocr.py
+29
-9
@@ -1,7 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
# adapted from https://github.com/deepseek-ai/DeepSeek-OCR/blob/main/DeepSeek-OCR-master/DeepSeek-OCR-vllm/process/image_process.py
4
+# and https://github.com/deepseek-ai/DeepSeek-OCR-2/blob/main/DeepSeek-OCR2-master/DeepSeek-OCR2-vllm/process/image_process.py
5
import math
6
+from typing import Literal
7
8
import torch
9
import torchvision.transforms as T
@@ -156,10 +158,19 @@ class DeepseekOCRProcessor(ProcessorMixin):
158
sft_format: str = "deepseek",
159
mask_prompt: bool = True,
160
ignore_id: int = -100,
161
+ image_size: int = IMAGE_SIZE,
162
+ base_size: int = BASE_SIZE,
163
+ strategy: Literal["v1", "v2"] = "v1",
164
**kwargs,
165
):
161
- self.image_size = IMAGE_SIZE
162
- self.base_size = BASE_SIZE
166
+ self.image_size = image_size
167
+ self.base_size = base_size
168
+
169
+ # image token calculation strategy for
170
+ # Deepseek-OCR and Deepseek-OCR-2
171
+ self.strategy = strategy
172
+ assert strategy in ["v1", "v2"], "Only 'v1' and 'v2' strategies are supported."
173
+
174
self.patch_size = 16
175
self.image_mean = image_mean
176
self.image_std = image_std
@@ -317,16 +328,16 @@ class DeepseekOCRProcessor(ProcessorMixin):
328
image_shapes.append(image.size)
329
330
images_crop_raw = []
320
- if image.size[0] <= 640 and image.size[1] <= 640:
331
+ if image.size[0] <= self.image_size and image.size[1] <= self.image_size:
332
crop_ratio = [1, 1]
333
elif cropping:
334
images_crop_raw, crop_ratio = dynamic_preprocess(
324
- image, image_size=IMAGE_SIZE
335
+ image, image_size=self.image_size
336
)
337
else:
338
crop_ratio = [1, 1]
339
329
- if self.image_size <= 640 and not cropping:
340
+ if not cropping:
341
image = image.resize((self.image_size, self.image_size))
342
343
global_view = ImageOps.pad(
@@ -350,12 +361,21 @@ class DeepseekOCRProcessor(ProcessorMixin):
361
(self.base_size // self.patch_size) / self.downsample_ratio
362
)
363
353
- tokenized_image = (
354
- [self.image_token_id] * num_queries_base + [self.image_token_id]
355
- ) * num_queries_base
364
+ num_tokens_base = (
365
+ (num_queries_base * (num_queries_base + 1))
366
+ if self.strategy == "v1"
367
+ else num_queries_base * num_queries_base
368
+ )
369
+ tokenized_image = [self.image_token_id] * num_tokens_base
370
+
371
tokenized_image += [self.image_token_id]
372
if num_width_tiles > 1 or num_height_tiles > 1:
358
- local_row = [self.image_token_id] * (num_queries * num_width_tiles + 1)
373
+ num_tokens_per_row = (
374
+ num_queries * num_width_tiles + 1
375
+ if self.strategy == "v1"
376
+ else num_queries * num_width_tiles
377
+ )
378
+ local_row = [self.image_token_id] * num_tokens_per_row
379
tokenized_image += local_row * (num_queries * num_height_tiles)
380
tokenized_str += tokenized_image
381
images_seq_mask += [True] * len(tokenized_image)
vllm/transformers_utils/processors/deepseek_ocr2.py
deleted
-320
@@ -1,320 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-# adapted from https://github.com/deepseek-ai/DeepSeek-OCR/blob/main/DeepSeek-OCR-master/DeepSeek-OCR-vllm/process/image_process.py
4
-import math
5
-
6
-import torch
7
-from PIL import Image, ImageOps
8
-from transformers import AutoProcessor, BatchFeature, LlamaTokenizerFast
9
-from transformers.processing_utils import ProcessorMixin
10
-
11
-from vllm.transformers_utils.processors.deepseek_ocr import (
12
- ImageTransform,
13
- dynamic_preprocess,
14
-)
15
-
16
-BASE_SIZE = 1024
17
-IMAGE_SIZE = 768
18
-CROP_MODE = True
19
-MIN_CROPS = 2
20
-MAX_CROPS = 6
21
-
22
-
23
-class DeepseekOCR2Processor(ProcessorMixin):
24
- tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast")
25
- attributes = ["tokenizer"]
26
-
27
- def __init__(
28
- self,
29
- tokenizer: LlamaTokenizerFast,
30
- patch_size: int = 16,
31
- downsample_ratio: int = 4,
32
- image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
33
- image_std: tuple[float, float, float] = (0.5, 0.5, 0.5),
34
- normalize: bool = True,
35
- image_token: str = "<image>",
36
- pad_token: str = "<｜▁pad▁｜>",
37
- add_special_token: bool = False,
38
- sft_format: str = "deepseek",
39
- mask_prompt: bool = True,
40
- ignore_id: int = -100,
41
- **kwargs,
42
- ):
43
- self.image_size = IMAGE_SIZE
44
- self.base_size = BASE_SIZE
45
- self.patch_size = 16
46
- self.image_mean = image_mean
47
- self.image_std = image_std
48
- self.normalize = normalize
49
- self.downsample_ratio = 4
50
-
51
- self.image_transform = ImageTransform(
52
- mean=image_mean, std=image_std, normalize=normalize
53
- )
54
-
55
- self.tokenizer = tokenizer
56
- self.tokenizer.padding_side = "left" # must set this，padding side with make a difference in batch inference # noqa: E501
57
-
58
- # add the pad_token as special token to use 'tokenizer.pad_token'
59
- # and 'tokenizer.pad_token_id'
60
- if self.tokenizer.pad_token is None:
61
- self.tokenizer.add_special_tokens({"pad_token": pad_token})
62
-
63
- # add image token
64
- self.image_token_id = self.tokenizer.vocab.get(image_token)
65
- self.image_token = image_token
66
- self.pad_token = pad_token
67
- self.add_special_token = add_special_token
68
- self.sft_format = sft_format
69
- self.mask_prompt = mask_prompt
70
- self.ignore_id = ignore_id
71
-
72
- super().__init__(
73
- tokenizer,
74
- **kwargs,
75
- )
76
-
77
- @property
78
- def bos_id(self):
79
- return self.tokenizer.bos_token_id
80
-
81
- @property
82
- def eos_id(self):
83
- return self.tokenizer.eos_token_id
84
-
85
- @property
86
- def pad_id(self):
87
- return self.tokenizer.pad_token_id
88
-
89
- def encode(self, text: str, bos: bool = True, eos: bool = False):
90
- t = self.tokenizer.encode(text, add_special_tokens=False)
91
- if bos:
92
- t = [self.bos_id] + t
93
- if eos:
94
- t = t + [self.eos_id]
95
- return t
96
-
97
- def decode(self, t: list[int], **kwargs) -> str:
98
- return self.tokenizer.decode(t, **kwargs)
99
-
100
- def process_one(
101
- self,
102
- prompt: str,
103
- images: list[Image.Image],
104
- crop_mode: bool = CROP_MODE,
105
- ):
106
- """
107
-
108
- Args:
109
- prompt (str): the formatted prompt;
110
- images (List[ImageType]): the list of images;
111
- crop_mode (bool): if True, then crop the image;
112
-
113
- Returns:
114
- outputs (BaseProcessorOutput): the output of the processor,
115
- - input_ids (torch.LongTensor): [N + image tokens]
116
- - target_ids (torch.LongTensor): [N + image tokens]
117
- - pixel_values (torch.FloatTensor): [n_patches, 3, H, W]
118
- - image_id (int): the id of the image token
119
- - num_image_tokens (List[int]): the number of image tokens
120
- """
121
-
122
- assert prompt is not None and images is not None, (
123
- "prompt and images must be used at the same time."
124
- )
125
-
126
- sft_format = prompt
127
-
128
- (
129
- input_ids,
130
- pixel_values,
131
- images_crop,
132
- images_seq_mask,
133
- images_spatial_crop,
134
- num_image_tokens,
135
- _,
136
- ) = self.tokenize_with_images(
137
- conversation=sft_format,
138
- images=images,
139
- bos=True,
140
- eos=True,
141
- cropping=crop_mode,
142
- )
143
-
144
- prepare = BatchFeature(
145
- data=dict(
146
- input_ids=input_ids,
147
- pixel_values=pixel_values,
148
- images_crop=images_crop,
149
- images_seq_mask=images_seq_mask,
150
- images_spatial_crop=images_spatial_crop,
151
- num_image_tokens=num_image_tokens,
152
- ),
153
- tensor_type="pt",
154
- )
155
- return prepare
156
-
157
- def __call__(
158
- self,
159
- *,
160
- prompt: str,
161
- images: list[Image.Image],
162
- crop_mode: bool = CROP_MODE,
163
- **kwargs,
164
- ):
165
- prepare = self.process_one(
166
- prompt=prompt,
167
- images=images,
168
- crop_mode=crop_mode,
169
- )
170
-
171
- return prepare
172
-
173
- def tokenize_with_images(
174
- self,
175
- conversation: str,
176
- images: list[Image.Image],
177
- bos: bool = True,
178
- eos: bool = True,
179
- cropping: bool = True,
180
- ):
181
- """Tokenize text with <image> tags."""
182
-
183
- assert conversation.count(self.image_token) == len(images)
184
- text_splits = conversation.split(self.image_token)
185
- images_list, images_crop_list, images_seq_mask, images_spatial_crop = (
186
- [],
187
- [],
188
- [],
189
- [],
190
- )
191
- image_shapes = []
192
- num_image_tokens = []
193
- tokenized_str = []
194
- for text_sep, image in zip(text_splits, images):
195
- tokenized_sep = self.encode(text_sep, bos=False, eos=False)
196
- tokenized_str += tokenized_sep
197
- images_seq_mask += [False] * len(tokenized_sep)
198
-
199
- image_shapes.append(image.size)
200
-
201
- images_crop_raw = []
202
- if image.size[0] <= 768 and image.size[1] <= 768:
203
- crop_ratio = [1, 1]
204
- elif cropping:
205
- images_crop_raw, crop_ratio = dynamic_preprocess(
206
- image, image_size=IMAGE_SIZE
207
- )
208
- else:
209
- crop_ratio = [1, 1]
210
-
211
- if self.image_size <= 768 and not cropping:
212
- image = image.resize((self.image_size, self.image_size))
213
-
214
- global_view = ImageOps.pad(
215
- image,
216
- (self.base_size, self.base_size),
217
- color=tuple(int(x * 255) for x in self.image_transform.mean),
218
- )
219
- images_list.append(self.image_transform(global_view))
220
-
221
- num_width_tiles, num_height_tiles = crop_ratio
222
- images_spatial_crop.append([num_width_tiles, num_height_tiles])
223
-
224
- if num_width_tiles > 1 or num_height_tiles > 1:
225
- for cropped_image in images_crop_raw:
226
- images_crop_list.append(self.image_transform(cropped_image))
227
-
228
- num_queries = math.ceil(
229
- (self.image_size // self.patch_size) / self.downsample_ratio
230
- )
231
- num_queries_base = math.ceil(
232
- (self.base_size // self.patch_size) / self.downsample_ratio
233
- )
234
-
235
- tokenized_image = (
236
- [self.image_token_id] * num_queries_base
237
- ) * num_queries_base
238
- tokenized_image += [self.image_token_id]
239
- if num_width_tiles > 1 or num_height_tiles > 1:
240
- local_row = [self.image_token_id] * (num_queries * num_width_tiles)
241
- tokenized_image += local_row * (num_queries * num_height_tiles)
242
- tokenized_str += tokenized_image
243
- images_seq_mask += [True] * len(tokenized_image)
244
- num_image_tokens.append(len(tokenized_image))
245
-
246
- """process the last text split"""
247
- tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False)
248
- tokenized_str += tokenized_sep
249
- images_seq_mask += [False] * len(tokenized_sep)
250
-
251
- """add the bos and eos tokens"""
252
- if bos:
253
- tokenized_str = [self.bos_id] + tokenized_str
254
- images_seq_mask = [False] + images_seq_mask
255
- if eos:
256
- tokenized_str = tokenized_str + [self.eos_id]
257
- images_seq_mask = images_seq_mask + [False]
258
-
259
- assert len(tokenized_str) == len(images_seq_mask), (
260
- f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} "
261
- f"is not equal to images_seq_mask's length {len(images_seq_mask)}."
262
- )
263
-
264
- masked_tokenized_str = []
265
- for token_index in tokenized_str:
266
- if token_index != self.image_token_id:
267
- masked_tokenized_str.append(token_index)
268
- else:
269
- masked_tokenized_str.append(self.ignore_id)
270
-
271
- assert (
272
- len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str)
273
- ), (
274
- f"tokenized_str's length {len(tokenized_str)}, "
275
- f"input_ids' length {len(masked_tokenized_str)}, "
276
- f"images_seq_mask's length {len(images_seq_mask)}, are not equal."
277
- )
278
-
279
- input_ids = torch.LongTensor(tokenized_str)
280
- target_ids = torch.LongTensor(masked_tokenized_str)
281
- images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)
282
-
283
- # set input_ids < 0 | input_ids == self.image_token_id as ignore_id
284
- target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = (
285
- self.ignore_id
286
- )
287
- input_ids[input_ids < 0] = self.pad_id
288
-
289
- # Remove the ending eos token
290
- assert input_ids[-1] == self.eos_id
291
- input_ids = input_ids[:-1]
292
- target_ids = target_ids[:-1]
293
- images_seq_mask = images_seq_mask[:-1]
294
-
295
- if len(images_list) == 0:
296
- pixel_values = torch.zeros((0, 3, self.base_size, self.base_size))
297
- images_spatial_crop = torch.zeros((0, 2), dtype=torch.long)
298
- images_crop = torch.zeros((0, 3, self.image_size, self.image_size))
299
- else:
300
- pixel_values = torch.stack(images_list, dim=0)
301
- images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)
302
- if images_crop_list:
303
- images_crop = torch.stack(images_crop_list, dim=0)
304
- else:
305
- images_crop = torch.zeros((0, 3, self.image_size, self.image_size))
306
-
307
- input_ids = input_ids.unsqueeze(0)
308
-
309
- return (
310
- input_ids,
311
- pixel_values,
312
- images_crop,
313
- images_seq_mask,
314
- images_spatial_crop,
315
- num_image_tokens,
316
- image_shapes,
317
- )
318
-
319
-
320
-AutoProcessor.register("DeepseekOCR2Processor", DeepseekOCR2Processor)