A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 04:00 UTC
[1/N] Reorganize multimodal processing code (#32327)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 14, 2026 at 23:25 UTC
9ea07b41da169f727a2eb7302adec4c724319522
76 files changed
+717
-670
docs/api/README.md
-4
@@ -82,10 +82,6 @@ Internal data structures.
82
83
- [vllm.multimodal.processing][]
84
85
-### Memory Profiling
86
-
87
-- [vllm.multimodal.profiling][]
88
-
85
### Registry
86
87
- [vllm.multimodal.registry][]
docs/contributing/model/multimodal.md
+4
-6
@@ -116,12 +116,10 @@ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
116
117
## 3. Specify dummy inputs
118
119
-Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.profiling.BaseDummyInputsBuilder] to construct dummy inputs for
120
-HF processing as well as memory profiling.
119
+Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] to construct dummy inputs for
120
+HF processing. The processed outputs are also used for memory profiling.
121
122
-### For memory profiling
123
-
124
-Override the abstract methods [get_dummy_text][vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_text] and [get_dummy_mm_data][vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_mm_data] to construct dummy inputs for memory profiling. These dummy inputs should result in the worst-case memory usage of the model so that vLLM can reserve the correct amount of memory for it.
122
+Override the abstract methods [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] and [get_dummy_mm_data][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_mm_data] to construct dummy inputs. These dummy inputs should result in the worst-case memory usage of the model so that vLLM can reserve the correct amount of memory for it.
123
124
Assuming that the memory usage increases with the number of tokens, the dummy inputs can be constructed to maximize the number of output embeddings, which is the same number as placeholder feature tokens.
125
@@ -803,7 +801,7 @@ Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies
801
## 5. Register processor-related classes
802
803
After you have defined [BaseProcessingInfo][vllm.multimodal.processing.BaseProcessingInfo] (Step 2),
806
-[BaseDummyInputsBuilder][vllm.multimodal.profiling.BaseDummyInputsBuilder] (Step 3),
804
+[BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] (Step 3),
805
and [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor] (Step 4),
806
decorate the model class with [MULTIMODAL_REGISTRY.register_processor][vllm.multimodal.registry.MultiModalRegistry.register_processor]
807
to register them to the multi-modal registry:
docs/design/mm_processing.md
+1
-1
@@ -43,7 +43,7 @@ Moreover, since the tokenized text has not passed through the HF processor, we h
43
44
### Dummy text
45
46
-We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_text]. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data.
46
+We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data.
47
48
### Automatic prompt updating
49
tests/multimodal/test_processing.py
+2
-2
@@ -9,8 +9,8 @@ import pytest
9
10
from vllm.config import ModelConfig
11
from vllm.multimodal import MULTIMODAL_REGISTRY
12
-from vllm.multimodal.processing import (
13
- InputProcessingContext,
12
+from vllm.multimodal.processing.context import InputProcessingContext
13
+from vllm.multimodal.processing.processor import (
14
PlaceholderFeaturesInfo,
15
PromptIndexTargets,
16
PromptInsertion,
vllm/benchmarks/mm_processor.py
+1
-1
@@ -24,7 +24,7 @@ import numpy as np
24
25
from vllm.benchmarks.throughput import get_requests
26
from vllm.engine.arg_utils import EngineArgs
27
-from vllm.multimodal.processing import (
27
+from vllm.multimodal.processing.context import (
28
get_timing_stats_from_engine_client,
29
)
30
from vllm.utils.gc_utils import freeze_gc_heap
vllm/model_executor/models/aria.py
+1
-1
@@ -30,12 +30,12 @@ from vllm.multimodal.inputs import (
30
)
31
from vllm.multimodal.parse import MultiModalDataItems
32
from vllm.multimodal.processing import (
33
+ BaseDummyInputsBuilder,
34
BaseMultiModalProcessor,
35
BaseProcessingInfo,
36
PromptReplacement,
37
PromptUpdate,
38
)
38
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
39
from vllm.sequence import IntermediateTensors
40
from vllm.utils.tensor_schema import TensorSchema, TensorShape
41
vllm/model_executor/models/audioflamingo3.py
+1
-1
@@ -47,13 +47,13 @@ from vllm.multimodal.parse import (
47
MultiModalDataParser,
48
)
49
from vllm.multimodal.processing import (
50
+ BaseDummyInputsBuilder,
51
BaseMultiModalProcessor,
52
BaseProcessingInfo,
53
PromptReplacement,
54
PromptUpdate,
55
PromptUpdateDetails,
56
)
56
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
57
from vllm.sequence import IntermediateTensors
58
from vllm.utils.tensor_schema import TensorSchema, TensorShape
59
vllm/model_executor/models/aya_vision.py
+6
-3
@@ -18,17 +18,20 @@ from transformers.models.got_ocr2.image_processing_got_ocr2 import (
18
from vllm.config import VllmConfig
19
from vllm.config.multimodal import BaseDummyOptions
20
from vllm.multimodal import MULTIMODAL_REGISTRY
21
-from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargsItems
21
+from vllm.multimodal.inputs import (
22
+ MultiModalDataDict,
23
+ MultiModalFieldConfig,
24
+ MultiModalKwargsItems,
25
+)
26
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
27
from vllm.multimodal.processing import (
28
+ BaseDummyInputsBuilder,
29
BaseMultiModalProcessor,
30
BaseProcessingInfo,
26
- MultiModalFieldConfig,
31
PromptReplacement,
32
PromptUpdate,
33
PromptUpdateDetails,
34
)
31
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
from vllm.utils.tensor_schema import TensorSchema, TensorShape
37
vllm/model_executor/models/bagel.py
+1
-1
@@ -30,11 +30,11 @@ from vllm.multimodal.inputs import (
30
)
31
from vllm.multimodal.parse import MultiModalDataItems
32
from vllm.multimodal.processing import (
33
+ BaseDummyInputsBuilder,
34
BaseMultiModalProcessor,
35
BaseProcessingInfo,
36
PromptReplacement,
37
)
37
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
38
from vllm.sequence import IntermediateTensors
39
from vllm.transformers_utils.processors.bagel import BagelProcessor
40
from vllm.utils.tensor_schema import TensorSchema
vllm/model_executor/models/blip2.py
+1
-1
@@ -25,13 +25,13 @@ from vllm.multimodal.inputs import (
25
)
26
from vllm.multimodal.parse import MultiModalDataItems
27
from vllm.multimodal.processing import (
28
+ BaseDummyInputsBuilder,
29
BaseMultiModalProcessor,
30
BaseProcessingInfo,
31
PromptIndexTargets,
32
PromptInsertion,
33
PromptUpdate,
34
)
34
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
from vllm.utils.tensor_schema import TensorSchema, TensorShape
37
vllm/model_executor/models/chameleon.py
+1
-1
@@ -49,13 +49,13 @@ from vllm.multimodal.inputs import (
49
)
50
from vllm.multimodal.parse import MultiModalDataItems
51
from vllm.multimodal.processing import (
52
+ BaseDummyInputsBuilder,
53
BaseMultiModalProcessor,
54
BaseProcessingInfo,
55
PromptReplacement,
56
PromptUpdate,
57
PromptUpdateDetails,
58
)
58
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
59
from vllm.sequence import IntermediateTensors
60
from vllm.utils.tensor_schema import TensorSchema, TensorShape
61
vllm/model_executor/models/clip.py
+1
-1
@@ -41,13 +41,13 @@ from vllm.multimodal.inputs import (
41
)
42
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
43
from vllm.multimodal.processing import (
44
+ BaseDummyInputsBuilder,
45
BaseMultiModalProcessor,
46
BaseProcessingInfo,
47
PromptIndexTargets,
48
PromptReplacement,
49
PromptUpdate,
50
)
50
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
51
from vllm.sequence import IntermediateTensors
52
from vllm.utils.tensor_schema import TensorSchema, TensorShape
53
vllm/model_executor/models/cohere2_vision.py
+6
-3
@@ -27,17 +27,20 @@ from vllm.model_executor.layers.linear import (
27
from vllm.model_executor.layers.quantization import QuantizationConfig
28
from vllm.model_executor.layers.quantization.awq import AWQConfig
29
from vllm.multimodal import MULTIMODAL_REGISTRY
30
-from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargsItems
30
+from vllm.multimodal.inputs import (
31
+ MultiModalDataDict,
32
+ MultiModalFieldConfig,
33
+ MultiModalKwargsItems,
34
+)
35
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
36
from vllm.multimodal.processing import (
37
+ BaseDummyInputsBuilder,
38
BaseMultiModalProcessor,
39
BaseProcessingInfo,
35
- MultiModalFieldConfig,
40
PromptReplacement,
41
PromptUpdate,
42
PromptUpdateDetails,
43
)
40
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
44
from vllm.sequence import IntermediateTensors
45
from vllm.utils.tensor_schema import TensorSchema, TensorShape
46
vllm/model_executor/models/deepseek_ocr.py
+1
-1
@@ -39,12 +39,12 @@ from vllm.multimodal.parse import (
39
MultiModalDataItems,
40
)
41
from vllm.multimodal.processing import (
42
+ BaseDummyInputsBuilder,
43
BaseMultiModalProcessor,
44
BaseProcessingInfo,
45
PromptReplacement,
46
PromptUpdate,
47
)
47
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
48
from vllm.sampling_params import SamplingParams
49
from vllm.sequence import IntermediateTensors
50
from vllm.tokenizers import cached_tokenizer_from_config
vllm/model_executor/models/deepseek_vl2.py
+2
-2
@@ -32,14 +32,14 @@ from vllm.multimodal.parse import (
32
ImageSize,
33
MultiModalDataItems,
34
)
35
-from vllm.multimodal.processing import (
35
+from vllm.multimodal.processing import BaseDummyInputsBuilder
36
+from vllm.multimodal.processing.processor import (
37
BaseMultiModalProcessor,
38
BaseProcessingInfo,
39
MultiModalProcessingInfo,
40
PromptReplacement,
41
PromptUpdate,
42
)
42
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
43
from vllm.sequence import IntermediateTensors
44
from vllm.tokenizers import cached_tokenizer_from_config
45
from vllm.transformers_utils.configs.deepseek_vl2 import (
vllm/model_executor/models/ernie45_vl.py
+1
-1
@@ -65,12 +65,12 @@ from vllm.multimodal.inputs import (
65
)
66
from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser
67
from vllm.multimodal.processing import (
68
+ BaseDummyInputsBuilder,
69
BaseMultiModalProcessor,
70
BaseProcessingInfo,
71
PromptReplacement,
72
PromptUpdate,
73
)
73
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
74
from vllm.sequence import IntermediateTensors
75
from vllm.utils.tensor_schema import TensorSchema, TensorShape
76
from vllm.v1.attention.backends.registry import AttentionBackendEnum
vllm/model_executor/models/fuyu.py
+1
-1
@@ -38,13 +38,13 @@ from vllm.multimodal.inputs import (
38
)
39
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
40
from vllm.multimodal.processing import (
41
+ BaseDummyInputsBuilder,
42
BaseMultiModalProcessor,
43
BaseProcessingInfo,
44
PromptReplacement,
45
PromptUpdate,
46
PromptUpdateDetails,
47
)
47
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
48
from vllm.sequence import IntermediateTensors
49
from vllm.utils.tensor_schema import TensorSchema, TensorShape
50
vllm/model_executor/models/gemma3_mm.py
+2
-2
@@ -21,7 +21,8 @@ from vllm.multimodal.inputs import (
21
MultiModalKwargsItems,
22
)
23
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
24
-from vllm.multimodal.processing import (
24
+from vllm.multimodal.processing import BaseDummyInputsBuilder
25
+from vllm.multimodal.processing.processor import (
26
BaseMultiModalProcessor,
27
BaseProcessingInfo,
28
MultiModalPromptUpdates,
@@ -32,7 +33,6 @@ from vllm.multimodal.processing import (
33
PromptUpdateDetails,
34
replace_token_matches,
35
)
35
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
36
from vllm.sequence import IntermediateTensors
37
from vllm.utils.tensor_schema import TensorSchema, TensorShape
38
vllm/model_executor/models/gemma3n_mm.py
+2
-2
@@ -41,7 +41,8 @@ from vllm.multimodal.parse import (
41
MultiModalDataItems,
42
MultiModalDataParser,
43
)
44
-from vllm.multimodal.processing import (
44
+from vllm.multimodal.processing import BaseDummyInputsBuilder
45
+from vllm.multimodal.processing.processor import (
46
BaseMultiModalProcessor,
47
BaseProcessingInfo,
48
MultiModalPromptUpdates,
@@ -52,7 +53,6 @@ from vllm.multimodal.processing import (
53
PromptUpdateDetails,
54
replace_token_matches,
55
)
55
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
56
from vllm.sequence import IntermediateTensors
57
from vllm.utils.tensor_schema import TensorSchema, TensorShape
58
vllm/model_executor/models/glm4_1v.py
+1
-1
@@ -79,13 +79,13 @@ from vllm.multimodal.inputs import (
79
)
80
from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser
81
from vllm.multimodal.processing import (
82
+ BaseDummyInputsBuilder,
83
BaseMultiModalProcessor,
84
BaseProcessingInfo,
85
PromptReplacement,
86
PromptUpdate,
87
PromptUpdateDetails,
88
)
88
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
89
from vllm.sequence import IntermediateTensors
90
from vllm.utils.tensor_schema import TensorSchema, TensorShape
91
from vllm.v1.attention.backends.registry import AttentionBackendEnum
vllm/model_executor/models/glm4v.py
+1
-1
@@ -43,12 +43,12 @@ from vllm.multimodal.inputs import (
43
)
44
from vllm.multimodal.parse import MultiModalDataItems
45
from vllm.multimodal.processing import (
46
+ BaseDummyInputsBuilder,
47
BaseMultiModalProcessor,
48
BaseProcessingInfo,
49
PromptReplacement,
50
PromptUpdate,
51
)
51
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
52
from vllm.sequence import IntermediateTensors
53
from vllm.transformers_utils.configs import ChatGLMConfig
54
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/glmasr.py
+1
-1
@@ -39,13 +39,13 @@ from vllm.multimodal.parse import (
39
MultiModalDataParser,
40
)
41
from vllm.multimodal.processing import (
42
+ BaseDummyInputsBuilder,
43
BaseMultiModalProcessor,
44
BaseProcessingInfo,
45
PromptReplacement,
46
PromptUpdate,
47
PromptUpdateDetails,
48
)
48
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
49
from vllm.sequence import IntermediateTensors
50
from vllm.tokenizers import cached_tokenizer_from_config
51
from vllm.transformers_utils.processor import cached_processor_from_config
vllm/model_executor/models/granite_speech.py
+1
-1
@@ -52,12 +52,12 @@ from vllm.multimodal.parse import (
52
MultiModalDataParser,
53
)
54
from vllm.multimodal.processing import (
55
+ BaseDummyInputsBuilder,
56
BaseMultiModalProcessor,
57
BaseProcessingInfo,
58
PromptReplacement,
59
PromptUpdate,
60
)
60
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
61
from vllm.sequence import IntermediateTensors
62
from vllm.tokenizers import cached_tokenizer_from_config
63
from vllm.transformers_utils.processor import cached_processor_from_config
vllm/model_executor/models/h2ovl.py
+1
-1
@@ -22,7 +22,7 @@ from vllm.multimodal.parse import (
22
ImageProcessorItems,
23
MultiModalDataItems,
24
)
25
-from vllm.multimodal.processing import (
25
+from vllm.multimodal.processing.processor import (
26
MultiModalProcessingInfo,
27
PromptReplacement,
28
PromptUpdate,
vllm/model_executor/models/hunyuan_vision.py
+1
-1
@@ -66,12 +66,12 @@ from vllm.multimodal.parse import (
66
MultiModalDataParser,
67
)
68
from vllm.multimodal.processing import (
69
+ BaseDummyInputsBuilder,
70
BaseMultiModalProcessor,
71
BaseProcessingInfo,
72
PromptReplacement,
73
PromptUpdate,
74
)
74
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
75
from vllm.sequence import IntermediateTensors
76
from vllm.transformers_utils.configs.hunyuan_vl import (
77
HunYuanVLConfig,
vllm/model_executor/models/hyperclovax_vision.py
+1
-1
@@ -29,13 +29,13 @@ from vllm.multimodal.inputs import (
29
)
30
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
31
from vllm.multimodal.processing import (
32
+ BaseDummyInputsBuilder,
33
BaseMultiModalProcessor,
34
BaseProcessingInfo,
35
InputProcessingContext,
36
PromptReplacement,
37
PromptUpdate,
38
)
38
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
39
from vllm.sequence import IntermediateTensors
40
from vllm.utils.tensor_schema import TensorSchema, TensorShape
41
vllm/model_executor/models/idefics3.py
+2
-3
@@ -42,16 +42,15 @@ from vllm.multimodal.inputs import (
42
MultiModalFieldConfig,
43
MultiModalKwargsItems,
44
)
45
-from vllm.multimodal.parse import ImageProcessorItems, ImageSize
45
+from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
46
from vllm.multimodal.processing import (
47
+ BaseDummyInputsBuilder,
48
BaseMultiModalProcessor,
49
BaseProcessingInfo,
49
- MultiModalDataItems,
50
PromptReplacement,
51
PromptUpdate,
52
PromptUpdateDetails,
53
)
54
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
54
from vllm.sequence import IntermediateTensors
55
from vllm.utils.tensor_schema import TensorSchema, TensorShape
56
vllm/model_executor/models/interns1.py
+1
-1
@@ -39,13 +39,13 @@ from vllm.multimodal.parse import (
39
MultiModalDataItems,
40
)
41
from vllm.multimodal.processing import (
42
+ BaseDummyInputsBuilder,
43
BaseMultiModalProcessor,
44
BaseProcessingInfo,
45
PromptReplacement,
46
PromptUpdate,
47
PromptUpdateDetails,
48
)
48
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
49
from vllm.sequence import IntermediateTensors
50
from vllm.transformers_utils.processor import cached_video_processor_from_config
51
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/internvl.py
+1
-1
@@ -41,13 +41,13 @@ from vllm.multimodal.parse import (
41
MultiModalDataItems,
42
)
43
from vllm.multimodal.processing import (
44
+ BaseDummyInputsBuilder,
45
BaseMultiModalProcessor,
46
BaseProcessingInfo,
47
PromptReplacement,
48
PromptUpdate,
49
PromptUpdateDetails,
50
)
50
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
51
from vllm.sequence import IntermediateTensors
52
from vllm.tokenizers import TokenizerLike
53
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/isaac.py
+1
-1
@@ -55,13 +55,13 @@ from vllm.multimodal.inputs import (
55
)
56
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
57
from vllm.multimodal.processing import (
58
+ BaseDummyInputsBuilder,
59
BaseMultiModalProcessor,
60
BaseProcessingInfo,
61
PromptReplacement,
62
PromptUpdate,
63
PromptUpdateDetails,
64
)
64
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
65
from vllm.sequence import IntermediateTensors
66
from vllm.tokenizers import get_tokenizer
67
from vllm.tokenizers.hf import get_cached_tokenizer
vllm/model_executor/models/kanana_v.py
+1
-1
@@ -28,12 +28,12 @@ from vllm.multimodal.inputs import (
28
)
29
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
30
from vllm.multimodal.processing import (
31
+ BaseDummyInputsBuilder,
32
BaseMultiModalProcessor,
33
BaseProcessingInfo,
34
PromptReplacement,
35
PromptUpdate,
36
)
36
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
37
from vllm.sequence import IntermediateTensors
38
from vllm.utils.import_utils import resolve_obj_by_qualname
39
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/keye.py
+1
-1
@@ -56,12 +56,12 @@ from vllm.multimodal.parse import (
56
MultiModalDataParser,
57
)
58
from vllm.multimodal.processing import (
59
+ BaseDummyInputsBuilder,
60
BaseMultiModalProcessor,
61
BaseProcessingInfo,
62
PromptReplacement,
63
PromptUpdate,
64
)
64
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
65
from vllm.sequence import IntermediateTensors
66
from vllm.utils.tensor_schema import TensorSchema, TensorShape
67
vllm/model_executor/models/kimi_vl.py
+1
-1
@@ -82,12 +82,12 @@ from vllm.multimodal.parse import (
82
MultiModalDataItems,
83
)
84
from vllm.multimodal.processing import (
85
+ BaseDummyInputsBuilder,
86
BaseMultiModalProcessor,
87
BaseProcessingInfo,
88
PromptReplacement,
89
PromptUpdate,
90
)
90
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
91
from vllm.sequence import IntermediateTensors
92
from vllm.transformers_utils.configs import KimiVLConfig, MoonViTConfig
93
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/lfm2_vl.py
+1
-1
@@ -34,12 +34,12 @@ from vllm.multimodal.inputs import (
34
)
35
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
36
from vllm.multimodal.processing import (
37
+ BaseDummyInputsBuilder,
38
BaseMultiModalProcessor,
39
BaseProcessingInfo,
40
PromptReplacement,
41
PromptUpdateDetails,
42
)
42
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
43
from vllm.sequence import IntermediateTensors
44
from vllm.utils.tensor_schema import TensorSchema, TensorShape
45
vllm/model_executor/models/lightonocr.py
+1
-1
@@ -31,12 +31,12 @@ from vllm.multimodal.cache import BaseMultiModalProcessorCache
31
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
32
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
33
from vllm.multimodal.processing import (
34
+ BaseDummyInputsBuilder,
35
BaseMultiModalProcessor,
36
PromptReplacement,
37
PromptUpdate,
38
PromptUpdateDetails,
39
)
39
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
40
41
_I = TypeVar("_I", bound=Mistral3ProcessingInfo)
42
vllm/model_executor/models/llava.py
+1
-1
@@ -39,6 +39,7 @@ from vllm.multimodal.parse import (
39
MultiModalDataItems,
40
)
41
from vllm.multimodal.processing import (
42
+ BaseDummyInputsBuilder,
43
BaseMultiModalProcessor,
44
BaseProcessingInfo,
45
InputProcessingContext,
@@ -46,7 +47,6 @@ from vllm.multimodal.processing import (
47
PromptUpdate,
48
PromptUpdateDetails,
49
)
49
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
50
from vllm.sequence import IntermediateTensors
51
from vllm.utils.tensor_schema import TensorSchema, TensorShape
52
vllm/model_executor/models/llava_next_video.py
+1
-1
@@ -26,12 +26,12 @@ from vllm.multimodal.parse import (
26
VideoProcessorItems,
27
)
28
from vllm.multimodal.processing import (
29
+ BaseDummyInputsBuilder,
30
BaseMultiModalProcessor,
31
BaseProcessingInfo,
32
PromptReplacement,
33
PromptUpdate,
34
)
34
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
from vllm.utils.collection_utils import is_list_of
37
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/midashenglm.py
+1
-1
@@ -54,13 +54,13 @@ from vllm.multimodal.inputs import (
54
)
55
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
56
from vllm.multimodal.processing import (
57
+ BaseDummyInputsBuilder,
58
BaseMultiModalProcessor,
59
BaseProcessingInfo,
60
PromptReplacement,
61
PromptUpdate,
62
PromptUpdateDetails,
63
)
63
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
64
from vllm.sequence import IntermediateTensors
65
from vllm.transformers_utils.configs.midashenglm import DashengConfig
66
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/minicpmv.py
+2
-2
@@ -71,7 +71,8 @@ from vllm.multimodal.parse import (
71
VideoItem,
72
VideoProcessorItems,
73
)
74
-from vllm.multimodal.processing import (
74
+from vllm.multimodal.processing import BaseDummyInputsBuilder
75
+from vllm.multimodal.processing.processor import (
76
BaseMultiModalProcessor,
77
BaseProcessingInfo,
78
PromptReplacement,
@@ -80,7 +81,6 @@ from vllm.multimodal.processing import (
81
ResolvedPromptUpdate,
82
_seq2text,
83
)
83
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
84
from vllm.platforms import current_platform
85
from vllm.sequence import IntermediateTensors
86
from vllm.utils.collection_utils import flatten_2d_lists
vllm/model_executor/models/mistral3.py
+1
-1
@@ -31,6 +31,7 @@ from vllm.multimodal.inputs import (
31
)
32
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
33
from vllm.multimodal.processing import (
34
+ BaseDummyInputsBuilder,
35
BaseMultiModalProcessor,
36
BaseProcessingInfo,
37
InputProcessingContext,
@@ -38,7 +39,6 @@ from vllm.multimodal.processing import (
39
PromptUpdate,
40
PromptUpdateDetails,
41
)
41
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
42
from vllm.sequence import IntermediateTensors
43
from vllm.utils.tensor_schema import TensorSchema, TensorShape
44
vllm/model_executor/models/mllama4.py
+1
-1
@@ -58,6 +58,7 @@ from vllm.multimodal.inputs import (
58
)
59
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
60
from vllm.multimodal.processing import (
61
+ BaseDummyInputsBuilder,
62
BaseMultiModalProcessor,
63
BaseProcessingInfo,
64
InputProcessingContext,
@@ -65,7 +66,6 @@ from vllm.multimodal.processing import (
66
PromptUpdate,
67
PromptUpdateDetails,
68
)
68
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
69
from vllm.sequence import IntermediateTensors
70
from vllm.utils.tensor_schema import TensorSchema, TensorShape
71
vllm/model_executor/models/molmo.py
+1
-1
@@ -54,6 +54,7 @@ from vllm.multimodal.inputs import (
54
)
55
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
56
from vllm.multimodal.processing import (
57
+ BaseDummyInputsBuilder,
58
BaseMultiModalProcessor,
59
BaseProcessingInfo,
60
PromptIndexTargets,
@@ -61,7 +62,6 @@ from vllm.multimodal.processing import (
62
PromptUpdate,
63
PromptUpdateDetails,
64
)
64
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
65
from vllm.sequence import IntermediateTensors
66
from vllm.utils.tensor_schema import TensorSchema, TensorShape
67
vllm/model_executor/models/molmo2.py
+1
-1
@@ -67,13 +67,13 @@ from vllm.multimodal.parse import (
67
MultiModalDataParser,
68
)
69
from vllm.multimodal.processing import (
70
+ BaseDummyInputsBuilder,
71
BaseMultiModalProcessor,
72
BaseProcessingInfo,
73
PromptReplacement,
74
PromptUpdate,
75
PromptUpdateDetails,
76
)
76
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
77
from vllm.sequence import IntermediateTensors
78
from vllm.utils.math_utils import round_down
79
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/nano_nemotron_vl.py
+2
-2
@@ -62,7 +62,8 @@ from vllm.multimodal.parse import (
62
MultiModalDataItems,
63
MultiModalDataParser,
64
)
65
-from vllm.multimodal.processing import (
65
+from vllm.multimodal.processing import BaseDummyInputsBuilder
66
+from vllm.multimodal.processing.processor import (
67
BaseMultiModalProcessor,
68
BaseProcessingInfo,
69
PromptReplacement,
@@ -70,7 +71,6 @@ from vllm.multimodal.processing import (
71
PromptUpdateDetails,
72
_seq2tokens,
73
)
73
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
74
from vllm.sequence import IntermediateTensors
75
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
76
from vllm.transformers_utils.configs.radio import RadioConfig
vllm/model_executor/models/nemotron_parse.py
+3
-3
@@ -53,14 +53,14 @@ from vllm.multimodal.inputs import (
53
)
54
from vllm.multimodal.parse import MultiModalDataItems
55
from vllm.multimodal.processing import (
56
+ BaseDummyInputsBuilder,
57
BaseProcessingInfo,
58
EncDecMultiModalProcessor,
59
PromptReplacement,
60
PromptUpdate,
61
)
61
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
62
from vllm.transformers_utils.configs.radio import RadioConfig
63
-from vllm.transformers_utils.tokenizer import AnyTokenizer
63
+from vllm.transformers_utils.tokenizer import TokenizerLike
64
from vllm.utils.tensor_schema import TensorSchema, TensorShape
65
from vllm.v1.attention.backend import AttentionType
66
@@ -558,7 +558,7 @@ class NemotronParseProcessor:
558
def __init__(
559
self,
560
config: PretrainedConfig,
561
- tokenizer: AnyTokenizer,
561
+ tokenizer: TokenizerLike,
562
**kwargs,
563
) -> None:
564
super().__init__()
vllm/model_executor/models/ovis.py
+1
-1
@@ -48,11 +48,11 @@ from vllm.multimodal.inputs import (
48
)
49
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
50
from vllm.multimodal.processing import (
51
+ BaseDummyInputsBuilder,
52
BaseMultiModalProcessor,
53
BaseProcessingInfo,
54
PromptReplacement,
55
)
55
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
56
from vllm.sequence import IntermediateTensors
57
from vllm.transformers_utils.processors.ovis import OvisProcessor
58
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/ovis2_5.py
+1
-1
@@ -30,11 +30,11 @@ from vllm.multimodal.inputs import (
30
)
31
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
32
from vllm.multimodal.processing import (
33
+ BaseDummyInputsBuilder,
34
BaseMultiModalProcessor,
35
BaseProcessingInfo,
36
PromptReplacement,
37
)
37
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
38
from vllm.sequence import IntermediateTensors
39
from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor
40
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/paddleocr_vl.py
+1
-1
@@ -63,12 +63,12 @@ from vllm.multimodal.parse import (
63
MultiModalDataItems,
64
)
65
from vllm.multimodal.processing import (
66
+ BaseDummyInputsBuilder,
67
BaseMultiModalProcessor,
68
BaseProcessingInfo,
69
PromptReplacement,
70
PromptUpdate,
71
)
71
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
72
from vllm.sequence import IntermediateTensors
73
from vllm.utils.tensor_schema import TensorSchema, TensorShape
74
from vllm.v1.attention.backends.registry import AttentionBackendEnum
vllm/model_executor/models/paligemma.py
+1
-1
@@ -24,6 +24,7 @@ from vllm.multimodal.parse import (
24
MultiModalDataItems,
25
)
26
from vllm.multimodal.processing import (
27
+ BaseDummyInputsBuilder,
28
BaseMultiModalProcessor,
29
BaseProcessingInfo,
30
PromptIndexTargets,
@@ -31,7 +32,6 @@ from vllm.multimodal.processing import (
32
PromptUpdate,
33
PromptUpdateDetails,
34
)
34
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
from vllm.utils.tensor_schema import TensorSchema, TensorShape
37
vllm/model_executor/models/phi3v.py
+2
-2
@@ -45,7 +45,8 @@ from vllm.multimodal.parse import (
45
ImageSize,
46
MultiModalDataItems,
47
)
48
-from vllm.multimodal.processing import (
48
+from vllm.multimodal.processing import BaseDummyInputsBuilder
49
+from vllm.multimodal.processing.processor import (
50
BaseMultiModalProcessor,
51
BaseProcessingInfo,
52
MultiModalPromptUpdates,
@@ -54,7 +55,6 @@ from vllm.multimodal.processing import (
55
PromptUpdate,
56
ResolvedPromptUpdate,
57
)
57
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
58
from vllm.sequence import IntermediateTensors
59
from vllm.utils.tensor_schema import TensorSchema, TensorShape
60
vllm/model_executor/models/phi4mm.py
+2
-2
@@ -40,14 +40,14 @@ from vllm.multimodal.parse import (
40
MultiModalDataItems,
41
MultiModalDataParser,
42
)
43
-from vllm.multimodal.processing import (
43
+from vllm.multimodal.processing import BaseDummyInputsBuilder
44
+from vllm.multimodal.processing.processor import (
45
BaseMultiModalProcessor,
46
BaseProcessingInfo,
47
PromptReplacement,
48
PromptUpdate,
49
ResolvedPromptUpdate,
50
)
50
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
51
from vllm.sequence import IntermediateTensors
52
from vllm.utils.tensor_schema import TensorSchema, TensorShape
53
vllm/model_executor/models/pixtral.py
+2
-2
@@ -48,7 +48,8 @@ from vllm.multimodal.inputs import (
48
NestedTensors,
49
)
50
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
51
-from vllm.multimodal.processing import (
51
+from vllm.multimodal.processing import BaseDummyInputsBuilder, ProcessorInputs
52
+from vllm.multimodal.processing.processor import (
53
BaseMultiModalProcessor,
54
BaseProcessingInfo,
55
MultiModalProcessingInfo,
@@ -56,7 +57,6 @@ from vllm.multimodal.processing import (
57
PromptUpdate,
58
PromptUpdateDetails,
59
)
59
-from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
60
from vllm.platforms import current_platform
61
from vllm.sequence import IntermediateTensors
62
from vllm.tokenizers import cached_tokenizer_from_config
vllm/model_executor/models/qwen2_5_omni_thinker.py
+2
-2
@@ -78,14 +78,14 @@ from vllm.multimodal.parse import (
78
MultiModalDataItems,
79
MultiModalDataParser,
80
)
81
-from vllm.multimodal.processing import (
81
+from vllm.multimodal.processing import BaseDummyInputsBuilder
82
+from vllm.multimodal.processing.processor import (
83
BaseMultiModalProcessor,
84
MultiModalPromptUpdates,
85
PlaceholderFeaturesInfo,
86
PromptReplacement,
87
PromptUpdate,
88
)
88
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
89
from vllm.sequence import IntermediateTensors
90
from vllm.utils.tensor_schema import TensorSchema, TensorShape
91
vllm/model_executor/models/qwen2_audio.py
+1
-1
@@ -54,13 +54,13 @@ from vllm.multimodal.parse import (
54
MultiModalDataParser,
55
)
56
from vllm.multimodal.processing import (
57
+ BaseDummyInputsBuilder,
58
BaseMultiModalProcessor,
59
BaseProcessingInfo,
60
PromptReplacement,
61
PromptUpdate,
62
PromptUpdateDetails,
63
)
63
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
64
from vllm.sequence import IntermediateTensors
65
from vllm.utils.tensor_schema import TensorSchema, TensorShape
66
vllm/model_executor/models/qwen2_vl.py
+1
-1
@@ -80,12 +80,12 @@ from vllm.multimodal.parse import (
80
MultiModalDataParser,
81
)
82
from vllm.multimodal.processing import (
83
+ BaseDummyInputsBuilder,
84
BaseMultiModalProcessor,
85
BaseProcessingInfo,
86
PromptReplacement,
87
PromptUpdate,
88
)
88
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
89
from vllm.sequence import IntermediateTensors
90
from vllm.tokenizers import TokenizerLike
91
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+1
-1
@@ -71,7 +71,7 @@ from vllm.model_executor.models.qwen2_audio import Qwen2AudioProcessingInfo
71
from vllm.multimodal import MULTIMODAL_REGISTRY
72
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItems
73
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
74
-from vllm.multimodal.processing import (
74
+from vllm.multimodal.processing.processor import (
75
MultiModalPromptUpdates,
76
PlaceholderFeaturesInfo,
77
PromptReplacement,
vllm/model_executor/models/qwen3_vl.py
+1
-1
@@ -83,12 +83,12 @@ from vllm.multimodal.inputs import (
83
)
84
from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser
85
from vllm.multimodal.processing import (
86
+ BaseDummyInputsBuilder,
87
BaseMultiModalProcessor,
88
PromptReplacement,
89
PromptUpdate,
90
PromptUpdateDetails,
91
)
91
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
92
from vllm.sequence import IntermediateTensors
93
from vllm.utils.collection_utils import is_list_of
94
from vllm.v1.attention.backends.registry import AttentionBackendEnum
vllm/model_executor/models/qwen_vl.py
+1
-1
@@ -42,13 +42,13 @@ from vllm.multimodal.inputs import (
42
)
43
from vllm.multimodal.parse import MultiModalDataItems
44
from vllm.multimodal.processing import (
45
+ BaseDummyInputsBuilder,
46
BaseMultiModalProcessor,
47
BaseProcessingInfo,
48
PromptReplacement,
49
PromptUpdate,
50
PromptUpdateDetails,
51
)
51
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
52
from vllm.sequence import IntermediateTensors
53
from vllm.utils.tensor_schema import TensorSchema, TensorShape
54
vllm/model_executor/models/siglip.py
+1
-1
@@ -46,13 +46,13 @@ from vllm.multimodal.inputs import (
46
)
47
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
48
from vllm.multimodal.processing import (
49
+ BaseDummyInputsBuilder,
50
BaseMultiModalProcessor,
51
BaseProcessingInfo,
52
PromptIndexTargets,
53
PromptReplacement,
54
PromptUpdate,
55
)
55
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
56
from vllm.sequence import IntermediateTensors
57
from vllm.utils.tensor_schema import TensorSchema, TensorShape
58
vllm/model_executor/models/skyworkr1v.py
+1
-1
@@ -39,13 +39,13 @@ from vllm.multimodal.parse import (
39
MultiModalDataItems,
40
)
41
from vllm.multimodal.processing import (
42
+ BaseDummyInputsBuilder,
43
BaseMultiModalProcessor,
44
BaseProcessingInfo,
45
PromptReplacement,
46
PromptUpdate,
47
PromptUpdateDetails,
48
)
48
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
49
from vllm.sequence import IntermediateTensors
50
from vllm.tokenizers import TokenizerLike
51
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/step3_vl.py
+1
-1
@@ -35,13 +35,13 @@ from vllm.multimodal.inputs import (
35
)
36
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
37
from vllm.multimodal.processing import (
38
+ BaseDummyInputsBuilder,
39
BaseMultiModalProcessor,
40
BaseProcessingInfo,
41
PromptReplacement,
42
PromptUpdate,
43
PromptUpdateDetails,
44
)
44
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
45
from vllm.sequence import IntermediateTensors
46
from vllm.tokenizers import TokenizerLike
47
from vllm.transformers_utils.configs import Step3VisionEncoderConfig
vllm/model_executor/models/tarsier.py
+1
-1
@@ -34,13 +34,13 @@ from vllm.multimodal.parse import (
34
MultiModalDataItems,
35
)
36
from vllm.multimodal.processing import (
37
+ BaseDummyInputsBuilder,
38
BaseMultiModalProcessor,
39
BaseProcessingInfo,
40
InputProcessingContext,
41
PromptReplacement,
42
PromptUpdate,
43
)
43
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
44
from vllm.sequence import IntermediateTensors
45
from vllm.utils.tensor_schema import TensorSchema, TensorShape
46
vllm/model_executor/models/terratorch.py
+1
-1
@@ -56,11 +56,11 @@ from vllm.multimodal.parse import (
56
MultiModalDataParser,
57
)
58
from vllm.multimodal.processing import (
59
+ BaseDummyInputsBuilder,
60
BaseMultiModalProcessor,
61
BaseProcessingInfo,
62
PromptUpdate,
63
)
63
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
64
from vllm.sequence import IntermediateTensors
65
66
from .interfaces import IsAttentionFree, MultiModalEmbeddings, SupportsMultiModal
vllm/model_executor/models/transformers/multimodal.py
+5
-2
@@ -35,8 +35,11 @@ from vllm.multimodal.inputs import (
35
PlaceholderRange,
36
)
37
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
38
-from vllm.multimodal.processing import BaseMultiModalProcessor, BaseProcessingInfo
39
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
38
+from vllm.multimodal.processing import (
39
+ BaseDummyInputsBuilder,
40
+ BaseMultiModalProcessor,
41
+ BaseProcessingInfo,
42
+)
43
from vllm.platforms import current_platform
44
from vllm.sequence import IntermediateTensors
45
vllm/model_executor/models/ultravox.py
+1
-1
@@ -36,12 +36,12 @@ from vllm.multimodal.inputs import (
36
)
37
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
38
from vllm.multimodal.processing import (
39
+ BaseDummyInputsBuilder,
40
BaseMultiModalProcessor,
41
BaseProcessingInfo,
42
PromptReplacement,
43
PromptUpdate,
44
)
44
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
45
from vllm.sequence import IntermediateTensors
46
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
47
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/model_executor/models/voxtral.py
+2
-2
@@ -47,14 +47,14 @@ from vllm.multimodal.parse import (
47
MultiModalDataItems,
48
MultiModalDataParser,
49
)
50
-from vllm.multimodal.processing import (
50
+from vllm.multimodal.processing import BaseDummyInputsBuilder, ProcessorInputs
51
+from vllm.multimodal.processing.processor import (
52
BaseMultiModalProcessor,
53
BaseProcessingInfo,
54
MultiModalProcessingInfo,
55
PromptReplacement,
56
PromptUpdate,
57
)
57
-from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
58
from vllm.sequence import IntermediateTensors
59
from vllm.tokenizers import cached_tokenizer_from_config
60
from vllm.tokenizers.mistral import MistralTokenizer
vllm/model_executor/models/voxtral_streaming.py
+2
-2
@@ -30,11 +30,11 @@ from vllm.multimodal.inputs import (
30
MultiModalKwargsOptionalItems,
31
)
32
from vllm.multimodal.parse import MultiModalDataItems
33
-from vllm.multimodal.processing import (
33
+from vllm.multimodal.processing import BaseDummyInputsBuilder
34
+from vllm.multimodal.processing.processor import (
35
MultiModalPromptUpdates,
36
PlaceholderFeaturesInfo,
37
)
37
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
38
from vllm.sequence import IntermediateTensors
39
from vllm.tokenizers import cached_tokenizer_from_config
40
vllm/model_executor/models/whisper.py
+1
-1
@@ -49,12 +49,12 @@ from vllm.multimodal.inputs import (
49
)
50
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
51
from vllm.multimodal.processing import (
52
+ BaseDummyInputsBuilder,
53
BaseProcessingInfo,
54
EncDecMultiModalProcessor,
55
PromptReplacement,
56
PromptUpdate,
57
)
57
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
58
from vllm.transformers_utils.processor import cached_processor_from_config
59
from vllm.utils.jsontree import json_map_leaves
60
from vllm.utils.tensor_schema import TensorSchema, TensorShape
vllm/multimodal/cache.py
+1
-1
@@ -34,7 +34,7 @@ from .inputs import (
34
if TYPE_CHECKING:
35
from vllm.config import ModelConfig, VllmConfig
36
37
- from .processing import ResolvedPromptUpdate
37
+ from .processing.processor import ResolvedPromptUpdate
38
from .registry import MultiModalRegistry
39
40
logger = init_logger(__name__)
vllm/multimodal/inputs.py
+9
-5
@@ -33,8 +33,6 @@ if TYPE_CHECKING:
33
from transformers.feature_extraction_utils import BatchFeature
34
35
from .base import MediaWithBytes
36
- from .processing import MultiModalHashes
37
-
36
else:
37
torch = LazyLoader("torch", globals(), "torch")
38
@@ -979,9 +977,15 @@ MultiModalKwargsOptionalItems: TypeAlias = (
977
)
978
979
980
+MultiModalHashes = dict[str, list[str]]
981
+"""
982
+A dictionary containing per-item hashes for each modality.
983
+"""
984
+
985
+
986
MultiModalPlaceholderDict: TypeAlias = Mapping[str, Sequence[PlaceholderRange]]
987
"""
984
-A dictionary containing placeholder ranges for each modality.
988
+A dictionary containing per-item placeholder ranges for each modality.
989
"""
990
991
@@ -1001,10 +1005,10 @@ class MultiModalInputs(TypedDict):
1005
mm_kwargs: MultiModalKwargsOptionalItems
1006
"""Keyword arguments to be directly passed to the model after batching."""
1007
1004
- mm_hashes: "MultiModalHashes"
1008
+ mm_hashes: MultiModalHashes
1009
"""The hashes of the multi-modal data."""
1010
1007
- mm_placeholders: "MultiModalPlaceholderDict"
1011
+ mm_placeholders: MultiModalPlaceholderDict
1012
"""
1013
For each modality, information about the placeholder tokens in
1014
`prompt_token_ids`.
vllm/multimodal/processing/__init__.py
new
+27
@@ -0,0 +1,27 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from .context import BaseProcessingInfo, InputProcessingContext
4
+from .dummy_inputs import BaseDummyInputsBuilder, ProcessorInputs
5
+from .processor import (
6
+ BaseMultiModalProcessor,
7
+ EncDecMultiModalProcessor,
8
+ PromptIndexTargets,
9
+ PromptInsertion,
10
+ PromptReplacement,
11
+ PromptUpdate,
12
+ PromptUpdateDetails,
13
+)
14
+
15
+__all__ = [
16
+ "BaseProcessingInfo",
17
+ "InputProcessingContext",
18
+ "BaseDummyInputsBuilder",
19
+ "ProcessorInputs",
20
+ "BaseMultiModalProcessor",
21
+ "EncDecMultiModalProcessor",
22
+ "PromptUpdate",
23
+ "PromptIndexTargets",
24
+ "PromptUpdateDetails",
25
+ "PromptInsertion",
26
+ "PromptReplacement",
27
+]
vllm/multimodal/processing/context.py
new
+558
@@ -0,0 +1,558 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import contextvars
4
+import threading
5
+import time
6
+from abc import abstractmethod
7
+from collections.abc import Generator, Mapping
8
+from contextlib import contextmanager
9
+from dataclasses import dataclass, field
10
+from typing import (
11
+ TYPE_CHECKING,
12
+ Any,
13
+ overload,
14
+)
15
+
16
+import torch
17
+from typing_extensions import TypeVar
18
+
19
+from vllm.logger import init_logger
20
+from vllm.tokenizers import TokenizerLike
21
+from vllm.transformers_utils.processor import cached_processor_from_config
22
+from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
23
+from vllm.utils.jsontree import JSONTree, json_map_leaves
24
+
25
+if TYPE_CHECKING:
26
+ from transformers.configuration_utils import PretrainedConfig
27
+ from transformers.feature_extraction_utils import BatchFeature
28
+ from transformers.processing_utils import ProcessorMixin
29
+
30
+ from vllm.config import ModelConfig, ObservabilityConfig
31
+else:
32
+ PretrainedConfig = object
33
+ BatchFeature = object
34
+ ProcessorMixin = object
35
+
36
+ ModelConfig = object
37
+ ObservabilityConfig = object
38
+
39
+logger = init_logger(__name__)
40
+
41
+
42
+_request_id_context: contextvars.ContextVar[str | None] = contextvars.ContextVar(
43
+ "_request_id_context", default=None
44
+)
45
+
46
+
47
+def get_current_request_id() -> str | None:
48
+ """Get the current request_id from the context, if available."""
49
+ return _request_id_context.get()
50
+
51
+
52
+@contextmanager
53
+def set_request_id(request_id: str) -> Generator[None, None, None]:
54
+ """Context manager to set the request_id for the current context."""
55
+ token = _request_id_context.set(request_id)
56
+ try:
57
+ yield
58
+ finally:
59
+ _request_id_context.reset(token)
60
+
61
+
62
+@dataclass
63
+class MultiModalProcessorTimingStats:
64
+ """Per-request timing statistics for multimodal processor stages."""
65
+
66
+ hf_processor_time: float = 0.0
67
+ """Time spent in HuggingFace processor calls (seconds)."""
68
+
69
+ hashing_time: float = 0.0
70
+ """Time spent computing multimodal item hashes (seconds)."""
71
+
72
+ cache_lookup_time: float = 0.0
73
+ """Time spent in cache lookups and merges (seconds)."""
74
+
75
+ prompt_update_time: float = 0.0
76
+ """Time spent applying prompt updates and finding placeholders (seconds)."""
77
+
78
+ total_time: float = 0.0
79
+ """Total processing time (seconds)."""
80
+
81
+ def to_dict(self) -> dict[str, float]:
82
+ """Convert stats to a dictionary for JSON serialization."""
83
+ return {
84
+ "hf_processor_time": self.hf_processor_time,
85
+ "hashing_time": self.hashing_time,
86
+ "cache_lookup_time": self.cache_lookup_time,
87
+ "prompt_update_time": self.prompt_update_time,
88
+ "total_time": self.total_time,
89
+ }
90
+
91
+
92
+def get_timing_stats_from_engine_client(
93
+ engine_client: Any,
94
+) -> dict[str, dict[str, float]]:
95
+ """
96
+ Get all timing stats from the context associated with the engine client.
97
+
98
+ Args:
99
+ engine_client: The engine client that has input_processor.
100
+
101
+ Returns:
102
+ A dictionary mapping request_id to stats dict.
103
+ """
104
+ try:
105
+ if not engine_client.vllm_config.observability_config.enable_mm_processor_stats:
106
+ return {}
107
+ except (AttributeError, RuntimeError):
108
+ return {}
109
+
110
+ try:
111
+ input_processor = engine_client.input_processor
112
+ input_preprocessor = input_processor.input_preprocessor
113
+
114
+ if hasattr(input_preprocessor, "_get_mm_processor"):
115
+ mm_processor = input_preprocessor._get_mm_processor()
116
+ if mm_processor is not None and hasattr(mm_processor, "info"):
117
+ ctx = mm_processor.info.ctx
118
+ return ctx.get_all_timing_stats()
119
+ except (AttributeError, RuntimeError):
120
+ pass
121
+
122
+ return {}
123
+
124
+
125
+@contextmanager
126
+def timed_operation(ctx: "InputProcessingContext", stage_name: str):
127
+ """
128
+ Context manager to time an operation using the context's timing stats.
129
+
130
+ The request_id is automatically retrieved from the context variable,
131
+ so it doesn't need to be passed as a parameter.
132
+
133
+ Args:
134
+ ctx: The InputProcessingContext containing the timing stats registry.
135
+ stage_name: Name of the stage being timed.
136
+ """
137
+ request_id = get_current_request_id()
138
+ if ctx is None or request_id is None:
139
+ yield
140
+ return
141
+
142
+ stats = ctx.get_timing_stats(request_id)
143
+ if stats is None:
144
+ yield
145
+ return
146
+
147
+ start_time = time.perf_counter()
148
+ try:
149
+ yield
150
+ finally:
151
+ elapsed = time.perf_counter() - start_time
152
+ if stage_name == "hf_processor":
153
+ stats.hf_processor_time += elapsed
154
+ elif stage_name == "hashing":
155
+ stats.hashing_time += elapsed
156
+ elif stage_name == "cache_lookup":
157
+ stats.cache_lookup_time += elapsed
158
+ elif stage_name == "prompt_update":
159
+ stats.prompt_update_time += elapsed
160
+ stats.total_time += elapsed
161
+
162
+
163
+_T = TypeVar("_T")
164
+_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig)
165
+_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
166
+
167
+
168
+@dataclass(frozen=True)
169
+class InputProcessingContext:
170
+ """
171
+ Contains information about the model which may be used to
172
+ modify the inputs.
173
+ """
174
+
175
+ model_config: ModelConfig
176
+ """The configuration of the model."""
177
+
178
+ tokenizer: TokenizerLike | None
179
+ """The tokenizer used to tokenize the inputs."""
180
+
181
+ observability_config: "ObservabilityConfig | None" = field(
182
+ default=None, compare=False, repr=False
183
+ )
184
+ """Configuration for observability features."""
185
+
186
+ timing_stats_registry: dict[str, MultiModalProcessorTimingStats] = field(
187
+ default_factory=dict, compare=False, repr=False
188
+ )
189
+ """Registry for storing timing stats keyed by request_id."""
190
+
191
+ _timing_stats_registry_lock: threading.Lock = field(
192
+ default_factory=threading.Lock, compare=False, repr=False
193
+ )
194
+ """Lock for thread-safe access to timing_stats_registry."""
195
+
196
+ def get_tokenizer(self) -> TokenizerLike:
197
+ if self.tokenizer is None:
198
+ raise ValueError(
199
+ "You cannot pass text prompts when `skip_tokenizer_init=True`"
200
+ )
201
+
202
+ return self.tokenizer
203
+
204
+ @overload
205
+ def get_hf_config(self, /) -> PretrainedConfig: ...
206
+
207
+ @overload
208
+ def get_hf_config(
209
+ self,
210
+ typ: type[_C] | tuple[type[_C], ...],
211
+ /,
212
+ ) -> _C: ...
213
+
214
+ def get_hf_config(
215
+ self,
216
+ typ: type[Any] | tuple[type[Any], ...] | None = None,
217
+ /,
218
+ ) -> Any:
219
+ """
220
+ Get the HuggingFace configuration
221
+ (`transformers.PretrainedConfig`) of the model,
222
+ additionally checking its type.
223
+
224
+ Raises:
225
+ TypeError: If the configuration is not of the specified type.
226
+ """
227
+ if typ is None:
228
+ from transformers.configuration_utils import PretrainedConfig
229
+
230
+ typ = PretrainedConfig
231
+
232
+ hf_config = self.model_config.hf_config
233
+ if not isinstance(hf_config, typ):
234
+ raise TypeError(
235
+ "Invalid type of HuggingFace config. "
236
+ f"Expected type: {typ}, but "
237
+ f"found type: {type(hf_config)}"
238
+ )
239
+
240
+ return hf_config
241
+
242
+ def get_hf_image_processor_config(self) -> dict[str, Any]:
243
+ """
244
+ Get the HuggingFace image processor configuration of the model.
245
+ """
246
+ return self.model_config.hf_image_processor_config
247
+
248
+ def get_mm_config(self):
249
+ """
250
+ Get the multimodal config of the model.
251
+
252
+ Raises:
253
+ RuntimeError: If the model is not a multimodal model.
254
+ """
255
+ mm_config = self.model_config.multimodal_config
256
+ if mm_config is None:
257
+ raise RuntimeError("Not a multimodal model")
258
+
259
+ return mm_config
260
+
261
+ @overload
262
+ def get_hf_processor(self, /, **kwargs: object) -> ProcessorMixin: ...
263
+
264
+ @overload
265
+ def get_hf_processor(
266
+ self,
267
+ typ: type[_P] | tuple[type[_P], ...],
268
+ /,
269
+ **kwargs: object,
270
+ ) -> _P: ...
271
+
272
+ def get_hf_processor(
273
+ self,
274
+ typ: type[Any] | tuple[type[Any], ...] | None = None,
275
+ /,
276
+ **kwargs: object,
277
+ ) -> Any:
278
+ """
279
+ Get the HuggingFace processor
280
+ (`transformers.ProcessorMixin`) of the model,
281
+ additionally checking its type.
282
+
283
+ Raises:
284
+ TypeError: If the processor is not of the specified type.
285
+ """
286
+ if typ is None:
287
+ from transformers.processing_utils import ProcessorMixin
288
+
289
+ typ = ProcessorMixin
290
+
291
+ from vllm.tokenizers.mistral import MistralTokenizer
292
+
293
+ tokenizer = self.tokenizer
294
+ if isinstance(tokenizer, MistralTokenizer):
295
+ tokenizer = tokenizer.transformers_tokenizer
296
+
297
+ return cached_processor_from_config(
298
+ self.model_config,
299
+ processor_cls=typ,
300
+ tokenizer=tokenizer,
301
+ **kwargs,
302
+ )
303
+
304
+ def init_processor(
305
+ self,
306
+ typ: type[_T],
307
+ /,
308
+ **kwargs: object,
309
+ ) -> _T:
310
+ """
311
+ Initialize a HuggingFace-like processor class, merging the
312
+ keyword arguments with those in the model's configuration.
313
+ """
314
+ mm_config = self.model_config.get_multimodal_config()
315
+ base_kwargs = mm_config.mm_processor_kwargs
316
+ if base_kwargs is None:
317
+ base_kwargs = {}
318
+
319
+ merged_kwargs = {**base_kwargs, **kwargs}
320
+
321
+ return typ(**merged_kwargs)
322
+
323
+ def _postprocess_output(
324
+ self,
325
+ output: JSONTree,
326
+ ) -> JSONTree:
327
+ def _postprocess_one(x: object):
328
+ if isinstance(x, torch.Tensor): # noqa: SIM102
329
+ # This mimics the behavior of transformers.BatchFeature
330
+ if x.is_floating_point():
331
+ x = x.to(dtype=self.model_config.dtype)
332
+
333
+ return x
334
+
335
+ return json_map_leaves(_postprocess_one, output)
336
+
337
+ def call_hf_processor(
338
+ self,
339
+ hf_processor: ProcessorMixin,
340
+ data: Mapping[str, object],
341
+ kwargs: Mapping[str, object] = {},
342
+ *,
343
+ num_tries: int = 1,
344
+ max_tries: int = 5,
345
+ ) -> BatchFeature | JSONTree:
346
+ """
347
+ Call `hf_processor` on the prompt `data`
348
+ (text, image, audio...) with configurable options `kwargs`.
349
+ """
350
+ assert callable(hf_processor)
351
+
352
+ mm_config = self.model_config.get_multimodal_config()
353
+ merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)
354
+
355
+ allowed_kwargs = get_allowed_kwarg_only_overrides(
356
+ hf_processor,
357
+ merged_kwargs,
358
+ requires_kw_only=False,
359
+ allow_var_kwargs=True,
360
+ )
361
+
362
+ try:
363
+ output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
364
+ except Exception as exc:
365
+ # See https://github.com/huggingface/tokenizers/issues/537
366
+ if (
367
+ isinstance(exc, RuntimeError)
368
+ and exc
369
+ and exc.args[0] == "Already borrowed"
370
+ and num_tries < max_tries
371
+ ):
372
+ logger.warning(
373
+ "Failed to acquire tokenizer in current thread. "
374
+ "Retrying (%d/%d)...",
375
+ num_tries,
376
+ max_tries,
377
+ )
378
+ time.sleep(0.5)
379
+ return self.call_hf_processor(
380
+ hf_processor,
381
+ data,
382
+ kwargs,
383
+ num_tries=num_tries + 1,
384
+ max_tries=max_tries,
385
+ )
386
+
387
+ msg = (
388
+ f"Failed to apply {type(hf_processor).__name__} "
389
+ f"on data={data} with kwargs={allowed_kwargs}"
390
+ )
391
+
392
+ raise ValueError(msg) from exc
393
+
394
+ # this emulates output.to(dtype=self.model_config.dtype)
395
+ from transformers.feature_extraction_utils import BatchFeature
396
+
397
+ if isinstance(output, BatchFeature):
398
+ output_ = self._postprocess_output(output.data)
399
+ return BatchFeature(output_)
400
+
401
+ logger.warning_once(
402
+ "%s did not return `BatchFeature`. "
403
+ "Make sure to match the behaviour of `ProcessorMixin` when "
404
+ "implementing custom processors.",
405
+ type(hf_processor).__name__,
406
+ )
407
+
408
+ return self._postprocess_output(output)
409
+
410
+ def get_timing_stats(
411
+ self, request_id: str
412
+ ) -> MultiModalProcessorTimingStats | None:
413
+ """
414
+ Get timing stats for a request.
415
+ """
416
+ if (
417
+ self.observability_config is None
418
+ or not self.observability_config.enable_mm_processor_stats
419
+ ):
420
+ return None
421
+ with self._timing_stats_registry_lock:
422
+ return self.timing_stats_registry.get(request_id)
423
+
424
+ def create_timing_stats(self, request_id: str) -> MultiModalProcessorTimingStats:
425
+ """
426
+ Create and store timing stats in the registry for a request.
427
+
428
+ This should be called at the start of processing for a request.
429
+ The stats object is created immediately and stored in the registry.
430
+ """
431
+ if (
432
+ self.observability_config is None
433
+ or not self.observability_config.enable_mm_processor_stats
434
+ ):
435
+ return MultiModalProcessorTimingStats()
436
+
437
+ with self._timing_stats_registry_lock:
438
+ if request_id in self.timing_stats_registry:
439
+ raise ValueError(
440
+ f"Timing stats already exist for request_id: {request_id}"
441
+ )
442
+ stats = MultiModalProcessorTimingStats()
443
+ self.timing_stats_registry[request_id] = stats
444
+ return stats
445
+
446
+ def clear_timing_stats_registry(self) -> int:
447
+ """
448
+ Clear all stats from the registry. Returns the number of stats cleared.
449
+ """
450
+ if (
451
+ self.observability_config is None
452
+ or not self.observability_config.enable_mm_processor_stats
453
+ ):
454
+ return 0
455
+ with self._timing_stats_registry_lock:
456
+ count = len(self.timing_stats_registry)
457
+ self.timing_stats_registry.clear()
458
+ return count
459
+
460
+ def get_all_timing_stats(self) -> dict[str, dict[str, float]]:
461
+ """
462
+ Get all timing stats as a dictionary for API endpoints.
463
+ """
464
+ if (
465
+ self.observability_config is None
466
+ or not self.observability_config.enable_mm_processor_stats
467
+ ):
468
+ return {}
469
+ with self._timing_stats_registry_lock:
470
+ return {
471
+ rid: stats.to_dict()
472
+ for rid, stats in self.timing_stats_registry.items()
473
+ }
474
+
475
+
476
+class BaseProcessingInfo:
477
+ """Base class to provide the information necessary for data processing."""
478
+
479
+ def __init__(self, ctx: InputProcessingContext) -> None:
480
+ super().__init__()
481
+
482
+ self.ctx = ctx
483
+
484
+ @property
485
+ def model_id(self) -> str:
486
+ return self.ctx.model_config.model
487
+
488
+ def get_tokenizer(self) -> TokenizerLike:
489
+ return self.ctx.get_tokenizer()
490
+
491
+ def get_hf_config(self) -> PretrainedConfig:
492
+ return self.ctx.get_hf_config()
493
+
494
+ def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
495
+ """
496
+ Subclasses can override this method to handle
497
+ specific kwargs from model config or user inputs.
498
+ """
499
+ return self.ctx.get_hf_processor(**kwargs)
500
+
501
+ @property
502
+ def skip_prompt_length_check(self) -> bool:
503
+ return False
504
+
505
+ @abstractmethod
506
+ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
507
+ """
508
+ Return the maximum supported number of items for each modality.
509
+
510
+ A value of `None` means unlimited number of items.
511
+
512
+ Omitting a modality from the returned dictionary means that
513
+ it is not supported at all.
514
+ """
515
+ raise NotImplementedError
516
+
517
+ def get_allowed_mm_limits(self) -> Mapping[str, int]:
518
+ """Return the maximum allowed number of items for each modality."""
519
+ supported_mm_limits = self.get_supported_mm_limits()
520
+ mm_config = self.ctx.get_mm_config()
521
+
522
+ allowed_limits = dict[str, int]()
523
+ for modality, supported_limit in supported_mm_limits.items():
524
+ user_limit = mm_config.get_limit_per_prompt(modality)
525
+
526
+ allowed_limits[modality] = (
527
+ user_limit
528
+ if supported_limit is None
529
+ else min(user_limit, supported_limit)
530
+ )
531
+
532
+ return allowed_limits
533
+
534
+ def get_mm_max_tokens_per_item(
535
+ self,
536
+ seq_len: int,
537
+ mm_counts: Mapping[str, int],
538
+ ) -> Mapping[str, int] | None:
539
+ """
540
+ Return the maximum number of tokens per item of for each modality.
541
+
542
+ When `None` (the default) is returned, vLLM will generate dummy inputs
543
+ (images/videos) at maximum possible sizes and process them to determine
544
+ the maximum token count per modality.
545
+
546
+ This approach works but can be very slow for certain models (e.g.,
547
+ Qwen2.5-VL), leading to very long startup time. For better performance,
548
+ each model can override this method to return pre-computed maximum token
549
+ counts, avoiding the need for dummy input generation and processing.
550
+
551
+ Note:
552
+ The maximum number of tokens per item of each modality returned
553
+ from this function should respect the model's maximum sequence
554
+ length and the maximum number of items of each modality allowed,
555
+ and agree with dummy inputs (images/videos) at maximum possible
556
+ sizes.
557
+ """
558
+ return None
vllm/multimodal/processing/dummy_inputs.py
renamed
+4
-8
@@ -3,7 +3,7 @@
3
from abc import ABC, abstractmethod
4
from collections.abc import Mapping
5
from dataclasses import dataclass, field
6
-from typing import TYPE_CHECKING, Generic
6
+from typing import Generic, TypeVar
7
8
import numpy as np
9
import numpy.typing as npt
@@ -17,14 +17,10 @@ from vllm.config.multimodal import (
17
)
18
from vllm.logger import init_logger
19
20
-from .inputs import MultiModalDataDict
20
+from ..inputs import MultiModalDataDict
21
+from .context import BaseProcessingInfo
22
22
-if TYPE_CHECKING:
23
- from .processing import _I
24
-else:
25
- from typing import TypeVar
26
-
27
- _I = TypeVar("_I")
23
+_I = TypeVar("_I", bound=BaseProcessingInfo)
24
25
logger = init_logger(__name__)
26
vllm/multimodal/processing/processor.py
renamed
+16
-556
@@ -1,24 +1,18 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import contextvars
4
-import threading
5
-import time
3
from abc import ABC, abstractmethod
4
from collections import defaultdict
5
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
9
-from contextlib import contextmanager
6
from dataclasses import dataclass, field, replace
7
from enum import Enum
8
from functools import lru_cache
9
from typing import (
10
TYPE_CHECKING,
15
- Any,
11
Generic,
12
NamedTuple,
13
Protocol,
14
TypeAlias,
15
cast,
21
- overload,
16
)
17
18
import regex as re
@@ -27,16 +21,14 @@ from typing_extensions import TypeVar, assert_never
21
22
from vllm.logger import init_logger
23
from vllm.tokenizers import TokenizerLike
30
-from vllm.transformers_utils.processor import cached_processor_from_config
24
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
32
-from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
33
-from vllm.utils.jsontree import JSONTree, json_map_leaves
25
35
-from .hasher import MultiModalHasher
36
-from .inputs import (
26
+from ..hasher import MultiModalHasher
27
+from ..inputs import (
28
MultiModalDataDict,
29
MultiModalEncDecInputs,
30
MultiModalFieldConfig,
31
+ MultiModalHashes,
32
MultiModalInputs,
33
MultiModalKwargsItem,
34
MultiModalKwargsItems,
@@ -44,29 +36,21 @@ from .inputs import (
36
MultiModalUUIDDict,
37
PlaceholderRange,
38
)
47
-from .parse import (
39
+from ..parse import (
40
DictEmbeddingItems,
41
EmbeddingItems,
42
MultiModalDataItems,
43
MultiModalDataParser,
44
)
53
-from .profiling import BaseDummyInputsBuilder
45
+from .context import BaseProcessingInfo, get_current_request_id, timed_operation
46
+from .dummy_inputs import BaseDummyInputsBuilder
47
48
if TYPE_CHECKING:
56
- from transformers.configuration_utils import PretrainedConfig
49
from transformers.feature_extraction_utils import BatchFeature
58
- from transformers.processing_utils import ProcessorMixin
50
60
- from vllm.config import ModelConfig, ObservabilityConfig
61
-
62
- from .cache import BaseMultiModalProcessorCache
51
+ from ..cache import BaseMultiModalProcessorCache
52
else:
64
- PretrainedConfig = object
53
BatchFeature = object
66
- ProcessorMixin = object
67
-
68
- ModelConfig = object
69
- ObservabilityConfig = object
54
55
BaseMultiModalProcessorCache = object
56
@@ -74,126 +58,6 @@ logger = init_logger(__name__)
58
59
_S = TypeVar("_S", str, list[int])
60
77
-_request_id_context: contextvars.ContextVar[str | None] = contextvars.ContextVar(
78
- "_request_id_context", default=None
79
-)
80
-
81
-
82
-def get_current_request_id() -> str | None:
83
- """Get the current request_id from the context, if available."""
84
- return _request_id_context.get()
85
-
86
-
87
-@contextmanager
88
-def set_request_id(request_id: str) -> Generator[None, None, None]:
89
- """Context manager to set the request_id for the current context."""
90
- token = _request_id_context.set(request_id)
91
- try:
92
- yield
93
- finally:
94
- _request_id_context.reset(token)
95
-
96
-
97
-@dataclass
98
-class MultiModalProcessorTimingStats:
99
- """Per-request timing statistics for multimodal processor stages."""
100
-
101
- hf_processor_time: float = 0.0
102
- """Time spent in HuggingFace processor calls (seconds)."""
103
-
104
- hashing_time: float = 0.0
105
- """Time spent computing multimodal item hashes (seconds)."""
106
-
107
- cache_lookup_time: float = 0.0
108
- """Time spent in cache lookups and merges (seconds)."""
109
-
110
- prompt_update_time: float = 0.0
111
- """Time spent applying prompt updates and finding placeholders (seconds)."""
112
-
113
- total_time: float = 0.0
114
- """Total processing time (seconds)."""
115
-
116
- def to_dict(self) -> dict[str, float]:
117
- """Convert stats to a dictionary for JSON serialization."""
118
- return {
119
- "hf_processor_time": self.hf_processor_time,
120
- "hashing_time": self.hashing_time,
121
- "cache_lookup_time": self.cache_lookup_time,
122
- "prompt_update_time": self.prompt_update_time,
123
- "total_time": self.total_time,
124
- }
125
-
126
-
127
-def get_timing_stats_from_engine_client(
128
- engine_client: Any,
129
-) -> dict[str, dict[str, float]]:
130
- """
131
- Get all timing stats from the context associated with the engine client.
132
-
133
- Args:
134
- engine_client: The engine client that has input_processor.
135
-
136
- Returns:
137
- A dictionary mapping request_id to stats dict.
138
- """
139
- try:
140
- if not engine_client.vllm_config.observability_config.enable_mm_processor_stats:
141
- return {}
142
- except (AttributeError, RuntimeError):
143
- return {}
144
-
145
- try:
146
- input_processor = engine_client.input_processor
147
- input_preprocessor = input_processor.input_preprocessor
148
-
149
- if hasattr(input_preprocessor, "_get_mm_processor"):
150
- mm_processor = input_preprocessor._get_mm_processor()
151
- if mm_processor is not None and hasattr(mm_processor, "info"):
152
- ctx = mm_processor.info.ctx
153
- return ctx.get_all_timing_stats()
154
- except (AttributeError, RuntimeError):
155
- pass
156
-
157
- return {}
158
-
159
-
160
-@contextmanager
161
-def _timed_operation(ctx: "InputProcessingContext", stage_name: str):
162
- """
163
- Context manager to time an operation using the context's timing stats.
164
-
165
- The request_id is automatically retrieved from the context variable,
166
- so it doesn't need to be passed as a parameter.
167
-
168
- Args:
169
- ctx: The InputProcessingContext containing the timing stats registry.
170
- stage_name: Name of the stage being timed.
171
- """
172
- request_id = get_current_request_id()
173
- if ctx is None or request_id is None:
174
- yield
175
- return
176
-
177
- stats = ctx.get_timing_stats(request_id)
178
- if stats is None:
179
- yield
180
- return
181
-
182
- start_time = time.perf_counter()
183
- try:
184
- yield
185
- finally:
186
- elapsed = time.perf_counter() - start_time
187
- if stage_name == "hf_processor":
188
- stats.hf_processor_time += elapsed
189
- elif stage_name == "hashing":
190
- stats.hashing_time += elapsed
191
- elif stage_name == "cache_lookup":
192
- stats.cache_lookup_time += elapsed
193
- elif stage_name == "prompt_update":
194
- stats.prompt_update_time += elapsed
195
- stats.total_time += elapsed
196
-
61
62
PromptSeq: TypeAlias = str | list[int]
63
"""A token sequence (list of token IDs) or text."""
@@ -1073,412 +937,6 @@ def find_mm_placeholders(
937
return dict(full_groupby_modality(it))
938
939
1076
-_T = TypeVar("_T")
1077
-_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig)
1078
-_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
1079
-
1080
-
1081
-@dataclass(frozen=True)
1082
-class InputProcessingContext:
1083
- """
1084
- Contains information about the model which may be used to
1085
- modify the inputs.
1086
- """
1087
-
1088
- model_config: ModelConfig
1089
- """The configuration of the model."""
1090
-
1091
- tokenizer: TokenizerLike | None
1092
- """The tokenizer used to tokenize the inputs."""
1093
-
1094
- observability_config: "ObservabilityConfig | None" = field(
1095
- default=None, compare=False, repr=False
1096
- )
1097
- """Configuration for observability features."""
1098
-
1099
- timing_stats_registry: dict[str, MultiModalProcessorTimingStats] = field(
1100
- default_factory=dict, compare=False, repr=False
1101
- )
1102
- """Registry for storing timing stats keyed by request_id."""
1103
-
1104
- _timing_stats_registry_lock: threading.Lock = field(
1105
- default_factory=threading.Lock, compare=False, repr=False
1106
- )
1107
- """Lock for thread-safe access to timing_stats_registry."""
1108
-
1109
- def get_tokenizer(self) -> TokenizerLike:
1110
- if self.tokenizer is None:
1111
- raise ValueError(
1112
- "You cannot pass text prompts when `skip_tokenizer_init=True`"
1113
- )
1114
-
1115
- return self.tokenizer
1116
-
1117
- @overload
1118
- def get_hf_config(self, /) -> PretrainedConfig: ...
1119
-
1120
- @overload
1121
- def get_hf_config(
1122
- self,
1123
- typ: type[_C] | tuple[type[_C], ...],
1124
- /,
1125
- ) -> _C: ...
1126
-
1127
- def get_hf_config(
1128
- self,
1129
- typ: type[Any] | tuple[type[Any], ...] | None = None,
1130
- /,
1131
- ) -> Any:
1132
- """
1133
- Get the HuggingFace configuration
1134
- (`transformers.PretrainedConfig`) of the model,
1135
- additionally checking its type.
1136
-
1137
- Raises:
1138
- TypeError: If the configuration is not of the specified type.
1139
- """
1140
- if typ is None:
1141
- from transformers.configuration_utils import PretrainedConfig
1142
-
1143
- typ = PretrainedConfig
1144
-
1145
- hf_config = self.model_config.hf_config
1146
- if not isinstance(hf_config, typ):
1147
- raise TypeError(
1148
- "Invalid type of HuggingFace config. "
1149
- f"Expected type: {typ}, but "
1150
- f"found type: {type(hf_config)}"
1151
- )
1152
-
1153
- return hf_config
1154
-
1155
- def get_hf_image_processor_config(self) -> dict[str, Any]:
1156
- """
1157
- Get the HuggingFace image processor configuration of the model.
1158
- """
1159
- return self.model_config.hf_image_processor_config
1160
-
1161
- def get_mm_config(self):
1162
- """
1163
- Get the multimodal config of the model.
1164
-
1165
- Raises:
1166
- RuntimeError: If the model is not a multimodal model.
1167
- """
1168
- mm_config = self.model_config.multimodal_config
1169
- if mm_config is None:
1170
- raise RuntimeError("Not a multimodal model")
1171
-
1172
- return mm_config
1173
-
1174
- @overload
1175
- def get_hf_processor(self, /, **kwargs: object) -> ProcessorMixin: ...
1176
-
1177
- @overload
1178
- def get_hf_processor(
1179
- self,
1180
- typ: type[_P] | tuple[type[_P], ...],
1181
- /,
1182
- **kwargs: object,
1183
- ) -> _P: ...
1184
-
1185
- def get_hf_processor(
1186
- self,
1187
- typ: type[Any] | tuple[type[Any], ...] | None = None,
1188
- /,
1189
- **kwargs: object,
1190
- ) -> Any:
1191
- """
1192
- Get the HuggingFace processor
1193
- (`transformers.ProcessorMixin`) of the model,
1194
- additionally checking its type.
1195
-
1196
- Raises:
1197
- TypeError: If the processor is not of the specified type.
1198
- """
1199
- if typ is None:
1200
- from transformers.processing_utils import ProcessorMixin
1201
-
1202
- typ = ProcessorMixin
1203
-
1204
- from vllm.tokenizers.mistral import MistralTokenizer
1205
-
1206
- tokenizer = self.tokenizer
1207
- if isinstance(tokenizer, MistralTokenizer):
1208
- tokenizer = tokenizer.transformers_tokenizer
1209
-
1210
- return cached_processor_from_config(
1211
- self.model_config,
1212
- processor_cls=typ,
1213
- tokenizer=tokenizer,
1214
- **kwargs,
1215
- )
1216
-
1217
- def init_processor(
1218
- self,
1219
- typ: type[_T],
1220
- /,
1221
- **kwargs: object,
1222
- ) -> _T:
1223
- """
1224
- Initialize a HuggingFace-like processor class, merging the
1225
- keyword arguments with those in the model's configuration.
1226
- """
1227
- mm_config = self.model_config.get_multimodal_config()
1228
- base_kwargs = mm_config.mm_processor_kwargs
1229
- if base_kwargs is None:
1230
- base_kwargs = {}
1231
-
1232
- merged_kwargs = {**base_kwargs, **kwargs}
1233
-
1234
- return typ(**merged_kwargs)
1235
-
1236
- def _postprocess_output(
1237
- self,
1238
- output: JSONTree,
1239
- ) -> JSONTree:
1240
- def _postprocess_one(x: object):
1241
- if isinstance(x, torch.Tensor): # noqa: SIM102
1242
- # This mimics the behavior of transformers.BatchFeature
1243
- if x.is_floating_point():
1244
- x = x.to(dtype=self.model_config.dtype)
1245
-
1246
- return x
1247
-
1248
- return json_map_leaves(_postprocess_one, output)
1249
-
1250
- def call_hf_processor(
1251
- self,
1252
- hf_processor: ProcessorMixin,
1253
- data: Mapping[str, object],
1254
- kwargs: Mapping[str, object] = {},
1255
- *,
1256
- num_tries: int = 1,
1257
- max_tries: int = 5,
1258
- ) -> BatchFeature | JSONTree:
1259
- """
1260
- Call `hf_processor` on the prompt `data`
1261
- (text, image, audio...) with configurable options `kwargs`.
1262
- """
1263
- assert callable(hf_processor)
1264
-
1265
- mm_config = self.model_config.get_multimodal_config()
1266
- merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)
1267
-
1268
- allowed_kwargs = get_allowed_kwarg_only_overrides(
1269
- hf_processor,
1270
- merged_kwargs,
1271
- requires_kw_only=False,
1272
- allow_var_kwargs=True,
1273
- )
1274
-
1275
- try:
1276
- output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
1277
- except Exception as exc:
1278
- # See https://github.com/huggingface/tokenizers/issues/537
1279
- if (
1280
- isinstance(exc, RuntimeError)
1281
- and exc
1282
- and exc.args[0] == "Already borrowed"
1283
- and num_tries < max_tries
1284
- ):
1285
- logger.warning(
1286
- "Failed to acquire tokenizer in current thread. "
1287
- "Retrying (%d/%d)...",
1288
- num_tries,
1289
- max_tries,
1290
- )
1291
- time.sleep(0.5)
1292
- return self.call_hf_processor(
1293
- hf_processor,
1294
- data,
1295
- kwargs,
1296
- num_tries=num_tries + 1,
1297
- max_tries=max_tries,
1298
- )
1299
-
1300
- msg = (
1301
- f"Failed to apply {type(hf_processor).__name__} "
1302
- f"on data={data} with kwargs={allowed_kwargs}"
1303
- )
1304
-
1305
- raise ValueError(msg) from exc
1306
-
1307
- # this emulates output.to(dtype=self.model_config.dtype)
1308
- from transformers.feature_extraction_utils import BatchFeature
1309
-
1310
- if isinstance(output, BatchFeature):
1311
- output_ = self._postprocess_output(output.data)
1312
- return BatchFeature(output_)
1313
-
1314
- logger.warning_once(
1315
- "%s did not return `BatchFeature`. "
1316
- "Make sure to match the behaviour of `ProcessorMixin` when "
1317
- "implementing custom processors.",
1318
- type(hf_processor).__name__,
1319
- )
1320
-
1321
- return self._postprocess_output(output)
1322
-
1323
- def get_timing_stats(
1324
- self, request_id: str
1325
- ) -> MultiModalProcessorTimingStats | None:
1326
- """
1327
- Get timing stats for a request.
1328
- """
1329
- if (
1330
- self.observability_config is None
1331
- or not self.observability_config.enable_mm_processor_stats
1332
- ):
1333
- return None
1334
- with self._timing_stats_registry_lock:
1335
- return self.timing_stats_registry.get(request_id)
1336
-
1337
- def create_timing_stats(self, request_id: str) -> MultiModalProcessorTimingStats:
1338
- """
1339
- Create and store timing stats in the registry for a request.
1340
-
1341
- This should be called at the start of processing for a request.
1342
- The stats object is created immediately and stored in the registry.
1343
- """
1344
- if (
1345
- self.observability_config is None
1346
- or not self.observability_config.enable_mm_processor_stats
1347
- ):
1348
- return MultiModalProcessorTimingStats()
1349
-
1350
- with self._timing_stats_registry_lock:
1351
- if request_id in self.timing_stats_registry:
1352
- raise ValueError(
1353
- f"Timing stats already exist for request_id: {request_id}"
1354
- )
1355
- stats = MultiModalProcessorTimingStats()
1356
- self.timing_stats_registry[request_id] = stats
1357
- return stats
1358
-
1359
- def clear_timing_stats_registry(self) -> int:
1360
- """
1361
- Clear all stats from the registry. Returns the number of stats cleared.
1362
- """
1363
- if (
1364
- self.observability_config is None
1365
- or not self.observability_config.enable_mm_processor_stats
1366
- ):
1367
- return 0
1368
- with self._timing_stats_registry_lock:
1369
- count = len(self.timing_stats_registry)
1370
- self.timing_stats_registry.clear()
1371
- return count
1372
-
1373
- def get_all_timing_stats(self) -> dict[str, dict[str, float]]:
1374
- """
1375
- Get all timing stats as a dictionary for API endpoints.
1376
- """
1377
- if (
1378
- self.observability_config is None
1379
- or not self.observability_config.enable_mm_processor_stats
1380
- ):
1381
- return {}
1382
- with self._timing_stats_registry_lock:
1383
- return {
1384
- rid: stats.to_dict()
1385
- for rid, stats in self.timing_stats_registry.items()
1386
- }
1387
-
1388
-
1389
-class BaseProcessingInfo:
1390
- """Base class to provide the information necessary for data processing."""
1391
-
1392
- def __init__(self, ctx: InputProcessingContext) -> None:
1393
- super().__init__()
1394
-
1395
- self.ctx = ctx
1396
-
1397
- @property
1398
- def model_id(self) -> str:
1399
- return self.ctx.model_config.model
1400
-
1401
- def get_tokenizer(self) -> TokenizerLike:
1402
- return self.ctx.get_tokenizer()
1403
-
1404
- def get_hf_config(self) -> PretrainedConfig:
1405
- return self.ctx.get_hf_config()
1406
-
1407
- def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
1408
- """
1409
- Subclasses can override this method to handle
1410
- specific kwargs from model config or user inputs.
1411
- """
1412
- return self.ctx.get_hf_processor(**kwargs)
1413
-
1414
- @property
1415
- def skip_prompt_length_check(self) -> bool:
1416
- return False
1417
-
1418
- @abstractmethod
1419
- def get_supported_mm_limits(self) -> Mapping[str, int | None]:
1420
- """
1421
- Return the maximum supported number of items for each modality.
1422
-
1423
- A value of `None` means unlimited number of items.
1424
-
1425
- Omitting a modality from the returned dictionary means that
1426
- it is not supported at all.
1427
- """
1428
- raise NotImplementedError
1429
-
1430
- def get_allowed_mm_limits(self) -> Mapping[str, int]:
1431
- """Return the maximum allowed number of items for each modality."""
1432
- supported_mm_limits = self.get_supported_mm_limits()
1433
- mm_config = self.ctx.get_mm_config()
1434
-
1435
- allowed_limits = dict[str, int]()
1436
- for modality, supported_limit in supported_mm_limits.items():
1437
- user_limit = mm_config.get_limit_per_prompt(modality)
1438
-
1439
- allowed_limits[modality] = (
1440
- user_limit
1441
- if supported_limit is None
1442
- else min(user_limit, supported_limit)
1443
- )
1444
-
1445
- return allowed_limits
1446
-
1447
- def get_mm_max_tokens_per_item(
1448
- self,
1449
- seq_len: int,
1450
- mm_counts: Mapping[str, int],
1451
- ) -> Mapping[str, int] | None:
1452
- """
1453
- Return the maximum number of tokens per item of for each modality.
1454
-
1455
- When `None` (the default) is returned, vLLM will generate dummy inputs
1456
- (images/videos) at maximum possible sizes and process them to determine
1457
- the maximum token count per modality.
1458
-
1459
- This approach works but can be very slow for certain models (e.g.,
1460
- Qwen2.5-VL), leading to very long startup time. For better performance,
1461
- each model can override this method to return pre-computed maximum token
1462
- counts, avoiding the need for dummy input generation and processing.
1463
-
1464
- Note:
1465
- The maximum number of tokens per item of each modality returned
1466
- from this function should respect the model's maximum sequence
1467
- length and the maximum number of items of each modality allowed,
1468
- and agree with dummy inputs (images/videos) at maximum possible
1469
- sizes.
1470
- """
1471
- return None
1472
-
1473
-
1474
-_I = TypeVar("_I", bound=BaseProcessingInfo)
1475
-
1476
-MultiModalHashes = dict[str, list[str]]
1477
-"""
1478
-A collection of the multi-modal hash for each item, with a similar structure as
1479
-[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
1480
-"""
1481
-
940
MultiModalIsCached = dict[str, list[bool]]
941
"""
942
A collection of the `is_cached` flag for each item, with a similar structure as
@@ -1499,6 +957,8 @@ For an item `MultiModalPromptUpdates[k][i]`,
957
`ResolvedPromptUpdate` instances have been applied.
958
"""
959
960
+_I = TypeVar("_I", bound=BaseProcessingInfo)
961
+
962
963
class MultiModalProcessingInfo(NamedTuple):
964
kwargs: MultiModalKwargsOptionalItems
@@ -1732,7 +1192,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1192
Call the HF processor on the prompt text and
1193
associated multi-modal data.
1194
"""
1735
- with _timed_operation(self.info.ctx, "hf_processor"):
1195
+ with timed_operation(self.info.ctx, "hf_processor"):
1196
return self.info.ctx.call_hf_processor(
1197
self.info.get_hf_processor(**mm_kwargs),
1198
dict(text=prompt, **mm_data),
@@ -1841,7 +1301,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1301
1302
Since HF processor requires that text and multi-modal items
1303
correspond to each other, we generate dummy text using
1844
- [`DummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder]
1304
+ [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
1305
to go along with the multi-modal data.
1306
"""
1307
mm_counts = mm_items.get_all_counts()
@@ -2085,7 +1545,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1545
)
1546
1547
# Use overrides if provided; fallback to data-dependent hashing.
2088
- with _timed_operation(self.info.ctx, "hashing"):
1548
+ with timed_operation(self.info.ctx, "hashing"):
1549
mm_hashes = self._hash_mm_items(
1550
mm_data_items,
1551
hf_processor_mm_kwargs,
@@ -2132,7 +1592,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1592
mm_uuids=mm_uuids,
1593
)
1594
2135
- with _timed_operation(self.info.ctx, "hashing"):
1595
+ with timed_operation(self.info.ctx, "hashing"):
1596
mm_hashes = self._hash_mm_items(
1597
mm_data_items,
1598
hf_processor_mm_kwargs,
@@ -2140,7 +1600,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1600
mm_uuids=mm_uuids,
1601
)
1602
2143
- with _timed_operation(self.info.ctx, "cache_lookup"):
1603
+ with timed_operation(self.info.ctx, "cache_lookup"):
1604
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
1605
cache=cache,
1606
mm_data_items=mm_data_items,
@@ -2175,7 +1635,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1635
mm_missing_kwargs,
1636
)
1637
2178
- with _timed_operation(self.info.ctx, "cache_lookup"):
1638
+ with timed_operation(self.info.ctx, "cache_lookup"):
1639
mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
1640
cache,
1641
mm_hashes=mm_hashes,
@@ -2386,7 +1846,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1846
)
1847
1848
# NOTE: tokenization_kwargs are not required to init processor
2389
- with _timed_operation(self.info.ctx, "prompt_update"):
1849
+ with timed_operation(self.info.ctx, "prompt_update"):
1850
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
1851
mm_items=mm_items,
1852
prompt_ids=prompt_ids,
vllm/multimodal/registry.py
+2
-2
@@ -12,11 +12,11 @@ from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
12
from .cache import BaseMultiModalProcessorCache
13
from .inputs import MultiModalInputs
14
from .processing import (
15
+ BaseDummyInputsBuilder,
16
BaseMultiModalProcessor,
17
BaseProcessingInfo,
18
InputProcessingContext,
19
)
19
-from .profiling import BaseDummyInputsBuilder
20
21
if TYPE_CHECKING:
22
from vllm.config import ModelConfig, ObservabilityConfig
@@ -45,7 +45,7 @@ class ProcessingInfoFactory(Protocol[_I_co]):
45
class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]
46
"""
47
Constructs a
48
- [`BaseDummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder]
48
+ [`BaseDummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
49
instance from the context.
50
"""
51
vllm/v1/engine/input_processor.py
+1
-1
@@ -17,7 +17,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
17
from vllm.multimodal.cache import processor_cache_from_config
18
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict
19
from vllm.multimodal.parse import MultiModalDataParser
20
-from vllm.multimodal.processing import set_request_id
20
+from vllm.multimodal.processing.context import set_request_id
21
from vllm.multimodal.utils import argsort_mm_positions
22
from vllm.pooling_params import PoolingParams
23
from vllm.sampling_params import _SAMPLING_EPS, SamplingParams