A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Limit CPU threads for image transformations in InternVL to reduce cpu contention. (#24519)
Signed-off-by: li-jinpeng <3332126450@qq.com> Co-authored-by: Roger Wang <hey@rogerw.io>
li-jinpeng committed
Sep 10, 2025 at 16:45 UTC
267c80d31f6b77092a5d5903da64556ac15c4d4d
1 file changed
+16
-1
vllm/model_executor/models/internvl.py
+16
-1
@@ -7,6 +7,7 @@
7
# Copyright (c) 2023 OpenGVLab
8
# Licensed under The MIT License [see LICENSE for details]
9
# --------------------------------------------------------
10
+import os
11
from abc import ABC, abstractmethod
12
from collections.abc import Iterable, Mapping, Sequence
13
from typing import Annotated, Any, Literal, Optional, TypeVar, Union
@@ -37,6 +38,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
38
from vllm.multimodal.profiling import BaseDummyInputsBuilder
39
from vllm.sequence import IntermediateTensors
40
from vllm.transformers_utils.tokenizer import AnyTokenizer
41
+from vllm.utils import set_default_torch_num_threads
42
from vllm.utils.tensor_schema import TensorSchema, TensorShape
43
44
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
@@ -115,13 +117,26 @@ InternVLVideoInputs = Union[InternVLVideoPixelInputs,
117
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
118
def build_transform(input_size: int):
119
MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
118
- return T.Compose([
120
+ transform = T.Compose([
121
T.Lambda(lambda img: convert_image_mode(img, 'RGB')),
122
T.Resize((input_size, input_size),
123
interpolation=T.InterpolationMode.BICUBIC),
124
T.ToTensor(),
125
T.Normalize(mean=MEAN, std=STD)
126
])
127
+ # Image transformation operations (which include tensor computations
128
+ # on the CPU) can occupy a substantial number of CPU cores, introducing
129
+ # overhead due to CPU contention. This issue becomes particularly
130
+ # noticeable when deploying multiple vLLM instances on a single machine.
131
+ # Therefore, it is necessary to limit the number of threads allocated to
132
+ # image transformation tasks.
133
+ num_threads = int(os.environ.get("OMP_NUM_THREADS", "1"))
134
+
135
+ def apply(img):
136
+ with set_default_torch_num_threads(num_threads):
137
+ return transform(img)
138
+
139
+ return apply
140
141
142
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B