A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 06:30 UTC
[Misc] Set default torch num threads for input processing (#31879)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Jan 12, 2026 at 10:28 UTC
16abe6b85a3fbf6312afe3514797ec741c10a178
2 files changed
+12
-16
vllm/model_executor/models/internvl.py
+1
-15
@@ -7,7 +7,6 @@
7
# Copyright (c) 2023 OpenGVLab
8
# Licensed under The MIT License [see LICENSE for details]
9
# --------------------------------------------------------
10
-import os
10
from abc import ABC, abstractmethod
11
from collections.abc import Iterable, Mapping, Sequence
12
from typing import Annotated, Any, Literal, TypeAlias, TypeVar
@@ -52,7 +51,6 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
51
from vllm.sequence import IntermediateTensors
52
from vllm.tokenizers import TokenizerLike
53
from vllm.utils.tensor_schema import TensorSchema, TensorShape
55
-from vllm.utils.torch_utils import set_default_torch_num_threads
54
55
from .interfaces import (
56
MultiModalEmbeddings,
@@ -143,19 +141,7 @@ def build_transform(input_size: int):
141
T.Normalize(mean=MEAN, std=STD),
142
]
143
)
146
- # Image transformation operations (which include tensor computations
147
- # on the CPU) can occupy a substantial number of CPU cores, introducing
148
- # overhead due to CPU contention. This issue becomes particularly
149
- # noticeable when deploying multiple vLLM instances on a single machine.
150
- # Therefore, it is necessary to limit the number of threads allocated to
151
- # image transformation tasks.
152
- num_threads = int(os.environ.get("OMP_NUM_THREADS", "1"))
153
-
154
- def apply(img):
155
- with set_default_torch_num_threads(num_threads):
156
- return transform(img)
157
-
158
- return apply
144
+ return transform
145
146
147
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
vllm/v1/engine/input_processor.py
+11
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import os
5
import time
6
from collections.abc import Mapping
7
from typing import Any, Literal, cast
@@ -23,6 +24,7 @@ from vllm.sampling_params import _SAMPLING_EPS, SamplingParams
24
from vllm.tokenizers import TokenizerLike
25
from vllm.tokenizers.mistral import MistralTokenizer
26
from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
27
+from vllm.utils.torch_utils import set_default_torch_num_threads
28
from vllm.v1.engine import EngineCoreRequest
29
from vllm.v1.metrics.stats import MultiModalCacheStats
30
from vllm.v1.structured_output.backend_guidance import (
@@ -493,7 +495,15 @@ class InputProcessor:
495
# 1. Tokenize text prompt, with LoRA request if one exists.
496
# 2. For multimodal models with a merged preprocessor, preprocess
497
# multimodal data and expand prompt token ids accordingly.
496
- with set_request_id(request_id):
498
+ num_threads = int(os.environ.get("OMP_NUM_THREADS", "1"))
499
+ if "OMP_NUM_THREADS" not in os.environ:
500
+ logger.debug_once(
501
+ "OMP_NUM_THREADS is not set; defaulting Torch threads to %d for "
502
+ "input preprocessing.",
503
+ num_threads,
504
+ )
505
+
506
+ with set_request_id(request_id), set_default_torch_num_threads(num_threads):
507
processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(
508
prompt,
509
tokenization_kwargs=tokenization_kwargs,