A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Chore]:Extract math and argparse utilities to separate modules (#27188)
Signed-off-by: Yeshwanth Surya <yeshsurya@gmail.com> Signed-off-by: Yeshwanth N <yeshsurya@gmail.com> Signed-off-by: yeshsurya <yeshsurya@gmail.com>
Yeshwanth N committed
Oct 26, 2025 at 16:33 UTC
71b1c8b66758ed6730707f270db3c05a831dd4af
125 files changed
+715
-639
benchmarks/benchmark_block_pool.py
+1
-1
@@ -5,7 +5,7 @@ import gc
5
from benchmark_utils import TimeCollector
6
from tabulate import tabulate
7
8
-from vllm.utils import FlexibleArgumentParser
8
+from vllm.utils.argparse_utils import FlexibleArgumentParser
9
from vllm.v1.core.block_pool import BlockPool
10
11
benchmarks/benchmark_long_document_qa_throughput.py
+1
-1
@@ -46,7 +46,7 @@ import time
46
47
from vllm import LLM, SamplingParams
48
from vllm.engine.arg_utils import EngineArgs
49
-from vllm.utils import FlexibleArgumentParser
49
+from vllm.utils.argparse_utils import FlexibleArgumentParser
50
51
52
def test_long_document_qa(llm=None, sampling_params=None, prompts=None):
benchmarks/benchmark_ngram_proposer.py
+1
-1
@@ -19,7 +19,7 @@ from vllm.config import (
19
VllmConfig,
20
)
21
from vllm.platforms import current_platform
22
-from vllm.utils import FlexibleArgumentParser
22
+from vllm.utils.argparse_utils import FlexibleArgumentParser
23
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
24
from vllm.v1.worker.gpu_input_batch import InputBatch
25
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
benchmarks/benchmark_prefix_caching.py
+1
-1
@@ -37,7 +37,7 @@ from transformers import PreTrainedTokenizerBase
37
38
from vllm import LLM, SamplingParams
39
from vllm.engine.arg_utils import EngineArgs
40
-from vllm.utils import FlexibleArgumentParser
40
+from vllm.utils.argparse_utils import FlexibleArgumentParser
41
42
try:
43
from vllm.transformers_utils.tokenizer import get_tokenizer
benchmarks/benchmark_prioritization.py
+1
-1
@@ -11,7 +11,7 @@ import time
11
from transformers import AutoTokenizer, PreTrainedTokenizerBase
12
13
from vllm.engine.arg_utils import EngineArgs
14
-from vllm.utils import FlexibleArgumentParser
14
+from vllm.utils.argparse_utils import FlexibleArgumentParser
15
16
17
# Select a equi-probable random priority
benchmarks/benchmark_serving_structured_output.py
+1
-1
@@ -51,7 +51,7 @@ except ImportError:
51
from backend_request_func import get_tokenizer
52
53
try:
54
- from vllm.utils import FlexibleArgumentParser
54
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
55
except ImportError:
56
from argparse import ArgumentParser as FlexibleArgumentParser
57
benchmarks/cutlass_benchmarks/sparse_benchmarks.py
+1
-1
@@ -15,7 +15,7 @@ from utils import make_rand_sparse_tensors
15
from weight_shapes import WEIGHT_SHAPES
16
17
from vllm import _custom_ops as ops
18
-from vllm.utils import FlexibleArgumentParser
18
+from vllm.utils.argparse_utils import FlexibleArgumentParser
19
20
DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
21
DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+2
-1
@@ -18,7 +18,8 @@ from vllm import _custom_ops as ops
18
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
19
w8a8_triton_block_scaled_mm,
20
)
21
-from vllm.utils import FlexibleArgumentParser, cdiv
21
+from vllm.utils.argparse_utils import FlexibleArgumentParser
22
+from vllm.utils.math_utils import cdiv
23
24
DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
25
DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
benchmarks/kernels/bench_per_token_quant_fp8.py
+1
-1
@@ -10,7 +10,7 @@ import torch
10
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
11
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
12
from vllm.triton_utils import triton
13
-from vllm.utils import FlexibleArgumentParser
13
+from vllm.utils.argparse_utils import FlexibleArgumentParser
14
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
15
16
benchmarks/kernels/benchmark_activation.py
+1
-1
@@ -10,7 +10,7 @@ import vllm.model_executor.layers.activation # noqa F401
10
from vllm.model_executor.custom_op import CustomOp
11
from vllm.platforms import current_platform
12
from vllm.triton_utils import triton
13
-from vllm.utils import FlexibleArgumentParser
13
+from vllm.utils.argparse_utils import FlexibleArgumentParser
14
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
15
16
batch_size_range = [1, 16, 32, 64, 128]
benchmarks/kernels/benchmark_bitblas.py
+1
-1
@@ -28,7 +28,7 @@ except ImportError as e:
28
29
from bitblas import Matmul, MatmulConfig, auto_detect_nvidia_target
30
31
-from vllm.utils import FlexibleArgumentParser
31
+from vllm.utils.argparse_utils import FlexibleArgumentParser
32
33
parser = FlexibleArgumentParser(
34
description="Benchmark BitBLAS int4 on a specific target."
benchmarks/kernels/benchmark_cutlass_fp4_moe.py
+1
-1
@@ -20,7 +20,7 @@ from vllm.model_executor.layers.fused_moe.config import (
20
from vllm.model_executor.layers.fused_moe.cutlass_moe import cutlass_moe_fp4
21
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk
22
from vllm.scalar_type import scalar_types
23
-from vllm.utils import FlexibleArgumentParser
23
+from vllm.utils.argparse_utils import FlexibleArgumentParser
24
25
WEIGHT_SHAPES_MOE = {
26
"nvidia/DeepSeek-R1-FP4": [
benchmarks/kernels/benchmark_cutlass_moe_fp8.py
+1
-1
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_confi
14
from vllm.model_executor.layers.fused_moe.cutlass_moe import cutlass_moe_fp8
15
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk
16
from vllm.platforms import current_platform
17
-from vllm.utils import FlexibleArgumentParser
17
+from vllm.utils.argparse_utils import FlexibleArgumentParser
18
19
# Weight shapes for different models: [num_experts, topk, hidden_size,
20
# intermediate_size]
benchmarks/kernels/benchmark_device_communicators.py
+1
-1
@@ -39,7 +39,7 @@ from vllm.distributed.device_communicators.pynccl_allocator import (
39
)
40
from vllm.distributed.device_communicators.symm_mem import SymmMemCommunicator
41
from vllm.logger import init_logger
42
-from vllm.utils import FlexibleArgumentParser
42
+from vllm.utils.argparse_utils import FlexibleArgumentParser
43
44
logger = init_logger(__name__)
45
benchmarks/kernels/benchmark_grouped_gemm_cutlass.py
+1
-1
@@ -13,7 +13,7 @@ from vllm.model_executor.layers.fused_moe.fused_moe import (
13
fused_experts,
14
fused_topk,
15
)
16
-from vllm.utils import FlexibleArgumentParser
16
+from vllm.utils.argparse_utils import FlexibleArgumentParser
17
18
DEFAULT_MODELS = [
19
"nm-testing/Mixtral-8x7B-Instruct-v0.1",
benchmarks/kernels/benchmark_layernorm.py
+1
-1
@@ -7,7 +7,7 @@ import torch
7
8
from vllm.model_executor.layers.layernorm import RMSNorm
9
from vllm.platforms import current_platform
10
-from vllm.utils import FlexibleArgumentParser
10
+from vllm.utils.argparse_utils import FlexibleArgumentParser
11
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
12
13
benchmarks/kernels/benchmark_lora.py
+1
-1
@@ -25,7 +25,7 @@ if HAS_TRITON:
25
from vllm.lora.ops.triton_ops import LoRAKernelMeta, lora_expand, lora_shrink
26
from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
27
28
-from vllm.utils import FlexibleArgumentParser
28
+from vllm.utils.argparse_utils import FlexibleArgumentParser
29
30
DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
31
DEFAULT_TP_SIZES = [1]
benchmarks/kernels/benchmark_machete.py
+1
-1
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
33
quantize_weights,
34
)
35
from vllm.scalar_type import ScalarType, scalar_types
36
-from vllm.utils import FlexibleArgumentParser
36
+from vllm.utils.argparse_utils import FlexibleArgumentParser
37
38
DEFAULT_MODELS = ["meta-llama/Llama-3-8b", "meta-llama/Llama-2-70b-hf"]
39
DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512, 1024]
benchmarks/kernels/benchmark_marlin.py
+1
-1
@@ -44,7 +44,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
44
sort_weights,
45
)
46
from vllm.scalar_type import ScalarType, scalar_types
47
-from vllm.utils import FlexibleArgumentParser
47
+from vllm.utils.argparse_utils import FlexibleArgumentParser
48
49
DEFAULT_MODELS = ["meta-llama/Llama-2-7b-hf/TP1"]
50
DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
benchmarks/kernels/benchmark_moe.py
+1
-1
@@ -22,7 +22,7 @@ from vllm.model_executor.layers.fused_moe.fused_moe import *
22
from vllm.platforms import current_platform
23
from vllm.transformers_utils.config import get_config
24
from vllm.triton_utils import triton
25
-from vllm.utils import FlexibleArgumentParser
25
+from vllm.utils.argparse_utils import FlexibleArgumentParser
26
27
FP8_DTYPE = current_platform.fp8_dtype()
28
benchmarks/kernels/benchmark_moe_permute_unpermute.py
+1
-1
@@ -17,7 +17,7 @@ from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
17
)
18
from vllm.model_executor.layers.fused_moe.utils import _fp8_quantize
19
from vllm.platforms import current_platform
20
-from vllm.utils import FlexibleArgumentParser
20
+from vllm.utils.argparse_utils import FlexibleArgumentParser
21
22
FP8_DTYPE = current_platform.fp8_dtype()
23
benchmarks/kernels/benchmark_mrope.py
+1
-1
@@ -39,7 +39,7 @@ import torch
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.platforms import current_platform
41
from vllm.transformers_utils.config import get_config
42
-from vllm.utils import FlexibleArgumentParser
42
+from vllm.utils.argparse_utils import FlexibleArgumentParser
43
44
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
45
benchmarks/kernels/benchmark_paged_attention.py
+1
-1
@@ -9,7 +9,7 @@ import torch
9
from vllm import _custom_ops as ops
10
from vllm.logger import init_logger
11
from vllm.platforms import current_platform
12
-from vllm.utils import FlexibleArgumentParser
12
+from vllm.utils.argparse_utils import FlexibleArgumentParser
13
from vllm.utils.torch_utils import (
14
STR_DTYPE_TO_TORCH_DTYPE,
15
create_kv_caches_with_random,
benchmarks/kernels/benchmark_quant.py
+1
-1
@@ -7,7 +7,7 @@ import torch
7
8
from vllm import _custom_ops as ops
9
from vllm.platforms import current_platform
10
-from vllm.utils import FlexibleArgumentParser
10
+from vllm.utils.argparse_utils import FlexibleArgumentParser
11
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
12
13
benchmarks/kernels/benchmark_reshape_and_cache.py
+1
-1
@@ -9,7 +9,7 @@ from tabulate import tabulate
9
from vllm import _custom_ops as ops
10
from vllm.logger import init_logger
11
from vllm.platforms import current_platform
12
-from vllm.utils import FlexibleArgumentParser
12
+from vllm.utils.argparse_utils import FlexibleArgumentParser
13
from vllm.utils.torch_utils import (
14
STR_DTYPE_TO_TORCH_DTYPE,
15
create_kv_caches_with_random,
benchmarks/kernels/benchmark_reshape_and_cache_flash.py
+1
-1
@@ -12,7 +12,7 @@ from vllm.attention.ops.triton_reshape_and_cache_flash import (
12
)
13
from vllm.logger import init_logger
14
from vllm.platforms import current_platform
15
-from vllm.utils import FlexibleArgumentParser
15
+from vllm.utils.argparse_utils import FlexibleArgumentParser
16
from vllm.utils.torch_utils import (
17
STR_DTYPE_TO_TORCH_DTYPE,
18
create_kv_caches_with_random_flash,
benchmarks/kernels/benchmark_rope.py
+1
-1
@@ -8,7 +8,7 @@ import torch
8
9
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
10
from vllm.platforms import current_platform
11
-from vllm.utils import FlexibleArgumentParser
11
+from vllm.utils.argparse_utils import FlexibleArgumentParser
12
13
14
def benchmark_rope_kernels_multi_lora(
benchmarks/kernels/benchmark_trtllm_decode_attention.py
+1
-1
@@ -8,7 +8,7 @@ from datetime import datetime
8
import flashinfer
9
import torch
10
11
-from vllm.utils import round_up
11
+from vllm.utils.math_utils import round_up
12
13
FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
14
FP8_DTYPE = torch.float8_e4m3fn
benchmarks/kernels/benchmark_trtllm_prefill_attention.py
+1
-1
@@ -8,7 +8,7 @@ from datetime import datetime
8
import flashinfer
9
import torch
10
11
-from vllm.utils import round_up
11
+from vllm.utils.math_utils import round_up
12
13
FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
14
FP8_DTYPE = torch.float8_e4m3fn
benchmarks/kernels/benchmark_w8a8_block_fp8.py
+1
-1
@@ -18,7 +18,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
18
)
19
from vllm.platforms import current_platform
20
from vllm.triton_utils import triton
21
-from vllm.utils import FlexibleArgumentParser
21
+from vllm.utils.argparse_utils import FlexibleArgumentParser
22
23
mp.set_start_method("spawn", force=True)
24
benchmarks/kernels/graph_machete_bench.py
+1
-1
@@ -11,7 +11,7 @@ import regex as re
11
import seaborn as sns
12
from torch.utils.benchmark import Measurement as TMeasurement
13
14
-from vllm.utils import FlexibleArgumentParser
14
+from vllm.utils.argparse_utils import FlexibleArgumentParser
15
16
if __name__ == "__main__":
17
parser = FlexibleArgumentParser(
benchmarks/overheads/benchmark_hashing.py
+1
-1
@@ -5,7 +5,7 @@ import cProfile
5
import pstats
6
7
from vllm import LLM, SamplingParams
8
-from vllm.utils import FlexibleArgumentParser
8
+from vllm.utils.argparse_utils import FlexibleArgumentParser
9
10
# A very long prompt, total number of tokens is about 15k.
11
LONG_PROMPT = ["You are an expert in large language models, aren't you?"] * 1000
examples/offline_inference/audio_language.py
+1
-1
@@ -18,7 +18,7 @@ from transformers import AutoTokenizer
18
from vllm import LLM, EngineArgs, SamplingParams
19
from vllm.assets.audio import AudioAsset
20
from vllm.lora.request import LoRARequest
21
-from vllm.utils import FlexibleArgumentParser
21
+from vllm.utils.argparse_utils import FlexibleArgumentParser
22
23
audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
24
question_per_audio_count = {
examples/offline_inference/basic/chat.py
+1
-1
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from vllm import LLM, EngineArgs
5
-from vllm.utils import FlexibleArgumentParser
5
+from vllm.utils.argparse_utils import FlexibleArgumentParser
6
7
8
def create_parser():
examples/offline_inference/basic/classify.py
+1
-1
@@ -4,7 +4,7 @@
4
from argparse import Namespace
5
6
from vllm import LLM, EngineArgs
7
-from vllm.utils import FlexibleArgumentParser
7
+from vllm.utils.argparse_utils import FlexibleArgumentParser
8
9
10
def parse_args():
examples/offline_inference/basic/embed.py
+1
-1
@@ -4,7 +4,7 @@
4
from argparse import Namespace
5
6
from vllm import LLM, EngineArgs
7
-from vllm.utils import FlexibleArgumentParser
7
+from vllm.utils.argparse_utils import FlexibleArgumentParser
8
9
10
def parse_args():
examples/offline_inference/basic/generate.py
+1
-1
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from vllm import LLM, EngineArgs
5
-from vllm.utils import FlexibleArgumentParser
5
+from vllm.utils.argparse_utils import FlexibleArgumentParser
6
7
8
def create_parser():
examples/offline_inference/basic/reward.py
+1
-1
@@ -4,7 +4,7 @@
4
from argparse import Namespace
5
6
from vllm import LLM, EngineArgs
7
-from vllm.utils import FlexibleArgumentParser
7
+from vllm.utils.argparse_utils import FlexibleArgumentParser
8
9
10
def parse_args():
examples/offline_inference/basic/score.py
+1
-1
@@ -4,7 +4,7 @@
4
from argparse import Namespace
5
6
from vllm import LLM, EngineArgs
7
-from vllm.utils import FlexibleArgumentParser
7
+from vllm.utils.argparse_utils import FlexibleArgumentParser
8
9
10
def parse_args():
examples/offline_inference/encoder_decoder_multimodal.py
+1
-1
@@ -13,7 +13,7 @@ from typing import NamedTuple
13
14
from vllm import LLM, EngineArgs, PromptType, SamplingParams
15
from vllm.assets.audio import AudioAsset
16
-from vllm.utils import FlexibleArgumentParser
16
+from vllm.utils.argparse_utils import FlexibleArgumentParser
17
18
19
class ModelRequestData(NamedTuple):
examples/offline_inference/llm_engine_example.py
+1
-1
@@ -8,7 +8,7 @@ for processing prompts with various sampling parameters.
8
import argparse
9
10
from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams
11
-from vllm.utils import FlexibleArgumentParser
11
+from vllm.utils.argparse_utils import FlexibleArgumentParser
12
13
14
def create_test_prompts() -> list[tuple[str, SamplingParams]]:
examples/offline_inference/load_sharded_state.py
+1
-1
@@ -25,7 +25,7 @@ python load_sharded_state.py \
25
import dataclasses
26
27
from vllm import LLM, EngineArgs, SamplingParams
28
-from vllm.utils import FlexibleArgumentParser
28
+from vllm.utils.argparse_utils import FlexibleArgumentParser
29
30
31
def parse_args():
examples/offline_inference/pooling/embed_jina_embeddings_v3.py
+1
-1
@@ -4,7 +4,7 @@
4
from argparse import Namespace
5
6
from vllm import LLM, EngineArgs
7
-from vllm.utils import FlexibleArgumentParser
7
+from vllm.utils.argparse_utils import FlexibleArgumentParser
8
9
10
def parse_args():
examples/offline_inference/pooling/embed_matryoshka_fy.py
+1
-1
@@ -4,7 +4,7 @@
4
from argparse import Namespace
5
6
from vllm import LLM, EngineArgs, PoolingParams
7
-from vllm.utils import FlexibleArgumentParser
7
+from vllm.utils.argparse_utils import FlexibleArgumentParser
8
9
10
def parse_args():
examples/offline_inference/pooling/multi_vector_retrieval.py
+1
-1
@@ -4,7 +4,7 @@
4
from argparse import Namespace
5
6
from vllm import LLM, EngineArgs
7
-from vllm.utils import FlexibleArgumentParser
7
+from vllm.utils.argparse_utils import FlexibleArgumentParser
8
9
10
def parse_args():
examples/offline_inference/pooling/ner.py
+1
-1
@@ -5,7 +5,7 @@
5
from argparse import Namespace
6
7
from vllm import LLM, EngineArgs
8
-from vllm.utils import FlexibleArgumentParser
8
+from vllm.utils.argparse_utils import FlexibleArgumentParser
9
10
11
def parse_args():
examples/offline_inference/profiling_tpu/profiling.py
+1
-1
@@ -13,7 +13,7 @@ from tqdm import tqdm
13
from vllm import LLM, SamplingParams
14
from vllm.engine.arg_utils import EngineArgs
15
from vllm.inputs import PromptType
16
-from vllm.utils import FlexibleArgumentParser
16
+from vllm.utils.argparse_utils import FlexibleArgumentParser
17
18
DURATION_MS = int(os.getenv("VLLM_TPU_PROFILE_DURATION_MS", 3000))
19
DELAY_MS = int(os.getenv("VLLM_TPU_PROFILE_DELAY_MS", 0))
examples/offline_inference/qwen2_5_omni/only_thinker.py
+1
-1
@@ -13,7 +13,7 @@ from vllm.assets.audio import AudioAsset
13
from vllm.assets.image import ImageAsset
14
from vllm.assets.video import VideoAsset
15
from vllm.multimodal.image import convert_image_mode
16
-from vllm.utils import FlexibleArgumentParser
16
+from vllm.utils.argparse_utils import FlexibleArgumentParser
17
18
19
class QueryResult(NamedTuple):
examples/offline_inference/save_sharded_state.py
+1
-1
@@ -30,7 +30,7 @@ from pathlib import Path
30
31
from vllm import LLM, EngineArgs
32
from vllm.model_executor.model_loader import ShardedStateLoader
33
-from vllm.utils import FlexibleArgumentParser
33
+from vllm.utils.argparse_utils import FlexibleArgumentParser
34
35
36
def parse_args():
examples/offline_inference/spec_decode.py
+1
-1
@@ -9,7 +9,7 @@ from vllm.inputs import TokensPrompt
9
from vllm.v1.metrics.reader import Counter, Vector
10
11
try:
12
- from vllm.utils import FlexibleArgumentParser
12
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
13
except ImportError:
14
from argparse import ArgumentParser as FlexibleArgumentParser
15
examples/offline_inference/vision_language.py
+1
-1
@@ -22,7 +22,7 @@ from vllm.assets.image import ImageAsset
22
from vllm.assets.video import VideoAsset
23
from vllm.lora.request import LoRARequest
24
from vllm.multimodal.image import convert_image_mode
25
-from vllm.utils import FlexibleArgumentParser
25
+from vllm.utils.argparse_utils import FlexibleArgumentParser
26
27
28
class ModelRequestData(NamedTuple):
examples/offline_inference/vision_language_multi_image.py
+1
-1
@@ -18,7 +18,7 @@ from transformers import AutoProcessor, AutoTokenizer
18
from vllm import LLM, EngineArgs, SamplingParams
19
from vllm.lora.request import LoRARequest
20
from vllm.multimodal.utils import fetch_image
21
-from vllm.utils import FlexibleArgumentParser
21
+from vllm.utils.argparse_utils import FlexibleArgumentParser
22
23
QUESTION = "What is the content of each image?"
24
IMAGE_URLS = [
examples/offline_inference/vision_language_pooling.py
+1
-1
@@ -18,7 +18,7 @@ from PIL.Image import Image
18
from vllm import LLM, EngineArgs
19
from vllm.entrypoints.score_utils import ScoreMultiModalParam
20
from vllm.multimodal.utils import fetch_image
21
-from vllm.utils import FlexibleArgumentParser
21
+from vllm.utils.argparse_utils import FlexibleArgumentParser
22
23
ROOT_DIR = Path(__file__).parent.parent.parent
24
EXAMPLES_DIR = ROOT_DIR / "examples"
examples/online_serving/openai_chat_completion_client_for_multimodal.py
+1
-1
@@ -26,7 +26,7 @@ import requests
26
from openai import OpenAI
27
from utils import get_first_model
28
29
-from vllm.utils import FlexibleArgumentParser
29
+from vllm.utils.argparse_utils import FlexibleArgumentParser
30
31
# Modify OpenAI's API key and API base to use vLLM's API server.
32
openai_api_key = "EMPTY"
examples/others/tensorize_vllm_model.py
+1
-1
@@ -16,7 +16,7 @@ from vllm.model_executor.model_loader.tensorizer import (
16
tensorize_vllm_model,
17
tensorizer_kwargs_arg,
18
)
19
-from vllm.utils import FlexibleArgumentParser
19
+from vllm.utils.argparse_utils import FlexibleArgumentParser
20
21
logger = logging.getLogger()
22
tests/engine/test_arg_utils.py
+1
-1
@@ -22,7 +22,7 @@ from vllm.engine.arg_utils import (
22
optional_type,
23
parse_type,
24
)
25
-from vllm.utils import FlexibleArgumentParser
25
+from vllm.utils.argparse_utils import FlexibleArgumentParser
26
27
28
@pytest.mark.parametrize(
tests/entrypoints/openai/test_cli_args.py
+1
-1
@@ -7,7 +7,7 @@ import pytest
7
8
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
9
from vllm.entrypoints.openai.serving_models import LoRAModulePath
10
-from vllm.utils import FlexibleArgumentParser
10
+from vllm.utils.argparse_utils import FlexibleArgumentParser
11
12
from ...utils import VLLM_PATH
13
tests/kernels/attention/test_deepgemm_attention.py
+1
-1
@@ -6,7 +6,6 @@ import pytest
6
import torch
7
8
from vllm.platforms import current_platform
9
-from vllm.utils import cdiv
9
from vllm.utils.deep_gemm import (
10
_ceil_to_ue8m0,
11
calc_diff,
@@ -16,6 +15,7 @@ from vllm.utils.deep_gemm import (
15
get_paged_mqa_logits_metadata,
16
)
17
from vllm.utils.import_utils import has_deep_gemm
18
+from vllm.utils.math_utils import cdiv
19
20
21
def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
tests/kernels/attention/test_flashinfer_trtllm_attention.py
+1
-1
@@ -10,7 +10,7 @@ from tests.kernels.quantization.nvfp4_utils import (
10
get_nvfp4_global_scale,
11
)
12
from vllm.platforms import current_platform
13
-from vllm.utils import round_up
13
+from vllm.utils.math_utils import round_up
14
15
if not current_platform.is_device_capability(100):
16
pytest.skip(
tests/kernels/attention/test_mla_decode_cpu.py
+1
-1
@@ -7,7 +7,7 @@ from torch import Tensor
7
8
import vllm._custom_ops as ops
9
from vllm.platforms import current_platform
10
-from vllm.utils import cdiv
10
+from vllm.utils.math_utils import cdiv
11
12
13
def ref_mla(
tests/kernels/attention/test_triton_decode_attention.py
+1
-1
@@ -5,7 +5,7 @@ import pytest
5
import torch
6
7
from vllm.attention.ops.triton_decode_attention import decode_attention_fwd
8
-from vllm.utils import cdiv
8
+from vllm.utils.math_utils import cdiv
9
10
11
@pytest.mark.parametrize("B", [3, 5])
tests/kernels/moe/test_cutlass_grouped_gemm.py
+1
-1
@@ -13,8 +13,8 @@ from tests.kernels.moe.utils import per_token_cast_to_fp8
13
from tests.kernels.utils import baseline_scaled_mm
14
from vllm import _custom_ops as ops
15
from vllm.platforms import current_platform
16
-from vllm.utils import cdiv
16
from vllm.utils.deep_gemm import per_block_cast_to_fp8
17
+from vllm.utils.math_utils import cdiv
18
19
20
@pytest.mark.parametrize(
tests/kernels/moe/test_gpt_oss_triton_kernels.py
+1
-1
@@ -27,7 +27,7 @@ from vllm.model_executor.layers.fused_moe.gpt_oss_triton_kernels_moe import (
27
triton_kernel_moe_forward,
28
)
29
from vllm.model_executor.layers.utils import shuffle_weight
30
-from vllm.utils import round_up
30
+from vllm.utils.math_utils import round_up
31
32
33
def deshuffle(w: torch.Tensor):
tests/kernels/moe/test_moe_align_block_size.py
+1
-1
@@ -13,7 +13,7 @@ from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
13
moe_align_block_size,
14
)
15
from vllm.platforms import current_platform
16
-from vllm.utils import round_up
16
+from vllm.utils.math_utils import round_up
17
18
NUM_TOKENS = [1, 3, 256, 2256, 4096]
19
NUM_EXPERTS = [32, 160, 256, 257]
tests/kernels/moe/test_pplx_cutlass_moe.py
+1
-1
@@ -13,7 +13,7 @@ from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassBatchedExper
13
from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk
14
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEModularKernel
15
from vllm.platforms import current_platform
16
-from vllm.utils import cdiv
16
+from vllm.utils.math_utils import cdiv
17
18
from ...utils import multi_gpu_test
19
from .parallel_utils import ProcessGroupInfo, parallel_launch
tests/kernels/moe/test_pplx_moe.py
+1
-1
@@ -45,7 +45,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
45
TopKWeightAndReduceDelegate,
46
)
47
from vllm.platforms import current_platform
48
-from vllm.utils import round_up
48
+from vllm.utils.math_utils import round_up
49
50
from ...utils import multi_gpu_test
51
from .parallel_utils import ProcessGroupInfo, parallel_launch
tests/kernels/moe/test_silu_mul_fp8_quant_deep_gemm.py
+1
-1
@@ -8,7 +8,7 @@ from vllm.model_executor.layers.fused_moe.batched_deep_gemm_moe import (
8
persistent_masked_m_silu_mul_quant,
9
)
10
from vllm.platforms import current_platform
11
-from vllm.utils import cdiv
11
+from vllm.utils.math_utils import cdiv
12
13
fp8_dtype = torch.float8_e4m3fn
14
tests/kernels/moe/utils.py
+1
-1
@@ -16,8 +16,8 @@ from vllm.model_executor.layers.fused_moe.fused_batched_moe import (
16
)
17
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEModularKernel
18
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
19
-from vllm.utils import round_up
19
from vllm.utils.deep_gemm import per_block_cast_to_fp8
20
+from vllm.utils.math_utils import round_up
21
22
23
def triton_moe(
tests/kernels/quant_utils.py
+1
-1
@@ -6,7 +6,7 @@ import torch
6
7
from vllm.model_executor.layers.quantization.utils.quant_utils import group_broadcast
8
from vllm.platforms import current_platform
9
-from vllm.utils import round_up
9
+from vllm.utils.math_utils import round_up
10
11
# Using the default value (240.0) from pytorch will cause accuracy
12
# issue on dynamic quantization models. Here use 224.0 for rocm.
tests/kernels/quantization/test_cutlass_scaled_mm.py
+1
-1
@@ -13,7 +13,7 @@ import torch
13
from tests.kernels.utils import baseline_scaled_mm, opcheck, to_fp8, to_int8
14
from vllm import _custom_ops as ops
15
from vllm.platforms import current_platform
16
-from vllm.utils import cdiv
16
+from vllm.utils.math_utils import cdiv
17
18
MNK_FACTORS = [
19
(1, 256, 128),
tests/v1/attention/test_attention_backends.py
+1
-1
@@ -18,7 +18,7 @@ from tests.v1.attention.utils import (
18
from vllm.attention.backends.registry import _Backend
19
from vllm.config import ModelConfig
20
from vllm.platforms import current_platform
21
-from vllm.utils import cdiv
21
+from vllm.utils.math_utils import cdiv
22
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, is_torch_equal_or_newer
23
from vllm.v1.attention.backends.utils import (
24
CommonAttentionMetadata,
tests/v1/attention/test_mla_backends.py
+1
-1
@@ -22,7 +22,7 @@ from vllm import _custom_ops as ops
22
from vllm.attention.backends.registry import _Backend
23
from vllm.attention.ops.flashmla import is_flashmla_dense_supported
24
from vllm.config.vllm import set_current_vllm_config
25
-from vllm.utils import cdiv
25
+from vllm.utils.math_utils import cdiv
26
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
27
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
28
from vllm.v1.kv_cache_interface import FullAttentionSpec
tests/v1/attention/test_sparse_mla_backends.py
+1
-1
@@ -23,7 +23,7 @@ from tests.v1.attention.utils import (
23
from vllm import _custom_ops as ops
24
from vllm.attention.ops import flashmla
25
from vllm.model_executor.layers.linear import ColumnParallelLinear
26
-from vllm.utils import cdiv
26
+from vllm.utils.math_utils import cdiv
27
from vllm.v1.attention.backends.mla.flashmla_sparse import FlashMLASparseBackend
28
from vllm.v1.attention.backends.mla.indexer import split_prefill_chunks
29
tests/v1/engine/test_engine_args.py
+1
-1
@@ -8,7 +8,7 @@ import pytest
8
from vllm.config import VllmConfig
9
from vllm.engine.arg_utils import EngineArgs
10
from vllm.usage.usage_lib import UsageContext
11
-from vllm.utils import FlexibleArgumentParser
11
+from vllm.utils.argparse_utils import FlexibleArgumentParser
12
13
14
def test_prefix_caching_from_cli():
vllm/attention/layers/cross_attention.py
+1
-1
@@ -16,7 +16,7 @@ from vllm.attention.layer import Attention
16
from vllm.attention.selector import get_attn_backend
17
from vllm.config import CacheConfig, VllmConfig
18
from vllm.logger import init_logger
19
-from vllm.utils import cdiv
19
+from vllm.utils.math_utils import cdiv
20
from vllm.v1.attention.backends.utils import (
21
CommonAttentionMetadata,
22
subclass_attention_backend,
vllm/attention/ops/pallas_kv_cache_update.py
+1
-1
@@ -7,7 +7,7 @@ import jax
7
from jax.experimental import pallas as pl
8
from jax.experimental.pallas import tpu as pltpu
9
10
-from vllm.utils import cdiv
10
+from vllm.utils.math_utils import cdiv
11
12
13
def _kv_cache_update_kernel(
vllm/attention/ops/rocm_aiter_paged_attn.py
+1
-1
@@ -6,7 +6,7 @@ import torch
6
7
from vllm.attention.ops.paged_attn import PagedAttention
8
from vllm.platforms import current_platform
9
-from vllm.utils import cdiv
9
+from vllm.utils.math_utils import cdiv
10
11
FP8_DTYPE = current_platform.fp8_dtype()
12
vllm/benchmarks/datasets.py
+1
-1
@@ -58,7 +58,7 @@ except ImportError:
58
librosa = PlaceholderModule("librosa")
59
60
try:
61
- from vllm.utils import FlexibleArgumentParser
61
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
62
except ImportError:
63
from argparse import ArgumentParser as FlexibleArgumentParser
64
vllm/compilation/fusion_attn.py
+1
-1
@@ -19,7 +19,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
19
kStaticTensorScale,
20
)
21
from vllm.platforms import current_platform
22
-from vllm.utils import round_up
22
+from vllm.utils.math_utils import round_up
23
24
from .fusion import QUANT_OPS, empty_bf16, empty_fp32, empty_i32
25
from .fx_utils import is_func
vllm/engine/arg_utils.py
+2
-1
@@ -82,7 +82,8 @@ from vllm.transformers_utils.config import (
82
maybe_override_with_speculators,
83
)
84
from vllm.transformers_utils.utils import check_gguf_file
85
-from vllm.utils import FlexibleArgumentParser, is_in_ray_actor
85
+from vllm.utils import is_in_ray_actor
86
+from vllm.utils.argparse_utils import FlexibleArgumentParser
87
from vllm.utils.mem_constants import GiB_bytes
88
from vllm.utils.network_utils import get_ip
89
from vllm.v1.sample.logits_processor import LogitsProcessor
vllm/entrypoints/api_server.py
+2
-1
@@ -26,7 +26,8 @@ from vllm.entrypoints.utils import with_cancellation
26
from vllm.logger import init_logger
27
from vllm.sampling_params import SamplingParams
28
from vllm.usage.usage_lib import UsageContext
29
-from vllm.utils import FlexibleArgumentParser, random_uuid, set_ulimit
29
+from vllm.utils import random_uuid, set_ulimit
30
+from vllm.utils.argparse_utils import FlexibleArgumentParser
31
from vllm.version import __version__ as VLLM_VERSION
32
33
logger = init_logger("vllm.entrypoints.api_server")
vllm/entrypoints/cli/benchmark/main.py
+1
-1
@@ -9,7 +9,7 @@ from vllm.entrypoints.cli.types import CLISubcommand
9
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
10
11
if typing.TYPE_CHECKING:
12
- from vllm.utils import FlexibleArgumentParser
12
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
13
else:
14
FlexibleArgumentParser = argparse.ArgumentParser
15
vllm/entrypoints/cli/collect_env.py
+1
-1
@@ -8,7 +8,7 @@ from vllm.collect_env import main as collect_env_main
8
from vllm.entrypoints.cli.types import CLISubcommand
9
10
if typing.TYPE_CHECKING:
11
- from vllm.utils import FlexibleArgumentParser
11
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
12
else:
13
FlexibleArgumentParser = argparse.ArgumentParser
14
vllm/entrypoints/cli/main.py
+1
-1
@@ -20,7 +20,7 @@ def main():
20
import vllm.entrypoints.cli.run_batch
21
import vllm.entrypoints.cli.serve
22
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG, cli_env_setup
23
- from vllm.utils import FlexibleArgumentParser
23
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
24
25
CMD_MODULES = [
26
vllm.entrypoints.cli.openai,
vllm/entrypoints/cli/openai.py
+1
-1
@@ -13,7 +13,7 @@ from openai.types.chat import ChatCompletionMessageParam
13
from vllm.entrypoints.cli.types import CLISubcommand
14
15
if TYPE_CHECKING:
16
- from vllm.utils import FlexibleArgumentParser
16
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
17
else:
18
FlexibleArgumentParser = argparse.ArgumentParser
19
vllm/entrypoints/cli/run_batch.py
+1
-1
@@ -11,7 +11,7 @@ from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
11
from vllm.logger import init_logger
12
13
if typing.TYPE_CHECKING:
14
- from vllm.utils import FlexibleArgumentParser
14
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
15
else:
16
FlexibleArgumentParser = argparse.ArgumentParser
17
vllm/entrypoints/cli/types.py
+1
-1
@@ -5,7 +5,7 @@ import argparse
5
import typing
6
7
if typing.TYPE_CHECKING:
8
- from vllm.utils import FlexibleArgumentParser
8
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
9
else:
10
FlexibleArgumentParser = argparse.ArgumentParser
11
vllm/entrypoints/openai/cli_args.py
+1
-1
@@ -29,7 +29,7 @@ from vllm.entrypoints.constants import (
29
from vllm.entrypoints.openai.serving_models import LoRAModulePath
30
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
31
from vllm.logger import init_logger
32
-from vllm.utils import FlexibleArgumentParser
32
+from vllm.utils.argparse_utils import FlexibleArgumentParser
33
34
logger = init_logger(__name__)
35
vllm/entrypoints/openai/run_batch.py
+2
-1
@@ -32,7 +32,8 @@ from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingM
32
from vllm.entrypoints.openai.serving_score import ServingScores
33
from vllm.logger import init_logger
34
from vllm.reasoning import ReasoningParserManager
35
-from vllm.utils import FlexibleArgumentParser, random_uuid
35
+from vllm.utils import random_uuid
36
+from vllm.utils.argparse_utils import FlexibleArgumentParser
37
from vllm.version import __version__ as VLLM_VERSION
38
39
logger = init_logger(__name__)
vllm/entrypoints/utils.py
+1
-1
@@ -31,7 +31,7 @@ from vllm.entrypoints.openai.serving_models import LoRAModulePath
31
from vllm.logger import init_logger
32
from vllm.platforms import current_platform
33
from vllm.transformers_utils.tokenizers import MistralTokenizer
34
-from vllm.utils import FlexibleArgumentParser
34
+from vllm.utils.argparse_utils import FlexibleArgumentParser
35
36
logger = init_logger(__name__)
37
vllm/model_executor/layers/fla/ops/layernorm_guard.py
+1
-1
@@ -21,7 +21,7 @@ import torch.nn.functional as F
21
from einops import rearrange
22
23
from vllm.triton_utils import tl, triton
24
-from vllm.utils import cdiv, next_power_of_2
24
+from vllm.utils.math_utils import cdiv, next_power_of_2
25
26
from .utils import input_guard
27
vllm/model_executor/layers/fused_moe/config.py
+1
-1
@@ -14,9 +14,9 @@ from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
14
OCP_MX_Scheme,
15
)
16
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
17
-from vllm.utils import cdiv
17
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
18
from vllm.utils.import_utils import has_triton_kernels
19
+from vllm.utils.math_utils import cdiv
20
21
logger = init_logger(__name__)
22
vllm/model_executor/layers/fused_moe/deep_gemm_utils.py
+1
-1
@@ -10,8 +10,8 @@ import torch
10
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
11
from vllm.model_executor.layers.fused_moe.utils import count_expert_num_tokens
12
from vllm.triton_utils import tl, triton
13
-from vllm.utils import round_up
13
from vllm.utils.deep_gemm import get_mk_alignment_for_contiguous_layout
14
+from vllm.utils.math_utils import round_up
15
16
17
def expert_num_tokens_round_up_and_sum(
vllm/model_executor/layers/fused_moe/deepep_ht_prepare_finalize.py
+1
-1
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
12
TopKWeightAndReduceDelegate,
13
)
14
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
15
-from vllm.utils import round_up
15
+from vllm.utils.math_utils import round_up
16
from vllm.v1.worker.ubatching import (
17
dbo_current_ubatch_id,
18
dbo_enabled,
vllm/model_executor/layers/fused_moe/layer.py
+1
-1
@@ -55,9 +55,9 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
55
from vllm.model_executor.utils import set_weight_attrs
56
from vllm.platforms import current_platform
57
from vllm.platforms.interface import CpuArchEnum
58
-from vllm.utils import cdiv, round_up
58
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
59
from vllm.utils.import_utils import has_deep_ep, has_pplx
60
+from vllm.utils.math_utils import cdiv, round_up
61
from vllm.utils.torch_utils import current_stream, direct_register_custom_op
62
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
63
vllm/model_executor/layers/fused_moe/modular_kernel.py
+1
-1
@@ -16,7 +16,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
16
count_expert_num_tokens,
17
disable_inplace,
18
)
19
-from vllm.utils import cdiv
19
+from vllm.utils.math_utils import cdiv
20
from vllm.v1.worker.ubatching import (
21
dbo_current_ubatch_id,
22
dbo_enabled,
vllm/model_executor/layers/fused_moe/moe_align_block_size.py
+1
-1
@@ -5,7 +5,7 @@ import torch
5
6
from vllm import _custom_ops as ops
7
from vllm.triton_utils import triton
8
-from vllm.utils import round_up
8
+from vllm.utils.math_utils import round_up
9
10
11
def moe_align_block_size(
vllm/model_executor/layers/fused_moe/pplx_prepare_finalize.py
+1
-1
@@ -15,7 +15,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
15
_validate_scale_shape,
16
moe_kernel_quantize_input,
17
)
18
-from vllm.utils import cdiv, round_up
18
+from vllm.utils.math_utils import cdiv, round_up
19
20
logger = init_logger(__name__)
21
vllm/model_executor/layers/fused_moe/utils.py
+1
-1
@@ -23,8 +23,8 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
23
mxfp8_e4m3_quantize,
24
)
25
from vllm.triton_utils import tl, triton
26
-from vllm.utils import cdiv
26
from vllm.utils.flashinfer import flashinfer_fp4_quantize
27
+from vllm.utils.math_utils import cdiv
28
from vllm.utils.torch_utils import is_torch_equal_or_newer
29
30
vllm/model_executor/model_loader/tensorizer.py
+1
-1
@@ -26,7 +26,7 @@ from vllm.config import ModelConfig, ParallelConfig, VllmConfig, set_current_vll
26
from vllm.logger import init_logger
27
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
28
from vllm.platforms import current_platform
29
-from vllm.utils import FlexibleArgumentParser
29
+from vllm.utils.argparse_utils import FlexibleArgumentParser
30
from vllm.utils.import_utils import PlaceholderModule
31
32
if TYPE_CHECKING:
vllm/model_executor/models/config.py
+1
-1
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING
6
import vllm.envs as envs
7
from vllm.logger import init_logger
8
from vllm.model_executor.models import ModelRegistry
9
-from vllm.utils import cdiv, round_up
9
+from vllm.utils.math_utils import cdiv, round_up
10
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
11
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
12
vllm/model_executor/models/gpt_oss.py
+1
-1
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
33
from vllm.model_executor.models.utils import sequence_parallel_chunk
34
from vllm.sequence import IntermediateTensors
35
-from vllm.utils import cdiv
35
+from vllm.utils.math_utils import cdiv
36
37
from .interfaces import SupportsEagle3, SupportsLoRA, SupportsPP
38
from .utils import (
vllm/model_executor/models/utils.py
+1
-1
@@ -22,7 +22,7 @@ from vllm.logger import init_logger
22
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
from vllm.multimodal import NestedTensors
24
from vllm.sequence import IntermediateTensors
25
-from vllm.utils import cdiv
25
+from vllm.utils.math_utils import cdiv
26
from vllm.utils.platform_utils import (
27
is_pin_memory_available,
28
is_uva_available,
vllm/platforms/interface.py
+1
-1
@@ -22,7 +22,7 @@ if TYPE_CHECKING:
22
from vllm.inputs import ProcessorInputs, PromptType
23
from vllm.pooling_params import PoolingParams
24
from vllm.sampling_params import SamplingParams
25
- from vllm.utils import FlexibleArgumentParser
25
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
26
else:
27
FlexibleArgumentParser = object
28
vllm/utils/__init__.py
+48
-516
@@ -6,42 +6,56 @@ import datetime
6
import enum
7
import getpass
8
import inspect
9
-import json
9
import multiprocessing
10
import os
11
import signal
12
import sys
13
import tempfile
15
-import textwrap
14
import threading
15
import traceback
16
import uuid
17
import warnings
18
import weakref
21
-from argparse import (
22
- Action,
23
- ArgumentDefaultsHelpFormatter,
24
- ArgumentParser,
25
- ArgumentTypeError,
26
- RawDescriptionHelpFormatter,
27
- _ArgumentGroup,
28
-)
29
-from collections import defaultdict
19
from collections.abc import Callable
31
-from functools import partial, wraps
20
+from functools import cache, partial, wraps
21
from typing import TYPE_CHECKING, Any, TypeVar
22
23
import cloudpickle
24
import psutil
36
-import regex as re
25
import torch
38
-import yaml
26
27
import vllm.envs as envs
28
from vllm.logger import enable_trace_function_call, init_logger
29
from vllm.ray.lazy_utils import is_in_ray_actor
30
+
31
+# Import utilities from specialized modules for backward compatibility
32
+from vllm.utils.argparse_utils import (
33
+ FlexibleArgumentParser,
34
+ SortedHelpFormatter,
35
+ StoreBoolean,
36
+)
37
+from vllm.utils.math_utils import (
38
+ cdiv,
39
+ next_power_of_2,
40
+ prev_power_of_2,
41
+ round_down,
42
+ round_up,
43
+)
44
from vllm.utils.platform_utils import cuda_is_initialized, xpu_is_initialized
45
46
+__all__ = [
47
+ # Argparse utilities
48
+ "FlexibleArgumentParser",
49
+ "SortedHelpFormatter",
50
+ "StoreBoolean",
51
+ # Math utilities
52
+ "cdiv",
53
+ "next_power_of_2",
54
+ "prev_power_of_2",
55
+ "round_down",
56
+ "round_up",
57
+]
58
+
59
_DEPRECATED_MAPPINGS = {
60
"cprofile": "profiling",
61
"cprofile_context": "profiling",
@@ -139,31 +153,31 @@ def random_uuid() -> str:
153
return str(uuid.uuid4().hex)
154
155
142
-def cdiv(a: int, b: int) -> int:
143
- """Ceiling division."""
144
- return -(a // -b)
145
-
146
-
147
-def next_power_of_2(n) -> int:
148
- """The next power of 2 (inclusive)"""
149
- if n < 1:
150
- return 1
151
- return 1 << (n - 1).bit_length()
152
-
156
+def update_environment_variables(envs: dict[str, str]):
157
+ for k, v in envs.items():
158
+ if k in os.environ and os.environ[k] != v:
159
+ logger.warning(
160
+ "Overwriting environment variable %s from '%s' to '%s'",
161
+ k,
162
+ os.environ[k],
163
+ v,
164
+ )
165
+ os.environ[k] = v
166
154
-def prev_power_of_2(n: int) -> int:
155
- """The previous power of 2 (inclusive)"""
156
- if n <= 0:
157
- return 0
158
- return 1 << (n.bit_length() - 1)
167
168
+@cache
169
+def is_pin_memory_available() -> bool:
170
+ from vllm.platforms import current_platform
171
161
-def round_up(x: int, y: int) -> int:
162
- return ((x + y - 1) // y) * y
172
+ return current_platform.is_pin_memory_available()
173
174
165
-def round_down(x: int, y: int) -> int:
166
- return (x // y) * y
175
+@cache
176
+def is_uva_available() -> bool:
177
+ """Check if Unified Virtual Addressing (UVA) is available."""
178
+ # UVA requires pinned memory.
179
+ # TODO: Add more requirements for UVA if needed.
180
+ return is_pin_memory_available()
181
182
183
# TODO: This function can be removed if transformer_modules classes are
@@ -214,488 +228,6 @@ def weak_bind(
228
return weak_bound
229
230
217
-class StoreBoolean(Action):
218
- def __call__(self, parser, namespace, values, option_string=None):
219
- if values.lower() == "true":
220
- setattr(namespace, self.dest, True)
221
- elif values.lower() == "false":
222
- setattr(namespace, self.dest, False)
223
- else:
224
- raise ValueError(
225
- f"Invalid boolean value: {values}. Expected 'true' or 'false'."
226
- )
227
-
228
-
229
-class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter):
230
- """SortedHelpFormatter that sorts arguments by their option strings."""
231
-
232
- def _split_lines(self, text, width):
233
- """
234
- 1. Sentences split across lines have their single newlines removed.
235
- 2. Paragraphs and explicit newlines are split into separate lines.
236
- 3. Each line is wrapped to the specified width (width of terminal).
237
- """
238
- # The patterns also include whitespace after the newline
239
- single_newline = re.compile(r"(?<!\n)\n(?!\n)\s*")
240
- multiple_newlines = re.compile(r"\n{2,}\s*")
241
- text = single_newline.sub(" ", text)
242
- lines = re.split(multiple_newlines, text)
243
- return sum([textwrap.wrap(line, width) for line in lines], [])
244
-
245
- def add_arguments(self, actions):
246
- actions = sorted(actions, key=lambda x: x.option_strings)
247
- super().add_arguments(actions)
248
-
249
-
250
-class FlexibleArgumentParser(ArgumentParser):
251
- """ArgumentParser that allows both underscore and dash in names."""
252
-
253
- _deprecated: set[Action] = set()
254
- _json_tip: str = (
255
- "When passing JSON CLI arguments, the following sets of arguments "
256
- "are equivalent:\n"
257
- ' --json-arg \'{"key1": "value1", "key2": {"key3": "value2"}}\'\n'
258
- " --json-arg.key1 value1 --json-arg.key2.key3 value2\n\n"
259
- "Additionally, list elements can be passed individually using +:\n"
260
- ' --json-arg \'{"key4": ["value3", "value4", "value5"]}\'\n'
261
- " --json-arg.key4+ value3 --json-arg.key4+='value4,value5'\n\n"
262
- )
263
- _search_keyword: str | None = None
264
-
265
- def __init__(self, *args, **kwargs):
266
- # Set the default "formatter_class" to SortedHelpFormatter
267
- if "formatter_class" not in kwargs:
268
- kwargs["formatter_class"] = SortedHelpFormatter
269
- # Pop kwarg "add_json_tip" to control whether to add the JSON tip
270
- self.add_json_tip = kwargs.pop("add_json_tip", True)
271
- super().__init__(*args, **kwargs)
272
-
273
- if sys.version_info < (3, 13):
274
- # Enable the deprecated kwarg for Python 3.12 and below
275
-
276
- def parse_known_args(self, args=None, namespace=None):
277
- if args is not None and "--disable-log-requests" in args:
278
- # Special case warning because the warning below won't trigger
279
- # if –-disable-log-requests because its value is default.
280
- logger.warning_once(
281
- "argument '--disable-log-requests' is deprecated and "
282
- "replaced with '--enable-log-requests'. This will be "
283
- "removed in v0.12.0."
284
- )
285
- namespace, args = super().parse_known_args(args, namespace)
286
- for action in FlexibleArgumentParser._deprecated:
287
- if (
288
- hasattr(namespace, dest := action.dest)
289
- and getattr(namespace, dest) != action.default
290
- ):
291
- logger.warning_once("argument '%s' is deprecated", dest)
292
- return namespace, args
293
-
294
- def add_argument(self, *args, **kwargs):
295
- deprecated = kwargs.pop("deprecated", False)
296
- action = super().add_argument(*args, **kwargs)
297
- if deprecated:
298
- FlexibleArgumentParser._deprecated.add(action)
299
- return action
300
-
301
- class _FlexibleArgumentGroup(_ArgumentGroup):
302
- def add_argument(self, *args, **kwargs):
303
- deprecated = kwargs.pop("deprecated", False)
304
- action = super().add_argument(*args, **kwargs)
305
- if deprecated:
306
- FlexibleArgumentParser._deprecated.add(action)
307
- return action
308
-
309
- def add_argument_group(self, *args, **kwargs):
310
- group = self._FlexibleArgumentGroup(self, *args, **kwargs)
311
- self._action_groups.append(group)
312
- return group
313
-
314
- def format_help(self):
315
- # Only use custom help formatting for bottom level parsers
316
- if self._subparsers is not None:
317
- return super().format_help()
318
-
319
- formatter = self._get_formatter()
320
-
321
- # Handle keyword search of the args
322
- if (search_keyword := self._search_keyword) is not None:
323
- # Normalise the search keyword
324
- search_keyword = search_keyword.lower().replace("_", "-")
325
- # Return full help if searching for 'all'
326
- if search_keyword == "all":
327
- self.epilog = self._json_tip
328
- return super().format_help()
329
-
330
- # Return group help if searching for a group title
331
- for group in self._action_groups:
332
- if group.title and group.title.lower() == search_keyword:
333
- formatter.start_section(group.title)
334
- formatter.add_text(group.description)
335
- formatter.add_arguments(group._group_actions)
336
- formatter.end_section()
337
- formatter.add_text(self._json_tip)
338
- return formatter.format_help()
339
-
340
- # Return matched args if searching for an arg name
341
- matched_actions = []
342
- for group in self._action_groups:
343
- for action in group._group_actions:
344
- # search option name
345
- if any(
346
- search_keyword in opt.lower() for opt in action.option_strings
347
- ):
348
- matched_actions.append(action)
349
- if matched_actions:
350
- formatter.start_section(f"Arguments matching '{search_keyword}'")
351
- formatter.add_arguments(matched_actions)
352
- formatter.end_section()
353
- formatter.add_text(self._json_tip)
354
- return formatter.format_help()
355
-
356
- # No match found
357
- formatter.add_text(
358
- f"No group or arguments matching '{search_keyword}'.\n"
359
- "Use '--help' to see available groups or "
360
- "'--help=all' to see all available parameters."
361
- )
362
- return formatter.format_help()
363
-
364
- # usage
365
- formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups)
366
-
367
- # description
368
- formatter.add_text(self.description)
369
-
370
- # positionals, optionals and user-defined groups
371
- formatter.start_section("Config Groups")
372
- config_groups = ""
373
- for group in self._action_groups:
374
- if not group._group_actions:
375
- continue
376
- title = group.title
377
- description = group.description or ""
378
- config_groups += f"{title: <24}{description}\n"
379
- formatter.add_text(config_groups)
380
- formatter.end_section()
381
-
382
- # epilog
383
- formatter.add_text(self.epilog)
384
-
385
- # determine help from format above
386
- return formatter.format_help()
387
-
388
- def parse_args( # type: ignore[override]
389
- self,
390
- args: list[str] | None = None,
391
- namespace: Namespace | None = None,
392
- ):
393
- if args is None:
394
- args = sys.argv[1:]
395
-
396
- # Check for --model in command line arguments first
397
- if args and args[0] == "serve":
398
- try:
399
- model_idx = next(
400
- i
401
- for i, arg in enumerate(args)
402
- if arg == "--model" or arg.startswith("--model=")
403
- )
404
- logger.warning(
405
- "With `vllm serve`, you should provide the model as a "
406
- "positional argument or in a config file instead of via "
407
- "the `--model` option. "
408
- "The `--model` option will be removed in v0.13."
409
- )
410
-
411
- if args[model_idx] == "--model":
412
- model_tag = args[model_idx + 1]
413
- rest_start_idx = model_idx + 2
414
- else:
415
- model_tag = args[model_idx].removeprefix("--model=")
416
- rest_start_idx = model_idx + 1
417
-
418
- # Move <model> to the front, e,g:
419
- # [Before]
420
- # vllm serve -tp 2 --model <model> --enforce-eager --port 8001
421
- # [After]
422
- # vllm serve <model> -tp 2 --enforce-eager --port 8001
423
- args = [
424
- "serve",
425
- model_tag,
426
- *args[1:model_idx],
427
- *args[rest_start_idx:],
428
- ]
429
- print("args", args)
430
- except StopIteration:
431
- pass
432
-
433
- if "--config" in args:
434
- args = self._pull_args_from_config(args)
435
-
436
- def repl(match: re.Match) -> str:
437
- """Replaces underscores with dashes in the matched string."""
438
- return match.group(0).replace("_", "-")
439
-
440
- # Everything between the first -- and the first .
441
- pattern = re.compile(r"(?<=--)[^\.]*")
442
-
443
- # Convert underscores to dashes and vice versa in argument names
444
- processed_args = list[str]()
445
- for i, arg in enumerate(args):
446
- if arg.startswith("--help="):
447
- FlexibleArgumentParser._search_keyword = arg.split("=", 1)[-1].lower()
448
- processed_args.append("--help")
449
- elif arg.startswith("--"):
450
- if "=" in arg:
451
- key, value = arg.split("=", 1)
452
- key = pattern.sub(repl, key, count=1)
453
- processed_args.append(f"{key}={value}")
454
- else:
455
- key = pattern.sub(repl, arg, count=1)
456
- processed_args.append(key)
457
- elif arg.startswith("-O") and arg != "-O" and arg[2] != ".":
458
- # allow -O flag to be used without space, e.g. -O3 or -Odecode
459
- # -O.<...> handled later
460
- # also handle -O=<mode> here
461
- mode = arg[3:] if arg[2] == "=" else arg[2:]
462
- processed_args.append(f"-O.mode={mode}")
463
- elif (
464
- arg == "-O"
465
- and i + 1 < len(args)
466
- and args[i + 1] in {"0", "1", "2", "3"}
467
- ):
468
- # Convert -O <n> to -O.mode <n>
469
- processed_args.append("-O.mode")
470
- else:
471
- processed_args.append(arg)
472
-
473
- def create_nested_dict(keys: list[str], value: str) -> dict[str, Any]:
474
- """Creates a nested dictionary from a list of keys and a value.
475
-
476
- For example, `keys = ["a", "b", "c"]` and `value = 1` will create:
477
- `{"a": {"b": {"c": 1}}}`
478
- """
479
- nested_dict: Any = value
480
- for key in reversed(keys):
481
- nested_dict = {key: nested_dict}
482
- return nested_dict
483
-
484
- def recursive_dict_update(
485
- original: dict[str, Any],
486
- update: dict[str, Any],
487
- ) -> set[str]:
488
- """Recursively updates a dictionary with another dictionary.
489
- Returns a set of duplicate keys that were overwritten.
490
- """
491
- duplicates = set[str]()
492
- for k, v in update.items():
493
- if isinstance(v, dict) and isinstance(original.get(k), dict):
494
- nested_duplicates = recursive_dict_update(original[k], v)
495
- duplicates |= {f"{k}.{d}" for d in nested_duplicates}
496
- elif isinstance(v, list) and isinstance(original.get(k), list):
497
- original[k] += v
498
- else:
499
- if k in original:
500
- duplicates.add(k)
501
- original[k] = v
502
- return duplicates
503
-
504
- delete = set[int]()
505
- dict_args = defaultdict[str, dict[str, Any]](dict)
506
- duplicates = set[str]()
507
- for i, processed_arg in enumerate(processed_args):
508
- if i in delete: # skip if value from previous arg
509
- continue
510
-
511
- if processed_arg.startswith("-") and "." in processed_arg:
512
- if "=" in processed_arg:
513
- processed_arg, value_str = processed_arg.split("=", 1)
514
- if "." not in processed_arg:
515
- # False positive, '.' was only in the value
516
- continue
517
- else:
518
- value_str = processed_args[i + 1]
519
- delete.add(i + 1)
520
-
521
- if processed_arg.endswith("+"):
522
- processed_arg = processed_arg[:-1]
523
- value_str = json.dumps(list(value_str.split(",")))
524
-
525
- key, *keys = processed_arg.split(".")
526
- try:
527
- value = json.loads(value_str)
528
- except json.decoder.JSONDecodeError:
529
- value = value_str
530
-
531
- # Merge all values with the same key into a single dict
532
- arg_dict = create_nested_dict(keys, value)
533
- arg_duplicates = recursive_dict_update(dict_args[key], arg_dict)
534
- duplicates |= {f"{key}.{d}" for d in arg_duplicates}
535
- delete.add(i)
536
- # Filter out the dict args we set to None
537
- processed_args = [a for i, a in enumerate(processed_args) if i not in delete]
538
- if duplicates:
539
- logger.warning("Found duplicate keys %s", ", ".join(duplicates))
540
-
541
- # Add the dict args back as if they were originally passed as JSON
542
- for dict_arg, dict_value in dict_args.items():
543
- processed_args.append(dict_arg)
544
- processed_args.append(json.dumps(dict_value))
545
-
546
- return super().parse_args(processed_args, namespace)
547
-
548
- def check_port(self, value):
549
- try:
550
- value = int(value)
551
- except ValueError:
552
- msg = "Port must be an integer"
553
- raise ArgumentTypeError(msg) from None
554
-
555
- if not (1024 <= value <= 65535):
556
- raise ArgumentTypeError("Port must be between 1024 and 65535")
557
-
558
- return value
559
-
560
- def _pull_args_from_config(self, args: list[str]) -> list[str]:
561
- """Method to pull arguments specified in the config file
562
- into the command-line args variable.
563
-
564
- The arguments in config file will be inserted between
565
- the argument list.
566
-
567
- example:
568
- ```yaml
569
- port: 12323
570
- tensor-parallel-size: 4
571
- ```
572
- ```python
573
- $: vllm {serve,chat,complete} "facebook/opt-12B" \
574
- --config config.yaml -tp 2
575
- $: args = [
576
- "serve,chat,complete",
577
- "facebook/opt-12B",
578
- '--config', 'config.yaml',
579
- '-tp', '2'
580
- ]
581
- $: args = [
582
- "serve,chat,complete",
583
- "facebook/opt-12B",
584
- '--port', '12323',
585
- '--tensor-parallel-size', '4',
586
- '-tp', '2'
587
- ]
588
- ```
589
-
590
- Please note how the config args are inserted after the sub command.
591
- this way the order of priorities is maintained when these are args
592
- parsed by super().
593
- """
594
- assert args.count("--config") <= 1, "More than one config file specified!"
595
-
596
- index = args.index("--config")
597
- if index == len(args) - 1:
598
- raise ValueError(
599
- "No config file specified! \
600
- Please check your command-line arguments."
601
- )
602
-
603
- file_path = args[index + 1]
604
-
605
- config_args = self.load_config_file(file_path)
606
-
607
- # 0th index might be the sub command {serve,chat,complete,...}
608
- # optionally followed by model_tag (only for serve)
609
- # followed by config args
610
- # followed by rest of cli args.
611
- # maintaining this order will enforce the precedence
612
- # of cli > config > defaults
613
- if args[0].startswith("-"):
614
- # No sub command (e.g., api_server entry point)
615
- args = config_args + args[0:index] + args[index + 2 :]
616
- elif args[0] == "serve":
617
- model_in_cli = len(args) > 1 and not args[1].startswith("-")
618
- model_in_config = any(arg == "--model" for arg in config_args)
619
-
620
- if not model_in_cli and not model_in_config:
621
- raise ValueError(
622
- "No model specified! Please specify model either "
623
- "as a positional argument or in a config file."
624
- )
625
-
626
- if model_in_cli:
627
- # Model specified as positional arg, keep CLI version
628
- args = (
629
- [args[0]]
630
- + [args[1]]
631
- + config_args
632
- + args[2:index]
633
- + args[index + 2 :]
634
- )
635
- else:
636
- # No model in CLI, use config if available
637
- args = [args[0]] + config_args + args[1:index] + args[index + 2 :]
638
- else:
639
- args = [args[0]] + config_args + args[1:index] + args[index + 2 :]
640
-
641
- return args
642
-
643
- def load_config_file(self, file_path: str) -> list[str]:
644
- """Loads a yaml file and returns the key value pairs as a
645
- flattened list with argparse like pattern
646
- ```yaml
647
- port: 12323
648
- tensor-parallel-size: 4
649
- ```
650
- returns:
651
- processed_args: list[str] = [
652
- '--port': '12323',
653
- '--tensor-parallel-size': '4'
654
- ]
655
- """
656
- extension: str = file_path.split(".")[-1]
657
- if extension not in ("yaml", "yml"):
658
- raise ValueError(
659
- "Config file must be of a yaml/yml type.\
660
- %s supplied",
661
- extension,
662
- )
663
-
664
- # only expecting a flat dictionary of atomic types
665
- processed_args: list[str] = []
666
-
667
- config: dict[str, int | str] = {}
668
- try:
669
- with open(file_path) as config_file:
670
- config = yaml.safe_load(config_file)
671
- except Exception as ex:
672
- logger.error(
673
- "Unable to read the config file at %s. \
674
- Make sure path is correct",
675
- file_path,
676
- )
677
- raise ex
678
-
679
- store_boolean_arguments = [
680
- action.dest for action in self._actions if isinstance(action, StoreBoolean)
681
- ]
682
-
683
- for key, value in config.items():
684
- if isinstance(value, bool) and key not in store_boolean_arguments:
685
- if value:
686
- processed_args.append("--" + key)
687
- elif isinstance(value, list):
688
- if value:
689
- processed_args.append("--" + key)
690
- for item in value:
691
- processed_args.append(str(item))
692
- else:
693
- processed_args.append("--" + key)
694
- processed_args.append(str(value))
695
-
696
- return processed_args
697
-
698
-
231
class AtomicCounter:
232
"""An atomic, thread-safe counter"""
233
vllm/utils/argparse_utils.py
new
+507
@@ -0,0 +1,507 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Argument parsing utilities for vLLM."""
4
+
5
+import json
6
+import sys
7
+import textwrap
8
+from argparse import (
9
+ Action,
10
+ ArgumentDefaultsHelpFormatter,
11
+ ArgumentParser,
12
+ ArgumentTypeError,
13
+ RawDescriptionHelpFormatter,
14
+ _ArgumentGroup,
15
+)
16
+from collections import defaultdict
17
+from typing import TYPE_CHECKING, Any
18
+
19
+import regex as re
20
+import yaml
21
+
22
+from vllm.logger import init_logger
23
+
24
+if TYPE_CHECKING:
25
+ from argparse import Namespace
26
+else:
27
+ Namespace = object
28
+
29
+logger = init_logger(__name__)
30
+
31
+
32
+class StoreBoolean(Action):
33
+ def __call__(self, parser, namespace, values, option_string=None):
34
+ if values.lower() == "true":
35
+ setattr(namespace, self.dest, True)
36
+ elif values.lower() == "false":
37
+ setattr(namespace, self.dest, False)
38
+ else:
39
+ raise ValueError(
40
+ f"Invalid boolean value: {values}. Expected 'true' or 'false'."
41
+ )
42
+
43
+
44
+class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter):
45
+ """SortedHelpFormatter that sorts arguments by their option strings."""
46
+
47
+ def _split_lines(self, text, width):
48
+ """
49
+ 1. Sentences split across lines have their single newlines removed.
50
+ 2. Paragraphs and explicit newlines are split into separate lines.
51
+ 3. Each line is wrapped to the specified width (width of terminal).
52
+ """
53
+ # The patterns also include whitespace after the newline
54
+ single_newline = re.compile(r"(?<!\n)\n(?!\n)\s*")
55
+ multiple_newlines = re.compile(r"\n{2,}\s*")
56
+ text = single_newline.sub(" ", text)
57
+ lines = re.split(multiple_newlines, text)
58
+ return sum([textwrap.wrap(line, width) for line in lines], [])
59
+
60
+ def add_arguments(self, actions):
61
+ actions = sorted(actions, key=lambda x: x.option_strings)
62
+ super().add_arguments(actions)
63
+
64
+
65
+class FlexibleArgumentParser(ArgumentParser):
66
+ """ArgumentParser that allows both underscore and dash in names."""
67
+
68
+ _deprecated: set[Action] = set()
69
+ _json_tip: str = (
70
+ "When passing JSON CLI arguments, the following sets of arguments "
71
+ "are equivalent:\n"
72
+ ' --json-arg \'{"key1": "value1", "key2": {"key3": "value2"}}\'\n'
73
+ " --json-arg.key1 value1 --json-arg.key2.key3 value2\n\n"
74
+ "Additionally, list elements can be passed individually using +:\n"
75
+ ' --json-arg \'{"key4": ["value3", "value4", "value5"]}\'\n'
76
+ " --json-arg.key4+ value3 --json-arg.key4+='value4,value5'\n\n"
77
+ )
78
+ _search_keyword: str | None = None
79
+
80
+ def __init__(self, *args, **kwargs):
81
+ # Set the default "formatter_class" to SortedHelpFormatter
82
+ if "formatter_class" not in kwargs:
83
+ kwargs["formatter_class"] = SortedHelpFormatter
84
+ # Pop kwarg "add_json_tip" to control whether to add the JSON tip
85
+ self.add_json_tip = kwargs.pop("add_json_tip", True)
86
+ super().__init__(*args, **kwargs)
87
+
88
+ if sys.version_info < (3, 13):
89
+ # Enable the deprecated kwarg for Python 3.12 and below
90
+
91
+ def parse_known_args(self, args=None, namespace=None):
92
+ if args is not None and "--disable-log-requests" in args:
93
+ # Special case warning because the warning below won't trigger
94
+ # if –-disable-log-requests because its value is default.
95
+ logger.warning_once(
96
+ "argument '--disable-log-requests' is deprecated and "
97
+ "replaced with '--enable-log-requests'. This will be "
98
+ "removed in v0.12.0."
99
+ )
100
+ namespace, args = super().parse_known_args(args, namespace)
101
+ for action in FlexibleArgumentParser._deprecated:
102
+ if (
103
+ hasattr(namespace, dest := action.dest)
104
+ and getattr(namespace, dest) != action.default
105
+ ):
106
+ logger.warning_once("argument '%s' is deprecated", dest)
107
+ return namespace, args
108
+
109
+ def add_argument(self, *args, **kwargs):
110
+ deprecated = kwargs.pop("deprecated", False)
111
+ action = super().add_argument(*args, **kwargs)
112
+ if deprecated:
113
+ FlexibleArgumentParser._deprecated.add(action)
114
+ return action
115
+
116
+ class _FlexibleArgumentGroup(_ArgumentGroup):
117
+ def add_argument(self, *args, **kwargs):
118
+ deprecated = kwargs.pop("deprecated", False)
119
+ action = super().add_argument(*args, **kwargs)
120
+ if deprecated:
121
+ FlexibleArgumentParser._deprecated.add(action)
122
+ return action
123
+
124
+ def add_argument_group(self, *args, **kwargs):
125
+ group = self._FlexibleArgumentGroup(self, *args, **kwargs)
126
+ self._action_groups.append(group)
127
+ return group
128
+
129
+ def format_help(self):
130
+ # Only use custom help formatting for bottom level parsers
131
+ if self._subparsers is not None:
132
+ return super().format_help()
133
+
134
+ formatter = self._get_formatter()
135
+
136
+ # Handle keyword search of the args
137
+ if (search_keyword := self._search_keyword) is not None:
138
+ # Normalise the search keyword
139
+ search_keyword = search_keyword.lower().replace("_", "-")
140
+ # Return full help if searching for 'all'
141
+ if search_keyword == "all":
142
+ self.epilog = self._json_tip
143
+ return super().format_help()
144
+
145
+ # Return group help if searching for a group title
146
+ for group in self._action_groups:
147
+ if group.title and group.title.lower() == search_keyword:
148
+ formatter.start_section(group.title)
149
+ formatter.add_text(group.description)
150
+ formatter.add_arguments(group._group_actions)
151
+ formatter.end_section()
152
+ formatter.add_text(self._json_tip)
153
+ return formatter.format_help()
154
+
155
+ # Return matched args if searching for an arg name
156
+ matched_actions = []
157
+ for group in self._action_groups:
158
+ for action in group._group_actions:
159
+ # search option name
160
+ if any(
161
+ search_keyword in opt.lower() for opt in action.option_strings
162
+ ):
163
+ matched_actions.append(action)
164
+ if matched_actions:
165
+ formatter.start_section(f"Arguments matching '{search_keyword}'")
166
+ formatter.add_arguments(matched_actions)
167
+ formatter.end_section()
168
+ formatter.add_text(self._json_tip)
169
+ return formatter.format_help()
170
+
171
+ # No match found
172
+ formatter.add_text(
173
+ f"No group or arguments matching '{search_keyword}'.\n"
174
+ "Use '--help' to see available groups or "
175
+ "'--help=all' to see all available parameters."
176
+ )
177
+ return formatter.format_help()
178
+
179
+ # usage
180
+ formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups)
181
+
182
+ # description
183
+ formatter.add_text(self.description)
184
+
185
+ # positionals, optionals and user-defined groups
186
+ formatter.start_section("Config Groups")
187
+ config_groups = ""
188
+ for group in self._action_groups:
189
+ if not group._group_actions:
190
+ continue
191
+ title = group.title
192
+ description = group.description or ""
193
+ config_groups += f"{title: <24}{description}\n"
194
+ formatter.add_text(config_groups)
195
+ formatter.end_section()
196
+
197
+ # epilog
198
+ formatter.add_text(self.epilog)
199
+
200
+ # determine help from format above
201
+ return formatter.format_help()
202
+
203
+ def parse_args( # type: ignore[override]
204
+ self,
205
+ args: list[str] | None = None,
206
+ namespace: Namespace | None = None,
207
+ ):
208
+ if args is None:
209
+ args = sys.argv[1:]
210
+
211
+ # Check for --model in command line arguments first
212
+ if args and args[0] == "serve":
213
+ try:
214
+ model_idx = next(
215
+ i
216
+ for i, arg in enumerate(args)
217
+ if arg == "--model" or arg.startswith("--model=")
218
+ )
219
+ logger.warning(
220
+ "With `vllm serve`, you should provide the model as a "
221
+ "positional argument or in a config file instead of via "
222
+ "the `--model` option. "
223
+ "The `--model` option will be removed in v0.13."
224
+ )
225
+
226
+ if args[model_idx] == "--model":
227
+ model_tag = args[model_idx + 1]
228
+ rest_start_idx = model_idx + 2
229
+ else:
230
+ model_tag = args[model_idx].removeprefix("--model=")
231
+ rest_start_idx = model_idx + 1
232
+
233
+ # Move <model> to the front, e,g:
234
+ # [Before]
235
+ # vllm serve -tp 2 --model <model> --enforce-eager --port 8001
236
+ # [After]
237
+ # vllm serve <model> -tp 2 --enforce-eager --port 8001
238
+ args = [
239
+ "serve",
240
+ model_tag,
241
+ *args[1:model_idx],
242
+ *args[rest_start_idx:],
243
+ ]
244
+ except StopIteration:
245
+ pass
246
+
247
+ if "--config" in args:
248
+ args = self._pull_args_from_config(args)
249
+
250
+ def repl(match: re.Match) -> str:
251
+ """Replaces underscores with dashes in the matched string."""
252
+ return match.group(0).replace("_", "-")
253
+
254
+ # Everything between the first -- and the first .
255
+ pattern = re.compile(r"(?<=--)[^\.]*")
256
+
257
+ # Convert underscores to dashes and vice versa in argument names
258
+ processed_args = list[str]()
259
+ for i, arg in enumerate(args):
260
+ if arg.startswith("--help="):
261
+ FlexibleArgumentParser._search_keyword = arg.split("=", 1)[-1].lower()
262
+ processed_args.append("--help")
263
+ elif arg.startswith("--"):
264
+ if "=" in arg:
265
+ key, value = arg.split("=", 1)
266
+ key = pattern.sub(repl, key, count=1)
267
+ processed_args.append(f"{key}={value}")
268
+ else:
269
+ key = pattern.sub(repl, arg, count=1)
270
+ processed_args.append(key)
271
+ elif arg.startswith("-O") and arg != "-O" and arg[2] != ".":
272
+ # allow -O flag to be used without space, e.g. -O3 or -Odecode
273
+ # -O.<...> handled later
274
+ # also handle -O=<mode> here
275
+ mode = arg[3:] if arg[2] == "=" else arg[2:]
276
+ processed_args.append(f"-O.mode={mode}")
277
+ elif (
278
+ arg == "-O"
279
+ and i + 1 < len(args)
280
+ and args[i + 1] in {"0", "1", "2", "3"}
281
+ ):
282
+ # Convert -O <n> to -O.mode <n>
283
+ processed_args.append("-O.mode")
284
+ else:
285
+ processed_args.append(arg)
286
+
287
+ def create_nested_dict(keys: list[str], value: str) -> dict[str, Any]:
288
+ """Creates a nested dictionary from a list of keys and a value.
289
+
290
+ For example, `keys = ["a", "b", "c"]` and `value = 1` will create:
291
+ `{"a": {"b": {"c": 1}}}`
292
+ """
293
+ nested_dict: Any = value
294
+ for key in reversed(keys):
295
+ nested_dict = {key: nested_dict}
296
+ return nested_dict
297
+
298
+ def recursive_dict_update(
299
+ original: dict[str, Any],
300
+ update: dict[str, Any],
301
+ ) -> set[str]:
302
+ """Recursively updates a dictionary with another dictionary.
303
+ Returns a set of duplicate keys that were overwritten.
304
+ """
305
+ duplicates = set[str]()
306
+ for k, v in update.items():
307
+ if isinstance(v, dict) and isinstance(original.get(k), dict):
308
+ nested_duplicates = recursive_dict_update(original[k], v)
309
+ duplicates |= {f"{k}.{d}" for d in nested_duplicates}
310
+ elif isinstance(v, list) and isinstance(original.get(k), list):
311
+ original[k] += v
312
+ else:
313
+ if k in original:
314
+ duplicates.add(k)
315
+ original[k] = v
316
+ return duplicates
317
+
318
+ delete = set[int]()
319
+ dict_args = defaultdict[str, dict[str, Any]](dict)
320
+ duplicates = set[str]()
321
+ for i, processed_arg in enumerate(processed_args):
322
+ if i in delete: # skip if value from previous arg
323
+ continue
324
+
325
+ if processed_arg.startswith("-") and "." in processed_arg:
326
+ if "=" in processed_arg:
327
+ processed_arg, value_str = processed_arg.split("=", 1)
328
+ if "." not in processed_arg:
329
+ # False positive, '.' was only in the value
330
+ continue
331
+ else:
332
+ value_str = processed_args[i + 1]
333
+ delete.add(i + 1)
334
+
335
+ if processed_arg.endswith("+"):
336
+ processed_arg = processed_arg[:-1]
337
+ value_str = json.dumps(list(value_str.split(",")))
338
+
339
+ key, *keys = processed_arg.split(".")
340
+ try:
341
+ value = json.loads(value_str)
342
+ except json.decoder.JSONDecodeError:
343
+ value = value_str
344
+
345
+ # Merge all values with the same key into a single dict
346
+ arg_dict = create_nested_dict(keys, value)
347
+ arg_duplicates = recursive_dict_update(dict_args[key], arg_dict)
348
+ duplicates |= {f"{key}.{d}" for d in arg_duplicates}
349
+ delete.add(i)
350
+ # Filter out the dict args we set to None
351
+ processed_args = [a for i, a in enumerate(processed_args) if i not in delete]
352
+ if duplicates:
353
+ logger.warning("Found duplicate keys %s", ", ".join(duplicates))
354
+
355
+ # Add the dict args back as if they were originally passed as JSON
356
+ for dict_arg, dict_value in dict_args.items():
357
+ processed_args.append(dict_arg)
358
+ processed_args.append(json.dumps(dict_value))
359
+
360
+ return super().parse_args(processed_args, namespace)
361
+
362
+ def check_port(self, value):
363
+ try:
364
+ value = int(value)
365
+ except ValueError:
366
+ msg = "Port must be an integer"
367
+ raise ArgumentTypeError(msg) from None
368
+
369
+ if not (1024 <= value <= 65535):
370
+ raise ArgumentTypeError("Port must be between 1024 and 65535")
371
+
372
+ return value
373
+
374
+ def _pull_args_from_config(self, args: list[str]) -> list[str]:
375
+ """Method to pull arguments specified in the config file
376
+ into the command-line args variable.
377
+
378
+ The arguments in config file will be inserted between
379
+ the argument list.
380
+
381
+ example:
382
+ ```yaml
383
+ port: 12323
384
+ tensor-parallel-size: 4
385
+ ```
386
+ ```python
387
+ $: vllm {serve,chat,complete} "facebook/opt-12B" \
388
+ --config config.yaml -tp 2
389
+ $: args = [
390
+ "serve,chat,complete",
391
+ "facebook/opt-12B",
392
+ '--config', 'config.yaml',
393
+ '-tp', '2'
394
+ ]
395
+ $: args = [
396
+ "serve,chat,complete",
397
+ "facebook/opt-12B",
398
+ '--port', '12323',
399
+ '--tensor-parallel-size', '4',
400
+ '-tp', '2'
401
+ ]
402
+ ```
403
+
404
+ Please note how the config args are inserted after the sub command.
405
+ this way the order of priorities is maintained when these are args
406
+ parsed by super().
407
+ """
408
+ assert args.count("--config") <= 1, "More than one config file specified!"
409
+
410
+ index = args.index("--config")
411
+ if index == len(args) - 1:
412
+ raise ValueError(
413
+ "No config file specified! \
414
+ Please check your command-line arguments."
415
+ )
416
+
417
+ file_path = args[index + 1]
418
+
419
+ config_args = self.load_config_file(file_path)
420
+
421
+ # 0th index might be the sub command {serve,chat,complete,...}
422
+ # optionally followed by model_tag (only for serve)
423
+ # followed by config args
424
+ # followed by rest of cli args.
425
+ # maintaining this order will enforce the precedence
426
+ # of cli > config > defaults
427
+ if args[0].startswith("-"):
428
+ # No sub command (e.g., api_server entry point)
429
+ args = config_args + args[0:index] + args[index + 2 :]
430
+ elif args[0] == "serve":
431
+ model_in_cli = len(args) > 1 and not args[1].startswith("-")
432
+ model_in_config = any(arg == "--model" for arg in config_args)
433
+
434
+ if not model_in_cli and not model_in_config:
435
+ raise ValueError(
436
+ "No model specified! Please specify model either "
437
+ "as a positional argument or in a config file."
438
+ )
439
+
440
+ if model_in_cli:
441
+ # Model specified as positional arg, keep CLI version
442
+ args = (
443
+ [args[0]]
444
+ + [args[1]]
445
+ + config_args
446
+ + args[2:index]
447
+ + args[index + 2 :]
448
+ )
449
+ else:
450
+ # No model in CLI, use config if available
451
+ args = [args[0]] + config_args + args[1:index] + args[index + 2 :]
452
+ else:
453
+ args = [args[0]] + config_args + args[1:index] + args[index + 2 :]
454
+
455
+ return args
456
+
457
+ def load_config_file(self, file_path: str) -> list[str]:
458
+ """Loads a yaml file and returns the key value pairs as a
459
+ flattened list with argparse like pattern
460
+ ```yaml
461
+ port: 12323
462
+ tensor-parallel-size: 4
463
+ ```
464
+ returns:
465
+ processed_args: list[str] = [
466
+ '--port': '12323',
467
+ '--tensor-parallel-size': '4'
468
+ ]
469
+ """
470
+ extension: str = file_path.split(".")[-1]
471
+ if extension not in ("yaml", "yml"):
472
+ raise ValueError(
473
+ f"Config file must be of a yaml/yml type. {extension} supplied"
474
+ )
475
+
476
+ # only expecting a flat dictionary of atomic types
477
+ processed_args: list[str] = []
478
+
479
+ config: dict[str, int | str] = {}
480
+ try:
481
+ with open(file_path) as config_file:
482
+ config = yaml.safe_load(config_file)
483
+ except Exception as ex:
484
+ logger.error(
485
+ "Unable to read the config file at %s. Check path correctness",
486
+ file_path,
487
+ )
488
+ raise ex
489
+
490
+ store_boolean_arguments = [
491
+ action.dest for action in self._actions if isinstance(action, StoreBoolean)
492
+ ]
493
+
494
+ for key, value in config.items():
495
+ if isinstance(value, bool) and key not in store_boolean_arguments:
496
+ if value:
497
+ processed_args.append("--" + key)
498
+ elif isinstance(value, list):
499
+ if value:
500
+ processed_args.append("--" + key)
501
+ for item in value:
502
+ processed_args.append(str(item))
503
+ else:
504
+ processed_args.append("--" + key)
505
+ processed_args.append(str(value))
506
+
507
+ return processed_args
vllm/utils/deep_gemm.py
+1
-1
@@ -16,8 +16,8 @@ import torch
16
import vllm.envs as envs
17
from vllm.logger import logger
18
from vllm.platforms import current_platform
19
-from vllm.utils import cdiv
19
from vllm.utils.import_utils import has_deep_gemm
20
+from vllm.utils.math_utils import cdiv
21
22
23
@functools.cache
vllm/utils/math_utils.py
new
+32
@@ -0,0 +1,32 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Math utility functions for vLLM."""
4
+
5
+
6
+def cdiv(a: int, b: int) -> int:
7
+ """Ceiling division."""
8
+ return -(a // -b)
9
+
10
+
11
+def next_power_of_2(n: int) -> int:
12
+ """The next power of 2 (inclusive)"""
13
+ if n < 1:
14
+ return 1
15
+ return 1 << (n - 1).bit_length()
16
+
17
+
18
+def prev_power_of_2(n: int) -> int:
19
+ """The previous power of 2 (inclusive)"""
20
+ if n <= 0:
21
+ return 0
22
+ return 1 << (n.bit_length() - 1)
23
+
24
+
25
+def round_up(x: int, y: int) -> int:
26
+ """Round up x to the nearest multiple of y."""
27
+ return ((x + y - 1) // y) * y
28
+
29
+
30
+def round_down(x: int, y: int) -> int:
31
+ """Round down x to the nearest multiple of y."""
32
+ return (x // y) * y
vllm/v1/attention/backends/flash_attn.py
+1
-1
@@ -37,7 +37,7 @@ from vllm.logger import init_logger
37
from vllm.model_executor.layers.batch_invariant import (
38
vllm_is_batch_invariant,
39
)
40
-from vllm.utils import cdiv
40
+from vllm.utils.math_utils import cdiv
41
from vllm.v1.attention.backends.utils import (
42
AttentionCGSupport,
43
AttentionMetadataBuilder,
vllm/v1/attention/backends/flashinfer.py
+1
-1
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
34
)
35
from vllm.platforms import current_platform
36
from vllm.triton_utils import tl, triton
37
-from vllm.utils import cdiv
37
from vllm.utils.flashinfer import (
38
can_use_trtllm_attention,
39
flashinfer_disable_q_quantization,
40
use_trtllm_attention,
41
)
42
+from vllm.utils.math_utils import cdiv
43
from vllm.utils.platform_utils import is_pin_memory_available
44
from vllm.v1.attention.backends.utils import (
45
AttentionCGSupport,
vllm/v1/attention/backends/flex_attention.py
+1
-1
@@ -28,7 +28,7 @@ from vllm.logger import init_logger
28
from vllm.model_executor.layers.batch_invariant import (
29
vllm_is_batch_invariant,
30
)
31
-from vllm.utils import cdiv
31
+from vllm.utils.math_utils import cdiv
32
from vllm.utils.torch_utils import is_torch_equal_or_newer
33
from vllm.v1.attention.backends.utils import (
34
AttentionMetadataBuilder,
vllm/v1/attention/backends/mamba2_attn.py
+1
-1
@@ -7,7 +7,7 @@ import torch
7
8
from vllm.attention.backends.abstract import AttentionBackend
9
from vllm.config import VllmConfig
10
-from vllm.utils import cdiv
10
+from vllm.utils.math_utils import cdiv
11
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder
12
from vllm.v1.attention.backends.utils import (
13
PAD_SLOT_ID,
vllm/v1/attention/backends/mla/common.py
+1
-1
@@ -220,8 +220,8 @@ from vllm.model_executor.layers.linear import (
220
UnquantizedLinearMethod,
221
)
222
from vllm.platforms import current_platform
223
-from vllm.utils import cdiv, round_down
223
from vllm.utils.flashinfer import has_nvidia_artifactory
224
+from vllm.utils.math_utils import cdiv, round_down
225
from vllm.v1.attention.backends.utils import (
226
AttentionMetadataBuilder,
227
CommonAttentionMetadata,
vllm/v1/attention/backends/mla/flashmla_sparse.py
+1
-1
@@ -22,7 +22,7 @@ from vllm.config import VllmConfig
22
from vllm.logger import init_logger
23
from vllm.platforms import current_platform
24
from vllm.triton_utils import tl, triton
25
-from vllm.utils import cdiv
25
+from vllm.utils.math_utils import cdiv
26
from vllm.v1.attention.backends.mla.common import MLACommonBaseImpl
27
from vllm.v1.attention.backends.utils import (
28
AttentionCGSupport,
vllm/v1/attention/backends/mla/rocm_aiter_mla.py
+1
-1
@@ -10,7 +10,7 @@ import vllm.envs as envs
10
from vllm.attention.backends.abstract import AttentionLayer
11
from vllm.attention.ops.rocm_aiter_mla import aiter_mla_decode_fwd
12
from vllm.config import VllmConfig
13
-from vllm.utils import cdiv
13
+from vllm.utils.math_utils import cdiv
14
from vllm.v1.attention.backends.mla.common import (
15
MLACommonBackend,
16
MLACommonDecodeMetadata,
vllm/v1/attention/backends/pallas.py
+1
-1
@@ -13,7 +13,7 @@ from vllm.attention.backends.abstract import (
13
)
14
from vllm.config import VllmConfig
15
from vllm.logger import init_logger
16
-from vllm.utils import cdiv, next_power_of_2
16
+from vllm.utils.math_utils import cdiv, next_power_of_2
17
18
logger = init_logger(__name__)
19
vllm/v1/attention/backends/utils.py
+1
-1
@@ -21,7 +21,7 @@ import torch
21
from typing_extensions import runtime_checkable
22
23
from vllm.config import VllmConfig, get_layers_from_vllm_config
24
-from vllm.utils import cdiv
24
+from vllm.utils.math_utils import cdiv
25
26
if TYPE_CHECKING:
27
from vllm.attention.backends.abstract import AttentionImpl
vllm/v1/core/kv_cache_utils.py
+1
-1
@@ -12,8 +12,8 @@ from typing import Any, NewType, TypeAlias
12
from vllm import envs
13
from vllm.config import VllmConfig
14
from vllm.logger import init_logger
15
-from vllm.utils import cdiv
15
from vllm.utils.hashing import sha256_cbor
16
+from vllm.utils.math_utils import cdiv
17
from vllm.utils.mem_constants import GiB_bytes
18
from vllm.v1.kv_cache_interface import (
19
ChunkedLocalAttentionSpec,
vllm/v1/core/single_type_kv_cache_manager.py
+1
-1
@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
5
from collections import defaultdict
6
from collections.abc import Sequence
7
8
-from vllm.utils import cdiv
8
+from vllm.utils.math_utils import cdiv
9
from vllm.v1.core.block_pool import BlockPool
10
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock
11
from vllm.v1.kv_cache_interface import (
vllm/v1/engine/async_llm.py
+2
-1
@@ -29,10 +29,11 @@ from vllm.tracing import init_tracer
29
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
30
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
31
from vllm.usage.usage_lib import UsageContext
32
-from vllm.utils import Device, cdiv
32
+from vllm.utils import Device
33
from vllm.utils.async_utils import cancel_task_threadsafe
34
from vllm.utils.collection_utils import as_list
35
from vllm.utils.func_utils import deprecate_kwargs
36
+from vllm.utils.math_utils import cdiv
37
from vllm.v1.engine import EngineCoreRequest
38
from vllm.v1.engine.core_client import EngineCoreClient
39
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
vllm/v1/kv_cache_interface.py
+1
-1
@@ -10,7 +10,7 @@ from typing_extensions import Self
10
11
from vllm.config import VllmConfig
12
from vllm.logger import init_logger
13
-from vllm.utils import cdiv
13
+from vllm.utils.math_utils import cdiv
14
from vllm.utils.torch_utils import get_dtype_size
15
16
logger = init_logger(__name__)
vllm/v1/worker/block_table.py
+1
-1
@@ -6,7 +6,7 @@ import torch
6
7
from vllm.distributed import get_dcp_group
8
from vllm.logger import init_logger
9
-from vllm.utils import cdiv
9
+from vllm.utils.math_utils import cdiv
10
from vllm.v1.utils import CpuGpuBuffer
11
12
logger = init_logger(__name__)
vllm/v1/worker/gpu_model_runner.py
+1
-2
@@ -70,12 +70,11 @@ from vllm.sampling_params import SamplingType
70
from vllm.sequence import IntermediateTensors
71
from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
72
from vllm.utils import (
73
- cdiv,
73
check_use_alibi,
74
length_from_prompt_token_ids_or_embeds,
76
- round_up,
75
)
76
from vllm.utils.jsontree import json_map_leaves
77
+from vllm.utils.math_utils import cdiv, round_up
78
from vllm.utils.mem_constants import GiB_bytes
79
from vllm.utils.mem_utils import DeviceMemoryProfiler
80
from vllm.utils.platform_utils import is_pin_memory_available
vllm/v1/worker/tpu_model_runner.py
+2
-1
@@ -53,7 +53,8 @@ from vllm.multimodal.inputs import (
53
from vllm.multimodal.utils import group_mm_kwargs_by_modality
54
from vllm.sequence import IntermediateTensors
55
from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
56
-from vllm.utils import LayerBlockType, cdiv, prev_power_of_2
56
+from vllm.utils import LayerBlockType
57
+from vllm.utils.math_utils import cdiv, prev_power_of_2
58
from vllm.utils.platform_utils import is_pin_memory_available
59
from vllm.v1.attention.backends.pallas import (
60
TPU_STR_DTYPE_TO_TORCH_DTYPE,
vllm/v1/worker/tpu_worker.py
+1
-1
@@ -25,7 +25,7 @@ from vllm.model_executor import set_random_seed
25
from vllm.platforms import current_platform
26
from vllm.platforms.tpu import USE_TPU_INFERENCE
27
from vllm.tasks import SupportedTask
28
-from vllm.utils import cdiv
28
+from vllm.utils.math_utils import cdiv
29
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
30
from vllm.v1.core.sched.output import SchedulerOutput
31
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig, KVCacheSpec