A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Move utils to avoid conflicts with stdlib, and move tests (#27169)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 19, 2025 at 20:20 UTC
d31f7844f8d4e312f0521a69f7aa27c94103b2db
52 files changed
+246
-237
tests/conftest.py
+1
-1
@@ -60,7 +60,7 @@ from vllm.multimodal.utils import fetch_image
60
from vllm.outputs import RequestOutput
61
from vllm.sampling_params import BeamSearchParams
62
from vllm.transformers_utils.utils import maybe_model_redirect
63
-from vllm.utils.collections import is_list_of
63
+from vllm.utils.collection_utils import is_list_of
64
from vllm.utils.torch_utils import set_default_torch_num_threads
65
66
logger = init_logger(__name__)
tests/lora/test_add_lora.py
+1
-1
@@ -12,7 +12,7 @@ from vllm.entrypoints.openai.api_server import (
12
from vllm.inputs import TextPrompt
13
from vllm.lora.request import LoRARequest
14
from vllm.sampling_params import SamplingParams
15
-from vllm.utils.asyncio import merge_async_iterators
15
+from vllm.utils.async_utils import merge_async_iterators
16
17
MODEL_PATH = "zai-org/chatglm3-6b"
18
LORA_RANK = 64
tests/models/multimodal/generation/test_common.py
+1
-1
@@ -17,7 +17,7 @@ from transformers import (
17
)
18
19
from vllm.platforms import current_platform
20
-from vllm.utils.functools import identity
20
+from vllm.utils.func_utils import identity
21
22
from ....conftest import (
23
IMAGE_ASSETS,
tests/models/multimodal/generation/vlm_utils/model_utils.py
+1
-1
@@ -25,7 +25,7 @@ from transformers import (
25
from transformers.video_utils import VideoMetadata
26
27
from vllm.logprobs import SampleLogprobs
28
-from vllm.utils.collections import is_list_of
28
+from vllm.utils.collection_utils import is_list_of
29
30
from .....conftest import HfRunner, ImageAsset, ImageTestAssets
31
from .types import RunnerOutput
tests/models/multimodal/processing/test_tensor_schema.py
+1
-1
@@ -34,7 +34,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
34
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
35
from vllm.multimodal.utils import group_mm_kwargs_by_modality
36
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
37
-from vllm.utils.collections import is_list_of
37
+from vllm.utils.collection_utils import is_list_of
38
from vllm.utils.torch_utils import set_default_torch_dtype
39
40
from ...registry import _MULTIMODAL_EXAMPLE_MODELS, HF_EXAMPLE_MODELS
tests/utils_/test_async_utils.py
+1
-1
@@ -5,7 +5,7 @@ from collections.abc import AsyncIterator
5
6
import pytest
7
8
-from vllm.utils.asyncio import merge_async_iterators
8
+from vllm.utils.async_utils import merge_async_iterators
9
10
11
async def _mock_async_iterator(idx: int):
tests/utils_/test_collection_utils.py
renamed
+1
-1
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import pytest
4
5
-from vllm.utils.collections import swap_dict_values
5
+from vllm.utils.collection_utils import swap_dict_values
6
7
8
@pytest.mark.parametrize(
tests/utils_/test_func_utils.py
+1
-1
@@ -4,7 +4,7 @@
4
5
import pytest
6
7
-from vllm.utils.functools import deprecate_kwargs, supports_kw
7
+from vllm.utils.func_utils import deprecate_kwargs, supports_kw
8
9
from ..utils import error_on_warning
10
tests/utils_/test_hashing.py
new
+25
@@ -0,0 +1,25 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import hashlib
4
+import pickle
5
+
6
+import pytest
7
+
8
+from vllm.utils.hashing import sha256
9
+
10
+
11
+@pytest.mark.parametrize("input", [(), ("abc",), (None,), (None, bool, [1, 2, 3])])
12
+def test_sha256(input: tuple):
13
+ digest = sha256(input)
14
+ assert digest is not None
15
+ assert isinstance(digest, bytes)
16
+ assert digest != b""
17
+
18
+ input_bytes = pickle.dumps(input, protocol=pickle.HIGHEST_PROTOCOL)
19
+ assert digest == hashlib.sha256(input_bytes).digest()
20
+
21
+ # hashing again, returns the same value
22
+ assert digest == sha256(input)
23
+
24
+ # hashing different input, returns different value
25
+ assert digest != sha256(input + (1,))
tests/utils_/test_mem_utils.py
new
+63
@@ -0,0 +1,63 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import torch
4
+from vllm_test_utils.monitor import monitor
5
+
6
+from vllm.utils.mem_utils import MemorySnapshot, memory_profiling
7
+
8
+from ..utils import create_new_process_for_each_test
9
+
10
+
11
+@create_new_process_for_each_test()
12
+def test_memory_profiling():
13
+ # Fake out some model loading + inference memory usage to test profiling
14
+ # Memory used by other processes will show up as cuda usage outside of torch
15
+ from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
16
+
17
+ lib = CudaRTLibrary()
18
+ # 512 MiB allocation outside of this instance
19
+ handle1 = lib.cudaMalloc(512 * 1024 * 1024)
20
+
21
+ baseline_snapshot = MemorySnapshot()
22
+
23
+ # load weights
24
+
25
+ weights = torch.randn(128, 1024, 1024, device="cuda", dtype=torch.float32)
26
+
27
+ weights_memory = 128 * 1024 * 1024 * 4 # 512 MiB
28
+
29
+ def measure_current_non_torch():
30
+ free, total = torch.cuda.mem_get_info()
31
+ current_used = total - free
32
+ current_torch = torch.cuda.memory_reserved()
33
+ current_non_torch = current_used - current_torch
34
+ return current_non_torch
35
+
36
+ with (
37
+ memory_profiling(
38
+ baseline_snapshot=baseline_snapshot, weights_memory=weights_memory
39
+ ) as result,
40
+ monitor(measure_current_non_torch) as monitored_values,
41
+ ):
42
+ # make a memory spike, 1 GiB
43
+ spike = torch.randn(256, 1024, 1024, device="cuda", dtype=torch.float32)
44
+ del spike
45
+
46
+ # Add some extra non-torch memory 256 MiB (simulate NCCL)
47
+ handle2 = lib.cudaMalloc(256 * 1024 * 1024)
48
+
49
+ # this is an analytic value, it is exact,
50
+ # we only have 256 MiB non-torch memory increase
51
+ measured_diff = monitored_values.values[-1] - monitored_values.values[0]
52
+ assert measured_diff == 256 * 1024 * 1024
53
+
54
+ # Check that the memory usage is within 5% of the expected values
55
+ # 5% tolerance is caused by cuda runtime.
56
+ # we cannot control cuda runtime in the granularity of bytes,
57
+ # which causes a small error (<10 MiB in practice)
58
+ non_torch_ratio = result.non_torch_increase / (256 * 1024 * 1024) # noqa
59
+ assert abs(non_torch_ratio - 1) <= 0.05
60
+ assert result.torch_peak_increase == 1024 * 1024 * 1024
61
+ del weights
62
+ lib.cudaFree(handle1)
63
+ lib.cudaFree(handle2)
tests/utils_/test_torch_utils.py
new
+104
@@ -0,0 +1,104 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import pytest
4
+import torch
5
+
6
+from vllm.utils.torch_utils import (
7
+ common_broadcastable_dtype,
8
+ current_stream,
9
+ is_lossless_cast,
10
+)
11
+
12
+
13
+@pytest.mark.parametrize(
14
+ ("src_dtype", "tgt_dtype", "expected_result"),
15
+ [
16
+ # Different precision_levels
17
+ (torch.bool, torch.int8, True),
18
+ (torch.bool, torch.float16, True),
19
+ (torch.bool, torch.complex32, True),
20
+ (torch.int64, torch.bool, False),
21
+ (torch.int64, torch.float16, True),
22
+ (torch.int64, torch.complex32, True),
23
+ (torch.float64, torch.bool, False),
24
+ (torch.float64, torch.int8, False),
25
+ (torch.float64, torch.complex32, True),
26
+ (torch.complex128, torch.bool, False),
27
+ (torch.complex128, torch.int8, False),
28
+ (torch.complex128, torch.float16, False),
29
+ # precision_level=0
30
+ (torch.bool, torch.bool, True),
31
+ # precision_level=1
32
+ (torch.int8, torch.int16, True),
33
+ (torch.int16, torch.int8, False),
34
+ (torch.uint8, torch.int8, False),
35
+ (torch.int8, torch.uint8, False),
36
+ # precision_level=2
37
+ (torch.float16, torch.float32, True),
38
+ (torch.float32, torch.float16, False),
39
+ (torch.bfloat16, torch.float32, True),
40
+ (torch.float32, torch.bfloat16, False),
41
+ # precision_level=3
42
+ (torch.complex32, torch.complex64, True),
43
+ (torch.complex64, torch.complex32, False),
44
+ ],
45
+)
46
+def test_is_lossless_cast(src_dtype, tgt_dtype, expected_result):
47
+ assert is_lossless_cast(src_dtype, tgt_dtype) == expected_result
48
+
49
+
50
+@pytest.mark.parametrize(
51
+ ("dtypes", "expected_result"),
52
+ [
53
+ ([torch.bool], torch.bool),
54
+ ([torch.bool, torch.int8], torch.int8),
55
+ ([torch.bool, torch.int8, torch.float16], torch.float16),
56
+ ([torch.bool, torch.int8, torch.float16, torch.complex32], torch.complex32), # noqa: E501
57
+ ],
58
+)
59
+def test_common_broadcastable_dtype(dtypes, expected_result):
60
+ assert common_broadcastable_dtype(dtypes) == expected_result
61
+
62
+
63
+def test_current_stream_multithread():
64
+ import threading
65
+
66
+ if not torch.cuda.is_available():
67
+ pytest.skip("CUDA not available")
68
+
69
+ main_default_stream = torch.cuda.current_stream()
70
+ child_stream = torch.cuda.Stream()
71
+
72
+ thread_stream_ready = threading.Event()
73
+ thread_can_exit = threading.Event()
74
+
75
+ def child_thread_func():
76
+ with torch.cuda.stream(child_stream):
77
+ thread_stream_ready.set()
78
+ thread_can_exit.wait(timeout=10)
79
+
80
+ child_thread = threading.Thread(target=child_thread_func)
81
+ child_thread.start()
82
+
83
+ try:
84
+ assert thread_stream_ready.wait(timeout=5), (
85
+ "Child thread failed to enter stream context in time"
86
+ )
87
+
88
+ main_current_stream = current_stream()
89
+
90
+ assert main_current_stream != child_stream, (
91
+ "Main thread's current_stream was contaminated by child thread"
92
+ )
93
+ assert main_current_stream == main_default_stream, (
94
+ "Main thread's current_stream is not the default stream"
95
+ )
96
+
97
+ # Notify child thread it can exit
98
+ thread_can_exit.set()
99
+
100
+ finally:
101
+ # Ensure child thread exits properly
102
+ child_thread.join(timeout=5)
103
+ if child_thread.is_alive():
104
+ pytest.fail("Child thread failed to exit properly")
tests/utils_/test_utils.py
-176
@@ -2,10 +2,8 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
# ruff: noqa
4
5
-import hashlib
5
import json
6
import os
8
-import pickle
7
import tempfile
8
from pathlib import Path
9
from unittest.mock import patch
@@ -14,7 +12,6 @@ import pytest
12
import torch
13
import yaml
14
from transformers import AutoTokenizer
17
-from vllm_test_utils.monitor import monitor
15
16
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
17
from vllm.transformers_utils.detokenizer_utils import convert_ids_list_to_tokens
@@ -24,13 +21,6 @@ from vllm.utils import (
21
bind_kv_cache,
22
unique_filepath,
23
)
27
-from vllm.utils.hashing import sha256
28
-from vllm.utils.torch_utils import (
29
- common_broadcastable_dtype,
30
- current_stream,
31
- is_lossless_cast,
32
-)
33
-from vllm.utils.mem_utils import MemorySnapshot, memory_profiling
24
from ..utils import create_new_process_for_each_test, flat_product
25
26
@@ -267,61 +257,6 @@ def test_duplicate_dict_args(caplog_vllm, parser):
257
assert "-O.mode" in caplog_vllm.text
258
259
270
-@create_new_process_for_each_test()
271
-def test_memory_profiling():
272
- # Fake out some model loading + inference memory usage to test profiling
273
- # Memory used by other processes will show up as cuda usage outside of torch
274
- from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
275
-
276
- lib = CudaRTLibrary()
277
- # 512 MiB allocation outside of this instance
278
- handle1 = lib.cudaMalloc(512 * 1024 * 1024)
279
-
280
- baseline_snapshot = MemorySnapshot()
281
-
282
- # load weights
283
-
284
- weights = torch.randn(128, 1024, 1024, device="cuda", dtype=torch.float32)
285
-
286
- weights_memory = 128 * 1024 * 1024 * 4 # 512 MiB
287
-
288
- def measure_current_non_torch():
289
- free, total = torch.cuda.mem_get_info()
290
- current_used = total - free
291
- current_torch = torch.cuda.memory_reserved()
292
- current_non_torch = current_used - current_torch
293
- return current_non_torch
294
-
295
- with (
296
- memory_profiling(
297
- baseline_snapshot=baseline_snapshot, weights_memory=weights_memory
298
- ) as result,
299
- monitor(measure_current_non_torch) as monitored_values,
300
- ):
301
- # make a memory spike, 1 GiB
302
- spike = torch.randn(256, 1024, 1024, device="cuda", dtype=torch.float32)
303
- del spike
304
-
305
- # Add some extra non-torch memory 256 MiB (simulate NCCL)
306
- handle2 = lib.cudaMalloc(256 * 1024 * 1024)
307
-
308
- # this is an analytic value, it is exact,
309
- # we only have 256 MiB non-torch memory increase
310
- measured_diff = monitored_values.values[-1] - monitored_values.values[0]
311
- assert measured_diff == 256 * 1024 * 1024
312
-
313
- # Check that the memory usage is within 5% of the expected values
314
- # 5% tolerance is caused by cuda runtime.
315
- # we cannot control cuda runtime in the granularity of bytes,
316
- # which causes a small error (<10 MiB in practice)
317
- non_torch_ratio = result.non_torch_increase / (256 * 1024 * 1024) # noqa
318
- assert abs(non_torch_ratio - 1) <= 0.05
319
- assert result.torch_peak_increase == 1024 * 1024 * 1024
320
- del weights
321
- lib.cudaFree(handle1)
322
- lib.cudaFree(handle2)
323
-
324
-
260
def test_bind_kv_cache():
261
from vllm.attention import Attention
262
@@ -403,56 +338,6 @@ def test_bind_kv_cache_pp():
338
assert ctx["layers.0.self_attn"].kv_cache[1] is kv_cache[1][0]
339
340
406
-@pytest.mark.parametrize(
407
- ("src_dtype", "tgt_dtype", "expected_result"),
408
- [
409
- # Different precision_levels
410
- (torch.bool, torch.int8, True),
411
- (torch.bool, torch.float16, True),
412
- (torch.bool, torch.complex32, True),
413
- (torch.int64, torch.bool, False),
414
- (torch.int64, torch.float16, True),
415
- (torch.int64, torch.complex32, True),
416
- (torch.float64, torch.bool, False),
417
- (torch.float64, torch.int8, False),
418
- (torch.float64, torch.complex32, True),
419
- (torch.complex128, torch.bool, False),
420
- (torch.complex128, torch.int8, False),
421
- (torch.complex128, torch.float16, False),
422
- # precision_level=0
423
- (torch.bool, torch.bool, True),
424
- # precision_level=1
425
- (torch.int8, torch.int16, True),
426
- (torch.int16, torch.int8, False),
427
- (torch.uint8, torch.int8, False),
428
- (torch.int8, torch.uint8, False),
429
- # precision_level=2
430
- (torch.float16, torch.float32, True),
431
- (torch.float32, torch.float16, False),
432
- (torch.bfloat16, torch.float32, True),
433
- (torch.float32, torch.bfloat16, False),
434
- # precision_level=3
435
- (torch.complex32, torch.complex64, True),
436
- (torch.complex64, torch.complex32, False),
437
- ],
438
-)
439
-def test_is_lossless_cast(src_dtype, tgt_dtype, expected_result):
440
- assert is_lossless_cast(src_dtype, tgt_dtype) == expected_result
441
-
442
-
443
-@pytest.mark.parametrize(
444
- ("dtypes", "expected_result"),
445
- [
446
- ([torch.bool], torch.bool),
447
- ([torch.bool, torch.int8], torch.int8),
448
- ([torch.bool, torch.int8, torch.float16], torch.float16),
449
- ([torch.bool, torch.int8, torch.float16, torch.complex32], torch.complex32), # noqa: E501
450
- ],
451
-)
452
-def test_common_broadcastable_dtype(dtypes, expected_result):
453
- assert common_broadcastable_dtype(dtypes) == expected_result
454
-
455
-
341
def test_model_specification(
342
parser_with_config, cli_config_file, cli_config_file_with_model
343
):
@@ -535,23 +420,6 @@ def test_model_specification(
420
assert args.port == 12312
421
422
538
-@pytest.mark.parametrize("input", [(), ("abc",), (None,), (None, bool, [1, 2, 3])])
539
-def test_sha256(input: tuple):
540
- digest = sha256(input)
541
- assert digest is not None
542
- assert isinstance(digest, bytes)
543
- assert digest != b""
544
-
545
- input_bytes = pickle.dumps(input, protocol=pickle.HIGHEST_PROTOCOL)
546
- assert digest == hashlib.sha256(input_bytes).digest()
547
-
548
- # hashing again, returns the same value
549
- assert digest == sha256(input)
550
-
551
- # hashing different input, returns different value
552
- assert digest != sha256(input + (1,))
553
-
554
-
423
def test_convert_ids_list_to_tokens():
424
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-1.5B-Instruct")
425
token_ids = tokenizer.encode("Hello, world!")
@@ -561,50 +429,6 @@ def test_convert_ids_list_to_tokens():
429
assert tokens == ["Hello", ",", " world", "!"]
430
431
564
-def test_current_stream_multithread():
565
- import threading
566
-
567
- if not torch.cuda.is_available():
568
- pytest.skip("CUDA not available")
569
-
570
- main_default_stream = torch.cuda.current_stream()
571
- child_stream = torch.cuda.Stream()
572
-
573
- thread_stream_ready = threading.Event()
574
- thread_can_exit = threading.Event()
575
-
576
- def child_thread_func():
577
- with torch.cuda.stream(child_stream):
578
- thread_stream_ready.set()
579
- thread_can_exit.wait(timeout=10)
580
-
581
- child_thread = threading.Thread(target=child_thread_func)
582
- child_thread.start()
583
-
584
- try:
585
- assert thread_stream_ready.wait(timeout=5), (
586
- "Child thread failed to enter stream context in time"
587
- )
588
-
589
- main_current_stream = current_stream()
590
-
591
- assert main_current_stream != child_stream, (
592
- "Main thread's current_stream was contaminated by child thread"
593
- )
594
- assert main_current_stream == main_default_stream, (
595
- "Main thread's current_stream is not the default stream"
596
- )
597
-
598
- # Notify child thread it can exit
599
- thread_can_exit.set()
600
-
601
- finally:
602
- # Ensure child thread exits properly
603
- child_thread.join(timeout=5)
604
- if child_thread.is_alive():
605
- pytest.fail("Child thread failed to exit properly")
606
-
607
-
432
def test_load_config_file(tmp_path):
433
# Define the configuration data
434
config_data = {
tools/pre_commit/check_pickle_imports.py
+4
-5
@@ -23,13 +23,14 @@ ALLOWED_FILES = {
23
"vllm/transformers_utils/config.py",
24
"vllm/model_executor/models/registry.py",
25
"vllm/compilation/caching.py",
26
- "tests/utils_/test_utils.py",
27
- "tests/tokenization/test_cached_tokenizer.py",
26
"vllm/distributed/utils.py",
27
"vllm/distributed/parallel_state.py",
28
"vllm/distributed/device_communicators/all_reduce_utils.py",
29
"vllm/distributed/device_communicators/shm_broadcast.py",
30
"vllm/distributed/device_communicators/shm_object_storage.py",
31
+ "vllm/utils/hashing.py",
32
+ "tests/utils_/test_hashing.py",
33
+ "tests/tokenization/test_cached_tokenizer.py",
34
"benchmarks/kernels/graph_machete_bench.py",
35
"benchmarks/kernels/benchmark_lora.py",
36
"benchmarks/kernels/benchmark_machete.py",
@@ -40,10 +41,8 @@ ALLOWED_FILES = {
41
"vllm/executor/mp_distributed_executor.py",
42
"vllm/executor/ray_distributed_executor.py",
43
"vllm/entrypoints/llm.py",
43
- "tests/utils.py",
44
- # pickle and cloudpickle
44
"vllm/utils/__init__.py",
46
- "vllm/utils/hashing.py",
45
+ "tests/utils.py",
46
}
47
48
PICKLE_RE = re.compile(
vllm/benchmarks/throughput.py
+1
-1
@@ -34,7 +34,7 @@ from vllm.inputs import TextPrompt, TokensPrompt
34
from vllm.lora.request import LoRARequest
35
from vllm.outputs import RequestOutput
36
from vllm.sampling_params import BeamSearchParams
37
-from vllm.utils.asyncio import merge_async_iterators
37
+from vllm.utils.async_utils import merge_async_iterators
38
39
40
def run_vllm(
vllm/entrypoints/chat_utils.py
+1
-1
@@ -51,7 +51,7 @@ from vllm.transformers_utils.chat_templates import get_chat_template_fallback_pa
51
from vllm.transformers_utils.processor import cached_get_processor
52
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
53
from vllm.utils import random_uuid
54
-from vllm.utils.functools import supports_kw
54
+from vllm.utils.func_utils import supports_kw
55
56
logger = init_logger(__name__)
57
vllm/entrypoints/llm.py
+1
-1
@@ -76,7 +76,7 @@ from vllm.transformers_utils.tokenizer import (
76
)
77
from vllm.usage.usage_lib import UsageContext
78
from vllm.utils import Counter, Device
79
-from vllm.utils.collections import as_iter, is_list_of
79
+from vllm.utils.collection_utils import as_iter, is_list_of
80
from vllm.v1.engine import EngineCoreRequest
81
from vllm.v1.engine.llm_engine import LLMEngine
82
from vllm.v1.sample.logits_processor import LogitsProcessor
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -70,7 +70,7 @@ from vllm.transformers_utils.tokenizers import (
70
truncate_tool_call_ids,
71
validate_request_params,
72
)
73
-from vllm.utils.collections import as_list
73
+from vllm.utils.collection_utils import as_list
74
75
logger = init_logger(__name__)
76
vllm/entrypoints/openai/serving_completion.py
+2
-2
@@ -34,8 +34,8 @@ from vllm.logprobs import Logprob
34
from vllm.outputs import RequestOutput
35
from vllm.sampling_params import BeamSearchParams, SamplingParams
36
from vllm.transformers_utils.tokenizer import AnyTokenizer
37
-from vllm.utils.asyncio import merge_async_iterators
38
-from vllm.utils.collections import as_list
37
+from vllm.utils.async_utils import merge_async_iterators
38
+from vllm.utils.collection_utils import as_list
39
40
logger = init_logger(__name__)
41
vllm/entrypoints/openai/serving_embedding.py
+2
-2
@@ -39,8 +39,8 @@ from vllm.outputs import (
39
RequestOutput,
40
)
41
from vllm.pooling_params import PoolingParams
42
-from vllm.utils.asyncio import merge_async_iterators
43
-from vllm.utils.collections import chunk_list
42
+from vllm.utils.async_utils import merge_async_iterators
43
+from vllm.utils.collection_utils import chunk_list
44
45
logger = init_logger(__name__)
46
vllm/entrypoints/openai/serving_engine.py
+2
-2
@@ -91,13 +91,13 @@ from vllm.tracing import (
91
)
92
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
93
from vllm.utils import random_uuid
94
-from vllm.utils.asyncio import (
94
+from vllm.utils.async_utils import (
95
AsyncMicrobatchTokenizer,
96
collect_from_async_generator,
97
make_async,
98
merge_async_iterators,
99
)
100
-from vllm.utils.collections import is_list_of
100
+from vllm.utils.collection_utils import is_list_of
101
from vllm.v1.engine import EngineCoreRequest
102
103
logger = init_logger(__name__)
vllm/entrypoints/openai/serving_pooling.py
+1
-1
@@ -36,7 +36,7 @@ from vllm.entrypoints.utils import _validate_truncation_size
36
from vllm.logger import init_logger
37
from vllm.outputs import PoolingOutput, PoolingRequestOutput
38
from vllm.tasks import SupportedTask
39
-from vllm.utils.asyncio import merge_async_iterators
39
+from vllm.utils.async_utils import merge_async_iterators
40
41
logger = init_logger(__name__)
42
vllm/entrypoints/openai/serving_score.py
+1
-1
@@ -37,7 +37,7 @@ from vllm.logger import init_logger
37
from vllm.lora.request import LoRARequest
38
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
39
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
40
-from vllm.utils.asyncio import make_async, merge_async_iterators
40
+from vllm.utils.async_utils import make_async, merge_async_iterators
41
42
logger = init_logger(__name__)
43
vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py
+1
-1
@@ -12,7 +12,7 @@ from vllm.entrypoints.openai.protocol import (
12
)
13
from vllm.logger import init_logger
14
from vllm.transformers_utils.tokenizer import AnyTokenizer
15
-from vllm.utils.collections import is_list_of
15
+from vllm.utils.collection_utils import is_list_of
16
from vllm.utils.import_utils import import_from_path
17
18
logger = init_logger(__name__)
vllm/entrypoints/renderer.py
+1
-1
@@ -17,7 +17,7 @@ from vllm.inputs.data import TextPrompt as EngineTextPrompt
17
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
18
from vllm.inputs.parse import get_prompt_components, parse_raw_prompts
19
from vllm.transformers_utils.tokenizer import AnyTokenizer
20
-from vllm.utils.asyncio import AsyncMicrobatchTokenizer
20
+from vllm.utils.async_utils import AsyncMicrobatchTokenizer
21
22
23
@dataclass(frozen=True)
vllm/executor/executor_base.py
+1
-1
@@ -17,7 +17,7 @@ from vllm.logger import init_logger
17
from vllm.lora.request import LoRARequest
18
from vllm.sequence import ExecuteModelRequest
19
from vllm.tasks import SupportedTask
20
-from vllm.utils.asyncio import make_async
20
+from vllm.utils.async_utils import make_async
21
from vllm.v1.outputs import SamplerOutput
22
from vllm.v1.worker.worker_base import WorkerBase
23
vllm/executor/ray_distributed_executor.py
+1
-1
@@ -19,7 +19,7 @@ from vllm.logger import init_logger
19
from vllm.platforms import current_platform
20
from vllm.ray.ray_env import get_env_vars_to_copy
21
from vllm.sequence import ExecuteModelRequest
22
-from vllm.utils.asyncio import make_async
22
+from vllm.utils.async_utils import make_async
23
from vllm.utils.network_utils import (
24
get_distributed_init_method,
25
get_ip,
vllm/inputs/parse.py
+1
-1
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Literal, NamedTuple, TypeAlias, TypedDict, cas
5
6
from typing_extensions import TypeIs
7
8
-from vllm.utils.collections import is_list_of
8
+from vllm.utils.collection_utils import is_list_of
9
10
from .data import (
11
EmbedsPrompt,
vllm/model_executor/layers/activation.py
+1
-1
@@ -17,7 +17,7 @@ from vllm.logger import init_logger
17
from vllm.model_executor.custom_op import CustomOp
18
from vllm.model_executor.utils import set_weight_attrs
19
from vllm.platforms import current_platform
20
-from vllm.utils.collections import LazyDict
20
+from vllm.utils.collection_utils import LazyDict
21
22
logger = init_logger(__name__)
23
vllm/model_executor/layers/fused_moe/deep_gemm_moe.py
+1
-1
@@ -31,7 +31,7 @@ from vllm.utils.deep_gemm import (
31
get_mk_alignment_for_contiguous_layout,
32
m_grouped_fp8_gemm_nt_contiguous,
33
)
34
-from vllm.utils.functools import run_once
34
+from vllm.utils.func_utils import run_once
35
36
logger = init_logger(__name__)
37
vllm/model_executor/layers/quantization/gptq.py
+1
-1
@@ -28,7 +28,7 @@ from vllm.model_executor.parameter import (
28
RowvLLMParameter,
29
)
30
from vllm.transformers_utils.config import get_safetensors_params_metadata
31
-from vllm.utils.collections import is_list_of
31
+from vllm.utils.collection_utils import is_list_of
32
33
if TYPE_CHECKING:
34
from vllm.model_executor.layers.quantization import QuantizationMethods
vllm/model_executor/layers/quantization/gptq_marlin.py
+1
-1
@@ -57,7 +57,7 @@ from vllm.model_executor.parameter import (
57
from vllm.platforms import current_platform
58
from vllm.scalar_type import scalar_types
59
from vllm.transformers_utils.config import get_safetensors_params_metadata
60
-from vllm.utils.collections import is_list_of
60
+from vllm.utils.collection_utils import is_list_of
61
62
logger = init_logger(__name__)
63
vllm/model_executor/models/deepseek_vl2.py
+1
-1
@@ -48,7 +48,7 @@ from vllm.transformers_utils.configs.deepseek_vl2 import (
48
)
49
from vllm.transformers_utils.processors.deepseek_vl2 import DeepseekVLV2Processor
50
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
51
-from vllm.utils.collections import is_list_of
51
+from vllm.utils.collection_utils import is_list_of
52
from vllm.utils.tensor_schema import TensorSchema, TensorShape
53
from vllm.utils.torch_utils import set_default_torch_dtype
54
vllm/model_executor/models/interfaces.py
+1
-1
@@ -24,7 +24,7 @@ from vllm.inputs import TokensPrompt
24
from vllm.inputs.data import PromptType
25
from vllm.logger import init_logger
26
from vllm.model_executor.layers.quantization import QuantizationConfig
27
-from vllm.utils.functools import supports_kw
27
+from vllm.utils.func_utils import supports_kw
28
29
from .interfaces_base import VllmModel, is_pooling_model
30
vllm/model_executor/models/interfaces_base.py
+1
-1
@@ -15,7 +15,7 @@ import torch.nn as nn
15
from typing_extensions import TypeIs, TypeVar
16
17
from vllm.logger import init_logger
18
-from vllm.utils.functools import supports_kw
18
+from vllm.utils.func_utils import supports_kw
19
20
if TYPE_CHECKING:
21
from vllm.config import VllmConfig
vllm/model_executor/models/llava_next_video.py
+1
-1
@@ -33,7 +33,7 @@ from vllm.multimodal.processing import (
33
)
34
from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
-from vllm.utils.collections import is_list_of
36
+from vllm.utils.collection_utils import is_list_of
37
from vllm.utils.tensor_schema import TensorSchema, TensorShape
38
39
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
vllm/model_executor/models/minicpmv.py
+1
-1
@@ -85,7 +85,7 @@ from vllm.multimodal.processing import (
85
from vllm.multimodal.profiling import BaseDummyInputsBuilder
86
from vllm.platforms import current_platform
87
from vllm.sequence import IntermediateTensors
88
-from vllm.utils.collections import flatten_2d_lists
88
+from vllm.utils.collection_utils import flatten_2d_lists
89
from vllm.utils.tensor_schema import TensorSchema, TensorShape
90
from vllm.utils.torch_utils import set_default_torch_dtype
91
vllm/model_executor/models/qwen3_vl.py
+1
-1
@@ -79,7 +79,7 @@ from vllm.multimodal.processing import (
79
)
80
from vllm.multimodal.profiling import BaseDummyInputsBuilder
81
from vllm.sequence import IntermediateTensors
82
-from vllm.utils.collections import is_list_of
82
+from vllm.utils.collection_utils import is_list_of
83
84
from .interfaces import (
85
MultiModalEmbeddings,
vllm/multimodal/inputs.py
+1
-1
@@ -22,7 +22,7 @@ from typing import (
22
import numpy as np
23
from typing_extensions import NotRequired, TypeVar, deprecated
24
25
-from vllm.utils.collections import full_groupby, is_list_of
25
+from vllm.utils.collection_utils import full_groupby, is_list_of
26
from vllm.utils.import_utils import LazyLoader
27
from vllm.utils.jsontree import json_map_leaves
28
vllm/multimodal/parse.py
+1
-1
@@ -19,7 +19,7 @@ import numpy as np
19
import torch
20
from typing_extensions import assert_never
21
22
-from vllm.utils.collections import is_list_of
22
+from vllm.utils.collection_utils import is_list_of
23
from vllm.utils.import_utils import LazyLoader
24
25
from .audio import AudioResampler
vllm/multimodal/processing.py
+3
-3
@@ -25,8 +25,8 @@ from typing_extensions import TypeVar, assert_never
25
from vllm.logger import init_logger
26
from vllm.transformers_utils.processor import cached_processor_from_config
27
from vllm.transformers_utils.tokenizer import AnyTokenizer, decode_tokens, encode_tokens
28
-from vllm.utils.collections import flatten_2d_lists, full_groupby
29
-from vllm.utils.functools import get_allowed_kwarg_only_overrides
28
+from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
29
+from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
30
from vllm.utils.jsontree import JSONTree, json_map_leaves
31
32
from .hasher import MultiModalHasher
@@ -486,7 +486,7 @@ _M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp)
486
def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
487
"""
488
Convenience function to apply
489
- [`full_groupby`][vllm.utils.collections.full_groupby]
489
+ [`full_groupby`][vllm.utils.collection_utils.full_groupby]
490
based on modality.
491
"""
492
return full_groupby(values, key=lambda x: x.modality)
vllm/multimodal/registry.py
+1
-1
@@ -9,7 +9,7 @@ import torch.nn as nn
9
from vllm.config.multimodal import BaseDummyOptions
10
from vllm.logger import init_logger
11
from vllm.transformers_utils.tokenizer import AnyTokenizer, cached_tokenizer_from_config
12
-from vllm.utils.collections import ClassRegistry
12
+from vllm.utils.collection_utils import ClassRegistry
13
14
from .cache import BaseMultiModalProcessorCache
15
from .processing import (
vllm/reasoning/abs_reasoning_parsers.py
+1
-1
@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any
9
10
from vllm.entrypoints.tool_server import ToolServer
11
from vllm.logger import init_logger
12
-from vllm.utils.collections import is_list_of
12
+from vllm.utils.collection_utils import is_list_of
13
from vllm.utils.import_utils import import_from_path
14
15
if TYPE_CHECKING:
vllm/tracing.py
+1
-1
@@ -5,7 +5,7 @@ import os
5
from collections.abc import Mapping
6
7
from vllm.logger import init_logger
8
-from vllm.utils.functools import run_once
8
+from vllm.utils.func_utils import run_once
9
10
TRACE_HEADERS = ["traceparent", "tracestate"]
11
vllm/transformers_utils/processor.py
+1
-1
@@ -16,7 +16,7 @@ from transformers.processing_utils import ProcessorMixin
16
from transformers.video_processing_utils import BaseVideoProcessor
17
from typing_extensions import TypeVar
18
19
-from vllm.utils.functools import get_allowed_kwarg_only_overrides
19
+from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
20
21
if TYPE_CHECKING:
22
from vllm.config import ModelConfig
vllm/utils/__init__.py
-6
@@ -1122,9 +1122,6 @@ def warn_for_unimplemented_methods(cls: type[T]) -> type[T]:
1122
return cls
1123
1124
1125
-## moved to vllm.utils.profiling (imported at module top)
1126
-
1127
-
1125
# Only relevant for models using ALiBi (e.g, MPT)
1126
def check_use_alibi(model_config: ModelConfig) -> bool:
1127
cfg = model_config.hf_text_config
@@ -1150,9 +1147,6 @@ def check_use_alibi(model_config: ModelConfig) -> bool:
1147
)
1148
1149
1153
-## moved to vllm.utils.hashing
1154
-
1155
-
1150
@cache
1151
def _has_module(module_name: str) -> bool:
1152
"""Return True if *module_name* can be found in the current environment.
vllm/utils/async_utils.py
renamed
vllm/utils/collection_utils.py
renamed
vllm/utils/func_utils.py
renamed
vllm/v1/engine/async_llm.py
+3
-3
@@ -30,9 +30,9 @@ from vllm.transformers_utils.config import maybe_register_config_serialize_by_va
30
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
31
from vllm.usage.usage_lib import UsageContext
32
from vllm.utils import Device, cdiv
33
-from vllm.utils.asyncio import cancel_task_threadsafe
34
-from vllm.utils.collections import as_list
35
-from vllm.utils.functools import deprecate_kwargs
33
+from vllm.utils.async_utils import cancel_task_threadsafe
34
+from vllm.utils.collection_utils import as_list
35
+from vllm.utils.func_utils import deprecate_kwargs
36
from vllm.v1.engine import EngineCoreRequest
37
from vllm.v1.engine.core_client import EngineCoreClient
38
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
vllm/v1/engine/core_client.py
+1
-1
@@ -23,7 +23,7 @@ from vllm.config import VllmConfig
23
from vllm.logger import init_logger
24
from vllm.lora.request import LoRARequest
25
from vllm.tasks import SupportedTask
26
-from vllm.utils.asyncio import in_loop
26
+from vllm.utils.async_utils import in_loop
27
from vllm.utils.network_utils import (
28
close_sockets,
29
get_open_port,
vllm/v1/worker/gpu_input_batch.py
+1
-1
@@ -13,7 +13,7 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec
13
from vllm.pooling_params import PoolingParams
14
from vllm.sampling_params import SamplingParams, SamplingType
15
from vllm.utils import length_from_prompt_token_ids_or_embeds
16
-from vllm.utils.collections import swap_dict_values
16
+from vllm.utils.collection_utils import swap_dict_values
17
from vllm.v1.outputs import LogprobsTensors
18
from vllm.v1.pool.metadata import PoolingMetadata
19
from vllm.v1.sample.logits_processor import (
vllm/v1/worker/tpu_input_batch.py
+1
-1
@@ -10,7 +10,7 @@ import torch
10
from vllm.lora.request import LoRARequest
11
from vllm.sampling_params import SamplingType
12
from vllm.utils import length_from_prompt_token_ids_or_embeds
13
-from vllm.utils.collections import swap_dict_values
13
+from vllm.utils.collection_utils import swap_dict_values
14
from vllm.v1.outputs import LogprobsTensors
15
from vllm.v1.worker.block_table import MultiGroupBlockTable
16
from vllm.v1.worker.gpu_input_batch import CachedRequestState