A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Chore] Separate out `vllm.utils.func` (#26904)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 15, 2025 at 21:03 UTC
136a17fe6edacba292d899af4cfd424e0ca98d9f
18 files changed
+407
-371
tests/models/multimodal/generation/test_common.py
+1
-1
@@ -17,7 +17,7 @@ from transformers import (
17
)
18
19
from vllm.platforms import current_platform
20
-from vllm.utils import identity
20
+from vllm.utils.func import identity
21
22
from ....conftest import (
23
IMAGE_ASSETS,
tests/utils_/test_func_utils.py
new
+97
@@ -0,0 +1,97 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# ruff: noqa
4
+
5
+import pytest
6
+
7
+from vllm.utils.func import deprecate_kwargs, supports_kw
8
+
9
+from ..utils import error_on_warning
10
+
11
+
12
+def test_deprecate_kwargs_always():
13
+ @deprecate_kwargs("old_arg", is_deprecated=True)
14
+ def dummy(*, old_arg: object = None, new_arg: object = None):
15
+ pass
16
+
17
+ with pytest.warns(DeprecationWarning, match="'old_arg'"):
18
+ dummy(old_arg=1)
19
+
20
+ with error_on_warning(DeprecationWarning):
21
+ dummy(new_arg=1)
22
+
23
+
24
+def test_deprecate_kwargs_never():
25
+ @deprecate_kwargs("old_arg", is_deprecated=False)
26
+ def dummy(*, old_arg: object = None, new_arg: object = None):
27
+ pass
28
+
29
+ with error_on_warning(DeprecationWarning):
30
+ dummy(old_arg=1)
31
+
32
+ with error_on_warning(DeprecationWarning):
33
+ dummy(new_arg=1)
34
+
35
+
36
+def test_deprecate_kwargs_dynamic():
37
+ is_deprecated = True
38
+
39
+ @deprecate_kwargs("old_arg", is_deprecated=lambda: is_deprecated)
40
+ def dummy(*, old_arg: object = None, new_arg: object = None):
41
+ pass
42
+
43
+ with pytest.warns(DeprecationWarning, match="'old_arg'"):
44
+ dummy(old_arg=1)
45
+
46
+ with error_on_warning(DeprecationWarning):
47
+ dummy(new_arg=1)
48
+
49
+ is_deprecated = False
50
+
51
+ with error_on_warning(DeprecationWarning):
52
+ dummy(old_arg=1)
53
+
54
+ with error_on_warning(DeprecationWarning):
55
+ dummy(new_arg=1)
56
+
57
+
58
+def test_deprecate_kwargs_additional_message():
59
+ @deprecate_kwargs("old_arg", is_deprecated=True, additional_message="abcd")
60
+ def dummy(*, old_arg: object = None, new_arg: object = None):
61
+ pass
62
+
63
+ with pytest.warns(DeprecationWarning, match="abcd"):
64
+ dummy(old_arg=1)
65
+
66
+
67
+@pytest.mark.parametrize(
68
+ ("callable", "kw_name", "requires_kw_only", "allow_var_kwargs", "is_supported"),
69
+ [
70
+ # Tests for positional argument support
71
+ (lambda foo: None, "foo", True, True, False),
72
+ (lambda foo: None, "foo", False, True, True),
73
+ # Tests for positional or keyword / keyword only
74
+ (lambda foo=100: None, "foo", True, True, False),
75
+ (lambda *, foo: None, "foo", False, True, True),
76
+ # Tests to make sure the names of variadic params are NOT supported
77
+ (lambda *args: None, "args", False, True, False),
78
+ (lambda **kwargs: None, "kwargs", False, True, False),
79
+ # Tests for if we allow var kwargs to add support
80
+ (lambda foo: None, "something_else", False, True, False),
81
+ (lambda foo, **kwargs: None, "something_else", False, True, True),
82
+ (lambda foo, **kwargs: None, "kwargs", True, True, False),
83
+ (lambda foo, **kwargs: None, "foo", True, True, False),
84
+ ],
85
+)
86
+def test_supports_kw(
87
+ callable, kw_name, requires_kw_only, allow_var_kwargs, is_supported
88
+):
89
+ assert (
90
+ supports_kw(
91
+ callable=callable,
92
+ kw_name=kw_name,
93
+ requires_kw_only=requires_kw_only,
94
+ allow_var_kwargs=allow_var_kwargs,
95
+ )
96
+ == is_supported
97
+ )
tests/utils_/test_jsontree.py
new
+32
@@ -0,0 +1,32 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from vllm.utils.jsontree import json_count_leaves
4
+
5
+
6
+def test_json_count_leaves():
7
+ """Test json_count_leaves function from jsontree utility."""
8
+
9
+ # Single leaf values
10
+ assert json_count_leaves(42) == 1
11
+ assert json_count_leaves("hello") == 1
12
+ assert json_count_leaves(None) == 1
13
+
14
+ # Empty containers
15
+ assert json_count_leaves([]) == 0
16
+ assert json_count_leaves({}) == 0
17
+ assert json_count_leaves(()) == 0
18
+
19
+ # Flat structures
20
+ assert json_count_leaves([1, 2, 3]) == 3
21
+ assert json_count_leaves({"a": 1, "b": 2}) == 2
22
+ assert json_count_leaves((1, 2, 3)) == 3
23
+
24
+ # Nested structures
25
+ nested_dict = {"a": 1, "b": {"c": 2, "d": 3}}
26
+ assert json_count_leaves(nested_dict) == 3
27
+
28
+ nested_list = [1, [2, 3], 4]
29
+ assert json_count_leaves(nested_list) == 4
30
+
31
+ mixed_nested = {"list": [1, 2], "dict": {"x": 3}, "value": 4}
32
+ assert json_count_leaves(mixed_nested) == 4
tests/utils_/test_utils.py
+1
-121
@@ -30,7 +30,6 @@ from vllm.utils import (
30
bind_kv_cache,
31
common_broadcastable_dtype,
32
current_stream,
33
- deprecate_kwargs,
33
get_open_port,
34
get_tcp_uri,
35
is_lossless_cast,
@@ -42,12 +41,11 @@ from vllm.utils import (
41
sha256,
42
split_host_port,
43
split_zmq_path,
45
- supports_kw,
44
swap_dict_values,
45
unique_filepath,
46
)
47
50
-from ..utils import create_new_process_for_each_test, error_on_warning
48
+from ..utils import create_new_process_for_each_test
49
50
51
@pytest.mark.asyncio
@@ -83,61 +81,6 @@ async def test_merge_async_iterators():
81
raise AssertionError() from e
82
83
86
-def test_deprecate_kwargs_always():
87
- @deprecate_kwargs("old_arg", is_deprecated=True)
88
- def dummy(*, old_arg: object = None, new_arg: object = None):
89
- pass
90
-
91
- with pytest.warns(DeprecationWarning, match="'old_arg'"):
92
- dummy(old_arg=1)
93
-
94
- with error_on_warning(DeprecationWarning):
95
- dummy(new_arg=1)
96
-
97
-
98
-def test_deprecate_kwargs_never():
99
- @deprecate_kwargs("old_arg", is_deprecated=False)
100
- def dummy(*, old_arg: object = None, new_arg: object = None):
101
- pass
102
-
103
- with error_on_warning(DeprecationWarning):
104
- dummy(old_arg=1)
105
-
106
- with error_on_warning(DeprecationWarning):
107
- dummy(new_arg=1)
108
-
109
-
110
-def test_deprecate_kwargs_dynamic():
111
- is_deprecated = True
112
-
113
- @deprecate_kwargs("old_arg", is_deprecated=lambda: is_deprecated)
114
- def dummy(*, old_arg: object = None, new_arg: object = None):
115
- pass
116
-
117
- with pytest.warns(DeprecationWarning, match="'old_arg'"):
118
- dummy(old_arg=1)
119
-
120
- with error_on_warning(DeprecationWarning):
121
- dummy(new_arg=1)
122
-
123
- is_deprecated = False
124
-
125
- with error_on_warning(DeprecationWarning):
126
- dummy(old_arg=1)
127
-
128
- with error_on_warning(DeprecationWarning):
129
- dummy(new_arg=1)
130
-
131
-
132
-def test_deprecate_kwargs_additional_message():
133
- @deprecate_kwargs("old_arg", is_deprecated=True, additional_message="abcd")
134
- def dummy(*, old_arg: object = None, new_arg: object = None):
135
- pass
136
-
137
- with pytest.warns(DeprecationWarning, match="abcd"):
138
- dummy(old_arg=1)
139
-
140
-
84
def test_get_open_port(monkeypatch: pytest.MonkeyPatch):
85
with monkeypatch.context() as m:
86
m.setenv("VLLM_PORT", "5678")
@@ -383,39 +326,6 @@ def test_duplicate_dict_args(caplog_vllm, parser):
326
assert "-O.mode" in caplog_vllm.text
327
328
386
-@pytest.mark.parametrize(
387
- "callable,kw_name,requires_kw_only,allow_var_kwargs,is_supported",
388
- [
389
- # Tests for positional argument support
390
- (lambda foo: None, "foo", True, True, False),
391
- (lambda foo: None, "foo", False, True, True),
392
- # Tests for positional or keyword / keyword only
393
- (lambda foo=100: None, "foo", True, True, False),
394
- (lambda *, foo: None, "foo", False, True, True),
395
- # Tests to make sure the names of variadic params are NOT supported
396
- (lambda *args: None, "args", False, True, False),
397
- (lambda **kwargs: None, "kwargs", False, True, False),
398
- # Tests for if we allow var kwargs to add support
399
- (lambda foo: None, "something_else", False, True, False),
400
- (lambda foo, **kwargs: None, "something_else", False, True, True),
401
- (lambda foo, **kwargs: None, "kwargs", True, True, False),
402
- (lambda foo, **kwargs: None, "foo", True, True, False),
403
- ],
404
-)
405
-def test_supports_kw(
406
- callable, kw_name, requires_kw_only, allow_var_kwargs, is_supported
407
-):
408
- assert (
409
- supports_kw(
410
- callable=callable,
411
- kw_name=kw_name,
412
- requires_kw_only=requires_kw_only,
413
- allow_var_kwargs=allow_var_kwargs,
414
- )
415
- == is_supported
416
- )
417
-
418
-
329
@create_new_process_for_each_test()
330
def test_memory_profiling():
331
# Fake out some model loading + inference memory usage to test profiling
@@ -863,36 +773,6 @@ def test_join_host_port():
773
assert join_host_port("::1", 5555) == "[::1]:5555"
774
775
866
-def test_json_count_leaves():
867
- """Test json_count_leaves function from jsontree utility."""
868
- from vllm.utils.jsontree import json_count_leaves
869
-
870
- # Single leaf values
871
- assert json_count_leaves(42) == 1
872
- assert json_count_leaves("hello") == 1
873
- assert json_count_leaves(None) == 1
874
-
875
- # Empty containers
876
- assert json_count_leaves([]) == 0
877
- assert json_count_leaves({}) == 0
878
- assert json_count_leaves(()) == 0
879
-
880
- # Flat structures
881
- assert json_count_leaves([1, 2, 3]) == 3
882
- assert json_count_leaves({"a": 1, "b": 2}) == 2
883
- assert json_count_leaves((1, 2, 3)) == 3
884
-
885
- # Nested structures
886
- nested_dict = {"a": 1, "b": {"c": 2, "d": 3}}
887
- assert json_count_leaves(nested_dict) == 3
888
-
889
- nested_list = [1, [2, 3], 4]
890
- assert json_count_leaves(nested_list) == 4
891
-
892
- mixed_nested = {"list": [1, 2], "dict": {"x": 3}, "value": 4}
893
- assert json_count_leaves(mixed_nested) == 4
894
-
895
-
776
def test_convert_ids_list_to_tokens():
777
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-1.5B-Instruct")
778
token_ids = tokenizer.encode("Hello, world!")
vllm/entrypoints/chat_utils.py
+2
-1
@@ -50,7 +50,8 @@ from vllm.multimodal.utils import MediaConnector
50
from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
51
from vllm.transformers_utils.processor import cached_get_processor
52
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
53
-from vllm.utils import random_uuid, supports_kw
53
+from vllm.utils import random_uuid
54
+from vllm.utils.func import supports_kw
55
56
logger = init_logger(__name__)
57
vllm/entrypoints/openai/serving_engine.py
+1
-1
@@ -94,10 +94,10 @@ from vllm.utils import (
94
AsyncMicrobatchTokenizer,
95
collect_from_async_generator,
96
is_list_of,
97
- make_async,
97
merge_async_iterators,
98
random_uuid,
99
)
100
+from vllm.utils.func import make_async
101
from vllm.v1.engine import EngineCoreRequest
102
103
logger = init_logger(__name__)
vllm/entrypoints/openai/serving_score.py
+2
-1
@@ -37,7 +37,8 @@ from vllm.logger import init_logger
37
from vllm.lora.request import LoRARequest
38
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
39
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
40
-from vllm.utils import make_async, merge_async_iterators
40
+from vllm.utils import merge_async_iterators
41
+from vllm.utils.func import make_async
42
43
logger = init_logger(__name__)
44
vllm/executor/executor_base.py
+1
-1
@@ -17,7 +17,7 @@ from vllm.logger import init_logger
17
from vllm.lora.request import LoRARequest
18
from vllm.sequence import ExecuteModelRequest
19
from vllm.tasks import SupportedTask
20
-from vllm.utils import make_async
20
+from vllm.utils.func import make_async
21
from vllm.v1.outputs import SamplerOutput
22
from vllm.v1.worker.worker_base import WorkerBase
23
vllm/executor/ray_distributed_executor.py
+1
-1
@@ -24,8 +24,8 @@ from vllm.utils import (
24
get_distributed_init_method,
25
get_ip,
26
get_open_port,
27
- make_async,
27
)
28
+from vllm.utils.func import make_async
29
from vllm.v1.outputs import SamplerOutput
30
31
if ray is not None:
vllm/model_executor/layers/fused_moe/deep_gemm_moe.py
+2
-1
@@ -27,8 +27,9 @@ from vllm.model_executor.layers.fused_moe.utils import _resize_cache
27
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
28
per_token_group_quant_fp8,
29
)
30
-from vllm.utils import has_deep_gemm, run_once
30
+from vllm.utils import has_deep_gemm
31
from vllm.utils.deep_gemm import m_grouped_fp8_gemm_nt_contiguous
32
+from vllm.utils.func import run_once
33
34
logger = init_logger(__name__)
35
vllm/model_executor/models/interfaces.py
+1
-1
@@ -24,7 +24,7 @@ from vllm.inputs import TokensPrompt
24
from vllm.inputs.data import PromptType
25
from vllm.logger import init_logger
26
from vllm.model_executor.layers.quantization import QuantizationConfig
27
-from vllm.utils import supports_kw
27
+from vllm.utils.func import supports_kw
28
29
from .interfaces_base import VllmModel, is_pooling_model
30
vllm/model_executor/models/interfaces_base.py
+1
-1
@@ -15,7 +15,7 @@ import torch.nn as nn
15
from typing_extensions import TypeIs, TypeVar
16
17
from vllm.logger import init_logger
18
-from vllm.utils import supports_kw
18
+from vllm.utils.func import supports_kw
19
20
if TYPE_CHECKING:
21
from vllm.config import VllmConfig
vllm/multimodal/processing.py
+2
-1
@@ -25,7 +25,8 @@ from typing_extensions import TypeVar, assert_never
25
from vllm.logger import init_logger
26
from vllm.transformers_utils.processor import cached_processor_from_config
27
from vllm.transformers_utils.tokenizer import AnyTokenizer, decode_tokens, encode_tokens
28
-from vllm.utils import flatten_2d_lists, full_groupby, get_allowed_kwarg_only_overrides
28
+from vllm.utils import flatten_2d_lists, full_groupby
29
+from vllm.utils.func import get_allowed_kwarg_only_overrides
30
from vllm.utils.jsontree import JSONTree, json_map_leaves
31
32
from .hasher import MultiModalHasher
vllm/tracing.py
+1
-1
@@ -5,7 +5,7 @@ import os
5
from collections.abc import Mapping
6
7
from vllm.logger import init_logger
8
-from vllm.utils import run_once
8
+from vllm.utils.func import run_once
9
10
TRACE_HEADERS = ["traceparent", "tracestate"]
11
vllm/transformers_utils/processor.py
+1
-1
@@ -16,7 +16,7 @@ from transformers.processing_utils import ProcessorMixin
16
from transformers.video_processing_utils import BaseVideoProcessor
17
from typing_extensions import TypeVar
18
19
-from vllm.utils import get_allowed_kwarg_only_overrides
19
+from vllm.utils.func import get_allowed_kwarg_only_overrides
20
21
if TYPE_CHECKING:
22
from vllm.config import ModelConfig
vllm/utils/__init__.py
+1
-237
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import asyncio
5
-import concurrent
5
import contextlib
6
import datetime
7
import enum
@@ -43,7 +42,6 @@ from asyncio import FIRST_COMPLETED, AbstractEventLoop, Task
42
from collections import UserDict, defaultdict
43
from collections.abc import (
44
AsyncGenerator,
46
- Awaitable,
45
Callable,
46
Collection,
47
Generator,
@@ -85,7 +83,7 @@ from packaging import version
83
from packaging.version import Version
84
from torch.library import Library
85
from transformers.tokenization_utils_base import BatchEncoding
88
-from typing_extensions import Never, ParamSpec, TypeIs, assert_never
86
+from typing_extensions import Never, TypeIs, assert_never
87
88
import vllm.envs as envs
89
from vllm.logger import enable_trace_function_call, init_logger
@@ -174,7 +172,6 @@ def set_default_torch_num_threads(num_threads: int):
172
torch.set_num_threads(old_num_threads)
173
174
177
-P = ParamSpec("P")
175
T = TypeVar("T")
176
U = TypeVar("U")
177
@@ -452,24 +449,6 @@ def in_loop(event_loop: AbstractEventLoop) -> bool:
449
return False
450
451
455
-def make_async(
456
- func: Callable[P, T], executor: concurrent.futures.Executor | None = None
457
-) -> Callable[P, Awaitable[T]]:
458
- """Take a blocking function, and run it on in an executor thread.
459
-
460
- This function prevents the blocking function from blocking the
461
- asyncio event loop.
462
- The code in this function needs to be thread safe.
463
- """
464
-
465
- def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> asyncio.Future:
466
- loop = asyncio.get_event_loop()
467
- p_func = partial(func, *args, **kwargs)
468
- return loop.run_in_executor(executor=executor, func=p_func)
469
-
470
- return _async_wrapper
471
-
472
-
452
async def merge_async_iterators(
453
*iterators: AsyncGenerator[T, None],
454
) -> AsyncGenerator[tuple[int, T], None]:
@@ -1254,90 +1233,6 @@ def enable_trace_function_call_for_thread(vllm_config: VllmConfig) -> None:
1233
enable_trace_function_call(log_path)
1234
1235
1257
-# `functools` helpers
1258
-def identity(value: T, **kwargs) -> T:
1259
- """Returns the first provided value."""
1260
- return value
1261
-
1262
-
1263
-F = TypeVar("F", bound=Callable[..., Any])
1264
-
1265
-
1266
-def deprecate_args(
1267
- start_index: int,
1268
- is_deprecated: bool | Callable[[], bool] = True,
1269
- additional_message: str | None = None,
1270
-) -> Callable[[F], F]:
1271
- if not callable(is_deprecated):
1272
- is_deprecated = partial(identity, is_deprecated)
1273
-
1274
- def wrapper(fn: F) -> F:
1275
- params = inspect.signature(fn).parameters
1276
- pos_types = (
1277
- inspect.Parameter.POSITIONAL_ONLY,
1278
- inspect.Parameter.POSITIONAL_OR_KEYWORD,
1279
- )
1280
- pos_kws = [kw for kw, param in params.items() if param.kind in pos_types]
1281
-
1282
- @wraps(fn)
1283
- def inner(*args, **kwargs):
1284
- if is_deprecated():
1285
- deprecated_args = pos_kws[start_index : len(args)]
1286
- if deprecated_args:
1287
- msg = (
1288
- f"The positional arguments {deprecated_args} are "
1289
- "deprecated and will be removed in a future update."
1290
- )
1291
- if additional_message is not None:
1292
- msg += f" {additional_message}"
1293
-
1294
- warnings.warn(
1295
- DeprecationWarning(msg),
1296
- stacklevel=3, # The inner function takes up one level
1297
- )
1298
-
1299
- return fn(*args, **kwargs)
1300
-
1301
- return inner # type: ignore
1302
-
1303
- return wrapper
1304
-
1305
-
1306
-def deprecate_kwargs(
1307
- *kws: str,
1308
- is_deprecated: bool | Callable[[], bool] = True,
1309
- additional_message: str | None = None,
1310
-) -> Callable[[F], F]:
1311
- deprecated_kws = set(kws)
1312
-
1313
- if not callable(is_deprecated):
1314
- is_deprecated = partial(identity, is_deprecated)
1315
-
1316
- def wrapper(fn: F) -> F:
1317
- @wraps(fn)
1318
- def inner(*args, **kwargs):
1319
- if is_deprecated():
1320
- deprecated_kwargs = kwargs.keys() & deprecated_kws
1321
- if deprecated_kwargs:
1322
- msg = (
1323
- f"The keyword arguments {deprecated_kwargs} are "
1324
- "deprecated and will be removed in a future update."
1325
- )
1326
- if additional_message is not None:
1327
- msg += f" {additional_message}"
1328
-
1329
- warnings.warn(
1330
- DeprecationWarning(msg),
1331
- stacklevel=3, # The inner function takes up one level
1332
- )
1333
-
1334
- return fn(*args, **kwargs)
1335
-
1336
- return inner # type: ignore
1337
-
1338
- return wrapper
1339
-
1340
-
1236
@lru_cache(maxsize=8)
1237
def _cuda_device_count_stateless(cuda_visible_devices: str | None = None) -> int:
1238
# Note: cuda_visible_devices is not used, but we keep it as an argument for
@@ -1426,21 +1321,6 @@ def weak_bind(
1321
return weak_bound
1322
1323
1429
-def run_once(f: Callable[P, None]) -> Callable[P, None]:
1430
- def wrapper(*args: P.args, **kwargs: P.kwargs) -> None:
1431
- if wrapper.has_run: # type: ignore[attr-defined]
1432
- return
1433
-
1434
- with wrapper.lock: # type: ignore[attr-defined]
1435
- if not wrapper.has_run: # type: ignore[attr-defined]
1436
- wrapper.has_run = True # type: ignore[attr-defined]
1437
- return f(*args, **kwargs)
1438
-
1439
- wrapper.has_run = False # type: ignore[attr-defined]
1440
- wrapper.lock = threading.Lock() # type: ignore[attr-defined]
1441
- return wrapper
1442
-
1443
-
1324
class StoreBoolean(Action):
1325
def __call__(self, parser, namespace, values, option_string=None):
1326
if values.lower() == "true":
@@ -1929,122 +1809,6 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args, **kwarg
1809
return await task(*args, **kwargs)
1810
1811
1932
-@lru_cache
1933
-def supports_kw(
1934
- callable: Callable[..., object],
1935
- kw_name: str,
1936
- *,
1937
- requires_kw_only: bool = False,
1938
- allow_var_kwargs: bool = True,
1939
-) -> bool:
1940
- """Check if a keyword is a valid kwarg for a callable; if requires_kw_only
1941
- disallows kwargs names that can also be positional arguments.
1942
- """
1943
- params = inspect.signature(callable).parameters
1944
- if not params:
1945
- return False
1946
-
1947
- param_val = params.get(kw_name)
1948
-
1949
- # Types where the it may be valid, i.e., explicitly defined & nonvariadic
1950
- passable_kw_types = set(
1951
- (
1952
- inspect.Parameter.POSITIONAL_ONLY,
1953
- inspect.Parameter.POSITIONAL_OR_KEYWORD,
1954
- inspect.Parameter.KEYWORD_ONLY,
1955
- )
1956
- )
1957
-
1958
- if param_val:
1959
- is_sig_param = param_val.kind in passable_kw_types
1960
- # We want kwargs only, but this is passable as a positional arg
1961
- if (
1962
- requires_kw_only
1963
- and is_sig_param
1964
- and param_val.kind != inspect.Parameter.KEYWORD_ONLY
1965
- ):
1966
- return False
1967
- if (requires_kw_only and param_val.kind == inspect.Parameter.KEYWORD_ONLY) or (
1968
- not requires_kw_only and is_sig_param
1969
- ):
1970
- return True
1971
-
1972
- # If we're okay with var-kwargs, it's supported as long as
1973
- # the kw_name isn't something like *args, **kwargs
1974
- if allow_var_kwargs:
1975
- # Get the last param; type is ignored here because params is a proxy
1976
- # mapping, but it wraps an ordered dict, and they appear in order.
1977
- # Ref: https://docs.python.org/3/library/inspect.html#inspect.Signature.parameters
1978
- last_param = params[next(reversed(params))] # type: ignore
1979
- return (
1980
- last_param.kind == inspect.Parameter.VAR_KEYWORD
1981
- and last_param.name != kw_name
1982
- )
1983
-
1984
- return False
1985
-
1986
-
1987
-def get_allowed_kwarg_only_overrides(
1988
- callable: Callable[..., object],
1989
- overrides: Mapping[str, object] | None,
1990
- *,
1991
- requires_kw_only: bool = True,
1992
- allow_var_kwargs: bool = False,
1993
-) -> dict[str, Any]:
1994
- """
1995
- Given a callable which has one or more keyword only params and a dict
1996
- mapping param names to values, drop values that can be not be kwarg
1997
- expanded to overwrite one or more keyword-only args. This is used in a
1998
- few places to handle custom processor overrides for multimodal models,
1999
- e.g., for profiling when processor options provided by the user
2000
- may affect the number of mm tokens per instance.
2001
-
2002
- Args:
2003
- callable: Callable which takes 0 or more keyword only arguments.
2004
- If None is provided, all overrides names are allowed.
2005
- overrides: Potential overrides to be used when invoking the callable.
2006
- allow_var_kwargs: Allows overrides that are expandable for var kwargs.
2007
-
2008
- Returns:
2009
- Dictionary containing the kwargs to be leveraged which may be used
2010
- to overwrite one or more keyword only arguments when invoking the
2011
- callable.
2012
- """
2013
- if not overrides:
2014
- return {}
2015
-
2016
- # Drop any mm_processor_kwargs provided by the user that
2017
- # are not kwargs, unless it can fit it var_kwargs param
2018
- filtered_overrides = {
2019
- kwarg_name: val
2020
- for kwarg_name, val in overrides.items()
2021
- if supports_kw(
2022
- callable,
2023
- kwarg_name,
2024
- requires_kw_only=requires_kw_only,
2025
- allow_var_kwargs=allow_var_kwargs,
2026
- )
2027
- }
2028
-
2029
- # If anything is dropped, log a warning
2030
- dropped_keys = overrides.keys() - filtered_overrides.keys()
2031
- if dropped_keys:
2032
- if requires_kw_only:
2033
- logger.warning(
2034
- "The following intended overrides are not keyword-only args "
2035
- "and will be dropped: %s",
2036
- dropped_keys,
2037
- )
2038
- else:
2039
- logger.warning(
2040
- "The following intended overrides are not keyword args "
2041
- "and will be dropped: %s",
2042
- dropped_keys,
2043
- )
2044
-
2045
- return filtered_overrides
2046
-
2047
-
1812
# Using dynamo with vLLM doesn't really work well with PyTorch versions < 2.4.0.
1813
# In particular, the FakeScalarType is not supported for earlier versions of
1814
# PyTorch which breaks dynamo for any ops registered using ScalarType.
vllm/utils/func.py
new
+258
@@ -0,0 +1,258 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Contains helpers that are applied to functions.
5
+
6
+This is similar in concept to the `functools` module.
7
+"""
8
+
9
+import asyncio
10
+import concurrent.futures
11
+import inspect
12
+import threading
13
+import warnings
14
+from collections.abc import Awaitable, Callable, Mapping
15
+from functools import lru_cache, partial, wraps
16
+from typing import Any, TypeVar
17
+
18
+from typing_extensions import ParamSpec
19
+
20
+from vllm.logger import init_logger
21
+
22
+logger = init_logger(__name__)
23
+
24
+
25
+P = ParamSpec("P")
26
+T = TypeVar("T")
27
+F = TypeVar("F", bound=Callable[..., Any])
28
+
29
+
30
+def identity(value: T, **kwargs) -> T:
31
+ """Returns the first provided value."""
32
+ return value
33
+
34
+
35
+def make_async(
36
+ func: Callable[P, T],
37
+ executor: concurrent.futures.Executor | None = None,
38
+) -> Callable[P, Awaitable[T]]:
39
+ """
40
+ Take a blocking function, and run it on in an executor thread.
41
+
42
+ This function prevents the blocking function from blocking the
43
+ asyncio event loop.
44
+ The code in this function needs to be thread safe.
45
+ """
46
+
47
+ def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> asyncio.Future[T]:
48
+ loop = asyncio.get_event_loop()
49
+ p_func = partial(func, *args, **kwargs)
50
+ return loop.run_in_executor(executor=executor, func=p_func)
51
+
52
+ return _async_wrapper
53
+
54
+
55
+def run_once(f: Callable[P, None]) -> Callable[P, None]:
56
+ def wrapper(*args: P.args, **kwargs: P.kwargs) -> None:
57
+ if wrapper.has_run: # type: ignore[attr-defined]
58
+ return
59
+
60
+ with wrapper.lock: # type: ignore[attr-defined]
61
+ if not wrapper.has_run: # type: ignore[attr-defined]
62
+ wrapper.has_run = True # type: ignore[attr-defined]
63
+ return f(*args, **kwargs)
64
+
65
+ wrapper.has_run = False # type: ignore[attr-defined]
66
+ wrapper.lock = threading.Lock() # type: ignore[attr-defined]
67
+ return wrapper
68
+
69
+
70
+def deprecate_args(
71
+ start_index: int,
72
+ is_deprecated: bool | Callable[[], bool] = True,
73
+ additional_message: str | None = None,
74
+) -> Callable[[F], F]:
75
+ if not callable(is_deprecated):
76
+ is_deprecated = partial(identity, is_deprecated)
77
+
78
+ def wrapper(fn: F) -> F:
79
+ params = inspect.signature(fn).parameters
80
+ pos_types = (
81
+ inspect.Parameter.POSITIONAL_ONLY,
82
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
83
+ )
84
+ pos_kws = [kw for kw, param in params.items() if param.kind in pos_types]
85
+
86
+ @wraps(fn)
87
+ def inner(*args, **kwargs):
88
+ if is_deprecated():
89
+ deprecated_args = pos_kws[start_index : len(args)]
90
+ if deprecated_args:
91
+ msg = (
92
+ f"The positional arguments {deprecated_args} are "
93
+ "deprecated and will be removed in a future update."
94
+ )
95
+ if additional_message is not None:
96
+ msg += f" {additional_message}"
97
+
98
+ warnings.warn(
99
+ DeprecationWarning(msg),
100
+ stacklevel=3, # The inner function takes up one level
101
+ )
102
+
103
+ return fn(*args, **kwargs)
104
+
105
+ return inner # type: ignore
106
+
107
+ return wrapper
108
+
109
+
110
+def deprecate_kwargs(
111
+ *kws: str,
112
+ is_deprecated: bool | Callable[[], bool] = True,
113
+ additional_message: str | None = None,
114
+) -> Callable[[F], F]:
115
+ deprecated_kws = set(kws)
116
+
117
+ if not callable(is_deprecated):
118
+ is_deprecated = partial(identity, is_deprecated)
119
+
120
+ def wrapper(fn: F) -> F:
121
+ @wraps(fn)
122
+ def inner(*args, **kwargs):
123
+ if is_deprecated():
124
+ deprecated_kwargs = kwargs.keys() & deprecated_kws
125
+ if deprecated_kwargs:
126
+ msg = (
127
+ f"The keyword arguments {deprecated_kwargs} are "
128
+ "deprecated and will be removed in a future update."
129
+ )
130
+ if additional_message is not None:
131
+ msg += f" {additional_message}"
132
+
133
+ warnings.warn(
134
+ DeprecationWarning(msg),
135
+ stacklevel=3, # The inner function takes up one level
136
+ )
137
+
138
+ return fn(*args, **kwargs)
139
+
140
+ return inner # type: ignore
141
+
142
+ return wrapper
143
+
144
+
145
+@lru_cache
146
+def supports_kw(
147
+ callable: Callable[..., object],
148
+ kw_name: str,
149
+ *,
150
+ requires_kw_only: bool = False,
151
+ allow_var_kwargs: bool = True,
152
+) -> bool:
153
+ """Check if a keyword is a valid kwarg for a callable; if requires_kw_only
154
+ disallows kwargs names that can also be positional arguments.
155
+ """
156
+ params = inspect.signature(callable).parameters
157
+ if not params:
158
+ return False
159
+
160
+ param_val = params.get(kw_name)
161
+
162
+ # Types where the it may be valid, i.e., explicitly defined & nonvariadic
163
+ passable_kw_types = set(
164
+ (
165
+ inspect.Parameter.POSITIONAL_ONLY,
166
+ inspect.Parameter.POSITIONAL_OR_KEYWORD,
167
+ inspect.Parameter.KEYWORD_ONLY,
168
+ )
169
+ )
170
+
171
+ if param_val:
172
+ is_sig_param = param_val.kind in passable_kw_types
173
+ # We want kwargs only, but this is passable as a positional arg
174
+ if (
175
+ requires_kw_only
176
+ and is_sig_param
177
+ and param_val.kind != inspect.Parameter.KEYWORD_ONLY
178
+ ):
179
+ return False
180
+ if (requires_kw_only and param_val.kind == inspect.Parameter.KEYWORD_ONLY) or (
181
+ not requires_kw_only and is_sig_param
182
+ ):
183
+ return True
184
+
185
+ # If we're okay with var-kwargs, it's supported as long as
186
+ # the kw_name isn't something like *args, **kwargs
187
+ if allow_var_kwargs:
188
+ # Get the last param; type is ignored here because params is a proxy
189
+ # mapping, but it wraps an ordered dict, and they appear in order.
190
+ # Ref: https://docs.python.org/3/library/inspect.html#inspect.Signature.parameters
191
+ last_param = params[next(reversed(params))] # type: ignore
192
+ return (
193
+ last_param.kind == inspect.Parameter.VAR_KEYWORD
194
+ and last_param.name != kw_name
195
+ )
196
+
197
+ return False
198
+
199
+
200
+def get_allowed_kwarg_only_overrides(
201
+ callable: Callable[..., object],
202
+ overrides: Mapping[str, object] | None,
203
+ *,
204
+ requires_kw_only: bool = True,
205
+ allow_var_kwargs: bool = False,
206
+) -> dict[str, Any]:
207
+ """
208
+ Given a callable which has one or more keyword only params and a dict
209
+ mapping param names to values, drop values that can be not be kwarg
210
+ expanded to overwrite one or more keyword-only args. This is used in a
211
+ few places to handle custom processor overrides for multimodal models,
212
+ e.g., for profiling when processor options provided by the user
213
+ may affect the number of mm tokens per instance.
214
+
215
+ Args:
216
+ callable: Callable which takes 0 or more keyword only arguments.
217
+ If None is provided, all overrides names are allowed.
218
+ overrides: Potential overrides to be used when invoking the callable.
219
+ allow_var_kwargs: Allows overrides that are expandable for var kwargs.
220
+
221
+ Returns:
222
+ Dictionary containing the kwargs to be leveraged which may be used
223
+ to overwrite one or more keyword only arguments when invoking the
224
+ callable.
225
+ """
226
+ if not overrides:
227
+ return {}
228
+
229
+ # Drop any mm_processor_kwargs provided by the user that
230
+ # are not kwargs, unless it can fit it var_kwargs param
231
+ filtered_overrides = {
232
+ kwarg_name: val
233
+ for kwarg_name, val in overrides.items()
234
+ if supports_kw(
235
+ callable,
236
+ kwarg_name,
237
+ requires_kw_only=requires_kw_only,
238
+ allow_var_kwargs=allow_var_kwargs,
239
+ )
240
+ }
241
+
242
+ # If anything is dropped, log a warning
243
+ dropped_keys = overrides.keys() - filtered_overrides.keys()
244
+ if dropped_keys:
245
+ if requires_kw_only:
246
+ logger.warning(
247
+ "The following intended overrides are not keyword-only args "
248
+ "and will be dropped: %s",
249
+ dropped_keys,
250
+ )
251
+ else:
252
+ logger.warning(
253
+ "The following intended overrides are not keyword args "
254
+ "and will be dropped: %s",
255
+ dropped_keys,
256
+ )
257
+
258
+ return filtered_overrides
vllm/v1/engine/async_llm.py
+2
-1
@@ -29,7 +29,8 @@ from vllm.tracing import init_tracer
29
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
30
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
31
from vllm.usage.usage_lib import UsageContext
32
-from vllm.utils import Device, as_list, cancel_task_threadsafe, cdiv, deprecate_kwargs
32
+from vllm.utils import Device, as_list, cancel_task_threadsafe, cdiv
33
+from vllm.utils.func import deprecate_kwargs
34
from vllm.v1.engine import EngineCoreRequest
35
from vllm.v1.engine.core_client import EngineCoreClient
36
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError