A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
Fix interaction between `Optional` and `Annotated` in CLI typing (#19093)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Yikun Jiang <yikun@apache.org>
Harry Mellor committed
Jun 3, 2025 at 22:07 UTC
6865fe0074771ed56c1cb2eca047a8e74ab53ce9
2 files changed
+34
-10
tests/engine/test_arg_utils.py
+15
-3
@@ -5,14 +5,14 @@ import json
5
from argparse import ArgumentError, ArgumentTypeError
6
from contextlib import nullcontext
7
from dataclasses import dataclass, field
8
-from typing import Literal, Optional
8
+from typing import Annotated, Literal, Optional
9
10
import pytest
11
12
from vllm.config import CompilationConfig, config
13
from vllm.engine.arg_utils import (EngineArgs, contains_type, get_kwargs,
14
- get_type, is_not_builtin, is_type,
15
- literal_to_kwargs, nullable_kvs,
14
+ get_type, get_type_hints, is_not_builtin,
15
+ is_type, literal_to_kwargs, nullable_kvs,
16
optional_type, parse_type)
17
from vllm.utils import FlexibleArgumentParser
18
@@ -160,6 +160,18 @@ def test_is_not_builtin(type_hint, expected):
160
assert is_not_builtin(type_hint) == expected
161
162
163
+@pytest.mark.parametrize(
164
+ ("type_hint", "expected"), [
165
+ (Annotated[int, "annotation"], {int}),
166
+ (Optional[int], {int, type(None)}),
167
+ (Annotated[Optional[int], "annotation"], {int, type(None)}),
168
+ (Optional[Annotated[int, "annotation"]], {int, type(None)}),
169
+ ],
170
+ ids=["Annotated", "Optional", "Annotated_Optional", "Optional_Annotated"])
171
+def test_get_type_hints(type_hint, expected):
172
+ assert get_type_hints(type_hint) == expected
173
+
174
+
175
def test_get_kwargs():
176
kwargs = get_kwargs(DummyConfig)
177
print(kwargs)
vllm/engine/arg_utils.py
+19
-7
@@ -15,7 +15,7 @@ from typing import (Annotated, Any, Callable, Dict, List, Literal, Optional,
15
16
import regex as re
17
import torch
18
-from pydantic import SkipValidation, TypeAdapter, ValidationError
18
+from pydantic import TypeAdapter, ValidationError
19
from typing_extensions import TypeIs, deprecated
20
21
import vllm.envs as envs
@@ -151,17 +151,29 @@ def is_not_builtin(type_hint: TypeHint) -> bool:
151
return type_hint.__module__ != "builtins"
152
153
154
+def get_type_hints(type_hint: TypeHint) -> set[TypeHint]:
155
+ """Extract type hints from Annotated or Union type hints."""
156
+ type_hints: set[TypeHint] = set()
157
+ origin = get_origin(type_hint)
158
+ args = get_args(type_hint)
159
+
160
+ if origin is Annotated:
161
+ type_hints.update(get_type_hints(args[0]))
162
+ elif origin is Union:
163
+ for arg in args:
164
+ type_hints.update(get_type_hints(arg))
165
+ else:
166
+ type_hints.add(type_hint)
167
+
168
+ return type_hints
169
+
170
+
171
def get_kwargs(cls: ConfigType) -> dict[str, Any]:
172
cls_docs = get_attr_docs(cls)
173
kwargs = {}
174
for field in fields(cls):
175
# Get the set of possible types for the field
159
- type_hints: set[TypeHint] = set()
160
- if get_origin(field.type) in {Union, Annotated}:
161
- predicate = lambda arg: not isinstance(arg, SkipValidation)
162
- type_hints.update(filter(predicate, get_args(field.type)))
163
- else:
164
- type_hints.add(field.type)
176
+ type_hints: set[TypeHint] = get_type_hints(field.type)
177
178
# If the field is a dataclass, we can use the model_validate_json
179
generator = (th for th in type_hints if is_dataclass(th))