A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Fix `nullable_kvs` fallback (#16837)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Apr 18, 2025 at 13:58 UTC
686623c5e7a0ee0c7679c052ced565dd83055709
8 files changed
+27
-19
tests/engine/test_arg_utils.py
+1
-1
@@ -10,7 +10,7 @@ from vllm.utils import FlexibleArgumentParser
10
11
12
@pytest.mark.parametrize(("arg", "expected"), [
13
- (None, None),
13
+ (None, dict()),
14
("image=16", {
15
"image": 16
16
}),
tests/entrypoints/openai/test_audio.py
+3
-1
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+import json
4
+
5
import openai
6
import pytest
7
import pytest_asyncio
@@ -27,7 +29,7 @@ def server():
29
"--enforce-eager",
30
"--trust-remote-code",
31
"--limit-mm-per-prompt",
30
- str({"audio": MAXIMUM_AUDIOS}),
32
+ json.dumps({"audio": MAXIMUM_AUDIOS}),
33
]
34
35
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
tests/entrypoints/openai/test_video.py
+3
-1
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+import json
4
+
5
import openai
6
import pytest
7
import pytest_asyncio
@@ -31,7 +33,7 @@ def server():
33
"--enforce-eager",
34
"--trust-remote-code",
35
"--limit-mm-per-prompt",
34
- str({"video": MAXIMUM_VIDEOS}),
36
+ json.dumps({"video": MAXIMUM_VIDEOS}),
37
]
38
39
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
tests/entrypoints/openai/test_vision.py
+3
-1
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+import json
4
+
5
import openai
6
import pytest
7
import pytest_asyncio
@@ -35,7 +37,7 @@ def server():
37
"--enforce-eager",
38
"--trust-remote-code",
39
"--limit-mm-per-prompt",
38
- str({"image": MAXIMUM_IMAGES}),
40
+ json.dumps({"image": MAXIMUM_IMAGES}),
41
]
42
43
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
tests/entrypoints/openai/test_vision_embedding.py
+3
-1
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+import json
4
+
5
import pytest
6
import requests
7
from PIL import Image
@@ -37,7 +39,7 @@ def server():
39
"--enforce-eager",
40
"--trust-remote-code",
41
"--limit-mm-per-prompt",
40
- str({"image": MAXIMUM_IMAGES}),
42
+ json.dumps({"image": MAXIMUM_IMAGES}),
43
"--chat-template",
44
str(vlm2vec_jinja_path),
45
]
tests/models/decoder_only/audio_language/test_ultravox.py
+2
-1
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+import json
4
from typing import Optional
5
6
import numpy as np
@@ -50,7 +51,7 @@ def server(request, audio_assets):
51
args = [
52
"--dtype", "bfloat16", "--max-model-len", "4096", "--enforce-eager",
53
"--limit-mm-per-prompt",
53
- str({"audio": len(audio_assets)}), "--trust-remote-code"
54
+ json.dumps({"audio": len(audio_assets)}), "--trust-remote-code"
55
] + [
56
f"--{key.replace('_','-')}={value}"
57
for key, value in request.param.items()
vllm/config.py
+3
-4
@@ -10,7 +10,6 @@ import sys
10
import textwrap
11
import warnings
12
from collections import Counter
13
-from collections.abc import Mapping
13
from contextlib import contextmanager
14
from dataclasses import (MISSING, dataclass, field, fields, is_dataclass,
15
replace)
@@ -355,7 +354,7 @@ class ModelConfig:
354
disable_cascade_attn: bool = False,
355
skip_tokenizer_init: bool = False,
356
served_model_name: Optional[Union[str, list[str]]] = None,
358
- limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
357
+ limit_mm_per_prompt: Optional[dict[str, int]] = None,
358
use_async_output_proc: bool = True,
359
config_format: ConfigFormat = ConfigFormat.AUTO,
360
hf_token: Optional[Union[bool, str]] = None,
@@ -578,7 +577,7 @@ class ModelConfig:
577
self.tokenizer = s3_tokenizer.dir
578
579
def _init_multimodal_config(
581
- self, limit_mm_per_prompt: Optional[Mapping[str, int]]
580
+ self, limit_mm_per_prompt: Optional[dict[str, int]]
581
) -> Optional["MultiModalConfig"]:
582
if self.registry.is_multimodal_model(self.architectures):
583
return MultiModalConfig(limit_per_prompt=limit_mm_per_prompt or {})
@@ -2730,7 +2729,7 @@ class PromptAdapterConfig:
2729
class MultiModalConfig:
2730
"""Controls the behavior of multimodal models."""
2731
2733
- limit_per_prompt: Mapping[str, int] = field(default_factory=dict)
2732
+ limit_per_prompt: dict[str, int] = field(default_factory=dict)
2733
"""
2734
The maximum number of input items allowed per prompt for each modality.
2735
This should be a JSON string that will be parsed into a dictionary.
vllm/engine/arg_utils.py
+9
-9
@@ -7,7 +7,7 @@ import json
7
import re
8
import threading
9
from dataclasses import MISSING, dataclass, fields
10
-from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal, Mapping,
10
+from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal,
11
Optional, Tuple, Type, TypeVar, Union, cast, get_args,
12
get_origin)
13
@@ -112,14 +112,14 @@ def nullable_kvs(val: str) -> Optional[dict[str, int]]:
112
113
114
def optional_dict(val: str) -> Optional[dict[str, int]]:
115
- try:
115
+ if re.match("^{.*}$", val):
116
return optional_arg(val, json.loads)
117
- except ValueError:
118
- logger.warning(
119
- "Failed to parse JSON string. Attempting to parse as "
120
- "comma-separated key=value pairs. This will be deprecated in a "
121
- "future release.")
122
- return nullable_kvs(val)
117
+
118
+ logger.warning(
119
+ "Failed to parse JSON string. Attempting to parse as "
120
+ "comma-separated key=value pairs. This will be deprecated in a "
121
+ "future release.")
122
+ return nullable_kvs(val)
123
124
125
@dataclass
@@ -191,7 +191,7 @@ class EngineArgs:
191
TokenizerPoolConfig.pool_type
192
tokenizer_pool_extra_config: dict[str, Any] = \
193
get_field(TokenizerPoolConfig, "extra_config")
194
- limit_mm_per_prompt: Mapping[str, int] = \
194
+ limit_mm_per_prompt: dict[str, int] = \
195
get_field(MultiModalConfig, "limit_per_prompt")
196
mm_processor_kwargs: Optional[Dict[str, Any]] = None
197
disable_mm_preprocessor_cache: bool = False