A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Move `DeviceConfig`, `ObservabilityConfig`, `SpeechToTextConfig` to their own files (#25564)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Sep 24, 2025 at 14:59 UTC
8938774c79f185035bc3de5f19cfc7abaa242a5a
4 files changed
+215
-183
vllm/config/__init__.py
+3
-183
@@ -27,6 +27,7 @@ from vllm.config.cache import (BlockSize, CacheConfig, CacheDType, MambaDType,
27
PrefixCachingHashAlgo)
28
from vllm.config.compilation import (CompilationConfig, CompilationLevel,
29
CUDAGraphMode, PassConfig)
30
+from vllm.config.device import Device, DeviceConfig
31
from vllm.config.kv_events import KVEventsConfig
32
from vllm.config.kv_transfer import KVTransferConfig
33
from vllm.config.load import LoadConfig
@@ -38,11 +39,13 @@ from vllm.config.model import (ConvertOption, HfOverrides, LogprobsMode,
39
try_match_architecture_defaults)
40
from vllm.config.multimodal import (MMCacheType, MMEncoderTPMode,
41
MultiModalConfig)
42
+from vllm.config.observability import DetailedTraceModules, ObservabilityConfig
43
from vllm.config.parallel import (DistributedExecutorBackend, EPLBConfig,
44
ParallelConfig)
45
from vllm.config.pooler import PoolerConfig
46
from vllm.config.scheduler import RunnerType, SchedulerConfig, SchedulerPolicy
47
from vllm.config.speculative import SpeculativeConfig
48
+from vllm.config.speech_to_text import SpeechToTextConfig
49
from vllm.config.structured_outputs import StructuredOutputsConfig
50
from vllm.config.utils import ConfigType, config, get_attr_docs, is_init_field
51
from vllm.logger import init_logger
@@ -81,158 +84,6 @@ class SupportsMetricsInfo(Protocol):
84
...
85
86
84
-Device = Literal["auto", "cuda", "cpu", "tpu", "xpu"]
85
-
86
-
87
-@config
88
-@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
89
-class DeviceConfig:
90
- """Configuration for the device to use for vLLM execution."""
91
-
92
- device: SkipValidation[Optional[Union[Device, torch.device]]] = "auto"
93
- """Device type for vLLM execution.
94
- This parameter is deprecated and will be
95
- removed in a future release.
96
- It will now be set automatically based
97
- on the current platform."""
98
- device_type: str = field(init=False)
99
- """Device type from the current platform. This is set in
100
- `__post_init__`."""
101
-
102
- def compute_hash(self) -> str:
103
- """
104
- WARNING: Whenever a new field is added to this config,
105
- ensure that it is included in the factors list if
106
- it affects the computation graph.
107
-
108
- Provide a hash that uniquely identifies all the configs
109
- that affect the structure of the computation
110
- graph from input ids/embeddings to the final hidden states,
111
- excluding anything before input ids/embeddings and after
112
- the final hidden states.
113
- """
114
- # no factors to consider.
115
- # the device/platform information will be summarized
116
- # by torch/vllm automatically.
117
- factors: list[Any] = []
118
- hash_str = hashlib.md5(str(factors).encode(),
119
- usedforsecurity=False).hexdigest()
120
- return hash_str
121
-
122
- def __post_init__(self):
123
- if self.device == "auto":
124
- # Automated device type detection
125
- from vllm.platforms import current_platform
126
- self.device_type = current_platform.device_type
127
- if not self.device_type:
128
- raise RuntimeError(
129
- "Failed to infer device type, please set "
130
- "the environment variable `VLLM_LOGGING_LEVEL=DEBUG` "
131
- "to turn on verbose logging to help debug the issue.")
132
- else:
133
- # Device type is assigned explicitly
134
- if isinstance(self.device, str):
135
- self.device_type = self.device
136
- elif isinstance(self.device, torch.device):
137
- self.device_type = self.device.type
138
-
139
- # Some device types require processing inputs on CPU
140
- if self.device_type in ["tpu"]:
141
- self.device = None
142
- else:
143
- # Set device with device type
144
- self.device = torch.device(self.device_type)
145
-
146
-
147
-DetailedTraceModules = Literal["model", "worker", "all"]
148
-
149
-
150
-@config
151
-@dataclass
152
-class ObservabilityConfig:
153
- """Configuration for observability - metrics and tracing."""
154
-
155
- show_hidden_metrics_for_version: Optional[str] = None
156
- """Enable deprecated Prometheus metrics that have been hidden since the
157
- specified version. For example, if a previously deprecated metric has been
158
- hidden since the v0.7.0 release, you use
159
- `--show-hidden-metrics-for-version=0.7` as a temporary escape hatch while
160
- you migrate to new metrics. The metric is likely to be removed completely
161
- in an upcoming release."""
162
-
163
- @cached_property
164
- def show_hidden_metrics(self) -> bool:
165
- """Check if the hidden metrics should be shown."""
166
- if self.show_hidden_metrics_for_version is None:
167
- return False
168
- return version._prev_minor_version_was(
169
- self.show_hidden_metrics_for_version)
170
-
171
- otlp_traces_endpoint: Optional[str] = None
172
- """Target URL to which OpenTelemetry traces will be sent."""
173
-
174
- collect_detailed_traces: Optional[list[DetailedTraceModules]] = None
175
- """It makes sense to set this only if `--otlp-traces-endpoint` is set. If
176
- set, it will collect detailed traces for the specified modules. This
177
- involves use of possibly costly and or blocking operations and hence might
178
- have a performance impact.
179
-
180
- Note that collecting detailed timing information for each request can be
181
- expensive."""
182
-
183
- @cached_property
184
- def collect_model_forward_time(self) -> bool:
185
- """Whether to collect model forward time for the request."""
186
- return (self.collect_detailed_traces is not None
187
- and ("model" in self.collect_detailed_traces
188
- or "all" in self.collect_detailed_traces))
189
-
190
- @cached_property
191
- def collect_model_execute_time(self) -> bool:
192
- """Whether to collect model execute time for the request."""
193
- return (self.collect_detailed_traces is not None
194
- and ("worker" in self.collect_detailed_traces
195
- or "all" in self.collect_detailed_traces))
196
-
197
- def compute_hash(self) -> str:
198
- """
199
- WARNING: Whenever a new field is added to this config,
200
- ensure that it is included in the factors list if
201
- it affects the computation graph.
202
-
203
- Provide a hash that uniquely identifies all the configs
204
- that affect the structure of the computation
205
- graph from input ids/embeddings to the final hidden states,
206
- excluding anything before input ids/embeddings and after
207
- the final hidden states.
208
- """
209
- # no factors to consider.
210
- # this config will not affect the computation graph.
211
- factors: list[Any] = []
212
- hash_str = hashlib.md5(str(factors).encode(),
213
- usedforsecurity=False).hexdigest()
214
- return hash_str
215
-
216
- def __post_init__(self):
217
- if (self.collect_detailed_traces is not None
218
- and len(self.collect_detailed_traces) == 1
219
- and "," in self.collect_detailed_traces[0]):
220
- self._parse_collect_detailed_traces()
221
-
222
- from vllm.tracing import is_otel_available, otel_import_error_traceback
223
- if not is_otel_available() and self.otlp_traces_endpoint is not None:
224
- raise ValueError(
225
- "OpenTelemetry is not available. Unable to configure "
226
- "'otlp_traces_endpoint'. Ensure OpenTelemetry packages are "
227
- f"installed. Original error:\n{otel_import_error_traceback}")
228
-
229
- def _parse_collect_detailed_traces(self):
230
- assert isinstance(self.collect_detailed_traces, list)
231
- self.collect_detailed_traces = cast(
232
- list[DetailedTraceModules],
233
- self.collect_detailed_traces[0].split(","))
234
-
235
-
87
@config
88
@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
89
class VllmConfig:
@@ -1009,37 +860,6 @@ def get_layers_from_vllm_config(
860
}
861
862
1012
-@config
1013
-@dataclass
1014
-class SpeechToTextConfig:
1015
- """Configuration for speech-to-text models."""
1016
-
1017
- sample_rate: float = 16_000
1018
- """Sample rate (Hz) to resample input audio to. Most speech models expect
1019
- 16kHz audio input. The input audio will be automatically resampled to this
1020
- rate before processing."""
1021
-
1022
- max_audio_clip_s: int = 30
1023
- """Maximum duration in seconds for a single audio clip without chunking.
1024
- Audio longer than this will be split into smaller chunks if
1025
- `allow_audio_chunking` evaluates to True, otherwise it will be rejected."""
1026
-
1027
- overlap_chunk_second: int = 1
1028
- """Overlap duration in seconds between consecutive audio chunks when
1029
- splitting long audio. This helps maintain context across chunk boundaries
1030
- and improves transcription quality at split points."""
1031
-
1032
- min_energy_split_window_size: Optional[int] = 1600
1033
- """Window size in samples for finding low-energy (quiet) regions to split
1034
- audio chunks. The algorithm looks for the quietest moment within this
1035
- window to minimize cutting through speech. Default 1600 samples ≈ 100ms
1036
- at 16kHz. If None, no chunking will be done."""
1037
-
1038
- @property
1039
- def allow_audio_chunking(self) -> bool:
1040
- return self.min_energy_split_window_size is not None
1041
-
1042
-
863
def update_config(config: DataclassInstanceT,
864
overrides: dict[str, Any]) -> DataclassInstanceT:
865
processed_overrides = {}
vllm/config/device.py
new
+74
@@ -0,0 +1,74 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import hashlib
5
+from dataclasses import field
6
+from typing import Any, Literal, Optional, Union
7
+
8
+import torch
9
+from pydantic import ConfigDict, SkipValidation
10
+from pydantic.dataclasses import dataclass
11
+
12
+from vllm.config.utils import config
13
+
14
+Device = Literal["auto", "cuda", "cpu", "tpu", "xpu"]
15
+
16
+
17
+@config
18
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
19
+class DeviceConfig:
20
+ """Configuration for the device to use for vLLM execution."""
21
+
22
+ device: SkipValidation[Optional[Union[Device, torch.device]]] = "auto"
23
+ """Device type for vLLM execution.
24
+ This parameter is deprecated and will be
25
+ removed in a future release.
26
+ It will now be set automatically based
27
+ on the current platform."""
28
+ device_type: str = field(init=False)
29
+ """Device type from the current platform. This is set in
30
+ `__post_init__`."""
31
+
32
+ def compute_hash(self) -> str:
33
+ """
34
+ WARNING: Whenever a new field is added to this config,
35
+ ensure that it is included in the factors list if
36
+ it affects the computation graph.
37
+
38
+ Provide a hash that uniquely identifies all the configs
39
+ that affect the structure of the computation
40
+ graph from input ids/embeddings to the final hidden states,
41
+ excluding anything before input ids/embeddings and after
42
+ the final hidden states.
43
+ """
44
+ # no factors to consider.
45
+ # the device/platform information will be summarized
46
+ # by torch/vllm automatically.
47
+ factors: list[Any] = []
48
+ hash_str = hashlib.md5(str(factors).encode(),
49
+ usedforsecurity=False).hexdigest()
50
+ return hash_str
51
+
52
+ def __post_init__(self):
53
+ if self.device == "auto":
54
+ # Automated device type detection
55
+ from vllm.platforms import current_platform
56
+ self.device_type = current_platform.device_type
57
+ if not self.device_type:
58
+ raise RuntimeError(
59
+ "Failed to infer device type, please set "
60
+ "the environment variable `VLLM_LOGGING_LEVEL=DEBUG` "
61
+ "to turn on verbose logging to help debug the issue.")
62
+ else:
63
+ # Device type is assigned explicitly
64
+ if isinstance(self.device, str):
65
+ self.device_type = self.device
66
+ elif isinstance(self.device, torch.device):
67
+ self.device_type = self.device.type
68
+
69
+ # Some device types require processing inputs on CPU
70
+ if self.device_type in ["tpu"]:
71
+ self.device = None
72
+ else:
73
+ # Set device with device type
74
+ self.device = torch.device(self.device_type)
vllm/config/observability.py
new
+99
@@ -0,0 +1,99 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import hashlib
5
+from functools import cached_property
6
+from typing import Any, Literal, Optional, cast
7
+
8
+from pydantic.dataclasses import dataclass
9
+
10
+from vllm import version
11
+from vllm.config.utils import config
12
+
13
+DetailedTraceModules = Literal["model", "worker", "all"]
14
+
15
+
16
+@config
17
+@dataclass
18
+class ObservabilityConfig:
19
+ """Configuration for observability - metrics and tracing."""
20
+
21
+ show_hidden_metrics_for_version: Optional[str] = None
22
+ """Enable deprecated Prometheus metrics that have been hidden since the
23
+ specified version. For example, if a previously deprecated metric has been
24
+ hidden since the v0.7.0 release, you use
25
+ `--show-hidden-metrics-for-version=0.7` as a temporary escape hatch while
26
+ you migrate to new metrics. The metric is likely to be removed completely
27
+ in an upcoming release."""
28
+
29
+ @cached_property
30
+ def show_hidden_metrics(self) -> bool:
31
+ """Check if the hidden metrics should be shown."""
32
+ if self.show_hidden_metrics_for_version is None:
33
+ return False
34
+ return version._prev_minor_version_was(
35
+ self.show_hidden_metrics_for_version)
36
+
37
+ otlp_traces_endpoint: Optional[str] = None
38
+ """Target URL to which OpenTelemetry traces will be sent."""
39
+
40
+ collect_detailed_traces: Optional[list[DetailedTraceModules]] = None
41
+ """It makes sense to set this only if `--otlp-traces-endpoint` is set. If
42
+ set, it will collect detailed traces for the specified modules. This
43
+ involves use of possibly costly and or blocking operations and hence might
44
+ have a performance impact.
45
+
46
+ Note that collecting detailed timing information for each request can be
47
+ expensive."""
48
+
49
+ @cached_property
50
+ def collect_model_forward_time(self) -> bool:
51
+ """Whether to collect model forward time for the request."""
52
+ return (self.collect_detailed_traces is not None
53
+ and ("model" in self.collect_detailed_traces
54
+ or "all" in self.collect_detailed_traces))
55
+
56
+ @cached_property
57
+ def collect_model_execute_time(self) -> bool:
58
+ """Whether to collect model execute time for the request."""
59
+ return (self.collect_detailed_traces is not None
60
+ and ("worker" in self.collect_detailed_traces
61
+ or "all" in self.collect_detailed_traces))
62
+
63
+ def compute_hash(self) -> str:
64
+ """
65
+ WARNING: Whenever a new field is added to this config,
66
+ ensure that it is included in the factors list if
67
+ it affects the computation graph.
68
+
69
+ Provide a hash that uniquely identifies all the configs
70
+ that affect the structure of the computation
71
+ graph from input ids/embeddings to the final hidden states,
72
+ excluding anything before input ids/embeddings and after
73
+ the final hidden states.
74
+ """
75
+ # no factors to consider.
76
+ # this config will not affect the computation graph.
77
+ factors: list[Any] = []
78
+ hash_str = hashlib.md5(str(factors).encode(),
79
+ usedforsecurity=False).hexdigest()
80
+ return hash_str
81
+
82
+ def __post_init__(self):
83
+ if (self.collect_detailed_traces is not None
84
+ and len(self.collect_detailed_traces) == 1
85
+ and "," in self.collect_detailed_traces[0]):
86
+ self._parse_collect_detailed_traces()
87
+
88
+ from vllm.tracing import is_otel_available, otel_import_error_traceback
89
+ if not is_otel_available() and self.otlp_traces_endpoint is not None:
90
+ raise ValueError(
91
+ "OpenTelemetry is not available. Unable to configure "
92
+ "'otlp_traces_endpoint'. Ensure OpenTelemetry packages are "
93
+ f"installed. Original error:\n{otel_import_error_traceback}")
94
+
95
+ def _parse_collect_detailed_traces(self):
96
+ assert isinstance(self.collect_detailed_traces, list)
97
+ self.collect_detailed_traces = cast(
98
+ list[DetailedTraceModules],
99
+ self.collect_detailed_traces[0].split(","))
vllm/config/speech_to_text.py
new
+39
@@ -0,0 +1,39 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from typing import Optional
5
+
6
+from pydantic.dataclasses import dataclass
7
+
8
+from vllm.config.utils import config
9
+
10
+
11
+@config
12
+@dataclass
13
+class SpeechToTextConfig:
14
+ """Configuration for speech-to-text models."""
15
+
16
+ sample_rate: float = 16_000
17
+ """Sample rate (Hz) to resample input audio to. Most speech models expect
18
+ 16kHz audio input. The input audio will be automatically resampled to this
19
+ rate before processing."""
20
+
21
+ max_audio_clip_s: int = 30
22
+ """Maximum duration in seconds for a single audio clip without chunking.
23
+ Audio longer than this will be split into smaller chunks if
24
+ `allow_audio_chunking` evaluates to True, otherwise it will be rejected."""
25
+
26
+ overlap_chunk_second: int = 1
27
+ """Overlap duration in seconds between consecutive audio chunks when
28
+ splitting long audio. This helps maintain context across chunk boundaries
29
+ and improves transcription quality at split points."""
30
+
31
+ min_energy_split_window_size: Optional[int] = 1600
32
+ """Window size in samples for finding low-energy (quiet) regions to split
33
+ audio chunks. The algorithm looks for the quietest moment within this
34
+ window to minimize cutting through speech. Default 1600 samples ≈ 100ms
35
+ at 16kHz. If None, no chunking will be done."""
36
+
37
+ @property
38
+ def allow_audio_chunking(self) -> bool:
39
+ return self.min_energy_split_window_size is not None