A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 04:30 UTC
[Config] Clean up SchedulerConfig initialization (#28665)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 14, 2025 at 22:41 UTC
511a6b611d2e7e6b13e09c050147b367434f1a54
9 files changed
+181
-162
tests/models/language/generation/test_hybrid.py
+6
-1
@@ -348,9 +348,14 @@ def test_fp32_cache_state(
348
349
350
# Helper functions for the APC tests
351
-def _get_vllm_runner_params(model, max_model_len, tensor_parallel_size=1):
351
+def _get_vllm_runner_params(
352
+ model: str,
353
+ max_model_len: int,
354
+ tensor_parallel_size: int = 1,
355
+):
356
return {
357
"model_name": model,
358
+ "enable_chunked_prefill": True,
359
"enable_prefix_caching": False,
360
"max_model_len": max_model_len,
361
"tensor_parallel_size": tensor_parallel_size,
tests/v1/core/test_scheduler.py
+2
@@ -2256,6 +2256,8 @@ def test_chunked_prefill_disabled_for_encoder_decoder(
2256
scheduler_config = SchedulerConfig(
2257
enable_chunked_prefill=enable_chunked_prefill,
2258
is_encoder_decoder=is_encoder_decoder,
2259
+ # Must <= max_num_batched_tokens if chunked prefill is disabled
2260
+ max_model_len=SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS,
2261
)
2262
2263
# `is_encoder_decoder` should only be used during construction
tests/v1/sample/test_logprobs.py
+1
@@ -47,6 +47,7 @@ def vllm_model(vllm_runner, request) -> Generator[VllmRunner, None, None]:
47
max_num_batched_tokens=16,
48
max_num_seqs=16,
49
max_model_len=128,
50
+ enable_chunked_prefill=True,
51
enforce_eager=True,
52
# TODO: enable this once we support it for
53
# prompt logprobs.
vllm/config/scheduler.py
+33
-69
@@ -4,7 +4,7 @@
4
import hashlib
5
from collections.abc import Callable
6
from dataclasses import InitVar
7
-from typing import TYPE_CHECKING, Any, Literal, cast
7
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
8
9
from pydantic import Field, field_validator, model_validator
10
from pydantic.dataclasses import dataclass
@@ -12,11 +12,6 @@ from typing_extensions import Self
12
13
from vllm.config.utils import config
14
from vllm.logger import init_logger
15
-from vllm.utils import (
16
- DEFAULT_MAX_NUM_BATCHED_TOKENS,
17
- MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,
18
- POOLING_MODEL_MAX_NUM_BATCHED_TOKENS,
19
-)
15
from vllm.utils.import_utils import resolve_obj_by_qualname
16
17
if TYPE_CHECKING:
@@ -33,25 +28,32 @@ SchedulerPolicy = Literal["fcfs", "priority"]
28
class SchedulerConfig:
29
"""Scheduler configuration."""
30
31
+ DEFAULT_MAX_NUM_BATCHED_TOKENS: ClassVar[int] = 2048
32
+ DEFAULT_MAX_NUM_SEQS: ClassVar[int] = 128
33
+
34
runner_type: RunnerType = "generate"
35
"""The runner type to launch for the model."""
36
39
- max_num_batched_tokens: int = Field(default=None, ge=1)
37
+ max_num_batched_tokens: int = Field(default=DEFAULT_MAX_NUM_BATCHED_TOKENS, ge=1)
38
"""Maximum number of tokens to be processed in a single iteration.
39
42
- This config has no static default. If left unspecified by the user, it will
43
- be set in `EngineArgs.create_engine_config` based on the usage context."""
40
+ The default value here is mainly for convenience when testing.
41
+ In real usage, this should be set in `EngineArgs.create_engine_config`.
42
+ """
43
45
- max_num_seqs: int = Field(default=None, ge=1)
44
+ max_num_seqs: int = Field(default=DEFAULT_MAX_NUM_SEQS, ge=1)
45
"""Maximum number of sequences to be processed in a single iteration.
46
48
- This config has no static default. If left unspecified by the user, it will
49
- be set in `EngineArgs.create_engine_config` based on the usage context."""
47
+ The default value here is mainly for convenience when testing.
48
+ In real usage, this should be set in `EngineArgs.create_engine_config`.
49
+ """
50
51
- max_model_len: int = Field(default=None, ge=1)
52
- """Maximum length of a sequence (including prompt and generated text). This
53
- is primarily set in `ModelConfig` and that value should be manually
54
- duplicated here."""
51
+ max_model_len: int = Field(default=8192, ge=1)
52
+ """Maximum length of a sequence (including prompt and generated text).
53
+
54
+ The default value here is mainly for convenience when testing.
55
+ In real usage, this should duplicate `ModelConfig.max_model_len` via
56
+ `EngineArgs`."""
57
58
max_num_partial_prefills: int = Field(default=1, ge=1)
59
"""For chunked prefill, the maximum number of sequences that can be
@@ -76,9 +78,13 @@ class SchedulerConfig:
78
NOTE: This will be replaced by speculative config in the future; it is
79
present to enable correctness tests until then."""
80
79
- enable_chunked_prefill: bool = Field(default=None)
81
+ enable_chunked_prefill: bool = True
82
"""If True, prefill requests can be chunked based
81
- on the remaining max_num_batched_tokens."""
83
+ on the remaining `max_num_batched_tokens`.
84
+
85
+ The default value here is mainly for convenience when testing.
86
+ In real usage, this should be set in `EngineArgs.create_engine_config`.
87
+ """
88
89
is_multimodal_model: bool = False
90
"""True if the model is multimodal."""
@@ -111,9 +117,6 @@ class SchedulerConfig:
117
- "priority" means requests are handled based on given priority (lower
118
value means earlier handling) and time of arrival deciding any ties)."""
119
114
- chunked_prefill_enabled: bool = Field(init=False)
115
- """True if chunked prefill is enabled."""
116
-
120
disable_chunked_mm_input: bool = False
121
"""If set to true and chunked prefill is enabled, we do not want to
122
partially schedule a multimodal item. Only used in V1
@@ -188,15 +191,7 @@ class SchedulerConfig:
191
hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest()
192
return hash_str
193
191
- @field_validator(
192
- "max_num_batched_tokens",
193
- "max_num_seqs",
194
- "max_model_len",
195
- "enable_chunked_prefill",
196
- "scheduler_cls",
197
- "async_scheduling",
198
- mode="wrap",
199
- )
194
+ @field_validator("scheduler_cls", "async_scheduling", mode="wrap")
195
@classmethod
196
def _skip_none_validation(cls, value: Any, handler: Callable) -> Any:
197
"""Skip validation if the value is `None` when initialisation is delayed."""
@@ -205,16 +200,9 @@ class SchedulerConfig:
200
return handler(value)
201
202
def __post_init__(self, is_encoder_decoder: bool) -> None:
208
- if self.max_model_len is None:
209
- self.max_model_len = 8192
210
-
211
- if self.max_num_seqs is None:
212
- self.max_num_seqs = 128
213
-
203
if is_encoder_decoder:
204
# Chunked prefill should be disabled for encoder-decoder models.
205
self.disable_chunked_mm_input = True
217
- self.chunked_prefill_enabled = False
206
self.enable_chunked_prefill = False
207
self.long_prefill_token_threshold = 0
208
logger.info(
@@ -222,37 +210,6 @@ class SchedulerConfig:
210
" prefix caching; disabling both."
211
)
212
225
- if self.max_num_batched_tokens is None:
226
- if self.enable_chunked_prefill:
227
- self.max_num_batched_tokens = DEFAULT_MAX_NUM_BATCHED_TOKENS
228
- else:
229
- # If max_model_len is too short, use
230
- # DEFAULT_MAX_NUM_BATCHED_TOKENS as the default value
231
- # for higher throughput.
232
- self.max_num_batched_tokens = max(
233
- self.max_model_len, DEFAULT_MAX_NUM_BATCHED_TOKENS
234
- )
235
-
236
- if self.runner_type == "pooling":
237
- # Choose specific value for higher throughput
238
- self.max_num_batched_tokens = max(
239
- self.max_num_batched_tokens,
240
- POOLING_MODEL_MAX_NUM_BATCHED_TOKENS,
241
- )
242
- if self.is_multimodal_model:
243
- # The value needs to be at least the number of multimodal tokens
244
- self.max_num_batched_tokens = max(
245
- self.max_num_batched_tokens,
246
- MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,
247
- )
248
-
249
- # When using default settings,
250
- # Ensure max_num_batched_tokens does not exceed model limit.
251
- # Some models (e.g., Whisper) have embeddings tied to max length.
252
- self.max_num_batched_tokens = min(
253
- self.max_num_seqs * self.max_model_len, self.max_num_batched_tokens
254
- )
255
-
213
self.max_num_encoder_input_tokens = self.max_num_batched_tokens
214
self.encoder_cache_size = self.max_num_batched_tokens
215
@@ -262,7 +219,6 @@ class SchedulerConfig:
219
self.max_num_batched_tokens,
220
)
221
265
- self.chunked_prefill_enabled = self.enable_chunked_prefill
222
if self.max_num_partial_prefills > 1:
223
if self.long_prefill_token_threshold == 0:
224
self.long_prefill_token_threshold = int(self.max_model_len * 0.04)
@@ -276,6 +232,14 @@ class SchedulerConfig:
232
self.long_prefill_token_threshold,
233
)
234
235
+ @property
236
+ def chunked_prefill_enabled(self) -> bool:
237
+ return self.enable_chunked_prefill
238
+
239
+ @chunked_prefill_enabled.setter
240
+ def chunked_prefill_enabled(self, value: bool):
241
+ self.enable_chunked_prefill = value
242
+
243
@model_validator(mode="after")
244
def _verify_args(self) -> Self:
245
if (
vllm/engine/arg_utils.py
+135
-73
@@ -428,11 +428,11 @@ class EngineArgs:
428
cpu_offload_gb: float = CacheConfig.cpu_offload_gb
429
gpu_memory_utilization: float = CacheConfig.gpu_memory_utilization
430
kv_cache_memory_bytes: int | None = CacheConfig.kv_cache_memory_bytes
431
- max_num_batched_tokens: int | None = SchedulerConfig.max_num_batched_tokens
431
+ max_num_batched_tokens: int | None = None
432
max_num_partial_prefills: int = SchedulerConfig.max_num_partial_prefills
433
max_long_partial_prefills: int = SchedulerConfig.max_long_partial_prefills
434
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
435
- max_num_seqs: int | None = SchedulerConfig.max_num_seqs
435
+ max_num_seqs: int | None = None
436
max_logprobs: int = ModelConfig.max_logprobs
437
logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
438
disable_log_stats: bool = False
@@ -485,7 +485,7 @@ class EngineArgs:
485
model_loader_extra_config: dict = get_field(LoadConfig, "model_loader_extra_config")
486
ignore_patterns: str | list[str] = get_field(LoadConfig, "ignore_patterns")
487
488
- enable_chunked_prefill: bool | None = SchedulerConfig.enable_chunked_prefill
488
+ enable_chunked_prefill: bool | None = None
489
disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input
490
491
disable_hybrid_kv_cache_manager: bool = (
@@ -1738,41 +1738,41 @@ class EngineArgs:
1738
)
1739
_raise_unsupported_error(feature_name=name)
1740
1741
- def _set_default_args(
1742
- self, usage_context: UsageContext, model_config: ModelConfig
1743
- ) -> None:
1744
- """Set Default Arguments for V1 Engine."""
1745
-
1746
- # V1 uses chunked prefills and prefix caching by default
1747
- # for non-pooling tasks.
1748
- # For pooling tasks the default is False
1741
+ @classmethod
1742
+ def get_chunked_prefill_prefix_caching_defaults(
1743
+ cls,
1744
+ model_config: ModelConfig,
1745
+ ) -> tuple[bool, bool]:
1746
if model_config.runner_type != "pooling":
1750
- self.enable_chunked_prefill = True
1751
-
1752
- if self.enable_prefix_caching is None:
1753
- # Disable prefix caching default for hybrid models
1754
- # since the feature is still experimental.
1755
- if model_config.is_hybrid:
1756
- self.enable_prefix_caching = False
1757
- else:
1758
- self.enable_prefix_caching = True
1747
+ default_chunked_prefill = True
1748
+
1749
+ # Disable prefix caching default for hybrid models
1750
+ # since the feature is still experimental.
1751
+ default_prefix_caching = not model_config.is_hybrid
1752
else:
1753
+ assert model_config.pooler_config is not None
1754
+
1755
pooling_type = model_config.pooler_config.pooling_type
1761
- is_causal = getattr(model_config.hf_config, "is_causal", True)
1756
incremental_prefill_supported = (
1757
pooling_type is not None
1758
and pooling_type.lower() == "last"
1765
- and bool(is_causal)
1759
+ and getattr(model_config.hf_config, "is_causal", True)
1760
)
1761
1768
- action = "Enabling" if incremental_prefill_supported else "Disabling"
1762
+ default_chunked_prefill = incremental_prefill_supported
1763
+ default_prefix_caching = incremental_prefill_supported
1764
+
1765
+ return default_chunked_prefill, default_prefix_caching
1766
+
1767
+ @classmethod
1768
+ def get_batch_defaults(
1769
+ cls,
1770
+ world_size: int,
1771
+ ) -> tuple[dict[UsageContext | None, int], dict[UsageContext | None, int]]:
1772
+ from vllm.usage.usage_lib import UsageContext
1773
1770
- if self.enable_chunked_prefill is None:
1771
- self.enable_chunked_prefill = incremental_prefill_supported
1772
- logger.info("(%s) chunked prefill by default", action)
1773
- if self.enable_prefix_caching is None:
1774
- self.enable_prefix_caching = incremental_prefill_supported
1775
- logger.info("(%s) prefix caching by default", action)
1774
+ default_max_num_batched_tokens: dict[UsageContext | None, int]
1775
+ default_max_num_seqs: dict[UsageContext | None, int]
1776
1777
# When no user override, set the default values based on the usage
1778
# context.
@@ -1793,8 +1793,6 @@ class EngineArgs:
1793
# NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces
1794
# throughput, see PR #17885 for more details.
1795
# So here we do an extra device name check to prevent such regression.
1796
- from vllm.usage.usage_lib import UsageContext
1797
-
1796
if device_memory >= 70 * GiB_bytes and "a100" not in device_name:
1797
# For GPUs like H100 and MI300x, use larger default values.
1798
default_max_num_batched_tokens = {
@@ -1818,22 +1816,26 @@ class EngineArgs:
1816
1817
# tpu specific default values.
1818
if current_platform.is_tpu():
1821
- default_max_num_batched_tokens_tpu = {
1822
- UsageContext.LLM_CLASS: {
1823
- "V6E": 2048,
1824
- "V5E": 1024,
1825
- "V5P": 512,
1826
- },
1827
- UsageContext.OPENAI_API_SERVER: {
1828
- "V6E": 1024,
1829
- "V5E": 512,
1830
- "V5P": 256,
1831
- },
1832
- }
1819
+ chip_name = current_platform.get_device_name()
1820
+
1821
+ if chip_name == "V6E":
1822
+ default_max_num_batched_tokens = {
1823
+ UsageContext.LLM_CLASS: 2048,
1824
+ UsageContext.OPENAI_API_SERVER: 1024,
1825
+ }
1826
+ elif chip_name == "V5E":
1827
+ default_max_num_batched_tokens = {
1828
+ UsageContext.LLM_CLASS: 1024,
1829
+ UsageContext.OPENAI_API_SERVER: 512,
1830
+ }
1831
+ elif chip_name == "V5P":
1832
+ default_max_num_batched_tokens = {
1833
+ UsageContext.LLM_CLASS: 512,
1834
+ UsageContext.OPENAI_API_SERVER: 256,
1835
+ }
1836
1837
# cpu specific default values.
1838
if current_platform.is_cpu():
1836
- world_size = self.pipeline_parallel_size * self.tensor_parallel_size
1839
default_max_num_batched_tokens = {
1840
UsageContext.LLM_CLASS: 4096 * world_size,
1841
UsageContext.OPENAI_API_SERVER: 2048 * world_size,
@@ -1843,44 +1845,104 @@ class EngineArgs:
1845
UsageContext.OPENAI_API_SERVER: 128 * world_size,
1846
}
1847
1846
- use_context_value = usage_context.value if usage_context else None
1847
- if (
1848
- self.max_num_batched_tokens is None
1849
- and usage_context in default_max_num_batched_tokens
1848
+ return default_max_num_batched_tokens, default_max_num_seqs
1849
+
1850
+ def _set_default_args(
1851
+ self, usage_context: UsageContext, model_config: ModelConfig
1852
+ ) -> None:
1853
+ """Set Default Arguments for V1 Engine."""
1854
+ (
1855
+ default_chunked_prefill,
1856
+ default_prefix_caching,
1857
+ ) = self.get_chunked_prefill_prefix_caching_defaults(model_config)
1858
+
1859
+ if self.enable_chunked_prefill is None:
1860
+ self.enable_chunked_prefill = default_chunked_prefill
1861
+
1862
+ logger.debug(
1863
+ "%s chunked prefill by default",
1864
+ "Enabling" if default_chunked_prefill else "Disabling",
1865
+ )
1866
+ elif (
1867
+ model_config.runner_type == "pooling"
1868
+ and self.enable_chunked_prefill
1869
+ and not default_chunked_prefill
1870
):
1851
- if current_platform.is_tpu():
1852
- chip_name = current_platform.get_device_name()
1853
- if chip_name in default_max_num_batched_tokens_tpu[usage_context]:
1854
- self.max_num_batched_tokens = default_max_num_batched_tokens_tpu[
1855
- usage_context
1856
- ][chip_name]
1857
- else:
1858
- self.max_num_batched_tokens = default_max_num_batched_tokens[
1859
- usage_context
1860
- ]
1861
- else:
1862
- if not self.enable_chunked_prefill:
1863
- self.max_num_batched_tokens = model_config.max_model_len
1864
- else:
1865
- self.max_num_batched_tokens = default_max_num_batched_tokens[
1866
- usage_context
1867
- ]
1871
+ logger.warning(
1872
+ "This model does not officially support chunked prefill. "
1873
+ "Enabling this manually may cause the engine to crash "
1874
+ "or produce incorrect outputs.",
1875
+ )
1876
+
1877
+ if self.enable_prefix_caching is None:
1878
+ self.enable_prefix_caching = default_prefix_caching
1879
+
1880
logger.debug(
1869
- "Setting max_num_batched_tokens to %d for %s usage context.",
1881
+ "%s prefix caching by default",
1882
+ "Enabling" if default_prefix_caching else "Disabling",
1883
+ )
1884
+ elif (
1885
+ model_config.runner_type == "pooling"
1886
+ and self.enable_prefix_caching
1887
+ and not default_prefix_caching
1888
+ ):
1889
+ logger.warning(
1890
+ "This model does not officially support prefix caching. "
1891
+ "Enabling this manually may cause the engine to crash "
1892
+ "or produce incorrect outputs.",
1893
+ )
1894
+
1895
+ world_size = self.pipeline_parallel_size * self.tensor_parallel_size
1896
+ (
1897
+ default_max_num_batched_tokens,
1898
+ default_max_num_seqs,
1899
+ ) = self.get_batch_defaults(world_size)
1900
+
1901
+ orig_max_num_batched_tokens = self.max_num_batched_tokens
1902
+ orig_max_num_seqs = self.max_num_seqs
1903
+
1904
+ if self.max_num_batched_tokens is None:
1905
+ self.max_num_batched_tokens = default_max_num_batched_tokens.get(
1906
+ usage_context,
1907
+ SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS,
1908
+ )
1909
+
1910
+ if self.max_num_seqs is None:
1911
+ self.max_num_seqs = default_max_num_seqs.get(
1912
+ usage_context,
1913
+ SchedulerConfig.DEFAULT_MAX_NUM_SEQS,
1914
+ )
1915
+
1916
+ if orig_max_num_batched_tokens is None:
1917
+ if not self.enable_chunked_prefill:
1918
+ # If max_model_len is too short, use the default for higher throughput.
1919
+ self.max_num_batched_tokens = max(
1920
+ model_config.max_model_len,
1921
+ self.max_num_batched_tokens,
1922
+ )
1923
+
1924
+ # When using default settings,
1925
+ # Ensure max_num_batched_tokens does not exceed model limit.
1926
+ # Some models (e.g., Whisper) have embeddings tied to max length.
1927
+ self.max_num_batched_tokens = min(
1928
+ self.max_num_seqs * model_config.max_model_len,
1929
self.max_num_batched_tokens,
1871
- use_context_value,
1930
)
1931
1874
- if self.max_num_seqs is None and usage_context in default_max_num_seqs:
1875
- self.max_num_seqs = min(
1876
- default_max_num_seqs[usage_context],
1877
- self.max_num_batched_tokens or sys.maxsize,
1932
+ logger.debug(
1933
+ "Defaulting max_num_batched_tokens to %d for %s usage context.",
1934
+ self.max_num_batched_tokens,
1935
+ usage_context.value if usage_context else None,
1936
)
1937
1938
+ if orig_max_num_seqs is None:
1939
+ assert self.max_num_batched_tokens is not None # For type checking
1940
+ self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens)
1941
+
1942
logger.debug(
1881
- "Setting max_num_seqs to %d for %s usage context.",
1943
+ "Defaulting max_num_seqs to %d for %s usage context.",
1944
self.max_num_seqs,
1883
- use_context_value,
1945
+ usage_context.value if usage_context else None,
1946
)
1947
1948
vllm/platforms/cpu.py
+1
-3
@@ -15,7 +15,6 @@ import torch
15
16
from vllm import envs
17
from vllm.logger import init_logger
18
-from vllm.utils import DEFAULT_MAX_NUM_BATCHED_TOKENS
18
19
from .interface import CpuArchEnum, Platform, PlatformEnum
20
@@ -339,10 +338,9 @@ class CpuPlatform(Platform):
338
"prefill and prefix caching to be disabled."
339
)
340
vllm_config.scheduler_config.enable_chunked_prefill = False
342
- vllm_config.scheduler_config.chunked_prefill_enabled = False
341
vllm_config.scheduler_config.max_num_batched_tokens = max(
342
vllm_config.scheduler_config.max_model_len,
345
- DEFAULT_MAX_NUM_BATCHED_TOKENS,
343
+ vllm_config.scheduler_config.DEFAULT_MAX_NUM_BATCHED_TOKENS,
344
)
345
346
@classmethod
vllm/platforms/tpu.py
+1
-3
@@ -10,7 +10,6 @@ from tpu_info import device
10
from vllm.inputs import ProcessorInputs, PromptType
11
from vllm.logger import init_logger
12
from vllm.sampling_params import SamplingParams, SamplingType
13
-from vllm.utils import DEFAULT_MAX_NUM_BATCHED_TOKENS
13
14
from .interface import Platform, PlatformEnum
15
@@ -186,10 +185,9 @@ class TpuPlatform(Platform):
185
"prefill and prefix caching to be disabled."
186
)
187
vllm_config.scheduler_config.enable_chunked_prefill = False
189
- vllm_config.scheduler_config.chunked_prefill_enabled = False
188
vllm_config.scheduler_config.max_num_batched_tokens = max(
189
vllm_config.scheduler_config.max_model_len,
192
- DEFAULT_MAX_NUM_BATCHED_TOKENS,
190
+ vllm_config.scheduler_config.DEFAULT_MAX_NUM_BATCHED_TOKENS,
191
)
192
193
@classmethod
vllm/platforms/xpu.py
+1
-3
@@ -9,7 +9,6 @@ import torch
9
10
import vllm.envs as envs
11
from vllm.logger import init_logger
12
-from vllm.utils import DEFAULT_MAX_NUM_BATCHED_TOKENS
12
13
from .interface import DeviceCapability, Platform, PlatformEnum
14
@@ -185,10 +184,9 @@ class XPUPlatform(Platform):
184
"prefill and prefix caching to be disabled."
185
)
186
vllm_config.scheduler_config.enable_chunked_prefill = False
188
- vllm_config.scheduler_config.chunked_prefill_enabled = False
187
vllm_config.scheduler_config.max_num_batched_tokens = max(
188
vllm_config.scheduler_config.max_model_len,
191
- DEFAULT_MAX_NUM_BATCHED_TOKENS,
189
+ vllm_config.scheduler_config.DEFAULT_MAX_NUM_BATCHED_TOKENS,
190
)
191
192
@classmethod
vllm/utils/__init__.py
+1
-10
@@ -3,7 +3,7 @@
3
4
import uuid
5
import warnings
6
-from typing import Any, TypeVar
6
+from typing import Any
7
8
import torch
9
@@ -39,12 +39,6 @@ def __dir__() -> list[str]:
39
40
logger = init_logger(__name__)
41
42
-# This value is chosen to have a balance between ITL and TTFT. Note it is
43
-# not optimized for throughput.
44
-DEFAULT_MAX_NUM_BATCHED_TOKENS = 2048
45
-POOLING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
46
-MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120
47
-
42
# Constants related to forcing the attention backend selection
43
44
# String name of register which may be set in order to
@@ -60,9 +54,6 @@ STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
54
STR_INVALID_VAL: str = "INVALID"
55
56
63
-T = TypeVar("T")
64
-
65
-
57
def random_uuid() -> str:
58
return str(uuid.uuid4().hex)
59