@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ff8ebb2d7

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[V0 Deprecation] Remove async_output_proc, preemption mode, delay factor (#25334)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Sep 21, 2025 at 08:52 UTC 0ff8ebb2d700b2e39457a661ef979b0da2ad73b3

15 files changed +12 -210

tests/detokenizer/test_stop_strings.py

+5 -41

@@ -32,10 +32,6 @@ def _test_stopping(llm: LLM,

32 assert output.stop_reason == expected_reason

35 -def _set_async_mode(llm, is_async):

36 - llm.llm_engine.scheduler[0].use_async_output_proc = is_async

37 -

38 -

35 def _stop_basic(llm):

36 _test_stopping(llm,

37 stop=["."],

@@ -103,40 +99,8 @@ def test_stop_strings():

99 # async output processing below.

100 llm = LLM(MODEL, enforce_eager=envs.VLLM_USE_V1)

101

106 - if envs.VLLM_USE_V1:

107 - _stop_basic(llm)

108 - else:

109 - _set_async_mode(llm, True)

110 - _stop_basic(llm)

111 -

112 - _set_async_mode(llm, False)

113 - _stop_basic(llm)

114 -

115 - if envs.VLLM_USE_V1:

116 - _stop_multi_tokens(llm)

117 - else:

118 - _set_async_mode(llm, True)

119 - _stop_multi_tokens(llm)

120 -

121 - _set_async_mode(llm, False)

122 - _stop_multi_tokens(llm)

123 -

124 - if envs.VLLM_USE_V1:

125 - _stop_partial_token(llm)

126 - else:

127 - _set_async_mode(llm, True)

128 - _stop_partial_token(llm)

129 -

130 - _set_async_mode(llm, False)

131 - _stop_partial_token(llm)

132 -

133 - if envs.VLLM_USE_V1:

134 - # FIXME: this does not respect include_in_output=False

135 - # _stop_token_id(llm)

136 - pass

137 - else:

138 - _set_async_mode(llm, True)

139 - _stop_token_id(llm)

140 -

141 - _set_async_mode(llm, False)

142 - _stop_token_id(llm)

102 + _stop_basic(llm)

103 + _stop_multi_tokens(llm)

104 + _stop_partial_token(llm)

105 + # FIXME: this does not respect include_in_output=False

106 + # _stop_token_id(llm)

tests/v1/engine/test_processor_multi_modal_uuids.py

-10

@@ -6,7 +6,6 @@ import pytest

6 from vllm.assets.image import ImageAsset

7 from vllm.assets.video import VideoAsset

8 from vllm.config import CacheConfig, DeviceConfig, ModelConfig, VllmConfig

9 -from vllm.platforms.interface import UnspecifiedPlatform

9 from vllm.sampling_params import SamplingParams

10 from vllm.v1.engine import processor as processor_mod

11 from vllm.v1.engine.processor import Processor

@@ -33,15 +32,6 @@ def _mk_processor(monkeypatch,

32 "__post_init__",

33 lambda self, *args: None,

34 raising=True)

36 - monkeypatch.setattr(UnspecifiedPlatform,

37 - "is_async_output_supported",

38 - classmethod(lambda cls, enforce_eager: True),

39 - raising=True)

40 - monkeypatch.setattr(

41 - ModelConfig,

42 - "verify_async_output_proc",

43 - lambda self, parallel_config, speculative_config, device_config: None,

44 - raising=True)

35 monkeypatch.setattr(ModelConfig,

36 "verify_with_parallel_config",

37 lambda self, parallel_config: None,

tests/v1/test_oracle.py

-18

@@ -29,24 +29,6 @@ def test_unsupported_configs(monkeypatch):

29 },

30 ).create_engine_config()

32 - with pytest.raises(NotImplementedError):

33 - AsyncEngineArgs(

34 - model=MODEL,

35 - preemption_mode="swap",

36 - ).create_engine_config()

37 -

38 - with pytest.raises(NotImplementedError):

39 - AsyncEngineArgs(

40 - model=MODEL,

41 - disable_async_output_proc=True,

42 - ).create_engine_config()

43 -

44 - with pytest.raises(NotImplementedError):

45 - AsyncEngineArgs(

46 - model=MODEL,

47 - scheduler_delay_factor=1.2,

48 - ).create_engine_config()

49 -

33 def test_enable_by_default_fallback(monkeypatch):

34 with monkeypatch.context() as m:

vllm/config/__init__.py

-4

@@ -454,9 +454,6 @@ class VllmConfig:

454 self.try_verify_and_update_config()

455

456 if self.model_config is not None:

457 - self.model_config.verify_async_output_proc(self.parallel_config,

458 - self.speculative_config,

459 - self.device_config)

457 self.model_config.verify_with_parallel_config(self.parallel_config)

458 self.model_config.verify_dual_chunk_attention_config(

459 self.load_config)

@@ -877,7 +874,6 @@ class VllmConfig:

874 f"served_model_name={self.model_config.served_model_name}, "

875 f"enable_prefix_caching={self.cache_config.enable_prefix_caching}, "

876 f"chunked_prefill_enabled={self.scheduler_config.chunked_prefill_enabled}, " # noqa

880 - f"use_async_output_proc={self.model_config.use_async_output_proc}, "

877 f"pooler_config={self.model_config.pooler_config!r}, "

878 f"compilation_config={self.compilation_config!r}")

879

vllm/config/model.py

+6 -42

@@ -223,8 +223,6 @@ class ModelConfig:

223 that this name(s) will also be used in `model_name` tag content of

224 prometheus metrics, if multiple names provided, metrics tag will take the

225 first one."""

226 - use_async_output_proc: bool = True

227 - """Whether to use async output processor."""

226 config_format: Union[str, ConfigFormat] = "auto"

227 """The format of the model config to load:\n

228 - "auto" will try to load the config in hf format if available else it

@@ -1119,37 +1117,6 @@ class ModelConfig:

1117 raise ValueError("please set VLLM_ATTENTION_BACKEND to "

1118 f"{STR_DUAL_CHUNK_FLASH_ATTN_VAL}")

1119

1122 - def verify_async_output_proc(self, parallel_config, speculative_config,

1123 - device_config) -> None:

1124 - if not self.use_async_output_proc:

1125 - # Nothing to check

1126 - return

1127 -

1128 - if parallel_config.pipeline_parallel_size > 1:

1129 - self.use_async_output_proc = False

1130 - return

1131 -

1132 - # Reminder: Please update docs/features/compatibility_matrix.md

1133 - # If the feature combo become valid

1134 - from vllm.platforms import current_platform

1135 - if not current_platform.is_async_output_supported(self.enforce_eager):

1136 - self.use_async_output_proc = False

1137 - return

1138 -

1139 - if envs.VLLM_USE_RAY_SPMD_WORKER:

1140 - self.use_async_output_proc = False

1141 - return

1142 -

1143 - # Async postprocessor is not necessary for pooling models

1144 - # since there is no token generation

1145 - if self.runner_type == "pooling":

1146 - self.use_async_output_proc = False

1147 -

1148 - # Reminder: Please update docs/features/compatibility_matrix.md

1149 - # If the feature combo become valid

1150 - if speculative_config:

1151 - self.use_async_output_proc = False

1152 -

1120 def verify_with_parallel_config(

1121 self,

1122 parallel_config: ParallelConfig,

@@ -1173,15 +1140,12 @@ class ModelConfig:

1140 self._verify_with_expert_parallelism()

1141

1142 pipeline_parallel_size = parallel_config.pipeline_parallel_size

1176 - if pipeline_parallel_size > 1:

1177 - if not self.registry.is_pp_supported_model(self.architectures,

1178 - self):

1179 - raise NotImplementedError(

1180 - "Pipeline parallelism is not supported for this model. "

1181 - "Supported models implement the `SupportsPP` interface.")

1182 -

1183 - if self.use_async_output_proc:

1184 - self.use_async_output_proc = False

1143 + if (pipeline_parallel_size > 1

1144 + and not self.registry.is_pp_supported_model(

1145 + self.architectures, self)):

1146 + raise NotImplementedError(

1147 + "Pipeline parallelism is not supported for this model. "

1148 + "Supported models implement the `SupportsPP` interface.")

1149

1150 def get_sliding_window(self) -> Optional[int]:

1151 """Get the sliding window size from the HF text config if present."""

vllm/config/scheduler.py

+1 -14

@@ -3,7 +3,7 @@

4 import hashlib

5 from dataclasses import field

6 -from typing import Any, Literal, Optional, Union

6 +from typing import Any, Literal, Union

8 from pydantic import SkipValidation, model_validator

9 from pydantic.dataclasses import dataclass

@@ -18,7 +18,6 @@ from vllm.utils import (DEFAULT_MAX_NUM_BATCHED_TOKENS,

18 logger = init_logger(__name__)

20 RunnerType = Literal["generate", "pooling", "draft"]

21 -PreemptionMode = Literal["swap", "recompute"]

21 SchedulerPolicy = Literal["fcfs", "priority"]

@@ -78,10 +77,6 @@ class SchedulerConfig:

77 3. more than one value (e.g. 1 2 128) is provided, then the capture list

78 will follow the provided list."""

81 - delay_factor: float = 0.0

82 - """Apply a delay (of delay factor multiplied by previous

83 - prompt latency) before scheduling next prompt."""

84 -

80 enable_chunked_prefill: SkipValidation[bool] = None # type: ignore

81 """If True, prefill requests can be chunked based

82 on the remaining max_num_batched_tokens."""

@@ -103,14 +98,6 @@ class SchedulerConfig:

98 NOTE: This is not currently configurable. It will be overridden by

99 max_num_batched_tokens in case max multimodal embedding size is larger."""

100

106 - preemption_mode: Optional[PreemptionMode] = None

107 - """Whether to perform preemption by swapping or

108 - recomputation. If not specified, we determine the mode as follows:

109 - We use recomputation by default since it incurs lower overhead than

110 - swapping. However, when the sequence group has multiple sequences

111 - (e.g., beam search), recomputation is not currently supported. In

112 - such a case, we use swapping instead."""

113 -

101 send_delta_data: bool = False

102 """Private API. If used, scheduler sends delta data to

103 workers instead of an entire data. It should be enabled only

vllm/engine/arg_utils.py

-34

@@ -409,9 +409,7 @@ class EngineArgs:

409 get_field(LoadConfig, "model_loader_extra_config")

410 ignore_patterns: Optional[Union[str,

411 List[str]]] = LoadConfig.ignore_patterns

412 - preemption_mode: Optional[str] = SchedulerConfig.preemption_mode

412

414 - scheduler_delay_factor: float = SchedulerConfig.delay_factor

413 enable_chunked_prefill: Optional[

414 bool] = SchedulerConfig.enable_chunked_prefill

415 disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input

@@ -439,7 +437,6 @@ class EngineArgs:

437 ObservabilityConfig.otlp_traces_endpoint

438 collect_detailed_traces: Optional[list[DetailedTraceModules]] = \

439 ObservabilityConfig.collect_detailed_traces

442 - disable_async_output_proc: bool = not ModelConfig.use_async_output_proc

440 scheduling_policy: SchedulerPolicy = SchedulerConfig.policy

441 scheduler_cls: Union[str, Type[object]] = SchedulerConfig.scheduler_cls

442

@@ -561,14 +558,6 @@ class EngineArgs:

558 **model_kwargs["enable_prompt_embeds"])

559 model_group.add_argument("--served-model-name",

560 **model_kwargs["served_model_name"])

564 - # This one is a special case because it is the

565 - # opposite of ModelConfig.use_async_output_proc

566 - model_group.add_argument(

567 - "--disable-async-output-proc",

568 - action="store_true",

569 - default=EngineArgs.disable_async_output_proc,

570 - help="Disable async output processing. This may result in "

571 - "lower performance.")

561 model_group.add_argument("--config-format",

562 **model_kwargs["config_format"])

563 # This one is a special case because it can bool

@@ -897,10 +886,6 @@ class EngineArgs:

886 **scheduler_kwargs["long_prefill_token_threshold"])

887 scheduler_group.add_argument("--num-lookahead-slots",

888 **scheduler_kwargs["num_lookahead_slots"])

900 - scheduler_group.add_argument("--scheduler-delay-factor",

901 - **scheduler_kwargs["delay_factor"])

902 - scheduler_group.add_argument("--preemption-mode",

903 - **scheduler_kwargs["preemption_mode"])

889 # multi-step scheduling has been removed; corresponding arguments

890 # are no longer supported.

891 scheduler_group.add_argument("--scheduling-policy",

@@ -1029,7 +1014,6 @@ class EngineArgs:

1014 interleave_mm_strings=self.interleave_mm_strings,

1015 media_io_kwargs=self.media_io_kwargs,

1016 skip_mm_profiling=self.skip_mm_profiling,

1032 - use_async_output_proc=not self.disable_async_output_proc,

1017 config_format=self.config_format,

1018 mm_processor_kwargs=self.mm_processor_kwargs,

1019 mm_processor_cache_gb=self.mm_processor_cache_gb,

@@ -1395,11 +1379,9 @@ class EngineArgs:

1379 max_model_len=model_config.max_model_len,

1380 cuda_graph_sizes=self.cuda_graph_sizes,

1381 num_lookahead_slots=num_lookahead_slots,

1398 - delay_factor=self.scheduler_delay_factor,

1382 enable_chunked_prefill=self.enable_chunked_prefill,

1383 disable_chunked_mm_input=self.disable_chunked_mm_input,

1384 is_multimodal_model=model_config.is_multimodal_model,

1402 - preemption_mode=self.preemption_mode,

1385 send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER

1386 and parallel_config.use_ray),

1387 policy=self.scheduling_policy,

@@ -1492,22 +1474,6 @@ class EngineArgs:

1474 recommend_to_remove=False)

1475 return False

1476

1495 - if self.preemption_mode != SchedulerConfig.preemption_mode:

1496 - _raise_or_fallback(feature_name="--preemption-mode",

1497 - recommend_to_remove=True)

1498 - return False

1499 -

1500 - if (self.disable_async_output_proc

1501 - != EngineArgs.disable_async_output_proc):

1502 - _raise_or_fallback(feature_name="--disable-async-output-proc",

1503 - recommend_to_remove=True)

1504 - return False

1505 -

1506 - if self.scheduler_delay_factor != SchedulerConfig.delay_factor:

1507 - _raise_or_fallback(feature_name="--scheduler-delay-factor",

1508 - recommend_to_remove=True)

1509 - return False

1510 -

1477 # No Mamba or Encoder-Decoder so far.

1478 if not model_config.is_v1_compatible:

1479 _raise_or_fallback(feature_name=model_config.architectures,

vllm/entrypoints/llm.py

-4

@@ -137,8 +137,6 @@ class LLM:

137 back to the eager mode.

138 disable_custom_all_reduce: See

139 [ParallelConfig][vllm.config.ParallelConfig].

140 - disable_async_output_proc: Disable async output processing.

141 - This may result in lower performance.

140 hf_token: The token to use as HTTP bearer authorization for remote files

141 . If `True`, will use the token generated when running

142 `huggingface-cli login` (stored in `~/.huggingface`).

@@ -188,7 +186,6 @@ class LLM:

186 enforce_eager: bool = False,

187 max_seq_len_to_capture: int = 8192,

188 disable_custom_all_reduce: bool = False,

191 - disable_async_output_proc: bool = False,

189 hf_token: Optional[Union[bool, str]] = None,

190 hf_overrides: Optional[HfOverrides] = None,

191 mm_processor_kwargs: Optional[dict[str, Any]] = None,

@@ -286,7 +283,6 @@ class LLM:

283 enforce_eager=enforce_eager,

284 max_seq_len_to_capture=max_seq_len_to_capture,

285 disable_custom_all_reduce=disable_custom_all_reduce,

289 - disable_async_output_proc=disable_async_output_proc,

286 hf_token=hf_token,

287 hf_overrides=hf_overrides,

288 mm_processor_kwargs=mm_processor_kwargs,

vllm/executor/uniproc_executor.py

-4

@@ -137,10 +137,6 @@ class ExecutorWithExternalLauncher(UniProcExecutor):

137 def _init_executor(self) -> None:

138 """Initialize the worker and load the model.

139 """

140 - assert self.vllm_config.scheduler_config.delay_factor == 0.0, \

141 - ("ExecutorWithExternalLauncher needs deterministic "

142 - "execution, so it"

143 - "does not support delay_factor in scheduling")

140 if envs.VLLM_USE_V1:

141 assert not envs.VLLM_ENABLE_V1_MULTIPROCESSING, \

142 ("To get deterministic execution in V1, "

vllm/platforms/cpu.py

-4

@@ -126,10 +126,6 @@ class CpuPlatform(Platform):

126 """

127 torch.cpu.set_device(device)

128

129 - @classmethod

130 - def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:

131 - return False

132 -

129 @classmethod

130 def inference_mode(cls):

131 return torch.no_grad()

vllm/platforms/cuda.py

-10

@@ -96,16 +96,6 @@ class CudaPlatformBase(Platform):

96 def get_device_total_memory(cls, device_id: int = 0) -> int:

97 raise NotImplementedError

99 - @classmethod

100 - def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:

101 - if enforce_eager and not envs.VLLM_USE_V1:

102 - logger.warning(

103 - "To see benefits of async output processing, enable CUDA "

104 - "graph. Since, enforce-eager is enabled, async output "

105 - "processor cannot be used")

106 - return False

107 - return True

108 -

99 @classmethod

100 def is_fully_connected(cls, device_ids: list[int]) -> bool:

101 raise NotImplementedError

vllm/platforms/interface.py

-7

@@ -275,13 +275,6 @@ class Platform:

275 """Get the total memory of a device in bytes."""

276 raise NotImplementedError

277

278 - @classmethod

279 - def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:

280 - """

281 - Check if the current platform supports async output.

282 - """

283 - raise NotImplementedError

284 -

278 @classmethod

279 def inference_mode(cls):

280 """A device-specific wrapper of `torch.inference_mode`.

vllm/platforms/rocm.py

-10

@@ -310,16 +310,6 @@ class RocmPlatform(Platform):

310 device_props = torch.cuda.get_device_properties(device_id)

311 return device_props.total_memory

312

313 - @classmethod

314 - def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:

315 - if enforce_eager and not envs.VLLM_USE_V1:

316 - logger.warning(

317 - "To see benefits of async output processing, enable CUDA "

318 - "graph. Since, enforce-eager is enabled, async output "

319 - "processor cannot be used")

320 - return False

321 - return True

322 -

313 @classmethod

314 def check_and_update_config(cls, vllm_config: "VllmConfig") -> None:

315 from vllm.config.compilation import CUDAGraphMode

vllm/platforms/tpu.py

-4

@@ -75,10 +75,6 @@ class TpuPlatform(Platform):

75 def get_device_total_memory(cls, device_id: int = 0) -> int:

76 raise NotImplementedError

78 - @classmethod

79 - def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:

80 - return False

81 -

78 @classmethod

79 def get_punica_wrapper(cls) -> str:

80 return "vllm.lora.punica_wrapper.punica_tpu.PunicaWrapperTPU"

vllm/platforms/xpu.py

-4

@@ -98,10 +98,6 @@ class XPUPlatform(Platform):

98 device_props = torch.xpu.get_device_properties(device_id)

99 return device_props.total_memory

100

101 - @classmethod

102 - def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:

103 - return True

104 -

101 @classmethod

102 def inference_mode(cls):

103 return torch.no_grad()