5fe77aecfc · tduncan/Nvidia-Optimization-Stack-VLLM

benchmarks/attention_benchmarks/mla_runner.py

-6

@@ -409,10 +409,6 @@ def _build_attention_metadata(

409 seq_lens_cpu = torch.tensor(kv_lens, dtype=torch.int32)

410 seq_lens_gpu = seq_lens_cpu.to(device)

411

412 - # Build num_computed_tokens (context length for each request)

413 - context_lens = [kv_len - q_len for q_len, kv_len in zip(q_lens, kv_lens)]

414 - num_computed_tokens_cpu = torch.tensor(context_lens, dtype=torch.int32)

415 -

412 # Build block table

413 num_blocks_per_req = [(kv + block_size - 1) // block_size for kv in kv_lens]

414 max_num_blocks = max(num_blocks_per_req)

@@ -454,8 +450,6 @@ def _build_attention_metadata(

450 query_start_loc_cpu=q_start_cpu,

451 seq_lens=seq_lens_gpu,

452 seq_lens_cpu_upper_bound=seq_lens_cpu,

457 - _seq_lens_cpu=seq_lens_cpu,

458 - _num_computed_tokens_cpu=num_computed_tokens_cpu,

453 slot_mapping=slot_mapping,

454 block_table_tensor=block_table_gpu,

455 dcp_local_seq_lens=None,

tests/engine/test_arg_utils.py

-18

@@ -561,24 +561,6 @@ def test_prefix_cache_default():

561 assert engine_args.prefix_cache_retention_interval == 64

562

563

564 -def test_prefix_cache_retention_interval_from_deprecated_env(

565 - monkeypatch, caplog, disable_log_dedup

566 -):

567 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")

568 -

569 - engine_args = EngineArgs()

570 -

571 - assert engine_args.prefix_cache_retention_interval == 64

572 - assert "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in caplog.text

573 - assert "deprecated" in caplog.text

574 - assert "prefix_cache_retention_interval" in caplog.text

575 -

576 - parser = EngineArgs.add_cli_args(FlexibleArgumentParser())

577 - args = parser.parse_args(["--prefix-cache-retention-interval", "32"])

578 - engine_args = EngineArgs.from_cli_args(args)

579 - assert engine_args.prefix_cache_retention_interval == 32

580 -

581 -

564 @pytest.mark.parametrize(

565 ("arg", "expected", "option"),

566 [

tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py

+2 -2

@@ -81,7 +81,7 @@ def _make_common_metadata():

81 query_start_loc=query_start_loc,

82 query_start_loc_cpu=query_start_loc,

83 seq_lens=seq_lens,

84 - _seq_lens_cpu=seq_lens,

84 + seq_lens_cpu_upper_bound=seq_lens,

85 num_reqs=2,

86 num_actual_tokens=2,

87 max_query_len=1,

@@ -99,7 +99,7 @@ def _make_mixed_common_metadata():

99 query_start_loc=query_start_loc,

100 query_start_loc_cpu=query_start_loc,

101 seq_lens=seq_lens,

102 - _seq_lens_cpu=seq_lens,

102 + seq_lens_cpu_upper_bound=seq_lens,

103 num_reqs=2,

104 num_actual_tokens=5,

105 max_query_len=4,

tests/kernels/test_compressor_kv_cache.py

+6 -2

@@ -705,9 +705,13 @@ def test_gfx950_compressed_cache_canonicalizes_nonfinite(writer: str) -> None:

705 ],

706 )

707 def test_get_c128_boundary(starts, query_start_loc, expected):

708 + query_start_loc_tensor = torch.tensor(query_start_loc)

709 + query_lens = query_start_loc_tensor[1:] - query_start_loc_tensor[:-1]

710 metadata = SimpleNamespace(

709 - _num_computed_tokens_cpu=None if starts is None else torch.tensor(starts),

710 - query_start_loc_cpu=torch.tensor(query_start_loc),

711 + seq_lens_cpu_upper_bound=(

712 + None if starts is None else torch.tensor(starts) + query_lens

713 + ),

714 + query_start_loc_cpu=query_start_loc_tensor,

715 )

716 assert _get_c128_boundary(metadata) is expected

717

tests/v1/attention/test_chunked_local_attention.py

+2 -1

@@ -181,7 +181,8 @@ def test_local_attention_virtual_batches(test_data: LocalAttentionTestData):

181

182 # Convert to numpy for easier comparison

183 actual_q_seqlens = np.diff(result.query_start_loc_cpu.numpy())

184 - actual_k_seqlens = result.seq_lens_cpu.numpy()

184 + assert result.seq_lens_cpu_upper_bound is not None

185 + actual_k_seqlens = result.seq_lens_cpu_upper_bound.numpy()

186

187 # Check that all query lengths are less than or equal to attn_chunk_size

188 assert all(q_len <= attn_chunk_size for q_len in actual_q_seqlens)

tests/v1/attention/utils.py

-9

@@ -85,13 +85,6 @@ def create_common_attn_metadata(

85 seq_lens_cpu = seq_lens.cpu()

86 max_seq_len = int(seq_lens_cpu.max())

87

88 - # Create computed tokens (context length for each sequence)

89 - context_lens = [

90 - batch_spec.seq_lens[i] - batch_spec.query_lens[i]

91 - for i in range(batch_spec.batch_size)

92 - ]

93 - num_computed_tokens_cpu = torch.tensor(context_lens, dtype=torch.int32)

94 -

88 # Create block table and slot mapping

89 max_blocks = (max(batch_spec.seq_lens) + block_size - 1) // block_size

90 if arange_block_indices:

@@ -122,8 +115,6 @@ def create_common_attn_metadata(

115 query_start_loc_cpu=query_start_loc_cpu,

116 seq_lens=seq_lens,

117 seq_lens_cpu_upper_bound=seq_lens_cpu,

125 - _seq_lens_cpu=seq_lens_cpu,

126 - _num_computed_tokens_cpu=num_computed_tokens_cpu,

118 num_reqs=batch_spec.batch_size,

119 num_actual_tokens=num_tokens,

120 max_query_len=max_query_len,

tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py

+2 -2

@@ -2386,8 +2386,8 @@ def test_retention_snapshots_handed_off_with_exact_block_ids():

2386 def test_snapshot_handoff_dense_default_retention():

2387 """With dense (default) retention, every materialized mamba boundary

2388 state block is handed off — regular mamba-align + prefix-match-unit

2389 - deployments get store-able boundary snapshots without setting

2390 - VLLM_PREFIX_CACHE_RETENTION_INTERVAL."""

2389 + deployments get store-able boundary snapshots without setting a

2390 + prefix-cache retention interval."""

2391 hash_block_size = 2

2392 block_size = 4

2393 manager = make_kv_cache_manager(

tests/v1/e2e/spec_decode/draft_model/test_async.py

+5 -71

@@ -1,70 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""

4 -Test that verifies no implicit GPU-CPU synchronization occurs during

5 -speculative decoding generation under expected conditions.

6 -"""

7 -

8 -import multiprocessing

9 -import sys

10 -import traceback

3 +"""Tests for sync-free speculative decoding with async scheduling."""

4

5 import pytest

13 -import torch

14 -

15 -

16 -@pytest.fixture

17 -def sync_tracker():

18 - """

19 - Fixture that patches CommonAttentionMetadata.seq_lens_cpu to detect

20 - lazy init syncs. Prints stack traces immediately when syncs occur.

21 - """

22 - from vllm.v1.attention.backend import CommonAttentionMetadata

23 -

24 - # Shared counter for cross-process communication (inherited by fork)

25 - sync_count = multiprocessing.Value("i", 0)

26 -

27 - # Save original property

28 - original_prop = CommonAttentionMetadata.seq_lens_cpu

29 - original_fget = original_prop.fget

30 -

31 - # Create tracking wrapper

32 - def tracking_seq_lens_cpu(self):

33 - if self._seq_lens_cpu is None:

34 - # Increment counter

35 - with sync_count.get_lock():

36 - sync_count.value += 1

37 - count = sync_count.value

38 - # Print stack trace immediately (shows in subprocess output)

39 - print(f"\n{'=' * 60}", file=sys.stderr)

40 - print(f"SYNC #{count}: seq_lens_cpu lazy init triggered!", file=sys.stderr)

41 - print(f"{'=' * 60}", file=sys.stderr)

42 - traceback.print_stack(file=sys.stderr)

43 - print(f"{'=' * 60}\n", file=sys.stderr)

44 - sys.stderr.flush()

45 - return original_fget(self)

46 -

47 - # Apply patch

48 - CommonAttentionMetadata.seq_lens_cpu = property(tracking_seq_lens_cpu)

49 -

50 - class SyncTracker:

51 - @property

52 - def count(self) -> int:

53 - return sync_count.value

54 -

55 - def assert_no_sync(self, msg: str = ""):

56 - count = sync_count.value

57 - assert count == 0, (

58 - f"Unexpected GPU-CPU sync: seq_lens_cpu lazy init triggered "

59 - f"{count} times. See stack traces above. {msg}"

60 - )

61 -

62 - yield SyncTracker()

63 -

64 - # Restore original property

65 - CommonAttentionMetadata.seq_lens_cpu = original_prop

66 - torch._dynamo.reset()

67 -

6

7 # Test configurations: (model, spec_model, method, num_spec_tokens, backend_env)

8 SPEC_DECODE_CONFIGS = [

@@ -97,18 +35,16 @@ SPEC_DECODE_CONFIGS = [

35 SPEC_DECODE_CONFIGS,

36 )

37 def test_no_sync_with_spec_decode(

100 - sync_tracker,

38 model: str,

39 spec_model: str,

40 method: str,

41 num_spec_tokens: int,

42 + monkeypatch,

43 vllm_runner,

44 ):

107 - """

108 - Test that no implicit GPU-CPU sync occurs during speculative decoding

109 - generation.

110 - """

111 - # Import vLLM AFTER sync_tracker fixture has applied the patch

45 + """Test generation while the worker rejects unintended GPU-CPU syncs."""

46 + monkeypatch.setenv("VLLM_GPU_SYNC_CHECK", "error")

47 +

48 from vllm import SamplingParams

49 from vllm.config import CompilationConfig

50

@@ -158,5 +94,3 @@ def test_no_sync_with_spec_decode(

94 assert outputs[0].outputs[0].text, (

95 f"{method} target={model}: generated completion text is empty"

96 )

161 -

162 - sync_tracker.assert_no_sync()

vllm/config/attention.py

+1 -24

@@ -7,12 +7,9 @@ from typing import Any, Literal

7 from pydantic import field_validator

8

9 from vllm.config.utils import config

10 -from vllm.logger import init_logger

10 from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum

11 from vllm.v1.attention.backends.registry import AttentionBackendEnum

12

14 -logger = init_logger(__name__)

15 -

13 IndexerKVDType = Literal["auto", "bf16", "fp8", "mxfp4", "nvfp4"]

14 MiniMaxM3MSADecodeBackend = Literal["triton", "cutlass"]

15

@@ -64,10 +61,6 @@ class AttentionConfig:

61 use_prefill_query_quantization: bool = False

62 """If set, quantize query for attention in prefill."""

63

67 - use_fp4_indexer_cache: bool | None = None

68 - """Deprecated alias for `indexer_kv_dtype`; use that instead. True maps to

69 - `mxfp4`, False is a no-op (it selected the model default already)."""

70 -

64 indexer_kv_dtype: IndexerKVDType = "auto"

65 """Data type for the sparse-attention indexer K cache. "auto" picks the

66 model's default (bf16 for MiniMax M3, fp8 for the DeepSeek sparse

@@ -116,20 +109,6 @@ class AttentionConfig:

109 # layers still use the platform's normal automatic backend.

110 self.backend = None

111

119 - if self.use_fp4_indexer_cache is not None:

120 - logger.warning(

121 - "use_fp4_indexer_cache is deprecated and will be removed in "

122 - "v0.29. Use indexer_kv_dtype instead (True -> 'mxfp4')."

123 - )

124 - if self.use_fp4_indexer_cache:

125 - if self.indexer_kv_dtype not in ("auto", "mxfp4"):

126 - raise ValueError(

127 - "use_fp4_indexer_cache=True conflicts with "

128 - f"indexer_kv_dtype={self.indexer_kv_dtype!r}. Set only "

129 - "indexer_kv_dtype."

130 - )

131 - self.indexer_kv_dtype = "mxfp4"

132 -

112 def resolve_indexer_kv_dtype(self, default: IndexerKVDType) -> IndexerKVDType:

113 """Resolve `indexer_kv_dtype`, substituting `default` for "auto"."""

114 if self.indexer_kv_dtype == "auto":

@@ -146,9 +125,7 @@ class AttentionConfig:

125 """

126 from vllm.config.utils import get_hash_factors, hash_factors

127

149 - # Folded into indexer_kv_dtype by __post_init__.

150 - ignored_factors: set[str] = {"use_fp4_indexer_cache"}

151 - factors = get_hash_factors(self, ignored_factors)

128 + factors = get_hash_factors(self, set())

129 return hash_factors(factors)

130

131 @field_validator("backend", mode="before")

vllm/config/cache.py

+2 -13

@@ -8,7 +8,7 @@ from typing import Any, ClassVar, Literal

8

9 from pydantic import Field, field_validator, model_validator

10

11 -from vllm.config.utils import config, get_from_deprecated_env_if_set

11 +from vllm.config.utils import config

12 from vllm.logger import init_logger

13 from vllm.utils.torch_utils import (

14 is_quantized_kv_cache,

@@ -58,15 +58,6 @@ CacheDType = Literal[

58 ]

59

60

61 -def _get_prefix_cache_retention_interval() -> int | None:

62 - env_value = get_from_deprecated_env_if_set(

63 - "VLLM_PREFIX_CACHE_RETENTION_INTERVAL",

64 - "v0.29",

65 - "prefix_cache_retention_interval",

66 - )

67 - return 0 if env_value is None else int(env_value)

68 -

69 -

61 MambaDType = Literal["auto", "float32", "float16", "bfloat16"]

62 MambaCacheMode = Literal["all", "align", "none"]

63 PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"]

@@ -154,9 +145,7 @@ class CacheConfig:

145 security risk tolerance against the performance benefits before turning this on.

146 - "xxhash_cbor" combines canonical CBOR serialization with xxHash for

147 reproducible hashing. Requires the optional ``xxhash`` package."""

157 - prefix_cache_retention_interval: int | None = Field(

158 - default_factory=_get_prefix_cache_retention_interval, ge=0

159 - )

148 + prefix_cache_retention_interval: int | None = Field(default=0, ge=0)

149 """Token interval between retained sliding-window and Mamba prefix-cache

150 checkpoints. ``0`` retains only semantic checkpoints, including the latest

151 replay boundary and shared-prefix junctions. Positive values additionally

vllm/config/multimodal.py

+3 -15

@@ -3,7 +3,7 @@

3

4 from collections.abc import Mapping

5 from pathlib import Path

6 -from typing import Any, Literal, TypeAlias, TypedDict, cast, final

6 +from typing import Any, Literal, TypeAlias, TypedDict, final

7

8 import torch

9 from pydantic import ConfigDict, Field, field_validator, model_validator

@@ -11,7 +11,7 @@ from pydantic.dataclasses import dataclass

11

12 import vllm.envs as envs

13 from vllm.config.ec_transfer import ECTransferConfig

14 -from vllm.config.utils import config, get_from_deprecated_env_if_set

14 +from vllm.config.utils import config

15 from vllm.logger import init_logger

16 from vllm.utils.hashing import safe_hash

17 from vllm.v1.attention.backends.registry import AttentionBackendEnum

@@ -75,16 +75,6 @@ MMProcessorDevice: TypeAlias = str

75 `"xpu"` on XPU). Validated against that set by the CLI."""

76

77

78 -def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:

79 - env_value = get_from_deprecated_env_if_set(

80 - "VLLM_MM_HASHER_ALGORITHM",

81 - "v0.27",

82 - "mm_hasher_algorithm",

83 - )

84 - env_value = "blake3" if env_value is None else env_value

85 - return cast(MMHasherAlgorithm, env_value.lower())

86 -

87 -

78 MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]

79 """

80 A dictionary containing an entry for each modality type of dummy data.

@@ -164,9 +154,7 @@ class MultiModalConfig:

154 mm_processor_cache_type: MMCacheType = "lru"

155 """Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,

156 use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""

167 - mm_hasher_algorithm: MMHasherAlgorithm = Field(

168 - default_factory=_get_mm_hasher_algorithm

169 - )

157 + mm_hasher_algorithm: MMHasherAlgorithm = "blake3"

158 """Hash algorithm to use for multi-modal input caching. Use `"sha256"` or

159 `"sha512"` for FIPS-compliant deployments."""

160 mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0)

vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py

+2 -2

@@ -305,7 +305,7 @@ class SchedulerOffloadConfig(NamedTuple):

305 if retention_interval is not None:

306 if retention_interval < 0:

307 raise ValueError(

308 - f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL "

308 + f"prefix_cache_retention_interval "

309 f"({retention_interval}) must be non-negative."

310 )

311 for config in kv_group_configs:

@@ -314,7 +314,7 @@ class SchedulerOffloadConfig(NamedTuple):

314 and retention_interval % config.tokens_per_chunk != 0

315 ):

316 raise ValueError(

317 - f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL "

317 + f"prefix_cache_retention_interval "

318 f"({retention_interval}) must be a multiple of "

319 f"tokens_per_chunk ({config.tokens_per_chunk})."

320 )

vllm/envs.py

-18

@@ -86,7 +86,6 @@ if TYPE_CHECKING:

86 VLLM_MAX_IMAGE_PIXELS: int = 178_956_970

87 VLLM_VIDEO_LOADER_BACKEND: str = "opencv"

88 VLLM_MEDIA_CONNECTOR: str = "http"

89 - VLLM_MM_HASHER_ALGORITHM: str = "blake3"

89 VLLM_TARGET_DEVICE: str = "cuda"

90 VLLM_MAIN_CUDA_VERSION: str = "13.0"

91 VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest"

@@ -325,7 +324,6 @@ if TYPE_CHECKING:

324 VLLM_LORA_ENABLE_DUAL_STREAM: bool = False

325 VLLM_GPU_NIC_PCIE_MAPPING: str = ""

326 VLLM_NIC_SELECTION_VARS: str = ""

328 - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: int | None = None

327 VLLM_ENABLE_HPC_OPS: bool = False

328

329

@@ -1070,17 +1068,6 @@ environment_variables: dict[str, Callable[[], Any]] = {

1068 # imported at runtime.

1069 # If a non-existing backend is used, an AssertionError will be thrown.

1070 "VLLM_MEDIA_CONNECTOR": lambda: os.getenv("VLLM_MEDIA_CONNECTOR", "http"),

1073 - # Hash algorithm for multimodal content hashing.

1074 - # - "blake3": Default, fast cryptographic hash (not FIPS 140-3 compliant)

1075 - # - "sha256": FIPS 140-3 compliant, widely supported

1076 - # - "sha512": FIPS 140-3 compliant, faster on 64-bit systems

1077 - # Use sha256 or sha512 for FIPS compliance in government/enterprise deployments

1078 - "VLLM_MM_HASHER_ALGORITHM": env_with_choices(

1079 - "VLLM_MM_HASHER_ALGORITHM",

1080 - "blake3",

1081 - ["blake3", "sha256", "sha512"],

1082 - case_sensitive=False,

1083 - ),

1071 # Path to the XLA persistent cache directory.

1072 # Only used for XLA devices such as TPUs.

1073 "VLLM_XLA_CACHE_PATH": lambda: os.path.expanduser(

@@ -1172,11 +1159,6 @@ environment_variables: dict[str, Callable[[], Any]] = {

1159 if "VLLM_PLUGINS" not in os.environ

1160 else os.environ["VLLM_PLUGINS"].split(",")

1161 ),

1175 - "VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: (

1176 - int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"])

1177 - if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ

1178 - else None

1179 - ),

1162 # a local directory to look in for unrecognized LoRA adapters.

1163 # only works if plugins are enabled and

1164 # VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled.

vllm/model_executor/layers/attention/cross_attention.py

+1 -1

@@ -115,7 +115,7 @@ def create_cross_attention_backend(

115 # needed here to know how many tokens to attend to from the cached

116 # cross-attention KV cache.

117 new_metadata.seq_lens = common_attn_metadata.encoder_seq_lens

118 - new_metadata._seq_lens_cpu = torch.from_numpy(

118 + new_metadata.seq_lens_cpu_upper_bound = torch.from_numpy(

119 common_attn_metadata.encoder_seq_lens_cpu

120 )

121

vllm/model_executor/models/whisper_causal.py

+2 -4

@@ -161,10 +161,8 @@ def create_whisper_attention_backend_with_block_pooling(

161 new_common_attn_metadata.query_start_loc *= block_pool_size

162 new_common_attn_metadata.query_start_loc_cpu *= block_pool_size

163 new_common_attn_metadata.seq_lens *= block_pool_size

164 - if new_common_attn_metadata._seq_lens_cpu is not None:

165 - new_common_attn_metadata._seq_lens_cpu *= block_pool_size

166 - if new_common_attn_metadata._num_computed_tokens_cpu is not None:

167 - new_common_attn_metadata._num_computed_tokens_cpu *= block_pool_size

164 + if new_common_attn_metadata.seq_lens_cpu_upper_bound is not None:

165 + new_common_attn_metadata.seq_lens_cpu_upper_bound *= block_pool_size

166 new_common_attn_metadata.num_actual_tokens *= block_pool_size

167 new_common_attn_metadata.max_query_len *= block_pool_size

168 new_common_attn_metadata.max_seq_len *= block_pool_size

vllm/models/deepseek_v4/compressor.py

+9 -3

@@ -43,10 +43,12 @@ def _prefer_two_stage_compressor() -> bool:

43

44

45 def _get_c128_boundary(metadata: CommonAttentionMetadata) -> bool | None:

46 - starts = metadata._num_computed_tokens_cpu

47 - if starts is None:

46 + seq_lens_cpu = metadata.seq_lens_cpu_upper_bound

47 + if seq_lens_cpu is None:

48 return None

49

50 + query_lens = metadata.query_start_loc_cpu[1:] - metadata.query_start_loc_cpu[:-1]

51 + starts = seq_lens_cpu - query_lens

52 starts_list = starts.tolist()

53 query_start_loc = metadata.query_start_loc_cpu.tolist()

54 return any(

@@ -116,6 +118,10 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder):

118 _, _, num_decode_tokens, _ = split_decodes_and_prefills(

119 common_attn_metadata, decode_threshold=1

120 )

121 + async_spec_decode = (

122 + self.vllm_config.scheduler_config.async_scheduling

123 + and self.vllm_config.speculative_config is not None

124 + )

125 return CompressorMetadata(

126 block_table=common_attn_metadata.block_table_tensor.clamp_(min=0),

127 slot_mapping=common_attn_metadata.slot_mapping,

@@ -124,7 +130,7 @@ class CompressorMetadataBuilder(AttentionMetadataBuilder):

130 num_decode_tokens=num_decode_tokens,

131 c128_boundary=(

132 _get_c128_boundary(common_attn_metadata)

127 - if self.block_size == 8

133 + if self.block_size == 8 and not async_spec_decode

134 else None

135 ),

136 )

vllm/platforms/rocm.py

-10

@@ -130,14 +130,6 @@ def _sync_hip_cuda_env_vars():

130 hip_val = os.environ.get("HIP_VISIBLE_DEVICES") or None

131 cuda_val = os.environ.get("CUDA_VISIBLE_DEVICES") or None

132

133 - if cuda_val is not None:

134 - logger.warning_once(

135 - "Using CUDA_VISIBLE_DEVICES on ROCm is deprecated and support "

136 - "will be removed in vLLM v0.26.0. Please use HIP_VISIBLE_DEVICES "

137 - "instead.",

138 - scope="process",

139 - )

140 -

133 if hip_val is not None and cuda_val is not None:

134 if hip_val != cuda_val:

135 raise ValueError(

@@ -148,8 +140,6 @@ def _sync_hip_cuda_env_vars():

140 )

141 elif hip_val is not None:

142 os.environ["CUDA_VISIBLE_DEVICES"] = hip_val

151 - elif cuda_val is not None:

152 - os.environ["HIP_VISIBLE_DEVICES"] = cuda_val

143

144

145 # Sync at import time - catches misconfigurations from process start.

vllm/v1/attention/backend.py

+1 -41

@@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Any, ClassVar, Generic, Protocol, TypeVar

8

9 import numpy as np

10 import torch

11 -from typing_extensions import deprecated

11

12 from vllm.model_executor.layers.quantization.utils.quant_utils import (

13 kFp8Dynamic64Sym,

@@ -452,10 +451,6 @@ class CommonAttentionMetadata:

451 at the current decode run's last full-state write. write_pos counts from

452 here, so a preemption-resumed request re-anchors past the prompt boundary."""

453

455 - # WARNING: Deprecated fields. Will be removed in a future release (v0.15.0)

456 - _seq_lens_cpu: torch.Tensor | None = None

457 - _num_computed_tokens_cpu: torch.Tensor | None = None

458 -

454 _num_computed_tokens_cache: torch.Tensor | None = None

455 _token_to_req_indices_cache: torch.Tensor | None = None

456

@@ -469,36 +464,6 @@ class CommonAttentionMetadata:

464 def replace(self, **kwargs) -> "CommonAttentionMetadata":

465 return replace(self, **kwargs)

466

472 - @property

473 - @deprecated(

474 - """

475 - Prefer using device seq_lens directly to avoid implicit H<>D sync.

476 - If a CPU copy is needed, use `seq_lens.cpu()` instead.

477 - Will be removed in a future release, please migrate as soon as possible.

478 - """

479 - )

480 - def seq_lens_cpu(self) -> torch.Tensor:

481 - if self._seq_lens_cpu is None:

482 - self._seq_lens_cpu = self.seq_lens.to("cpu")

483 - return self._seq_lens_cpu

484 -

485 - @property

486 - @deprecated(

487 - """

488 - Prefer using device seq_lens directly to avoid implicit H<>D sync which breaks full

489 - async scheduling. If a CPU copy is needed, it can be derived from

490 - query_start_loc_cpu and seq_lens.

491 - Will be removed in a future release, please migrate as soon as possible.

492 - """

493 - )

494 - def num_computed_tokens_cpu(self) -> torch.Tensor:

495 - if self._num_computed_tokens_cpu is None:

496 - query_seq_lens = (

497 - self.query_start_loc_cpu[1:] - self.query_start_loc_cpu[:-1]

498 - )

499 - self._num_computed_tokens_cpu = self.seq_lens_cpu - query_seq_lens

500 - return self._num_computed_tokens_cpu

501 -

467 def compute_num_computed_tokens(self) -> torch.Tensor:

468 """Compute num_computed_tokens on device (seq_lens - query_lens)."""

469 if self._num_computed_tokens_cache is None:

@@ -542,12 +507,6 @@ class CommonAttentionMetadata:

507 query_start_loc=self.query_start_loc[: num_actual_reqs + 1],

508 query_start_loc_cpu=self.query_start_loc_cpu[: num_actual_reqs + 1],

509 seq_lens=self.seq_lens[:num_actual_reqs],

545 - _seq_lens_cpu=self._seq_lens_cpu[:num_actual_reqs]

546 - if self._seq_lens_cpu is not None

547 - else None,

548 - _num_computed_tokens_cpu=self._num_computed_tokens_cpu[:num_actual_reqs]

549 - if self._num_computed_tokens_cpu is not None

550 - else None,

510 num_reqs=num_actual_reqs,

511 num_actual_tokens=num_actual_tokens,

512 max_query_len=self.max_query_len,

@@ -564,6 +523,7 @@ class CommonAttentionMetadata:

523 encoder_seq_lens_cpu=maybe_slice_reqs(self.encoder_seq_lens_cpu),

524 dcp_local_seq_lens=maybe_slice_reqs(self.dcp_local_seq_lens),

525 dcp_local_seq_lens_cpu=maybe_slice_reqs(self.dcp_local_seq_lens_cpu),

526 + seq_lens_cpu_upper_bound=maybe_slice_reqs(self.seq_lens_cpu_upper_bound),

527 is_prefilling=maybe_slice_reqs(self.is_prefilling),

528 rswa_prefix_lens=maybe_slice_reqs(self.rswa_prefix_lens),

529 replayssm_decode_base_cpu=maybe_slice_reqs(self.replayssm_decode_base_cpu),

vllm/v1/attention/backends/flashinfer.py

+1 -1

@@ -1421,7 +1421,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):

1421 needs_seq_lens_cpu = self.use_dcp or use_cascade or not all_uses_trtllm

1422 if needs_seq_lens_cpu:

1423 with gpu_sync_allowed():

1424 - seq_lens_cpu = common_attn_metadata.seq_lens_cpu

1424 + seq_lens_cpu = common_attn_metadata.seq_lens.cpu()

1425 else:

1426 seq_lens_cpu = None

1427

vllm/v1/attention/backends/mamba_attn.py

+13 -9

@@ -633,13 +633,21 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):

633

634 if self.use_replayssm and not self.use_flashinfer_replayssm and num_decodes > 0:

635 decode_base_cpu = common_attn_metadata.replayssm_decode_base_cpu

636 - num_computed_tokens_cpu = common_attn_metadata._num_computed_tokens_cpu

637 - if decode_base_cpu is None or num_computed_tokens_cpu is None:

636 + seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound

637 + async_spec_decode = (

638 + self.vllm_config.scheduler_config.async_scheduling

639 + and self.vllm_config.speculative_config is not None

640 + )

641 + if decode_base_cpu is None or seq_lens_cpu is None or async_spec_decode:

642 raise ValueError(

639 - "--use-replayssm requires CPU decode-base and "

640 - "computed-token counts to derive decode write positions"

643 + "--use-replayssm requires exact CPU sequence lengths and "

644 + "decode-base counts to derive decode write positions"

645 )

642 - num_computed_d = num_computed_tokens_cpu[:num_decodes]

646 + query_lens_cpu = (

647 + common_attn_metadata.query_start_loc_cpu[1 : num_decodes + 1]

648 + - common_attn_metadata.query_start_loc_cpu[:num_decodes]

649 + )

650 + num_computed_d = seq_lens_cpu[:num_decodes] - query_lens_cpu

651 decode_base_d = decode_base_cpu[:num_decodes]

652 align_mode = self.vllm_config.cache_config.mamba_cache_mode == "align"

653 block_size = self.kv_cache_spec.block_size

@@ -656,10 +664,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):

664 # write_pos counts decode steps since the ring's last full-state

665 # write (the anchor), so a resumed request re-anchors correctly.

666 decode_steps_cpu = num_computed_d - effective_base

659 - query_lens_cpu = (

660 - common_attn_metadata.query_start_loc_cpu[1 : num_decodes + 1]

661 - - common_attn_metadata.query_start_loc_cpu[:num_decodes]

662 - )

667 valid_decode_rows = query_lens_cpu > 0

668 # A single-token prefill row replayed as decode (query_len==1 with

669 # prior state) has decode_steps < 0; force it to a one-token flush

vllm/v1/attention/backends/mla/indexer.py

+2 -2

@@ -695,7 +695,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):

695 if self.vllm_config.speculative_config

696 else 0

697 )

698 - self.use_fp4_indexer_cache = dsa_indexer_uses_fp4(self.vllm_config)

698 + self.indexer_uses_fp4 = dsa_indexer_uses_fp4(self.vllm_config)

699

700 next_n = self.num_speculative_tokens + 1

701 self.decode_threshold = next_n

@@ -708,7 +708,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):

708 self.use_flattening,

709 self.supports_varlen,

710 next_n,

711 - self.use_fp4_indexer_cache,

711 + self.indexer_uses_fp4,

712 )

713

714 sm_count = num_compute_units(self.device.index)

vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py

+5 -2

@@ -17,6 +17,7 @@ from vllm.model_executor.layers.attention.mla_attention import (

17 )

18 from vllm.platforms import current_platform

19 from vllm.triton_utils import tl, triton

20 +from vllm.utils.gpu_sync_debug import gpu_sync_allowed

21 from vllm.utils.torch_utils import np_to_pinned_tensor

22 from vllm.v1.attention.backend import (

23 AttentionBackend,

@@ -638,12 +639,14 @@ class ROCMAiterMLASparseMetadataBuilder(

639 reduce_partial_map = None

640 if self._use_persistent_metadata and not use_triton_sparse:

641 num_reqs = common_attn_metadata.num_reqs

642 + with gpu_sync_allowed():

643 + seq_lens_cpu = common_attn_metadata.seq_lens[:num_reqs].cpu().numpy()

644 clamped_seq_lens = np.minimum(

642 - common_attn_metadata.seq_lens_cpu[:num_reqs].numpy(),

645 + seq_lens_cpu,

646 self.topk_tokens,

647 )

648 clamped_context_lens = np.minimum(

646 - common_attn_metadata.seq_lens_cpu[:num_reqs].numpy() - seg_lengths,

649 + seq_lens_cpu - seg_lengths,

650 self.topk_tokens,

651 )

652 metadata_key = (

vllm/v1/attention/backends/utils.py

+2 -8

@@ -478,7 +478,7 @@ def make_local_attention_virtual_batches(

478 query_start_loc_np = common_attn_metadata.query_start_loc_cpu.numpy()

479 with gpu_sync_allowed():

480 # TODO see https://github.com/vllm-project/vllm/pull/31852

481 - seq_lens_np = common_attn_metadata.seq_lens_cpu.numpy()

481 + seq_lens_np = common_attn_metadata.seq_lens.cpu().numpy()

482 block_table = common_attn_metadata.block_table_tensor

483 device = common_attn_metadata.query_start_loc.device

484

@@ -541,8 +541,6 @@ def make_local_attention_virtual_batches(

541 # seqlens_k_local = [4, 2, 4, 4, 4, 1, 4, 1]

542 seqlens_k_local = np.full(cu_num_blocks[-1], attn_chunk_size, dtype=np.int32)

543 seqlens_k_local[cu_num_blocks - 1] = tokens_in_last_block

544 - num_computed_tokens_local = seqlens_k_local - seqlens_q_local

545 -

544 k_seqstarts_absolute = np.repeat(seq_lens_np, local_blocks) - (

545 rarange * attn_chunk_size + np.repeat(tokens_in_last_block, local_blocks)

546 )

@@ -612,9 +610,7 @@ def make_local_attention_virtual_batches(

610 block_table_tensor=block_table_local,

611 slot_mapping=common_attn_metadata.slot_mapping,

612 causal=True,

615 - seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound,

616 - _seq_lens_cpu=seq_lens_cpu,

617 - _num_computed_tokens_cpu=torch.from_numpy(num_computed_tokens_local),

613 + seq_lens_cpu_upper_bound=seq_lens_cpu,

614 ), make_block_table

615

616

@@ -692,8 +688,6 @@ def make_kv_sharing_fast_prefill_common_attn_metadata(

688 slot_mapping=common_attn_metadata.slot_mapping,

689 causal=True,

690 seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound,

695 - _seq_lens_cpu=common_attn_metadata._seq_lens_cpu,

696 - _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu,

691 )

692 return common_attn_metadata

693

vllm/v1/spec_decode/dflash.py

-2

@@ -192,8 +192,6 @@ class DFlashProposer(SpecDecodeBaseProposer):

192 torch.from_numpy(self.token_arange_np[: batch_size + 1]).clone()

193 * num_query_per_req

194 ),

195 - _seq_lens_cpu=None,

196 - _num_computed_tokens_cpu=None,

195 seq_lens_cpu_upper_bound=new_seq_lens_cpu_upper_bound,

196 num_reqs=cad.num_reqs,

197 num_actual_tokens=num_query_total,

vllm/v1/spec_decode/llm_base_proposer.py

-11

@@ -669,9 +669,6 @@ class SpecDecodeBaseProposer:

669 # (i.e., not the first proposal).

670 if self.num_speculative_tokens > 1 and num_rejected_tokens_gpu is not None:

671 common_attn_metadata.seq_lens -= num_rejected_tokens_gpu

672 - # Invalidate the CPU-side shadows to avoid H<>D sync.

673 - common_attn_metadata._seq_lens_cpu = None

674 - common_attn_metadata._num_computed_tokens_cpu = None

672

673 block_size = self.block_size

674 assert block_size > 0, "block_size has not been initialized."

@@ -798,10 +795,6 @@ class SpecDecodeBaseProposer:

795 self.max_model_len,

796 )

797

801 - if common_attn_metadata._seq_lens_cpu is not None:

802 - common_attn_metadata._seq_lens_cpu += 1

803 - if common_attn_metadata._num_computed_tokens_cpu is not None:

804 - common_attn_metadata._num_computed_tokens_cpu += 1

798 if common_attn_metadata.seq_lens_cpu_upper_bound is not None:

799 common_attn_metadata.seq_lens_cpu_upper_bound += 1

800

@@ -1134,8 +1127,6 @@ class SpecDecodeBaseProposer:

1127 query_start_loc=common_attn_metadata.query_start_loc,

1128 seq_lens=common_attn_metadata.seq_lens,

1129 query_start_loc_cpu=query_start_loc_cpu,

1137 - _seq_lens_cpu=common_attn_metadata._seq_lens_cpu,

1138 - _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu,

1130 seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound,

1131 num_reqs=common_attn_metadata.num_reqs,

1132 num_actual_tokens=total_num_tokens,

@@ -1245,8 +1236,6 @@ class SpecDecodeBaseProposer:

1236 query_start_loc=async_tensor_h2d(new_query_start_loc_cpu, device=device),

1237 seq_lens=async_tensor_h2d(new_seq_lens_cpu, device=device),

1238 query_start_loc_cpu=new_query_start_loc_cpu,

1248 - _seq_lens_cpu=new_seq_lens_cpu,

1249 - _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu,

1239 seq_lens_cpu_upper_bound=new_seq_lens_cpu,

1240 num_reqs=common_attn_metadata.num_reqs,

1241 num_actual_tokens=total_num_tokens,

vllm/v1/spec_decode/step3p5.py

-2

@@ -386,8 +386,6 @@ class Step3p5MTPProposer(EagleProposer):

386

387 if self.num_speculative_tokens > 1 and num_rejected_tokens_gpu is not None:

388 common_attn_metadata.seq_lens -= num_rejected_tokens_gpu

389 - common_attn_metadata._seq_lens_cpu = None

390 - common_attn_metadata._num_computed_tokens_cpu = None

389

390 block_size = self.block_size

391 assert block_size > 0, "block_size has not been initialized."

vllm/v1/worker/gpu_model_runner.py

+1 -9

@@ -2377,8 +2377,7 @@ class GPUModelRunner(

2377 num_prompt_tokens_cpu = self.input_batch.num_prompt_tokens_cpu_tensor[

2378 :num_reqs_padded

2379 ]

2380 - seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs_padded]

2381 - seq_lens_cpu_upper_bound = seq_lens_cpu

2380 + seq_lens_cpu_upper_bound = self.optimistic_seq_lens_cpu[:num_reqs_padded]

2381

2382 # is_prefilling: True if request is still in prefill phase.

2383 # Used by mamba backends to distinguish actual decodes from

@@ -2388,11 +2387,6 @@ class GPUModelRunner(

2387 # misclassify padding as prefill in CUDA graph mode.

2388 is_prefilling[num_reqs:] = False

2389

2391 - if self.use_async_spec_decode:

2392 - # GPU tensors are authoritative in async mode.

2393 - seq_lens_cpu = None

2394 - num_computed_tokens_cpu = None

2395 -

2390 # Compute mm_prefix bidirectional ranges before building

2391 # attention metadata so builders handle them during build().

2392 # By default, ranges exceeding sliding_window are skipped to prevent

@@ -2466,8 +2460,6 @@ class GPUModelRunner(

2460 query_start_loc=self.query_start_loc.gpu[: num_reqs_padded + 1],

2461 query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs_padded + 1],

2462 seq_lens=self.seq_lens[:num_reqs_padded],

2469 - _seq_lens_cpu=seq_lens_cpu,

2470 - _num_computed_tokens_cpu=num_computed_tokens_cpu,

2463 seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,

2464 replayssm_decode_base_cpu=replayssm_decode_base_cpu,

2465 num_reqs=num_reqs_padded,

vllm/v1/worker/ubatch_utils.py

-17

@@ -280,23 +280,11 @@ def _make_metadata_with_slice(

280 query_start_loc[1:] -= tokens_skipped

281 query_start_loc_cpu[1:] -= tokens_skipped

282 seq_lens = attn_metadata.seq_lens[request_slice]

283 - # Read raw fields to avoid triggering the deprecated D2H-syncing properties.

284 - seq_lens_cpu = (

285 - attn_metadata._seq_lens_cpu[request_slice]

286 - if attn_metadata._seq_lens_cpu is not None

287 - else None

288 - )

283 seq_lens_cpu_upper_bound = (

284 attn_metadata.seq_lens_cpu_upper_bound[request_slice]

285 if attn_metadata.seq_lens_cpu_upper_bound is not None

286 else None

287 )

294 - num_computed_tokens_cpu = (

295 - attn_metadata._num_computed_tokens_cpu[request_slice]

296 - if attn_metadata._num_computed_tokens_cpu is not None

297 - else None

298 - )

299 -

288 if splits_last_request:

289 # NOTE: We use start_locs (the original query_start_loc_cpu) to calculate

290 # the tokens skipped because query_start_loc_cpu might have been modified

@@ -309,9 +297,6 @@ def _make_metadata_with_slice(

297 # (not cudagraph compatible)

298 seq_lens = seq_lens.clone()

299 seq_lens[-1] -= tokens_skipped

312 - if seq_lens_cpu is not None:

313 - seq_lens_cpu = seq_lens_cpu.clone()

314 - seq_lens_cpu[-1] -= tokens_skipped

300 if seq_lens_cpu_upper_bound is not None:

301 seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound.clone()

302 seq_lens_cpu_upper_bound[-1] -= tokens_skipped

@@ -346,8 +331,6 @@ def _make_metadata_with_slice(

331 block_table_tensor=block_table_tensor,

332 slot_mapping=slot_mapping,

333 seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,

349 - _seq_lens_cpu=seq_lens_cpu,

350 - _num_computed_tokens_cpu=num_computed_tokens_cpu,

334 )

335

336

[Deprecation] Deprecate items scheduled for 0.29 (#55353)