Disable dynamic speculative decoding when DP is enabled (#45963)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Tyler Michael Smith committed
Jul 7, 2026 at 08:52 UTC
93e2ab71119ff08805adc93be75196450382b088
4 files changed
+60
-1
docs/features/speculative_decoding/dynamic_speculative_decoding.md
+1
@@ -73,3 +73,4 @@ VLLM_USE_V2_MODEL_RUNNER=0 vllm serve meta-llama/Llama-3.1-8B-Instruct \
73
74
* Tested with Eagle, Eagle-3, and DFlash. Other SD methods may or may not work out of the box
75
* Full Cudagraph only works with Model Runner V2. MRv1 only supports piece-wise cuda graph with this feature
76
+* Not compatible with data parallelism (`--data-parallel-size > 1`). Each DP rank schedules independently, so ranks can pick different K values, causing DP collective divergence and deadlocks. When DP is enabled, vLLM automatically disables `num_speculative_tokens_per_batch_size` and falls back to the static `num_speculative_tokens` value.
tests/v1/core/utils.py
+10
-1
@@ -58,6 +58,8 @@ def create_scheduler(
58
skip_tokenizer_init: bool = False,
59
async_scheduling: bool = False,
60
pipeline_parallel_size: int = 1,
61
+ data_parallel_size: int = 1,
62
+ num_speculative_tokens_per_batch_size: list[tuple[int, int, int]] | None = None,
63
use_ec_connector: bool = False,
64
ec_role: str | None = None,
65
use_v2_model_runner: bool | None = None,
@@ -130,6 +132,10 @@ def create_scheduler(
132
spec_kwargs: dict = dict(
133
model="ngram", num_speculative_tokens=num_speculative_tokens
134
)
135
+ if num_speculative_tokens_per_batch_size is not None:
136
+ spec_kwargs["num_speculative_tokens_per_batch_size"] = (
137
+ num_speculative_tokens_per_batch_size
138
+ )
139
if speculative_method is not None:
140
spec_kwargs["method"] = speculative_method
141
spec_kwargs["prompt_lookup_max"] = num_speculative_tokens
@@ -150,7 +156,10 @@ def create_scheduler(
156
scheduler_config=scheduler_config,
157
model_config=model_config,
158
cache_config=cache_config,
153
- parallel_config=ParallelConfig(pipeline_parallel_size=pipeline_parallel_size),
159
+ parallel_config=ParallelConfig(
160
+ pipeline_parallel_size=pipeline_parallel_size,
161
+ data_parallel_size=data_parallel_size,
162
+ ),
163
kv_transfer_config=kv_transfer_config,
164
speculative_config=speculative_config,
165
ec_transfer_config=ec_transfer_config,
tests/v1/spec_decode/test_dynamic_sd.py
+29
@@ -2,6 +2,8 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""Regression tests for the Dynamic SD batch-size schedule helpers."""
4
5
+import logging
6
+
7
import pytest
8
9
from tests.v1.core.utils import create_requests, create_scheduler
@@ -187,6 +189,33 @@ def test_scheduler_falls_back_to_static_k_when_dsd_not_configured():
189
assert output.num_spec_tokens_to_schedule == 3
190
191
192
+def test_dynamic_sd_is_disabled_with_data_parallel(caplog_vllm):
193
+ with caplog_vllm.at_level(logging.WARNING, logger="vllm"):
194
+ scheduler = create_scheduler(
195
+ max_num_seqs=256,
196
+ max_num_batched_tokens=2560,
197
+ num_speculative_tokens=3,
198
+ num_speculative_tokens_per_batch_size=[
199
+ (1, 16, 3),
200
+ (64, 128, 2),
201
+ (256, 4096, 0),
202
+ ],
203
+ data_parallel_size=2,
204
+ )
205
+
206
+ speculative_config = scheduler.vllm_config.speculative_config
207
+ assert speculative_config is not None
208
+ assert speculative_config.num_speculative_tokens_per_batch_size is None
209
+ assert scheduler.dynamic_sd_lookup is None
210
+ assert "Dynamic speculative decoding is not supported with data parallelism" in (
211
+ caplog_vllm.text
212
+ )
213
+
214
+ output = _add_requests_and_schedule(scheduler, 256)
215
+ assert len(output.num_scheduled_tokens) == 256
216
+ assert output.num_spec_tokens_to_schedule == 3
217
+
218
+
219
def test_scheduler_uses_static_k_when_no_requests_are_scheduled():
220
scheduler = _make_scheduler_with_dynamic_sd(
221
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
vllm/config/vllm.py
+20
@@ -787,6 +787,25 @@ class VllmConfig:
787
)
788
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
789
790
+ def _maybe_disable_dynamic_sd_for_data_parallel(self) -> None:
791
+ speculative_config = self.speculative_config
792
+ if (
793
+ speculative_config is None
794
+ or not speculative_config.uses_dynamic_speculative_decoding()
795
+ or self.parallel_config.data_parallel_size <= 1
796
+ ):
797
+ return
798
+
799
+ logger.warning_once(
800
+ "Dynamic speculative decoding is not supported with data "
801
+ "parallelism because data-parallel ranks can select different "
802
+ "speculative-token counts, causing DP divergence and deadlocks. "
803
+ "Disabling num_speculative_tokens_per_batch_size and falling back "
804
+ "to static num_speculative_tokens=%d.",
805
+ speculative_config.num_speculative_tokens,
806
+ )
807
+ speculative_config.num_speculative_tokens_per_batch_size = None
808
+
809
def _post_init_kv_transfer_config(self) -> None:
810
"""Update KVTransferConfig based on top-level configs in VllmConfig.
811
@@ -1201,6 +1220,7 @@ class VllmConfig:
1220
"optimization level defaults."
1221
)
1222
1223
+ self._maybe_disable_dynamic_sd_for_data_parallel()
1224
self._maybe_override_dynamic_sd_cudagraph_mode()
1225
1226
if (