A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 09:15 UTC
[Frontend] Add --spec-method/--spec-model/--spec-tokens CLI aliases (#42476)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
Michael Goin committed
May 18, 2026 at 20:22 UTC
239b5ff30cf46f9196149c888a20be2096fdff03
2 files changed
+34
vllm/engine/arg_utils.py
+24
@@ -607,6 +607,9 @@ class EngineArgs:
607
reasoning_parser_plugin: str | None = None
608
609
speculative_config: dict[str, Any] | None = None
610
+ spec_method: str | None = None
611
+ spec_model: str | None = None
612
+ spec_tokens: int | None = None
613
614
show_hidden_metrics_for_version: str | None = (
615
ObservabilityConfig.show_hidden_metrics_for_version
@@ -1435,6 +1438,12 @@ class EngineArgs:
1438
vllm_group.add_argument(
1439
"--speculative-config", "-sc", **vllm_kwargs["speculative_config"]
1440
)
1441
+ speculative_kwargs = get_kwargs(SpeculativeConfig)
1442
+ vllm_group.add_argument("--spec-method", **speculative_kwargs["method"])
1443
+ vllm_group.add_argument("--spec-model", **speculative_kwargs["model"])
1444
+ vllm_group.add_argument(
1445
+ "--spec-tokens", **speculative_kwargs["num_speculative_tokens"]
1446
+ )
1447
vllm_group.add_argument(
1448
"--kv-transfer-config", **vllm_kwargs["kv_transfer_config"]
1449
)
@@ -1634,6 +1643,21 @@ class EngineArgs:
1643
"""Initializes and returns a SpeculativeConfig object based on
1644
`speculative_config`.
1645
"""
1646
+ for flag, key, value in (
1647
+ ("--spec-method", "method", self.spec_method),
1648
+ ("--spec-model", "model", self.spec_model),
1649
+ ("--spec-tokens", "num_speculative_tokens", self.spec_tokens),
1650
+ ):
1651
+ if value is None:
1652
+ continue
1653
+ if self.speculative_config is None:
1654
+ self.speculative_config = {}
1655
+ if key in self.speculative_config:
1656
+ raise ValueError(
1657
+ f"{flag} and --speculative-config['{key}'] are mutually exclusive"
1658
+ )
1659
+ self.speculative_config[key] = value
1660
+
1661
if self.speculative_config is None:
1662
return None
1663
vllm/entrypoints/llm.py
+10
@@ -188,6 +188,10 @@ class LLM(PoolingOfflineMixin):
188
dictionary or an AttentionConfig instance. If a dictionary, it will
189
be converted to an AttentionConfig. Allows specifying the attention
190
backend and other attention-related settings.
191
+ spec_method: Top-level alias for `speculative_config["method"]`.
192
+ spec_model: Top-level alias for `speculative_config["model"]`.
193
+ spec_tokens: Top-level alias for
194
+ `speculative_config["num_speculative_tokens"]`.
195
**kwargs: Arguments for [`EngineArgs`][vllm.EngineArgs].
196
197
Note:
@@ -236,6 +240,9 @@ class LLM(PoolingOfflineMixin):
240
compilation_config: int | dict[str, Any] | CompilationConfig | None = None,
241
quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None,
242
logits_processors: list[str | type[LogitsProcessor]] | None = None,
243
+ spec_method: str | None = None,
244
+ spec_model: str | None = None,
245
+ spec_tokens: int | None = None,
246
**kwargs: Any,
247
) -> None:
248
"""LLM constructor."""
@@ -357,6 +364,9 @@ class LLM(PoolingOfflineMixin):
364
compilation_config=compilation_config_instance,
365
quantization_config=quantization_config,
366
logits_processors=logits_processors,
367
+ spec_method=spec_method,
368
+ spec_model=spec_model,
369
+ spec_tokens=spec_tokens,
370
**kwargs,
371
)
372