A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[V1][Spec Decode] Remove deprecated spec decode config params (#15466)
Signed-off-by: Shangming Cai <caishangming@linux.alibaba.com>
shangmingc committed
Apr 1, 2025 at 00:19 UTC
239b7befddb486d121779a7f80d1f5d9dcdd248f
10 files changed
+125
-220
.buildkite/nightly-benchmarks/tests/serving-tests.json
+6
-4
@@ -63,10 +63,12 @@
63
"model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
64
"disable_log_requests": "",
65
"tensor_parallel_size": 4,
66
- "swap_space": 16,
67
- "speculative_model": "turboderp/Qwama-0.5B-Instruct",
68
- "num_speculative_tokens": 4,
69
- "speculative_draft_tensor_parallel_size": 1
66
+ "swap_space": 16,
67
+ "speculative_config": {
68
+ "model": "turboderp/Qwama-0.5B-Instruct",
69
+ "num_speculative_tokens": 4,
70
+ "draft_tensor_parallel_size": 1
71
+ }
72
},
73
"client_parameters": {
74
"model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
docs/source/features/spec_decode.md
+1
-1
@@ -52,7 +52,7 @@ python -m vllm.entrypoints.openai.api_server --host 0.0.0.0 --port 8000 --model
52
```
53
54
:::{warning}
55
-Note: Please use `--speculative_config` to set all configurations related to speculative decoding. The previous method of specifying the model through `--speculative_model` and adding related parameters (e.g., `--num_speculative_tokens`) separately will be deprecated in the next release.
55
+Note: Please use `--speculative_config` to set all configurations related to speculative decoding. The previous method of specifying the model through `--speculative_model` and adding related parameters (e.g., `--num_speculative_tokens`) separately has been deprecated now.
56
:::
57
58
Then use a client:
examples/offline_inference/eagle.py
+6
-4
@@ -69,10 +69,12 @@ llm = LLM(
69
max_model_len=max_model_len,
70
max_num_seqs=args.max_num_seqs,
71
gpu_memory_utilization=0.8,
72
- speculative_model=eagle_dir,
73
- num_speculative_tokens=args.num_spec_tokens,
74
- speculative_draft_tensor_parallel_size=args.draft_tp,
75
- speculative_max_model_len=max_model_len,
72
+ speculative_config={
73
+ "model": eagle_dir,
74
+ "num_speculative_tokens": args.num_spec_tokens,
75
+ "draft_tensor_parallel_size": args.draft_tp,
76
+ "max_model_len": max_model_len,
77
+ },
78
disable_log_stats=False,
79
)
80
tests/metrics/test_metrics.py
+8
-4
@@ -248,8 +248,10 @@ def test_metric_spec_decode(
248
dtype=dtype,
249
disable_log_stats=False,
250
gpu_memory_utilization=0.4,
251
- speculative_model=model,
252
- num_speculative_tokens=k,
251
+ speculative_config={
252
+ "model": model,
253
+ "num_speculative_tokens": k,
254
+ },
255
) as vllm_model:
256
257
# Force log interval to be 0 to catch all metrics.
@@ -300,8 +302,10 @@ def test_metric_spec_decode_interval(
302
dtype=dtype,
303
disable_log_stats=False,
304
gpu_memory_utilization=0.4,
303
- speculative_model=model,
304
- num_speculative_tokens=k,
305
+ speculative_config={
306
+ "model": model,
307
+ "num_speculative_tokens": k,
308
+ },
309
enforce_eager=True,
310
)
311
tests/models/test_initialization.py
+4
-2
@@ -54,8 +54,10 @@ def test_can_initialize(model_arch):
54
model_info.default,
55
tokenizer=model_info.tokenizer,
56
tokenizer_mode=model_info.tokenizer_mode,
57
- speculative_model=model_info.speculative_model,
58
- num_speculative_tokens=1 if model_info.speculative_model else None,
57
+ speculative_config={
58
+ "model": model_info.speculative_model,
59
+ "num_speculative_tokens": 1,
60
+ } if model_info.speculative_model else None,
61
trust_remote_code=model_info.trust_remote_code,
62
load_format="dummy",
63
hf_overrides=hf_overrides,
tests/spec_decode/e2e/test_integration_dist_tp2.py
+70
-10
@@ -3,6 +3,7 @@
3
tensor parallelism.
4
"""
5
6
+import json
7
from typing import Optional
8
9
import pytest
@@ -28,14 +29,14 @@ from .conftest import run_equality_correctness_test_tp
29
@pytest.mark.parametrize("test_llm_kwargs", [
30
[
31
"--speculative_config",
31
- str({
32
+ json.dumps({
33
"model": "JackFram/llama-68m",
34
"num_speculative_tokens": 3,
35
}),
36
],
37
[
38
"--speculative_config",
38
- str({
39
+ json.dumps({
40
"model": "ngram",
41
"num_speculative_tokens": 5,
42
"prompt_lookup_max": 3,
@@ -88,7 +89,7 @@ def test_target_model_tp_gt_1(common_llm_kwargs, per_test_common_llm_kwargs,
89
"model, test_llm_kwargs",
90
[("JackFram/llama-68m", [
91
"--speculative_config",
91
- str({
92
+ json.dumps({
93
"model": "JackFram/llama-68m",
94
"num_speculative_tokens": 5,
95
"draft_tensor_parallel_size": 1,
@@ -96,7 +97,7 @@ def test_target_model_tp_gt_1(common_llm_kwargs, per_test_common_llm_kwargs,
97
]),
98
("ibm-granite/granite-3b-code-instruct", [
99
"--speculative_config",
99
- str({
100
+ json.dumps({
101
"model": "ibm-granite/granite-3b-code-instruct",
102
"num_speculative_tokens": 5,
103
"draft_tensor_parallel_size": 1,
@@ -147,20 +148,20 @@ def test_draft_model_tp_lt_target_model_tp2(model, common_llm_kwargs,
148
@pytest.mark.parametrize("model, test_llm_kwargs",
149
[("JackFram/llama-68m", [
150
"--speculative_config",
150
- str({
151
+ json.dumps({
152
"model": "JackFram/llama-68m",
153
"num_speculative_tokens": 3,
154
}),
155
]),
156
("JackFram/llama-68m", [
157
"--speculative_config",
157
- str({
158
+ json.dumps({
159
"model": "JackFram/llama-68m",
160
"num_speculative_tokens": 3,
161
"draft_tensor_parallel_size": 1,
162
}),
163
])])
163
-@pytest.mark.parametrize("logprobs", [None, 2])
164
+@pytest.mark.parametrize("logprobs", [None])
165
@pytest.mark.parametrize("batch_size", [2])
166
@pytest.mark.parametrize("seed", [1])
167
def test_spec_decode_chunked_prefill_tp2(model, common_llm_kwargs,
@@ -171,9 +172,68 @@ def test_spec_decode_chunked_prefill_tp2(model, common_llm_kwargs,
172
"""Verify spec decode works well with same and different TP size for
173
the draft model with chunked prefill.
174
"""
174
- if logprobs:
175
- test_llm_kwargs.extend(
176
- ["--disable_logprobs_during_spec_decoding", "False"])
175
+ run_equality_correctness_test_tp(model,
176
+ common_llm_kwargs,
177
+ per_test_common_llm_kwargs,
178
+ baseline_llm_kwargs,
179
+ test_llm_kwargs,
180
+ batch_size,
181
+ max_output_len=32,
182
+ seed=seed,
183
+ temperature=0.0,
184
+ logprobs=logprobs)
185
+
186
+
187
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
188
+ reason="Need at least 2 GPUs to run the test.")
189
+@pytest.mark.parametrize(
190
+ "common_llm_kwargs",
191
+ [[
192
+ # Skip cuda graph recording for fast test.
193
+ "--enforce-eager",
194
+ "--tensor_parallel_size",
195
+ "2",
196
+
197
+ # precision
198
+ "--dtype",
199
+ "bfloat16",
200
+ ]])
201
+@pytest.mark.parametrize(
202
+ "per_test_common_llm_kwargs",
203
+ [["--enable-chunked-prefill", "False"],
204
+ [
205
+ "--enable-chunked-prefill", "True", "--max-num-batched-tokens", "4",
206
+ "--max-num-seqs", "4"
207
+ ]])
208
+@pytest.mark.parametrize("baseline_llm_kwargs", [[]])
209
+@pytest.mark.parametrize("model, test_llm_kwargs",
210
+ [("JackFram/llama-68m", [
211
+ "--speculative_config",
212
+ json.dumps({
213
+ "model": "JackFram/llama-68m",
214
+ "num_speculative_tokens": 3,
215
+ "disable_logprobs": False,
216
+ }),
217
+ ]),
218
+ ("JackFram/llama-68m", [
219
+ "--speculative_config",
220
+ json.dumps({
221
+ "model": "JackFram/llama-68m",
222
+ "num_speculative_tokens": 3,
223
+ "draft_tensor_parallel_size": 1,
224
+ "disable_logprobs": False,
225
+ }),
226
+ ])])
227
+@pytest.mark.parametrize("logprobs", [2])
228
+@pytest.mark.parametrize("batch_size", [2])
229
+@pytest.mark.parametrize("seed", [1])
230
+def test_spec_decode_chunked_prefill_tp2_with_logprobs(
231
+ model, common_llm_kwargs, per_test_common_llm_kwargs,
232
+ baseline_llm_kwargs, test_llm_kwargs, logprobs: Optional[int],
233
+ batch_size: int, seed: int):
234
+ """Verify spec decode works well with same and different TP size for
235
+ the draft model with chunked prefill.
236
+ """
237
run_equality_correctness_test_tp(model,
238
common_llm_kwargs,
239
per_test_common_llm_kwargs,
tests/spec_decode/e2e/test_integration_dist_tp4.py
+4
-2
@@ -3,6 +3,8 @@
3
tensor parallelism.
4
"""
5
6
+import json
7
+
8
import openai
9
import pytest
10
import torch
@@ -33,7 +35,7 @@ SPEC_MODEL = "JackFram/llama-68m"
35
#TODO(wooyeon): add spec_draft_dp=2 case
36
[
37
"--speculative_config",
36
- str({
38
+ json.dumps({
39
"model": f"{SPEC_MODEL}",
40
"num_speculative_tokens": 5,
41
"draft_tensor_parallel_size": 1,
@@ -80,7 +82,7 @@ def test_draft_model_tp_lt_target_model_tp4(common_llm_kwargs,
82
# Artificially limit the draft model max model len; this forces vLLM
83
# to skip speculation once the sequences grow beyond 32-k tokens.
84
"--speculative_config",
83
- str({
85
+ json.dumps({
86
"model": f"{SPEC_MODEL}",
87
"num_speculative_tokens": 5,
88
"max_model_len": 32,
tests/v1/test_oracle.py
+3
-1
@@ -49,7 +49,9 @@ def test_unsupported_configs(monkeypatch):
49
with pytest.raises(NotImplementedError):
50
AsyncEngineArgs(
51
model=MODEL,
52
- speculative_model=MODEL,
52
+ speculative_config={
53
+ "model": MODEL,
54
+ },
55
).create_engine_config()
56
57
with pytest.raises(NotImplementedError):
vllm/config.py
+9
-10
@@ -2047,14 +2047,13 @@ class SpeculativeConfig:
2047
2048
def __post_init__(self):
2049
2050
- # Note: After next release, the method parameter will be used to
2051
- # specify the speculative method, which helps to extend the
2052
- # configuration of non-model-based proposers, and the model parameter
2053
- # will be used when the draft model or head is needed.
2054
- # If users do not specify the method, the speculative method will
2055
- # be detected automatically if possible. If the speculative method can
2056
- # not be detected, it will be considered as the draft-model-based
2057
- # method by default.
2050
+ # Note: "method" is a new parameter that helps to extend the
2051
+ # configuration of non-model-based proposers, and the "model" parameter
2052
+ # will be used to set the draft model, eagle head, or additional weight
2053
+ # when needed. If users do not specify "method", the speculative method
2054
+ # will be detected automatically if possible. If the speculative method
2055
+ # can not be detected, it will be considered as the "draft_model" by
2056
+ # default.
2057
2058
if self.model is None and self.num_speculative_tokens is not None:
2059
# TODO(Shangming): Refactor mtp configuration logic when supporting
@@ -2069,8 +2068,8 @@ class SpeculativeConfig:
2068
raise ValueError("num_speculative_tokens was provided without "
2069
"speculative model.")
2070
2072
- # Automatically configure the ngram method during configuration
2073
- # refactoring to ensure a smooth transition.
2071
+ # Automatically configure the method for ngram when "model" is used
2072
+ # instead of "method"
2073
if self.method is None and (self.model is not None
2074
and self.model in ("ngram", "[ngram]")):
2075
self.method = "ngram"
vllm/engine/arg_utils.py
+14
-182
@@ -181,22 +181,7 @@ class EngineArgs:
181
guided_decoding_backend: str = 'xgrammar'
182
logits_processor_pattern: Optional[str] = None
183
184
- speculative_config: Optional[Union[str, Dict[str, Any]]] = None
185
-
186
- # TODO(Shangming): Deprecate these out-of-date params after next release
187
- speculative_model: Optional[str] = None
188
- speculative_model_quantization: Optional[str] = None
189
- speculative_draft_tensor_parallel_size: Optional[int] = None
190
- num_speculative_tokens: Optional[int] = None
191
- speculative_disable_mqa_scorer: Optional[bool] = False
192
- speculative_max_model_len: Optional[int] = None
193
- speculative_disable_by_batch_size: Optional[int] = None
194
- ngram_prompt_lookup_max: Optional[int] = None
195
- ngram_prompt_lookup_min: Optional[int] = None
196
- spec_decoding_acceptance_method: str = 'rejection_sampler'
197
- typical_acceptance_sampler_posterior_threshold: Optional[float] = None
198
- typical_acceptance_sampler_posterior_alpha: Optional[float] = None
199
- disable_logprobs_during_spec_decoding: Optional[bool] = None
184
+ speculative_config: Optional[Dict[str, Any]] = None
185
186
qlora_adapter_name_or_path: Optional[str] = None
187
show_hidden_metrics_for_version: Optional[str] = None
@@ -793,122 +778,10 @@ class EngineArgs:
778
help='If set, the prefill requests can be chunked based on the '
779
'max_num_batched_tokens.')
780
parser.add_argument('--speculative-config',
796
- type=nullable_str,
781
+ type=json.loads,
782
default=None,
783
help='The configurations for speculative decoding.'
784
' Should be a JSON string.')
800
- parser.add_argument(
801
- '--speculative-model',
802
- type=nullable_str,
803
- default=EngineArgs.speculative_model,
804
- help=
805
- 'The name of the draft model to be used in speculative decoding.')
806
- # Quantization settings for speculative model.
807
- parser.add_argument(
808
- '--speculative-model-quantization',
809
- type=nullable_str,
810
- choices=[*QUANTIZATION_METHODS, None],
811
- default=EngineArgs.speculative_model_quantization,
812
- help='Method used to quantize the weights of speculative model. '
813
- 'If None, we first check the `quantization_config` '
814
- 'attribute in the model config file. If that is '
815
- 'None, we assume the model weights are not '
816
- 'quantized and use `dtype` to determine the data '
817
- 'type of the weights.')
818
- parser.add_argument(
819
- '--num-speculative-tokens',
820
- type=int,
821
- default=EngineArgs.num_speculative_tokens,
822
- help='The number of speculative tokens to sample from '
823
- 'the draft model in speculative decoding.')
824
- parser.add_argument(
825
- '--speculative-disable-mqa-scorer',
826
- action='store_true',
827
- help=
828
- 'If set to True, the MQA scorer will be disabled in speculative '
829
- ' and fall back to batch expansion')
830
- parser.add_argument(
831
- '--speculative-draft-tensor-parallel-size',
832
- '-spec-draft-tp',
833
- type=int,
834
- default=EngineArgs.speculative_draft_tensor_parallel_size,
835
- help='Number of tensor parallel replicas for '
836
- 'the draft model in speculative decoding.')
837
-
838
- parser.add_argument(
839
- '--speculative-max-model-len',
840
- type=int,
841
- default=EngineArgs.speculative_max_model_len,
842
- help='The maximum sequence length supported by the '
843
- 'draft model. Sequences over this length will skip '
844
- 'speculation.')
845
-
846
- parser.add_argument(
847
- '--speculative-disable-by-batch-size',
848
- type=int,
849
- default=EngineArgs.speculative_disable_by_batch_size,
850
- help='Disable speculative decoding for new incoming requests '
851
- 'if the number of enqueue requests is larger than this value.')
852
-
853
- parser.add_argument(
854
- '--ngram-prompt-lookup-max',
855
- type=int,
856
- default=EngineArgs.ngram_prompt_lookup_max,
857
- help='Max size of window for ngram prompt lookup in speculative '
858
- 'decoding.')
859
-
860
- parser.add_argument(
861
- '--ngram-prompt-lookup-min',
862
- type=int,
863
- default=EngineArgs.ngram_prompt_lookup_min,
864
- help='Min size of window for ngram prompt lookup in speculative '
865
- 'decoding.')
866
-
867
- parser.add_argument(
868
- '--spec-decoding-acceptance-method',
869
- type=str,
870
- default=EngineArgs.spec_decoding_acceptance_method,
871
- choices=['rejection_sampler', 'typical_acceptance_sampler'],
872
- help='Specify the acceptance method to use during draft token '
873
- 'verification in speculative decoding. Two types of acceptance '
874
- 'routines are supported: '
875
- '1) RejectionSampler which does not allow changing the '
876
- 'acceptance rate of draft tokens, '
877
- '2) TypicalAcceptanceSampler which is configurable, allowing for '
878
- 'a higher acceptance rate at the cost of lower quality, '
879
- 'and vice versa.')
880
-
881
- parser.add_argument(
882
- '--typical-acceptance-sampler-posterior-threshold',
883
- type=float,
884
- default=EngineArgs.typical_acceptance_sampler_posterior_threshold,
885
- help='Set the lower bound threshold for the posterior '
886
- 'probability of a token to be accepted. This threshold is '
887
- 'used by the TypicalAcceptanceSampler to make sampling decisions '
888
- 'during speculative decoding.')
889
-
890
- parser.add_argument(
891
- '--typical-acceptance-sampler-posterior-alpha',
892
- type=float,
893
- default=EngineArgs.typical_acceptance_sampler_posterior_alpha,
894
- help='A scaling factor for the entropy-based threshold for token '
895
- 'acceptance in the TypicalAcceptanceSampler. Typically defaults '
896
- 'to sqrt of --typical-acceptance-sampler-posterior-threshold '
897
- 'i.e. 0.3')
898
-
899
- parser.add_argument(
900
- '--disable-logprobs-during-spec-decoding',
901
- action=StoreBoolean,
902
- default=EngineArgs.disable_logprobs_during_spec_decoding,
903
- nargs="?",
904
- const="True",
905
- help='If set to True, token log probabilities are not returned '
906
- 'during speculative decoding. If set to False, log probabilities '
907
- 'are returned according to the settings in SamplingParams. If '
908
- 'not specified, it defaults to True. Disabling log probabilities '
909
- 'during speculative decoding reduces latency by skipping logprob '
910
- 'calculation in proposal sampling, target sampling, and after '
911
- 'accepted tokens are determined.')
785
786
parser.add_argument('--model-loader-extra-config',
787
type=nullable_str,
@@ -1221,58 +1094,14 @@ class EngineArgs:
1094
This function utilizes `speculative_config` to create a
1095
SpeculativeConfig object. The `speculative_config` can either be
1096
provided as a JSON string input via CLI arguments or directly as a
1224
- dictionary from the engine. If `speculative_config` is not set, this
1225
- function will attempt to construct a configuration dictionary using
1226
- certain parameters, which are scheduled for deprecation in the next
1227
- release. Note that in next releases, `speculative_config` must be
1228
- provided, and the deprecated standalone speculative-related parameters
1229
- will be removed.
1097
+ dictionary from the engine.
1098
"""
1099
if self.speculative_config is None:
1232
- if (self.speculative_model is None
1233
- and self.num_speculative_tokens is None):
1234
- return None
1235
-
1236
- # TODO(Shangming): Deprecate this way of setting SpeculativeConfig,
1237
- # only allow '--speculative-config' after next release
1238
- logger.warning_once(
1239
- "Please use '--speculative-config' to set all configurations "
1240
- "related to speculative decoding. The current method of "
1241
- "specifying the model through '--speculative-model' and "
1242
- "adding related parameters (e.g., '--num-speculative-tokens') "
1243
- "separately will be deprecated in the next release.")
1244
-
1245
- spec_config_dict = {
1246
- "model": self.speculative_model,
1247
- "quantization": self.speculative_model_quantization,
1248
- "max_model_len": self.speculative_max_model_len,
1249
- "draft_tensor_parallel_size":
1250
- self.speculative_draft_tensor_parallel_size,
1251
- "num_speculative_tokens": self.num_speculative_tokens,
1252
- "disable_mqa_scorer": self.speculative_disable_mqa_scorer,
1253
- "disable_by_batch_size":
1254
- self.speculative_disable_by_batch_size,
1255
- "prompt_lookup_max": self.ngram_prompt_lookup_max,
1256
- "prompt_lookup_min": self.ngram_prompt_lookup_min,
1257
- "acceptance_method": self.spec_decoding_acceptance_method,
1258
- "posterior_threshold":
1259
- self.typical_acceptance_sampler_posterior_threshold,
1260
- "posterior_alpha":
1261
- self.typical_acceptance_sampler_posterior_alpha,
1262
- "disable_logprobs": self.disable_logprobs_during_spec_decoding,
1263
- }
1100
+ return None
1101
1265
- self.speculative_config = spec_config_dict
1266
- else:
1267
- if isinstance(self.speculative_config, str):
1268
- import ast
1269
- self.speculative_config = ast.literal_eval(
1270
- self.speculative_config)
1102
# Note(Shangming): These parameters are not obtained from the cli arg
1103
# '--speculative-config' and must be passed in when creating the engine
1104
# config.
1274
-
1275
- assert isinstance(self.speculative_config, dict)
1105
self.speculative_config.update({
1106
"target_model_config": target_model_config,
1107
"target_parallel_config": target_parallel_config,
@@ -1638,11 +1467,15 @@ class EngineArgs:
1467
return False
1468
1469
# Only Ngram speculative decoding so far.
1641
- if (self.speculative_model is not None
1642
- or self.num_speculative_tokens is not None):
1470
+ is_ngram_enabled = False
1471
+ if self.speculative_config is not None:
1472
# This is supported but experimental (handled below).
1644
- if self.speculative_model in ("ngram", "[ngram]"):
1645
- pass
1473
+ if (("method" in self.speculative_config
1474
+ and self.speculative_config["method"] in ("ngram", "[ngram]"))
1475
+ or
1476
+ ("model" in self.speculative_config and
1477
+ self.speculative_config["model"] in ("ngram", "[ngram]"))):
1478
+ is_ngram_enabled = True
1479
else:
1480
_raise_or_fallback(feature_name="Speculative Decoding",
1481
recommend_to_remove=False)
@@ -1691,8 +1524,7 @@ class EngineArgs:
1524
return False
1525
1526
# ngram is supported on V1, but off by default for now.
1694
- if self.speculative_model in (
1695
- "ngram", "[ngram]") and _warn_or_fallback("ngram"):
1527
+ if is_ngram_enabled and _warn_or_fallback("ngram"):
1528
return False
1529
1530
# Non-CUDA is supported on V1, but off by default for now.
@@ -1721,7 +1553,7 @@ class EngineArgs:
1553
is_gpu = current_platform.is_cuda()
1554
use_sliding_window = (model_config.get_sliding_window()
1555
is not None)
1724
- use_spec_decode = self.speculative_model is not None
1556
+ use_spec_decode = self.speculative_config is not None
1557
1558
if (is_gpu and not use_sliding_window and not use_spec_decode
1559
and not self.enable_lora