A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Speculative Decoding] MLPSpeculator Tensor Parallel support (1/2) (#6050)
Co-authored-by: Sirej Dua <sirej.dua@databricks.com> Co-authored-by: Sirej Dua <Sirej Dua>
Sirej Dua committed
Jul 2, 2024 at 07:20 UTC
15aba081f33e6d048422df6dcdb94301d08d13e6
3 files changed
+35
-25
tests/spec_decode/e2e/test_integration_dist_tp2.py
+24
-12
@@ -70,10 +70,6 @@ def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
70
@pytest.mark.parametrize(
71
"common_llm_kwargs",
72
[{
73
- # Use a small model for a fast test.
74
- # Note this is repeated in the test body; to initialize a tokenizer.
75
- "model": "JackFram/llama-68m",
76
-
73
# Skip cuda graph recording for fast test.
74
"enforce_eager": True,
75
@@ -88,15 +84,31 @@ def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
84
# second run of the test to fail with internal NCCL error.
85
"use_async": True,
86
}])
91
-@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
87
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
93
-@pytest.mark.parametrize("test_llm_kwargs", [
94
- {
95
- "speculative_model": "JackFram/llama-68m",
96
- "num_speculative_tokens": 5,
97
- "speculative_draft_tensor_parallel_size": 1,
98
- },
99
-])
88
+@pytest.mark.parametrize(
89
+ "per_test_common_llm_kwargs, test_llm_kwargs",
90
+ [
91
+ (
92
+ {
93
+ # Use a small model for a fast test.
94
+ # Note this is repeated in the test body; to initialize a
95
+ # tokenizer.
96
+ "model": "JackFram/llama-68m",
97
+ },
98
+ {
99
+ "speculative_model": "JackFram/llama-68m",
100
+ "num_speculative_tokens": 5,
101
+ "speculative_draft_tensor_parallel_size": 1,
102
+ }),
103
+ ({
104
+ "model": "ibm-granite/granite-3b-code-instruct",
105
+ }, {
106
+ "speculative_model":
107
+ "ibm-granite/granite-3b-code-instruct-accelerator",
108
+ "num_speculative_tokens": 5,
109
+ "speculative_draft_tensor_parallel_size": 1,
110
+ })
111
+ ])
112
@pytest.mark.parametrize("batch_size", [2])
113
@pytest.mark.parametrize("seed", [1])
114
def test_draft_model_tp_lt_target_model_tp2(test_llm_generator,
vllm/config.py
-6
@@ -957,12 +957,6 @@ class SpeculativeConfig:
957
)
958
959
draft_hf_config = draft_model_config.hf_config
960
- if (draft_hf_config.model_type == "mlp_speculator"
961
- and target_parallel_config.world_size != 1):
962
- # MLPSpeculator TP support will be added very soon
963
- raise ValueError(
964
- "Speculative decoding with mlp_speculator models does not "
965
- "yet support distributed inferencing (TP > 1).")
960
961
if (num_speculative_tokens is not None
962
and hasattr(draft_hf_config, "num_lookahead_tokens")):
vllm/spec_decode/spec_decode_worker.py
+11
-7
@@ -113,24 +113,28 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
113
draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
114
115
disable_bonus_tokens = True
116
+
117
if ngram_prompt_lookup_max > 0:
118
disable_bonus_tokens = False
119
proposer_worker = NGramWorker(**draft_worker_kwargs)
120
proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
121
ngram_prompt_lookup_max)
121
- elif draft_worker_kwargs[
122
- "model_config"].hf_config.model_type == "mlp_speculator":
123
- proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
124
- disable_bonus_tokens = False
122
else:
123
draft_parallel_config: ParallelConfig = draft_worker_kwargs[
124
'parallel_config']
125
draft_tp = draft_parallel_config.tensor_parallel_size
126
target_tp = scorer_worker.parallel_config.tensor_parallel_size
127
131
- if draft_tp == 1:
132
- draft_worker_kwargs["model_runner_cls"] = TP1DraftModelRunner
133
- proposer_worker = MultiStepWorker(**draft_worker_kwargs)
128
+ if draft_worker_kwargs[
129
+ "model_config"].hf_config.model_type == "mlp_speculator":
130
+ disable_bonus_tokens = False
131
+ proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
132
+ else:
133
+ if draft_tp == 1:
134
+ draft_worker_kwargs[
135
+ "model_runner_cls"] = TP1DraftModelRunner
136
+ proposer_worker = MultiStepWorker(**draft_worker_kwargs)
137
+
138
proposer_worker = SmallerTpProposerWorker.maybe_wrap_worker(
139
proposer_worker, draft_tp, target_tp)
140