@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 15aba081f3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Speculative Decoding] MLPSpeculator Tensor Parallel support (1/2) (#6050)

Co-authored-by: Sirej Dua <sirej.dua@databricks.com> Co-authored-by: Sirej Dua <Sirej Dua>

Sirej Dua committed Jul 2, 2024 at 07:20 UTC 15aba081f33e6d048422df6dcdb94301d08d13e6
3 files changed +35 -25
tests/spec_decode/e2e/test_integration_dist_tp2.py
+24 -12
@@ -70,10 +70,6 @@ def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
70 @pytest.mark.parametrize(
71 "common_llm_kwargs",
72 [{
73 - # Use a small model for a fast test.
74 - # Note this is repeated in the test body; to initialize a tokenizer.
75 - "model": "JackFram/llama-68m",
76 -
73 # Skip cuda graph recording for fast test.
74 "enforce_eager": True,
75
@@ -88,15 +84,31 @@ def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
84 # second run of the test to fail with internal NCCL error.
85 "use_async": True,
86 }])
91 -@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
87 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
93 -@pytest.mark.parametrize("test_llm_kwargs", [
94 - {
95 - "speculative_model": "JackFram/llama-68m",
96 - "num_speculative_tokens": 5,
97 - "speculative_draft_tensor_parallel_size": 1,
98 - },
99 -])
88 +@pytest.mark.parametrize(
89 + "per_test_common_llm_kwargs, test_llm_kwargs",
90 + [
91 + (
92 + {
93 + # Use a small model for a fast test.
94 + # Note this is repeated in the test body; to initialize a
95 + # tokenizer.
96 + "model": "JackFram/llama-68m",
97 + },
98 + {
99 + "speculative_model": "JackFram/llama-68m",
100 + "num_speculative_tokens": 5,
101 + "speculative_draft_tensor_parallel_size": 1,
102 + }),
103 + ({
104 + "model": "ibm-granite/granite-3b-code-instruct",
105 + }, {
106 + "speculative_model":
107 + "ibm-granite/granite-3b-code-instruct-accelerator",
108 + "num_speculative_tokens": 5,
109 + "speculative_draft_tensor_parallel_size": 1,
110 + })
111 + ])
112 @pytest.mark.parametrize("batch_size", [2])
113 @pytest.mark.parametrize("seed", [1])
114 def test_draft_model_tp_lt_target_model_tp2(test_llm_generator,
vllm/config.py
-6
@@ -957,12 +957,6 @@ class SpeculativeConfig:
957 )
958
959 draft_hf_config = draft_model_config.hf_config
960 - if (draft_hf_config.model_type == "mlp_speculator"
961 - and target_parallel_config.world_size != 1):
962 - # MLPSpeculator TP support will be added very soon
963 - raise ValueError(
964 - "Speculative decoding with mlp_speculator models does not "
965 - "yet support distributed inferencing (TP > 1).")
960
961 if (num_speculative_tokens is not None
962 and hasattr(draft_hf_config, "num_lookahead_tokens")):
vllm/spec_decode/spec_decode_worker.py
+11 -7
@@ -113,24 +113,28 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
113 draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
114
115 disable_bonus_tokens = True
116 +
117 if ngram_prompt_lookup_max > 0:
118 disable_bonus_tokens = False
119 proposer_worker = NGramWorker(**draft_worker_kwargs)
120 proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
121 ngram_prompt_lookup_max)
121 - elif draft_worker_kwargs[
122 - "model_config"].hf_config.model_type == "mlp_speculator":
123 - proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
124 - disable_bonus_tokens = False
122 else:
123 draft_parallel_config: ParallelConfig = draft_worker_kwargs[
124 'parallel_config']
125 draft_tp = draft_parallel_config.tensor_parallel_size
126 target_tp = scorer_worker.parallel_config.tensor_parallel_size
127
131 - if draft_tp == 1:
132 - draft_worker_kwargs["model_runner_cls"] = TP1DraftModelRunner
133 - proposer_worker = MultiStepWorker(**draft_worker_kwargs)
128 + if draft_worker_kwargs[
129 + "model_config"].hf_config.model_type == "mlp_speculator":
130 + disable_bonus_tokens = False
131 + proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
132 + else:
133 + if draft_tp == 1:
134 + draft_worker_kwargs[
135 + "model_runner_cls"] = TP1DraftModelRunner
136 + proposer_worker = MultiStepWorker(**draft_worker_kwargs)
137 +
138 proposer_worker = SmallerTpProposerWorker.maybe_wrap_worker(
139 proposer_worker, draft_tp, target_tp)
140