A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Speculative Decoding] Medusa Implementation with Top-1 proposer (#4978)
Abhinav Goyal committed
Jul 10, 2024 at 07:04 UTC
2416b26e119b9d1932ba30790ecaddfac1ae4143
9 files changed
+587
-4
tests/spec_decode/e2e/test_medusa_correctness.py
new
+226
@@ -0,0 +1,226 @@
1
+"""This docstring details important information on the testing methodology.
2
+
3
+Most of the tests rely on "greedy equality", where we expect the output of
4
+speculative decoding on a sequence to exactly match the output of normal non-
5
+speculative decoding.
6
+
7
+Since speculative decoding with rejection sampling guarantees that the output
8
+distribution matches the target model's output distribution (up to hardware
9
+numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
10
+equality.
11
+
12
+However, we still need to verify below scenario could be passed:
13
+ * Batch size 1 greedy equality
14
+ * Batch size >1 greedy equality
15
+ * Test greedy equality under preemption
16
+ * Test greedy equality under various number of speculative tokens.
17
+
18
+With those tests, we can say at least, Medusa would not break the
19
+correctess for the target model outputs.
20
+"""
21
+
22
+import pytest
23
+
24
+from .conftest import run_greedy_equality_correctness_test
25
+
26
+# main model
27
+# lmsys/vicuna-7b-v1.3 was to be used but it's causing
28
+# OOM in CI pipeline, so using a smaller model.
29
+MAIN_MODEL = "JackFram/llama-68m"
30
+
31
+# speculative model
32
+SPEC_MODEL = "abhigoyal/vllm-medusa-llama-68m-random"
33
+
34
+# max. number of speculative tokens: this corresponds to
35
+# num_heads in the config.json of the speculator model.
36
+MAX_SPEC_TOKENS = 5
37
+
38
+# precision
39
+PRECISION = "float32"
40
+
41
+
42
+@pytest.mark.parametrize(
43
+ "common_llm_kwargs",
44
+ [{
45
+ # Skip cuda graph recording for fast test.
46
+ "enforce_eager": True,
47
+
48
+ # Required for spec decode.
49
+ "use_v2_block_manager": True,
50
+
51
+ # Print spec metrics.
52
+ "disable_log_stats": False,
53
+
54
+ # Precision
55
+ "dtype": PRECISION,
56
+
57
+ # Main model
58
+ "model": MAIN_MODEL,
59
+ }])
60
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
61
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
62
+@pytest.mark.parametrize("test_llm_kwargs", [
63
+ {
64
+ "speculative_model": SPEC_MODEL,
65
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
66
+ },
67
+])
68
+@pytest.mark.parametrize("output_len", [
69
+ 128,
70
+])
71
+@pytest.mark.parametrize("batch_size", [1, 32])
72
+@pytest.mark.parametrize("seed", [1])
73
+def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
74
+ batch_size: int, output_len: int):
75
+ """Verify greedy equality with different batch size."""
76
+ run_greedy_equality_correctness_test(baseline_llm_generator,
77
+ test_llm_generator,
78
+ batch_size,
79
+ max_output_len=output_len,
80
+ force_output_len=True)
81
+
82
+
83
+@pytest.mark.parametrize(
84
+ "common_llm_kwargs",
85
+ [{
86
+ "block_size": 8,
87
+ # 2 for small prompt, 256//8 for generated.
88
+ "num_gpu_blocks_override": 2 + 256 // 8,
89
+ "max_model_len": (2 + 256 // 8) * 8,
90
+
91
+ # Skip cuda graph recording for fast test.
92
+ "enforce_eager": True,
93
+
94
+ # Required for spec decode.
95
+ "use_v2_block_manager": True,
96
+
97
+ # Precision
98
+ "dtype": PRECISION,
99
+
100
+ # Main model
101
+ "model": MAIN_MODEL,
102
+ }])
103
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
104
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
105
+@pytest.mark.parametrize("test_llm_kwargs", [
106
+ {
107
+ "speculative_model": SPEC_MODEL,
108
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
109
+ },
110
+])
111
+@pytest.mark.parametrize(
112
+ "output_len",
113
+ [
114
+ # Use small output len for fast test.
115
+ 128,
116
+ ])
117
+@pytest.mark.parametrize("batch_size", [4])
118
+@pytest.mark.parametrize("seed", [1])
119
+def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
120
+ test_llm_generator,
121
+ batch_size: int,
122
+ output_len: int):
123
+ """Verify greedy equality, even when some sequences are preempted mid-
124
+ generation.
125
+ """
126
+ run_greedy_equality_correctness_test(baseline_llm_generator,
127
+ test_llm_generator,
128
+ batch_size,
129
+ max_output_len=output_len,
130
+ force_output_len=True)
131
+
132
+
133
+@pytest.mark.parametrize(
134
+ "common_llm_kwargs",
135
+ [{
136
+ # Skip cuda graph recording for fast test.
137
+ "enforce_eager": True,
138
+
139
+ # Required for spec decode.
140
+ "use_v2_block_manager": True,
141
+
142
+ # Precision
143
+ "dtype": PRECISION,
144
+
145
+ # Main model
146
+ "model": MAIN_MODEL,
147
+ }])
148
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
149
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
150
+@pytest.mark.parametrize(
151
+ "test_llm_kwargs",
152
+ [
153
+ {
154
+ "speculative_model": SPEC_MODEL,
155
+ "num_speculative_tokens": k,
156
+ }
157
+ # Try a range of num. speculative tokens
158
+ for k in range(1, 1 + MAX_SPEC_TOKENS)
159
+ ])
160
+@pytest.mark.parametrize("batch_size", [2])
161
+@pytest.mark.parametrize(
162
+ "output_len",
163
+ [
164
+ # Use smaller output len for fast test.
165
+ 32,
166
+ ])
167
+@pytest.mark.parametrize("seed", [1])
168
+def test_mlp_different_k(baseline_llm_generator, test_llm_generator,
169
+ batch_size: int, output_len: int):
170
+ """Verify that mlp speculative decoding produces exact equality
171
+ to without spec decode with different values of num_speculative_tokens.
172
+ """
173
+ run_greedy_equality_correctness_test(baseline_llm_generator,
174
+ test_llm_generator,
175
+ batch_size,
176
+ max_output_len=output_len,
177
+ force_output_len=True)
178
+
179
+
180
+@pytest.mark.parametrize(
181
+ "common_llm_kwargs",
182
+ [{
183
+ # Skip cuda graph recording for fast test.
184
+ "enforce_eager": True,
185
+
186
+ # Required for spec decode.
187
+ "use_v2_block_manager": True,
188
+
189
+ # Precision
190
+ "dtype": PRECISION,
191
+
192
+ # Main model
193
+ "model": MAIN_MODEL,
194
+ }])
195
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
196
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
197
+@pytest.mark.parametrize("test_llm_kwargs",
198
+ [{
199
+ "speculative_model": SPEC_MODEL,
200
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
201
+ "speculative_disable_by_batch_size": 4
202
+ }])
203
+@pytest.mark.parametrize("batch_size", [1, 5])
204
+@pytest.mark.parametrize(
205
+ "output_len",
206
+ [
207
+ # Use smaller output len for fast test.
208
+ 32,
209
+ ])
210
+@pytest.mark.parametrize("seed", [1])
211
+def test_mlp_disable_queue(baseline_llm_generator, test_llm_generator,
212
+ batch_size: int, output_len: int):
213
+ """Verify that mlp speculative decoding produces exact equality
214
+ to without spec decode when speculation is disabled for large
215
+ batch sizes.
216
+ """
217
+ run_greedy_equality_correctness_test(baseline_llm_generator,
218
+ test_llm_generator,
219
+ batch_size,
220
+ max_output_len=output_len,
221
+ force_output_len=True)
222
+
223
+
224
+if __name__ == "__main__":
225
+ import pytest
226
+ pytest.main([__file__])
vllm/model_executor/models/__init__.py
+1
@@ -64,6 +64,7 @@ _GENERATION_MODELS = {
64
"ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
65
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
66
"Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
67
+ "MedusaModel": ("medusa", "Medusa"),
68
"MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
69
"JambaForCausalLM": ("jamba", "JambaForCausalLM")
70
}
vllm/model_executor/models/medusa.py
new
+159
@@ -0,0 +1,159 @@
1
+from typing import Iterable, List, Optional, Tuple
2
+
3
+import torch
4
+import torch.nn as nn
5
+
6
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
7
+from vllm.model_executor.layers.vocab_parallel_embedding import (
8
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
9
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
10
+from vllm.model_executor.sampling_metadata import SamplingMetadata
11
+from vllm.sequence import SamplerOutput
12
+from vllm.transformers_utils.configs.medusa import MedusaConfig
13
+
14
+
15
+class ResidualBlock(nn.Module):
16
+
17
+ def __init__(self, hidden_size: int, num_layers: int) -> None:
18
+ super().__init__()
19
+
20
+ self.layers = nn.ModuleList([
21
+ nn.Linear(hidden_size, hidden_size, bias=False)
22
+ for _ in range(num_layers)
23
+ ])
24
+ self.act = nn.SiLU()
25
+
26
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
27
+ for layer in self.layers:
28
+ x = x + self.act(layer(x))
29
+ return x
30
+
31
+
32
+class Medusa(nn.Module):
33
+
34
+ def __init__(self, config: MedusaConfig, **_) -> None:
35
+ super().__init__()
36
+ self.config = config
37
+ self.blocks = nn.ModuleList([
38
+ ResidualBlock(hidden_size=self.config.hidden_size,
39
+ num_layers=self.config.num_hidden_layers)
40
+ for _ in range(self.config.num_heads)
41
+ ])
42
+ self.orig_vocab_size = config.vocab_size
43
+ self.truncated_vocab_size = config.truncated_vocab_size
44
+ self.unpadded_vocab_size = self.truncated_vocab_size
45
+
46
+ self.lm_heads = nn.ModuleList([
47
+ ParallelLMHead(
48
+ self.unpadded_vocab_size,
49
+ config.hidden_size,
50
+ org_num_embeddings=self.truncated_vocab_size,
51
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE,
52
+ ) for _ in range(self.config.num_heads)
53
+ ])
54
+
55
+ logit_scale = getattr(config, "logit_scale", 1.0)
56
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
57
+ self.truncated_vocab_size,
58
+ logit_scale)
59
+
60
+ self.token_map = None
61
+
62
+ def forward(self, hidden_states: torch.Tensor) -> List[torch.Tensor]:
63
+ return [block(hidden_states) for block in self.blocks]
64
+
65
+ def compute_logits(
66
+ self, hidden_states: List[torch.Tensor],
67
+ sampling_metadata: SamplingMetadata) -> List[torch.Tensor]:
68
+ logits = []
69
+
70
+ for hs, lm_head in zip(hidden_states, self.lm_heads):
71
+ _logits = self.logits_processor(lm_head, hs, sampling_metadata)
72
+
73
+ if self.token_map is None:
74
+ logits.append(_logits)
75
+ else:
76
+ logits.append(-torch.inf * torch.ones(
77
+ size=(*_logits.shape[:-1], self.orig_vocab_size),
78
+ device=_logits.device,
79
+ dtype=_logits.dtype))
80
+
81
+ logits[-1][..., self.token_map] = _logits
82
+
83
+ return logits
84
+
85
+ def sample(
86
+ self,
87
+ logits: List[torch.Tensor],
88
+ sampling_metadata: SamplingMetadata,
89
+ ) -> List[SamplerOutput]:
90
+ logits = torch.stack(logits, dim=0).float()
91
+ logprobs = torch.log_softmax(logits, dim=-1)
92
+ token_ids = logits.argmax(-1) # support only top-1 for now
93
+ probs = torch.softmax(logits, dim=-1)
94
+
95
+ token_id_list = []
96
+ token_prob_list = []
97
+ token_logprob_list = []
98
+
99
+ for idx, seq_group in enumerate(sampling_metadata.seq_groups):
100
+ token_id_list.append(token_ids[:, seq_group.sample_indices])
101
+ token_prob_list.append(probs[:, seq_group.sample_indices])
102
+ token_logprob_list.append(logprobs[:, seq_group.sample_indices])
103
+
104
+ outputs: List[Optional[SamplerOutput]] = []
105
+ for idx in range(len(sampling_metadata.seq_groups)):
106
+ outputs.append(
107
+ SamplerOutput(
108
+ outputs=None,
109
+ sampled_token_probs=token_prob_list[idx].squeeze(1),
110
+ logprobs=token_logprob_list[idx].squeeze(1),
111
+ sampled_token_ids=token_id_list[idx].squeeze(1),
112
+ ))
113
+
114
+ return outputs
115
+
116
+ def generate_proposals(
117
+ self,
118
+ previous_hidden_states: torch.Tensor,
119
+ sampling_metadata: SamplingMetadata,
120
+ ) -> List[SamplerOutput]:
121
+ return self.sample(
122
+ logits=self.compute_logits(
123
+ hidden_states=self.forward(previous_hidden_states),
124
+ sampling_metadata=sampling_metadata,
125
+ ),
126
+ sampling_metadata=sampling_metadata,
127
+ )
128
+
129
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
130
+ params_dict = dict(self.named_parameters())
131
+
132
+ weights_map = {}
133
+
134
+ for name, loaded_weight in weights:
135
+ name = name.replace("medusa_heads.", "")
136
+
137
+ if name == "token_map":
138
+ if self.truncated_vocab_size < self.orig_vocab_size:
139
+ self.token_map = nn.Parameter(loaded_weight,
140
+ requires_grad=False)
141
+ elif name in params_dict:
142
+ weights_map[name] = loaded_weight
143
+
144
+ for name, loaded_weight in weights_map.items():
145
+ if "lm_head" in name and self.token_map is not None and\
146
+ loaded_weight.shape[0] > self.token_map.shape[0]:
147
+
148
+ loaded_weight = loaded_weight[self.token_map]
149
+
150
+ param = params_dict[name]
151
+ weight_loader = getattr(param, "weight_loader",
152
+ default_weight_loader)
153
+ weight_loader(param, loaded_weight)
154
+
155
+ if self.token_map is not None:
156
+ self.token_map.to(device=self.lm_heads[0].weight.device)
157
+
158
+ assert (self.truncated_vocab_size
159
+ == self.orig_vocab_size) or (self.token_map is not None)
vllm/spec_decode/medusa_worker.py
new
+127
@@ -0,0 +1,127 @@
1
+import weakref
2
+from typing import List, Optional, Tuple
3
+
4
+import torch
5
+
6
+from vllm.model_executor import SamplingMetadata
7
+from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
8
+ SequenceGroupMetadata)
9
+from vllm.spec_decode.interfaces import SpeculativeProposals
10
+from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
11
+from vllm.spec_decode.top1_proposer import Top1Proposer
12
+from vllm.worker.worker import Worker
13
+
14
+
15
+class MedusaWorker(NonLLMProposerWorkerBase, Worker):
16
+ """Worker for Medusa.
17
+ """
18
+
19
+ def __init__(self, *args, **kwargs):
20
+ super().__init__(*args, **kwargs)
21
+
22
+ # Lazy initialization list.
23
+ self._proposer: Top1Proposer
24
+
25
+ def init_device(self):
26
+ super().init_device()
27
+
28
+ self._proposer = Top1Proposer(
29
+ weakref.proxy(self), # type: ignore[arg-type]
30
+ self.device,
31
+ self.vocab_size,
32
+ max_proposal_len=self.max_model_len,
33
+ )
34
+
35
+ def set_include_gpu_probs_tensor(self):
36
+ pass
37
+
38
+ @torch.inference_mode()
39
+ def sampler_output(
40
+ self,
41
+ execute_model_req: ExecuteModelRequest,
42
+ sample_len: int,
43
+ ) -> Tuple[List[SamplerOutput], bool]:
44
+ """Run the model forward pass to generate sample_len future tokens.
45
+ Returns the list of sampler output, one per layer, along with indicator
46
+ of whether torch tensor in sampler output need to be transposed in
47
+ latter sampler_output_to_torch logic.
48
+
49
+ For medusa worker, this indicator shall be False.
50
+ """
51
+ self._raise_if_unsupported(execute_model_req)
52
+
53
+ seq_group_metadata_list = execute_model_req.seq_group_metadata_list
54
+
55
+ seq_lens, query_lens = self._prepare_input_tensors(
56
+ seq_group_metadata_list)
57
+
58
+ sampling_metadata = SamplingMetadata.prepare(
59
+ seq_group_metadata_list, seq_lens, query_lens, self.device,
60
+ self.model_runner.pin_memory)
61
+
62
+ model_outputs = self.model_runner.model.generate_proposals(
63
+ previous_hidden_states=execute_model_req.previous_hidden_states.
64
+ hidden_states,
65
+ sampling_metadata=sampling_metadata)
66
+
67
+ return model_outputs, False
68
+
69
+ def _prepare_input_tensors(
70
+ self,
71
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
72
+ ) -> Tuple[List[int], List[int]]:
73
+ if not seq_group_metadata_list:
74
+ return [], []
75
+
76
+ seq_lens: List[int] = []
77
+ query_lens: List[int] = []
78
+
79
+ for seq_group_metadata in seq_group_metadata_list:
80
+ is_prompt = seq_group_metadata.is_prompt
81
+
82
+ for seq_data in seq_group_metadata.seq_data.values():
83
+ seq_data_len = seq_data.get_len()
84
+ if is_prompt:
85
+ context_len = seq_data.get_num_computed_tokens()
86
+ seq_len = min(
87
+ seq_data_len,
88
+ context_len + seq_group_metadata.token_chunk_size)
89
+ seq_lens.append(seq_len)
90
+ query_lens.append(seq_len - context_len)
91
+ else:
92
+ seq_lens.append(seq_data_len)
93
+ query_lens.append(1)
94
+
95
+ return seq_lens, query_lens
96
+
97
+ def get_spec_proposals(
98
+ self,
99
+ execute_model_req: ExecuteModelRequest,
100
+ ) -> SpeculativeProposals:
101
+ """Produce speculations given an input batch of sequences. The number of
102
+ speculative tokens per sequence is determined by max_proposal_len.
103
+ """
104
+
105
+ return self._proposer.get_spec_proposals(execute_model_req)
106
+
107
+ def _raise_if_unsupported(
108
+ self,
109
+ execute_model_req: ExecuteModelRequest,
110
+ ) -> None:
111
+ """MedusaWorker does not yet implement support for cache swap
112
+ operations or beam search.
113
+ """
114
+ if any([
115
+ execute_model_req.blocks_to_swap_in,
116
+ execute_model_req.blocks_to_swap_out,
117
+ execute_model_req.blocks_to_copy
118
+ ]):
119
+ raise NotImplementedError(
120
+ "MedusaWorker does not support cache operations")
121
+
122
+ if any(
123
+ len(seq_group_metadata.seq_data.keys()) != 1
124
+ for seq_group_metadata in
125
+ execute_model_req.seq_group_metadata_list):
126
+ raise NotImplementedError(
127
+ "MedusaWorker does not support beam search.")
vllm/spec_decode/spec_decode_worker.py
+5
@@ -18,6 +18,7 @@ from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
18
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
19
from vllm.spec_decode.interfaces import (SpeculativeProposals,
20
SpeculativeScorer, SpeculativeScores)
21
+from vllm.spec_decode.medusa_worker import MedusaWorker
22
from vllm.spec_decode.metrics import AsyncMetricsCollector
23
from vllm.spec_decode.mlp_speculator_worker import MLPSpeculatorWorker
24
from vllm.spec_decode.multi_step_worker import MultiStepWorker
@@ -129,6 +130,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
130
"model_config"].hf_config.model_type == "mlp_speculator":
131
disable_bonus_tokens = False
132
proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
133
+ elif draft_worker_kwargs[
134
+ "model_config"].hf_config.model_type == "medusa":
135
+ disable_bonus_tokens = False
136
+ proposer_worker = MedusaWorker(**draft_worker_kwargs)
137
else:
138
if draft_tp == 1:
139
draft_worker_kwargs[
vllm/transformers_utils/config.py
+4
-2
@@ -6,8 +6,9 @@ from transformers import GenerationConfig, PretrainedConfig
6
from vllm.envs import VLLM_USE_MODELSCOPE
7
from vllm.logger import init_logger
8
from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9
- JAISConfig, MLPSpeculatorConfig,
10
- MPTConfig, RWConfig)
9
+ JAISConfig, MedusaConfig,
10
+ MLPSpeculatorConfig, MPTConfig,
11
+ RWConfig)
12
13
if VLLM_USE_MODELSCOPE:
14
from modelscope import AutoConfig
@@ -24,6 +25,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
25
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
26
"jais": JAISConfig,
27
"mlp_speculator": MLPSpeculatorConfig,
28
+ "medusa": MedusaConfig,
29
}
30
31
for name, cls in _CONFIG_REGISTRY.items():
vllm/transformers_utils/configs/__init__.py
+2
@@ -5,6 +5,7 @@ from vllm.transformers_utils.configs.dbrx import DbrxConfig
5
# `FalconConfig` class from the official HuggingFace transformers library.
6
from vllm.transformers_utils.configs.falcon import RWConfig
7
from vllm.transformers_utils.configs.jais import JAISConfig
8
+from vllm.transformers_utils.configs.medusa import MedusaConfig
9
from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
10
from vllm.transformers_utils.configs.mpt import MPTConfig
11
@@ -14,5 +15,6 @@ __all__ = [
15
"MPTConfig",
16
"RWConfig",
17
"JAISConfig",
18
+ "MedusaConfig",
19
"MLPSpeculatorConfig",
20
]
vllm/transformers_utils/configs/medusa.py
new
+60
@@ -0,0 +1,60 @@
1
+import os
2
+from typing import Optional, Union
3
+
4
+from transformers import PretrainedConfig
5
+
6
+
7
+class MedusaConfig(PretrainedConfig):
8
+ model_type = "medusa"
9
+
10
+ def __init__(self,
11
+ hidden_size: int = 4096,
12
+ vocab_size: int = 32001,
13
+ num_heads: int = 5,
14
+ num_hidden_layers: int = 1,
15
+ max_paths: int = 64,
16
+ topk: int = 10,
17
+ truncated_vocab_size: Optional[int] = None,
18
+ **kwargs):
19
+
20
+ self.hidden_size = hidden_size
21
+ self.vocab_size = vocab_size
22
+ self.num_heads = num_heads
23
+ self.num_hidden_layers = num_hidden_layers
24
+ self.max_paths = max_paths
25
+ self.topk = topk
26
+ self.max_seq_len = int(2**20)
27
+ self.truncated_vocab_size = vocab_size if truncated_vocab_size is None\
28
+ else truncated_vocab_size
29
+ if "architectures" not in kwargs:
30
+ kwargs["architectures"] = ["MedusaModel"]
31
+
32
+ super().__init__(**kwargs)
33
+
34
+ @classmethod
35
+ def from_pretrained(
36
+ cls,
37
+ pretrained_model_name_or_path: Union[str, os.PathLike],
38
+ **kwargs,
39
+ ) -> "MedusaConfig":
40
+ config_dict, kwargs = cls.get_config_dict(
41
+ pretrained_model_name_or_path, **kwargs)
42
+ for k in list(config_dict.keys()):
43
+ if 'num' in k:
44
+ if 'heads' in k:
45
+ config_dict["num_heads"] = config_dict.pop(k)
46
+ elif 'layers' in k:
47
+ config_dict["num_hidden_layers"] = config_dict.pop(k)
48
+ return cls.from_dict(config_dict, **kwargs)
49
+
50
+ @property
51
+ def num_attention_heads(self):
52
+ return 0
53
+
54
+ @property
55
+ def num_lookahead_tokens(self):
56
+ return self.num_heads
57
+
58
+ @num_lookahead_tokens.setter
59
+ def num_lookahead_tokens(self, num_lookahead_tokens: int):
60
+ self.num_heads = num_lookahead_tokens
vllm/worker/worker.py
+3
-2
@@ -78,8 +78,9 @@ class Worker(LocalOrDistributedWorkerBase):
78
speculative_args = {} if speculative_config is None \
79
or (speculative_config.draft_model_config.model ==
80
model_config.model) \
81
- or (speculative_config.draft_model_config.hf_config.model_type !=
82
- "mlp_speculator") else {"return_hidden_states": True}
81
+ or (speculative_config.draft_model_config.hf_config.model_type
82
+ not in ["medusa", "mlp_speculator"]) \
83
+ else {"return_hidden_states": True}
84
85
ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
86
if model_runner_cls is not None: