@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2416b26e11
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Speculative Decoding] Medusa Implementation with Top-1 proposer (#4978)

Abhinav Goyal committed Jul 10, 2024 at 07:04 UTC 2416b26e119b9d1932ba30790ecaddfac1ae4143
9 files changed +587 -4
tests/spec_decode/e2e/test_medusa_correctness.py new
+226
@@ -0,0 +1,226 @@
1 +"""This docstring details important information on the testing methodology.
2 +
3 +Most of the tests rely on "greedy equality", where we expect the output of
4 +speculative decoding on a sequence to exactly match the output of normal non-
5 +speculative decoding.
6 +
7 +Since speculative decoding with rejection sampling guarantees that the output
8 +distribution matches the target model's output distribution (up to hardware
9 +numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
10 +equality.
11 +
12 +However, we still need to verify below scenario could be passed:
13 + * Batch size 1 greedy equality
14 + * Batch size >1 greedy equality
15 + * Test greedy equality under preemption
16 + * Test greedy equality under various number of speculative tokens.
17 +
18 +With those tests, we can say at least, Medusa would not break the
19 +correctess for the target model outputs.
20 +"""
21 +
22 +import pytest
23 +
24 +from .conftest import run_greedy_equality_correctness_test
25 +
26 +# main model
27 +# lmsys/vicuna-7b-v1.3 was to be used but it's causing
28 +# OOM in CI pipeline, so using a smaller model.
29 +MAIN_MODEL = "JackFram/llama-68m"
30 +
31 +# speculative model
32 +SPEC_MODEL = "abhigoyal/vllm-medusa-llama-68m-random"
33 +
34 +# max. number of speculative tokens: this corresponds to
35 +# num_heads in the config.json of the speculator model.
36 +MAX_SPEC_TOKENS = 5
37 +
38 +# precision
39 +PRECISION = "float32"
40 +
41 +
42 +@pytest.mark.parametrize(
43 + "common_llm_kwargs",
44 + [{
45 + # Skip cuda graph recording for fast test.
46 + "enforce_eager": True,
47 +
48 + # Required for spec decode.
49 + "use_v2_block_manager": True,
50 +
51 + # Print spec metrics.
52 + "disable_log_stats": False,
53 +
54 + # Precision
55 + "dtype": PRECISION,
56 +
57 + # Main model
58 + "model": MAIN_MODEL,
59 + }])
60 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
61 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
62 +@pytest.mark.parametrize("test_llm_kwargs", [
63 + {
64 + "speculative_model": SPEC_MODEL,
65 + "num_speculative_tokens": MAX_SPEC_TOKENS,
66 + },
67 +])
68 +@pytest.mark.parametrize("output_len", [
69 + 128,
70 +])
71 +@pytest.mark.parametrize("batch_size", [1, 32])
72 +@pytest.mark.parametrize("seed", [1])
73 +def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
74 + batch_size: int, output_len: int):
75 + """Verify greedy equality with different batch size."""
76 + run_greedy_equality_correctness_test(baseline_llm_generator,
77 + test_llm_generator,
78 + batch_size,
79 + max_output_len=output_len,
80 + force_output_len=True)
81 +
82 +
83 +@pytest.mark.parametrize(
84 + "common_llm_kwargs",
85 + [{
86 + "block_size": 8,
87 + # 2 for small prompt, 256//8 for generated.
88 + "num_gpu_blocks_override": 2 + 256 // 8,
89 + "max_model_len": (2 + 256 // 8) * 8,
90 +
91 + # Skip cuda graph recording for fast test.
92 + "enforce_eager": True,
93 +
94 + # Required for spec decode.
95 + "use_v2_block_manager": True,
96 +
97 + # Precision
98 + "dtype": PRECISION,
99 +
100 + # Main model
101 + "model": MAIN_MODEL,
102 + }])
103 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
104 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
105 +@pytest.mark.parametrize("test_llm_kwargs", [
106 + {
107 + "speculative_model": SPEC_MODEL,
108 + "num_speculative_tokens": MAX_SPEC_TOKENS,
109 + },
110 +])
111 +@pytest.mark.parametrize(
112 + "output_len",
113 + [
114 + # Use small output len for fast test.
115 + 128,
116 + ])
117 +@pytest.mark.parametrize("batch_size", [4])
118 +@pytest.mark.parametrize("seed", [1])
119 +def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
120 + test_llm_generator,
121 + batch_size: int,
122 + output_len: int):
123 + """Verify greedy equality, even when some sequences are preempted mid-
124 + generation.
125 + """
126 + run_greedy_equality_correctness_test(baseline_llm_generator,
127 + test_llm_generator,
128 + batch_size,
129 + max_output_len=output_len,
130 + force_output_len=True)
131 +
132 +
133 +@pytest.mark.parametrize(
134 + "common_llm_kwargs",
135 + [{
136 + # Skip cuda graph recording for fast test.
137 + "enforce_eager": True,
138 +
139 + # Required for spec decode.
140 + "use_v2_block_manager": True,
141 +
142 + # Precision
143 + "dtype": PRECISION,
144 +
145 + # Main model
146 + "model": MAIN_MODEL,
147 + }])
148 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
149 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
150 +@pytest.mark.parametrize(
151 + "test_llm_kwargs",
152 + [
153 + {
154 + "speculative_model": SPEC_MODEL,
155 + "num_speculative_tokens": k,
156 + }
157 + # Try a range of num. speculative tokens
158 + for k in range(1, 1 + MAX_SPEC_TOKENS)
159 + ])
160 +@pytest.mark.parametrize("batch_size", [2])
161 +@pytest.mark.parametrize(
162 + "output_len",
163 + [
164 + # Use smaller output len for fast test.
165 + 32,
166 + ])
167 +@pytest.mark.parametrize("seed", [1])
168 +def test_mlp_different_k(baseline_llm_generator, test_llm_generator,
169 + batch_size: int, output_len: int):
170 + """Verify that mlp speculative decoding produces exact equality
171 + to without spec decode with different values of num_speculative_tokens.
172 + """
173 + run_greedy_equality_correctness_test(baseline_llm_generator,
174 + test_llm_generator,
175 + batch_size,
176 + max_output_len=output_len,
177 + force_output_len=True)
178 +
179 +
180 +@pytest.mark.parametrize(
181 + "common_llm_kwargs",
182 + [{
183 + # Skip cuda graph recording for fast test.
184 + "enforce_eager": True,
185 +
186 + # Required for spec decode.
187 + "use_v2_block_manager": True,
188 +
189 + # Precision
190 + "dtype": PRECISION,
191 +
192 + # Main model
193 + "model": MAIN_MODEL,
194 + }])
195 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
196 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
197 +@pytest.mark.parametrize("test_llm_kwargs",
198 + [{
199 + "speculative_model": SPEC_MODEL,
200 + "num_speculative_tokens": MAX_SPEC_TOKENS,
201 + "speculative_disable_by_batch_size": 4
202 + }])
203 +@pytest.mark.parametrize("batch_size", [1, 5])
204 +@pytest.mark.parametrize(
205 + "output_len",
206 + [
207 + # Use smaller output len for fast test.
208 + 32,
209 + ])
210 +@pytest.mark.parametrize("seed", [1])
211 +def test_mlp_disable_queue(baseline_llm_generator, test_llm_generator,
212 + batch_size: int, output_len: int):
213 + """Verify that mlp speculative decoding produces exact equality
214 + to without spec decode when speculation is disabled for large
215 + batch sizes.
216 + """
217 + run_greedy_equality_correctness_test(baseline_llm_generator,
218 + test_llm_generator,
219 + batch_size,
220 + max_output_len=output_len,
221 + force_output_len=True)
222 +
223 +
224 +if __name__ == "__main__":
225 + import pytest
226 + pytest.main([__file__])
vllm/model_executor/models/__init__.py
+1
@@ -64,6 +64,7 @@ _GENERATION_MODELS = {
64 "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
65 "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
66 "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
67 + "MedusaModel": ("medusa", "Medusa"),
68 "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
69 "JambaForCausalLM": ("jamba", "JambaForCausalLM")
70 }
vllm/model_executor/models/medusa.py new
+159
@@ -0,0 +1,159 @@
1 +from typing import Iterable, List, Optional, Tuple
2 +
3 +import torch
4 +import torch.nn as nn
5 +
6 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
7 +from vllm.model_executor.layers.vocab_parallel_embedding import (
8 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
9 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
10 +from vllm.model_executor.sampling_metadata import SamplingMetadata
11 +from vllm.sequence import SamplerOutput
12 +from vllm.transformers_utils.configs.medusa import MedusaConfig
13 +
14 +
15 +class ResidualBlock(nn.Module):
16 +
17 + def __init__(self, hidden_size: int, num_layers: int) -> None:
18 + super().__init__()
19 +
20 + self.layers = nn.ModuleList([
21 + nn.Linear(hidden_size, hidden_size, bias=False)
22 + for _ in range(num_layers)
23 + ])
24 + self.act = nn.SiLU()
25 +
26 + def forward(self, x: torch.Tensor) -> torch.Tensor:
27 + for layer in self.layers:
28 + x = x + self.act(layer(x))
29 + return x
30 +
31 +
32 +class Medusa(nn.Module):
33 +
34 + def __init__(self, config: MedusaConfig, **_) -> None:
35 + super().__init__()
36 + self.config = config
37 + self.blocks = nn.ModuleList([
38 + ResidualBlock(hidden_size=self.config.hidden_size,
39 + num_layers=self.config.num_hidden_layers)
40 + for _ in range(self.config.num_heads)
41 + ])
42 + self.orig_vocab_size = config.vocab_size
43 + self.truncated_vocab_size = config.truncated_vocab_size
44 + self.unpadded_vocab_size = self.truncated_vocab_size
45 +
46 + self.lm_heads = nn.ModuleList([
47 + ParallelLMHead(
48 + self.unpadded_vocab_size,
49 + config.hidden_size,
50 + org_num_embeddings=self.truncated_vocab_size,
51 + padding_size=DEFAULT_VOCAB_PADDING_SIZE,
52 + ) for _ in range(self.config.num_heads)
53 + ])
54 +
55 + logit_scale = getattr(config, "logit_scale", 1.0)
56 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
57 + self.truncated_vocab_size,
58 + logit_scale)
59 +
60 + self.token_map = None
61 +
62 + def forward(self, hidden_states: torch.Tensor) -> List[torch.Tensor]:
63 + return [block(hidden_states) for block in self.blocks]
64 +
65 + def compute_logits(
66 + self, hidden_states: List[torch.Tensor],
67 + sampling_metadata: SamplingMetadata) -> List[torch.Tensor]:
68 + logits = []
69 +
70 + for hs, lm_head in zip(hidden_states, self.lm_heads):
71 + _logits = self.logits_processor(lm_head, hs, sampling_metadata)
72 +
73 + if self.token_map is None:
74 + logits.append(_logits)
75 + else:
76 + logits.append(-torch.inf * torch.ones(
77 + size=(*_logits.shape[:-1], self.orig_vocab_size),
78 + device=_logits.device,
79 + dtype=_logits.dtype))
80 +
81 + logits[-1][..., self.token_map] = _logits
82 +
83 + return logits
84 +
85 + def sample(
86 + self,
87 + logits: List[torch.Tensor],
88 + sampling_metadata: SamplingMetadata,
89 + ) -> List[SamplerOutput]:
90 + logits = torch.stack(logits, dim=0).float()
91 + logprobs = torch.log_softmax(logits, dim=-1)
92 + token_ids = logits.argmax(-1) # support only top-1 for now
93 + probs = torch.softmax(logits, dim=-1)
94 +
95 + token_id_list = []
96 + token_prob_list = []
97 + token_logprob_list = []
98 +
99 + for idx, seq_group in enumerate(sampling_metadata.seq_groups):
100 + token_id_list.append(token_ids[:, seq_group.sample_indices])
101 + token_prob_list.append(probs[:, seq_group.sample_indices])
102 + token_logprob_list.append(logprobs[:, seq_group.sample_indices])
103 +
104 + outputs: List[Optional[SamplerOutput]] = []
105 + for idx in range(len(sampling_metadata.seq_groups)):
106 + outputs.append(
107 + SamplerOutput(
108 + outputs=None,
109 + sampled_token_probs=token_prob_list[idx].squeeze(1),
110 + logprobs=token_logprob_list[idx].squeeze(1),
111 + sampled_token_ids=token_id_list[idx].squeeze(1),
112 + ))
113 +
114 + return outputs
115 +
116 + def generate_proposals(
117 + self,
118 + previous_hidden_states: torch.Tensor,
119 + sampling_metadata: SamplingMetadata,
120 + ) -> List[SamplerOutput]:
121 + return self.sample(
122 + logits=self.compute_logits(
123 + hidden_states=self.forward(previous_hidden_states),
124 + sampling_metadata=sampling_metadata,
125 + ),
126 + sampling_metadata=sampling_metadata,
127 + )
128 +
129 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
130 + params_dict = dict(self.named_parameters())
131 +
132 + weights_map = {}
133 +
134 + for name, loaded_weight in weights:
135 + name = name.replace("medusa_heads.", "")
136 +
137 + if name == "token_map":
138 + if self.truncated_vocab_size < self.orig_vocab_size:
139 + self.token_map = nn.Parameter(loaded_weight,
140 + requires_grad=False)
141 + elif name in params_dict:
142 + weights_map[name] = loaded_weight
143 +
144 + for name, loaded_weight in weights_map.items():
145 + if "lm_head" in name and self.token_map is not None and\
146 + loaded_weight.shape[0] > self.token_map.shape[0]:
147 +
148 + loaded_weight = loaded_weight[self.token_map]
149 +
150 + param = params_dict[name]
151 + weight_loader = getattr(param, "weight_loader",
152 + default_weight_loader)
153 + weight_loader(param, loaded_weight)
154 +
155 + if self.token_map is not None:
156 + self.token_map.to(device=self.lm_heads[0].weight.device)
157 +
158 + assert (self.truncated_vocab_size
159 + == self.orig_vocab_size) or (self.token_map is not None)
vllm/spec_decode/medusa_worker.py new
+127
@@ -0,0 +1,127 @@
1 +import weakref
2 +from typing import List, Optional, Tuple
3 +
4 +import torch
5 +
6 +from vllm.model_executor import SamplingMetadata
7 +from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
8 + SequenceGroupMetadata)
9 +from vllm.spec_decode.interfaces import SpeculativeProposals
10 +from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
11 +from vllm.spec_decode.top1_proposer import Top1Proposer
12 +from vllm.worker.worker import Worker
13 +
14 +
15 +class MedusaWorker(NonLLMProposerWorkerBase, Worker):
16 + """Worker for Medusa.
17 + """
18 +
19 + def __init__(self, *args, **kwargs):
20 + super().__init__(*args, **kwargs)
21 +
22 + # Lazy initialization list.
23 + self._proposer: Top1Proposer
24 +
25 + def init_device(self):
26 + super().init_device()
27 +
28 + self._proposer = Top1Proposer(
29 + weakref.proxy(self), # type: ignore[arg-type]
30 + self.device,
31 + self.vocab_size,
32 + max_proposal_len=self.max_model_len,
33 + )
34 +
35 + def set_include_gpu_probs_tensor(self):
36 + pass
37 +
38 + @torch.inference_mode()
39 + def sampler_output(
40 + self,
41 + execute_model_req: ExecuteModelRequest,
42 + sample_len: int,
43 + ) -> Tuple[List[SamplerOutput], bool]:
44 + """Run the model forward pass to generate sample_len future tokens.
45 + Returns the list of sampler output, one per layer, along with indicator
46 + of whether torch tensor in sampler output need to be transposed in
47 + latter sampler_output_to_torch logic.
48 +
49 + For medusa worker, this indicator shall be False.
50 + """
51 + self._raise_if_unsupported(execute_model_req)
52 +
53 + seq_group_metadata_list = execute_model_req.seq_group_metadata_list
54 +
55 + seq_lens, query_lens = self._prepare_input_tensors(
56 + seq_group_metadata_list)
57 +
58 + sampling_metadata = SamplingMetadata.prepare(
59 + seq_group_metadata_list, seq_lens, query_lens, self.device,
60 + self.model_runner.pin_memory)
61 +
62 + model_outputs = self.model_runner.model.generate_proposals(
63 + previous_hidden_states=execute_model_req.previous_hidden_states.
64 + hidden_states,
65 + sampling_metadata=sampling_metadata)
66 +
67 + return model_outputs, False
68 +
69 + def _prepare_input_tensors(
70 + self,
71 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
72 + ) -> Tuple[List[int], List[int]]:
73 + if not seq_group_metadata_list:
74 + return [], []
75 +
76 + seq_lens: List[int] = []
77 + query_lens: List[int] = []
78 +
79 + for seq_group_metadata in seq_group_metadata_list:
80 + is_prompt = seq_group_metadata.is_prompt
81 +
82 + for seq_data in seq_group_metadata.seq_data.values():
83 + seq_data_len = seq_data.get_len()
84 + if is_prompt:
85 + context_len = seq_data.get_num_computed_tokens()
86 + seq_len = min(
87 + seq_data_len,
88 + context_len + seq_group_metadata.token_chunk_size)
89 + seq_lens.append(seq_len)
90 + query_lens.append(seq_len - context_len)
91 + else:
92 + seq_lens.append(seq_data_len)
93 + query_lens.append(1)
94 +
95 + return seq_lens, query_lens
96 +
97 + def get_spec_proposals(
98 + self,
99 + execute_model_req: ExecuteModelRequest,
100 + ) -> SpeculativeProposals:
101 + """Produce speculations given an input batch of sequences. The number of
102 + speculative tokens per sequence is determined by max_proposal_len.
103 + """
104 +
105 + return self._proposer.get_spec_proposals(execute_model_req)
106 +
107 + def _raise_if_unsupported(
108 + self,
109 + execute_model_req: ExecuteModelRequest,
110 + ) -> None:
111 + """MedusaWorker does not yet implement support for cache swap
112 + operations or beam search.
113 + """
114 + if any([
115 + execute_model_req.blocks_to_swap_in,
116 + execute_model_req.blocks_to_swap_out,
117 + execute_model_req.blocks_to_copy
118 + ]):
119 + raise NotImplementedError(
120 + "MedusaWorker does not support cache operations")
121 +
122 + if any(
123 + len(seq_group_metadata.seq_data.keys()) != 1
124 + for seq_group_metadata in
125 + execute_model_req.seq_group_metadata_list):
126 + raise NotImplementedError(
127 + "MedusaWorker does not support beam search.")
vllm/spec_decode/spec_decode_worker.py
+5
@@ -18,6 +18,7 @@ from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
18 from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
19 from vllm.spec_decode.interfaces import (SpeculativeProposals,
20 SpeculativeScorer, SpeculativeScores)
21 +from vllm.spec_decode.medusa_worker import MedusaWorker
22 from vllm.spec_decode.metrics import AsyncMetricsCollector
23 from vllm.spec_decode.mlp_speculator_worker import MLPSpeculatorWorker
24 from vllm.spec_decode.multi_step_worker import MultiStepWorker
@@ -129,6 +130,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
130 "model_config"].hf_config.model_type == "mlp_speculator":
131 disable_bonus_tokens = False
132 proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
133 + elif draft_worker_kwargs[
134 + "model_config"].hf_config.model_type == "medusa":
135 + disable_bonus_tokens = False
136 + proposer_worker = MedusaWorker(**draft_worker_kwargs)
137 else:
138 if draft_tp == 1:
139 draft_worker_kwargs[
vllm/transformers_utils/config.py
+4 -2
@@ -6,8 +6,9 @@ from transformers import GenerationConfig, PretrainedConfig
6 from vllm.envs import VLLM_USE_MODELSCOPE
7 from vllm.logger import init_logger
8 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
9 - JAISConfig, MLPSpeculatorConfig,
10 - MPTConfig, RWConfig)
9 + JAISConfig, MedusaConfig,
10 + MLPSpeculatorConfig, MPTConfig,
11 + RWConfig)
12
13 if VLLM_USE_MODELSCOPE:
14 from modelscope import AutoConfig
@@ -24,6 +25,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
25 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
26 "jais": JAISConfig,
27 "mlp_speculator": MLPSpeculatorConfig,
28 + "medusa": MedusaConfig,
29 }
30
31 for name, cls in _CONFIG_REGISTRY.items():
vllm/transformers_utils/configs/__init__.py
+2
@@ -5,6 +5,7 @@ from vllm.transformers_utils.configs.dbrx import DbrxConfig
5 # `FalconConfig` class from the official HuggingFace transformers library.
6 from vllm.transformers_utils.configs.falcon import RWConfig
7 from vllm.transformers_utils.configs.jais import JAISConfig
8 +from vllm.transformers_utils.configs.medusa import MedusaConfig
9 from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
10 from vllm.transformers_utils.configs.mpt import MPTConfig
11
@@ -14,5 +15,6 @@ __all__ = [
15 "MPTConfig",
16 "RWConfig",
17 "JAISConfig",
18 + "MedusaConfig",
19 "MLPSpeculatorConfig",
20 ]
vllm/transformers_utils/configs/medusa.py new
+60
@@ -0,0 +1,60 @@
1 +import os
2 +from typing import Optional, Union
3 +
4 +from transformers import PretrainedConfig
5 +
6 +
7 +class MedusaConfig(PretrainedConfig):
8 + model_type = "medusa"
9 +
10 + def __init__(self,
11 + hidden_size: int = 4096,
12 + vocab_size: int = 32001,
13 + num_heads: int = 5,
14 + num_hidden_layers: int = 1,
15 + max_paths: int = 64,
16 + topk: int = 10,
17 + truncated_vocab_size: Optional[int] = None,
18 + **kwargs):
19 +
20 + self.hidden_size = hidden_size
21 + self.vocab_size = vocab_size
22 + self.num_heads = num_heads
23 + self.num_hidden_layers = num_hidden_layers
24 + self.max_paths = max_paths
25 + self.topk = topk
26 + self.max_seq_len = int(2**20)
27 + self.truncated_vocab_size = vocab_size if truncated_vocab_size is None\
28 + else truncated_vocab_size
29 + if "architectures" not in kwargs:
30 + kwargs["architectures"] = ["MedusaModel"]
31 +
32 + super().__init__(**kwargs)
33 +
34 + @classmethod
35 + def from_pretrained(
36 + cls,
37 + pretrained_model_name_or_path: Union[str, os.PathLike],
38 + **kwargs,
39 + ) -> "MedusaConfig":
40 + config_dict, kwargs = cls.get_config_dict(
41 + pretrained_model_name_or_path, **kwargs)
42 + for k in list(config_dict.keys()):
43 + if 'num' in k:
44 + if 'heads' in k:
45 + config_dict["num_heads"] = config_dict.pop(k)
46 + elif 'layers' in k:
47 + config_dict["num_hidden_layers"] = config_dict.pop(k)
48 + return cls.from_dict(config_dict, **kwargs)
49 +
50 + @property
51 + def num_attention_heads(self):
52 + return 0
53 +
54 + @property
55 + def num_lookahead_tokens(self):
56 + return self.num_heads
57 +
58 + @num_lookahead_tokens.setter
59 + def num_lookahead_tokens(self, num_lookahead_tokens: int):
60 + self.num_heads = num_lookahead_tokens
vllm/worker/worker.py
+3 -2
@@ -78,8 +78,9 @@ class Worker(LocalOrDistributedWorkerBase):
78 speculative_args = {} if speculative_config is None \
79 or (speculative_config.draft_model_config.model ==
80 model_config.model) \
81 - or (speculative_config.draft_model_config.hf_config.model_type !=
82 - "mlp_speculator") else {"return_hidden_states": True}
81 + or (speculative_config.draft_model_config.hf_config.model_type
82 + not in ["medusa", "mlp_speculator"]) \
83 + else {"return_hidden_states": True}
84
85 ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
86 if model_runner_cls is not None: