A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Model] MLPSpeculator speculative decoding support (#4947)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: Davis Wertheimer <Davis.Wertheimer@ibm.com>
Joshua Rosenkranz committed
Jun 20, 2024 at 20:23 UTC
b12518d3cf4326dfcd10a09780913b86c19fcf1a
18 files changed
+523
-40
examples/offline_inference_mlpspeculator.py
new
+59
@@ -0,0 +1,59 @@
1
+import gc
2
+import time
3
+from typing import List
4
+
5
+from vllm import LLM, SamplingParams
6
+
7
+
8
+def time_generation(llm: LLM, prompts: List[str],
9
+ sampling_params: SamplingParams):
10
+ # Generate texts from the prompts. The output is a list of RequestOutput
11
+ # objects that contain the prompt, generated text, and other information.
12
+ # Warmup first
13
+ llm.generate(prompts, sampling_params)
14
+ llm.generate(prompts, sampling_params)
15
+ start = time.time()
16
+ outputs = llm.generate(prompts, sampling_params)
17
+ end = time.time()
18
+ print((end - start) / sum([len(o.outputs[0].token_ids) for o in outputs]))
19
+ # Print the outputs.
20
+ for output in outputs:
21
+ generated_text = output.outputs[0].text
22
+ print(f"text: {generated_text!r}")
23
+
24
+
25
+if __name__ == "__main__":
26
+
27
+ template = (
28
+ "Below is an instruction that describes a task. Write a response "
29
+ "that appropriately completes the request.\n\n### Instruction:\n{}"
30
+ "\n\n### Response:\n")
31
+
32
+ # Sample prompts.
33
+ prompts = [
34
+ "Write about the president of the United States.",
35
+ ]
36
+ prompts = [template.format(prompt) for prompt in prompts]
37
+ # Create a sampling params object.
38
+ sampling_params = SamplingParams(temperature=0.0, max_tokens=200)
39
+
40
+ # Create an LLM without spec decoding
41
+ llm = LLM(model="meta-llama/Llama-2-13b-chat-hf")
42
+
43
+ print("Without speculation")
44
+ time_generation(llm, prompts, sampling_params)
45
+
46
+ del llm
47
+ gc.collect()
48
+
49
+ # Create an LLM with spec decoding
50
+ llm = LLM(
51
+ model="meta-llama/Llama-2-13b-chat-hf",
52
+ speculative_model="ibm-fms/llama-13b-accelerator",
53
+ # These are currently required for MLPSpeculator decoding
54
+ use_v2_block_manager=True,
55
+ enforce_eager=True,
56
+ )
57
+
58
+ print("With speculation")
59
+ time_generation(llm, prompts, sampling_params)
tests/spec_decode/test_spec_decode_worker.py
+6
-2
@@ -456,7 +456,9 @@ def test_k_equals_zero(k: int, batch_size: int):
456
rejection_sampler.token_id_dtype = torch.int64
457
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
458
459
- target_worker.execute_model.return_value = [MagicMock(spec=SamplerOutput)]
459
+ sampler_output = MagicMock(spec=SamplerOutput)
460
+ sampler_output.hidden_states = None
461
+ target_worker.execute_model.return_value = [sampler_output]
462
463
draft_worker.device = 'cuda'
464
target_worker.device = 'cuda'
@@ -497,7 +499,9 @@ def test_empty_input_batch(k: int, batch_size: int):
499
rejection_sampler.token_id_dtype = torch.int64
500
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
501
500
- target_worker.execute_model.return_value = [MagicMock(spec=SamplerOutput)]
502
+ sampler_output = MagicMock(spec=SamplerOutput)
503
+ sampler_output.hidden_states = None
504
+ target_worker.execute_model.return_value = [sampler_output]
505
506
draft_worker.device = 'cuda'
507
target_worker.device = 'cuda'
tests/spec_decode/test_utils.py
+2
-2
@@ -2,8 +2,8 @@ from unittest.mock import MagicMock
2
3
import pytest
4
5
-from vllm.sequence import SequenceGroupMetadata
6
-from vllm.spec_decode.util import get_all_seq_ids, split_batch_by_proposal_len
5
+from vllm.sequence import SequenceGroupMetadata, get_all_seq_ids
6
+from vllm.spec_decode.util import split_batch_by_proposal_len
7
8
9
def test_get_all_seq_ids():
vllm/config.py
+39
-15
@@ -230,7 +230,8 @@ class ModelConfig:
230
self,
231
parallel_config: "ParallelConfig",
232
) -> None:
233
- total_num_attention_heads = self.hf_text_config.num_attention_heads
233
+ total_num_attention_heads = getattr(self.hf_text_config,
234
+ "num_attention_heads", 0)
235
tensor_parallel_size = parallel_config.tensor_parallel_size
236
if total_num_attention_heads % tensor_parallel_size != 0:
237
raise ValueError(
@@ -238,7 +239,8 @@ class ModelConfig:
239
" must be divisible by tensor parallel size "
240
f"({tensor_parallel_size}).")
241
241
- total_num_hidden_layers = self.hf_text_config.num_hidden_layers
242
+ total_num_hidden_layers = getattr(self.hf_text_config,
243
+ "num_hidden_layers", 0)
244
pipeline_parallel_size = parallel_config.pipeline_parallel_size
245
if total_num_hidden_layers % pipeline_parallel_size != 0:
246
raise ValueError(
@@ -341,8 +343,8 @@ class ModelConfig:
343
344
def get_num_attention_heads(self,
345
parallel_config: "ParallelConfig") -> int:
344
- return self.hf_text_config.num_attention_heads // \
345
- parallel_config.tensor_parallel_size
346
+ num_heads = getattr(self.hf_text_config, "num_attention_heads", 0)
347
+ return num_heads // parallel_config.tensor_parallel_size
348
349
def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
350
total_num_hidden_layers = self.hf_text_config.num_hidden_layers
@@ -818,7 +820,8 @@ class SpeculativeConfig:
820
speculative_model (Optional[str]): The name of the speculative
821
model, if provided.
822
num_speculative_tokens (Optional[int]): The number of speculative
821
- tokens, if provided.
823
+ tokens, if provided. Will default to the number in the draft
824
+ model config if present, otherwise is required.
825
speculative_max_model_len (Optional[int]): The maximum model len of
826
the speculative model. Used when testing the ability to skip
827
speculation for some sequences.
@@ -841,24 +844,18 @@ class SpeculativeConfig:
844
the necessary conditions are met, else None.
845
"""
846
844
- if speculative_model is None and num_speculative_tokens is None:
847
+ if speculative_model is None:
848
+ if num_speculative_tokens is not None:
849
+ raise ValueError("num_speculative_tokens was provided without "
850
+ "speculative_model.")
851
return None
852
847
- if speculative_model is not None and num_speculative_tokens is None:
848
- raise ValueError(
849
- "Expected both speculative_model and "
850
- "num_speculative_tokens to be provided, but found "
851
- f"{speculative_model=} and {num_speculative_tokens=}.")
852
-
853
if (speculative_disable_by_batch_size is not None
854
and speculative_disable_by_batch_size < 2):
855
raise ValueError("Expect the batch size threshold of disabling "
856
"speculative decoding is > 1, but got "
857
f"{speculative_disable_by_batch_size=}")
858
859
- assert (speculative_model is not None
860
- and num_speculative_tokens is not None)
861
-
859
if enable_chunked_prefill:
860
raise ValueError(
861
"Speculative decoding and chunked prefill are "
@@ -912,6 +909,27 @@ class SpeculativeConfig:
909
max_logprobs=target_model_config.max_logprobs,
910
)
911
912
+ if (draft_model_config.hf_config.model_type == "mlp_speculator"
913
+ and target_parallel_config.world_size != 1):
914
+ # MLPSpeculator TP support will be added very soon
915
+ raise ValueError(
916
+ "Speculative decoding with mlp_speculator models does not "
917
+ "yet support distributed inferencing (TP > 1).")
918
+
919
+ n_predict = getattr(draft_model_config.hf_config, "n_predict",
920
+ None)
921
+ if n_predict is not None:
922
+ if num_speculative_tokens is None:
923
+ # Default to max value defined in draft model config.
924
+ num_speculative_tokens = n_predict
925
+ elif num_speculative_tokens > n_predict:
926
+ # Verify provided value doesn't exceed the maximum
927
+ # supported by the draft model.
928
+ raise ValueError(
929
+ "Expected both speculative_model and "
930
+ "num_speculative_tokens to be provided, but found "
931
+ f"{speculative_model=} and {num_speculative_tokens=}.")
932
+
933
draft_model_config.max_model_len = (
934
SpeculativeConfig._maybe_override_draft_max_model_len(
935
speculative_max_model_len,
@@ -923,6 +941,12 @@ class SpeculativeConfig:
941
SpeculativeConfig.create_draft_parallel_config(
942
target_parallel_config))
943
944
+ if num_speculative_tokens is None:
945
+ raise ValueError(
946
+ "num_speculative_tokens must be provided with "
947
+ "speculative_model unless the draft model config contains an "
948
+ "n_predict parameter.")
949
+
950
return SpeculativeConfig(
951
draft_model_config,
952
draft_parallel_config,
vllm/model_executor/models/__init__.py
+1
@@ -60,6 +60,7 @@ _GENERATION_MODELS = {
60
"ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
61
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
62
"Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
63
+ "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
64
}
65
66
_EMBEDDING_MODELS = {
vllm/model_executor/models/mlp_speculator.py
new
+143
@@ -0,0 +1,143 @@
1
+import math
2
+from typing import Iterable, List, Tuple
3
+
4
+import torch
5
+import torch.nn as nn
6
+
7
+from vllm.model_executor import SamplingMetadata
8
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
9
+from vllm.model_executor.layers.sampler import Sampler
10
+from vllm.model_executor.layers.vocab_parallel_embedding import (
11
+ VocabParallelEmbedding)
12
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13
+from vllm.sequence import SamplerOutput
14
+
15
+
16
+class MLPSpeculatorLayerNorm(nn.Module):
17
+ """
18
+ A L2 normalization implementation
19
+ ...
20
+ Args
21
+ ----
22
+ normalized_shape : int
23
+ Dimensionality of input data (size of final tensor axis)
24
+ eps : float
25
+ Safety term to prevent division by zero. Make sure the chosen value
26
+ fits in the range of your encoding scheme
27
+ (i.e. fp16 requires eps >= 6e-8).
28
+ """
29
+
30
+ def __init__(
31
+ self,
32
+ normalized_shape,
33
+ eps=1e-06,
34
+ ):
35
+ super(MLPSpeculatorLayerNorm, self).__init__()
36
+ self.weight = nn.Parameter(torch.empty(normalized_shape))
37
+ self.bias = nn.Parameter(torch.empty(normalized_shape))
38
+ self.eps = eps
39
+
40
+ def forward(self, x):
41
+ xf = x
42
+ xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + self.eps)
43
+ x = xf.type_as(x)
44
+ x = self.weight * x
45
+ x = x + self.bias
46
+ return x
47
+
48
+
49
+class MLPSpeculator(nn.Module):
50
+
51
+ def __init__(self, config, **kwargs) -> None:
52
+ super().__init__()
53
+ self.n_predict = config.n_predict
54
+ self.vocab_size = config.vocab_size
55
+ self.emb_dim = config.emb_dim
56
+ self.inner_dim = config.inner_dim if config.inner_dim != 0 \
57
+ else config.emb_dim
58
+
59
+ self.max_speculative_tokens = getattr(config, "max_speculative_tokens",
60
+ self.n_predict)
61
+
62
+ self.emb = nn.ModuleList([
63
+ VocabParallelEmbedding(config.vocab_size,
64
+ self.inner_dim,
65
+ org_num_embeddings=config.vocab_size)
66
+ for _ in range(self.max_speculative_tokens)
67
+ ])
68
+
69
+ self.proj = nn.ModuleList([
70
+ nn.Linear((self.emb_dim if i == 0 else self.inner_dim),
71
+ self.inner_dim,
72
+ bias=False) for i in range(self.max_speculative_tokens)
73
+ ])
74
+
75
+ self.head = nn.ModuleList([
76
+ nn.Linear(self.inner_dim, self.vocab_size, bias=False)
77
+ for _ in range(self.max_speculative_tokens)
78
+ ])
79
+ self.ln = nn.ModuleList([
80
+ MLPSpeculatorLayerNorm(self.inner_dim)
81
+ for _ in range(self.max_speculative_tokens)
82
+ ])
83
+
84
+ self.state_weight = 0.5**(0.5 / config.n_predict)
85
+ self.emb_weight = math.sqrt(
86
+ (1 - self.state_weight**2) * (self.inner_dim / 2))
87
+ self.activation = nn.GELU()
88
+ self.config = config
89
+ self.logits_processor = LogitsProcessor(config.vocab_size,
90
+ config.vocab_size, 1.0)
91
+ self.sampler = Sampler()
92
+
93
+ def generate_proposals(
94
+ self,
95
+ input_ids: torch.Tensor,
96
+ previous_hidden_states: torch.Tensor,
97
+ num_predict_tokens: int,
98
+ sampling_metadata: SamplingMetadata,
99
+ ) -> List[SamplerOutput]:
100
+ if num_predict_tokens > self.max_speculative_tokens:
101
+ raise ValueError(f"Max speculative tokens for model is "
102
+ f"{self.max_speculative_tokens}, but "
103
+ f"{num_predict_tokens} were requested")
104
+
105
+ # b x 1 x d
106
+ previous_hidden_states = previous_hidden_states.unsqueeze(1)
107
+
108
+ # b x 1
109
+ last_tokens = input_ids.unsqueeze(1)
110
+
111
+ next_tokens = []
112
+
113
+ for head_index in range(num_predict_tokens):
114
+
115
+ # Project and predict
116
+ z = self.emb[head_index](last_tokens) # b k d
117
+ states = self.proj[head_index](previous_hidden_states)
118
+
119
+ # Weighted add of state_weight*state and emb_weight*z
120
+ # Let subsequent LN take care of denominator
121
+ # state_weight is close to 1, so shouldn't be any precision issues
122
+ states.add_(z, alpha=self.emb_weight / self.state_weight)
123
+
124
+ states = self.activation(self.ln[head_index](states)) # b k d
125
+ # TODO: not yet supporting top_k_tokens_per_head
126
+ previous_hidden_states = states
127
+
128
+ logits = self.logits_processor(self.head[head_index].weight,
129
+ states, sampling_metadata)
130
+
131
+ output = self.sampler(logits.flatten(0, 1), sampling_metadata)
132
+ last_tokens = output.sampled_token_ids
133
+ next_tokens.append(output)
134
+
135
+ return next_tokens
136
+
137
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
138
+ params_dict = dict(self.named_parameters())
139
+ for name, loaded_weight in weights:
140
+ param = params_dict[name.replace("speculator.", "")]
141
+ weight_loader = getattr(param, "weight_loader",
142
+ default_weight_loader)
143
+ weight_loader(param, loaded_weight)
vllm/sequence.py
+46
@@ -794,6 +794,9 @@ class SamplerOutput:
794
# Spec decode metrics populated by workers.
795
spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
796
797
+ # Optional last hidden states from the model.
798
+ hidden_states: Optional[torch.Tensor] = None
799
+
800
def __getitem__(self, idx: int):
801
return self.outputs[idx]
802
@@ -842,6 +845,46 @@ class PoolerOutput:
845
self.__class__) and self.outputs == other.outputs
846
847
848
+def get_all_seq_ids(
849
+ seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[int]:
850
+ """Given a list of SequenceGroupMetadata, create a list of all
851
+ sequence ids.
852
+ """
853
+ return [seq_id for sg in seq_group_metadata_list for seq_id in sg.seq_data]
854
+
855
+
856
+class HiddenStates:
857
+ """Hidden states corresponding to in-progress sequences.
858
+ Used in speculative decoding to pass hidden states from
859
+ the target model to the proposer model in the subsequent step.
860
+
861
+ seq_ids are the sequence ids of each entry of the batch
862
+ dimension of the hidden_states tensor"""
863
+
864
+ def __init__(self, seq_group_metadata_list: List[SequenceGroupMetadata],
865
+ hidden_states: torch.Tensor):
866
+ assert len(seq_group_metadata_list) == len(hidden_states)
867
+ self.seq_ids: List[int] = get_all_seq_ids(seq_group_metadata_list)
868
+ self.hidden_states: torch.Tensor = hidden_states
869
+
870
+ def update(self, seq_group_metadata_list: List[SequenceGroupMetadata],
871
+ hidden_states: torch.Tensor) -> None:
872
+ """Update hidden states from target model invocation."""
873
+ assert len(seq_group_metadata_list) == len(hidden_states)
874
+ self.seq_ids.extend(get_all_seq_ids(seq_group_metadata_list))
875
+ self.hidden_states = torch.cat([self.hidden_states, hidden_states])
876
+
877
+ def prune(self,
878
+ seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
879
+ """Prune to provided list of sequence ids."""
880
+ seq_ids = get_all_seq_ids(seq_group_metadata_list)
881
+ if seq_ids != self.seq_ids:
882
+ # Batch contents changed - prune removed sequences.
883
+ index = [self.seq_ids.index(seq_id) for seq_id in seq_ids]
884
+ self.hidden_states = self.hidden_states[index]
885
+ self.seq_ids = seq_ids
886
+
887
+
888
@dataclass
889
class ExecuteModelRequest:
890
"""The model execution request."""
@@ -857,6 +900,8 @@ class ExecuteModelRequest:
900
num_lookahead_slots: int = 0
901
# The number of requests in the running queue.
902
running_queue_size: int = 0
903
+ # Optional hidden states from prior step.
904
+ previous_hidden_states: Optional[HiddenStates] = None
905
906
def clone(
907
self, seq_group_metadata_list: List[SequenceGroupMetadata]
@@ -869,4 +914,5 @@ class ExecuteModelRequest:
914
blocks_to_copy=self.blocks_to_copy.copy(),
915
num_lookahead_slots=self.num_lookahead_slots,
916
running_queue_size=self.running_queue_size,
917
+ previous_hidden_states=self.previous_hidden_states,
918
)
vllm/spec_decode/batch_expansion.py
+3
-3
@@ -4,11 +4,10 @@ from typing import Iterator, List, Tuple
4
import torch
5
6
from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
7
- SequenceGroupMetadata)
7
+ SequenceGroupMetadata, get_all_seq_ids)
8
from vllm.spec_decode.interfaces import (SpeculativeProposals,
9
SpeculativeScorer, SpeculativeScores)
10
-from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
11
- sampler_output_to_torch,
10
+from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
11
split_batch_by_proposal_len)
12
from vllm.worker.worker_base import WorkerBase
13
@@ -98,6 +97,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
97
probs=all_probs,
98
token_ids=all_tokens,
99
logprobs=spec_logprobs,
100
+ hidden_states=target_sampler_output.hidden_states,
101
)
102
103
def _expand_batch(
vllm/spec_decode/interfaces.py
+4
@@ -1,5 +1,6 @@
1
from abc import ABC, abstractmethod
2
from dataclasses import dataclass
3
+from typing import Optional
4
5
import torch
6
@@ -46,6 +47,9 @@ class SpeculativeScores:
47
# tokens and also non-speculative normal decoding.
48
token_ids: torch.Tensor
49
50
+ # Optional last hidden states from the scoring model.
51
+ hidden_states: Optional[torch.Tensor] = None
52
+
53
def __repr__(self):
54
return (f"SpeculativeScores("
55
f"probs={self.probs.shape}, "
vllm/spec_decode/mlp_speculator_worker.py
new
+87
@@ -0,0 +1,87 @@
1
+from typing import List, Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm.model_executor import SamplingMetadata
6
+from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
7
+ SequenceGroupMetadata)
8
+from vllm.spec_decode.multi_step_worker import MultiStepWorker
9
+from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
10
+from vllm.worker.model_runner import ModelInput
11
+
12
+
13
+class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker):
14
+ """Worker for MLPSpeculator models.
15
+
16
+ Not currently compatible with LoRA or chunked prefill.
17
+ """
18
+
19
+ @torch.inference_mode()
20
+ def sampler_output(
21
+ self,
22
+ execute_model_req: ExecuteModelRequest,
23
+ sample_len: int,
24
+ ) -> Tuple[List[SamplerOutput], bool]:
25
+ """Run the model forward pass to generate sample_len future tokens.
26
+ Returns the list of sampler output, one per layer, along with indicator
27
+ of whether torch tensor in sampler output need to be transposed in
28
+ latter sampler_output_to_torch logic.
29
+
30
+ For mlp spec worker, this indicator shall be True.
31
+ """
32
+ self._raise_if_unsupported(execute_model_req)
33
+
34
+ seq_group_metadata_list = execute_model_req.seq_group_metadata_list
35
+
36
+ (input_tokens, seq_lens,
37
+ query_lens) = self._prepare_input_tensors(seq_group_metadata_list)
38
+
39
+ sampling_metadata = SamplingMetadata.prepare(
40
+ seq_group_metadata_list, seq_lens, query_lens, self.device,
41
+ self.model_runner.pin_memory)
42
+
43
+ model_outputs = self.model_runner.model.generate_proposals(
44
+ input_ids=input_tokens,
45
+ previous_hidden_states=execute_model_req.previous_hidden_states.
46
+ hidden_states,
47
+ num_predict_tokens=sample_len,
48
+ sampling_metadata=sampling_metadata)
49
+
50
+ assert len(model_outputs) == sample_len
51
+
52
+ return model_outputs, True
53
+
54
+ def _prepare_input_tensors(
55
+ self,
56
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
57
+ ) -> Tuple[torch.Tensor, List[int], List[int]]:
58
+ if not seq_group_metadata_list:
59
+ return ModelInput.empty(self.device)
60
+
61
+ input_tokens: List[int] = []
62
+ seq_lens: List[int] = []
63
+ query_lens: List[int] = []
64
+
65
+ for seq_group_metadata in seq_group_metadata_list:
66
+ is_prompt = seq_group_metadata.is_prompt
67
+
68
+ for seq_data in seq_group_metadata.seq_data.values():
69
+ seq_data_len = seq_data.get_len()
70
+ if is_prompt:
71
+ context_len = seq_data.get_num_computed_tokens()
72
+ seq_len = min(
73
+ seq_data_len,
74
+ context_len + seq_group_metadata.token_chunk_size)
75
+ tokens = seq_data.get_token_ids()[context_len:seq_len]
76
+ seq_lens.append(seq_len)
77
+ input_tokens.extend(tokens)
78
+ query_lens.append(seq_len - context_len)
79
+ else:
80
+ seq_lens.append(seq_data_len)
81
+ input_tokens.append(seq_data.get_last_token_id())
82
+ query_lens.append(1)
83
+
84
+ input_tokens_tensor = torch.tensor(input_tokens,
85
+ dtype=torch.long,
86
+ device=self.device)
87
+ return input_tokens_tensor, seq_lens, query_lens
vllm/spec_decode/spec_decode_worker.py
+40
-2
@@ -8,16 +8,18 @@ from vllm.distributed.communication_op import broadcast_tensor_dict
8
from vllm.logger import init_logger
9
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
10
from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
11
- SamplerOutput, SequenceGroupMetadata)
11
+ HiddenStates, SamplerOutput, SequenceGroupMetadata,
12
+ get_all_seq_ids)
13
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
14
from vllm.spec_decode.interfaces import (SpeculativeProposals,
15
SpeculativeScorer, SpeculativeScores)
16
from vllm.spec_decode.metrics import AsyncMetricsCollector
17
+from vllm.spec_decode.mlp_speculator_worker import MLPSpeculatorWorker
18
from vllm.spec_decode.multi_step_worker import MultiStepWorker
19
from vllm.spec_decode.ngram_worker import NGramWorker
20
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
21
from vllm.spec_decode.util import (create_sequence_group_output,
20
- get_all_num_logprobs, get_all_seq_ids,
22
+ get_all_num_logprobs,
23
get_sampled_token_logprobs, nvtx_range,
24
split_batch_by_proposal_len)
25
from vllm.worker.worker import Worker
@@ -104,6 +106,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
106
proposer_worker = NGramWorker(**draft_worker_kwargs)
107
proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
108
ngram_prompt_lookup_max)
109
+ elif draft_worker_kwargs[
110
+ "model_config"].hf_config.model_type == "mlp_speculator":
111
+ proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
112
+ disable_bonus_tokens = False
113
else:
114
proposer_worker = MultiStepWorker(**draft_worker_kwargs)
115
@@ -155,6 +161,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
161
# Lazy initiazliation.
162
self.scorer: SpeculativeScorer
163
164
+ # Hidden states from target model to pass to proposer
165
+ # in the subsequent step.
166
+ self.previous_hidden_states: Optional[HiddenStates] = None
167
+
168
def init_device(self) -> None:
169
"""Initialize both scorer and proposer models.
170
"""
@@ -337,6 +347,16 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
347
assert len(sampler_output) == 1
348
sampler_output = sampler_output[0]
349
350
+ # Store hidden states from target model execution.
351
+ hidden_states = sampler_output.hidden_states
352
+ if hidden_states is not None:
353
+ if self.previous_hidden_states is None:
354
+ self.previous_hidden_states = HiddenStates(
355
+ execute_model_req.seq_group_metadata_list, hidden_states)
356
+ else:
357
+ self.previous_hidden_states.update(
358
+ execute_model_req.seq_group_metadata_list, hidden_states)
359
+
360
# Clear device tensors from sampler output. This reduces communication
361
# overhead when the engine runs in a different process than the workers.
362
sampler_output.probs = None
@@ -383,6 +403,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
403
"""
404
assert num_lookahead_slots == execute_model_req.num_lookahead_slots
405
406
+ # Pass last hidden states from target model to proposer
407
+ execute_model_req.previous_hidden_states = self.previous_hidden_states
408
+ self.previous_hidden_states = None
409
+
410
# Generate proposals using draft worker.
411
proposals = self.proposer_worker.get_spec_proposals(execute_model_req)
412
@@ -466,6 +490,20 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
490
# metadata.
491
accepted_token_ids[original_indices] = accepted_token_ids.clone()
492
493
+ hidden_states = proposal_scores.hidden_states
494
+ if hidden_states is not None:
495
+ # Contract hidden states based on accepted tokens
496
+ hs_size = hidden_states.shape[1]
497
+ hidden_states = hidden_states.reshape(-1, max_proposal_len + 1,
498
+ hs_size)
499
+ accepted_index = accepted_token_ids + 1 # Convert -1 to 0
500
+ accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
501
+ index = accepted_index[:, None, None].expand(-1, 1, hs_size)
502
+ hidden_states = hidden_states.gather(1, index).squeeze(1) # b x d
503
+ # Store hidden states from target model for subsequent decode step
504
+ self.previous_hidden_states = HiddenStates(seq_group_metadata_list,
505
+ hidden_states)
506
+
507
return accepted_token_ids, logprobs
508
509
def _create_output_sampler_list(
vllm/spec_decode/top1_proposer.py
+4
@@ -65,9 +65,13 @@ class Top1Proposer(SpeculativeProposer):
65
# token_ids is like [batch] format in proposal_len size list,
66
# while if it is false, the format would be [proposal_len]
67
# in batch size list
68
+ hidden_states = execute_model_req.previous_hidden_states
69
+ if hidden_states is not None:
70
+ hidden_states.prune(nonzero_proposal_len_seqs)
71
nonzero_execute_model_req = ExecuteModelRequest(
72
seq_group_metadata_list=nonzero_proposal_len_seqs,
73
num_lookahead_slots=proposal_len,
74
+ previous_hidden_states=hidden_states,
75
)
76
maybe_sampler_output, transposed = self._worker.sampler_output(
77
execute_model_req=nonzero_execute_model_req,
vllm/spec_decode/util.py
-8
@@ -10,14 +10,6 @@ from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
10
SeqId = int
11
12
13
-def get_all_seq_ids(
14
- seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[SeqId]:
15
- """Given a list of SequenceGroupMetadata, create a list of all
16
- sequence ids.
17
- """
18
- return [seq_id for sg in seq_group_metadata_list for seq_id in sg.seq_data]
19
-
20
-
13
def get_all_num_logprobs(
14
seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[int]:
15
"""Given a list of SequenceGroupMetadata, create a list of all num_logprobs.
vllm/transformers_utils/config.py
+13
-5
@@ -1,3 +1,4 @@
1
+import contextlib
2
from typing import Dict, Optional, Type
3
4
from transformers import PretrainedConfig
@@ -5,7 +6,13 @@ from transformers import PretrainedConfig
6
from vllm.envs import VLLM_USE_MODELSCOPE
7
from vllm.logger import init_logger
8
from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
8
- JAISConfig, MPTConfig, RWConfig)
9
+ JAISConfig, MLPSpeculatorConfig,
10
+ MPTConfig, RWConfig)
11
+
12
+if VLLM_USE_MODELSCOPE:
13
+ from modelscope import AutoConfig
14
+else:
15
+ from transformers import AutoConfig
16
17
logger = init_logger(__name__)
18
@@ -16,8 +23,13 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
23
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
24
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
25
"jais": JAISConfig,
26
+ "mlp_speculator": MLPSpeculatorConfig,
27
}
28
29
+for name, cls in _CONFIG_REGISTRY.items():
30
+ with contextlib.suppress(ValueError):
31
+ AutoConfig.register(name, cls)
32
+
33
34
def get_config(model: str,
35
trust_remote_code: bool,
@@ -26,10 +38,6 @@ def get_config(model: str,
38
rope_scaling: Optional[dict] = None,
39
rope_theta: Optional[float] = None) -> PretrainedConfig:
40
try:
29
- if VLLM_USE_MODELSCOPE:
30
- from modelscope import AutoConfig
31
- else:
32
- from transformers import AutoConfig
41
config = AutoConfig.from_pretrained(
42
model,
43
trust_remote_code=trust_remote_code,
vllm/transformers_utils/configs/__init__.py
+2
@@ -5,6 +5,7 @@ from vllm.transformers_utils.configs.dbrx import DbrxConfig
5
# `FalconConfig` class from the official HuggingFace transformers library.
6
from vllm.transformers_utils.configs.falcon import RWConfig
7
from vllm.transformers_utils.configs.jais import JAISConfig
8
+from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
9
from vllm.transformers_utils.configs.mpt import MPTConfig
10
11
__all__ = [
@@ -13,4 +14,5 @@ __all__ = [
14
"MPTConfig",
15
"RWConfig",
16
"JAISConfig",
17
+ "MLPSpeculatorConfig",
18
]
vllm/transformers_utils/configs/mlp_speculator.py
new
+50
@@ -0,0 +1,50 @@
1
+from typing import List, Optional
2
+
3
+from transformers import PretrainedConfig
4
+
5
+
6
+class MLPSpeculatorConfig(PretrainedConfig):
7
+ model_type = "mlp_speculator"
8
+
9
+ attribute_map = {
10
+ "hidden_size": "emb_dim",
11
+ }
12
+
13
+ def __init__(self,
14
+ vocab_size: int = 32000,
15
+ emb_dim: int = 4096,
16
+ inner_dim: int = 0,
17
+ n_predict: int = 3,
18
+ top_k_tokens_per_head: Optional[List[int]] = None,
19
+ n_candidates: int = 5,
20
+ **kwargs):
21
+ """
22
+ Initialize an MLPSpeculatorConfig
23
+
24
+ Args:
25
+ vocab_size: int
26
+ the model vocab size
27
+ emb_dim: int
28
+ the model embedding dimension
29
+ inner_dim: int
30
+ the inner dimension of the model. If 0, will be the emb_dim.
31
+ n_predict: int
32
+ the number of lookaheads for the speculator
33
+ top_k_tokens_per_head: List[int]
34
+ Number of tokens to consider from each head when forming the
35
+ candidate tree.
36
+ For each candidate branch in the tree, head n produces topk[n]
37
+ additional sub-branches.
38
+ n_candidates: int
39
+ number of child candidates to create per sequence
40
+ """
41
+ if top_k_tokens_per_head is None:
42
+ top_k_tokens_per_head = [5, 4, 3]
43
+ assert len(top_k_tokens_per_head) == n_predict
44
+ self.vocab_size = vocab_size
45
+ self.emb_dim = emb_dim
46
+ self.inner_dim = inner_dim
47
+ self.n_predict = n_predict
48
+ self.top_k_tokens_per_head = top_k_tokens_per_head
49
+ self.n_candidates = n_candidates
50
+ super().__init__(**kwargs)
vllm/worker/model_runner.py
+15
-3
@@ -86,6 +86,7 @@ class ModelRunner:
86
kv_cache_dtype: Optional[str] = "auto",
87
is_driver_worker: bool = False,
88
vision_language_config: Optional[VisionLanguageConfig] = None,
89
+ return_hidden_states: bool = False,
90
):
91
self.model_config = model_config
92
self.parallel_config = parallel_config
@@ -96,6 +97,7 @@ class ModelRunner:
97
self.load_config = load_config
98
self.is_driver_worker = is_driver_worker
99
self.vision_language_config = vision_language_config
100
+ self.return_hidden_states = return_hidden_states
101
102
self.device = self.device_config.device
103
self.pin_memory = is_pin_memory_available()
@@ -116,15 +118,17 @@ class ModelRunner:
118
self.graph_block_tables = np.zeros(
119
(max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
120
dtype=np.int32)
121
+ num_attn_heads = self.model_config.get_num_attention_heads(
122
+ self.parallel_config)
123
self.attn_backend = get_attn_backend(
120
- self.model_config.get_num_attention_heads(self.parallel_config),
124
+ num_attn_heads,
125
self.model_config.get_head_size(),
126
self.model_config.get_num_kv_heads(self.parallel_config),
127
self.model_config.get_sliding_window(),
128
self.model_config.dtype,
129
self.kv_cache_dtype,
130
self.block_size,
127
- )
131
+ ) if num_attn_heads else None
132
133
# Create processor for multi-modal data
134
if self.vision_language_config is not None:
@@ -762,11 +766,19 @@ class ModelRunner:
766
return None
767
768
# Sample the next token.
765
- output = self.model.sample(
769
+ output: SamplerOutput = self.model.sample(
770
logits=logits,
771
sampling_metadata=sampling_metadata,
772
)
773
774
+ if self.return_hidden_states:
775
+ # we only need to pass hidden states of most recent token
776
+ assert seq_group_metadata_list is not None
777
+ if seq_group_metadata_list[0].is_prompt:
778
+ hidden_states = hidden_states.index_select(
779
+ 0, sampling_metadata.selected_token_indices)
780
+ output.hidden_states = hidden_states
781
+
782
return output
783
784
@torch.inference_mode()
vllm/worker/worker.py
+9
@@ -70,6 +70,14 @@ class Worker(WorkerBase):
70
assert not self.lora_config, (
71
"To be tested: vision language model with LoRA settings.")
72
73
+ # Return hidden states from target model if the draft model is an
74
+ # mlp_speculator
75
+ speculative_args = {} if speculative_config is None \
76
+ or (speculative_config.draft_model_config.model ==
77
+ model_config.model) \
78
+ or (speculative_config.draft_model_config.hf_config.model_type !=
79
+ "mlp_speculator") else {"return_hidden_states": True}
80
+
81
ModelRunnerClass = (EmbeddingModelRunner if
82
self.model_config.embedding_mode else ModelRunner)
83
self.model_runner = ModelRunnerClass(
@@ -83,6 +91,7 @@ class Worker(WorkerBase):
91
kv_cache_dtype=self.cache_config.cache_dtype,
92
is_driver_worker=is_driver_worker,
93
vision_language_config=vision_language_config,
94
+ **speculative_args,
95
)
96
# Uninitialized cache engine. Will be initialized by
97
# initialize_cache.