@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b12518d3cf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Model] MLPSpeculator speculative decoding support (#4947)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: Davis Wertheimer <Davis.Wertheimer@ibm.com>

Joshua Rosenkranz committed Jun 20, 2024 at 20:23 UTC b12518d3cf4326dfcd10a09780913b86c19fcf1a
18 files changed +523 -40
examples/offline_inference_mlpspeculator.py new
+59
@@ -0,0 +1,59 @@
1 +import gc
2 +import time
3 +from typing import List
4 +
5 +from vllm import LLM, SamplingParams
6 +
7 +
8 +def time_generation(llm: LLM, prompts: List[str],
9 + sampling_params: SamplingParams):
10 + # Generate texts from the prompts. The output is a list of RequestOutput
11 + # objects that contain the prompt, generated text, and other information.
12 + # Warmup first
13 + llm.generate(prompts, sampling_params)
14 + llm.generate(prompts, sampling_params)
15 + start = time.time()
16 + outputs = llm.generate(prompts, sampling_params)
17 + end = time.time()
18 + print((end - start) / sum([len(o.outputs[0].token_ids) for o in outputs]))
19 + # Print the outputs.
20 + for output in outputs:
21 + generated_text = output.outputs[0].text
22 + print(f"text: {generated_text!r}")
23 +
24 +
25 +if __name__ == "__main__":
26 +
27 + template = (
28 + "Below is an instruction that describes a task. Write a response "
29 + "that appropriately completes the request.\n\n### Instruction:\n{}"
30 + "\n\n### Response:\n")
31 +
32 + # Sample prompts.
33 + prompts = [
34 + "Write about the president of the United States.",
35 + ]
36 + prompts = [template.format(prompt) for prompt in prompts]
37 + # Create a sampling params object.
38 + sampling_params = SamplingParams(temperature=0.0, max_tokens=200)
39 +
40 + # Create an LLM without spec decoding
41 + llm = LLM(model="meta-llama/Llama-2-13b-chat-hf")
42 +
43 + print("Without speculation")
44 + time_generation(llm, prompts, sampling_params)
45 +
46 + del llm
47 + gc.collect()
48 +
49 + # Create an LLM with spec decoding
50 + llm = LLM(
51 + model="meta-llama/Llama-2-13b-chat-hf",
52 + speculative_model="ibm-fms/llama-13b-accelerator",
53 + # These are currently required for MLPSpeculator decoding
54 + use_v2_block_manager=True,
55 + enforce_eager=True,
56 + )
57 +
58 + print("With speculation")
59 + time_generation(llm, prompts, sampling_params)
tests/spec_decode/test_spec_decode_worker.py
+6 -2
@@ -456,7 +456,9 @@ def test_k_equals_zero(k: int, batch_size: int):
456 rejection_sampler.token_id_dtype = torch.int64
457 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
458
459 - target_worker.execute_model.return_value = [MagicMock(spec=SamplerOutput)]
459 + sampler_output = MagicMock(spec=SamplerOutput)
460 + sampler_output.hidden_states = None
461 + target_worker.execute_model.return_value = [sampler_output]
462
463 draft_worker.device = 'cuda'
464 target_worker.device = 'cuda'
@@ -497,7 +499,9 @@ def test_empty_input_batch(k: int, batch_size: int):
499 rejection_sampler.token_id_dtype = torch.int64
500 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
501
500 - target_worker.execute_model.return_value = [MagicMock(spec=SamplerOutput)]
502 + sampler_output = MagicMock(spec=SamplerOutput)
503 + sampler_output.hidden_states = None
504 + target_worker.execute_model.return_value = [sampler_output]
505
506 draft_worker.device = 'cuda'
507 target_worker.device = 'cuda'
tests/spec_decode/test_utils.py
+2 -2
@@ -2,8 +2,8 @@ from unittest.mock import MagicMock
2
3 import pytest
4
5 -from vllm.sequence import SequenceGroupMetadata
6 -from vllm.spec_decode.util import get_all_seq_ids, split_batch_by_proposal_len
5 +from vllm.sequence import SequenceGroupMetadata, get_all_seq_ids
6 +from vllm.spec_decode.util import split_batch_by_proposal_len
7
8
9 def test_get_all_seq_ids():
vllm/config.py
+39 -15
@@ -230,7 +230,8 @@ class ModelConfig:
230 self,
231 parallel_config: "ParallelConfig",
232 ) -> None:
233 - total_num_attention_heads = self.hf_text_config.num_attention_heads
233 + total_num_attention_heads = getattr(self.hf_text_config,
234 + "num_attention_heads", 0)
235 tensor_parallel_size = parallel_config.tensor_parallel_size
236 if total_num_attention_heads % tensor_parallel_size != 0:
237 raise ValueError(
@@ -238,7 +239,8 @@ class ModelConfig:
239 " must be divisible by tensor parallel size "
240 f"({tensor_parallel_size}).")
241
241 - total_num_hidden_layers = self.hf_text_config.num_hidden_layers
242 + total_num_hidden_layers = getattr(self.hf_text_config,
243 + "num_hidden_layers", 0)
244 pipeline_parallel_size = parallel_config.pipeline_parallel_size
245 if total_num_hidden_layers % pipeline_parallel_size != 0:
246 raise ValueError(
@@ -341,8 +343,8 @@ class ModelConfig:
343
344 def get_num_attention_heads(self,
345 parallel_config: "ParallelConfig") -> int:
344 - return self.hf_text_config.num_attention_heads // \
345 - parallel_config.tensor_parallel_size
346 + num_heads = getattr(self.hf_text_config, "num_attention_heads", 0)
347 + return num_heads // parallel_config.tensor_parallel_size
348
349 def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
350 total_num_hidden_layers = self.hf_text_config.num_hidden_layers
@@ -818,7 +820,8 @@ class SpeculativeConfig:
820 speculative_model (Optional[str]): The name of the speculative
821 model, if provided.
822 num_speculative_tokens (Optional[int]): The number of speculative
821 - tokens, if provided.
823 + tokens, if provided. Will default to the number in the draft
824 + model config if present, otherwise is required.
825 speculative_max_model_len (Optional[int]): The maximum model len of
826 the speculative model. Used when testing the ability to skip
827 speculation for some sequences.
@@ -841,24 +844,18 @@ class SpeculativeConfig:
844 the necessary conditions are met, else None.
845 """
846
844 - if speculative_model is None and num_speculative_tokens is None:
847 + if speculative_model is None:
848 + if num_speculative_tokens is not None:
849 + raise ValueError("num_speculative_tokens was provided without "
850 + "speculative_model.")
851 return None
852
847 - if speculative_model is not None and num_speculative_tokens is None:
848 - raise ValueError(
849 - "Expected both speculative_model and "
850 - "num_speculative_tokens to be provided, but found "
851 - f"{speculative_model=} and {num_speculative_tokens=}.")
852 -
853 if (speculative_disable_by_batch_size is not None
854 and speculative_disable_by_batch_size < 2):
855 raise ValueError("Expect the batch size threshold of disabling "
856 "speculative decoding is > 1, but got "
857 f"{speculative_disable_by_batch_size=}")
858
859 - assert (speculative_model is not None
860 - and num_speculative_tokens is not None)
861 -
859 if enable_chunked_prefill:
860 raise ValueError(
861 "Speculative decoding and chunked prefill are "
@@ -912,6 +909,27 @@ class SpeculativeConfig:
909 max_logprobs=target_model_config.max_logprobs,
910 )
911
912 + if (draft_model_config.hf_config.model_type == "mlp_speculator"
913 + and target_parallel_config.world_size != 1):
914 + # MLPSpeculator TP support will be added very soon
915 + raise ValueError(
916 + "Speculative decoding with mlp_speculator models does not "
917 + "yet support distributed inferencing (TP > 1).")
918 +
919 + n_predict = getattr(draft_model_config.hf_config, "n_predict",
920 + None)
921 + if n_predict is not None:
922 + if num_speculative_tokens is None:
923 + # Default to max value defined in draft model config.
924 + num_speculative_tokens = n_predict
925 + elif num_speculative_tokens > n_predict:
926 + # Verify provided value doesn't exceed the maximum
927 + # supported by the draft model.
928 + raise ValueError(
929 + "Expected both speculative_model and "
930 + "num_speculative_tokens to be provided, but found "
931 + f"{speculative_model=} and {num_speculative_tokens=}.")
932 +
933 draft_model_config.max_model_len = (
934 SpeculativeConfig._maybe_override_draft_max_model_len(
935 speculative_max_model_len,
@@ -923,6 +941,12 @@ class SpeculativeConfig:
941 SpeculativeConfig.create_draft_parallel_config(
942 target_parallel_config))
943
944 + if num_speculative_tokens is None:
945 + raise ValueError(
946 + "num_speculative_tokens must be provided with "
947 + "speculative_model unless the draft model config contains an "
948 + "n_predict parameter.")
949 +
950 return SpeculativeConfig(
951 draft_model_config,
952 draft_parallel_config,
vllm/model_executor/models/__init__.py
+1
@@ -60,6 +60,7 @@ _GENERATION_MODELS = {
60 "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
61 "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
62 "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
63 + "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
64 }
65
66 _EMBEDDING_MODELS = {
vllm/model_executor/models/mlp_speculator.py new
+143
@@ -0,0 +1,143 @@
1 +import math
2 +from typing import Iterable, List, Tuple
3 +
4 +import torch
5 +import torch.nn as nn
6 +
7 +from vllm.model_executor import SamplingMetadata
8 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
9 +from vllm.model_executor.layers.sampler import Sampler
10 +from vllm.model_executor.layers.vocab_parallel_embedding import (
11 + VocabParallelEmbedding)
12 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13 +from vllm.sequence import SamplerOutput
14 +
15 +
16 +class MLPSpeculatorLayerNorm(nn.Module):
17 + """
18 + A L2 normalization implementation
19 + ...
20 + Args
21 + ----
22 + normalized_shape : int
23 + Dimensionality of input data (size of final tensor axis)
24 + eps : float
25 + Safety term to prevent division by zero. Make sure the chosen value
26 + fits in the range of your encoding scheme
27 + (i.e. fp16 requires eps >= 6e-8).
28 + """
29 +
30 + def __init__(
31 + self,
32 + normalized_shape,
33 + eps=1e-06,
34 + ):
35 + super(MLPSpeculatorLayerNorm, self).__init__()
36 + self.weight = nn.Parameter(torch.empty(normalized_shape))
37 + self.bias = nn.Parameter(torch.empty(normalized_shape))
38 + self.eps = eps
39 +
40 + def forward(self, x):
41 + xf = x
42 + xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + self.eps)
43 + x = xf.type_as(x)
44 + x = self.weight * x
45 + x = x + self.bias
46 + return x
47 +
48 +
49 +class MLPSpeculator(nn.Module):
50 +
51 + def __init__(self, config, **kwargs) -> None:
52 + super().__init__()
53 + self.n_predict = config.n_predict
54 + self.vocab_size = config.vocab_size
55 + self.emb_dim = config.emb_dim
56 + self.inner_dim = config.inner_dim if config.inner_dim != 0 \
57 + else config.emb_dim
58 +
59 + self.max_speculative_tokens = getattr(config, "max_speculative_tokens",
60 + self.n_predict)
61 +
62 + self.emb = nn.ModuleList([
63 + VocabParallelEmbedding(config.vocab_size,
64 + self.inner_dim,
65 + org_num_embeddings=config.vocab_size)
66 + for _ in range(self.max_speculative_tokens)
67 + ])
68 +
69 + self.proj = nn.ModuleList([
70 + nn.Linear((self.emb_dim if i == 0 else self.inner_dim),
71 + self.inner_dim,
72 + bias=False) for i in range(self.max_speculative_tokens)
73 + ])
74 +
75 + self.head = nn.ModuleList([
76 + nn.Linear(self.inner_dim, self.vocab_size, bias=False)
77 + for _ in range(self.max_speculative_tokens)
78 + ])
79 + self.ln = nn.ModuleList([
80 + MLPSpeculatorLayerNorm(self.inner_dim)
81 + for _ in range(self.max_speculative_tokens)
82 + ])
83 +
84 + self.state_weight = 0.5**(0.5 / config.n_predict)
85 + self.emb_weight = math.sqrt(
86 + (1 - self.state_weight**2) * (self.inner_dim / 2))
87 + self.activation = nn.GELU()
88 + self.config = config
89 + self.logits_processor = LogitsProcessor(config.vocab_size,
90 + config.vocab_size, 1.0)
91 + self.sampler = Sampler()
92 +
93 + def generate_proposals(
94 + self,
95 + input_ids: torch.Tensor,
96 + previous_hidden_states: torch.Tensor,
97 + num_predict_tokens: int,
98 + sampling_metadata: SamplingMetadata,
99 + ) -> List[SamplerOutput]:
100 + if num_predict_tokens > self.max_speculative_tokens:
101 + raise ValueError(f"Max speculative tokens for model is "
102 + f"{self.max_speculative_tokens}, but "
103 + f"{num_predict_tokens} were requested")
104 +
105 + # b x 1 x d
106 + previous_hidden_states = previous_hidden_states.unsqueeze(1)
107 +
108 + # b x 1
109 + last_tokens = input_ids.unsqueeze(1)
110 +
111 + next_tokens = []
112 +
113 + for head_index in range(num_predict_tokens):
114 +
115 + # Project and predict
116 + z = self.emb[head_index](last_tokens) # b k d
117 + states = self.proj[head_index](previous_hidden_states)
118 +
119 + # Weighted add of state_weight*state and emb_weight*z
120 + # Let subsequent LN take care of denominator
121 + # state_weight is close to 1, so shouldn't be any precision issues
122 + states.add_(z, alpha=self.emb_weight / self.state_weight)
123 +
124 + states = self.activation(self.ln[head_index](states)) # b k d
125 + # TODO: not yet supporting top_k_tokens_per_head
126 + previous_hidden_states = states
127 +
128 + logits = self.logits_processor(self.head[head_index].weight,
129 + states, sampling_metadata)
130 +
131 + output = self.sampler(logits.flatten(0, 1), sampling_metadata)
132 + last_tokens = output.sampled_token_ids
133 + next_tokens.append(output)
134 +
135 + return next_tokens
136 +
137 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
138 + params_dict = dict(self.named_parameters())
139 + for name, loaded_weight in weights:
140 + param = params_dict[name.replace("speculator.", "")]
141 + weight_loader = getattr(param, "weight_loader",
142 + default_weight_loader)
143 + weight_loader(param, loaded_weight)
vllm/sequence.py
+46
@@ -794,6 +794,9 @@ class SamplerOutput:
794 # Spec decode metrics populated by workers.
795 spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
796
797 + # Optional last hidden states from the model.
798 + hidden_states: Optional[torch.Tensor] = None
799 +
800 def __getitem__(self, idx: int):
801 return self.outputs[idx]
802
@@ -842,6 +845,46 @@ class PoolerOutput:
845 self.__class__) and self.outputs == other.outputs
846
847
848 +def get_all_seq_ids(
849 + seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[int]:
850 + """Given a list of SequenceGroupMetadata, create a list of all
851 + sequence ids.
852 + """
853 + return [seq_id for sg in seq_group_metadata_list for seq_id in sg.seq_data]
854 +
855 +
856 +class HiddenStates:
857 + """Hidden states corresponding to in-progress sequences.
858 + Used in speculative decoding to pass hidden states from
859 + the target model to the proposer model in the subsequent step.
860 +
861 + seq_ids are the sequence ids of each entry of the batch
862 + dimension of the hidden_states tensor"""
863 +
864 + def __init__(self, seq_group_metadata_list: List[SequenceGroupMetadata],
865 + hidden_states: torch.Tensor):
866 + assert len(seq_group_metadata_list) == len(hidden_states)
867 + self.seq_ids: List[int] = get_all_seq_ids(seq_group_metadata_list)
868 + self.hidden_states: torch.Tensor = hidden_states
869 +
870 + def update(self, seq_group_metadata_list: List[SequenceGroupMetadata],
871 + hidden_states: torch.Tensor) -> None:
872 + """Update hidden states from target model invocation."""
873 + assert len(seq_group_metadata_list) == len(hidden_states)
874 + self.seq_ids.extend(get_all_seq_ids(seq_group_metadata_list))
875 + self.hidden_states = torch.cat([self.hidden_states, hidden_states])
876 +
877 + def prune(self,
878 + seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
879 + """Prune to provided list of sequence ids."""
880 + seq_ids = get_all_seq_ids(seq_group_metadata_list)
881 + if seq_ids != self.seq_ids:
882 + # Batch contents changed - prune removed sequences.
883 + index = [self.seq_ids.index(seq_id) for seq_id in seq_ids]
884 + self.hidden_states = self.hidden_states[index]
885 + self.seq_ids = seq_ids
886 +
887 +
888 @dataclass
889 class ExecuteModelRequest:
890 """The model execution request."""
@@ -857,6 +900,8 @@ class ExecuteModelRequest:
900 num_lookahead_slots: int = 0
901 # The number of requests in the running queue.
902 running_queue_size: int = 0
903 + # Optional hidden states from prior step.
904 + previous_hidden_states: Optional[HiddenStates] = None
905
906 def clone(
907 self, seq_group_metadata_list: List[SequenceGroupMetadata]
@@ -869,4 +914,5 @@ class ExecuteModelRequest:
914 blocks_to_copy=self.blocks_to_copy.copy(),
915 num_lookahead_slots=self.num_lookahead_slots,
916 running_queue_size=self.running_queue_size,
917 + previous_hidden_states=self.previous_hidden_states,
918 )
vllm/spec_decode/batch_expansion.py
+3 -3
@@ -4,11 +4,10 @@ from typing import Iterator, List, Tuple
4 import torch
5
6 from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
7 - SequenceGroupMetadata)
7 + SequenceGroupMetadata, get_all_seq_ids)
8 from vllm.spec_decode.interfaces import (SpeculativeProposals,
9 SpeculativeScorer, SpeculativeScores)
10 -from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
11 - sampler_output_to_torch,
10 +from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
11 split_batch_by_proposal_len)
12 from vllm.worker.worker_base import WorkerBase
13
@@ -98,6 +97,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
97 probs=all_probs,
98 token_ids=all_tokens,
99 logprobs=spec_logprobs,
100 + hidden_states=target_sampler_output.hidden_states,
101 )
102
103 def _expand_batch(
vllm/spec_decode/interfaces.py
+4
@@ -1,5 +1,6 @@
1 from abc import ABC, abstractmethod
2 from dataclasses import dataclass
3 +from typing import Optional
4
5 import torch
6
@@ -46,6 +47,9 @@ class SpeculativeScores:
47 # tokens and also non-speculative normal decoding.
48 token_ids: torch.Tensor
49
50 + # Optional last hidden states from the scoring model.
51 + hidden_states: Optional[torch.Tensor] = None
52 +
53 def __repr__(self):
54 return (f"SpeculativeScores("
55 f"probs={self.probs.shape}, "
vllm/spec_decode/mlp_speculator_worker.py new
+87
@@ -0,0 +1,87 @@
1 +from typing import List, Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm.model_executor import SamplingMetadata
6 +from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
7 + SequenceGroupMetadata)
8 +from vllm.spec_decode.multi_step_worker import MultiStepWorker
9 +from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
10 +from vllm.worker.model_runner import ModelInput
11 +
12 +
13 +class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker):
14 + """Worker for MLPSpeculator models.
15 +
16 + Not currently compatible with LoRA or chunked prefill.
17 + """
18 +
19 + @torch.inference_mode()
20 + def sampler_output(
21 + self,
22 + execute_model_req: ExecuteModelRequest,
23 + sample_len: int,
24 + ) -> Tuple[List[SamplerOutput], bool]:
25 + """Run the model forward pass to generate sample_len future tokens.
26 + Returns the list of sampler output, one per layer, along with indicator
27 + of whether torch tensor in sampler output need to be transposed in
28 + latter sampler_output_to_torch logic.
29 +
30 + For mlp spec worker, this indicator shall be True.
31 + """
32 + self._raise_if_unsupported(execute_model_req)
33 +
34 + seq_group_metadata_list = execute_model_req.seq_group_metadata_list
35 +
36 + (input_tokens, seq_lens,
37 + query_lens) = self._prepare_input_tensors(seq_group_metadata_list)
38 +
39 + sampling_metadata = SamplingMetadata.prepare(
40 + seq_group_metadata_list, seq_lens, query_lens, self.device,
41 + self.model_runner.pin_memory)
42 +
43 + model_outputs = self.model_runner.model.generate_proposals(
44 + input_ids=input_tokens,
45 + previous_hidden_states=execute_model_req.previous_hidden_states.
46 + hidden_states,
47 + num_predict_tokens=sample_len,
48 + sampling_metadata=sampling_metadata)
49 +
50 + assert len(model_outputs) == sample_len
51 +
52 + return model_outputs, True
53 +
54 + def _prepare_input_tensors(
55 + self,
56 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
57 + ) -> Tuple[torch.Tensor, List[int], List[int]]:
58 + if not seq_group_metadata_list:
59 + return ModelInput.empty(self.device)
60 +
61 + input_tokens: List[int] = []
62 + seq_lens: List[int] = []
63 + query_lens: List[int] = []
64 +
65 + for seq_group_metadata in seq_group_metadata_list:
66 + is_prompt = seq_group_metadata.is_prompt
67 +
68 + for seq_data in seq_group_metadata.seq_data.values():
69 + seq_data_len = seq_data.get_len()
70 + if is_prompt:
71 + context_len = seq_data.get_num_computed_tokens()
72 + seq_len = min(
73 + seq_data_len,
74 + context_len + seq_group_metadata.token_chunk_size)
75 + tokens = seq_data.get_token_ids()[context_len:seq_len]
76 + seq_lens.append(seq_len)
77 + input_tokens.extend(tokens)
78 + query_lens.append(seq_len - context_len)
79 + else:
80 + seq_lens.append(seq_data_len)
81 + input_tokens.append(seq_data.get_last_token_id())
82 + query_lens.append(1)
83 +
84 + input_tokens_tensor = torch.tensor(input_tokens,
85 + dtype=torch.long,
86 + device=self.device)
87 + return input_tokens_tensor, seq_lens, query_lens
vllm/spec_decode/spec_decode_worker.py
+40 -2
@@ -8,16 +8,18 @@ from vllm.distributed.communication_op import broadcast_tensor_dict
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
10 from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
11 - SamplerOutput, SequenceGroupMetadata)
11 + HiddenStates, SamplerOutput, SequenceGroupMetadata,
12 + get_all_seq_ids)
13 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
14 from vllm.spec_decode.interfaces import (SpeculativeProposals,
15 SpeculativeScorer, SpeculativeScores)
16 from vllm.spec_decode.metrics import AsyncMetricsCollector
17 +from vllm.spec_decode.mlp_speculator_worker import MLPSpeculatorWorker
18 from vllm.spec_decode.multi_step_worker import MultiStepWorker
19 from vllm.spec_decode.ngram_worker import NGramWorker
20 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
21 from vllm.spec_decode.util import (create_sequence_group_output,
20 - get_all_num_logprobs, get_all_seq_ids,
22 + get_all_num_logprobs,
23 get_sampled_token_logprobs, nvtx_range,
24 split_batch_by_proposal_len)
25 from vllm.worker.worker import Worker
@@ -104,6 +106,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
106 proposer_worker = NGramWorker(**draft_worker_kwargs)
107 proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
108 ngram_prompt_lookup_max)
109 + elif draft_worker_kwargs[
110 + "model_config"].hf_config.model_type == "mlp_speculator":
111 + proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
112 + disable_bonus_tokens = False
113 else:
114 proposer_worker = MultiStepWorker(**draft_worker_kwargs)
115
@@ -155,6 +161,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
161 # Lazy initiazliation.
162 self.scorer: SpeculativeScorer
163
164 + # Hidden states from target model to pass to proposer
165 + # in the subsequent step.
166 + self.previous_hidden_states: Optional[HiddenStates] = None
167 +
168 def init_device(self) -> None:
169 """Initialize both scorer and proposer models.
170 """
@@ -337,6 +347,16 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
347 assert len(sampler_output) == 1
348 sampler_output = sampler_output[0]
349
350 + # Store hidden states from target model execution.
351 + hidden_states = sampler_output.hidden_states
352 + if hidden_states is not None:
353 + if self.previous_hidden_states is None:
354 + self.previous_hidden_states = HiddenStates(
355 + execute_model_req.seq_group_metadata_list, hidden_states)
356 + else:
357 + self.previous_hidden_states.update(
358 + execute_model_req.seq_group_metadata_list, hidden_states)
359 +
360 # Clear device tensors from sampler output. This reduces communication
361 # overhead when the engine runs in a different process than the workers.
362 sampler_output.probs = None
@@ -383,6 +403,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
403 """
404 assert num_lookahead_slots == execute_model_req.num_lookahead_slots
405
406 + # Pass last hidden states from target model to proposer
407 + execute_model_req.previous_hidden_states = self.previous_hidden_states
408 + self.previous_hidden_states = None
409 +
410 # Generate proposals using draft worker.
411 proposals = self.proposer_worker.get_spec_proposals(execute_model_req)
412
@@ -466,6 +490,20 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
490 # metadata.
491 accepted_token_ids[original_indices] = accepted_token_ids.clone()
492
493 + hidden_states = proposal_scores.hidden_states
494 + if hidden_states is not None:
495 + # Contract hidden states based on accepted tokens
496 + hs_size = hidden_states.shape[1]
497 + hidden_states = hidden_states.reshape(-1, max_proposal_len + 1,
498 + hs_size)
499 + accepted_index = accepted_token_ids + 1 # Convert -1 to 0
500 + accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
501 + index = accepted_index[:, None, None].expand(-1, 1, hs_size)
502 + hidden_states = hidden_states.gather(1, index).squeeze(1) # b x d
503 + # Store hidden states from target model for subsequent decode step
504 + self.previous_hidden_states = HiddenStates(seq_group_metadata_list,
505 + hidden_states)
506 +
507 return accepted_token_ids, logprobs
508
509 def _create_output_sampler_list(
vllm/spec_decode/top1_proposer.py
+4
@@ -65,9 +65,13 @@ class Top1Proposer(SpeculativeProposer):
65 # token_ids is like [batch] format in proposal_len size list,
66 # while if it is false, the format would be [proposal_len]
67 # in batch size list
68 + hidden_states = execute_model_req.previous_hidden_states
69 + if hidden_states is not None:
70 + hidden_states.prune(nonzero_proposal_len_seqs)
71 nonzero_execute_model_req = ExecuteModelRequest(
72 seq_group_metadata_list=nonzero_proposal_len_seqs,
73 num_lookahead_slots=proposal_len,
74 + previous_hidden_states=hidden_states,
75 )
76 maybe_sampler_output, transposed = self._worker.sampler_output(
77 execute_model_req=nonzero_execute_model_req,
vllm/spec_decode/util.py
-8
@@ -10,14 +10,6 @@ from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
10 SeqId = int
11
12
13 -def get_all_seq_ids(
14 - seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[SeqId]:
15 - """Given a list of SequenceGroupMetadata, create a list of all
16 - sequence ids.
17 - """
18 - return [seq_id for sg in seq_group_metadata_list for seq_id in sg.seq_data]
19 -
20 -
13 def get_all_num_logprobs(
14 seq_group_metadata_list: List[SequenceGroupMetadata]) -> List[int]:
15 """Given a list of SequenceGroupMetadata, create a list of all num_logprobs.
vllm/transformers_utils/config.py
+13 -5
@@ -1,3 +1,4 @@
1 +import contextlib
2 from typing import Dict, Optional, Type
3
4 from transformers import PretrainedConfig
@@ -5,7 +6,13 @@ from transformers import PretrainedConfig
6 from vllm.envs import VLLM_USE_MODELSCOPE
7 from vllm.logger import init_logger
8 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
8 - JAISConfig, MPTConfig, RWConfig)
9 + JAISConfig, MLPSpeculatorConfig,
10 + MPTConfig, RWConfig)
11 +
12 +if VLLM_USE_MODELSCOPE:
13 + from modelscope import AutoConfig
14 +else:
15 + from transformers import AutoConfig
16
17 logger = init_logger(__name__)
18
@@ -16,8 +23,13 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
23 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
24 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
25 "jais": JAISConfig,
26 + "mlp_speculator": MLPSpeculatorConfig,
27 }
28
29 +for name, cls in _CONFIG_REGISTRY.items():
30 + with contextlib.suppress(ValueError):
31 + AutoConfig.register(name, cls)
32 +
33
34 def get_config(model: str,
35 trust_remote_code: bool,
@@ -26,10 +38,6 @@ def get_config(model: str,
38 rope_scaling: Optional[dict] = None,
39 rope_theta: Optional[float] = None) -> PretrainedConfig:
40 try:
29 - if VLLM_USE_MODELSCOPE:
30 - from modelscope import AutoConfig
31 - else:
32 - from transformers import AutoConfig
41 config = AutoConfig.from_pretrained(
42 model,
43 trust_remote_code=trust_remote_code,
vllm/transformers_utils/configs/__init__.py
+2
@@ -5,6 +5,7 @@ from vllm.transformers_utils.configs.dbrx import DbrxConfig
5 # `FalconConfig` class from the official HuggingFace transformers library.
6 from vllm.transformers_utils.configs.falcon import RWConfig
7 from vllm.transformers_utils.configs.jais import JAISConfig
8 +from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
9 from vllm.transformers_utils.configs.mpt import MPTConfig
10
11 __all__ = [
@@ -13,4 +14,5 @@ __all__ = [
14 "MPTConfig",
15 "RWConfig",
16 "JAISConfig",
17 + "MLPSpeculatorConfig",
18 ]
vllm/transformers_utils/configs/mlp_speculator.py new
+50
@@ -0,0 +1,50 @@
1 +from typing import List, Optional
2 +
3 +from transformers import PretrainedConfig
4 +
5 +
6 +class MLPSpeculatorConfig(PretrainedConfig):
7 + model_type = "mlp_speculator"
8 +
9 + attribute_map = {
10 + "hidden_size": "emb_dim",
11 + }
12 +
13 + def __init__(self,
14 + vocab_size: int = 32000,
15 + emb_dim: int = 4096,
16 + inner_dim: int = 0,
17 + n_predict: int = 3,
18 + top_k_tokens_per_head: Optional[List[int]] = None,
19 + n_candidates: int = 5,
20 + **kwargs):
21 + """
22 + Initialize an MLPSpeculatorConfig
23 +
24 + Args:
25 + vocab_size: int
26 + the model vocab size
27 + emb_dim: int
28 + the model embedding dimension
29 + inner_dim: int
30 + the inner dimension of the model. If 0, will be the emb_dim.
31 + n_predict: int
32 + the number of lookaheads for the speculator
33 + top_k_tokens_per_head: List[int]
34 + Number of tokens to consider from each head when forming the
35 + candidate tree.
36 + For each candidate branch in the tree, head n produces topk[n]
37 + additional sub-branches.
38 + n_candidates: int
39 + number of child candidates to create per sequence
40 + """
41 + if top_k_tokens_per_head is None:
42 + top_k_tokens_per_head = [5, 4, 3]
43 + assert len(top_k_tokens_per_head) == n_predict
44 + self.vocab_size = vocab_size
45 + self.emb_dim = emb_dim
46 + self.inner_dim = inner_dim
47 + self.n_predict = n_predict
48 + self.top_k_tokens_per_head = top_k_tokens_per_head
49 + self.n_candidates = n_candidates
50 + super().__init__(**kwargs)
vllm/worker/model_runner.py
+15 -3
@@ -86,6 +86,7 @@ class ModelRunner:
86 kv_cache_dtype: Optional[str] = "auto",
87 is_driver_worker: bool = False,
88 vision_language_config: Optional[VisionLanguageConfig] = None,
89 + return_hidden_states: bool = False,
90 ):
91 self.model_config = model_config
92 self.parallel_config = parallel_config
@@ -96,6 +97,7 @@ class ModelRunner:
97 self.load_config = load_config
98 self.is_driver_worker = is_driver_worker
99 self.vision_language_config = vision_language_config
100 + self.return_hidden_states = return_hidden_states
101
102 self.device = self.device_config.device
103 self.pin_memory = is_pin_memory_available()
@@ -116,15 +118,17 @@ class ModelRunner:
118 self.graph_block_tables = np.zeros(
119 (max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
120 dtype=np.int32)
121 + num_attn_heads = self.model_config.get_num_attention_heads(
122 + self.parallel_config)
123 self.attn_backend = get_attn_backend(
120 - self.model_config.get_num_attention_heads(self.parallel_config),
124 + num_attn_heads,
125 self.model_config.get_head_size(),
126 self.model_config.get_num_kv_heads(self.parallel_config),
127 self.model_config.get_sliding_window(),
128 self.model_config.dtype,
129 self.kv_cache_dtype,
130 self.block_size,
127 - )
131 + ) if num_attn_heads else None
132
133 # Create processor for multi-modal data
134 if self.vision_language_config is not None:
@@ -762,11 +766,19 @@ class ModelRunner:
766 return None
767
768 # Sample the next token.
765 - output = self.model.sample(
769 + output: SamplerOutput = self.model.sample(
770 logits=logits,
771 sampling_metadata=sampling_metadata,
772 )
773
774 + if self.return_hidden_states:
775 + # we only need to pass hidden states of most recent token
776 + assert seq_group_metadata_list is not None
777 + if seq_group_metadata_list[0].is_prompt:
778 + hidden_states = hidden_states.index_select(
779 + 0, sampling_metadata.selected_token_indices)
780 + output.hidden_states = hidden_states
781 +
782 return output
783
784 @torch.inference_mode()
vllm/worker/worker.py
+9
@@ -70,6 +70,14 @@ class Worker(WorkerBase):
70 assert not self.lora_config, (
71 "To be tested: vision language model with LoRA settings.")
72
73 + # Return hidden states from target model if the draft model is an
74 + # mlp_speculator
75 + speculative_args = {} if speculative_config is None \
76 + or (speculative_config.draft_model_config.model ==
77 + model_config.model) \
78 + or (speculative_config.draft_model_config.hf_config.model_type !=
79 + "mlp_speculator") else {"return_hidden_states": True}
80 +
81 ModelRunnerClass = (EmbeddingModelRunner if
82 self.model_config.embedding_mode else ModelRunner)
83 self.model_runner = ModelRunnerClass(
@@ -83,6 +91,7 @@ class Worker(WorkerBase):
91 kv_cache_dtype=self.cache_config.cache_dtype,
92 is_driver_worker=is_driver_worker,
93 vision_language_config=vision_language_config,
94 + **speculative_args,
95 )
96 # Uninitialized cache engine. Will be initialized by
97 # initialize_cache.