A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Doc] [SpecDecode] Update MLPSpeculator documentation (#7100)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Aug 6, 2024 at 01:29 UTC
789937af2edb6c1ff847c3cbf0c773fb06602a5f
2 files changed
+58
docs/source/models/spec_decode.rst
+49
@@ -69,6 +69,55 @@ matching n-grams in the prompt. For more information read `this thread. <https:/
69
generated_text = output.outputs[0].text
70
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
71
72
+Speculating using MLP speculators
73
+---------------------------------
74
+
75
+The following code configures vLLM to use speculative decoding where proposals are generated by
76
+draft models that conditioning draft predictions on both context vectors and sampled tokens.
77
+For more information see `this blog <https://pytorch.org/blog/hitchhikers-guide-speculative-decoding/>`_ or
78
+`this technical report <https://arxiv.org/abs/2404.19124>`_.
79
+
80
+.. code-block:: python
81
+
82
+ from vllm import LLM, SamplingParams
83
+
84
+ prompts = [
85
+ "The future of AI is",
86
+ ]
87
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
88
+
89
+ llm = LLM(
90
+ model="meta-llama/Meta-Llama-3.1-70B-Instruct",
91
+ tensor_parallel_size=4,
92
+ speculative_model="ibm-fms/llama3-70b-accelerator",
93
+ speculative_draft_tensor_parallel_size=1,
94
+ use_v2_block_manager=True,
95
+ )
96
+ outputs = llm.generate(prompts, sampling_params)
97
+
98
+ for output in outputs:
99
+ prompt = output.prompt
100
+ generated_text = output.outputs[0].text
101
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
102
+
103
+Note that these speculative models currently need to be run without tensor parallelism, although
104
+it is possible to run the main model using tensor parallelism (see example above). Since the
105
+speculative models are relatively small, we still see significant speedups. However, this
106
+limitation will be fixed in a future release.
107
+
108
+A variety of speculative models of this type are available on HF hub:
109
+
110
+* `llama-13b-accelerator <https://huggingface.co/ibm-fms/llama-13b-accelerator>`_
111
+* `llama3-8b-accelerator <https://huggingface.co/ibm-fms/llama3-8b-accelerator>`_
112
+* `codellama-34b-accelerator <https://huggingface.co/ibm-fms/codellama-34b-accelerator>`_
113
+* `llama2-70b-accelerator <https://huggingface.co/ibm-fms/llama2-70b-accelerator>`_
114
+* `llama3-70b-accelerator <https://huggingface.co/ibm-fms/llama3-70b-accelerator>`_
115
+* `granite-3b-code-instruct-accelerator <https://huggingface.co/ibm-granite/granite-3b-code-instruct-accelerator>`_
116
+* `granite-8b-code-instruct-accelerator <https://huggingface.co/ibm-granite/granite-8b-code-instruct-accelerator>`_
117
+* `granite-7b-instruct-accelerator <https://huggingface.co/ibm-granite/granite-7b-instruct-accelerator>`_
118
+* `granite-20b-code-instruct-accelerator <https://huggingface.co/ibm-granite/granite-20b-code-instruct-accelerator>`_
119
+
120
+
121
Resources for vLLM contributors
122
-------------------------------
123
* `A Hacker's Guide to Speculative Decoding in vLLM <https://www.youtube.com/watch?v=9wNAgpX6z_4>`_
vllm/model_executor/models/mlp_speculator.py
+9
@@ -56,6 +56,15 @@ class MLPSpeculatorLayerNorm(nn.Module):
56
57
58
class MLPSpeculator(nn.Module):
59
+ """
60
+ An implementation of the speculative models introduced in
61
+ "Accelerating Production LLMs with Combined Token/Embedding
62
+ Speculators"
63
+ https://arxiv.org/pdf/2404.19124
64
+
65
+ Trained speculators of this type are available on HF hub at:
66
+ https://huggingface.co/ibm-fms and https://huggingface.co/ibm-granite
67
+ """
68
69
def __init__(self, config: MLPSpeculatorConfig, **kwargs) -> None:
70
super().__init__()