| 1 | # N-Gram Speculation |
| 2 | |
| 3 | The following code configures vLLM to use speculative decoding where proposals are generated by |
| 4 | matching n-grams in the prompt. For more information, read [this thread](https://x.com/joao_gante/status/1747322413006643259). |
| 5 | |
| 6 | ```python |
| 7 | from vllm import LLM, SamplingParams |
| 8 | |
| 9 | prompts = ["The future of AI is"] |
| 10 | sampling_params = SamplingParams(temperature=0.8, top_p=0.95) |
| 11 | |
| 12 | llm = LLM( |
| 13 | model="Qwen/Qwen3-8B", |
| 14 | tensor_parallel_size=1, |
| 15 | speculative_config={ |
| 16 | "method": "ngram", |
| 17 | "num_speculative_tokens": 5, |
| 18 | "prompt_lookup_max": 4, |
| 19 | }, |
| 20 | ) |
| 21 | outputs = llm.generate(prompts, sampling_params) |
| 22 | |
| 23 | for output in outputs: |
| 24 | prompt = output.prompt |
| 25 | generated_text = output.outputs[0].text |
| 26 | print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") |
| 27 | ``` |