@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 32aa2059ad
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Docs] Convert rST to MyST (Markdown) (#11145)

Signed-off-by: Rafael Vasquez <rafvasq21@gmail.com>

Rafael Vasquez committed Dec 23, 2024 at 17:35 UTC 32aa2059addd97be1afce7a199d228191710c294
167 files changed +7870 -8138
.gitignore
+2
@@ -81,6 +81,8 @@ instance/
81 docs/_build/
82 docs/source/getting_started/examples/*.rst
83 !**/*.template.rst
84 +docs/source/getting_started/examples/*.md
85 +!**/*.template.md
86
87 # PyBuilder
88 .pybuilder/
Dockerfile
+1 -1
@@ -2,7 +2,7 @@
2 # to run the OpenAI compatible server.
3
4 # Please update any changes made here to
5 -# docs/source/dev/dockerfile/dockerfile.rst and
5 +# docs/source/dev/dockerfile/dockerfile.md and
6 # docs/source/assets/dev/dockerfile-stages-dependency.png
7
8 ARG CUDA_VERSION=12.4.1
docs/requirements-docs.txt
+1 -1
@@ -1,7 +1,7 @@
1 sphinx==6.2.1
2 sphinx-book-theme==1.0.1
3 sphinx-copybutton==0.5.2
4 -myst-parser==2.0.0
4 +myst-parser==3.0.1
5 sphinx-argparse==0.4.0
6 msgspec
7 cloudpickle
docs/source/automatic_prefix_caching/apc.md new
+102
@@ -0,0 +1,102 @@
1 +(apc)=
2 +
3 +# Introduction
4 +
5 +## What is Automatic Prefix Caching
6 +
7 +Automatic Prefix Caching (APC in short) caches the KV cache of existing queries, so that a new query can directly reuse the KV cache if it shares the same prefix with one of the existing queries, allowing the new query to skip the computation of the shared part.
8 +
9 +```{note}
10 +Technical details on how vLLM implements APC are in the next page.
11 +```
12 +
13 +## Enabling APC in vLLM
14 +
15 +Set `enable_prefix_caching=True` in vLLM engine to enable APC. Here is an example:
16 +
17 +```python
18 +import time
19 +from vllm import LLM, SamplingParams
20 +
21 +
22 +# A prompt containing a large markdown table. The table is randomly generated by GPT-4.
23 +LONG_PROMPT = "You are a helpful assistant in recognizes the content of tables in markdown format. Here is a table as follows.\n# Table\n" + """
24 +| ID | Name | Age | Occupation | Country | Email | Phone Number | Address |
25 +|-----|---------------|-----|---------------|---------------|------------------------|----------------|------------------------------|
26 +| 1 | John Doe | 29 | Engineer | USA | john.doe@example.com | 555-1234 | 123 Elm St, Springfield, IL |
27 +| 2 | Jane Smith | 34 | Doctor | Canada | jane.smith@example.com | 555-5678 | 456 Oak St, Toronto, ON |
28 +| 3 | Alice Johnson | 27 | Teacher | UK | alice.j@example.com | 555-8765 | 789 Pine St, London, UK |
29 +| 4 | Bob Brown | 45 | Artist | Australia | bob.b@example.com | 555-4321 | 321 Maple St, Sydney, NSW |
30 +| 5 | Carol White | 31 | Scientist | New Zealand | carol.w@example.com | 555-6789 | 654 Birch St, Wellington, NZ |
31 +| 6 | Dave Green | 28 | Lawyer | Ireland | dave.g@example.com | 555-3456 | 987 Cedar St, Dublin, IE |
32 +| 7 | Emma Black | 40 | Musician | USA | emma.b@example.com | 555-1111 | 246 Ash St, New York, NY |
33 +| 8 | Frank Blue | 37 | Chef | Canada | frank.b@example.com | 555-2222 | 135 Spruce St, Vancouver, BC |
34 +| 9 | Grace Yellow | 50 | Engineer | UK | grace.y@example.com | 555-3333 | 864 Fir St, Manchester, UK |
35 +| 10 | Henry Violet | 32 | Artist | Australia | henry.v@example.com | 555-4444 | 753 Willow St, Melbourne, VIC|
36 +| 11 | Irene Orange | 26 | Scientist | New Zealand | irene.o@example.com | 555-5555 | 912 Poplar St, Auckland, NZ |
37 +| 12 | Jack Indigo | 38 | Teacher | Ireland | jack.i@example.com | 555-6666 | 159 Elm St, Cork, IE |
38 +| 13 | Karen Red | 41 | Lawyer | USA | karen.r@example.com | 555-7777 | 357 Cedar St, Boston, MA |
39 +| 14 | Leo Brown | 30 | Chef | Canada | leo.b@example.com | 555-8888 | 246 Oak St, Calgary, AB |
40 +| 15 | Mia Green | 33 | Musician | UK | mia.g@example.com | 555-9999 | 975 Pine St, Edinburgh, UK |
41 +| 16 | Noah Yellow | 29 | Doctor | Australia | noah.y@example.com | 555-0000 | 864 Birch St, Brisbane, QLD |
42 +| 17 | Olivia Blue | 35 | Engineer | New Zealand | olivia.b@example.com | 555-1212 | 753 Maple St, Hamilton, NZ |
43 +| 18 | Peter Black | 42 | Artist | Ireland | peter.b@example.com | 555-3434 | 912 Fir St, Limerick, IE |
44 +| 19 | Quinn White | 28 | Scientist | USA | quinn.w@example.com | 555-5656 | 159 Willow St, Seattle, WA |
45 +| 20 | Rachel Red | 31 | Teacher | Canada | rachel.r@example.com | 555-7878 | 357 Poplar St, Ottawa, ON |
46 +| 21 | Steve Green | 44 | Lawyer | UK | steve.g@example.com | 555-9090 | 753 Elm St, Birmingham, UK |
47 +| 22 | Tina Blue | 36 | Musician | Australia | tina.b@example.com | 555-1213 | 864 Cedar St, Perth, WA |
48 +| 23 | Umar Black | 39 | Chef | New Zealand | umar.b@example.com | 555-3435 | 975 Spruce St, Christchurch, NZ|
49 +| 24 | Victor Yellow | 43 | Engineer | Ireland | victor.y@example.com | 555-5657 | 246 Willow St, Galway, IE |
50 +| 25 | Wendy Orange | 27 | Artist | USA | wendy.o@example.com | 555-7879 | 135 Elm St, Denver, CO |
51 +| 26 | Xavier Green | 34 | Scientist | Canada | xavier.g@example.com | 555-9091 | 357 Oak St, Montreal, QC |
52 +| 27 | Yara Red | 41 | Teacher | UK | yara.r@example.com | 555-1214 | 975 Pine St, Leeds, UK |
53 +| 28 | Zack Blue | 30 | Lawyer | Australia | zack.b@example.com | 555-3436 | 135 Birch St, Adelaide, SA |
54 +| 29 | Amy White | 33 | Musician | New Zealand | amy.w@example.com | 555-5658 | 159 Maple St, Wellington, NZ |
55 +| 30 | Ben Black | 38 | Chef | Ireland | ben.b@example.com | 555-7870 | 246 Fir St, Waterford, IE |
56 +"""
57 +
58 +
59 +def get_generation_time(llm, sampling_params, prompts):
60 + # time the generation
61 + start_time = time.time()
62 + output = llm.generate(prompts, sampling_params=sampling_params)
63 + end_time = time.time()
64 + # print the output and generation time
65 + print(f"Output: {output[0].outputs[0].text}")
66 + print(f"Generation time: {end_time - start_time} seconds.")
67 +
68 +
69 +# set enable_prefix_caching=True to enable APC
70 +llm = LLM(
71 + model='lmsys/longchat-13b-16k',
72 + enable_prefix_caching=True
73 +)
74 +
75 +sampling_params = SamplingParams(temperature=0, max_tokens=100)
76 +
77 +# Querying the age of John Doe
78 +get_generation_time(
79 + llm,
80 + sampling_params,
81 + LONG_PROMPT + "Question: what is the age of John Doe? Your answer: The age of John Doe is ",
82 +)
83 +
84 +# Querying the age of Zack Blue
85 +# This query will be faster since vllm avoids computing the KV cache of LONG_PROMPT again.
86 +get_generation_time(
87 + llm,
88 + sampling_params,
89 + LONG_PROMPT + "Question: what is the age of Zack Blue? Your answer: The age of Zack Blue is ",
90 +)
91 +```
92 +
93 +## Example workloads
94 +
95 +We describe two example workloads, where APC can provide huge performance benefit:
96 +
97 +- Long document query, where the user repeatedly queries the same long document (e.g. software manual or annual report) with different queries. In this case, instead of processing the long document again and again, APC allows vLLM to process this long document *only once*, and all future requests can avoid recomputing this long document by reusing its KV cache. This allows vLLM to serve future requests with much higher throughput and much lower latency.
98 +- Multi-round conversation, where the user may chat with the application multiple times in the same chatting session. In this case, instead of processing the whole chatting history again and again, APC allows vLLM to reuse the processing results of the chat history across all future rounds of conversation, allowing vLLM to serve future requests with much higher throughput and much lower latency.
99 +
100 +## Limits
101 +
102 +APC in general does not reduce the performance of vLLM. With that being said, APC only reduces the time of processing the queries (the prefilling phase) and does not reduce the time of generating new tokens (the decoding phase). So APC does not bring performance gain when vLLM spends most of the time generating answers to the queries (e.g. when the length of the answer is long), or new queries do not share the same prefix with any of existing queries (so that the computation cannot be reused).
docs/source/automatic_prefix_caching/apc.rst deleted
-110
@@ -1,110 +0,0 @@
1 -.. _apc:
2 -
3 -Introduction
4 -============
5 -
6 -What is Automatic Prefix Caching
7 ---------------------------------
8 -
9 -Automatic Prefix Caching (APC in short) caches the KV cache of existing queries, so that a new query can directly reuse the KV cache if it shares the same prefix with one of the existing queries, allowing the new query to skip the computation of the shared part.
10 -
11 -
12 -.. note::
13 -
14 - Technical details on how vLLM implements APC are in the next page.
15 -
16 -
17 -
18 -Enabling APC in vLLM
19 ---------------------
20 -
21 -Set ``enable_prefix_caching=True`` in vLLM engine to enable APC. Here is an example:
22 -
23 -.. code-block:: python
24 -
25 - import time
26 - from vllm import LLM, SamplingParams
27 -
28 -
29 - # A prompt containing a large markdown table. The table is randomly generated by GPT-4.
30 - LONG_PROMPT = "You are a helpful assistant in recognizes the content of tables in markdown format. Here is a table as follows.\n# Table\n" + """
31 - | ID | Name | Age | Occupation | Country | Email | Phone Number | Address |
32 - |-----|---------------|-----|---------------|---------------|------------------------|----------------|------------------------------|
33 - | 1 | John Doe | 29 | Engineer | USA | john.doe@example.com | 555-1234 | 123 Elm St, Springfield, IL |
34 - | 2 | Jane Smith | 34 | Doctor | Canada | jane.smith@example.com | 555-5678 | 456 Oak St, Toronto, ON |
35 - | 3 | Alice Johnson | 27 | Teacher | UK | alice.j@example.com | 555-8765 | 789 Pine St, London, UK |
36 - | 4 | Bob Brown | 45 | Artist | Australia | bob.b@example.com | 555-4321 | 321 Maple St, Sydney, NSW |
37 - | 5 | Carol White | 31 | Scientist | New Zealand | carol.w@example.com | 555-6789 | 654 Birch St, Wellington, NZ |
38 - | 6 | Dave Green | 28 | Lawyer | Ireland | dave.g@example.com | 555-3456 | 987 Cedar St, Dublin, IE |
39 - | 7 | Emma Black | 40 | Musician | USA | emma.b@example.com | 555-1111 | 246 Ash St, New York, NY |
40 - | 8 | Frank Blue | 37 | Chef | Canada | frank.b@example.com | 555-2222 | 135 Spruce St, Vancouver, BC |
41 - | 9 | Grace Yellow | 50 | Engineer | UK | grace.y@example.com | 555-3333 | 864 Fir St, Manchester, UK |
42 - | 10 | Henry Violet | 32 | Artist | Australia | henry.v@example.com | 555-4444 | 753 Willow St, Melbourne, VIC|
43 - | 11 | Irene Orange | 26 | Scientist | New Zealand | irene.o@example.com | 555-5555 | 912 Poplar St, Auckland, NZ |
44 - | 12 | Jack Indigo | 38 | Teacher | Ireland | jack.i@example.com | 555-6666 | 159 Elm St, Cork, IE |
45 - | 13 | Karen Red | 41 | Lawyer | USA | karen.r@example.com | 555-7777 | 357 Cedar St, Boston, MA |
46 - | 14 | Leo Brown | 30 | Chef | Canada | leo.b@example.com | 555-8888 | 246 Oak St, Calgary, AB |
47 - | 15 | Mia Green | 33 | Musician | UK | mia.g@example.com | 555-9999 | 975 Pine St, Edinburgh, UK |
48 - | 16 | Noah Yellow | 29 | Doctor | Australia | noah.y@example.com | 555-0000 | 864 Birch St, Brisbane, QLD |
49 - | 17 | Olivia Blue | 35 | Engineer | New Zealand | olivia.b@example.com | 555-1212 | 753 Maple St, Hamilton, NZ |
50 - | 18 | Peter Black | 42 | Artist | Ireland | peter.b@example.com | 555-3434 | 912 Fir St, Limerick, IE |
51 - | 19 | Quinn White | 28 | Scientist | USA | quinn.w@example.com | 555-5656 | 159 Willow St, Seattle, WA |
52 - | 20 | Rachel Red | 31 | Teacher | Canada | rachel.r@example.com | 555-7878 | 357 Poplar St, Ottawa, ON |
53 - | 21 | Steve Green | 44 | Lawyer | UK | steve.g@example.com | 555-9090 | 753 Elm St, Birmingham, UK |
54 - | 22 | Tina Blue | 36 | Musician | Australia | tina.b@example.com | 555-1213 | 864 Cedar St, Perth, WA |
55 - | 23 | Umar Black | 39 | Chef | New Zealand | umar.b@example.com | 555-3435 | 975 Spruce St, Christchurch, NZ|
56 - | 24 | Victor Yellow | 43 | Engineer | Ireland | victor.y@example.com | 555-5657 | 246 Willow St, Galway, IE |
57 - | 25 | Wendy Orange | 27 | Artist | USA | wendy.o@example.com | 555-7879 | 135 Elm St, Denver, CO |
58 - | 26 | Xavier Green | 34 | Scientist | Canada | xavier.g@example.com | 555-9091 | 357 Oak St, Montreal, QC |
59 - | 27 | Yara Red | 41 | Teacher | UK | yara.r@example.com | 555-1214 | 975 Pine St, Leeds, UK |
60 - | 28 | Zack Blue | 30 | Lawyer | Australia | zack.b@example.com | 555-3436 | 135 Birch St, Adelaide, SA |
61 - | 29 | Amy White | 33 | Musician | New Zealand | amy.w@example.com | 555-5658 | 159 Maple St, Wellington, NZ |
62 - | 30 | Ben Black | 38 | Chef | Ireland | ben.b@example.com | 555-7870 | 246 Fir St, Waterford, IE |
63 - """
64 -
65 -
66 - def get_generation_time(llm, sampling_params, prompts):
67 - # time the generation
68 - start_time = time.time()
69 - output = llm.generate(prompts, sampling_params=sampling_params)
70 - end_time = time.time()
71 - # print the output and generation time
72 - print(f"Output: {output[0].outputs[0].text}")
73 - print(f"Generation time: {end_time - start_time} seconds.")
74 -
75 -
76 - # set enable_prefix_caching=True to enable APC
77 - llm = LLM(
78 - model='lmsys/longchat-13b-16k',
79 - enable_prefix_caching=True
80 - )
81 -
82 - sampling_params = SamplingParams(temperature=0, max_tokens=100)
83 -
84 - # Querying the age of John Doe
85 - get_generation_time(
86 - llm,
87 - sampling_params,
88 - LONG_PROMPT + "Question: what is the age of John Doe? Your answer: The age of John Doe is ",
89 - )
90 -
91 - # Querying the age of Zack Blue
92 - # This query will be faster since vllm avoids computing the KV cache of LONG_PROMPT again.
93 - get_generation_time(
94 - llm,
95 - sampling_params,
96 - LONG_PROMPT + "Question: what is the age of Zack Blue? Your answer: The age of Zack Blue is ",
97 - )
98 -
99 -Example workloads
100 ------------------
101 -
102 -We describe two example workloads, where APC can provide huge performance benefit:
103 -
104 -- Long document query, where the user repeatedly queries the same long document (e.g. software manual or annual report) with different queries. In this case, instead of processing the long document again and again, APC allows vLLM to process this long document *only once*, and all future requests can avoid recomputing this long document by reusing its KV cache. This allows vLLM to serve future requests with much higher throughput and much lower latency.
105 -- Multi-round conversation, where the user may chat with the application multiple times in the same chatting session. In this case, instead of processing the whole chatting history again and again, APC allows vLLM to reuse the processing results of the chat history across all future rounds of conversation, allowing vLLM to serve future requests with much higher throughput and much lower latency.
106 -
107 -
108 -Limits
109 -------
110 -APC in general does not reduce the performance of vLLM. With that being said, APC only reduces the time of processing the queries (the prefilling phase) and does not reduce the time of generating new tokens (the decoding phase). So APC does not bring performance gain when vLLM spends most of the time generating answers to the queries (e.g. when the length of the answer is long), or new queries do not share the same prefix with any of existing queries (so that the computation cannot be reused).
docs/source/community/meetups.md new
+15
@@ -0,0 +1,15 @@
1 +(meetups)=
2 +
3 +# vLLM Meetups
4 +
5 +We host regular meetups in San Francisco Bay Area every 2 months. We will share the project updates from the vLLM team and have guest speakers from the industry to share their experience and insights. Please find the materials of our previous meetups below:
6 +
7 +- [The seventh vLLM meetup](https://lu.ma/h0qvrajz), with Snowflake, November 14th 2024. [[Slides]](https://docs.google.com/presentation/d/1e3CxQBV3JsfGp30SwyvS3eM_tW-ghOhJ9PAJGK6KR54/edit?usp=sharing)
8 +- [The sixth vLLM meetup](https://lu.ma/87q3nvnh), with NVIDIA, September 9th 2024. [[Slides]](https://docs.google.com/presentation/d/1wrLGwytQfaOTd5wCGSPNhoaW3nq0E-9wqyP7ny93xRs/edit?usp=sharing)
9 +- [The fifth vLLM meetup](https://lu.ma/lp0gyjqr), with AWS, July 24th 2024. [[Slides]](https://docs.google.com/presentation/d/1RgUD8aCfcHocghoP3zmXzck9vX3RCI9yfUAB2Bbcl4Y/edit?usp=sharing)
10 +- [The fourth vLLM meetup](https://lu.ma/agivllm), with Cloudflare and BentoML, June 11th 2024. [[Slides]](https://docs.google.com/presentation/d/1iJ8o7V2bQEi0BFEljLTwc5G1S10_Rhv3beed5oB0NJ4/edit?usp=sharing)
11 +- [The third vLLM meetup](https://robloxandvllmmeetup2024.splashthat.com/), with Roblox, April 2nd 2024. [[Slides]](https://docs.google.com/presentation/d/1A--47JAK4BJ39t954HyTkvtfwn0fkqtsL8NGFuslReM/edit?usp=sharing)
12 +- [The second vLLM meetup](https://lu.ma/ygxbpzhl), with IBM Research, January 31st 2024. [[Slides]](https://docs.google.com/presentation/d/12mI2sKABnUw5RBWXDYY-HtHth4iMSNcEoQ10jDQbxgA/edit?usp=sharing) [[Video (vLLM Update)]](https://youtu.be/Y0C-DUvEnZQ) [[Video (IBM Research & torch.compile)]](https://youtu.be/m0dMtFLI-dg)
13 +- [The first vLLM meetup](https://lu.ma/first-vllm-meetup), with a16z, October 5th 2023. [[Slides]](https://docs.google.com/presentation/d/1QL-XPFXiFpDBh86DbEegFXBXFXjix4v032GhShbKf3s/edit?usp=sharing)
14 +
15 +We are always looking for speakers and sponsors at San Francisco Bay Area and potentially other locations. If you are interested in speaking or sponsoring, please contact us at [vllm-questions@lists.berkeley.edu](mailto:vllm-questions@lists.berkeley.edu).
docs/source/community/meetups.rst deleted
-16
@@ -1,16 +0,0 @@
1 -.. _meetups:
2 -
3 -vLLM Meetups
4 -============
5 -
6 -We host regular meetups in San Francisco Bay Area every 2 months. We will share the project updates from the vLLM team and have guest speakers from the industry to share their experience and insights. Please find the materials of our previous meetups below:
7 -
8 -- `The seventh vLLM meetup <https://lu.ma/h0qvrajz>`__, with Snowflake, November 14th 2024. `[Slides] <https://docs.google.com/presentation/d/1e3CxQBV3JsfGp30SwyvS3eM_tW-ghOhJ9PAJGK6KR54/edit?usp=sharing>`__
9 -- `The sixth vLLM meetup <https://lu.ma/87q3nvnh>`__, with NVIDIA, September 9th 2024. `[Slides] <https://docs.google.com/presentation/d/1wrLGwytQfaOTd5wCGSPNhoaW3nq0E-9wqyP7ny93xRs/edit?usp=sharing>`__
10 -- `The fifth vLLM meetup <https://lu.ma/lp0gyjqr>`__, with AWS, July 24th 2024. `[Slides] <https://docs.google.com/presentation/d/1RgUD8aCfcHocghoP3zmXzck9vX3RCI9yfUAB2Bbcl4Y/edit?usp=sharing>`__
11 -- `The fourth vLLM meetup <https://lu.ma/agivllm>`__, with Cloudflare and BentoML, June 11th 2024. `[Slides] <https://docs.google.com/presentation/d/1iJ8o7V2bQEi0BFEljLTwc5G1S10_Rhv3beed5oB0NJ4/edit?usp=sharing>`__
12 -- `The third vLLM meetup <https://robloxandvllmmeetup2024.splashthat.com/>`__, with Roblox, April 2nd 2024. `[Slides] <https://docs.google.com/presentation/d/1A--47JAK4BJ39t954HyTkvtfwn0fkqtsL8NGFuslReM/edit?usp=sharing>`__
13 -- `The second vLLM meetup <https://lu.ma/ygxbpzhl>`__, with IBM Research, January 31st 2024. `[Slides] <https://docs.google.com/presentation/d/12mI2sKABnUw5RBWXDYY-HtHth4iMSNcEoQ10jDQbxgA/edit?usp=sharing>`__ `[Video (vLLM Update)] <https://youtu.be/Y0C-DUvEnZQ>`__ `[Video (IBM Research & torch.compile)] <https://youtu.be/m0dMtFLI-dg>`__
14 -- `The first vLLM meetup <https://lu.ma/first-vllm-meetup>`__, with a16z, October 5th 2023. `[Slides] <https://docs.google.com/presentation/d/1QL-XPFXiFpDBh86DbEegFXBXFXjix4v032GhShbKf3s/edit?usp=sharing>`__
15 -
16 -We are always looking for speakers and sponsors at San Francisco Bay Area and potentially other locations. If you are interested in speaking or sponsoring, please contact us at `vllm-questions@lists.berkeley.edu <mailto:vllm-questions@lists.berkeley.edu>`__.
docs/source/conf.py
+1 -1
@@ -51,7 +51,7 @@ templates_path = ['_templates']
51 # List of patterns, relative to source directory, that match files and
52 # directories to ignore when looking for source files.
53 # This pattern also affects html_static_path and html_extra_path.
54 -exclude_patterns: List[str] = ["**/*.template.rst"]
54 +exclude_patterns: List[str] = ["**/*.template.md"]
55
56 # Exclude the prompt "$" when copying code
57 copybutton_prompt_text = r"\$ "
docs/source/contributing/dockerfile/dockerfile.md new
+50
@@ -0,0 +1,50 @@
1 +# Dockerfile
2 +
3 +See [here](https://github.com/vllm-project/vllm/blob/main/Dockerfile) for the main Dockerfile to construct
4 +the image for running an OpenAI compatible server with vLLM. More information about deploying with Docker can be found [here](https://docs.vllm.ai/en/stable/serving/deploying_with_docker.html).
5 +
6 +Below is a visual representation of the multi-stage Dockerfile. The build graph contains the following nodes:
7 +
8 +- All build stages
9 +- The default build target (highlighted in grey)
10 +- External images (with dashed borders)
11 +
12 +The edges of the build graph represent:
13 +
14 +- FROM ... dependencies (with a solid line and a full arrow head)
15 +
16 +- COPY --from=... dependencies (with a dashed line and an empty arrow head)
17 +
18 +- RUN --mount=(.\*)from=... dependencies (with a dotted line and an empty diamond arrow head)
19 +
20 + > ```{figure} ../../assets/dev/dockerfile-stages-dependency.png
21 + > :align: center
22 + > :alt: query
23 + > :width: 100%
24 + > ```
25 + >
26 + > Made using: <https://github.com/patrickhoefler/dockerfilegraph>
27 + >
28 + > Commands to regenerate the build graph (make sure to run it **from the \`root\` directory of the vLLM repository** where the dockerfile is present):
29 + >
30 + > ```bash
31 + > dockerfilegraph -o png --legend --dpi 200 --max-label-length 50 --filename Dockerfile
32 + > ```
33 + >
34 + > or in case you want to run it directly with the docker image:
35 + >
36 + > ```bash
37 + > docker run \
38 + > --rm \
39 + > --user "$(id -u):$(id -g)" \
40 + > --workdir /workspace \
41 + > --volume "$(pwd)":/workspace \
42 + > ghcr.io/patrickhoefler/dockerfilegraph:alpine \
43 + > --output png \
44 + > --dpi 200 \
45 + > --max-label-length 50 \
46 + > --filename Dockerfile \
47 + > --legend
48 + > ```
49 + >
50 + > (To run it for a different file, you can pass in a different argument to the flag `--filename`.)
docs/source/contributing/dockerfile/dockerfile.rst deleted
-50
@@ -1,50 +0,0 @@
1 -Dockerfile
2 -====================
3 -
4 -See `here <https://github.com/vllm-project/vllm/blob/main/Dockerfile>`__ for the main Dockerfile to construct
5 -the image for running an OpenAI compatible server with vLLM. More information about deploying with Docker can be found `here <https://docs.vllm.ai/en/stable/serving/deploying_with_docker.html>`__.
6 -
7 -Below is a visual representation of the multi-stage Dockerfile. The build graph contains the following nodes:
8 -
9 -- All build stages
10 -- The default build target (highlighted in grey)
11 -- External images (with dashed borders)
12 -
13 -The edges of the build graph represent:
14 -
15 -- FROM ... dependencies (with a solid line and a full arrow head)
16 -- COPY --from=... dependencies (with a dashed line and an empty arrow head)
17 -- RUN --mount=(.*)from=... dependencies (with a dotted line and an empty diamond arrow head)
18 -
19 - .. figure:: ../../assets/dev/dockerfile-stages-dependency.png
20 - :alt: query
21 - :width: 100%
22 - :align: center
23 -
24 - Made using: https://github.com/patrickhoefler/dockerfilegraph
25 -
26 - Commands to regenerate the build graph (make sure to run it **from the `root` directory of the vLLM repository** where the dockerfile is present):
27 -
28 - .. code:: bash
29 -
30 - dockerfilegraph -o png --legend --dpi 200 --max-label-length 50 --filename Dockerfile
31 -
32 - or in case you want to run it directly with the docker image:
33 -
34 - .. code:: bash
35 -
36 - docker run \
37 - --rm \
38 - --user "$(id -u):$(id -g)" \
39 - --workdir /workspace \
40 - --volume "$(pwd)":/workspace \
41 - ghcr.io/patrickhoefler/dockerfilegraph:alpine \
42 - --output png \
43 - --dpi 200 \
44 - --max-label-length 50 \
45 - --filename Dockerfile \
46 - --legend
47 -
48 - (To run it for a different file, you can pass in a different argument to the flag `--filename`.)
49 -
50 -
\ No newline at end of file
docs/source/contributing/overview.md renamed
+62 -75
@@ -1,5 +1,4 @@
1 -Contributing to vLLM
2 -=====================
1 +# Contributing to vLLM
2
3 Thank you for your interest in contributing to vLLM! Our community is open to everyone and welcomes all kinds of contributions, no matter how small or large. There are several ways you can contribute to the project:
4
@@ -12,132 +11,121 @@ We also believe in the power of community support; thus, answering queries, offe
11
12 Finally, one of the most impactful ways to support us is by raising awareness about vLLM. Talk about it in your blog posts and highlight how it's driving your incredible projects. Express your support on social media if you're using vLLM, or simply offer your appreciation by starring our repository!
13
15 -License
16 --------
14 +## License
15
18 -See `LICENSE <https://github.com/vllm-project/vllm/tree/main/LICENSE>`_.
16 +See [LICENSE](https://github.com/vllm-project/vllm/tree/main/LICENSE).
17
20 -Developing
21 -----------
18 +## Developing
19
23 -Depending on the kind of development you'd like to do (e.g. Python, CUDA), you can choose to build vLLM with or without compilation. Check out the `building from source <https://docs.vllm.ai/en/latest/getting_started/installation.html#build-from-source>`_ documentation for details.
20 +Depending on the kind of development you'd like to do (e.g. Python, CUDA), you can choose to build vLLM with or without compilation. Check out the [building from source](https://docs.vllm.ai/en/latest/getting_started/installation.html#build-from-source) documentation for details.
21
25 -Testing
26 --------
22 +## Testing
23
28 -.. code-block:: bash
24 +```bash
25 +pip install -r requirements-dev.txt
26
30 - pip install -r requirements-dev.txt
27 +# linting and formatting
28 +bash format.sh
29 +# Static type checking
30 +mypy
31 +# Unit tests
32 +pytest tests/
33 +```
34
32 - # linting and formatting
33 - bash format.sh
34 - # Static type checking
35 - mypy
36 - # Unit tests
37 - pytest tests/
35 +```{note}
36 +Currently, the repository does not pass the `mypy` tests.
37 +```
38
39 -.. note:: Currently, the repository does not pass the ``mypy`` tests.
39 +# Contribution Guidelines
40
41 -Contribution Guidelines
42 -=======================
41 +## Issues
42
44 -Issues
45 -------
43 +If you encounter a bug or have a feature request, please [search existing issues](https://github.com/vllm-project/vllm/issues?q=is%3Aissue) first to see if it has already been reported. If not, please [file a new issue](https://github.com/vllm-project/vllm/issues/new/choose), providing as much relevant information as possible.
44
47 -If you encounter a bug or have a feature request, please `search existing issues <https://github.com/vllm-project/vllm/issues?q=is%3Aissue>`_ first to see if it has already been reported. If not, please `file a new issue <https://github.com/vllm-project/vllm/issues/new/choose>`_, providing as much relevant information as possible.
45 +```{important}
46 +If you discover a security vulnerability, please follow the instructions [here](https://github.com/vllm-project/vllm/tree/main/SECURITY.md#reporting-a-vulnerability).
47 +```
48
49 -.. important::
50 - If you discover a security vulnerability, please follow the instructions `here <https://github.com/vllm-project/vllm/tree/main/SECURITY.md#reporting-a-vulnerability>`_.
51 -
52 -Pull Requests & Code Reviews
53 -----------------------------
49 +## Pull Requests & Code Reviews
50
51 Thank you for your contribution to vLLM! Before submitting the pull request,
52 please ensure the PR meets the following criteria. This helps vLLM maintain the
53 code quality and improve the efficiency of the review process.
54
59 -DCO and Signed-off-by
60 -^^^^^^^^^^^^^^^^^^^^^
55 +### DCO and Signed-off-by
56
62 -When contributing changes to this project, you must agree to the `DCO <https://github.com/vllm-project/vllm/tree/main/DCO>`_.
63 -Commits must include a ``Signed-off-by:`` header which certifies agreement with
64 -the terms of the `DCO <https://github.com/vllm-project/vllm/tree/main/DCO>`_.
57 +When contributing changes to this project, you must agree to the [DCO](https://github.com/vllm-project/vllm/tree/main/DCO).
58 +Commits must include a `Signed-off-by:` header which certifies agreement with
59 +the terms of the [DCO](https://github.com/vllm-project/vllm/tree/main/DCO).
60
66 -Using ``-s`` with ``git commit`` will automatically add this header.
61 +Using `-s` with `git commit` will automatically add this header.
62
68 -PR Title and Classification
69 -^^^^^^^^^^^^^^^^^^^^^^^^^^^
63 +### PR Title and Classification
64
65 Only specific types of PRs will be reviewed. The PR title is prefixed
66 appropriately to indicate the type of change. Please use one of the following:
67
74 -- ``[Bugfix]`` for bug fixes.
75 -- ``[CI/Build]`` for build or continuous integration improvements.
76 -- ``[Doc]`` for documentation fixes and improvements.
77 -- ``[Model]`` for adding a new model or improving an existing model. Model name
68 +- `[Bugfix]` for bug fixes.
69 +- `[CI/Build]` for build or continuous integration improvements.
70 +- `[Doc]` for documentation fixes and improvements.
71 +- `[Model]` for adding a new model or improving an existing model. Model name
72 should appear in the title.
79 -- ``[Frontend]`` For changes on the vLLM frontend (e.g., OpenAI API server,
80 - ``LLM`` class, etc.)
81 -- ``[Kernel]`` for changes affecting CUDA kernels or other compute kernels.
82 -- ``[Core]`` for changes in the core vLLM logic (e.g., ``LLMEngine``,
83 - ``AsyncLLMEngine``, ``Scheduler``, etc.)
84 -- ``[Hardware][Vendor]`` for hardware-specific changes. Vendor name should
85 - appear in the prefix (e.g., ``[Hardware][AMD]``).
86 -- ``[Misc]`` for PRs that do not fit the above categories. Please use this
73 +- `[Frontend]` For changes on the vLLM frontend (e.g., OpenAI API server,
74 + `LLM` class, etc.)
75 +- `[Kernel]` for changes affecting CUDA kernels or other compute kernels.
76 +- `[Core]` for changes in the core vLLM logic (e.g., `LLMEngine`,
77 + `AsyncLLMEngine`, `Scheduler`, etc.)
78 +- `[Hardware][Vendor]` for hardware-specific changes. Vendor name should
79 + appear in the prefix (e.g., `[Hardware][AMD]`).
80 +- `[Misc]` for PRs that do not fit the above categories. Please use this
81 sparingly.
82
89 -.. note::
90 - If the PR spans more than one category, please include all relevant prefixes.
83 +```{note}
84 +If the PR spans more than one category, please include all relevant prefixes.
85 +```
86
92 -Code Quality
93 -^^^^^^^^^^^^
87 +### Code Quality
88
89 The PR needs to meet the following code quality standards:
90
97 -- We adhere to `Google Python style guide
98 - <https://google.github.io/styleguide/pyguide.html>`_ and `Google C++ style guide
99 - <https://google.github.io/styleguide/cppguide.html>`_.
100 -- Pass all linter checks. Please use `format.sh
101 - <https://github.com/vllm-project/vllm/blob/main/format.sh>`_ to format your
91 +- We adhere to [Google Python style guide](https://google.github.io/styleguide/pyguide.html) and [Google C++ style guide](https://google.github.io/styleguide/cppguide.html).
92 +- Pass all linter checks. Please use [format.sh](https://github.com/vllm-project/vllm/blob/main/format.sh) to format your
93 code.
94 - The code needs to be well-documented to ensure future contributors can easily
95 understand the code.
96 - Include sufficient tests to ensure the project stays correct and robust. This
97 includes both unit tests and integration tests.
107 -- Please add documentation to ``docs/source/`` if the PR modifies the
98 +- Please add documentation to `docs/source/` if the PR modifies the
99 user-facing behaviors of vLLM. It helps vLLM users understand and utilize the
100 new features or changes.
101
111 -Adding or Changing Kernels
112 -^^^^^^^^^^^^^^^^^^^^^^^^^^
102 +### Adding or Changing Kernels
103
104 Each custom kernel needs a schema and one or more implementations to be registered with PyTorch.
105
106 - Make sure custom ops are registered following PyTorch guidelines:
117 - `Custom C++ and CUDA Operators <https://pytorch.org/tutorials/advanced/cpp_custom_ops.html#cpp-custom-ops-tutorial>`_
118 - and `The Custom Operators Manual <https://docs.google.com/document/d/1_W62p8WJOQQUzPsJYa7s701JXt0qf2OfLub2sbkHOaU>`_.
119 -- Custom operations that return ``Tensors`` require meta-functions.
107 + [Custom C++ and CUDA Operators](https://pytorch.org/tutorials/advanced/cpp_custom_ops.html#cpp-custom-ops-tutorial)
108 + and [The Custom Operators Manual](https://docs.google.com/document/d/1_W62p8WJOQQUzPsJYa7s701JXt0qf2OfLub2sbkHOaU).
109 +- Custom operations that return `Tensors` require meta-functions.
110 Meta-functions should be implemented and registered in Python so that dynamic
111 dims can be handled automatically. See above documents for a description of
112 meta-functions.
123 -- Use `torch.library.opcheck() <https://pytorch.org/docs/stable/library.html#torch.library.opcheck>`_
113 +- Use [torch.library.opcheck()](https://pytorch.org/docs/stable/library.html#torch.library.opcheck)
114 to test the function registration and meta-function for any registered ops.
125 - See ``tests/kernels`` for examples.
115 + See `tests/kernels` for examples.
116 - When changing the C++ signature of an existing op, the schema must be updated
117 to reflect the changes.
118 - If a new custom type is needed, see the following document:
129 - `Custom Class Support in PT2 <https://docs.google.com/document/d/18fBMPuOJ0fY5ZQ6YyrHUppw9FA332CpNtgB6SOIgyuA>`_.
119 + [Custom Class Support in PT2](https://docs.google.com/document/d/18fBMPuOJ0fY5ZQ6YyrHUppw9FA332CpNtgB6SOIgyuA).
120
131 -Notes for Large Changes
132 -^^^^^^^^^^^^^^^^^^^^^^^
121 +### Notes for Large Changes
122
123 Please keep the changes as concise as possible. For major architectural changes
124 (>500 LOC excluding kernel/data/config/test), we would expect a GitHub issue
125 (RFC) discussing the technical design and justification. Otherwise, we will tag
137 -it with ``rfc-required`` and might not go through the PR.
126 +it with `rfc-required` and might not go through the PR.
127
139 -What to Expect for the Reviews
140 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
128 +### What to Expect for the Reviews
129
130 The goal of the vLLM team is to be a *transparent reviewing machine*. We would
131 like to make the review process transparent and efficient and make sure no
@@ -150,15 +138,14 @@ review process:
138 - After the PR is assigned, the reviewer will provide status updates every 2-3
139 days. If the PR is not reviewed within 7 days, please feel free to ping the
140 reviewer or the vLLM team.
153 -- After the review, the reviewer will put an ``action-required`` label on the PR
141 +- After the review, the reviewer will put an `action-required` label on the PR
142 if there are changes required. The contributor should address the comments and
143 ping the reviewer to re-review the PR.
144 - Please respond to all comments within a reasonable time frame. If a comment
145 isn't clear or you disagree with a suggestion, feel free to ask for
146 clarification or discuss the suggestion.
147
160 -Thank You
161 ----------
148 +## Thank You
149
150 Finally, thank you for taking the time to read these guidelines and for your interest in contributing to vLLM.
151 All of your contributions help make vLLM a great tool and community for everyone!
docs/source/contributing/profiling/profiling_index.md new
+41
@@ -0,0 +1,41 @@
1 +# Profiling vLLM
2 +
3 +We support tracing vLLM workers using the `torch.profiler` module. You can enable tracing by setting the `VLLM_TORCH_PROFILER_DIR` environment variable to the directory where you want to save the traces: `VLLM_TORCH_PROFILER_DIR=/mnt/traces/`
4 +
5 +The OpenAI server also needs to be started with the `VLLM_TORCH_PROFILER_DIR` environment variable set.
6 +
7 +When using `benchmarks/benchmark_serving.py`, you can enable profiling by passing the `--profile` flag.
8 +
9 +```{warning}
10 +Only enable profiling in a development environment.
11 +```
12 +
13 +Traces can be visualized using <https://ui.perfetto.dev/>.
14 +
15 +```{tip}
16 +Only send a few requests through vLLM when profiling, as the traces can get quite large. Also, no need to untar the traces, they can be viewed directly.
17 +```
18 +
19 +```{tip}
20 +To stop the profiler - it flushes out all the profile trace files to the directory. This takes time, for example for about 100 requests worth of data for a llama 70b, it takes about 10 minutes to flush out on a H100.
21 +Set the env variable VLLM_RPC_TIMEOUT to a big number before you start the server. Say something like 30 minutes.
22 +`export VLLM_RPC_TIMEOUT=1800000`
23 +```
24 +
25 +## Example commands and usage:
26 +
27 +### Offline Inference:
28 +
29 +Refer to [examples/offline_inference_with_profiler.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_with_profiler.py) for an example.
30 +
31 +### OpenAI Server:
32 +
33 +```bash
34 +VLLM_TORCH_PROFILER_DIR=./vllm_profile python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B
35 +```
36 +
37 +benchmark_serving.py:
38 +
39 +```bash
40 +python benchmarks/benchmark_serving.py --backend vllm --model meta-llama/Meta-Llama-3-70B --dataset-name sharegpt --dataset-path sharegpt.json --profile --num-prompts 2
41 +```
docs/source/contributing/profiling/profiling_index.rst deleted
-48
@@ -1,48 +0,0 @@
1 -==============
2 -Profiling vLLM
3 -==============
4 -
5 -We support tracing vLLM workers using the ``torch.profiler`` module. You can enable tracing by setting the ``VLLM_TORCH_PROFILER_DIR`` environment variable to the directory where you want to save the traces: ``VLLM_TORCH_PROFILER_DIR=/mnt/traces/``
6 -
7 -The OpenAI server also needs to be started with the ``VLLM_TORCH_PROFILER_DIR`` environment variable set.
8 -
9 -When using ``benchmarks/benchmark_serving.py``, you can enable profiling by passing the ``--profile`` flag.
10 -
11 -.. warning::
12 -
13 - Only enable profiling in a development environment.
14 -
15 -
16 -Traces can be visualized using https://ui.perfetto.dev/.
17 -
18 -.. tip::
19 -
20 - Only send a few requests through vLLM when profiling, as the traces can get quite large. Also, no need to untar the traces, they can be viewed directly.
21 -
22 -.. tip::
23 -
24 - To stop the profiler - it flushes out all the profile trace files to the directory. This takes time, for example for about 100 requests worth of data for a llama 70b, it takes about 10 minutes to flush out on a H100.
25 - Set the env variable VLLM_RPC_TIMEOUT to a big number before you start the server. Say something like 30 minutes.
26 - ``export VLLM_RPC_TIMEOUT=1800000``
27 -
28 -Example commands and usage:
29 -===========================
30 -
31 -Offline Inference:
32 -------------------
33 -
34 -Refer to `examples/offline_inference_with_profiler.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_with_profiler.py>`_ for an example.
35 -
36 -
37 -OpenAI Server:
38 ---------------
39 -
40 -.. code-block:: bash
41 -
42 - VLLM_TORCH_PROFILER_DIR=./vllm_profile python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B
43 -
44 -benchmark_serving.py:
45 -
46 -.. code-block:: bash
47 -
48 - python benchmarks/benchmark_serving.py --backend vllm --model meta-llama/Meta-Llama-3-70B --dataset-name sharegpt --dataset-path sharegpt.json --profile --num-prompts 2
\ No newline at end of file
docs/source/design/arch_overview.md renamed
+117 -133
@@ -1,25 +1,24 @@
1 -.. _arch_overview:
1 +(arch-overview)=
2
3 -Architecture Overview
4 -======================
3 +# Architecture Overview
4
5 This document provides an overview of the vLLM architecture.
6
8 -.. contents:: Table of Contents
9 - :local:
10 - :depth: 2
7 +```{contents} Table of Contents
8 +:depth: 2
9 +:local: true
10 +```
11
12 -Entrypoints
13 ------------
12 +## Entrypoints
13
14 vLLM provides a number of entrypoints for interacting with the system. The
15 following diagram shows the relationship between them.
16
18 -.. image:: /assets/design/arch_overview/entrypoints.excalidraw.png
19 - :alt: Entrypoints Diagram
17 +```{image} /assets/design/arch_overview/entrypoints.excalidraw.png
18 +:alt: Entrypoints Diagram
19 +```
20
21 -LLM Class
22 -^^^^^^^^^
21 +### LLM Class
22
23 The LLM class provides the primary Python interface for doing offline inference,
24 which is interacting with a model without using a separate model inference
@@ -27,75 +26,70 @@ server.
26
27 Here is a sample of `LLM` class usage:
28
30 -.. code-block:: python
29 +```python
30 +from vllm import LLM, SamplingParams
31
32 - from vllm import LLM, SamplingParams
32 +# Define a list of input prompts
33 +prompts = [
34 + "Hello, my name is",
35 + "The capital of France is",
36 + "The largest ocean is",
37 +]
38
34 - # Define a list of input prompts
35 - prompts = [
36 - "Hello, my name is",
37 - "The capital of France is",
38 - "The largest ocean is",
39 - ]
39 +# Define sampling parameters
40 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
41
41 - # Define sampling parameters
42 - sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
42 +# Initialize the LLM engine with the OPT-125M model
43 +llm = LLM(model="facebook/opt-125m")
44
44 - # Initialize the LLM engine with the OPT-125M model
45 - llm = LLM(model="facebook/opt-125m")
45 +# Generate outputs for the input prompts
46 +outputs = llm.generate(prompts, sampling_params)
47
47 - # Generate outputs for the input prompts
48 - outputs = llm.generate(prompts, sampling_params)
48 +# Print the generated outputs
49 +for output in outputs:
50 + prompt = output.prompt
51 + generated_text = output.outputs[0].text
52 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
53 +```
54
50 - # Print the generated outputs
51 - for output in outputs:
52 - prompt = output.prompt
53 - generated_text = output.outputs[0].text
54 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
55 -
56 -More API details can be found in the :doc:`Offline Inference
55 +More API details can be found in the {doc}`Offline Inference
56 </dev/offline_inference/offline_index>` section of the API docs.
57
59 -The code for the `LLM` class can be found in `vllm/entrypoints/llm.py
60 -<https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/llm.py>`_.
58 +The code for the `LLM` class can be found in [vllm/entrypoints/llm.py](https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/llm.py).
59
62 -OpenAI-compatible API server
63 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^
60 +### OpenAI-compatible API server
61
62 The second primary interface to vLLM is via its OpenAI-compatible API server.
63 This server can be started using the `vllm serve` command.
64
68 -.. code-block:: bash
69 -
70 - vllm serve <model>
65 +```bash
66 +vllm serve <model>
67 +```
68
72 -The code for the `vllm` CLI can be found in `vllm/scripts.py
73 -<https://github.com/vllm-project/vllm/blob/main/vllm/scripts.py>`_.
69 +The code for the `vllm` CLI can be found in [vllm/scripts.py](https://github.com/vllm-project/vllm/blob/main/vllm/scripts.py).
70
71 Sometimes you may see the API server entrypoint used directly instead of via the
72 `vllm` CLI command. For example:
73
78 -.. code-block:: bash
79 -
80 - python -m vllm.entrypoints.openai.api_server --model <model>
74 +```bash
75 +python -m vllm.entrypoints.openai.api_server --model <model>
76 +```
77
82 -That code can be found in `vllm/entrypoints/openai/api_server.py
83 -<https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/api_server.py>`_.
78 +That code can be found in [vllm/entrypoints/openai/api_server.py](https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/api_server.py).
79
85 -More details on the API server can be found in the :doc:`OpenAI Compatible
80 +More details on the API server can be found in the {doc}`OpenAI Compatible
81 Server </serving/openai_compatible_server>` document.
82
88 -LLM Engine
89 -----------
83 +## LLM Engine
84
85 The `LLMEngine` and `AsyncLLMEngine` classes are central to the functioning of
86 the vLLM system, handling model inference and asynchronous request processing.
87
94 -.. image:: /assets/design/arch_overview/llm_engine.excalidraw.png
95 - :alt: LLMEngine Diagram
88 +```{image} /assets/design/arch_overview/llm_engine.excalidraw.png
89 +:alt: LLMEngine Diagram
90 +```
91
97 -LLMEngine
98 -^^^^^^^^^
92 +### LLMEngine
93
94 The `LLMEngine` class is the core component of the vLLM engine. It is
95 responsible for receiving requests from clients and generating outputs from the
@@ -105,21 +99,15 @@ processing.
99
100 - **Input Processing**: Handles tokenization of input text using the specified
101 tokenizer.
108 -
102 - **Scheduling**: Chooses which requests are processed in each step.
110 -
103 - **Model Execution**: Manages the execution of the language model, including
104 distributed execution across multiple GPUs.
113 -
105 - **Output Processing**: Processes the outputs generated by the model, decoding the
106 token IDs from a language model into human-readable text.
107
117 -The code for `LLMEngine` can be found in `vllm/engine/llm_engine.py`_.
118 -
119 -.. _vllm/engine/llm_engine.py: https://github.com/vllm-project/vllm/tree/main/vllm/engine/llm_engine.py
108 +The code for `LLMEngine` can be found in [vllm/engine/llm_engine.py].
109
121 -AsyncLLMEngine
122 -^^^^^^^^^^^^^^
110 +### AsyncLLMEngine
111
112 The `AsyncLLMEngine` class is an asynchronous wrapper for the `LLMEngine` class.
113 It uses `asyncio` to create a background loop that continuously processes
@@ -128,54 +116,46 @@ can handle multiple concurrent requests and stream outputs to clients.
116
117 The OpenAI-compatible API server uses the `AsyncLLMEngine`. There is also a demo
118 API server that serves as a simpler example in
131 -`vllm/entrypoints/api_server.py`_.
132 -
133 -.. _vllm/entrypoints/api_server.py: https://github.com/vllm-project/vllm/tree/main/vllm/entrypoints/api_server.py
119 +[vllm/entrypoints/api_server.py].
120
135 -The code for `AsyncLLMEngine` can be found in `vllm/engine/async_llm_engine.py`_.
121 +The code for `AsyncLLMEngine` can be found in [vllm/engine/async_llm_engine.py].
122
137 -.. _vllm/engine/async_llm_engine.py: https://github.com/vllm-project/vllm/tree/main/vllm/engine/async_llm_engine.py
138 -
139 -Worker
140 -------
123 +## Worker
124
125 A worker is a process that runs the model inference. vLLM follows the common
126 practice of using one process to control one accelerator device, such as GPUs.
127 For example, if we use tensor parallelism of size 2 and pipeline parallelism of
128 size 2, we will have 4 workers in total. Workers are identified by their
146 -``rank`` and ``local_rank``. ``rank`` is used for global orchestration, while
147 -``local_rank`` is mainly used for assigning the accelerator device and accessing
129 +`rank` and `local_rank`. `rank` is used for global orchestration, while
130 +`local_rank` is mainly used for assigning the accelerator device and accessing
131 local resources such as the file system and shared memory.
132
150 -Model Runner
151 -------------
133 +## Model Runner
134
135 Every worker has one model runner object, responsible for loading and running
136 the model. Much of the model execution logic resides here, such as preparing
137 input tensors and capturing cudagraphs.
138
157 -Model
158 ------
139 +## Model
140
141 Every model runner object has one model object, which is the actual
161 -``torch.nn.Module`` instance. See :ref:`huggingface_integration` for how various
142 +`torch.nn.Module` instance. See [huggingface_integration](#huggingface-integration) for how various
143 configurations affect the class we ultimately get.
144
164 -Class Hierarchy
165 ----------------
145 +## Class Hierarchy
146
147 The following figure shows the class hierarchy of vLLM:
148
169 - .. figure:: /assets/design/hierarchy.png
170 - :alt: query
171 - :width: 100%
172 - :align: center
149 +> ```{figure} /assets/design/hierarchy.png
150 +> :align: center
151 +> :alt: query
152 +> :width: 100%
153 +> ```
154
155 There are several important design choices behind this class hierarchy:
156
176 -1. **Extensibility**: All classes in the hierarchy accept a configuration object
177 -containing all the necessary information. The `VllmConfig
178 -<https://github.com/vllm-project/vllm/blob/d1c6799b8870e513bf4f2305cbf6cda9fc3d773b/vllm/config.py#L2036>`__
157 +1\. **Extensibility**: All classes in the hierarchy accept a configuration object
158 +containing all the necessary information. The [VllmConfig](https://github.com/vllm-project/vllm/blob/d1c6799b8870e513bf4f2305cbf6cda9fc3d773b/vllm/config.py#L2036)
159 class is the main configuration object that is passed around. The class
160 hierarchy is quite deep, and every class needs to read the configuration it is
161 interested in. By encapsulating all configurations in one object, we can easily
@@ -188,7 +168,7 @@ the `VllmConfig` class, and the model runner can access it directly. We don't
168 need to change the constructor of the engine, worker, or model class to pass the
169 new configuration option.
170
191 -2. **Uniformity**: The model runner needs a unified interface to create and
171 +2\. **Uniformity**: The model runner needs a unified interface to create and
172 initialize the model. vLLM supports more than 50 types of popular open-source
173 models. Each model has its own initialization logic. If the constructor
174 signature varies with models, the model runner does not know how to call the
@@ -200,46 +180,46 @@ of a vision model and a language model. By making the constructor uniform, we
180 can easily create a vision model and a language model and compose them into a
181 vision-language model.
182
203 -.. note::
204 -
205 - To support this change, all vLLM models' signatures have been updated to:
206 -
207 - .. code-block:: python
208 -
209 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
210 -
211 - To avoid accidentally passing incorrect arguments, the constructor is now keyword-only. This ensures that the constructor will raise an error if old configurations are passed. vLLM developers have already made this change for all models within vLLM. For out-of-tree registered models, developers need to update their models, for example by adding shim code to adapt the old constructor signature to the new one:
212 -
213 - .. code-block:: python
214 -
215 - class MyOldModel(nn.Module):
216 - def __init__(
217 - self,
218 - config,
219 - cache_config: Optional[CacheConfig] = None,
220 - quant_config: Optional[QuantizationConfig] = None,
221 - lora_config: Optional[LoRAConfig] = None,
222 - prefix: str = "",
223 - ) -> None:
224 - ...
225 -
226 - from vllm.config import VllmConfig
227 - class MyNewModel(MyOldModel):
228 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
229 - config = vllm_config.model_config.hf_config
230 - cache_config = vllm_config.cache_config
231 - quant_config = vllm_config.quant_config
232 - lora_config = vllm_config.lora_config
233 - super().__init__(config, cache_config, quant_config, lora_config, prefix)
234 -
235 - if __version__ >= "0.6.4":
236 - MyModel = MyNewModel
237 - else:
238 - MyModel = MyOldModel
239 -
240 - This way, the model can work with both old and new versions of vLLM.
241 -
242 -3. **Sharding and Quantization at Initialization**: Certain features require
183 +````{note}
184 +To support this change, all vLLM models' signatures have been updated to:
185 +
186 +```python
187 +def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
188 +```
189 +
190 +To avoid accidentally passing incorrect arguments, the constructor is now keyword-only. This ensures that the constructor will raise an error if old configurations are passed. vLLM developers have already made this change for all models within vLLM. For out-of-tree registered models, developers need to update their models, for example by adding shim code to adapt the old constructor signature to the new one:
191 +
192 +```python
193 +class MyOldModel(nn.Module):
194 + def __init__(
195 + self,
196 + config,
197 + cache_config: Optional[CacheConfig] = None,
198 + quant_config: Optional[QuantizationConfig] = None,
199 + lora_config: Optional[LoRAConfig] = None,
200 + prefix: str = "",
201 + ) -> None:
202 + ...
203 +
204 +from vllm.config import VllmConfig
205 +class MyNewModel(MyOldModel):
206 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
207 + config = vllm_config.model_config.hf_config
208 + cache_config = vllm_config.cache_config
209 + quant_config = vllm_config.quant_config
210 + lora_config = vllm_config.lora_config
211 + super().__init__(config, cache_config, quant_config, lora_config, prefix)
212 +
213 +if __version__ >= "0.6.4":
214 + MyModel = MyNewModel
215 +else:
216 + MyModel = MyOldModel
217 +```
218 +
219 +This way, the model can work with both old and new versions of vLLM.
220 +````
221 +
222 +3\. **Sharding and Quantization at Initialization**: Certain features require
223 changing the model weights. For example, tensor parallelism needs to shard the
224 model weights, and quantization needs to quantize the model weights. There are
225 two possible ways to implement this feature. One way is to change the model
@@ -252,23 +232,27 @@ initialized, we need to load the full 810GB weights to every GPU and then shard
232 the weights, leading to a huge memory overhead. Instead, if we shard the weights
233 during the model initialization, every layer will only create a shard of the
234 weights it needs, leading to a much smaller memory overhead. The same idea
255 -applies to quantization. Note that we also add an additional argument ``prefix``
235 +applies to quantization. Note that we also add an additional argument `prefix`
236 to the model's constructor so that the model can initialize itself differently
237 based on the prefix. This is useful for non-uniform quantization, where
258 -different parts of the model are quantized differently. The ``prefix`` is
259 -usually an empty string for the top-level model and a string like ``"vision"``
260 -or ``"language"`` for the sub-models. In general, it matches the name of the
238 +different parts of the model are quantized differently. The `prefix` is
239 +usually an empty string for the top-level model and a string like `"vision"`
240 +or `"language"` for the sub-models. In general, it matches the name of the
241 module's state dict in the checkpoint file.
242
243 One disadvantage of this design is that it is hard to write unit tests for
244 individual components in vLLM because every component needs to be initialized by
245 a complete config object. We solve this problem by providing a default
246 initialization function that creates a default config object with all fields set
267 -to ``None``. If the component we want to test only cares about a few fields in
247 +to `None`. If the component we want to test only cares about a few fields in
248 the config object, we can create a default config object and set the fields we
249 care about. This way, we can test the component in isolation. Note that many
250 tests in vLLM are end-to-end tests that test the whole system, so this is not a
251 big problem.
252
273 -In summary, the complete config object ``VllmConfig`` can be treated as an
253 +In summary, the complete config object `VllmConfig` can be treated as an
254 engine-level global state that is shared among all vLLM classes.
255 +
256 +[vllm/engine/async_llm_engine.py]: https://github.com/vllm-project/vllm/tree/main/vllm/engine/async_llm_engine.py
257 +[vllm/engine/llm_engine.py]: https://github.com/vllm-project/vllm/tree/main/vllm/engine/llm_engine.py
258 +[vllm/entrypoints/api_server.py]: https://github.com/vllm-project/vllm/tree/main/vllm/entrypoints/api_server.py
docs/source/design/huggingface_integration.md new
+36
@@ -0,0 +1,36 @@
1 +(huggingface-integration)=
2 +
3 +# Integration with HuggingFace
4 +
5 +This document describes how vLLM integrates with HuggingFace libraries. We will explain step by step what happens under the hood when we run `vllm serve`.
6 +
7 +Let's say we want to serve the popular QWen model by running `vllm serve Qwen/Qwen2-7B`.
8 +
9 +1. The `model` argument is `Qwen/Qwen2-7B`. vLLM determines whether this model exists by checking for the corresponding config file `config.json`. See this [code snippet](https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#L162-L182) for the implementation. Within this process:
10 +
11 + - If the `model` argument corresponds to an existing local path, vLLM will load the config file directly from this path.
12 + - If the `model` argument is a HuggingFace model ID consisting of a username and model name, vLLM will first try to use the config file from the HuggingFace local cache, using the `model` argument as the model name and the `--revision` argument as the revision. See [their website](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hfhome) for more information on how the HuggingFace cache works.
13 + - If the `model` argument is a HuggingFace model ID but it is not found in the cache, vLLM will download the config file from the HuggingFace model hub. Refer to [this function](https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#L91) for the implementation. The input arguments include the `model` argument as the model name, the `--revision` argument as the revision, and the environment variable `HF_TOKEN` as the token to access the model hub. In our case, vLLM will download the [config.json](https://huggingface.co/Qwen/Qwen2-7B/blob/main/config.json) file.
14 +
15 +2. After confirming the existence of the model, vLLM loads its config file and converts it into a dictionary. See this [code snippet](https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#L185-L186) for the implementation.
16 +
17 +3. Next, vLLM [inspects](https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#L189) the `model_type` field in the config dictionary to [generate](https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#190-L216) the config object to use. There are some `model_type` values that vLLM directly supports; see [here](https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#L48) for the list. If the `model_type` is not in the list, vLLM will use [AutoConfig.from_pretrained](https://huggingface.co/docs/transformers/en/model_doc/auto#transformers.AutoConfig.from_pretrained) to load the config class, with `model`, `--revision`, and `--trust_remote_code` as the arguments. Please note that:
18 +
19 + - HuggingFace also has its own logic to determine the config class to use. It will again use the `model_type` field to search for the class name in the transformers library; see [here](https://github.com/huggingface/transformers/tree/main/src/transformers/models) for the list of supported models. If the `model_type` is not found, HuggingFace will use the `auto_map` field from the config JSON file to determine the class name. Specifically, it is the `AutoConfig` field under `auto_map`. See [DeepSeek](https://huggingface.co/deepseek-ai/DeepSeek-V2.5/blob/main/config.json) for an example.
20 + - The `AutoConfig` field under `auto_map` points to a module path in the model's repository. To create the config class, HuggingFace will import the module and use the `from_pretrained` method to load the config class. This can generally cause arbitrary code execution, so it is only executed when `--trust_remote_code` is enabled.
21 +
22 +4. Subsequently, vLLM applies some historical patches to the config object. These are mostly related to RoPE configuration; see [here](https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/transformers_utils/config.py#L244) for the implementation.
23 +
24 +5. Finally, vLLM can reach the model class we want to initialize. vLLM uses the `architectures` field in the config object to determine the model class to initialize, as it maintains the mapping from architecture name to model class in [its registry](https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/model_executor/models/registry.py#L80). If the architecture name is not found in the registry, it means this model architecture is not supported by vLLM. For `Qwen/Qwen2-7B`, the `architectures` field is `["Qwen2ForCausalLM"]`, which corresponds to the `Qwen2ForCausalLM` class in [vLLM's code](https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/model_executor/models/qwen2.py#L364). This class will initialize itself depending on various configs.
25 +
26 +Beyond that, there are two more things vLLM depends on HuggingFace for.
27 +
28 +1. **Tokenizer**: vLLM uses the tokenizer from HuggingFace to tokenize the input text. The tokenizer is loaded using [AutoTokenizer.from_pretrained](https://huggingface.co/docs/transformers/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained) with the `model` argument as the model name and the `--revision` argument as the revision. It is also possible to use a tokenizer from another model by specifying the `--tokenizer` argument in the `vllm serve` command. Other relevant arguments are `--tokenizer-revision` and `--tokenizer-mode`. Please check HuggingFace's documentation for the meaning of these arguments. This part of the logic can be found in the [get_tokenizer](https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/transformers_utils/tokenizer.py#L87) function. After obtaining the tokenizer, notably, vLLM will cache some expensive attributes of the tokenizer in [get_cached_tokenizer](https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/transformers_utils/tokenizer.py#L24).
29 +
30 +2. **Model weight**: vLLM downloads the model weight from the HuggingFace model hub using the `model` argument as the model name and the `--revision` argument as the revision. vLLM provides the argument `--load-format` to control what files to download from the model hub. By default, it will try to load the weights in the safetensors format and fall back to the PyTorch bin format if the safetensors format is not available. We can also pass `--load-format dummy` to skip downloading the weights.
31 +
32 + - It is recommended to use the safetensors format, as it is efficient for loading in distributed inference and also safe from arbitrary code execution. See the [documentation](https://huggingface.co/docs/safetensors/en/index) for more information on the safetensors format. This part of the logic can be found [here](https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/model_executor/model_loader/loader.py#L385). Please note that:
33 +
34 +This completes the integration between vLLM and HuggingFace.
35 +
36 +In summary, vLLM reads the config file `config.json`, tokenizer, and model weight from the HuggingFace model hub or a local directory. It uses the config class from either vLLM, HuggingFace transformers, or loads the config class from the model's repository.
docs/source/design/huggingface_integration.rst deleted
-40
@@ -1,40 +0,0 @@
1 -.. _huggingface_integration:
2 -
3 -Integration with HuggingFace
4 -===================================
5 -
6 -This document describes how vLLM integrates with HuggingFace libraries. We will explain step by step what happens under the hood when we run ``vllm serve``.
7 -
8 -Let's say we want to serve the popular QWen model by running ``vllm serve Qwen/Qwen2-7B``.
9 -
10 -1. The ``model`` argument is ``Qwen/Qwen2-7B``. vLLM determines whether this model exists by checking for the corresponding config file ``config.json``. See this `code snippet <https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#L162-L182>`__ for the implementation. Within this process:
11 -
12 - - If the ``model`` argument corresponds to an existing local path, vLLM will load the config file directly from this path.
13 -
14 - - If the ``model`` argument is a HuggingFace model ID consisting of a username and model name, vLLM will first try to use the config file from the HuggingFace local cache, using the ``model`` argument as the model name and the ``--revision`` argument as the revision. See `their website <https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hfhome>`__ for more information on how the HuggingFace cache works.
15 -
16 - - If the ``model`` argument is a HuggingFace model ID but it is not found in the cache, vLLM will download the config file from the HuggingFace model hub. Refer to `this function <https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#L91>`__ for the implementation. The input arguments include the ``model`` argument as the model name, the ``--revision`` argument as the revision, and the environment variable ``HF_TOKEN`` as the token to access the model hub. In our case, vLLM will download the `config.json <https://huggingface.co/Qwen/Qwen2-7B/blob/main/config.json>`__ file.
17 -
18 -2. After confirming the existence of the model, vLLM loads its config file and converts it into a dictionary. See this `code snippet <https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#L185-L186>`__ for the implementation.
19 -
20 -3. Next, vLLM `inspects <https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#L189>`__ the ``model_type`` field in the config dictionary to `generate <https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#190-L216>`__ the config object to use. There are some ``model_type`` values that vLLM directly supports; see `here <https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/transformers_utils/config.py#L48>`__ for the list. If the ``model_type`` is not in the list, vLLM will use `AutoConfig.from_pretrained <https://huggingface.co/docs/transformers/en/model_doc/auto#transformers.AutoConfig.from_pretrained>`__ to load the config class, with ``model``, ``--revision``, and ``--trust_remote_code`` as the arguments. Please note that:
21 -
22 - - HuggingFace also has its own logic to determine the config class to use. It will again use the ``model_type`` field to search for the class name in the transformers library; see `here <https://github.com/huggingface/transformers/tree/main/src/transformers/models>`__ for the list of supported models. If the ``model_type`` is not found, HuggingFace will use the ``auto_map`` field from the config JSON file to determine the class name. Specifically, it is the ``AutoConfig`` field under ``auto_map``. See `DeepSeek <https://huggingface.co/deepseek-ai/DeepSeek-V2.5/blob/main/config.json>`__ for an example.
23 -
24 - - The ``AutoConfig`` field under ``auto_map`` points to a module path in the model's repository. To create the config class, HuggingFace will import the module and use the ``from_pretrained`` method to load the config class. This can generally cause arbitrary code execution, so it is only executed when ``--trust_remote_code`` is enabled.
25 -
26 -4. Subsequently, vLLM applies some historical patches to the config object. These are mostly related to RoPE configuration; see `here <https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/transformers_utils/config.py#L244>`__ for the implementation.
27 -
28 -5. Finally, vLLM can reach the model class we want to initialize. vLLM uses the ``architectures`` field in the config object to determine the model class to initialize, as it maintains the mapping from architecture name to model class in `its registry <https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/model_executor/models/registry.py#L80>`__. If the architecture name is not found in the registry, it means this model architecture is not supported by vLLM. For ``Qwen/Qwen2-7B``, the ``architectures`` field is ``["Qwen2ForCausalLM"]``, which corresponds to the ``Qwen2ForCausalLM`` class in `vLLM's code <https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/model_executor/models/qwen2.py#L364>`__. This class will initialize itself depending on various configs.
29 -
30 -Beyond that, there are two more things vLLM depends on HuggingFace for.
31 -
32 -1. **Tokenizer**: vLLM uses the tokenizer from HuggingFace to tokenize the input text. The tokenizer is loaded using `AutoTokenizer.from_pretrained <https://huggingface.co/docs/transformers/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained>`__ with the ``model`` argument as the model name and the ``--revision`` argument as the revision. It is also possible to use a tokenizer from another model by specifying the ``--tokenizer`` argument in the ``vllm serve`` command. Other relevant arguments are ``--tokenizer-revision`` and ``--tokenizer-mode``. Please check HuggingFace's documentation for the meaning of these arguments. This part of the logic can be found in the `get_tokenizer <https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/transformers_utils/tokenizer.py#L87>`__ function. After obtaining the tokenizer, notably, vLLM will cache some expensive attributes of the tokenizer in `get_cached_tokenizer <https://github.com/vllm-project/vllm/blob/127c07480ecea15e4c2990820c457807ff78a057/vllm/transformers_utils/tokenizer.py#L24>`__.
33 -
34 -2. **Model weight**: vLLM downloads the model weight from the HuggingFace model hub using the ``model`` argument as the model name and the ``--revision`` argument as the revision. vLLM provides the argument ``--load-format`` to control what files to download from the model hub. By default, it will try to load the weights in the safetensors format and fall back to the PyTorch bin format if the safetensors format is not available. We can also pass ``--load-format dummy`` to skip downloading the weights.
35 -
36 - - It is recommended to use the safetensors format, as it is efficient for loading in distributed inference and also safe from arbitrary code execution. See the `documentation <https://huggingface.co/docs/safetensors/en/index>`__ for more information on the safetensors format. This part of the logic can be found `here <https://github.com/vllm-project/vllm/blob/10b67d865d92e376956345becafc249d4c3c0ab7/vllm/model_executor/model_loader/loader.py#L385>`__. Please note that:
37 -
38 -This completes the integration between vLLM and HuggingFace.
39 -
40 -In summary, vLLM reads the config file ``config.json``, tokenizer, and model weight from the HuggingFace model hub or a local directory. It uses the config class from either vLLM, HuggingFace transformers, or loads the config class from the model's repository.
docs/source/design/input_processing/input_processing_pipeline.md new
+19
@@ -0,0 +1,19 @@
1 +(input-processing-pipeline)=
2 +
3 +# Input Processing Pipeline
4 +
5 +1. Input data is passed to {class}`~vllm.LLMEngine` (or {class}`~vllm.AsyncLLMEngine`).
6 +
7 +2. Tokenize the data if necessary.
8 +
9 +3. Process the inputs using {meth}`INPUT_REGISTRY.process_input <vllm.inputs.registry.InputRegistry.process_input>`.
10 +
11 + - For example, add placeholder tokens to reserve KV cache for multi-modal embeddings.
12 +
13 +4. Send the processed inputs to {class}`~vllm.executor.executor_base.ExecutorBase`.
14 +
15 +5. Distribute the inputs via {class}`~vllm.worker.worker_base.WorkerBase` to {class}`~vllm.worker.model_runner_base.ModelRunnerBase`.
16 +
17 +6. If the data contains multi-modal data, convert it into keyword arguments using {meth}`MULTIMODAL_REGISTRY.map_input <vllm.multimodal.MultiModalRegistry.map_input>`.
18 +
19 + - For example, convert a {class}`PIL.Image.Image` input to its pixel values for a vision model.
docs/source/design/input_processing/input_processing_pipeline.rst deleted
-20
@@ -1,20 +0,0 @@
1 -.. _input_processing_pipeline:
2 -
3 -Input Processing Pipeline
4 -=========================
5 -
6 -1. Input data is passed to :class:`~vllm.LLMEngine` (or :class:`~vllm.AsyncLLMEngine`).
7 -
8 -2. Tokenize the data if necessary.
9 -
10 -3. Process the inputs using :meth:`INPUT_REGISTRY.process_input <vllm.inputs.registry.InputRegistry.process_input>`.
11 -
12 - - For example, add placeholder tokens to reserve KV cache for multi-modal embeddings.
13 -
14 -4. Send the processed inputs to :class:`~vllm.executor.executor_base.ExecutorBase`.
15 -
16 -5. Distribute the inputs via :class:`~vllm.worker.worker_base.WorkerBase` to :class:`~vllm.worker.model_runner_base.ModelRunnerBase`.
17 -
18 -6. If the data contains multi-modal data, convert it into keyword arguments using :meth:`MULTIMODAL_REGISTRY.map_input <vllm.multimodal.MultiModalRegistry.map_input>`.
19 -
20 - - For example, convert a :class:`PIL.Image.Image` input to its pixel values for a vision model.
docs/source/design/input_processing/model_inputs_index.md new
+43
@@ -0,0 +1,43 @@
1 +(input-processing)=
2 +
3 +# Input Processing
4 +
5 +```{eval-rst}
6 +.. currentmodule:: vllm.inputs
7 +```
8 +
9 +Each model can override parts of vLLM's [input processing pipeline](#input-processing-pipeline) via
10 +{data}`~vllm.inputs.INPUT_REGISTRY` and {data}`~vllm.multimodal.MULTIMODAL_REGISTRY`.
11 +
12 +Currently, this mechanism is only utilized in [multi-modal](#multi-modality) models for preprocessing multi-modal input
13 +data in addition to input prompt, but it can be extended to text-only language models when needed.
14 +
15 +## Guides
16 +
17 +```{toctree}
18 +:maxdepth: 1
19 +
20 +input_processing_pipeline
21 +```
22 +
23 +## Module Contents
24 +
25 +### LLM Engine Inputs
26 +
27 +```{eval-rst}
28 +.. autoclass:: vllm.inputs.DecoderOnlyInputs
29 + :members:
30 + :show-inheritance:
31 +```
32 +
33 +### Registry
34 +
35 +```{eval-rst}
36 +.. autodata:: vllm.inputs.INPUT_REGISTRY
37 +```
38 +
39 +```{eval-rst}
40 +.. automodule:: vllm.inputs.registry
41 + :members:
42 + :show-inheritance:
43 +```
docs/source/design/input_processing/model_inputs_index.rst deleted
-39
@@ -1,39 +0,0 @@
1 -.. _input_processing:
2 -
3 -Input Processing
4 -================
5 -
6 -.. currentmodule:: vllm.inputs
7 -
8 -Each model can override parts of vLLM's :ref:`input processing pipeline <input_processing_pipeline>` via
9 -:data:`~vllm.inputs.INPUT_REGISTRY` and :data:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
10 -
11 -Currently, this mechanism is only utilized in :ref:`multi-modal <multi_modality>` models for preprocessing multi-modal input
12 -data in addition to input prompt, but it can be extended to text-only language models when needed.
13 -
14 -Guides
15 -++++++
16 -
17 -.. toctree::
18 - :maxdepth: 1
19 -
20 - input_processing_pipeline
21 -
22 -Module Contents
23 -+++++++++++++++
24 -
25 -LLM Engine Inputs
26 ------------------
27 -
28 -.. autoclass:: vllm.inputs.DecoderOnlyInputs
29 - :members:
30 - :show-inheritance:
31 -
32 -Registry
33 ---------
34 -
35 -.. autodata:: vllm.inputs.INPUT_REGISTRY
36 -
37 -.. automodule:: vllm.inputs.registry
38 - :members:
39 - :show-inheritance:
docs/source/design/kernel/paged_attention.md new
+527
@@ -0,0 +1,527 @@
1 +# vLLM Paged Attention
2 +
3 +- Currently, vLLM utilizes its own implementation of a multi-head query
4 + attention kernel (`csrc/attention/attention_kernels.cu`).
5 + This kernel is designed to be compatible with
6 + vLLM's paged KV caches, where the key and value cache are stored in
7 + separate blocks (note that this block concept differs from the GPU
8 + thread block. So in a later document, I will refer to vLLM paged
9 + attention block as "block", while refer to GPU thread block as
10 + "thread block").
11 +- To achieve high performance, this kernel relies on a specially
12 + designed memory layout and access method, specifically when threads
13 + read data from global memory to shared memory. The purpose of this
14 + document is to provide a high-level explanation of the kernel
15 + implementation step by step, aiding those who wish to learn about the
16 + vLLM multi-head query attention kernel. After going through this
17 + document, users will likely have a better understanding and feel easier
18 + to follow the actual implementation.
19 +- Please note that this document may not cover all details, such as how
20 + to calculate the correct index for the corresponding data or the dot
21 + multiplication implementation. However, after reading this document
22 + and becoming familiar with the high-level logic flow, it should be
23 + easier for you to read the actual code and understand the details.
24 +
25 +## Inputs
26 +
27 +- The kernel function takes a list of arguments for the current thread
28 + to perform its assigned work. The three most important arguments are
29 + the input pointers `q`, `k_cache`, and `v_cache`, which point
30 + to query, key, and value data on global memory that need to be read
31 + and processed. The output pointer `out` points to global memory
32 + where the result should be written. These four pointers actually
33 + refer to multi-dimensional arrays, but each thread only accesses the
34 + portion of data assigned to it. I have omitted all other runtime
35 + parameters here for simplicity.
36 +
37 + ```cpp
38 + template<
39 + typename scalar_t,
40 + int HEAD_SIZE,
41 + int BLOCK_SIZE,
42 + int NUM_THREADS,
43 + int PARTITION_SIZE = 0>
44 + __device__ void paged_attention_kernel(
45 + ... // Other side args.
46 + const scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions, head_size]
47 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
48 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
49 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
50 + ... // Other side args.
51 + )
52 + ```
53 +
54 +- There are also a list of template arguments above the function
55 + signature that are determined during compilation time. `scalar_t`
56 + represents the data type of the query, key, and value data elements,
57 + such as FP16. `HEAD_SIZE` indicates the number of elements in each
58 + head. `BLOCK_SIZE` refers to the number of tokens in each block.
59 + `NUM_THREADS` denotes the number of threads in each thread block.
60 + `PARTITION_SIZE` represents the number of tensor parallel GPUs (For
61 + simplicity, we assume this is 0 and tensor parallel is disabled).
62 +
63 +- With these arguments, we need to perform a sequence of preparations.
64 + This includes calculating the current head index, block index, and
65 + other necessary variables. However, for now, we can ignore these
66 + preparations and proceed directly to the actual calculations. It will
67 + be easier to understand them once we grasp the entire flow.
68 +
69 +## Concepts
70 +
71 +- Just before we dive into the calculation flow, I want to describe a
72 + few concepts that are needed for later sections. However, you may
73 + skip this section and return later if you encounter any confusing
74 + terminologies.
75 +- **Sequence**: A sequence represents a client request. For example,
76 + the data pointed to by `q` has a shape of
77 + `[num_seqs, num_heads, head_size]`. That represents there are total
78 + `num_seqs` of query sequence data are pointed by `q`. Since this
79 + kernel is a single query attention kernel, each sequence only has one
80 + query token. Hence, the `num_seqs` equals the total number of tokens
81 + that are processed in the batch.
82 +- **Context**: The context consists of the generated tokens from the
83 + sequence. For instance, `["What", "is", "your"]` are the context
84 + tokens, and the input query token is `"name"`. The model might
85 + generate the token `"?"`.
86 +- **Vec**: The vec is a list of elements that are fetched and
87 + calculated together. For query and key data, the vec size
88 + (`VEC_SIZE`) is determined so that each thread group can fetch and
89 + calculate 16 bytes of data at a time. For value data, the vec size
90 + (`V_VEC_SIZE`) is determined so that each thread can fetch and
91 + calculate 16 bytes of data at a time. For example, if the
92 + `scalar_t` is FP16 (2 bytes) and `THREAD_GROUP_SIZE` is 2, the
93 + `VEC_SIZE` will be 4, while the `V_VEC_SIZE` will be 8.
94 +- **Thread group**: The thread group is a small group of
95 + threads(`THREAD_GROUP_SIZE`) that fetches and calculates one
96 + query token and one key token at a time. Each thread handles only a
97 + portion of the token data. The total number of elements processed by
98 + one thread group is referred as `x`. For example, if the thread
99 + group contains 2 threads and the head size is 8, then thread 0
100 + handles the query and key elements at index 0, 2, 4, 6, while thread
101 + 1 handles the elements at index 1, 3, 5, 7.
102 +- **Block**: The key and value cache data in vLLM are split into
103 + blocks. Each block stores data for a fixed number(`BLOCK_SIZE`)
104 + of tokens at one head. Each block may contain only a portion of the
105 + whole context tokens. For example, if the block size is 16 and the
106 + head size is 128, then for one head, one block can store 16 * 128 =
107 + 2048 elements.
108 +- **Warp**: A warp is a group of 32 threads(`WARP_SIZE`) that
109 + execute simultaneously on a stream multiprocessor (SM). In this
110 + kernel, each warp processes the calculation between one query token
111 + and key tokens of one entire block at a time (it may process multiple
112 + blocks in multiple iterations). For example, if there are 4 warps and
113 + 6 blocks for one context, the assignment would be like warp 0 handles
114 + the 0th, 4th blocks, warp 1 handles the 1st, 5th blocks, warp 2
115 + handles the 2nd block and warp 3 handles the 3rd block.
116 +- **Thread block**: A thread block is a group of
117 + threads(`NUM_THREADS`) that can access the same shared memory.
118 + Each thread block contains multiple warps(`NUM_WARPS`), and in
119 + this kernel, each thread block processes the calculation between one
120 + query token and key tokens of a whole context.
121 +- **Grid**: A grid is a collection of thread blocks and defines the
122 + shape of the collection. In this kernel, the shape is
123 + `(num_heads, num_seqs, max_num_partitions)`. Therefore, each thread
124 + block only handles the calculation for one head, one sequence, and
125 + one partition.
126 +
127 +## Query
128 +
129 +- This section will introduce how query data is stored in memory and
130 + fetched by each thread. As mentioned above, each thread group fetches
131 + one query token data, while each thread itself only handles a part of
132 + one query token data. Within each warp, every thread group will fetch
133 + the same query token data, but will multiply it with different key
134 + token data.
135 +
136 + ```cpp
137 + const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
138 + ```
139 +
140 + ```{figure} ../../assets/kernel/query.png
141 + :align: center
142 + :alt: query
143 + :width: 70%
144 +
145 + Query data of one token at one head
146 + ```
147 +
148 +- Each thread defines its own `q_ptr` which points to the assigned
149 + query token data on global memory. For example, if `VEC_SIZE` is 4
150 + and `HEAD_SIZE` is 128, the `q_ptr` points to data that contains
151 + total of 128 elements divided into 128 / 4 = 32 vecs.
152 +
153 + ```{figure} ../../assets/kernel/q_vecs.png
154 + :align: center
155 + :alt: q_vecs
156 + :width: 70%
157 +
158 + `q_vecs` for one thread group
159 + ```
160 +
161 + ```cpp
162 + __shared__ Q_vec q_vecs[THREAD_GROUP_SIZE][NUM_VECS_PER_THREAD];
163 + ```
164 +
165 +- Next, we need to read the global memory data pointed to by `q_ptr`
166 + into shared memory as `q_vecs`. It is important to note that each
167 + vecs is assigned to a different row. For example, if the
168 + `THREAD_GROUP_SIZE` is 2, thread 0 will handle the 0th row vecs,
169 + while thread 1 handles the 1st row vecs. By reading the query data in
170 + this way, neighboring threads like thread 0 and thread 1 can read
171 + neighbor memory, achieving the memory coalescing to improve
172 + performance.
173 +
174 +## Key
175 +
176 +- Similar to the "Query" section, this section introduces memory layout
177 + and assignment for keys. While each thread group only handle one
178 + query token one kernel run, it may handle multiple key tokens across
179 + multiple iterations. Meanwhile, each warp will process multiple blocks
180 + of key tokens in multiple iterations, ensuring that all context
181 + tokens are processed by the entire thread group after the kernel run.
182 + In this context, "handle" refers to performing the dot multiplication
183 + between query data and key data.
184 +
185 + ```cpp
186 + const scalar_t* k_ptr = k_cache + physical_block_number * kv_block_stride
187 + + kv_head_idx * kv_head_stride
188 + + physical_block_offset * x;
189 + ```
190 +
191 +- Unlike to `q_ptr`, `k_ptr` in each thread will point to different
192 + key token at different iterations. As shown above, that `k_ptr`
193 + points to key token data based on `k_cache` at assigned block,
194 + assigned head and assigned token.
195 +
196 + ```{figure} ../../assets/kernel/key.png
197 + :align: center
198 + :alt: key
199 + :width: 70%
200 +
201 + Key data of all context tokens at one head
202 + ```
203 +
204 +- The diagram above illustrates the memory layout for key data. It
205 + assumes that the `BLOCK_SIZE` is 16, `HEAD_SIZE` is 128, `x` is
206 + 8, `THREAD_GROUP_SIZE` is 2, and there are a total of 4 warps. Each
207 + rectangle represents all the elements for one key token at one head,
208 + which will be processed by one thread group. The left half shows the
209 + total 16 blocks of key token data for warp 0, while the right half
210 + represents the remaining key token data for other warps or
211 + iterations. Inside each rectangle, there are a total 32 vecs (128
212 + elements for one token) that will be processed by 2 threads (one
213 + thread group) separately.
214 +
215 + ```{figure} ../../assets/kernel/k_vecs.png
216 + :align: center
217 + :alt: k_vecs
218 + :width: 70%
219 +
220 + `k_vecs` for one thread
221 + ```
222 +
223 + ```cpp
224 + K_vec k_vecs[NUM_VECS_PER_THREAD]
225 + ```
226 +
227 +- Next, we need to read the key token data from `k_ptr` and store
228 + them on register memory as `k_vecs`. We use register memory for
229 + `k_vecs` because it will only be accessed by one thread once,
230 + whereas `q_vecs` will be accessed by multiple threads multiple
231 + times. Each `k_vecs` will contain multiple vectors for later
232 + calculation. Each vec will be set at each inner iteration. The
233 + assignment of vecs allows neighboring threads in a warp to read
234 + neighboring memory together, which again promotes the memory
235 + coalescing. For instance, thread 0 will read vec 0, while thread 1
236 + will read vec 1. In the next inner loop, thread 0 will read vec 2,
237 + while thread 1 will read vec 3, and so on.
238 +
239 +- You may still be a little confused about the overall flow. Don't
240 + worry, please keep reading the next "QK" section. It will illustrate
241 + the query and key calculation flow in a clearer and higher-level
242 + manner.
243 +
244 +## QK
245 +
246 +- As shown the pseudo code below, before the entire for loop block, we
247 + fetch the query data for one token and store it in `q_vecs`. Then,
248 + in the outer for loop, we iterate through different `k_ptrs` that
249 + point to different tokens and prepare the `k_vecs` in the inner for
250 + loop. Finally, we perform the dot multiplication between the
251 + `q_vecs` and each `k_vecs`.
252 +
253 + ```cpp
254 + q_vecs = ...
255 + for ... {
256 + k_ptr = ...
257 + for ... {
258 + k_vecs[i] = ...
259 + }
260 + ...
261 + float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs[thread_group_offset], k_vecs);
262 + }
263 + ```
264 +
265 +- As mentioned before, for each thread, it only fetches part of the
266 + query and key token data at a time. However, there will be a cross
267 + thread group reduction happen in the `Qk_dot<>::dot` . So `qk`
268 + returned here is not just between part of the query and key token dot
269 + multiplication, but actually a full result between entire query and
270 + key token data.
271 +
272 +- For example, if the value of `HEAD_SIZE` is 128 and
273 + `THREAD_GROUP_SIZE` is 2, each thread's `k_vecs` will contain
274 + total 64 elements. However, the returned `qk` is actually the
275 + result of dot multiplication between 128 query elements and 128 key
276 + elements. If you want to learn more about the details of the dot
277 + multiplication and reduction, you may refer to the implementation of
278 + `Qk_dot<>::dot`. However, for the sake of simplicity, I will not
279 + cover it in this document.
280 +
281 +## Softmax
282 +
283 +- Next, we need to calculate the normalized softmax for all `qk`s,
284 + as shown above, where each $x$ represents a `qk`. To do this,
285 + we must obtain the reduced value of `qk_max`($m(x)$) and
286 + the `exp_sum`($\ell(x)$) of all `qk`s. The reduction
287 + should be performed across the entire thread block, encompassing
288 + results between the query token and all context key tokens.
289 +
290 + ```{math}
291 + :nowrap: true
292 +
293 + \begin{gather*}
294 + m(x):=\max _i \quad x_i \\ \quad f(x):=\left[\begin{array}{lll}e^{x_1-m(x)} & \ldots & e^{x_B-m(x)}\end{array}\right]\\ \quad \ell(x):=\sum_i f(x)_i \\
295 + \quad \operatorname{softmax}(x):=\frac{f(x)}{\ell(x)}
296 + \end{gather*}
297 + ```
298 +
299 +### `qk_max` and `logits`
300 +
301 +- Just right after we get the `qk` result, we can set the temporary
302 + `logits` result with `qk` (In the end, the `logits` should
303 + store the normalized softmax result). Also we can compare and collect
304 + the `qk_max` for all `qk`s that are calculated by current
305 + thread group.
306 +
307 + ```cpp
308 + if (thread_group_offset == 0) {
309 + const bool mask = token_idx >= context_len;
310 + logits[token_idx - start_token_idx] = mask ? 0.f : qk;
311 + qk_max = mask ? qk_max : fmaxf(qk_max, qk);
312 + }
313 + ```
314 +
315 +- Please note that the `logits` here is on shared memory, so each
316 + thread group will set the fields for its own assigned context tokens.
317 + Overall, the size of logits should be number of context tokens.
318 +
319 + ```cpp
320 + for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
321 + qk_max = fmaxf(qk_max, VLLM_SHFL_XOR_SYNC(qk_max, mask));
322 + }
323 +
324 + if (lane == 0) {
325 + red_smem[warp_idx] = qk_max;
326 + }
327 + ```
328 +
329 +- Then we need to get the reduced `qk_max` across each warp. The main
330 + idea is to make threads in warp to communicate with each other and
331 + get the final max `qk` .
332 +
333 + ```cpp
334 + for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
335 + qk_max = fmaxf(qk_max, VLLM_SHFL_XOR_SYNC(qk_max, mask));
336 + }
337 + qk_max = VLLM_SHFL_SYNC(qk_max, 0);
338 + ```
339 +
340 +- Finally, we can get the reduced `qk_max` from whole thread block by
341 + compare the `qk_max` from all warps in this thread block. Then we
342 + need to broadcast the final result to each thread.
343 +
344 +### `exp_sum`
345 +
346 +- Similar to `qk_max`, we need to get the reduced sum value from the
347 + entire thread block too.
348 +
349 + ```cpp
350 + for (int i = thread_idx; i < num_tokens; i += NUM_THREADS) {
351 + float val = __expf(logits[i] - qk_max);
352 + logits[i] = val;
353 + exp_sum += val;
354 + }
355 + ...
356 + exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
357 + ```
358 +
359 +- Firstly, sum all exp values from each thread group, and meanwhile,
360 + convert each entry of `logits` from `qk` to `exp(qk - qk_max)`.
361 + Please note, the `qk_max` here is already the max `qk` across the
362 + whole thread block. And then we can do reduction for `exp_sum`
363 + across whole thread block just like the `qk_max`.
364 +
365 + ```cpp
366 + const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
367 + for (int i = thread_idx; i < num_tokens; i += NUM_THREADS) {
368 + logits[i] *= inv_sum;
369 + }
370 + ```
371 +
372 +- Finally, with the reduced `qk_max` and `exp_sum`, we can obtain
373 + the final normalized softmax result as `logits`. This `logits`
374 + variable will be used for dot multiplication with the value data in
375 + later steps. Now, it should store the normalized softmax result of
376 + `qk` for all assigned context tokens.
377 +
378 +## Value
379 +
380 +```{figure} ../../assets/kernel/value.png
381 +:align: center
382 +:alt: value
383 +:width: 70%
384 +
385 +Value data of all context tokens at one head
386 +```
387 +
388 +```{figure} ../../assets/kernel/logits_vec.png
389 +:align: center
390 +:alt: logits_vec
391 +:width: 50%
392 +
393 +`logits_vec` for one thread
394 +```
395 +
396 +```{figure} ../../assets/kernel/v_vec.png
397 +:align: center
398 +:alt: v_vec
399 +:width: 70%
400 +
401 +List of `v_vec` for one thread
402 +```
403 +
404 +- Now we need to retrieve the value data and perform dot multiplication
405 + with `logits`. Unlike query and key, there is no thread group
406 + concept for value data. As shown in diagram, different from key token
407 + memory layout, elements from the same column correspond to the same
408 + value token. For one block of value data, there are `HEAD_SIZE` of
409 + rows and `BLOCK_SIZE` of columns that are split into multiple
410 + `v_vecs`.
411 +
412 +- Each thread always fetches `V_VEC_SIZE` elements from the same
413 + `V_VEC_SIZE` of tokens at a time. As a result, a single thread
414 + retrieves multiple `v_vec`s from different rows and the same
415 + columns through multiple inner iterations. For each `v_vec`, it
416 + needs to be dot multiplied with the corresponding `logits_vec`,
417 + which is also `V_VEC_SIZE` elements from `logits`. Overall, with
418 + multiple inner iterations, each warp will process one block of value
419 + tokens. And with multiple outer iterations, the whole context value
420 + tokens are processd
421 +
422 + ```cpp
423 + float accs[NUM_ROWS_PER_THREAD];
424 + for ... { // Iteration over different blocks.
425 + logits_vec = ...
426 + for ... { // Iteration over different rows.
427 + v_vec = ...
428 + ...
429 + accs[i] += dot(logits_vec, v_vec);
430 + }
431 + }
432 + ```
433 +
434 +- As shown in the above pseudo code, in the outer loop, similar to
435 + `k_ptr`, `logits_vec` iterates over different blocks and reads
436 + `V_VEC_SIZE` elements from `logits`. In the inner loop, each
437 + thread reads `V_VEC_SIZE` elements from the same tokens as a
438 + `v_vec` and performs dot multiplication. It is important to note
439 + that in each inner iteration, the thread fetches different head
440 + position elements for the same tokens. The dot result is then
441 + accumulated in `accs`. Therefore, each entry of `accs` is mapped
442 + to a head position assigned to the current thread.
443 +
444 +- For example, if `BLOCK_SIZE` is 16 and `V_VEC_SIZE` is 8, each
445 + thread fetches 8 value elements for 8 tokens at a time. Each element
446 + is from different tokens at the same head position. If `HEAD_SIZE`
447 + is 128 and `WARP_SIZE` is 32, for each inner loop, a warp needs to
448 + fetch `WARP_SIZE * V_VEC_SIZE = 256` elements. This means there are
449 + a total of 128 * 16 / 256 = 8 inner iterations for a warp to handle
450 + a whole block of value tokens. And each `accs` in each thread
451 + contains 8 elements that accumulated at 8 different head positions.
452 + For the thread 0, the `accs` variable will have 8 elements, which
453 + are 0th, 32th … 224th elements of a value head that are accumulated
454 + from all assigned 8 tokens.
455 +
456 +## LV
457 +
458 +- Now, we need to perform reduction for `accs` within each warp. This
459 + process allows each thread to accumulate the `accs` for the
460 + assigned head positions of all tokens in one block.
461 +
462 + ```cpp
463 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
464 + float acc = accs[i];
465 + for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
466 + acc += VLLM_SHFL_XOR_SYNC(acc, mask);
467 + }
468 + accs[i] = acc;
469 + }
470 + ```
471 +
472 +- Next, we perform reduction for `accs` across all warps, allowing
473 + each thread to have the accumulation of `accs` for the assigned
474 + head positions of all context tokens. Please note that each `accs`
475 + in every thread only stores the accumulation for a portion of
476 + elements of the entire head for all context tokens. However, overall,
477 + all results for output have been calculated but are just stored in
478 + different thread register memory.
479 +
480 + ```cpp
481 + float* out_smem = reinterpret_cast<float*>(shared_mem);
482 + for (int i = NUM_WARPS; i > 1; i /= 2) {
483 + // Upper warps write to shared memory.
484 + ...
485 + float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
486 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
487 + ...
488 + dst[row_idx] = accs[i];
489 + }
490 +
491 + // Lower warps update the output.
492 + const float* src = &out_smem[warp_idx * HEAD_SIZE];
493 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
494 + ...
495 + accs[i] += src[row_idx];
496 + }
497 +
498 + // Write out the accs.
499 + }
500 + ```
501 +
502 +## Output
503 +
504 +- Now we can write all of calculated result from local register memory
505 + to final output global memory.
506 +
507 + ```cpp
508 + scalar_t* out_ptr = out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
509 + + head_idx * max_num_partitions * HEAD_SIZE
510 + + partition_idx * HEAD_SIZE;
511 + ```
512 +
513 +- First, we need to define the `out_ptr` variable, which points to
514 + the start address of the assigned sequence and assigned head.
515 +
516 + ```cpp
517 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
518 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
519 + if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
520 + from_float(*(out_ptr + row_idx), accs[i]);
521 + }
522 + }
523 + ```
524 +
525 +- Finally, we need to iterate over different assigned head positions
526 + and write out the corresponding accumulated result based on the
527 + `out_ptr`.
docs/source/design/kernel/paged_attention.rst deleted
-525
@@ -1,525 +0,0 @@
1 -vLLM Paged Attention
2 -====================
3 -
4 -- Currently, vLLM utilizes its own implementation of a multi-head query
5 - attention kernel (``csrc/attention/attention_kernels.cu``).
6 - This kernel is designed to be compatible with
7 - vLLM's paged KV caches, where the key and value cache are stored in
8 - separate blocks (note that this block concept differs from the GPU
9 - thread block. So in a later document, I will refer to vLLM paged
10 - attention block as "block", while refer to GPU thread block as
11 - "thread block").
12 -- To achieve high performance, this kernel relies on a specially
13 - designed memory layout and access method, specifically when threads
14 - read data from global memory to shared memory. The purpose of this
15 - document is to provide a high-level explanation of the kernel
16 - implementation step by step, aiding those who wish to learn about the
17 - vLLM multi-head query attention kernel. After going through this
18 - document, users will likely have a better understanding and feel easier
19 - to follow the actual implementation.
20 -- Please note that this document may not cover all details, such as how
21 - to calculate the correct index for the corresponding data or the dot
22 - multiplication implementation. However, after reading this document
23 - and becoming familiar with the high-level logic flow, it should be
24 - easier for you to read the actual code and understand the details.
25 -
26 -Inputs
27 -------
28 -
29 -- The kernel function takes a list of arguments for the current thread
30 - to perform its assigned work. The three most important arguments are
31 - the input pointers ``q``, ``k_cache``, and ``v_cache``, which point
32 - to query, key, and value data on global memory that need to be read
33 - and processed. The output pointer ``out`` points to global memory
34 - where the result should be written. These four pointers actually
35 - refer to multi-dimensional arrays, but each thread only accesses the
36 - portion of data assigned to it. I have omitted all other runtime
37 - parameters here for simplicity.
38 -
39 - .. code:: cpp
40 -
41 - template<
42 - typename scalar_t,
43 - int HEAD_SIZE,
44 - int BLOCK_SIZE,
45 - int NUM_THREADS,
46 - int PARTITION_SIZE = 0>
47 - __device__ void paged_attention_kernel(
48 - ... // Other side args.
49 - const scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions, head_size]
50 - const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
51 - const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
52 - const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
53 - ... // Other side args.
54 - )
55 -
56 -- There are also a list of template arguments above the function
57 - signature that are determined during compilation time. ``scalar_t``
58 - represents the data type of the query, key, and value data elements,
59 - such as FP16. ``HEAD_SIZE`` indicates the number of elements in each
60 - head. ``BLOCK_SIZE`` refers to the number of tokens in each block.
61 - ``NUM_THREADS`` denotes the number of threads in each thread block.
62 - ``PARTITION_SIZE`` represents the number of tensor parallel GPUs (For
63 - simplicity, we assume this is 0 and tensor parallel is disabled).
64 -- With these arguments, we need to perform a sequence of preparations.
65 - This includes calculating the current head index, block index, and
66 - other necessary variables. However, for now, we can ignore these
67 - preparations and proceed directly to the actual calculations. It will
68 - be easier to understand them once we grasp the entire flow.
69 -
70 -Concepts
71 ---------
72 -
73 -- Just before we dive into the calculation flow, I want to describe a
74 - few concepts that are needed for later sections. However, you may
75 - skip this section and return later if you encounter any confusing
76 - terminologies.
77 -- **Sequence**: A sequence represents a client request. For example,
78 - the data pointed to by ``q`` has a shape of
79 - ``[num_seqs, num_heads, head_size]``. That represents there are total
80 - ``num_seqs`` of query sequence data are pointed by ``q``. Since this
81 - kernel is a single query attention kernel, each sequence only has one
82 - query token. Hence, the ``num_seqs`` equals the total number of tokens
83 - that are processed in the batch.
84 -- **Context**: The context consists of the generated tokens from the
85 - sequence. For instance, ``["What", "is", "your"]`` are the context
86 - tokens, and the input query token is ``"name"``. The model might
87 - generate the token ``"?"``.
88 -- **Vec**: The vec is a list of elements that are fetched and
89 - calculated together. For query and key data, the vec size
90 - (``VEC_SIZE``) is determined so that each thread group can fetch and
91 - calculate 16 bytes of data at a time. For value data, the vec size
92 - (``V_VEC_SIZE``) is determined so that each thread can fetch and
93 - calculate 16 bytes of data at a time. For example, if the
94 - ``scalar_t`` is FP16 (2 bytes) and ``THREAD_GROUP_SIZE`` is 2, the
95 - ``VEC_SIZE`` will be 4, while the ``V_VEC_SIZE`` will be 8.
96 -- **Thread group**: The thread group is a small group of
97 - threads(\ ``THREAD_GROUP_SIZE``) that fetches and calculates one
98 - query token and one key token at a time. Each thread handles only a
99 - portion of the token data. The total number of elements processed by
100 - one thread group is referred as ``x``. For example, if the thread
101 - group contains 2 threads and the head size is 8, then thread 0
102 - handles the query and key elements at index 0, 2, 4, 6, while thread
103 - 1 handles the elements at index 1, 3, 5, 7.
104 -- **Block**: The key and value cache data in vLLM are split into
105 - blocks. Each block stores data for a fixed number(\ ``BLOCK_SIZE``)
106 - of tokens at one head. Each block may contain only a portion of the
107 - whole context tokens. For example, if the block size is 16 and the
108 - head size is 128, then for one head, one block can store 16 \* 128 =
109 - 2048 elements.
110 -- **Warp**: A warp is a group of 32 threads(\ ``WARP_SIZE``) that
111 - execute simultaneously on a stream multiprocessor (SM). In this
112 - kernel, each warp processes the calculation between one query token
113 - and key tokens of one entire block at a time (it may process multiple
114 - blocks in multiple iterations). For example, if there are 4 warps and
115 - 6 blocks for one context, the assignment would be like warp 0 handles
116 - the 0th, 4th blocks, warp 1 handles the 1st, 5th blocks, warp 2
117 - handles the 2nd block and warp 3 handles the 3rd block.
118 -- **Thread block**: A thread block is a group of
119 - threads(\ ``NUM_THREADS``) that can access the same shared memory.
120 - Each thread block contains multiple warps(\ ``NUM_WARPS``), and in
121 - this kernel, each thread block processes the calculation between one
122 - query token and key tokens of a whole context.
123 -- **Grid**: A grid is a collection of thread blocks and defines the
124 - shape of the collection. In this kernel, the shape is
125 - ``(num_heads, num_seqs, max_num_partitions)``. Therefore, each thread
126 - block only handles the calculation for one head, one sequence, and
127 - one partition.
128 -
129 -Query
130 ------
131 -
132 -- This section will introduce how query data is stored in memory and
133 - fetched by each thread. As mentioned above, each thread group fetches
134 - one query token data, while each thread itself only handles a part of
135 - one query token data. Within each warp, every thread group will fetch
136 - the same query token data, but will multiply it with different key
137 - token data.
138 -
139 - .. code:: cpp
140 -
141 - const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
142 -
143 - .. figure:: ../../assets/kernel/query.png
144 - :alt: query
145 - :width: 70%
146 - :align: center
147 -
148 - Query data of one token at one head
149 -
150 -- Each thread defines its own ``q_ptr`` which points to the assigned
151 - query token data on global memory. For example, if ``VEC_SIZE`` is 4
152 - and ``HEAD_SIZE`` is 128, the ``q_ptr`` points to data that contains
153 - total of 128 elements divided into 128 / 4 = 32 vecs.
154 -
155 - .. figure:: ../../assets/kernel/q_vecs.png
156 - :alt: q_vecs
157 - :width: 70%
158 - :align: center
159 -
160 - ``q_vecs`` for one thread group
161 -
162 - .. code:: cpp
163 -
164 - __shared__ Q_vec q_vecs[THREAD_GROUP_SIZE][NUM_VECS_PER_THREAD];
165 -
166 -- Next, we need to read the global memory data pointed to by ``q_ptr``
167 - into shared memory as ``q_vecs``. It is important to note that each
168 - vecs is assigned to a different row. For example, if the
169 - ``THREAD_GROUP_SIZE`` is 2, thread 0 will handle the 0th row vecs,
170 - while thread 1 handles the 1st row vecs. By reading the query data in
171 - this way, neighboring threads like thread 0 and thread 1 can read
172 - neighbor memory, achieving the memory coalescing to improve
173 - performance.
174 -
175 -Key
176 ----
177 -
178 -- Similar to the "Query" section, this section introduces memory layout
179 - and assignment for keys. While each thread group only handle one
180 - query token one kernel run, it may handle multiple key tokens across
181 - multiple iterations. Meanwhile, each warp will process multiple blocks
182 - of key tokens in multiple iterations, ensuring that all context
183 - tokens are processed by the entire thread group after the kernel run.
184 - In this context, "handle" refers to performing the dot multiplication
185 - between query data and key data.
186 -
187 - .. code:: cpp
188 -
189 - const scalar_t* k_ptr = k_cache + physical_block_number * kv_block_stride
190 - + kv_head_idx * kv_head_stride
191 - + physical_block_offset * x;
192 -
193 -- Unlike to ``q_ptr``, ``k_ptr`` in each thread will point to different
194 - key token at different iterations. As shown above, that ``k_ptr``
195 - points to key token data based on ``k_cache`` at assigned block,
196 - assigned head and assigned token.
197 -
198 - .. figure:: ../../assets/kernel/key.png
199 - :alt: key
200 - :width: 70%
201 - :align: center
202 -
203 - Key data of all context tokens at one head
204 -
205 -- The diagram above illustrates the memory layout for key data. It
206 - assumes that the ``BLOCK_SIZE`` is 16, ``HEAD_SIZE`` is 128, ``x`` is
207 - 8, ``THREAD_GROUP_SIZE`` is 2, and there are a total of 4 warps. Each
208 - rectangle represents all the elements for one key token at one head,
209 - which will be processed by one thread group. The left half shows the
210 - total 16 blocks of key token data for warp 0, while the right half
211 - represents the remaining key token data for other warps or
212 - iterations. Inside each rectangle, there are a total 32 vecs (128
213 - elements for one token) that will be processed by 2 threads (one
214 - thread group) separately.
215 -
216 - .. figure:: ../../assets/kernel/k_vecs.png
217 - :alt: k_vecs
218 - :width: 70%
219 - :align: center
220 -
221 - ``k_vecs`` for one thread
222 -
223 - .. code:: cpp
224 -
225 - K_vec k_vecs[NUM_VECS_PER_THREAD]
226 -
227 -- Next, we need to read the key token data from ``k_ptr`` and store
228 - them on register memory as ``k_vecs``. We use register memory for
229 - ``k_vecs`` because it will only be accessed by one thread once,
230 - whereas ``q_vecs`` will be accessed by multiple threads multiple
231 - times. Each ``k_vecs`` will contain multiple vectors for later
232 - calculation. Each vec will be set at each inner iteration. The
233 - assignment of vecs allows neighboring threads in a warp to read
234 - neighboring memory together, which again promotes the memory
235 - coalescing. For instance, thread 0 will read vec 0, while thread 1
236 - will read vec 1. In the next inner loop, thread 0 will read vec 2,
237 - while thread 1 will read vec 3, and so on.
238 -- You may still be a little confused about the overall flow. Don't
239 - worry, please keep reading the next "QK" section. It will illustrate
240 - the query and key calculation flow in a clearer and higher-level
241 - manner.
242 -
243 -QK
244 ----
245 -
246 -- As shown the pseudo code below, before the entire for loop block, we
247 - fetch the query data for one token and store it in ``q_vecs``. Then,
248 - in the outer for loop, we iterate through different ``k_ptrs`` that
249 - point to different tokens and prepare the ``k_vecs`` in the inner for
250 - loop. Finally, we perform the dot multiplication between the
251 - ``q_vecs`` and each ``k_vecs``.
252 -
253 - .. code:: cpp
254 -
255 - q_vecs = ...
256 - for ... {
257 - k_ptr = ...
258 - for ... {
259 - k_vecs[i] = ...
260 - }
261 - ...
262 - float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs[thread_group_offset], k_vecs);
263 - }
264 -
265 -- As mentioned before, for each thread, it only fetches part of the
266 - query and key token data at a time. However, there will be a cross
267 - thread group reduction happen in the ``Qk_dot<>::dot`` . So ``qk``
268 - returned here is not just between part of the query and key token dot
269 - multiplication, but actually a full result between entire query and
270 - key token data.
271 -- For example, if the value of ``HEAD_SIZE`` is 128 and
272 - ``THREAD_GROUP_SIZE`` is 2, each thread's ``k_vecs`` will contain
273 - total 64 elements. However, the returned ``qk`` is actually the
274 - result of dot multiplication between 128 query elements and 128 key
275 - elements. If you want to learn more about the details of the dot
276 - multiplication and reduction, you may refer to the implementation of
277 - ``Qk_dot<>::dot``. However, for the sake of simplicity, I will not
278 - cover it in this document.
279 -
280 -Softmax
281 --------
282 -
283 -- Next, we need to calculate the normalized softmax for all ``qk``\ s,
284 - as shown above, where each :math:`x` represents a ``qk``. To do this,
285 - we must obtain the reduced value of ``qk_max``\ (:math:`m(x)`) and
286 - the ``exp_sum``\ (:math:`\ell(x)`) of all ``qk``\ s. The reduction
287 - should be performed across the entire thread block, encompassing
288 - results between the query token and all context key tokens.
289 -
290 - .. math::
291 - :nowrap:
292 -
293 - \begin{gather*}
294 - m(x):=\max _i \quad x_i \\ \quad f(x):=\left[\begin{array}{lll}e^{x_1-m(x)} & \ldots & e^{x_B-m(x)}\end{array}\right]\\ \quad \ell(x):=\sum_i f(x)_i \\
295 - \quad \operatorname{softmax}(x):=\frac{f(x)}{\ell(x)}
296 - \end{gather*}
297 -
298 -``qk_max`` and ``logits``
299 -~~~~~~~~~~~~~~~~~~~~~~~~~
300 -
301 -- Just right after we get the ``qk`` result, we can set the temporary
302 - ``logits`` result with ``qk`` (In the end, the ``logits`` should
303 - store the normalized softmax result). Also we can compare and collect
304 - the ``qk_max`` for all ``qk``\ s that are calculated by current
305 - thread group.
306 -
307 - .. code:: cpp
308 -
309 - if (thread_group_offset == 0) {
310 - const bool mask = token_idx >= context_len;
311 - logits[token_idx - start_token_idx] = mask ? 0.f : qk;
312 - qk_max = mask ? qk_max : fmaxf(qk_max, qk);
313 - }
314 -
315 -- Please note that the ``logits`` here is on shared memory, so each
316 - thread group will set the fields for its own assigned context tokens.
317 - Overall, the size of logits should be number of context tokens.
318 -
319 - .. code:: cpp
320 -
321 - for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
322 - qk_max = fmaxf(qk_max, VLLM_SHFL_XOR_SYNC(qk_max, mask));
323 - }
324 -
325 - if (lane == 0) {
326 - red_smem[warp_idx] = qk_max;
327 - }
328 -
329 -- Then we need to get the reduced ``qk_max`` across each warp. The main
330 - idea is to make threads in warp to communicate with each other and
331 - get the final max ``qk`` .
332 -
333 - .. code:: cpp
334 -
335 - for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
336 - qk_max = fmaxf(qk_max, VLLM_SHFL_XOR_SYNC(qk_max, mask));
337 - }
338 - qk_max = VLLM_SHFL_SYNC(qk_max, 0);
339 -
340 -- Finally, we can get the reduced ``qk_max`` from whole thread block by
341 - compare the ``qk_max`` from all warps in this thread block. Then we
342 - need to broadcast the final result to each thread.
343 -
344 -``exp_sum``
345 -~~~~~~~~~~~
346 -
347 -- Similar to ``qk_max``, we need to get the reduced sum value from the
348 - entire thread block too.
349 -
350 - .. code:: cpp
351 -
352 - for (int i = thread_idx; i < num_tokens; i += NUM_THREADS) {
353 - float val = __expf(logits[i] - qk_max);
354 - logits[i] = val;
355 - exp_sum += val;
356 - }
357 - ...
358 - exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
359 -
360 -- Firstly, sum all exp values from each thread group, and meanwhile,
361 - convert each entry of ``logits`` from ``qk`` to ``exp(qk - qk_max)``.
362 - Please note, the ``qk_max`` here is already the max ``qk`` across the
363 - whole thread block. And then we can do reduction for ``exp_sum``
364 - across whole thread block just like the ``qk_max``.
365 -
366 - .. code:: cpp
367 -
368 - const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
369 - for (int i = thread_idx; i < num_tokens; i += NUM_THREADS) {
370 - logits[i] *= inv_sum;
371 - }
372 -
373 -- Finally, with the reduced ``qk_max`` and ``exp_sum``, we can obtain
374 - the final normalized softmax result as ``logits``. This ``logits``
375 - variable will be used for dot multiplication with the value data in
376 - later steps. Now, it should store the normalized softmax result of
377 - ``qk`` for all assigned context tokens.
378 -
379 -Value
380 ------
381 -
382 -.. figure:: ../../assets/kernel/value.png
383 - :alt: value
384 - :width: 70%
385 - :align: center
386 -
387 - Value data of all context tokens at one head
388 -
389 -.. figure:: ../../assets/kernel/logits_vec.png
390 - :alt: logits_vec
391 - :width: 50%
392 - :align: center
393 -
394 - ``logits_vec`` for one thread
395 -
396 -.. figure:: ../../assets/kernel/v_vec.png
397 - :alt: v_vec
398 - :width: 70%
399 - :align: center
400 -
401 - List of ``v_vec`` for one thread
402 -
403 -- Now we need to retrieve the value data and perform dot multiplication
404 - with ``logits``. Unlike query and key, there is no thread group
405 - concept for value data. As shown in diagram, different from key token
406 - memory layout, elements from the same column correspond to the same
407 - value token. For one block of value data, there are ``HEAD_SIZE`` of
408 - rows and ``BLOCK_SIZE`` of columns that are split into multiple
409 - ``v_vecs``.
410 -- Each thread always fetches ``V_VEC_SIZE`` elements from the same
411 - ``V_VEC_SIZE`` of tokens at a time. As a result, a single thread
412 - retrieves multiple ``v_vec``\ s from different rows and the same
413 - columns through multiple inner iterations. For each ``v_vec``, it
414 - needs to be dot multiplied with the corresponding ``logits_vec``,
415 - which is also ``V_VEC_SIZE`` elements from ``logits``. Overall, with
416 - multiple inner iterations, each warp will process one block of value
417 - tokens. And with multiple outer iterations, the whole context value
418 - tokens are processd
419 -
420 - .. code:: cpp
421 -
422 - float accs[NUM_ROWS_PER_THREAD];
423 - for ... { // Iteration over different blocks.
424 - logits_vec = ...
425 - for ... { // Iteration over different rows.
426 - v_vec = ...
427 - ...
428 - accs[i] += dot(logits_vec, v_vec);
429 - }
430 - }
431 -
432 -- As shown in the above pseudo code, in the outer loop, similar to
433 - ``k_ptr``, ``logits_vec`` iterates over different blocks and reads
434 - ``V_VEC_SIZE`` elements from ``logits``. In the inner loop, each
435 - thread reads ``V_VEC_SIZE`` elements from the same tokens as a
436 - ``v_vec`` and performs dot multiplication. It is important to note
437 - that in each inner iteration, the thread fetches different head
438 - position elements for the same tokens. The dot result is then
439 - accumulated in ``accs``. Therefore, each entry of ``accs`` is mapped
440 - to a head position assigned to the current thread.
441 -- For example, if ``BLOCK_SIZE`` is 16 and ``V_VEC_SIZE`` is 8, each
442 - thread fetches 8 value elements for 8 tokens at a time. Each element
443 - is from different tokens at the same head position. If ``HEAD_SIZE``
444 - is 128 and ``WARP_SIZE`` is 32, for each inner loop, a warp needs to
445 - fetch ``WARP_SIZE * V_VEC_SIZE = 256`` elements. This means there are
446 - a total of 128 \* 16 / 256 = 8 inner iterations for a warp to handle
447 - a whole block of value tokens. And each ``accs`` in each thread
448 - contains 8 elements that accumulated at 8 different head positions.
449 - For the thread 0, the ``accs`` variable will have 8 elements, which
450 - are 0th, 32th … 224th elements of a value head that are accumulated
451 - from all assigned 8 tokens.
452 -
453 -LV
454 ----
455 -- Now, we need to perform reduction for ``accs`` within each warp. This
456 - process allows each thread to accumulate the ``accs`` for the
457 - assigned head positions of all tokens in one block.
458 -
459 - .. code:: cpp
460 -
461 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
462 - float acc = accs[i];
463 - for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
464 - acc += VLLM_SHFL_XOR_SYNC(acc, mask);
465 - }
466 - accs[i] = acc;
467 - }
468 -
469 -- Next, we perform reduction for ``accs`` across all warps, allowing
470 - each thread to have the accumulation of ``accs`` for the assigned
471 - head positions of all context tokens. Please note that each ``accs``
472 - in every thread only stores the accumulation for a portion of
473 - elements of the entire head for all context tokens. However, overall,
474 - all results for output have been calculated but are just stored in
475 - different thread register memory.
476 -
477 - .. code:: cpp
478 -
479 - float* out_smem = reinterpret_cast<float*>(shared_mem);
480 - for (int i = NUM_WARPS; i > 1; i /= 2) {
481 - // Upper warps write to shared memory.
482 - ...
483 - float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
484 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
485 - ...
486 - dst[row_idx] = accs[i];
487 - }
488 -
489 - // Lower warps update the output.
490 - const float* src = &out_smem[warp_idx * HEAD_SIZE];
491 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
492 - ...
493 - accs[i] += src[row_idx];
494 - }
495 -
496 - // Write out the accs.
497 - }
498 -
499 -Output
500 -------
501 -
502 -- Now we can write all of calculated result from local register memory
503 - to final output global memory.
504 -
505 - .. code:: cpp
506 -
507 - scalar_t* out_ptr = out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
508 - + head_idx * max_num_partitions * HEAD_SIZE
509 - + partition_idx * HEAD_SIZE;
510 -
511 -- First, we need to define the ``out_ptr`` variable, which points to
512 - the start address of the assigned sequence and assigned head.
513 -
514 - .. code:: cpp
515 -
516 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
517 - const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
518 - if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
519 - from_float(*(out_ptr + row_idx), accs[i]);
520 - }
521 - }
522 -
523 -- Finally, we need to iterate over different assigned head positions
524 - and write out the corresponding accumulated result based on the
525 - ``out_ptr``.
docs/source/design/multimodal/adding_multimodal_plugin.md new
+16
@@ -0,0 +1,16 @@
1 +(adding-multimodal-plugin)=
2 +
3 +# Adding a Multimodal Plugin
4 +
5 +This document teaches you how to add a new modality to vLLM.
6 +
7 +Each modality in vLLM is represented by a {class}`~vllm.multimodal.MultiModalPlugin` and registered to {data}`~vllm.multimodal.MULTIMODAL_REGISTRY`.
8 +For vLLM to recognize a new modality type, you have to create a new plugin and then pass it to {meth}`~vllm.multimodal.MultiModalRegistry.register_plugin`.
9 +
10 +The remainder of this document details how to define custom {class}`~vllm.multimodal.MultiModalPlugin` s.
11 +
12 +```{note}
13 +This article is a work in progress.
14 +```
15 +
16 +% TODO: Add more instructions on how to add new plugins once embeddings is in.
docs/source/design/multimodal/adding_multimodal_plugin.rst deleted
-17
@@ -1,17 +0,0 @@
1 -.. _adding_multimodal_plugin:
2 -
3 -Adding a Multimodal Plugin
4 -==========================
5 -
6 -This document teaches you how to add a new modality to vLLM.
7 -
8 -Each modality in vLLM is represented by a :class:`~vllm.multimodal.MultiModalPlugin` and registered to :data:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
9 -For vLLM to recognize a new modality type, you have to create a new plugin and then pass it to :meth:`~vllm.multimodal.MultiModalRegistry.register_plugin`.
10 -
11 -The remainder of this document details how to define custom :class:`~vllm.multimodal.MultiModalPlugin` s.
12 -
13 -.. note::
14 - This article is a work in progress.
15 -
16 -..
17 - TODO: Add more instructions on how to add new plugins once embeddings is in.
docs/source/design/multimodal/multimodal_index.md renamed
+39 -22
@@ -1,66 +1,83 @@
1 -.. _multi_modality:
1 +(multi-modality)=
2
3 -Multi-Modality
4 -==============
3 +# Multi-Modality
4
5 +```{eval-rst}
6 .. currentmodule:: vllm.multimodal
7 -
8 -vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
7 +```
8
10 -Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_mm_models>`
11 -via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptType`.
9 +vLLM provides experimental support for multi-modal models through the {mod}`vllm.multimodal` package.
10 +
11 +Multi-modal inputs can be passed alongside text and token prompts to [supported models](#supported-mm-models)
12 +via the `multi_modal_data` field in {class}`vllm.inputs.PromptType`.
13
14 Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14 -by following :ref:`this guide <adding_multimodal_plugin>`.
15 +by following [this guide](#adding-multimodal-plugin).
16
16 -Looking to add your own multi-modal model? Please follow the instructions listed :ref:`here <enabling_multimodal_inputs>`.
17 +Looking to add your own multi-modal model? Please follow the instructions listed [here](#enabling-multimodal-inputs).
18
18 -Guides
19 -++++++
19 +## Guides
20
21 -.. toctree::
22 - :maxdepth: 1
21 +```{toctree}
22 +:maxdepth: 1
23
24 - adding_multimodal_plugin
24 +adding_multimodal_plugin
25 +```
26
26 -Module Contents
27 -+++++++++++++++
27 +## Module Contents
28
29 +```{eval-rst}
30 .. automodule:: vllm.multimodal
31 +```
32
31 -Registry
32 ---------
33 +### Registry
34
35 +```{eval-rst}
36 .. autodata:: vllm.multimodal.MULTIMODAL_REGISTRY
37 +```
38
39 +```{eval-rst}
40 .. autoclass:: vllm.multimodal.MultiModalRegistry
41 :members:
42 :show-inheritance:
43 +```
44
40 -Base Classes
41 -------------
45 +### Base Classes
46
47 +```{eval-rst}
48 .. autodata:: vllm.multimodal.NestedTensors
49 +```
50
51 +```{eval-rst}
52 .. autodata:: vllm.multimodal.BatchedTensorInputs
53 +```
54
55 +```{eval-rst}
56 .. autoclass:: vllm.multimodal.MultiModalDataBuiltins
57 :members:
58 :show-inheritance:
59 +```
60
61 +```{eval-rst}
62 .. autodata:: vllm.multimodal.MultiModalDataDict
63 +```
64
65 +```{eval-rst}
66 .. autoclass:: vllm.multimodal.MultiModalKwargs
67 :members:
68 :show-inheritance:
69 +```
70
71 +```{eval-rst}
72 .. autoclass:: vllm.multimodal.MultiModalPlugin
73 :members:
74 :show-inheritance:
75 +```
76
61 -Image Classes
62 --------------
77 +### Image Classes
78
79 +```{eval-rst}
80 .. automodule:: vllm.multimodal.image
81 :members:
82 :show-inheritance:
83 +```
docs/source/design/plugin_system.md new
+54
@@ -0,0 +1,54 @@
1 +(plugin-system)=
2 +
3 +# vLLM's Plugin System
4 +
5 +The community frequently requests the ability to extend vLLM with custom features. To facilitate this, vLLM includes a plugin system that allows users to add custom features without modifying the vLLM codebase. This document explains how plugins work in vLLM and how to create a plugin for vLLM.
6 +
7 +## How Plugins Work in vLLM
8 +
9 +Plugins are user-registered code that vLLM executes. Given vLLM's architecture (see [](#arch-overview)), multiple processes may be involved, especially when using distributed inference with various parallelism techniques. To enable plugins successfully, every process created by vLLM needs to load the plugin. This is done by the [load_general_plugins](https://github.com/vllm-project/vllm/blob/c76ac49d266e27aa3fea84ef2df1f813d24c91c7/vllm/plugins/__init__.py#L16) function in the `vllm.plugins` module. This function is called for every process created by vLLM before it starts any work.
10 +
11 +## How vLLM Discovers Plugins
12 +
13 +vLLM's plugin system uses the standard Python `entry_points` mechanism. This mechanism allows developers to register functions in their Python packages for use by other packages. An example of a plugin:
14 +
15 +```python
16 +# inside `setup.py` file
17 +from setuptools import setup
18 +
19 +setup(name='vllm_add_dummy_model',
20 + version='0.1',
21 + packages=['vllm_add_dummy_model'],
22 + entry_points={
23 + 'vllm.general_plugins':
24 + ["register_dummy_model = vllm_add_dummy_model:register"]
25 + })
26 +
27 +# inside `vllm_add_dummy_model.py` file
28 +def register():
29 + from vllm import ModelRegistry
30 +
31 + if "MyLlava" not in ModelRegistry.get_supported_archs():
32 + ModelRegistry.register_model("MyLlava",
33 + "vllm_add_dummy_model.my_llava:MyLlava")
34 +```
35 +
36 +For more information on adding entry points to your package, please check the [official documentation](https://setuptools.pypa.io/en/latest/userguide/entry_point.html).
37 +
38 +Every plugin has three parts:
39 +
40 +1. **Plugin group**: The name of the entry point group. vLLM uses the entry point group `vllm.general_plugins` to register general plugins. This is the key of `entry_points` in the `setup.py` file. Always use `vllm.general_plugins` for vLLM's general plugins.
41 +2. **Plugin name**: The name of the plugin. This is the value in the dictionary of the `entry_points` dictionary. In the example above, the plugin name is `register_dummy_model`. Plugins can be filtered by their names using the `VLLM_PLUGINS` environment variable. To load only a specific plugin, set `VLLM_PLUGINS` to the plugin name.
42 +3. **Plugin value**: The fully qualified name of the function to register in the plugin system. In the example above, the plugin value is `vllm_add_dummy_model:register`, which refers to a function named `register` in the `vllm_add_dummy_model` module.
43 +
44 +## What Can Plugins Do?
45 +
46 +Currently, the primary use case for plugins is to register custom, out-of-the-tree models into vLLM. This is done by calling `ModelRegistry.register_model` to register the model. In the future, the plugin system may be extended to support more features, such as swapping in custom implementations for certain classes in vLLM.
47 +
48 +## Guidelines for Writing Plugins
49 +
50 +- **Being re-entrant**: The function specified in the entry point should be re-entrant, meaning it can be called multiple times without causing issues. This is necessary because the function might be called multiple times in some processes.
51 +
52 +## Compatibility Guarantee
53 +
54 +vLLM guarantees the interface of documented plugins, such as `ModelRegistry.register_model`, will always be available for plugins to register models. However, it is the responsibility of plugin developers to ensure their plugins are compatible with the version of vLLM they are targeting. For example, `"vllm_add_dummy_model.my_llava:MyLlava"` should be compatible with the version of vLLM that the plugin targets. The interface for the model may change during vLLM's development.
docs/source/design/plugin_system.rst deleted
-62
@@ -1,62 +0,0 @@
1 -.. _plugin_system:
2 -
3 -vLLM's Plugin System
4 -====================
5 -
6 -The community frequently requests the ability to extend vLLM with custom features. To facilitate this, vLLM includes a plugin system that allows users to add custom features without modifying the vLLM codebase. This document explains how plugins work in vLLM and how to create a plugin for vLLM.
7 -
8 -How Plugins Work in vLLM
9 -------------------------
10 -
11 -Plugins are user-registered code that vLLM executes. Given vLLM's architecture (see :ref:`arch_overview`), multiple processes may be involved, especially when using distributed inference with various parallelism techniques. To enable plugins successfully, every process created by vLLM needs to load the plugin. This is done by the `load_general_plugins <https://github.com/vllm-project/vllm/blob/c76ac49d266e27aa3fea84ef2df1f813d24c91c7/vllm/plugins/__init__.py#L16>`__ function in the ``vllm.plugins`` module. This function is called for every process created by vLLM before it starts any work.
12 -
13 -How vLLM Discovers Plugins
14 ---------------------------
15 -
16 -vLLM's plugin system uses the standard Python ``entry_points`` mechanism. This mechanism allows developers to register functions in their Python packages for use by other packages. An example of a plugin:
17 -
18 -.. code-block:: python
19 -
20 - # inside `setup.py` file
21 - from setuptools import setup
22 -
23 - setup(name='vllm_add_dummy_model',
24 - version='0.1',
25 - packages=['vllm_add_dummy_model'],
26 - entry_points={
27 - 'vllm.general_plugins':
28 - ["register_dummy_model = vllm_add_dummy_model:register"]
29 - })
30 -
31 - # inside `vllm_add_dummy_model.py` file
32 - def register():
33 - from vllm import ModelRegistry
34 -
35 - if "MyLlava" not in ModelRegistry.get_supported_archs():
36 - ModelRegistry.register_model("MyLlava",
37 - "vllm_add_dummy_model.my_llava:MyLlava")
38 -
39 -For more information on adding entry points to your package, please check the `official documentation <https://setuptools.pypa.io/en/latest/userguide/entry_point.html>`__.
40 -
41 -Every plugin has three parts:
42 -
43 -1. **Plugin group**: The name of the entry point group. vLLM uses the entry point group ``vllm.general_plugins`` to register general plugins. This is the key of ``entry_points`` in the ``setup.py`` file. Always use ``vllm.general_plugins`` for vLLM's general plugins.
44 -
45 -2. **Plugin name**: The name of the plugin. This is the value in the dictionary of the ``entry_points`` dictionary. In the example above, the plugin name is ``register_dummy_model``. Plugins can be filtered by their names using the ``VLLM_PLUGINS`` environment variable. To load only a specific plugin, set ``VLLM_PLUGINS`` to the plugin name.
46 -
47 -3. **Plugin value**: The fully qualified name of the function to register in the plugin system. In the example above, the plugin value is ``vllm_add_dummy_model:register``, which refers to a function named ``register`` in the ``vllm_add_dummy_model`` module.
48 -
49 -What Can Plugins Do?
50 ---------------------
51 -
52 -Currently, the primary use case for plugins is to register custom, out-of-the-tree models into vLLM. This is done by calling ``ModelRegistry.register_model`` to register the model. In the future, the plugin system may be extended to support more features, such as swapping in custom implementations for certain classes in vLLM.
53 -
54 -Guidelines for Writing Plugins
55 -------------------------------
56 -
57 -- **Being re-entrant**: The function specified in the entry point should be re-entrant, meaning it can be called multiple times without causing issues. This is necessary because the function might be called multiple times in some processes.
58 -
59 -Compatibility Guarantee
60 ------------------------
61 -
62 -vLLM guarantees the interface of documented plugins, such as ``ModelRegistry.register_model``, will always be available for plugins to register models. However, it is the responsibility of plugin developers to ensure their plugins are compatible with the version of vLLM they are targeting. For example, ``"vllm_add_dummy_model.my_llava:MyLlava"`` should be compatible with the version of vLLM that the plugin targets. The interface for the model may change during vLLM's development.
docs/source/dev/engine/async_llm_engine.md renamed
+3 -2
@@ -1,6 +1,7 @@
1 -AsyncLLMEngine
2 -=================================
1 +# AsyncLLMEngine
2
3 +```{eval-rst}
4 .. autoclass:: vllm.AsyncLLMEngine
5 :members:
6 :show-inheritance:
7 +```
docs/source/dev/engine/engine_index.md new
+17
@@ -0,0 +1,17 @@
1 +# vLLM Engine
2 +
3 +```{eval-rst}
4 +.. automodule:: vllm.engine
5 +```
6 +
7 +```{eval-rst}
8 +.. currentmodule:: vllm.engine
9 +```
10 +
11 +```{toctree}
12 +:caption: Engines
13 +:maxdepth: 2
14 +
15 +llm_engine
16 +async_llm_engine
17 +```
docs/source/dev/engine/engine_index.rst deleted
-13
@@ -1,13 +0,0 @@
1 -vLLM Engine
2 -=================================
3 -
4 -.. automodule:: vllm.engine
5 -.. currentmodule:: vllm.engine
6 -
7 -.. toctree::
8 - :maxdepth: 2
9 - :caption: Engines
10 -
11 - llm_engine
12 - async_llm_engine
13 -
docs/source/dev/engine/llm_engine.md renamed
+3 -2
@@ -1,6 +1,7 @@
1 -LLMEngine
2 -=================================
1 +# LLMEngine
2
3 +```{eval-rst}
4 .. autoclass:: vllm.LLMEngine
5 :members:
6 :show-inheritance:
7 +```
docs/source/dev/offline_inference/llm.md renamed
+3 -2
@@ -1,6 +1,7 @@
1 -LLM Class
2 -=========
1 +# LLM Class
2
3 +```{eval-rst}
4 .. autoclass:: vllm.LLM
5 :members:
6 :show-inheritance:
7 +```
docs/source/dev/offline_inference/llm_inputs.md renamed
+7 -2
@@ -1,14 +1,19 @@
1 -LLM Inputs
2 -==========
1 +# LLM Inputs
2
3 +```{eval-rst}
4 .. autodata:: vllm.inputs.PromptType
5 +```
6
7 +```{eval-rst}
8 .. autoclass:: vllm.inputs.TextPrompt
9 :show-inheritance:
10 :members:
11 :member-order: bysource
12 +```
13
14 +```{eval-rst}
15 .. autoclass:: vllm.inputs.TokensPrompt
16 :show-inheritance:
17 :members:
18 :member-order: bysource
19 +```
docs/source/dev/offline_inference/offline_index.md new
+8
@@ -0,0 +1,8 @@
1 +# Offline Inference
2 +
3 +```{toctree}
4 +:maxdepth: 1
5 +
6 +llm
7 +llm_inputs
8 +```
docs/source/dev/offline_inference/offline_index.rst deleted
-8
@@ -1,8 +0,0 @@
1 -Offline Inference
2 -=================================
3 -
4 -.. toctree::
5 - :maxdepth: 1
6 -
7 - llm
8 - llm_inputs
docs/source/dev/pooling_params.md renamed
+3 -2
@@ -1,5 +1,6 @@
1 -Pooling Parameters
2 -==================
1 +# Pooling Parameters
2
3 +```{eval-rst}
4 .. autoclass:: vllm.PoolingParams
5 :members:
6 +```
docs/source/dev/sampling_params.md renamed
+3 -2
@@ -1,5 +1,6 @@
1 -Sampling Parameters
2 -===================
1 +# Sampling Parameters
2
3 +```{eval-rst}
4 .. autoclass:: vllm.SamplingParams
5 :members:
6 +```
docs/source/generate_examples.py
+9 -15
@@ -15,18 +15,12 @@ def fix_case(text: str) -> str:
15 return text
16
17
18 -def underline(title: str, character: str = "=") -> str:
19 - return f"{title}\n{character * len(title)}"
20 -
21 -
18 def generate_title(filename: str) -> str:
19 # Turn filename into a title
20 title = filename.replace("_", " ").title()
21 # Handle acronyms and names
22 title = fix_case(title)
27 - # Underline title
28 - title = underline(title)
29 - return title
23 + return f"# {title}"
24
25
26 def generate_examples():
@@ -38,7 +32,7 @@ def generate_examples():
32
33 # Destination paths
34 doc_dir = root_dir / "docs/source/getting_started/examples"
41 - doc_paths = [doc_dir / f"{path.stem}.rst" for path in script_paths]
35 + doc_paths = [doc_dir / f"{path.stem}.md" for path in script_paths]
36
37 # Generate the example docs for each example script
38 for script_path, doc_path in zip(script_paths, doc_paths):
@@ -46,16 +40,16 @@ def generate_examples():
40 # Make script_path relative to doc_path and call it include_path
41 include_path = '../../../..' / script_path.relative_to(root_dir)
42 content = (f"{generate_title(doc_path.stem)}\n\n"
49 - f"Source {script_url}.\n\n"
50 - f".. literalinclude:: {include_path}\n"
51 - " :language: python\n"
52 - " :linenos:\n")
43 + f"Source: <{script_url}>.\n\n"
44 + f"```{{literalinclude}} {include_path}\n"
45 + ":language: python\n"
46 + ":linenos:\n```")
47 with open(doc_path, "w+") as f:
48 f.write(content)
49
50 # Generate the toctree for the example scripts
57 - with open(doc_dir / "examples_index.template.rst") as f:
51 + with open(doc_dir / "examples_index.template.md") as f:
52 examples_index = f.read()
59 - with open(doc_dir / "examples_index.rst", "w+") as f:
60 - example_docs = "\n ".join(path.stem for path in script_paths)
53 + with open(doc_dir / "examples_index.md", "w+") as f:
54 + example_docs = "\n".join(path.stem + ".md" for path in script_paths)
55 f.write(examples_index.replace(r"%EXAMPLE_DOCS%", example_docs))
docs/source/getting_started/amd-installation.md new
+163
@@ -0,0 +1,163 @@
1 +(installation-rocm)=
2 +
3 +# Installation with ROCm
4 +
5 +vLLM supports AMD GPUs with ROCm 6.2.
6 +
7 +## Requirements
8 +
9 +- OS: Linux
10 +- Python: 3.9 -- 3.12
11 +- GPU: MI200s (gfx90a), MI300 (gfx942), Radeon RX 7900 series (gfx1100)
12 +- ROCm 6.2
13 +
14 +Installation options:
15 +
16 +1. [Build from source with docker](#build-from-source-docker-rocm)
17 +2. [Build from source](#build-from-source-rocm)
18 +
19 +(build-from-source-docker-rocm)=
20 +
21 +## Option 1: Build from source with docker (recommended)
22 +
23 +You can build and install vLLM from source.
24 +
25 +First, build a docker image from [Dockerfile.rocm](https://github.com/vllm-project/vllm/blob/main/Dockerfile.rocm) and launch a docker container from the image.
26 +It is important that the user kicks off the docker build using buildkit. Either the user put DOCKER_BUILDKIT=1 as environment variable when calling docker build command, or the user needs to setup buildkit in the docker daemon configuration /etc/docker/daemon.json as follows and restart the daemon:
27 +
28 +```console
29 +{
30 + "features": {
31 + "buildkit": true
32 + }
33 +}
34 +```
35 +
36 +[Dockerfile.rocm](https://github.com/vllm-project/vllm/blob/main/Dockerfile.rocm) uses ROCm 6.2 by default, but also supports ROCm 5.7, 6.0 and 6.1 in older vLLM branches.
37 +It provides flexibility to customize the build of docker image using the following arguments:
38 +
39 +- `BASE_IMAGE`: specifies the base image used when running `docker build`, specifically the PyTorch on ROCm base image.
40 +- `BUILD_FA`: specifies whether to build CK flash-attention. The default is 1. For [Radeon RX 7900 series (gfx1100)](https://rocm.docs.amd.com/projects/radeon/en/latest/index.html), this should be set to 0 before flash-attention supports this target.
41 +- `FX_GFX_ARCHS`: specifies the GFX architecture that is used to build CK flash-attention, for example, `gfx90a;gfx942` for MI200 and MI300. The default is `gfx90a;gfx942`
42 +- `FA_BRANCH`: specifies the branch used to build the CK flash-attention in [ROCm's flash-attention repo](https://github.com/ROCmSoftwarePlatform/flash-attention). The default is `ae7928c`
43 +- `BUILD_TRITON`: specifies whether to build triton flash-attention. The default value is 1.
44 +
45 +Their values can be passed in when running `docker build` with `--build-arg` options.
46 +
47 +To build vllm on ROCm 6.2 for MI200 and MI300 series, you can use the default:
48 +
49 +```console
50 +$ DOCKER_BUILDKIT=1 docker build -f Dockerfile.rocm -t vllm-rocm .
51 +```
52 +
53 +To build vllm on ROCm 6.2 for Radeon RX7900 series (gfx1100), you should specify `BUILD_FA` as below:
54 +
55 +```console
56 +$ DOCKER_BUILDKIT=1 docker build --build-arg BUILD_FA="0" -f Dockerfile.rocm -t vllm-rocm .
57 +```
58 +
59 +To run the above docker image `vllm-rocm`, use the below command:
60 +
61 +```console
62 +$ docker run -it \
63 + --network=host \
64 + --group-add=video \
65 + --ipc=host \
66 + --cap-add=SYS_PTRACE \
67 + --security-opt seccomp=unconfined \
68 + --device /dev/kfd \
69 + --device /dev/dri \
70 + -v <path/to/model>:/app/model \
71 + vllm-rocm \
72 + bash
73 +```
74 +
75 +Where the `<path/to/model>` is the location where the model is stored, for example, the weights for llama2 or llama3 models.
76 +
77 +(build-from-source-rocm)=
78 +
79 +## Option 2: Build from source
80 +
81 +0. Install prerequisites (skip if you are already in an environment/docker with the following installed):
82 +
83 +- [ROCm](https://rocm.docs.amd.com/en/latest/deploy/linux/index.html)
84 +- [PyTorch](https://pytorch.org/)
85 +
86 +For installing PyTorch, you can start from a fresh docker image, e.g, `rocm/pytorch:rocm6.2_ubuntu20.04_py3.9_pytorch_release_2.3.0`, `rocm/pytorch-nightly`.
87 +
88 +Alternatively, you can install PyTorch using PyTorch wheels. You can check PyTorch installation guide in PyTorch [Getting Started](https://pytorch.org/get-started/locally/)
89 +
90 +1. Install [Triton flash attention for ROCm](https://github.com/ROCm/triton)
91 +
92 +Install ROCm's Triton flash attention (the default triton-mlir branch) following the instructions from [ROCm/triton](https://github.com/ROCm/triton/blob/triton-mlir/README.md)
93 +
94 +```console
95 +$ python3 -m pip install ninja cmake wheel pybind11
96 +$ pip uninstall -y triton
97 +$ git clone https://github.com/OpenAI/triton.git
98 +$ cd triton
99 +$ git checkout e192dba
100 +$ cd python
101 +$ pip3 install .
102 +$ cd ../..
103 +```
104 +
105 +```{note}
106 +- If you see HTTP issue related to downloading packages during building triton, please try again as the HTTP error is intermittent.
107 +```
108 +
109 +2. Optionally, if you choose to use CK flash attention, you can install [flash attention for ROCm](https://github.com/ROCm/flash-attention/tree/ck_tile)
110 +
111 +Install ROCm's flash attention (v2.5.9.post1) following the instructions from [ROCm/flash-attention](https://github.com/ROCm/flash-attention/tree/ck_tile#amd-gpurocm-support)
112 +Alternatively, wheels intended for vLLM use can be accessed under the releases.
113 +
114 +For example, for ROCm 6.2, suppose your gfx arch is `gfx90a`. To get your gfx architecture, run `rocminfo |grep gfx`.
115 +
116 +```console
117 +$ git clone https://github.com/ROCm/flash-attention.git
118 +$ cd flash-attention
119 +$ git checkout 3cea2fb
120 +$ git submodule update --init
121 +$ GPU_ARCHS="gfx90a" python3 setup.py install
122 +$ cd ..
123 +```
124 +
125 +```{note}
126 +- You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
127 +```
128 +
129 +3. Build vLLM. For example, vLLM on ROCM 6.2 can be built with the following steps:
130 +
131 +```bash
132 +$ pip install --upgrade pip
133 +
134 +# Install PyTorch
135 +$ pip uninstall torch -y
136 +$ pip install --no-cache-dir --pre torch==2.6.0.dev20241024 --index-url https://download.pytorch.org/whl/nightly/rocm6.2
137 +
138 +# Build & install AMD SMI
139 +$ pip install /opt/rocm/share/amd_smi
140 +
141 +# Install dependencies
142 +$ pip install --upgrade numba scipy huggingface-hub[cli]
143 +$ pip install "numpy<2"
144 +$ pip install -r requirements-rocm.txt
145 +
146 +# Build vLLM for MI210/MI250/MI300.
147 +$ export PYTORCH_ROCM_ARCH="gfx90a;gfx942"
148 +$ python3 setup.py develop
149 +```
150 +
151 +This may take 5-10 minutes. Currently, {code}`pip install .` does not work for ROCm installation.
152 +
153 +```{tip}
154 +- Triton flash attention is used by default. For benchmarking purposes, it is recommended to run a warm up step before collecting perf numbers.
155 +- Triton flash attention does not currently support sliding window attention. If using half precision, please use CK flash-attention for sliding window support.
156 +- To use CK flash-attention or PyTorch naive attention, please use this flag `export VLLM_USE_TRITON_FLASH_ATTN=0` to turn off triton flash attention.
157 +- The ROCm version of PyTorch, ideally, should match the ROCm driver version.
158 +```
159 +
160 +```{tip}
161 +- For MI300x (gfx942) users, to achieve optimal performance, please refer to [MI300x tuning guide](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/index.html) for performance optimization and tuning tips on system and workflow level.
162 + For vLLM, please refer to [vLLM performance optimization](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/workload.html#vllm-performance-optimization).
163 +```
docs/source/getting_started/amd-installation.rst deleted
-178
@@ -1,178 +0,0 @@
1 -.. _installation_rocm:
2 -
3 -Installation with ROCm
4 -======================
5 -
6 -vLLM supports AMD GPUs with ROCm 6.2.
7 -
8 -Requirements
9 -------------
10 -
11 -* OS: Linux
12 -* Python: 3.9 -- 3.12
13 -* GPU: MI200s (gfx90a), MI300 (gfx942), Radeon RX 7900 series (gfx1100)
14 -* ROCm 6.2
15 -
16 -Installation options:
17 -
18 -#. :ref:`Build from source with docker <build_from_source_docker_rocm>`
19 -#. :ref:`Build from source <build_from_source_rocm>`
20 -
21 -.. _build_from_source_docker_rocm:
22 -
23 -Option 1: Build from source with docker (recommended)
24 ------------------------------------------------------
25 -
26 -You can build and install vLLM from source.
27 -
28 -First, build a docker image from `Dockerfile.rocm <https://github.com/vllm-project/vllm/blob/main/Dockerfile.rocm>`_ and launch a docker container from the image.
29 -It is important that the user kicks off the docker build using buildkit. Either the user put DOCKER_BUILDKIT=1 as environment variable when calling docker build command, or the user needs to setup buildkit in the docker daemon configuration /etc/docker/daemon.json as follows and restart the daemon:
30 -
31 -.. code-block:: console
32 -
33 - {
34 - "features": {
35 - "buildkit": true
36 - }
37 - }
38 -
39 -
40 -`Dockerfile.rocm <https://github.com/vllm-project/vllm/blob/main/Dockerfile.rocm>`_ uses ROCm 6.2 by default, but also supports ROCm 5.7, 6.0 and 6.1 in older vLLM branches.
41 -It provides flexibility to customize the build of docker image using the following arguments:
42 -
43 -* `BASE_IMAGE`: specifies the base image used when running ``docker build``, specifically the PyTorch on ROCm base image.
44 -* `BUILD_FA`: specifies whether to build CK flash-attention. The default is 1. For `Radeon RX 7900 series (gfx1100) <https://rocm.docs.amd.com/projects/radeon/en/latest/index.html>`_, this should be set to 0 before flash-attention supports this target.
45 -* `FX_GFX_ARCHS`: specifies the GFX architecture that is used to build CK flash-attention, for example, `gfx90a;gfx942` for MI200 and MI300. The default is `gfx90a;gfx942`
46 -* `FA_BRANCH`: specifies the branch used to build the CK flash-attention in `ROCm's flash-attention repo <https://github.com/ROCmSoftwarePlatform/flash-attention>`_. The default is `ae7928c`
47 -* `BUILD_TRITON`: specifies whether to build triton flash-attention. The default value is 1.
48 -
49 -Their values can be passed in when running ``docker build`` with ``--build-arg`` options.
50 -
51 -
52 -To build vllm on ROCm 6.2 for MI200 and MI300 series, you can use the default:
53 -
54 -.. code-block:: console
55 -
56 - $ DOCKER_BUILDKIT=1 docker build -f Dockerfile.rocm -t vllm-rocm .
57 -
58 -To build vllm on ROCm 6.2 for Radeon RX7900 series (gfx1100), you should specify ``BUILD_FA`` as below:
59 -
60 -.. code-block:: console
61 -
62 - $ DOCKER_BUILDKIT=1 docker build --build-arg BUILD_FA="0" -f Dockerfile.rocm -t vllm-rocm .
63 -
64 -To run the above docker image ``vllm-rocm``, use the below command:
65 -
66 -.. code-block:: console
67 -
68 - $ docker run -it \
69 - --network=host \
70 - --group-add=video \
71 - --ipc=host \
72 - --cap-add=SYS_PTRACE \
73 - --security-opt seccomp=unconfined \
74 - --device /dev/kfd \
75 - --device /dev/dri \
76 - -v <path/to/model>:/app/model \
77 - vllm-rocm \
78 - bash
79 -
80 -Where the `<path/to/model>` is the location where the model is stored, for example, the weights for llama2 or llama3 models.
81 -
82 -
83 -.. _build_from_source_rocm:
84 -
85 -Option 2: Build from source
86 ----------------------------
87 -
88 -0. Install prerequisites (skip if you are already in an environment/docker with the following installed):
89 -
90 -- `ROCm <https://rocm.docs.amd.com/en/latest/deploy/linux/index.html>`_
91 -- `PyTorch <https://pytorch.org/>`_
92 -
93 -For installing PyTorch, you can start from a fresh docker image, e.g, `rocm/pytorch:rocm6.2_ubuntu20.04_py3.9_pytorch_release_2.3.0`, `rocm/pytorch-nightly`.
94 -
95 -Alternatively, you can install PyTorch using PyTorch wheels. You can check PyTorch installation guide in PyTorch `Getting Started <https://pytorch.org/get-started/locally/>`_
96 -
97 -
98 -1. Install `Triton flash attention for ROCm <https://github.com/ROCm/triton>`_
99 -
100 -Install ROCm's Triton flash attention (the default triton-mlir branch) following the instructions from `ROCm/triton <https://github.com/ROCm/triton/blob/triton-mlir/README.md>`_
101 -
102 - .. code-block:: console
103 -
104 - $ python3 -m pip install ninja cmake wheel pybind11
105 - $ pip uninstall -y triton
106 - $ git clone https://github.com/OpenAI/triton.git
107 - $ cd triton
108 - $ git checkout e192dba
109 - $ cd python
110 - $ pip3 install .
111 - $ cd ../..
112 -
113 -.. note::
114 - - If you see HTTP issue related to downloading packages during building triton, please try again as the HTTP error is intermittent.
115 -
116 -
117 -2. Optionally, if you choose to use CK flash attention, you can install `flash attention for ROCm <https://github.com/ROCm/flash-attention/tree/ck_tile>`_
118 -
119 -
120 -Install ROCm's flash attention (v2.5.9.post1) following the instructions from `ROCm/flash-attention <https://github.com/ROCm/flash-attention/tree/ck_tile#amd-gpurocm-support>`_
121 -Alternatively, wheels intended for vLLM use can be accessed under the releases.
122 -
123 -For example, for ROCm 6.2, suppose your gfx arch is `gfx90a`.
124 -Note to get your gfx architecture, run `rocminfo |grep gfx`.
125 -
126 - .. code-block:: console
127 -
128 - $ git clone https://github.com/ROCm/flash-attention.git
129 - $ cd flash-attention
130 - $ git checkout 3cea2fb
131 - $ git submodule update --init
132 - $ GPU_ARCHS="gfx90a" python3 setup.py install
133 - $ cd ..
134 -
135 -.. note::
136 - - You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
137 -
138 -3. Build vLLM.
139 -
140 - For example, vLLM on ROCM 6.2 can be built with the following steps:
141 -
142 - .. code-block:: console
143 -
144 - $ pip install --upgrade pip
145 -
146 - $ # Install PyTorch
147 - $ pip uninstall torch -y
148 - $ pip install --no-cache-dir --pre torch==2.6.0.dev20241024 --index-url https://download.pytorch.org/whl/nightly/rocm6.2
149 -
150 - $ # Build & install AMD SMI
151 - $ pip install /opt/rocm/share/amd_smi
152 -
153 - $ # Install dependencies
154 - $ pip install --upgrade numba scipy huggingface-hub[cli]
155 - $ pip install "numpy<2"
156 - $ pip install -r requirements-rocm.txt
157 -
158 - $ # Build vLLM for MI210/MI250/MI300.
159 - $ export PYTORCH_ROCM_ARCH="gfx90a;gfx942"
160 - $ python3 setup.py develop
161 -
162 -
163 - This may take 5-10 minutes. Currently, :code:`pip install .` does not work for ROCm installation.
164 -
165 -
166 -.. tip::
167 -
168 - - Triton flash attention is used by default. For benchmarking purposes, it is recommended to run a warm up step before collecting perf numbers.
169 - - Triton flash attention does not currently support sliding window attention. If using half precision, please use CK flash-attention for sliding window support.
170 - - To use CK flash-attention or PyTorch naive attention, please use this flag ``export VLLM_USE_TRITON_FLASH_ATTN=0`` to turn off triton flash attention.
171 - - The ROCm version of PyTorch, ideally, should match the ROCm driver version.
172 -
173 -
174 -.. tip::
175 - - For MI300x (gfx942) users, to achieve optimal performance, please refer to `MI300x tuning guide <https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/index.html>`_ for performance optimization and tuning tips on system and workflow level.
176 - For vLLM, please refer to `vLLM performance optimization <https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/workload.html#vllm-performance-optimization>`_.
177 -
178 -
docs/source/getting_started/arm-installation.md new
+46
@@ -0,0 +1,46 @@
1 +(installation-arm)=
2 +
3 +# Installation for ARM CPUs
4 +
5 +vLLM has been adapted to work on ARM64 CPUs with NEON support, leveraging the CPU backend initially developed for the x86 platform. This guide provides installation instructions specific to ARM. For additional details on supported features, refer to the x86 platform documentation covering:
6 +
7 +- CPU backend inference capabilities
8 +- Relevant runtime environment variables
9 +- Performance optimization tips
10 +
11 +ARM CPU backend currently supports Float32, FP16 and BFloat16 datatypes.
12 +Contents:
13 +
14 +1. [Requirements](#arm-backend-requirements)
15 +2. [Quick Start with Dockerfile](#arm-backend-quick-start-dockerfile)
16 +3. [Building from Source](#build-arm-backend-from-source)
17 +
18 +(arm-backend-requirements)=
19 +
20 +## Requirements
21 +
22 +- **Operating System**: Linux or macOS
23 +- **Compiler**: gcc/g++ >= 12.3.0 (optional, but recommended)
24 +- **Instruction Set Architecture (ISA)**: NEON support is required
25 +
26 +(arm-backend-quick-start-dockerfile)=
27 +
28 +## Quick Start with Dockerfile
29 +
30 +You can quickly set up vLLM on ARM using Docker:
31 +
32 +```console
33 +$ docker build -f Dockerfile.arm -t vllm-cpu-env --shm-size=4g .
34 +$ docker run -it \
35 + --rm \
36 + --network=host \
37 + --cpuset-cpus=<cpu-id-list, optional> \
38 + --cpuset-mems=<memory-node, optional> \
39 + vllm-cpu-env
40 +```
41 +
42 +(build-arm-backend-from-source)=
43 +
44 +## Building from Source
45 +
46 +To build vLLM from source on Ubuntu 22.04 or other Linux distributions, follow a similar process as with x86. Testing has been conducted on AWS Graviton3 instances for compatibility.
docs/source/getting_started/arm-installation.rst deleted
-50
@@ -1,50 +0,0 @@
1 -.. _installation_arm:
2 -
3 -Installation for ARM CPUs
4 -=========================
5 -
6 -vLLM has been adapted to work on ARM64 CPUs with NEON support, leveraging the CPU backend initially developed for the x86 platform. This guide provides installation instructions specific to ARM. For additional details on supported features, refer to the x86 platform documentation covering:
7 -
8 -* CPU backend inference capabilities
9 -* Relevant runtime environment variables
10 -* Performance optimization tips
11 -
12 -ARM CPU backend currently supports Float32, FP16 and BFloat16 datatypes.
13 -Contents:
14 -
15 -1. :ref:`Requirements <arm_backend_requirements>`
16 -2. :ref:`Quick Start with Dockerfile <arm_backend_quick_start_dockerfile>`
17 -3. :ref:`Building from Source <build_arm_backend_from_source>`
18 -
19 -.. _arm_backend_requirements:
20 -
21 -Requirements
22 -------------
23 -
24 -* **Operating System**: Linux or macOS
25 -* **Compiler**: gcc/g++ >= 12.3.0 (optional, but recommended)
26 -* **Instruction Set Architecture (ISA)**: NEON support is required
27 -
28 -.. _arm_backend_quick_start_dockerfile:
29 -
30 -Quick Start with Dockerfile
31 ----------------------------
32 -
33 -You can quickly set up vLLM on ARM using Docker:
34 -
35 -.. code-block:: console
36 -
37 - $ docker build -f Dockerfile.arm -t vllm-cpu-env --shm-size=4g .
38 - $ docker run -it \
39 - --rm \
40 - --network=host \
41 - --cpuset-cpus=<cpu-id-list, optional> \
42 - --cpuset-mems=<memory-node, optional> \
43 - vllm-cpu-env
44 -
45 -.. _build_arm_backend_from_source:
46 -
47 -Building from Source
48 ---------------------
49 -
50 -To build vLLM from source on Ubuntu 22.04 or other Linux distributions, follow a similar process as with x86. Testing has been conducted on AWS Graviton3 instances for compatibility.
docs/source/getting_started/cpu-installation.md new
+154
@@ -0,0 +1,154 @@
1 +(installation-cpu)=
2 +
3 +# Installation with CPU
4 +
5 +vLLM initially supports basic model inferencing and serving on x86 CPU platform, with data types FP32, FP16 and BF16. vLLM CPU backend supports the following vLLM features:
6 +
7 +- Tensor Parallel
8 +- Model Quantization (`INT8 W8A8, AWQ`)
9 +- Chunked-prefill
10 +- Prefix-caching
11 +- FP8-E5M2 KV-Caching (TODO)
12 +
13 +Table of contents:
14 +
15 +1. [Requirements](#cpu-backend-requirements)
16 +2. [Quick start using Dockerfile](#cpu-backend-quick-start-dockerfile)
17 +3. [Build from source](#build-cpu-backend-from-source)
18 +4. [Related runtime environment variables](#env-intro)
19 +5. [Intel Extension for PyTorch](#ipex-guidance)
20 +6. [Performance tips](#cpu-backend-performance-tips)
21 +
22 +(cpu-backend-requirements)=
23 +
24 +## Requirements
25 +
26 +- OS: Linux
27 +- Compiler: gcc/g++>=12.3.0 (optional, recommended)
28 +- Instruction set architecture (ISA) requirement: AVX512 (optional, recommended)
29 +
30 +(cpu-backend-quick-start-dockerfile)=
31 +
32 +## Quick start using Dockerfile
33 +
34 +```console
35 +$ docker build -f Dockerfile.cpu -t vllm-cpu-env --shm-size=4g .
36 +$ docker run -it \
37 + --rm \
38 + --network=host \
39 + --cpuset-cpus=<cpu-id-list, optional> \
40 + --cpuset-mems=<memory-node, optional> \
41 + vllm-cpu-env
42 +```
43 +
44 +(build-cpu-backend-from-source)=
45 +
46 +## Build from source
47 +
48 +- First, install recommended compiler. We recommend to use `gcc/g++ >= 12.3.0` as the default compiler to avoid potential problems. For example, on Ubuntu 22.4, you can run:
49 +
50 +```console
51 +$ sudo apt-get update -y
52 +$ sudo apt-get install -y gcc-12 g++-12 libnuma-dev
53 +$ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
54 +```
55 +
56 +- Second, install Python packages for vLLM CPU backend building:
57 +
58 +```console
59 +$ pip install --upgrade pip
60 +$ pip install cmake>=3.26 wheel packaging ninja "setuptools-scm>=8" numpy
61 +$ pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
62 +```
63 +
64 +- Finally, build and install vLLM CPU backend:
65 +
66 +```console
67 +$ VLLM_TARGET_DEVICE=cpu python setup.py install
68 +```
69 +
70 +```{note}
71 +- AVX512_BF16 is an extension ISA provides native BF16 data type conversion and vector product instructions, will brings some performance improvement compared with pure AVX512. The CPU backend build script will check the host CPU flags to determine whether to enable AVX512_BF16.
72 +- If you want to force enable AVX512_BF16 for the cross-compilation, please set environment variable VLLM_CPU_AVX512BF16=1 before the building.
73 +```
74 +
75 +(env-intro)=
76 +
77 +## Related runtime environment variables
78 +
79 +- `VLLM_CPU_KVCACHE_SPACE`: specify the KV Cache size (e.g, `VLLM_CPU_KVCACHE_SPACE=40` means 40 GB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users.
80 +- `VLLM_CPU_OMP_THREADS_BIND`: specify the CPU cores dedicated to the OpenMP threads. For example, `VLLM_CPU_OMP_THREADS_BIND=0-31` means there will be 32 OpenMP threads bound on 0-31 CPU cores. `VLLM_CPU_OMP_THREADS_BIND=0-31|32-63` means there will be 2 tensor parallel processes, 32 OpenMP threads of rank0 are bound on 0-31 CPU cores, and the OpenMP threads of rank1 are bound on 32-63 CPU cores.
81 +
82 +(ipex-guidance)=
83 +
84 +## Intel Extension for PyTorch
85 +
86 +- [Intel Extension for PyTorch (IPEX)](https://github.com/intel/intel-extension-for-pytorch) extends PyTorch with up-to-date features optimizations for an extra performance boost on Intel hardware.
87 +
88 +(cpu-backend-performance-tips)=
89 +
90 +## Performance tips
91 +
92 +- We highly recommend to use TCMalloc for high performance memory allocation and better cache locality. For example, on Ubuntu 22.4, you can run:
93 +
94 +```console
95 +$ sudo apt-get install libtcmalloc-minimal4 # install TCMalloc library
96 +$ find / -name *libtcmalloc* # find the dynamic link library path
97 +$ export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$LD_PRELOAD # prepend the library to LD_PRELOAD
98 +$ python examples/offline_inference.py # run vLLM
99 +```
100 +
101 +- When using the online serving, it is recommended to reserve 1-2 CPU cores for the serving framework to avoid CPU oversubscription. For example, on a platform with 32 physical CPU cores, reserving CPU 30 and 31 for the framework and using CPU 0-29 for OpenMP:
102 +
103 +```console
104 +$ export VLLM_CPU_KVCACHE_SPACE=40
105 +$ export VLLM_CPU_OMP_THREADS_BIND=0-29
106 +$ vllm serve facebook/opt-125m
107 +```
108 +
109 +- If using vLLM CPU backend on a machine with hyper-threading, it is recommended to bind only one OpenMP thread on each physical CPU core using `VLLM_CPU_OMP_THREADS_BIND`. On a hyper-threading enabled platform with 16 logical CPU cores / 8 physical CPU cores:
110 +
111 +```console
112 +$ lscpu -e # check the mapping between logical CPU cores and physical CPU cores
113 +
114 +# The "CPU" column means the logical CPU core IDs, and the "CORE" column means the physical core IDs. On this platform, two logical cores are sharing one physical core.
115 +CPU NODE SOCKET CORE L1d:L1i:L2:L3 ONLINE MAXMHZ MINMHZ MHZ
116 +0 0 0 0 0:0:0:0 yes 2401.0000 800.0000 800.000
117 +1 0 0 1 1:1:1:0 yes 2401.0000 800.0000 800.000
118 +2 0 0 2 2:2:2:0 yes 2401.0000 800.0000 800.000
119 +3 0 0 3 3:3:3:0 yes 2401.0000 800.0000 800.000
120 +4 0 0 4 4:4:4:0 yes 2401.0000 800.0000 800.000
121 +5 0 0 5 5:5:5:0 yes 2401.0000 800.0000 800.000
122 +6 0 0 6 6:6:6:0 yes 2401.0000 800.0000 800.000
123 +7 0 0 7 7:7:7:0 yes 2401.0000 800.0000 800.000
124 +8 0 0 0 0:0:0:0 yes 2401.0000 800.0000 800.000
125 +9 0 0 1 1:1:1:0 yes 2401.0000 800.0000 800.000
126 +10 0 0 2 2:2:2:0 yes 2401.0000 800.0000 800.000
127 +11 0 0 3 3:3:3:0 yes 2401.0000 800.0000 800.000
128 +12 0 0 4 4:4:4:0 yes 2401.0000 800.0000 800.000
129 +13 0 0 5 5:5:5:0 yes 2401.0000 800.0000 800.000
130 +14 0 0 6 6:6:6:0 yes 2401.0000 800.0000 800.000
131 +15 0 0 7 7:7:7:0 yes 2401.0000 800.0000 800.000
132 +
133 +# On this platform, it is recommend to only bind openMP threads on logical CPU cores 0-7 or 8-15
134 +$ export VLLM_CPU_OMP_THREADS_BIND=0-7
135 +$ python examples/offline_inference.py
136 +```
137 +
138 +- If using vLLM CPU backend on a multi-socket machine with NUMA, be aware to set CPU cores using `VLLM_CPU_OMP_THREADS_BIND` to avoid cross NUMA node memory access.
139 +
140 +## CPU Backend Considerations
141 +
142 +- The CPU backend significantly differs from the GPU backend since the vLLM architecture was originally optimized for GPU use. A number of optimizations are needed to enhance its performance.
143 +
144 +- Decouple the HTTP serving components from the inference components. In a GPU backend configuration, the HTTP serving and tokenization tasks operate on the CPU, while inference runs on the GPU, which typically does not pose a problem. However, in a CPU-based setup, the HTTP serving and tokenization can cause significant context switching and reduced cache efficiency. Therefore, it is strongly recommended to segregate these two components for improved performance.
145 +
146 +- On CPU based setup with NUMA enabled, the memory access performance may be largely impacted by the [topology](https://github.com/intel/intel-extension-for-pytorch/blob/main/docs/tutorials/performance_tuning/tuning_guide.md#non-uniform-memory-access-numa). For NUMA architecture, two optimizations are to recommended: Tensor Parallel or Data Parallel.
147 +
148 + - Using Tensor Parallel for a latency constraints deployment: following GPU backend design, a Megatron-LM's parallel algorithm will be used to shard the model, based on the number of NUMA nodes (e.g. TP = 2 for a two NUMA node system). With [TP feature on CPU](https://github.com/vllm-project/vllm/pull/6125) merged, Tensor Parallel is supported for serving and offline inferencing. In general each NUMA node is treated as one GPU card. Below is the example script to enable Tensor Parallel = 2 for serving:
149 +
150 + ```console
151 + $ VLLM_CPU_KVCACHE_SPACE=40 VLLM_CPU_OMP_THREADS_BIND="0-31|32-63" vllm serve meta-llama/Llama-2-7b-chat-hf -tp=2 --distributed-executor-backend mp
152 + ```
153 +
154 + - Using Data Parallel for maximum throughput: to launch an LLM serving endpoint on each NUMA node along with one additional load balancer to dispatch the requests to those endpoints. Common solutions like [Nginx](../serving/deploying_with_nginx) or HAProxy are recommended. Anyscale Ray project provides the feature on LLM [serving](https://docs.ray.io/en/latest/serve/index.html). Here is the example to setup a scalable LLM serving with [Ray Serve](https://github.com/intel/llm-on-ray/blob/main/docs/setup.md).
docs/source/getting_started/cpu-installation.rst deleted
-164
@@ -1,164 +0,0 @@
1 -.. _installation_cpu:
2 -
3 -Installation with CPU
4 -========================
5 -
6 -vLLM initially supports basic model inferencing and serving on x86 CPU platform, with data types FP32, FP16 and BF16. vLLM CPU backend supports the following vLLM features:
7 -
8 -- Tensor Parallel
9 -- Model Quantization (``INT8 W8A8, AWQ``)
10 -- Chunked-prefill
11 -- Prefix-caching
12 -- FP8-E5M2 KV-Caching (TODO)
13 -
14 -Table of contents:
15 -
16 -#. :ref:`Requirements <cpu_backend_requirements>`
17 -#. :ref:`Quick start using Dockerfile <cpu_backend_quick_start_dockerfile>`
18 -#. :ref:`Build from source <build_cpu_backend_from_source>`
19 -#. :ref:`Related runtime environment variables <env_intro>`
20 -#. :ref:`Intel Extension for PyTorch <ipex_guidance>`
21 -#. :ref:`Performance tips <cpu_backend_performance_tips>`
22 -
23 -.. _cpu_backend_requirements:
24 -
25 -Requirements
26 -------------
27 -
28 -* OS: Linux
29 -* Compiler: gcc/g++>=12.3.0 (optional, recommended)
30 -* Instruction set architecture (ISA) requirement: AVX512 (optional, recommended)
31 -
32 -.. _cpu_backend_quick_start_dockerfile:
33 -
34 -Quick start using Dockerfile
35 -----------------------------
36 -
37 -.. code-block:: console
38 -
39 - $ docker build -f Dockerfile.cpu -t vllm-cpu-env --shm-size=4g .
40 - $ docker run -it \
41 - --rm \
42 - --network=host \
43 - --cpuset-cpus=<cpu-id-list, optional> \
44 - --cpuset-mems=<memory-node, optional> \
45 - vllm-cpu-env
46 -
47 -.. _build_cpu_backend_from_source:
48 -
49 -Build from source
50 ------------------
51 -
52 -- First, install recommended compiler. We recommend to use ``gcc/g++ >= 12.3.0`` as the default compiler to avoid potential problems. For example, on Ubuntu 22.4, you can run:
53 -
54 -.. code-block:: console
55 -
56 - $ sudo apt-get update -y
57 - $ sudo apt-get install -y gcc-12 g++-12 libnuma-dev
58 - $ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
59 -
60 -- Second, install Python packages for vLLM CPU backend building:
61 -
62 -.. code-block:: console
63 -
64 - $ pip install --upgrade pip
65 - $ pip install cmake>=3.26 wheel packaging ninja "setuptools-scm>=8" numpy
66 - $ pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
67 -
68 -- Finally, build and install vLLM CPU backend:
69 -
70 -.. code-block:: console
71 -
72 - $ VLLM_TARGET_DEVICE=cpu python setup.py install
73 -
74 -.. note::
75 - - AVX512_BF16 is an extension ISA provides native BF16 data type conversion and vector product instructions, will brings some performance improvement compared with pure AVX512. The CPU backend build script will check the host CPU flags to determine whether to enable AVX512_BF16.
76 -
77 - - If you want to force enable AVX512_BF16 for the cross-compilation, please set environment variable VLLM_CPU_AVX512BF16=1 before the building.
78 -
79 -.. _env_intro:
80 -
81 -Related runtime environment variables
82 --------------------------------------
83 -
84 -- ``VLLM_CPU_KVCACHE_SPACE``: specify the KV Cache size (e.g, ``VLLM_CPU_KVCACHE_SPACE=40`` means 40 GB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users.
85 -
86 -- ``VLLM_CPU_OMP_THREADS_BIND``: specify the CPU cores dedicated to the OpenMP threads. For example, ``VLLM_CPU_OMP_THREADS_BIND=0-31`` means there will be 32 OpenMP threads bound on 0-31 CPU cores. ``VLLM_CPU_OMP_THREADS_BIND=0-31|32-63`` means there will be 2 tensor parallel processes, 32 OpenMP threads of rank0 are bound on 0-31 CPU cores, and the OpenMP threads of rank1 are bound on 32-63 CPU cores.
87 -
88 -.. _ipex_guidance:
89 -
90 -Intel Extension for PyTorch
91 ----------------------------
92 -
93 -- `Intel Extension for PyTorch (IPEX) <https://github.com/intel/intel-extension-for-pytorch>`_ extends PyTorch with up-to-date features optimizations for an extra performance boost on Intel hardware.
94 -
95 -.. _cpu_backend_performance_tips:
96 -
97 -Performance tips
98 ------------------
99 -
100 -- We highly recommend to use TCMalloc for high performance memory allocation and better cache locality. For example, on Ubuntu 22.4, you can run:
101 -
102 -.. code-block:: console
103 -
104 - $ sudo apt-get install libtcmalloc-minimal4 # install TCMalloc library
105 - $ find / -name *libtcmalloc* # find the dynamic link library path
106 - $ export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$LD_PRELOAD # prepend the library to LD_PRELOAD
107 - $ python examples/offline_inference.py # run vLLM
108 -
109 -- When using the online serving, it is recommended to reserve 1-2 CPU cores for the serving framework to avoid CPU oversubscription. For example, on a platform with 32 physical CPU cores, reserving CPU 30 and 31 for the framework and using CPU 0-29 for OpenMP:
110 -
111 -.. code-block:: console
112 -
113 - $ export VLLM_CPU_KVCACHE_SPACE=40
114 - $ export VLLM_CPU_OMP_THREADS_BIND=0-29
115 - $ vllm serve facebook/opt-125m
116 -
117 -- If using vLLM CPU backend on a machine with hyper-threading, it is recommended to bind only one OpenMP thread on each physical CPU core using ``VLLM_CPU_OMP_THREADS_BIND``. On a hyper-threading enabled platform with 16 logical CPU cores / 8 physical CPU cores:
118 -
119 -.. code-block:: console
120 -
121 - $ lscpu -e # check the mapping between logical CPU cores and physical CPU cores
122 -
123 - # The "CPU" column means the logical CPU core IDs, and the "CORE" column means the physical core IDs. On this platform, two logical cores are sharing one physical core.
124 - CPU NODE SOCKET CORE L1d:L1i:L2:L3 ONLINE MAXMHZ MINMHZ MHZ
125 - 0 0 0 0 0:0:0:0 yes 2401.0000 800.0000 800.000
126 - 1 0 0 1 1:1:1:0 yes 2401.0000 800.0000 800.000
127 - 2 0 0 2 2:2:2:0 yes 2401.0000 800.0000 800.000
128 - 3 0 0 3 3:3:3:0 yes 2401.0000 800.0000 800.000
129 - 4 0 0 4 4:4:4:0 yes 2401.0000 800.0000 800.000
130 - 5 0 0 5 5:5:5:0 yes 2401.0000 800.0000 800.000
131 - 6 0 0 6 6:6:6:0 yes 2401.0000 800.0000 800.000
132 - 7 0 0 7 7:7:7:0 yes 2401.0000 800.0000 800.000
133 - 8 0 0 0 0:0:0:0 yes 2401.0000 800.0000 800.000
134 - 9 0 0 1 1:1:1:0 yes 2401.0000 800.0000 800.000
135 - 10 0 0 2 2:2:2:0 yes 2401.0000 800.0000 800.000
136 - 11 0 0 3 3:3:3:0 yes 2401.0000 800.0000 800.000
137 - 12 0 0 4 4:4:4:0 yes 2401.0000 800.0000 800.000
138 - 13 0 0 5 5:5:5:0 yes 2401.0000 800.0000 800.000
139 - 14 0 0 6 6:6:6:0 yes 2401.0000 800.0000 800.000
140 - 15 0 0 7 7:7:7:0 yes 2401.0000 800.0000 800.000
141 -
142 - # On this platform, it is recommend to only bind openMP threads on logical CPU cores 0-7 or 8-15
143 - $ export VLLM_CPU_OMP_THREADS_BIND=0-7
144 - $ python examples/offline_inference.py
145 -
146 -- If using vLLM CPU backend on a multi-socket machine with NUMA, be aware to set CPU cores using ``VLLM_CPU_OMP_THREADS_BIND`` to avoid cross NUMA node memory access.
147 -
148 -CPU Backend Considerations
149 ---------------------------
150 -
151 -- The CPU backend significantly differs from the GPU backend since the vLLM architecture was originally optimized for GPU use. A number of optimizations are needed to enhance its performance.
152 -
153 -- Decouple the HTTP serving components from the inference components. In a GPU backend configuration, the HTTP serving and tokenization tasks operate on the CPU, while inference runs on the GPU, which typically does not pose a problem. However, in a CPU-based setup, the HTTP serving and tokenization can cause significant context switching and reduced cache efficiency. Therefore, it is strongly recommended to segregate these two components for improved performance.
154 -
155 -- On CPU based setup with NUMA enabled, the memory access performance may be largely impacted by the `topology <https://github.com/intel/intel-extension-for-pytorch/blob/main/docs/tutorials/performance_tuning/tuning_guide.md#non-uniform-memory-access-numa>`_. For NUMA architecture, two optimizations are to recommended: Tensor Parallel or Data Parallel.
156 -
157 - * Using Tensor Parallel for a latency constraints deployment: following GPU backend design, a Megatron-LM's parallel algorithm will be used to shard the model, based on the number of NUMA nodes (e.g. TP = 2 for a two NUMA node system). With `TP feature on CPU <https://github.com/vllm-project/vllm/pull/6125>`_ merged, Tensor Parallel is supported for serving and offline inferencing. In general each NUMA node is treated as one GPU card. Below is the example script to enable Tensor Parallel = 2 for serving:
158 -
159 - .. code-block:: console
160 -
161 - $ VLLM_CPU_KVCACHE_SPACE=40 VLLM_CPU_OMP_THREADS_BIND="0-31|32-63" vllm serve meta-llama/Llama-2-7b-chat-hf -tp=2 --distributed-executor-backend mp
162 -
163 -
164 - * Using Data Parallel for maximum throughput: to launch an LLM serving endpoint on each NUMA node along with one additional load balancer to dispatch the requests to those endpoints. Common solutions like `Nginx <../serving/deploying_with_nginx.html>`_ or HAProxy are recommended. Anyscale Ray project provides the feature on LLM `serving <https://docs.ray.io/en/latest/serve/index.html>`_. Here is the example to setup a scalable LLM serving with `Ray Serve <https://github.com/intel/llm-on-ray/blob/main/docs/setup.md>`_.
\ No newline at end of file
docs/source/getting_started/debugging.md new
+199
@@ -0,0 +1,199 @@
1 +(debugging)=
2 +
3 +# Debugging Tips
4 +
5 +This document outlines some debugging strategies you can consider. If you think you've discovered a bug, please [search existing issues](https://github.com/vllm-project/vllm/issues?q=is%3Aissue) first to see if it has already been reported. If not, please [file a new issue](https://github.com/vllm-project/vllm/issues/new/choose), providing as much relevant information as possible.
6 +
7 +```{note}
8 +Once you've debugged a problem, remember to turn off any debugging environment variables defined, or simply start a new shell to avoid being affected by lingering debugging settings. Otherwise, the system might be slow with debugging functionalities left activated.
9 +```
10 +
11 +## Hangs downloading a model
12 +
13 +If the model isn't already downloaded to disk, vLLM will download it from the internet which can take time and depend on your internet connection.
14 +It's recommended to download the model first using the [huggingface-cli](https://huggingface.co/docs/huggingface_hub/en/guides/cli) and passing the local path to the model to vLLM. This way, you can isolate the issue.
15 +
16 +## Hangs loading a model from disk
17 +
18 +If the model is large, it can take a long time to load it from disk. Pay attention to where you store the model. Some clusters have shared filesystems across nodes, e.g. a distributed filesystem or a network filesystem, which can be slow.
19 +It'd be better to store the model in a local disk. Additionally, have a look at the CPU memory usage, when the model is too large it might take a lot of CPU memory, slowing down the operating system because it needs to frequently swap between disk and memory.
20 +
21 +```{note}
22 +To isolate the model downloading and loading issue, you can use the `--load-format dummy` argument to skip loading the model weights. This way, you can check if the model downloading and loading is the bottleneck.
23 +```
24 +
25 +## Model is too large
26 +
27 +If the model is too large to fit in a single GPU, you might want to [consider tensor parallelism](https://docs.vllm.ai/en/latest/serving/distributed_serving.html#distributed-inference-and-serving) to split the model across multiple GPUs. In that case, every process will read the whole model and split it into chunks, which makes the disk reading time even longer (proportional to the size of tensor parallelism). You can convert the model checkpoint to a sharded checkpoint using [this example](https://docs.vllm.ai/en/latest/getting_started/examples/save_sharded_state.html) . The conversion process might take some time, but later you can load the sharded checkpoint much faster. The model loading time should remain constant regardless of the size of tensor parallelism.
28 +
29 +## Enable more logging
30 +
31 +If other strategies don't solve the problem, it's likely that the vLLM instance is stuck somewhere. You can use the following environment variables to help debug the issue:
32 +
33 +- `export VLLM_LOGGING_LEVEL=DEBUG` to turn on more logging.
34 +- `export CUDA_LAUNCH_BLOCKING=1` to identify which CUDA kernel is causing the problem.
35 +- `export NCCL_DEBUG=TRACE` to turn on more logging for NCCL.
36 +- `export VLLM_TRACE_FUNCTION=1` to record all function calls for inspection in the log files to tell which function crashes or hangs.
37 +
38 +## Incorrect network setup
39 +
40 +The vLLM instance cannot get the correct IP address if you have a complicated network config. You can find a log such as `DEBUG 06-10 21:32:17 parallel_state.py:88] world_size=8 rank=0 local_rank=0 distributed_init_method=tcp://xxx.xxx.xxx.xxx:54641 backend=nccl` and the IP address should be the correct one.
41 +If it's not, override the IP address using the environment variable `export VLLM_HOST_IP=<your_ip_address>`.
42 +
43 +You might also need to set `export NCCL_SOCKET_IFNAME=<your_network_interface>` and `export GLOO_SOCKET_IFNAME=<your_network_interface>` to specify the network interface for the IP address.
44 +
45 +## Error near `self.graph.replay()`
46 +
47 +If vLLM crashes and the error trace captures it somewhere around `self.graph.replay()` in `vllm/worker/model_runner.py`, it is a CUDA error inside CUDAGraph.
48 +To identify the particular CUDA operation that causes the error, you can add `--enforce-eager` to the command line, or `enforce_eager=True` to the {class}`~vllm.LLM` class to disable the CUDAGraph optimization and isolate the exact CUDA operation that causes the error.
49 +
50 +## Incorrect hardware/driver
51 +
52 +If GPU/CPU communication cannot be established, you can use the following Python script and follow the instructions below to confirm whether the GPU/CPU communication is working correctly.
53 +
54 +```python
55 +# Test PyTorch NCCL
56 +import torch
57 +import torch.distributed as dist
58 +dist.init_process_group(backend="nccl")
59 +local_rank = dist.get_rank() % torch.cuda.device_count()
60 +torch.cuda.set_device(local_rank)
61 +data = torch.FloatTensor([1,] * 128).to("cuda")
62 +dist.all_reduce(data, op=dist.ReduceOp.SUM)
63 +torch.cuda.synchronize()
64 +value = data.mean().item()
65 +world_size = dist.get_world_size()
66 +assert value == world_size, f"Expected {world_size}, got {value}"
67 +
68 +print("PyTorch NCCL is successful!")
69 +
70 +# Test PyTorch GLOO
71 +gloo_group = dist.new_group(ranks=list(range(world_size)), backend="gloo")
72 +cpu_data = torch.FloatTensor([1,] * 128)
73 +dist.all_reduce(cpu_data, op=dist.ReduceOp.SUM, group=gloo_group)
74 +value = cpu_data.mean().item()
75 +assert value == world_size, f"Expected {world_size}, got {value}"
76 +
77 +print("PyTorch GLOO is successful!")
78 +
79 +if world_size <= 1:
80 + exit()
81 +
82 +# Test vLLM NCCL, with cuda graph
83 +from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
84 +
85 +pynccl = PyNcclCommunicator(group=gloo_group, device=local_rank)
86 +# pynccl is enabled by default for 0.6.5+,
87 +# but for 0.6.4 and below, we need to enable it manually.
88 +# keep the code for backward compatibility when because people
89 +# prefer to read the latest documentation.
90 +pynccl.disabled = False
91 +
92 +s = torch.cuda.Stream()
93 +with torch.cuda.stream(s):
94 + data.fill_(1)
95 + pynccl.all_reduce(data, stream=s)
96 + value = data.mean().item()
97 + assert value == world_size, f"Expected {world_size}, got {value}"
98 +
99 +print("vLLM NCCL is successful!")
100 +
101 +g = torch.cuda.CUDAGraph()
102 +with torch.cuda.graph(cuda_graph=g, stream=s):
103 + pynccl.all_reduce(data, stream=torch.cuda.current_stream())
104 +
105 +data.fill_(1)
106 +g.replay()
107 +torch.cuda.current_stream().synchronize()
108 +value = data.mean().item()
109 +assert value == world_size, f"Expected {world_size}, got {value}"
110 +
111 +print("vLLM NCCL with cuda graph is successful!")
112 +
113 +dist.destroy_process_group(gloo_group)
114 +dist.destroy_process_group()
115 +```
116 +
117 +If you are testing with a single node, adjust `--nproc-per-node` to the number of GPUs you want to use:
118 +
119 +```console
120 +$ NCCL_DEBUG=TRACE torchrun --nproc-per-node=<number-of-GPUs> test.py
121 +```
122 +
123 +If you are testing with multi-nodes, adjust `--nproc-per-node` and `--nnodes` according to your setup and set `MASTER_ADDR` to the correct IP address of the master node, reachable from all nodes. Then, run:
124 +
125 +```console
126 +$ NCCL_DEBUG=TRACE torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=$MASTER_ADDR test.py
127 +```
128 +
129 +If the script runs successfully, you should see the message `sanity check is successful!`.
130 +
131 +If the test script hangs or crashes, usually it means the hardware/drivers are broken in some sense. You should try to contact your system administrator or hardware vendor for further assistance. As a common workaround, you can try to tune some NCCL environment variables, such as `export NCCL_P2P_DISABLE=1` to see if it helps. Please check [their documentation](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html) for more information. Please only use these environment variables as a temporary workaround, as they might affect the performance of the system. The best solution is still to fix the hardware/drivers so that the test script can run successfully.
132 +
133 +```{note}
134 +A multi-node environment is more complicated than a single-node one. If you see errors such as `torch.distributed.DistNetworkError`, it is likely that the network/DNS setup is incorrect. In that case, you can manually assign node rank and specify the IP via command line arguments:
135 +
136 +- In the first node, run `NCCL_DEBUG=TRACE torchrun --nnodes 2 --nproc-per-node=2 --node-rank 0 --master_addr $MASTER_ADDR test.py`.
137 +- In the second node, run `NCCL_DEBUG=TRACE torchrun --nnodes 2 --nproc-per-node=2 --node-rank 1 --master_addr $MASTER_ADDR test.py`.
138 +
139 +Adjust `--nproc-per-node`, `--nnodes`, and `--node-rank` according to your setup, being sure to execute different commands (with different `--node-rank`) on different nodes.
140 +```
141 +
142 +## Python multiprocessing
143 +
144 +### `RuntimeError` Exception
145 +
146 +If you have seen a warning in your logs like this:
147 +
148 +```console
149 +WARNING 12-11 14:50:37 multiproc_worker_utils.py:281] CUDA was previously
150 + initialized. We must use the `spawn` multiprocessing start method. Setting
151 + VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See
152 + https://docs.vllm.ai/en/latest/getting_started/debugging.html#python-multiprocessing
153 + for more information.
154 +```
155 +
156 +or an error from Python that looks like this:
157 +
158 +```console
159 +RuntimeError:
160 + An attempt has been made to start a new process before the
161 + current process has finished its bootstrapping phase.
162 +
163 + This probably means that you are not using fork to start your
164 + child processes and you have forgotten to use the proper idiom
165 + in the main module:
166 +
167 + if __name__ == '__main__':
168 + freeze_support()
169 + ...
170 +
171 + The "freeze_support()" line can be omitted if the program
172 + is not going to be frozen to produce an executable.
173 +
174 + To fix this issue, refer to the "Safe importing of main module"
175 + section in https://docs.python.org/3/library/multiprocessing.html
176 +```
177 +
178 +then you must update your Python code to guard usage of `vllm` behind a `if
179 +__name__ == '__main__':` block. For example, instead of this:
180 +
181 +```python
182 +import vllm
183 +
184 +llm = vllm.LLM(...)
185 +```
186 +
187 +try this instead:
188 +
189 +```python
190 +if __name__ == '__main__':
191 + import vllm
192 +
193 + llm = vllm.LLM(...)
194 +```
195 +
196 +## Known Issues
197 +
198 +- In `v0.5.2`, `v0.5.3`, and `v0.5.3.post1`, there is a bug caused by [zmq](https://github.com/zeromq/pyzmq/issues/2000) , which can occasionally cause vLLM to hang depending on the machine configuration. The solution is to upgrade to the latest version of `vllm` to include the [fix](https://github.com/vllm-project/vllm/pull/6759).
199 +- To circumvent a NCCL [bug](https://github.com/NVIDIA/nccl/issues/1234) , all vLLM processes will set an environment variable ``NCCL_CUMEM_ENABLE=0`` to disable NCCL's ``cuMem`` allocator. It does not affect performance but only gives memory benefits. When external processes want to set up a NCCL connection with vLLM's processes, they should also set this environment variable, otherwise, inconsistent environment setup will cause NCCL to hang or crash, as observed in the [RLHF integration](https://github.com/OpenRLHF/OpenRLHF/pull/604) and the [discussion](https://github.com/vllm-project/vllm/issues/5723#issuecomment-2554389656) .
docs/source/getting_started/debugging.rst deleted
-203
@@ -1,203 +0,0 @@
1 -.. _debugging:
2 -
3 -===============
4 -Debugging Tips
5 -===============
6 -
7 -This document outlines some debugging strategies you can consider. If you think you've discovered a bug, please `search existing issues <https://github.com/vllm-project/vllm/issues?q=is%3Aissue>`_ first to see if it has already been reported. If not, please `file a new issue <https://github.com/vllm-project/vllm/issues/new/choose>`_, providing as much relevant information as possible.
8 -
9 -.. note::
10 -
11 - Once you've debugged a problem, remember to turn off any debugging environment variables defined, or simply start a new shell to avoid being affected by lingering debugging settings. Otherwise, the system might be slow with debugging functionalities left activated.
12 -
13 -Hangs downloading a model
14 -----------------------------------------
15 -If the model isn't already downloaded to disk, vLLM will download it from the internet which can take time and depend on your internet connection.
16 -It's recommended to download the model first using the `huggingface-cli <https://huggingface.co/docs/huggingface_hub/en/guides/cli>`_ and passing the local path to the model to vLLM. This way, you can isolate the issue.
17 -
18 -Hangs loading a model from disk
19 -----------------------------------------
20 -If the model is large, it can take a long time to load it from disk. Pay attention to where you store the model. Some clusters have shared filesystems across nodes, e.g. a distributed filesystem or a network filesystem, which can be slow.
21 -It'd be better to store the model in a local disk. Additionally, have a look at the CPU memory usage, when the model is too large it might take a lot of CPU memory, slowing down the operating system because it needs to frequently swap between disk and memory.
22 -
23 -.. note::
24 -
25 - To isolate the model downloading and loading issue, you can use the ``--load-format dummy`` argument to skip loading the model weights. This way, you can check if the model downloading and loading is the bottleneck.
26 -
27 -Model is too large
28 -----------------------------------------
29 -If the model is too large to fit in a single GPU, you might want to `consider tensor parallelism <https://docs.vllm.ai/en/latest/serving/distributed_serving.html#distributed-inference-and-serving>`_ to split the model across multiple GPUs. In that case, every process will read the whole model and split it into chunks, which makes the disk reading time even longer (proportional to the size of tensor parallelism). You can convert the model checkpoint to a sharded checkpoint using `this example <https://docs.vllm.ai/en/latest/getting_started/examples/save_sharded_state.html>`_ . The conversion process might take some time, but later you can load the sharded checkpoint much faster. The model loading time should remain constant regardless of the size of tensor parallelism.
30 -
31 -Enable more logging
32 -----------------------------------------
33 -If other strategies don't solve the problem, it's likely that the vLLM instance is stuck somewhere. You can use the following environment variables to help debug the issue:
34 -
35 -- ``export VLLM_LOGGING_LEVEL=DEBUG`` to turn on more logging.
36 -- ``export CUDA_LAUNCH_BLOCKING=1`` to identify which CUDA kernel is causing the problem.
37 -- ``export NCCL_DEBUG=TRACE`` to turn on more logging for NCCL.
38 -- ``export VLLM_TRACE_FUNCTION=1`` to record all function calls for inspection in the log files to tell which function crashes or hangs.
39 -
40 -Incorrect network setup
41 -----------------------------------------
42 -The vLLM instance cannot get the correct IP address if you have a complicated network config. You can find a log such as ``DEBUG 06-10 21:32:17 parallel_state.py:88] world_size=8 rank=0 local_rank=0 distributed_init_method=tcp://xxx.xxx.xxx.xxx:54641 backend=nccl`` and the IP address should be the correct one.
43 -If it's not, override the IP address using the environment variable ``export VLLM_HOST_IP=<your_ip_address>``.
44 -
45 -You might also need to set ``export NCCL_SOCKET_IFNAME=<your_network_interface>`` and ``export GLOO_SOCKET_IFNAME=<your_network_interface>`` to specify the network interface for the IP address.
46 -
47 -Error near ``self.graph.replay()``
48 -----------------------------------------
49 -If vLLM crashes and the error trace captures it somewhere around ``self.graph.replay()`` in ``vllm/worker/model_runner.py``, it is a CUDA error inside CUDAGraph.
50 -To identify the particular CUDA operation that causes the error, you can add ``--enforce-eager`` to the command line, or ``enforce_eager=True`` to the :class:`~vllm.LLM` class to disable the CUDAGraph optimization and isolate the exact CUDA operation that causes the error.
51 -
52 -Incorrect hardware/driver
53 -----------------------------------------
54 -If GPU/CPU communication cannot be established, you can use the following Python script and follow the instructions below to confirm whether the GPU/CPU communication is working correctly.
55 -
56 -.. code-block:: python
57 -
58 - # Test PyTorch NCCL
59 - import torch
60 - import torch.distributed as dist
61 - dist.init_process_group(backend="nccl")
62 - local_rank = dist.get_rank() % torch.cuda.device_count()
63 - torch.cuda.set_device(local_rank)
64 - data = torch.FloatTensor([1,] * 128).to("cuda")
65 - dist.all_reduce(data, op=dist.ReduceOp.SUM)
66 - torch.cuda.synchronize()
67 - value = data.mean().item()
68 - world_size = dist.get_world_size()
69 - assert value == world_size, f"Expected {world_size}, got {value}"
70 -
71 - print("PyTorch NCCL is successful!")
72 -
73 - # Test PyTorch GLOO
74 - gloo_group = dist.new_group(ranks=list(range(world_size)), backend="gloo")
75 - cpu_data = torch.FloatTensor([1,] * 128)
76 - dist.all_reduce(cpu_data, op=dist.ReduceOp.SUM, group=gloo_group)
77 - value = cpu_data.mean().item()
78 - assert value == world_size, f"Expected {world_size}, got {value}"
79 -
80 - print("PyTorch GLOO is successful!")
81 -
82 - if world_size <= 1:
83 - exit()
84 -
85 - # Test vLLM NCCL, with cuda graph
86 - from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
87 -
88 - pynccl = PyNcclCommunicator(group=gloo_group, device=local_rank)
89 - # pynccl is enabled by default for 0.6.5+,
90 - # but for 0.6.4 and below, we need to enable it manually.
91 - # keep the code for backward compatibility when because people
92 - # prefer to read the latest documentation.
93 - pynccl.disabled = False
94 -
95 - s = torch.cuda.Stream()
96 - with torch.cuda.stream(s):
97 - data.fill_(1)
98 - pynccl.all_reduce(data, stream=s)
99 - value = data.mean().item()
100 - assert value == world_size, f"Expected {world_size}, got {value}"
101 -
102 - print("vLLM NCCL is successful!")
103 -
104 - g = torch.cuda.CUDAGraph()
105 - with torch.cuda.graph(cuda_graph=g, stream=s):
106 - pynccl.all_reduce(data, stream=torch.cuda.current_stream())
107 -
108 - data.fill_(1)
109 - g.replay()
110 - torch.cuda.current_stream().synchronize()
111 - value = data.mean().item()
112 - assert value == world_size, f"Expected {world_size}, got {value}"
113 -
114 - print("vLLM NCCL with cuda graph is successful!")
115 -
116 - dist.destroy_process_group(gloo_group)
117 - dist.destroy_process_group()
118 -
119 -If you are testing with a single node, adjust ``--nproc-per-node`` to the number of GPUs you want to use:
120 -
121 -.. code-block:: console
122 -
123 - $ NCCL_DEBUG=TRACE torchrun --nproc-per-node=<number-of-GPUs> test.py
124 -
125 -If you are testing with multi-nodes, adjust ``--nproc-per-node`` and ``--nnodes`` according to your setup and set ``MASTER_ADDR`` to the correct IP address of the master node, reachable from all nodes. Then, run:
126 -
127 -.. code-block:: console
128 -
129 - $ NCCL_DEBUG=TRACE torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=$MASTER_ADDR test.py
130 -
131 -If the script runs successfully, you should see the message ``sanity check is successful!``.
132 -
133 -If the test script hangs or crashes, usually it means the hardware/drivers are broken in some sense. You should try to contact your system administrator or hardware vendor for further assistance. As a common workaround, you can try to tune some NCCL environment variables, such as ``export NCCL_P2P_DISABLE=1`` to see if it helps. Please check `their documentation <https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html>`__ for more information. Please only use these environment variables as a temporary workaround, as they might affect the performance of the system. The best solution is still to fix the hardware/drivers so that the test script can run successfully.
134 -
135 -.. note::
136 -
137 - A multi-node environment is more complicated than a single-node one. If you see errors such as ``torch.distributed.DistNetworkError``, it is likely that the network/DNS setup is incorrect. In that case, you can manually assign node rank and specify the IP via command line arguments:
138 -
139 - - In the first node, run ``NCCL_DEBUG=TRACE torchrun --nnodes 2 --nproc-per-node=2 --node-rank 0 --master_addr $MASTER_ADDR test.py``.
140 - - In the second node, run ``NCCL_DEBUG=TRACE torchrun --nnodes 2 --nproc-per-node=2 --node-rank 1 --master_addr $MASTER_ADDR test.py``.
141 -
142 - Adjust ``--nproc-per-node``, ``--nnodes``, and ``--node-rank`` according to your setup, being sure to execute different commands (with different ``--node-rank``) on different nodes.
143 -
144 -Python multiprocessing
145 -----------------------
146 -
147 -`RuntimeError` Exception
148 -^^^^^^^^^^^^^^^^^^^^^^^^
149 -
150 -If you have seen a warning in your logs like this:
151 -
152 -.. code-block:: console
153 -
154 - WARNING 12-11 14:50:37 multiproc_worker_utils.py:281] CUDA was previously
155 - initialized. We must use the `spawn` multiprocessing start method. Setting
156 - VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See
157 - https://docs.vllm.ai/en/latest/getting_started/debugging.html#python-multiprocessing
158 - for more information.
159 -
160 -or an error from Python that looks like this:
161 -
162 -.. code-block:: console
163 -
164 - RuntimeError:
165 - An attempt has been made to start a new process before the
166 - current process has finished its bootstrapping phase.
167 -
168 - This probably means that you are not using fork to start your
169 - child processes and you have forgotten to use the proper idiom
170 - in the main module:
171 -
172 - if __name__ == '__main__':
173 - freeze_support()
174 - ...
175 -
176 - The "freeze_support()" line can be omitted if the program
177 - is not going to be frozen to produce an executable.
178 -
179 - To fix this issue, refer to the "Safe importing of main module"
180 - section in https://docs.python.org/3/library/multiprocessing.html
181 -
182 -then you must update your Python code to guard usage of ``vllm`` behind a ``if
183 -__name__ == '__main__':`` block. For example, instead of this:
184 -
185 -.. code-block:: python
186 -
187 - import vllm
188 -
189 - llm = vllm.LLM(...)
190 -
191 -try this instead:
192 -
193 -.. code-block:: python
194 -
195 - if __name__ == '__main__':
196 - import vllm
197 -
198 - llm = vllm.LLM(...)
199 -
200 -Known Issues
201 -----------------------------------------
202 -- In ``v0.5.2``, ``v0.5.3``, and ``v0.5.3.post1``, there is a bug caused by `zmq <https://github.com/zeromq/pyzmq/issues/2000>`_ , which can occasionally cause vLLM to hang depending on the machine configuration. The solution is to upgrade to the latest version of ``vllm`` to include the `fix <https://github.com/vllm-project/vllm/pull/6759>`_.
203 -- To circumvent a NCCL `bug <https://github.com/NVIDIA/nccl/issues/1234>`__ , all vLLM processes will set an environment variable ``NCCL_CUMEM_ENABLE=0`` to disable NCCL's ``cuMem`` allocator. It does not affect performance but only gives memory benefits. When external processes want to set up a NCCL connection with vLLM's processes, they should also set this environment variable, otherwise, inconsistent environment setup will cause NCCL to hang or crash, as observed in `the RLHF integration <https://github.com/OpenRLHF/OpenRLHF/pull/604>`__ and the `discussion <https://github.com/vllm-project/vllm/issues/5723#issuecomment-2554389656>`__ .
docs/source/getting_started/examples/examples_index.template.md new
+8
@@ -0,0 +1,8 @@
1 +# Examples
2 +
3 +```{toctree}
4 +:maxdepth: 1
5 +:caption: Scripts
6 +
7 +%EXAMPLE_DOCS%
8 +```
\ No newline at end of file
docs/source/getting_started/examples/examples_index.template.rst deleted
-8
@@ -1,8 +0,0 @@
1 -Examples
2 -=================================
3 -
4 -.. toctree::
5 - :maxdepth: 1
6 - :caption: Scripts
7 -
8 - %EXAMPLE_DOCS%
docs/source/getting_started/gaudi-installation.md new
+388
@@ -0,0 +1,388 @@
1 +# Installation with Intel® Gaudi® AI Accelerators
2 +
3 +This README provides instructions on running vLLM with Intel Gaudi devices.
4 +
5 +## Requirements and Installation
6 +
7 +Please follow the instructions provided in the [Gaudi Installation
8 +Guide](https://docs.habana.ai/en/latest/Installation_Guide/index.html)
9 +to set up the execution environment. To achieve the best performance,
10 +please follow the methods outlined in the [Optimizing Training Platform
11 +Guide](https://docs.habana.ai/en/latest/PyTorch/Model_Optimization_PyTorch/Optimization_in_Training_Platform.html).
12 +
13 +### Requirements
14 +
15 +- OS: Ubuntu 22.04 LTS
16 +- Python: 3.10
17 +- Intel Gaudi accelerator
18 +- Intel Gaudi software version 1.18.0
19 +
20 +### Quick start using Dockerfile
21 +
22 +```console
23 +$ docker build -f Dockerfile.hpu -t vllm-hpu-env .
24 +$ docker run -it --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --net=host --rm vllm-hpu-env
25 +```
26 +
27 +```{tip}
28 +If you're observing the following error: `docker: Error response from daemon: Unknown runtime specified habana.`, please refer to "Install Using Containers" section of [Intel Gaudi Software Stack and Driver Installation](https://docs.habana.ai/en/v1.18.0/Installation_Guide/Bare_Metal_Fresh_OS.html). Make sure you have `habana-container-runtime` package installed and that `habana` container runtime is registered.
29 +```
30 +
31 +### Build from source
32 +
33 +#### Environment verification
34 +
35 +To verify that the Intel Gaudi software was correctly installed, run:
36 +
37 +```console
38 +$ hl-smi # verify that hl-smi is in your PATH and each Gaudi accelerator is visible
39 +$ apt list --installed | grep habana # verify that habanalabs-firmware-tools, habanalabs-graph, habanalabs-rdma-core, habanalabs-thunk and habanalabs-container-runtime are installed
40 +$ pip list | grep habana # verify that habana-torch-plugin, habana-torch-dataloader, habana-pyhlml and habana-media-loader are installed
41 +$ pip list | grep neural # verify that neural_compressor is installed
42 +```
43 +
44 +Refer to [Intel Gaudi Software Stack
45 +Verification](https://docs.habana.ai/en/latest/Installation_Guide/SW_Verification.html#platform-upgrade)
46 +for more details.
47 +
48 +#### Run Docker Image
49 +
50 +It is highly recommended to use the latest Docker image from Intel Gaudi
51 +vault. Refer to the [Intel Gaudi
52 +documentation](https://docs.habana.ai/en/latest/Installation_Guide/Bare_Metal_Fresh_OS.html#pull-prebuilt-containers)
53 +for more details.
54 +
55 +Use the following commands to run a Docker image:
56 +
57 +```console
58 +$ docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest
59 +$ docker run -it --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --net=host --ipc=host vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest
60 +```
61 +
62 +#### Build and Install vLLM
63 +
64 +To build and install vLLM from source, run:
65 +
66 +```console
67 +$ git clone https://github.com/vllm-project/vllm.git
68 +$ cd vllm
69 +$ python setup.py develop
70 +```
71 +
72 +Currently, the latest features and performance optimizations are developed in Gaudi's [vLLM-fork](https://github.com/HabanaAI/vllm-fork) and we periodically upstream them to vLLM main repo. To install latest [HabanaAI/vLLM-fork](https://github.com/HabanaAI/vllm-fork), run the following:
73 +
74 +```console
75 +$ git clone https://github.com/HabanaAI/vllm-fork.git
76 +$ cd vllm-fork
77 +$ git checkout habana_main
78 +$ python setup.py develop
79 +```
80 +
81 +## Supported Features
82 +
83 +- [Offline batched
84 + inference](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#offline-batched-inference)
85 +- Online inference via [OpenAI-Compatible
86 + Server](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server)
87 +- HPU autodetection - no need to manually select device within vLLM
88 +- Paged KV cache with algorithms enabled for Intel Gaudi accelerators
89 +- Custom Intel Gaudi implementations of Paged Attention, KV cache ops,
90 + prefill attention, Root Mean Square Layer Normalization, Rotary
91 + Positional Encoding
92 +- Tensor parallelism support for multi-card inference
93 +- Inference with [HPU Graphs](https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html)
94 + for accelerating low-batch latency and throughput
95 +- Attention with Linear Biases (ALiBi)
96 +
97 +## Unsupported Features
98 +
99 +- Beam search
100 +- LoRA adapters
101 +- Quantization
102 +- Prefill chunking (mixed-batch inferencing)
103 +
104 +## Supported Configurations
105 +
106 +The following configurations have been validated to be function with
107 +Gaudi2 devices. Configurations that are not listed may or may not work.
108 +
109 +- [meta-llama/Llama-2-7b](https://huggingface.co/meta-llama/Llama-2-7b)
110 + on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
111 + datatype with random or greedy sampling
112 +- [meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf)
113 + on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
114 + datatype with random or greedy sampling
115 +- [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B)
116 + on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
117 + datatype with random or greedy sampling
118 +- [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct)
119 + on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
120 + datatype with random or greedy sampling
121 +- [meta-llama/Meta-Llama-3.1-8B](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B)
122 + on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
123 + datatype with random or greedy sampling
124 +- [meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct)
125 + on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
126 + datatype with random or greedy sampling
127 +- [meta-llama/Llama-2-70b](https://huggingface.co/meta-llama/Llama-2-70b)
128 + with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
129 +- [meta-llama/Llama-2-70b-chat-hf](https://huggingface.co/meta-llama/Llama-2-70b-chat-hf)
130 + with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
131 +- [meta-llama/Meta-Llama-3-70B](https://huggingface.co/meta-llama/Meta-Llama-3-70B)
132 + with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
133 +- [meta-llama/Meta-Llama-3-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct)
134 + with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
135 +- [meta-llama/Meta-Llama-3.1-70B](https://huggingface.co/meta-llama/Meta-Llama-3.1-70B)
136 + with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
137 +- [meta-llama/Meta-Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct)
138 + with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
139 +
140 +## Performance Tuning
141 +
142 +### Execution modes
143 +
144 +Currently in vLLM for HPU we support four execution modes, depending on selected HPU PyTorch Bridge backend (via `PT_HPU_LAZY_MODE` environment variable), and `--enforce-eager` flag.
145 +
146 +```{eval-rst}
147 +.. list-table:: vLLM execution modes
148 + :widths: 25 25 50
149 + :header-rows: 1
150 +
151 + * - ``PT_HPU_LAZY_MODE``
152 + - ``enforce_eager``
153 + - execution mode
154 + * - 0
155 + - 0
156 + - torch.compile
157 + * - 0
158 + - 1
159 + - PyTorch eager mode
160 + * - 1
161 + - 0
162 + - HPU Graphs
163 + * - 1
164 + - 1
165 + - PyTorch lazy mode
166 +```
167 +
168 +```{warning}
169 +In 1.18.0, all modes utilizing `PT_HPU_LAZY_MODE=0` are highly experimental and should be only used for validating functional correctness. Their performance will be improved in the next releases. For obtaining the best performance in 1.18.0, please use HPU Graphs, or PyTorch lazy mode.
170 +```
171 +
172 +### Bucketing mechanism
173 +
174 +Intel Gaudi accelerators work best when operating on models with fixed tensor shapes. [Intel Gaudi Graph Compiler](https://docs.habana.ai/en/latest/Gaudi_Overview/Intel_Gaudi_Software_Suite.html#graph-compiler-and-runtime) is responsible for generating optimized binary code that implements the given model topology on Gaudi. In its default configuration, the produced binary code may be heavily dependent on input and output tensor shapes, and can require graph recompilation when encountering differently shaped tensors within the same topology. While the resulting binaries utilize Gaudi efficiently, the compilation itself may introduce a noticeable overhead in end-to-end execution.
175 +In a dynamic inference serving scenario, there is a need to minimize the number of graph compilations and reduce the risk of graph compilation occurring during server runtime. Currently it is achieved by "bucketing" model's forward pass across two dimensions - `batch_size` and `sequence_length`.
176 +
177 +```{note}
178 +Bucketing allows us to reduce the number of required graphs significantly, but it does not handle any graph compilation and device code generation - this is done in warmup and HPUGraph capture phase.
179 +```
180 +
181 +Bucketing ranges are determined with 3 parameters - `min`, `step` and `max`. They can be set separately for prompt and decode phase, and for batch size and sequence length dimension. These parameters can be observed in logs during vLLM startup:
182 +
183 +```
184 +INFO 08-01 21:37:59 hpu_model_runner.py:493] Prompt bucket config (min, step, max_warmup) bs:[1, 32, 4], seq:[128, 128, 1024]
185 +INFO 08-01 21:37:59 hpu_model_runner.py:499] Generated 24 prompt buckets: [(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (2, 768), (2, 896), (2, 1024), (4, 128), (4, 256), (4, 384), (4, 512), (4, 640), (4, 768), (4, 896), (4, 1024)]
186 +INFO 08-01 21:37:59 hpu_model_runner.py:504] Decode bucket config (min, step, max_warmup) bs:[1, 128, 4], seq:[128, 128, 2048]
187 +INFO 08-01 21:37:59 hpu_model_runner.py:509] Generated 48 decode buckets: [(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (1, 1152), (1, 1280), (1, 1408), (1, 1536), (1, 1664), (1, 1792), (1, 1920), (1, 2048), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (2, 768), (2, 896), (2, 1024), (2, 1152), (2, 1280), (2, 1408), (2, 1536), (2, 1664), (2, 1792), (2, 1920), (2, 2048), (4, 128), (4, 256), (4, 384), (4, 512), (4, 640), (4, 768), (4, 896), (4, 1024), (4, 1152), (4, 1280), (4, 1408), (4, 1536), (4, 1664), (4, 1792), (4, 1920), (4, 2048)]
188 +```
189 +
190 +`min` determines the lowest value of the bucket. `step` determines the interval between buckets, and `max` determines the upper bound of the bucket. Furthermore, interval between `min` and `step` has special handling - `min` gets multiplied by consecutive powers of two, until `step` gets reached. We call this the ramp-up phase and it is used for handling lower batch sizes with minimum wastage, while allowing larger padding on larger batch sizes.
191 +
192 +Example (with ramp-up)
193 +
194 +```
195 +min = 2, step = 32, max = 64
196 +=> ramp_up = (2, 4, 8, 16)
197 +=> stable = (32, 64)
198 +=> buckets = ramp_up + stable => (2, 4, 8, 16, 32, 64)
199 +```
200 +
201 +Example (without ramp-up)
202 +
203 +```
204 +min = 128, step = 128, max = 512
205 +=> ramp_up = ()
206 +=> stable = (128, 256, 384, 512)
207 +=> buckets = ramp_up + stable => (128, 256, 384, 512)
208 +```
209 +
210 +In the logged scenario, 24 buckets were generated for prompt (prefill) runs, and 48 buckets for decode runs. Each bucket corresponds to a separate optimized device binary for a given model with specified tensor shapes. Whenever a batch of requests is processed, it is padded across batch and sequence length dimension to the smallest possible bucket.
211 +
212 +```{warning}
213 +If a request exceeds maximum bucket size in any dimension, it will be processed without padding, and its processing may require a graph compilation, potentially significantly increasing end-to-end latency. The boundaries of the buckets are user-configurable via environment variables, and upper bucket boundaries can be increased to avoid such scenario.
214 +```
215 +
216 +As an example, if a request of 3 sequences, with max sequence length of 412 comes in to an idle vLLM server, it will be padded executed as `(4, 512)` prefill bucket, as `batch_size` (number of sequences) will be padded to 4 (closest batch_size dimension higher than 3), and max sequence length will be padded to 512 (closest sequence length dimension higher than 412). After prefill stage, it will be executed as `(4, 512)` decode bucket and will continue as that bucket until either batch dimension changes (due to request being finished) - in which case it will become a `(2, 512)` bucket, or context length increases above 512 tokens, in which case it will become `(4, 640)` bucket.
217 +
218 +```{note}
219 +Bucketing is transparent to a client - padding in sequence length dimension is never returned to the client, and padding in batch dimension does not create new requests.
220 +```
221 +
222 +### Warmup
223 +
224 +Warmup is an optional, but highly recommended step occurring before vLLM server starts listening. It executes a forward pass for each bucket with dummy data. The goal is to pre-compile all graphs and not incur any graph compilation overheads within bucket boundaries during server runtime. Each warmup step is logged during vLLM startup:
225 +
226 +```
227 +INFO 08-01 22:26:47 hpu_model_runner.py:1066] [Warmup][Prompt][1/24] batch_size:4 seq_len:1024 free_mem:79.16 GiB
228 +INFO 08-01 22:26:47 hpu_model_runner.py:1066] [Warmup][Prompt][2/24] batch_size:4 seq_len:896 free_mem:55.43 GiB
229 +INFO 08-01 22:26:48 hpu_model_runner.py:1066] [Warmup][Prompt][3/24] batch_size:4 seq_len:768 free_mem:55.43 GiB
230 +...
231 +INFO 08-01 22:26:59 hpu_model_runner.py:1066] [Warmup][Prompt][24/24] batch_size:1 seq_len:128 free_mem:55.43 GiB
232 +INFO 08-01 22:27:00 hpu_model_runner.py:1066] [Warmup][Decode][1/48] batch_size:4 seq_len:2048 free_mem:55.43 GiB
233 +INFO 08-01 22:27:00 hpu_model_runner.py:1066] [Warmup][Decode][2/48] batch_size:4 seq_len:1920 free_mem:55.43 GiB
234 +INFO 08-01 22:27:01 hpu_model_runner.py:1066] [Warmup][Decode][3/48] batch_size:4 seq_len:1792 free_mem:55.43 GiB
235 +...
236 +INFO 08-01 22:27:16 hpu_model_runner.py:1066] [Warmup][Decode][47/48] batch_size:2 seq_len:128 free_mem:55.43 GiB
237 +INFO 08-01 22:27:16 hpu_model_runner.py:1066] [Warmup][Decode][48/48] batch_size:1 seq_len:128 free_mem:55.43 GiB
238 +```
239 +
240 +This example uses the same buckets as in *Bucketing mechanism* section. Each output line corresponds to execution of a single bucket. When bucket is executed for the first time, its graph is compiled and can be reused later on, skipping further graph compilations.
241 +
242 +```{tip}
243 +Compiling all the buckets might take some time and can be turned off with `VLLM_SKIP_WARMUP=true` environment variable. Keep in mind that if you do that, you may face graph compilations once executing a given bucket for the first time. It is fine to disable warmup for development, but it's highly recommended to enable it in deployment.
244 +```
245 +
246 +### HPU Graph capture
247 +
248 +[HPU Graphs](https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html) are currently the most performant execution method of vLLM on Intel Gaudi. When HPU Graphs are enabled, execution graphs will be traced (recorded) ahead of time (after performing warmup), to be later replayed during inference, significantly reducing host overheads. Recording can take large amounts of memory, which needs to be taken into account when allocating KV cache. Enabling HPU Graphs will impact the number of available KV cache blocks, but vLLM provides user-configurable variables to control memory management.
249 +
250 +When HPU Graphs are being used, they share the common memory pool ("usable memory") as KV cache, determined by `gpu_memory_utilization` flag (`0.9` by default).
251 +Before KV cache gets allocated, model weights are loaded onto the device, and a forward pass of the model is executed on dummy data, to estimate memory usage.
252 +Only after that, `gpu_memory_utilization` flag is utilized - at its default value, will mark 90% of free device memory at that point as usable.
253 +Next, KV cache gets allocated, model is warmed up, and HPU Graphs are captured.
254 +Environment variable `VLLM_GRAPH_RESERVED_MEM` defines the ratio of memory reserved for HPU Graphs capture.
255 +With its default value (`VLLM_GRAPH_RESERVED_MEM=0.1`), 10% of usable memory will be reserved for graph capture (later referred to as "usable graph memory"), and the remaining 90% will be utilized for KV cache.
256 +Environment variable `VLLM_GRAPH_PROMPT_RATIO` determines the ratio of usable graph memory reserved for prefill and decode graphs. By default (`VLLM_GRAPH_PROMPT_RATIO=0.3`), both stages have equal memory constraints.
257 +Lower value corresponds to less usable graph memory reserved for prefill stage, e.g. `VLLM_GRAPH_PROMPT_RATIO=0.2` will reserve 20% of usable graph memory for prefill graphs, and 80% of usable graph memory for decode graphs.
258 +
259 +```{note}
260 +`gpu_memory_utilization` does not correspond to the absolute memory usage across HPU. It specifies the memory margin after loading the model and performing a profile run. If device has 100 GiB of total memory, and 50 GiB of free memory after loading model weights and executing profiling run, `gpu_memory_utilization` at its default value will mark 90% of 50 GiB as usable, leaving 5 GiB of margin, regardless of total device memory.
261 +```
262 +
263 +User can also configure the strategy for capturing HPU Graphs for prompt and decode stages separately. Strategy affects the order of capturing graphs. There are two strategies implemented:
264 +\- `max_bs` - graph capture queue will sorted in descending order by their batch sizes. Buckets with equal batch sizes are sorted by sequence length in ascending order (e.g. `(64, 128)`, `(64, 256)`, `(32, 128)`, `(32, 256)`, `(1, 128)`, `(1,256)`), default strategy for decode
265 +\- `min_tokens` - graph capture queue will be sorted in ascending order by the number of tokens each graph processes (`batch_size*sequence_length`), default strategy for prompt
266 +
267 +When there's large amount of requests pending, vLLM scheduler will attempt to fill the maximum batch size for decode as soon as possible. When a request is finished, decode batch size decreases. When that happens, vLLM will attempt to schedule a prefill iteration for requests in the waiting queue, to fill the decode batch size to its previous state. This means that in a full load scenario, decode batch size is often at its maximum, which makes large batch size HPU Graphs crucial to capture, as reflected by `max_bs` strategy. On the other hand, prefills will be executed most frequently with very low batch sizes (1-4), which is reflected in `min_tokens` strategy.
268 +
269 +```{note}
270 +`VLLM_GRAPH_PROMPT_RATIO` does not set a hard limit on memory taken by graphs for each stage (prefill and decode). vLLM will first attempt to use up entirety of usable prefill graph memory (usable graph memory * `VLLM_GRAPH_PROMPT_RATIO`) for capturing prefill HPU Graphs, next it will attempt do the same for decode graphs and usable decode graph memory pool. If one stage is fully captured, and there is unused memory left within usable graph memory pool, vLLM will attempt further graph capture for the other stage, until no more HPU Graphs can be captured without exceeding reserved memory pool. The behavior on that mechanism can be observed in the example below.
271 +```
272 +
273 +Each described step is logged by vLLM server, as follows (negative values correspond to memory being released):
274 +
275 +```
276 +INFO 08-02 17:37:44 hpu_model_runner.py:493] Prompt bucket config (min, step, max_warmup) bs:[1, 32, 4], seq:[128, 128, 1024]
277 +INFO 08-02 17:37:44 hpu_model_runner.py:499] Generated 24 prompt buckets: [(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (2, 768), (2, 896), (2, 1024), (4, 128), (4, 256), (4, 384), (4, 512), (4, 640), (4, 768), (4, 896), (4, 1024)]
278 +INFO 08-02 17:37:44 hpu_model_runner.py:504] Decode bucket config (min, step, max_warmup) bs:[1, 128, 4], seq:[128, 128, 2048]
279 +INFO 08-02 17:37:44 hpu_model_runner.py:509] Generated 48 decode buckets: [(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (1, 1152), (1, 1280), (1, 1408), (1, 1536), (1, 1664), (1, 1792), (1, 1920), (1, 2048), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (2, 768), (2, 896), (2, 1024), (2, 1152), (2, 1280), (2, 1408), (2, 1536), (2, 1664), (2, 1792), (2, 1920), (2, 2048), (4, 128), (4, 256), (4, 384), (4, 512), (4, 640), (4, 768), (4, 896), (4, 1024), (4, 1152), (4, 1280), (4, 1408), (4, 1536), (4, 1664), (4, 1792), (4, 1920), (4, 2048)]
280 +INFO 08-02 17:37:52 hpu_model_runner.py:430] Pre-loading model weights on hpu:0 took 14.97 GiB of device memory (14.97 GiB/94.62 GiB used) and 2.95 GiB of host memory (475.2 GiB/1007 GiB used)
281 +INFO 08-02 17:37:52 hpu_model_runner.py:438] Wrapping in HPU Graph took 0 B of device memory (14.97 GiB/94.62 GiB used) and -252 KiB of host memory (475.2 GiB/1007 GiB used)
282 +INFO 08-02 17:37:52 hpu_model_runner.py:442] Loading model weights took in total 14.97 GiB of device memory (14.97 GiB/94.62 GiB used) and 2.95 GiB of host memory (475.2 GiB/1007 GiB used)
283 +INFO 08-02 17:37:54 hpu_worker.py:134] Model profiling run took 504 MiB of device memory (15.46 GiB/94.62 GiB used) and 180.9 MiB of host memory (475.4 GiB/1007 GiB used)
284 +INFO 08-02 17:37:54 hpu_worker.py:158] Free device memory: 79.16 GiB, 39.58 GiB usable (gpu_memory_utilization=0.5), 15.83 GiB reserved for HPUGraphs (VLLM_GRAPH_RESERVED_MEM=0.4), 23.75 GiB reserved for KV cache
285 +INFO 08-02 17:37:54 hpu_executor.py:85] # HPU blocks: 1519, # CPU blocks: 0
286 +INFO 08-02 17:37:54 hpu_worker.py:190] Initializing cache engine took 23.73 GiB of device memory (39.2 GiB/94.62 GiB used) and -1.238 MiB of host memory (475.4 GiB/1007 GiB used)
287 +INFO 08-02 17:37:54 hpu_model_runner.py:1066] [Warmup][Prompt][1/24] batch_size:4 seq_len:1024 free_mem:55.43 GiB
288 +...
289 +INFO 08-02 17:38:22 hpu_model_runner.py:1066] [Warmup][Decode][48/48] batch_size:1 seq_len:128 free_mem:55.43 GiB
290 +INFO 08-02 17:38:22 hpu_model_runner.py:1159] Using 15.85 GiB/55.43 GiB of free device memory for HPUGraphs, 7.923 GiB for prompt and 7.923 GiB for decode (VLLM_GRAPH_PROMPT_RATIO=0.3)
291 +INFO 08-02 17:38:22 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][1/24] batch_size:1 seq_len:128 free_mem:55.43 GiB
292 +...
293 +INFO 08-02 17:38:26 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][11/24] batch_size:1 seq_len:896 free_mem:48.77 GiB
294 +INFO 08-02 17:38:27 hpu_model_runner.py:1066] [Warmup][Graph/Decode][1/48] batch_size:4 seq_len:128 free_mem:47.51 GiB
295 +...
296 +INFO 08-02 17:38:41 hpu_model_runner.py:1066] [Warmup][Graph/Decode][48/48] batch_size:1 seq_len:2048 free_mem:47.35 GiB
297 +INFO 08-02 17:38:41 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][12/24] batch_size:4 seq_len:256 free_mem:47.35 GiB
298 +INFO 08-02 17:38:42 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][13/24] batch_size:2 seq_len:512 free_mem:45.91 GiB
299 +INFO 08-02 17:38:42 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][14/24] batch_size:1 seq_len:1024 free_mem:44.48 GiB
300 +INFO 08-02 17:38:43 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][15/24] batch_size:2 seq_len:640 free_mem:43.03 GiB
301 +INFO 08-02 17:38:43 hpu_model_runner.py:1128] Graph/Prompt captured:15 (62.5%) used_mem:14.03 GiB buckets:[(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (4, 128), (4, 256)]
302 +INFO 08-02 17:38:43 hpu_model_runner.py:1128] Graph/Decode captured:48 (100.0%) used_mem:161.9 MiB buckets:[(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (1, 1152), (1, 1280), (1, 1408), (1, 1536), (1, 1664), (1, 1792), (1, 1920), (1, 2048), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (2, 768), (2, 896), (2, 1024), (2, 1152), (2, 1280), (2, 1408), (2, 1536), (2, 1664), (2, 1792), (2, 1920), (2, 2048), (4, 128), (4, 256), (4, 384), (4, 512), (4, 640), (4, 768), (4, 896), (4, 1024), (4, 1152), (4, 1280), (4, 1408), (4, 1536), (4, 1664), (4, 1792), (4, 1920), (4, 2048)]
303 +INFO 08-02 17:38:43 hpu_model_runner.py:1206] Warmup finished in 49 secs, allocated 14.19 GiB of device memory
304 +INFO 08-02 17:38:43 hpu_executor.py:91] init_cache_engine took 37.92 GiB of device memory (53.39 GiB/94.62 GiB used) and 57.86 MiB of host memory (475.4 GiB/1007 GiB used)
305 +```
306 +
307 +### Recommended vLLM Parameters
308 +
309 +- We recommend running inference on Gaudi 2 with `block_size` of 128
310 + for BF16 data type. Using default values (16, 32) might lead to
311 + sub-optimal performance due to Matrix Multiplication Engine
312 + under-utilization (see [Gaudi
313 + Architecture](https://docs.habana.ai/en/latest/Gaudi_Overview/Gaudi_Architecture.html)).
314 +- For max throughput on Llama 7B, we recommend running with batch size
315 + of 128 or 256 and max context length of 2048 with HPU Graphs enabled.
316 + If you encounter out-of-memory issues, see troubleshooting section.
317 +
318 +### Environment variables
319 +
320 +**Diagnostic and profiling knobs:**
321 +
322 +- `VLLM_PROFILER_ENABLED`: if `true`, high level profiler will be enabled. Resulting JSON traces can be viewed in [perfetto.habana.ai](https://perfetto.habana.ai/#!/viewer). Disabled by default.
323 +- `VLLM_HPU_LOG_STEP_GRAPH_COMPILATION`: if `true`, will log graph compilations per each vLLM engine step, only when there was any - highly recommended to use alongside `PT_HPU_METRICS_GC_DETAILS=1`. Disabled by default.
324 +- `VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL`: if `true`, will log graph compilations per each vLLM engine step, always, even if there were none. Disabled by default.
325 +- `VLLM_HPU_LOG_STEP_CPU_FALLBACKS`: if `true`, will log cpu fallbacks per each vLLM engine step, only when there was any. Disabled by default.
326 +- `VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL`: if `true`, will log cpu fallbacks per each vLLM engine step, always, even if there were none. Disabled by default.
327 +
328 +**Performance tuning knobs:**
329 +
330 +- `VLLM_SKIP_WARMUP`: if `true`, warmup will be skipped, `false` by default
331 +
332 +- `VLLM_GRAPH_RESERVED_MEM`: percentage of memory dedicated for HPUGraph capture, `0.1` by default
333 +
334 +- `VLLM_GRAPH_PROMPT_RATIO`: percentage of reserved graph memory dedicated for prompt graphs, `0.3` by default
335 +
336 +- `VLLM_GRAPH_PROMPT_STRATEGY`: strategy determining order of prompt graph capture, `min_tokens` or `max_bs`, `min_tokens` by default
337 +
338 +- `VLLM_GRAPH_DECODE_STRATEGY`: strategy determining order of decode graph capture, `min_tokens` or `max_bs`, `max_bs` by default
339 +
340 +- `VLLM_{phase}_{dim}_BUCKET_{param}` - collection of 12 environment variables configuring ranges of bucketing mechanism
341 +
342 + - `{phase}` is either `PROMPT` or `DECODE`
343 +
344 + - `{dim}` is either `BS`, `SEQ` or `BLOCK`
345 +
346 + - `{param}` is either `MIN`, `STEP` or `MAX`
347 +
348 + - Default values:
349 +
350 + - Prompt:
351 + : - batch size min (`VLLM_PROMPT_BS_BUCKET_MIN`): `1`
352 + - batch size step (`VLLM_PROMPT_BS_BUCKET_STEP`): `min(max_num_seqs, 32)`
353 + - batch size max (`VLLM_PROMPT_BS_BUCKET_MAX`): `min(max_num_seqs, 64)`
354 + - sequence length min (`VLLM_PROMPT_SEQ_BUCKET_MIN`): `block_size`
355 + - sequence length step (`VLLM_PROMPT_SEQ_BUCKET_STEP`): `block_size`
356 + - sequence length max (`VLLM_PROMPT_SEQ_BUCKET_MAX`): `max_model_len`
357 + - Decode:
358 + : - batch size min (`VLLM_DECODE_BS_BUCKET_MIN`): `1`
359 + - batch size step (`VLLM_DECODE_BS_BUCKET_STEP`): `min(max_num_seqs, 32)`
360 + - batch size max (`VLLM_DECODE_BS_BUCKET_MAX`): `max_num_seqs`
361 + - sequence length min (`VLLM_DECODE_BLOCK_BUCKET_MIN`): `block_size`
362 + - sequence length step (`VLLM_DECODE_BLOCK_BUCKET_STEP`): `block_size`
363 + - sequence length max (`VLLM_DECODE_BLOCK_BUCKET_MAX`): `max(128, (max_num_seqs*max_model_len)/block_size)`
364 +
365 +Additionally, there are HPU PyTorch Bridge environment variables impacting vLLM execution:
366 +
367 +- `PT_HPU_LAZY_MODE`: if `0`, PyTorch Eager backend for Gaudi will be used, if `1` PyTorch Lazy backend for Gaudi will be used, `1` is default
368 +- `PT_HPU_ENABLE_LAZY_COLLECTIVES`: required to be `true` for tensor parallel inference with HPU Graphs
369 +
370 +## Troubleshooting: Tweaking HPU Graphs
371 +
372 +If you experience device out-of-memory issues or want to attempt
373 +inference at higher batch sizes, try tweaking HPU Graphs by following
374 +the below:
375 +
376 +- Tweak `gpu_memory_utilization` knob. It will decrease the
377 + allocation of KV cache, leaving some headroom for capturing graphs
378 + with larger batch size. By default `gpu_memory_utilization` is set
379 + to 0.9. It attempts to allocate ~90% of HBM left for KV cache after
380 + short profiling run. Note that decreasing reduces the number of KV
381 + cache blocks you have available, and therefore reduces the effective
382 + maximum number of tokens you can handle at a given time.
383 +- If this method is not efficient, you can disable `HPUGraph`
384 + completely. With HPU Graphs disabled, you are trading latency and
385 + throughput at lower batches for potentially higher throughput on
386 + higher batches. You can do that by adding `--enforce-eager` flag to
387 + server (for online inference), or by passing `enforce_eager=True`
388 + argument to LLM constructor (for offline inference).
docs/source/getting_started/gaudi-installation.rst deleted
-402
@@ -1,402 +0,0 @@
1 -Installation with Intel® Gaudi® AI Accelerators
2 -===============================================
3 -
4 -This README provides instructions on running vLLM with Intel Gaudi devices.
5 -
6 -Requirements and Installation
7 ------------------------------
8 -
9 -Please follow the instructions provided in the `Gaudi Installation
10 -Guide <https://docs.habana.ai/en/latest/Installation_Guide/index.html>`__
11 -to set up the execution environment. To achieve the best performance,
12 -please follow the methods outlined in the `Optimizing Training Platform
13 -Guide <https://docs.habana.ai/en/latest/PyTorch/Model_Optimization_PyTorch/Optimization_in_Training_Platform.html>`__.
14 -
15 -Requirements
16 -~~~~~~~~~~~~
17 -
18 -- OS: Ubuntu 22.04 LTS
19 -- Python: 3.10
20 -- Intel Gaudi accelerator
21 -- Intel Gaudi software version 1.18.0
22 -
23 -
24 -Quick start using Dockerfile
25 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~
26 -.. code:: console
27 -
28 - $ docker build -f Dockerfile.hpu -t vllm-hpu-env .
29 - $ docker run -it --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --net=host --rm vllm-hpu-env
30 -
31 -
32 -.. tip::
33 - If you're observing the following error: ``docker: Error response from daemon: Unknown runtime specified habana.``, please refer to "Install Using Containers" section of `Intel Gaudi Software Stack and Driver Installation <https://docs.habana.ai/en/v1.18.0/Installation_Guide/Bare_Metal_Fresh_OS.html>`__. Make sure you have ``habana-container-runtime`` package installed and that ``habana`` container runtime is registered.
34 -
35 -
36 -Build from source
37 -~~~~~~~~~~~~~~~~~
38 -
39 -Environment verification
40 -^^^^^^^^^^^^^^^^^^^^^^^^
41 -
42 -To verify that the Intel Gaudi software was correctly installed, run:
43 -
44 -.. code:: console
45 -
46 - $ hl-smi # verify that hl-smi is in your PATH and each Gaudi accelerator is visible
47 - $ apt list --installed | grep habana # verify that habanalabs-firmware-tools, habanalabs-graph, habanalabs-rdma-core, habanalabs-thunk and habanalabs-container-runtime are installed
48 - $ pip list | grep habana # verify that habana-torch-plugin, habana-torch-dataloader, habana-pyhlml and habana-media-loader are installed
49 - $ pip list | grep neural # verify that neural_compressor is installed
50 -
51 -Refer to `Intel Gaudi Software Stack
52 -Verification <https://docs.habana.ai/en/latest/Installation_Guide/SW_Verification.html#platform-upgrade>`__
53 -for more details.
54 -
55 -Run Docker Image
56 -^^^^^^^^^^^^^^^^
57 -
58 -It is highly recommended to use the latest Docker image from Intel Gaudi
59 -vault. Refer to the `Intel Gaudi
60 -documentation <https://docs.habana.ai/en/latest/Installation_Guide/Bare_Metal_Fresh_OS.html#pull-prebuilt-containers>`__
61 -for more details.
62 -
63 -Use the following commands to run a Docker image:
64 -
65 -.. code:: console
66 -
67 - $ docker pull vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest
68 - $ docker run -it --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --net=host --ipc=host vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest
69 -
70 -Build and Install vLLM
71 -^^^^^^^^^^^^^^^^^^^^^^
72 -
73 -To build and install vLLM from source, run:
74 -
75 -.. code:: console
76 -
77 - $ git clone https://github.com/vllm-project/vllm.git
78 - $ cd vllm
79 - $ python setup.py develop
80 -
81 -
82 -Currently, the latest features and performance optimizations are developed in Gaudi's `vLLM-fork <https://github.com/HabanaAI/vllm-fork>`__ and we periodically upstream them to vLLM main repo. To install latest `HabanaAI/vLLM-fork <https://github.com/HabanaAI/vllm-fork>`__, run the following:
83 -
84 -.. code:: console
85 -
86 - $ git clone https://github.com/HabanaAI/vllm-fork.git
87 - $ cd vllm-fork
88 - $ git checkout habana_main
89 - $ python setup.py develop
90 -
91 -
92 -Supported Features
93 -------------------
94 -
95 -- `Offline batched
96 - inference <https://docs.vllm.ai/en/latest/getting_started/quickstart.html#offline-batched-inference>`__
97 -- Online inference via `OpenAI-Compatible
98 - Server <https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server>`__
99 -- HPU autodetection - no need to manually select device within vLLM
100 -- Paged KV cache with algorithms enabled for Intel Gaudi accelerators
101 -- Custom Intel Gaudi implementations of Paged Attention, KV cache ops,
102 - prefill attention, Root Mean Square Layer Normalization, Rotary
103 - Positional Encoding
104 -- Tensor parallelism support for multi-card inference
105 -- Inference with `HPU Graphs <https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html>`__
106 - for accelerating low-batch latency and throughput
107 -- Attention with Linear Biases (ALiBi)
108 -
109 -Unsupported Features
110 ---------------------
111 -
112 -- Beam search
113 -- LoRA adapters
114 -- Quantization
115 -- Prefill chunking (mixed-batch inferencing)
116 -
117 -Supported Configurations
118 -------------------------
119 -
120 -The following configurations have been validated to be function with
121 -Gaudi2 devices. Configurations that are not listed may or may not work.
122 -
123 -- `meta-llama/Llama-2-7b <https://huggingface.co/meta-llama/Llama-2-7b>`__
124 - on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
125 - datatype with random or greedy sampling
126 -- `meta-llama/Llama-2-7b-chat-hf <https://huggingface.co/meta-llama/Llama-2-7b-chat-hf>`__
127 - on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
128 - datatype with random or greedy sampling
129 -- `meta-llama/Meta-Llama-3-8B <https://huggingface.co/meta-llama/Meta-Llama-3-8B>`__
130 - on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
131 - datatype with random or greedy sampling
132 -- `meta-llama/Meta-Llama-3-8B-Instruct <https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct>`__
133 - on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
134 - datatype with random or greedy sampling
135 -- `meta-llama/Meta-Llama-3.1-8B <https://huggingface.co/meta-llama/Meta-Llama-3.1-8B>`__
136 - on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
137 - datatype with random or greedy sampling
138 -- `meta-llama/Meta-Llama-3.1-8B-Instruct <https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct>`__
139 - on single HPU, or with tensor parallelism on 2x and 8x HPU, BF16
140 - datatype with random or greedy sampling
141 -- `meta-llama/Llama-2-70b <https://huggingface.co/meta-llama/Llama-2-70b>`__
142 - with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
143 -- `meta-llama/Llama-2-70b-chat-hf <https://huggingface.co/meta-llama/Llama-2-70b-chat-hf>`__
144 - with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
145 -- `meta-llama/Meta-Llama-3-70B <https://huggingface.co/meta-llama/Meta-Llama-3-70B>`__
146 - with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
147 -- `meta-llama/Meta-Llama-3-70B-Instruct <https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct>`__
148 - with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
149 -- `meta-llama/Meta-Llama-3.1-70B <https://huggingface.co/meta-llama/Meta-Llama-3.1-70B>`__
150 - with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
151 -- `meta-llama/Meta-Llama-3.1-70B-Instruct <https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct>`__
152 - with tensor parallelism on 8x HPU, BF16 datatype with random or greedy sampling
153 -
154 -Performance Tuning
155 -------------------
156 -
157 -Execution modes
158 -~~~~~~~~~~~~~~~
159 -
160 -Currently in vLLM for HPU we support four execution modes, depending on selected HPU PyTorch Bridge backend (via ``PT_HPU_LAZY_MODE`` environment variable), and ``--enforce-eager`` flag.
161 -
162 -.. list-table:: vLLM execution modes
163 - :widths: 25 25 50
164 - :header-rows: 1
165 -
166 - * - ``PT_HPU_LAZY_MODE``
167 - - ``enforce_eager``
168 - - execution mode
169 - * - 0
170 - - 0
171 - - torch.compile
172 - * - 0
173 - - 1
174 - - PyTorch eager mode
175 - * - 1
176 - - 0
177 - - HPU Graphs
178 - * - 1
179 - - 1
180 - - PyTorch lazy mode
181 -
182 -.. warning::
183 - In 1.18.0, all modes utilizing ``PT_HPU_LAZY_MODE=0`` are highly experimental and should be only used for validating functional correctness. Their performance will be improved in the next releases. For obtaining the best performance in 1.18.0, please use HPU Graphs, or PyTorch lazy mode.
184 -
185 -
186 -Bucketing mechanism
187 -~~~~~~~~~~~~~~~~~~~
188 -
189 -Intel Gaudi accelerators work best when operating on models with fixed tensor shapes. `Intel Gaudi Graph Compiler <https://docs.habana.ai/en/latest/Gaudi_Overview/Intel_Gaudi_Software_Suite.html#graph-compiler-and-runtime>`__ is responsible for generating optimized binary code that implements the given model topology on Gaudi. In its default configuration, the produced binary code may be heavily dependent on input and output tensor shapes, and can require graph recompilation when encountering differently shaped tensors within the same topology. While the resulting binaries utilize Gaudi efficiently, the compilation itself may introduce a noticeable overhead in end-to-end execution.
190 -In a dynamic inference serving scenario, there is a need to minimize the number of graph compilations and reduce the risk of graph compilation occurring during server runtime. Currently it is achieved by "bucketing" model's forward pass across two dimensions - ``batch_size`` and ``sequence_length``.
191 -
192 -.. note::
193 - Bucketing allows us to reduce the number of required graphs significantly, but it does not handle any graph compilation and device code generation - this is done in warmup and HPUGraph capture phase.
194 -
195 -Bucketing ranges are determined with 3 parameters - ``min``, ``step`` and ``max``. They can be set separately for prompt and decode phase, and for batch size and sequence length dimension. These parameters can be observed in logs during vLLM startup:
196 -
197 -.. code-block::
198 -
199 - INFO 08-01 21:37:59 hpu_model_runner.py:493] Prompt bucket config (min, step, max_warmup) bs:[1, 32, 4], seq:[128, 128, 1024]
200 - INFO 08-01 21:37:59 hpu_model_runner.py:499] Generated 24 prompt buckets: [(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (2, 768), (2, 896), (2, 1024), (4, 128), (4, 256), (4, 384), (4, 512), (4, 640), (4, 768), (4, 896), (4, 1024)]
201 - INFO 08-01 21:37:59 hpu_model_runner.py:504] Decode bucket config (min, step, max_warmup) bs:[1, 128, 4], seq:[128, 128, 2048]
202 - INFO 08-01 21:37:59 hpu_model_runner.py:509] Generated 48 decode buckets: [(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (1, 1152), (1, 1280), (1, 1408), (1, 1536), (1, 1664), (1, 1792), (1, 1920), (1, 2048), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (2, 768), (2, 896), (2, 1024), (2, 1152), (2, 1280), (2, 1408), (2, 1536), (2, 1664), (2, 1792), (2, 1920), (2, 2048), (4, 128), (4, 256), (4, 384), (4, 512), (4, 640), (4, 768), (4, 896), (4, 1024), (4, 1152), (4, 1280), (4, 1408), (4, 1536), (4, 1664), (4, 1792), (4, 1920), (4, 2048)]
203 -
204 -``min`` determines the lowest value of the bucket. ``step`` determines the interval between buckets, and ``max`` determines the upper bound of the bucket. Furthermore, interval between ``min`` and ``step`` has special handling - ``min`` gets multiplied by consecutive powers of two, until ``step`` gets reached. We call this the ramp-up phase and it is used for handling lower batch sizes with minimum wastage, while allowing larger padding on larger batch sizes.
205 -
206 -Example (with ramp-up)
207 -
208 -.. code-block::
209 -
210 - min = 2, step = 32, max = 64
211 - => ramp_up = (2, 4, 8, 16)
212 - => stable = (32, 64)
213 - => buckets = ramp_up + stable => (2, 4, 8, 16, 32, 64)
214 -
215 -Example (without ramp-up)
216 -
217 -.. code-block::
218 -
219 - min = 128, step = 128, max = 512
220 - => ramp_up = ()
221 - => stable = (128, 256, 384, 512)
222 - => buckets = ramp_up + stable => (128, 256, 384, 512)
223 -
224 -
225 -In the logged scenario, 24 buckets were generated for prompt (prefill) runs, and 48 buckets for decode runs. Each bucket corresponds to a separate optimized device binary for a given model with specified tensor shapes. Whenever a batch of requests is processed, it is padded across batch and sequence length dimension to the smallest possible bucket.
226 -
227 -.. warning::
228 - If a request exceeds maximum bucket size in any dimension, it will be processed without padding, and its processing may require a graph compilation, potentially significantly increasing end-to-end latency. The boundaries of the buckets are user-configurable via environment variables, and upper bucket boundaries can be increased to avoid such scenario.
229 -
230 -As an example, if a request of 3 sequences, with max sequence length of 412 comes in to an idle vLLM server, it will be padded executed as ``(4, 512)`` prefill bucket, as ``batch_size`` (number of sequences) will be padded to 4 (closest batch_size dimension higher than 3), and max sequence length will be padded to 512 (closest sequence length dimension higher than 412). After prefill stage, it will be executed as ``(4, 512)`` decode bucket and will continue as that bucket until either batch dimension changes (due to request being finished) - in which case it will become a ``(2, 512)`` bucket, or context length increases above 512 tokens, in which case it will become ``(4, 640)`` bucket.
231 -
232 -.. note::
233 - Bucketing is transparent to a client - padding in sequence length dimension is never returned to the client, and padding in batch dimension does not create new requests.
234 -
235 -Warmup
236 -~~~~~~
237 -
238 -Warmup is an optional, but highly recommended step occurring before vLLM server starts listening. It executes a forward pass for each bucket with dummy data. The goal is to pre-compile all graphs and not incur any graph compilation overheads within bucket boundaries during server runtime. Each warmup step is logged during vLLM startup:
239 -
240 -.. code-block::
241 -
242 - INFO 08-01 22:26:47 hpu_model_runner.py:1066] [Warmup][Prompt][1/24] batch_size:4 seq_len:1024 free_mem:79.16 GiB
243 - INFO 08-01 22:26:47 hpu_model_runner.py:1066] [Warmup][Prompt][2/24] batch_size:4 seq_len:896 free_mem:55.43 GiB
244 - INFO 08-01 22:26:48 hpu_model_runner.py:1066] [Warmup][Prompt][3/24] batch_size:4 seq_len:768 free_mem:55.43 GiB
245 - ...
246 - INFO 08-01 22:26:59 hpu_model_runner.py:1066] [Warmup][Prompt][24/24] batch_size:1 seq_len:128 free_mem:55.43 GiB
247 - INFO 08-01 22:27:00 hpu_model_runner.py:1066] [Warmup][Decode][1/48] batch_size:4 seq_len:2048 free_mem:55.43 GiB
248 - INFO 08-01 22:27:00 hpu_model_runner.py:1066] [Warmup][Decode][2/48] batch_size:4 seq_len:1920 free_mem:55.43 GiB
249 - INFO 08-01 22:27:01 hpu_model_runner.py:1066] [Warmup][Decode][3/48] batch_size:4 seq_len:1792 free_mem:55.43 GiB
250 - ...
251 - INFO 08-01 22:27:16 hpu_model_runner.py:1066] [Warmup][Decode][47/48] batch_size:2 seq_len:128 free_mem:55.43 GiB
252 - INFO 08-01 22:27:16 hpu_model_runner.py:1066] [Warmup][Decode][48/48] batch_size:1 seq_len:128 free_mem:55.43 GiB
253 -
254 -This example uses the same buckets as in *Bucketing mechanism* section. Each output line corresponds to execution of a single bucket. When bucket is executed for the first time, its graph is compiled and can be reused later on, skipping further graph compilations.
255 -
256 -.. tip::
257 - Compiling all the buckets might take some time and can be turned off with ``VLLM_SKIP_WARMUP=true`` environment variable. Keep in mind that if you do that, you may face graph compilations once executing a given bucket for the first time. It is fine to disable warmup for development, but it's highly recommended to enable it in deployment.
258 -
259 -HPU Graph capture
260 -~~~~~~~~~~~~~~~~~
261 -
262 -`HPU Graphs <https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html>`__ are currently the most performant execution method of vLLM on Intel Gaudi. When HPU Graphs are enabled, execution graphs will be traced (recorded) ahead of time (after performing warmup), to be later replayed during inference, significantly reducing host overheads. Recording can take large amounts of memory, which needs to be taken into account when allocating KV cache. Enabling HPU Graphs will impact the number of available KV cache blocks, but vLLM provides user-configurable variables to control memory management.
263 -
264 -
265 -When HPU Graphs are being used, they share the common memory pool ("usable memory") as KV cache, determined by ``gpu_memory_utilization`` flag (``0.9`` by default).
266 -Before KV cache gets allocated, model weights are loaded onto the device, and a forward pass of the model is executed on dummy data, to estimate memory usage.
267 -Only after that, ``gpu_memory_utilization`` flag is utilized - at its default value, will mark 90% of free device memory at that point as usable.
268 -Next, KV cache gets allocated, model is warmed up, and HPU Graphs are captured.
269 -Environment variable ``VLLM_GRAPH_RESERVED_MEM`` defines the ratio of memory reserved for HPU Graphs capture.
270 -With its default value (``VLLM_GRAPH_RESERVED_MEM=0.1``), 10% of usable memory will be reserved for graph capture (later referred to as "usable graph memory"), and the remaining 90% will be utilized for KV cache.
271 -Environment variable ``VLLM_GRAPH_PROMPT_RATIO`` determines the ratio of usable graph memory reserved for prefill and decode graphs. By default (``VLLM_GRAPH_PROMPT_RATIO=0.3``), both stages have equal memory constraints.
272 -Lower value corresponds to less usable graph memory reserved for prefill stage, e.g. ``VLLM_GRAPH_PROMPT_RATIO=0.2`` will reserve 20% of usable graph memory for prefill graphs, and 80% of usable graph memory for decode graphs.
273 -
274 -.. note::
275 - ``gpu_memory_utilization`` does not correspond to the absolute memory usage across HPU. It specifies the memory margin after loading the model and performing a profile run. If device has 100 GiB of total memory, and 50 GiB of free memory after loading model weights and executing profiling run, ``gpu_memory_utilization`` at its default value will mark 90% of 50 GiB as usable, leaving 5 GiB of margin, regardless of total device memory.
276 -
277 -User can also configure the strategy for capturing HPU Graphs for prompt and decode stages separately. Strategy affects the order of capturing graphs. There are two strategies implemented:
278 -- ``max_bs`` - graph capture queue will sorted in descending order by their batch sizes. Buckets with equal batch sizes are sorted by sequence length in ascending order (e.g. ``(64, 128)``, ``(64, 256)``, ``(32, 128)``, ``(32, 256)``, ``(1, 128)``, ``(1,256)``), default strategy for decode
279 -- ``min_tokens`` - graph capture queue will be sorted in ascending order by the number of tokens each graph processes (``batch_size*sequence_length``), default strategy for prompt
280 -
281 -When there's large amount of requests pending, vLLM scheduler will attempt to fill the maximum batch size for decode as soon as possible. When a request is finished, decode batch size decreases. When that happens, vLLM will attempt to schedule a prefill iteration for requests in the waiting queue, to fill the decode batch size to its previous state. This means that in a full load scenario, decode batch size is often at its maximum, which makes large batch size HPU Graphs crucial to capture, as reflected by ``max_bs`` strategy. On the other hand, prefills will be executed most frequently with very low batch sizes (1-4), which is reflected in ``min_tokens`` strategy.
282 -
283 -
284 -.. note::
285 - ``VLLM_GRAPH_PROMPT_RATIO`` does not set a hard limit on memory taken by graphs for each stage (prefill and decode). vLLM will first attempt to use up entirety of usable prefill graph memory (usable graph memory * ``VLLM_GRAPH_PROMPT_RATIO``) for capturing prefill HPU Graphs, next it will attempt do the same for decode graphs and usable decode graph memory pool. If one stage is fully captured, and there is unused memory left within usable graph memory pool, vLLM will attempt further graph capture for the other stage, until no more HPU Graphs can be captured without exceeding reserved memory pool. The behavior on that mechanism can be observed in the example below.
286 -
287 -
288 -Each described step is logged by vLLM server, as follows (negative values correspond to memory being released):
289 -
290 -.. code-block::
291 -
292 - INFO 08-02 17:37:44 hpu_model_runner.py:493] Prompt bucket config (min, step, max_warmup) bs:[1, 32, 4], seq:[128, 128, 1024]
293 - INFO 08-02 17:37:44 hpu_model_runner.py:499] Generated 24 prompt buckets: [(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (2, 768), (2, 896), (2, 1024), (4, 128), (4, 256), (4, 384), (4, 512), (4, 640), (4, 768), (4, 896), (4, 1024)]
294 - INFO 08-02 17:37:44 hpu_model_runner.py:504] Decode bucket config (min, step, max_warmup) bs:[1, 128, 4], seq:[128, 128, 2048]
295 - INFO 08-02 17:37:44 hpu_model_runner.py:509] Generated 48 decode buckets: [(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (1, 1152), (1, 1280), (1, 1408), (1, 1536), (1, 1664), (1, 1792), (1, 1920), (1, 2048), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (2, 768), (2, 896), (2, 1024), (2, 1152), (2, 1280), (2, 1408), (2, 1536), (2, 1664), (2, 1792), (2, 1920), (2, 2048), (4, 128), (4, 256), (4, 384), (4, 512), (4, 640), (4, 768), (4, 896), (4, 1024), (4, 1152), (4, 1280), (4, 1408), (4, 1536), (4, 1664), (4, 1792), (4, 1920), (4, 2048)]
296 - INFO 08-02 17:37:52 hpu_model_runner.py:430] Pre-loading model weights on hpu:0 took 14.97 GiB of device memory (14.97 GiB/94.62 GiB used) and 2.95 GiB of host memory (475.2 GiB/1007 GiB used)
297 - INFO 08-02 17:37:52 hpu_model_runner.py:438] Wrapping in HPU Graph took 0 B of device memory (14.97 GiB/94.62 GiB used) and -252 KiB of host memory (475.2 GiB/1007 GiB used)
298 - INFO 08-02 17:37:52 hpu_model_runner.py:442] Loading model weights took in total 14.97 GiB of device memory (14.97 GiB/94.62 GiB used) and 2.95 GiB of host memory (475.2 GiB/1007 GiB used)
299 - INFO 08-02 17:37:54 hpu_worker.py:134] Model profiling run took 504 MiB of device memory (15.46 GiB/94.62 GiB used) and 180.9 MiB of host memory (475.4 GiB/1007 GiB used)
300 - INFO 08-02 17:37:54 hpu_worker.py:158] Free device memory: 79.16 GiB, 39.58 GiB usable (gpu_memory_utilization=0.5), 15.83 GiB reserved for HPUGraphs (VLLM_GRAPH_RESERVED_MEM=0.4), 23.75 GiB reserved for KV cache
301 - INFO 08-02 17:37:54 hpu_executor.py:85] # HPU blocks: 1519, # CPU blocks: 0
302 - INFO 08-02 17:37:54 hpu_worker.py:190] Initializing cache engine took 23.73 GiB of device memory (39.2 GiB/94.62 GiB used) and -1.238 MiB of host memory (475.4 GiB/1007 GiB used)
303 - INFO 08-02 17:37:54 hpu_model_runner.py:1066] [Warmup][Prompt][1/24] batch_size:4 seq_len:1024 free_mem:55.43 GiB
304 - ...
305 - INFO 08-02 17:38:22 hpu_model_runner.py:1066] [Warmup][Decode][48/48] batch_size:1 seq_len:128 free_mem:55.43 GiB
306 - INFO 08-02 17:38:22 hpu_model_runner.py:1159] Using 15.85 GiB/55.43 GiB of free device memory for HPUGraphs, 7.923 GiB for prompt and 7.923 GiB for decode (VLLM_GRAPH_PROMPT_RATIO=0.3)
307 - INFO 08-02 17:38:22 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][1/24] batch_size:1 seq_len:128 free_mem:55.43 GiB
308 - ...
309 - INFO 08-02 17:38:26 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][11/24] batch_size:1 seq_len:896 free_mem:48.77 GiB
310 - INFO 08-02 17:38:27 hpu_model_runner.py:1066] [Warmup][Graph/Decode][1/48] batch_size:4 seq_len:128 free_mem:47.51 GiB
311 - ...
312 - INFO 08-02 17:38:41 hpu_model_runner.py:1066] [Warmup][Graph/Decode][48/48] batch_size:1 seq_len:2048 free_mem:47.35 GiB
313 - INFO 08-02 17:38:41 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][12/24] batch_size:4 seq_len:256 free_mem:47.35 GiB
314 - INFO 08-02 17:38:42 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][13/24] batch_size:2 seq_len:512 free_mem:45.91 GiB
315 - INFO 08-02 17:38:42 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][14/24] batch_size:1 seq_len:1024 free_mem:44.48 GiB
316 - INFO 08-02 17:38:43 hpu_model_runner.py:1066] [Warmup][Graph/Prompt][15/24] batch_size:2 seq_len:640 free_mem:43.03 GiB
317 - INFO 08-02 17:38:43 hpu_model_runner.py:1128] Graph/Prompt captured:15 (62.5%) used_mem:14.03 GiB buckets:[(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (4, 128), (4, 256)]
318 - INFO 08-02 17:38:43 hpu_model_runner.py:1128] Graph/Decode captured:48 (100.0%) used_mem:161.9 MiB buckets:[(1, 128), (1, 256), (1, 384), (1, 512), (1, 640), (1, 768), (1, 896), (1, 1024), (1, 1152), (1, 1280), (1, 1408), (1, 1536), (1, 1664), (1, 1792), (1, 1920), (1, 2048), (2, 128), (2, 256), (2, 384), (2, 512), (2, 640), (2, 768), (2, 896), (2, 1024), (2, 1152), (2, 1280), (2, 1408), (2, 1536), (2, 1664), (2, 1792), (2, 1920), (2, 2048), (4, 128), (4, 256), (4, 384), (4, 512), (4, 640), (4, 768), (4, 896), (4, 1024), (4, 1152), (4, 1280), (4, 1408), (4, 1536), (4, 1664), (4, 1792), (4, 1920), (4, 2048)]
319 - INFO 08-02 17:38:43 hpu_model_runner.py:1206] Warmup finished in 49 secs, allocated 14.19 GiB of device memory
320 - INFO 08-02 17:38:43 hpu_executor.py:91] init_cache_engine took 37.92 GiB of device memory (53.39 GiB/94.62 GiB used) and 57.86 MiB of host memory (475.4 GiB/1007 GiB used)
321 -
322 -
323 -Recommended vLLM Parameters
324 -~~~~~~~~~~~~~~~~~~~~~~~~~~~
325 -
326 -- We recommend running inference on Gaudi 2 with ``block_size`` of 128
327 - for BF16 data type. Using default values (16, 32) might lead to
328 - sub-optimal performance due to Matrix Multiplication Engine
329 - under-utilization (see `Gaudi
330 - Architecture <https://docs.habana.ai/en/latest/Gaudi_Overview/Gaudi_Architecture.html>`__).
331 -- For max throughput on Llama 7B, we recommend running with batch size
332 - of 128 or 256 and max context length of 2048 with HPU Graphs enabled.
333 - If you encounter out-of-memory issues, see troubleshooting section.
334 -
335 -Environment variables
336 -~~~~~~~~~~~~~~~~~~~~~
337 -
338 -**Diagnostic and profiling knobs:**
339 -
340 -- ``VLLM_PROFILER_ENABLED``: if ``true``, high level profiler will be enabled. Resulting JSON traces can be viewed in `perfetto.habana.ai <https://perfetto.habana.ai/#!/viewer>`__. Disabled by default.
341 -- ``VLLM_HPU_LOG_STEP_GRAPH_COMPILATION``: if ``true``, will log graph compilations per each vLLM engine step, only when there was any - highly recommended to use alongside ``PT_HPU_METRICS_GC_DETAILS=1``. Disabled by default.
342 -- ``VLLM_HPU_LOG_STEP_GRAPH_COMPILATION_ALL``: if ``true``, will log graph compilations per each vLLM engine step, always, even if there were none. Disabled by default.
343 -- ``VLLM_HPU_LOG_STEP_CPU_FALLBACKS``: if ``true``, will log cpu fallbacks per each vLLM engine step, only when there was any. Disabled by default.
344 -- ``VLLM_HPU_LOG_STEP_CPU_FALLBACKS_ALL``: if ``true``, will log cpu fallbacks per each vLLM engine step, always, even if there were none. Disabled by default.
345 -
346 -**Performance tuning knobs:**
347 -
348 -- ``VLLM_SKIP_WARMUP``: if ``true``, warmup will be skipped, ``false`` by default
349 -- ``VLLM_GRAPH_RESERVED_MEM``: percentage of memory dedicated for HPUGraph capture, ``0.1`` by default
350 -- ``VLLM_GRAPH_PROMPT_RATIO``: percentage of reserved graph memory dedicated for prompt graphs, ``0.3`` by default
351 -- ``VLLM_GRAPH_PROMPT_STRATEGY``: strategy determining order of prompt graph capture, ``min_tokens`` or ``max_bs``, ``min_tokens`` by default
352 -- ``VLLM_GRAPH_DECODE_STRATEGY``: strategy determining order of decode graph capture, ``min_tokens`` or ``max_bs``, ``max_bs`` by default
353 -- ``VLLM_{phase}_{dim}_BUCKET_{param}`` - collection of 12 environment variables configuring ranges of bucketing mechanism
354 -
355 - - ``{phase}`` is either ``PROMPT`` or ``DECODE``
356 - - ``{dim}`` is either ``BS``, ``SEQ`` or ``BLOCK``
357 - - ``{param}`` is either ``MIN``, ``STEP`` or ``MAX``
358 - - Default values:
359 -
360 - - Prompt:
361 - - batch size min (``VLLM_PROMPT_BS_BUCKET_MIN``): ``1``
362 - - batch size step (``VLLM_PROMPT_BS_BUCKET_STEP``): ``min(max_num_seqs, 32)``
363 - - batch size max (``VLLM_PROMPT_BS_BUCKET_MAX``): ``min(max_num_seqs, 64)``
364 - - sequence length min (``VLLM_PROMPT_SEQ_BUCKET_MIN``): ``block_size``
365 - - sequence length step (``VLLM_PROMPT_SEQ_BUCKET_STEP``): ``block_size``
366 - - sequence length max (``VLLM_PROMPT_SEQ_BUCKET_MAX``): ``max_model_len``
367 -
368 - - Decode:
369 - - batch size min (``VLLM_DECODE_BS_BUCKET_MIN``): ``1``
370 - - batch size step (``VLLM_DECODE_BS_BUCKET_STEP``): ``min(max_num_seqs, 32)``
371 - - batch size max (``VLLM_DECODE_BS_BUCKET_MAX``): ``max_num_seqs``
372 - - sequence length min (``VLLM_DECODE_BLOCK_BUCKET_MIN``): ``block_size``
373 - - sequence length step (``VLLM_DECODE_BLOCK_BUCKET_STEP``): ``block_size``
374 - - sequence length max (``VLLM_DECODE_BLOCK_BUCKET_MAX``): ``max(128, (max_num_seqs*max_model_len)/block_size)``
375 -
376 -
377 -Additionally, there are HPU PyTorch Bridge environment variables impacting vLLM execution:
378 -
379 -- ``PT_HPU_LAZY_MODE``: if ``0``, PyTorch Eager backend for Gaudi will be used, if ``1`` PyTorch Lazy backend for Gaudi will be used, ``1`` is default
380 -- ``PT_HPU_ENABLE_LAZY_COLLECTIVES``: required to be ``true`` for tensor parallel inference with HPU Graphs
381 -
382 -Troubleshooting: Tweaking HPU Graphs
383 -------------------------------------
384 -
385 -If you experience device out-of-memory issues or want to attempt
386 -inference at higher batch sizes, try tweaking HPU Graphs by following
387 -the below:
388 -
389 -- Tweak ``gpu_memory_utilization`` knob. It will decrease the
390 - allocation of KV cache, leaving some headroom for capturing graphs
391 - with larger batch size. By default ``gpu_memory_utilization`` is set
392 - to 0.9. It attempts to allocate ~90% of HBM left for KV cache after
393 - short profiling run. Note that decreasing reduces the number of KV
394 - cache blocks you have available, and therefore reduces the effective
395 - maximum number of tokens you can handle at a given time.
396 -
397 -- If this method is not efficient, you can disable ``HPUGraph``
398 - completely. With HPU Graphs disabled, you are trading latency and
399 - throughput at lower batches for potentially higher throughput on
400 - higher batches. You can do that by adding ``--enforce-eager`` flag to
401 - server (for online inference), or by passing ``enforce_eager=True``
402 - argument to LLM constructor (for offline inference).
docs/source/getting_started/installation.md new
+199
@@ -0,0 +1,199 @@
1 +(installation)=
2 +
3 +# Installation
4 +
5 +vLLM is a Python library that also contains pre-compiled C++ and CUDA (12.1) binaries.
6 +
7 +## Requirements
8 +
9 +- OS: Linux
10 +- Python: 3.9 -- 3.12
11 +- GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, L4, H100, etc.)
12 +
13 +## Install released versions
14 +
15 +You can install vLLM using pip:
16 +
17 +```console
18 +$ # (Recommended) Create a new conda environment.
19 +$ conda create -n myenv python=3.12 -y
20 +$ conda activate myenv
21 +
22 +$ # Install vLLM with CUDA 12.1.
23 +$ pip install vllm
24 +```
25 +
26 +```{note}
27 +Although we recommend using `conda` to create and manage Python environments, it is highly recommended to use `pip` to install vLLM. This is because `pip` can install `torch` with separate library packages like `NCCL`, while `conda` installs `torch` with statically linked `NCCL`. This can cause issues when vLLM tries to use `NCCL`. See [this issue](https://github.com/vllm-project/vllm/issues/8420) for more details.
28 +```
29 +
30 +````{note}
31 +As of now, vLLM's binaries are compiled with CUDA 12.1 and public PyTorch release versions by default.
32 +We also provide vLLM binaries compiled with CUDA 11.8 and public PyTorch release versions:
33 +
34 +```console
35 +$ # Install vLLM with CUDA 11.8.
36 +$ export VLLM_VERSION=0.6.1.post1
37 +$ export PYTHON_VERSION=310
38 +$ pip install https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}+cu118-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}-manylinux1_x86_64.whl --extra-index-url https://download.pytorch.org/whl/cu118
39 +```
40 +
41 +In order to be performant, vLLM has to compile many cuda kernels. The compilation unfortunately introduces binary incompatibility with other CUDA versions and PyTorch versions, even for the same PyTorch version with different building configurations.
42 +
43 +Therefore, it is recommended to install vLLM with a **fresh new** conda environment. If either you have a different CUDA version or you want to use an existing PyTorch installation, you need to build vLLM from source. See below for instructions.
44 +````
45 +
46 +(install-the-latest-code)=
47 +
48 +## Install the latest code
49 +
50 +LLM inference is a fast-evolving field, and the latest code may contain bug fixes, performance improvements, and new features that are not released yet. To allow users to try the latest code without waiting for the next release, vLLM provides wheels for Linux running on a x86 platform with CUDA 12 for every commit since `v0.5.3`. You can download and install it with the following command:
51 +
52 +```console
53 +$ pip install https://vllm-wheels.s3.us-west-2.amazonaws.com/nightly/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl
54 +```
55 +
56 +If you want to access the wheels for previous commits, you can specify the commit hash in the URL:
57 +
58 +```console
59 +$ export VLLM_COMMIT=33f460b17a54acb3b6cc0b03f4a17876cff5eafd # use full commit hash from the main branch
60 +$ pip install https://vllm-wheels.s3.us-west-2.amazonaws.com/${VLLM_COMMIT}/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl
61 +```
62 +
63 +Note that the wheels are built with Python 3.8 ABI (see [PEP 425](https://peps.python.org/pep-0425/) for more details about ABI), so **they are compatible with Python 3.8 and later**. The version string in the wheel file name (`1.0.0.dev`) is just a placeholder to have a unified URL for the wheels. The actual versions of wheels are contained in the wheel metadata. Although we don't support Python 3.8 any more (because PyTorch 2.5 dropped support for Python 3.8), the wheels are still built with Python 3.8 ABI to keep the same wheel name as before.
64 +
65 +Another way to access the latest code is to use the docker images:
66 +
67 +```console
68 +$ export VLLM_COMMIT=33f460b17a54acb3b6cc0b03f4a17876cff5eafd # use full commit hash from the main branch
69 +$ docker pull public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:${VLLM_COMMIT}
70 +```
71 +
72 +These docker images are used for CI and testing only, and they are not intended for production use. They will be expired after several days.
73 +
74 +The latest code can contain bugs and may not be stable. Please use it with caution.
75 +
76 +(build-from-source)=
77 +
78 +## Build from source
79 +
80 +(python-only-build)=
81 +
82 +### Python-only build (without compilation)
83 +
84 +If you only need to change Python code, you can build and install vLLM without compilation. Using `pip`'s [`--editable` flag](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs), changes you make to the code will be reflected when you run vLLM:
85 +
86 +```console
87 +$ git clone https://github.com/vllm-project/vllm.git
88 +$ cd vllm
89 +$ VLLM_USE_PRECOMPILED=1 pip install --editable .
90 +```
91 +
92 +This will download the latest nightly wheel and use the compiled libraries from there in the install.
93 +
94 +The `VLLM_PRECOMPILED_WHEEL_LOCATION` environment variable can be used instead of `VLLM_USE_PRECOMPILED` to specify a custom path or URL to the wheel file. For example, to use the [0.6.1.post1 PyPi wheel](https://pypi.org/project/vllm/#files):
95 +
96 +```console
97 +$ export VLLM_PRECOMPILED_WHEEL_LOCATION=https://files.pythonhosted.org/packages/4a/4c/ee65ba33467a4c0de350ce29fbae39b9d0e7fcd887cc756fa993654d1228/vllm-0.6.3.post1-cp38-abi3-manylinux1_x86_64.whl
98 +$ pip install --editable .
99 +```
100 +
101 +You can find more information about vLLM's wheels [above](#install-the-latest-code).
102 +
103 +```{note}
104 +There is a possibility that your source code may have a different commit ID compared to the latest vLLM wheel, which could potentially lead to unknown errors.
105 +It is recommended to use the same commit ID for the source code as the vLLM wheel you have installed. Please refer to [the section above](#install-the-latest-code) for instructions on how to install a specified wheel.
106 +```
107 +
108 +### Full build (with compilation)
109 +
110 +If you want to modify C++ or CUDA code, you'll need to build vLLM from source. This can take several minutes:
111 +
112 +```console
113 +$ git clone https://github.com/vllm-project/vllm.git
114 +$ cd vllm
115 +$ pip install -e .
116 +```
117 +
118 +```{tip}
119 +Building from source requires a lot of compilation. If you are building from source repeatedly, it's more efficient to cache the compilation results.
120 +
121 +For example, you can install [ccache](https://github.com/ccache/ccache) using `conda install ccache` or `apt install ccache` .
122 +As long as `which ccache` command can find the `ccache` binary, it will be used automatically by the build system. After the first build, subsequent builds will be much faster.
123 +
124 +[sccache](https://github.com/mozilla/sccache) works similarly to `ccache`, but has the capability to utilize caching in remote storage environments.
125 +The following environment variables can be set to configure the vLLM `sccache` remote: `SCCACHE_BUCKET=vllm-build-sccache SCCACHE_REGION=us-west-2 SCCACHE_S3_NO_CREDENTIALS=1`. We also recommend setting `SCCACHE_IDLE_TIMEOUT=0`.
126 +```
127 +
128 +#### Use an existing PyTorch installation
129 +
130 +There are scenarios where the PyTorch dependency cannot be easily installed via pip, e.g.:
131 +
132 +- Building vLLM with PyTorch nightly or a custom PyTorch build.
133 +- Building vLLM with aarch64 and CUDA (GH200), where the PyTorch wheels are not available on PyPI. Currently, only the PyTorch nightly has wheels for aarch64 with CUDA. You can run `pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124` to [install PyTorch nightly](https://pytorch.org/get-started/locally/), and then build vLLM on top of it.
134 +
135 +To build vLLM using an existing PyTorch installation:
136 +
137 +```console
138 +$ git clone https://github.com/vllm-project/vllm.git
139 +$ cd vllm
140 +$ python use_existing_torch.py
141 +$ pip install -r requirements-build.txt
142 +$ pip install -e . --no-build-isolation
143 +```
144 +
145 +#### Use the local cutlass for compilation
146 +
147 +Currently, before starting the build process, vLLM fetches cutlass code from GitHub. However, there may be scenarios where you want to use a local version of cutlass instead.
148 +To achieve this, you can set the environment variable VLLM_CUTLASS_SRC_DIR to point to your local cutlass directory.
149 +
150 +```console
151 +$ git clone https://github.com/vllm-project/vllm.git
152 +$ cd vllm
153 +$ VLLM_CUTLASS_SRC_DIR=/path/to/cutlass pip install -e .
154 +```
155 +
156 +#### Troubleshooting
157 +
158 +To avoid your system being overloaded, you can limit the number of compilation jobs
159 +to be run simultaneously, via the environment variable `MAX_JOBS`. For example:
160 +
161 +```console
162 +$ export MAX_JOBS=6
163 +$ pip install -e .
164 +```
165 +
166 +This is especially useful when you are building on less powerful machines. For example, when you use WSL it only [assigns 50% of the total memory by default](https://learn.microsoft.com/en-us/windows/wsl/wsl-config#main-wsl-settings), so using `export MAX_JOBS=1` can avoid compiling multiple files simultaneously and running out of memory.
167 +A side effect is a much slower build process.
168 +
169 +Additionally, if you have trouble building vLLM, we recommend using the NVIDIA PyTorch Docker image.
170 +
171 +```console
172 +$ # Use `--ipc=host` to make sure the shared memory is large enough.
173 +$ docker run --gpus all -it --rm --ipc=host nvcr.io/nvidia/pytorch:23.10-py3
174 +```
175 +
176 +If you don't want to use docker, it is recommended to have a full installation of CUDA Toolkit. You can download and install it from [the official website](https://developer.nvidia.com/cuda-toolkit-archive). After installation, set the environment variable `CUDA_HOME` to the installation path of CUDA Toolkit, and make sure that the `nvcc` compiler is in your `PATH`, e.g.:
177 +
178 +```console
179 +$ export CUDA_HOME=/usr/local/cuda
180 +$ export PATH="${CUDA_HOME}/bin:$PATH"
181 +```
182 +
183 +Here is a sanity check to verify that the CUDA Toolkit is correctly installed:
184 +
185 +```console
186 +$ nvcc --version # verify that nvcc is in your PATH
187 +$ ${CUDA_HOME}/bin/nvcc --version # verify that nvcc is in your CUDA_HOME
188 +```
189 +
190 +### Unsupported OS build
191 +
192 +vLLM can fully run only on Linux but for development purposes, you can still build it on other systems (for example, macOS), allowing for imports and a more convenient development environment. The binaries will not be compiled and won't work on non-Linux systems.
193 +
194 +Simply disable the `VLLM_TARGET_DEVICE` environment variable before installing:
195 +
196 +```console
197 +$ export VLLM_TARGET_DEVICE=empty
198 +$ pip install -e .
199 +```
docs/source/getting_started/installation.rst deleted
-214
@@ -1,214 +0,0 @@
1 -.. _installation:
2 -
3 -============
4 -Installation
5 -============
6 -
7 -vLLM is a Python library that also contains pre-compiled C++ and CUDA (12.1) binaries.
8 -
9 -Requirements
10 -============
11 -
12 -* OS: Linux
13 -* Python: 3.9 -- 3.12
14 -* GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, L4, H100, etc.)
15 -
16 -Install released versions
17 -=========================
18 -
19 -You can install vLLM using pip:
20 -
21 -.. code-block:: console
22 -
23 - $ # (Recommended) Create a new conda environment.
24 - $ conda create -n myenv python=3.12 -y
25 - $ conda activate myenv
26 -
27 - $ # Install vLLM with CUDA 12.1.
28 - $ pip install vllm
29 -
30 -.. note::
31 -
32 - Although we recommend using ``conda`` to create and manage Python environments, it is highly recommended to use ``pip`` to install vLLM. This is because ``pip`` can install ``torch`` with separate library packages like ``NCCL``, while ``conda`` installs ``torch`` with statically linked ``NCCL``. This can cause issues when vLLM tries to use ``NCCL``. See `this issue <https://github.com/vllm-project/vllm/issues/8420>`_ for more details.
33 -
34 -.. note::
35 -
36 - As of now, vLLM's binaries are compiled with CUDA 12.1 and public PyTorch release versions by default.
37 - We also provide vLLM binaries compiled with CUDA 11.8 and public PyTorch release versions:
38 -
39 - .. code-block:: console
40 -
41 - $ # Install vLLM with CUDA 11.8.
42 - $ export VLLM_VERSION=0.6.1.post1
43 - $ export PYTHON_VERSION=310
44 - $ pip install https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}+cu118-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}-manylinux1_x86_64.whl --extra-index-url https://download.pytorch.org/whl/cu118
45 -
46 - In order to be performant, vLLM has to compile many cuda kernels. The compilation unfortunately introduces binary incompatibility with other CUDA versions and PyTorch versions, even for the same PyTorch version with different building configurations.
47 -
48 - Therefore, it is recommended to install vLLM with a **fresh new** conda environment. If either you have a different CUDA version or you want to use an existing PyTorch installation, you need to build vLLM from source. See below for instructions.
49 -
50 -
51 -.. _install-the-latest-code:
52 -
53 -Install the latest code
54 -=======================
55 -
56 -LLM inference is a fast-evolving field, and the latest code may contain bug fixes, performance improvements, and new features that are not released yet. To allow users to try the latest code without waiting for the next release, vLLM provides wheels for Linux running on a x86 platform with CUDA 12 for every commit since ``v0.5.3``. You can download and install it with the following command:
57 -
58 -.. code-block:: console
59 -
60 - $ pip install https://vllm-wheels.s3.us-west-2.amazonaws.com/nightly/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl
61 -
62 -If you want to access the wheels for previous commits, you can specify the commit hash in the URL:
63 -
64 -.. code-block:: console
65 -
66 - $ export VLLM_COMMIT=33f460b17a54acb3b6cc0b03f4a17876cff5eafd # use full commit hash from the main branch
67 - $ pip install https://vllm-wheels.s3.us-west-2.amazonaws.com/${VLLM_COMMIT}/vllm-1.0.0.dev-cp38-abi3-manylinux1_x86_64.whl
68 -
69 -Note that the wheels are built with Python 3.8 ABI (see `PEP 425 <https://peps.python.org/pep-0425/>`_ for more details about ABI), so **they are compatible with Python 3.8 and later**. The version string in the wheel file name (``1.0.0.dev``) is just a placeholder to have a unified URL for the wheels. The actual versions of wheels are contained in the wheel metadata. Although we don't support Python 3.8 any more (because PyTorch 2.5 dropped support for Python 3.8), the wheels are still built with Python 3.8 ABI to keep the same wheel name as before.
70 -
71 -Another way to access the latest code is to use the docker images:
72 -
73 -.. code-block:: console
74 -
75 - $ export VLLM_COMMIT=33f460b17a54acb3b6cc0b03f4a17876cff5eafd # use full commit hash from the main branch
76 - $ docker pull public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:${VLLM_COMMIT}
77 -
78 -These docker images are used for CI and testing only, and they are not intended for production use. They will be expired after several days.
79 -
80 -The latest code can contain bugs and may not be stable. Please use it with caution.
81 -
82 -.. _build_from_source:
83 -
84 -Build from source
85 -=================
86 -
87 -.. _python-only-build:
88 -
89 -Python-only build (without compilation)
90 ----------------------------------------
91 -
92 -If you only need to change Python code, you can build and install vLLM without compilation. Using `pip's ``--editable`` flag <https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs>`_, changes you make to the code will be reflected when you run vLLM:
93 -
94 -.. code-block:: console
95 -
96 - $ git clone https://github.com/vllm-project/vllm.git
97 - $ cd vllm
98 - $ VLLM_USE_PRECOMPILED=1 pip install --editable .
99 -
100 -This will download the latest nightly wheel and use the compiled libraries from there in the install.
101 -
102 -The ``VLLM_PRECOMPILED_WHEEL_LOCATION`` environment variable can be used instead of ``VLLM_USE_PRECOMPILED`` to specify a custom path or URL to the wheel file. For example, to use the `0.6.1.post1 PyPi wheel <https://pypi.org/project/vllm/#files>`_:
103 -
104 -.. code-block:: console
105 -
106 - $ export VLLM_PRECOMPILED_WHEEL_LOCATION=https://files.pythonhosted.org/packages/4a/4c/ee65ba33467a4c0de350ce29fbae39b9d0e7fcd887cc756fa993654d1228/vllm-0.6.3.post1-cp38-abi3-manylinux1_x86_64.whl
107 - $ pip install --editable .
108 -
109 -You can find more information about vLLM's wheels `above <#install-the-latest-code>`_.
110 -
111 -.. note::
112 -
113 - There is a possibility that your source code may have a different commit ID compared to the latest vLLM wheel, which could potentially lead to unknown errors.
114 - It is recommended to use the same commit ID for the source code as the vLLM wheel you have installed. Please refer to `the section above <#install-the-latest-code>`_ for instructions on how to install a specified wheel.
115 -
116 -Full build (with compilation)
117 ------------------------------
118 -
119 -If you want to modify C++ or CUDA code, you'll need to build vLLM from source. This can take several minutes:
120 -
121 -.. code-block:: console
122 -
123 - $ git clone https://github.com/vllm-project/vllm.git
124 - $ cd vllm
125 - $ pip install -e .
126 -
127 -.. tip::
128 -
129 - Building from source requires a lot of compilation. If you are building from source repeatedly, it's more efficient to cache the compilation results.
130 -
131 - For example, you can install `ccache <https://github.com/ccache/ccache>`_ using ``conda install ccache`` or ``apt install ccache`` .
132 - As long as ``which ccache`` command can find the ``ccache`` binary, it will be used automatically by the build system. After the first build, subsequent builds will be much faster.
133 -
134 - `sccache <https://github.com/mozilla/sccache>`_ works similarly to ``ccache``, but has the capability to utilize caching in remote storage environments.
135 - The following environment variables can be set to configure the vLLM ``sccache`` remote: ``SCCACHE_BUCKET=vllm-build-sccache SCCACHE_REGION=us-west-2 SCCACHE_S3_NO_CREDENTIALS=1``. We also recommend setting ``SCCACHE_IDLE_TIMEOUT=0``.
136 -
137 -
138 -Use an existing PyTorch installation
139 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
140 -There are scenarios where the PyTorch dependency cannot be easily installed via pip, e.g.:
141 -
142 -* Building vLLM with PyTorch nightly or a custom PyTorch build.
143 -* Building vLLM with aarch64 and CUDA (GH200), where the PyTorch wheels are not available on PyPI. Currently, only the PyTorch nightly has wheels for aarch64 with CUDA. You can run ``pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124`` to `install PyTorch nightly <https://pytorch.org/get-started/locally/>`_, and then build vLLM on top of it.
144 -
145 -To build vLLM using an existing PyTorch installation:
146 -
147 -.. code-block:: console
148 -
149 - $ git clone https://github.com/vllm-project/vllm.git
150 - $ cd vllm
151 - $ python use_existing_torch.py
152 - $ pip install -r requirements-build.txt
153 - $ pip install -e . --no-build-isolation
154 -
155 -
156 -Use the local cutlass for compilation
157 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
158 -Currently, before starting the build process, vLLM fetches cutlass code from GitHub. However, there may be scenarios where you want to use a local version of cutlass instead.
159 -To achieve this, you can set the environment variable VLLM_CUTLASS_SRC_DIR to point to your local cutlass directory.
160 -
161 -.. code-block:: console
162 -
163 - $ git clone https://github.com/vllm-project/vllm.git
164 - $ cd vllm
165 - $ VLLM_CUTLASS_SRC_DIR=/path/to/cutlass pip install -e .
166 -
167 -
168 -Troubleshooting
169 -~~~~~~~~~~~~~~~
170 -
171 -To avoid your system being overloaded, you can limit the number of compilation jobs
172 -to be run simultaneously, via the environment variable ``MAX_JOBS``. For example:
173 -
174 -.. code-block:: console
175 -
176 - $ export MAX_JOBS=6
177 - $ pip install -e .
178 -
179 -This is especially useful when you are building on less powerful machines. For example, when you use WSL it only `assigns 50% of the total memory by default <https://learn.microsoft.com/en-us/windows/wsl/wsl-config#main-wsl-settings>`_, so using ``export MAX_JOBS=1`` can avoid compiling multiple files simultaneously and running out of memory.
180 -A side effect is a much slower build process.
181 -
182 -Additionally, if you have trouble building vLLM, we recommend using the NVIDIA PyTorch Docker image.
183 -
184 -.. code-block:: console
185 -
186 - $ # Use `--ipc=host` to make sure the shared memory is large enough.
187 - $ docker run --gpus all -it --rm --ipc=host nvcr.io/nvidia/pytorch:23.10-py3
188 -
189 -If you don't want to use docker, it is recommended to have a full installation of CUDA Toolkit. You can download and install it from `the official website <https://developer.nvidia.com/cuda-toolkit-archive>`_. After installation, set the environment variable ``CUDA_HOME`` to the installation path of CUDA Toolkit, and make sure that the ``nvcc`` compiler is in your ``PATH``, e.g.:
190 -
191 -.. code-block:: console
192 -
193 - $ export CUDA_HOME=/usr/local/cuda
194 - $ export PATH="${CUDA_HOME}/bin:$PATH"
195 -
196 -Here is a sanity check to verify that the CUDA Toolkit is correctly installed:
197 -
198 -.. code-block:: console
199 -
200 - $ nvcc --version # verify that nvcc is in your PATH
201 - $ ${CUDA_HOME}/bin/nvcc --version # verify that nvcc is in your CUDA_HOME
202 -
203 -
204 -Unsupported OS build
205 ---------------------
206 -
207 -vLLM can fully run only on Linux but for development purposes, you can still build it on other systems (for example, macOS), allowing for imports and a more convenient development environment. The binaries will not be compiled and won't work on non-Linux systems.
208 -
209 -Simply disable the ``VLLM_TARGET_DEVICE`` environment variable before installing:
210 -
211 -.. code-block:: console
212 -
213 - $ export VLLM_TARGET_DEVICE=empty
214 - $ pip install -e .
docs/source/getting_started/neuron-installation.md new
+132
@@ -0,0 +1,132 @@
1 +(installation-neuron)=
2 +
3 +# Installation with Neuron
4 +
5 +vLLM 0.3.3 onwards supports model inferencing and serving on AWS Trainium/Inferentia with Neuron SDK with continuous batching.
6 +Paged Attention and Chunked Prefill are currently in development and will be available soon.
7 +Data types currently supported in Neuron SDK are FP16 and BF16.
8 +
9 +## Requirements
10 +
11 +- OS: Linux
12 +- Python: 3.9 -- 3.11
13 +- Accelerator: NeuronCore_v2 (in trn1/inf2 instances)
14 +- Pytorch 2.0.1/2.1.1
15 +- AWS Neuron SDK 2.16/2.17 (Verified on python 3.8)
16 +
17 +Installation steps:
18 +
19 +- [Build from source](#build-from-source-neuron)
20 +
21 + - [Step 0. Launch Trn1/Inf2 instances](#launch-instances)
22 + - [Step 1. Install drivers and tools](#install-drivers)
23 + - [Step 2. Install transformers-neuronx and its dependencies](#install-tnx)
24 + - [Step 3. Install vLLM from source](#install-vllm)
25 +
26 +(build-from-source-neuron)=
27 +
28 +```{note}
29 +The currently supported version of Pytorch for Neuron installs `triton` version `2.1.0`. This is incompatible with vLLM >= 0.5.3. You may see an error `cannot import name 'default_dump_dir...`. To work around this, run a `pip install --upgrade triton==3.0.0` after installing the vLLM wheel.
30 +```
31 +
32 +## Build from source
33 +
34 +Following instructions are applicable to Neuron SDK 2.16 and beyond.
35 +
36 +(launch-instances)=
37 +
38 +### Step 0. Launch Trn1/Inf2 instances
39 +
40 +Here are the steps to launch trn1/inf2 instances, in order to install [PyTorch Neuron ("torch-neuronx") Setup on Ubuntu 22.04 LTS](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/setup/neuron-setup/pytorch/neuronx/ubuntu/torch-neuronx-ubuntu22.html).
41 +
42 +- Please follow the instructions at [launch an Amazon EC2 Instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EC2_GetStarted.html#ec2-launch-instance) to launch an instance. When choosing the instance type at the EC2 console, please make sure to select the correct instance type.
43 +- To get more information about instances sizes and pricing see: [Trn1 web page](https://aws.amazon.com/ec2/instance-types/trn1/), [Inf2 web page](https://aws.amazon.com/ec2/instance-types/inf2/)
44 +- Select Ubuntu Server 22.04 TLS AMI
45 +- When launching a Trn1/Inf2, please adjust your primary EBS volume size to a minimum of 512GB.
46 +- After launching the instance, follow the instructions in [Connect to your instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AccessingInstancesLinux.html) to connect to the instance
47 +
48 +(install-drivers)=
49 +
50 +### Step 1. Install drivers and tools
51 +
52 +The installation of drivers and tools wouldn't be necessary, if [Deep Learning AMI Neuron](https://docs.aws.amazon.com/dlami/latest/devguide/appendix-ami-release-notes.html) is installed. In case the drivers and tools are not installed on the operating system, follow the steps below:
53 +
54 +```console
55 +# Configure Linux for Neuron repository updates
56 +. /etc/os-release
57 +sudo tee /etc/apt/sources.list.d/neuron.list > /dev/null <<EOF
58 +deb https://apt.repos.neuron.amazonaws.com ${VERSION_CODENAME} main
59 +EOF
60 +wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | sudo apt-key add -
61 +
62 +# Update OS packages
63 +sudo apt-get update -y
64 +
65 +# Install OS headers
66 +sudo apt-get install linux-headers-$(uname -r) -y
67 +
68 +# Install git
69 +sudo apt-get install git -y
70 +
71 +# install Neuron Driver
72 +sudo apt-get install aws-neuronx-dkms=2.* -y
73 +
74 +# Install Neuron Runtime
75 +sudo apt-get install aws-neuronx-collectives=2.* -y
76 +sudo apt-get install aws-neuronx-runtime-lib=2.* -y
77 +
78 +# Install Neuron Tools
79 +sudo apt-get install aws-neuronx-tools=2.* -y
80 +
81 +# Add PATH
82 +export PATH=/opt/aws/neuron/bin:$PATH
83 +```
84 +
85 +(install-tnx)=
86 +
87 +### Step 2. Install transformers-neuronx and its dependencies
88 +
89 +[transformers-neuronx](https://github.com/aws-neuron/transformers-neuronx) will be the backend to support inference on trn1/inf2 instances.
90 +Follow the steps below to install transformer-neuronx package and its dependencies.
91 +
92 +```console
93 +# Install Python venv
94 +sudo apt-get install -y python3.10-venv g++
95 +
96 +# Create Python venv
97 +python3.10 -m venv aws_neuron_venv_pytorch
98 +
99 +# Activate Python venv
100 +source aws_neuron_venv_pytorch/bin/activate
101 +
102 +# Install Jupyter notebook kernel
103 +pip install ipykernel
104 +python3.10 -m ipykernel install --user --name aws_neuron_venv_pytorch --display-name "Python (torch-neuronx)"
105 +pip install jupyter notebook
106 +pip install environment_kernels
107 +
108 +# Set pip repository pointing to the Neuron repository
109 +python -m pip config set global.extra-index-url https://pip.repos.neuron.amazonaws.com
110 +
111 +# Install wget, awscli
112 +python -m pip install wget
113 +python -m pip install awscli
114 +
115 +# Update Neuron Compiler and Framework
116 +python -m pip install --upgrade neuronx-cc==2.* --pre torch-neuronx==2.1.* torchvision transformers-neuronx
117 +```
118 +
119 +(install-vllm)=
120 +
121 +### Step 3. Install vLLM from source
122 +
123 +Once neuronx-cc and transformers-neuronx packages are installed, we will be able to install vllm as follows:
124 +
125 +```console
126 +$ git clone https://github.com/vllm-project/vllm.git
127 +$ cd vllm
128 +$ pip install -U -r requirements-neuron.txt
129 +$ VLLM_TARGET_DEVICE="neuron" pip install .
130 +```
131 +
132 +If neuron packages are detected correctly in the installation process, `vllm-0.3.0+neuron212` will be installed.
docs/source/getting_started/neuron-installation.rst deleted
-140
@@ -1,140 +0,0 @@
1 -.. _installation_neuron:
2 -
3 -Installation with Neuron
4 -========================
5 -
6 -vLLM 0.3.3 onwards supports model inferencing and serving on AWS Trainium/Inferentia with Neuron SDK with continuous batching.
7 -Paged Attention and Chunked Prefill are currently in development and will be available soon.
8 -Data types currently supported in Neuron SDK are FP16 and BF16.
9 -
10 -Requirements
11 -------------
12 -
13 -* OS: Linux
14 -* Python: 3.9 -- 3.11
15 -* Accelerator: NeuronCore_v2 (in trn1/inf2 instances)
16 -* Pytorch 2.0.1/2.1.1
17 -* AWS Neuron SDK 2.16/2.17 (Verified on python 3.8)
18 -
19 -Installation steps:
20 -
21 -- :ref:`Build from source <build_from_source_neuron>`
22 -
23 - - :ref:`Step 0. Launch Trn1/Inf2 instances <launch_instances>`
24 - - :ref:`Step 1. Install drivers and tools <install_drivers>`
25 - - :ref:`Step 2. Install transformers-neuronx and its dependencies <install_tnx>`
26 - - :ref:`Step 3. Install vLLM from source <install_vllm>`
27 -
28 -.. _build_from_source_neuron:
29 -
30 -.. note::
31 -
32 - The currently supported version of Pytorch for Neuron installs `triton` version `2.1.0`. This is incompatible with vLLM >= 0.5.3. You may see an error `cannot import name 'default_dump_dir...`. To work around this, run a `pip install --upgrade triton==3.0.0` after installing the vLLM wheel.
33 -
34 -Build from source
35 ------------------
36 -
37 -Following instructions are applicable to Neuron SDK 2.16 and beyond.
38 -
39 -.. _launch_instances:
40 -
41 -Step 0. Launch Trn1/Inf2 instances
42 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
43 -
44 -Here are the steps to launch trn1/inf2 instances, in order to install `PyTorch Neuron ("torch-neuronx") Setup on Ubuntu 22.04 LTS <https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/setup/neuron-setup/pytorch/neuronx/ubuntu/torch-neuronx-ubuntu22.html>`_.
45 -
46 -- Please follow the instructions at `launch an Amazon EC2 Instance <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EC2_GetStarted.html#ec2-launch-instance>`_ to launch an instance. When choosing the instance type at the EC2 console, please make sure to select the correct instance type.
47 -- To get more information about instances sizes and pricing see: `Trn1 web page <https://aws.amazon.com/ec2/instance-types/trn1/>`_, `Inf2 web page <https://aws.amazon.com/ec2/instance-types/inf2/>`_
48 -- Select Ubuntu Server 22.04 TLS AMI
49 -- When launching a Trn1/Inf2, please adjust your primary EBS volume size to a minimum of 512GB.
50 -- After launching the instance, follow the instructions in `Connect to your instance <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AccessingInstancesLinux.html>`_ to connect to the instance
51 -
52 -.. _install_drivers:
53 -
54 -Step 1. Install drivers and tools
55 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
56 -
57 -The installation of drivers and tools wouldn't be necessary, if `Deep Learning AMI Neuron <https://docs.aws.amazon.com/dlami/latest/devguide/appendix-ami-release-notes.html>`_ is installed. In case the drivers and tools are not installed on the operating system, follow the steps below:
58 -
59 -.. code-block:: console
60 -
61 - # Configure Linux for Neuron repository updates
62 - . /etc/os-release
63 - sudo tee /etc/apt/sources.list.d/neuron.list > /dev/null <<EOF
64 - deb https://apt.repos.neuron.amazonaws.com ${VERSION_CODENAME} main
65 - EOF
66 - wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | sudo apt-key add -
67 -
68 - # Update OS packages
69 - sudo apt-get update -y
70 -
71 - # Install OS headers
72 - sudo apt-get install linux-headers-$(uname -r) -y
73 -
74 - # Install git
75 - sudo apt-get install git -y
76 -
77 - # install Neuron Driver
78 - sudo apt-get install aws-neuronx-dkms=2.* -y
79 -
80 - # Install Neuron Runtime
81 - sudo apt-get install aws-neuronx-collectives=2.* -y
82 - sudo apt-get install aws-neuronx-runtime-lib=2.* -y
83 -
84 - # Install Neuron Tools
85 - sudo apt-get install aws-neuronx-tools=2.* -y
86 -
87 - # Add PATH
88 - export PATH=/opt/aws/neuron/bin:$PATH
89 -
90 -
91 -.. _install_tnx:
92 -
93 -Step 2. Install transformers-neuronx and its dependencies
94 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
95 -
96 -`transformers-neuronx <https://github.com/aws-neuron/transformers-neuronx>`_ will be the backend to support inference on trn1/inf2 instances.
97 -Follow the steps below to install transformer-neuronx package and its dependencies.
98 -
99 -.. code-block:: console
100 -
101 - # Install Python venv
102 - sudo apt-get install -y python3.10-venv g++
103 -
104 - # Create Python venv
105 - python3.10 -m venv aws_neuron_venv_pytorch
106 -
107 - # Activate Python venv
108 - source aws_neuron_venv_pytorch/bin/activate
109 -
110 - # Install Jupyter notebook kernel
111 - pip install ipykernel
112 - python3.10 -m ipykernel install --user --name aws_neuron_venv_pytorch --display-name "Python (torch-neuronx)"
113 - pip install jupyter notebook
114 - pip install environment_kernels
115 -
116 - # Set pip repository pointing to the Neuron repository
117 - python -m pip config set global.extra-index-url https://pip.repos.neuron.amazonaws.com
118 -
119 - # Install wget, awscli
120 - python -m pip install wget
121 - python -m pip install awscli
122 -
123 - # Update Neuron Compiler and Framework
124 - python -m pip install --upgrade neuronx-cc==2.* --pre torch-neuronx==2.1.* torchvision transformers-neuronx
125 -
126 -.. _install_vllm:
127 -
128 -Step 3. Install vLLM from source
129 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
130 -
131 -Once neuronx-cc and transformers-neuronx packages are installed, we will be able to install vllm as follows:
132 -
133 -.. code-block:: console
134 -
135 - $ git clone https://github.com/vllm-project/vllm.git
136 - $ cd vllm
137 - $ pip install -U -r requirements-neuron.txt
138 - $ VLLM_TARGET_DEVICE="neuron" pip install .
139 -
140 -If neuron packages are detected correctly in the installation process, ``vllm-0.3.0+neuron212`` will be installed.
docs/source/getting_started/openvino-installation.md new
+104
@@ -0,0 +1,104 @@
1 +(installation-openvino)=
2 +
3 +# Installation with OpenVINO
4 +
5 +vLLM powered by OpenVINO supports all LLM models from {doc}`vLLM supported models list <../models/supported_models>` and can perform optimal model serving on all x86-64 CPUs with, at least, AVX2 support, as well as on both integrated and discrete Intel® GPUs ([the list of supported GPUs](https://docs.openvino.ai/2024/about-openvino/release-notes-openvino/system-requirements.html#gpu)). OpenVINO vLLM backend supports the following advanced vLLM features:
6 +
7 +- Prefix caching (`--enable-prefix-caching`)
8 +- Chunked prefill (`--enable-chunked-prefill`)
9 +
10 +**Table of contents**:
11 +
12 +- [Requirements](#openvino-backend-requirements)
13 +- [Quick start using Dockerfile](#openvino-backend-quick-start-dockerfile)
14 +- [Build from source](#install-openvino-backend-from-source)
15 +- [Performance tips](#openvino-backend-performance-tips)
16 +- [Limitations](#openvino-backend-limitations)
17 +
18 +(openvino-backend-requirements)=
19 +
20 +## Requirements
21 +
22 +- OS: Linux
23 +- Instruction set architecture (ISA) requirement: at least AVX2.
24 +
25 +(openvino-backend-quick-start-dockerfile)=
26 +
27 +## Quick start using Dockerfile
28 +
29 +```console
30 +$ docker build -f Dockerfile.openvino -t vllm-openvino-env .
31 +$ docker run -it --rm vllm-openvino-env
32 +```
33 +
34 +(install-openvino-backend-from-source)=
35 +
36 +## Install from source
37 +
38 +- First, install Python. For example, on Ubuntu 22.04, you can run:
39 +
40 + ```console
41 + $ sudo apt-get update -y
42 + $ sudo apt-get install python3
43 + ```
44 +
45 +- Second, install prerequisites vLLM OpenVINO backend installation:
46 +
47 + ```console
48 + $ pip install --upgrade pip
49 + $ pip install -r requirements-build.txt --extra-index-url https://download.pytorch.org/whl/cpu
50 + ```
51 +
52 +- Finally, install vLLM with OpenVINO backend:
53 +
54 + ```console
55 + $ PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" VLLM_TARGET_DEVICE=openvino python -m pip install -v .
56 + ```
57 +
58 +- [Optional] To use vLLM OpenVINO backend with a GPU device, ensure your system is properly set up. Follow the instructions provided here: [https://docs.openvino.ai/2024/get-started/configurations/configurations-intel-gpu.html](https://docs.openvino.ai/2024/get-started/configurations/configurations-intel-gpu.html).
59 +
60 +(openvino-backend-performance-tips)=
61 +
62 +## Performance tips
63 +
64 +### vLLM OpenVINO backend environment variables
65 +
66 +- `VLLM_OPENVINO_DEVICE` to specify which device utilize for the inference. If there are multiple GPUs in the system, additional indexes can be used to choose the proper one (e.g, `VLLM_OPENVINO_DEVICE=GPU.1`). If the value is not specified, CPU device is used by default.
67 +- `VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON` to enable U8 weights compression during model loading stage. By default, compression is turned off. You can also export model with different compression techniques using `optimum-cli` and pass exported folder as `<model_id>`
68 +
69 +### CPU performance tips
70 +
71 +CPU uses the following environment variables to control behavior:
72 +
73 +- `VLLM_OPENVINO_KVCACHE_SPACE` to specify the KV Cache size (e.g, `VLLM_OPENVINO_KVCACHE_SPACE=40` means 40 GB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users.
74 +- `VLLM_OPENVINO_CPU_KV_CACHE_PRECISION=u8` to control KV cache precision. By default, FP16 / BF16 is used depending on platform.
75 +
76 +To enable better TPOT / TTFT latency, you can use vLLM's chunked prefill feature (`--enable-chunked-prefill`). Based on the experiments, the recommended batch size is `256` (`--max-num-batched-tokens`)
77 +
78 +OpenVINO best known configuration for CPU is:
79 +
80 +```console
81 +$ VLLM_OPENVINO_KVCACHE_SPACE=100 VLLM_OPENVINO_CPU_KV_CACHE_PRECISION=u8 VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON \
82 + python3 vllm/benchmarks/benchmark_throughput.py --model meta-llama/Llama-2-7b-chat-hf --dataset vllm/benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json --enable-chunked-prefill --max-num-batched-tokens 256
83 +```
84 +
85 +### GPU performance tips
86 +
87 +GPU device implements the logic for automatic detection of available GPU memory and, by default, tries to reserve as much memory as possible for the KV cache (taking into account `gpu_memory_utilization` option). However, this behavior can be overridden by explicitly specifying the desired amount of memory for the KV cache using `VLLM_OPENVINO_KVCACHE_SPACE` environment variable (e.g, `VLLM_OPENVINO_KVCACHE_SPACE=8` means 8 GB space for KV cache).
88 +
89 +Currently, the best performance using GPU can be achieved with the default vLLM execution parameters for models with quantized weights (8 and 4-bit integer data types are supported) and `preemption-mode=swap`.
90 +
91 +OpenVINO best known configuration for GPU is:
92 +
93 +```console
94 +$ VLLM_OPENVINO_DEVICE=GPU VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON \
95 + python3 vllm/benchmarks/benchmark_throughput.py --model meta-llama/Llama-2-7b-chat-hf --dataset vllm/benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json
96 +```
97 +
98 +(openvino-backend-limitations)=
99 +
100 +## Limitations
101 +
102 +- LoRA serving is not supported.
103 +- Only LLM models are currently supported. LLaVa and encoder-decoder models are not currently enabled in vLLM OpenVINO integration.
104 +- Tensor and pipeline parallelism are not currently enabled in vLLM integration.
docs/source/getting_started/openvino-installation.rst deleted
-116
@@ -1,116 +0,0 @@
1 -.. _installation_openvino:
2 -
3 -Installation with OpenVINO
4 -==========================
5 -
6 -vLLM powered by OpenVINO supports all LLM models from :doc:`vLLM supported models list <../models/supported_models>` and can perform optimal model serving on all x86-64 CPUs with, at least, AVX2 support, as well as on both integrated and discrete Intel® GPUs (`the list of supported GPUs <https://docs.openvino.ai/2024/about-openvino/release-notes-openvino/system-requirements.html#gpu>`_). OpenVINO vLLM backend supports the following advanced vLLM features:
7 -
8 -- Prefix caching (``--enable-prefix-caching``)
9 -- Chunked prefill (``--enable-chunked-prefill``)
10 -
11 -**Table of contents**:
12 -
13 -- :ref:`Requirements <openvino_backend_requirements>`
14 -- :ref:`Quick start using Dockerfile <openvino_backend_quick_start_dockerfile>`
15 -- :ref:`Build from source <install_openvino_backend_from_source>`
16 -- :ref:`Performance tips <openvino_backend_performance_tips>`
17 -- :ref:`Limitations <openvino_backend_limitations>`
18 -
19 -.. _openvino_backend_requirements:
20 -
21 -Requirements
22 -------------
23 -
24 -* OS: Linux
25 -* Instruction set architecture (ISA) requirement: at least AVX2.
26 -
27 -.. _openvino_backend_quick_start_dockerfile:
28 -
29 -Quick start using Dockerfile
30 -----------------------------
31 -
32 -.. code-block:: console
33 -
34 - $ docker build -f Dockerfile.openvino -t vllm-openvino-env .
35 - $ docker run -it --rm vllm-openvino-env
36 -
37 -.. _install_openvino_backend_from_source:
38 -
39 -Install from source
40 --------------------
41 -
42 -- First, install Python. For example, on Ubuntu 22.04, you can run:
43 -
44 - .. code-block:: console
45 -
46 - $ sudo apt-get update -y
47 - $ sudo apt-get install python3
48 -
49 -- Second, install prerequisites vLLM OpenVINO backend installation:
50 -
51 - .. code-block:: console
52 -
53 - $ pip install --upgrade pip
54 - $ pip install -r requirements-build.txt --extra-index-url https://download.pytorch.org/whl/cpu
55 -
56 -- Finally, install vLLM with OpenVINO backend:
57 -
58 - .. code-block:: console
59 -
60 - $ PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" VLLM_TARGET_DEVICE=openvino python -m pip install -v .
61 -
62 -- [Optional] To use vLLM OpenVINO backend with a GPU device, ensure your system is properly set up. Follow the instructions provided here: `https://docs.openvino.ai/2024/get-started/configurations/configurations-intel-gpu.html <https://docs.openvino.ai/2024/get-started/configurations/configurations-intel-gpu.html>`_.
63 -
64 -.. _openvino_backend_performance_tips:
65 -
66 -Performance tips
67 -----------------
68 -
69 -vLLM OpenVINO backend environment variables
70 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
71 -
72 -- ``VLLM_OPENVINO_DEVICE`` to specify which device utilize for the inference. If there are multiple GPUs in the system, additional indexes can be used to choose the proper one (e.g, ``VLLM_OPENVINO_DEVICE=GPU.1``). If the value is not specified, CPU device is used by default.
73 -
74 -- ``VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON`` to enable U8 weights compression during model loading stage. By default, compression is turned off. You can also export model with different compression techniques using `optimum-cli` and pass exported folder as `<model_id>`
75 -
76 -CPU performance tips
77 -~~~~~~~~~~~~~~~~~~~~
78 -
79 -CPU uses the following environment variables to control behavior:
80 -
81 -- ``VLLM_OPENVINO_KVCACHE_SPACE`` to specify the KV Cache size (e.g, ``VLLM_OPENVINO_KVCACHE_SPACE=40`` means 40 GB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users.
82 -
83 -- ``VLLM_OPENVINO_CPU_KV_CACHE_PRECISION=u8`` to control KV cache precision. By default, FP16 / BF16 is used depending on platform.
84 -
85 -To enable better TPOT / TTFT latency, you can use vLLM's chunked prefill feature (``--enable-chunked-prefill``). Based on the experiments, the recommended batch size is ``256`` (``--max-num-batched-tokens``)
86 -
87 -OpenVINO best known configuration for CPU is:
88 -
89 -.. code-block:: console
90 -
91 - $ VLLM_OPENVINO_KVCACHE_SPACE=100 VLLM_OPENVINO_CPU_KV_CACHE_PRECISION=u8 VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON \
92 - python3 vllm/benchmarks/benchmark_throughput.py --model meta-llama/Llama-2-7b-chat-hf --dataset vllm/benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json --enable-chunked-prefill --max-num-batched-tokens 256
93 -
94 -GPU performance tips
95 -~~~~~~~~~~~~~~~~~~~~
96 -GPU device implements the logic for automatic detection of available GPU memory and, by default, tries to reserve as much memory as possible for the KV cache (taking into account ``gpu_memory_utilization`` option). However, this behavior can be overridden by explicitly specifying the desired amount of memory for the KV cache using ``VLLM_OPENVINO_KVCACHE_SPACE`` environment variable (e.g, ``VLLM_OPENVINO_KVCACHE_SPACE=8`` means 8 GB space for KV cache).
97 -
98 -Currently, the best performance using GPU can be achieved with the default vLLM execution parameters for models with quantized weights (8 and 4-bit integer data types are supported) and `preemption-mode=swap`.
99 -
100 -OpenVINO best known configuration for GPU is:
101 -
102 -.. code-block:: console
103 -
104 - $ VLLM_OPENVINO_DEVICE=GPU VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON \
105 - python3 vllm/benchmarks/benchmark_throughput.py --model meta-llama/Llama-2-7b-chat-hf --dataset vllm/benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json
106 -
107 -.. _openvino_backend_limitations:
108 -
109 -Limitations
110 ------------
111 -
112 -- LoRA serving is not supported.
113 -
114 -- Only LLM models are currently supported. LLaVa and encoder-decoder models are not currently enabled in vLLM OpenVINO integration.
115 -
116 -- Tensor and pipeline parallelism are not currently enabled in vLLM integration.
docs/source/getting_started/quickstart.md new
+174
@@ -0,0 +1,174 @@
1 +(quickstart)=
2 +
3 +# Quickstart
4 +
5 +This guide will help you quickly get started with vLLM to:
6 +
7 +- [Run offline batched inference](#offline-batched-inference)
8 +- [Run OpenAI-compatible inference](#openai-compatible-server)
9 +
10 +## Prerequisites
11 +
12 +- OS: Linux
13 +- Python: 3.9 -- 3.12
14 +- GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, L4, H100, etc.)
15 +
16 +## Installation
17 +
18 +You can install vLLM using pip. It's recommended to use [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/getting-started.html) to create and manage Python environments.
19 +
20 +```console
21 +$ conda create -n myenv python=3.10 -y
22 +$ conda activate myenv
23 +$ pip install vllm
24 +```
25 +
26 +Please refer to the {ref}`installation documentation <installation>` for more details on installing vLLM.
27 +
28 +(offline-batched-inference)=
29 +
30 +## Offline Batched Inference
31 +
32 +With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). The example script for this section can be found [here](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference.py).
33 +
34 +The first line of this example imports the classes {class}`~vllm.LLM` and {class}`~vllm.SamplingParams`:
35 +
36 +- {class}`~vllm.LLM` is the main class for running offline inference with vLLM engine.
37 +- {class}`~vllm.SamplingParams` specifies the parameters for the sampling process.
38 +
39 +```python
40 +from vllm import LLM, SamplingParams
41 +```
42 +
43 +The next section defines a list of input prompts and sampling parameters for text generation. The [sampling temperature](https://arxiv.org/html/2402.05201v1) is set to `0.8` and the [nucleus sampling probability](https://en.wikipedia.org/wiki/Top-p_sampling) is set to `0.95`. You can find more information about the sampling parameters [here](https://docs.vllm.ai/en/stable/dev/sampling_params.html).
44 +
45 +```python
46 +prompts = [
47 + "Hello, my name is",
48 + "The president of the United States is",
49 + "The capital of France is",
50 + "The future of AI is",
51 +]
52 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
53 +```
54 +
55 +The {class}`~vllm.LLM` class initializes vLLM's engine and the [OPT-125M model](https://arxiv.org/abs/2205.01068) for offline inference. The list of supported models can be found [here](#supported-models).
56 +
57 +```python
58 +llm = LLM(model="facebook/opt-125m")
59 +```
60 +
61 +```{note}
62 +By default, vLLM downloads models from [HuggingFace](https://huggingface.co/). If you would like to use models from [ModelScope](https://www.modelscope.cn), set the environment variable `VLLM_USE_MODELSCOPE` before initializing the engine.
63 +```
64 +
65 +Now, the fun part! The outputs are generated using `llm.generate`. It adds the input prompts to the vLLM engine's waiting queue and executes the vLLM engine to generate the outputs with high throughput. The outputs are returned as a list of `RequestOutput` objects, which include all of the output tokens.
66 +
67 +```python
68 +outputs = llm.generate(prompts, sampling_params)
69 +
70 +for output in outputs:
71 + prompt = output.prompt
72 + generated_text = output.outputs[0].text
73 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
74 +```
75 +
76 +(openai-compatible-server)=
77 +
78 +## OpenAI-Compatible Server
79 +
80 +vLLM can be deployed as a server that implements the OpenAI API protocol. This allows vLLM to be used as a drop-in replacement for applications using OpenAI API.
81 +By default, it starts the server at `http://localhost:8000`. You can specify the address with `--host` and `--port` arguments. The server currently hosts one model at a time and implements endpoints such as [list models](https://platform.openai.com/docs/api-reference/models/list), [create chat completion](https://platform.openai.com/docs/api-reference/chat/completions/create), and [create completion](https://platform.openai.com/docs/api-reference/completions/create) endpoints.
82 +
83 +Run the following command to start the vLLM server with the [Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct) model:
84 +
85 +```console
86 +$ vllm serve Qwen/Qwen2.5-1.5B-Instruct
87 +```
88 +
89 +```{note}
90 +By default, the server uses a predefined chat template stored in the tokenizer. You can learn about overriding it [here](https://github.com/vllm-project/vllm/blob/main/docs/source/serving/openai_compatible_server.md#chat-template).
91 +```
92 +
93 +This server can be queried in the same format as OpenAI API. For example, to list the models:
94 +
95 +```console
96 +$ curl http://localhost:8000/v1/models
97 +```
98 +
99 +You can pass in the argument `--api-key` or environment variable `VLLM_API_KEY` to enable the server to check for API key in the header.
100 +
101 +### OpenAI Completions API with vLLM
102 +
103 +Once your server is started, you can query the model with input prompts:
104 +
105 +```console
106 +$ curl http://localhost:8000/v1/completions \
107 +$ -H "Content-Type: application/json" \
108 +$ -d '{
109 +$ "model": "Qwen/Qwen2.5-1.5B-Instruct",
110 +$ "prompt": "San Francisco is a",
111 +$ "max_tokens": 7,
112 +$ "temperature": 0
113 +$ }'
114 +```
115 +
116 +Since this server is compatible with OpenAI API, you can use it as a drop-in replacement for any applications using OpenAI API. For example, another way to query the server is via the `openai` python package:
117 +
118 +```python
119 +from openai import OpenAI
120 +
121 +# Modify OpenAI's API key and API base to use vLLM's API server.
122 +openai_api_key = "EMPTY"
123 +openai_api_base = "http://localhost:8000/v1"
124 +client = OpenAI(
125 + api_key=openai_api_key,
126 + base_url=openai_api_base,
127 +)
128 +completion = client.completions.create(model="Qwen/Qwen2.5-1.5B-Instruct",
129 + prompt="San Francisco is a")
130 +print("Completion result:", completion)
131 +```
132 +
133 +A more detailed client example can be found [here](https://github.com/vllm-project/vllm/blob/main/examples/openai_completion_client.py).
134 +
135 +### OpenAI Chat Completions API with vLLM
136 +
137 +vLLM is designed to also support the OpenAI Chat Completions API. The chat interface is a more dynamic, interactive way to communicate with the model, allowing back-and-forth exchanges that can be stored in the chat history. This is useful for tasks that require context or more detailed explanations.
138 +
139 +You can use the [create chat completion](https://platform.openai.com/docs/api-reference/chat/completions/create) endpoint to interact with the model:
140 +
141 +```console
142 +$ curl http://localhost:8000/v1/chat/completions \
143 +$ -H "Content-Type: application/json" \
144 +$ -d '{
145 +$ "model": "Qwen/Qwen2.5-1.5B-Instruct",
146 +$ "messages": [
147 +$ {"role": "system", "content": "You are a helpful assistant."},
148 +$ {"role": "user", "content": "Who won the world series in 2020?"}
149 +$ ]
150 +$ }'
151 +```
152 +
153 +Alternatively, you can use the `openai` python package:
154 +
155 +```python
156 +from openai import OpenAI
157 +# Set OpenAI's API key and API base to use vLLM's API server.
158 +openai_api_key = "EMPTY"
159 +openai_api_base = "http://localhost:8000/v1"
160 +
161 +client = OpenAI(
162 + api_key=openai_api_key,
163 + base_url=openai_api_base,
164 +)
165 +
166 +chat_response = client.chat.completions.create(
167 + model="Qwen/Qwen2.5-1.5B-Instruct",
168 + messages=[
169 + {"role": "system", "content": "You are a helpful assistant."},
170 + {"role": "user", "content": "Tell me a joke."},
171 + ]
172 +)
173 +print("Chat response:", chat_response)
174 +```
docs/source/getting_started/quickstart.rst deleted
-181
@@ -1,181 +0,0 @@
1 -.. _quickstart:
2 -
3 -==========
4 -Quickstart
5 -==========
6 -
7 -This guide will help you quickly get started with vLLM to:
8 -
9 -* :ref:`Run offline batched inference <offline_batched_inference>`
10 -* :ref:`Run OpenAI-compatible inference <openai_compatible_server>`
11 -
12 -Prerequisites
13 ---------------
14 -- OS: Linux
15 -- Python: 3.9 -- 3.12
16 -- GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, L4, H100, etc.)
17 -
18 -Installation
19 ---------------
20 -
21 -You can install vLLM using pip. It's recommended to use `conda <https://docs.conda.io/projects/conda/en/latest/user-guide/getting-started.html>`_ to create and manage Python environments.
22 -
23 -.. code-block:: console
24 -
25 - $ conda create -n myenv python=3.10 -y
26 - $ conda activate myenv
27 - $ pip install vllm
28 -
29 -Please refer to the :ref:`installation documentation <installation>` for more details on installing vLLM.
30 -
31 -.. _offline_batched_inference:
32 -
33 -Offline Batched Inference
34 --------------------------
35 -
36 -With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). The example script for this section can be found `here <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference.py>`__.
37 -
38 -The first line of this example imports the classes :class:`~vllm.LLM` and :class:`~vllm.SamplingParams`:
39 -
40 -- :class:`~vllm.LLM` is the main class for running offline inference with vLLM engine.
41 -- :class:`~vllm.SamplingParams` specifies the parameters for the sampling process.
42 -
43 -.. code-block:: python
44 -
45 - from vllm import LLM, SamplingParams
46 -
47 -The next section defines a list of input prompts and sampling parameters for text generation. The `sampling temperature <https://arxiv.org/html/2402.05201v1>`_ is set to ``0.8`` and the `nucleus sampling probability <https://en.wikipedia.org/wiki/Top-p_sampling>`_ is set to ``0.95``. You can find more information about the sampling parameters `here <https://docs.vllm.ai/en/stable/dev/sampling_params.html>`__.
48 -
49 -.. code-block:: python
50 -
51 - prompts = [
52 - "Hello, my name is",
53 - "The president of the United States is",
54 - "The capital of France is",
55 - "The future of AI is",
56 - ]
57 - sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
58 -
59 -The :class:`~vllm.LLM` class initializes vLLM's engine and the `OPT-125M model <https://arxiv.org/abs/2205.01068>`_ for offline inference. The list of supported models can be found :ref:`here <supported_models>`.
60 -
61 -.. code-block:: python
62 -
63 - llm = LLM(model="facebook/opt-125m")
64 -
65 -.. note::
66 -
67 - By default, vLLM downloads models from `HuggingFace <https://huggingface.co/>`_. If you would like to use models from `ModelScope <https://www.modelscope.cn>`_, set the environment variable ``VLLM_USE_MODELSCOPE`` before initializing the engine.
68 -
69 -Now, the fun part! The outputs are generated using ``llm.generate``. It adds the input prompts to the vLLM engine's waiting queue and executes the vLLM engine to generate the outputs with high throughput. The outputs are returned as a list of ``RequestOutput`` objects, which include all of the output tokens.
70 -
71 -.. code-block:: python
72 -
73 - outputs = llm.generate(prompts, sampling_params)
74 -
75 - for output in outputs:
76 - prompt = output.prompt
77 - generated_text = output.outputs[0].text
78 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
79 -
80 -.. _openai_compatible_server:
81 -
82 -OpenAI-Compatible Server
83 -------------------------
84 -
85 -vLLM can be deployed as a server that implements the OpenAI API protocol. This allows vLLM to be used as a drop-in replacement for applications using OpenAI API.
86 -By default, it starts the server at ``http://localhost:8000``. You can specify the address with ``--host`` and ``--port`` arguments. The server currently hosts one model at a time and implements endpoints such as `list models <https://platform.openai.com/docs/api-reference/models/list>`_, `create chat completion <https://platform.openai.com/docs/api-reference/chat/completions/create>`_, and `create completion <https://platform.openai.com/docs/api-reference/completions/create>`_ endpoints.
87 -
88 -Run the following command to start the vLLM server with the `Qwen2.5-1.5B-Instruct <https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct>`_ model:
89 -
90 -.. code-block:: console
91 -
92 - $ vllm serve Qwen/Qwen2.5-1.5B-Instruct
93 -
94 -.. note::
95 -
96 - By default, the server uses a predefined chat template stored in the tokenizer. You can learn about overriding it `here <https://github.com/vllm-project/vllm/blob/main/docs/source/serving/openai_compatible_server.md#chat-template>`__.
97 -
98 -This server can be queried in the same format as OpenAI API. For example, to list the models:
99 -
100 -.. code-block:: console
101 -
102 - $ curl http://localhost:8000/v1/models
103 -
104 -You can pass in the argument ``--api-key`` or environment variable ``VLLM_API_KEY`` to enable the server to check for API key in the header.
105 -
106 -OpenAI Completions API with vLLM
107 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
108 -
109 -Once your server is started, you can query the model with input prompts:
110 -
111 -.. code-block:: console
112 -
113 - $ curl http://localhost:8000/v1/completions \
114 - $ -H "Content-Type: application/json" \
115 - $ -d '{
116 - $ "model": "Qwen/Qwen2.5-1.5B-Instruct",
117 - $ "prompt": "San Francisco is a",
118 - $ "max_tokens": 7,
119 - $ "temperature": 0
120 - $ }'
121 -
122 -Since this server is compatible with OpenAI API, you can use it as a drop-in replacement for any applications using OpenAI API. For example, another way to query the server is via the ``openai`` python package:
123 -
124 -.. code-block:: python
125 -
126 - from openai import OpenAI
127 -
128 - # Modify OpenAI's API key and API base to use vLLM's API server.
129 - openai_api_key = "EMPTY"
130 - openai_api_base = "http://localhost:8000/v1"
131 - client = OpenAI(
132 - api_key=openai_api_key,
133 - base_url=openai_api_base,
134 - )
135 - completion = client.completions.create(model="Qwen/Qwen2.5-1.5B-Instruct",
136 - prompt="San Francisco is a")
137 - print("Completion result:", completion)
138 -
139 -A more detailed client example can be found `here <https://github.com/vllm-project/vllm/blob/main/examples/openai_completion_client.py>`__.
140 -
141 -OpenAI Chat Completions API with vLLM
142 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
143 -
144 -vLLM is designed to also support the OpenAI Chat Completions API. The chat interface is a more dynamic, interactive way to communicate with the model, allowing back-and-forth exchanges that can be stored in the chat history. This is useful for tasks that require context or more detailed explanations.
145 -
146 -You can use the `create chat completion <https://platform.openai.com/docs/api-reference/chat/completions/create>`_ endpoint to interact with the model:
147 -
148 -.. code-block:: console
149 -
150 - $ curl http://localhost:8000/v1/chat/completions \
151 - $ -H "Content-Type: application/json" \
152 - $ -d '{
153 - $ "model": "Qwen/Qwen2.5-1.5B-Instruct",
154 - $ "messages": [
155 - $ {"role": "system", "content": "You are a helpful assistant."},
156 - $ {"role": "user", "content": "Who won the world series in 2020?"}
157 - $ ]
158 - $ }'
159 -
160 -Alternatively, you can use the ``openai`` python package:
161 -
162 -.. code-block:: python
163 -
164 - from openai import OpenAI
165 - # Set OpenAI's API key and API base to use vLLM's API server.
166 - openai_api_key = "EMPTY"
167 - openai_api_base = "http://localhost:8000/v1"
168 -
169 - client = OpenAI(
170 - api_key=openai_api_key,
171 - base_url=openai_api_base,
172 - )
173 -
174 - chat_response = client.chat.completions.create(
175 - model="Qwen/Qwen2.5-1.5B-Instruct",
176 - messages=[
177 - {"role": "system", "content": "You are a helpful assistant."},
178 - {"role": "user", "content": "Tell me a joke."},
179 - ]
180 - )
181 - print("Chat response:", chat_response)
docs/source/getting_started/tpu-installation.md new
+193
@@ -0,0 +1,193 @@
1 +(installation-tpu)=
2 +
3 +# Installation with TPU
4 +
5 +Tensor Processing Units (TPUs) are Google's custom-developed application-specific
6 +integrated circuits (ASICs) used to accelerate machine learning workloads. TPUs
7 +are available in different versions each with different hardware specifications.
8 +For more information about TPUs, see [TPU System Architecture](https://cloud.google.com/tpu/docs/system-architecture-tpu-vm).
9 +For more information on the TPU versions supported with vLLM, see:
10 +
11 +- [TPU v6e](https://cloud.google.com/tpu/docs/v6e)
12 +- [TPU v5e](https://cloud.google.com/tpu/docs/v5e)
13 +- [TPU v5p](https://cloud.google.com/tpu/docs/v5p)
14 +- [TPU v4](https://cloud.google.com/tpu/docs/v4)
15 +
16 +These TPU versions allow you to configure the physical arrangements of the TPU
17 +chips. This can improve throughput and networking performance. For more
18 +information see:
19 +
20 +- [TPU v6e topologies](https://cloud.google.com/tpu/docs/v6e#configurations)
21 +- [TPU v5e topologies](https://cloud.google.com/tpu/docs/v5e#tpu-v5e-config)
22 +- [TPU v5p topologies](https://cloud.google.com/tpu/docs/v5p#tpu-v5p-config)
23 +- [TPU v4 topologies](https://cloud.google.com/tpu/docs/v4#tpu-v4-config)
24 +
25 +In order for you to use Cloud TPUs you need to have TPU quota granted to your
26 +Google Cloud Platform project. TPU quotas specify how many TPUs you can use in a
27 +GPC project and are specified in terms of TPU version, the number of TPU you
28 +want to use, and quota type. For more information, see [TPU quota](https://cloud.google.com/tpu/docs/quota#tpu_quota).
29 +
30 +For TPU pricing information, see [Cloud TPU pricing](https://cloud.google.com/tpu/pricing).
31 +
32 +You may need additional persistent storage for your TPU VMs. For more
33 +information, see [Storage options for Cloud TPU data](https://cloud.devsite.corp.google.com/tpu/docs/storage-options).
34 +
35 +## Requirements
36 +
37 +- Google Cloud TPU VM
38 +- TPU versions: v6e, v5e, v5p, v4
39 +- Python: 3.10 or newer
40 +
41 +### Provision Cloud TPUs
42 +
43 +You can provision Cloud TPUs using the [Cloud TPU API](https://cloud.google.com/tpu/docs/reference/rest)
44 +or the [queued resources](https://cloud.google.com/tpu/docs/queued-resources)
45 +API. This section shows how to create TPUs using the queued resource API. For
46 +more information about using the Cloud TPU API, see [Create a Cloud TPU using the Create Node API](https://cloud.google.com/tpu/docs/managing-tpus-tpu-vm#create-node-api).
47 +Queued resources enable you to request Cloud TPU resources in a queued manner.
48 +When you request queued resources, the request is added to a queue maintained by
49 +the Cloud TPU service. When the requested resource becomes available, it's
50 +assigned to your Google Cloud project for your immediate exclusive use.
51 +
52 +```{note}
53 +In all of the following commands, replace the ALL CAPS parameter names with
54 +appropriate values. See the parameter descriptions table for more information.
55 +```
56 +
57 +## Provision a Cloud TPU with the queued resource API
58 +
59 +Create a TPU v5e with 4 TPU chips:
60 +
61 +```console
62 +gcloud alpha compute tpus queued-resources create QUEUED_RESOURCE_ID \
63 +--node-id TPU_NAME \
64 +--project PROJECT_ID \
65 +--zone ZONE \
66 +--accelerator-type ACCELERATOR_TYPE \
67 +--runtime-version RUNTIME_VERSION \
68 +--service-account SERVICE_ACCOUNT
69 +```
70 +
71 +```{eval-rst}
72 +.. list-table:: Parameter descriptions
73 + :header-rows: 1
74 +
75 + * - Parameter name
76 + - Description
77 + * - QUEUED_RESOURCE_ID
78 + - The user-assigned ID of the queued resource request.
79 + * - TPU_NAME
80 + - The user-assigned name of the TPU which is created when the queued
81 + resource request is allocated.
82 + * - PROJECT_ID
83 + - Your Google Cloud project
84 + * - ZONE
85 + - The GCP zone where you want to create your Cloud TPU. The value you use
86 + depends on the version of TPUs you are using. For more information, see
87 + `TPU regions and zones <https://cloud.google.com/tpu/docs/regions-zones>`_
88 + * - ACCELERATOR_TYPE
89 + - The TPU version you want to use. Specify the TPU version, for example
90 + `v5litepod-4` specifies a v5e TPU with 4 cores. For more information,
91 + see `TPU versions <https://cloud.devsite.corp.google.com/tpu/docs/system-architecture-tpu-vm#versions>`_.
92 + * - RUNTIME_VERSION
93 + - The TPU VM runtime version to use. For more information see `TPU VM images <https://cloud.google.com/tpu/docs/runtimes>`_.
94 + * - SERVICE_ACCOUNT
95 + - The email address for your service account. You can find it in the IAM
96 + Cloud Console under *Service Accounts*. For example:
97 + `tpu-service-account@<your_project_ID>.iam.gserviceaccount.com`
98 +```
99 +
100 +Connect to your TPU using SSH:
101 +
102 +```bash
103 +gcloud compute tpus tpu-vm ssh TPU_NAME --zone ZONE
104 +```
105 +
106 +Install Miniconda
107 +
108 +```bash
109 +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
110 +bash Miniconda3-latest-Linux-x86_64.sh
111 +source ~/.bashrc
112 +```
113 +
114 +Create and activate a Conda environment for vLLM:
115 +
116 +```bash
117 +conda create -n vllm python=3.10 -y
118 +conda activate vllm
119 +```
120 +
121 +Clone the vLLM repository and go to the vLLM directory:
122 +
123 +```bash
124 +git clone https://github.com/vllm-project/vllm.git && cd vllm
125 +```
126 +
127 +Uninstall the existing `torch` and `torch_xla` packages:
128 +
129 +```bash
130 +pip uninstall torch torch-xla -y
131 +```
132 +
133 +Install build dependencies:
134 +
135 +```bash
136 +pip install -r requirements-tpu.txt
137 +sudo apt-get install libopenblas-base libopenmpi-dev libomp-dev
138 +```
139 +
140 +Run the setup script:
141 +
142 +```bash
143 +VLLM_TARGET_DEVICE="tpu" python setup.py develop
144 +```
145 +
146 +## Provision Cloud TPUs with GKE
147 +
148 +For more information about using TPUs with GKE, see
149 +<https://cloud.google.com/kubernetes-engine/docs/how-to/tpus>
150 +<https://cloud.google.com/kubernetes-engine/docs/concepts/tpus>
151 +<https://cloud.google.com/kubernetes-engine/docs/concepts/plan-tpus>
152 +
153 +(build-docker-tpu)=
154 +
155 +## Build a docker image with {code}`Dockerfile.tpu`
156 +
157 +You can use [Dockerfile.tpu](https://github.com/vllm-project/vllm/blob/main/Dockerfile.tpu)
158 +to build a Docker image with TPU support.
159 +
160 +```console
161 +$ docker build -f Dockerfile.tpu -t vllm-tpu .
162 +```
163 +
164 +Run the Docker image with the following command:
165 +
166 +```console
167 +$ # Make sure to add `--privileged --net host --shm-size=16G`.
168 +$ docker run --privileged --net host --shm-size=16G -it vllm-tpu
169 +```
170 +
171 +```{note}
172 +Since TPU relies on XLA which requires static shapes, vLLM bucketizes the
173 +possible input shapes and compiles an XLA graph for each shape. The
174 +compilation time may take 20~30 minutes in the first run. However, the
175 +compilation time reduces to ~5 minutes afterwards because the XLA graphs are
176 +cached in the disk (in {code}`VLLM_XLA_CACHE_PATH` or {code}`~/.cache/vllm/xla_cache` by default).
177 +```
178 +
179 +````{tip}
180 +If you encounter the following error:
181 +
182 +```console
183 +from torch._C import * # noqa: F403
184 +ImportError: libopenblas.so.0: cannot open shared object file: No such
185 +file or directory
186 +```
187 +
188 +Install OpenBLAS with the following command:
189 +
190 +```console
191 +$ sudo apt-get install libopenblas-base libopenmpi-dev libomp-dev
192 +```
193 +````
docs/source/getting_started/tpu-installation.rst deleted
-200
@@ -1,200 +0,0 @@
1 -.. _installation_tpu:
2 -
3 -#####################
4 -Installation with TPU
5 -#####################
6 -
7 -Tensor Processing Units (TPUs) are Google's custom-developed application-specific
8 -integrated circuits (ASICs) used to accelerate machine learning workloads. TPUs
9 -are available in different versions each with different hardware specifications.
10 -For more information about TPUs, see `TPU System Architecture <https://cloud.google.com/tpu/docs/system-architecture-tpu-vm>`_.
11 -For more information on the TPU versions supported with vLLM, see:
12 -
13 -* `TPU v6e <https://cloud.google.com/tpu/docs/v6e>`_
14 -* `TPU v5e <https://cloud.google.com/tpu/docs/v5e>`_
15 -* `TPU v5p <https://cloud.google.com/tpu/docs/v5p>`_
16 -* `TPU v4 <https://cloud.google.com/tpu/docs/v4>`_
17 -
18 -These TPU versions allow you to configure the physical arrangements of the TPU
19 -chips. This can improve throughput and networking performance. For more
20 -information see:
21 -
22 -* `TPU v6e topologies <https://cloud.google.com/tpu/docs/v6e#configurations>`_
23 -* `TPU v5e topologies <https://cloud.google.com/tpu/docs/v5e#tpu-v5e-config>`_
24 -* `TPU v5p topologies <https://cloud.google.com/tpu/docs/v5p#tpu-v5p-config>`_
25 -* `TPU v4 topologies <https://cloud.google.com/tpu/docs/v4#tpu-v4-config>`_
26 -
27 -In order for you to use Cloud TPUs you need to have TPU quota granted to your
28 -Google Cloud Platform project. TPU quotas specify how many TPUs you can use in a
29 -GPC project and are specified in terms of TPU version, the number of TPU you
30 -want to use, and quota type. For more information, see `TPU quota <https://cloud.google.com/tpu/docs/quota#tpu_quota>`_.
31 -
32 -For TPU pricing information, see `Cloud TPU pricing <https://cloud.google.com/tpu/pricing>`_.
33 -
34 -You may need additional persistent storage for your TPU VMs. For more
35 -information, see `Storage options for Cloud TPU data <https://cloud.devsite.corp.google.com/tpu/docs/storage-options>`_.
36 -
37 -Requirements
38 -------------
39 -
40 -* Google Cloud TPU VM
41 -* TPU versions: v6e, v5e, v5p, v4
42 -* Python: 3.10 or newer
43 -
44 -Provision Cloud TPUs
45 -====================
46 -
47 -You can provision Cloud TPUs using the `Cloud TPU API <https://cloud.google.com/tpu/docs/reference/rest>`_
48 -or the `queued resources <https://cloud.google.com/tpu/docs/queued-resources>`_
49 -API. This section shows how to create TPUs using the queued resource API. For
50 -more information about using the Cloud TPU API, see `Create a Cloud TPU using the Create Node API <https://cloud.google.com/tpu/docs/managing-tpus-tpu-vm#create-node-api>`_.
51 -Queued resources enable you to request Cloud TPU resources in a queued manner.
52 -When you request queued resources, the request is added to a queue maintained by
53 -the Cloud TPU service. When the requested resource becomes available, it's
54 -assigned to your Google Cloud project for your immediate exclusive use.
55 -
56 -.. note::
57 - In all of the following commands, replace the ALL CAPS parameter names with
58 - appropriate values. See the parameter descriptions table for more information.
59 -
60 -Provision a Cloud TPU with the queued resource API
61 ---------------------------------------------------
62 -Create a TPU v5e with 4 TPU chips:
63 -
64 -.. code-block:: console
65 -
66 - gcloud alpha compute tpus queued-resources create QUEUED_RESOURCE_ID \
67 - --node-id TPU_NAME \
68 - --project PROJECT_ID \
69 - --zone ZONE \
70 - --accelerator-type ACCELERATOR_TYPE \
71 - --runtime-version RUNTIME_VERSION \
72 - --service-account SERVICE_ACCOUNT
73 -
74 -
75 -.. list-table:: Parameter descriptions
76 - :header-rows: 1
77 -
78 - * - Parameter name
79 - - Description
80 - * - QUEUED_RESOURCE_ID
81 - - The user-assigned ID of the queued resource request.
82 - * - TPU_NAME
83 - - The user-assigned name of the TPU which is created when the queued
84 - resource request is allocated.
85 - * - PROJECT_ID
86 - - Your Google Cloud project
87 - * - ZONE
88 - - The GCP zone where you want to create your Cloud TPU. The value you use
89 - depends on the version of TPUs you are using. For more information, see
90 - `TPU regions and zones <https://cloud.google.com/tpu/docs/regions-zones>`_
91 - * - ACCELERATOR_TYPE
92 - - The TPU version you want to use. Specify the TPU version, for example
93 - `v5litepod-4` specifies a v5e TPU with 4 cores. For more information,
94 - see `TPU versions <https://cloud.devsite.corp.google.com/tpu/docs/system-architecture-tpu-vm#versions>`_.
95 - * - RUNTIME_VERSION
96 - - The TPU VM runtime version to use. For more information see `TPU VM images <https://cloud.google.com/tpu/docs/runtimes>`_.
97 - * - SERVICE_ACCOUNT
98 - - The email address for your service account. You can find it in the IAM
99 - Cloud Console under *Service Accounts*. For example:
100 - `tpu-service-account@<your_project_ID>.iam.gserviceaccount.com`
101 -
102 -Connect to your TPU using SSH:
103 -
104 -.. code-block:: bash
105 -
106 - gcloud compute tpus tpu-vm ssh TPU_NAME --zone ZONE
107 -
108 -Install Miniconda
109 -
110 -.. code-block:: bash
111 -
112 - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
113 - bash Miniconda3-latest-Linux-x86_64.sh
114 - source ~/.bashrc
115 -
116 -Create and activate a Conda environment for vLLM:
117 -
118 -.. code-block:: bash
119 -
120 - conda create -n vllm python=3.10 -y
121 - conda activate vllm
122 -
123 -Clone the vLLM repository and go to the vLLM directory:
124 -
125 -.. code-block:: bash
126 -
127 - git clone https://github.com/vllm-project/vllm.git && cd vllm
128 -
129 -Uninstall the existing `torch` and `torch_xla` packages:
130 -
131 -.. code-block:: bash
132 -
133 - pip uninstall torch torch-xla -y
134 -
135 -Install build dependencies:
136 -
137 -.. code-block:: bash
138 -
139 - pip install -r requirements-tpu.txt
140 - sudo apt-get install libopenblas-base libopenmpi-dev libomp-dev
141 -
142 -Run the setup script:
143 -
144 -.. code-block:: bash
145 -
146 - VLLM_TARGET_DEVICE="tpu" python setup.py develop
147 -
148 -
149 -Provision Cloud TPUs with GKE
150 ------------------------------
151 -
152 -For more information about using TPUs with GKE, see
153 -https://cloud.google.com/kubernetes-engine/docs/how-to/tpus
154 -https://cloud.google.com/kubernetes-engine/docs/concepts/tpus
155 -https://cloud.google.com/kubernetes-engine/docs/concepts/plan-tpus
156 -
157 -.. _build_docker_tpu:
158 -
159 -Build a docker image with :code:`Dockerfile.tpu`
160 -------------------------------------------------
161 -
162 -You can use `Dockerfile.tpu <https://github.com/vllm-project/vllm/blob/main/Dockerfile.tpu>`_
163 -to build a Docker image with TPU support.
164 -
165 -.. code-block:: console
166 -
167 - $ docker build -f Dockerfile.tpu -t vllm-tpu .
168 -
169 -Run the Docker image with the following command:
170 -
171 -.. code-block:: console
172 -
173 - $ # Make sure to add `--privileged --net host --shm-size=16G`.
174 - $ docker run --privileged --net host --shm-size=16G -it vllm-tpu
175 -
176 -.. note::
177 -
178 - Since TPU relies on XLA which requires static shapes, vLLM bucketizes the
179 - possible input shapes and compiles an XLA graph for each shape. The
180 - compilation time may take 20~30 minutes in the first run. However, the
181 - compilation time reduces to ~5 minutes afterwards because the XLA graphs are
182 - cached in the disk (in :code:`VLLM_XLA_CACHE_PATH` or :code:`~/.cache/vllm/xla_cache` by default).
183 -
184 -.. tip::
185 -
186 - If you encounter the following error:
187 -
188 - .. code-block:: console
189 -
190 - from torch._C import * # noqa: F403
191 - ImportError: libopenblas.so.0: cannot open shared object file: No such
192 - file or directory
193 -
194 -
195 - Install OpenBLAS with the following command:
196 -
197 - .. code-block:: console
198 -
199 - $ sudo apt-get install libopenblas-base libopenmpi-dev libomp-dev
200 -
docs/source/getting_started/xpu-installation.md new
+74
@@ -0,0 +1,74 @@
1 +(installation-xpu)=
2 +
3 +# Installation with XPU
4 +
5 +vLLM initially supports basic model inferencing and serving on Intel GPU platform.
6 +
7 +Table of contents:
8 +
9 +1. [Requirements](#xpu-backend-requirements)
10 +2. [Quick start using Dockerfile](#xpu-backend-quick-start-dockerfile)
11 +3. [Build from source](#build-xpu-backend-from-source)
12 +
13 +(xpu-backend-requirements)=
14 +
15 +## Requirements
16 +
17 +- OS: Linux
18 +- Supported Hardware: Intel Data Center GPU, Intel ARC GPU
19 +- OneAPI requirements: oneAPI 2024.2
20 +
21 +(xpu-backend-quick-start-dockerfile)=
22 +
23 +## Quick start using Dockerfile
24 +
25 +```console
26 +$ docker build -f Dockerfile.xpu -t vllm-xpu-env --shm-size=4g .
27 +$ docker run -it \
28 + --rm \
29 + --network=host \
30 + --device /dev/dri \
31 + -v /dev/dri/by-path:/dev/dri/by-path \
32 + vllm-xpu-env
33 +```
34 +
35 +(build-xpu-backend-from-source)=
36 +
37 +## Build from source
38 +
39 +- First, install required driver and intel OneAPI 2024.2 or later.
40 +- Second, install Python packages for vLLM XPU backend building:
41 +
42 +```console
43 +$ source /opt/intel/oneapi/setvars.sh
44 +$ pip install --upgrade pip
45 +$ pip install -v -r requirements-xpu.txt
46 +```
47 +
48 +- Finally, build and install vLLM XPU backend:
49 +
50 +```console
51 +$ VLLM_TARGET_DEVICE=xpu python setup.py install
52 +```
53 +
54 +```{note}
55 +- FP16 is the default data type in the current XPU backend. The BF16 data
56 + type will be supported in the future.
57 +```
58 +
59 +## Distributed inference and serving
60 +
61 +XPU platform supports tensor-parallel inference/serving and also supports pipeline parallel as a beta feature for online serving. We requires Ray as the distributed runtime backend. For example, a reference execution likes following:
62 +
63 +```console
64 +$ python -m vllm.entrypoints.openai.api_server \
65 +$ --model=facebook/opt-13b \
66 +$ --dtype=bfloat16 \
67 +$ --device=xpu \
68 +$ --max_model_len=1024 \
69 +$ --distributed-executor-backend=ray \
70 +$ --pipeline-parallel-size=2 \
71 +$ -tp=8
72 +```
73 +
74 +By default, a ray instance will be launched automatically if no existing one is detected in system, with `num-gpus` equals to `parallel_config.world_size`. We recommend properly starting a ray cluster before execution, referring helper [script](https://github.com/vllm-project/vllm/tree/main/examples/run_cluster.sh).
docs/source/getting_started/xpu-installation.rst deleted
-80
@@ -1,80 +0,0 @@
1 -.. _installation_xpu:
2 -
3 -Installation with XPU
4 -========================
5 -
6 -vLLM initially supports basic model inferencing and serving on Intel GPU platform.
7 -
8 -Table of contents:
9 -
10 -#. :ref:`Requirements <xpu_backend_requirements>`
11 -#. :ref:`Quick start using Dockerfile <xpu_backend_quick_start_dockerfile>`
12 -#. :ref:`Build from source <build_xpu_backend_from_source>`
13 -
14 -.. _xpu_backend_requirements:
15 -
16 -Requirements
17 -------------
18 -
19 -* OS: Linux
20 -* Supported Hardware: Intel Data Center GPU, Intel ARC GPU
21 -* OneAPI requirements: oneAPI 2024.2
22 -
23 -.. _xpu_backend_quick_start_dockerfile:
24 -
25 -Quick start using Dockerfile
26 -----------------------------
27 -
28 -.. code-block:: console
29 -
30 - $ docker build -f Dockerfile.xpu -t vllm-xpu-env --shm-size=4g .
31 - $ docker run -it \
32 - --rm \
33 - --network=host \
34 - --device /dev/dri \
35 - -v /dev/dri/by-path:/dev/dri/by-path \
36 - vllm-xpu-env
37 -
38 -.. _build_xpu_backend_from_source:
39 -
40 -Build from source
41 ------------------
42 -
43 -- First, install required driver and intel OneAPI 2024.2 or later.
44 -
45 -- Second, install Python packages for vLLM XPU backend building:
46 -
47 -.. code-block:: console
48 -
49 - $ source /opt/intel/oneapi/setvars.sh
50 - $ pip install --upgrade pip
51 - $ pip install -v -r requirements-xpu.txt
52 -
53 -- Finally, build and install vLLM XPU backend:
54 -
55 -.. code-block:: console
56 -
57 - $ VLLM_TARGET_DEVICE=xpu python setup.py install
58 -
59 -.. note::
60 - - FP16 is the default data type in the current XPU backend. The BF16 data
61 - type will be supported in the future.
62 -
63 -
64 -Distributed inference and serving
65 ----------------------------------
66 -
67 -XPU platform supports tensor-parallel inference/serving and also supports pipeline parallel as a beta feature for online serving. We requires Ray as the distributed runtime backend. For example, a reference execution likes following:
68 -
69 -.. code-block:: console
70 -
71 - $ python -m vllm.entrypoints.openai.api_server \
72 - $ --model=facebook/opt-13b \
73 - $ --dtype=bfloat16 \
74 - $ --device=xpu \
75 - $ --max_model_len=1024 \
76 - $ --distributed-executor-backend=ray \
77 - $ --pipeline-parallel-size=2 \
78 - $ -tp=8
79 -
80 -By default, a ray instance will be launched automatically if no existing one is detected in system, with ``num-gpus`` equals to ``parallel_config.world_size``. We recommend properly starting a ray cluster before execution, referring helper `script <https://github.com/vllm-project/vllm/tree/main/examples/run_cluster.sh>`_.
docs/source/index.md new
+200
@@ -0,0 +1,200 @@
1 +# Welcome to vLLM!
2 +
3 +```{figure} ./assets/logos/vllm-logo-text-light.png
4 +:align: center
5 +:alt: vLLM
6 +:class: no-scaled-link
7 +:width: 60%
8 +```
9 +
10 +```{raw} html
11 +<p style="text-align:center">
12 +<strong>Easy, fast, and cheap LLM serving for everyone
13 +</strong>
14 +</p>
15 +
16 +<p style="text-align:center">
17 +<script async defer src="https://buttons.github.io/buttons.js"></script>
18 +<a class="github-button" href="https://github.com/vllm-project/vllm" data-show-count="true" data-size="large" aria-label="Star">Star</a>
19 +<a class="github-button" href="https://github.com/vllm-project/vllm/subscription" data-icon="octicon-eye" data-size="large" aria-label="Watch">Watch</a>
20 +<a class="github-button" href="https://github.com/vllm-project/vllm/fork" data-icon="octicon-repo-forked" data-size="large" aria-label="Fork">Fork</a>
21 +</p>
22 +```
23 +
24 +vLLM is a fast and easy-to-use library for LLM inference and serving.
25 +
26 +vLLM is fast with:
27 +
28 +- State-of-the-art serving throughput
29 +- Efficient management of attention key and value memory with **PagedAttention**
30 +- Continuous batching of incoming requests
31 +- Fast model execution with CUDA/HIP graph
32 +- Quantization: [GPTQ](https://arxiv.org/abs/2210.17323), [AWQ](https://arxiv.org/abs/2306.00978), INT4, INT8, and FP8
33 +- Optimized CUDA kernels, including integration with FlashAttention and FlashInfer.
34 +- Speculative decoding
35 +- Chunked prefill
36 +
37 +vLLM is flexible and easy to use with:
38 +
39 +- Seamless integration with popular HuggingFace models
40 +- High-throughput serving with various decoding algorithms, including *parallel sampling*, *beam search*, and more
41 +- Tensor parallelism and pipeline parallelism support for distributed inference
42 +- Streaming outputs
43 +- OpenAI-compatible API server
44 +- Support NVIDIA GPUs, AMD CPUs and GPUs, Intel CPUs, Gaudi® accelerators and GPUs, PowerPC CPUs, TPU, and AWS Trainium and Inferentia Accelerators.
45 +- Prefix caching support
46 +- Multi-lora support
47 +
48 +For more information, check out the following:
49 +
50 +- [vLLM announcing blog post](https://vllm.ai) (intro to PagedAttention)
51 +- [vLLM paper](https://arxiv.org/abs/2309.06180) (SOSP 2023)
52 +- [How continuous batching enables 23x throughput in LLM inference while reducing p50 latency](https://www.anyscale.com/blog/continuous-batching-llm-inference) by Cade Daniel et al.
53 +- {ref}`vLLM Meetups <meetups>`.
54 +
55 +## Documentation
56 +
57 +```{toctree}
58 +:caption: Getting Started
59 +:maxdepth: 1
60 +
61 +getting_started/installation
62 +getting_started/amd-installation
63 +getting_started/openvino-installation
64 +getting_started/cpu-installation
65 +getting_started/gaudi-installation
66 +getting_started/arm-installation
67 +getting_started/neuron-installation
68 +getting_started/tpu-installation
69 +getting_started/xpu-installation
70 +getting_started/quickstart
71 +getting_started/debugging
72 +getting_started/examples/examples_index
73 +```
74 +
75 +```{toctree}
76 +:caption: Serving
77 +:maxdepth: 1
78 +
79 +serving/openai_compatible_server
80 +serving/deploying_with_docker
81 +serving/deploying_with_k8s
82 +serving/deploying_with_helm
83 +serving/deploying_with_nginx
84 +serving/distributed_serving
85 +serving/metrics
86 +serving/integrations
87 +serving/tensorizer
88 +serving/runai_model_streamer
89 +```
90 +
91 +```{toctree}
92 +:caption: Models
93 +:maxdepth: 1
94 +
95 +models/supported_models
96 +models/generative_models
97 +models/pooling_models
98 +models/adding_model
99 +models/enabling_multimodal_inputs
100 +```
101 +
102 +```{toctree}
103 +:caption: Usage
104 +:maxdepth: 1
105 +
106 +usage/lora
107 +usage/multimodal_inputs
108 +usage/tool_calling
109 +usage/structured_outputs
110 +usage/spec_decode
111 +usage/compatibility_matrix
112 +usage/performance
113 +usage/faq
114 +usage/engine_args
115 +usage/env_vars
116 +usage/usage_stats
117 +usage/disagg_prefill
118 +```
119 +
120 +```{toctree}
121 +:caption: Quantization
122 +:maxdepth: 1
123 +
124 +quantization/supported_hardware
125 +quantization/auto_awq
126 +quantization/bnb
127 +quantization/gguf
128 +quantization/int8
129 +quantization/fp8
130 +quantization/fp8_e5m2_kvcache
131 +quantization/fp8_e4m3_kvcache
132 +```
133 +
134 +```{toctree}
135 +:caption: Automatic Prefix Caching
136 +:maxdepth: 1
137 +
138 +automatic_prefix_caching/apc
139 +automatic_prefix_caching/details
140 +```
141 +
142 +```{toctree}
143 +:caption: Performance
144 +:maxdepth: 1
145 +
146 +performance/benchmarks
147 +```
148 +
149 +% Community: User community resources
150 +
151 +```{toctree}
152 +:caption: Community
153 +:maxdepth: 1
154 +
155 +community/meetups
156 +community/sponsors
157 +```
158 +
159 +% API Documentation: API reference aimed at vllm library usage
160 +
161 +```{toctree}
162 +:caption: API Documentation
163 +:maxdepth: 2
164 +
165 +dev/sampling_params
166 +dev/pooling_params
167 +dev/offline_inference/offline_index
168 +dev/engine/engine_index
169 +```
170 +
171 +% Design: docs about vLLM internals
172 +
173 +```{toctree}
174 +:caption: Design
175 +:maxdepth: 2
176 +
177 +design/arch_overview
178 +design/huggingface_integration
179 +design/plugin_system
180 +design/input_processing/model_inputs_index
181 +design/kernel/paged_attention
182 +design/multimodal/multimodal_index
183 +design/multiprocessing
184 +```
185 +
186 +% For Developers: contributing to the vLLM project
187 +
188 +```{toctree}
189 +:caption: For Developers
190 +:maxdepth: 2
191 +
192 +contributing/overview
193 +contributing/profiling/profiling_index
194 +contributing/dockerfile/dockerfile
195 +```
196 +
197 +# Indices and tables
198 +
199 +- {ref}`genindex`
200 +- {ref}`modindex`
docs/source/index.rst deleted
-194
@@ -1,194 +0,0 @@
1 -Welcome to vLLM!
2 -================
3 -
4 -.. figure:: ./assets/logos/vllm-logo-text-light.png
5 - :width: 60%
6 - :align: center
7 - :alt: vLLM
8 - :class: no-scaled-link
9 -
10 -.. raw:: html
11 -
12 - <p style="text-align:center">
13 - <strong>Easy, fast, and cheap LLM serving for everyone
14 - </strong>
15 - </p>
16 -
17 - <p style="text-align:center">
18 - <script async defer src="https://buttons.github.io/buttons.js"></script>
19 - <a class="github-button" href="https://github.com/vllm-project/vllm" data-show-count="true" data-size="large" aria-label="Star">Star</a>
20 - <a class="github-button" href="https://github.com/vllm-project/vllm/subscription" data-icon="octicon-eye" data-size="large" aria-label="Watch">Watch</a>
21 - <a class="github-button" href="https://github.com/vllm-project/vllm/fork" data-icon="octicon-repo-forked" data-size="large" aria-label="Fork">Fork</a>
22 - </p>
23 -
24 -
25 -
26 -vLLM is a fast and easy-to-use library for LLM inference and serving.
27 -
28 -vLLM is fast with:
29 -
30 -* State-of-the-art serving throughput
31 -* Efficient management of attention key and value memory with **PagedAttention**
32 -* Continuous batching of incoming requests
33 -* Fast model execution with CUDA/HIP graph
34 -* Quantization: `GPTQ <https://arxiv.org/abs/2210.17323>`_, `AWQ <https://arxiv.org/abs/2306.00978>`_, INT4, INT8, and FP8
35 -* Optimized CUDA kernels, including integration with FlashAttention and FlashInfer.
36 -* Speculative decoding
37 -* Chunked prefill
38 -
39 -vLLM is flexible and easy to use with:
40 -
41 -* Seamless integration with popular HuggingFace models
42 -* High-throughput serving with various decoding algorithms, including *parallel sampling*, *beam search*, and more
43 -* Tensor parallelism and pipeline parallelism support for distributed inference
44 -* Streaming outputs
45 -* OpenAI-compatible API server
46 -* Support NVIDIA GPUs, AMD CPUs and GPUs, Intel CPUs, Gaudi® accelerators and GPUs, PowerPC CPUs, TPU, and AWS Trainium and Inferentia Accelerators.
47 -* Prefix caching support
48 -* Multi-lora support
49 -
50 -For more information, check out the following:
51 -
52 -* `vLLM announcing blog post <https://vllm.ai>`_ (intro to PagedAttention)
53 -* `vLLM paper <https://arxiv.org/abs/2309.06180>`_ (SOSP 2023)
54 -* `How continuous batching enables 23x throughput in LLM inference while reducing p50 latency <https://www.anyscale.com/blog/continuous-batching-llm-inference>`_ by Cade Daniel et al.
55 -* :ref:`vLLM Meetups <meetups>`.
56 -
57 -
58 -Documentation
59 --------------
60 -
61 -.. toctree::
62 - :maxdepth: 1
63 - :caption: Getting Started
64 -
65 - getting_started/installation
66 - getting_started/amd-installation
67 - getting_started/openvino-installation
68 - getting_started/cpu-installation
69 - getting_started/gaudi-installation
70 - getting_started/arm-installation
71 - getting_started/neuron-installation
72 - getting_started/tpu-installation
73 - getting_started/xpu-installation
74 - getting_started/quickstart
75 - getting_started/debugging
76 - getting_started/examples/examples_index
77 -
78 -.. toctree::
79 - :maxdepth: 1
80 - :caption: Serving
81 -
82 - serving/openai_compatible_server
83 - serving/deploying_with_docker
84 - serving/deploying_with_k8s
85 - serving/deploying_with_helm
86 - serving/deploying_with_nginx
87 - serving/distributed_serving
88 - serving/metrics
89 - serving/integrations
90 - serving/tensorizer
91 - serving/runai_model_streamer
92 -
93 -.. toctree::
94 - :maxdepth: 1
95 - :caption: Models
96 -
97 - models/supported_models
98 - models/generative_models
99 - models/pooling_models
100 - models/adding_model
101 - models/enabling_multimodal_inputs
102 -
103 -.. toctree::
104 - :maxdepth: 1
105 - :caption: Usage
106 -
107 - usage/lora
108 - usage/multimodal_inputs
109 - usage/tool_calling
110 - usage/structured_outputs
111 - usage/spec_decode
112 - usage/compatibility_matrix
113 - usage/performance
114 - usage/faq
115 - usage/engine_args
116 - usage/env_vars
117 - usage/usage_stats
118 - usage/disagg_prefill
119 -
120 -.. toctree::
121 - :maxdepth: 1
122 - :caption: Quantization
123 -
124 - quantization/supported_hardware
125 - quantization/auto_awq
126 - quantization/bnb
127 - quantization/gguf
128 - quantization/int8
129 - quantization/fp8
130 - quantization/fp8_e5m2_kvcache
131 - quantization/fp8_e4m3_kvcache
132 -
133 -.. toctree::
134 - :maxdepth: 1
135 - :caption: Automatic Prefix Caching
136 -
137 - automatic_prefix_caching/apc
138 - automatic_prefix_caching/details
139 -
140 -.. toctree::
141 - :maxdepth: 1
142 - :caption: Performance
143 -
144 - performance/benchmarks
145 -
146 -.. Community: User community resources
147 -
148 -.. toctree::
149 - :maxdepth: 1
150 - :caption: Community
151 -
152 - community/meetups
153 - community/sponsors
154 -
155 -.. API Documentation: API reference aimed at vllm library usage
156 -
157 -.. toctree::
158 - :maxdepth: 2
159 - :caption: API Documentation
160 -
161 - dev/sampling_params
162 - dev/pooling_params
163 - dev/offline_inference/offline_index
164 - dev/engine/engine_index
165 -
166 -.. Design: docs about vLLM internals
167 -
168 -.. toctree::
169 - :maxdepth: 2
170 - :caption: Design
171 -
172 - design/arch_overview
173 - design/huggingface_integration
174 - design/plugin_system
175 - design/input_processing/model_inputs_index
176 - design/kernel/paged_attention
177 - design/multimodal/multimodal_index
178 - design/multiprocessing
179 -
180 -.. For Developers: contributing to the vLLM project
181 -
182 -.. toctree::
183 - :maxdepth: 2
184 - :caption: For Developers
185 -
186 - contributing/overview
187 - contributing/profiling/profiling_index
188 - contributing/dockerfile/dockerfile
189 -
190 -Indices and tables
191 -==================
192 -
193 -* :ref:`genindex`
194 -* :ref:`modindex`
docs/source/models/adding_model.md new
+155
@@ -0,0 +1,155 @@
1 +(adding-a-new-model)=
2 +
3 +# Adding a New Model
4 +
5 +This document provides a high-level guide on integrating a [HuggingFace Transformers](https://github.com/huggingface/transformers) model into vLLM.
6 +
7 +```{note}
8 +The complexity of adding a new model depends heavily on the model's architecture.
9 +The process is considerably straightforward if the model shares a similar architecture with an existing model in vLLM.
10 +However, for models that include new operators (e.g., a new attention mechanism), the process can be a bit more complex.
11 +```
12 +
13 +```{note}
14 +By default, vLLM models do not support multi-modal inputs. To enable multi-modal support,
15 +please follow [this guide](#enabling-multimodal-inputs) after implementing the model here.
16 +```
17 +
18 +```{tip}
19 +If you are encountering issues while integrating your model into vLLM, feel free to open an issue on our [GitHub](https://github.com/vllm-project/vllm/issues) repository.
20 +We will be happy to help you out!
21 +```
22 +
23 +## 0. Fork the vLLM repository
24 +
25 +Start by forking our [GitHub] repository and then [build it from source](#build-from-source).
26 +This gives you the ability to modify the codebase and test your model.
27 +
28 +```{tip}
29 +If you don't want to fork the repository and modify vLLM's codebase, please refer to the "Out-of-Tree Model Integration" section below.
30 +```
31 +
32 +## 1. Bring your model code
33 +
34 +Clone the PyTorch model code from the HuggingFace Transformers repository and put it into the [vllm/model_executor/models](https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models) directory.
35 +For instance, vLLM's [OPT model](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/opt.py) was adapted from the HuggingFace's [modeling_opt.py](https://github.com/huggingface/transformers/blob/main/src/transformers/models/opt/modeling_opt.py) file.
36 +
37 +```{warning}
38 +When copying the model code, make sure to review and adhere to the code's copyright and licensing terms.
39 +```
40 +
41 +## 2. Make your code compatible with vLLM
42 +
43 +To ensure compatibility with vLLM, your model must meet the following requirements:
44 +
45 +### Initialization Code
46 +
47 +All vLLM modules within the model must include a `prefix` argument in their constructor. This `prefix` is typically the full name of the module in the model's state dictionary and is crucial for:
48 +
49 +- Runtime support: vLLM's attention operators are registered in a model's state by their full names. Each attention operator must have a unique prefix as its layer name to avoid conflicts.
50 +- Non-uniform quantization support: A quantized checkpoint can selectively quantize certain layers while keeping others in full precision. By providing the `prefix` during initialization, vLLM can match the current layer's `prefix` with the quantization configuration to determine if the layer should be initialized in quantized mode.
51 +
52 +The initialization code should look like this:
53 +
54 +```python
55 +from torch import nn
56 +from vllm.config import VllmConfig
57 +from vllm.attention import Attention
58 +
59 +class MyAttention(nn.Module):
60 + def __init__(self, vllm_config: VllmConfig, prefix: str):
61 + super().__init__()
62 + self.attn = Attention(prefix=f"{prefix}.attn")
63 +
64 +class MyDecoderLayer(nn.Module):
65 + def __init__(self, vllm_config: VllmConfig, prefix: str):
66 + super().__init__()
67 + self.self_attn = MyAttention(prefix=f"{prefix}.self_attn")
68 +
69 +class MyModel(nn.Module):
70 + def __init__(self, vllm_config: VllmConfig, prefix: str):
71 + super().__init__()
72 + self.layers = nn.ModuleList(
73 + [MyDecoderLayer(vllm_config, prefix=f"{prefix}.layers.{i}") for i in range(vllm_config.model_config.hf_config.num_hidden_layers)]
74 + )
75 +
76 +class MyModelForCausalLM(nn.Module):
77 + def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
78 + super().__init__()
79 + self.model = MyModel(vllm_config, prefix=f"{prefix}.model")
80 +```
81 +
82 +### Computation Code
83 +
84 +Rewrite the {meth}`~torch.nn.Module.forward` method of your model to remove any unnecessary code, such as training-specific code. Modify the input parameters to treat `input_ids` and `positions` as flattened tensors with a single batch size dimension, without a max-sequence length dimension.
85 +
86 +```python
87 +def forward(
88 + self,
89 + input_ids: torch.Tensor,
90 + positions: torch.Tensor,
91 + kv_caches: List[torch.Tensor],
92 + attn_metadata: AttentionMetadata,
93 +) -> torch.Tensor:
94 + ...
95 +```
96 +
97 +```{note}
98 +Currently, vLLM supports the basic multi-head attention mechanism and its variant with rotary positional embeddings.
99 +If your model employs a different attention mechanism, you will need to implement a new attention layer in vLLM.
100 +```
101 +
102 +For reference, check out the [LLAMA model](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llama.py). vLLM already supports a large number of models. It is recommended to find a model similar to yours and adapt it to your model's architecture. Check out the [vLLM models](https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models) directory for more examples.
103 +
104 +## 3. (Optional) Implement tensor parallelism and quantization support
105 +
106 +If your model is too large to fit into a single GPU, you can use tensor parallelism to manage it.
107 +To do this, substitute your model's linear and embedding layers with their tensor-parallel versions.
108 +For the embedding layer, you can simply replace {class}`torch.nn.Embedding` with {code}`VocabParallelEmbedding`. For the output LM head, you can use {code}`ParallelLMHead`.
109 +When it comes to the linear layers, we provide the following options to parallelize them:
110 +
111 +- {code}`ReplicatedLinear`: Replicates the inputs and weights across multiple GPUs. No memory saving.
112 +- {code}`RowParallelLinear`: The input tensor is partitioned along the hidden dimension. The weight matrix is partitioned along the rows (input dimension). An *all-reduce* operation is performed after the matrix multiplication to reduce the results. Typically used for the second FFN layer and the output linear transformation of the attention layer.
113 +- {code}`ColumnParallelLinear`: The input tensor is replicated. The weight matrix is partitioned along the columns (output dimension). The result is partitioned along the column dimension. Typically used for the first FFN layer and the separated QKV transformation of the attention layer in the original Transformer.
114 +- {code}`MergedColumnParallelLinear`: Column-parallel linear that merges multiple {code}`ColumnParallelLinear` operators. Typically used for the first FFN layer with weighted activation functions (e.g., SiLU). This class handles the sharded weight loading logic of multiple weight matrices.
115 +- {code}`QKVParallelLinear`: Parallel linear layer for the query, key, and value projections of the multi-head and grouped-query attention mechanisms. When number of key/value heads are less than the world size, this class replicates the key/value heads properly. This class handles the weight loading and replication of the weight matrices.
116 +
117 +Note that all the linear layers above take {code}`linear_method` as an input. vLLM will set this parameter according to different quantization schemes to support weight quantization.
118 +
119 +## 4. Implement the weight loading logic
120 +
121 +You now need to implement the {code}`load_weights` method in your {code}`*ForCausalLM` class.
122 +This method should load the weights from the HuggingFace's checkpoint file and assign them to the corresponding layers in your model. Specifically, for {code}`MergedColumnParallelLinear` and {code}`QKVParallelLinear` layers, if the original model has separated weight matrices, you need to load the different parts separately.
123 +
124 +## 5. Register your model
125 +
126 +Finally, register your {code}`*ForCausalLM` class to the {code}`_VLLM_MODELS` in [vllm/model_executor/models/registry.py](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/registry.py).
127 +
128 +## 6. Out-of-Tree Model Integration
129 +
130 +You can integrate a model without modifying the vLLM codebase. Steps 2, 3, and 4 are still required, but you can skip steps 1 and 5. Instead, write a plugin to register your model. For general introduction of the plugin system, see [plugin-system](#plugin-system).
131 +
132 +To register the model, use the following code:
133 +
134 +```python
135 +from vllm import ModelRegistry
136 +from your_code import YourModelForCausalLM
137 +ModelRegistry.register_model("YourModelForCausalLM", YourModelForCausalLM)
138 +```
139 +
140 +If your model imports modules that initialize CUDA, consider lazy-importing it to avoid errors like {code}`RuntimeError: Cannot re-initialize CUDA in forked subprocess`:
141 +
142 +```python
143 +from vllm import ModelRegistry
144 +
145 +ModelRegistry.register_model("YourModelForCausalLM", "your_code:YourModelForCausalLM")
146 +```
147 +
148 +```{important}
149 +If your model is a multimodal model, ensure the model class implements the {class}`~vllm.model_executor.models.interfaces.SupportsMultiModal` interface.
150 +Read more about that [here](#enabling-multimodal-inputs).
151 +```
152 +
153 +```{note}
154 +Although you can directly put these code snippets in your script using `vllm.LLM`, the recommended way is to place these snippets in a vLLM plugin. This ensures compatibility with various vLLM features like distributed inference and the API server.
155 +```
docs/source/models/adding_model.rst deleted
-159
@@ -1,159 +0,0 @@
1 -.. _adding_a_new_model:
2 -
3 -Adding a New Model
4 -==================
5 -
6 -This document provides a high-level guide on integrating a `HuggingFace Transformers <https://github.com/huggingface/transformers>`_ model into vLLM.
7 -
8 -.. note::
9 - The complexity of adding a new model depends heavily on the model's architecture.
10 - The process is considerably straightforward if the model shares a similar architecture with an existing model in vLLM.
11 - However, for models that include new operators (e.g., a new attention mechanism), the process can be a bit more complex.
12 -
13 -.. note::
14 - By default, vLLM models do not support multi-modal inputs. To enable multi-modal support,
15 - please follow :ref:`this guide <enabling_multimodal_inputs>` after implementing the model here.
16 -
17 -.. tip::
18 - If you are encountering issues while integrating your model into vLLM, feel free to open an issue on our `GitHub <https://github.com/vllm-project/vllm/issues>`_ repository.
19 - We will be happy to help you out!
20 -
21 -
22 -0. Fork the vLLM repository
23 ---------------------------------
24 -
25 -Start by forking our `GitHub`_ repository and then :ref:`build it from source <build_from_source>`.
26 -This gives you the ability to modify the codebase and test your model.
27 -
28 -.. tip::
29 - If you don't want to fork the repository and modify vLLM's codebase, please refer to the "Out-of-Tree Model Integration" section below.
30 -
31 -1. Bring your model code
32 -------------------------
33 -
34 -Clone the PyTorch model code from the HuggingFace Transformers repository and put it into the `vllm/model_executor/models <https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models>`_ directory.
35 -For instance, vLLM's `OPT model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/opt.py>`_ was adapted from the HuggingFace's `modeling_opt.py <https://github.com/huggingface/transformers/blob/main/src/transformers/models/opt/modeling_opt.py>`_ file.
36 -
37 -.. warning::
38 - When copying the model code, make sure to review and adhere to the code's copyright and licensing terms.
39 -
40 -
41 -2. Make your code compatible with vLLM
42 ---------------------------------------
43 -
44 -To ensure compatibility with vLLM, your model must meet the following requirements:
45 -
46 -Initialization Code
47 -^^^^^^^^^^^^^^^^^^^
48 -
49 -All vLLM modules within the model must include a ``prefix`` argument in their constructor. This ``prefix`` is typically the full name of the module in the model's state dictionary and is crucial for:
50 -
51 -* Runtime support: vLLM's attention operators are registered in a model's state by their full names. Each attention operator must have a unique prefix as its layer name to avoid conflicts.
52 -* Non-uniform quantization support: A quantized checkpoint can selectively quantize certain layers while keeping others in full precision. By providing the ``prefix`` during initialization, vLLM can match the current layer's ``prefix`` with the quantization configuration to determine if the layer should be initialized in quantized mode.
53 -
54 -The initialization code should look like this:
55 -
56 -.. code-block:: python
57 -
58 - from torch import nn
59 - from vllm.config import VllmConfig
60 - from vllm.attention import Attention
61 -
62 - class MyAttention(nn.Module):
63 - def __init__(self, vllm_config: VllmConfig, prefix: str):
64 - super().__init__()
65 - self.attn = Attention(prefix=f"{prefix}.attn")
66 -
67 - class MyDecoderLayer(nn.Module):
68 - def __init__(self, vllm_config: VllmConfig, prefix: str):
69 - super().__init__()
70 - self.self_attn = MyAttention(prefix=f"{prefix}.self_attn")
71 -
72 - class MyModel(nn.Module):
73 - def __init__(self, vllm_config: VllmConfig, prefix: str):
74 - super().__init__()
75 - self.layers = nn.ModuleList(
76 - [MyDecoderLayer(vllm_config, prefix=f"{prefix}.layers.{i}") for i in range(vllm_config.model_config.hf_config.num_hidden_layers)]
77 - )
78 -
79 - class MyModelForCausalLM(nn.Module):
80 - def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
81 - super().__init__()
82 - self.model = MyModel(vllm_config, prefix=f"{prefix}.model")
83 -
84 -Computation Code
85 -^^^^^^^^^^^^^^^^
86 -
87 -Rewrite the :meth:`~torch.nn.Module.forward` method of your model to remove any unnecessary code, such as training-specific code. Modify the input parameters to treat ``input_ids`` and ``positions`` as flattened tensors with a single batch size dimension, without a max-sequence length dimension.
88 -
89 -.. code-block:: python
90 -
91 - def forward(
92 - self,
93 - input_ids: torch.Tensor,
94 - positions: torch.Tensor,
95 - kv_caches: List[torch.Tensor],
96 - attn_metadata: AttentionMetadata,
97 - ) -> torch.Tensor:
98 - ...
99 -
100 -.. note::
101 - Currently, vLLM supports the basic multi-head attention mechanism and its variant with rotary positional embeddings.
102 - If your model employs a different attention mechanism, you will need to implement a new attention layer in vLLM.
103 -
104 -For reference, check out the `LLAMA model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llama.py>`__. vLLM already supports a large number of models. It is recommended to find a model similar to yours and adapt it to your model's architecture. Check out the `vLLM models <https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models>`__ directory for more examples.
105 -
106 -3. (Optional) Implement tensor parallelism and quantization support
107 --------------------------------------------------------------------
108 -
109 -If your model is too large to fit into a single GPU, you can use tensor parallelism to manage it.
110 -To do this, substitute your model's linear and embedding layers with their tensor-parallel versions.
111 -For the embedding layer, you can simply replace :class:`torch.nn.Embedding` with :code:`VocabParallelEmbedding`. For the output LM head, you can use :code:`ParallelLMHead`.
112 -When it comes to the linear layers, we provide the following options to parallelize them:
113 -
114 -* :code:`ReplicatedLinear`: Replicates the inputs and weights across multiple GPUs. No memory saving.
115 -* :code:`RowParallelLinear`: The input tensor is partitioned along the hidden dimension. The weight matrix is partitioned along the rows (input dimension). An *all-reduce* operation is performed after the matrix multiplication to reduce the results. Typically used for the second FFN layer and the output linear transformation of the attention layer.
116 -* :code:`ColumnParallelLinear`: The input tensor is replicated. The weight matrix is partitioned along the columns (output dimension). The result is partitioned along the column dimension. Typically used for the first FFN layer and the separated QKV transformation of the attention layer in the original Transformer.
117 -* :code:`MergedColumnParallelLinear`: Column-parallel linear that merges multiple :code:`ColumnParallelLinear` operators. Typically used for the first FFN layer with weighted activation functions (e.g., SiLU). This class handles the sharded weight loading logic of multiple weight matrices.
118 -* :code:`QKVParallelLinear`: Parallel linear layer for the query, key, and value projections of the multi-head and grouped-query attention mechanisms. When number of key/value heads are less than the world size, this class replicates the key/value heads properly. This class handles the weight loading and replication of the weight matrices.
119 -
120 -Note that all the linear layers above take :code:`linear_method` as an input. vLLM will set this parameter according to different quantization schemes to support weight quantization.
121 -
122 -4. Implement the weight loading logic
123 --------------------------------------
124 -
125 -You now need to implement the :code:`load_weights` method in your :code:`*ForCausalLM` class.
126 -This method should load the weights from the HuggingFace's checkpoint file and assign them to the corresponding layers in your model. Specifically, for :code:`MergedColumnParallelLinear` and :code:`QKVParallelLinear` layers, if the original model has separated weight matrices, you need to load the different parts separately.
127 -
128 -5. Register your model
129 -----------------------
130 -
131 -Finally, register your :code:`*ForCausalLM` class to the :code:`_VLLM_MODELS` in `vllm/model_executor/models/registry.py <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/registry.py>`_.
132 -
133 -6. Out-of-Tree Model Integration
134 ---------------------------------
135 -
136 -You can integrate a model without modifying the vLLM codebase. Steps 2, 3, and 4 are still required, but you can skip steps 1 and 5. Instead, write a plugin to register your model. For general introduction of the plugin system, see :ref:`plugin_system`.
137 -
138 -To register the model, use the following code:
139 -
140 -.. code-block:: python
141 -
142 - from vllm import ModelRegistry
143 - from your_code import YourModelForCausalLM
144 - ModelRegistry.register_model("YourModelForCausalLM", YourModelForCausalLM)
145 -
146 -If your model imports modules that initialize CUDA, consider lazy-importing it to avoid errors like :code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`:
147 -
148 -.. code-block:: python
149 -
150 - from vllm import ModelRegistry
151 -
152 - ModelRegistry.register_model("YourModelForCausalLM", "your_code:YourModelForCausalLM")
153 -
154 -.. important::
155 - If your model is a multimodal model, ensure the model class implements the :class:`~vllm.model_executor.models.interfaces.SupportsMultiModal` interface.
156 - Read more about that :ref:`here <enabling_multimodal_inputs>`.
157 -
158 -.. note::
159 - Although you can directly put these code snippets in your script using ``vllm.LLM``, the recommended way is to place these snippets in a vLLM plugin. This ensures compatibility with various vLLM features like distributed inference and the API server.
docs/source/models/enabling_multimodal_inputs.md new
+143
@@ -0,0 +1,143 @@
1 +(enabling-multimodal-inputs)=
2 +
3 +# Enabling Multimodal Inputs
4 +
5 +This document walks you through the steps to extend a vLLM model so that it accepts [multi-modal inputs](#multimodal-inputs).
6 +
7 +```{seealso}
8 +[Adding a New Model](adding-a-new-model)
9 +```
10 +
11 +## 1. Update the base vLLM model
12 +
13 +It is assumed that you have already implemented the model in vLLM according to [these steps](#adding-a-new-model).
14 +Further update the model as follows:
15 +
16 +- Implement the {class}`~vllm.model_executor.models.interfaces.SupportsMultiModal` interface.
17 +
18 + ```diff
19 + + from vllm.model_executor.models.interfaces import SupportsMultiModal
20 +
21 + - class YourModelForImage2Seq(nn.Module):
22 + + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
23 + ```
24 +
25 + ```{note}
26 + The model class does not have to be named {code}`*ForCausalLM`.
27 + Check out [the HuggingFace Transformers documentation](https://huggingface.co/docs/transformers/model_doc/auto#multimodal) for some examples.
28 + ```
29 +
30 +- If you haven't already done so, reserve a keyword parameter in {meth}`~torch.nn.Module.forward`
31 + for each input tensor that corresponds to a multi-modal input, as shown in the following example:
32 +
33 + ```diff
34 + def forward(
35 + self,
36 + input_ids: torch.Tensor,
37 + positions: torch.Tensor,
38 + kv_caches: List[torch.Tensor],
39 + attn_metadata: AttentionMetadata,
40 + + pixel_values: torch.Tensor,
41 + ) -> SamplerOutput:
42 + ```
43 +
44 +## 2. Register input mappers
45 +
46 +For each modality type that the model accepts as input, decorate the model class with {meth}`MULTIMODAL_REGISTRY.register_input_mapper <vllm.multimodal.MultiModalRegistry.register_input_mapper>`.
47 +This decorator accepts a function that maps multi-modal inputs to the keyword arguments you have previously defined in {meth}`~torch.nn.Module.forward`.
48 +
49 +```diff
50 + from vllm.model_executor.models.interfaces import SupportsMultiModal
51 ++ from vllm.multimodal import MULTIMODAL_REGISTRY
52 +
53 ++ @MULTIMODAL_REGISTRY.register_image_input_mapper()
54 + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
55 +```
56 +
57 +A default mapper is available for each modality in the core vLLM library. This input mapper will be used if you do not provide your own function.
58 +
59 +```{seealso}
60 +[Input Processing Pipeline](#input-processing-pipeline)
61 +```
62 +
63 +## 3. Register maximum number of multi-modal tokens
64 +
65 +For each modality type that the model accepts as input, calculate the maximum possible number of tokens per data item
66 +and register it via {meth}`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_max_multimodal_tokens>`.
67 +
68 +```diff
69 + from vllm.inputs import INPUT_REGISTRY
70 + from vllm.model_executor.models.interfaces import SupportsMultiModal
71 + from vllm.multimodal import MULTIMODAL_REGISTRY
72 +
73 + @MULTIMODAL_REGISTRY.register_image_input_mapper()
74 ++ @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
75 + @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
76 + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
77 +```
78 +
79 +Here are some examples:
80 +
81 +- Image inputs (static feature size): [LLaVA-1.5 Model](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava.py)
82 +- Image inputs (dynamic feature size): [LLaVA-NeXT Model](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava_next.py)
83 +
84 +```{seealso}
85 +[Input Processing Pipeline](#input-processing-pipeline)
86 +```
87 +
88 +## 4. (Optional) Register dummy data
89 +
90 +During startup, dummy data is passed to the vLLM model to allocate memory. This only consists of text input by default, which may not be applicable to multi-modal models.
91 +In such cases, you can define your own dummy data by registering a factory method via {meth}`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_dummy_data>`.
92 +
93 +```diff
94 + from vllm.inputs import INPUT_REGISTRY
95 + from vllm.model_executor.models.interfaces import SupportsMultiModal
96 + from vllm.multimodal import MULTIMODAL_REGISTRY
97 +
98 + @MULTIMODAL_REGISTRY.register_image_input_mapper()
99 + @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
100 ++ @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
101 + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
102 +```
103 +
104 +```{note}
105 +The dummy data should have the maximum possible number of multi-modal tokens, as described in the previous step.
106 +```
107 +
108 +Here are some examples:
109 +
110 +- Image inputs (static feature size): [LLaVA-1.5 Model](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava.py)
111 +- Image inputs (dynamic feature size): [LLaVA-NeXT Model](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava_next.py)
112 +
113 +```{seealso}
114 +[Input Processing Pipeline](#input-processing-pipeline)
115 +```
116 +
117 +## 5. (Optional) Register input processor
118 +
119 +Sometimes, there is a need to process inputs at the {class}`~vllm.LLMEngine` level before they are passed to the model executor.
120 +This is often due to the fact that unlike implementations in HuggingFace Transformers, the reshaping and/or expansion of multi-modal embeddings needs to take place outside model's {meth}`~torch.nn.Module.forward` call.
121 +You can register input processors via {meth}`INPUT_REGISTRY.register_input_processor <vllm.inputs.registry.InputRegistry.register_input_processor>`.
122 +
123 +```diff
124 + from vllm.inputs import INPUT_REGISTRY
125 + from vllm.model_executor.models.interfaces import SupportsMultiModal
126 + from vllm.multimodal import MULTIMODAL_REGISTRY
127 +
128 + @MULTIMODAL_REGISTRY.register_image_input_mapper()
129 + @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
130 + @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
131 ++ @INPUT_REGISTRY.register_input_processor(<your_input_processor>)
132 + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
133 +```
134 +
135 +A common use case of input processors is inserting placeholder tokens to leverage the vLLM framework for attention mask generation.
136 +Here are some examples:
137 +
138 +- Insert static number of image tokens: [LLaVA-1.5 Model](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava.py)
139 +- Insert dynamic number of image tokens: [LLaVA-NeXT Model](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava_next.py)
140 +
141 +```{seealso}
142 +[Input Processing Pipeline](#input-processing-pipeline)
143 +```
docs/source/models/enabling_multimodal_inputs.rst deleted
-147
@@ -1,147 +0,0 @@
1 -.. _enabling_multimodal_inputs:
2 -
3 -Enabling Multimodal Inputs
4 -==========================
5 -
6 -This document walks you through the steps to extend a vLLM model so that it accepts :ref:`multi-modal inputs <multimodal_inputs>`.
7 -
8 -.. seealso::
9 - :ref:`adding_a_new_model`
10 -
11 -
12 -1. Update the base vLLM model
13 ------------------------------
14 -
15 -It is assumed that you have already implemented the model in vLLM according to :ref:`these steps <adding_a_new_model>`.
16 -Further update the model as follows:
17 -
18 -- Implement the :class:`~vllm.model_executor.models.interfaces.SupportsMultiModal` interface.
19 -
20 - .. code-block:: diff
21 -
22 - + from vllm.model_executor.models.interfaces import SupportsMultiModal
23 -
24 - - class YourModelForImage2Seq(nn.Module):
25 - + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
26 -
27 - .. note::
28 - The model class does not have to be named :code:`*ForCausalLM`.
29 - Check out `the HuggingFace Transformers documentation <https://huggingface.co/docs/transformers/model_doc/auto#multimodal>`__ for some examples.
30 -
31 -- If you haven't already done so, reserve a keyword parameter in :meth:`~torch.nn.Module.forward`
32 - for each input tensor that corresponds to a multi-modal input, as shown in the following example:
33 -
34 - .. code-block:: diff
35 -
36 - def forward(
37 - self,
38 - input_ids: torch.Tensor,
39 - positions: torch.Tensor,
40 - kv_caches: List[torch.Tensor],
41 - attn_metadata: AttentionMetadata,
42 - + pixel_values: torch.Tensor,
43 - ) -> SamplerOutput:
44 -
45 -
46 -2. Register input mappers
47 --------------------------
48 -
49 -For each modality type that the model accepts as input, decorate the model class with :meth:`MULTIMODAL_REGISTRY.register_input_mapper <vllm.multimodal.MultiModalRegistry.register_input_mapper>`.
50 -This decorator accepts a function that maps multi-modal inputs to the keyword arguments you have previously defined in :meth:`~torch.nn.Module.forward`.
51 -
52 -.. code-block:: diff
53 -
54 - from vllm.model_executor.models.interfaces import SupportsMultiModal
55 - + from vllm.multimodal import MULTIMODAL_REGISTRY
56 -
57 - + @MULTIMODAL_REGISTRY.register_image_input_mapper()
58 - class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
59 -
60 -A default mapper is available for each modality in the core vLLM library. This input mapper will be used if you do not provide your own function.
61 -
62 -.. seealso::
63 - :ref:`input_processing_pipeline`
64 -
65 -
66 -3. Register maximum number of multi-modal tokens
67 -------------------------------------------------
68 -
69 -For each modality type that the model accepts as input, calculate the maximum possible number of tokens per data item
70 -and register it via :meth:`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_max_multimodal_tokens>`.
71 -
72 -.. code-block:: diff
73 -
74 - from vllm.inputs import INPUT_REGISTRY
75 - from vllm.model_executor.models.interfaces import SupportsMultiModal
76 - from vllm.multimodal import MULTIMODAL_REGISTRY
77 -
78 - @MULTIMODAL_REGISTRY.register_image_input_mapper()
79 - + @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
80 - @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
81 - class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
82 -
83 -Here are some examples:
84 -
85 -- Image inputs (static feature size): `LLaVA-1.5 Model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava.py>`__
86 -- Image inputs (dynamic feature size): `LLaVA-NeXT Model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava_next.py>`__
87 -
88 -.. seealso::
89 - :ref:`input_processing_pipeline`
90 -
91 -
92 -4. (Optional) Register dummy data
93 ----------------------------------
94 -
95 -During startup, dummy data is passed to the vLLM model to allocate memory. This only consists of text input by default, which may not be applicable to multi-modal models.
96 -In such cases, you can define your own dummy data by registering a factory method via :meth:`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_dummy_data>`.
97 -
98 -.. code-block:: diff
99 -
100 - from vllm.inputs import INPUT_REGISTRY
101 - from vllm.model_executor.models.interfaces import SupportsMultiModal
102 - from vllm.multimodal import MULTIMODAL_REGISTRY
103 -
104 - @MULTIMODAL_REGISTRY.register_image_input_mapper()
105 - @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
106 - + @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
107 - class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
108 -
109 -.. note::
110 - The dummy data should have the maximum possible number of multi-modal tokens, as described in the previous step.
111 -
112 -Here are some examples:
113 -
114 -- Image inputs (static feature size): `LLaVA-1.5 Model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava.py>`__
115 -- Image inputs (dynamic feature size): `LLaVA-NeXT Model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava_next.py>`__
116 -
117 -.. seealso::
118 - :ref:`input_processing_pipeline`
119 -
120 -
121 -5. (Optional) Register input processor
122 ---------------------------------------
123 -
124 -Sometimes, there is a need to process inputs at the :class:`~vllm.LLMEngine` level before they are passed to the model executor.
125 -This is often due to the fact that unlike implementations in HuggingFace Transformers, the reshaping and/or expansion of multi-modal embeddings needs to take place outside model's :meth:`~torch.nn.Module.forward` call.
126 -You can register input processors via :meth:`INPUT_REGISTRY.register_input_processor <vllm.inputs.registry.InputRegistry.register_input_processor>`.
127 -
128 -.. code-block:: diff
129 -
130 - from vllm.inputs import INPUT_REGISTRY
131 - from vllm.model_executor.models.interfaces import SupportsMultiModal
132 - from vllm.multimodal import MULTIMODAL_REGISTRY
133 -
134 - @MULTIMODAL_REGISTRY.register_image_input_mapper()
135 - @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
136 - @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
137 - + @INPUT_REGISTRY.register_input_processor(<your_input_processor>)
138 - class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
139 -
140 -A common use case of input processors is inserting placeholder tokens to leverage the vLLM framework for attention mask generation.
141 -Here are some examples:
142 -
143 -- Insert static number of image tokens: `LLaVA-1.5 Model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava.py>`__
144 -- Insert dynamic number of image tokens: `LLaVA-NeXT Model <https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llava_next.py>`__
145 -
146 -.. seealso::
147 - :ref:`input_processing_pipeline`
docs/source/models/generative_models.md new
+138
@@ -0,0 +1,138 @@
1 +(generative-models)=
2 +
3 +# Generative Models
4 +
5 +vLLM provides first-class support for generative models, which covers most of LLMs.
6 +
7 +In vLLM, generative models implement the {class}`~vllm.model_executor.models.VllmModelForTextGeneration` interface.
8 +Based on the final hidden states of the input, these models output log probabilities of the tokens to generate,
9 +which are then passed through {class}`~vllm.model_executor.layers.Sampler` to obtain the final text.
10 +
11 +## Offline Inference
12 +
13 +The {class}`~vllm.LLM` class provides various methods for offline inference.
14 +See [Engine Arguments](#engine-args) for a list of options when initializing the model.
15 +
16 +For generative models, the only supported {code}`task` option is {code}`"generate"`.
17 +Usually, this is automatically inferred so you don't have to specify it.
18 +
19 +### `LLM.generate`
20 +
21 +The {class}`~vllm.LLM.generate` method is available to all generative models in vLLM.
22 +It is similar to [its counterpart in HF Transformers](https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate),
23 +except that tokenization and detokenization are also performed automatically.
24 +
25 +```python
26 +llm = LLM(model="facebook/opt-125m")
27 +outputs = llm.generate("Hello, my name is")
28 +
29 +for output in outputs:
30 + prompt = output.prompt
31 + generated_text = output.outputs[0].text
32 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
33 +```
34 +
35 +You can optionally control the language generation by passing {class}`~vllm.SamplingParams`.
36 +For example, you can use greedy sampling by setting {code}`temperature=0`:
37 +
38 +```python
39 +llm = LLM(model="facebook/opt-125m")
40 +params = SamplingParams(temperature=0)
41 +outputs = llm.generate("Hello, my name is", params)
42 +
43 +for output in outputs:
44 + prompt = output.prompt
45 + generated_text = output.outputs[0].text
46 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
47 +```
48 +
49 +A code example can be found in [examples/offline_inference.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference.py).
50 +
51 +### `LLM.beam_search`
52 +
53 +The {class}`~vllm.LLM.beam_search` method implements [beam search](https://huggingface.co/docs/transformers/en/generation_strategies#beam-search-decoding) on top of {class}`~vllm.LLM.generate`.
54 +For example, to search using 5 beams and output at most 50 tokens:
55 +
56 +```python
57 +llm = LLM(model="facebook/opt-125m")
58 +params = BeamSearchParams(beam_width=5, max_tokens=50)
59 +outputs = llm.generate("Hello, my name is", params)
60 +
61 +for output in outputs:
62 + prompt = output.prompt
63 + generated_text = output.outputs[0].text
64 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
65 +```
66 +
67 +### `LLM.chat`
68 +
69 +The {class}`~vllm.LLM.chat` method implements chat functionality on top of {class}`~vllm.LLM.generate`.
70 +In particular, it accepts input similar to [OpenAI Chat Completions API](https://platform.openai.com/docs/api-reference/chat)
71 +and automatically applies the model's [chat template](https://huggingface.co/docs/transformers/en/chat_templating) to format the prompt.
72 +
73 +```{important}
74 +In general, only instruction-tuned models have a chat template.
75 +Base models may perform poorly as they are not trained to respond to the chat conversation.
76 +```
77 +
78 +```python
79 +llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
80 +conversation = [
81 + {
82 + "role": "system",
83 + "content": "You are a helpful assistant"
84 + },
85 + {
86 + "role": "user",
87 + "content": "Hello"
88 + },
89 + {
90 + "role": "assistant",
91 + "content": "Hello! How can I assist you today?"
92 + },
93 + {
94 + "role": "user",
95 + "content": "Write an essay about the importance of higher education.",
96 + },
97 +]
98 +outputs = llm.chat(conversation)
99 +
100 +for output in outputs:
101 + prompt = output.prompt
102 + generated_text = output.outputs[0].text
103 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
104 +```
105 +
106 +A code example can be found in [examples/offline_inference_chat.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_chat.py).
107 +
108 +If the model doesn't have a chat template or you want to specify another one,
109 +you can explicitly pass a chat template:
110 +
111 +```python
112 +from vllm.entrypoints.chat_utils import load_chat_template
113 +
114 +# You can find a list of existing chat templates under `examples/`
115 +custom_template = load_chat_template(chat_template="<path_to_template>")
116 +print("Loaded chat template:", custom_template)
117 +
118 +outputs = llm.chat(conversation, chat_template=custom_template)
119 +```
120 +
121 +## Online Inference
122 +
123 +Our [OpenAI Compatible Server](../serving/openai_compatible_server) can be used for online inference.
124 +Please click on the above link for more details on how to launch the server.
125 +
126 +### Completions API
127 +
128 +Our Completions API is similar to `LLM.generate` but only accepts text.
129 +It is compatible with [OpenAI Completions API](https://platform.openai.com/docs/api-reference/completions)
130 +so that you can use OpenAI client to interact with it.
131 +A code example can be found in [examples/openai_completion_client.py](https://github.com/vllm-project/vllm/blob/main/examples/openai_completion_client.py).
132 +
133 +### Chat API
134 +
135 +Our Chat API is similar to `LLM.chat`, accepting both text and [multi-modal inputs](#multimodal-inputs).
136 +It is compatible with [OpenAI Chat Completions API](https://platform.openai.com/docs/api-reference/chat)
137 +so that you can use OpenAI client to interact with it.
138 +A code example can be found in [examples/openai_chat_completion_client.py](https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client.py).
docs/source/models/generative_models.rst deleted
-146
@@ -1,146 +0,0 @@
1 -.. _generative_models:
2 -
3 -Generative Models
4 -=================
5 -
6 -vLLM provides first-class support for generative models, which covers most of LLMs.
7 -
8 -In vLLM, generative models implement the :class:`~vllm.model_executor.models.VllmModelForTextGeneration` interface.
9 -Based on the final hidden states of the input, these models output log probabilities of the tokens to generate,
10 -which are then passed through :class:`~vllm.model_executor.layers.Sampler` to obtain the final text.
11 -
12 -Offline Inference
13 ------------------
14 -
15 -The :class:`~vllm.LLM` class provides various methods for offline inference.
16 -See :ref:`Engine Arguments <engine_args>` for a list of options when initializing the model.
17 -
18 -For generative models, the only supported :code:`task` option is :code:`"generate"`.
19 -Usually, this is automatically inferred so you don't have to specify it.
20 -
21 -``LLM.generate``
22 -^^^^^^^^^^^^^^^^
23 -
24 -The :class:`~vllm.LLM.generate` method is available to all generative models in vLLM.
25 -It is similar to `its counterpart in HF Transformers <https://huggingface.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate>`__,
26 -except that tokenization and detokenization are also performed automatically.
27 -
28 -.. code-block:: python
29 -
30 - llm = LLM(model="facebook/opt-125m")
31 - outputs = llm.generate("Hello, my name is")
32 -
33 - for output in outputs:
34 - prompt = output.prompt
35 - generated_text = output.outputs[0].text
36 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
37 -
38 -You can optionally control the language generation by passing :class:`~vllm.SamplingParams`.
39 -For example, you can use greedy sampling by setting :code:`temperature=0`:
40 -
41 -.. code-block:: python
42 -
43 - llm = LLM(model="facebook/opt-125m")
44 - params = SamplingParams(temperature=0)
45 - outputs = llm.generate("Hello, my name is", params)
46 -
47 - for output in outputs:
48 - prompt = output.prompt
49 - generated_text = output.outputs[0].text
50 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
51 -
52 -A code example can be found in `examples/offline_inference.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference.py>`_.
53 -
54 -``LLM.beam_search``
55 -^^^^^^^^^^^^^^^^^^^
56 -
57 -The :class:`~vllm.LLM.beam_search` method implements `beam search <https://huggingface.co/docs/transformers/en/generation_strategies#beam-search-decoding>`__ on top of :class:`~vllm.LLM.generate`.
58 -For example, to search using 5 beams and output at most 50 tokens:
59 -
60 -.. code-block:: python
61 -
62 - llm = LLM(model="facebook/opt-125m")
63 - params = BeamSearchParams(beam_width=5, max_tokens=50)
64 - outputs = llm.generate("Hello, my name is", params)
65 -
66 - for output in outputs:
67 - prompt = output.prompt
68 - generated_text = output.outputs[0].text
69 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
70 -
71 -``LLM.chat``
72 -^^^^^^^^^^^^
73 -
74 -The :class:`~vllm.LLM.chat` method implements chat functionality on top of :class:`~vllm.LLM.generate`.
75 -In particular, it accepts input similar to `OpenAI Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`__
76 -and automatically applies the model's `chat template <https://huggingface.co/docs/transformers/en/chat_templating>`__ to format the prompt.
77 -
78 -.. important::
79 -
80 - In general, only instruction-tuned models have a chat template.
81 - Base models may perform poorly as they are not trained to respond to the chat conversation.
82 -
83 -.. code-block:: python
84 -
85 - llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
86 - conversation = [
87 - {
88 - "role": "system",
89 - "content": "You are a helpful assistant"
90 - },
91 - {
92 - "role": "user",
93 - "content": "Hello"
94 - },
95 - {
96 - "role": "assistant",
97 - "content": "Hello! How can I assist you today?"
98 - },
99 - {
100 - "role": "user",
101 - "content": "Write an essay about the importance of higher education.",
102 - },
103 - ]
104 - outputs = llm.chat(conversation)
105 -
106 - for output in outputs:
107 - prompt = output.prompt
108 - generated_text = output.outputs[0].text
109 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
110 -
111 -A code example can be found in `examples/offline_inference_chat.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_chat.py>`_.
112 -
113 -If the model doesn't have a chat template or you want to specify another one,
114 -you can explicitly pass a chat template:
115 -
116 -.. code-block:: python
117 -
118 - from vllm.entrypoints.chat_utils import load_chat_template
119 -
120 - # You can find a list of existing chat templates under `examples/`
121 - custom_template = load_chat_template(chat_template="<path_to_template>")
122 - print("Loaded chat template:", custom_template)
123 -
124 - outputs = llm.chat(conversation, chat_template=custom_template)
125 -
126 -Online Inference
127 -----------------
128 -
129 -Our `OpenAI Compatible Server <../serving/openai_compatible_server>`__ can be used for online inference.
130 -Please click on the above link for more details on how to launch the server.
131 -
132 -Completions API
133 -^^^^^^^^^^^^^^^
134 -
135 -Our Completions API is similar to ``LLM.generate`` but only accepts text.
136 -It is compatible with `OpenAI Completions API <https://platform.openai.com/docs/api-reference/completions>`__
137 -so that you can use OpenAI client to interact with it.
138 -A code example can be found in `examples/openai_completion_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_completion_client.py>`_.
139 -
140 -Chat API
141 -^^^^^^^^
142 -
143 -Our Chat API is similar to ``LLM.chat``, accepting both text and :ref:`multi-modal inputs <multimodal_inputs>`.
144 -It is compatible with `OpenAI Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`__
145 -so that you can use OpenAI client to interact with it.
146 -A code example can be found in `examples/openai_chat_completion_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client.py>`_.
docs/source/models/pooling_models.md new
+127
@@ -0,0 +1,127 @@
1 +(pooling-models)=
2 +
3 +# Pooling Models
4 +
5 +vLLM also supports pooling models, including embedding, reranking and reward models.
6 +
7 +In vLLM, pooling models implement the {class}`~vllm.model_executor.models.VllmModelForPooling` interface.
8 +These models use a {class}`~vllm.model_executor.layers.Pooler` to extract the final hidden states of the input
9 +before returning them.
10 +
11 +```{note}
12 +We currently support pooling models primarily as a matter of convenience.
13 +As shown in the [Compatibility Matrix](#compatibility-matrix), most vLLM features are not applicable to
14 +pooling models as they only work on the generation or decode stage, so performance may not improve as much.
15 +```
16 +
17 +## Offline Inference
18 +
19 +The {class}`~vllm.LLM` class provides various methods for offline inference.
20 +See [Engine Arguments](#engine-args) for a list of options when initializing the model.
21 +
22 +For pooling models, we support the following {code}`task` options:
23 +
24 +- Embedding ({code}`"embed"` / {code}`"embedding"`)
25 +- Classification ({code}`"classify"`)
26 +- Sentence Pair Scoring ({code}`"score"`)
27 +- Reward Modeling ({code}`"reward"`)
28 +
29 +The selected task determines the default {class}`~vllm.model_executor.layers.Pooler` that is used:
30 +
31 +- Embedding: Extract only the hidden states corresponding to the last token, and apply normalization.
32 +- Classification: Extract only the hidden states corresponding to the last token, and apply softmax.
33 +- Sentence Pair Scoring: Extract only the hidden states corresponding to the last token, and apply softmax.
34 +- Reward Modeling: Extract all of the hidden states and return them directly.
35 +
36 +When loading [Sentence Transformers](https://huggingface.co/sentence-transformers) models,
37 +we attempt to override the default pooler based on its Sentence Transformers configuration file ({code}`modules.json`).
38 +
39 +You can customize the model's pooling method via the {code}`override_pooler_config` option,
40 +which takes priority over both the model's and Sentence Transformers's defaults.
41 +
42 +### `LLM.encode`
43 +
44 +The {class}`~vllm.LLM.encode` method is available to all pooling models in vLLM.
45 +It returns the extracted hidden states directly, which is useful for reward models.
46 +
47 +```python
48 +llm = LLM(model="Qwen/Qwen2.5-Math-RM-72B", task="reward")
49 +(output,) = llm.encode("Hello, my name is")
50 +
51 +data = output.outputs.data
52 +print(f"Data: {data!r}")
53 +```
54 +
55 +### `LLM.embed`
56 +
57 +The {class}`~vllm.LLM.embed` method outputs an embedding vector for each prompt.
58 +It is primarily designed for embedding models.
59 +
60 +```python
61 +llm = LLM(model="intfloat/e5-mistral-7b-instruct", task="embed")
62 +(output,) = llm.embed("Hello, my name is")
63 +
64 +embeds = output.outputs.embedding
65 +print(f"Embeddings: {embeds!r} (size={len(embeds)})")
66 +```
67 +
68 +A code example can be found in [examples/offline_inference_embedding.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_embedding.py).
69 +
70 +### `LLM.classify`
71 +
72 +The {class}`~vllm.LLM.classify` method outputs a probability vector for each prompt.
73 +It is primarily designed for classification models.
74 +
75 +```python
76 +llm = LLM(model="jason9693/Qwen2.5-1.5B-apeach", task="classify")
77 +(output,) = llm.classify("Hello, my name is")
78 +
79 +probs = output.outputs.probs
80 +print(f"Class Probabilities: {probs!r} (size={len(probs)})")
81 +```
82 +
83 +A code example can be found in [examples/offline_inference_classification.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_classification.py).
84 +
85 +### `LLM.score`
86 +
87 +The {class}`~vllm.LLM.score` method outputs similarity scores between sentence pairs.
88 +It is primarily designed for [cross-encoder models](https://www.sbert.net/examples/applications/cross-encoder/README.html).
89 +These types of models serve as rerankers between candidate query-document pairs in RAG systems.
90 +
91 +```{note}
92 +vLLM can only perform the model inference component (e.g. embedding, reranking) of RAG.
93 +To handle RAG at a higher level, you should use integration frameworks such as [LangChain](https://github.com/langchain-ai/langchain).
94 +```
95 +
96 +```python
97 +llm = LLM(model="BAAI/bge-reranker-v2-m3", task="score")
98 +(output,) = llm.score("What is the capital of France?",
99 + "The capital of Brazil is Brasilia.")
100 +
101 +score = output.outputs.score
102 +print(f"Score: {score}")
103 +```
104 +
105 +A code example can be found in [examples/offline_inference_scoring.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_scoring.py).
106 +
107 +## Online Inference
108 +
109 +Our [OpenAI Compatible Server](../serving/openai_compatible_server.md) can be used for online inference.
110 +Please click on the above link for more details on how to launch the server.
111 +
112 +### Embeddings API
113 +
114 +Our Embeddings API is similar to `LLM.embed`, accepting both text and [multi-modal inputs](#multimodal-inputs).
115 +
116 +The text-only API is compatible with [OpenAI Embeddings API](https://platform.openai.com/docs/api-reference/embeddings)
117 +so that you can use OpenAI client to interact with it.
118 +A code example can be found in [examples/openai_embedding_client.py](https://github.com/vllm-project/vllm/blob/main/examples/openai_embedding_client.py).
119 +
120 +The multi-modal API is an extension of the [OpenAI Embeddings API](https://platform.openai.com/docs/api-reference/embeddings)
121 +that incorporates [OpenAI Chat Completions API](https://platform.openai.com/docs/api-reference/chat),
122 +so it is not part of the OpenAI standard. Please see [](#multimodal-inputs) for more details on how to use it.
123 +
124 +### Score API
125 +
126 +Our Score API is similar to `LLM.score`.
127 +Please see [this page](#score-api) for more details on how to use it.
docs/source/models/pooling_models.rst deleted
-136
@@ -1,136 +0,0 @@
1 -.. _pooling_models:
2 -
3 -Pooling Models
4 -==============
5 -
6 -vLLM also supports pooling models, including embedding, reranking and reward models.
7 -
8 -In vLLM, pooling models implement the :class:`~vllm.model_executor.models.VllmModelForPooling` interface.
9 -These models use a :class:`~vllm.model_executor.layers.Pooler` to extract the final hidden states of the input
10 -before returning them.
11 -
12 -.. note::
13 -
14 - We currently support pooling models primarily as a matter of convenience.
15 - As shown in the :ref:`Compatibility Matrix <compatibility_matrix>`, most vLLM features are not applicable to
16 - pooling models as they only work on the generation or decode stage, so performance may not improve as much.
17 -
18 -Offline Inference
19 ------------------
20 -
21 -The :class:`~vllm.LLM` class provides various methods for offline inference.
22 -See :ref:`Engine Arguments <engine_args>` for a list of options when initializing the model.
23 -
24 -For pooling models, we support the following :code:`task` options:
25 -
26 -- Embedding (:code:`"embed"` / :code:`"embedding"`)
27 -- Classification (:code:`"classify"`)
28 -- Sentence Pair Scoring (:code:`"score"`)
29 -- Reward Modeling (:code:`"reward"`)
30 -
31 -The selected task determines the default :class:`~vllm.model_executor.layers.Pooler` that is used:
32 -
33 -- Embedding: Extract only the hidden states corresponding to the last token, and apply normalization.
34 -- Classification: Extract only the hidden states corresponding to the last token, and apply softmax.
35 -- Sentence Pair Scoring: Extract only the hidden states corresponding to the last token, and apply softmax.
36 -- Reward Modeling: Extract all of the hidden states and return them directly.
37 -
38 -When loading `Sentence Transformers <https://huggingface.co/sentence-transformers>`__ models,
39 -we attempt to override the default pooler based on its Sentence Transformers configuration file (:code:`modules.json`).
40 -
41 -You can customize the model's pooling method via the :code:`override_pooler_config` option,
42 -which takes priority over both the model's and Sentence Transformers's defaults.
43 -
44 -``LLM.encode``
45 -^^^^^^^^^^^^^^
46 -
47 -The :class:`~vllm.LLM.encode` method is available to all pooling models in vLLM.
48 -It returns the extracted hidden states directly, which is useful for reward models.
49 -
50 -.. code-block:: python
51 -
52 - llm = LLM(model="Qwen/Qwen2.5-Math-RM-72B", task="reward")
53 - (output,) = llm.encode("Hello, my name is")
54 -
55 - data = output.outputs.data
56 - print(f"Data: {data!r}")
57 -
58 -``LLM.embed``
59 -^^^^^^^^^^^^^
60 -
61 -The :class:`~vllm.LLM.embed` method outputs an embedding vector for each prompt.
62 -It is primarily designed for embedding models.
63 -
64 -.. code-block:: python
65 -
66 - llm = LLM(model="intfloat/e5-mistral-7b-instruct", task="embed")
67 - (output,) = llm.embed("Hello, my name is")
68 -
69 - embeds = output.outputs.embedding
70 - print(f"Embeddings: {embeds!r} (size={len(embeds)})")
71 -
72 -A code example can be found in `examples/offline_inference_embedding.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_embedding.py>`_.
73 -
74 -``LLM.classify``
75 -^^^^^^^^^^^^^^^^
76 -
77 -The :class:`~vllm.LLM.classify` method outputs a probability vector for each prompt.
78 -It is primarily designed for classification models.
79 -
80 -.. code-block:: python
81 -
82 - llm = LLM(model="jason9693/Qwen2.5-1.5B-apeach", task="classify")
83 - (output,) = llm.classify("Hello, my name is")
84 -
85 - probs = output.outputs.probs
86 - print(f"Class Probabilities: {probs!r} (size={len(probs)})")
87 -
88 -A code example can be found in `examples/offline_inference_classification.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_classification.py>`_.
89 -
90 -``LLM.score``
91 -^^^^^^^^^^^^^
92 -
93 -The :class:`~vllm.LLM.score` method outputs similarity scores between sentence pairs.
94 -It is primarily designed for `cross-encoder models <https://www.sbert.net/examples/applications/cross-encoder/README.html>`__.
95 -These types of models serve as rerankers between candidate query-document pairs in RAG systems.
96 -
97 -.. note::
98 -
99 - vLLM can only perform the model inference component (e.g. embedding, reranking) of RAG.
100 - To handle RAG at a higher level, you should use integration frameworks such as `LangChain <https://github.com/langchain-ai/langchain>`_.
101 -
102 -.. code-block:: python
103 -
104 - llm = LLM(model="BAAI/bge-reranker-v2-m3", task="score")
105 - (output,) = llm.score("What is the capital of France?",
106 - "The capital of Brazil is Brasilia.")
107 -
108 - score = output.outputs.score
109 - print(f"Score: {score}")
110 -
111 -A code example can be found in `examples/offline_inference_scoring.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_scoring.py>`_.
112 -
113 -Online Inference
114 -----------------
115 -
116 -Our `OpenAI Compatible Server <../serving/openai_compatible_server>`__ can be used for online inference.
117 -Please click on the above link for more details on how to launch the server.
118 -
119 -Embeddings API
120 -^^^^^^^^^^^^^^
121 -
122 -Our Embeddings API is similar to ``LLM.embed``, accepting both text and :ref:`multi-modal inputs <multimodal_inputs>`.
123 -
124 -The text-only API is compatible with `OpenAI Embeddings API <https://platform.openai.com/docs/api-reference/embeddings>`__
125 -so that you can use OpenAI client to interact with it.
126 -A code example can be found in `examples/openai_embedding_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_embedding_client.py>`_.
127 -
128 -The multi-modal API is an extension of the `OpenAI Embeddings API <https://platform.openai.com/docs/api-reference/embeddings>`__
129 -that incorporates `OpenAI Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`__,
130 -so it is not part of the OpenAI standard. Please see :ref:`this page <multimodal_inputs>` for more details on how to use it.
131 -
132 -Score API
133 -^^^^^^^^^
134 -
135 -Our Score API is similar to ``LLM.score``.
136 -Please see `this page <../serving/openai_compatible_server.html#score-api-for-cross-encoder-models>`__ for more details on how to use it.
docs/source/models/supported_models.md renamed
+205 -196
@@ -1,84 +1,78 @@
1 -.. _supported_models:
1 +(supported-models)=
2
3 -Supported Models
4 -================
3 +# Supported Models
4
5 vLLM supports generative and pooling models across various tasks.
7 -If a model supports more than one task, you can set the task via the :code:`--task` argument.
6 +If a model supports more than one task, you can set the task via the {code}`--task` argument.
7
8 For each task, we list the model architectures that have been implemented in vLLM.
9 Alongside each architecture, we include some popular models that use it.
10
12 -Loading a Model
13 -^^^^^^^^^^^^^^^
11 +## Loading a Model
12
15 -HuggingFace Hub
16 -+++++++++++++++
13 +### HuggingFace Hub
14
18 -By default, vLLM loads models from `HuggingFace (HF) Hub <https://huggingface.co/models>`_.
15 +By default, vLLM loads models from [HuggingFace (HF) Hub](https://huggingface.co/models).
16
20 -To determine whether a given model is supported, you can check the :code:`config.json` file inside the HF repository.
21 -If the :code:`"architectures"` field contains a model architecture listed below, then it should be supported in theory.
17 +To determine whether a given model is supported, you can check the {code}`config.json` file inside the HF repository.
18 +If the {code}`"architectures"` field contains a model architecture listed below, then it should be supported in theory.
19
23 -.. tip::
24 - The easiest way to check if your model is really supported at runtime is to run the program below:
20 +````{tip}
21 +The easiest way to check if your model is really supported at runtime is to run the program below:
22
26 - .. code-block:: python
23 +```python
24 +from vllm import LLM
25
28 - from vllm import LLM
26 +# For generative models (task=generate) only
27 +llm = LLM(model=..., task="generate") # Name or path of your model
28 +output = llm.generate("Hello, my name is")
29 +print(output)
30
30 - # For generative models (task=generate) only
31 - llm = LLM(model=..., task="generate") # Name or path of your model
32 - output = llm.generate("Hello, my name is")
33 - print(output)
31 +# For pooling models (task={embed,classify,reward}) only
32 +llm = LLM(model=..., task="embed") # Name or path of your model
33 +output = llm.encode("Hello, my name is")
34 +print(output)
35 +```
36
35 - # For pooling models (task={embed,classify,reward}) only
36 - llm = LLM(model=..., task="embed") # Name or path of your model
37 - output = llm.encode("Hello, my name is")
38 - print(output)
37 +If vLLM successfully returns text (for generative models) or hidden states (for pooling models), it indicates that your model is supported.
38 +````
39
40 - If vLLM successfully returns text (for generative models) or hidden states (for pooling models), it indicates that your model is supported.
40 +Otherwise, please refer to [Adding a New Model](#adding-a-new-model) and [Enabling Multimodal Inputs](#enabling-multimodal-inputs) for instructions on how to implement your model in vLLM.
41 +Alternatively, you can [open an issue on GitHub](https://github.com/vllm-project/vllm/issues/new/choose) to request vLLM support.
42
42 -Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` and :ref:`Enabling Multimodal Inputs <enabling_multimodal_inputs>`
43 -for instructions on how to implement your model in vLLM.
44 -Alternatively, you can `open an issue on GitHub <https://github.com/vllm-project/vllm/issues/new/choose>`_ to request vLLM support.
43 +### ModelScope
44
46 -ModelScope
47 -++++++++++
45 +To use models from [ModelScope](https://www.modelscope.cn) instead of HuggingFace Hub, set an environment variable:
46
49 -To use models from `ModelScope <https://www.modelscope.cn>`_ instead of HuggingFace Hub, set an environment variable:
47 +```shell
48 +$ export VLLM_USE_MODELSCOPE=True
49 +```
50
51 -.. code-block:: shell
51 +And use with {code}`trust_remote_code=True`.
52
53 - $ export VLLM_USE_MODELSCOPE=True
53 +```python
54 +from vllm import LLM
55
55 -And use with :code:`trust_remote_code=True`.
56 +llm = LLM(model=..., revision=..., task=..., trust_remote_code=True)
57
57 -.. code-block:: python
58 +# For generative models (task=generate) only
59 +output = llm.generate("Hello, my name is")
60 +print(output)
61
59 - from vllm import LLM
62 +# For pooling models (task={embed,classify,reward}) only
63 +output = llm.encode("Hello, my name is")
64 +print(output)
65 +```
66
61 - llm = LLM(model=..., revision=..., task=..., trust_remote_code=True)
67 +## List of Text-only Language Models
68
63 - # For generative models (task=generate) only
64 - output = llm.generate("Hello, my name is")
65 - print(output)
69 +### Generative Models
70
67 - # For pooling models (task={embed,classify,reward}) only
68 - output = llm.encode("Hello, my name is")
69 - print(output)
71 +See [this page](#generative-models) for more information on how to use generative models.
72
71 -List of Text-only Language Models
72 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
73 -
74 -Generative Models
75 -+++++++++++++++++
76 -
77 -See :ref:`this page <generative_models>` for more information on how to use generative models.
78 -
79 -Text Generation (``--task generate``)
80 --------------------------------------
73 +#### Text Generation (`--task generate`)
74
75 +```{eval-rst}
76 .. list-table::
77 :widths: 25 25 50 5 5
78 :header-rows: 1
@@ -86,8 +80,8 @@ Text Generation (``--task generate``)
80 * - Architecture
81 - Models
82 - Example HF Models
89 - - :ref:`LoRA <lora>`
90 - - :ref:`PP <distributed_serving>`
83 + - :ref:`LoRA <lora-adapter>`
84 + - :ref:`PP <distributed-serving>`
85 * - :code:`AquilaForCausalLM`
86 - Aquila, Aquila2
87 - :code:`BAAI/Aquila-7B`, :code:`BAAI/AquilaChat-7B`, etc.
@@ -111,8 +105,8 @@ Text Generation (``--task generate``)
105 * - :code:`BartForConditionalGeneration`
106 - BART
107 - :code:`facebook/bart-base`, :code:`facebook/bart-large-cnn`, etc.
114 - -
115 - -
108 + -
109 + -
110 * - :code:`ChatGLMModel`
111 - ChatGLM
112 - :code:`THUDM/chatglm2-6b`, :code:`THUDM/chatglm3-6b`, etc.
@@ -136,12 +130,12 @@ Text Generation (``--task generate``)
130 * - :code:`DeepseekForCausalLM`
131 - DeepSeek
132 - :code:`deepseek-ai/deepseek-llm-67b-base`, :code:`deepseek-ai/deepseek-llm-7b-chat` etc.
139 - -
133 + -
134 - ✅︎
135 * - :code:`DeepseekV2ForCausalLM`
136 - DeepSeek-V2
137 - :code:`deepseek-ai/DeepSeek-V2`, :code:`deepseek-ai/DeepSeek-V2-Chat` etc.
144 - -
138 + -
139 - ✅︎
140 * - :code:`ExaoneForCausalLM`
141 - EXAONE-3
@@ -316,7 +310,7 @@ Text Generation (``--task generate``)
310 * - :code:`PersimmonForCausalLM`
311 - Persimmon
312 - :code:`adept/persimmon-8b-base`, :code:`adept/persimmon-8b-chat`, etc.
319 - -
313 + -
314 - ✅︎
315 * - :code:`QWenLMHeadModel`
316 - Qwen
@@ -358,29 +352,32 @@ Text Generation (``--task generate``)
352 - :code:`xverse/XVERSE-7B-Chat`, :code:`xverse/XVERSE-13B-Chat`, :code:`xverse/XVERSE-65B-Chat`, etc.
353 - ✅︎
354 - ✅︎
355 +```
356
362 -.. note::
363 - Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.
357 +```{note}
358 +Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.
359 +```
360
365 -Pooling Models
366 -++++++++++++++
361 +### Pooling Models
362
368 -See :ref:`this page <pooling_models>` for more information on how to use pooling models.
363 +See [this page](pooling-models) for more information on how to use pooling models.
364
370 -.. important::
371 - Since some model architectures support both generative and pooling tasks,
372 - you should explicitly specify the task type to ensure that the model is used in pooling mode instead of generative mode.
365 +```{important}
366 +Since some model architectures support both generative and pooling tasks,
367 +you should explicitly specify the task type to ensure that the model is used in pooling mode instead of generative mode.
368 +```
369
374 -Text Embedding (``--task embed``)
375 ----------------------------------
370 +#### Text Embedding (`--task embed`)
371
377 -Any text generation model can be converted into an embedding model by passing :code:`--task embed`.
372 +Any text generation model can be converted into an embedding model by passing {code}`--task embed`.
373
379 -.. note::
380 - To get the best results, you should use pooling models that are specifically trained as such.
374 +```{note}
375 +To get the best results, you should use pooling models that are specifically trained as such.
376 +```
377
378 The following table lists those that are tested in vLLM.
379
380 +```{eval-rst}
381 .. list-table::
382 :widths: 25 25 50 5 5
383 :header-rows: 1
@@ -388,17 +385,17 @@ The following table lists those that are tested in vLLM.
385 * - Architecture
386 - Models
387 - Example HF Models
391 - - :ref:`LoRA <lora>`
392 - - :ref:`PP <distributed_serving>`
388 + - :ref:`LoRA <lora-adapter>`
389 + - :ref:`PP <distributed-serving>`
390 * - :code:`BertModel`
391 - BERT-based
392 - :code:`BAAI/bge-base-en-v1.5`, etc.
396 - -
397 - -
393 + -
394 + -
395 * - :code:`Gemma2Model`
396 - Gemma2-based
397 - :code:`BAAI/bge-multilingual-gemma2`, etc.
401 - -
398 + -
399 - ✅︎
400 * - :code:`GritLM`
401 - GritLM
@@ -418,28 +415,31 @@ The following table lists those that are tested in vLLM.
415 * - :code:`RobertaModel`, :code:`RobertaForMaskedLM`
416 - RoBERTa-based
417 - :code:`sentence-transformers/all-roberta-large-v1`, :code:`sentence-transformers/all-roberta-large-v1`, etc.
421 - -
422 - -
418 + -
419 + -
420 * - :code:`XLMRobertaModel`
421 - XLM-RoBERTa-based
422 - :code:`intfloat/multilingual-e5-large`, etc.
426 - -
427 - -
423 + -
424 + -
425 +```
426
429 -.. note::
430 - :code:`ssmits/Qwen2-7B-Instruct-embed-base` has an improperly defined Sentence Transformers config.
431 - You should manually set mean pooling by passing :code:`--override-pooler-config '{"pooling_type": "MEAN"}'`.
427 +```{note}
428 +{code}`ssmits/Qwen2-7B-Instruct-embed-base` has an improperly defined Sentence Transformers config.
429 +You should manually set mean pooling by passing {code}`--override-pooler-config '{"pooling_type": "MEAN"}'`.
430 +```
431
433 -.. note::
434 - Unlike base Qwen2, :code:`Alibaba-NLP/gte-Qwen2-7B-instruct` uses bi-directional attention.
435 - You can set :code:`--hf-overrides '{"is_causal": false}'` to change the attention mask accordingly.
432 +```{note}
433 +Unlike base Qwen2, {code}`Alibaba-NLP/gte-Qwen2-7B-instruct` uses bi-directional attention.
434 +You can set {code}`--hf-overrides '{"is_causal": false}'` to change the attention mask accordingly.
435
437 - On the other hand, its 1.5B variant (:code:`Alibaba-NLP/gte-Qwen2-1.5B-instruct`) uses causal attention
438 - despite being described otherwise on its model card.
436 +On the other hand, its 1.5B variant ({code}`Alibaba-NLP/gte-Qwen2-1.5B-instruct`) uses causal attention
437 +despite being described otherwise on its model card.
438 +```
439
440 -Reward Modeling (``--task reward``)
441 ------------------------------------
440 +#### Reward Modeling (`--task reward`)
441
442 +```{eval-rst}
443 .. list-table::
444 :widths: 25 25 50 5 5
445 :header-rows: 1
@@ -447,8 +447,8 @@ Reward Modeling (``--task reward``)
447 * - Architecture
448 - Models
449 - Example HF Models
450 - - :ref:`LoRA <lora>`
451 - - :ref:`PP <distributed_serving>`
450 + - :ref:`LoRA <lora-adapter>`
451 + - :ref:`PP <distributed-serving>`
452 * - :code:`LlamaForCausalLM`
453 - Llama-based
454 - :code:`peiyi9979/math-shepherd-mistral-7b-prm`, etc.
@@ -459,14 +459,16 @@ Reward Modeling (``--task reward``)
459 - :code:`Qwen/Qwen2.5-Math-RM-72B`, etc.
460 - ✅︎
461 - ✅︎
462 +```
463
463 -.. important::
464 - For process-supervised reward models such as :code:`peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
465 - e.g.: :code:`--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
464 +```{important}
465 +For process-supervised reward models such as {code}`peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
466 +e.g.: {code}`--override-pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
467 +```
468
467 -Classification (``--task classify``)
468 -------------------------------------
469 +#### Classification (`--task classify`)
470
471 +```{eval-rst}
472 .. list-table::
473 :widths: 25 25 50 5 5
474 :header-rows: 1
@@ -474,8 +476,8 @@ Classification (``--task classify``)
476 * - Architecture
477 - Models
478 - Example HF Models
477 - - :ref:`LoRA <lora>`
478 - - :ref:`PP <distributed_serving>`
479 + - :ref:`LoRA <lora-adapter>`
480 + - :ref:`PP <distributed-serving>`
481 * - :code:`JambaForSequenceClassification`
482 - Jamba
483 - :code:`ai21labs/Jamba-tiny-reward-dev`, etc.
@@ -486,10 +488,11 @@ Classification (``--task classify``)
488 - :code:`jason9693/Qwen2.5-1.5B-apeach`, etc.
489 - ✅︎
490 - ✅︎
491 +```
492
490 -Sentence Pair Scoring (``--task score``)
491 -----------------------------------------
493 +#### Sentence Pair Scoring (`--task score`)
494
495 +```{eval-rst}
496 .. list-table::
497 :widths: 25 25 50 5 5
498 :header-rows: 1
@@ -497,54 +500,53 @@ Sentence Pair Scoring (``--task score``)
500 * - Architecture
501 - Models
502 - Example HF Models
500 - - :ref:`LoRA <lora>`
501 - - :ref:`PP <distributed_serving>`
503 + - :ref:`LoRA <lora-adapter>`
504 + - :ref:`PP <distributed-serving>`
505 * - :code:`BertForSequenceClassification`
506 - BERT-based
507 - :code:`cross-encoder/ms-marco-MiniLM-L-6-v2`, etc.
505 - -
506 - -
508 + -
509 + -
510 * - :code:`RobertaForSequenceClassification`
511 - RoBERTa-based
512 - :code:`cross-encoder/quora-roberta-base`, etc.
510 - -
511 - -
513 + -
514 + -
515 * - :code:`XLMRobertaForSequenceClassification`
516 - XLM-RoBERTa-based
517 - :code:`BAAI/bge-reranker-v2-m3`, etc.
515 - -
516 - -
518 + -
519 + -
520 +```
521
518 -.. _supported_mm_models:
522 +(supported-mm-models)=
523
520 -List of Multimodal Language Models
521 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
524 +## List of Multimodal Language Models
525
526 The following modalities are supported depending on the model:
527
525 -- **T**\ ext
526 -- **I**\ mage
527 -- **V**\ ideo
528 -- **A**\ udio
528 +- **T**ext
529 +- **I**mage
530 +- **V**ideo
531 +- **A**udio
532
530 -Any combination of modalities joined by :code:`+` are supported.
533 +Any combination of modalities joined by {code}`+` are supported.
534
532 -- e.g.: :code:`T + I` means that the model supports text-only, image-only, and text-with-image inputs.
535 +- e.g.: {code}`T + I` means that the model supports text-only, image-only, and text-with-image inputs.
536
534 -On the other hand, modalities separated by :code:`/` are mutually exclusive.
537 +On the other hand, modalities separated by {code}`/` are mutually exclusive.
538
536 -- e.g.: :code:`T / I` means that the model supports text-only and image-only inputs, but not text-with-image inputs.
539 +- e.g.: {code}`T / I` means that the model supports text-only and image-only inputs, but not text-with-image inputs.
540
538 -See :ref:`this page <multimodal_inputs>` on how to pass multi-modal inputs to the model.
541 +See [this page](#multimodal-inputs) on how to pass multi-modal inputs to the model.
542
540 -Generative Models
541 -+++++++++++++++++
543 +### Generative Models
544
543 -See :ref:`this page <generative_models>` for more information on how to use generative models.
545 +See [this page](#generative-models) for more information on how to use generative models.
546
545 -Text Generation (``--task generate``)
546 --------------------------------------
547 +#### Text Generation (`--task generate`)
548
549 +```{eval-rst}
550 .. list-table::
551 :widths: 25 25 15 20 5 5 5
552 :header-rows: 1
@@ -553,63 +555,63 @@ Text Generation (``--task generate``)
555 - Models
556 - Inputs
557 - Example HF Models
556 - - :ref:`LoRA <lora>`
557 - - :ref:`PP <distributed_serving>`
558 + - :ref:`LoRA <lora-adapter>`
559 + - :ref:`PP <distributed-serving>`
560 - V1
561 * - :code:`AriaForConditionalGeneration`
562 - Aria
563 - T + I
564 - :code:`rhymes-ai/Aria`
563 - -
565 + -
566 - ✅︎
565 - -
567 + -
568 * - :code:`Blip2ForConditionalGeneration`
569 - BLIP-2
570 - T + I\ :sup:`E`
571 - :code:`Salesforce/blip2-opt-2.7b`, :code:`Salesforce/blip2-opt-6.7b`, etc.
572 -
573 - ✅︎
572 - -
574 + -
575 * - :code:`ChameleonForConditionalGeneration`
576 - Chameleon
577 - T + I
578 - :code:`facebook/chameleon-7b` etc.
577 - -
579 + -
580 - ✅︎
579 - -
581 + -
582 * - :code:`FuyuForCausalLM`
583 - Fuyu
584 - T + I
585 - :code:`adept/fuyu-8b` etc.
584 - -
586 + -
587 - ✅︎
586 - -
588 + -
589 * - :code:`ChatGLMModel`
590 - GLM-4V
591 - T + I
592 - :code:`THUDM/glm-4v-9b` etc.
593 - ✅︎
594 - ✅︎
593 - -
595 + -
596 * - :code:`H2OVLChatModel`
597 - H2OVL
598 - T + I\ :sup:`E+`
599 - :code:`h2oai/h2ovl-mississippi-800m`, :code:`h2oai/h2ovl-mississippi-2b`, etc.
598 - -
600 + -
601 - ✅︎
600 - -
602 + -
603 * - :code:`Idefics3ForConditionalGeneration`
604 - Idefics3
605 - T + I
606 - :code:`HuggingFaceM4/Idefics3-8B-Llama3` etc.
607 - ✅︎
608 -
607 - -
609 + -
610 * - :code:`InternVLChatModel`
611 - InternVL 2.5, Mono-InternVL, InternVL 2.0
612 - T + I\ :sup:`E+`
613 - :code:`OpenGVLab/InternVL2_5-4B`, :code:`OpenGVLab/Mono-InternVL-2B`, :code:`OpenGVLab/InternVL2-4B`, etc.
612 - -
614 + -
615 - ✅︎
616 - ✅︎
617 * - :code:`LlavaForConditionalGeneration`
@@ -625,28 +627,28 @@ Text Generation (``--task generate``)
627 - :code:`llava-hf/llava-v1.6-mistral-7b-hf`, :code:`llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
628 -
629 - ✅︎
628 - -
630 + -
631 * - :code:`LlavaNextVideoForConditionalGeneration`
632 - LLaVA-NeXT-Video
633 - T + V
634 - :code:`llava-hf/LLaVA-NeXT-Video-7B-hf`, etc.
635 -
636 - ✅︎
635 - -
637 + -
638 * - :code:`LlavaOnevisionForConditionalGeneration`
639 - LLaVA-Onevision
640 - T + I\ :sup:`+` + V\ :sup:`+`
641 - :code:`llava-hf/llava-onevision-qwen2-7b-ov-hf`, :code:`llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc.
642 -
643 - ✅︎
642 - -
644 + -
645 * - :code:`MiniCPMV`
646 - MiniCPM-V
647 - T + I\ :sup:`E+`
648 - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
649 - ✅︎
650 - ✅︎
649 - -
651 + -
652 * - :code:`MllamaForConditionalGeneration`
653 - Llama 3.2
654 - T + I\ :sup:`+`
@@ -665,7 +667,7 @@ Text Generation (``--task generate``)
667 - NVLM-D 1.0
668 - T + I\ :sup:`E+`
669 - :code:`nvidia/NVLM-D-72B`, etc.
668 - -
670 + -
671 - ✅︎
672 - ✅︎
673 * - :code:`PaliGemmaForConditionalGeneration`
@@ -674,7 +676,7 @@ Text Generation (``--task generate``)
676 - :code:`google/paligemma-3b-pt-224`, :code:`google/paligemma-3b-mix-224`, :code:`google/paligemma2-3b-ft-docci-448`, etc.
677 -
678 - ✅︎
677 - -
679 + -
680 * - :code:`Phi3VForCausalLM`
681 - Phi-3-Vision, Phi-3.5-Vision
682 - T + I\ :sup:`E+`
@@ -702,70 +704,79 @@ Text Generation (``--task generate``)
704 - :code:`Qwen/Qwen2-Audio-7B-Instruct`
705 -
706 - ✅︎
705 - -
707 + -
708 * - :code:`Qwen2VLForConditionalGeneration`
709 - Qwen2-VL
710 - T + I\ :sup:`E+` + V\ :sup:`E+`
711 - :code:`Qwen/Qwen2-VL-2B-Instruct`, :code:`Qwen/Qwen2-VL-7B-Instruct`, :code:`Qwen/Qwen2-VL-72B-Instruct`, etc.
712 - ✅︎
713 - ✅︎
712 - -
714 + -
715 * - :code:`UltravoxModel`
716 - Ultravox
717 - T + A\ :sup:`E+`
718 - :code:`fixie-ai/ultravox-v0_3`
719 -
720 - ✅︎
719 - -
720 -
721 -| :sup:`E` Pre-computed embeddings can be inputted for this modality.
722 -| :sup:`+` Multiple items can be inputted per text prompt for this modality.
721 + -
722 +```
723
724 -.. important::
725 - To enable multiple multi-modal items per text prompt, you have to set :code:`limit_mm_per_prompt` (offline inference)
726 - or :code:`--limit-mm-per-prompt` (online inference). For example, to enable passing up to 4 images per text prompt:
724 +```{eval-rst}
725 +:sup:`E` Pre-computed embeddings can be inputted for this modality.
726
728 - .. code-block:: python
727 +:sup:`+` Multiple items can be inputted per text prompt for this modality.
728 +```
729
730 - llm = LLM(
731 - model="Qwen/Qwen2-VL-7B-Instruct",
732 - limit_mm_per_prompt={"image": 4},
733 - )
730 +````{important}
731 +To enable multiple multi-modal items per text prompt, you have to set {code}`limit_mm_per_prompt` (offline inference)
732 +or {code}`--limit-mm-per-prompt` (online inference). For example, to enable passing up to 4 images per text prompt:
733
735 - .. code-block:: bash
734 +```python
735 +llm = LLM(
736 + model="Qwen/Qwen2-VL-7B-Instruct",
737 + limit_mm_per_prompt={"image": 4},
738 +)
739 +```
740
737 - vllm serve Qwen/Qwen2-VL-7B-Instruct --limit-mm-per-prompt image=4
741 +```bash
742 +vllm serve Qwen/Qwen2-VL-7B-Instruct --limit-mm-per-prompt image=4
743 +```
744 +````
745
739 -.. note::
740 - vLLM currently only supports adding LoRA to the language backbone of multimodal models.
746 +```{note}
747 +vLLM currently only supports adding LoRA to the language backbone of multimodal models.
748 +```
749
742 -.. note::
743 - To use :code:`TIGER-Lab/Mantis-8B-siglip-llama3`, you have to install their GitHub repo (:code:`pip install git+https://github.com/TIGER-AI-Lab/Mantis.git`)
744 - and pass :code:`--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
750 +```{note}
751 +To use {code}`TIGER-Lab/Mantis-8B-siglip-llama3`, you have to install their GitHub repo ({code}`pip install git+https://github.com/TIGER-AI-Lab/Mantis.git`)
752 +and pass {code}`--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
753 +```
754
746 -.. note::
747 - The official :code:`openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
748 - For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
755 +```{note}
756 +The official {code}`openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork ({code}`HwwwH/MiniCPM-V-2`) for now.
757 +For more details, please see: <https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630>
758 +```
759
750 -Pooling Models
751 -++++++++++++++
760 +### Pooling Models
761
753 -See :ref:`this page <pooling_models>` for more information on how to use pooling models.
762 +See [this page](pooling-models) for more information on how to use pooling models.
763
755 -.. important::
756 - Since some model architectures support both generative and pooling tasks,
757 - you should explicitly specify the task type to ensure that the model is used in pooling mode instead of generative mode.
764 +```{important}
765 +Since some model architectures support both generative and pooling tasks,
766 +you should explicitly specify the task type to ensure that the model is used in pooling mode instead of generative mode.
767 +```
768
759 -Text Embedding (``--task embed``)
760 ----------------------------------
769 +#### Text Embedding (`--task embed`)
770
762 -Any text generation model can be converted into an embedding model by passing :code:`--task embed`.
771 +Any text generation model can be converted into an embedding model by passing {code}`--task embed`.
772
764 -.. note::
765 - To get the best results, you should use pooling models that are specifically trained as such.
773 +```{note}
774 +To get the best results, you should use pooling models that are specifically trained as such.
775 +```
776
777 The following table lists those that are tested in vLLM.
778
779 +```{eval-rst}
780 .. list-table::
781 :widths: 25 25 15 25 5 5
782 :header-rows: 1
@@ -774,13 +785,13 @@ The following table lists those that are tested in vLLM.
785 - Models
786 - Inputs
787 - Example HF Models
777 - - :ref:`LoRA <lora>`
778 - - :ref:`PP <distributed_serving>`
788 + - :ref:`LoRA <lora-adapter>`
789 + - :ref:`PP <distributed-serving>`
790 * - :code:`LlavaNextForConditionalGeneration`
791 - LLaVA-NeXT-based
792 - T / I
793 - :code:`royokong/e5-v`
783 - -
794 + -
795 - ✅︎
796 * - :code:`Phi3VForCausalLM`
797 - Phi-3-Vision-based
@@ -792,27 +803,25 @@ The following table lists those that are tested in vLLM.
803 - Qwen2-VL-based
804 - T + I
805 - :code:`MrLight/dse-qwen2-2b-mrl-v1`
795 - -
806 + -
807 - ✅︎
808 +```
809
798 -----
810 +______________________________________________________________________
811
800 -Model Support Policy
801 -=====================
812 +# Model Support Policy
813
814 At vLLM, we are committed to facilitating the integration and support of third-party models within our ecosystem. Our approach is designed to balance the need for robustness and the practical limitations of supporting a wide range of models. Here’s how we manage third-party model support:
815
816 1. **Community-Driven Support**: We encourage community contributions for adding new models. When a user requests support for a new model, we welcome pull requests (PRs) from the community. These contributions are evaluated primarily on the sensibility of the output they generate, rather than strict consistency with existing implementations such as those in transformers. **Call for contribution:** PRs coming directly from model vendors are greatly appreciated!
806 -
817 2. **Best-Effort Consistency**: While we aim to maintain a level of consistency between the models implemented in vLLM and other frameworks like transformers, complete alignment is not always feasible. Factors like acceleration techniques and the use of low-precision computations can introduce discrepancies. Our commitment is to ensure that the implemented models are functional and produce sensible results.
818
809 -.. tip::
810 - When comparing the output of :code:`model.generate` from HuggingFace Transformers with the output of :code:`llm.generate` from vLLM, note that the former reads the model's generation config file (i.e., `generation_config.json <https://github.com/huggingface/transformers/blob/19dabe96362803fb0a9ae7073d03533966598b17/src/transformers/generation/utils.py#L1945>`__) and applies the default parameters for generation, while the latter only uses the parameters passed to the function. Ensure all sampling parameters are identical when comparing outputs.
819 +```{tip}
820 +When comparing the output of {code}`model.generate` from HuggingFace Transformers with the output of {code}`llm.generate` from vLLM, note that the former reads the model's generation config file (i.e., [generation_config.json](https://github.com/huggingface/transformers/blob/19dabe96362803fb0a9ae7073d03533966598b17/src/transformers/generation/utils.py#L1945)) and applies the default parameters for generation, while the latter only uses the parameters passed to the function. Ensure all sampling parameters are identical when comparing outputs.
821 +```
822
823 3. **Issue Resolution and Model Updates**: Users are encouraged to report any bugs or issues they encounter with third-party models. Proposed fixes should be submitted via PRs, with a clear explanation of the problem and the rationale behind the proposed solution. If a fix for one model impacts another, we rely on the community to highlight and address these cross-model dependencies. Note: for bugfix PRs, it is good etiquette to inform the original author to seek their feedback.
813 -
824 4. **Monitoring and Updates**: Users interested in specific models should monitor the commit history for those models (e.g., by tracking changes in the main/vllm/model_executor/models directory). This proactive approach helps users stay informed about updates and changes that may affect the models they use.
815 -
825 5. **Selective Focus**: Our resources are primarily directed towards models with significant user interest and impact. Models that are less frequently used may receive less attention, and we rely on the community to play a more active role in their upkeep and improvement.
826
827 Through this approach, vLLM fosters a collaborative environment where both the core development team and the broader community contribute to the robustness and diversity of the third-party models supported in our ecosystem.
@@ -821,7 +830,7 @@ Note that, as an inference engine, vLLM does not introduce new models. Therefore
830
831 We have the following levels of testing for models:
832
824 -1. **Strict Consistency**: We compare the output of the model with the output of the model in the HuggingFace Transformers library under greedy decoding. This is the most stringent test. Please refer to `models tests <https://github.com/vllm-project/vllm/blob/main/tests/models>`_ for the models that have passed this test.
833 +1. **Strict Consistency**: We compare the output of the model with the output of the model in the HuggingFace Transformers library under greedy decoding. This is the most stringent test. Please refer to [models tests](https://github.com/vllm-project/vllm/blob/main/tests/models) for the models that have passed this test.
834 2. **Output Sensibility**: We check if the output of the model is sensible and coherent, by measuring the perplexity of the output and checking for any obvious errors. This is a less stringent test.
826 -3. **Runtime Functionality**: We check if the model can be loaded and run without errors. This is the least stringent test. Please refer to `functionality tests <https://github.com/vllm-project/vllm/tree/main/tests>`_ and `examples <https://github.com/vllm-project/vllm/tree/main/examples>`_ for the models that have passed this test.
835 +3. **Runtime Functionality**: We check if the model can be loaded and run without errors. This is the least stringent test. Please refer to [functionality tests](https://github.com/vllm-project/vllm/tree/main/tests) and [examples](https://github.com/vllm-project/vllm/tree/main/examples) for the models that have passed this test.
836 4. **Community Feedback**: We rely on the community to provide feedback on the models. If a model is broken or not working as expected, we encourage users to raise issues to report it or open pull requests to fix it. The rest of the models fall under this category.
docs/source/performance/benchmarks.md new
+28
@@ -0,0 +1,28 @@
1 +(benchmarks)=
2 +
3 +# Benchmark Suites
4 +
5 +vLLM contains two sets of benchmarks:
6 +
7 +- [Performance benchmarks](#performance-benchmarks)
8 +- [Nightly benchmarks](#nightly-benchmarks)
9 +
10 +(performance-benchmarks)=
11 +
12 +## Performance Benchmarks
13 +
14 +The performance benchmarks are used for development to confirm whether new changes improve performance under various workloads. They are triggered on every commit with both the `perf-benchmarks` and `ready` labels, and when a PR is merged into vLLM.
15 +
16 +The latest performance results are hosted on the public [vLLM Performance Dashboard](https://perf.vllm.ai).
17 +
18 +More information on the performance benchmarks and their parameters can be found [here](https://github.com/vllm-project/vllm/blob/main/.buildkite/nightly-benchmarks/performance-benchmarks-descriptions.md).
19 +
20 +(nightly-benchmarks)=
21 +
22 +## Nightly Benchmarks
23 +
24 +These compare vLLM's performance against alternatives (`tgi`, `trt-llm`, and `lmdeploy`) when there are major updates of vLLM (e.g., bumping up to a new version). They are primarily intended for consumers to evaluate when to choose vLLM over other options and are triggered on every commit with both the `perf-benchmarks` and `nightly-benchmarks` labels.
25 +
26 +The latest nightly benchmark results are shared in major release blog posts such as [vLLM v0.6.0](https://blog.vllm.ai/2024/09/05/perf-update.html).
27 +
28 +More information on the nightly benchmarks and their parameters can be found [here](https://github.com/vllm-project/vllm/blob/main/.buildkite/nightly-benchmarks/nightly-descriptions.md).
docs/source/performance/benchmarks.rst deleted
-33
@@ -1,33 +0,0 @@
1 -.. _benchmarks:
2 -
3 -================
4 -Benchmark Suites
5 -================
6 -
7 -vLLM contains two sets of benchmarks:
8 -
9 -+ :ref:`Performance benchmarks <performance_benchmarks>`
10 -+ :ref:`Nightly benchmarks <nightly_benchmarks>`
11 -
12 -
13 -.. _performance_benchmarks:
14 -
15 -Performance Benchmarks
16 -----------------------
17 -
18 -The performance benchmarks are used for development to confirm whether new changes improve performance under various workloads. They are triggered on every commit with both the ``perf-benchmarks`` and ``ready`` labels, and when a PR is merged into vLLM.
19 -
20 -The latest performance results are hosted on the public `vLLM Performance Dashboard <https://perf.vllm.ai>`_.
21 -
22 -More information on the performance benchmarks and their parameters can be found `here <https://github.com/vllm-project/vllm/blob/main/.buildkite/nightly-benchmarks/performance-benchmarks-descriptions.md>`__.
23 -
24 -.. _nightly_benchmarks:
25 -
26 -Nightly Benchmarks
27 -------------------
28 -
29 -These compare vLLM's performance against alternatives (``tgi``, ``trt-llm``, and ``lmdeploy``) when there are major updates of vLLM (e.g., bumping up to a new version). They are primarily intended for consumers to evaluate when to choose vLLM over other options and are triggered on every commit with both the ``perf-benchmarks`` and ``nightly-benchmarks`` labels.
30 -
31 -The latest nightly benchmark results are shared in major release blog posts such as `vLLM v0.6.0 <https://blog.vllm.ai/2024/09/05/perf-update.html>`_.
32 -
33 -More information on the nightly benchmarks and their parameters can be found `here <https://github.com/vllm-project/vllm/blob/main/.buildkite/nightly-benchmarks/nightly-descriptions.md>`__.
\ No newline at end of file
docs/source/quantization/auto_awq.md new
+78
@@ -0,0 +1,78 @@
1 +(auto-awq)=
2 +
3 +# AutoAWQ
4 +
5 +```{warning}
6 +Please note that AWQ support in vLLM is under-optimized at the moment. We would recommend using the unquantized version of the model for better
7 +accuracy and higher throughput. Currently, you can use AWQ as a way to reduce memory footprint. As of now, it is more suitable for low latency
8 +inference with small number of concurrent requests. vLLM's AWQ implementation have lower throughput than unquantized version.
9 +```
10 +
11 +To create a new 4-bit quantized model, you can leverage [AutoAWQ](https://github.com/casper-hansen/AutoAWQ).
12 +Quantizing reduces the model's precision from FP16 to INT4 which effectively reduces the file size by ~70%.
13 +The main benefits are lower latency and memory usage.
14 +
15 +You can quantize your own models by installing AutoAWQ or picking one of the [400+ models on Huggingface](https://huggingface.co/models?sort=trending&search=awq).
16 +
17 +```console
18 +$ pip install autoawq
19 +```
20 +
21 +After installing AutoAWQ, you are ready to quantize a model. Here is an example of how to quantize `mistralai/Mistral-7B-Instruct-v0.2`:
22 +
23 +```python
24 +from awq import AutoAWQForCausalLM
25 +from transformers import AutoTokenizer
26 +
27 +model_path = 'mistralai/Mistral-7B-Instruct-v0.2'
28 +quant_path = 'mistral-instruct-v0.2-awq'
29 +quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" }
30 +
31 +# Load model
32 +model = AutoAWQForCausalLM.from_pretrained(
33 + model_path, **{"low_cpu_mem_usage": True, "use_cache": False}
34 +)
35 +tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
36 +
37 +# Quantize
38 +model.quantize(tokenizer, quant_config=quant_config)
39 +
40 +# Save quantized model
41 +model.save_quantized(quant_path)
42 +tokenizer.save_pretrained(quant_path)
43 +
44 +print(f'Model is quantized and saved at "{quant_path}"')
45 +```
46 +
47 +To run an AWQ model with vLLM, you can use [TheBloke/Llama-2-7b-Chat-AWQ](https://huggingface.co/TheBloke/Llama-2-7b-Chat-AWQ) with the following command:
48 +
49 +```console
50 +$ python examples/llm_engine_example.py --model TheBloke/Llama-2-7b-Chat-AWQ --quantization awq
51 +```
52 +
53 +AWQ models are also supported directly through the LLM entrypoint:
54 +
55 +```python
56 +from vllm import LLM, SamplingParams
57 +
58 +# Sample prompts.
59 +prompts = [
60 + "Hello, my name is",
61 + "The president of the United States is",
62 + "The capital of France is",
63 + "The future of AI is",
64 +]
65 +# Create a sampling params object.
66 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
67 +
68 +# Create an LLM.
69 +llm = LLM(model="TheBloke/Llama-2-7b-Chat-AWQ", quantization="AWQ")
70 +# Generate texts from the prompts. The output is a list of RequestOutput objects
71 +# that contain the prompt, generated text, and other information.
72 +outputs = llm.generate(prompts, sampling_params)
73 +# Print the outputs.
74 +for output in outputs:
75 + prompt = output.prompt
76 + generated_text = output.outputs[0].text
77 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
78 +```
docs/source/quantization/auto_awq.rst deleted
-79
@@ -1,79 +0,0 @@
1 -.. _auto_awq:
2 -
3 -AutoAWQ
4 -==================
5 -
6 -.. warning::
7 -
8 - Please note that AWQ support in vLLM is under-optimized at the moment. We would recommend using the unquantized version of the model for better
9 - accuracy and higher throughput. Currently, you can use AWQ as a way to reduce memory footprint. As of now, it is more suitable for low latency
10 - inference with small number of concurrent requests. vLLM's AWQ implementation have lower throughput than unquantized version.
11 -
12 -To create a new 4-bit quantized model, you can leverage `AutoAWQ <https://github.com/casper-hansen/AutoAWQ>`_.
13 -Quantizing reduces the model's precision from FP16 to INT4 which effectively reduces the file size by ~70%.
14 -The main benefits are lower latency and memory usage.
15 -
16 -You can quantize your own models by installing AutoAWQ or picking one of the `400+ models on Huggingface <https://huggingface.co/models?sort=trending&search=awq>`_.
17 -
18 -.. code-block:: console
19 -
20 - $ pip install autoawq
21 -
22 -After installing AutoAWQ, you are ready to quantize a model. Here is an example of how to quantize `mistralai/Mistral-7B-Instruct-v0.2`:
23 -
24 -.. code-block:: python
25 -
26 - from awq import AutoAWQForCausalLM
27 - from transformers import AutoTokenizer
28 -
29 - model_path = 'mistralai/Mistral-7B-Instruct-v0.2'
30 - quant_path = 'mistral-instruct-v0.2-awq'
31 - quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" }
32 -
33 - # Load model
34 - model = AutoAWQForCausalLM.from_pretrained(
35 - model_path, **{"low_cpu_mem_usage": True, "use_cache": False}
36 - )
37 - tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
38 -
39 - # Quantize
40 - model.quantize(tokenizer, quant_config=quant_config)
41 -
42 - # Save quantized model
43 - model.save_quantized(quant_path)
44 - tokenizer.save_pretrained(quant_path)
45 -
46 - print(f'Model is quantized and saved at "{quant_path}"')
47 -
48 -To run an AWQ model with vLLM, you can use `TheBloke/Llama-2-7b-Chat-AWQ <https://huggingface.co/TheBloke/Llama-2-7b-Chat-AWQ>`_ with the following command:
49 -
50 -.. code-block:: console
51 -
52 - $ python examples/llm_engine_example.py --model TheBloke/Llama-2-7b-Chat-AWQ --quantization awq
53 -
54 -AWQ models are also supported directly through the LLM entrypoint:
55 -
56 -.. code-block:: python
57 -
58 - from vllm import LLM, SamplingParams
59 -
60 - # Sample prompts.
61 - prompts = [
62 - "Hello, my name is",
63 - "The president of the United States is",
64 - "The capital of France is",
65 - "The future of AI is",
66 - ]
67 - # Create a sampling params object.
68 - sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
69 -
70 - # Create an LLM.
71 - llm = LLM(model="TheBloke/Llama-2-7b-Chat-AWQ", quantization="AWQ")
72 - # Generate texts from the prompts. The output is a list of RequestOutput objects
73 - # that contain the prompt, generated text, and other information.
74 - outputs = llm.generate(prompts, sampling_params)
75 - # Print the outputs.
76 - for output in outputs:
77 - prompt = output.prompt
78 - generated_text = output.outputs[0].text
79 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
docs/source/quantization/bnb.md new
+39
@@ -0,0 +1,39 @@
1 +(bits-and-bytes)=
2 +
3 +# BitsAndBytes
4 +
5 +vLLM now supports [BitsAndBytes](https://github.com/TimDettmers/bitsandbytes) for more efficient model inference.
6 +BitsAndBytes quantizes models to reduce memory usage and enhance performance without significantly sacrificing accuracy.
7 +Compared to other quantization methods, BitsAndBytes eliminates the need for calibrating the quantized model with input data.
8 +
9 +Below are the steps to utilize BitsAndBytes with vLLM.
10 +
11 +```console
12 +$ pip install bitsandbytes>=0.45.0
13 +```
14 +
15 +vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint.
16 +
17 +You can find bitsandbytes quantized models on <https://huggingface.co/models?other=bitsandbytes>.
18 +And usually, these repositories have a config.json file that includes a quantization_config section.
19 +
20 +## Read quantized checkpoint.
21 +
22 +```python
23 +from vllm import LLM
24 +import torch
25 +# unsloth/tinyllama-bnb-4bit is a pre-quantized checkpoint.
26 +model_id = "unsloth/tinyllama-bnb-4bit"
27 +llm = LLM(model=model_id, dtype=torch.bfloat16, trust_remote_code=True, \
28 +quantization="bitsandbytes", load_format="bitsandbytes")
29 +```
30 +
31 +## Inflight quantization: load as 4bit quantization
32 +
33 +```python
34 +from vllm import LLM
35 +import torch
36 +model_id = "huggyllama/llama-7b"
37 +llm = LLM(model=model_id, dtype=torch.bfloat16, trust_remote_code=True, \
38 +quantization="bitsandbytes", load_format="bitsandbytes")
39 +```
docs/source/quantization/bnb.rst deleted
-43
@@ -1,43 +0,0 @@
1 -.. _bits_and_bytes:
2 -
3 -BitsAndBytes
4 -==================
5 -
6 -vLLM now supports `BitsAndBytes <https://github.com/TimDettmers/bitsandbytes>`_ for more efficient model inference.
7 -BitsAndBytes quantizes models to reduce memory usage and enhance performance without significantly sacrificing accuracy.
8 -Compared to other quantization methods, BitsAndBytes eliminates the need for calibrating the quantized model with input data.
9 -
10 -Below are the steps to utilize BitsAndBytes with vLLM.
11 -
12 -.. code-block:: console
13 -
14 - $ pip install bitsandbytes>=0.45.0
15 -
16 -vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint.
17 -
18 -You can find bitsandbytes quantized models on https://huggingface.co/models?other=bitsandbytes.
19 -And usually, these repositories have a config.json file that includes a quantization_config section.
20 -
21 -Read quantized checkpoint.
22 ---------------------------
23 -
24 -.. code-block:: python
25 -
26 - from vllm import LLM
27 - import torch
28 - # unsloth/tinyllama-bnb-4bit is a pre-quantized checkpoint.
29 - model_id = "unsloth/tinyllama-bnb-4bit"
30 - llm = LLM(model=model_id, dtype=torch.bfloat16, trust_remote_code=True, \
31 - quantization="bitsandbytes", load_format="bitsandbytes")
32 -
33 -Inflight quantization: load as 4bit quantization
34 -------------------------------------------------
35 -
36 -.. code-block:: python
37 -
38 - from vllm import LLM
39 - import torch
40 - model_id = "huggyllama/llama-7b"
41 - llm = LLM(model=model_id, dtype=torch.bfloat16, trust_remote_code=True, \
42 - quantization="bitsandbytes", load_format="bitsandbytes")
43 -
docs/source/quantization/fp8.md new
+192
@@ -0,0 +1,192 @@
1 +(fp8)=
2 +
3 +# FP8 W8A8
4 +
5 +vLLM supports FP8 (8-bit floating point) weight and activation quantization using hardware acceleration on GPUs such as Nvidia H100 and AMD MI300x.
6 +Currently, only Hopper and Ada Lovelace GPUs are officially supported for W8A8.
7 +Ampere GPUs are supported for W8A16 (weight-only FP8) utilizing Marlin kernels.
8 +Quantization of models with FP8 allows for a 2x reduction in model memory requirements and up to a 1.6x improvement in throughput with minimal impact on accuracy.
9 +
10 +Please visit the HF collection of [quantized FP8 checkpoints of popular LLMs ready to use with vLLM](https://huggingface.co/collections/neuralmagic/fp8-llms-for-vllm-666742ed2b78b7ac8df13127).
11 +
12 +The FP8 types typically supported in hardware have two distinct representations, each useful in different scenarios:
13 +
14 +- **E4M3**: Consists of 1 sign bit, 4 exponent bits, and 3 bits of mantissa. It can store values up to +/-448 and `nan`.
15 +- **E5M2**: Consists of 1 sign bit, 5 exponent bits, and 2 bits of mantissa. It can store values up to +/-57344, +/- `inf`, and `nan`. The tradeoff for the increased dynamic range is lower precision of the stored values.
16 +
17 +```{note}
18 +FP8 computation is supported on NVIDIA GPUs with compute capability > 8.9 (Ada Lovelace, Hopper).
19 +FP8 models will run on compute capability > 8.0 (Ampere) as weight-only W8A16, utilizing FP8 Marlin.
20 +```
21 +
22 +## Quick Start with Online Dynamic Quantization
23 +
24 +Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8"` in the command line or setting `quantization="fp8"` in the LLM constructor.
25 +
26 +In this mode, all Linear modules (except for the final `lm_head`) have their weights quantized down to FP8_E4M3 precision with a per-tensor scale. Activations have their minimum and maximum values calculated during each forward pass to provide a dynamic per-tensor scale for high accuracy. As a result, latency improvements are limited in this mode.
27 +
28 +```python
29 +from vllm import LLM
30 +model = LLM("facebook/opt-125m", quantization="fp8")
31 +# INFO 06-10 17:55:42 model_runner.py:157] Loading model weights took 0.1550 GB
32 +result = model.generate("Hello, my name is")
33 +```
34 +
35 +```{warning}
36 +Currently, we load the model at original precision before quantizing down to 8-bits, so you need enough memory to load the whole model.
37 +```
38 +
39 +## Installation
40 +
41 +To produce performant FP8 quantized models with vLLM, you'll need to install the [llm-compressor](https://github.com/vllm-project/llm-compressor/) library:
42 +
43 +```console
44 +$ pip install llmcompressor
45 +```
46 +
47 +## Quantization Process
48 +
49 +The quantization process involves three main steps:
50 +
51 +1. Loading the model
52 +2. Applying quantization
53 +3. Evaluating accuracy in vLLM
54 +
55 +### 1. Loading the Model
56 +
57 +Use `SparseAutoModelForCausalLM`, which wraps `AutoModelForCausalLM`, for saving and loading quantized models:
58 +
59 +```python
60 +from llmcompressor.transformers import SparseAutoModelForCausalLM
61 +from transformers import AutoTokenizer
62 +
63 +MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"
64 +
65 +model = SparseAutoModelForCausalLM.from_pretrained(
66 + MODEL_ID, device_map="auto", torch_dtype="auto")
67 +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
68 +```
69 +
70 +### 2. Applying Quantization
71 +
72 +For FP8 quantization, we can recover accuracy with simple RTN quantization. We recommend targeting all `Linear` layers using the `FP8_DYNAMIC` scheme, which uses:
73 +
74 +- Static, per-channel quantization on the weights
75 +- Dynamic, per-token quantization on the activations
76 +
77 +Since simple RTN does not require data for weight quantization and the activations are quantized dynamically, we do not need any calibration data for this quantization flow.
78 +
79 +```python
80 +from llmcompressor.transformers import oneshot
81 +from llmcompressor.modifiers.quantization import QuantizationModifier
82 +
83 +# Configure the simple PTQ quantization
84 +recipe = QuantizationModifier(
85 + targets="Linear", scheme="FP8_DYNAMIC", ignore=["lm_head"])
86 +
87 +# Apply the quantization algorithm.
88 +oneshot(model=model, recipe=recipe)
89 +
90 +# Save the model.
91 +SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-Dynamic"
92 +model.save_pretrained(SAVE_DIR)
93 +tokenizer.save_pretrained(SAVE_DIR)
94 +```
95 +
96 +### 3. Evaluating Accuracy
97 +
98 +Install `vllm` and `lm-evaluation-harness`:
99 +
100 +```console
101 +$ pip install vllm lm-eval==0.4.4
102 +```
103 +
104 +Load and run the model in `vllm`:
105 +
106 +```python
107 +from vllm import LLM
108 +model = LLM("./Meta-Llama-3-8B-Instruct-FP8-Dynamic")
109 +model.generate("Hello my name is")
110 +```
111 +
112 +Evaluate accuracy with `lm_eval` (for example on 250 samples of `gsm8k`):
113 +
114 +```{note}
115 +Quantized models can be sensitive to the presence of the `bos` token. `lm_eval` does not add a `bos` token by default, so make sure to include the `add_bos_token=True` argument when running your evaluations.
116 +```
117 +
118 +```console
119 +$ MODEL=$PWD/Meta-Llama-3-8B-Instruct-FP8-Dynamic
120 +$ lm_eval \
121 + --model vllm \
122 + --model_args pretrained=$MODEL,add_bos_token=True \
123 + --tasks gsm8k --num_fewshot 5 --batch_size auto --limit 250
124 +```
125 +
126 +Here's an example of the resulting scores:
127 +
128 +```text
129 +|Tasks|Version| Filter |n-shot| Metric | |Value| |Stderr|
130 +|-----|------:|----------------|-----:|-----------|---|----:|---|-----:|
131 +|gsm8k| 3|flexible-extract| 5|exact_match|↑ |0.768|± |0.0268|
132 +| | |strict-match | 5|exact_match|↑ |0.768|± |0.0268|
133 +```
134 +
135 +## Troubleshooting and Support
136 +
137 +If you encounter any issues or have feature requests, please open an issue on the `vllm-project/llm-compressor` GitHub repository.
138 +
139 +## Deprecated Flow
140 +
141 +```{note}
142 +The following information is preserved for reference and search purposes.
143 +The quantization method described below is deprecated in favor of the `llmcompressor` method described above.
144 +```
145 +
146 +For static per-tensor offline quantization to FP8, please install the [AutoFP8 library](https://github.com/neuralmagic/autofp8).
147 +
148 +```bash
149 +git clone https://github.com/neuralmagic/AutoFP8.git
150 +pip install -e AutoFP8
151 +```
152 +
153 +This package introduces the `AutoFP8ForCausalLM` and `BaseQuantizeConfig` objects for managing how your model will be compressed.
154 +
155 +## Offline Quantization with Static Activation Scaling Factors
156 +
157 +You can use AutoFP8 with calibration data to produce per-tensor static scales for both the weights and activations by enabling the `activation_scheme="static"` argument.
158 +
159 +```python
160 +from datasets import load_dataset
161 +from transformers import AutoTokenizer
162 +from auto_fp8 import AutoFP8ForCausalLM, BaseQuantizeConfig
163 +
164 +pretrained_model_dir = "meta-llama/Meta-Llama-3-8B-Instruct"
165 +quantized_model_dir = "Meta-Llama-3-8B-Instruct-FP8"
166 +
167 +tokenizer = AutoTokenizer.from_pretrained(pretrained_model_dir, use_fast=True)
168 +tokenizer.pad_token = tokenizer.eos_token
169 +
170 +# Load and tokenize 512 dataset samples for calibration of activation scales
171 +ds = load_dataset("mgoin/ultrachat_2k", split="train_sft").select(range(512))
172 +examples = [tokenizer.apply_chat_template(batch["messages"], tokenize=False) for batch in ds]
173 +examples = tokenizer(examples, padding=True, truncation=True, return_tensors="pt").to("cuda")
174 +
175 +# Define quantization config with static activation scales
176 +quantize_config = BaseQuantizeConfig(quant_method="fp8", activation_scheme="static")
177 +
178 +# Load the model, quantize, and save checkpoint
179 +model = AutoFP8ForCausalLM.from_pretrained(pretrained_model_dir, quantize_config)
180 +model.quantize(examples)
181 +model.save_quantized(quantized_model_dir)
182 +```
183 +
184 +Your model checkpoint with quantized weights and activations should be available at `Meta-Llama-3-8B-Instruct-FP8/`.
185 +Finally, you can load the quantized model checkpoint directly in vLLM.
186 +
187 +```python
188 +from vllm import LLM
189 +model = LLM(model="Meta-Llama-3-8B-Instruct-FP8/")
190 +# INFO 06-10 21:15:41 model_runner.py:159] Loading model weights took 8.4596 GB
191 +result = model.generate("Hello, my name is")
192 +```
docs/source/quantization/fp8.rst deleted
-204
@@ -1,204 +0,0 @@
1 -.. _fp8:
2 -
3 -FP8 W8A8
4 -==================
5 -
6 -vLLM supports FP8 (8-bit floating point) weight and activation quantization using hardware acceleration on GPUs such as Nvidia H100 and AMD MI300x.
7 -Currently, only Hopper and Ada Lovelace GPUs are officially supported for W8A8.
8 -Ampere GPUs are supported for W8A16 (weight-only FP8) utilizing Marlin kernels.
9 -Quantization of models with FP8 allows for a 2x reduction in model memory requirements and up to a 1.6x improvement in throughput with minimal impact on accuracy.
10 -
11 -Please visit the HF collection of `quantized FP8 checkpoints of popular LLMs ready to use with vLLM <https://huggingface.co/collections/neuralmagic/fp8-llms-for-vllm-666742ed2b78b7ac8df13127>`_.
12 -
13 -The FP8 types typically supported in hardware have two distinct representations, each useful in different scenarios:
14 -
15 -- **E4M3**: Consists of 1 sign bit, 4 exponent bits, and 3 bits of mantissa. It can store values up to +/-448 and ``nan``.
16 -- **E5M2**: Consists of 1 sign bit, 5 exponent bits, and 2 bits of mantissa. It can store values up to +/-57344, +/- ``inf``, and ``nan``. The tradeoff for the increased dynamic range is lower precision of the stored values.
17 -
18 -.. note::
19 -
20 - FP8 computation is supported on NVIDIA GPUs with compute capability > 8.9 (Ada Lovelace, Hopper).
21 - FP8 models will run on compute capability > 8.0 (Ampere) as weight-only W8A16, utilizing FP8 Marlin.
22 -
23 -Quick Start with Online Dynamic Quantization
24 ---------------------------------------------
25 -
26 -Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying ``--quantization="fp8"`` in the command line or setting ``quantization="fp8"`` in the LLM constructor.
27 -
28 -In this mode, all Linear modules (except for the final ``lm_head``) have their weights quantized down to FP8_E4M3 precision with a per-tensor scale. Activations have their minimum and maximum values calculated during each forward pass to provide a dynamic per-tensor scale for high accuracy. As a result, latency improvements are limited in this mode.
29 -
30 -.. code-block:: python
31 -
32 - from vllm import LLM
33 - model = LLM("facebook/opt-125m", quantization="fp8")
34 - # INFO 06-10 17:55:42 model_runner.py:157] Loading model weights took 0.1550 GB
35 - result = model.generate("Hello, my name is")
36 -
37 -.. warning::
38 -
39 - Currently, we load the model at original precision before quantizing down to 8-bits, so you need enough memory to load the whole model.
40 -
41 -Installation
42 -------------
43 -
44 -To produce performant FP8 quantized models with vLLM, you'll need to install the `llm-compressor <https://github.com/vllm-project/llm-compressor/>`_ library:
45 -
46 -.. code-block:: console
47 -
48 - $ pip install llmcompressor
49 -
50 -Quantization Process
51 ---------------------
52 -
53 -The quantization process involves three main steps:
54 -
55 -1. Loading the model
56 -2. Applying quantization
57 -3. Evaluating accuracy in vLLM
58 -
59 -1. Loading the Model
60 -^^^^^^^^^^^^^^^^^^^^
61 -
62 -Use ``SparseAutoModelForCausalLM``, which wraps ``AutoModelForCausalLM``, for saving and loading quantized models:
63 -
64 -.. code-block:: python
65 -
66 - from llmcompressor.transformers import SparseAutoModelForCausalLM
67 - from transformers import AutoTokenizer
68 -
69 - MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"
70 -
71 - model = SparseAutoModelForCausalLM.from_pretrained(
72 - MODEL_ID, device_map="auto", torch_dtype="auto")
73 - tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
74 -
75 -2. Applying Quantization
76 -^^^^^^^^^^^^^^^^^^^^^^^^
77 -
78 -For FP8 quantization, we can recover accuracy with simple RTN quantization. We recommend targeting all ``Linear`` layers using the ``FP8_DYNAMIC`` scheme, which uses:
79 -
80 -- Static, per-channel quantization on the weights
81 -- Dynamic, per-token quantization on the activations
82 -
83 -Since simple RTN does not require data for weight quantization and the activations are quantized dynamically, we do not need any calibration data for this quantization flow.
84 -
85 -.. code-block:: python
86 -
87 - from llmcompressor.transformers import oneshot
88 - from llmcompressor.modifiers.quantization import QuantizationModifier
89 -
90 - # Configure the simple PTQ quantization
91 - recipe = QuantizationModifier(
92 - targets="Linear", scheme="FP8_DYNAMIC", ignore=["lm_head"])
93 -
94 - # Apply the quantization algorithm.
95 - oneshot(model=model, recipe=recipe)
96 -
97 - # Save the model.
98 - SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-Dynamic"
99 - model.save_pretrained(SAVE_DIR)
100 - tokenizer.save_pretrained(SAVE_DIR)
101 -
102 -3. Evaluating Accuracy
103 -^^^^^^^^^^^^^^^^^^^^^^
104 -
105 -Install ``vllm`` and ``lm-evaluation-harness``:
106 -
107 -.. code-block:: console
108 -
109 - $ pip install vllm lm-eval==0.4.4
110 -
111 -Load and run the model in ``vllm``:
112 -
113 -.. code-block:: python
114 -
115 - from vllm import LLM
116 - model = LLM("./Meta-Llama-3-8B-Instruct-FP8-Dynamic")
117 - model.generate("Hello my name is")
118 -
119 -Evaluate accuracy with ``lm_eval`` (for example on 250 samples of ``gsm8k``):
120 -
121 -.. note::
122 -
123 - Quantized models can be sensitive to the presence of the ``bos`` token. ``lm_eval`` does not add a ``bos`` token by default, so make sure to include the ``add_bos_token=True`` argument when running your evaluations.
124 -
125 -.. code-block:: console
126 -
127 - $ MODEL=$PWD/Meta-Llama-3-8B-Instruct-FP8-Dynamic
128 - $ lm_eval \
129 - --model vllm \
130 - --model_args pretrained=$MODEL,add_bos_token=True \
131 - --tasks gsm8k --num_fewshot 5 --batch_size auto --limit 250
132 -
133 -Here's an example of the resulting scores:
134 -
135 -.. code-block:: text
136 -
137 - |Tasks|Version| Filter |n-shot| Metric | |Value| |Stderr|
138 - |-----|------:|----------------|-----:|-----------|---|----:|---|-----:|
139 - |gsm8k| 3|flexible-extract| 5|exact_match|↑ |0.768|± |0.0268|
140 - | | |strict-match | 5|exact_match|↑ |0.768|± |0.0268|
141 -
142 -Troubleshooting and Support
143 ----------------------------
144 -
145 -If you encounter any issues or have feature requests, please open an issue on the ``vllm-project/llm-compressor`` GitHub repository.
146 -
147 -
148 -Deprecated Flow
149 -------------------
150 -
151 -.. note::
152 -
153 - The following information is preserved for reference and search purposes.
154 - The quantization method described below is deprecated in favor of the ``llmcompressor`` method described above.
155 -
156 -For static per-tensor offline quantization to FP8, please install the `AutoFP8 library <https://github.com/neuralmagic/autofp8>`_.
157 -
158 -.. code-block:: bash
159 -
160 - git clone https://github.com/neuralmagic/AutoFP8.git
161 - pip install -e AutoFP8
162 -
163 -This package introduces the ``AutoFP8ForCausalLM`` and ``BaseQuantizeConfig`` objects for managing how your model will be compressed.
164 -
165 -Offline Quantization with Static Activation Scaling Factors
166 ------------------------------------------------------------
167 -
168 -You can use AutoFP8 with calibration data to produce per-tensor static scales for both the weights and activations by enabling the ``activation_scheme="static"`` argument.
169 -
170 -.. code-block:: python
171 -
172 - from datasets import load_dataset
173 - from transformers import AutoTokenizer
174 - from auto_fp8 import AutoFP8ForCausalLM, BaseQuantizeConfig
175 -
176 - pretrained_model_dir = "meta-llama/Meta-Llama-3-8B-Instruct"
177 - quantized_model_dir = "Meta-Llama-3-8B-Instruct-FP8"
178 -
179 - tokenizer = AutoTokenizer.from_pretrained(pretrained_model_dir, use_fast=True)
180 - tokenizer.pad_token = tokenizer.eos_token
181 -
182 - # Load and tokenize 512 dataset samples for calibration of activation scales
183 - ds = load_dataset("mgoin/ultrachat_2k", split="train_sft").select(range(512))
184 - examples = [tokenizer.apply_chat_template(batch["messages"], tokenize=False) for batch in ds]
185 - examples = tokenizer(examples, padding=True, truncation=True, return_tensors="pt").to("cuda")
186 -
187 - # Define quantization config with static activation scales
188 - quantize_config = BaseQuantizeConfig(quant_method="fp8", activation_scheme="static")
189 -
190 - # Load the model, quantize, and save checkpoint
191 - model = AutoFP8ForCausalLM.from_pretrained(pretrained_model_dir, quantize_config)
192 - model.quantize(examples)
193 - model.save_quantized(quantized_model_dir)
194 -
195 -Your model checkpoint with quantized weights and activations should be available at ``Meta-Llama-3-8B-Instruct-FP8/``.
196 -Finally, you can load the quantized model checkpoint directly in vLLM.
197 -
198 -.. code-block:: python
199 -
200 - from vllm import LLM
201 - model = LLM(model="Meta-Llama-3-8B-Instruct-FP8/")
202 - # INFO 06-10 21:15:41 model_runner.py:159] Loading model weights took 8.4596 GB
203 - result = model.generate("Hello, my name is")
204 -
docs/source/quantization/fp8_e4m3_kvcache.md new
+44
@@ -0,0 +1,44 @@
1 +(fp8-e4m3-kvcache)=
2 +
3 +# FP8 E4M3 KV Cache
4 +
5 +Quantizing the KV cache to FP8 reduces its memory footprint. This increases the number of tokens that can be stored in the cache,
6 +improving throughput. OCP (Open Compute Project www.opencompute.org) specifies two common 8-bit floating point data formats: E5M2
7 +(5 exponent bits and 2 mantissa bits) and E4M3FN (4 exponent bits and 3 mantissa bits), often shortened as E4M3. One benefit of
8 +the E4M3 format over E5M2 is that floating point numbers are represented in higher precision. However, the small dynamic range of
9 +FP8 E4M3 (±240.0 can be represented) typically necessitates the use of a higher-precision (typically FP32) scaling factor alongside
10 +each quantized tensor. For now, only per-tensor (scalar) scaling factors are supported. Development is ongoing to support scaling
11 +factors of a finer granularity (e.g. per-channel).
12 +
13 +These scaling factors can be specified by passing an optional quantization param JSON to the LLM engine at load time. If
14 +this JSON is not specified, scaling factors default to 1.0. These scaling factors are typically obtained when running an
15 +unquantized model through a quantizer tool (e.g. AMD quantizer or NVIDIA AMMO).
16 +
17 +To install AMMO (AlgorithMic Model Optimization):
18 +
19 +```console
20 +$ pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com nvidia-ammo
21 +```
22 +
23 +Studies have shown that FP8 E4M3 quantization typically only minimally degrades inference accuracy. The most recent silicon
24 +offerings e.g. AMD MI300, NVIDIA Hopper or later support native hardware conversion to and from fp32, fp16, bf16, etc.
25 +Thus, LLM inference is greatly accelerated with minimal accuracy loss.
26 +
27 +Here is an example of how to enable this feature:
28 +
29 +```python
30 +# two float8_e4m3fn kv cache scaling factor files are provided under tests/fp8_kv, please refer to
31 +# https://github.com/vllm-project/vllm/blob/main/examples/fp8/README.md to generate kv_cache_scales.json of your own.
32 +
33 +from vllm import LLM, SamplingParams
34 +sampling_params = SamplingParams(temperature=1.3, top_p=0.8)
35 +llm = LLM(model="meta-llama/Llama-2-7b-chat-hf",
36 + kv_cache_dtype="fp8",
37 + quantization_param_path="./tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json")
38 +prompt = "London is the capital of"
39 +out = llm.generate(prompt, sampling_params)[0].outputs[0].text
40 +print(out)
41 +
42 +# output w/ scaling factors: England, the United Kingdom, and one of the world's leading financial,
43 +# output w/o scaling factors: England, located in the southeastern part of the country. It is known
44 +```
docs/source/quantization/fp8_e4m3_kvcache.rst deleted
-47
@@ -1,47 +0,0 @@
1 -.. _fp8_e4m3_kvcache:
2 -
3 -FP8 E4M3 KV Cache
4 -==================
5 -
6 -Quantizing the KV cache to FP8 reduces its memory footprint. This increases the number of tokens that can be stored in the cache,
7 -improving throughput. OCP (Open Compute Project www.opencompute.org) specifies two common 8-bit floating point data formats: E5M2
8 -(5 exponent bits and 2 mantissa bits) and E4M3FN (4 exponent bits and 3 mantissa bits), often shortened as E4M3. One benefit of
9 -the E4M3 format over E5M2 is that floating point numbers are represented in higher precision. However, the small dynamic range of
10 -FP8 E4M3 (±240.0 can be represented) typically necessitates the use of a higher-precision (typically FP32) scaling factor alongside
11 -each quantized tensor. For now, only per-tensor (scalar) scaling factors are supported. Development is ongoing to support scaling
12 -factors of a finer granularity (e.g. per-channel).
13 -
14 -These scaling factors can be specified by passing an optional quantization param JSON to the LLM engine at load time. If
15 -this JSON is not specified, scaling factors default to 1.0. These scaling factors are typically obtained when running an
16 -unquantized model through a quantizer tool (e.g. AMD quantizer or NVIDIA AMMO).
17 -
18 -To install AMMO (AlgorithMic Model Optimization):
19 -
20 -.. code-block:: console
21 -
22 - $ pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com nvidia-ammo
23 -
24 -Studies have shown that FP8 E4M3 quantization typically only minimally degrades inference accuracy. The most recent silicon
25 -offerings e.g. AMD MI300, NVIDIA Hopper or later support native hardware conversion to and from fp32, fp16, bf16, etc.
26 -Thus, LLM inference is greatly accelerated with minimal accuracy loss.
27 -
28 -
29 -Here is an example of how to enable this feature:
30 -
31 -.. code-block:: python
32 -
33 - # two float8_e4m3fn kv cache scaling factor files are provided under tests/fp8_kv, please refer to
34 - # https://github.com/vllm-project/vllm/blob/main/examples/fp8/README.md to generate kv_cache_scales.json of your own.
35 -
36 - from vllm import LLM, SamplingParams
37 - sampling_params = SamplingParams(temperature=1.3, top_p=0.8)
38 - llm = LLM(model="meta-llama/Llama-2-7b-chat-hf",
39 - kv_cache_dtype="fp8",
40 - quantization_param_path="./tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json")
41 - prompt = "London is the capital of"
42 - out = llm.generate(prompt, sampling_params)[0].outputs[0].text
43 - print(out)
44 -
45 - # output w/ scaling factors: England, the United Kingdom, and one of the world's leading financial,
46 - # output w/o scaling factors: England, located in the southeastern part of the country. It is known
47 -
docs/source/quantization/fp8_e5m2_kvcache.md new
+31
@@ -0,0 +1,31 @@
1 +(fp8-kv-cache)=
2 +
3 +# FP8 E5M2 KV Cache
4 +
5 +The int8/int4 quantization scheme requires additional scale GPU memory storage, which reduces the expected GPU memory benefits.
6 +The FP8 data format retains 2~3 mantissa bits and can convert float/fp16/bfloat16 and fp8 to each other.
7 +
8 +Here is an example of how to enable this feature:
9 +
10 +```python
11 +from vllm import LLM, SamplingParams
12 +# Sample prompts.
13 +prompts = [
14 + "Hello, my name is",
15 + "The president of the United States is",
16 + "The capital of France is",
17 + "The future of AI is",
18 +]
19 +# Create a sampling params object.
20 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
21 +# Create an LLM.
22 +llm = LLM(model="facebook/opt-125m", kv_cache_dtype="fp8")
23 +# Generate texts from the prompts. The output is a list of RequestOutput objects
24 +# that contain the prompt, generated text, and other information.
25 +outputs = llm.generate(prompts, sampling_params)
26 +# Print the outputs.
27 +for output in outputs:
28 + prompt = output.prompt
29 + generated_text = output.outputs[0].text
30 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
31 +```
docs/source/quantization/fp8_e5m2_kvcache.rst deleted
-34
@@ -1,34 +0,0 @@
1 -.. _fp8_kv_cache:
2 -
3 -FP8 E5M2 KV Cache
4 -==================
5 -
6 -The int8/int4 quantization scheme requires additional scale GPU memory storage, which reduces the expected GPU memory benefits.
7 -The FP8 data format retains 2~3 mantissa bits and can convert float/fp16/bfloat16 and fp8 to each other.
8 -
9 -Here is an example of how to enable this feature:
10 -
11 -.. code-block:: python
12 -
13 - from vllm import LLM, SamplingParams
14 - # Sample prompts.
15 - prompts = [
16 - "Hello, my name is",
17 - "The president of the United States is",
18 - "The capital of France is",
19 - "The future of AI is",
20 - ]
21 - # Create a sampling params object.
22 - sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
23 - # Create an LLM.
24 - llm = LLM(model="facebook/opt-125m", kv_cache_dtype="fp8")
25 - # Generate texts from the prompts. The output is a list of RequestOutput objects
26 - # that contain the prompt, generated text, and other information.
27 - outputs = llm.generate(prompts, sampling_params)
28 - # Print the outputs.
29 - for output in outputs:
30 - prompt = output.prompt
31 - generated_text = output.outputs[0].text
32 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
33 -
34 -
docs/source/quantization/gguf.md new
+72
@@ -0,0 +1,72 @@
1 +(gguf)=
2 +
3 +# GGUF
4 +
5 +```{warning}
6 +Please note that GGUF support in vLLM is highly experimental and under-optimized at the moment, it might be incompatible with other features. Currently, you can use GGUF as a way to reduce memory footprint. If you encounter any issues, please report them to the vLLM team.
7 +```
8 +
9 +```{warning}
10 +Currently, vllm only supports loading single-file GGUF models. If you have a multi-files GGUF model, you can use [gguf-split](https://github.com/ggerganov/llama.cpp/pull/6135) tool to merge them to a single-file model.
11 +```
12 +
13 +To run a GGUF model with vLLM, you can download and use the local GGUF model from [TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF) with the following command:
14 +
15 +```console
16 +$ wget https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf
17 +$ # We recommend using the tokenizer from base model to avoid long-time and buggy tokenizer conversion.
18 +$ vllm serve ./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf --tokenizer TinyLlama/TinyLlama-1.1B-Chat-v1.0
19 +```
20 +
21 +You can also add `--tensor-parallel-size 2` to enable tensor parallelism inference with 2 GPUs:
22 +
23 +```console
24 +$ # We recommend using the tokenizer from base model to avoid long-time and buggy tokenizer conversion.
25 +$ vllm serve ./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf --tokenizer TinyLlama/TinyLlama-1.1B-Chat-v1.0 --tensor-parallel-size 2
26 +```
27 +
28 +```{warning}
29 +We recommend using the tokenizer from base model instead of GGUF model. Because the tokenizer conversion from GGUF is time-consuming and unstable, especially for some models with large vocab size.
30 +```
31 +
32 +You can also use the GGUF model directly through the LLM entrypoint:
33 +
34 +```python
35 +from vllm import LLM, SamplingParams
36 +
37 +# In this script, we demonstrate how to pass input to the chat method:
38 +conversation = [
39 + {
40 + "role": "system",
41 + "content": "You are a helpful assistant"
42 + },
43 + {
44 + "role": "user",
45 + "content": "Hello"
46 + },
47 + {
48 + "role": "assistant",
49 + "content": "Hello! How can I assist you today?"
50 + },
51 + {
52 + "role": "user",
53 + "content": "Write an essay about the importance of higher education.",
54 + },
55 +]
56 +
57 +# Create a sampling params object.
58 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
59 +
60 +# Create an LLM.
61 +llm = LLM(model="./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
62 + tokenizer="TinyLlama/TinyLlama-1.1B-Chat-v1.0")
63 +# Generate texts from the prompts. The output is a list of RequestOutput objects
64 +# that contain the prompt, generated text, and other information.
65 +outputs = llm.chat(conversation, sampling_params)
66 +
67 +# Print the outputs.
68 +for output in outputs:
69 + prompt = output.prompt
70 + generated_text = output.outputs[0].text
71 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
72 +```
docs/source/quantization/gguf.rst deleted
-73
@@ -1,73 +0,0 @@
1 -.. _gguf:
2 -
3 -GGUF
4 -==================
5 -
6 -.. warning::
7 -
8 - Please note that GGUF support in vLLM is highly experimental and under-optimized at the moment, it might be incompatible with other features. Currently, you can use GGUF as a way to reduce memory footprint. If you encounter any issues, please report them to the vLLM team.
9 -
10 -.. warning::
11 -
12 - Currently, vllm only supports loading single-file GGUF models. If you have a multi-files GGUF model, you can use `gguf-split <https://github.com/ggerganov/llama.cpp/pull/6135>`_ tool to merge them to a single-file model.
13 -
14 -To run a GGUF model with vLLM, you can download and use the local GGUF model from `TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF <https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF>`_ with the following command:
15 -
16 -.. code-block:: console
17 -
18 - $ wget https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf
19 - $ # We recommend using the tokenizer from base model to avoid long-time and buggy tokenizer conversion.
20 - $ vllm serve ./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf --tokenizer TinyLlama/TinyLlama-1.1B-Chat-v1.0
21 -
22 -You can also add ``--tensor-parallel-size 2`` to enable tensor parallelism inference with 2 GPUs:
23 -
24 -.. code-block:: console
25 -
26 - $ # We recommend using the tokenizer from base model to avoid long-time and buggy tokenizer conversion.
27 - $ vllm serve ./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf --tokenizer TinyLlama/TinyLlama-1.1B-Chat-v1.0 --tensor-parallel-size 2
28 -
29 -.. warning::
30 -
31 - We recommend using the tokenizer from base model instead of GGUF model. Because the tokenizer conversion from GGUF is time-consuming and unstable, especially for some models with large vocab size.
32 -
33 -You can also use the GGUF model directly through the LLM entrypoint:
34 -
35 -.. code-block:: python
36 -
37 - from vllm import LLM, SamplingParams
38 -
39 - # In this script, we demonstrate how to pass input to the chat method:
40 - conversation = [
41 - {
42 - "role": "system",
43 - "content": "You are a helpful assistant"
44 - },
45 - {
46 - "role": "user",
47 - "content": "Hello"
48 - },
49 - {
50 - "role": "assistant",
51 - "content": "Hello! How can I assist you today?"
52 - },
53 - {
54 - "role": "user",
55 - "content": "Write an essay about the importance of higher education.",
56 - },
57 - ]
58 -
59 - # Create a sampling params object.
60 - sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
61 -
62 - # Create an LLM.
63 - llm = LLM(model="./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
64 - tokenizer="TinyLlama/TinyLlama-1.1B-Chat-v1.0")
65 - # Generate texts from the prompts. The output is a list of RequestOutput objects
66 - # that contain the prompt, generated text, and other information.
67 - outputs = llm.chat(conversation, sampling_params)
68 -
69 - # Print the outputs.
70 - for output in outputs:
71 - prompt = output.prompt
72 - generated_text = output.outputs[0].text
73 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
docs/source/quantization/int8.md new
+136
@@ -0,0 +1,136 @@
1 +(int8)=
2 +
3 +# INT8 W8A8
4 +
5 +vLLM supports quantizing weights and activations to INT8 for memory savings and inference acceleration.
6 +This quantization method is particularly useful for reducing model size while maintaining good performance.
7 +
8 +Please visit the HF collection of [quantized INT8 checkpoints of popular LLMs ready to use with vLLM](https://huggingface.co/collections/neuralmagic/int8-llms-for-vllm-668ec32c049dca0369816415).
9 +
10 +```{note}
11 +INT8 computation is supported on NVIDIA GPUs with compute capability > 7.5 (Turing, Ampere, Ada Lovelace, Hopper).
12 +```
13 +
14 +## Prerequisites
15 +
16 +To use INT8 quantization with vLLM, you'll need to install the [llm-compressor](https://github.com/vllm-project/llm-compressor/) library:
17 +
18 +```console
19 +$ pip install llmcompressor
20 +```
21 +
22 +## Quantization Process
23 +
24 +The quantization process involves four main steps:
25 +
26 +1. Loading the model
27 +2. Preparing calibration data
28 +3. Applying quantization
29 +4. Evaluating accuracy in vLLM
30 +
31 +### 1. Loading the Model
32 +
33 +Use `SparseAutoModelForCausalLM`, which wraps `AutoModelForCausalLM`, for saving and loading quantized models:
34 +
35 +```python
36 +from llmcompressor.transformers import SparseAutoModelForCausalLM
37 +from transformers import AutoTokenizer
38 +
39 +MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"
40 +model = SparseAutoModelForCausalLM.from_pretrained(
41 + MODEL_ID, device_map="auto", torch_dtype="auto",
42 +)
43 +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
44 +```
45 +
46 +### 2. Preparing Calibration Data
47 +
48 +When quantizing activations to INT8, you need sample data to estimate the activation scales.
49 +It's best to use calibration data that closely matches your deployment data.
50 +For a general-purpose instruction-tuned model, you can use a dataset like `ultrachat`:
51 +
52 +```python
53 +from datasets import load_dataset
54 +
55 +NUM_CALIBRATION_SAMPLES = 512
56 +MAX_SEQUENCE_LENGTH = 2048
57 +
58 +# Load and preprocess the dataset
59 +ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft")
60 +ds = ds.shuffle(seed=42).select(range(NUM_CALIBRATION_SAMPLES))
61 +
62 +def preprocess(example):
63 + return {"text": tokenizer.apply_chat_template(example["messages"], tokenize=False)}
64 +ds = ds.map(preprocess)
65 +
66 +def tokenize(sample):
67 + return tokenizer(sample["text"], padding=False, max_length=MAX_SEQUENCE_LENGTH, truncation=True, add_special_tokens=False)
68 +ds = ds.map(tokenize, remove_columns=ds.column_names)
69 +```
70 +
71 +### 3. Applying Quantization
72 +
73 +Now, apply the quantization algorithms:
74 +
75 +```python
76 +from llmcompressor.transformers import oneshot
77 +from llmcompressor.modifiers.quantization import GPTQModifier
78 +from llmcompressor.modifiers.smoothquant import SmoothQuantModifier
79 +
80 +# Configure the quantization algorithms
81 +recipe = [
82 + SmoothQuantModifier(smoothing_strength=0.8),
83 + GPTQModifier(targets="Linear", scheme="W8A8", ignore=["lm_head"]),
84 +]
85 +
86 +# Apply quantization
87 +oneshot(
88 + model=model,
89 + dataset=ds,
90 + recipe=recipe,
91 + max_seq_length=MAX_SEQUENCE_LENGTH,
92 + num_calibration_samples=NUM_CALIBRATION_SAMPLES,
93 +)
94 +
95 +# Save the compressed model
96 +SAVE_DIR = MODEL_ID.split("/")[1] + "-W8A8-Dynamic-Per-Token"
97 +model.save_pretrained(SAVE_DIR, save_compressed=True)
98 +tokenizer.save_pretrained(SAVE_DIR)
99 +```
100 +
101 +This process creates a W8A8 model with weights and activations quantized to 8-bit integers.
102 +
103 +### 4. Evaluating Accuracy
104 +
105 +After quantization, you can load and run the model in vLLM:
106 +
107 +```python
108 +from vllm import LLM
109 +model = LLM("./Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Per-Token")
110 +```
111 +
112 +To evaluate accuracy, you can use `lm_eval`:
113 +
114 +```console
115 +$ lm_eval --model vllm \
116 + --model_args pretrained="./Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Per-Token",add_bos_token=true \
117 + --tasks gsm8k \
118 + --num_fewshot 5 \
119 + --limit 250 \
120 + --batch_size 'auto'
121 +```
122 +
123 +```{note}
124 +Quantized models can be sensitive to the presence of the `bos` token. Make sure to include the `add_bos_token=True` argument when running evaluations.
125 +```
126 +
127 +## Best Practices
128 +
129 +- Start with 512 samples for calibration data (increase if accuracy drops)
130 +- Use a sequence length of 2048 as a starting point
131 +- Employ the chat template or instruction template that the model was trained with
132 +- If you've fine-tuned a model, consider using a sample of your training data for calibration
133 +
134 +## Troubleshooting and Support
135 +
136 +If you encounter any issues or have feature requests, please open an issue on the `vllm-project/llm-compressor` GitHub repository.
docs/source/quantization/int8.rst deleted
-145
@@ -1,145 +0,0 @@
1 -.. _int8:
2 -
3 -INT8 W8A8
4 -==================
5 -
6 -vLLM supports quantizing weights and activations to INT8 for memory savings and inference acceleration.
7 -This quantization method is particularly useful for reducing model size while maintaining good performance.
8 -
9 -Please visit the HF collection of `quantized INT8 checkpoints of popular LLMs ready to use with vLLM <https://huggingface.co/collections/neuralmagic/int8-llms-for-vllm-668ec32c049dca0369816415>`_.
10 -
11 -.. note::
12 -
13 - INT8 computation is supported on NVIDIA GPUs with compute capability > 7.5 (Turing, Ampere, Ada Lovelace, Hopper).
14 -
15 -Prerequisites
16 --------------
17 -
18 -To use INT8 quantization with vLLM, you'll need to install the `llm-compressor <https://github.com/vllm-project/llm-compressor/>`_ library:
19 -
20 -.. code-block:: console
21 -
22 - $ pip install llmcompressor
23 -
24 -Quantization Process
25 ---------------------
26 -
27 -The quantization process involves four main steps:
28 -
29 -1. Loading the model
30 -2. Preparing calibration data
31 -3. Applying quantization
32 -4. Evaluating accuracy in vLLM
33 -
34 -1. Loading the Model
35 -^^^^^^^^^^^^^^^^^^^^
36 -
37 -Use ``SparseAutoModelForCausalLM``, which wraps ``AutoModelForCausalLM``, for saving and loading quantized models:
38 -
39 -.. code-block:: python
40 -
41 - from llmcompressor.transformers import SparseAutoModelForCausalLM
42 - from transformers import AutoTokenizer
43 -
44 - MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"
45 - model = SparseAutoModelForCausalLM.from_pretrained(
46 - MODEL_ID, device_map="auto", torch_dtype="auto",
47 - )
48 - tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
49 -
50 -2. Preparing Calibration Data
51 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
52 -
53 -When quantizing activations to INT8, you need sample data to estimate the activation scales.
54 -It's best to use calibration data that closely matches your deployment data.
55 -For a general-purpose instruction-tuned model, you can use a dataset like ``ultrachat``:
56 -
57 -.. code-block:: python
58 -
59 - from datasets import load_dataset
60 -
61 - NUM_CALIBRATION_SAMPLES = 512
62 - MAX_SEQUENCE_LENGTH = 2048
63 -
64 - # Load and preprocess the dataset
65 - ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft")
66 - ds = ds.shuffle(seed=42).select(range(NUM_CALIBRATION_SAMPLES))
67 -
68 - def preprocess(example):
69 - return {"text": tokenizer.apply_chat_template(example["messages"], tokenize=False)}
70 - ds = ds.map(preprocess)
71 -
72 - def tokenize(sample):
73 - return tokenizer(sample["text"], padding=False, max_length=MAX_SEQUENCE_LENGTH, truncation=True, add_special_tokens=False)
74 - ds = ds.map(tokenize, remove_columns=ds.column_names)
75 -
76 -3. Applying Quantization
77 -^^^^^^^^^^^^^^^^^^^^^^^^
78 -
79 -Now, apply the quantization algorithms:
80 -
81 -.. code-block:: python
82 -
83 - from llmcompressor.transformers import oneshot
84 - from llmcompressor.modifiers.quantization import GPTQModifier
85 - from llmcompressor.modifiers.smoothquant import SmoothQuantModifier
86 -
87 - # Configure the quantization algorithms
88 - recipe = [
89 - SmoothQuantModifier(smoothing_strength=0.8),
90 - GPTQModifier(targets="Linear", scheme="W8A8", ignore=["lm_head"]),
91 - ]
92 -
93 - # Apply quantization
94 - oneshot(
95 - model=model,
96 - dataset=ds,
97 - recipe=recipe,
98 - max_seq_length=MAX_SEQUENCE_LENGTH,
99 - num_calibration_samples=NUM_CALIBRATION_SAMPLES,
100 - )
101 -
102 - # Save the compressed model
103 - SAVE_DIR = MODEL_ID.split("/")[1] + "-W8A8-Dynamic-Per-Token"
104 - model.save_pretrained(SAVE_DIR, save_compressed=True)
105 - tokenizer.save_pretrained(SAVE_DIR)
106 -
107 -This process creates a W8A8 model with weights and activations quantized to 8-bit integers.
108 -
109 -4. Evaluating Accuracy
110 -^^^^^^^^^^^^^^^^^^^^^^
111 -
112 -After quantization, you can load and run the model in vLLM:
113 -
114 -.. code-block:: python
115 -
116 - from vllm import LLM
117 - model = LLM("./Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Per-Token")
118 -
119 -To evaluate accuracy, you can use ``lm_eval``:
120 -
121 -.. code-block:: console
122 -
123 - $ lm_eval --model vllm \
124 - --model_args pretrained="./Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Per-Token",add_bos_token=true \
125 - --tasks gsm8k \
126 - --num_fewshot 5 \
127 - --limit 250 \
128 - --batch_size 'auto'
129 -
130 -.. note::
131 -
132 - Quantized models can be sensitive to the presence of the ``bos`` token. Make sure to include the ``add_bos_token=True`` argument when running evaluations.
133 -
134 -Best Practices
135 ---------------
136 -
137 -- Start with 512 samples for calibration data (increase if accuracy drops)
138 -- Use a sequence length of 2048 as a starting point
139 -- Employ the chat template or instruction template that the model was trained with
140 -- If you've fine-tuned a model, consider using a sample of your training data for calibration
141 -
142 -Troubleshooting and Support
143 ----------------------------
144 -
145 -If you encounter any issues or have feature requests, please open an issue on the ``vllm-project/llm-compressor`` GitHub repository.
docs/source/quantization/supported_hardware.md renamed
+132 -132
@@ -1,132 +1,132 @@
1 -.. _supported_hardware_for_quantization:
2 -
3 -Supported Hardware for Quantization Kernels
4 -===========================================
5 -
6 -The table below shows the compatibility of various quantization implementations with different hardware platforms in vLLM:
7 -
8 -.. list-table::
9 - :header-rows: 1
10 - :widths: 20 8 8 8 8 8 8 8 8 8 8
11 -
12 - * - Implementation
13 - - Volta
14 - - Turing
15 - - Ampere
16 - - Ada
17 - - Hopper
18 - - AMD GPU
19 - - Intel GPU
20 - - x86 CPU
21 - - AWS Inferentia
22 - - Google TPU
23 - * - AWQ
24 - - ✗
25 - - ✅︎
26 - - ✅︎
27 - - ✅︎
28 - - ✅︎
29 - - ✗
30 - - ✅︎
31 - - ✅︎
32 - - ✗
33 - - ✗
34 - * - GPTQ
35 - - ✅︎
36 - - ✅︎
37 - - ✅︎
38 - - ✅︎
39 - - ✅︎
40 - - ✗
41 - - ✅︎
42 - - ✅︎
43 - - ✗
44 - - ✗
45 - * - Marlin (GPTQ/AWQ/FP8)
46 - - ✗
47 - - ✗
48 - - ✅︎
49 - - ✅︎
50 - - ✅︎
51 - - ✗
52 - - ✗
53 - - ✗
54 - - ✗
55 - - ✗
56 - * - INT8 (W8A8)
57 - - ✗
58 - - ✅︎
59 - - ✅︎
60 - - ✅︎
61 - - ✅︎
62 - - ✗
63 - - ✗
64 - - ✅︎
65 - - ✗
66 - - ✗
67 - * - FP8 (W8A8)
68 - - ✗
69 - - ✗
70 - - ✗
71 - - ✅︎
72 - - ✅︎
73 - - ✅︎
74 - - ✗
75 - - ✗
76 - - ✗
77 - - ✗
78 - * - AQLM
79 - - ✅︎
80 - - ✅︎
81 - - ✅︎
82 - - ✅︎
83 - - ✅︎
84 - - ✗
85 - - ✗
86 - - ✗
87 - - ✗
88 - - ✗
89 - * - bitsandbytes
90 - - ✅︎
91 - - ✅︎
92 - - ✅︎
93 - - ✅︎
94 - - ✅︎
95 - - ✗
96 - - ✗
97 - - ✗
98 - - ✗
99 - - ✗
100 - * - DeepSpeedFP
101 - - ✅︎
102 - - ✅︎
103 - - ✅︎
104 - - ✅︎
105 - - ✅︎
106 - - ✗
107 - - ✗
108 - - ✗
109 - - ✗
110 - - ✗
111 - * - GGUF
112 - - ✅︎
113 - - ✅︎
114 - - ✅︎
115 - - ✅︎
116 - - ✅︎
117 - - ✗
118 - - ✗
119 - - ✗
120 - - ✗
121 - - ✗
122 -
123 -Notes:
124 -^^^^^^
125 -
126 -- Volta refers to SM 7.0, Turing to SM 7.5, Ampere to SM 8.0/8.6, Ada to SM 8.9, and Hopper to SM 9.0.
127 -- "✅︎" indicates that the quantization method is supported on the specified hardware.
128 -- "✗" indicates that the quantization method is not supported on the specified hardware.
129 -
130 -Please note that this compatibility chart may be subject to change as vLLM continues to evolve and expand its support for different hardware platforms and quantization methods.
131 -
132 -For the most up-to-date information on hardware support and quantization methods, please check the `quantization directory <https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization>`_ or consult with the vLLM development team.
1 +(supported-hardware-for-quantization)=
2 +
3 +# Supported Hardware for Quantization Kernels
4 +
5 +The table below shows the compatibility of various quantization implementations with different hardware platforms in vLLM:
6 +
7 +```{eval-rst}
8 +.. list-table::
9 + :header-rows: 1
10 + :widths: 20 8 8 8 8 8 8 8 8 8 8
11 +
12 + * - Implementation
13 + - Volta
14 + - Turing
15 + - Ampere
16 + - Ada
17 + - Hopper
18 + - AMD GPU
19 + - Intel GPU
20 + - x86 CPU
21 + - AWS Inferentia
22 + - Google TPU
23 + * - AWQ
24 + - ✗
25 + - ✅︎
26 + - ✅︎
27 + - ✅︎
28 + - ✅︎
29 + - ✗
30 + - ✅︎
31 + - ✅︎
32 + - ✗
33 + - ✗
34 + * - GPTQ
35 + - ✅︎
36 + - ✅︎
37 + - ✅︎
38 + - ✅︎
39 + - ✅︎
40 + - ✗
41 + - ✅︎
42 + - ✅︎
43 + - ✗
44 + - ✗
45 + * - Marlin (GPTQ/AWQ/FP8)
46 + - ✗
47 + - ✗
48 + - ✅︎
49 + - ✅︎
50 + - ✅︎
51 + - ✗
52 + - ✗
53 + - ✗
54 + - ✗
55 + - ✗
56 + * - INT8 (W8A8)
57 + - ✗
58 + - ✅︎
59 + - ✅︎
60 + - ✅︎
61 + - ✅︎
62 + - ✗
63 + - ✗
64 + - ✅︎
65 + - ✗
66 + - ✗
67 + * - FP8 (W8A8)
68 + - ✗
69 + - ✗
70 + - ✗
71 + - ✅︎
72 + - ✅︎
73 + - ✅︎
74 + - ✗
75 + - ✗
76 + - ✗
77 + - ✗
78 + * - AQLM
79 + - ✅︎
80 + - ✅︎
81 + - ✅︎
82 + - ✅︎
83 + - ✅︎
84 + - ✗
85 + - ✗
86 + - ✗
87 + - ✗
88 + - ✗
89 + * - bitsandbytes
90 + - ✅︎
91 + - ✅︎
92 + - ✅︎
93 + - ✅︎
94 + - ✅︎
95 + - ✗
96 + - ✗
97 + - ✗
98 + - ✗
99 + - ✗
100 + * - DeepSpeedFP
101 + - ✅︎
102 + - ✅︎
103 + - ✅︎
104 + - ✅︎
105 + - ✅︎
106 + - ✗
107 + - ✗
108 + - ✗
109 + - ✗
110 + - ✗
111 + * - GGUF
112 + - ✅︎
113 + - ✅︎
114 + - ✅︎
115 + - ✅︎
116 + - ✅︎
117 + - ✗
118 + - ✗
119 + - ✗
120 + - ✗
121 + - ✗
122 +```
123 +
124 +## Notes:
125 +
126 +- Volta refers to SM 7.0, Turing to SM 7.5, Ampere to SM 8.0/8.6, Ada to SM 8.9, and Hopper to SM 9.0.
127 +- "✅︎" indicates that the quantization method is supported on the specified hardware.
128 +- "✗" indicates that the quantization method is not supported on the specified hardware.
129 +
130 +Please note that this compatibility chart may be subject to change as vLLM continues to evolve and expand its support for different hardware platforms and quantization methods.
131 +
132 +For the most up-to-date information on hardware support and quantization methods, please check the [quantization directory](https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization) or consult with the vLLM development team.
docs/source/serving/deploying_with_bentoml.md new
+7
@@ -0,0 +1,7 @@
1 +(deploying-with-bentoml)=
2 +
3 +# Deploying with BentoML
4 +
5 +[BentoML](https://github.com/bentoml/BentoML) allows you to deploy a large language model (LLM) server with vLLM as the backend, which exposes OpenAI-compatible endpoints. You can serve the model locally or containerize it as an OCI-complicant image and deploy it on Kubernetes.
6 +
7 +For details, see the tutorial [vLLM inference in the BentoML documentation](https://docs.bentoml.com/en/latest/use-cases/large-language-models/vllm.html).
docs/source/serving/deploying_with_bentoml.rst deleted
-8
@@ -1,8 +0,0 @@
1 -.. _deploying_with_bentoml:
2 -
3 -Deploying with BentoML
4 -======================
5 -
6 -`BentoML <https://github.com/bentoml/BentoML>`_ allows you to deploy a large language model (LLM) server with vLLM as the backend, which exposes OpenAI-compatible endpoints. You can serve the model locally or containerize it as an OCI-complicant image and deploy it on Kubernetes.
7 -
8 -For details, see the tutorial `vLLM inference in the BentoML documentation <https://docs.bentoml.com/en/latest/use-cases/large-language-models/vllm.html>`_.
\ No newline at end of file
docs/source/serving/deploying_with_cerebrium.md new
+109
@@ -0,0 +1,109 @@
1 +(deploying-with-cerebrium)=
2 +
3 +# Deploying with Cerebrium
4 +
5 +```{raw} html
6 +<p align="center">
7 + <img src="https://i.ibb.co/hHcScTT/Screenshot-2024-06-13-at-10-14-54.png" alt="vLLM_plus_cerebrium"/>
8 +</p>
9 +```
10 +
11 +vLLM can be run on a cloud based GPU machine with [Cerebrium](https://www.cerebrium.ai/), a serverless AI infrastructure platform that makes it easier for companies to build and deploy AI based applications.
12 +
13 +To install the Cerebrium client, run:
14 +
15 +```console
16 +$ pip install cerebrium
17 +$ cerebrium login
18 +```
19 +
20 +Next, create your Cerebrium project, run:
21 +
22 +```console
23 +$ cerebrium init vllm-project
24 +```
25 +
26 +Next, to install the required packages, add the following to your cerebrium.toml:
27 +
28 +```toml
29 +[cerebrium.deployment]
30 +docker_base_image_url = "nvidia/cuda:12.1.1-runtime-ubuntu22.04"
31 +
32 +[cerebrium.dependencies.pip]
33 +vllm = "latest"
34 +```
35 +
36 +Next, let us add our code to handle inference for the LLM of your choice(`mistralai/Mistral-7B-Instruct-v0.1` for this example), add the following code to your main.py\`:
37 +
38 +```python
39 +from vllm import LLM, SamplingParams
40 +
41 +llm = LLM(model="mistralai/Mistral-7B-Instruct-v0.1")
42 +
43 +def run(prompts: list[str], temperature: float = 0.8, top_p: float = 0.95):
44 +
45 + sampling_params = SamplingParams(temperature=temperature, top_p=top_p)
46 + outputs = llm.generate(prompts, sampling_params)
47 +
48 + # Print the outputs.
49 + results = []
50 + for output in outputs:
51 + prompt = output.prompt
52 + generated_text = output.outputs[0].text
53 + results.append({"prompt": prompt, "generated_text": generated_text})
54 +
55 + return {"results": results}
56 +```
57 +
58 +Then, run the following code to deploy it to the cloud
59 +
60 +```console
61 +$ cerebrium deploy
62 +```
63 +
64 +If successful, you should be returned a CURL command that you can call inference against. Just remember to end the url with the function name you are calling (in our case /run)
65 +
66 +```python
67 +curl -X POST https://api.cortex.cerebrium.ai/v4/p-xxxxxx/vllm/run \
68 + -H 'Content-Type: application/json' \
69 + -H 'Authorization: <JWT TOKEN>' \
70 + --data '{
71 + "prompts": [
72 + "Hello, my name is",
73 + "The president of the United States is",
74 + "The capital of France is",
75 + "The future of AI is"
76 + ]
77 + }'
78 +```
79 +
80 +You should get a response like:
81 +
82 +```python
83 +{
84 + "run_id": "52911756-3066-9ae8-bcc9-d9129d1bd262",
85 + "result": {
86 + "result": [
87 + {
88 + "prompt": "Hello, my name is",
89 + "generated_text": " Sarah, and I'm a teacher. I teach elementary school students. One of"
90 + },
91 + {
92 + "prompt": "The president of the United States is",
93 + "generated_text": " elected every four years. This is a democratic system.\n\n5. What"
94 + },
95 + {
96 + "prompt": "The capital of France is",
97 + "generated_text": " Paris.\n"
98 + },
99 + {
100 + "prompt": "The future of AI is",
101 + "generated_text": " bright, but it's important to approach it with a balanced and nuanced perspective."
102 + }
103 + ]
104 + },
105 + "run_time_ms": 152.53663063049316
106 +}
107 +```
108 +
109 +You now have an autoscaling endpoint where you only pay for the compute you use!
docs/source/serving/deploying_with_cerebrium.rst deleted
-112
@@ -1,112 +0,0 @@
1 -.. _deploying_with_cerebrium:
2 -
3 -Deploying with Cerebrium
4 -============================
5 -
6 -.. raw:: html
7 -
8 - <p align="center">
9 - <img src="https://i.ibb.co/hHcScTT/Screenshot-2024-06-13-at-10-14-54.png" alt="vLLM_plus_cerebrium"/>
10 - </p>
11 -
12 -vLLM can be run on a cloud based GPU machine with `Cerebrium <https://www.cerebrium.ai/>`__, a serverless AI infrastructure platform that makes it easier for companies to build and deploy AI based applications.
13 -
14 -To install the Cerebrium client, run:
15 -
16 -.. code-block:: console
17 -
18 - $ pip install cerebrium
19 - $ cerebrium login
20 -
21 -Next, create your Cerebrium project, run:
22 -
23 -.. code-block:: console
24 -
25 - $ cerebrium init vllm-project
26 -
27 -Next, to install the required packages, add the following to your cerebrium.toml:
28 -
29 -.. code-block:: toml
30 -
31 - [cerebrium.deployment]
32 - docker_base_image_url = "nvidia/cuda:12.1.1-runtime-ubuntu22.04"
33 -
34 - [cerebrium.dependencies.pip]
35 - vllm = "latest"
36 -
37 -Next, let us add our code to handle inference for the LLM of your choice(`mistralai/Mistral-7B-Instruct-v0.1` for this example), add the following code to your main.py`:
38 -
39 -.. code-block:: python
40 -
41 - from vllm import LLM, SamplingParams
42 -
43 - llm = LLM(model="mistralai/Mistral-7B-Instruct-v0.1")
44 -
45 - def run(prompts: list[str], temperature: float = 0.8, top_p: float = 0.95):
46 -
47 - sampling_params = SamplingParams(temperature=temperature, top_p=top_p)
48 - outputs = llm.generate(prompts, sampling_params)
49 -
50 - # Print the outputs.
51 - results = []
52 - for output in outputs:
53 - prompt = output.prompt
54 - generated_text = output.outputs[0].text
55 - results.append({"prompt": prompt, "generated_text": generated_text})
56 -
57 - return {"results": results}
58 -
59 -
60 -Then, run the following code to deploy it to the cloud
61 -
62 -.. code-block:: console
63 -
64 - $ cerebrium deploy
65 -
66 -If successful, you should be returned a CURL command that you can call inference against. Just remember to end the url with the function name you are calling (in our case /run)
67 -
68 -.. code-block:: python
69 -
70 - curl -X POST https://api.cortex.cerebrium.ai/v4/p-xxxxxx/vllm/run \
71 - -H 'Content-Type: application/json' \
72 - -H 'Authorization: <JWT TOKEN>' \
73 - --data '{
74 - "prompts": [
75 - "Hello, my name is",
76 - "The president of the United States is",
77 - "The capital of France is",
78 - "The future of AI is"
79 - ]
80 - }'
81 -
82 -You should get a response like:
83 -
84 -.. code-block:: python
85 -
86 - {
87 - "run_id": "52911756-3066-9ae8-bcc9-d9129d1bd262",
88 - "result": {
89 - "result": [
90 - {
91 - "prompt": "Hello, my name is",
92 - "generated_text": " Sarah, and I'm a teacher. I teach elementary school students. One of"
93 - },
94 - {
95 - "prompt": "The president of the United States is",
96 - "generated_text": " elected every four years. This is a democratic system.\n\n5. What"
97 - },
98 - {
99 - "prompt": "The capital of France is",
100 - "generated_text": " Paris.\n"
101 - },
102 - {
103 - "prompt": "The future of AI is",
104 - "generated_text": " bright, but it's important to approach it with a balanced and nuanced perspective."
105 - }
106 - ]
107 - },
108 - "run_time_ms": 152.53663063049316
109 - }
110 -
111 -You now have an autoscaling endpoint where you only pay for the compute you use!
112 -
docs/source/serving/deploying_with_docker.md new
+81
@@ -0,0 +1,81 @@
1 +(deploying-with-docker)=
2 +
3 +# Deploying with Docker
4 +
5 +## Use vLLM's Official Docker Image
6 +
7 +vLLM offers an official Docker image for deployment.
8 +The image can be used to run OpenAI compatible server and is available on Docker Hub as [vllm/vllm-openai](https://hub.docker.com/r/vllm/vllm-openai/tags).
9 +
10 +```console
11 +$ docker run --runtime nvidia --gpus all \
12 + -v ~/.cache/huggingface:/root/.cache/huggingface \
13 + --env "HUGGING_FACE_HUB_TOKEN=<secret>" \
14 + -p 8000:8000 \
15 + --ipc=host \
16 + vllm/vllm-openai:latest \
17 + --model mistralai/Mistral-7B-v0.1
18 +```
19 +
20 +```{note}
21 +You can either use the `ipc=host` flag or `--shm-size` flag to allow the
22 +container to access the host's shared memory. vLLM uses PyTorch, which uses shared
23 +memory to share data between processes under the hood, particularly for tensor parallel inference.
24 +```
25 +
26 +## Building vLLM's Docker Image from Source
27 +
28 +You can build and run vLLM from source via the provided [Dockerfile](https://github.com/vllm-project/vllm/blob/main/Dockerfile). To build vLLM:
29 +
30 +```console
31 +$ # optionally specifies: --build-arg max_jobs=8 --build-arg nvcc_threads=2
32 +$ DOCKER_BUILDKIT=1 docker build . --target vllm-openai --tag vllm/vllm-openai
33 +```
34 +
35 +```{note}
36 +By default vLLM will build for all GPU types for widest distribution. If you are just building for the
37 +current GPU type the machine is running on, you can add the argument `--build-arg torch_cuda_arch_list=""`
38 +for vLLM to find the current GPU type and build for that.
39 +```
40 +
41 +## Building for Arm64/aarch64
42 +
43 +A docker container can be built for aarch64 systems such as the Nvidia Grace-Hopper. At time of this writing, this requires the use
44 +of PyTorch Nightly and should be considered **experimental**. Using the flag `--platform "linux/arm64"` will attempt to build for arm64.
45 +
46 +```{note}
47 +Multiple modules must be compiled, so this process can take a while. Recommend using `--build-arg max_jobs=` & `--build-arg nvcc_threads=`
48 +flags to speed up build process. However, ensure your `max_jobs` is substantially larger than `nvcc_threads` to get the most benefits.
49 +Keep an eye on memory usage with parallel jobs as it can be substantial (see example below).
50 +```
51 +
52 +```console
53 +# Example of building on Nvidia GH200 server. (Memory usage: ~15GB, Build time: ~1475s / ~25 min, Image size: 6.93GB)
54 +$ python3 use_existing_torch.py
55 +$ DOCKER_BUILDKIT=1 docker build . \
56 + --target vllm-openai \
57 + --platform "linux/arm64" \
58 + -t vllm/vllm-gh200-openai:latest \
59 + --build-arg max_jobs=66 \
60 + --build-arg nvcc_threads=2 \
61 + --build-arg torch_cuda_arch_list="9.0+PTX" \
62 + --build-arg vllm_fa_cmake_gpu_arches="90-real"
63 +```
64 +
65 +## Use the custom-built vLLM Docker image
66 +
67 +To run vLLM with the custom-built Docker image:
68 +
69 +```console
70 +$ docker run --runtime nvidia --gpus all \
71 + -v ~/.cache/huggingface:/root/.cache/huggingface \
72 + -p 8000:8000 \
73 + --env "HUGGING_FACE_HUB_TOKEN=<secret>" \
74 + vllm/vllm-openai <args...>
75 +```
76 +
77 +The argument `vllm/vllm-openai` specifies the image to run, and should be replaced with the name of the custom-built image (the `-t` tag from the build command).
78 +
79 +```{note}
80 +**For version 0.4.1 and 0.4.2 only** - the vLLM docker images under these versions are supposed to be run under the root user since a library under the root user's home directory, i.e. `/root/.config/vllm/nccl/cu12/libnccl.so.2.18.1` is required to be loaded during runtime. If you are running the container under a different user, you may need to first change the permissions of the library (and all the parent directories) to allow the user to access it, then run vLLM with environment variable `VLLM_NCCL_SO_PATH=/root/.config/vllm/nccl/cu12/libnccl.so.2.18.1` .
81 +```
docs/source/serving/deploying_with_docker.rst deleted
-88
@@ -1,88 +0,0 @@
1 -.. _deploying_with_docker:
2 -
3 -Deploying with Docker
4 -============================
5 -
6 -Use vLLM's Official Docker Image
7 ---------------------------------
8 -
9 -vLLM offers an official Docker image for deployment.
10 -The image can be used to run OpenAI compatible server and is available on Docker Hub as `vllm/vllm-openai <https://hub.docker.com/r/vllm/vllm-openai/tags>`_.
11 -
12 -.. code-block:: console
13 -
14 - $ docker run --runtime nvidia --gpus all \
15 - -v ~/.cache/huggingface:/root/.cache/huggingface \
16 - --env "HUGGING_FACE_HUB_TOKEN=<secret>" \
17 - -p 8000:8000 \
18 - --ipc=host \
19 - vllm/vllm-openai:latest \
20 - --model mistralai/Mistral-7B-v0.1
21 -
22 -
23 -.. note::
24 -
25 - You can either use the ``ipc=host`` flag or ``--shm-size`` flag to allow the
26 - container to access the host's shared memory. vLLM uses PyTorch, which uses shared
27 - memory to share data between processes under the hood, particularly for tensor parallel inference.
28 -
29 -
30 -Building vLLM's Docker Image from Source
31 -----------------------------------------
32 -
33 -You can build and run vLLM from source via the provided `Dockerfile <https://github.com/vllm-project/vllm/blob/main/Dockerfile>`_. To build vLLM:
34 -
35 -.. code-block:: console
36 -
37 - $ # optionally specifies: --build-arg max_jobs=8 --build-arg nvcc_threads=2
38 - $ DOCKER_BUILDKIT=1 docker build . --target vllm-openai --tag vllm/vllm-openai
39 -
40 -.. note::
41 -
42 - By default vLLM will build for all GPU types for widest distribution. If you are just building for the
43 - current GPU type the machine is running on, you can add the argument ``--build-arg torch_cuda_arch_list=""``
44 - for vLLM to find the current GPU type and build for that.
45 -
46 -Building for Arm64/aarch64
47 ---------------------------
48 -
49 -A docker container can be built for aarch64 systems such as the Nvidia Grace-Hopper. At time of this writing, this requires the use
50 -of PyTorch Nightly and should be considered **experimental**. Using the flag ``--platform "linux/arm64"`` will attempt to build for arm64.
51 -
52 -.. note::
53 -
54 - Multiple modules must be compiled, so this process can take a while. Recommend using ``--build-arg max_jobs=`` & ``--build-arg nvcc_threads=``
55 - flags to speed up build process. However, ensure your ``max_jobs`` is substantially larger than ``nvcc_threads`` to get the most benefits.
56 - Keep an eye on memory usage with parallel jobs as it can be substantial (see example below).
57 -
58 -.. code-block:: console
59 -
60 - # Example of building on Nvidia GH200 server. (Memory usage: ~15GB, Build time: ~1475s / ~25 min, Image size: 6.93GB)
61 - $ python3 use_existing_torch.py
62 - $ DOCKER_BUILDKIT=1 docker build . \
63 - --target vllm-openai \
64 - --platform "linux/arm64" \
65 - -t vllm/vllm-gh200-openai:latest \
66 - --build-arg max_jobs=66 \
67 - --build-arg nvcc_threads=2 \
68 - --build-arg torch_cuda_arch_list="9.0+PTX" \
69 - --build-arg vllm_fa_cmake_gpu_arches="90-real"
70 -
71 -Use the custom-built vLLM Docker image
72 ---------------------------------------
73 -
74 -To run vLLM with the custom-built Docker image:
75 -
76 -.. code-block:: console
77 -
78 - $ docker run --runtime nvidia --gpus all \
79 - -v ~/.cache/huggingface:/root/.cache/huggingface \
80 - -p 8000:8000 \
81 - --env "HUGGING_FACE_HUB_TOKEN=<secret>" \
82 - vllm/vllm-openai <args...>
83 -
84 -The argument ``vllm/vllm-openai`` specifies the image to run, and should be replaced with the name of the custom-built image (the ``-t`` tag from the build command).
85 -
86 -.. note::
87 -
88 - **For version 0.4.1 and 0.4.2 only** - the vLLM docker images under these versions are supposed to be run under the root user since a library under the root user's home directory, i.e. ``/root/.config/vllm/nccl/cu12/libnccl.so.2.18.1`` is required to be loaded during runtime. If you are running the container under a different user, you may need to first change the permissions of the library (and all the parent directories) to allow the user to access it, then run vLLM with environment variable ``VLLM_NCCL_SO_PATH=/root/.config/vllm/nccl/cu12/libnccl.so.2.18.1`` .
docs/source/serving/deploying_with_dstack.md new
+102
@@ -0,0 +1,102 @@
1 +(deploying-with-dstack)=
2 +
3 +# Deploying with dstack
4 +
5 +```{raw} html
6 +<p align="center">
7 + <img src="https://i.ibb.co/71kx6hW/vllm-dstack.png" alt="vLLM_plus_dstack"/>
8 +</p>
9 +```
10 +
11 +vLLM can be run on a cloud based GPU machine with [dstack](https://dstack.ai/), an open-source framework for running LLMs on any cloud. This tutorial assumes that you have already configured credentials, gateway, and GPU quotas on your cloud environment.
12 +
13 +To install dstack client, run:
14 +
15 +```console
16 +$ pip install "dstack[all]
17 +$ dstack server
18 +```
19 +
20 +Next, to configure your dstack project, run:
21 +
22 +```console
23 +$ mkdir -p vllm-dstack
24 +$ cd vllm-dstack
25 +$ dstack init
26 +```
27 +
28 +Next, to provision a VM instance with LLM of your choice(`NousResearch/Llama-2-7b-chat-hf` for this example), create the following `serve.dstack.yml` file for the dstack `Service`:
29 +
30 +```yaml
31 +type: service
32 +
33 +python: "3.11"
34 +env:
35 + - MODEL=NousResearch/Llama-2-7b-chat-hf
36 +port: 8000
37 +resources:
38 + gpu: 24GB
39 +commands:
40 + - pip install vllm
41 + - vllm serve $MODEL --port 8000
42 +model:
43 + format: openai
44 + type: chat
45 + name: NousResearch/Llama-2-7b-chat-hf
46 +```
47 +
48 +Then, run the following CLI for provisioning:
49 +
50 +```console
51 +$ dstack run . -f serve.dstack.yml
52 +
53 +⠸ Getting run plan...
54 + Configuration serve.dstack.yml
55 + Project deep-diver-main
56 + User deep-diver
57 + Min resources 2..xCPU, 8GB.., 1xGPU (24GB)
58 + Max price -
59 + Max duration -
60 + Spot policy auto
61 + Retry policy no
62 +
63 + # BACKEND REGION INSTANCE RESOURCES SPOT PRICE
64 + 1 gcp us-central1 g2-standard-4 4xCPU, 16GB, 1xL4 (24GB), 100GB (disk) yes $0.223804
65 + 2 gcp us-east1 g2-standard-4 4xCPU, 16GB, 1xL4 (24GB), 100GB (disk) yes $0.223804
66 + 3 gcp us-west1 g2-standard-4 4xCPU, 16GB, 1xL4 (24GB), 100GB (disk) yes $0.223804
67 + ...
68 + Shown 3 of 193 offers, $5.876 max
69 +
70 +Continue? [y/n]: y
71 +⠙ Submitting run...
72 +⠏ Launching spicy-treefrog-1 (pulling)
73 +spicy-treefrog-1 provisioning completed (running)
74 +Service is published at ...
75 +```
76 +
77 +After the provisioning, you can interact with the model by using the OpenAI SDK:
78 +
79 +```python
80 +from openai import OpenAI
81 +
82 +client = OpenAI(
83 + base_url="https://gateway.<gateway domain>",
84 + api_key="<YOUR-DSTACK-SERVER-ACCESS-TOKEN>"
85 +)
86 +
87 +completion = client.chat.completions.create(
88 + model="NousResearch/Llama-2-7b-chat-hf",
89 + messages=[
90 + {
91 + "role": "user",
92 + "content": "Compose a poem that explains the concept of recursion in programming.",
93 + }
94 + ]
95 +)
96 +
97 +print(completion.choices[0].message.content)
98 +```
99 +
100 +```{note}
101 +dstack automatically handles authentication on the gateway using dstack's tokens. Meanwhile, if you don't want to configure a gateway, you can provision dstack `Task` instead of `Service`. The `Task` is for development purpose only. If you want to know more about hands-on materials how to serve vLLM using dstack, check out [this repository](https://github.com/dstackai/dstack-examples/tree/main/deployment/vllm)
102 +```
docs/source/serving/deploying_with_dstack.rst deleted
-103
@@ -1,103 +0,0 @@
1 -.. _deploying_with_dstack:
2 -
3 -Deploying with dstack
4 -============================
5 -
6 -.. raw:: html
7 -
8 - <p align="center">
9 - <img src="https://i.ibb.co/71kx6hW/vllm-dstack.png" alt="vLLM_plus_dstack"/>
10 - </p>
11 -
12 -vLLM can be run on a cloud based GPU machine with `dstack <https://dstack.ai/>`__, an open-source framework for running LLMs on any cloud. This tutorial assumes that you have already configured credentials, gateway, and GPU quotas on your cloud environment.
13 -
14 -To install dstack client, run:
15 -
16 -.. code-block:: console
17 -
18 - $ pip install "dstack[all]
19 - $ dstack server
20 -
21 -Next, to configure your dstack project, run:
22 -
23 -.. code-block:: console
24 -
25 - $ mkdir -p vllm-dstack
26 - $ cd vllm-dstack
27 - $ dstack init
28 -
29 -Next, to provision a VM instance with LLM of your choice(`NousResearch/Llama-2-7b-chat-hf` for this example), create the following `serve.dstack.yml` file for the dstack `Service`:
30 -
31 -.. code-block:: yaml
32 -
33 - type: service
34 -
35 - python: "3.11"
36 - env:
37 - - MODEL=NousResearch/Llama-2-7b-chat-hf
38 - port: 8000
39 - resources:
40 - gpu: 24GB
41 - commands:
42 - - pip install vllm
43 - - vllm serve $MODEL --port 8000
44 - model:
45 - format: openai
46 - type: chat
47 - name: NousResearch/Llama-2-7b-chat-hf
48 -
49 -Then, run the following CLI for provisioning:
50 -
51 -.. code-block:: console
52 -
53 - $ dstack run . -f serve.dstack.yml
54 -
55 - ⠸ Getting run plan...
56 - Configuration serve.dstack.yml
57 - Project deep-diver-main
58 - User deep-diver
59 - Min resources 2..xCPU, 8GB.., 1xGPU (24GB)
60 - Max price -
61 - Max duration -
62 - Spot policy auto
63 - Retry policy no
64 -
65 - # BACKEND REGION INSTANCE RESOURCES SPOT PRICE
66 - 1 gcp us-central1 g2-standard-4 4xCPU, 16GB, 1xL4 (24GB), 100GB (disk) yes $0.223804
67 - 2 gcp us-east1 g2-standard-4 4xCPU, 16GB, 1xL4 (24GB), 100GB (disk) yes $0.223804
68 - 3 gcp us-west1 g2-standard-4 4xCPU, 16GB, 1xL4 (24GB), 100GB (disk) yes $0.223804
69 - ...
70 - Shown 3 of 193 offers, $5.876 max
71 -
72 - Continue? [y/n]: y
73 - ⠙ Submitting run...
74 - ⠏ Launching spicy-treefrog-1 (pulling)
75 - spicy-treefrog-1 provisioning completed (running)
76 - Service is published at ...
77 -
78 -After the provisioning, you can interact with the model by using the OpenAI SDK:
79 -
80 -.. code-block:: python
81 -
82 - from openai import OpenAI
83 -
84 - client = OpenAI(
85 - base_url="https://gateway.<gateway domain>",
86 - api_key="<YOUR-DSTACK-SERVER-ACCESS-TOKEN>"
87 - )
88 -
89 - completion = client.chat.completions.create(
90 - model="NousResearch/Llama-2-7b-chat-hf",
91 - messages=[
92 - {
93 - "role": "user",
94 - "content": "Compose a poem that explains the concept of recursion in programming.",
95 - }
96 - ]
97 - )
98 -
99 - print(completion.choices[0].message.content)
100 -
101 -.. note::
102 -
103 - dstack automatically handles authentication on the gateway using dstack's tokens. Meanwhile, if you don't want to configure a gateway, you can provision dstack `Task` instead of `Service`. The `Task` is for development purpose only. If you want to know more about hands-on materials how to serve vLLM using dstack, check out `this repository <https://github.com/dstackai/dstack-examples/tree/main/deployment/vllm>`__
docs/source/serving/deploying_with_helm.md renamed
+21 -23
@@ -1,7 +1,6 @@
1 -.. _deploying_with_helm:
1 +(deploying-with-helm)=
2
3 -Deploying with Helm
4 -===================
3 +# Deploying with Helm
4
5 A Helm chart to deploy vLLM for Kubernetes
6
@@ -9,44 +8,42 @@ Helm is a package manager for Kubernetes. It will help you to deploy vLLM on k8s
8
9 This guide will walk you through the process of deploying vLLM with Helm, including the necessary prerequisites, steps for helm install and documentation on architecture and values file.
10
12 -Prerequisites
13 --------------
11 +## Prerequisites
12 +
13 Before you begin, ensure that you have the following:
14
15 - A running Kubernetes cluster
17 -- NVIDIA Kubernetes Device Plugin (``k8s-device-plugin``): This can be found at `https://github.com/NVIDIA/k8s-device-plugin <https://github.com/NVIDIA/k8s-device-plugin>`__
16 +- NVIDIA Kubernetes Device Plugin (`k8s-device-plugin`): This can be found at [https://github.com/NVIDIA/k8s-device-plugin](https://github.com/NVIDIA/k8s-device-plugin)
17 - Available GPU resources in your cluster
18 - S3 with the model which will be deployed
19
21 -Installing the chart
22 ---------------------
23 -
24 -To install the chart with the release name ``test-vllm``:
25 -
26 -.. code-block:: console
20 +## Installing the chart
21
28 - helm upgrade --install --create-namespace --namespace=ns-vllm test-vllm . -f values.yaml --set secrets.s3endpoint=$ACCESS_POINT --set secrets.s3bucketname=$BUCKET --set secrets.s3accesskeyid=$ACCESS_KEY --set secrets.s3accesskey=$SECRET_KEY
22 +To install the chart with the release name `test-vllm`:
23
30 -Uninstalling the Chart
31 -----------------------
24 +```console
25 +helm upgrade --install --create-namespace --namespace=ns-vllm test-vllm . -f values.yaml --set secrets.s3endpoint=$ACCESS_POINT --set secrets.s3bucketname=$BUCKET --set secrets.s3accesskeyid=$ACCESS_KEY --set secrets.s3accesskey=$SECRET_KEY
26 +```
27
33 -To uninstall the ``test-vllm`` deployment:
28 +## Uninstalling the Chart
29
35 -.. code-block:: console
30 +To uninstall the `test-vllm` deployment:
31
37 - helm uninstall test-vllm --namespace=ns-vllm
32 +```console
33 +helm uninstall test-vllm --namespace=ns-vllm
34 +```
35
36 The command removes all the Kubernetes components associated with the
37 chart **including persistent volumes** and deletes the release.
38
42 -Architecture
43 -------------
39 +## Architecture
40
45 -.. image:: architecture_helm_deployment.png
41 +```{image} architecture_helm_deployment.png
42 +```
43
47 -Values
48 -------
44 +## Values
45
46 +```{eval-rst}
47 .. list-table:: Values
48 :widths: 25 25 25 25
49 :header-rows: 1
@@ -251,3 +248,4 @@ Values
248 - string
249 - test
250 - Release name
251 +```
docs/source/serving/deploying_with_k8s.md new
+171
@@ -0,0 +1,171 @@
1 +(deploying-with-k8s)=
2 +
3 +# Deploying with Kubernetes
4 +
5 +Using Kubernetes to deploy vLLM is a scalable and efficient way to serve machine learning models. This guide will walk you through the process of deploying vLLM with Kubernetes, including the necessary prerequisites, steps for deployment, and testing.
6 +
7 +## Prerequisites
8 +
9 +Before you begin, ensure that you have the following:
10 +
11 +- A running Kubernetes cluster
12 +- NVIDIA Kubernetes Device Plugin (`k8s-device-plugin`): This can be found at `https://github.com/NVIDIA/k8s-device-plugin/`
13 +- Available GPU resources in your cluster
14 +
15 +## Deployment Steps
16 +
17 +1. **Create a PVC , Secret and Deployment for vLLM**
18 +
19 +PVC is used to store the model cache and it is optional, you can use hostPath or other storage options
20 +
21 +```yaml
22 +apiVersion: v1
23 +kind: PersistentVolumeClaim
24 +metadata:
25 + name: mistral-7b
26 + namespace: default
27 +spec:
28 + accessModes:
29 + - ReadWriteOnce
30 + resources:
31 + requests:
32 + storage: 50Gi
33 + storageClassName: default
34 + volumeMode: Filesystem
35 +```
36 +
37 +Secret is optional and only required for accessing gated models, you can skip this step if you are not using gated models
38 +
39 +```yaml
40 +apiVersion: v1
41 +kind: Secret
42 +metadata:
43 + name: hf-token-secret
44 + namespace: default
45 +type: Opaque
46 +data:
47 + token: "REPLACE_WITH_TOKEN"
48 +```
49 +
50 +Create a deployment file for vLLM to run the model server. The following example deploys the `Mistral-7B-Instruct-v0.3` model:
51 +
52 +```yaml
53 +apiVersion: apps/v1
54 +kind: Deployment
55 +metadata:
56 + name: mistral-7b
57 + namespace: default
58 + labels:
59 + app: mistral-7b
60 +spec:
61 + replicas: 1
62 + selector:
63 + matchLabels:
64 + app: mistral-7b
65 + template:
66 + metadata:
67 + labels:
68 + app: mistral-7b
69 + spec:
70 + volumes:
71 + - name: cache-volume
72 + persistentVolumeClaim:
73 + claimName: mistral-7b
74 + # vLLM needs to access the host's shared memory for tensor parallel inference.
75 + - name: shm
76 + emptyDir:
77 + medium: Memory
78 + sizeLimit: "2Gi"
79 + containers:
80 + - name: mistral-7b
81 + image: vllm/vllm-openai:latest
82 + command: ["/bin/sh", "-c"]
83 + args: [
84 + "vllm serve mistralai/Mistral-7B-Instruct-v0.3 --trust-remote-code --enable-chunked-prefill --max_num_batched_tokens 1024"
85 + ]
86 + env:
87 + - name: HUGGING_FACE_HUB_TOKEN
88 + valueFrom:
89 + secretKeyRef:
90 + name: hf-token-secret
91 + key: token
92 + ports:
93 + - containerPort: 8000
94 + resources:
95 + limits:
96 + cpu: "10"
97 + memory: 20G
98 + nvidia.com/gpu: "1"
99 + requests:
100 + cpu: "2"
101 + memory: 6G
102 + nvidia.com/gpu: "1"
103 + volumeMounts:
104 + - mountPath: /root/.cache/huggingface
105 + name: cache-volume
106 + - name: shm
107 + mountPath: /dev/shm
108 + livenessProbe:
109 + httpGet:
110 + path: /health
111 + port: 8000
112 + initialDelaySeconds: 60
113 + periodSeconds: 10
114 + readinessProbe:
115 + httpGet:
116 + path: /health
117 + port: 8000
118 + initialDelaySeconds: 60
119 + periodSeconds: 5
120 +```
121 +
122 +2. **Create a Kubernetes Service for vLLM**
123 +
124 +Next, create a Kubernetes Service file to expose the `mistral-7b` deployment:
125 +
126 +```yaml
127 +apiVersion: v1
128 +kind: Service
129 +metadata:
130 + name: mistral-7b
131 + namespace: default
132 +spec:
133 + ports:
134 + - name: http-mistral-7b
135 + port: 80
136 + protocol: TCP
137 + targetPort: 8000
138 + # The label selector should match the deployment labels & it is useful for prefix caching feature
139 + selector:
140 + app: mistral-7b
141 + sessionAffinity: None
142 + type: ClusterIP
143 +```
144 +
145 +3. **Deploy and Test**
146 +
147 +Apply the deployment and service configurations using `kubectl apply -f <filename>`:
148 +
149 +```console
150 +kubectl apply -f deployment.yaml
151 +kubectl apply -f service.yaml
152 +```
153 +
154 +To test the deployment, run the following `curl` command:
155 +
156 +```console
157 +curl http://mistral-7b.default.svc.cluster.local/v1/completions \
158 + -H "Content-Type: application/json" \
159 + -d '{
160 + "model": "mistralai/Mistral-7B-Instruct-v0.3",
161 + "prompt": "San Francisco is a",
162 + "max_tokens": 7,
163 + "temperature": 0
164 + }'
165 +```
166 +
167 +If the service is correctly deployed, you should receive a response from the vLLM model.
168 +
169 +## Conclusion
170 +
171 +Deploying vLLM with Kubernetes allows for efficient scaling and management of ML models leveraging GPU resources. By following the steps outlined above, you should be able to set up and test a vLLM deployment within your Kubernetes cluster. If you encounter any issues or have suggestions, please feel free to contribute to the documentation.
docs/source/serving/deploying_with_k8s.rst deleted
-175
@@ -1,175 +0,0 @@
1 -.. _deploying_with_k8s:
2 -
3 -Deploying with Kubernetes
4 -==========================
5 -
6 -Using Kubernetes to deploy vLLM is a scalable and efficient way to serve machine learning models. This guide will walk you through the process of deploying vLLM with Kubernetes, including the necessary prerequisites, steps for deployment, and testing.
7 -
8 -Prerequisites
9 --------------
10 -Before you begin, ensure that you have the following:
11 -
12 -- A running Kubernetes cluster
13 -- NVIDIA Kubernetes Device Plugin (`k8s-device-plugin`): This can be found at `https://github.com/NVIDIA/k8s-device-plugin/`
14 -- Available GPU resources in your cluster
15 -
16 -Deployment Steps
17 -----------------
18 -
19 -1. **Create a PVC , Secret and Deployment for vLLM**
20 -
21 -
22 -PVC is used to store the model cache and it is optional, you can use hostPath or other storage options
23 -
24 -.. code-block:: yaml
25 -
26 - apiVersion: v1
27 - kind: PersistentVolumeClaim
28 - metadata:
29 - name: mistral-7b
30 - namespace: default
31 - spec:
32 - accessModes:
33 - - ReadWriteOnce
34 - resources:
35 - requests:
36 - storage: 50Gi
37 - storageClassName: default
38 - volumeMode: Filesystem
39 -
40 -Secret is optional and only required for accessing gated models, you can skip this step if you are not using gated models
41 -
42 -.. code-block:: yaml
43 -
44 - apiVersion: v1
45 - kind: Secret
46 - metadata:
47 - name: hf-token-secret
48 - namespace: default
49 - type: Opaque
50 - data:
51 - token: "REPLACE_WITH_TOKEN"
52 -
53 -
54 -Create a deployment file for vLLM to run the model server. The following example deploys the `Mistral-7B-Instruct-v0.3` model:
55 -
56 -.. code-block:: yaml
57 -
58 - apiVersion: apps/v1
59 - kind: Deployment
60 - metadata:
61 - name: mistral-7b
62 - namespace: default
63 - labels:
64 - app: mistral-7b
65 - spec:
66 - replicas: 1
67 - selector:
68 - matchLabels:
69 - app: mistral-7b
70 - template:
71 - metadata:
72 - labels:
73 - app: mistral-7b
74 - spec:
75 - volumes:
76 - - name: cache-volume
77 - persistentVolumeClaim:
78 - claimName: mistral-7b
79 - # vLLM needs to access the host's shared memory for tensor parallel inference.
80 - - name: shm
81 - emptyDir:
82 - medium: Memory
83 - sizeLimit: "2Gi"
84 - containers:
85 - - name: mistral-7b
86 - image: vllm/vllm-openai:latest
87 - command: ["/bin/sh", "-c"]
88 - args: [
89 - "vllm serve mistralai/Mistral-7B-Instruct-v0.3 --trust-remote-code --enable-chunked-prefill --max_num_batched_tokens 1024"
90 - ]
91 - env:
92 - - name: HUGGING_FACE_HUB_TOKEN
93 - valueFrom:
94 - secretKeyRef:
95 - name: hf-token-secret
96 - key: token
97 - ports:
98 - - containerPort: 8000
99 - resources:
100 - limits:
101 - cpu: "10"
102 - memory: 20G
103 - nvidia.com/gpu: "1"
104 - requests:
105 - cpu: "2"
106 - memory: 6G
107 - nvidia.com/gpu: "1"
108 - volumeMounts:
109 - - mountPath: /root/.cache/huggingface
110 - name: cache-volume
111 - - name: shm
112 - mountPath: /dev/shm
113 - livenessProbe:
114 - httpGet:
115 - path: /health
116 - port: 8000
117 - initialDelaySeconds: 60
118 - periodSeconds: 10
119 - readinessProbe:
120 - httpGet:
121 - path: /health
122 - port: 8000
123 - initialDelaySeconds: 60
124 - periodSeconds: 5
125 -
126 -2. **Create a Kubernetes Service for vLLM**
127 -
128 -Next, create a Kubernetes Service file to expose the `mistral-7b` deployment:
129 -
130 -.. code-block:: yaml
131 -
132 - apiVersion: v1
133 - kind: Service
134 - metadata:
135 - name: mistral-7b
136 - namespace: default
137 - spec:
138 - ports:
139 - - name: http-mistral-7b
140 - port: 80
141 - protocol: TCP
142 - targetPort: 8000
143 - # The label selector should match the deployment labels & it is useful for prefix caching feature
144 - selector:
145 - app: mistral-7b
146 - sessionAffinity: None
147 - type: ClusterIP
148 -
149 -3. **Deploy and Test**
150 -
151 -Apply the deployment and service configurations using ``kubectl apply -f <filename>``:
152 -
153 -.. code-block:: console
154 -
155 - kubectl apply -f deployment.yaml
156 - kubectl apply -f service.yaml
157 -
158 -To test the deployment, run the following ``curl`` command:
159 -
160 -.. code-block:: console
161 -
162 - curl http://mistral-7b.default.svc.cluster.local/v1/completions \
163 - -H "Content-Type: application/json" \
164 - -d '{
165 - "model": "mistralai/Mistral-7B-Instruct-v0.3",
166 - "prompt": "San Francisco is a",
167 - "max_tokens": 7,
168 - "temperature": 0
169 - }'
170 -
171 -If the service is correctly deployed, you should receive a response from the vLLM model.
172 -
173 -Conclusion
174 -----------
175 -Deploying vLLM with Kubernetes allows for efficient scaling and management of ML models leveraging GPU resources. By following the steps outlined above, you should be able to set up and test a vLLM deployment within your Kubernetes cluster. If you encounter any issues or have suggestions, please feel free to contribute to the documentation.
docs/source/serving/deploying_with_kserve.md new
+7
@@ -0,0 +1,7 @@
1 +(deploying-with-kserve)=
2 +
3 +# Deploying with KServe
4 +
5 +vLLM can be deployed with [KServe](https://github.com/kserve/kserve) on Kubernetes for highly scalable distributed model serving.
6 +
7 +Please see [this guide](https://kserve.github.io/website/latest/modelserving/v1beta1/llm/huggingface/) for more details on using vLLM with KServe.
docs/source/serving/deploying_with_kserve.rst deleted
-8
@@ -1,8 +0,0 @@
1 -.. _deploying_with_kserve:
2 -
3 -Deploying with KServe
4 -============================
5 -
6 -vLLM can be deployed with `KServe <https://github.com/kserve/kserve>`_ on Kubernetes for highly scalable distributed model serving.
7 -
8 -Please see `this guide <https://kserve.github.io/website/latest/modelserving/v1beta1/llm/huggingface/>`_ for more details on using vLLM with KServe.
docs/source/serving/deploying_with_kubeai.md new
+15
@@ -0,0 +1,15 @@
1 +(deploying-with-kubeai)=
2 +
3 +# Deploying with KubeAI
4 +
5 +[KubeAI](https://github.com/substratusai/kubeai) is a Kubernetes operator that enables you to deploy and manage AI models on Kubernetes. It provides a simple and scalable way to deploy vLLM in production. Functionality such as scale-from-zero, load based autoscaling, model caching, and much more is provided out of the box with zero external dependencies.
6 +
7 +Please see the Installation Guides for environment specific instructions:
8 +
9 +- [Any Kubernetes Cluster](https://www.kubeai.org/installation/any/)
10 +- [EKS](https://www.kubeai.org/installation/eks/)
11 +- [GKE](https://www.kubeai.org/installation/gke/)
12 +
13 +Once you have KubeAI installed, you can
14 +[configure text generation models](https://www.kubeai.org/how-to/configure-text-generation-models/)
15 +using vLLM.
docs/source/serving/deploying_with_kubeai.rst deleted
-17
@@ -1,17 +0,0 @@
1 -.. _deploying_with_kubeai:
2 -
3 -Deploying with KubeAI
4 -=====================
5 -
6 -`KubeAI <https://github.com/substratusai/kubeai>`_ is a Kubernetes operator that enables you to deploy and manage AI models on Kubernetes. It provides a simple and scalable way to deploy vLLM in production. Functionality such as scale-from-zero, load based autoscaling, model caching, and much more is provided out of the box with zero external dependencies.
7 -
8 -
9 -Please see the Installation Guides for environment specific instructions:
10 -
11 -* `Any Kubernetes Cluster <https://www.kubeai.org/installation/any/>`_
12 -* `EKS <https://www.kubeai.org/installation/eks/>`_
13 -* `GKE <https://www.kubeai.org/installation/gke/>`_
14 -
15 -Once you have KubeAI installed, you can
16 -`configure text generation models <https://www.kubeai.org/how-to/configure-text-generation-models/>`_
17 -using vLLM.
\ No newline at end of file
docs/source/serving/deploying_with_lws.md new
+11
@@ -0,0 +1,11 @@
1 +(deploying-with-lws)=
2 +
3 +# Deploying with LWS
4 +
5 +LeaderWorkerSet (LWS) is a Kubernetes API that aims to address common deployment patterns of AI/ML inference workloads.
6 +A major use case is for multi-host/multi-node distributed inference.
7 +
8 +vLLM can be deployed with [LWS](https://github.com/kubernetes-sigs/lws) on Kubernetes for distributed model serving.
9 +
10 +Please see [this guide](https://github.com/kubernetes-sigs/lws/tree/main/docs/examples/vllm) for more details on
11 +deploying vLLM on Kubernetes using LWS.
docs/source/serving/deploying_with_lws.rst deleted
-12
@@ -1,12 +0,0 @@
1 -.. _deploying_with_lws:
2 -
3 -Deploying with LWS
4 -============================
5 -
6 -LeaderWorkerSet (LWS) is a Kubernetes API that aims to address common deployment patterns of AI/ML inference workloads.
7 -A major use case is for multi-host/multi-node distributed inference.
8 -
9 -vLLM can be deployed with `LWS <https://github.com/kubernetes-sigs/lws>`_ on Kubernetes for distributed model serving.
10 -
11 -Please see `this guide <https://github.com/kubernetes-sigs/lws/tree/main/docs/examples/vllm>`_ for more details on
12 -deploying vLLM on Kubernetes using LWS.
docs/source/serving/deploying_with_nginx.md new
+133
@@ -0,0 +1,133 @@
1 +(nginxloadbalancer)=
2 +
3 +# Deploying with Nginx Loadbalancer
4 +
5 +This document shows how to launch multiple vLLM serving containers and use Nginx to act as a load balancer between the servers.
6 +
7 +Table of contents:
8 +
9 +1. [Build Nginx Container](#nginxloadbalancer-nginx-build)
10 +2. [Create Simple Nginx Config file](#nginxloadbalancer-nginx-conf)
11 +3. [Build vLLM Container](#nginxloadbalancer-nginx-vllm-container)
12 +4. [Create Docker Network](#nginxloadbalancer-nginx-docker-network)
13 +5. [Launch vLLM Containers](#nginxloadbalancer-nginx-launch-container)
14 +6. [Launch Nginx](#nginxloadbalancer-nginx-launch-nginx)
15 +7. [Verify That vLLM Servers Are Ready](#nginxloadbalancer-nginx-verify-nginx)
16 +
17 +(nginxloadbalancer-nginx-build)=
18 +
19 +## Build Nginx Container
20 +
21 +This guide assumes that you have just cloned the vLLM project and you're currently in the vllm root directory.
22 +
23 +```console
24 +export vllm_root=`pwd`
25 +```
26 +
27 +Create a file named `Dockerfile.nginx`:
28 +
29 +```console
30 +FROM nginx:latest
31 +RUN rm /etc/nginx/conf.d/default.conf
32 +EXPOSE 80
33 +CMD ["nginx", "-g", "daemon off;"]
34 +```
35 +
36 +Build the container:
37 +
38 +```console
39 +docker build . -f Dockerfile.nginx --tag nginx-lb
40 +```
41 +
42 +(nginxloadbalancer-nginx-conf)=
43 +
44 +## Create Simple Nginx Config file
45 +
46 +Create a file named `nginx_conf/nginx.conf`. Note that you can add as many servers as you'd like. In the below example we'll start with two. To add more, add another `server vllmN:8000 max_fails=3 fail_timeout=10000s;` entry to `upstream backend`.
47 +
48 +```console
49 +upstream backend {
50 + least_conn;
51 + server vllm0:8000 max_fails=3 fail_timeout=10000s;
52 + server vllm1:8000 max_fails=3 fail_timeout=10000s;
53 +}
54 +server {
55 + listen 80;
56 + location / {
57 + proxy_pass http://backend;
58 + proxy_set_header Host $host;
59 + proxy_set_header X-Real-IP $remote_addr;
60 + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
61 + proxy_set_header X-Forwarded-Proto $scheme;
62 + }
63 +}
64 +```
65 +
66 +(nginxloadbalancer-nginx-vllm-container)=
67 +
68 +## Build vLLM Container
69 +
70 +```console
71 +cd $vllm_root
72 +docker build -f Dockerfile . --tag vllm
73 +```
74 +
75 +If you are behind proxy, you can pass the proxy settings to the docker build command as shown below:
76 +
77 +```console
78 +cd $vllm_root
79 +docker build -f Dockerfile . --tag vllm --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy
80 +```
81 +
82 +(nginxloadbalancer-nginx-docker-network)=
83 +
84 +## Create Docker Network
85 +
86 +```console
87 +docker network create vllm_nginx
88 +```
89 +
90 +(nginxloadbalancer-nginx-launch-container)=
91 +
92 +## Launch vLLM Containers
93 +
94 +Notes:
95 +
96 +- If you have your HuggingFace models cached somewhere else, update `hf_cache_dir` below.
97 +- If you don't have an existing HuggingFace cache you will want to start `vllm0` and wait for the model to complete downloading and the server to be ready. This will ensure that `vllm1` can leverage the model you just downloaded and it won't have to be downloaded again.
98 +- The below example assumes GPU backend used. If you are using CPU backend, remove `--gpus all`, add `VLLM_CPU_KVCACHE_SPACE` and `VLLM_CPU_OMP_THREADS_BIND` environment variables to the docker run command.
99 +- Adjust the model name that you want to use in your vLLM servers if you don't want to use `Llama-2-7b-chat-hf`.
100 +
101 +```console
102 +mkdir -p ~/.cache/huggingface/hub/
103 +hf_cache_dir=~/.cache/huggingface/
104 +docker run -itd --ipc host --privileged --network vllm_nginx --gpus all --shm-size=10.24gb -v $hf_cache_dir:/root/.cache/huggingface/ -p 8081:8000 --name vllm0 vllm --model meta-llama/Llama-2-7b-chat-hf
105 +docker run -itd --ipc host --privileged --network vllm_nginx --gpus all --shm-size=10.24gb -v $hf_cache_dir:/root/.cache/huggingface/ -p 8082:8000 --name vllm1 vllm --model meta-llama/Llama-2-7b-chat-hf
106 +```
107 +
108 +```{note}
109 +If you are behind proxy, you can pass the proxy settings to the docker run command via `-e http_proxy=$http_proxy -e https_proxy=$https_proxy`.
110 +```
111 +
112 +(nginxloadbalancer-nginx-launch-nginx)=
113 +
114 +## Launch Nginx
115 +
116 +```console
117 +docker run -itd -p 8000:80 --network vllm_nginx -v ./nginx_conf/:/etc/nginx/conf.d/ --name nginx-lb nginx-lb:latest
118 +```
119 +
120 +(nginxloadbalancer-nginx-verify-nginx)=
121 +
122 +## Verify That vLLM Servers Are Ready
123 +
124 +```console
125 +docker logs vllm0 | grep Uvicorn
126 +docker logs vllm1 | grep Uvicorn
127 +```
128 +
129 +Both outputs should look like this:
130 +
131 +```console
132 +INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
133 +```
docs/source/serving/deploying_with_nginx.rst deleted
-142
@@ -1,142 +0,0 @@
1 -.. _nginxloadbalancer:
2 -
3 -Deploying with Nginx Loadbalancer
4 -=================================
5 -
6 -This document shows how to launch multiple vLLM serving containers and use Nginx to act as a load balancer between the servers.
7 -
8 -Table of contents:
9 -
10 -#. :ref:`Build Nginx Container <nginxloadbalancer_nginx_build>`
11 -#. :ref:`Create Simple Nginx Config file <nginxloadbalancer_nginx_conf>`
12 -#. :ref:`Build vLLM Container <nginxloadbalancer_nginx_vllm_container>`
13 -#. :ref:`Create Docker Network <nginxloadbalancer_nginx_docker_network>`
14 -#. :ref:`Launch vLLM Containers <nginxloadbalancer_nginx_launch_container>`
15 -#. :ref:`Launch Nginx <nginxloadbalancer_nginx_launch_nginx>`
16 -#. :ref:`Verify That vLLM Servers Are Ready <nginxloadbalancer_nginx_verify_nginx>`
17 -
18 -.. _nginxloadbalancer_nginx_build:
19 -
20 -Build Nginx Container
21 ----------------------
22 -
23 -This guide assumes that you have just cloned the vLLM project and you're currently in the vllm root directory.
24 -
25 -.. code-block:: console
26 -
27 - export vllm_root=`pwd`
28 -
29 -Create a file named ``Dockerfile.nginx``:
30 -
31 -.. code-block:: console
32 -
33 - FROM nginx:latest
34 - RUN rm /etc/nginx/conf.d/default.conf
35 - EXPOSE 80
36 - CMD ["nginx", "-g", "daemon off;"]
37 -
38 -Build the container:
39 -
40 -.. code-block:: console
41 -
42 - docker build . -f Dockerfile.nginx --tag nginx-lb
43 -
44 -.. _nginxloadbalancer_nginx_conf:
45 -
46 -Create Simple Nginx Config file
47 --------------------------------
48 -
49 -Create a file named ``nginx_conf/nginx.conf``. Note that you can add as many servers as you'd like. In the below example we'll start with two. To add more, add another ``server vllmN:8000 max_fails=3 fail_timeout=10000s;`` entry to ``upstream backend``.
50 -
51 -.. code-block:: console
52 -
53 - upstream backend {
54 - least_conn;
55 - server vllm0:8000 max_fails=3 fail_timeout=10000s;
56 - server vllm1:8000 max_fails=3 fail_timeout=10000s;
57 - }
58 - server {
59 - listen 80;
60 - location / {
61 - proxy_pass http://backend;
62 - proxy_set_header Host $host;
63 - proxy_set_header X-Real-IP $remote_addr;
64 - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
65 - proxy_set_header X-Forwarded-Proto $scheme;
66 - }
67 - }
68 -
69 -.. _nginxloadbalancer_nginx_vllm_container:
70 -
71 -Build vLLM Container
72 ---------------------
73 -
74 -.. code-block:: console
75 -
76 - cd $vllm_root
77 - docker build -f Dockerfile . --tag vllm
78 -
79 -
80 -If you are behind proxy, you can pass the proxy settings to the docker build command as shown below:
81 -
82 -.. code-block:: console
83 -
84 - cd $vllm_root
85 - docker build -f Dockerfile . --tag vllm --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy
86 -
87 -.. _nginxloadbalancer_nginx_docker_network:
88 -
89 -Create Docker Network
90 ----------------------
91 -
92 -.. code-block:: console
93 -
94 - docker network create vllm_nginx
95 -
96 -
97 -.. _nginxloadbalancer_nginx_launch_container:
98 -
99 -Launch vLLM Containers
100 -----------------------
101 -
102 -Notes:
103 -
104 -* If you have your HuggingFace models cached somewhere else, update ``hf_cache_dir`` below.
105 -* If you don't have an existing HuggingFace cache you will want to start ``vllm0`` and wait for the model to complete downloading and the server to be ready. This will ensure that ``vllm1`` can leverage the model you just downloaded and it won't have to be downloaded again.
106 -* The below example assumes GPU backend used. If you are using CPU backend, remove ``--gpus all``, add ``VLLM_CPU_KVCACHE_SPACE`` and ``VLLM_CPU_OMP_THREADS_BIND`` environment variables to the docker run command.
107 -* Adjust the model name that you want to use in your vLLM servers if you don't want to use ``Llama-2-7b-chat-hf``.
108 -
109 -.. code-block:: console
110 -
111 - mkdir -p ~/.cache/huggingface/hub/
112 - hf_cache_dir=~/.cache/huggingface/
113 - docker run -itd --ipc host --privileged --network vllm_nginx --gpus all --shm-size=10.24gb -v $hf_cache_dir:/root/.cache/huggingface/ -p 8081:8000 --name vllm0 vllm --model meta-llama/Llama-2-7b-chat-hf
114 - docker run -itd --ipc host --privileged --network vllm_nginx --gpus all --shm-size=10.24gb -v $hf_cache_dir:/root/.cache/huggingface/ -p 8082:8000 --name vllm1 vllm --model meta-llama/Llama-2-7b-chat-hf
115 -
116 -.. note::
117 - If you are behind proxy, you can pass the proxy settings to the docker run command via ``-e http_proxy=$http_proxy -e https_proxy=$https_proxy``.
118 -
119 -.. _nginxloadbalancer_nginx_launch_nginx:
120 -
121 -Launch Nginx
122 -------------
123 -
124 -.. code-block:: console
125 -
126 - docker run -itd -p 8000:80 --network vllm_nginx -v ./nginx_conf/:/etc/nginx/conf.d/ --name nginx-lb nginx-lb:latest
127 -
128 -.. _nginxloadbalancer_nginx_verify_nginx:
129 -
130 -Verify That vLLM Servers Are Ready
131 -----------------------------------
132 -
133 -.. code-block:: console
134 -
135 - docker logs vllm0 | grep Uvicorn
136 - docker logs vllm1 | grep Uvicorn
137 -
138 -Both outputs should look like this:
139 -
140 -.. code-block:: console
141 -
142 - INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
docs/source/serving/deploying_with_triton.md new
+5
@@ -0,0 +1,5 @@
1 +(deploying-with-triton)=
2 +
3 +# Deploying with NVIDIA Triton
4 +
5 +The [Triton Inference Server](https://github.com/triton-inference-server) hosts a tutorial demonstrating how to quickly deploy a simple [facebook/opt-125m](https://huggingface.co/facebook/opt-125m) model using vLLM. Please see [Deploying a vLLM model in Triton](https://github.com/triton-inference-server/tutorials/blob/main/Quick_Deploy/vLLM/README.md#deploying-a-vllm-model-in-triton) for more details.
docs/source/serving/deploying_with_triton.rst deleted
-6
@@ -1,6 +0,0 @@
1 -.. _deploying_with_triton:
2 -
3 -Deploying with NVIDIA Triton
4 -============================
5 -
6 -The `Triton Inference Server <https://github.com/triton-inference-server>`_ hosts a tutorial demonstrating how to quickly deploy a simple `facebook/opt-125m <https://huggingface.co/facebook/opt-125m>`_ model using vLLM. Please see `Deploying a vLLM model in Triton <https://github.com/triton-inference-server/tutorials/blob/main/Quick_Deploy/vLLM/README.md#deploying-a-vllm-model-in-triton>`_ for more details.
docs/source/serving/distributed_serving.md new
+105
@@ -0,0 +1,105 @@
1 +(distributed-serving)=
2 +
3 +# Distributed Inference and Serving
4 +
5 +## How to decide the distributed inference strategy?
6 +
7 +Before going into the details of distributed inference and serving, let's first make it clear when to use distributed inference and what are the strategies available. The common practice is:
8 +
9 +- **Single GPU (no distributed inference)**: If your model fits in a single GPU, you probably don't need to use distributed inference. Just use the single GPU to run the inference.
10 +- **Single-Node Multi-GPU (tensor parallel inference)**: If your model is too large to fit in a single GPU, but it can fit in a single node with multiple GPUs, you can use tensor parallelism. The tensor parallel size is the number of GPUs you want to use. For example, if you have 4 GPUs in a single node, you can set the tensor parallel size to 4.
11 +- **Multi-Node Multi-GPU (tensor parallel plus pipeline parallel inference)**: If your model is too large to fit in a single node, you can use tensor parallel together with pipeline parallelism. The tensor parallel size is the number of GPUs you want to use in each node, and the pipeline parallel size is the number of nodes you want to use. For example, if you have 16 GPUs in 2 nodes (8GPUs per node), you can set the tensor parallel size to 8 and the pipeline parallel size to 2.
12 +
13 +In short, you should increase the number of GPUs and the number of nodes until you have enough GPU memory to hold the model. The tensor parallel size should be the number of GPUs in each node, and the pipeline parallel size should be the number of nodes.
14 +
15 +After adding enough GPUs and nodes to hold the model, you can run vLLM first, which will print some logs like `# GPU blocks: 790`. Multiply the number by `16` (the block size), and you can get roughly the maximum number of tokens that can be served on the current configuration. If this number is not satisfying, e.g. you want higher throughput, you can further increase the number of GPUs or nodes, until the number of blocks is enough.
16 +
17 +```{note}
18 +There is one edge case: if the model fits in a single node with multiple GPUs, but the number of GPUs cannot divide the model size evenly, you can use pipeline parallelism, which splits the model along layers and supports uneven splits. In this case, the tensor parallel size should be 1 and the pipeline parallel size should be the number of GPUs.
19 +```
20 +
21 +## Details for Distributed Inference and Serving
22 +
23 +vLLM supports distributed tensor-parallel and pipeline-parallel inference and serving. Currently, we support [Megatron-LM's tensor parallel algorithm](https://arxiv.org/pdf/1909.08053.pdf). We manage the distributed runtime with either [Ray](https://github.com/ray-project/ray) or python native multiprocessing. Multiprocessing can be used when deploying on a single node, multi-node inferencing currently requires Ray.
24 +
25 +Multiprocessing will be used by default when not running in a Ray placement group and if there are sufficient GPUs available on the same node for the configured {code}`tensor_parallel_size`, otherwise Ray will be used. This default can be overridden via the {code}`LLM` class {code}`distributed-executor-backend` argument or {code}`--distributed-executor-backend` API server argument. Set it to {code}`mp` for multiprocessing or {code}`ray` for Ray. It's not required for Ray to be installed for the multiprocessing case.
26 +
27 +To run multi-GPU inference with the {code}`LLM` class, set the {code}`tensor_parallel_size` argument to the number of GPUs you want to use. For example, to run inference on 4 GPUs:
28 +
29 +```python
30 +from vllm import LLM
31 +llm = LLM("facebook/opt-13b", tensor_parallel_size=4)
32 +output = llm.generate("San Franciso is a")
33 +```
34 +
35 +To run multi-GPU serving, pass in the {code}`--tensor-parallel-size` argument when starting the server. For example, to run API server on 4 GPUs:
36 +
37 +```console
38 +$ vllm serve facebook/opt-13b \
39 +$ --tensor-parallel-size 4
40 +```
41 +
42 +You can also additionally specify {code}`--pipeline-parallel-size` to enable pipeline parallelism. For example, to run API server on 8 GPUs with pipeline parallelism and tensor parallelism:
43 +
44 +```console
45 +$ vllm serve gpt2 \
46 +$ --tensor-parallel-size 4 \
47 +$ --pipeline-parallel-size 2
48 +```
49 +
50 +## Multi-Node Inference and Serving
51 +
52 +If a single node does not have enough GPUs to hold the model, you can run the model using multiple nodes. It is important to make sure the execution environment is the same on all nodes, including the model path, the Python environment. The recommended way is to use docker images to ensure the same environment, and hide the heterogeneity of the host machines via mapping them into the same docker configuration.
53 +
54 +The first step, is to start containers and organize them into a cluster. We have provided a helper [script](https://github.com/vllm-project/vllm/tree/main/examples/run_cluster.sh) to start the cluster. Please note, this script launches docker without administrative privileges that would be required to access GPU performance counters when running profiling and tracing tools. For that purpose, the script can have `CAP_SYS_ADMIN` to the docker container by using the `--cap-add` option in the docker run command.
55 +
56 +Pick a node as the head node, and run the following command:
57 +
58 +```console
59 +$ bash run_cluster.sh \
60 +$ vllm/vllm-openai \
61 +$ ip_of_head_node \
62 +$ --head \
63 +$ /path/to/the/huggingface/home/in/this/node
64 +```
65 +
66 +On the rest of the worker nodes, run the following command:
67 +
68 +```console
69 +$ bash run_cluster.sh \
70 +$ vllm/vllm-openai \
71 +$ ip_of_head_node \
72 +$ --worker \
73 +$ /path/to/the/huggingface/home/in/this/node
74 +```
75 +
76 +Then you get a ray cluster of containers. Note that you need to keep the shells running these commands alive to hold the cluster. Any shell disconnect will terminate the cluster. In addition, please note that the argument `ip_of_head_node` should be the IP address of the head node, which is accessible by all the worker nodes. A common misunderstanding is to use the IP address of the worker node, which is not correct.
77 +
78 +Then, on any node, use `docker exec -it node /bin/bash` to enter the container, execute `ray status` to check the status of the Ray cluster. You should see the right number of nodes and GPUs.
79 +
80 +After that, on any node, you can use vLLM as usual, just as you have all the GPUs on one node. The common practice is to set the tensor parallel size to the number of GPUs in each node, and the pipeline parallel size to the number of nodes. For example, if you have 16 GPUs in 2 nodes (8GPUs per node), you can set the tensor parallel size to 8 and the pipeline parallel size to 2:
81 +
82 +```console
83 +$ vllm serve /path/to/the/model/in/the/container \
84 +$ --tensor-parallel-size 8 \
85 +$ --pipeline-parallel-size 2
86 +```
87 +
88 +You can also use tensor parallel without pipeline parallel, just set the tensor parallel size to the number of GPUs in the cluster. For example, if you have 16 GPUs in 2 nodes (8GPUs per node), you can set the tensor parallel size to 16:
89 +
90 +```console
91 +$ vllm serve /path/to/the/model/in/the/container \
92 +$ --tensor-parallel-size 16
93 +```
94 +
95 +To make tensor parallel performant, you should make sure the communication between nodes is efficient, e.g. using high-speed network cards like Infiniband. To correctly set up the cluster to use Infiniband, append additional arguments like `--privileged -e NCCL_IB_HCA=mlx5` to the `run_cluster.sh` script. Please contact your system administrator for more information on how to set up the flags. One way to confirm if the Infiniband is working is to run vLLM with `NCCL_DEBUG=TRACE` environment variable set, e.g. `NCCL_DEBUG=TRACE vllm serve ...` and check the logs for the NCCL version and the network used. If you find `[send] via NET/Socket` in the logs, it means NCCL uses raw TCP Socket, which is not efficient for cross-node tensor parallel. If you find `[send] via NET/IB/GDRDMA` in the logs, it means NCCL uses Infiniband with GPU-Direct RDMA, which is efficient.
96 +
97 +```{warning}
98 +After you start the Ray cluster, you'd better also check the GPU-GPU communication between nodes. It can be non-trivial to set up. Please refer to the [sanity check script](https://docs.vllm.ai/en/latest/getting_started/debugging.html) for more information. If you need to set some environment variables for the communication configuration, you can append them to the `run_cluster.sh` script, e.g. `-e NCCL_SOCKET_IFNAME=eth0`. Note that setting environment variables in the shell (e.g. `NCCL_SOCKET_IFNAME=eth0 vllm serve ...`) only works for the processes in the same node, not for the processes in the other nodes. Setting environment variables when you create the cluster is the recommended way. See the [discussion](https://github.com/vllm-project/vllm/issues/6803) for more information.
99 +```
100 +
101 +```{warning}
102 +Please make sure you downloaded the model to all the nodes (with the same path), or the model is downloaded to some distributed file system that is accessible by all nodes.
103 +
104 +When you use huggingface repo id to refer to the model, you should append your huggingface token to the `run_cluster.sh` script, e.g. `-e HF_TOKEN=`. The recommended way is to download the model first, and then use the path to refer to the model.
105 +```
docs/source/serving/distributed_serving.rst deleted
-107
@@ -1,107 +0,0 @@
1 -.. _distributed_serving:
2 -
3 -Distributed Inference and Serving
4 -=================================
5 -
6 -How to decide the distributed inference strategy?
7 --------------------------------------------------
8 -
9 -Before going into the details of distributed inference and serving, let's first make it clear when to use distributed inference and what are the strategies available. The common practice is:
10 -
11 -- **Single GPU (no distributed inference)**: If your model fits in a single GPU, you probably don't need to use distributed inference. Just use the single GPU to run the inference.
12 -- **Single-Node Multi-GPU (tensor parallel inference)**: If your model is too large to fit in a single GPU, but it can fit in a single node with multiple GPUs, you can use tensor parallelism. The tensor parallel size is the number of GPUs you want to use. For example, if you have 4 GPUs in a single node, you can set the tensor parallel size to 4.
13 -- **Multi-Node Multi-GPU (tensor parallel plus pipeline parallel inference)**: If your model is too large to fit in a single node, you can use tensor parallel together with pipeline parallelism. The tensor parallel size is the number of GPUs you want to use in each node, and the pipeline parallel size is the number of nodes you want to use. For example, if you have 16 GPUs in 2 nodes (8GPUs per node), you can set the tensor parallel size to 8 and the pipeline parallel size to 2.
14 -
15 -In short, you should increase the number of GPUs and the number of nodes until you have enough GPU memory to hold the model. The tensor parallel size should be the number of GPUs in each node, and the pipeline parallel size should be the number of nodes.
16 -
17 -After adding enough GPUs and nodes to hold the model, you can run vLLM first, which will print some logs like ``# GPU blocks: 790``. Multiply the number by ``16`` (the block size), and you can get roughly the maximum number of tokens that can be served on the current configuration. If this number is not satisfying, e.g. you want higher throughput, you can further increase the number of GPUs or nodes, until the number of blocks is enough.
18 -
19 -.. note::
20 - There is one edge case: if the model fits in a single node with multiple GPUs, but the number of GPUs cannot divide the model size evenly, you can use pipeline parallelism, which splits the model along layers and supports uneven splits. In this case, the tensor parallel size should be 1 and the pipeline parallel size should be the number of GPUs.
21 -
22 -Details for Distributed Inference and Serving
23 -----------------------------------------------
24 -
25 -vLLM supports distributed tensor-parallel and pipeline-parallel inference and serving. Currently, we support `Megatron-LM's tensor parallel algorithm <https://arxiv.org/pdf/1909.08053.pdf>`_. We manage the distributed runtime with either `Ray <https://github.com/ray-project/ray>`_ or python native multiprocessing. Multiprocessing can be used when deploying on a single node, multi-node inferencing currently requires Ray.
26 -
27 -Multiprocessing will be used by default when not running in a Ray placement group and if there are sufficient GPUs available on the same node for the configured :code:`tensor_parallel_size`, otherwise Ray will be used. This default can be overridden via the :code:`LLM` class :code:`distributed-executor-backend` argument or :code:`--distributed-executor-backend` API server argument. Set it to :code:`mp` for multiprocessing or :code:`ray` for Ray. It's not required for Ray to be installed for the multiprocessing case.
28 -
29 -To run multi-GPU inference with the :code:`LLM` class, set the :code:`tensor_parallel_size` argument to the number of GPUs you want to use. For example, to run inference on 4 GPUs:
30 -
31 -.. code-block:: python
32 -
33 - from vllm import LLM
34 - llm = LLM("facebook/opt-13b", tensor_parallel_size=4)
35 - output = llm.generate("San Franciso is a")
36 -
37 -To run multi-GPU serving, pass in the :code:`--tensor-parallel-size` argument when starting the server. For example, to run API server on 4 GPUs:
38 -
39 -.. code-block:: console
40 -
41 - $ vllm serve facebook/opt-13b \
42 - $ --tensor-parallel-size 4
43 -
44 -You can also additionally specify :code:`--pipeline-parallel-size` to enable pipeline parallelism. For example, to run API server on 8 GPUs with pipeline parallelism and tensor parallelism:
45 -
46 -.. code-block:: console
47 -
48 - $ vllm serve gpt2 \
49 - $ --tensor-parallel-size 4 \
50 - $ --pipeline-parallel-size 2
51 -
52 -Multi-Node Inference and Serving
53 ---------------------------------
54 -
55 -If a single node does not have enough GPUs to hold the model, you can run the model using multiple nodes. It is important to make sure the execution environment is the same on all nodes, including the model path, the Python environment. The recommended way is to use docker images to ensure the same environment, and hide the heterogeneity of the host machines via mapping them into the same docker configuration.
56 -
57 -The first step, is to start containers and organize them into a cluster. We have provided a helper `script <https://github.com/vllm-project/vllm/tree/main/examples/run_cluster.sh>`_ to start the cluster. Please note, this script launches docker without administrative privileges that would be required to access GPU performance counters when running profiling and tracing tools. For that purpose, the script can have ``CAP_SYS_ADMIN`` to the docker container by using the ``--cap-add`` option in the docker run command.
58 -
59 -Pick a node as the head node, and run the following command:
60 -
61 -.. code-block:: console
62 -
63 - $ bash run_cluster.sh \
64 - $ vllm/vllm-openai \
65 - $ ip_of_head_node \
66 - $ --head \
67 - $ /path/to/the/huggingface/home/in/this/node
68 -
69 -On the rest of the worker nodes, run the following command:
70 -
71 -.. code-block:: console
72 -
73 - $ bash run_cluster.sh \
74 - $ vllm/vllm-openai \
75 - $ ip_of_head_node \
76 - $ --worker \
77 - $ /path/to/the/huggingface/home/in/this/node
78 -
79 -Then you get a ray cluster of containers. Note that you need to keep the shells running these commands alive to hold the cluster. Any shell disconnect will terminate the cluster. In addition, please note that the argument ``ip_of_head_node`` should be the IP address of the head node, which is accessible by all the worker nodes. A common misunderstanding is to use the IP address of the worker node, which is not correct.
80 -
81 -Then, on any node, use ``docker exec -it node /bin/bash`` to enter the container, execute ``ray status`` to check the status of the Ray cluster. You should see the right number of nodes and GPUs.
82 -
83 -After that, on any node, you can use vLLM as usual, just as you have all the GPUs on one node. The common practice is to set the tensor parallel size to the number of GPUs in each node, and the pipeline parallel size to the number of nodes. For example, if you have 16 GPUs in 2 nodes (8GPUs per node), you can set the tensor parallel size to 8 and the pipeline parallel size to 2:
84 -
85 -.. code-block:: console
86 -
87 - $ vllm serve /path/to/the/model/in/the/container \
88 - $ --tensor-parallel-size 8 \
89 - $ --pipeline-parallel-size 2
90 -
91 -You can also use tensor parallel without pipeline parallel, just set the tensor parallel size to the number of GPUs in the cluster. For example, if you have 16 GPUs in 2 nodes (8GPUs per node), you can set the tensor parallel size to 16:
92 -
93 -.. code-block:: console
94 -
95 - $ vllm serve /path/to/the/model/in/the/container \
96 - $ --tensor-parallel-size 16
97 -
98 -To make tensor parallel performant, you should make sure the communication between nodes is efficient, e.g. using high-speed network cards like Infiniband. To correctly set up the cluster to use Infiniband, append additional arguments like ``--privileged -e NCCL_IB_HCA=mlx5`` to the ``run_cluster.sh`` script. Please contact your system administrator for more information on how to set up the flags. One way to confirm if the Infiniband is working is to run vLLM with ``NCCL_DEBUG=TRACE`` environment variable set, e.g. ``NCCL_DEBUG=TRACE vllm serve ...`` and check the logs for the NCCL version and the network used. If you find ``[send] via NET/Socket`` in the logs, it means NCCL uses raw TCP Socket, which is not efficient for cross-node tensor parallel. If you find ``[send] via NET/IB/GDRDMA`` in the logs, it means NCCL uses Infiniband with GPU-Direct RDMA, which is efficient.
99 -
100 -.. warning::
101 - After you start the Ray cluster, you'd better also check the GPU-GPU communication between nodes. It can be non-trivial to set up. Please refer to the `sanity check script <https://docs.vllm.ai/en/latest/getting_started/debugging.html>`_ for more information. If you need to set some environment variables for the communication configuration, you can append them to the ``run_cluster.sh`` script, e.g. ``-e NCCL_SOCKET_IFNAME=eth0``. Note that setting environment variables in the shell (e.g. ``NCCL_SOCKET_IFNAME=eth0 vllm serve ...``) only works for the processes in the same node, not for the processes in the other nodes. Setting environment variables when you create the cluster is the recommended way. See the `discussion <https://github.com/vllm-project/vllm/issues/6803>`_ for more information.
102 -
103 -.. warning::
104 -
105 - Please make sure you downloaded the model to all the nodes (with the same path), or the model is downloaded to some distributed file system that is accessible by all nodes.
106 -
107 - When you use huggingface repo id to refer to the model, you should append your huggingface token to the ``run_cluster.sh`` script, e.g. ``-e HF_TOKEN=``. The recommended way is to download the model first, and then use the path to refer to the model.
docs/source/serving/integrations.md new
+17
@@ -0,0 +1,17 @@
1 +# Integrations
2 +
3 +```{toctree}
4 +:maxdepth: 1
5 +
6 +run_on_sky
7 +deploying_with_kserve
8 +deploying_with_kubeai
9 +deploying_with_triton
10 +deploying_with_bentoml
11 +deploying_with_cerebrium
12 +deploying_with_lws
13 +deploying_with_dstack
14 +serving_with_langchain
15 +serving_with_llamaindex
16 +serving_with_llamastack
17 +```
docs/source/serving/integrations.rst deleted
-17
@@ -1,17 +0,0 @@
1 -Integrations
2 -------------
3 -
4 -.. toctree::
5 - :maxdepth: 1
6 -
7 - run_on_sky
8 - deploying_with_kserve
9 - deploying_with_kubeai
10 - deploying_with_triton
11 - deploying_with_bentoml
12 - deploying_with_cerebrium
13 - deploying_with_lws
14 - deploying_with_dstack
15 - serving_with_langchain
16 - serving_with_llamaindex
17 - serving_with_llamastack
docs/source/serving/metrics.md new
+38
@@ -0,0 +1,38 @@
1 +# Production Metrics
2 +
3 +vLLM exposes a number of metrics that can be used to monitor the health of the
4 +system. These metrics are exposed via the `/metrics` endpoint on the vLLM
5 +OpenAI compatible API server.
6 +
7 +You can start the server using Python, or using [Docker](deploying_with_docker.md):
8 +
9 +```console
10 +$ vllm serve unsloth/Llama-3.2-1B-Instruct
11 +```
12 +
13 +Then query the endpoint to get the latest metrics from the server:
14 +
15 +```console
16 +$ curl http://0.0.0.0:8000/metrics
17 +
18 +# HELP vllm:iteration_tokens_total Histogram of number of tokens per engine_step.
19 +# TYPE vllm:iteration_tokens_total histogram
20 +vllm:iteration_tokens_total_sum{model_name="unsloth/Llama-3.2-1B-Instruct"} 0.0
21 +vllm:iteration_tokens_total_bucket{le="1.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
22 +vllm:iteration_tokens_total_bucket{le="8.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
23 +vllm:iteration_tokens_total_bucket{le="16.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
24 +vllm:iteration_tokens_total_bucket{le="32.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
25 +vllm:iteration_tokens_total_bucket{le="64.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
26 +vllm:iteration_tokens_total_bucket{le="128.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
27 +vllm:iteration_tokens_total_bucket{le="256.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
28 +vllm:iteration_tokens_total_bucket{le="512.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
29 +...
30 +```
31 +
32 +The following metrics are exposed:
33 +
34 +```{literalinclude} ../../../vllm/engine/metrics.py
35 +:end-before: end-metrics-definitions
36 +:language: python
37 +:start-after: begin-metrics-definitions
38 +```
docs/source/serving/metrics.rst deleted
-38
@@ -1,38 +0,0 @@
1 -Production Metrics
2 -==================
3 -
4 -vLLM exposes a number of metrics that can be used to monitor the health of the
5 -system. These metrics are exposed via the ``/metrics`` endpoint on the vLLM
6 -OpenAI compatible API server.
7 -
8 -You can start the server using Python, or using [Docker](deploying_with_docker.rst):
9 -
10 -.. code-block:: console
11 -
12 - $ vllm serve unsloth/Llama-3.2-1B-Instruct
13 -
14 -Then query the endpoint to get the latest metrics from the server:
15 -
16 -.. code-block:: console
17 -
18 - $ curl http://0.0.0.0:8000/metrics
19 -
20 - # HELP vllm:iteration_tokens_total Histogram of number of tokens per engine_step.
21 - # TYPE vllm:iteration_tokens_total histogram
22 - vllm:iteration_tokens_total_sum{model_name="unsloth/Llama-3.2-1B-Instruct"} 0.0
23 - vllm:iteration_tokens_total_bucket{le="1.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
24 - vllm:iteration_tokens_total_bucket{le="8.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
25 - vllm:iteration_tokens_total_bucket{le="16.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
26 - vllm:iteration_tokens_total_bucket{le="32.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
27 - vllm:iteration_tokens_total_bucket{le="64.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
28 - vllm:iteration_tokens_total_bucket{le="128.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
29 - vllm:iteration_tokens_total_bucket{le="256.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
30 - vllm:iteration_tokens_total_bucket{le="512.0",model_name="unsloth/Llama-3.2-1B-Instruct"} 3.0
31 - ...
32 -
33 -The following metrics are exposed:
34 -
35 -.. literalinclude:: ../../../vllm/engine/metrics.py
36 - :language: python
37 - :start-after: begin-metrics-definitions
38 - :end-before: end-metrics-definitions
docs/source/serving/openai_compatible_server.md
+11 -11
@@ -2,7 +2,7 @@
2
3 vLLM provides an HTTP server that implements OpenAI's [Completions](https://platform.openai.com/docs/api-reference/completions) and [Chat](https://platform.openai.com/docs/api-reference/chat) API, and more!
4
5 -You can start the server via the [`vllm serve`](#vllm-serve) command, or through [Docker](deploying_with_docker.rst):
5 +You can start the server via the [`vllm serve`](#vllm-serve) command, or through [Docker](deploying_with_docker.md):
6 ```bash
7 vllm serve NousResearch/Meta-Llama-3-8B-Instruct --dtype auto --api-key token-abc123
8 ```
@@ -30,20 +30,20 @@ print(completion.choices[0].message)
30 We currently support the following OpenAI APIs:
31
32 - [Completions API](#completions-api) (`/v1/completions`)
33 - - Only applicable to [text generation models](../models/generative_models.rst) (`--task generate`).
33 + - Only applicable to [text generation models](../models/generative_models.md) (`--task generate`).
34 - *Note: `suffix` parameter is not supported.*
35 - [Chat Completions API](#chat-api) (`/v1/chat/completions`)
36 - - Only applicable to [text generation models](../models/generative_models.rst) (`--task generate`) with a [chat template](#chat-template).
36 + - Only applicable to [text generation models](../models/generative_models.md) (`--task generate`) with a [chat template](#chat-template).
37 - *Note: `parallel_tool_calls` and `user` parameters are ignored.*
38 - [Embeddings API](#embeddings-api) (`/v1/embeddings`)
39 - - Only applicable to [embedding models](../models/pooling_models.rst) (`--task embed`).
39 + - Only applicable to [embedding models](../models/pooling_models.md) (`--task embed`).
40
41 In addition, we have the following custom APIs:
42
43 - [Tokenizer API](#tokenizer-api) (`/tokenize`, `/detokenize`)
44 - Applicable to any model with a tokenizer.
45 - [Score API](#score-api) (`/score`)
46 - - Only applicable to [cross-encoder models](../models/pooling_models.rst) (`--task score`).
46 + - Only applicable to [cross-encoder models](../models/pooling_models.md) (`--task score`).
47
48 (chat-template)=
49 ## Chat Template
@@ -183,7 +183,7 @@ Refer to [OpenAI's API reference](https://platform.openai.com/docs/api-reference
183
184 #### Extra parameters
185
186 -The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
186 +The following [sampling parameters (click through to see documentation)](../dev/sampling_params.md) are supported.
187
188 ```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
189 :language: python
@@ -206,12 +206,12 @@ Refer to [OpenAI's API reference](https://platform.openai.com/docs/api-reference
206
207 We support both [Vision](https://platform.openai.com/docs/guides/vision)- and
208 [Audio](https://platform.openai.com/docs/guides/audio?audio-generation-quickstart-example=audio-in)-related parameters;
209 -see our [Multimodal Inputs](../usage/multimodal_inputs.rst) guide for more information.
209 +see our [Multimodal Inputs](../usage/multimodal_inputs.md) guide for more information.
210 - *Note: `image_url.detail` parameter is not supported.*
211
212 #### Extra parameters
213
214 -The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
214 +The following [sampling parameters (click through to see documentation)](../dev/sampling_params.md) are supported.
215
216 ```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
217 :language: python
@@ -236,12 +236,12 @@ If the model has a [chat template](#chat-template), you can replace `inputs` wit
236 which will be treated as a single prompt to the model.
237
238 ```{tip}
239 -This enables multi-modal inputs to be passed to embedding models, see [this page](../usage/multimodal_inputs.rst) for details.
239 +This enables multi-modal inputs to be passed to embedding models, see [this page](../usage/multimodal_inputs.md) for details.
240 ```
241
242 #### Extra parameters
243
244 -The following [pooling parameters (click through to see documentation)](../dev/pooling_params.rst) are supported.
244 +The following [pooling parameters (click through to see documentation)](../dev/pooling_params.md) are supported.
245
246 ```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
247 :language: python
@@ -418,7 +418,7 @@ Response:
418
419 #### Extra parameters
420
421 -The following [pooling parameters (click through to see documentation)](../dev/pooling_params.rst) are supported.
421 +The following [pooling parameters (click through to see documentation)](../dev/pooling_params.md) are supported.
422
423 ```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
424 :language: python
docs/source/serving/run_on_sky.md new
+345
@@ -0,0 +1,345 @@
1 +(on-cloud)=
2 +
3 +# Deploying and scaling up with SkyPilot
4 +
5 +```{raw} html
6 +<p align="center">
7 + <img src="https://imgur.com/yxtzPEu.png" alt="vLLM"/>
8 +</p>
9 +```
10 +
11 +vLLM can be **run and scaled to multiple service replicas on clouds and Kubernetes** with [SkyPilot](https://github.com/skypilot-org/skypilot), an open-source framework for running LLMs on any cloud. More examples for various open models, such as Llama-3, Mixtral, etc, can be found in [SkyPilot AI gallery](https://skypilot.readthedocs.io/en/latest/gallery/index.html).
12 +
13 +## Prerequisites
14 +
15 +- Go to the [HuggingFace model page](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) and request access to the model {code}`meta-llama/Meta-Llama-3-8B-Instruct`.
16 +- Check that you have installed SkyPilot ([docs](https://skypilot.readthedocs.io/en/latest/getting-started/installation.html)).
17 +- Check that {code}`sky check` shows clouds or Kubernetes are enabled.
18 +
19 +```console
20 +pip install skypilot-nightly
21 +sky check
22 +```
23 +
24 +## Run on a single instance
25 +
26 +See the vLLM SkyPilot YAML for serving, [serving.yaml](https://github.com/skypilot-org/skypilot/blob/master/llm/vllm/serve.yaml).
27 +
28 +```yaml
29 +resources:
30 + accelerators: {L4, A10g, A10, L40, A40, A100, A100-80GB} # We can use cheaper accelerators for 8B model.
31 + use_spot: True
32 + disk_size: 512 # Ensure model checkpoints can fit.
33 + disk_tier: best
34 + ports: 8081 # Expose to internet traffic.
35 +
36 +envs:
37 + MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct
38 + HF_TOKEN: <your-huggingface-token> # Change to your own huggingface token, or use --env to pass.
39 +
40 +setup: |
41 + conda create -n vllm python=3.10 -y
42 + conda activate vllm
43 +
44 + pip install vllm==0.4.0.post1
45 + # Install Gradio for web UI.
46 + pip install gradio openai
47 + pip install flash-attn==2.5.7
48 +
49 +run: |
50 + conda activate vllm
51 + echo 'Starting vllm api server...'
52 + python -u -m vllm.entrypoints.openai.api_server \
53 + --port 8081 \
54 + --model $MODEL_NAME \
55 + --trust-remote-code \
56 + --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \
57 + 2>&1 | tee api_server.log &
58 +
59 + echo 'Waiting for vllm api server to start...'
60 + while ! `cat api_server.log | grep -q 'Uvicorn running on'`; do sleep 1; done
61 +
62 + echo 'Starting gradio server...'
63 + git clone https://github.com/vllm-project/vllm.git || true
64 + python vllm/examples/gradio_openai_chatbot_webserver.py \
65 + -m $MODEL_NAME \
66 + --port 8811 \
67 + --model-url http://localhost:8081/v1 \
68 + --stop-token-ids 128009,128001
69 +```
70 +
71 +Start the serving the Llama-3 8B model on any of the candidate GPUs listed (L4, A10g, ...):
72 +
73 +```console
74 +HF_TOKEN="your-huggingface-token" sky launch serving.yaml --env HF_TOKEN
75 +```
76 +
77 +Check the output of the command. There will be a shareable gradio link (like the last line of the following). Open it in your browser to use the LLaMA model to do the text completion.
78 +
79 +```console
80 +(task, pid=7431) Running on public URL: https://<gradio-hash>.gradio.live
81 +```
82 +
83 +**Optional**: Serve the 70B model instead of the default 8B and use more GPU:
84 +
85 +```console
86 +HF_TOKEN="your-huggingface-token" sky launch serving.yaml --gpus A100:8 --env HF_TOKEN --env MODEL_NAME=meta-llama/Meta-Llama-3-70B-Instruct
87 +```
88 +
89 +## Scale up to multiple replicas
90 +
91 +SkyPilot can scale up the service to multiple service replicas with built-in autoscaling, load-balancing and fault-tolerance. You can do it by adding a services section to the YAML file.
92 +
93 +```yaml
94 +service:
95 + replicas: 2
96 + # An actual request for readiness probe.
97 + readiness_probe:
98 + path: /v1/chat/completions
99 + post_data:
100 + model: $MODEL_NAME
101 + messages:
102 + - role: user
103 + content: Hello! What is your name?
104 + max_completion_tokens: 1
105 +```
106 +
107 +```{raw} html
108 +<details>
109 +<summary>Click to see the full recipe YAML</summary>
110 +```
111 +
112 +```yaml
113 +service:
114 + replicas: 2
115 + # An actual request for readiness probe.
116 + readiness_probe:
117 + path: /v1/chat/completions
118 + post_data:
119 + model: $MODEL_NAME
120 + messages:
121 + - role: user
122 + content: Hello! What is your name?
123 + max_completion_tokens: 1
124 +
125 +resources:
126 + accelerators: {L4, A10g, A10, L40, A40, A100, A100-80GB} # We can use cheaper accelerators for 8B model.
127 + use_spot: True
128 + disk_size: 512 # Ensure model checkpoints can fit.
129 + disk_tier: best
130 + ports: 8081 # Expose to internet traffic.
131 +
132 +envs:
133 + MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct
134 + HF_TOKEN: <your-huggingface-token> # Change to your own huggingface token, or use --env to pass.
135 +
136 +setup: |
137 + conda create -n vllm python=3.10 -y
138 + conda activate vllm
139 +
140 + pip install vllm==0.4.0.post1
141 + # Install Gradio for web UI.
142 + pip install gradio openai
143 + pip install flash-attn==2.5.7
144 +
145 +run: |
146 + conda activate vllm
147 + echo 'Starting vllm api server...'
148 + python -u -m vllm.entrypoints.openai.api_server \
149 + --port 8081 \
150 + --model $MODEL_NAME \
151 + --trust-remote-code \
152 + --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \
153 + 2>&1 | tee api_server.log
154 +```
155 +
156 +```{raw} html
157 +</details>
158 +```
159 +
160 +Start the serving the Llama-3 8B model on multiple replicas:
161 +
162 +```console
163 +HF_TOKEN="your-huggingface-token" sky serve up -n vllm serving.yaml --env HF_TOKEN
164 +```
165 +
166 +Wait until the service is ready:
167 +
168 +```console
169 +watch -n10 sky serve status vllm
170 +```
171 +
172 +```{raw} html
173 +<details>
174 +<summary>Example outputs:</summary>
175 +```
176 +
177 +```console
178 +Services
179 +NAME VERSION UPTIME STATUS REPLICAS ENDPOINT
180 +vllm 1 35s READY 2/2 xx.yy.zz.100:30001
181 +
182 +Service Replicas
183 +SERVICE_NAME ID VERSION IP LAUNCHED RESOURCES STATUS REGION
184 +vllm 1 1 xx.yy.zz.121 18 mins ago 1x GCP([Spot]{'L4': 1}) READY us-east4
185 +vllm 2 1 xx.yy.zz.245 18 mins ago 1x GCP([Spot]{'L4': 1}) READY us-east4
186 +```
187 +
188 +```{raw} html
189 +</details>
190 +```
191 +
192 +After the service is READY, you can find a single endpoint for the service and access the service with the endpoint:
193 +
194 +```console
195 +ENDPOINT=$(sky serve status --endpoint 8081 vllm)
196 +curl -L http://$ENDPOINT/v1/chat/completions \
197 + -H "Content-Type: application/json" \
198 + -d '{
199 + "model": "meta-llama/Meta-Llama-3-8B-Instruct",
200 + "messages": [
201 + {
202 + "role": "system",
203 + "content": "You are a helpful assistant."
204 + },
205 + {
206 + "role": "user",
207 + "content": "Who are you?"
208 + }
209 + ],
210 + "stop_token_ids": [128009, 128001]
211 + }'
212 +```
213 +
214 +To enable autoscaling, you could replace the `replicas` with the following configs in `service`:
215 +
216 +```yaml
217 +service:
218 + replica_policy:
219 + min_replicas: 2
220 + max_replicas: 4
221 + target_qps_per_replica: 2
222 +```
223 +
224 +This will scale the service up to when the QPS exceeds 2 for each replica.
225 +
226 +```{raw} html
227 +<details>
228 +<summary>Click to see the full recipe YAML</summary>
229 +```
230 +
231 +```yaml
232 +service:
233 + replica_policy:
234 + min_replicas: 2
235 + max_replicas: 4
236 + target_qps_per_replica: 2
237 + # An actual request for readiness probe.
238 + readiness_probe:
239 + path: /v1/chat/completions
240 + post_data:
241 + model: $MODEL_NAME
242 + messages:
243 + - role: user
244 + content: Hello! What is your name?
245 + max_completion_tokens: 1
246 +
247 +resources:
248 + accelerators: {L4, A10g, A10, L40, A40, A100, A100-80GB} # We can use cheaper accelerators for 8B model.
249 + use_spot: True
250 + disk_size: 512 # Ensure model checkpoints can fit.
251 + disk_tier: best
252 + ports: 8081 # Expose to internet traffic.
253 +
254 +envs:
255 + MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct
256 + HF_TOKEN: <your-huggingface-token> # Change to your own huggingface token, or use --env to pass.
257 +
258 +setup: |
259 + conda create -n vllm python=3.10 -y
260 + conda activate vllm
261 +
262 + pip install vllm==0.4.0.post1
263 + # Install Gradio for web UI.
264 + pip install gradio openai
265 + pip install flash-attn==2.5.7
266 +
267 +run: |
268 + conda activate vllm
269 + echo 'Starting vllm api server...'
270 + python -u -m vllm.entrypoints.openai.api_server \
271 + --port 8081 \
272 + --model $MODEL_NAME \
273 + --trust-remote-code \
274 + --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \
275 + 2>&1 | tee api_server.log
276 +```
277 +
278 +```{raw} html
279 +</details>
280 +```
281 +
282 +To update the service with the new config:
283 +
284 +```console
285 +HF_TOKEN="your-huggingface-token" sky serve update vllm serving.yaml --env HF_TOKEN
286 +```
287 +
288 +To stop the service:
289 +
290 +```console
291 +sky serve down vllm
292 +```
293 +
294 +### **Optional**: Connect a GUI to the endpoint
295 +
296 +It is also possible to access the Llama-3 service with a separate GUI frontend, so the user requests send to the GUI will be load-balanced across replicas.
297 +
298 +```{raw} html
299 +<details>
300 +<summary>Click to see the full GUI YAML</summary>
301 +```
302 +
303 +```yaml
304 +envs:
305 + MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct
306 + ENDPOINT: x.x.x.x:3031 # Address of the API server running vllm.
307 +
308 +resources:
309 + cpus: 2
310 +
311 +setup: |
312 + conda create -n vllm python=3.10 -y
313 + conda activate vllm
314 +
315 + # Install Gradio for web UI.
316 + pip install gradio openai
317 +
318 +run: |
319 + conda activate vllm
320 + export PATH=$PATH:/sbin
321 +
322 + echo 'Starting gradio server...'
323 + git clone https://github.com/vllm-project/vllm.git || true
324 + python vllm/examples/gradio_openai_chatbot_webserver.py \
325 + -m $MODEL_NAME \
326 + --port 8811 \
327 + --model-url http://$ENDPOINT/v1 \
328 + --stop-token-ids 128009,128001 | tee ~/gradio.log
329 +```
330 +
331 +```{raw} html
332 +</details>
333 +```
334 +
335 +1. Start the chat web UI:
336 +
337 +```console
338 +sky launch -c gui ./gui.yaml --env ENDPOINT=$(sky serve status --endpoint vllm)
339 +```
340 +
341 +2. Then, we can access the GUI at the returned gradio link:
342 +
343 +```console
344 +| INFO | stdout | Running on public URL: https://6141e84201ce0bb4ed.gradio.live
345 +```
docs/source/serving/run_on_sky.rst deleted
-366
@@ -1,366 +0,0 @@
1 -.. _on_cloud:
2 -
3 -Deploying and scaling up with SkyPilot
4 -================================================
5 -
6 -.. raw:: html
7 -
8 - <p align="center">
9 - <img src="https://imgur.com/yxtzPEu.png" alt="vLLM"/>
10 - </p>
11 -
12 -vLLM can be **run and scaled to multiple service replicas on clouds and Kubernetes** with `SkyPilot <https://github.com/skypilot-org/skypilot>`__, an open-source framework for running LLMs on any cloud. More examples for various open models, such as Llama-3, Mixtral, etc, can be found in `SkyPilot AI gallery <https://skypilot.readthedocs.io/en/latest/gallery/index.html>`__.
13 -
14 -
15 -Prerequisites
16 --------------
17 -
18 -- Go to the `HuggingFace model page <https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct>`__ and request access to the model :code:`meta-llama/Meta-Llama-3-8B-Instruct`.
19 -- Check that you have installed SkyPilot (`docs <https://skypilot.readthedocs.io/en/latest/getting-started/installation.html>`__).
20 -- Check that :code:`sky check` shows clouds or Kubernetes are enabled.
21 -
22 -.. code-block:: console
23 -
24 - pip install skypilot-nightly
25 - sky check
26 -
27 -
28 -Run on a single instance
29 -------------------------
30 -
31 -See the vLLM SkyPilot YAML for serving, `serving.yaml <https://github.com/skypilot-org/skypilot/blob/master/llm/vllm/serve.yaml>`__.
32 -
33 -.. code-block:: yaml
34 -
35 - resources:
36 - accelerators: {L4, A10g, A10, L40, A40, A100, A100-80GB} # We can use cheaper accelerators for 8B model.
37 - use_spot: True
38 - disk_size: 512 # Ensure model checkpoints can fit.
39 - disk_tier: best
40 - ports: 8081 # Expose to internet traffic.
41 -
42 - envs:
43 - MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct
44 - HF_TOKEN: <your-huggingface-token> # Change to your own huggingface token, or use --env to pass.
45 -
46 - setup: |
47 - conda create -n vllm python=3.10 -y
48 - conda activate vllm
49 -
50 - pip install vllm==0.4.0.post1
51 - # Install Gradio for web UI.
52 - pip install gradio openai
53 - pip install flash-attn==2.5.7
54 -
55 - run: |
56 - conda activate vllm
57 - echo 'Starting vllm api server...'
58 - python -u -m vllm.entrypoints.openai.api_server \
59 - --port 8081 \
60 - --model $MODEL_NAME \
61 - --trust-remote-code \
62 - --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \
63 - 2>&1 | tee api_server.log &
64 -
65 - echo 'Waiting for vllm api server to start...'
66 - while ! `cat api_server.log | grep -q 'Uvicorn running on'`; do sleep 1; done
67 -
68 - echo 'Starting gradio server...'
69 - git clone https://github.com/vllm-project/vllm.git || true
70 - python vllm/examples/gradio_openai_chatbot_webserver.py \
71 - -m $MODEL_NAME \
72 - --port 8811 \
73 - --model-url http://localhost:8081/v1 \
74 - --stop-token-ids 128009,128001
75 -
76 -Start the serving the Llama-3 8B model on any of the candidate GPUs listed (L4, A10g, ...):
77 -
78 -.. code-block:: console
79 -
80 - HF_TOKEN="your-huggingface-token" sky launch serving.yaml --env HF_TOKEN
81 -
82 -Check the output of the command. There will be a shareable gradio link (like the last line of the following). Open it in your browser to use the LLaMA model to do the text completion.
83 -
84 -.. code-block:: console
85 -
86 - (task, pid=7431) Running on public URL: https://<gradio-hash>.gradio.live
87 -
88 -**Optional**: Serve the 70B model instead of the default 8B and use more GPU:
89 -
90 -.. code-block:: console
91 -
92 - HF_TOKEN="your-huggingface-token" sky launch serving.yaml --gpus A100:8 --env HF_TOKEN --env MODEL_NAME=meta-llama/Meta-Llama-3-70B-Instruct
93 -
94 -
95 -Scale up to multiple replicas
96 ------------------------------
97 -
98 -SkyPilot can scale up the service to multiple service replicas with built-in autoscaling, load-balancing and fault-tolerance. You can do it by adding a services section to the YAML file.
99 -
100 -.. code-block:: yaml
101 -
102 - service:
103 - replicas: 2
104 - # An actual request for readiness probe.
105 - readiness_probe:
106 - path: /v1/chat/completions
107 - post_data:
108 - model: $MODEL_NAME
109 - messages:
110 - - role: user
111 - content: Hello! What is your name?
112 - max_completion_tokens: 1
113 -
114 -.. raw:: html
115 -
116 - <details>
117 - <summary>Click to see the full recipe YAML</summary>
118 -
119 -
120 -.. code-block:: yaml
121 -
122 - service:
123 - replicas: 2
124 - # An actual request for readiness probe.
125 - readiness_probe:
126 - path: /v1/chat/completions
127 - post_data:
128 - model: $MODEL_NAME
129 - messages:
130 - - role: user
131 - content: Hello! What is your name?
132 - max_completion_tokens: 1
133 -
134 - resources:
135 - accelerators: {L4, A10g, A10, L40, A40, A100, A100-80GB} # We can use cheaper accelerators for 8B model.
136 - use_spot: True
137 - disk_size: 512 # Ensure model checkpoints can fit.
138 - disk_tier: best
139 - ports: 8081 # Expose to internet traffic.
140 -
141 - envs:
142 - MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct
143 - HF_TOKEN: <your-huggingface-token> # Change to your own huggingface token, or use --env to pass.
144 -
145 - setup: |
146 - conda create -n vllm python=3.10 -y
147 - conda activate vllm
148 -
149 - pip install vllm==0.4.0.post1
150 - # Install Gradio for web UI.
151 - pip install gradio openai
152 - pip install flash-attn==2.5.7
153 -
154 - run: |
155 - conda activate vllm
156 - echo 'Starting vllm api server...'
157 - python -u -m vllm.entrypoints.openai.api_server \
158 - --port 8081 \
159 - --model $MODEL_NAME \
160 - --trust-remote-code \
161 - --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \
162 - 2>&1 | tee api_server.log
163 -
164 -.. raw:: html
165 -
166 - </details>
167 -
168 -Start the serving the Llama-3 8B model on multiple replicas:
169 -
170 -.. code-block:: console
171 -
172 - HF_TOKEN="your-huggingface-token" sky serve up -n vllm serving.yaml --env HF_TOKEN
173 -
174 -
175 -Wait until the service is ready:
176 -
177 -.. code-block:: console
178 -
179 - watch -n10 sky serve status vllm
180 -
181 -
182 -.. raw:: html
183 -
184 - <details>
185 - <summary>Example outputs:</summary>
186 -
187 -.. code-block:: console
188 -
189 - Services
190 - NAME VERSION UPTIME STATUS REPLICAS ENDPOINT
191 - vllm 1 35s READY 2/2 xx.yy.zz.100:30001
192 -
193 - Service Replicas
194 - SERVICE_NAME ID VERSION IP LAUNCHED RESOURCES STATUS REGION
195 - vllm 1 1 xx.yy.zz.121 18 mins ago 1x GCP([Spot]{'L4': 1}) READY us-east4
196 - vllm 2 1 xx.yy.zz.245 18 mins ago 1x GCP([Spot]{'L4': 1}) READY us-east4
197 -
198 -.. raw:: html
199 -
200 - </details>
201 -
202 -After the service is READY, you can find a single endpoint for the service and access the service with the endpoint:
203 -
204 -.. code-block:: console
205 -
206 - ENDPOINT=$(sky serve status --endpoint 8081 vllm)
207 - curl -L http://$ENDPOINT/v1/chat/completions \
208 - -H "Content-Type: application/json" \
209 - -d '{
210 - "model": "meta-llama/Meta-Llama-3-8B-Instruct",
211 - "messages": [
212 - {
213 - "role": "system",
214 - "content": "You are a helpful assistant."
215 - },
216 - {
217 - "role": "user",
218 - "content": "Who are you?"
219 - }
220 - ],
221 - "stop_token_ids": [128009, 128001]
222 - }'
223 -
224 -To enable autoscaling, you could replace the `replicas` with the following configs in `service`:
225 -
226 -.. code-block:: yaml
227 -
228 - service:
229 - replica_policy:
230 - min_replicas: 2
231 - max_replicas: 4
232 - target_qps_per_replica: 2
233 -
234 -This will scale the service up to when the QPS exceeds 2 for each replica.
235 -
236 -
237 -.. raw:: html
238 -
239 - <details>
240 - <summary>Click to see the full recipe YAML</summary>
241 -
242 -
243 -.. code-block:: yaml
244 -
245 - service:
246 - replica_policy:
247 - min_replicas: 2
248 - max_replicas: 4
249 - target_qps_per_replica: 2
250 - # An actual request for readiness probe.
251 - readiness_probe:
252 - path: /v1/chat/completions
253 - post_data:
254 - model: $MODEL_NAME
255 - messages:
256 - - role: user
257 - content: Hello! What is your name?
258 - max_completion_tokens: 1
259 -
260 - resources:
261 - accelerators: {L4, A10g, A10, L40, A40, A100, A100-80GB} # We can use cheaper accelerators for 8B model.
262 - use_spot: True
263 - disk_size: 512 # Ensure model checkpoints can fit.
264 - disk_tier: best
265 - ports: 8081 # Expose to internet traffic.
266 -
267 - envs:
268 - MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct
269 - HF_TOKEN: <your-huggingface-token> # Change to your own huggingface token, or use --env to pass.
270 -
271 - setup: |
272 - conda create -n vllm python=3.10 -y
273 - conda activate vllm
274 -
275 - pip install vllm==0.4.0.post1
276 - # Install Gradio for web UI.
277 - pip install gradio openai
278 - pip install flash-attn==2.5.7
279 -
280 - run: |
281 - conda activate vllm
282 - echo 'Starting vllm api server...'
283 - python -u -m vllm.entrypoints.openai.api_server \
284 - --port 8081 \
285 - --model $MODEL_NAME \
286 - --trust-remote-code \
287 - --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \
288 - 2>&1 | tee api_server.log
289 -
290 -
291 -.. raw:: html
292 -
293 - </details>
294 -
295 -To update the service with the new config:
296 -
297 -.. code-block:: console
298 -
299 - HF_TOKEN="your-huggingface-token" sky serve update vllm serving.yaml --env HF_TOKEN
300 -
301 -
302 -To stop the service:
303 -
304 -.. code-block:: console
305 -
306 - sky serve down vllm
307 -
308 -
309 -**Optional**: Connect a GUI to the endpoint
310 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
311 -
312 -
313 -It is also possible to access the Llama-3 service with a separate GUI frontend, so the user requests send to the GUI will be load-balanced across replicas.
314 -
315 -.. raw:: html
316 -
317 - <details>
318 - <summary>Click to see the full GUI YAML</summary>
319 -
320 -.. code-block:: yaml
321 -
322 - envs:
323 - MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct
324 - ENDPOINT: x.x.x.x:3031 # Address of the API server running vllm.
325 -
326 - resources:
327 - cpus: 2
328 -
329 - setup: |
330 - conda create -n vllm python=3.10 -y
331 - conda activate vllm
332 -
333 - # Install Gradio for web UI.
334 - pip install gradio openai
335 -
336 - run: |
337 - conda activate vllm
338 - export PATH=$PATH:/sbin
339 -
340 - echo 'Starting gradio server...'
341 - git clone https://github.com/vllm-project/vllm.git || true
342 - python vllm/examples/gradio_openai_chatbot_webserver.py \
343 - -m $MODEL_NAME \
344 - --port 8811 \
345 - --model-url http://$ENDPOINT/v1 \
346 - --stop-token-ids 128009,128001 | tee ~/gradio.log
347 -
348 -
349 -.. raw:: html
350 -
351 - </details>
352 -
353 -1. Start the chat web UI:
354 -
355 -.. code-block:: console
356 -
357 - sky launch -c gui ./gui.yaml --env ENDPOINT=$(sky serve status --endpoint vllm)
358 -
359 -
360 -2. Then, we can access the GUI at the returned gradio link:
361 -
362 -.. code-block:: console
363 -
364 - | INFO | stdout | Running on public URL: https://6141e84201ce0bb4ed.gradio.live
365 -
366 -
docs/source/serving/runai_model_streamer.md new
+53
@@ -0,0 +1,53 @@
1 +(runai-model-streamer)=
2 +
3 +# Loading Models with Run:ai Model Streamer
4 +
5 +Run:ai Model Streamer is a library to read tensors in concurrency, while streaming it to GPU memory.
6 +Further reading can be found in [Run:ai Model Streamer Documentation](https://github.com/run-ai/runai-model-streamer/blob/master/docs/README.md).
7 +
8 +vLLM supports loading weights in Safetensors format using the Run:ai Model Streamer.
9 +You first need to install vLLM RunAI optional dependency:
10 +
11 +```console
12 +$ pip3 install vllm[runai]
13 +```
14 +
15 +To run it as an OpenAI-compatible server, add the `--load-format runai_streamer` flag:
16 +
17 +```console
18 +$ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer
19 +```
20 +
21 +To run model from AWS S3 object store run:
22 +
23 +```console
24 +$ vllm serve s3://core-llm/Llama-3-8b --load-format runai_streamer
25 +```
26 +
27 +To run model from a S3 compatible object store run:
28 +
29 +```console
30 +$ RUNAI_STREAMER_S3_USE_VIRTUAL_ADDRESSING=0 AWS_EC2_METADATA_DISABLED=true AWS_ENDPOINT_URL=https://storage.googleapis.com vllm serve s3://core-llm/Llama-3-8b --load-format runai_streamer
31 +```
32 +
33 +## Tunable parameters
34 +
35 +You can tune parameters using `--model-loader-extra-config`:
36 +
37 +You can tune `concurrency` that controls the level of concurrency and number of OS threads reading tensors from the file to the CPU buffer.
38 +For reading from S3, it will be the number of client instances the host is opening to the S3 server.
39 +
40 +```console
41 +$ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer --model-loader-extra-config '{"concurrency":16}'
42 +```
43 +
44 +You can controls the size of the CPU Memory buffer to which tensors are read from the file, and limit this size.
45 +You can read further about CPU buffer memory limiting [here](https://github.com/run-ai/runai-model-streamer/blob/master/docs/src/env-vars.md#runai_streamer_memory_limit).
46 +
47 +```console
48 +$ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer --model-loader-extra-config '{"memory_limit":5368709120}'
49 +```
50 +
51 +```{note}
52 +For further instructions about tunable parameters and additional parameters configurable through environment variables, read the [Environment Variables Documentation](https://github.com/run-ai/runai-model-streamer/blob/master/docs/src/env-vars.md).
53 +```
docs/source/serving/runai_model_streamer.rst deleted
-53
@@ -1,53 +0,0 @@
1 -.. _runai_model_streamer:
2 -
3 -Loading Models with Run:ai Model Streamer
4 -=========================================
5 -Run:ai Model Streamer is a library to read tensors in concurrency, while streaming it to GPU memory.
6 -Further reading can be found in `Run:ai Model Streamer Documentation <https://github.com/run-ai/runai-model-streamer/blob/master/docs/README.md>`_.
7 -
8 -vLLM supports loading weights in Safetensors format using the Run:ai Model Streamer.
9 -You first need to install vLLM RunAI optional dependency:
10 -
11 -.. code-block:: console
12 -
13 - $ pip3 install vllm[runai]
14 -
15 -To run it as an OpenAI-compatible server, add the `--load-format runai_streamer` flag:
16 -
17 -.. code-block:: console
18 -
19 - $ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer
20 -
21 -To run model from AWS S3 object store run:
22 -
23 -.. code-block:: console
24 -
25 - $ vllm serve s3://core-llm/Llama-3-8b --load-format runai_streamer
26 -
27 -
28 -To run model from a S3 compatible object store run:
29 -
30 -.. code-block:: console
31 -
32 - $ RUNAI_STREAMER_S3_USE_VIRTUAL_ADDRESSING=0 AWS_EC2_METADATA_DISABLED=true AWS_ENDPOINT_URL=https://storage.googleapis.com vllm serve s3://core-llm/Llama-3-8b --load-format runai_streamer
33 -
34 -Tunable parameters
35 -------------------
36 -You can tune parameters using `--model-loader-extra-config`:
37 -
38 -You can tune `concurrency` that controls the level of concurrency and number of OS threads reading tensors from the file to the CPU buffer.
39 -For reading from S3, it will be the number of client instances the host is opening to the S3 server.
40 -
41 - .. code-block:: console
42 -
43 - $ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer --model-loader-extra-config '{"concurrency":16}'
44 -
45 -You can controls the size of the CPU Memory buffer to which tensors are read from the file, and limit this size.
46 -You can read further about CPU buffer memory limiting `here <https://github.com/run-ai/runai-model-streamer/blob/master/docs/src/env-vars.md#runai_streamer_memory_limit>`_.
47 -
48 - .. code-block:: console
49 -
50 - $ vllm serve /home/meta-llama/Llama-3.2-3B-Instruct --load-format runai_streamer --model-loader-extra-config '{"memory_limit":5368709120}'
51 -
52 -.. note::
53 - For further instructions about tunable parameters and additional parameters configurable through environment variables, read the `Environment Variables Documentation <https://github.com/run-ai/runai-model-streamer/blob/master/docs/src/env-vars.md>`_.
docs/source/serving/serving_with_langchain.md new
+30
@@ -0,0 +1,30 @@
1 +(run-on-langchain)=
2 +
3 +# Serving with Langchain
4 +
5 +vLLM is also available via [Langchain](https://github.com/langchain-ai/langchain) .
6 +
7 +To install langchain, run
8 +
9 +```console
10 +$ pip install langchain langchain_community -q
11 +```
12 +
13 +To run inference on a single or multiple GPUs, use `VLLM` class from `langchain`.
14 +
15 +```python
16 +from langchain_community.llms import VLLM
17 +
18 +llm = VLLM(model="mosaicml/mpt-7b",
19 + trust_remote_code=True, # mandatory for hf models
20 + max_new_tokens=128,
21 + top_k=10,
22 + top_p=0.95,
23 + temperature=0.8,
24 + # tensor_parallel_size=... # for distributed inference
25 +)
26 +
27 +print(llm("What is the capital of France ?"))
28 +```
29 +
30 +Please refer to this [Tutorial](https://python.langchain.com/docs/integrations/llms/vllm) for more details.
docs/source/serving/serving_with_langchain.rst deleted
-31
@@ -1,31 +0,0 @@
1 -.. _run_on_langchain:
2 -
3 -Serving with Langchain
4 -============================
5 -
6 -vLLM is also available via `Langchain <https://github.com/langchain-ai/langchain>`_ .
7 -
8 -To install langchain, run
9 -
10 -.. code-block:: console
11 -
12 - $ pip install langchain langchain_community -q
13 -
14 -To run inference on a single or multiple GPUs, use ``VLLM`` class from ``langchain``.
15 -
16 -.. code-block:: python
17 -
18 - from langchain_community.llms import VLLM
19 -
20 - llm = VLLM(model="mosaicml/mpt-7b",
21 - trust_remote_code=True, # mandatory for hf models
22 - max_new_tokens=128,
23 - top_k=10,
24 - top_p=0.95,
25 - temperature=0.8,
26 - # tensor_parallel_size=... # for distributed inference
27 - )
28 -
29 - print(llm("What is the capital of France ?"))
30 -
31 -Please refer to this `Tutorial <https://python.langchain.com/docs/integrations/llms/vllm>`_ for more details.
docs/source/serving/serving_with_llamaindex.md new
+26
@@ -0,0 +1,26 @@
1 +(run-on-llamaindex)=
2 +
3 +# Serving with llama_index
4 +
5 +vLLM is also available via [llama_index](https://github.com/run-llama/llama_index) .
6 +
7 +To install llamaindex, run
8 +
9 +```console
10 +$ pip install llama-index-llms-vllm -q
11 +```
12 +
13 +To run inference on a single or multiple GPUs, use `Vllm` class from `llamaindex`.
14 +
15 +```python
16 +from llama_index.llms.vllm import Vllm
17 +
18 +llm = Vllm(
19 + model="microsoft/Orca-2-7b",
20 + tensor_parallel_size=4,
21 + max_new_tokens=100,
22 + vllm_kwargs={"swap_space": 1, "gpu_memory_utilization": 0.5},
23 +)
24 +```
25 +
26 +Please refer to this [Tutorial](https://docs.llamaindex.ai/en/latest/examples/llm/vllm/) for more details.
docs/source/serving/serving_with_llamaindex.rst deleted
-27
@@ -1,27 +0,0 @@
1 -.. _run_on_llamaindex:
2 -
3 -Serving with llama_index
4 -============================
5 -
6 -vLLM is also available via `llama_index <https://github.com/run-llama/llama_index>`_ .
7 -
8 -To install llamaindex, run
9 -
10 -.. code-block:: console
11 -
12 - $ pip install llama-index-llms-vllm -q
13 -
14 -To run inference on a single or multiple GPUs, use ``Vllm`` class from ``llamaindex``.
15 -
16 -.. code-block:: python
17 -
18 - from llama_index.llms.vllm import Vllm
19 -
20 - llm = Vllm(
21 - model="microsoft/Orca-2-7b",
22 - tensor_parallel_size=4,
23 - max_new_tokens=100,
24 - vllm_kwargs={"swap_space": 1, "gpu_memory_utilization": 0.5},
25 - )
26 -
27 -Please refer to this `Tutorial <https://docs.llamaindex.ai/en/latest/examples/llm/vllm/>`_ for more details.
docs/source/serving/serving_with_llamastack.md new
+38
@@ -0,0 +1,38 @@
1 +(run-on-llamastack)=
2 +
3 +# Serving with Llama Stack
4 +
5 +vLLM is also available via [Llama Stack](https://github.com/meta-llama/llama-stack) .
6 +
7 +To install Llama Stack, run
8 +
9 +```console
10 +$ pip install llama-stack -q
11 +```
12 +
13 +## Inference using OpenAI Compatible API
14 +
15 +Then start Llama Stack server pointing to your vLLM server with the following configuration:
16 +
17 +```yaml
18 +inference:
19 + - provider_id: vllm0
20 + provider_type: remote::vllm
21 + config:
22 + url: http://127.0.0.1:8000
23 +```
24 +
25 +Please refer to [this guide](https://llama-stack.readthedocs.io/en/latest/distributions/self_hosted_distro/remote-vllm.html) for more details on this remote vLLM provider.
26 +
27 +## Inference via Embedded vLLM
28 +
29 +An [inline vLLM provider](https://github.com/meta-llama/llama-stack/tree/main/llama_stack/providers/inline/inference/vllm)
30 +is also available. This is a sample of configuration using that method:
31 +
32 +```yaml
33 +inference
34 + - provider_type: vllm
35 + config:
36 + model: Llama3.1-8B-Instruct
37 + tensor_parallel_size: 4
38 +```
docs/source/serving/serving_with_llamastack.rst deleted
-42
@@ -1,42 +0,0 @@
1 -.. _run_on_llamastack:
2 -
3 -Serving with Llama Stack
4 -============================
5 -
6 -vLLM is also available via `Llama Stack <https://github.com/meta-llama/llama-stack>`_ .
7 -
8 -To install Llama Stack, run
9 -
10 -.. code-block:: console
11 -
12 - $ pip install llama-stack -q
13 -
14 -Inference using OpenAI Compatible API
15 --------------------------------------
16 -
17 -Then start Llama Stack server pointing to your vLLM server with the following configuration:
18 -
19 -.. code-block:: yaml
20 -
21 - inference:
22 - - provider_id: vllm0
23 - provider_type: remote::vllm
24 - config:
25 - url: http://127.0.0.1:8000
26 -
27 -Please refer to `this guide <https://llama-stack.readthedocs.io/en/latest/distributions/self_hosted_distro/remote-vllm.html>`_ for more details on this remote vLLM provider.
28 -
29 -Inference via Embedded vLLM
30 ----------------------------
31 -
32 -An `inline vLLM provider
33 -<https://github.com/meta-llama/llama-stack/tree/main/llama_stack/providers/inline/inference/vllm>`_
34 -is also available. This is a sample of configuration using that method:
35 -
36 -.. code-block:: yaml
37 -
38 - inference
39 - - provider_type: vllm
40 - config:
41 - model: Llama3.1-8B-Instruct
42 - tensor_parallel_size: 4
docs/source/serving/tensorizer.md new
+16
@@ -0,0 +1,16 @@
1 +(tensorizer)=
2 +
3 +# Loading Models with CoreWeave's Tensorizer
4 +
5 +vLLM supports loading models with [CoreWeave's Tensorizer](https://docs.coreweave.com/coreweave-machine-learning-and-ai/inference/tensorizer).
6 +vLLM model tensors that have been serialized to disk, an HTTP/HTTPS endpoint, or S3 endpoint can be deserialized
7 +at runtime extremely quickly directly to the GPU, resulting in significantly
8 +shorter Pod startup times and CPU memory usage. Tensor encryption is also supported.
9 +
10 +For more information on CoreWeave's Tensorizer, please refer to
11 +[CoreWeave's Tensorizer documentation](https://github.com/coreweave/tensorizer). For more information on serializing a vLLM model, as well a general usage guide to using Tensorizer with vLLM, see
12 +the [vLLM example script](https://docs.vllm.ai/en/stable/getting_started/examples/tensorize_vllm_model.html).
13 +
14 +```{note}
15 +Note that to use this feature you will need to install `tensorizer` by running `pip install vllm[tensorizer]`.
16 +```
docs/source/serving/tensorizer.rst deleted
-15
@@ -1,15 +0,0 @@
1 -.. _tensorizer:
2 -
3 -Loading Models with CoreWeave's Tensorizer
4 -==========================================
5 -vLLM supports loading models with `CoreWeave's Tensorizer <https://docs.coreweave.com/coreweave-machine-learning-and-ai/inference/tensorizer>`_.
6 -vLLM model tensors that have been serialized to disk, an HTTP/HTTPS endpoint, or S3 endpoint can be deserialized
7 -at runtime extremely quickly directly to the GPU, resulting in significantly
8 -shorter Pod startup times and CPU memory usage. Tensor encryption is also supported.
9 -
10 -For more information on CoreWeave's Tensorizer, please refer to
11 -`CoreWeave's Tensorizer documentation <https://github.com/coreweave/tensorizer>`_. For more information on serializing a vLLM model, as well a general usage guide to using Tensorizer with vLLM, see
12 -the `vLLM example script <https://docs.vllm.ai/en/stable/getting_started/examples/tensorize_vllm_model.html>`_.
13 -
14 -.. note::
15 - Note that to use this feature you will need to install `tensorizer` by running `pip install vllm[tensorizer]`.
docs/source/usage/compatibility_matrix.md new
+468
@@ -0,0 +1,468 @@
1 +(compatibility-matrix)=
2 +
3 +# Compatibility Matrix
4 +
5 +The tables below show mutually exclusive features and the support on some hardware.
6 +
7 +```{note}
8 +Check the '✗' with links to see tracking issue for unsupported feature/hardware combination.
9 +```
10 +
11 +## Feature x Feature
12 +
13 +```{raw} html
14 +<style>
15 + /* Make smaller to try to improve readability */
16 + td {
17 + font-size: 0.8rem;
18 + text-align: center;
19 + }
20 +
21 + th {
22 + text-align: center;
23 + font-size: 0.8rem;
24 + }
25 +</style>
26 +```
27 +
28 +```{list-table}
29 + :header-rows: 1
30 + :stub-columns: 1
31 + :widths: auto
32 +
33 + * - Feature
34 + - [CP](#chunked-prefill)
35 + - [APC](#apc)
36 + - [LoRA](#lora-adapter)
37 + - <abbr title="Prompt Adapter">prmpt adptr</abbr>
38 + - [SD](#spec_decode)
39 + - CUDA graph
40 + - <abbr title="Pooling Models">pooling</abbr>
41 + - <abbr title="Encoder-Decoder Models">enc-dec</abbr>
42 + - <abbr title="Logprobs">logP</abbr>
43 + - <abbr title="Prompt Logprobs">prmpt logP</abbr>
44 + - <abbr title="Async Output Processing">async output</abbr>
45 + - multi-step
46 + - <abbr title="Multimodal Inputs">mm</abbr>
47 + - best-of
48 + - beam-search
49 + - <abbr title="Guided Decoding">guided dec</abbr>
50 + * - [CP](#chunked-prefill)
51 + -
52 + -
53 + -
54 + -
55 + -
56 + -
57 + -
58 + -
59 + -
60 + -
61 + -
62 + -
63 + -
64 + -
65 + -
66 + -
67 + * - [APC](#apc)
68 + - ✅
69 + -
70 + -
71 + -
72 + -
73 + -
74 + -
75 + -
76 + -
77 + -
78 + -
79 + -
80 + -
81 + -
82 + -
83 + -
84 + * - [LoRA](#lora-adapter)
85 + - [✗](https://github.com/vllm-project/vllm/pull/9057)
86 + - ✅
87 + -
88 + -
89 + -
90 + -
91 + -
92 + -
93 + -
94 + -
95 + -
96 + -
97 + -
98 + -
99 + -
100 + -
101 + * - <abbr title="Prompt Adapter">prmpt adptr</abbr>
102 + - ✅
103 + - ✅
104 + - ✅
105 + -
106 + -
107 + -
108 + -
109 + -
110 + -
111 + -
112 + -
113 + -
114 + -
115 + -
116 + -
117 + -
118 + * - [SD](#spec_decode)
119 + - ✅
120 + - ✅
121 + - ✗
122 + - ✅
123 + -
124 + -
125 + -
126 + -
127 + -
128 + -
129 + -
130 + -
131 + -
132 + -
133 + -
134 + -
135 + * - CUDA graph
136 + - ✅
137 + - ✅
138 + - ✅
139 + - ✅
140 + - ✅
141 + -
142 + -
143 + -
144 + -
145 + -
146 + -
147 + -
148 + -
149 + -
150 + -
151 + -
152 + * - <abbr title="Pooling Models">pooling</abbr>
153 + - ✗
154 + - ✗
155 + - ✗
156 + - ✗
157 + - ✗
158 + - ✗
159 + -
160 + -
161 + -
162 + -
163 + -
164 + -
165 + -
166 + -
167 + -
168 + -
169 + * - <abbr title="Encoder-Decoder Models">enc-dec</abbr>
170 + - ✗
171 + - [✗](https://github.com/vllm-project/vllm/issues/7366)
172 + - ✗
173 + - ✗
174 + - [✗](https://github.com/vllm-project/vllm/issues/7366)
175 + - ✅
176 + - ✅
177 + -
178 + -
179 + -
180 + -
181 + -
182 + -
183 + -
184 + -
185 + -
186 + * - <abbr title="Logprobs">logP</abbr>
187 + - ✅
188 + - ✅
189 + - ✅
190 + - ✅
191 + - ✅
192 + - ✅
193 + - ✗
194 + - ✅
195 + -
196 + -
197 + -
198 + -
199 + -
200 + -
201 + -
202 + -
203 + * - <abbr title="Prompt Logprobs">prmpt logP</abbr>
204 + - ✅
205 + - ✅
206 + - ✅
207 + - ✅
208 + - [✗](https://github.com/vllm-project/vllm/pull/8199)
209 + - ✅
210 + - ✗
211 + - ✅
212 + - ✅
213 + -
214 + -
215 + -
216 + -
217 + -
218 + -
219 + -
220 + * - <abbr title="Async Output Processing">async output</abbr>
221 + - ✅
222 + - ✅
223 + - ✅
224 + - ✅
225 + - ✗
226 + - ✅
227 + - ✗
228 + - ✗
229 + - ✅
230 + - ✅
231 + -
232 + -
233 + -
234 + -
235 + -
236 + -
237 + * - multi-step
238 + - ✗
239 + - ✅
240 + - ✗
241 + - ✅
242 + - ✗
243 + - ✅
244 + - ✗
245 + - ✗
246 + - ✅
247 + - [✗](https://github.com/vllm-project/vllm/issues/8198)
248 + - ✅
249 + -
250 + -
251 + -
252 + -
253 + -
254 + * - <abbr title="Multimodal Inputs">mm</abbr>
255 + - ✅
256 + - [✗](https://github.com/vllm-project/vllm/pull/8348)
257 + - [✗](https://github.com/vllm-project/vllm/pull/7199)
258 + - ?
259 + - ?
260 + - ✅
261 + - ✅
262 + - ✅
263 + - ✅
264 + - ✅
265 + - ✅
266 + - ?
267 + -
268 + -
269 + -
270 + -
271 + * - best-of
272 + - ✅
273 + - ✅
274 + - ✅
275 + - ✅
276 + - [✗](https://github.com/vllm-project/vllm/issues/6137)
277 + - ✅
278 + - ✗
279 + - ✅
280 + - ✅
281 + - ✅
282 + - ?
283 + - [✗](https://github.com/vllm-project/vllm/issues/7968)
284 + - ✅
285 + -
286 + -
287 + -
288 + * - beam-search
289 + - ✅
290 + - ✅
291 + - ✅
292 + - ✅
293 + - [✗](https://github.com/vllm-project/vllm/issues/6137)
294 + - ✅
295 + - ✗
296 + - ✅
297 + - ✅
298 + - ✅
299 + - ?
300 + - [✗](https://github.com/vllm-project/vllm/issues/7968>)
301 + - ?
302 + - ✅
303 + -
304 + -
305 + * - <abbr title="Guided Decoding">guided dec</abbr>
306 + - ✅
307 + - ✅
308 + - ?
309 + - ?
310 + - ✅
311 + - ✅
312 + - ✗
313 + - ?
314 + - ✅
315 + - ✅
316 + - ✅
317 + - [✗](https://github.com/vllm-project/vllm/issues/9893)
318 + - ?
319 + - ✅
320 + - ✅
321 + -
322 +
323 +```
324 +
325 +### Feature x Hardware
326 +
327 +```{list-table}
328 + :header-rows: 1
329 + :stub-columns: 1
330 + :widths: auto
331 +
332 + * - Feature
333 + - Volta
334 + - Turing
335 + - Ampere
336 + - Ada
337 + - Hopper
338 + - CPU
339 + - AMD
340 + * - [CP](#chunked-prefill)
341 + - [✗](https://github.com/vllm-project/vllm/issues/2729)
342 + - ✅
343 + - ✅
344 + - ✅
345 + - ✅
346 + - ✅
347 + - ✅
348 + * - [APC](#apc)
349 + - [✗](https://github.com/vllm-project/vllm/issues/3687)
350 + - ✅
351 + - ✅
352 + - ✅
353 + - ✅
354 + - ✅
355 + - ✅
356 + * - [LoRA](#lora-adapter)
357 + - ✅
358 + - ✅
359 + - ✅
360 + - ✅
361 + - ✅
362 + - [✗](https://github.com/vllm-project/vllm/pull/4830)
363 + - ✅
364 + * - <abbr title="Prompt Adapter">prmpt adptr</abbr>
365 + - ✅
366 + - ✅
367 + - ✅
368 + - ✅
369 + - ✅
370 + - [✗](https://github.com/vllm-project/vllm/issues/8475)
371 + - ✅
372 + * - [SD](#spec_decode)
373 + - ✅
374 + - ✅
375 + - ✅
376 + - ✅
377 + - ✅
378 + - ✅
379 + - ✅
380 + * - CUDA graph
381 + - ✅
382 + - ✅
383 + - ✅
384 + - ✅
385 + - ✅
386 + - ✗
387 + - ✅
388 + * - <abbr title="Pooling Models">pooling</abbr>
389 + - ✅
390 + - ✅
391 + - ✅
392 + - ✅
393 + - ✅
394 + - ✅
395 + - ?
396 + * - <abbr title="Encoder-Decoder Models">enc-dec</abbr>
397 + - ✅
398 + - ✅
399 + - ✅
400 + - ✅
401 + - ✅
402 + - ✅
403 + - ✗
404 + * - <abbr title="Multimodal Inputs">mm</abbr>
405 + - ✅
406 + - ✅
407 + - ✅
408 + - ✅
409 + - ✅
410 + - ✅
411 + - ✅
412 + * - <abbr title="Logprobs">logP</abbr>
413 + - ✅
414 + - ✅
415 + - ✅
416 + - ✅
417 + - ✅
418 + - ✅
419 + - ✅
420 + * - <abbr title="Prompt Logprobs">prmpt logP</abbr>
421 + - ✅
422 + - ✅
423 + - ✅
424 + - ✅
425 + - ✅
426 + - ✅
427 + - ✅
428 + * - <abbr title="Async Output Processing">async output</abbr>
429 + - ✅
430 + - ✅
431 + - ✅
432 + - ✅
433 + - ✅
434 + - ✗
435 + - ✗
436 + * - multi-step
437 + - ✅
438 + - ✅
439 + - ✅
440 + - ✅
441 + - ✅
442 + - [✗](https://github.com/vllm-project/vllm/issues/8477)
443 + - ✅
444 + * - best-of
445 + - ✅
446 + - ✅
447 + - ✅
448 + - ✅
449 + - ✅
450 + - ✅
451 + - ✅
452 + * - beam-search
453 + - ✅
454 + - ✅
455 + - ✅
456 + - ✅
457 + - ✅
458 + - ✅
459 + - ✅
460 + * - <abbr title="Guided Decoding">guided dec</abbr>
461 + - ✅
462 + - ✅
463 + - ✅
464 + - ✅
465 + - ✅
466 + - ✅
467 + - ✅
468 +```
docs/source/usage/compatibility_matrix.rst deleted
-468
@@ -1,468 +0,0 @@
1 -.. _compatibility_matrix:
2 -
3 -Compatibility Matrix
4 -====================
5 -
6 -The tables below show mutually exclusive features and the support on some hardware.
7 -
8 -.. note::
9 -
10 - Check the '✗' with links to see tracking issue for unsupported feature/hardware combination.
11 -
12 -Feature x Feature
13 ------------------
14 -
15 -
16 -.. raw:: html
17 -
18 - <style>
19 - /* Make smaller to try to improve readability */
20 - td {
21 - font-size: 0.8rem;
22 - text-align: center;
23 - }
24 -
25 - th {
26 - text-align: center;
27 - font-size: 0.8rem;
28 - }
29 - </style>
30 -
31 -.. list-table::
32 - :header-rows: 1
33 - :widths: auto
34 -
35 - * - Feature
36 - - :ref:`CP <chunked-prefill>`
37 - - :ref:`APC <apc>`
38 - - :ref:`LoRA <lora>`
39 - - :abbr:`prmpt adptr (Prompt Adapter)`
40 - - :ref:`SD <spec_decode>`
41 - - CUDA graph
42 - - :abbr:`pooling (Pooling Models)`
43 - - :abbr:`enc-dec (Encoder-Decoder Models)`
44 - - :abbr:`logP (Logprobs)`
45 - - :abbr:`prmpt logP (Prompt Logprobs)`
46 - - :abbr:`async output (Async Output Processing)`
47 - - multi-step
48 - - :abbr:`mm (Multimodal Inputs)`
49 - - best-of
50 - - beam-search
51 - - :abbr:`guided dec (Guided Decoding)`
52 - * - :ref:`CP <chunked-prefill>`
53 - -
54 - -
55 - -
56 - -
57 - -
58 - -
59 - -
60 - -
61 - -
62 - -
63 - -
64 - -
65 - -
66 - -
67 - -
68 - -
69 - * - :ref:`APC <apc>`
70 - - ✅
71 - -
72 - -
73 - -
74 - -
75 - -
76 - -
77 - -
78 - -
79 - -
80 - -
81 - -
82 - -
83 - -
84 - -
85 - -
86 - * - :ref:`LoRA <lora>`
87 - - `✗ <https://github.com/vllm-project/vllm/pull/9057>`__
88 - - ✅
89 - -
90 - -
91 - -
92 - -
93 - -
94 - -
95 - -
96 - -
97 - -
98 - -
99 - -
100 - -
101 - -
102 - -
103 - * - :abbr:`prmpt adptr (Prompt Adapter)`
104 - - ✅
105 - - ✅
106 - - ✅
107 - -
108 - -
109 - -
110 - -
111 - -
112 - -
113 - -
114 - -
115 - -
116 - -
117 - -
118 - -
119 - -
120 - * - :ref:`SD <spec_decode>`
121 - - ✅
122 - - ✅
123 - - ✗
124 - - ✅
125 - -
126 - -
127 - -
128 - -
129 - -
130 - -
131 - -
132 - -
133 - -
134 - -
135 - -
136 - -
137 - * - CUDA graph
138 - - ✅
139 - - ✅
140 - - ✅
141 - - ✅
142 - - ✅
143 - -
144 - -
145 - -
146 - -
147 - -
148 - -
149 - -
150 - -
151 - -
152 - -
153 - -
154 - * - :abbr:`pooling (Pooling Models)`
155 - - ✗
156 - - ✗
157 - - ✗
158 - - ✗
159 - - ✗
160 - - ✗
161 - -
162 - -
163 - -
164 - -
165 - -
166 - -
167 - -
168 - -
169 - -
170 - -
171 - * - :abbr:`enc-dec (Encoder-Decoder Models)`
172 - - ✗
173 - - `✗ <https://github.com/vllm-project/vllm/issues/7366>`__
174 - - ✗
175 - - ✗
176 - - `✗ <https://github.com/vllm-project/vllm/issues/7366>`__
177 - - ✅
178 - - ✅
179 - -
180 - -
181 - -
182 - -
183 - -
184 - -
185 - -
186 - -
187 - -
188 - * - :abbr:`logP (Logprobs)`
189 - - ✅
190 - - ✅
191 - - ✅
192 - - ✅
193 - - ✅
194 - - ✅
195 - - ✗
196 - - ✅
197 - -
198 - -
199 - -
200 - -
201 - -
202 - -
203 - -
204 - -
205 - * - :abbr:`prmpt logP (Prompt Logprobs)`
206 - - ✅
207 - - ✅
208 - - ✅
209 - - ✅
210 - - `✗ <https://github.com/vllm-project/vllm/pull/8199>`__
211 - - ✅
212 - - ✗
213 - - ✅
214 - - ✅
215 - -
216 - -
217 - -
218 - -
219 - -
220 - -
221 - -
222 - * - :abbr:`async output (Async Output Processing)`
223 - - ✅
224 - - ✅
225 - - ✅
226 - - ✅
227 - - ✗
228 - - ✅
229 - - ✗
230 - - ✗
231 - - ✅
232 - - ✅
233 - -
234 - -
235 - -
236 - -
237 - -
238 - -
239 - * - multi-step
240 - - ✗
241 - - ✅
242 - - ✗
243 - - ✅
244 - - ✗
245 - - ✅
246 - - ✗
247 - - ✗
248 - - ✅
249 - - `✗ <https://github.com/vllm-project/vllm/issues/8198>`__
250 - - ✅
251 - -
252 - -
253 - -
254 - -
255 - -
256 - * - :abbr:`mm (Multimodal Inputs)`
257 - - ✅
258 - - `✗ <https://github.com/vllm-project/vllm/pull/8348>`__
259 - - `✗ <https://github.com/vllm-project/vllm/pull/7199>`__
260 - - ?
261 - - ?
262 - - ✅
263 - - ✅
264 - - ✅
265 - - ✅
266 - - ✅
267 - - ✅
268 - - ?
269 - -
270 - -
271 - -
272 - -
273 - * - best-of
274 - - ✅
275 - - ✅
276 - - ✅
277 - - ✅
278 - - `✗ <https://github.com/vllm-project/vllm/issues/6137>`__
279 - - ✅
280 - - ✗
281 - - ✅
282 - - ✅
283 - - ✅
284 - - ?
285 - - `✗ <https://github.com/vllm-project/vllm/issues/7968>`__
286 - - ✅
287 - -
288 - -
289 - -
290 - * - beam-search
291 - - ✅
292 - - ✅
293 - - ✅
294 - - ✅
295 - - `✗ <https://github.com/vllm-project/vllm/issues/6137>`__
296 - - ✅
297 - - ✗
298 - - ✅
299 - - ✅
300 - - ✅
301 - - ?
302 - - `✗ <https://github.com/vllm-project/vllm/issues/7968>`__
303 - - ?
304 - - ✅
305 - -
306 - -
307 - * - :abbr:`guided dec (Guided Decoding)`
308 - - ✅
309 - - ✅
310 - - ?
311 - - ?
312 - - ✅
313 - - ✅
314 - - ✗
315 - - ?
316 - - ✅
317 - - ✅
318 - - ✅
319 - - `✗ <https://github.com/vllm-project/vllm/issues/9893>`__
320 - - ?
321 - - ✅
322 - - ✅
323 - -
324 -
325 -
326 -Feature x Hardware
327 -^^^^^^^^^^^^^^^^^^
328 -
329 -.. list-table::
330 - :header-rows: 1
331 - :widths: auto
332 -
333 - * - Feature
334 - - Volta
335 - - Turing
336 - - Ampere
337 - - Ada
338 - - Hopper
339 - - CPU
340 - - AMD
341 - * - :ref:`CP <chunked-prefill>`
342 - - `✗ <https://github.com/vllm-project/vllm/issues/2729>`__
343 - - ✅
344 - - ✅
345 - - ✅
346 - - ✅
347 - - ✅
348 - - ✅
349 - * - :ref:`APC <apc>`
350 - - `✗ <https://github.com/vllm-project/vllm/issues/3687>`__
351 - - ✅
352 - - ✅
353 - - ✅
354 - - ✅
355 - - ✅
356 - - ✅
357 - * - :ref:`LoRA <lora>`
358 - - ✅
359 - - ✅
360 - - ✅
361 - - ✅
362 - - ✅
363 - - `✗ <https://github.com/vllm-project/vllm/pull/4830>`__
364 - - ✅
365 - * - :abbr:`prmpt adptr (Prompt Adapter)`
366 - - ✅
367 - - ✅
368 - - ✅
369 - - ✅
370 - - ✅
371 - - `✗ <https://github.com/vllm-project/vllm/issues/8475>`__
372 - - ✅
373 - * - :ref:`SD <spec_decode>`
374 - - ✅
375 - - ✅
376 - - ✅
377 - - ✅
378 - - ✅
379 - - ✅
380 - - ✅
381 - * - CUDA graph
382 - - ✅
383 - - ✅
384 - - ✅
385 - - ✅
386 - - ✅
387 - - ✗
388 - - ✅
389 - * - :abbr:`pooling (Pooling Models)`
390 - - ✅
391 - - ✅
392 - - ✅
393 - - ✅
394 - - ✅
395 - - ✅
396 - - ?
397 - * - :abbr:`enc-dec (Encoder-Decoder Models)`
398 - - ✅
399 - - ✅
400 - - ✅
401 - - ✅
402 - - ✅
403 - - ✅
404 - - ✗
405 - * - :abbr:`mm (Multimodal Inputs)`
406 - - ✅
407 - - ✅
408 - - ✅
409 - - ✅
410 - - ✅
411 - - ✅
412 - - ✅
413 - * - :abbr:`logP (Logprobs)`
414 - - ✅
415 - - ✅
416 - - ✅
417 - - ✅
418 - - ✅
419 - - ✅
420 - - ✅
421 - * - :abbr:`prmpt logP (Prompt Logprobs)`
422 - - ✅
423 - - ✅
424 - - ✅
425 - - ✅
426 - - ✅
427 - - ✅
428 - - ✅
429 - * - :abbr:`async output (Async Output Processing)`
430 - - ✅
431 - - ✅
432 - - ✅
433 - - ✅
434 - - ✅
435 - - ✗
436 - - ✗
437 - * - multi-step
438 - - ✅
439 - - ✅
440 - - ✅
441 - - ✅
442 - - ✅
443 - - `✗ <https://github.com/vllm-project/vllm/issues/8477>`__
444 - - ✅
445 - * - best-of
446 - - ✅
447 - - ✅
448 - - ✅
449 - - ✅
450 - - ✅
451 - - ✅
452 - - ✅
453 - * - beam-search
454 - - ✅
455 - - ✅
456 - - ✅
457 - - ✅
458 - - ✅
459 - - ✅
460 - - ✅
461 - * - :abbr:`guided dec (Guided Decoding)`
462 - - ✅
463 - - ✅
464 - - ✅
465 - - ✅
466 - - ✅
467 - - ✅
468 - - ✅
docs/source/usage/disagg_prefill.md new
+64
@@ -0,0 +1,64 @@
1 +(disagg-prefill)=
2 +
3 +# Disaggregated prefilling (experimental)
4 +
5 +This page introduces you the disaggregated prefilling feature in vLLM. This feature is experimental and subject to change.
6 +
7 +## Why disaggregated prefilling?
8 +
9 +Two main reasons:
10 +
11 +- **Tuning time-to-first-token (TTFT) and inter-token-latency (ITL) separately**. Disaggregated prefilling put prefill and decode phase of LLM inference inside different vLLM instances. This gives you the flexibility to assign different parallel strategies (e.g. `tp` and `pp`) to tune TTFT without affecting ITL, or to tune ITL without affecting TTFT.
12 +- **Controlling tail ITL**. Without disaggregated prefilling, vLLM may insert some prefill jobs during the decoding of one request. This results in higher tail latency. Disaggregated prefilling helps you solve this issue and control tail ITL. Chunked prefill with a proper chunk size also can achieve the same goal, but in practice it's hard to figure out the correct chunk size value. So disaggregated prefilling is a much more reliable way to control tail ITL.
13 +
14 +```{note}
15 +Disaggregated prefill DOES NOT improve throughput.
16 +```
17 +
18 +## Usage example
19 +
20 +Please refer to `examples/disaggregated_prefill.sh` for the example usage of disaggregated prefilling.
21 +
22 +## Benchmarks
23 +
24 +Please refer to `benchmarks/disagg_benchmarks/` for disaggregated prefilling benchmarks.
25 +
26 +## Development
27 +
28 +We implement disaggregated prefilling by running 2 vLLM instances. One for prefill (we call it prefill instance) and one for decode (we call it decode instance), and then use a connector to transfer the prefill KV caches and results from prefill instance to decode instance.
29 +
30 +All disaggregated prefilling implementation is under `vllm/distributed/kv_transfer`.
31 +
32 +Key abstractions for disaggregated prefilling:
33 +
34 +- **Connector**: Connector allows **kv consumer** to retrieve the KV caches of a batch of request from **kv producer**.
35 +- **LookupBuffer**: LookupBuffer provides two API: `insert` KV cache and `drop_select` KV cache. The semantics of `insert` and `drop_select` are similar to SQL, where `insert` inserts a KV cache into the buffer, and `drop_select` returns the KV cache that matches the given condition and drop it from the buffer.
36 +- **Pipe**: A single-direction FIFO pipe for tensor transmission. It supports `send_tensor` and `recv_tensor`.
37 +
38 +```{note}
39 +`insert` is non-blocking operation but `drop_select` is blocking operation.
40 +```
41 +
42 +Here is a figure illustrating how the above 3 abstractions are organized:
43 +
44 +```{image} /assets/usage/disagg_prefill/abstraction.jpg
45 +:alt: Disaggregated prefilling abstractions
46 +```
47 +
48 +The workflow of disaggregated prefilling is as follows:
49 +
50 +```{image} /assets/usage/disagg_prefill/overview.jpg
51 +:alt: Disaggregated prefilling workflow
52 +```
53 +
54 +The `buffer` corresponds to `insert` API in LookupBuffer, and the `drop_select` corresponds to `drop_select` API in LookupBuffer.
55 +
56 +## Third-party contributions
57 +
58 +Disaggregated prefilling is highly related to infrastructure, so vLLM relies on third-party connectors for production-level disaggregated prefilling (and vLLM team will actively review and merge new PRs for third-party connectors).
59 +
60 +We recommend three ways of implementations:
61 +
62 +- **Fully-customized connector**: Implement your own `Connector`, and call third-party libraries to send and receive KV caches, and many many more (like editing vLLM's model input to perform customized prefilling, etc). This approach gives you the most control, but at the risk of being incompatible with future vLLM versions.
63 +- **Database-like connector**: Implement your own `LookupBuffer` and support the `insert` and `drop_select` APIs just like SQL.
64 +- **Distributed P2P connector**: Implement your own `Pipe` and support the `send_tensor` and `recv_tensor` APIs, just like `torch.distributed`.
docs/source/usage/disagg_prefill.rst deleted
-69
@@ -1,69 +0,0 @@
1 -.. _disagg_prefill:
2 -
3 -Disaggregated prefilling (experimental)
4 -=======================================
5 -
6 -This page introduces you the disaggregated prefilling feature in vLLM. This feature is experimental and subject to change.
7 -
8 -Why disaggregated prefilling?
9 ------------------------------
10 -
11 -Two main reasons:
12 -
13 -* **Tuning time-to-first-token (TTFT) and inter-token-latency (ITL) separately**. Disaggregated prefilling put prefill and decode phase of LLM inference inside different vLLM instances. This gives you the flexibility to assign different parallel strategies (e.g. ``tp`` and ``pp``) to tune TTFT without affecting ITL, or to tune ITL without affecting TTFT.
14 -* **Controlling tail ITL**. Without disaggregated prefilling, vLLM may insert some prefill jobs during the decoding of one request. This results in higher tail latency. Disaggregated prefilling helps you solve this issue and control tail ITL. Chunked prefill with a proper chunk size also can achieve the same goal, but in practice it's hard to figure out the correct chunk size value. So disaggregated prefilling is a much more reliable way to control tail ITL.
15 -
16 -.. note::
17 - Disaggregated prefill DOES NOT improve throughput.
18 -
19 -Usage example
20 --------------
21 -
22 -Please refer to ``examples/disaggregated_prefill.sh`` for the example usage of disaggregated prefilling.
23 -
24 -
25 -Benchmarks
26 -----------
27 -
28 -Please refer to ``benchmarks/disagg_benchmarks/`` for disaggregated prefilling benchmarks.
29 -
30 -
31 -Development
32 ------------
33 -
34 -We implement disaggregated prefilling by running 2 vLLM instances. One for prefill (we call it prefill instance) and one for decode (we call it decode instance), and then use a connector to transfer the prefill KV caches and results from prefill instance to decode instance.
35 -
36 -All disaggregated prefilling implementation is under ``vllm/distributed/kv_transfer``.
37 -
38 -Key abstractions for disaggregated prefilling:
39 -
40 -* **Connector**: Connector allows **kv consumer** to retrieve the KV caches of a batch of request from **kv producer**.
41 -* **LookupBuffer**: LookupBuffer provides two API: ``insert`` KV cache and ``drop_select`` KV cache. The semantics of ``insert`` and ``drop_select`` are similar to SQL, where ``insert`` inserts a KV cache into the buffer, and ``drop_select`` returns the KV cache that matches the given condition and drop it from the buffer.
42 -* **Pipe**: A single-direction FIFO pipe for tensor transmission. It supports ``send_tensor`` and ``recv_tensor``.
43 -
44 -.. note::
45 - ``insert`` is non-blocking operation but ``drop_select`` is blocking operation.
46 -
47 -Here is a figure illustrating how the above 3 abstractions are organized:
48 -
49 -.. image:: /assets/usage/disagg_prefill/abstraction.jpg
50 - :alt: Disaggregated prefilling abstractions
51 -
52 -The workflow of disaggregated prefilling is as follows:
53 -
54 -.. image:: /assets/usage/disagg_prefill/overview.jpg
55 - :alt: Disaggregated prefilling workflow
56 -
57 -The ``buffer`` corresponds to ``insert`` API in LookupBuffer, and the ``drop_select`` corresponds to ``drop_select`` API in LookupBuffer.
58 -
59 -
60 -Third-party contributions
61 --------------------------
62 -
63 -Disaggregated prefilling is highly related to infrastructure, so vLLM relies on third-party connectors for production-level disaggregated prefilling (and vLLM team will actively review and merge new PRs for third-party connectors).
64 -
65 -We recommend three ways of implementations:
66 -
67 -* **Fully-customized connector**: Implement your own ``Connector``, and call third-party libraries to send and receive KV caches, and many many more (like editing vLLM's model input to perform customized prefilling, etc). This approach gives you the most control, but at the risk of being incompatible with future vLLM versions.
68 -* **Database-like connector**: Implement your own ``LookupBuffer`` and support the ``insert`` and ``drop_select`` APIs just like SQL.
69 -* **Distributed P2P connector**: Implement your own ``Pipe`` and support the ``send_tensor`` and ``recv_tensor`` APIs, just like `torch.distributed`.
docs/source/usage/engine_args.md renamed
+8 -6
@@ -1,23 +1,25 @@
1 -.. _engine_args:
1 +(engine-args)=
2
3 -Engine Arguments
4 -================
3 +# Engine Arguments
4
5 Below, you can find an explanation of every engine argument for vLLM:
6
7 +```{eval-rst}
8 .. argparse::
9 :module: vllm.engine.arg_utils
10 :func: _engine_args_parser
11 :prog: vllm serve
12 :nodefaultconst:
13 +```
14
14 -Async Engine Arguments
15 -----------------------
15 +## Async Engine Arguments
16
17 Below are the additional arguments related to the asynchronous engine:
18
19 +```{eval-rst}
20 .. argparse::
21 :module: vllm.engine.arg_utils
22 :func: _async_engine_args_parser
23 :prog: vllm serve
23 - :nodefaultconst:
\ No newline at end of file
24 + :nodefaultconst:
25 +```
docs/source/usage/env_vars.md new
+15
@@ -0,0 +1,15 @@
1 +# Environment Variables
2 +
3 +vLLM uses the following environment variables to configure the system:
4 +
5 +```{warning}
6 +Please note that `VLLM_PORT` and `VLLM_HOST_IP` set the port and ip for vLLM's **internal usage**. It is not the port and ip for the API server. If you use `--host $VLLM_HOST_IP` and `--port $VLLM_PORT` to start the API server, it will not work.
7 +
8 +All environment variables used by vLLM are prefixed with `VLLM_`. **Special care should be taken for Kubernetes users**: please do not name the service as `vllm`, otherwise environment variables set by Kubernetes might conflict with vLLM's environment variables, because [Kubernetes sets environment variables for each service with the capitalized service name as the prefix](https://kubernetes.io/docs/concepts/services-networking/service/#environment-variables).
9 +```
10 +
11 +```{literalinclude} ../../../vllm/envs.py
12 +:end-before: end-env-vars-definition
13 +:language: python
14 +:start-after: begin-env-vars-definition
15 +```
docs/source/usage/env_vars.rst deleted
-14
@@ -1,14 +0,0 @@
1 -Environment Variables
2 -========================
3 -
4 -vLLM uses the following environment variables to configure the system:
5 -
6 -.. warning::
7 - Please note that ``VLLM_PORT`` and ``VLLM_HOST_IP`` set the port and ip for vLLM's **internal usage**. It is not the port and ip for the API server. If you use ``--host $VLLM_HOST_IP`` and ``--port $VLLM_PORT`` to start the API server, it will not work.
8 -
9 - All environment variables used by vLLM are prefixed with ``VLLM_``. **Special care should be taken for Kubernetes users**: please do not name the service as ``vllm``, otherwise environment variables set by Kubernetes might conflict with vLLM's environment variables, because `Kubernetes sets environment variables for each service with the capitalized service name as the prefix <https://kubernetes.io/docs/concepts/services-networking/service/#environment-variables>`_.
10 -
11 -.. literalinclude:: ../../../vllm/envs.py
12 - :language: python
13 - :start-after: begin-env-vars-definition
14 - :end-before: end-env-vars-definition
docs/source/usage/faq.md renamed
+15 -16
@@ -1,34 +1,33 @@
1 -.. _faq:
1 +(faq)=
2
3 -Frequently Asked Questions
4 -===========================
3 +# Frequently Asked Questions
4
6 - Q: How can I serve multiple models on a single port using the OpenAI API?
5 +> Q: How can I serve multiple models on a single port using the OpenAI API?
6
7 A: Assuming that you're referring to using OpenAI compatible server to serve multiple models at once, that is not currently supported, you can run multiple instances of the server (each serving a different model) at the same time, and have another layer to route the incoming request to the correct server accordingly.
8
10 -----------------------------------------
9 +______________________________________________________________________
10
12 - Q: Which model to use for offline inference embedding?
11 +> Q: Which model to use for offline inference embedding?
12
14 -A: You can try `e5-mistral-7b-instruct <https://huggingface.co/intfloat/e5-mistral-7b-instruct>`__ and `BAAI/bge-base-en-v1.5 <https://huggingface.co/BAAI/bge-base-en-v1.5>`__;
15 -more are listed :ref:`here <supported_models>`.
13 +A: You can try [e5-mistral-7b-instruct](https://huggingface.co/intfloat/e5-mistral-7b-instruct) and [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5);
14 +more are listed [here](#supported-models).
15
17 -By extracting hidden states, vLLM can automatically convert text generation models like `Llama-3-8B <https://huggingface.co/meta-llama/Meta-Llama-3-8B>`__,
18 -`Mistral-7B-Instruct-v0.3 <https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3>`__ into embedding models,
16 +By extracting hidden states, vLLM can automatically convert text generation models like [Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B),
17 +[Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) into embedding models,
18 but they are expected be inferior to models that are specifically trained on embedding tasks.
19
21 -----------------------------------------
20 +______________________________________________________________________
21
23 - Q: Can the output of a prompt vary across runs in vLLM?
22 +> Q: Can the output of a prompt vary across runs in vLLM?
23
24 A: Yes, it can. vLLM does not guarantee stable log probabilities (logprobs) for the output tokens. Variations in logprobs may occur due to
26 -numerical instability in Torch operations or non-deterministic behavior in batched Torch operations when batching changes. For more details,
27 -see the `Numerical Accuracy section <https://pytorch.org/docs/stable/notes/numerical_accuracy.html#batched-computations-or-slice-computations>`_.
25 +numerical instability in Torch operations or non-deterministic behavior in batched Torch operations when batching changes. For more details,
26 +see the [Numerical Accuracy section](https://pytorch.org/docs/stable/notes/numerical_accuracy.html#batched-computations-or-slice-computations).
27
28 In vLLM, the same requests might be batched differently due to factors such as other concurrent requests,
30 -changes in batch size, or batch expansion in speculative decoding. These batching variations, combined with numerical instability of Torch operations,
31 -can lead to slightly different logit/logprob values at each step. Such differences can accumulate, potentially resulting in
29 +changes in batch size, or batch expansion in speculative decoding. These batching variations, combined with numerical instability of Torch operations,
30 +can lead to slightly different logit/logprob values at each step. Such differences can accumulate, potentially resulting in
31 different tokens being sampled. Once a different token is sampled, further divergence is likely.
32
33 **Mitigation Strategies**
docs/source/usage/lora.md new
+215
@@ -0,0 +1,215 @@
1 +(lora-adapter)=
2 +
3 +# LoRA Adapters
4 +
5 +This document shows you how to use [LoRA adapters](https://arxiv.org/abs/2106.09685) with vLLM on top of a base model.
6 +
7 +LoRA adapters can be used with any vLLM model that implements {class}`~vllm.model_executor.models.interfaces.SupportsLoRA`.
8 +
9 +Adapters can be efficiently served on a per request basis with minimal overhead. First we download the adapter(s) and save
10 +them locally with
11 +
12 +```python
13 +from huggingface_hub import snapshot_download
14 +
15 +sql_lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
16 +```
17 +
18 +Then we instantiate the base model and pass in the `enable_lora=True` flag:
19 +
20 +```python
21 +from vllm import LLM, SamplingParams
22 +from vllm.lora.request import LoRARequest
23 +
24 +llm = LLM(model="meta-llama/Llama-2-7b-hf", enable_lora=True)
25 +```
26 +
27 +We can now submit the prompts and call `llm.generate` with the `lora_request` parameter. The first parameter
28 +of `LoRARequest` is a human identifiable name, the second parameter is a globally unique ID for the adapter and
29 +the third parameter is the path to the LoRA adapter.
30 +
31 +```python
32 +sampling_params = SamplingParams(
33 + temperature=0,
34 + max_tokens=256,
35 + stop=["[/assistant]"]
36 +)
37 +
38 +prompts = [
39 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
40 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
41 +]
42 +
43 +outputs = llm.generate(
44 + prompts,
45 + sampling_params,
46 + lora_request=LoRARequest("sql_adapter", 1, sql_lora_path)
47 +)
48 +```
49 +
50 +Check out [examples/multilora_inference.py](https://github.com/vllm-project/vllm/blob/main/examples/multilora_inference.py)
51 +for an example of how to use LoRA adapters with the async engine and how to use more advanced configuration options.
52 +
53 +## Serving LoRA Adapters
54 +
55 +LoRA adapted models can also be served with the Open-AI compatible vLLM server. To do so, we use
56 +`--lora-modules {name}={path} {name}={path}` to specify each LoRA module when we kickoff the server:
57 +
58 +```bash
59 +vllm serve meta-llama/Llama-2-7b-hf \
60 + --enable-lora \
61 + --lora-modules sql-lora=$HOME/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/snapshots/0dfa347e8877a4d4ed19ee56c140fa518470028c/
62 +```
63 +
64 +```{note}
65 +The commit ID `0dfa347e8877a4d4ed19ee56c140fa518470028c` may change over time. Please check the latest commit ID in your environment to ensure you are using the correct one.
66 +```
67 +
68 +The server entrypoint accepts all other LoRA configuration parameters (`max_loras`, `max_lora_rank`, `max_cpu_loras`,
69 +etc.), which will apply to all forthcoming requests. Upon querying the `/models` endpoint, we should see our LoRA along
70 +with its base model:
71 +
72 +```bash
73 +curl localhost:8000/v1/models | jq .
74 +{
75 + "object": "list",
76 + "data": [
77 + {
78 + "id": "meta-llama/Llama-2-7b-hf",
79 + "object": "model",
80 + ...
81 + },
82 + {
83 + "id": "sql-lora",
84 + "object": "model",
85 + ...
86 + }
87 + ]
88 +}
89 +```
90 +
91 +Requests can specify the LoRA adapter as if it were any other model via the `model` request parameter. The requests will be
92 +processed according to the server-wide LoRA configuration (i.e. in parallel with base model requests, and potentially other
93 +LoRA adapter requests if they were provided and `max_loras` is set high enough).
94 +
95 +The following is an example request
96 +
97 +```bash
98 +curl http://localhost:8000/v1/completions \
99 + -H "Content-Type: application/json" \
100 + -d '{
101 + "model": "sql-lora",
102 + "prompt": "San Francisco is a",
103 + "max_tokens": 7,
104 + "temperature": 0
105 + }' | jq
106 +```
107 +
108 +## Dynamically serving LoRA Adapters
109 +
110 +In addition to serving LoRA adapters at server startup, the vLLM server now supports dynamically loading and unloading
111 +LoRA adapters at runtime through dedicated API endpoints. This feature can be particularly useful when the flexibility
112 +to change models on-the-fly is needed.
113 +
114 +Note: Enabling this feature in production environments is risky as user may participate model adapter management.
115 +
116 +To enable dynamic LoRA loading and unloading, ensure that the environment variable `VLLM_ALLOW_RUNTIME_LORA_UPDATING`
117 +is set to `True`. When this option is enabled, the API server will log a warning to indicate that dynamic loading is active.
118 +
119 +```bash
120 +export VLLM_ALLOW_RUNTIME_LORA_UPDATING=True
121 +```
122 +
123 +Loading a LoRA Adapter:
124 +
125 +To dynamically load a LoRA adapter, send a POST request to the `/v1/load_lora_adapter` endpoint with the necessary
126 +details of the adapter to be loaded. The request payload should include the name and path to the LoRA adapter.
127 +
128 +Example request to load a LoRA adapter:
129 +
130 +```bash
131 +curl -X POST http://localhost:8000/v1/load_lora_adapter \
132 +-H "Content-Type: application/json" \
133 +-d '{
134 + "lora_name": "sql_adapter",
135 + "lora_path": "/path/to/sql-lora-adapter"
136 +}'
137 +```
138 +
139 +Upon a successful request, the API will respond with a 200 OK status code. If an error occurs, such as if the adapter
140 +cannot be found or loaded, an appropriate error message will be returned.
141 +
142 +Unloading a LoRA Adapter:
143 +
144 +To unload a LoRA adapter that has been previously loaded, send a POST request to the `/v1/unload_lora_adapter` endpoint
145 +with the name or ID of the adapter to be unloaded.
146 +
147 +Example request to unload a LoRA adapter:
148 +
149 +```bash
150 +curl -X POST http://localhost:8000/v1/unload_lora_adapter \
151 +-H "Content-Type: application/json" \
152 +-d '{
153 + "lora_name": "sql_adapter"
154 +}'
155 +```
156 +
157 +## New format for `--lora-modules`
158 +
159 +In the previous version, users would provide LoRA modules via the following format, either as a key-value pair or in JSON format. For example:
160 +
161 +```bash
162 +--lora-modules sql-lora=$HOME/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/snapshots/0dfa347e8877a4d4ed19ee56c140fa518470028c/
163 +```
164 +
165 +This would only include the `name` and `path` for each LoRA module, but did not provide a way to specify a `base_model_name`.
166 +Now, you can specify a base_model_name alongside the name and path using JSON format. For example:
167 +
168 +```bash
169 +--lora-modules '{"name": "sql-lora", "path": "/path/to/lora", "base_model_name": "meta-llama/Llama-2-7b"}'
170 +```
171 +
172 +To provide the backward compatibility support, you can still use the old key-value format (name=path), but the `base_model_name` will remain unspecified in that case.
173 +
174 +## Lora model lineage in model card
175 +
176 +The new format of `--lora-modules` is mainly to support the display of parent model information in the model card. Here's an explanation of how your current response supports this:
177 +
178 +- The `parent` field of LoRA model `sql-lora` now links to its base model `meta-llama/Llama-2-7b-hf`. This correctly reflects the hierarchical relationship between the base model and the LoRA adapter.
179 +- The `root` field points to the artifact location of the lora adapter.
180 +
181 +```bash
182 +$ curl http://localhost:8000/v1/models
183 +
184 +{
185 + "object": "list",
186 + "data": [
187 + {
188 + "id": "meta-llama/Llama-2-7b-hf",
189 + "object": "model",
190 + "created": 1715644056,
191 + "owned_by": "vllm",
192 + "root": "~/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-hf/snapshots/01c7f73d771dfac7d292323805ebc428287df4f9/",
193 + "parent": null,
194 + "permission": [
195 + {
196 + .....
197 + }
198 + ]
199 + },
200 + {
201 + "id": "sql-lora",
202 + "object": "model",
203 + "created": 1715644056,
204 + "owned_by": "vllm",
205 + "root": "~/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/snapshots/0dfa347e8877a4d4ed19ee56c140fa518470028c/",
206 + "parent": meta-llama/Llama-2-7b-hf,
207 + "permission": [
208 + {
209 + ....
210 + }
211 + ]
212 + }
213 + ]
214 +}
215 +```
docs/source/usage/lora.rst deleted
-225
@@ -1,225 +0,0 @@
1 -.. _lora:
2 -
3 -LoRA Adapters
4 -=============
5 -
6 -This document shows you how to use `LoRA adapters <https://arxiv.org/abs/2106.09685>`_ with vLLM on top of a base model.
7 -
8 -LoRA adapters can be used with any vLLM model that implements :class:`~vllm.model_executor.models.interfaces.SupportsLoRA`.
9 -
10 -Adapters can be efficiently served on a per request basis with minimal overhead. First we download the adapter(s) and save
11 -them locally with
12 -
13 -.. code-block:: python
14 -
15 - from huggingface_hub import snapshot_download
16 -
17 - sql_lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
18 -
19 -
20 -Then we instantiate the base model and pass in the ``enable_lora=True`` flag:
21 -
22 -.. code-block:: python
23 -
24 - from vllm import LLM, SamplingParams
25 - from vllm.lora.request import LoRARequest
26 -
27 - llm = LLM(model="meta-llama/Llama-2-7b-hf", enable_lora=True)
28 -
29 -
30 -We can now submit the prompts and call ``llm.generate`` with the ``lora_request`` parameter. The first parameter
31 -of ``LoRARequest`` is a human identifiable name, the second parameter is a globally unique ID for the adapter and
32 -the third parameter is the path to the LoRA adapter.
33 -
34 -.. code-block:: python
35 -
36 - sampling_params = SamplingParams(
37 - temperature=0,
38 - max_tokens=256,
39 - stop=["[/assistant]"]
40 - )
41 -
42 - prompts = [
43 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
44 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
45 - ]
46 -
47 - outputs = llm.generate(
48 - prompts,
49 - sampling_params,
50 - lora_request=LoRARequest("sql_adapter", 1, sql_lora_path)
51 - )
52 -
53 -
54 -Check out `examples/multilora_inference.py <https://github.com/vllm-project/vllm/blob/main/examples/multilora_inference.py>`_
55 -for an example of how to use LoRA adapters with the async engine and how to use more advanced configuration options.
56 -
57 -Serving LoRA Adapters
58 ----------------------
59 -LoRA adapted models can also be served with the Open-AI compatible vLLM server. To do so, we use
60 -``--lora-modules {name}={path} {name}={path}`` to specify each LoRA module when we kickoff the server:
61 -
62 -.. code-block:: bash
63 -
64 - vllm serve meta-llama/Llama-2-7b-hf \
65 - --enable-lora \
66 - --lora-modules sql-lora=$HOME/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/snapshots/0dfa347e8877a4d4ed19ee56c140fa518470028c/
67 -
68 -.. note::
69 - The commit ID `0dfa347e8877a4d4ed19ee56c140fa518470028c` may change over time. Please check the latest commit ID in your environment to ensure you are using the correct one.
70 -
71 -The server entrypoint accepts all other LoRA configuration parameters (``max_loras``, ``max_lora_rank``, ``max_cpu_loras``,
72 -etc.), which will apply to all forthcoming requests. Upon querying the ``/models`` endpoint, we should see our LoRA along
73 -with its base model:
74 -
75 -.. code-block:: bash
76 -
77 - curl localhost:8000/v1/models | jq .
78 - {
79 - "object": "list",
80 - "data": [
81 - {
82 - "id": "meta-llama/Llama-2-7b-hf",
83 - "object": "model",
84 - ...
85 - },
86 - {
87 - "id": "sql-lora",
88 - "object": "model",
89 - ...
90 - }
91 - ]
92 - }
93 -
94 -Requests can specify the LoRA adapter as if it were any other model via the ``model`` request parameter. The requests will be
95 -processed according to the server-wide LoRA configuration (i.e. in parallel with base model requests, and potentially other
96 -LoRA adapter requests if they were provided and ``max_loras`` is set high enough).
97 -
98 -The following is an example request
99 -
100 -.. code-block:: bash
101 -
102 - curl http://localhost:8000/v1/completions \
103 - -H "Content-Type: application/json" \
104 - -d '{
105 - "model": "sql-lora",
106 - "prompt": "San Francisco is a",
107 - "max_tokens": 7,
108 - "temperature": 0
109 - }' | jq
110 -
111 -
112 -Dynamically serving LoRA Adapters
113 ----------------------------------
114 -
115 -In addition to serving LoRA adapters at server startup, the vLLM server now supports dynamically loading and unloading
116 -LoRA adapters at runtime through dedicated API endpoints. This feature can be particularly useful when the flexibility
117 -to change models on-the-fly is needed.
118 -
119 -Note: Enabling this feature in production environments is risky as user may participate model adapter management.
120 -
121 -To enable dynamic LoRA loading and unloading, ensure that the environment variable `VLLM_ALLOW_RUNTIME_LORA_UPDATING`
122 -is set to `True`. When this option is enabled, the API server will log a warning to indicate that dynamic loading is active.
123 -
124 -.. code-block:: bash
125 -
126 - export VLLM_ALLOW_RUNTIME_LORA_UPDATING=True
127 -
128 -
129 -Loading a LoRA Adapter:
130 -
131 -To dynamically load a LoRA adapter, send a POST request to the `/v1/load_lora_adapter` endpoint with the necessary
132 -details of the adapter to be loaded. The request payload should include the name and path to the LoRA adapter.
133 -
134 -Example request to load a LoRA adapter:
135 -
136 -.. code-block:: bash
137 -
138 - curl -X POST http://localhost:8000/v1/load_lora_adapter \
139 - -H "Content-Type: application/json" \
140 - -d '{
141 - "lora_name": "sql_adapter",
142 - "lora_path": "/path/to/sql-lora-adapter"
143 - }'
144 -
145 -Upon a successful request, the API will respond with a 200 OK status code. If an error occurs, such as if the adapter
146 -cannot be found or loaded, an appropriate error message will be returned.
147 -
148 -Unloading a LoRA Adapter:
149 -
150 -To unload a LoRA adapter that has been previously loaded, send a POST request to the `/v1/unload_lora_adapter` endpoint
151 -with the name or ID of the adapter to be unloaded.
152 -
153 -Example request to unload a LoRA adapter:
154 -
155 -.. code-block:: bash
156 -
157 - curl -X POST http://localhost:8000/v1/unload_lora_adapter \
158 - -H "Content-Type: application/json" \
159 - -d '{
160 - "lora_name": "sql_adapter"
161 - }'
162 -
163 -
164 -New format for `--lora-modules`
165 --------------------------------
166 -
167 -In the previous version, users would provide LoRA modules via the following format, either as a key-value pair or in JSON format. For example:
168 -
169 -.. code-block:: bash
170 -
171 - --lora-modules sql-lora=$HOME/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/snapshots/0dfa347e8877a4d4ed19ee56c140fa518470028c/
172 -
173 -This would only include the `name` and `path` for each LoRA module, but did not provide a way to specify a `base_model_name`.
174 -Now, you can specify a base_model_name alongside the name and path using JSON format. For example:
175 -
176 -.. code-block:: bash
177 -
178 - --lora-modules '{"name": "sql-lora", "path": "/path/to/lora", "base_model_name": "meta-llama/Llama-2-7b"}'
179 -
180 -To provide the backward compatibility support, you can still use the old key-value format (name=path), but the `base_model_name` will remain unspecified in that case.
181 -
182 -
183 -Lora model lineage in model card
184 ---------------------------------
185 -
186 -The new format of `--lora-modules` is mainly to support the display of parent model information in the model card. Here's an explanation of how your current response supports this:
187 -
188 -- The `parent` field of LoRA model `sql-lora` now links to its base model `meta-llama/Llama-2-7b-hf`. This correctly reflects the hierarchical relationship between the base model and the LoRA adapter.
189 -- The `root` field points to the artifact location of the lora adapter.
190 -
191 -.. code-block:: bash
192 -
193 - $ curl http://localhost:8000/v1/models
194 -
195 - {
196 - "object": "list",
197 - "data": [
198 - {
199 - "id": "meta-llama/Llama-2-7b-hf",
200 - "object": "model",
201 - "created": 1715644056,
202 - "owned_by": "vllm",
203 - "root": "~/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-hf/snapshots/01c7f73d771dfac7d292323805ebc428287df4f9/",
204 - "parent": null,
205 - "permission": [
206 - {
207 - .....
208 - }
209 - ]
210 - },
211 - {
212 - "id": "sql-lora",
213 - "object": "model",
214 - "created": 1715644056,
215 - "owned_by": "vllm",
216 - "root": "~/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/snapshots/0dfa347e8877a4d4ed19ee56c140fa518470028c/",
217 - "parent": meta-llama/Llama-2-7b-hf,
218 - "permission": [
219 - {
220 - ....
221 - }
222 - ]
223 - }
224 - ]
225 - }
docs/source/usage/multimodal_inputs.md new
+486
@@ -0,0 +1,486 @@
1 +(multimodal-inputs)=
2 +
3 +# Multimodal Inputs
4 +
5 +This page teaches you how to pass multi-modal inputs to [multi-modal models](#supported-mm-models) in vLLM.
6 +
7 +```{note}
8 +We are actively iterating on multi-modal support. See [this RFC](https://github.com/vllm-project/vllm/issues/4194) for upcoming changes,
9 +and [open an issue on GitHub](https://github.com/vllm-project/vllm/issues/new/choose) if you have any feedback or feature requests.
10 +```
11 +
12 +## Offline Inference
13 +
14 +To input multi-modal data, follow this schema in {class}`vllm.inputs.PromptType`:
15 +
16 +- `prompt`: The prompt should follow the format that is documented on HuggingFace.
17 +- `multi_modal_data`: This is a dictionary that follows the schema defined in {class}`vllm.multimodal.MultiModalDataDict`.
18 +
19 +### Image
20 +
21 +You can pass a single image to the {code}`'image'` field of the multi-modal dictionary, as shown in the following examples:
22 +
23 +```python
24 +llm = LLM(model="llava-hf/llava-1.5-7b-hf")
25 +
26 +# Refer to the HuggingFace repo for the correct format to use
27 +prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
28 +
29 +# Load the image using PIL.Image
30 +image = PIL.Image.open(...)
31 +
32 +# Single prompt inference
33 +outputs = llm.generate({
34 + "prompt": prompt,
35 + "multi_modal_data": {"image": image},
36 +})
37 +
38 +for o in outputs:
39 + generated_text = o.outputs[0].text
40 + print(generated_text)
41 +
42 +# Batch inference
43 +image_1 = PIL.Image.open(...)
44 +image_2 = PIL.Image.open(...)
45 +outputs = llm.generate(
46 + [
47 + {
48 + "prompt": "USER: <image>\nWhat is the content of this image?\nASSISTANT:",
49 + "multi_modal_data": {"image": image_1},
50 + },
51 + {
52 + "prompt": "USER: <image>\nWhat's the color of this image?\nASSISTANT:",
53 + "multi_modal_data": {"image": image_2},
54 + }
55 + ]
56 +)
57 +
58 +for o in outputs:
59 + generated_text = o.outputs[0].text
60 + print(generated_text)
61 +```
62 +
63 +A code example can be found in [examples/offline_inference_vision_language.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_vision_language.py).
64 +
65 +To substitute multiple images inside the same text prompt, you can pass in a list of images instead:
66 +
67 +```python
68 +llm = LLM(
69 + model="microsoft/Phi-3.5-vision-instruct",
70 + trust_remote_code=True, # Required to load Phi-3.5-vision
71 + max_model_len=4096, # Otherwise, it may not fit in smaller GPUs
72 + limit_mm_per_prompt={"image": 2}, # The maximum number to accept
73 +)
74 +
75 +# Refer to the HuggingFace repo for the correct format to use
76 +prompt = "<|user|>\n<|image_1|>\n<|image_2|>\nWhat is the content of each image?<|end|>\n<|assistant|>\n"
77 +
78 +# Load the images using PIL.Image
79 +image1 = PIL.Image.open(...)
80 +image2 = PIL.Image.open(...)
81 +
82 +outputs = llm.generate({
83 + "prompt": prompt,
84 + "multi_modal_data": {
85 + "image": [image1, image2]
86 + },
87 +})
88 +
89 +for o in outputs:
90 + generated_text = o.outputs[0].text
91 + print(generated_text)
92 +```
93 +
94 +A code example can be found in [examples/offline_inference_vision_language_multi_image.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_vision_language_multi_image.py).
95 +
96 +Multi-image input can be extended to perform video captioning. We show this with [Qwen2-VL](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct) as it supports videos:
97 +
98 +```python
99 +# Specify the maximum number of frames per video to be 4. This can be changed.
100 +llm = LLM("Qwen/Qwen2-VL-2B-Instruct", limit_mm_per_prompt={"image": 4})
101 +
102 +# Create the request payload.
103 +video_frames = ... # load your video making sure it only has the number of frames specified earlier.
104 +message = {
105 + "role": "user",
106 + "content": [
107 + {"type": "text", "text": "Describe this set of frames. Consider the frames to be a part of the same video."},
108 + ],
109 +}
110 +for i in range(len(video_frames)):
111 + base64_image = encode_image(video_frames[i]) # base64 encoding.
112 + new_image = {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}}
113 + message["content"].append(new_image)
114 +
115 +# Perform inference and log output.
116 +outputs = llm.chat([message])
117 +
118 +for o in outputs:
119 + generated_text = o.outputs[0].text
120 + print(generated_text)
121 +```
122 +
123 +### Video
124 +
125 +You can pass a list of NumPy arrays directly to the {code}`'video'` field of the multi-modal dictionary
126 +instead of using multi-image input.
127 +
128 +Please refer to [examples/offline_inference_vision_language.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_vision_language.py) for more details.
129 +
130 +### Audio
131 +
132 +You can pass a tuple {code}`(array, sampling_rate)` to the {code}`'audio'` field of the multi-modal dictionary.
133 +
134 +Please refer to [examples/offline_inference_audio_language.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_audio_language.py) for more details.
135 +
136 +### Embedding
137 +
138 +To input pre-computed embeddings belonging to a data type (i.e. image, video, or audio) directly to the language model,
139 +pass a tensor of shape {code}`(num_items, feature_size, hidden_size of LM)` to the corresponding field of the multi-modal dictionary.
140 +
141 +```python
142 +# Inference with image embeddings as input
143 +llm = LLM(model="llava-hf/llava-1.5-7b-hf")
144 +
145 +# Refer to the HuggingFace repo for the correct format to use
146 +prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
147 +
148 +# Embeddings for single image
149 +# torch.Tensor of shape (1, image_feature_size, hidden_size of LM)
150 +image_embeds = torch.load(...)
151 +
152 +outputs = llm.generate({
153 + "prompt": prompt,
154 + "multi_modal_data": {"image": image_embeds},
155 +})
156 +
157 +for o in outputs:
158 + generated_text = o.outputs[0].text
159 + print(generated_text)
160 +```
161 +
162 +For Qwen2-VL and MiniCPM-V, we accept additional parameters alongside the embeddings:
163 +
164 +```python
165 +# Construct the prompt based on your model
166 +prompt = ...
167 +
168 +# Embeddings for multiple images
169 +# torch.Tensor of shape (num_images, image_feature_size, hidden_size of LM)
170 +image_embeds = torch.load(...)
171 +
172 +# Qwen2-VL
173 +llm = LLM("Qwen/Qwen2-VL-2B-Instruct", limit_mm_per_prompt={"image": 4})
174 +mm_data = {
175 + "image": {
176 + "image_embeds": image_embeds,
177 + # image_grid_thw is needed to calculate positional encoding.
178 + "image_grid_thw": torch.load(...), # torch.Tensor of shape (1, 3),
179 + }
180 +}
181 +
182 +# MiniCPM-V
183 +llm = LLM("openbmb/MiniCPM-V-2_6", trust_remote_code=True, limit_mm_per_prompt={"image": 4})
184 +mm_data = {
185 + "image": {
186 + "image_embeds": image_embeds,
187 + # image_size_list is needed to calculate details of the sliced image.
188 + "image_size_list": [image.size for image in images], # list of image sizes
189 + }
190 +}
191 +
192 +outputs = llm.generate({
193 + "prompt": prompt,
194 + "multi_modal_data": mm_data,
195 +})
196 +
197 +for o in outputs:
198 + generated_text = o.outputs[0].text
199 + print(generated_text)
200 +```
201 +
202 +## Online Inference
203 +
204 +Our OpenAI-compatible server accepts multi-modal data via the [Chat Completions API](https://platform.openai.com/docs/api-reference/chat).
205 +
206 +```{important}
207 +A chat template is **required** to use Chat Completions API.
208 +
209 +Although most models come with a chat template, for others you have to define one yourself.
210 +The chat template can be inferred based on the documentation on the model's HuggingFace repo.
211 +For example, LLaVA-1.5 (`llava-hf/llava-1.5-7b-hf`) requires a chat template that can be found [here](https://github.com/vllm-project/vllm/blob/main/examples/template_llava.jinja).
212 +```
213 +
214 +### Image
215 +
216 +Image input is supported according to [OpenAI Vision API](https://platform.openai.com/docs/guides/vision).
217 +Here is a simple example using Phi-3.5-Vision.
218 +
219 +First, launch the OpenAI-compatible server:
220 +
221 +```bash
222 +vllm serve microsoft/Phi-3.5-vision-instruct --task generate \
223 + --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2
224 +```
225 +
226 +Then, you can use the OpenAI client as follows:
227 +
228 +```python
229 +from openai import OpenAI
230 +
231 +openai_api_key = "EMPTY"
232 +openai_api_base = "http://localhost:8000/v1"
233 +
234 +client = OpenAI(
235 + api_key=openai_api_key,
236 + base_url=openai_api_base,
237 +)
238 +
239 +# Single-image input inference
240 +image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
241 +
242 +chat_response = client.chat.completions.create(
243 + model="microsoft/Phi-3.5-vision-instruct",
244 + messages=[{
245 + "role": "user",
246 + "content": [
247 + # NOTE: The prompt formatting with the image token `<image>` is not needed
248 + # since the prompt will be processed automatically by the API server.
249 + {"type": "text", "text": "What’s in this image?"},
250 + {"type": "image_url", "image_url": {"url": image_url}},
251 + ],
252 + }],
253 +)
254 +print("Chat completion output:", chat_response.choices[0].message.content)
255 +
256 +# Multi-image input inference
257 +image_url_duck = "https://upload.wikimedia.org/wikipedia/commons/d/da/2015_Kaczka_krzy%C5%BCowka_w_wodzie_%28samiec%29.jpg"
258 +image_url_lion = "https://upload.wikimedia.org/wikipedia/commons/7/77/002_The_lion_king_Snyggve_in_the_Serengeti_National_Park_Photo_by_Giles_Laurent.jpg"
259 +
260 +chat_response = client.chat.completions.create(
261 + model="microsoft/Phi-3.5-vision-instruct",
262 + messages=[{
263 + "role": "user",
264 + "content": [
265 + {"type": "text", "text": "What are the animals in these images?"},
266 + {"type": "image_url", "image_url": {"url": image_url_duck}},
267 + {"type": "image_url", "image_url": {"url": image_url_lion}},
268 + ],
269 + }],
270 +)
271 +print("Chat completion output:", chat_response.choices[0].message.content)
272 +```
273 +
274 +A full code example can be found in [examples/openai_chat_completion_client_for_multimodal.py](https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client_for_multimodal.py).
275 +
276 +```{tip}
277 +Loading from local file paths is also supported on vLLM: You can specify the allowed local media path via `--allowed-local-media-path` when launching the API server/engine,
278 +and pass the file path as `url` in the API request.
279 +```
280 +
281 +```{tip}
282 +There is no need to place image placeholders in the text content of the API request - they are already represented by the image content.
283 +In fact, you can place image placeholders in the middle of the text by interleaving text and image content.
284 +```
285 +
286 +````{note}
287 +By default, the timeout for fetching images through HTTP URL is `5` seconds.
288 +You can override this by setting the environment variable:
289 +
290 +```console
291 +$ export VLLM_IMAGE_FETCH_TIMEOUT=<timeout>
292 +```
293 +````
294 +
295 +### Video
296 +
297 +Instead of {code}`image_url`, you can pass a video file via {code}`video_url`.
298 +
299 +You can use [these tests](https://github.com/vllm-project/vllm/blob/main/tests/entrypoints/openai/test_video.py) as reference.
300 +
301 +````{note}
302 +By default, the timeout for fetching videos through HTTP URL url is `30` seconds.
303 +You can override this by setting the environment variable:
304 +
305 +```console
306 +$ export VLLM_VIDEO_FETCH_TIMEOUT=<timeout>
307 +```
308 +````
309 +
310 +### Audio
311 +
312 +Audio input is supported according to [OpenAI Audio API](https://platform.openai.com/docs/guides/audio?audio-generation-quickstart-example=audio-in).
313 +Here is a simple example using Ultravox-v0.3.
314 +
315 +First, launch the OpenAI-compatible server:
316 +
317 +```bash
318 +vllm serve fixie-ai/ultravox-v0_3
319 +```
320 +
321 +Then, you can use the OpenAI client as follows:
322 +
323 +```python
324 +import base64
325 +import requests
326 +from openai import OpenAI
327 +from vllm.assets.audio import AudioAsset
328 +
329 +def encode_base64_content_from_url(content_url: str) -> str:
330 + """Encode a content retrieved from a remote url to base64 format."""
331 +
332 + with requests.get(content_url) as response:
333 + response.raise_for_status()
334 + result = base64.b64encode(response.content).decode('utf-8')
335 +
336 + return result
337 +
338 +openai_api_key = "EMPTY"
339 +openai_api_base = "http://localhost:8000/v1"
340 +
341 +client = OpenAI(
342 + api_key=openai_api_key,
343 + base_url=openai_api_base,
344 +)
345 +
346 +# Any format supported by librosa is supported
347 +audio_url = AudioAsset("winning_call").url
348 +audio_base64 = encode_base64_content_from_url(audio_url)
349 +
350 +chat_completion_from_base64 = client.chat.completions.create(
351 + messages=[{
352 + "role": "user",
353 + "content": [
354 + {
355 + "type": "text",
356 + "text": "What's in this audio?"
357 + },
358 + {
359 + "type": "input_audio",
360 + "input_audio": {
361 + "data": audio_base64,
362 + "format": "wav"
363 + },
364 + },
365 + ],
366 + }],
367 + model=model,
368 + max_completion_tokens=64,
369 +)
370 +
371 +result = chat_completion_from_base64.choices[0].message.content
372 +print("Chat completion output from input audio:", result)
373 +```
374 +
375 +Alternatively, you can pass {code}`audio_url`, which is the audio counterpart of {code}`image_url` for image input:
376 +
377 +```python
378 +chat_completion_from_url = client.chat.completions.create(
379 + messages=[{
380 + "role": "user",
381 + "content": [
382 + {
383 + "type": "text",
384 + "text": "What's in this audio?"
385 + },
386 + {
387 + "type": "audio_url",
388 + "audio_url": {
389 + "url": audio_url
390 + },
391 + },
392 + ],
393 + }],
394 + model=model,
395 + max_completion_tokens=64,
396 +)
397 +
398 +result = chat_completion_from_url.choices[0].message.content
399 +print("Chat completion output from audio url:", result)
400 +```
401 +
402 +A full code example can be found in [examples/openai_chat_completion_client_for_multimodal.py](https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client_for_multimodal.py).
403 +
404 +````{note}
405 +By default, the timeout for fetching audios through HTTP URL is `10` seconds.
406 +You can override this by setting the environment variable:
407 +
408 +```console
409 +$ export VLLM_AUDIO_FETCH_TIMEOUT=<timeout>
410 +```
411 +````
412 +
413 +### Embedding
414 +
415 +vLLM's Embeddings API is a superset of OpenAI's [Embeddings API](https://platform.openai.com/docs/api-reference/embeddings),
416 +where a list of chat `messages` can be passed instead of batched `inputs`. This enables multi-modal inputs to be passed to embedding models.
417 +
418 +```{tip}
419 +The schema of `messages` is exactly the same as in Chat Completions API.
420 +You can refer to the above tutorials for more details on how to pass each type of multi-modal data.
421 +```
422 +
423 +Usually, embedding models do not expect chat-based input, so we need to use a custom chat template to format the text and images.
424 +Refer to the examples below for illustration.
425 +
426 +Here is an end-to-end example using VLM2Vec. To serve the model:
427 +
428 +```bash
429 +vllm serve TIGER-Lab/VLM2Vec-Full --task embed \
430 + --trust-remote-code --max-model-len 4096 --chat-template examples/template_vlm2vec.jinja
431 +```
432 +
433 +```{important}
434 +Since VLM2Vec has the same model architecture as Phi-3.5-Vision, we have to explicitly pass `--task embed`
435 +to run this model in embedding mode instead of text generation mode.
436 +
437 +The custom chat template is completely different from the original one for this model,
438 +and can be found [here](https://github.com/vllm-project/vllm/blob/main/examples/template_vlm2vec.jinja).
439 +```
440 +
441 +Since the request schema is not defined by OpenAI client, we post a request to the server using the lower-level `requests` library:
442 +
443 +```python
444 +import requests
445 +
446 +image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
447 +
448 +response = requests.post(
449 + "http://localhost:8000/v1/embeddings",
450 + json={
451 + "model": "TIGER-Lab/VLM2Vec-Full",
452 + "messages": [{
453 + "role": "user",
454 + "content": [
455 + {"type": "image_url", "image_url": {"url": image_url}},
456 + {"type": "text", "text": "Represent the given image."},
457 + ],
458 + }],
459 + "encoding_format": "float",
460 + },
461 +)
462 +response.raise_for_status()
463 +response_json = response.json()
464 +print("Embedding output:", response_json["data"][0]["embedding"])
465 +```
466 +
467 +Below is another example, this time using the `MrLight/dse-qwen2-2b-mrl-v1` model.
468 +
469 +```bash
470 +vllm serve MrLight/dse-qwen2-2b-mrl-v1 --task embed \
471 + --trust-remote-code --max-model-len 8192 --chat-template examples/template_dse_qwen2_vl.jinja
472 +```
473 +
474 +```{important}
475 +Like with VLM2Vec, we have to explicitly pass `--task embed`.
476 +
477 +Additionally, `MrLight/dse-qwen2-2b-mrl-v1` requires an EOS token for embeddings, which is handled
478 +by [this custom chat template](https://github.com/vllm-project/vllm/blob/main/examples/template_dse_qwen2_vl.jinja).
479 +```
480 +
481 +```{important}
482 +Also important, `MrLight/dse-qwen2-2b-mrl-v1` requires a placeholder image of the minimum image size for text query embeddings. See the full code
483 +example below for details.
484 +```
485 +
486 +A full code example can be found in [examples/openai_chat_embedding_client_for_multimodal.py](https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_embedding_client_for_multimodal.py).
docs/source/usage/multimodal_inputs.rst deleted
-492
@@ -1,492 +0,0 @@
1 -.. _multimodal_inputs:
2 -
3 -Multimodal Inputs
4 -=================
5 -
6 -This page teaches you how to pass multi-modal inputs to :ref:`multi-modal models <supported_mm_models>` in vLLM.
7 -
8 -.. note::
9 - We are actively iterating on multi-modal support. See `this RFC <https://github.com/vllm-project/vllm/issues/4194>`_ for upcoming changes,
10 - and `open an issue on GitHub <https://github.com/vllm-project/vllm/issues/new/choose>`_ if you have any feedback or feature requests.
11 -
12 -Offline Inference
13 ------------------
14 -
15 -To input multi-modal data, follow this schema in :class:`vllm.inputs.PromptType`:
16 -
17 -* ``prompt``: The prompt should follow the format that is documented on HuggingFace.
18 -* ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
19 -
20 -Image
21 -^^^^^
22 -
23 -You can pass a single image to the :code:`'image'` field of the multi-modal dictionary, as shown in the following examples:
24 -
25 -.. code-block:: python
26 -
27 - llm = LLM(model="llava-hf/llava-1.5-7b-hf")
28 -
29 - # Refer to the HuggingFace repo for the correct format to use
30 - prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
31 -
32 - # Load the image using PIL.Image
33 - image = PIL.Image.open(...)
34 -
35 - # Single prompt inference
36 - outputs = llm.generate({
37 - "prompt": prompt,
38 - "multi_modal_data": {"image": image},
39 - })
40 -
41 - for o in outputs:
42 - generated_text = o.outputs[0].text
43 - print(generated_text)
44 -
45 - # Batch inference
46 - image_1 = PIL.Image.open(...)
47 - image_2 = PIL.Image.open(...)
48 - outputs = llm.generate(
49 - [
50 - {
51 - "prompt": "USER: <image>\nWhat is the content of this image?\nASSISTANT:",
52 - "multi_modal_data": {"image": image_1},
53 - },
54 - {
55 - "prompt": "USER: <image>\nWhat's the color of this image?\nASSISTANT:",
56 - "multi_modal_data": {"image": image_2},
57 - }
58 - ]
59 - )
60 -
61 - for o in outputs:
62 - generated_text = o.outputs[0].text
63 - print(generated_text)
64 -
65 -A code example can be found in `examples/offline_inference_vision_language.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_vision_language.py>`_.
66 -
67 -To substitute multiple images inside the same text prompt, you can pass in a list of images instead:
68 -
69 -.. code-block:: python
70 -
71 - llm = LLM(
72 - model="microsoft/Phi-3.5-vision-instruct",
73 - trust_remote_code=True, # Required to load Phi-3.5-vision
74 - max_model_len=4096, # Otherwise, it may not fit in smaller GPUs
75 - limit_mm_per_prompt={"image": 2}, # The maximum number to accept
76 - )
77 -
78 - # Refer to the HuggingFace repo for the correct format to use
79 - prompt = "<|user|>\n<|image_1|>\n<|image_2|>\nWhat is the content of each image?<|end|>\n<|assistant|>\n"
80 -
81 - # Load the images using PIL.Image
82 - image1 = PIL.Image.open(...)
83 - image2 = PIL.Image.open(...)
84 -
85 - outputs = llm.generate({
86 - "prompt": prompt,
87 - "multi_modal_data": {
88 - "image": [image1, image2]
89 - },
90 - })
91 -
92 - for o in outputs:
93 - generated_text = o.outputs[0].text
94 - print(generated_text)
95 -
96 -A code example can be found in `examples/offline_inference_vision_language_multi_image.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_vision_language_multi_image.py>`_.
97 -
98 -Multi-image input can be extended to perform video captioning. We show this with `Qwen2-VL <https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct>`_ as it supports videos:
99 -
100 -.. code-block:: python
101 -
102 - # Specify the maximum number of frames per video to be 4. This can be changed.
103 - llm = LLM("Qwen/Qwen2-VL-2B-Instruct", limit_mm_per_prompt={"image": 4})
104 -
105 - # Create the request payload.
106 - video_frames = ... # load your video making sure it only has the number of frames specified earlier.
107 - message = {
108 - "role": "user",
109 - "content": [
110 - {"type": "text", "text": "Describe this set of frames. Consider the frames to be a part of the same video."},
111 - ],
112 - }
113 - for i in range(len(video_frames)):
114 - base64_image = encode_image(video_frames[i]) # base64 encoding.
115 - new_image = {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}}
116 - message["content"].append(new_image)
117 -
118 - # Perform inference and log output.
119 - outputs = llm.chat([message])
120 -
121 - for o in outputs:
122 - generated_text = o.outputs[0].text
123 - print(generated_text)
124 -
125 -Video
126 -^^^^^
127 -
128 -You can pass a list of NumPy arrays directly to the :code:`'video'` field of the multi-modal dictionary
129 -instead of using multi-image input.
130 -
131 -Please refer to `examples/offline_inference_vision_language.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_vision_language.py>`_ for more details.
132 -
133 -Audio
134 -^^^^^
135 -
136 -You can pass a tuple :code:`(array, sampling_rate)` to the :code:`'audio'` field of the multi-modal dictionary.
137 -
138 -Please refer to `examples/offline_inference_audio_language.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_audio_language.py>`_ for more details.
139 -
140 -Embedding
141 -^^^^^^^^^
142 -
143 -To input pre-computed embeddings belonging to a data type (i.e. image, video, or audio) directly to the language model,
144 -pass a tensor of shape :code:`(num_items, feature_size, hidden_size of LM)` to the corresponding field of the multi-modal dictionary.
145 -
146 -.. code-block:: python
147 -
148 - # Inference with image embeddings as input
149 - llm = LLM(model="llava-hf/llava-1.5-7b-hf")
150 -
151 - # Refer to the HuggingFace repo for the correct format to use
152 - prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
153 -
154 - # Embeddings for single image
155 - # torch.Tensor of shape (1, image_feature_size, hidden_size of LM)
156 - image_embeds = torch.load(...)
157 -
158 - outputs = llm.generate({
159 - "prompt": prompt,
160 - "multi_modal_data": {"image": image_embeds},
161 - })
162 -
163 - for o in outputs:
164 - generated_text = o.outputs[0].text
165 - print(generated_text)
166 -
167 -For Qwen2-VL and MiniCPM-V, we accept additional parameters alongside the embeddings:
168 -
169 -.. code-block:: python
170 -
171 - # Construct the prompt based on your model
172 - prompt = ...
173 -
174 - # Embeddings for multiple images
175 - # torch.Tensor of shape (num_images, image_feature_size, hidden_size of LM)
176 - image_embeds = torch.load(...)
177 -
178 - # Qwen2-VL
179 - llm = LLM("Qwen/Qwen2-VL-2B-Instruct", limit_mm_per_prompt={"image": 4})
180 - mm_data = {
181 - "image": {
182 - "image_embeds": image_embeds,
183 - # image_grid_thw is needed to calculate positional encoding.
184 - "image_grid_thw": torch.load(...), # torch.Tensor of shape (1, 3),
185 - }
186 - }
187 -
188 - # MiniCPM-V
189 - llm = LLM("openbmb/MiniCPM-V-2_6", trust_remote_code=True, limit_mm_per_prompt={"image": 4})
190 - mm_data = {
191 - "image": {
192 - "image_embeds": image_embeds,
193 - # image_size_list is needed to calculate details of the sliced image.
194 - "image_size_list": [image.size for image in images], # list of image sizes
195 - }
196 - }
197 -
198 - outputs = llm.generate({
199 - "prompt": prompt,
200 - "multi_modal_data": mm_data,
201 - })
202 -
203 - for o in outputs:
204 - generated_text = o.outputs[0].text
205 - print(generated_text)
206 -
207 -Online Inference
208 -----------------
209 -
210 -Our OpenAI-compatible server accepts multi-modal data via the `Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`_.
211 -
212 -.. important::
213 - A chat template is **required** to use Chat Completions API.
214 -
215 - Although most models come with a chat template, for others you have to define one yourself.
216 - The chat template can be inferred based on the documentation on the model's HuggingFace repo.
217 - For example, LLaVA-1.5 (``llava-hf/llava-1.5-7b-hf``) requires a chat template that can be found `here <https://github.com/vllm-project/vllm/blob/main/examples/template_llava.jinja>`__.
218 -
219 -Image
220 -^^^^^
221 -
222 -Image input is supported according to `OpenAI Vision API <https://platform.openai.com/docs/guides/vision>`_.
223 -Here is a simple example using Phi-3.5-Vision.
224 -
225 -First, launch the OpenAI-compatible server:
226 -
227 -.. code-block:: bash
228 -
229 - vllm serve microsoft/Phi-3.5-vision-instruct --task generate \
230 - --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2
231 -
232 -Then, you can use the OpenAI client as follows:
233 -
234 -.. code-block:: python
235 -
236 - from openai import OpenAI
237 -
238 - openai_api_key = "EMPTY"
239 - openai_api_base = "http://localhost:8000/v1"
240 -
241 - client = OpenAI(
242 - api_key=openai_api_key,
243 - base_url=openai_api_base,
244 - )
245 -
246 - # Single-image input inference
247 - image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
248 -
249 - chat_response = client.chat.completions.create(
250 - model="microsoft/Phi-3.5-vision-instruct",
251 - messages=[{
252 - "role": "user",
253 - "content": [
254 - # NOTE: The prompt formatting with the image token `<image>` is not needed
255 - # since the prompt will be processed automatically by the API server.
256 - {"type": "text", "text": "What’s in this image?"},
257 - {"type": "image_url", "image_url": {"url": image_url}},
258 - ],
259 - }],
260 - )
261 - print("Chat completion output:", chat_response.choices[0].message.content)
262 -
263 - # Multi-image input inference
264 - image_url_duck = "https://upload.wikimedia.org/wikipedia/commons/d/da/2015_Kaczka_krzy%C5%BCowka_w_wodzie_%28samiec%29.jpg"
265 - image_url_lion = "https://upload.wikimedia.org/wikipedia/commons/7/77/002_The_lion_king_Snyggve_in_the_Serengeti_National_Park_Photo_by_Giles_Laurent.jpg"
266 -
267 - chat_response = client.chat.completions.create(
268 - model="microsoft/Phi-3.5-vision-instruct",
269 - messages=[{
270 - "role": "user",
271 - "content": [
272 - {"type": "text", "text": "What are the animals in these images?"},
273 - {"type": "image_url", "image_url": {"url": image_url_duck}},
274 - {"type": "image_url", "image_url": {"url": image_url_lion}},
275 - ],
276 - }],
277 - )
278 - print("Chat completion output:", chat_response.choices[0].message.content)
279 -
280 -A full code example can be found in `examples/openai_chat_completion_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client_for_multimodal.py>`_.
281 -
282 -.. tip::
283 - Loading from local file paths is also supported on vLLM: You can specify the allowed local media path via ``--allowed-local-media-path`` when launching the API server/engine,
284 - and pass the file path as ``url`` in the API request.
285 -
286 -.. tip::
287 - There is no need to place image placeholders in the text content of the API request - they are already represented by the image content.
288 - In fact, you can place image placeholders in the middle of the text by interleaving text and image content.
289 -
290 -.. note::
291 -
292 - By default, the timeout for fetching images through HTTP URL is ``5`` seconds.
293 - You can override this by setting the environment variable:
294 -
295 - .. code-block:: console
296 -
297 - $ export VLLM_IMAGE_FETCH_TIMEOUT=<timeout>
298 -
299 -Video
300 -^^^^^
301 -
302 -Instead of :code:`image_url`, you can pass a video file via :code:`video_url`.
303 -
304 -You can use `these tests <https://github.com/vllm-project/vllm/blob/main/tests/entrypoints/openai/test_video.py>`_ as reference.
305 -
306 -.. note::
307 -
308 - By default, the timeout for fetching videos through HTTP URL url is ``30`` seconds.
309 - You can override this by setting the environment variable:
310 -
311 - .. code-block:: console
312 -
313 - $ export VLLM_VIDEO_FETCH_TIMEOUT=<timeout>
314 -
315 -Audio
316 -^^^^^
317 -
318 -Audio input is supported according to `OpenAI Audio API <https://platform.openai.com/docs/guides/audio?audio-generation-quickstart-example=audio-in>`_.
319 -Here is a simple example using Ultravox-v0.3.
320 -
321 -First, launch the OpenAI-compatible server:
322 -
323 -.. code-block:: bash
324 -
325 - vllm serve fixie-ai/ultravox-v0_3
326 -
327 -Then, you can use the OpenAI client as follows:
328 -
329 -.. code-block:: python
330 -
331 - import base64
332 - import requests
333 - from openai import OpenAI
334 - from vllm.assets.audio import AudioAsset
335 -
336 - def encode_base64_content_from_url(content_url: str) -> str:
337 - """Encode a content retrieved from a remote url to base64 format."""
338 -
339 - with requests.get(content_url) as response:
340 - response.raise_for_status()
341 - result = base64.b64encode(response.content).decode('utf-8')
342 -
343 - return result
344 -
345 - openai_api_key = "EMPTY"
346 - openai_api_base = "http://localhost:8000/v1"
347 -
348 - client = OpenAI(
349 - api_key=openai_api_key,
350 - base_url=openai_api_base,
351 - )
352 -
353 - # Any format supported by librosa is supported
354 - audio_url = AudioAsset("winning_call").url
355 - audio_base64 = encode_base64_content_from_url(audio_url)
356 -
357 - chat_completion_from_base64 = client.chat.completions.create(
358 - messages=[{
359 - "role": "user",
360 - "content": [
361 - {
362 - "type": "text",
363 - "text": "What's in this audio?"
364 - },
365 - {
366 - "type": "input_audio",
367 - "input_audio": {
368 - "data": audio_base64,
369 - "format": "wav"
370 - },
371 - },
372 - ],
373 - }],
374 - model=model,
375 - max_completion_tokens=64,
376 - )
377 -
378 - result = chat_completion_from_base64.choices[0].message.content
379 - print("Chat completion output from input audio:", result)
380 -
381 -Alternatively, you can pass :code:`audio_url`, which is the audio counterpart of :code:`image_url` for image input:
382 -
383 -.. code-block:: python
384 -
385 - chat_completion_from_url = client.chat.completions.create(
386 - messages=[{
387 - "role": "user",
388 - "content": [
389 - {
390 - "type": "text",
391 - "text": "What's in this audio?"
392 - },
393 - {
394 - "type": "audio_url",
395 - "audio_url": {
396 - "url": audio_url
397 - },
398 - },
399 - ],
400 - }],
401 - model=model,
402 - max_completion_tokens=64,
403 - )
404 -
405 - result = chat_completion_from_url.choices[0].message.content
406 - print("Chat completion output from audio url:", result)
407 -
408 -A full code example can be found in `examples/openai_chat_completion_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client_for_multimodal.py>`_.
409 -
410 -.. note::
411 -
412 - By default, the timeout for fetching audios through HTTP URL is ``10`` seconds.
413 - You can override this by setting the environment variable:
414 -
415 - .. code-block:: console
416 -
417 - $ export VLLM_AUDIO_FETCH_TIMEOUT=<timeout>
418 -
419 -Embedding
420 -^^^^^^^^^
421 -
422 -vLLM's Embeddings API is a superset of OpenAI's `Embeddings API <https://platform.openai.com/docs/api-reference/embeddings>`_,
423 -where a list of chat ``messages`` can be passed instead of batched ``inputs``. This enables multi-modal inputs to be passed to embedding models.
424 -
425 -.. tip::
426 - The schema of ``messages`` is exactly the same as in Chat Completions API.
427 - You can refer to the above tutorials for more details on how to pass each type of multi-modal data.
428 -
429 -Usually, embedding models do not expect chat-based input, so we need to use a custom chat template to format the text and images.
430 -Refer to the examples below for illustration.
431 -
432 -Here is an end-to-end example using VLM2Vec. To serve the model:
433 -
434 -.. code-block:: bash
435 -
436 - vllm serve TIGER-Lab/VLM2Vec-Full --task embed \
437 - --trust-remote-code --max-model-len 4096 --chat-template examples/template_vlm2vec.jinja
438 -
439 -.. important::
440 -
441 - Since VLM2Vec has the same model architecture as Phi-3.5-Vision, we have to explicitly pass ``--task embed``
442 - to run this model in embedding mode instead of text generation mode.
443 -
444 - The custom chat template is completely different from the original one for this model,
445 - and can be found `here <https://github.com/vllm-project/vllm/blob/main/examples/template_vlm2vec.jinja>`__.
446 -
447 -Since the request schema is not defined by OpenAI client, we post a request to the server using the lower-level ``requests`` library:
448 -
449 -.. code-block:: python
450 -
451 - import requests
452 -
453 - image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
454 -
455 - response = requests.post(
456 - "http://localhost:8000/v1/embeddings",
457 - json={
458 - "model": "TIGER-Lab/VLM2Vec-Full",
459 - "messages": [{
460 - "role": "user",
461 - "content": [
462 - {"type": "image_url", "image_url": {"url": image_url}},
463 - {"type": "text", "text": "Represent the given image."},
464 - ],
465 - }],
466 - "encoding_format": "float",
467 - },
468 - )
469 - response.raise_for_status()
470 - response_json = response.json()
471 - print("Embedding output:", response_json["data"][0]["embedding"])
472 -
473 -Below is another example, this time using the ``MrLight/dse-qwen2-2b-mrl-v1`` model.
474 -
475 -.. code-block:: bash
476 -
477 - vllm serve MrLight/dse-qwen2-2b-mrl-v1 --task embed \
478 - --trust-remote-code --max-model-len 8192 --chat-template examples/template_dse_qwen2_vl.jinja
479 -
480 -.. important::
481 -
482 - Like with VLM2Vec, we have to explicitly pass ``--task embed``.
483 -
484 - Additionally, ``MrLight/dse-qwen2-2b-mrl-v1`` requires an EOS token for embeddings, which is handled
485 - by `this custom chat template <https://github.com/vllm-project/vllm/blob/main/examples/template_dse_qwen2_vl.jinja>`__.
486 -
487 -.. important::
488 -
489 - Also important, ``MrLight/dse-qwen2-2b-mrl-v1`` requires a placeholder image of the minimum image size for text query embeddings. See the full code
490 - example below for details.
491 -
492 -A full code example can be found in `examples/openai_chat_embedding_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_embedding_client_for_multimodal.py>`_.
docs/source/usage/performance.md renamed
+25 -26
@@ -1,16 +1,15 @@
1 -.. _performance:
1 +(performance)=
2
3 -Performance and Tuning
4 -======================
3 +# Performance and Tuning
4 +
5 +## Preemption
6
6 -Preemption
7 -----------
7 Due to the auto-regressive nature of transformer architecture, there are times when KV cache space is insufficient to handle all batched requests.
8 The vLLM can preempt requests to free up KV cache space for other requests. Preempted requests are recomputed when sufficient KV cache space becomes
9 available again. When this occurs, the following warning is printed:
10
11 ```
13 -WARNING 05-09 00:49:33 scheduler.py:1057] Sequence group 0 is preempted by PreemptionMode.SWAP mode because there is not enough KV cache space. This can affect the end-to-end performance. Increase gpu_memory_utilization or tensor_parallel_size to provide more KV cache memory. total_cumulative_preemption_cnt=1
12 +WARNING 05-09 00:49:33 scheduler.py:1057 Sequence group 0 is preempted by PreemptionMode.SWAP mode because there is not enough KV cache space. This can affect the end-to-end performance. Increase gpu_memory_utilization or tensor_parallel_size to provide more KV cache memory. total_cumulative_preemption_cnt=1
13 ```
14
15 While this mechanism ensures system robustness, preemption and recomputation can adversely affect end-to-end latency.
@@ -22,44 +21,44 @@ If you frequently encounter preemptions from the vLLM engine, consider the follo
21
22 You can also monitor the number of preemption requests through Prometheus metrics exposed by the vLLM. Additionally, you can log the cumulative number of preemption requests by setting disable_log_stats=False.
23
25 -.. _chunked-prefill:
24 +(chunked-prefill)=
25
27 -Chunked Prefill
28 ----------------
29 -vLLM supports an experimental feature chunked prefill. Chunked prefill allows to chunk large prefills into smaller chunks and batch them together with decode requests.
26 +## Chunked Prefill
27
31 -You can enable the feature by specifying ``--enable-chunked-prefill`` in the command line or setting ``enable_chunked_prefill=True`` in the LLM constructor.
28 +vLLM supports an experimental feature chunked prefill. Chunked prefill allows to chunk large prefills into smaller chunks and batch them together with decode requests.
29
33 -.. code-block:: python
30 +You can enable the feature by specifying `--enable-chunked-prefill` in the command line or setting `enable_chunked_prefill=True` in the LLM constructor.
31
35 - llm = LLM(model="meta-llama/Llama-2-7b-hf", enable_chunked_prefill=True)
36 - # Set max_num_batched_tokens to tune performance.
37 - # NOTE: 512 is the default max_num_batched_tokens for chunked prefill.
38 - # llm = LLM(model="meta-llama/Llama-2-7b-hf", enable_chunked_prefill=True, max_num_batched_tokens=512)
32 +```python
33 +llm = LLM(model="meta-llama/Llama-2-7b-hf", enable_chunked_prefill=True)
34 +# Set max_num_batched_tokens to tune performance.
35 +# NOTE: 512 is the default max_num_batched_tokens for chunked prefill.
36 +# llm = LLM(model="meta-llama/Llama-2-7b-hf", enable_chunked_prefill=True, max_num_batched_tokens=512)
37 +```
38
39 By default, vLLM scheduler prioritizes prefills and doesn't batch prefill and decode to the same batch.
40 This policy optimizes the TTFT (time to the first token), but incurs slower ITL (inter token latency) and inefficient GPU utilization.
41
42 Once chunked prefill is enabled, the policy is changed to prioritize decode requests.
43 It batches all pending decode requests to the batch before scheduling any prefill.
45 -When there are available token_budget (``max_num_batched_tokens``), it schedules pending prefills.
46 -If a last pending prefill request cannot fit into ``max_num_batched_tokens``, it chunks it.
44 +When there are available token_budget (`max_num_batched_tokens`), it schedules pending prefills.
45 +If a last pending prefill request cannot fit into `max_num_batched_tokens`, it chunks it.
46
47 This policy has two benefits:
48
49 - It improves ITL and generation decode because decode requests are prioritized.
50 - It helps achieve better GPU utilization by locating compute-bound (prefill) and memory-bound (decode) requests to the same batch.
51
53 -You can tune the performance by changing ``max_num_batched_tokens``.
52 +You can tune the performance by changing `max_num_batched_tokens`.
53 By default, it is set to 512, which has the best ITL on A100 in the initial benchmark (llama 70B and mixtral 8x22B).
55 -Smaller ``max_num_batched_tokens`` achieves better ITL because there are fewer prefills interrupting decodes.
56 -Higher ``max_num_batched_tokens`` achieves better TTFT as you can put more prefill to the batch.
54 +Smaller `max_num_batched_tokens` achieves better ITL because there are fewer prefills interrupting decodes.
55 +Higher `max_num_batched_tokens` achieves better TTFT as you can put more prefill to the batch.
56
58 -- If ``max_num_batched_tokens`` is the same as ``max_model_len``, that's almost the equivalent to the default scheduling policy (except that it still prioritizes decodes).
59 -- Note that the default value (512) of ``max_num_batched_tokens`` is optimized for ITL, and it may have lower throughput than the default scheduler.
57 +- If `max_num_batched_tokens` is the same as `max_model_len`, that's almost the equivalent to the default scheduling policy (except that it still prioritizes decodes).
58 +- Note that the default value (512) of `max_num_batched_tokens` is optimized for ITL, and it may have lower throughput than the default scheduler.
59
61 -We recommend you set ``max_num_batched_tokens > 2048`` for throughput.
60 +We recommend you set `max_num_batched_tokens > 2048` for throughput.
61
63 -See related papers for more details (https://arxiv.org/pdf/2401.08671 or https://arxiv.org/pdf/2308.16369).
62 +See related papers for more details (<https://arxiv.org/pdf/2401.08671> or <https://arxiv.org/pdf/2308.16369>).
63
65 -Please try out this feature and let us know your feedback via GitHub issues!
\ No newline at end of file
64 +Please try out this feature and let us know your feedback via GitHub issues!
docs/source/usage/spec_decode.md new
+205
@@ -0,0 +1,205 @@
1 +(spec-decode)=
2 +
3 +# Speculative decoding
4 +
5 +```{warning}
6 +Please note that speculative decoding in vLLM is not yet optimized and does
7 +not usually yield inter-token latency reductions for all prompt datasets or sampling parameters. The work
8 +to optimize it is ongoing and can be followed in [this issue.](https://github.com/vllm-project/vllm/issues/4630)
9 +```
10 +
11 +```{warning}
12 +Currently, speculative decoding in vLLM is not compatible with pipeline parallelism.
13 +```
14 +
15 +This document shows how to use [Speculative Decoding](https://x.com/karpathy/status/1697318534555336961) with vLLM.
16 +Speculative decoding is a technique which improves inter-token latency in memory-bound LLM inference.
17 +
18 +## Speculating with a draft model
19 +
20 +The following code configures vLLM in an offline mode to use speculative decoding with a draft model, speculating 5 tokens at a time.
21 +
22 +```python
23 +from vllm import LLM, SamplingParams
24 +
25 +prompts = [
26 + "The future of AI is",
27 +]
28 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
29 +
30 +llm = LLM(
31 + model="facebook/opt-6.7b",
32 + tensor_parallel_size=1,
33 + speculative_model="facebook/opt-125m",
34 + num_speculative_tokens=5,
35 +)
36 +outputs = llm.generate(prompts, sampling_params)
37 +
38 +for output in outputs:
39 + prompt = output.prompt
40 + generated_text = output.outputs[0].text
41 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
42 +```
43 +
44 +To perform the same with an online mode launch the server:
45 +
46 +```bash
47 +python -m vllm.entrypoints.openai.api_server --host 0.0.0.0 --port 8000 --model facebook/opt-6.7b \
48 + --seed 42 -tp 1 --speculative_model facebook/opt-125m --use-v2-block-manager \
49 + --num_speculative_tokens 5 --gpu_memory_utilization 0.8
50 +```
51 +
52 +Then use a client:
53 +
54 +```python
55 +from openai import OpenAI
56 +
57 +# Modify OpenAI's API key and API base to use vLLM's API server.
58 +openai_api_key = "EMPTY"
59 +openai_api_base = "http://localhost:8000/v1"
60 +
61 +client = OpenAI(
62 + # defaults to os.environ.get("OPENAI_API_KEY")
63 + api_key=openai_api_key,
64 + base_url=openai_api_base,
65 +)
66 +
67 +models = client.models.list()
68 +model = models.data[0].id
69 +
70 +# Completion API
71 +stream = False
72 +completion = client.completions.create(
73 + model=model,
74 + prompt="The future of AI is",
75 + echo=False,
76 + n=1,
77 + stream=stream,
78 +)
79 +
80 +print("Completion results:")
81 +if stream:
82 + for c in completion:
83 + print(c)
84 +else:
85 + print(completion)
86 +```
87 +
88 +## Speculating by matching n-grams in the prompt
89 +
90 +The following code configures vLLM to use speculative decoding where proposals are generated by
91 +matching n-grams in the prompt. For more information read [this thread.](https://x.com/joao_gante/status/1747322413006643259)
92 +
93 +```python
94 +from vllm import LLM, SamplingParams
95 +
96 +prompts = [
97 + "The future of AI is",
98 +]
99 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
100 +
101 +llm = LLM(
102 + model="facebook/opt-6.7b",
103 + tensor_parallel_size=1,
104 + speculative_model="[ngram]",
105 + num_speculative_tokens=5,
106 + ngram_prompt_lookup_max=4,
107 +)
108 +outputs = llm.generate(prompts, sampling_params)
109 +
110 +for output in outputs:
111 + prompt = output.prompt
112 + generated_text = output.outputs[0].text
113 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
114 +```
115 +
116 +## Speculating using MLP speculators
117 +
118 +The following code configures vLLM to use speculative decoding where proposals are generated by
119 +draft models that conditioning draft predictions on both context vectors and sampled tokens.
120 +For more information see [this blog](https://pytorch.org/blog/hitchhikers-guide-speculative-decoding/) or
121 +[this technical report](https://arxiv.org/abs/2404.19124).
122 +
123 +```python
124 +from vllm import LLM, SamplingParams
125 +
126 +prompts = [
127 + "The future of AI is",
128 +]
129 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
130 +
131 +llm = LLM(
132 + model="meta-llama/Meta-Llama-3.1-70B-Instruct",
133 + tensor_parallel_size=4,
134 + speculative_model="ibm-fms/llama3-70b-accelerator",
135 + speculative_draft_tensor_parallel_size=1,
136 +)
137 +outputs = llm.generate(prompts, sampling_params)
138 +
139 +for output in outputs:
140 + prompt = output.prompt
141 + generated_text = output.outputs[0].text
142 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
143 +```
144 +
145 +Note that these speculative models currently need to be run without tensor parallelism, although
146 +it is possible to run the main model using tensor parallelism (see example above). Since the
147 +speculative models are relatively small, we still see significant speedups. However, this
148 +limitation will be fixed in a future release.
149 +
150 +A variety of speculative models of this type are available on HF hub:
151 +
152 +- [llama-13b-accelerator](https://huggingface.co/ibm-fms/llama-13b-accelerator)
153 +- [llama3-8b-accelerator](https://huggingface.co/ibm-fms/llama3-8b-accelerator)
154 +- [codellama-34b-accelerator](https://huggingface.co/ibm-fms/codellama-34b-accelerator)
155 +- [llama2-70b-accelerator](https://huggingface.co/ibm-fms/llama2-70b-accelerator)
156 +- [llama3-70b-accelerator](https://huggingface.co/ibm-fms/llama3-70b-accelerator)
157 +- [granite-3b-code-instruct-accelerator](https://huggingface.co/ibm-granite/granite-3b-code-instruct-accelerator)
158 +- [granite-8b-code-instruct-accelerator](https://huggingface.co/ibm-granite/granite-8b-code-instruct-accelerator)
159 +- [granite-7b-instruct-accelerator](https://huggingface.co/ibm-granite/granite-7b-instruct-accelerator)
160 +- [granite-20b-code-instruct-accelerator](https://huggingface.co/ibm-granite/granite-20b-code-instruct-accelerator)
161 +
162 +## Lossless guarantees of Speculative Decoding
163 +
164 +In vLLM, speculative decoding aims to enhance inference efficiency while maintaining accuracy. This section addresses the lossless guarantees of
165 +speculative decoding, breaking down the guarantees into three key areas:
166 +
167 +1. **Theoretical Losslessness**
168 + \- Speculative decoding sampling is theoretically lossless up to the precision limits of hardware numerics. Floating-point errors might
169 + cause slight variations in output distributions, as discussed
170 + in [Accelerating Large Language Model Decoding with Speculative Sampling](https://arxiv.org/pdf/2302.01318)
171 +
172 +2. **Algorithmic Losslessness**
173 + \- vLLM’s implementation of speculative decoding is algorithmically validated to be lossless. Key validation tests include:
174 +
175 + > - **Rejection Sampler Convergence**: Ensures that samples from vLLM’s rejection sampler align with the target
176 + > distribution. [View Test Code](https://github.com/vllm-project/vllm/blob/47b65a550866c7ffbd076ecb74106714838ce7da/tests/samplers/test_rejection_sampler.py#L252)
177 + > - **Greedy Sampling Equality**: Confirms that greedy sampling with speculative decoding matches greedy sampling
178 + > without it. This verifies that vLLM's speculative decoding framework, when integrated with the vLLM forward pass and the vLLM rejection sampler,
179 + > provides a lossless guarantee. Almost all of the tests in [this directory](https://github.com/vllm-project/vllm/tree/b67ae00cdbbe1a58ffc8ff170f0c8d79044a684a/tests/spec_decode/e2e)
180 + > verify this property using [this assertion implementation](https://github.com/vllm-project/vllm/blob/b67ae00cdbbe1a58ffc8ff170f0c8d79044a684a/tests/spec_decode/e2e/conftest.py#L291)
181 +
182 +3. **vLLM Logprob Stability**
183 + \- vLLM does not currently guarantee stable token log probabilities (logprobs). This can result in different outputs for the
184 + same request across runs. For more details, see the FAQ section
185 + titled *Can the output of a prompt vary across runs in vLLM?* in the {ref}`FAQs <faq>`.
186 +
187 +**Conclusion**
188 +
189 +While vLLM strives to ensure losslessness in speculative decoding, variations in generated outputs with and without speculative decoding
190 +can occur due to following factors:
191 +
192 +- **Floating-Point Precision**: Differences in hardware numerical precision may lead to slight discrepancies in the output distribution.
193 +- **Batch Size and Numerical Stability**: Changes in batch size may cause variations in logprobs and output probabilities, potentially
194 + due to non-deterministic behavior in batched operations or numerical instability.
195 +
196 +**Mitigation Strategies**
197 +
198 +For mitigation strategies, please refer to the FAQ entry *Can the output of a prompt vary across runs in vLLM?* in the {ref}`FAQs <faq>`.
199 +
200 +## Resources for vLLM contributors
201 +
202 +- [A Hacker's Guide to Speculative Decoding in vLLM](https://www.youtube.com/watch?v=9wNAgpX6z_4)
203 +- [What is Lookahead Scheduling in vLLM?](https://docs.google.com/document/d/1Z9TvqzzBPnh5WHcRwjvK2UEeFeq5zMZb5mFE8jR0HCs/edit#heading=h.1fjfb0donq5a)
204 +- [Information on batch expansion](https://docs.google.com/document/d/1T-JaS2T1NRfdP51qzqpyakoCXxSXTtORppiwaj5asxA/edit#heading=h.kk7dq05lc6q8)
205 +- [Dynamic speculative decoding](https://github.com/vllm-project/vllm/issues/4565)
docs/source/usage/spec_decode.rst deleted
-210
@@ -1,210 +0,0 @@
1 -.. _spec_decode:
2 -
3 -Speculative decoding
4 -====================
5 -
6 -.. warning::
7 - Please note that speculative decoding in vLLM is not yet optimized and does
8 - not usually yield inter-token latency reductions for all prompt datasets or sampling parameters. The work
9 - to optimize it is ongoing and can be followed in `this issue. <https://github.com/vllm-project/vllm/issues/4630>`_
10 -
11 -.. warning::
12 - Currently, speculative decoding in vLLM is not compatible with pipeline parallelism.
13 -
14 -This document shows how to use `Speculative Decoding <https://x.com/karpathy/status/1697318534555336961>`_ with vLLM.
15 -Speculative decoding is a technique which improves inter-token latency in memory-bound LLM inference.
16 -
17 -Speculating with a draft model
18 -------------------------------
19 -
20 -The following code configures vLLM in an offline mode to use speculative decoding with a draft model, speculating 5 tokens at a time.
21 -
22 -.. code-block:: python
23 -
24 - from vllm import LLM, SamplingParams
25 -
26 - prompts = [
27 - "The future of AI is",
28 - ]
29 - sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
30 -
31 - llm = LLM(
32 - model="facebook/opt-6.7b",
33 - tensor_parallel_size=1,
34 - speculative_model="facebook/opt-125m",
35 - num_speculative_tokens=5,
36 - )
37 - outputs = llm.generate(prompts, sampling_params)
38 -
39 - for output in outputs:
40 - prompt = output.prompt
41 - generated_text = output.outputs[0].text
42 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
43 -
44 -To perform the same with an online mode launch the server:
45 -
46 -.. code-block:: bash
47 -
48 - python -m vllm.entrypoints.openai.api_server --host 0.0.0.0 --port 8000 --model facebook/opt-6.7b \
49 - --seed 42 -tp 1 --speculative_model facebook/opt-125m --use-v2-block-manager \
50 - --num_speculative_tokens 5 --gpu_memory_utilization 0.8
51 -
52 -Then use a client:
53 -
54 -.. code-block:: python
55 -
56 - from openai import OpenAI
57 -
58 - # Modify OpenAI's API key and API base to use vLLM's API server.
59 - openai_api_key = "EMPTY"
60 - openai_api_base = "http://localhost:8000/v1"
61 -
62 - client = OpenAI(
63 - # defaults to os.environ.get("OPENAI_API_KEY")
64 - api_key=openai_api_key,
65 - base_url=openai_api_base,
66 - )
67 -
68 - models = client.models.list()
69 - model = models.data[0].id
70 -
71 - # Completion API
72 - stream = False
73 - completion = client.completions.create(
74 - model=model,
75 - prompt="The future of AI is",
76 - echo=False,
77 - n=1,
78 - stream=stream,
79 - )
80 -
81 - print("Completion results:")
82 - if stream:
83 - for c in completion:
84 - print(c)
85 - else:
86 - print(completion)
87 -
88 -Speculating by matching n-grams in the prompt
89 ----------------------------------------------
90 -
91 -The following code configures vLLM to use speculative decoding where proposals are generated by
92 -matching n-grams in the prompt. For more information read `this thread. <https://x.com/joao_gante/status/1747322413006643259>`_
93 -
94 -.. code-block:: python
95 -
96 - from vllm import LLM, SamplingParams
97 -
98 - prompts = [
99 - "The future of AI is",
100 - ]
101 - sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
102 -
103 - llm = LLM(
104 - model="facebook/opt-6.7b",
105 - tensor_parallel_size=1,
106 - speculative_model="[ngram]",
107 - num_speculative_tokens=5,
108 - ngram_prompt_lookup_max=4,
109 - )
110 - outputs = llm.generate(prompts, sampling_params)
111 -
112 - for output in outputs:
113 - prompt = output.prompt
114 - generated_text = output.outputs[0].text
115 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
116 -
117 -Speculating using MLP speculators
118 ----------------------------------
119 -
120 -The following code configures vLLM to use speculative decoding where proposals are generated by
121 -draft models that conditioning draft predictions on both context vectors and sampled tokens.
122 -For more information see `this blog <https://pytorch.org/blog/hitchhikers-guide-speculative-decoding/>`_ or
123 -`this technical report <https://arxiv.org/abs/2404.19124>`_.
124 -
125 -.. code-block:: python
126 -
127 - from vllm import LLM, SamplingParams
128 -
129 - prompts = [
130 - "The future of AI is",
131 - ]
132 - sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
133 -
134 - llm = LLM(
135 - model="meta-llama/Meta-Llama-3.1-70B-Instruct",
136 - tensor_parallel_size=4,
137 - speculative_model="ibm-fms/llama3-70b-accelerator",
138 - speculative_draft_tensor_parallel_size=1,
139 - )
140 - outputs = llm.generate(prompts, sampling_params)
141 -
142 - for output in outputs:
143 - prompt = output.prompt
144 - generated_text = output.outputs[0].text
145 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
146 -
147 -Note that these speculative models currently need to be run without tensor parallelism, although
148 -it is possible to run the main model using tensor parallelism (see example above). Since the
149 -speculative models are relatively small, we still see significant speedups. However, this
150 -limitation will be fixed in a future release.
151 -
152 -A variety of speculative models of this type are available on HF hub:
153 -
154 -* `llama-13b-accelerator <https://huggingface.co/ibm-fms/llama-13b-accelerator>`_
155 -* `llama3-8b-accelerator <https://huggingface.co/ibm-fms/llama3-8b-accelerator>`_
156 -* `codellama-34b-accelerator <https://huggingface.co/ibm-fms/codellama-34b-accelerator>`_
157 -* `llama2-70b-accelerator <https://huggingface.co/ibm-fms/llama2-70b-accelerator>`_
158 -* `llama3-70b-accelerator <https://huggingface.co/ibm-fms/llama3-70b-accelerator>`_
159 -* `granite-3b-code-instruct-accelerator <https://huggingface.co/ibm-granite/granite-3b-code-instruct-accelerator>`_
160 -* `granite-8b-code-instruct-accelerator <https://huggingface.co/ibm-granite/granite-8b-code-instruct-accelerator>`_
161 -* `granite-7b-instruct-accelerator <https://huggingface.co/ibm-granite/granite-7b-instruct-accelerator>`_
162 -* `granite-20b-code-instruct-accelerator <https://huggingface.co/ibm-granite/granite-20b-code-instruct-accelerator>`_
163 -
164 -Lossless guarantees of Speculative Decoding
165 --------------------------------------------
166 -In vLLM, speculative decoding aims to enhance inference efficiency while maintaining accuracy. This section addresses the lossless guarantees of
167 -speculative decoding, breaking down the guarantees into three key areas:
168 -
169 -1. **Theoretical Losslessness**
170 - - Speculative decoding sampling is theoretically lossless up to the precision limits of hardware numerics. Floating-point errors might
171 - cause slight variations in output distributions, as discussed
172 - in `Accelerating Large Language Model Decoding with Speculative Sampling <https://arxiv.org/pdf/2302.01318>`_
173 -
174 -2. **Algorithmic Losslessness**
175 - - vLLM’s implementation of speculative decoding is algorithmically validated to be lossless. Key validation tests include:
176 -
177 - - **Rejection Sampler Convergence**: Ensures that samples from vLLM’s rejection sampler align with the target
178 - distribution. `View Test Code <https://github.com/vllm-project/vllm/blob/47b65a550866c7ffbd076ecb74106714838ce7da/tests/samplers/test_rejection_sampler.py#L252>`_
179 -
180 - - **Greedy Sampling Equality**: Confirms that greedy sampling with speculative decoding matches greedy sampling
181 - without it. This verifies that vLLM's speculative decoding framework, when integrated with the vLLM forward pass and the vLLM rejection sampler,
182 - provides a lossless guarantee. Almost all of the tests in `this directory <https://github.com/vllm-project/vllm/tree/b67ae00cdbbe1a58ffc8ff170f0c8d79044a684a/tests/spec_decode/e2e>`_
183 - verify this property using `this assertion implementation <https://github.com/vllm-project/vllm/blob/b67ae00cdbbe1a58ffc8ff170f0c8d79044a684a/tests/spec_decode/e2e/conftest.py#L291>`_
184 -
185 -3. **vLLM Logprob Stability**
186 - - vLLM does not currently guarantee stable token log probabilities (logprobs). This can result in different outputs for the
187 - same request across runs. For more details, see the FAQ section
188 - titled *Can the output of a prompt vary across runs in vLLM?* in the :ref:`FAQs <faq>`.
189 -
190 -
191 -**Conclusion**
192 -
193 -While vLLM strives to ensure losslessness in speculative decoding, variations in generated outputs with and without speculative decoding
194 -can occur due to following factors:
195 -
196 -- **Floating-Point Precision**: Differences in hardware numerical precision may lead to slight discrepancies in the output distribution.
197 -
198 -- **Batch Size and Numerical Stability**: Changes in batch size may cause variations in logprobs and output probabilities, potentially
199 - due to non-deterministic behavior in batched operations or numerical instability.
200 -
201 -**Mitigation Strategies**
202 -
203 -For mitigation strategies, please refer to the FAQ entry *Can the output of a prompt vary across runs in vLLM?* in the :ref:`FAQs <faq>`.
204 -
205 -Resources for vLLM contributors
206 --------------------------------
207 -* `A Hacker's Guide to Speculative Decoding in vLLM <https://www.youtube.com/watch?v=9wNAgpX6z_4>`_
208 -* `What is Lookahead Scheduling in vLLM? <https://docs.google.com/document/d/1Z9TvqzzBPnh5WHcRwjvK2UEeFeq5zMZb5mFE8jR0HCs/edit#heading=h.1fjfb0donq5a>`_
209 -* `Information on batch expansion <https://docs.google.com/document/d/1T-JaS2T1NRfdP51qzqpyakoCXxSXTtORppiwaj5asxA/edit#heading=h.kk7dq05lc6q8>`_
210 -* `Dynamic speculative decoding <https://github.com/vllm-project/vllm/issues/4565>`_
docs/source/usage/structured_outputs.md new
+260
@@ -0,0 +1,260 @@
1 +(structured-outputs)=
2 +
3 +# Structured Outputs
4 +
5 +vLLM supports the generation of structured outputs using [outlines](https://github.com/dottxt-ai/outlines) or [lm-format-enforcer](https://github.com/noamgat/lm-format-enforcer) as backends for the guided decoding.
6 +This document shows you some examples of the different options that are available to generate structured outputs.
7 +
8 +## Online Inference (OpenAI API)
9 +
10 +You can generate structured outputs using the OpenAI's [Completions](https://platform.openai.com/docs/api-reference/completions) and [Chat](https://platform.openai.com/docs/api-reference/chat) API.
11 +
12 +The following parameters are supported, which must be added as extra parameters:
13 +
14 +- `guided_choice`: the output will be exactly one of the choices.
15 +- `guided_regex`: the output will follow the regex pattern.
16 +- `guided_json`: the output will follow the JSON schema.
17 +- `guided_grammar`: the output will follow the context free grammar.
18 +- `guided_whitespace_pattern`: used to override the default whitespace pattern for guided json decoding.
19 +- `guided_decoding_backend`: used to select the guided decoding backend to use.
20 +
21 +You can see the complete list of supported parameters on the [OpenAI Compatible Server](../serving/openai_compatible_server.md) page.
22 +
23 +Now let´s see an example for each of the cases, starting with the `guided_choice`, as it´s the easiest one:
24 +
25 +```python
26 +from openai import OpenAI
27 +client = OpenAI(
28 + base_url="http://localhost:8000/v1",
29 + api_key="-",
30 +)
31 +
32 +completion = client.chat.completions.create(
33 + model="Qwen/Qwen2.5-3B-Instruct",
34 + messages=[
35 + {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}
36 + ],
37 + extra_body={"guided_choice": ["positive", "negative"]},
38 +)
39 +print(completion.choices[0].message.content)
40 +```
41 +
42 +The next example shows how to use the `guided_regex`. The idea is to generate an email address, given a simple regex template:
43 +
44 +```python
45 +completion = client.chat.completions.create(
46 + model="Qwen/Qwen2.5-3B-Instruct",
47 + messages=[
48 + {
49 + "role": "user",
50 + "content": "Generate an example email address for Alan Turing, who works in Enigma. End in .com and new line. Example result: alan.turing@enigma.com\n",
51 + }
52 + ],
53 + extra_body={"guided_regex": "\w+@\w+\.com\n", "stop": ["\n"]},
54 +)
55 +print(completion.choices[0].message.content)
56 +```
57 +
58 +One of the most relevant features in structured text generation is the option to generate a valid JSON with pre-defined fields and formats.
59 +For this we can use the `guided_json` parameter in two different ways:
60 +
61 +- Using directly a [JSON Schema](https://json-schema.org/)
62 +- Defining a [Pydantic model](https://docs.pydantic.dev/latest/) and then extracting the JSON Schema from it (which is normally an easier option).
63 +
64 +The next example shows how to use the `guided_json` parameter with a Pydantic model:
65 +
66 +```python
67 +from pydantic import BaseModel
68 +from enum import Enum
69 +
70 +class CarType(str, Enum):
71 + sedan = "sedan"
72 + suv = "SUV"
73 + truck = "Truck"
74 + coupe = "Coupe"
75 +
76 +
77 +class CarDescription(BaseModel):
78 + brand: str
79 + model: str
80 + car_type: CarType
81 +
82 +
83 +json_schema = CarDescription.model_json_schema()
84 +
85 +completion = client.chat.completions.create(
86 + model="Qwen/Qwen2.5-3B-Instruct",
87 + messages=[
88 + {
89 + "role": "user",
90 + "content": "Generate a JSON with the brand, model and car_type of the most iconic car from the 90's",
91 + }
92 + ],
93 + extra_body={"guided_json": json_schema},
94 +)
95 +print(completion.choices[0].message.content)
96 +```
97 +
98 +```{tip}
99 +While not strictly necessary, normally it´s better to indicate in the prompt that a JSON needs to be generated and which fields and how should the LLM fill them.
100 +This can improve the results notably in most cases.
101 +```
102 +
103 +Finally we have the `guided_grammar`, which probably is the most difficult one to use but it´s really powerful, as it allows us to define complete languages like SQL queries.
104 +It works by using a context free EBNF grammar, which for example we can use to define a specific format of simplified SQL queries, like in the example below:
105 +
106 +```python
107 +simplified_sql_grammar = """
108 + ?start: select_statement
109 +
110 + ?select_statement: "SELECT " column_list " FROM " table_name
111 +
112 + ?column_list: column_name ("," column_name)*
113 +
114 + ?table_name: identifier
115 +
116 + ?column_name: identifier
117 +
118 + ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
119 +"""
120 +
121 +completion = client.chat.completions.create(
122 + model="Qwen/Qwen2.5-3B-Instruct",
123 + messages=[
124 + {
125 + "role": "user",
126 + "content": "Generate an SQL query to show the 'username' and 'email' from the 'users' table.",
127 + }
128 + ],
129 + extra_body={"guided_grammar": simplified_sql_grammar},
130 +)
131 +print(completion.choices[0].message.content)
132 +```
133 +
134 +The complete code of the examples can be found on [examples/openai_chat_completion_structured_outputs.py](https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_structured_outputs.py).
135 +
136 +## Experimental Automatic Parsing (OpenAI API)
137 +
138 +This section covers the OpenAI beta wrapper over the `client.chat.completions.create()` method that provides richer integrations with Python specific types.
139 +
140 +At the time of writing (`openai==1.54.4`), this is a "beta" feature in the OpenAI client library. Code reference can be found [here](https://github.com/openai/openai-python/blob/52357cff50bee57ef442e94d78a0de38b4173fc2/src/openai/resources/beta/chat/completions.py#L100-L104).
141 +
142 +For the following examples, vLLM was setup using `vllm serve meta-llama/Llama-3.1-8B-Instruct`
143 +
144 +Here is a simple example demonstrating how to get structured output using Pydantic models:
145 +
146 +```python
147 +from pydantic import BaseModel
148 +from openai import OpenAI
149 +
150 +
151 +class Info(BaseModel):
152 + name: str
153 + age: int
154 +
155 +
156 +client = OpenAI(base_url="http://0.0.0.0:8000/v1", api_key="dummy")
157 +completion = client.beta.chat.completions.parse(
158 + model="meta-llama/Llama-3.1-8B-Instruct",
159 + messages=[
160 + {"role": "system", "content": "You are a helpful assistant."},
161 + {"role": "user", "content": "My name is Cameron, I'm 28. What's my name and age?"},
162 + ],
163 + response_format=Info,
164 + extra_body=dict(guided_decoding_backend="outlines"),
165 +)
166 +
167 +message = completion.choices[0].message
168 +print(message)
169 +assert message.parsed
170 +print("Name:", message.parsed.name)
171 +print("Age:", message.parsed.age)
172 +```
173 +
174 +Output:
175 +
176 +```console
177 +ParsedChatCompletionMessage[Testing](content='{"name": "Cameron", "age": 28}', refusal=None, role='assistant', audio=None, function_call=None, tool_calls=[], parsed=Testing(name='Cameron', age=28))
178 +Name: Cameron
179 +Age: 28
180 +```
181 +
182 +Here is a more complex example using nested Pydantic models to handle a step-by-step math solution:
183 +
184 +```python
185 +from typing import List
186 +from pydantic import BaseModel
187 +from openai import OpenAI
188 +
189 +
190 +class Step(BaseModel):
191 + explanation: str
192 + output: str
193 +
194 +
195 +class MathResponse(BaseModel):
196 + steps: List[Step]
197 + final_answer: str
198 +
199 +
200 +client = OpenAI(base_url="http://0.0.0.0:8000/v1", api_key="dummy")
201 +completion = client.beta.chat.completions.parse(
202 + model="meta-llama/Llama-3.1-8B-Instruct",
203 + messages=[
204 + {"role": "system", "content": "You are a helpful expert math tutor."},
205 + {"role": "user", "content": "Solve 8x + 31 = 2."},
206 + ],
207 + response_format=MathResponse,
208 + extra_body=dict(guided_decoding_backend="outlines"),
209 +)
210 +
211 +message = completion.choices[0].message
212 +print(message)
213 +assert message.parsed
214 +for i, step in enumerate(message.parsed.steps):
215 + print(f"Step #{i}:", step)
216 +print("Answer:", message.parsed.final_answer)
217 +```
218 +
219 +Output:
220 +
221 +```console
222 +ParsedChatCompletionMessage[MathResponse](content='{ "steps": [{ "explanation": "First, let\'s isolate the term with the variable \'x\'. To do this, we\'ll subtract 31 from both sides of the equation.", "output": "8x + 31 - 31 = 2 - 31"}, { "explanation": "By subtracting 31 from both sides, we simplify the equation to 8x = -29.", "output": "8x = -29"}, { "explanation": "Next, let\'s isolate \'x\' by dividing both sides of the equation by 8.", "output": "8x / 8 = -29 / 8"}], "final_answer": "x = -29/8" }', refusal=None, role='assistant', audio=None, function_call=None, tool_calls=[], parsed=MathResponse(steps=[Step(explanation="First, let's isolate the term with the variable 'x'. To do this, we'll subtract 31 from both sides of the equation.", output='8x + 31 - 31 = 2 - 31'), Step(explanation='By subtracting 31 from both sides, we simplify the equation to 8x = -29.', output='8x = -29'), Step(explanation="Next, let's isolate 'x' by dividing both sides of the equation by 8.", output='8x / 8 = -29 / 8')], final_answer='x = -29/8'))
223 +Step #0: explanation="First, let's isolate the term with the variable 'x'. To do this, we'll subtract 31 from both sides of the equation." output='8x + 31 - 31 = 2 - 31'
224 +Step #1: explanation='By subtracting 31 from both sides, we simplify the equation to 8x = -29.' output='8x = -29'
225 +Step #2: explanation="Next, let's isolate 'x' by dividing both sides of the equation by 8." output='8x / 8 = -29 / 8'
226 +Answer: x = -29/8
227 +```
228 +
229 +## Offline Inference
230 +
231 +Offline inference allows for the same types of guided decoding.
232 +To use it, we´ll need to configure the guided decoding using the class `GuidedDecodingParams` inside `SamplingParams`.
233 +The main available options inside `GuidedDecodingParams` are:
234 +
235 +- `json`
236 +- `regex`
237 +- `choice`
238 +- `grammar`
239 +- `backend`
240 +- `whitespace_pattern`
241 +
242 +These parameters can be used in the same way as the parameters from the Online Inference examples above.
243 +One example for the usage of the `choices` parameter is shown below:
244 +
245 +```python
246 +from vllm import LLM, SamplingParams
247 +from vllm.sampling_params import GuidedDecodingParams
248 +
249 +llm = LLM(model="HuggingFaceTB/SmolLM2-1.7B-Instruct")
250 +
251 +guided_decoding_params = GuidedDecodingParams(choice=["Positive", "Negative"])
252 +sampling_params = SamplingParams(guided_decoding=guided_decoding_params)
253 +outputs = llm.generate(
254 + prompts="Classify this sentiment: vLLM is wonderful!",
255 + sampling_params=sampling_params,
256 +)
257 +print(outputs[0].outputs[0].text)
258 +```
259 +
260 +A complete example with all options can be found in [examples/offline_inference_structured_outputs.py](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_structured_outputs.py).
docs/source/usage/structured_outputs.rst deleted
-267
@@ -1,267 +0,0 @@
1 -.. _structured_outputs:
2 -
3 -Structured Outputs
4 -==================
5 -
6 -vLLM supports the generation of structured outputs using `outlines <https://github.com/dottxt-ai/outlines>`_ or `lm-format-enforcer <https://github.com/noamgat/lm-format-enforcer>`_ as backends for the guided decoding.
7 -This document shows you some examples of the different options that are available to generate structured outputs.
8 -
9 -
10 -Online Inference (OpenAI API)
11 ------------------------------
12 -
13 -You can generate structured outputs using the OpenAI's `Completions <https://platform.openai.com/docs/api-reference/completions>`_ and `Chat <https://platform.openai.com/docs/api-reference/chat>`_ API.
14 -
15 -The following parameters are supported, which must be added as extra parameters:
16 -
17 -- ``guided_choice``: the output will be exactly one of the choices.
18 -- ``guided_regex``: the output will follow the regex pattern.
19 -- ``guided_json``: the output will follow the JSON schema.
20 -- ``guided_grammar``: the output will follow the context free grammar.
21 -- ``guided_whitespace_pattern``: used to override the default whitespace pattern for guided json decoding.
22 -- ``guided_decoding_backend``: used to select the guided decoding backend to use.
23 -
24 -You can see the complete list of supported parameters on the `OpenAI Compatible Server </../serving/openai_compatible_server.html>`_ page.
25 -
26 -Now let´s see an example for each of the cases, starting with the ``guided_choice``, as it´s the easiest one:
27 -
28 -.. code-block:: python
29 -
30 - from openai import OpenAI
31 - client = OpenAI(
32 - base_url="http://localhost:8000/v1",
33 - api_key="-",
34 - )
35 -
36 - completion = client.chat.completions.create(
37 - model="Qwen/Qwen2.5-3B-Instruct",
38 - messages=[
39 - {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}
40 - ],
41 - extra_body={"guided_choice": ["positive", "negative"]},
42 - )
43 - print(completion.choices[0].message.content)
44 -
45 -
46 -The next example shows how to use the ``guided_regex``. The idea is to generate an email address, given a simple regex template:
47 -
48 -.. code-block:: python
49 -
50 - completion = client.chat.completions.create(
51 - model="Qwen/Qwen2.5-3B-Instruct",
52 - messages=[
53 - {
54 - "role": "user",
55 - "content": "Generate an example email address for Alan Turing, who works in Enigma. End in .com and new line. Example result: alan.turing@enigma.com\n",
56 - }
57 - ],
58 - extra_body={"guided_regex": "\w+@\w+\.com\n", "stop": ["\n"]},
59 - )
60 - print(completion.choices[0].message.content)
61 -
62 -One of the most relevant features in structured text generation is the option to generate a valid JSON with pre-defined fields and formats.
63 -For this we can use the ``guided_json`` parameter in two different ways:
64 -
65 -- Using directly a `JSON Schema <https://json-schema.org/>`_
66 -- Defining a `Pydantic model <https://docs.pydantic.dev/latest/>`_ and then extracting the JSON Schema from it (which is normally an easier option).
67 -
68 -The next example shows how to use the ``guided_json`` parameter with a Pydantic model:
69 -
70 -.. code-block:: python
71 -
72 - from pydantic import BaseModel
73 - from enum import Enum
74 -
75 - class CarType(str, Enum):
76 - sedan = "sedan"
77 - suv = "SUV"
78 - truck = "Truck"
79 - coupe = "Coupe"
80 -
81 -
82 - class CarDescription(BaseModel):
83 - brand: str
84 - model: str
85 - car_type: CarType
86 -
87 -
88 - json_schema = CarDescription.model_json_schema()
89 -
90 - completion = client.chat.completions.create(
91 - model="Qwen/Qwen2.5-3B-Instruct",
92 - messages=[
93 - {
94 - "role": "user",
95 - "content": "Generate a JSON with the brand, model and car_type of the most iconic car from the 90's",
96 - }
97 - ],
98 - extra_body={"guided_json": json_schema},
99 - )
100 - print(completion.choices[0].message.content)
101 -
102 -.. tip::
103 - While not strictly necessary, normally it´s better to indicate in the prompt that a JSON needs to be generated and which fields and how should the LLM fill them.
104 - This can improve the results notably in most cases.
105 -
106 -
107 -Finally we have the ``guided_grammar``, which probably is the most difficult one to use but it´s really powerful, as it allows us to define complete languages like SQL queries.
108 -It works by using a context free EBNF grammar, which for example we can use to define a specific format of simplified SQL queries, like in the example below:
109 -
110 -.. code-block:: python
111 -
112 - simplified_sql_grammar = """
113 - ?start: select_statement
114 -
115 - ?select_statement: "SELECT " column_list " FROM " table_name
116 -
117 - ?column_list: column_name ("," column_name)*
118 -
119 - ?table_name: identifier
120 -
121 - ?column_name: identifier
122 -
123 - ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
124 - """
125 -
126 - completion = client.chat.completions.create(
127 - model="Qwen/Qwen2.5-3B-Instruct",
128 - messages=[
129 - {
130 - "role": "user",
131 - "content": "Generate an SQL query to show the 'username' and 'email' from the 'users' table.",
132 - }
133 - ],
134 - extra_body={"guided_grammar": simplified_sql_grammar},
135 - )
136 - print(completion.choices[0].message.content)
137 -
138 -The complete code of the examples can be found on `examples/openai_chat_completion_structured_outputs.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_structured_outputs.py>`_.
139 -
140 -Experimental Automatic Parsing (OpenAI API)
141 ---------------------------------------------
142 -
143 -This section covers the OpenAI beta wrapper over the ``client.chat.completions.create()`` method that provides richer integrations with Python specific types.
144 -
145 -At the time of writing (``openai==1.54.4``), this is a "beta" feature in the OpenAI client library. Code reference can be found `here <https://github.com/openai/openai-python/blob/52357cff50bee57ef442e94d78a0de38b4173fc2/src/openai/resources/beta/chat/completions.py#L100-L104>`_.
146 -
147 -For the following examples, vLLM was setup using ``vllm serve meta-llama/Llama-3.1-8B-Instruct``
148 -
149 -Here is a simple example demonstrating how to get structured output using Pydantic models:
150 -
151 -.. code-block:: python
152 -
153 - from pydantic import BaseModel
154 - from openai import OpenAI
155 -
156 -
157 - class Info(BaseModel):
158 - name: str
159 - age: int
160 -
161 -
162 - client = OpenAI(base_url="http://0.0.0.0:8000/v1", api_key="dummy")
163 - completion = client.beta.chat.completions.parse(
164 - model="meta-llama/Llama-3.1-8B-Instruct",
165 - messages=[
166 - {"role": "system", "content": "You are a helpful assistant."},
167 - {"role": "user", "content": "My name is Cameron, I'm 28. What's my name and age?"},
168 - ],
169 - response_format=Info,
170 - extra_body=dict(guided_decoding_backend="outlines"),
171 - )
172 -
173 - message = completion.choices[0].message
174 - print(message)
175 - assert message.parsed
176 - print("Name:", message.parsed.name)
177 - print("Age:", message.parsed.age)
178 -
179 -Output:
180 -
181 -.. code-block:: console
182 -
183 - ParsedChatCompletionMessage[Testing](content='{"name": "Cameron", "age": 28}', refusal=None, role='assistant', audio=None, function_call=None, tool_calls=[], parsed=Testing(name='Cameron', age=28))
184 - Name: Cameron
185 - Age: 28
186 -
187 -
188 -Here is a more complex example using nested Pydantic models to handle a step-by-step math solution:
189 -
190 -.. code-block:: python
191 -
192 - from typing import List
193 - from pydantic import BaseModel
194 - from openai import OpenAI
195 -
196 -
197 - class Step(BaseModel):
198 - explanation: str
199 - output: str
200 -
201 -
202 - class MathResponse(BaseModel):
203 - steps: List[Step]
204 - final_answer: str
205 -
206 -
207 - client = OpenAI(base_url="http://0.0.0.0:8000/v1", api_key="dummy")
208 - completion = client.beta.chat.completions.parse(
209 - model="meta-llama/Llama-3.1-8B-Instruct",
210 - messages=[
211 - {"role": "system", "content": "You are a helpful expert math tutor."},
212 - {"role": "user", "content": "Solve 8x + 31 = 2."},
213 - ],
214 - response_format=MathResponse,
215 - extra_body=dict(guided_decoding_backend="outlines"),
216 - )
217 -
218 - message = completion.choices[0].message
219 - print(message)
220 - assert message.parsed
221 - for i, step in enumerate(message.parsed.steps):
222 - print(f"Step #{i}:", step)
223 - print("Answer:", message.parsed.final_answer)
224 -
225 -Output:
226 -
227 -.. code-block:: console
228 -
229 - ParsedChatCompletionMessage[MathResponse](content='{ "steps": [{ "explanation": "First, let\'s isolate the term with the variable \'x\'. To do this, we\'ll subtract 31 from both sides of the equation.", "output": "8x + 31 - 31 = 2 - 31"}, { "explanation": "By subtracting 31 from both sides, we simplify the equation to 8x = -29.", "output": "8x = -29"}, { "explanation": "Next, let\'s isolate \'x\' by dividing both sides of the equation by 8.", "output": "8x / 8 = -29 / 8"}], "final_answer": "x = -29/8" }', refusal=None, role='assistant', audio=None, function_call=None, tool_calls=[], parsed=MathResponse(steps=[Step(explanation="First, let's isolate the term with the variable 'x'. To do this, we'll subtract 31 from both sides of the equation.", output='8x + 31 - 31 = 2 - 31'), Step(explanation='By subtracting 31 from both sides, we simplify the equation to 8x = -29.', output='8x = -29'), Step(explanation="Next, let's isolate 'x' by dividing both sides of the equation by 8.", output='8x / 8 = -29 / 8')], final_answer='x = -29/8'))
230 - Step #0: explanation="First, let's isolate the term with the variable 'x'. To do this, we'll subtract 31 from both sides of the equation." output='8x + 31 - 31 = 2 - 31'
231 - Step #1: explanation='By subtracting 31 from both sides, we simplify the equation to 8x = -29.' output='8x = -29'
232 - Step #2: explanation="Next, let's isolate 'x' by dividing both sides of the equation by 8." output='8x / 8 = -29 / 8'
233 - Answer: x = -29/8
234 -
235 -Offline Inference
236 ------------------
237 -
238 -Offline inference allows for the same types of guided decoding.
239 -To use it, we´ll need to configure the guided decoding using the class ``GuidedDecodingParams`` inside ``SamplingParams``.
240 -The main available options inside ``GuidedDecodingParams`` are:
241 -
242 -- ``json``
243 -- ``regex``
244 -- ``choice``
245 -- ``grammar``
246 -- ``backend``
247 -- ``whitespace_pattern``
248 -
249 -These parameters can be used in the same way as the parameters from the Online Inference examples above.
250 -One example for the usage of the ``choices`` parameter is shown below:
251 -
252 -.. code-block:: python
253 -
254 - from vllm import LLM, SamplingParams
255 - from vllm.sampling_params import GuidedDecodingParams
256 -
257 - llm = LLM(model="HuggingFaceTB/SmolLM2-1.7B-Instruct")
258 -
259 - guided_decoding_params = GuidedDecodingParams(choice=["Positive", "Negative"])
260 - sampling_params = SamplingParams(guided_decoding=guided_decoding_params)
261 - outputs = llm.generate(
262 - prompts="Classify this sentiment: vLLM is wonderful!",
263 - sampling_params=sampling_params,
264 - )
265 - print(outputs[0].outputs[0].text)
266 -
267 -A complete example with all options can be found in `examples/offline_inference_structured_outputs.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_structured_outputs.py>`_.
docs/source/usage/usage_stats.md
+1 -1
@@ -47,7 +47,7 @@ tail ~/.config/vllm/usage_stats.json
47
48 ## Opt-out of Usage Stats Collection
49
50 -You can opt-out of usage stats collection by setting the VLLM_NO_USAGE_STATS or DO_NOT_TRACK environment variable, or by creating a ~/.config/vllm/do_not_track file:
50 +You can opt-out of usage stats collection by setting the `VLLM_NO_USAGE_STATS` or `DO_NOT_TRACK` environment variable, or by creating a `~/.config/vllm/do_not_track` file:
51
52 ```bash
53 # Any of the following methods can disable usage stats collection
vllm/attention/backends/rocm_flash_attn.py
+1 -1
@@ -430,7 +430,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
430 Returns:
431 shape = [num_tokens, num_heads * head_size]
432 """
433 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
433 + # Reminder: Please update docs/source/usage/compatibility_matrix.md
434 # If the feature combo become valid
435 if attn_type != AttentionType.DECODER:
436 raise NotImplementedError("Encoder self-attention and "
vllm/config.py
+3 -3
@@ -638,7 +638,7 @@ class ModelConfig:
638 self.use_async_output_proc = False
639 return
640
641 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
641 + # Reminder: Please update docs/source/usage/compatibility_matrix.md
642 # If the feature combo become valid
643 if not current_platform.is_async_output_supported(self.enforce_eager):
644 logger.warning(
@@ -658,7 +658,7 @@ class ModelConfig:
658 if self.runner_type == "pooling":
659 self.use_async_output_proc = False
660
661 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
661 + # Reminder: Please update docs/source/usage/compatibility_matrix.md
662 # If the feature combo become valid
663 if speculative_config:
664 logger.warning("Async output processing is not supported with"
@@ -2058,7 +2058,7 @@ class LoRAConfig:
2058 model_config.quantization)
2059
2060 def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
2061 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
2061 + # Reminder: Please update docs/source/usage/compatibility_matrix.md
2062 # If the feature combo become valid
2063 if scheduler_config.chunked_prefill_enabled:
2064 logger.warning("LoRA with chunked prefill is still experimental "
vllm/engine/arg_utils.py
+1 -1
@@ -1148,7 +1148,7 @@ class EngineArgs:
1148 disable_logprobs=self.disable_logprobs_during_spec_decoding,
1149 )
1150
1151 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
1151 + # Reminder: Please update docs/source/usage/compatibility_matrix.md
1152 # If the feature combo become valid
1153 if self.num_scheduler_steps > 1:
1154 if speculative_config is not None:
vllm/engine/llm_engine.py
+1 -1
@@ -133,7 +133,7 @@ class LLMEngine:
133 and the :class:`AsyncLLMEngine` class wraps this class for online serving.
134
135 The config arguments are derived from :class:`~vllm.EngineArgs`. (See
136 - :ref:`engine_args`)
136 + :ref:`engine-args`)
137
138 Args:
139 model_config: The configuration related to the LLM model.
vllm/engine/output_processor/multi_step.py
+1 -1
@@ -65,7 +65,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
65 @staticmethod
66 @functools.lru_cache
67 def _log_prompt_logprob_unsupported_warning_once():
68 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
68 + # Reminder: Please update docs/source/usage/compatibility_matrix.md
69 # If the feature combo become valid
70 logger.warning(
71 "Prompt logprob is not supported by multi step workers. "
vllm/entrypoints/llm.py
+1 -1
@@ -115,7 +115,7 @@ class LLM:
115 integer, it is used as the level of compilation optimization. If it
116 is a dictionary, it can specify the full compilation configuration.
117 **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
118 - :ref:`engine_args`)
118 + :ref:`engine-args`)
119
120 Note:
121 This class is intended to be used for offline inference. For online
vllm/executor/cpu_executor.py
+1 -1
@@ -22,7 +22,7 @@ class CPUExecutor(ExecutorBase):
22
23 def _init_executor(self) -> None:
24 assert self.device_config.device_type == "cpu"
25 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
25 + # Reminder: Please update docs/source/usage/compatibility_matrix.md
26 # If the feature combo become valid
27 assert self.lora_config is None, "cpu backend doesn't support LoRA"
28
vllm/inputs/__init__.py
+1 -1
@@ -13,7 +13,7 @@ The global :class:`~InputRegistry` which is used by :class:`~vllm.LLMEngine`
13 to dispatch data processing according to the target model.
14
15 See also:
16 - :ref:`input_processing_pipeline`
16 + :ref:`input-processing-pipeline`
17 """
18
19 __all__ = [
vllm/inputs/registry.py
+3 -3
@@ -314,7 +314,7 @@ class InputRegistry:
314 The model is identified by ``model_config``.
315
316 See also:
317 - :ref:`enabling_multimodal_inputs`
317 + :ref:`enabling-multimodal-inputs`
318
319 Note:
320 This should be called after
@@ -391,7 +391,7 @@ class InputRegistry:
391 happens before :meth:`~vllm.multimodal.MultiModalRegistry.map_input`.
392
393 See also:
394 - :ref:`input_processing_pipeline`
394 + :ref:`input-processing-pipeline`
395 """
396
397 def wrapper(model_cls: N) -> N:
@@ -435,7 +435,7 @@ class InputRegistry:
435 The model is identified by ``model_config``.
436
437 See also:
438 - :ref:`input_processing_pipeline`
438 + :ref:`input-processing-pipeline`
439 """
440 # Avoid circular import
441 from vllm.model_executor.model_loader import get_model_architecture
vllm/multimodal/__init__.py
+1 -1
@@ -11,7 +11,7 @@ The global :class:`~MultiModalRegistry` is used by model runners to
11 dispatch data processing according to its modality and the target model.
12
13 See also:
14 - :ref:`input_processing_pipeline`
14 + :ref:`input-processing-pipeline`
15 """
16
17 __all__ = [
vllm/multimodal/base.py
+7 -7
@@ -50,7 +50,7 @@ class MultiModalPlugin(ABC):
50 (i.e., the modality of the data).
51
52 See also:
53 - :ref:`adding_multimodal_plugin`
53 + :ref:`adding-multimodal-plugin`
54 """
55
56 def __init__(self) -> None:
@@ -94,8 +94,8 @@ class MultiModalPlugin(ABC):
94 If `None` is provided, then the default input mapper is used instead.
95
96 See also:
97 - - :ref:`input_processing_pipeline`
98 - - :ref:`enabling_multimodal_inputs`
97 + - :ref:`input-processing-pipeline`
98 + - :ref:`enabling-multimodal-inputs`
99 """
100
101 def wrapper(model_cls: N) -> N:
@@ -130,8 +130,8 @@ class MultiModalPlugin(ABC):
130 TypeError: If the data type is not supported.
131
132 See also:
133 - - :ref:`input_processing_pipeline`
134 - - :ref:`enabling_multimodal_inputs`
133 + - :ref:`input-processing-pipeline`
134 + - :ref:`enabling-multimodal-inputs`
135 """
136
137 # Avoid circular import
@@ -190,7 +190,7 @@ class MultiModalPlugin(ABC):
190 If `None` is provided, then the default calculation is used instead.
191
192 See also:
193 - :ref:`enabling_multimodal_inputs`
193 + :ref:`enabling-multimodal-inputs`
194 """
195
196 def wrapper(model_cls: N) -> N:
@@ -222,7 +222,7 @@ class MultiModalPlugin(ABC):
222 The model is identified by ``model_config``.
223
224 See also:
225 - :ref:`enabling_multimodal_inputs`
225 + :ref:`enabling-multimodal-inputs`
226 """
227 # Avoid circular import
228 from vllm.model_executor.model_loader import get_model_architecture
vllm/multimodal/inputs.py
+1 -1
@@ -75,7 +75,7 @@ Note:
75 This dictionary also accepts modality keys defined outside
76 :class:`MultiModalDataBuiltins` as long as a customized plugin
77 is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
78 - Read more on that :ref:`here <adding_multimodal_plugin>`.
78 + Read more on that :ref:`here <adding-multimodal-plugin>`.
79 """
80
81
vllm/multimodal/registry.py
+3 -3
@@ -76,7 +76,7 @@ class MultiModalRegistry:
76 Register a multi-modal plugin so it can be recognized by vLLM.
77
78 See also:
79 - :ref:`adding_multimodal_plugin`
79 + :ref:`adding-multimodal-plugin`
80 """
81 data_type_key = plugin.get_data_key()
82
@@ -311,8 +311,8 @@ class MultiModalRegistry:
311 invoked to transform the data into a dictionary of model inputs.
312
313 See also:
314 - - :ref:`input_processing_pipeline`
315 - - :ref:`enabling_multimodal_inputs`
314 + - :ref:`input-processing-pipeline`
315 + - :ref:`enabling-multimodal-inputs`
316 """
317
318 def wrapper(model_cls: N) -> N:
vllm/platforms/cpu.py
+1 -1
@@ -50,7 +50,7 @@ class CpuPlatform(Platform):
50 import vllm.envs as envs
51 from vllm.utils import GiB_bytes
52 model_config = vllm_config.model_config
53 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
53 + # Reminder: Please update docs/source/usage/compatibility_matrix.md
54 # If the feature combo become valid
55 if not model_config.enforce_eager:
56 logger.warning(
vllm/scripts.py
+1 -1
@@ -165,7 +165,7 @@ def main():
165 required=False,
166 help="Read CLI options from a config file."
167 "Must be a YAML with the following options:"
168 - "https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#command-line-arguments-for-the-server"
168 + "https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#cli-reference"
169 )
170 serve_parser = make_arg_parser(serve_parser)
171 serve_parser.set_defaults(dispatch_function=serve)
vllm/spec_decode/spec_decode_worker.py
+1 -1
@@ -108,7 +108,7 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
108 return spec_decode_worker
109
110
111 -# Reminder: Please update docs/source/usage/compatibility_matrix.rst
111 +# Reminder: Please update docs/source/usage/compatibility_matrix.md
112 # If the feature combo become valid
113 class SpecDecodeWorker(LoraNotSupportedWorkerBase):
114 """Worker which implements speculative decoding.
vllm/utils.py
+1 -1
@@ -51,7 +51,7 @@ logger = init_logger(__name__)
51
52 # Exception strings for non-implemented encoder/decoder scenarios
53
54 -# Reminder: Please update docs/source/usage/compatibility_matrix.rst
54 +# Reminder: Please update docs/source/usage/compatibility_matrix.md
55 # If the feature combo become valid
56
57 STR_NOT_IMPL_ENC_DEC_SWA = \
vllm/worker/multi_step_model_runner.py
+1 -1
@@ -821,7 +821,7 @@ def _pythonize_sampler_output(
821
822 for sgdx, (seq_group,
823 sample_result) in enumerate(zip(seq_groups, samples_list)):
824 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
824 + # Reminder: Please update docs/source/usage/compatibility_matrix.md
825 # If the feature combo become valid
826 # (Check for Guided Decoding)
827 if seq_group.sampling_params.logits_processors:
vllm/worker/utils.py
+1 -1
@@ -13,7 +13,7 @@ def assert_enc_dec_mr_supported_scenario(
13 a supported scenario.
14 '''
15
16 - # Reminder: Please update docs/source/usage/compatibility_matrix.rst
16 + # Reminder: Please update docs/source/usage/compatibility_matrix.md
17 # If the feature combo become valid
18
19 if enc_dec_mr.cache_config.enable_prefix_caching: