A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Doc] Move examples and further reorganize user guide (#18666)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 26, 2025 at 22:38 UTC
82e2339b0632a4c787915210b5b57da13de26bf6
27 files changed
+31
-42
.buildkite/pyproject.toml
-5
@@ -6,11 +6,6 @@
6
7
[tool.ruff]
8
line-length = 88
9
-exclude = [
10
- # External file, leaving license intact
11
- "examples/other/fp8/quantizer/quantize.py",
12
- "vllm/vllm_flash_attn/flash_attn_interface.pyi"
13
-]
9
10
[tool.ruff.lint.per-file-ignores]
11
"vllm/third_party/**" = ["ALL"]
.buildkite/test-pipeline.yaml
+1
-1
@@ -246,7 +246,7 @@ steps:
246
- python3 offline_inference/vision_language.py --seed 0
247
- python3 offline_inference/vision_language_embedding.py --seed 0
248
- python3 offline_inference/vision_language_multi_image.py --seed 0
249
- - VLLM_USE_V1=0 python3 other/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 other/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
249
+ - VLLM_USE_V1=0 python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
250
- python3 offline_inference/encoder_decoder.py
251
- python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
252
- python3 offline_inference/basic/classify.py
.gitignore
+1
-1
@@ -146,7 +146,7 @@ venv.bak/
146
147
# mkdocs documentation
148
/site
149
-docs/getting_started/examples
149
+docs/examples
150
151
# mypy
152
.mypy_cache/
benchmarks/pyproject.toml
-5
@@ -6,11 +6,6 @@
6
7
[tool.ruff]
8
line-length = 88
9
-exclude = [
10
- # External file, leaving license intact
11
- "examples/other/fp8/quantizer/quantize.py",
12
- "vllm/vllm_flash_attn/flash_attn_interface.pyi"
13
-]
9
10
[tool.ruff.lint.per-file-ignores]
11
"vllm/third_party/**" = ["ALL"]
docs/.nav.yml
+4
-5
@@ -5,11 +5,9 @@ nav:
5
- getting_started/quickstart.md
6
- getting_started/installation
7
- Examples:
8
- - Offline Inference: getting_started/examples/offline_inference
9
- - Online Serving: getting_started/examples/online_serving
10
- - Others:
11
- - LMCache: getting_started/examples/lmcache
12
- - getting_started/examples/other/*
8
+ - Offline Inference: examples/offline_inference
9
+ - Online Serving: examples/online_serving
10
+ - Others: examples/others
11
- Quick Links:
12
- User Guide: usage/README.md
13
- Developer Guide: contributing/README.md
@@ -19,6 +17,7 @@ nav:
17
- Releases: https://github.com/vllm-project/vllm/releases
18
- User Guide:
19
- Summary: usage/README.md
20
+ - usage/v1_guide.md
21
- General:
22
- usage/*
23
- Inference and Serving:
docs/configuration/README.md
+7
-2
@@ -1,4 +1,9 @@
1
# Configuration Options
2
3
-This section lists the most common options for running the vLLM engine.
4
-For a full list, refer to the [configuration][configuration] page.
3
+This section lists the most common options for running vLLM.
4
+
5
+There are three main levels of configuration, from highest priority to lowest priority:
6
+
7
+- [Request parameters][completions-api] and [input arguments][sampling-params]
8
+- [Engine arguments](./engine_args.md)
9
+- [Environment variables](./env_vars.md)
docs/configuration/env_vars.md
renamed
docs/design/v1/metrics.md
+2
-2
@@ -61,7 +61,7 @@ These are documented under [Inferencing and Serving -> Production Metrics](../..
61
62
### Grafana Dashboard
63
64
-vLLM also provides [a reference example](https://docs.vllm.ai/en/latest/getting_started/examples/prometheus_grafana.html) for how to collect and store these metrics using Prometheus and visualize them using a Grafana dashboard.
64
+vLLM also provides [a reference example](https://docs.vllm.ai/en/latest/examples/prometheus_grafana.html) for how to collect and store these metrics using Prometheus and visualize them using a Grafana dashboard.
65
66
The subset of metrics exposed in the Grafana dashboard gives us an indication of which metrics are especially important:
67
@@ -673,7 +673,7 @@ v0 has support for OpenTelemetry tracing:
673
- [OpenTelemetry blog
674
post](https://opentelemetry.io/blog/2024/llm-observability/)
675
- [User-facing
676
- docs](https://docs.vllm.ai/en/latest/getting_started/examples/opentelemetry.html)
676
+ docs](https://docs.vllm.ai/en/latest/examples/opentelemetry.html)
677
- [Blog
678
post](https://medium.com/@ronen.schaffer/follow-the-trail-supercharging-vllm-with-opentelemetry-distributed-tracing-aa655229b46f)
679
- [IBM product
docs/mkdocs/hooks/generate_examples.py
+1
-1
@@ -9,7 +9,7 @@ from typing import Literal
9
ROOT_DIR = Path(__file__).parent.parent.parent.parent
10
ROOT_DIR_RELATIVE = '../../../../..'
11
EXAMPLE_DIR = ROOT_DIR / "examples"
12
-EXAMPLE_DOC_DIR = ROOT_DIR / "docs/getting_started/examples"
12
+EXAMPLE_DOC_DIR = ROOT_DIR / "docs/examples"
13
print(ROOT_DIR.resolve())
14
print(EXAMPLE_DIR.resolve())
15
print(EXAMPLE_DOC_DIR.resolve())
docs/models/extensions/tensorizer.md
+1
-1
@@ -10,7 +10,7 @@ shorter Pod startup times and CPU memory usage. Tensor encryption is also suppor
10
11
For more information on CoreWeave's Tensorizer, please refer to
12
[CoreWeave's Tensorizer documentation](https://github.com/coreweave/tensorizer). For more information on serializing a vLLM model, as well a general usage guide to using Tensorizer with vLLM, see
13
-the [vLLM example script](https://docs.vllm.ai/en/latest/getting_started/examples/tensorize_vllm_model.html).
13
+the [vLLM example script](https://docs.vllm.ai/en/latest/examples/tensorize_vllm_model.html).
14
15
!!! note
16
Note that to use this feature you will need to install `tensorizer` by running `pip install vllm[tensorizer]`.
docs/training/rlhf.md
+3
-3
@@ -6,6 +6,6 @@ vLLM can be used to generate the completions for RLHF. The best way to do this i
6
7
See the following basic examples to get started if you don't want to use an existing library:
8
9
-- [Training and inference processes are located on separate GPUs (inspired by OpenRLHF)](https://docs.vllm.ai/en/latest/getting_started/examples/rlhf.html)
10
-- [Training and inference processes are colocated on the same GPUs using Ray](https://docs.vllm.ai/en/latest/getting_started/examples/rlhf_colocate.html)
11
-- [Utilities for performing RLHF with vLLM](https://docs.vllm.ai/en/latest/getting_started/examples/rlhf_utils.html)
9
+- [Training and inference processes are located on separate GPUs (inspired by OpenRLHF)](../examples/offline_inference/rlhf.md)
10
+- [Training and inference processes are colocated on the same GPUs using Ray](../examples/offline_inference/rlhf_colocate.md)
11
+- [Utilities for performing RLHF with vLLM](../examples/offline_inference/rlhf_utils.md)
examples/others/lmcache/README.md
renamed
examples/others/lmcache/cpu_offload_lmcache.py
renamed
examples/others/lmcache/disagg_prefill_lmcache_v0.py
renamed
examples/others/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-decoder-config.yaml
renamed
examples/others/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-prefiller-config.yaml
renamed
examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh
renamed
examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py
renamed
examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_vllm_launcher.sh
renamed
examples/others/lmcache/kv_cache_sharing_lmcache_v1.py
renamed
examples/others/logging_configuration.md
renamed
examples/others/tensorize_vllm_model.py
renamed
+5
-5
@@ -28,7 +28,7 @@ https://github.com/coreweave/tensorizer
28
To serialize a model, install vLLM from source, then run something
29
like this from the root level of this repository:
30
31
-python examples/other/tensorize_vllm_model.py \
31
+python examples/others/tensorize_vllm_model.py \
32
--model facebook/opt-125m \
33
serialize \
34
--serialized-directory s3://my-bucket \
@@ -48,7 +48,7 @@ providing a `--keyfile` argument.
48
To deserialize a model, you can run something like this from the root
49
level of this repository:
50
51
-python examples/other/tensorize_vllm_model.py \
51
+python examples/others/tensorize_vllm_model.py \
52
--model EleutherAI/gpt-j-6B \
53
--dtype float16 \
54
deserialize \
@@ -66,11 +66,11 @@ shard's rank. Sharded models serialized with this script will be named as
66
model-rank-%03d.tensors
67
68
For more information on the available arguments for serializing, run
69
-`python -m examples.other.tensorize_vllm_model serialize --help`.
69
+`python -m examples.others.tensorize_vllm_model serialize --help`.
70
71
Or for deserializing:
72
73
-`python examples/other/tensorize_vllm_model.py deserialize --help`.
73
+`python examples/others/tensorize_vllm_model.py deserialize --help`.
74
75
Once a model is serialized, tensorizer can be invoked with the `LLM` class
76
directly to load models:
@@ -91,7 +91,7 @@ TensorizerConfig arguments desired.
91
In order to see all of the available arguments usable to configure
92
loading with tensorizer that are given to `TensorizerConfig`, run:
93
94
-`python examples/other/tensorize_vllm_model.py deserialize --help`
94
+`python examples/others/tensorize_vllm_model.py deserialize --help`
95
96
under the `tensorizer options` section. These can also be used for
97
deserialization in this example script, although `--tensorizer-uri` and
pyproject.toml
-5
@@ -62,11 +62,6 @@ ignore_patterns = [
62
[tool.ruff]
63
# Allow lines to be as long as 80.
64
line-length = 80
65
-exclude = [
66
- # External file, leaving license intact
67
- "examples/other/fp8/quantizer/quantize.py",
68
- "vllm/vllm_flash_attn/flash_attn_interface.pyi"
69
-]
65
66
[tool.ruff.lint.per-file-ignores]
67
"vllm/third_party/**" = ["ALL"]
requirements/common.txt
+1
-1
@@ -41,7 +41,7 @@ compressed-tensors == 0.9.4 # required for compressed-tensors
41
depyf==0.18.0 # required for profiling and debugging with compilation config
42
cloudpickle # allows pickling lambda functions in model_executor/models/registry.py
43
watchfiles # required for http server to monitor the updates of TLS files
44
-python-json-logger # Used by logging as per examples/other/logging_configuration.md
44
+python-json-logger # Used by logging as per examples/others/logging_configuration.md
45
scipy # Required for phi-4-multimodal-instruct
46
ninja # Required for xgrammar, rocm, tpu, xpu
47
opentelemetry-sdk>=1.26.0 # vllm.tracing
tests/lora/test_llama_tp.py
+1
-1
@@ -207,7 +207,7 @@ def test_tp2_serialize_and_deserialize_lora(tmp_path, sql_lora_files,
207
try:
208
result = subprocess.run([
209
sys.executable,
210
- f"{VLLM_PATH}/examples/other/tensorize_vllm_model.py", "--model",
210
+ f"{VLLM_PATH}/examples/others/tensorize_vllm_model.py", "--model",
211
MODEL_PATH, "--lora-path", lora_path, "--tensor-parallel-size",
212
str(tp_size), "serialize", "--serialized-directory",
213
str(tmp_path), "--suffix", suffix
vllm/model_executor/model_loader/tensorizer.py
+2
-2
@@ -251,7 +251,7 @@ class TensorizerArgs:
251
encryption_keyfile: File path to a binary file containing a
252
binary key to use for decryption. `None` (the default) means
253
no decryption. See the example script in
254
- examples/other/tensorize_vllm_model.py.
254
+ examples/others/tensorize_vllm_model.py.
255
s3_access_key_id: The access key for the S3 bucket. Can also be set via
256
the S3_ACCESS_KEY_ID environment variable.
257
s3_secret_access_key: The secret access key for the S3 bucket. Can also
@@ -469,7 +469,7 @@ def tensorizer_weights_iterator(
469
"loading on vLLM, as tensorizer is forced to load to CPU. "
470
"Consider deserializing a vLLM model instead for faster "
471
"load times. See the "
472
- "examples/other/tensorize_vllm_model.py example script "
472
+ "examples/others/tensorize_vllm_model.py example script "
473
"for serializing vLLM models.")
474
475
deserializer_args = tensorizer_args.deserializer_params
vllm/model_executor/model_loader/tensorizer_loader.py
+2
-2
@@ -48,7 +48,7 @@ class TensorizerLoader(BaseModelLoader):
48
"""Load a serialized model with tensorizer to the CPU.
49
50
This is only necessary when the model isn't vLLM-tensorized (see
51
- examples/other/tensorize_vllm_model.py) This should still
51
+ examples/others/tensorize_vllm_model.py) This should still
52
be faster than default HuggingFace loading, but will be slower than
53
loading a vLLM-tensorized model.
54
"""
@@ -68,7 +68,7 @@ class TensorizerLoader(BaseModelLoader):
68
"""Load a serialized model with tensorizer.
69
70
Expects a vLLM-tensorized model. See the
71
- examples/other/tensorize_vllm_model.py example script
71
+ examples/others/tensorize_vllm_model.py example script
72
for serializing vLLM models."""
73
74
device_config = vllm_config.device_config