@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a8208e6a81
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 4, 13:45 UTC

[Examples] Resettle features examples. (#40995)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>

wang.yuqi committed Apr 28, 2026 at 15:33 UTC a8208e6a81befd781b2a9a8b6b29fd61f5333c66
61 files changed +234 -234
.buildkite/test-amd.yaml
+17 -17
@@ -395,11 +395,11 @@ steps:
395 # Pooling models
396 - python3 pooling/embed/vision_embedding_offline.py --seed 0
397 # Features demo
398 - - python3 offline_inference/prefix_caching.py
398 + - python3 features/automatic_prefix_caching/prefix_caching_offline.py
399 - python3 offline_inference/llm_engine_example.py
400 - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
401 - - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
402 - - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
401 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
402 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
403
404 #---------------------------------------------------------- mi250 · kernels ----------------------------------------------------------#
405
@@ -1168,13 +1168,13 @@ steps:
1168 - vllm/v1/attention/backends/
1169 - vllm/v1/attention/selector.py
1170 - tests/distributed/test_context_parallel.py
1171 - - examples/offline_inference/data_parallel.py
1171 + - examples/features/data_parallel/data_parallel_offline.py
1172 - vllm/_aiter_ops.py
1173 - vllm/platforms/rocm.py
1174 commands:
1175 - export TORCH_NCCL_BLOCKING_WAIT=1
1176 - pytest -v -s tests/distributed/test_context_parallel.py
1177 - - VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
1177 + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
1178
1179 - label: Distributed Tests (4xA100-4xMI300) # TBD
1180 timeout_in_minutes: 180
@@ -1203,7 +1203,7 @@ steps:
1203 - tests/distributed/test_torchrun_example.py
1204 - tests/distributed/test_torchrun_example_moe.py
1205 - examples/rl/
1206 - - tests/examples/offline_inference/data_parallel.py
1206 + - tests/examples/features/data_parallel/data_parallel_offline.py
1207 - vllm/platforms/rocm.py
1208 commands:
1209 - export TORCH_NCCL_BLOCKING_WAIT=1
@@ -1213,7 +1213,7 @@ steps:
1213 - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
1214 - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
1215 - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
1216 - - python3 ../examples/offline_inference/data_parallel.py --enforce-eager
1216 + - python3 ../examples/features/data_parallel/data_parallel_offline.py --enforce-eager
1217 # rlhf examples
1218 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 ../examples/rl/rlhf_nccl.py
1219 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 ../examples/rl/rlhf_ipc.py
@@ -1266,7 +1266,7 @@ steps:
1266 optional: true
1267 working_dir: "/vllm-workspace/tests"
1268 source_file_dependencies:
1269 - - examples/offline_inference/torchrun_dp_example.py
1269 + - examples/features/torchrun/torchrun_dp_example_offline.py
1270 - vllm/config/parallel.py
1271 - vllm/distributed/
1272 - vllm/v1/engine/llm_engine.py
@@ -1275,7 +1275,7 @@ steps:
1275 - vllm/platforms/rocm.py
1276 commands:
1277 - export TORCH_NCCL_BLOCKING_WAIT=1
1278 - - torchrun --nproc-per-node=8 ../examples/offline_inference/torchrun_dp_example.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
1278 + - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
1279
1280 #-------------------------------------------------------- mi300 · entrypoints --------------------------------------------------------#
1281
@@ -1654,11 +1654,11 @@ steps:
1654 # Pooling models
1655 - python3 pooling/embed/vision_embedding_offline.py --seed 0
1656 # Features demo
1657 - - python3 offline_inference/prefix_caching.py
1657 + - python3 features/automatic_prefix_caching/prefix_caching_offline.py
1658 - python3 offline_inference/llm_engine_example.py
1659 - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
1660 - - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
1661 - - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
1660 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
1661 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
1662
1663 #---------------------------------------------------------- mi300 · kernels ----------------------------------------------------------#
1664
@@ -2302,7 +2302,7 @@ steps:
2302 commands:
2303 - export TORCH_NCCL_BLOCKING_WAIT=1
2304 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
2305 - - VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
2305 + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
2306 - pytest -v -s tests/v1/distributed/test_dbo.py
2307 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
2308 - pytest -v -s tests/distributed/test_packed_tensor.py
@@ -2713,7 +2713,7 @@ steps:
2713 - vllm/v1/attention/selector.py
2714 - tests/distributed/test_context_parallel.py
2715 - tests/v1/distributed/test_dbo.py
2716 - - examples/offline_inference/data_parallel.py
2716 + - examples/features/data_parallel/data_parallel_offline.py
2717 - vllm/_aiter_ops.py
2718 - vllm/platforms/rocm.py
2719 commands:
@@ -2937,11 +2937,11 @@ steps:
2937 # Pooling models
2938 - python3 pooling/embed/vision_embedding_offline.py --seed 0
2939 # Features demo
2940 - - python3 offline_inference/prefix_caching.py
2940 + - python3 features/automatic_prefix_caching/prefix_caching_offline.py
2941 - python3 offline_inference/llm_engine_example.py
2942 - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
2943 - - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
2944 - - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
2943 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
2944 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
2945
2946 #---------------------------------------------------------- mi355 · kernels ----------------------------------------------------------#
2947
.buildkite/test_areas/distributed.yaml
+7 -8
@@ -88,9 +88,8 @@ steps:
88 - vllm/distributed/
89 - tests/distributed/test_torchrun_example.py
90 - tests/distributed/test_torchrun_example_moe.py
91 - - examples/offline_inference/rlhf_colocate.py
91 - examples/rl/
93 - - tests/examples/offline_inference/data_parallel.py
92 + - tests/examples/features/data_parallel/data_parallel_offline.py
93 commands:
94 # https://github.com/NVIDIA/nccl/issues/1838
95 - export NCCL_CUMEM_HOST_ENABLE=0
@@ -107,7 +106,7 @@ steps:
106 # test with torchrun tp=2 and dp=2 with ep
107 - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
108 # test with internal dp
110 - - python3 examples/offline_inference/data_parallel.py --enforce-eager
109 + - python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager
110 # rlhf examples
111 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_nccl.py
112 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_ipc.py
@@ -159,7 +158,7 @@ steps:
158 num_devices: 8
159 working_dir: "/vllm-workspace/tests"
160 source_file_dependencies:
162 - - examples/offline_inference/torchrun_dp_example.py
161 + - examples/features/torchrun/torchrun_dp_example_offline.py
162 - vllm/config/parallel.py
163 - vllm/distributed/
164 - vllm/v1/engine/llm_engine.py
@@ -169,7 +168,7 @@ steps:
168 # https://github.com/NVIDIA/nccl/issues/1838
169 - export NCCL_CUMEM_HOST_ENABLE=0
170 # test with torchrun tp=2 and dp=4 with ep
172 - - torchrun --nproc-per-node=8 ../examples/offline_inference/torchrun_dp_example.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
171 + - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
172
173 - label: Distributed Tests (4 GPUs)(A100)
174 device: a100
@@ -194,7 +193,7 @@ steps:
193 commands:
194 - pytest -v -s tests/distributed/test_context_parallel.py
195 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
197 - - VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
196 + - VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
197 - pytest -v -s tests/v1/distributed/test_dbo.py
198 - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
199 - pytest -v -s tests/distributed/test_packed_tensor.py
@@ -222,9 +221,9 @@ steps:
221 - vllm/executor/
222 - vllm/model_executor/models/
223 - tests/distributed/
225 - - tests/examples/offline_inference/data_parallel.py
224 + - tests/examples/features/data_parallel/data_parallel_offline.py
225 commands:
227 - - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 $IMAGE_TAG "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code"
226 + - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 $IMAGE_TAG "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/features/data_parallel/data_parallel_offline.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/features/data_parallel/data_parallel_offline.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code"
227
228 - label: Pipeline + Context Parallelism (4 GPUs)
229 timeout_in_minutes: 60
.buildkite/test_areas/misc.yaml
+3 -3
@@ -120,12 +120,12 @@ steps:
120 # for pooling models
121 - python3 pooling/embed/vision_embedding_offline.py --seed 0
122 # for features demo
123 - - python3 offline_inference/prefix_caching.py
123 + - python3 features/automatic_prefix_caching/prefix_caching_offline.py
124 - python3 offline_inference/llm_engine_example.py
125 - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
126 - - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
126 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
127 # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
128 - - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
128 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
129
130 - label: Metrics, Tracing (2 GPUs)
131 timeout_in_minutes: 20
.buildkite/test_areas/model_runner_v2.yaml
+5 -4
@@ -31,8 +31,9 @@ steps:
31 - vllm/v1/worker/gpu/
32 - vllm/v1/core/sched/
33 - vllm/v1/worker/gpu_worker.py
34 - - examples/offline_inference/
34 - examples/basic/offline_inference/
35 + - examples/generate/multimodal/
36 + - examples/features/
37 - examples/pooling/embed/vision_embedding_offline.py
38 - examples/others/tensorize_vllm_model.py
39 commands:
@@ -51,12 +52,12 @@ steps:
52 # for pooling models
53 - python3 pooling/embed/vision_embedding_offline.py --seed 0
54 # for features demo
54 - - python3 offline_inference/prefix_caching.py
55 + - python3 features/automatic_prefix_caching/prefix_caching_offline.py
56 - python3 offline_inference/llm_engine_example.py
57 - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
57 - - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
58 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
59 # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
59 - - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
60 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
61
62 - label: Model Runner V2 Distributed (2 GPUs)
63 timeout_in_minutes: 45
.github/mergify.yml
+2 -3
@@ -308,8 +308,7 @@ pull_request_rules:
308 - files=benchmarks/benchmark_serving_structured_output.py
309 - files=benchmarks/run_structured_output_benchmark.sh
310 - files=docs/features/structured_outputs.md
311 - - files=examples/offline_inference/structured_outputs.py
312 - - files=examples/online_serving/structured_outputs/structured_outputs.py
311 + - files=^examples/features/structured_outputs/
312 - files~=^tests/v1/structured_output/
313 - files=tests/entrypoints/llm/test_struct_output_generate.py
314 - files~=^vllm/v1/structured_output/
@@ -325,7 +324,7 @@ pull_request_rules:
324 - or:
325 - files~=^vllm/v1/spec_decode/
326 - files~=^tests/v1/spec_decode/
328 - - files~=^examples/.*(spec_decode|mlpspeculator|eagle|speculation).*\.py
327 + - files=^examples/features/speculative_decoding/
328 - files~=^vllm/model_executor/models/.*eagle.*\.py
329 - files=vllm/model_executor/models/mlp_speculator.py
330 - files~=^vllm/transformers_utils/configs/(eagle|medusa|mlp_speculator)\.py
docs/cli/README.md
+2 -2
@@ -163,7 +163,7 @@ Running with a local file:
163
164 ```bash
165 vllm run-batch \
166 - -i offline_inference/openai_batch/openai_example_batch.jsonl \
166 + -i features/openai_batch/openai_example_batch.jsonl \
167 -o results.jsonl \
168 --model meta-llama/Meta-Llama-3-8B-Instruct
169 ```
@@ -172,7 +172,7 @@ Using remote file:
172
173 ```bash
174 vllm run-batch \
175 - -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl \
175 + -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/features/openai_batch/openai_example_batch.jsonl \
176 -o results.jsonl \
177 --model meta-llama/Meta-Llama-3-8B-Instruct
178 ```
docs/configuration/conserving_memory.md
+1 -1
@@ -23,7 +23,7 @@ llm = LLM(model="ibm-granite/granite-3.1-8b-instruct", tensor_parallel_size=2)
23 !!! note
24 With tensor parallelism enabled, each process will read the whole model and split it into chunks, which makes the disk reading time even longer (proportional to the size of tensor parallelism).
25
26 - You can convert the model checkpoint to a sharded checkpoint using [examples/offline_inference/save_sharded_state.py](../../examples/offline_inference/save_sharded_state.py). The conversion process might take some time, but later you can load the sharded checkpoint much faster. The model loading time should remain constant regardless of the size of tensor parallelism.
26 + You can convert the model checkpoint to a sharded checkpoint using [examples/features/sharded_state/load_sharded_state_offline.py](../../examples/features/sharded_state/load_sharded_state_offline.py). The conversion process might take some time, but later you can load the sharded checkpoint much faster. The model loading time should remain constant regardless of the size of tensor parallelism.
27
28 ## Quantization
29
docs/contributing/profiling.md
+1 -1
@@ -42,7 +42,7 @@ Traces can be visualized using <https://ui.perfetto.dev/>.
42
43 #### Offline Inference
44
45 -Refer to [examples/offline_inference/simple_profiling.py](../../examples/offline_inference/simple_profiling.py) for an example.
45 +Refer to [examples/features/profiling/simple_profiling_offline.py](../../examples/features/profiling/simple_profiling_offline.py) for an example.
46
47 #### OpenAI Server
48
docs/features/automatic_prefix_caching.md
+1 -1
@@ -11,7 +11,7 @@ Automatic Prefix Caching (APC in short) caches the KV cache of existing queries,
11
12 Set `enable_prefix_caching=True` in vLLM engine to enable APC. Here is an example:
13
14 -[examples/offline_inference/automatic_prefix_caching.py](../../examples/offline_inference/automatic_prefix_caching.py)
14 +[examples/features/automatic_prefix_caching/automatic_prefix_caching_offline.py](../../examples/features/automatic_prefix_caching/automatic_prefix_caching_offline.py)
15
16 ## Example workloads
17
docs/features/context_extension.md
+2 -2
@@ -6,12 +6,12 @@ This directory contains examples for extending the context length of models usin
6
7 ## Offline Inference Example
8
9 -The [`context_extension.py`](../../examples/offline_inference/context_extension) script demonstrates how to extend the context length of a Qwen model using the YARN method (rope_parameters) and run a simple chat example.
9 +The [`context_extension.py`](../../examples/features/context_extension/context_extension_offline.py) script demonstrates how to extend the context length of a Qwen model using the YARN method (rope_parameters) and run a simple chat example.
10
11 ### Usage
12
13 ```bash
14 -python examples/offline_inference/context_extension.py
14 +python examples/features/context_extension/context_extension_offline.py
15 ```
16
17 ## OpenAI Online Method
docs/features/lora.md
+1 -1
@@ -47,7 +47,7 @@ the third parameter is the path to the LoRA adapter.
47 )
48 ```
49
50 -Check out [examples/offline_inference/multilora_inference.py](../../examples/offline_inference/multilora_inference.py) for an example of how to use LoRA adapters with the async engine and how to use more advanced configuration options.
50 +Check out [examples/features/lora/multilora_offline.py](../../examples/features/lora/multilora_offline.py) for an example of how to use LoRA adapters with the async engine and how to use more advanced configuration options.
51
52 ## Serving LoRA Adapters
53
docs/features/prompt_embeds.md
+2 -2
@@ -16,7 +16,7 @@ To input multi-modal data, follow this schema in [vllm.inputs.EmbedsPrompt][]:
16
17 You can pass prompt embeddings from Hugging Face Transformers models to the `'prompt_embeds'` field of the prompt embedding dictionary, as shown in the following examples:
18
19 -[examples/offline_inference/prompt_embed_inference.py](../../examples/offline_inference/prompt_embed_inference.py)
19 +[examples/features/prompt_embed/prompt_embed_offline.py](../../examples/features/prompt_embed/prompt_embed_offline.py)
20
21 ## Online Serving
22
@@ -41,4 +41,4 @@ vllm serve meta-llama/Llama-3.2-1B-Instruct --runner generate \
41
42 Then, you can use the OpenAI client as follows:
43
44 -[examples/online_serving/prompt_embed_inference_with_openai_client.py](../../examples/online_serving/prompt_embed_inference_with_openai_client.py)
44 +[examples/features/prompt_embed/prompt_embed_inference_with_openai_client.py](../../examples/features/prompt_embed/prompt_embed_inference_with_openai_client.py)
docs/features/speculative_decoding/README.md
+1 -1
@@ -32,7 +32,7 @@ depend on your model family, traffic pattern, hardware, and sampling settings.
32 | Suffix decoding | Low to medium gain | Medium gain | No extra draft model; dynamic speculation depth. |
33
34 For reproducible measurements in your environment, use
35 -[`examples/offline_inference/spec_decode.py`](../../../examples/offline_inference/spec_decode.py)
35 +[`examples/features/speculative_decoding/spec_decode_offline.py`](../../../examples/features/speculative_decoding/spec_decode_offline.py)
36 or the [benchmark CLI guide](../../benchmarking/cli.md).
37
38 ## `--speculative-config` schema
docs/features/speculative_decoding/eagle.md
+1 -1
@@ -1,6 +1,6 @@
1 # EAGLE Draft Models
2
3 -The following code configures vLLM to use speculative decoding where proposals are generated by an [EAGLE (Extrapolation Algorithm for Greater Language-model Efficiency)](https://arxiv.org/pdf/2401.15077) based draft model. A more detailed example for offline mode, including how to extract request level acceptance rate, can be found in [examples/offline_inference/spec_decode.py](../../../examples/offline_inference/spec_decode.py)
3 +The following code configures vLLM to use speculative decoding where proposals are generated by an [EAGLE (Extrapolation Algorithm for Greater Language-model Efficiency)](https://arxiv.org/pdf/2401.15077) based draft model. A more detailed example for offline mode, including how to extract request level acceptance rate, can be found in [examples/features/speculative_decoding/spec_decode_offline.py](../../../examples/features/speculative_decoding/spec_decode_offline.py)
4
5 ## Eagle Drafter Example
6
docs/features/structured_outputs.md
+4 -4
@@ -165,7 +165,7 @@ As an example, we can use to define a specific format of simplified SQL queries:
165 print(completion.choices[0].message.content)
166 ```
167
168 -See also: [full example](../examples/online_serving/structured_outputs.md)
168 +See also: [full example](../../examples/features/structured_outputs/README.md)
169
170 ## Reasoning Outputs
171
@@ -208,7 +208,7 @@ Note that you can use reasoning with any provided structured outputs feature. Th
208 print("content: ", completion.choices[0].message.content)
209 ```
210
211 -See also: [full example](../examples/online_serving/structured_outputs.md)
211 +See also: [full example](../../examples/features/structured_outputs/README.md)
212
213 !!! note
214 When using Qwen3 Coder models with reasoning enabled, structured outputs might become disabled if the reasoning content does not get parsed into the `reasoning` field separately (v0.11.2+).
@@ -304,7 +304,7 @@ Step #2: explanation="Next, let's isolate 'x' by dividing both sides of the equa
304 Answer: x = -29/8
305 ```
306
307 -An example of using `structural_tag` can be found here: [examples/online_serving/structured_outputs](../../examples/online_serving/structured_outputs)
307 +An example of using `structural_tag` can be found here: [examples/features/structured_outputs](../../examples/features/structured_outputs/README.md)
308
309 ## Offline Inference
310
@@ -339,4 +339,4 @@ shown below:
339 print(outputs[0].outputs[0].text)
340 ```
341
342 -See also: [full example](../examples/online_serving/structured_outputs.md)
342 +See also: [full example](../../examples/features/structured_outputs/structured_outputs_offline.py)
docs/models/extensions/runai_model_streamer.md
+1 -1
@@ -101,7 +101,7 @@ vllm serve /path/to/sharded/model \
101 --model-loader-extra-config '{"pattern":"custom-model-rank-{rank}-part-{part}.safetensors"}'
102 ```
103
104 -To create sharded model files, you can use the script provided in [examples/offline_inference/save_sharded_state.py](../../../examples/offline_inference/save_sharded_state.py). This script demonstrates how to save a model in the sharded format that is compatible with the Run:ai Model Streamer sharded loader.
104 +To create sharded model files, you can use the script provided in [examples/features/sharded_state/save_sharded_state_offline.py](../../../examples/features/sharded_state/save_sharded_state_offline.py). This script demonstrates how to save a model in the sharded format that is compatible with the Run:ai Model Streamer sharded loader.
105
106 The sharded loader supports all the same tunable parameters as the regular Run:ai Model Streamer, including `concurrency` and `memory_limit`. These can be configured in the same way:
107
docs/serving/data_parallel_deployment.md
+1 -1
@@ -16,7 +16,7 @@ For MoE models, when any requests are in progress in any rank, we must ensure th
16
17 In all cases, it is beneficial to load-balance requests between DP ranks. For online deployments, this balancing can be optimized by taking into account the state of each DP engine - in particular its currently scheduled and waiting (queued) requests, and KV cache state. Each DP engine has an independent KV cache, and the benefit of prefix caching can be maximized by directing prompts intelligently.
18
19 -This document focuses on online deployments (with the API server). DP + EP is also supported for offline usage (via the LLM class), for an example see [examples/offline_inference/data_parallel.py](../../examples/offline_inference/data_parallel.py).
19 +This document focuses on online deployments (with the API server). DP + EP is also supported for offline usage (via the LLM class), for an example see [examples/features/data_parallel/data_parallel_offline.py](../../examples/features/data_parallel/data_parallel_offline.py).
20
21 There are two distinct modes supported for online deployments - self-contained with internal load balancing, or externally per-rank process deployment and load balancing.
22
docs/usage/reproducibility.md
+1 -1
@@ -7,7 +7,7 @@ reproducible results:
7 or enable [batch invariance](../features/batch_invariance.md) to make the outputs insensitive to scheduling.
8 - In online mode, you can only enable [batch invariance](../features/batch_invariance.md).
9
10 -Example: [examples/offline_inference/reproducibility.py](../../examples/offline_inference/reproducibility.py)
10 +Example: [examples/features/batch_invariance/reproducibility_offline.py](../../examples/features/batch_invariance/reproducibility_offline.py)
11
12 !!! warning
13
examples/features/automatic_prefix_caching/automatic_prefix_caching_offline.py renamed
+1 -1
@@ -15,7 +15,7 @@ compares the generation time for two queries that share the same prefix
15 but ask different questions.
16
17 Run:
18 -python examples/offline_inference/automatic_prefix_caching.py
18 +python examples/features/automatic_prefix_caching/automatic_prefix_caching_offline.py
19 """
20
21 import time
examples/features/automatic_prefix_caching/prefix_caching_offline.py renamed
examples/features/batch_invariance/reproducibility_offline.py renamed
examples/features/context_extension/context_extension_offline.py renamed
+1 -1
@@ -6,7 +6,7 @@ of a Qwen model using the YARN method (rope_parameters)
6 and run a simple chat example.
7
8 Usage:
9 - python examples/offline_inference/context_extension.py
9 + python examples/features/context_extension/context_extension_offline.py
10 """
11
12 from vllm import LLM, RequestOutput, SamplingParams
examples/features/data_parallel/data_parallel_offline.py renamed
+3 -3
@@ -3,14 +3,14 @@
3 """
4 Usage:
5 Single node:
6 - python examples/offline_inference/data_parallel.py \
6 + python examples/features/data_parallel/data_parallel_offline.py \
7 --model="ibm-research/PowerMoE-3b" \
8 -dp=2 \
9 -tp=2
10
11 Multi-node:
12 Node 0 (assume the node has ip of 10.99.48.128):
13 - python examples/offline_inference/data_parallel.py \
13 + python examples/features/data_parallel/data_parallel_offline.py \
14 --model="ibm-research/PowerMoE-3b" \
15 -dp=2 \
16 -tp=2 \
@@ -19,7 +19,7 @@ Multi-node:
19 --dp-master-addr=10.99.48.128 \
20 --dp-master-port=13345
21 Node 1:
22 - python examples/offline_inference/data_parallel.py \
22 + python examples/features/data_parallel/data_parallel_offline.py \
23 --model="ibm-research/PowerMoE-3b" \
24 -dp=2 \
25 -tp=2 \
examples/features/data_parallel/multi_instance_data_parallel.py renamed
+1 -1
@@ -12,7 +12,7 @@ from vllm.v1.metrics.loggers import AggregatedLoggingStatLogger
12 """
13 To run this example, run the following commands simultaneously with
14 different CUDA_VISIBLE_DEVICES:
15 - python examples/online_serving/multi_instance_data_parallel.py
15 + python examples/features/data_parallel/multi_instance_data_parallel.py
16
17 vllm serve ibm-research/PowerMoE-3b -dp 2 -dpr 1 \
18 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 62300 \
examples/features/kv_events/kv_events_subscriber.py renamed
examples/features/logits_processor/README.md renamed
+3 -3
@@ -9,7 +9,7 @@ This directory contains examples demonstrating how to use custom logits processo
9 Demonstrates how to instantiate vLLM with a custom logits processor class that operates at the batch level. The example uses a `DummyLogitsProcessor` that masks out all tokens except a specified `target_token` when passed via `SamplingParams.extra_args`.
10
11 ```bash
12 -python examples/offline_inference/logits_processor/custom.py
12 +python examples/features/logits_processor/custom.py
13 ```
14
15 ### `custom_req.py` — Request-level logits processor wrapper
@@ -17,7 +17,7 @@ python examples/offline_inference/logits_processor/custom.py
17 Shows how to wrap a request-level logits processor (which operates on individual requests) to be compatible with vLLM's batch-level logits processing interface.
18
19 ```bash
20 -python examples/offline_inference/logits_processor/custom_req.py
20 +python examples/features/logits_processor/custom_req.py
21 ```
22
23 ### `custom_req_init.py` — Request-level processor with engine config
@@ -25,7 +25,7 @@ python examples/offline_inference/logits_processor/custom_req.py
25 A special case of wrapping a request-level logits processor where the processor needs access to engine configuration or model metadata during initialization (e.g., vocabulary size, tokenizer info).
26
27 ```bash
28 -python examples/offline_inference/logits_processor/custom_req_init.py
28 +python examples/features/logits_processor/custom_req_init.py
29 ```
30
31 ## Key Concepts
examples/features/logits_processor/custom.py renamed
examples/features/logits_processor/custom_req.py renamed
examples/features/logits_processor/custom_req_init.py renamed
examples/features/lora/lora_with_quantization_offline.py renamed
examples/features/lora/multilora_offline.py renamed
examples/features/openai_batch/README.md renamed
+11 -11
@@ -8,7 +8,7 @@ This is a guide to performing batch inference using the OpenAI batch file format
8
9 The OpenAI batch file format consists of a series of json objects on new lines.
10
11 -[See here for an example file.](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl)
11 +[See here for an example file.](https://github.com/vllm-project/vllm/blob/main/examples/features/openai_batch/openai_example_batch.jsonl)
12
13 Each line represents a separate request. See the [OpenAI package reference](https://platform.openai.com/docs/api-reference/batch/requestInput) for more details.
14
@@ -30,13 +30,13 @@ We currently support `/v1/chat/completions`, `/v1/embeddings`, and `/v1/score` e
30 To follow along with this example, you can download the example batch, or create your own batch file in your working directory.
31
32 ```bash
33 -wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl
33 +wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/features/openai_batch/openai_example_batch.jsonl
34 ```
35
36 Once you've created your batch file it should look like this
37
38 ```bash
39 -cat offline_inference/openai_batch/openai_example_batch.jsonl
39 +cat features/openai_batch/openai_example_batch.jsonl
40 {"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
41 {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
42 ```
@@ -49,7 +49,7 @@ You can run the batch with the following command, which will write its results t
49
50 ```bash
51 python -m vllm.entrypoints.openai.run_batch \
52 - -i offline_inference/openai_batch/openai_example_batch.jsonl \
52 + -i features/openai_batch/openai_example_batch.jsonl \
53 -o results.jsonl \
54 --model meta-llama/Meta-Llama-3-8B-Instruct
55 ```
@@ -58,7 +58,7 @@ or use command-line:
58
59 ```bash
60 vllm run-batch \
61 - -i offline_inference/openai_batch/openai_example_batch.jsonl \
61 + -i features/openai_batch/openai_example_batch.jsonl \
62 -o results.jsonl \
63 --model meta-llama/Meta-Llama-3-8B-Instruct
64 ```
@@ -77,11 +77,11 @@ cat results.jsonl
77
78 The batch runner supports remote input and output urls that are accessible via http/https.
79
80 -For example, to run against our example input file located at `https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl`, you can run
80 +For example, to run against our example input file located at `https://raw.githubusercontent.com/vllm-project/vllm/main/examples/features/openai_batch/openai_example_batch.jsonl`, you can run
81
82 ```bash
83 python -m vllm.entrypoints.openai.run_batch \
84 - -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl \
84 + -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/features/openai_batch/openai_example_batch.jsonl \
85 -o results.jsonl \
86 --model meta-llama/Meta-Llama-3-8B-Instruct
87 ```
@@ -90,7 +90,7 @@ or use command-line:
90
91 ```bash
92 vllm run-batch \
93 - -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl \
93 + -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/features/openai_batch/openai_example_batch.jsonl \
94 -o results.jsonl \
95 --model meta-llama/Meta-Llama-3-8B-Instruct
96 ```
@@ -113,13 +113,13 @@ To integrate with cloud blob storage, we recommend using presigned urls.
113 To follow along with this example, you can download the example batch, or create your own batch file in your working directory.
114
115 ```bash
116 -wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai_batch/openai_example_batch.jsonl
116 +wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/features/openai_batch/openai_example_batch.jsonl
117 ```
118
119 Once you've created your batch file it should look like this
120
121 ```bash
122 -cat offline_inference/openai_batch/openai_example_batch.jsonl
122 +cat features/openai_batch/openai_example_batch.jsonl
123 {"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
124 {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
125 ```
@@ -127,7 +127,7 @@ cat offline_inference/openai_batch/openai_example_batch.jsonl
127 Now upload your batch file to your S3 bucket.
128
129 ```bash
130 -aws s3 cp offline_inference/openai_batch/openai_example_batch.jsonl s3://MY_BUCKET/MY_INPUT_FILE.jsonl
130 +aws s3 cp features/openai_batch/openai_example_batch.jsonl s3://MY_BUCKET/MY_INPUT_FILE.jsonl
131 ```
132
133 ### Step 2: Generate your presigned urls
examples/features/openai_batch/openai_example_batch.jsonl renamed
examples/features/pause_resume/data_parallel_pause_resume.py renamed
+135 -135
@@ -1,135 +1,135 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""
4 -Test pause/resume with Data Parallel (DP) via HTTP API.
5 -
6 -This example demonstrates coordinated pause/resume across multiple DP ranks.
7 -The pause synchronizes across all DP engines via all-reduce.
8 -
9 -Prerequisites:
10 - Start a vLLM server with data parallelism:
11 -
12 - $ VLLM_SERVER_DEV_MODE=1 vllm serve facebook/opt-125m \
13 - --enforce-eager \
14 - --data-parallel-size 4 \
15 - --tensor-parallel-size 1
16 -
17 - Then run this script:
18 -
19 - $ python data_parallel_pause_resume.py
20 -
21 -The test verifies pause works by:
22 -1. Starting a streaming generation request
23 -2. Pausing the server mid-generation
24 -3. Sleeping for PAUSE_DURATION seconds
25 -4. Resuming the server
26 -5. Verifying there was a gap in token generation matching the pause duration
27 -"""
28 -
29 -import argparse
30 -import threading
31 -import time
32 -
33 -import requests
34 -from openai import OpenAI
35 -
36 -BASE_URL = "http://localhost:8000"
37 -MODEL_NAME = "facebook/opt-125m"
38 -PAUSE_DURATION = 3.0
39 -
40 -
41 -def pause_generation(base_url: str, mode: str = "keep") -> None:
42 - """Pause generation via HTTP endpoint."""
43 - url = f"{base_url}/pause"
44 - response = requests.post(url, params={"mode": mode}, timeout=60)
45 - response.raise_for_status()
46 - print("Server paused")
47 -
48 -
49 -def resume_generation(base_url: str) -> None:
50 - """Resume generation via HTTP endpoint."""
51 - url = f"{base_url}/resume"
52 - response = requests.post(url, timeout=60)
53 - response.raise_for_status()
54 - print("Server resumed")
55 -
56 -
57 -def main():
58 - parser = argparse.ArgumentParser()
59 - parser.add_argument("--base-url", default=BASE_URL)
60 - parser.add_argument("--model", default=MODEL_NAME)
61 - args = parser.parse_args()
62 -
63 - client = OpenAI(
64 - base_url=f"{args.base_url}/v1",
65 - api_key="EMPTY",
66 - )
67 -
68 - prompt = "Write a long story about a dragon. Once upon a time"
69 - token_times: list[float] = []
70 - pause_token_idx = 0
71 - pause_triggered = threading.Event()
72 -
73 - def generator_thread():
74 - """Stream tokens and record timestamps."""
75 - stream = client.completions.create(
76 - model=args.model,
77 - prompt=prompt,
78 - max_tokens=50,
79 - stream=True,
80 - )
81 - for chunk in stream:
82 - if chunk.choices[0].text:
83 - token_times.append(time.monotonic())
84 - token_count = len(token_times)
85 - print(f"Token {token_count}: {chunk.choices[0].text!r}")
86 -
87 - # Signal controller after some tokens
88 - if token_count >= 5 and not pause_triggered.is_set():
89 - pause_triggered.set()
90 -
91 - def controller_thread():
92 - """Pause and resume the server."""
93 - nonlocal pause_token_idx
94 -
95 - # Wait for some tokens
96 - pause_triggered.wait()
97 -
98 - print(f"\nPausing server (keep mode) at token {len(token_times)}...")
99 - pause_generation(args.base_url, mode="keep")
100 - pause_token_idx = len(token_times)
101 - print(f"Sleeping for {PAUSE_DURATION}s...")
102 -
103 - time.sleep(PAUSE_DURATION)
104 -
105 - print("Resuming server...")
106 - resume_generation(args.base_url)
107 - print("Resumed!\n")
108 -
109 - # Run both threads
110 - gen_thread = threading.Thread(target=generator_thread)
111 - ctrl_thread = threading.Thread(target=controller_thread)
112 -
113 - gen_thread.start()
114 - ctrl_thread.start()
115 -
116 - gen_thread.join()
117 - ctrl_thread.join()
118 -
119 - # Check gap at the pause point
120 - if pause_token_idx < len(token_times):
121 - pause_gap = token_times[pause_token_idx] - token_times[pause_token_idx - 1]
122 - print(
123 - f"\nGap after pause (token {pause_token_idx} -> "
124 - f"{pause_token_idx + 1}): {pause_gap:.3f}s"
125 - )
126 - if pause_gap >= PAUSE_DURATION * 0.9:
127 - print("Test passed! Pause synchronized across DP ranks.")
128 - else:
129 - print(f"Test failed! Expected ~{PAUSE_DURATION}s gap, got {pause_gap:.3f}s")
130 - else:
131 - print("Test failed! No tokens were generated after resuming.")
132 -
133 -
134 -if __name__ == "__main__":
135 - main()
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Test pause/resume with Data Parallel (DP) via HTTP API.
5 +
6 +This example demonstrates coordinated pause/resume across multiple DP ranks.
7 +The pause synchronizes across all DP engines via all-reduce.
8 +
9 +Prerequisites:
10 + Start a vLLM server with data parallelism:
11 +
12 + $ VLLM_SERVER_DEV_MODE=1 vllm serve facebook/opt-125m \
13 + --enforce-eager \
14 + --data-parallel-size 4 \
15 + --tensor-parallel-size 1
16 +
17 + Then run this script:
18 +
19 + $ python data_parallel_pause_resume.py
20 +
21 +The test verifies pause works by:
22 +1. Starting a streaming generation request
23 +2. Pausing the server mid-generation
24 +3. Sleeping for PAUSE_DURATION seconds
25 +4. Resuming the server
26 +5. Verifying there was a gap in token generation matching the pause duration
27 +"""
28 +
29 +import argparse
30 +import threading
31 +import time
32 +
33 +import requests
34 +from openai import OpenAI
35 +
36 +BASE_URL = "http://localhost:8000"
37 +MODEL_NAME = "facebook/opt-125m"
38 +PAUSE_DURATION = 3.0
39 +
40 +
41 +def pause_generation(base_url: str, mode: str = "keep") -> None:
42 + """Pause generation via HTTP endpoint."""
43 + url = f"{base_url}/pause"
44 + response = requests.post(url, params={"mode": mode}, timeout=60)
45 + response.raise_for_status()
46 + print("Server paused")
47 +
48 +
49 +def resume_generation(base_url: str) -> None:
50 + """Resume generation via HTTP endpoint."""
51 + url = f"{base_url}/resume"
52 + response = requests.post(url, timeout=60)
53 + response.raise_for_status()
54 + print("Server resumed")
55 +
56 +
57 +def main():
58 + parser = argparse.ArgumentParser()
59 + parser.add_argument("--base-url", default=BASE_URL)
60 + parser.add_argument("--model", default=MODEL_NAME)
61 + args = parser.parse_args()
62 +
63 + client = OpenAI(
64 + base_url=f"{args.base_url}/v1",
65 + api_key="EMPTY",
66 + )
67 +
68 + prompt = "Write a long story about a dragon. Once upon a time"
69 + token_times: list[float] = []
70 + pause_token_idx = 0
71 + pause_triggered = threading.Event()
72 +
73 + def generator_thread():
74 + """Stream tokens and record timestamps."""
75 + stream = client.completions.create(
76 + model=args.model,
77 + prompt=prompt,
78 + max_tokens=50,
79 + stream=True,
80 + )
81 + for chunk in stream:
82 + if chunk.choices[0].text:
83 + token_times.append(time.monotonic())
84 + token_count = len(token_times)
85 + print(f"Token {token_count}: {chunk.choices[0].text!r}")
86 +
87 + # Signal controller after some tokens
88 + if token_count >= 5 and not pause_triggered.is_set():
89 + pause_triggered.set()
90 +
91 + def controller_thread():
92 + """Pause and resume the server."""
93 + nonlocal pause_token_idx
94 +
95 + # Wait for some tokens
96 + pause_triggered.wait()
97 +
98 + print(f"\nPausing server (keep mode) at token {len(token_times)}...")
99 + pause_generation(args.base_url, mode="keep")
100 + pause_token_idx = len(token_times)
101 + print(f"Sleeping for {PAUSE_DURATION}s...")
102 +
103 + time.sleep(PAUSE_DURATION)
104 +
105 + print("Resuming server...")
106 + resume_generation(args.base_url)
107 + print("Resumed!\n")
108 +
109 + # Run both threads
110 + gen_thread = threading.Thread(target=generator_thread)
111 + ctrl_thread = threading.Thread(target=controller_thread)
112 +
113 + gen_thread.start()
114 + ctrl_thread.start()
115 +
116 + gen_thread.join()
117 + ctrl_thread.join()
118 +
119 + # Check gap at the pause point
120 + if pause_token_idx < len(token_times):
121 + pause_gap = token_times[pause_token_idx] - token_times[pause_token_idx - 1]
122 + print(
123 + f"\nGap after pause (token {pause_token_idx} -> "
124 + f"{pause_token_idx + 1}): {pause_gap:.3f}s"
125 + )
126 + if pause_gap >= PAUSE_DURATION * 0.9:
127 + print("Test passed! Pause synchronized across DP ranks.")
128 + else:
129 + print(f"Test failed! Expected ~{PAUSE_DURATION}s gap, got {pause_gap:.3f}s")
130 + else:
131 + print("Test failed! No tokens were generated after resuming.")
132 +
133 +
134 +if __name__ == "__main__":
135 + main()
examples/features/pause_resume/pause_resume_offline.py renamed
examples/features/profiling/run_one_batch_offline.py renamed
examples/features/profiling/simple_profiling_offline.py renamed
examples/features/prompt_embed/prompt_embed_inference_with_openai_client.py renamed
+1 -1
@@ -15,7 +15,7 @@ vllm serve meta-llama/Llama-3.2-1B-Instruct \
15 --enable-prompt-embeds
16
17 Run the client:
18 -python examples/online_serving/prompt_embed_inference_with_openai_client.py
18 +python examples/features/prompt_embed/prompt_embed_inference_with_openai_client.py
19
20 Model: meta-llama/Llama-3.2-1B-Instruct
21 Note: This model is gated on Hugging Face Hub.
examples/features/prompt_embed/prompt_embed_offline.py renamed
+1 -1
@@ -15,7 +15,7 @@ Requirements:
15 - transformers
16
17 Run:
18 - python examples/offline_inference/prompt_embed_inference.py
18 + python examples/features/prompt_embed/prompt_embed_offline.py
19 """
20
21 import torch
examples/features/reset_kv/reset_kv_offline.py renamed
examples/features/sharded_state/load_sharded_state_offline.py renamed
+3 -3
@@ -3,16 +3,16 @@
3 """
4 Validates the loading of a model saved with the sharded_state format.
5 This script demonstrates how to load a model that was previously saved
6 -using save_sharded_state.py and validates it by running inference.
6 +using save_sharded_state_offline.py and validates it by running inference.
7 Example usage:
8 (First need to save a sharded_state mode)
9
10 -python save_sharded_state.py \
10 +python save_sharded_state_offline.py \
11 --model /path/to/load \
12 --tensor-parallel-size 8 \
13 --output /path/to/save/sharded/model
14
15 -python load_sharded_state.py \
15 +python load_sharded_state_offline.py \
16 --model /path/to/saved/sharded/model \
17 --load-format sharded_state \
18 --tensor-parallel-size 8 \
examples/features/sharded_state/save_sharded_state_offline.py renamed
+1 -1
@@ -7,7 +7,7 @@ read its own shard rather than the entire checkpoint.
7
8 Example usage:
9
10 -python save_sharded_state.py \
10 +python save_sharded_state_offline.py \
11 --model /path/to/load \
12 --tensor-parallel-size 8 \
13 --output /path/to/save
examples/features/speculative_decoding/extract_hidden_states_offline.py renamed
examples/features/speculative_decoding/mlpspeculator_offline.py renamed
examples/features/speculative_decoding/spec_decode_offline.py renamed
examples/features/structured_outputs/README.md renamed
+5 -5
@@ -20,7 +20,7 @@ vllm serve deepseek-ai/DeepSeek-R1-Distill-Qwen-7B \
20 If you want to run this script standalone with `uv`, you can use the following:
21
22 ```bash
23 -uvx --from git+https://github.com/vllm-project/vllm#subdirectory=examples/online_serving/structured_outputs \
23 +uvx --from git+https://github.com/vllm-project/vllm#subdirectory=examples/features/structured_outputs \
24 structured-outputs
25 ```
26
@@ -34,19 +34,19 @@ See [feature docs](https://docs.vllm.ai/en/latest/features/structured_outputs.ht
34 Run all constraints, non-streaming:
35
36 ```bash
37 -uv run structured_outputs.py
37 +uv run structured_outputs_offline.py
38 ```
39
40 Run all constraints, streaming:
41
42 ```bash
43 -uv run structured_outputs.py --stream
43 +uv run structured_outputs_offline.py --stream
44 ```
45
46 Run certain constraints, for example `structural_tag` and `regex`, streaming:
47
48 ```bash
49 -uv run structured_outputs.py \
49 +uv run structured_outputs_offline.py \
50 --constraint structural_tag regex \
51 --stream
52 ```
@@ -54,5 +54,5 @@ uv run structured_outputs.py \
54 Run all constraints, with reasoning models and streaming:
55
56 ```bash
57 -uv run structured_outputs.py --reasoning --stream
57 +uv run structured_outputs_offline.py --reasoning --stream
58 ```
examples/features/structured_outputs/pyproject.toml renamed
examples/features/structured_outputs/structured_outputs_client.py renamed
examples/features/structured_outputs/structured_outputs_offline.py renamed
examples/features/torchrun/torchrun_dp_example_offline.py renamed
+3 -3
@@ -7,15 +7,15 @@ no internal lb supported in external_launcher mode.
7
8 To run this example:
9 ```bash
10 -$ torchrun --nproc-per-node=2 examples/offline_inference/torchrun_dp_example.py
10 +$ torchrun --nproc-per-node=2 examples/features/torchrun/torchrun_dp_example_offline.py
11 ```
12
13 With custom parallelism settings:
14 ```bash
15 -$ torchrun --nproc-per-node=8 examples/offline_inference/torchrun_dp_example.py \
15 +$ torchrun --nproc-per-node=8 examples/features/torchrun/torchrun_dp_example_offline.py \
16 --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
17 ```
18 -"""
18 +""" # noqa: E501
19
20 import argparse
21
examples/features/torchrun/torchrun_example_offline.py renamed
+1 -1
@@ -4,7 +4,7 @@
4 experimental support for tensor-parallel inference with torchrun,
5 see https://github.com/vllm-project/vllm/issues/11400 for
6 the motivation and use case for this example.
7 -run the script with `torchrun --nproc-per-node=4 torchrun_example.py`,
7 +run the script with `torchrun --nproc-per-node=4 torchrun_example_offline.py`,
8 the argument `4` should match the product of `tensor_parallel_size` and
9 `pipeline_parallel_size` below. see `tests/distributed/test_torchrun_example.py`
10 for the unit test.
examples/rl/routed_experts_e2e.py renamed
+1 -1
@@ -9,7 +9,7 @@ Validates that:
9 3. Results are deterministic across runs (baseline vs reference).
10
11 Usage:
12 - python examples/offline_inference/routed_experts_e2e.py \
12 + python examples/rl/routed_experts_e2e.py \
13 --model Qwen/Qwen3-30B-A3B \
14 --tp 4 \
15 --max-model-len 4096 \
examples/rl/skip_loading_weights_in_engine_init.py renamed
tests/distributed/test_torchrun_example.py
+1 -1
@@ -1,7 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -# unit test for `examples/offline_inference/torchrun_example.py`
4 +# unit test for `examples/features/torchrun/torchrun_example_offline.py`
5 import os
6 import random
7
tests/distributed/test_torchrun_example_moe.py
+1 -1
@@ -1,7 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -# unit test for `examples/offline_inference/torchrun_example.py`
4 +# unit test for `examples/features/torchrun/torchrun_example_offline.py`
5 import os
6 import random
7
tests/v1/spec_decode/test_acceptance_length.py
+2 -1
@@ -43,7 +43,8 @@ class Eagle3ModelConfig:
43
44 # Model configurations for EAGLE3 acceptance length tests.
45 # Expected acceptance lengths are determined by running baseline benchmarks
46 -# using examples/offline_inference/spec_decode.py with the MT-Bench dataset.
46 +# using examples/features/speculative_decoding/spec_decode_offline.py
47 +# with the MT-Bench dataset.
48 EAGLE3_MODEL_CONFIGS = [
49 Eagle3ModelConfig(
50 verifier="meta-llama/Llama-3.1-8B-Instruct",
vllm/entrypoints/llm.py
+1 -1
@@ -334,7 +334,7 @@ class LLM:
334 f"LLM(data_parallel_size={_dp_size}) is not supported for single-"
335 "process usage and may hang. Please use "
336 "the explicit multi-process data-parallel example at "
337 - "'examples/offline_inference/data_parallel.py'."
337 + "'examples/features/data_parallel/data_parallel_offline.py'."
338 )
339
340 engine_args = EngineArgs(
vllm/model_executor/model_loader/sharded_state_loader.py
+2 -2
@@ -31,8 +31,8 @@ class ShardedStateLoader(BaseModelLoader):
31 Model loader that directly loads each worker's model state dict, which
32 enables a fast load path for large tensor-parallel models where each worker
33 only needs to read its own shard rather than the entire checkpoint. See
34 - `examples/offline_inference/save_sharded_state.py` for creating a sharded
35 - checkpoint.
34 + `examples/features/sharded_state/save_sharded_state_offline.py` for creating
35 + a sharded checkpoint.
36 """
37
38 DEFAULT_PATTERN = "model-rank-{rank}-part-{part}.safetensors"
vllm/model_executor/models/config.py
+1 -1
@@ -517,7 +517,7 @@ class NomicBertModelConfig(VerifyAndUpdateConfig):
517 "Nomic context extension is disabled. "
518 "Changing max_model_len from %s to %s. "
519 "To enable context extension, see: "
520 - "https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/context_extension.py",
520 + "https://github.com/vllm-project/vllm/tree/main/examples/features/context_extension/context_extension_offline.py",
521 max_model_len_before,
522 model_config.max_model_len,
523 )
vllm/v1/engine/utils.py
+1 -1
@@ -952,7 +952,7 @@ def get_engine_zmq_addresses(
952
953 # In offline mode there is an LLM instance per DP rank and
954 # one core engine per LLM, see
955 - # examples/offline_inference/data_parallel.py.
955 + # examples/features/data_parallel/data_parallel_offline.py.
956 offline_mode = local_start_index is not None
957
958 # client_local_only = True for cases where this front-end
vllm/v1/executor/uniproc_executor.py
+1 -1
@@ -147,7 +147,7 @@ class ExecutorWithExternalLauncher(UniProcExecutor):
147 offline inference with tensor parallelism.
148
149 see https://github.com/vllm-project/vllm/issues/11400 for
150 - the motivation, and examples/offline_inference/torchrun_example.py
150 + the motivation, and examples/features/torchrun/torchrun_example_offline.py
151 for the usage example.
152
153 The key idea: although it is tensor-parallel inference, we only