A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 01:30 UTC
[Examples][1/n] Resettle basic examples. (#35579)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
wang.yuqi committed
Mar 9, 2026 at 11:22 UTC
dcf8862fd47624ec48a6e3a06ff2bcc53dc4d4a0
26 files changed
+64
-65
.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh
+1
-1
@@ -34,7 +34,7 @@ function cpu_tests() {
34
# offline inference
35
docker exec cpu-test bash -c "
36
set -e
37
- python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m"
37
+ python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m"
38
39
# Run model tests
40
docker exec cpu-test bash -c "
.buildkite/scripts/hardware_ci/run-cpu-test-ppc64le.sh
+1
-1
@@ -27,7 +27,7 @@ function cpu_tests() {
27
podman exec -it "$container_id" bash -c "
28
export TORCH_COMPILE_DISABLE=1
29
set -xve
30
- python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m" >> "$HOME"/test_basic.log
30
+ python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m" >> "$HOME"/test_basic.log
31
32
# Run basic model test
33
podman exec -it "$container_id" bash -c "
.buildkite/scripts/hardware_ci/run-gh200-test.sh
+1
-1
@@ -25,5 +25,5 @@ remove_docker_container
25
26
# Run the image and test offline inference
27
docker run -e HF_TOKEN -e VLLM_WORKER_MULTIPROC_METHOD=spawn -v /root/.cache/huggingface:/root/.cache/huggingface --name gh200-test --gpus=all --entrypoint="" gh200-test bash -c '
28
- python3 examples/offline_inference/basic/generate.py --model meta-llama/Llama-3.2-1B
28
+ python3 examples/basic/offline_inference/generate.py --model meta-llama/Llama-3.2-1B
29
'
.buildkite/scripts/hardware_ci/run-hpu-test.sh
+1
-1
@@ -76,7 +76,7 @@ docker run --rm --runtime=habana --name="${container_name}" --network=host \
76
-e PT_HPU_LAZY_MODE=1 \
77
"${image_name}" \
78
/bin/bash -c '
79
- cd vllm; timeout 120s python -u examples/offline_inference/basic/generate.py --model facebook/opt-125m
79
+ cd vllm; timeout 120s python -u examples/basic/offline_inference/generate.py --model facebook/opt-125m
80
'
81
82
EXITCODE=$?
.buildkite/scripts/hardware_ci/run-xpu-test.sh
+9
-9
@@ -34,15 +34,15 @@ docker run \
34
set -e
35
echo $ZE_AFFINITY_MASK
36
pip install tblib==3.1.0
37
- python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
38
- python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
39
- python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend ray
40
- python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
41
- python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
42
- python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8
43
- python3 examples/offline_inference/basic/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager
44
- python3 examples/offline_inference/basic/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2
45
- python3 examples/offline_inference/basic/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
37
+ python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
38
+ python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
39
+ python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend ray
40
+ python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
41
+ python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
42
+ python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8
43
+ python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager
44
+ python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2
45
+ python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
46
cd tests
47
pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py
48
pytest -v -s v1/engine
.buildkite/test-amd.yaml
+15
-15
@@ -529,12 +529,12 @@ steps:
529
commands:
530
- pip install tensorizer # for tensorizer test
531
# for basic
532
- - python3 offline_inference/basic/chat.py
533
- - python3 offline_inference/basic/generate.py --model facebook/opt-125m
534
- - python3 offline_inference/basic/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
535
- - python3 offline_inference/basic/classify.py
536
- - python3 offline_inference/basic/embed.py
537
- - python3 offline_inference/basic/score.py
532
+ - python3 basic/offline_inference/chat.py
533
+ - python3 basic/offline_inference/generate.py --model facebook/opt-125m
534
+ - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
535
+ - python3 basic/offline_inference/classify.py
536
+ - python3 basic/offline_inference/embed.py
537
+ - python3 basic/offline_inference/score.py
538
# for multi-modal models
539
- python3 offline_inference/audio_language.py --seed 0
540
- python3 offline_inference/vision_language.py --seed 0
@@ -1169,7 +1169,7 @@ steps:
1169
- pytest -v -s tests/models/test_transformers.py
1170
# - pytest -v -s tests/models/multimodal/processing/
1171
- pytest -v -s tests/models/multimodal/test_mapping.py -k 'not (Gemma3 or Qwen2VL or Qwen2_5_VL)'
1172
- - python3 examples/offline_inference/basic/chat.py
1172
+ - python3 examples/basic/offline_inference/chat.py
1173
# - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
1174
# Whisper needs spawn method to avoid deadlock
1175
- VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
@@ -2208,12 +2208,12 @@ steps:
2208
commands:
2209
- pip install tensorizer # for tensorizer test
2210
# for basic
2211
- - python3 offline_inference/basic/chat.py
2212
- - python3 offline_inference/basic/generate.py --model facebook/opt-125m
2213
- - python3 offline_inference/basic/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
2214
- - python3 offline_inference/basic/classify.py
2215
- - python3 offline_inference/basic/embed.py
2216
- - python3 offline_inference/basic/score.py
2211
+ - python3 basic/offline_inference/chat.py
2212
+ - python3 basic/offline_inference/generate.py --model facebook/opt-125m
2213
+ - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
2214
+ - python3 basic/offline_inference/classify.py
2215
+ - python3 basic/offline_inference/embed.py
2216
+ - python3 basic/offline_inference/score.py
2217
# for multi-modal models
2218
- python3 offline_inference/audio_language.py --seed 0
2219
- python3 offline_inference/vision_language.py --seed 0
@@ -2789,7 +2789,7 @@ steps:
2789
- pytest -v -s tests/models/test_transformers.py
2790
# - pytest -v -s tests/models/multimodal/processing/
2791
- pytest -v -s tests/models/multimodal/test_mapping.py -k 'not (Gemma3 or Qwen2VL or Qwen2_5_VL)'
2792
- - python3 examples/offline_inference/basic/chat.py
2792
+ - python3 examples/basic/offline_inference/chat.py
2793
# - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
2794
# Whisper needs spawn method to avoid deadlock
2795
- VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
@@ -2816,7 +2816,7 @@ steps:
2816
- vllm/platforms/cuda.py
2817
commands:
2818
- rocm-smi
2819
- - python3 examples/offline_inference/basic/chat.py
2819
+ - python3 examples/basic/offline_inference/chat.py
2820
# Attention
2821
# num_heads2 broken by https://github.com/flashinfer-ai/flashinfer/issues/1353
2822
- pytest -v -s tests/kernels/attention/test_attention_selector.py
.buildkite/test_areas/kernels.yaml
+1
-1
@@ -96,7 +96,7 @@ steps:
96
- vllm/platforms/cuda.py
97
commands:
98
- nvidia-smi
99
- - python3 examples/offline_inference/basic/chat.py
99
+ - python3 examples/basic/offline_inference/chat.py
100
# Attention
101
# num_heads2 broken by https://github.com/flashinfer-ai/flashinfer/issues/1353
102
- pytest -v -s tests/kernels/attention/test_attention_selector.py
.buildkite/test_areas/misc.yaml
+7
-6
@@ -67,12 +67,13 @@ steps:
67
- examples/
68
commands:
69
- pip install tensorizer # for tensorizer test
70
- - python3 offline_inference/basic/chat.py # for basic
71
- - python3 offline_inference/basic/generate.py --model facebook/opt-125m
72
- - python3 offline_inference/basic/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
73
- - python3 offline_inference/basic/classify.py
74
- - python3 offline_inference/basic/embed.py
75
- - python3 offline_inference/basic/score.py
70
+ # for basic
71
+ - python3 basic/offline_inference/chat.py
72
+ - python3 basic/offline_inference/generate.py --model facebook/opt-125m
73
+ - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
74
+ - python3 basic/offline_inference/classify.py
75
+ - python3 basic/offline_inference/embed.py
76
+ - python3 basic/offline_inference/score.py
77
# for multi-modal models
78
- python3 offline_inference/audio_language.py --seed 0
79
- python3 offline_inference/vision_language.py --seed 0
.buildkite/test_areas/models_basic.yaml
+1
-1
@@ -65,7 +65,7 @@ steps:
65
- pytest -v -s tests/models/test_transformers.py
66
- pytest -v -s tests/models/multimodal/processing/
67
- pytest -v -s tests/models/multimodal/test_mapping.py
68
- - python3 examples/offline_inference/basic/chat.py
68
+ - python3 examples/basic/offline_inference/chat.py
69
- python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
70
# Whisper needs spawn method to avoid deadlock
71
- VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
docs/getting_started/installation/cpu.md
+1
-1
@@ -259,7 +259,7 @@ ON_CPU=1 SERVING_JSON=serving-tests-cpu-text.json DRY_RUN=1 MODEL_FILTER=meta-ll
259
260
# On this platform, it is recommended to only bind openMP threads on logical CPU cores 0-7 or 8-15
261
$ export VLLM_CPU_OMP_THREADS_BIND=0-7
262
- $ python examples/offline_inference/basic/basic.py
262
+ $ python examples/basic/offline_inference/basic.py
263
```
264
265
- When deploying vLLM CPU backend on a multi-socket machine with NUMA and enable tensor parallel or pipeline parallel, each NUMA node is treated as a TP/PP rank. So be aware to set CPU cores of a single rank on the same NUMA node to avoid cross NUMA node memory access.
docs/getting_started/quickstart.md
+2
-2
@@ -75,7 +75,7 @@ This guide will help you quickly get started with vLLM to perform:
75
76
## Offline Batched Inference
77
78
-With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). See the example script: [examples/offline_inference/basic/basic.py](../../examples/offline_inference/basic/basic.py)
78
+With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). See the example script: [examples/basic/offline_inference/basic.py](../../examples/basic/offline_inference/basic.py)
79
80
The first line of this example imports the classes [LLM][vllm.LLM] and [SamplingParams][vllm.SamplingParams]:
81
@@ -228,7 +228,7 @@ Since this server is compatible with OpenAI API, you can use it as a drop-in rep
228
print("Completion result:", completion)
229
```
230
231
-A more detailed client example can be found here: [examples/offline_inference/basic/basic.py](../../examples/offline_inference/basic/basic.py)
231
+A more detailed client example can be found here: [examples/basic/offline_inference/basic.py](../../examples/basic/offline_inference/basic.py)
232
233
### OpenAI Chat Completions API with vLLM
234
docs/models/generative_models.md
+2
-2
@@ -59,7 +59,7 @@ for output in outputs:
59
By default, vLLM will use sampling parameters recommended by model creator by applying the `generation_config.json` from the huggingface model repository if it exists. In most cases, this will provide you with the best results by default if [SamplingParams][vllm.SamplingParams] is not specified.
60
61
However, if vLLM's default sampling parameters are preferred, please pass `generation_config="vllm"` when creating the [LLM][vllm.LLM] instance.
62
-A code example can be found here: [examples/offline_inference/basic/basic.py](../../examples/offline_inference/basic/basic.py)
62
+A code example can be found here: [examples/basic/offline_inference/basic.py](../../examples/basic/offline_inference/basic.py)
63
64
### `LLM.beam_search`
65
@@ -121,7 +121,7 @@ and automatically applies the model's [chat template](https://huggingface.co/doc
121
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
122
```
123
124
-A code example can be found here: [examples/offline_inference/basic/chat.py](../../examples/offline_inference/basic/chat.py)
124
+A code example can be found here: [examples/basic/offline_inference/chat.py](../../examples/basic/offline_inference/chat.py)
125
126
If the model doesn't have a chat template or you want to specify another one,
127
you can explicitly pass a chat template:
docs/models/pooling_models.md
+4
-4
@@ -99,7 +99,7 @@ embeds = output.outputs.embedding
99
print(f"Embeddings: {embeds!r} (size={len(embeds)})")
100
```
101
102
-A code example can be found here: [examples/offline_inference/basic/embed.py](../../examples/offline_inference/basic/embed.py)
102
+A code example can be found here: [examples/basic/offline_inference/embed.py](../../examples/basic/offline_inference/embed.py)
103
104
### `LLM.classify`
105
@@ -116,7 +116,7 @@ probs = output.outputs.probs
116
print(f"Class Probabilities: {probs!r} (size={len(probs)})")
117
```
118
119
-A code example can be found here: [examples/offline_inference/basic/classify.py](../../examples/offline_inference/basic/classify.py)
119
+A code example can be found here: [examples/basic/offline_inference/classify.py](../../examples/basic/offline_inference/classify.py)
120
121
### `LLM.score`
122
@@ -140,7 +140,7 @@ score = output.outputs.score
140
print(f"Score: {score}")
141
```
142
143
-A code example can be found here: [examples/offline_inference/basic/score.py](../../examples/offline_inference/basic/score.py)
143
+A code example can be found here: [examples/basic/offline_inference/score.py](../../examples/basic/offline_inference/score.py)
144
145
### `LLM.reward`
146
@@ -156,7 +156,7 @@ data = output.outputs.data
156
print(f"Data: {data!r}")
157
```
158
159
-A code example can be found here: [examples/offline_inference/basic/reward.py](../../examples/offline_inference/basic/reward.py)
159
+A code example can be found here: [examples/basic/offline_inference/reward.py](../../examples/basic/offline_inference/reward.py)
160
161
### `LLM.encode`
162
docs/serving/openai_compatible_server.md
+2
-2
@@ -190,7 +190,7 @@ vllm serve NousResearch/Meta-Llama-3-8B-Instruct --enable-offline-docs
190
Our Completions API is compatible with [OpenAI's Completions API](https://platform.openai.com/docs/api-reference/completions);
191
you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it.
192
193
-Code example: [examples/online_serving/openai_completion_client.py](../../examples/online_serving/openai_completion_client.py)
193
+Code example: [examples/basic/online_serving/openai_completion_client.py](../../examples/basic/online_serving/openai_completion_client.py)
194
195
#### Extra parameters
196
@@ -221,7 +221,7 @@ see our [Multimodal Inputs](../features/multimodal_inputs.md) guide for more inf
221
222
- *Note: `image_url.detail` parameter is not supported.*
223
224
-Code example: [examples/online_serving/openai_chat_completion_client.py](../../examples/online_serving/openai_chat_completion_client.py)
224
+Code example: [examples/basic/online_serving/openai_chat_completion_client.py](../../examples/basic/online_serving/openai_chat_completion_client.py)
225
226
#### Extra parameters
227
examples/basic/offline_inference/README.md
renamed
+7
-7
@@ -1,4 +1,4 @@
1
-# Basic
1
+# Offline Inference
2
3
The `LLM` class provides the primary Python interface for doing offline inference, which is interacting with a model without using a separate model inference server.
4
@@ -7,31 +7,31 @@ The `LLM` class provides the primary Python interface for doing offline inferenc
7
The first script in this example shows the most basic usage of vLLM. If you are new to Python and vLLM, you should start here.
8
9
```bash
10
-python examples/offline_inference/basic/basic.py
10
+python examples/basic/offline_inference/basic.py
11
```
12
13
The rest of the scripts include an [argument parser](https://docs.python.org/3/library/argparse.html), which you can use to pass any arguments that are compatible with [`LLM`](https://docs.vllm.ai/en/latest/api/offline_inference/llm.html). Try running the script with `--help` for a list of all available arguments.
14
15
```bash
16
-python examples/offline_inference/basic/classify.py
16
+python examples/basic/offline_inference/classify.py
17
```
18
19
```bash
20
-python examples/offline_inference/basic/embed.py
20
+python examples/basic/offline_inference/embed.py
21
```
22
23
```bash
24
-python examples/offline_inference/basic/score.py
24
+python examples/basic/offline_inference/score.py
25
```
26
27
The chat and generate scripts also accept the [sampling parameters](https://docs.vllm.ai/en/latest/api/inference_params.html#sampling-parameters): `max_tokens`, `temperature`, `top_p` and `top_k`.
28
29
```bash
30
-python examples/offline_inference/basic/chat.py
30
+python examples/basic/offline_inference/chat.py
31
```
32
33
```bash
34
-python examples/offline_inference/basic/generate.py
34
+python examples/basic/offline_inference/generate.py
35
```
36
37
## Features
examples/basic/offline_inference/basic.py
renamed
examples/basic/offline_inference/chat.py
renamed
examples/basic/offline_inference/classify.py
renamed
examples/basic/offline_inference/embed.py
renamed
+3
-4
@@ -5,6 +5,7 @@ from argparse import Namespace
5
6
from vllm import LLM, EngineArgs
7
from vllm.utils.argparse_utils import FlexibleArgumentParser
8
+from vllm.utils.print_utils import print_embeddings
9
10
11
def parse_args():
@@ -39,10 +40,8 @@ def main(args: Namespace):
40
print("\nGenerated Outputs:\n" + "-" * 60)
41
for prompt, output in zip(prompts, outputs):
42
embeds = output.outputs.embedding
42
- embeds_trimmed = (
43
- (str(embeds[:16])[:-1] + ", ...]") if len(embeds) > 16 else embeds
44
- )
45
- print(f"Prompt: {prompt!r} \nEmbeddings: {embeds_trimmed} (size={len(embeds)})")
43
+ print(f"Prompt: {prompt!r}")
44
+ print_embeddings(embeds)
45
print("-" * 60)
46
47
examples/basic/offline_inference/generate.py
renamed
examples/basic/offline_inference/reward.py
renamed
+3
-4
@@ -5,6 +5,7 @@ from argparse import Namespace
5
6
from vllm import LLM, EngineArgs
7
from vllm.utils.argparse_utils import FlexibleArgumentParser
8
+from vllm.utils.print_utils import print_embeddings
9
10
11
def parse_args():
@@ -41,10 +42,8 @@ def main(args: Namespace):
42
print("\nGenerated Outputs:\n" + "-" * 60)
43
for prompt, output in zip(prompts, outputs):
44
rewards = output.outputs.data
44
- rewards_trimmed = (
45
- (str(rewards[:16])[:-1] + ", ...]") if len(rewards) > 16 else rewards
46
- )
47
- print(f"Prompt: {prompt!r} \nReward: {rewards_trimmed} (size={len(rewards)})")
45
+ print(f"Prompt: {prompt!r}")
46
+ print_embeddings(rewards, prefix="Reward")
47
print("-" * 60)
48
49
examples/basic/offline_inference/score.py
renamed
examples/basic/online_serving/openai_chat_completion_client.py
renamed
examples/basic/online_serving/openai_completion_client.py
renamed
tests/plugins_tests/test_platform_plugins.py
+1
-1
@@ -17,7 +17,7 @@ def test_platform_plugins():
17
example_file = os.path.join(
18
os.path.dirname(os.path.dirname(os.path.dirname(current_file))),
19
"examples",
20
- "offline_inference/basic/basic.py",
20
+ "basic/offline_inference/basic.py",
21
)
22
runpy.run_path(example_file)
23
vllm/utils/print_utils.py
+2
-2
@@ -2,6 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
-def print_embeddings(embeds: list[float]):
5
+def print_embeddings(embeds: list[float], prefix: str = "Embeddings"):
6
embeds_trimmed = (str(embeds[:4])[:-1] + ", ...]") if len(embeds) > 4 else embeds
7
- print(f"Embeddings: {embeds_trimmed} (size={len(embeds)})")
7
+ print(f"{prefix}: {embeds_trimmed} (size={len(embeds)})")