A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 15:00 UTC
[Examples] Resettle generate examples. (#36464)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Apr 27, 2026 at 15:48 UTC
8d8062d0a7014b4cde064024ae5d5a8715a833b3
36 files changed
+46
-50
.buildkite/test-amd.yaml
+14
-14
@@ -388,10 +388,10 @@ steps:
388
- python3 basic/offline_inference/embed.py
389
- python3 basic/offline_inference/score.py
390
# Multi-modal models
391
- - python3 offline_inference/audio_language.py --seed 0
392
- - python3 offline_inference/vision_language.py --seed 0
393
- - python3 offline_inference/vision_language_multi_image.py --seed 0
394
- - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
391
+ - python3 generate/multimodal/audio_language_offline.py --seed 0
392
+ - python3 generate/multimodal/vision_language_offline.py --seed 0
393
+ - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
394
+ - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
395
# Pooling models
396
- python3 pooling/embed/vision_embedding_offline.py --seed 0
397
# Features demo
@@ -1647,10 +1647,10 @@ steps:
1647
- python3 basic/offline_inference/embed.py
1648
- python3 basic/offline_inference/score.py
1649
# Multi-modal models
1650
- - python3 offline_inference/audio_language.py --seed 0
1651
- - python3 offline_inference/vision_language.py --seed 0
1652
- - python3 offline_inference/vision_language_multi_image.py --seed 0
1653
- - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
1650
+ - python3 generate/multimodal/audio_language_offline.py --seed 0
1651
+ - python3 generate/multimodal/vision_language_offline.py --seed 0
1652
+ - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
1653
+ - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
1654
# Pooling models
1655
- python3 pooling/embed/vision_embedding_offline.py --seed 0
1656
# Features demo
@@ -1951,8 +1951,8 @@ steps:
1951
- pytest -v -s tests/models/multimodal/processing/
1952
- pytest -v -s tests/models/multimodal/test_mapping.py
1953
- python3 examples/basic/offline_inference/chat.py
1954
- - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
1955
- - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
1954
+ - python3 examples/generate/multimodal/vision_language_offline.py --model-type qwen2_5_vl
1955
+ - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/generate/multimodal/audio_language_offline.py --model-type whisper
1956
1957
#------------------------------------------------------- mi300 · quantization --------------------------------------------------------#
1958
@@ -2930,10 +2930,10 @@ steps:
2930
- python3 basic/offline_inference/embed.py
2931
- python3 basic/offline_inference/score.py
2932
# Multi-modal models
2933
- - python3 offline_inference/audio_language.py --seed 0
2934
- - python3 offline_inference/vision_language.py --seed 0
2935
- - python3 offline_inference/vision_language_multi_image.py --seed 0
2936
- - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
2933
+ - python3 generate/multimodal/audio_language_offline.py --seed 0
2934
+ - python3 generate/multimodal/vision_language_offline.py --seed 0
2935
+ - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
2936
+ - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
2937
# Pooling models
2938
- python3 pooling/embed/vision_embedding_offline.py --seed 0
2939
# Features demo
.buildkite/test_areas/misc.yaml
+4
-4
@@ -113,10 +113,10 @@ steps:
113
- python3 basic/offline_inference/embed.py
114
- python3 basic/offline_inference/score.py
115
# for multi-modal models
116
- - python3 offline_inference/audio_language.py --seed 0
117
- - python3 offline_inference/vision_language.py --seed 0
118
- - python3 offline_inference/vision_language_multi_image.py --seed 0
119
- - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
116
+ - python3 generate/multimodal/audio_language_offline.py --seed 0
117
+ - python3 generate/multimodal/vision_language_offline.py --seed 0
118
+ - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
119
+ - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
120
# for pooling models
121
- python3 pooling/embed/vision_embedding_offline.py --seed 0
122
# for features demo
.buildkite/test_areas/model_runner_v2.yaml
+4
-4
@@ -44,10 +44,10 @@ steps:
44
#- python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 # TODO
45
#- python3 basic/offline_inference/embed.py # TODO
46
# for multi-modal models
47
- - python3 offline_inference/audio_language.py --seed 0
48
- - python3 offline_inference/vision_language.py --seed 0
49
- - python3 offline_inference/vision_language_multi_image.py --seed 0
50
- - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
47
+ - python3 generate/multimodal/audio_language_offline.py --seed 0
48
+ - python3 generate/multimodal/vision_language_offline.py --seed 0
49
+ - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
50
+ - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
51
# for pooling models
52
- python3 pooling/embed/vision_embedding_offline.py --seed 0
53
# for features demo
.buildkite/test_areas/models_basic.yaml
+5
-5
@@ -69,9 +69,9 @@ steps:
69
- pytest -v -s tests/models/multimodal/processing/
70
- pytest -v -s tests/models/multimodal/test_mapping.py
71
- python3 examples/basic/offline_inference/chat.py
72
- - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
72
+ - python3 examples/generate/multimodal/vision_language_offline.py --model-type qwen2_5_vl
73
# Whisper needs spawn method to avoid deadlock
74
- - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
74
+ - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/generate/multimodal/audio_language_offline.py --model-type whisper
75
76
- label: Transformers Backward Compatibility Models Test
77
working_dir: "/vllm-workspace/"
@@ -83,7 +83,7 @@ steps:
83
- pytest -v -s tests/models/test_transformers.py
84
- pytest -v -s tests/models/multimodal/processing/
85
- pytest -v -s tests/models/multimodal/test_mapping.py
86
- - python3 examples/offline_inference/basic/chat.py
87
- - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
86
+ - python3 examples/basic/offline_inference/chat.py
87
+ - python3 examples/generate/multimodal/vision_language_offline.py --model-type qwen2_5_vl
88
# Whisper needs spawn method to avoid deadlock
89
- - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
89
+ - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/generate/multimodal/audio_language_offline.py --model-type whisper
.github/mergify.yml
+1
-5
@@ -389,11 +389,7 @@ pull_request_rules:
389
- files~=^tests/entrypoints/anthropic/.*tool.*
390
- files~=^vllm/tool_parsers/
391
- files=docs/features/tool_calling.md
392
- - files~=^examples/tool_chat_*
393
- - files=examples/offline_inference/chat_with_tools.py
394
- - files=examples/online_serving/openai_chat_completion_client_with_tools_required.py
395
- - files=examples/online_serving/openai_chat_completion_tool_calls_with_reasoning.py
396
- - files=examples/online_serving/openai_chat_completion_client_with_tools.py
392
+ - files~=^examples/tool_calling/
393
actions:
394
label:
395
add:
docs/features/multimodal_inputs.md
+7
-7
@@ -68,7 +68,7 @@ You can pass a single image to the `'image'` field of the multi-modal dictionary
68
print(generated_text)
69
```
70
71
-Full example: [examples/offline_inference/vision_language.py](../../examples/offline_inference/vision_language.py)
71
+Full example: [examples/generate/multimodal/vision_language_offline.py](../../examples/generate/multimodal/vision_language_offline.py)
72
73
To substitute multiple images inside the same text prompt, you can pass in a list of images instead:
74
@@ -101,7 +101,7 @@ To substitute multiple images inside the same text prompt, you can pass in a lis
101
print(generated_text)
102
```
103
104
-Full example: [examples/offline_inference/vision_language_multi_image.py](../../examples/offline_inference/vision_language_multi_image.py)
104
+Full example: [examples/generate/multimodal/vision_language_multi_image_offline.py](../../examples/generate/multimodal/vision_language_multi_image_offline.py)
105
106
If using the [LLM.chat](../models/generative_models.md#llmchat) method, you can pass images directly in the message content using various formats: image URLs, PIL Image objects, or pre-computed embeddings:
107
@@ -287,13 +287,13 @@ Instead of NumPy arrays, you can also pass `'torch.Tensor'` instances, as shown
287
!!! note
288
'process_vision_info' is only applicable to Qwen2.5-VL and similar models.
289
290
-Full example: [examples/offline_inference/vision_language.py](../../examples/offline_inference/vision_language.py)
290
+Full example: [examples/generate/multimodal/vision_language_offline.py](../../examples/generate/multimodal/vision_language_offline.py)
291
292
### Audio Inputs
293
294
You can pass a tuple `(array, sampling_rate)` to the `'audio'` field of the multi-modal dictionary.
295
296
-Full example: [examples/offline_inference/audio_language.py](../../examples/offline_inference/audio_language.py)
296
+Full example: [examples/generate/multimodal/audio_language_offline.py](../../examples/generate/multimodal/audio_language_offline.py)
297
298
#### Chunking Long Audio for Transcription
299
@@ -674,7 +674,7 @@ Then, you can use the OpenAI client as follows:
674
print("Chat completion output:", chat_response.choices[0].message.content)
675
```
676
677
-Full example: [examples/online_serving/openai_chat_completion_client_for_multimodal.py](../../examples/online_serving/openai_chat_completion_client_for_multimodal.py)
677
+Full example: [examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py](../../examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py)
678
679
!!! tip
680
Loading from local file paths is also supported on vLLM: You can specify the allowed local media path via `--allowed-local-media-path` when launching the API server/engine,
@@ -745,7 +745,7 @@ Then, you can use the OpenAI client as follows:
745
print("Chat completion output from image url:", result)
746
```
747
748
-Full example: [examples/online_serving/openai_chat_completion_client_for_multimodal.py](../../examples/online_serving/openai_chat_completion_client_for_multimodal.py)
748
+Full example: [examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py](../../examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py)
749
750
!!! note
751
By default, the timeout for fetching videos through HTTP URL is `30` seconds.
@@ -958,7 +958,7 @@ Alternatively, you can pass `audio_url`, which is the audio counterpart of `imag
958
print("Chat completion output from audio url:", result)
959
```
960
961
-Full example: [examples/online_serving/openai_chat_completion_client_for_multimodal.py](../../examples/online_serving/openai_chat_completion_client_for_multimodal.py)
961
+Full example: [examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py](../../examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py)
962
963
!!! note
964
By default, the timeout for fetching audios through HTTP URL is `10` seconds.
docs/features/reasoning_outputs.md
+1
-1
@@ -202,7 +202,7 @@ The reasoning content is also available when both tool calling and the reasoning
202
print(f"Arguments: {tool_call.arguments}")
203
```
204
205
-For more examples, please refer to [examples/online_serving/openai_chat_completion_tool_calls_with_reasoning.py](../../examples/online_serving/openai_chat_completion_tool_calls_with_reasoning.py).
205
+For more examples, please refer to [examples/reasoning/openai_chat_completion_tool_calls_with_reasoning.py](../../examples/reasoning/openai_chat_completion_tool_calls_with_reasoning.py).
206
207
## Server-Level Default Chat Template Kwargs
208
docs/serving/openai_compatible_server.md
+3
-3
@@ -251,7 +251,7 @@ The following extra parameters are supported:
251
Our Responses API is compatible with [OpenAI's Responses API](https://platform.openai.com/docs/api-reference/responses);
252
you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it.
253
254
-Code example: [examples/online_serving/openai_responses_client_with_tools.py](../../examples/online_serving/openai_responses_client_with_tools.py)
254
+Code example: [examples/online_serving/openai_responses_client_with_tools.py](../../examples/tool_calling/openai_responses_client_with_tools.py)
255
256
#### Extra parameters
257
@@ -279,7 +279,7 @@ you can use the [official OpenAI Python client](https://github.com/openai/openai
279
!!! note
280
To use the Transcriptions API, please install with extra audio dependencies using `pip install vllm[audio]`.
281
282
-Code example: [examples/online_serving/openai_transcription_client.py](../../examples/online_serving/openai_transcription_client.py)
282
+Code example: [examples/speech_to_text/openai/openai_transcription_client.py](../../examples/speech_to_text/openai/openai_transcription_client.py)
283
284
NOTE: beam search is currently supported in the transcriptions endpoint for encoder-decoder multimodal models, e.g., whisper, but highly inefficient as work for handling the encoder/decoder cache is actively ongoing. This is an active point of ongoing optimization and will be handled properly in the very near future.
285
@@ -397,7 +397,7 @@ Please mind that the popular `openai/whisper-large-v3-turbo` model does not supp
397
!!! note
398
To use the Translation API, please install with extra audio dependencies using `pip install vllm[audio]`.
399
400
-Code example: [examples/online_serving/openai_translation_client.py](../../examples/online_serving/openai_translation_client.py)
400
+Code example: [examples/speech_to_text/openai/openai_translation_client.py](../../examples/speech_to_text/openai/openai_translation_client.py)
401
402
#### Extra Parameters
403
examples/generate/batched_chat_completions_online.py
renamed
examples/generate/multimodal/audio_language_offline.py
renamed
examples/generate/multimodal/encoder_decoder_multimodal_offline.py
renamed
examples/generate/multimodal/mistral-small_offline.py
renamed
examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py
renamed
examples/generate/multimodal/qwen2_5_omni/README.md
renamed
+6
-6
@@ -6,15 +6,15 @@ This folder provides several example scripts on how to inference Qwen2.5-Omni of
6
7
```bash
8
# Audio + image + video
9
-python examples/offline_inference/qwen2_5_omni/only_thinker.py \
9
+python examples/generate/multimodal/qwen2_5_omni/only_thinker.py \
10
-q mixed_modalities
11
12
# Read vision and audio inputs from a single video file
13
-python examples/offline_inference/qwen2_5_omni/only_thinker.py \
13
+python examples/generate/multimodal/qwen2_5_omni/only_thinker.py \
14
-q use_audio_in_video
15
16
# Multiple audios
17
-python examples/offline_inference/qwen2_5_omni/only_thinker.py \
17
+python examples/generate/multimodal/qwen2_5_omni/only_thinker.py \
18
-q multi_audios
19
```
20
@@ -24,16 +24,16 @@ You can also test Qwen2.5-Omni on a single modality:
24
25
```bash
26
# Process audio inputs
27
-python examples/offline_inference/audio_language.py \
27
+python examples/generate/multimodal/audio_language_offline.py \
28
--model-type qwen2_5_omni
29
30
# Process image inputs
31
-python examples/offline_inference/vision_language.py \
31
+python examples/generate/multimodal/vision_language_offline.py \
32
--modality image \
33
--model-type qwen2_5_omni
34
35
# Process video inputs
36
-python examples/offline_inference/vision_language.py \
36
+python examples/generate/multimodal/vision_language_offline.py \
37
--modality video \
38
--model-type qwen2_5_omni
39
```
examples/generate/multimodal/qwen2_5_omni/only_thinker.py
renamed
examples/generate/multimodal/qwen3_omni/only_thinker.py
renamed
examples/generate/multimodal/vision_language_multi_image_offline.py
renamed
examples/generate/multimodal/vision_language_offline.py
renamed
+1
-1
@@ -1402,7 +1402,7 @@ def run_mantis(questions: list[str], modality: str) -> ModelRequestData:
1402
# MiniCPM-V
1403
def run_minicpmv_base(questions: list[str], modality: str, model_name):
1404
assert modality in ["image", "video", "image+video"]
1405
- # If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language.py` # noqa
1405
+ # If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language_offline.py` # noqa
1406
1407
# 2.0
1408
# The official repo doesn't work yet, so we need to use a fork for now