A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Doc] Rename offline inference examples (#11927)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Jan 10, 2025 at 15:50 UTC
482cdc494e608b72303f49b56532f5c50b61cbdb
46 files changed
+46
-46
.buildkite/run-cpu-test.sh
+1
-1
@@ -30,7 +30,7 @@ function cpu_tests() {
30
# offline inference
31
docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" bash -c "
32
set -e
33
- python3 examples/offline_inference/offline_inference.py"
33
+ python3 examples/offline_inference/basic.py"
34
35
# Run basic model test
36
docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
.buildkite/run-gh200-test.sh
+1
-1
@@ -24,5 +24,5 @@ remove_docker_container
24
25
# Run the image and test offline inference
26
docker run --name gh200-test --gpus=all --entrypoint="" gh200-test bash -c '
27
- python3 examples/offline_inference/offline_inference.py
27
+ python3 examples/offline_inference/basic.py
28
'
.buildkite/run-hpu-test.sh
+1
-1
@@ -13,4 +13,4 @@ trap remove_docker_container EXIT
13
remove_docker_container
14
15
# Run the image and launch offline inference
16
-docker run --runtime=habana --name=hpu-test --network=host -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference/offline_inference.py
\ No newline at end of file
16
+docker run --runtime=habana --name=hpu-test --network=host -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference/basic.py
\ No newline at end of file
.buildkite/run-neuron-test.sh
+1
-1
@@ -51,4 +51,4 @@ docker run --rm -it --device=/dev/neuron0 --device=/dev/neuron1 --network host \
51
-e "NEURON_COMPILE_CACHE_URL=${NEURON_COMPILE_CACHE_MOUNT}" \
52
--name "${container_name}" \
53
${image_name} \
54
- /bin/bash -c "python3 /workspace/vllm/examples/offline_inference/offline_inference_neuron.py"
54
+ /bin/bash -c "python3 /workspace/vllm/examples/offline_inference/neuron.py"
.buildkite/run-openvino-test.sh
+1
-1
@@ -13,4 +13,4 @@ trap remove_docker_container EXIT
13
remove_docker_container
14
15
# Run the image and launch offline inference
16
-docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/examples/offline_inference/offline_inference.py
16
+docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/examples/offline_inference/basic.py
.buildkite/run-tpu-test.sh
+1
-1
@@ -23,4 +23,4 @@ docker run --privileged --net host --shm-size=16G -it \
23
&& pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py \
24
&& python3 /workspace/vllm/tests/tpu/test_compilation.py \
25
&& python3 /workspace/vllm/tests/tpu/test_quantization_accuracy.py \
26
- && python3 /workspace/vllm/examples/offline_inference/offline_inference_tpu.py"
26
+ && python3 /workspace/vllm/examples/offline_inference/tpu.py"
.buildkite/run-xpu-test.sh
+2
-2
@@ -14,6 +14,6 @@ remove_docker_container
14
15
# Run the image and test offline inference/tensor parallel
16
docker run --name xpu-test --device /dev/dri -v /dev/dri/by-path:/dev/dri/by-path --entrypoint="" xpu-test sh -c '
17
- python3 examples/offline_inference/offline_inference.py
18
- python3 examples/offline_inference/offline_inference_cli.py -tp 2
17
+ python3 examples/offline_inference/basic.py
18
+ python3 examples/offline_inference/cli.py -tp 2
19
'
.buildkite/test-pipeline.yaml
+10
-10
@@ -187,19 +187,19 @@ steps:
187
- examples/
188
commands:
189
- pip install tensorizer # for tensorizer test
190
- - python3 offline_inference/offline_inference.py
190
+ - python3 offline_inference/basic.py
191
- python3 offline_inference/cpu_offload.py
192
- - python3 offline_inference/offline_inference_chat.py
193
- - python3 offline_inference/offline_inference_with_prefix.py
192
+ - python3 offline_inference/chat.py
193
+ - python3 offline_inference/prefix_caching.py
194
- python3 offline_inference/llm_engine_example.py
195
- - python3 offline_inference/offline_inference_vision_language.py
196
- - python3 offline_inference/offline_inference_vision_language_multi_image.py
195
+ - python3 offline_inference/vision_language.py
196
+ - python3 offline_inference/vision_language_multi_image.py
197
- python3 other/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 other/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
198
- - python3 offline_inference/offline_inference_encoder_decoder.py
199
- - python3 offline_inference/offline_inference_classification.py
200
- - python3 offline_inference/offline_inference_embedding.py
201
- - python3 offline_inference/offline_inference_scoring.py
202
- - python3 offline_inference/offline_profile.py --model facebook/opt-125m run_num_steps --num-steps 2
198
+ - python3 offline_inference/encoder_decoder.py
199
+ - python3 offline_inference/classification.py
200
+ - python3 offline_inference/embedding.py
201
+ - python3 offline_inference/scoring.py
202
+ - python3 offline_inference/profiling.py --model facebook/opt-125m run_num_steps --num-steps 2
203
204
- label: Prefix Caching Test # 9min
205
mirror_hardwares: [amd]
docs/source/contributing/profiling/profiling_index.md
+1
-1
@@ -26,7 +26,7 @@ Set the env variable VLLM_RPC_TIMEOUT to a big number before you start the serve
26
27
### Offline Inference
28
29
-Refer to <gh-file:examples/offline_inference/offline_inference_with_profiler.py> for an example.
29
+Refer to <gh-file:examples/offline_inference/simple_profiling.py> for an example.
30
31
### OpenAI Server
32
docs/source/features/structured_outputs.md
+1
-1
@@ -257,4 +257,4 @@ outputs = llm.generate(
257
print(outputs[0].outputs[0].text)
258
```
259
260
-Full example: <gh-file:examples/offline_inference/offline_inference_structured_outputs.py>
260
+Full example: <gh-file:examples/offline_inference/structured_outputs.py>
docs/source/getting_started/installation/cpu-x86.md
+2
-2
@@ -95,7 +95,7 @@ $ VLLM_TARGET_DEVICE=cpu python setup.py install
95
$ sudo apt-get install libtcmalloc-minimal4 # install TCMalloc library
96
$ find / -name *libtcmalloc* # find the dynamic link library path
97
$ export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$LD_PRELOAD # prepend the library to LD_PRELOAD
98
-$ python examples/offline_inference/offline_inference.py # run vLLM
98
+$ python examples/offline_inference/basic.py # run vLLM
99
```
100
101
- When using the online serving, it is recommended to reserve 1-2 CPU cores for the serving framework to avoid CPU oversubscription. For example, on a platform with 32 physical CPU cores, reserving CPU 30 and 31 for the framework and using CPU 0-29 for OpenMP:
@@ -132,7 +132,7 @@ CPU NODE SOCKET CORE L1d:L1i:L2:L3 ONLINE MAXMHZ MINMHZ MHZ
132
133
# On this platform, it is recommend to only bind openMP threads on logical CPU cores 0-7 or 8-15
134
$ export VLLM_CPU_OMP_THREADS_BIND=0-7
135
-$ python examples/offline_inference/offline_inference.py
135
+$ python examples/offline_inference/basic.py
136
```
137
138
- If using vLLM CPU backend on a multi-socket machine with NUMA, be aware to set CPU cores using `VLLM_CPU_OMP_THREADS_BIND` to avoid cross NUMA node memory access.
docs/source/getting_started/quickstart.md
+1
-1
@@ -40,7 +40,7 @@ For non-CUDA platforms, please refer [here](#installation-index) for specific in
40
41
## Offline Batched Inference
42
43
-With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). See the example script: <gh-file:examples/offline_inference/offline_inference.py>
43
+With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). See the example script: <gh-file:examples/offline_inference/basic.py>
44
45
The first line of this example imports the classes {class}`~vllm.LLM` and {class}`~vllm.SamplingParams`:
46
docs/source/models/generative_models.md
+2
-2
@@ -46,7 +46,7 @@ for output in outputs:
46
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
47
```
48
49
-A code example can be found here: <gh-file:examples/offline_inference/offline_inference.py>
49
+A code example can be found here: <gh-file:examples/offline_inference/basic.py>
50
51
### `LLM.beam_search`
52
@@ -103,7 +103,7 @@ for output in outputs:
103
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
104
```
105
106
-A code example can be found here: <gh-file:examples/offline_inference/offline_inference_chat.py>
106
+A code example can be found here: <gh-file:examples/offline_inference/chat.py>
107
108
If the model doesn't have a chat template or you want to specify another one,
109
you can explicitly pass a chat template:
docs/source/models/pooling_models.md
+3
-3
@@ -88,7 +88,7 @@ embeds = output.outputs.embedding
88
print(f"Embeddings: {embeds!r} (size={len(embeds)})")
89
```
90
91
-A code example can be found here: <gh-file:examples/offline_inference/offline_inference_embedding.py>
91
+A code example can be found here: <gh-file:examples/offline_inference/embedding.py>
92
93
### `LLM.classify`
94
@@ -103,7 +103,7 @@ probs = output.outputs.probs
103
print(f"Class Probabilities: {probs!r} (size={len(probs)})")
104
```
105
106
-A code example can be found here: <gh-file:examples/offline_inference/offline_inference_classification.py>
106
+A code example can be found here: <gh-file:examples/offline_inference/classification.py>
107
108
### `LLM.score`
109
@@ -125,7 +125,7 @@ score = output.outputs.score
125
print(f"Score: {score}")
126
```
127
128
-A code example can be found here: <gh-file:examples/offline_inference/offline_inference_scoring.py>
128
+A code example can be found here: <gh-file:examples/offline_inference/scoring.py>
129
130
## Online Serving
131
docs/source/serving/multimodal_inputs.md
+4
-4
@@ -60,7 +60,7 @@ for o in outputs:
60
print(generated_text)
61
```
62
63
-Full example: <gh-file:examples/offline_inference/offline_inference_vision_language.py>
63
+Full example: <gh-file:examples/offline_inference/vision_language.py>
64
65
To substitute multiple images inside the same text prompt, you can pass in a list of images instead:
66
@@ -91,7 +91,7 @@ for o in outputs:
91
print(generated_text)
92
```
93
94
-Full example: <gh-file:examples/offline_inference/offline_inference_vision_language_multi_image.py>
94
+Full example: <gh-file:examples/offline_inference/vision_language_multi_image.py>
95
96
Multi-image input can be extended to perform video captioning. We show this with [Qwen2-VL](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct) as it supports videos:
97
@@ -125,13 +125,13 @@ for o in outputs:
125
You can pass a list of NumPy arrays directly to the `'video'` field of the multi-modal dictionary
126
instead of using multi-image input.
127
128
-Full example: <gh-file:examples/offline_inference/offline_inference_vision_language.py>
128
+Full example: <gh-file:examples/offline_inference/vision_language.py>
129
130
### Audio
131
132
You can pass a tuple `(array, sampling_rate)` to the `'audio'` field of the multi-modal dictionary.
133
134
-Full example: <gh-file:examples/offline_inference/offline_inference_audio_language.py>
134
+Full example: <gh-file:examples/offline_inference/audio_language.py>
135
136
### Embedding
137
examples/offline_inference/arctic.py
renamed
examples/offline_inference/audio_language.py
renamed
examples/offline_inference/basic.py
renamed
examples/offline_inference/basic_with_model_default_sampling.py
renamed
examples/offline_inference/chat.py
renamed
examples/offline_inference/chat_with_tools.py
renamed
examples/offline_inference/classification.py
renamed
examples/offline_inference/cli.py
renamed
examples/offline_inference/distributed.py
renamed
examples/offline_inference/embedding.py
renamed
examples/offline_inference/encoder_decoder.py
renamed
examples/offline_inference/florence2_inference.py
+1
-1
@@ -3,7 +3,7 @@ Demonstrate prompting of text-to-text
3
encoder/decoder models, specifically Florence-2
4
'''
5
# TODO(Isotr0py):
6
-# Move to offline_inference/offline_inference_vision_language.py
6
+# Move to offline_inference/vision_language.py
7
# after porting vision backbone
8
from vllm import LLM, SamplingParams
9
examples/offline_inference/mlpspeculator.py
renamed
examples/offline_inference/neuron.py
renamed
examples/offline_inference/neuron_int8_quantization.py
renamed
examples/offline_inference/openai/openai_batch.md
renamed
+9
-9
@@ -8,7 +8,7 @@ This is a guide to performing batch inference using the OpenAI batch file format
8
9
The OpenAI batch file format consists of a series of json objects on new lines.
10
11
-[See here for an example file.](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl)
11
+[See here for an example file.](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference/openai/openai_example_batch.jsonl)
12
13
Each line represents a separate request. See the [OpenAI package reference](https://platform.openai.com/docs/api-reference/batch/requestInput) for more details.
14
@@ -31,13 +31,13 @@ We currently only support `/v1/chat/completions` and `/v1/embeddings` endpoints
31
To follow along with this example, you can download the example batch, or create your own batch file in your working directory.
32
33
```
34
-wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl
34
+wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai/openai_example_batch.jsonl
35
```
36
37
Once you've created your batch file it should look like this
38
39
```
40
-$ cat offline_inference/offline_inference_openai/openai_example_batch.jsonl
40
+$ cat offline_inference/openai/openai_example_batch.jsonl
41
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
42
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
43
```
@@ -49,7 +49,7 @@ The batch running tool is designed to be used from the command line.
49
You can run the batch with the following command, which will write its results to a file called `results.jsonl`
50
51
```
52
-python -m vllm.entrypoints.openai.run_batch -i offline_inference/offline_inference_openai/openai_example_batch.jsonl -o results.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct
52
+python -m vllm.entrypoints.openai.run_batch -i offline_inference/openai/openai_example_batch.jsonl -o results.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct
53
```
54
55
### Step 3: Check your results
@@ -66,10 +66,10 @@ $ cat results.jsonl
66
67
The batch runner supports remote input and output urls that are accessible via http/https.
68
69
-For example, to run against our example input file located at `https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl`, you can run
69
+For example, to run against our example input file located at `https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai/openai_example_batch.jsonl`, you can run
70
71
```
72
-python -m vllm.entrypoints.openai.run_batch -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl -o results.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct
72
+python -m vllm.entrypoints.openai.run_batch -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai/openai_example_batch.jsonl -o results.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct
73
```
74
75
## Example 3: Integrating with AWS S3
@@ -90,13 +90,13 @@ To integrate with cloud blob storage, we recommend using presigned urls.
90
To follow along with this example, you can download the example batch, or create your own batch file in your working directory.
91
92
```
93
-wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl
93
+wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/openai/openai_example_batch.jsonl
94
```
95
96
Once you've created your batch file it should look like this
97
98
```
99
-$ cat offline_inference/offline_inference_openai/openai_example_batch.jsonl
99
+$ cat offline_inference/openai/openai_example_batch.jsonl
100
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
101
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
102
```
@@ -104,7 +104,7 @@ $ cat offline_inference/offline_inference_openai/openai_example_batch.jsonl
104
Now upload your batch file to your S3 bucket.
105
106
```
107
-aws s3 cp offline_inference/offline_inference_openai/openai_example_batch.jsonl s3://MY_BUCKET/MY_INPUT_FILE.jsonl
107
+aws s3 cp offline_inference/openai/openai_example_batch.jsonl s3://MY_BUCKET/MY_INPUT_FILE.jsonl
108
```
109
110
### Step 2: Generate your presigned urls
examples/offline_inference/openai/openai_example_batch.jsonl
renamed
examples/offline_inference/pixtral.py
renamed
examples/offline_inference/prefix_caching.py
renamed
examples/offline_inference/profiling.py
renamed
+1
-1
@@ -363,7 +363,7 @@ Profile a model
363
364
example:
365
```
366
- python examples/offline_inference/offline_profile.py \\
366
+ python examples/offline_inference/profiling.py \\
367
--model neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8 --batch-size 4 \\
368
--prompt-len 512 --max-num-batched-tokens 8196 --json Llama31-8b-FP8 \\
369
--enforce-eager run_num_steps -n 2
examples/offline_inference/scoring.py
renamed
examples/offline_inference/simple_profiling.py
renamed
examples/offline_inference/structured_outputs.py
renamed
examples/offline_inference/tpu.py
renamed
examples/offline_inference/vision_language.py
renamed
examples/offline_inference/vision_language_embedding.py
renamed
examples/offline_inference/vision_language_multi_image.py
renamed
examples/offline_inference/whisper.py
renamed
tests/plugins_tests/test_platform_plugins.py
+1
-1
@@ -5,7 +5,7 @@ def test_platform_plugins():
5
import os
6
example_file = os.path.join(
7
os.path.dirname(os.path.dirname(os.path.dirname(current_file))),
8
- "examples", "offline_inference/offline_inference.py")
8
+ "examples", "offline_inference/basic.py")
9
runpy.run_path(example_file)
10
11
# check if the plugin is loaded correctly
tools/profiler/print_layerwise_table.py
+1
-1
@@ -31,7 +31,7 @@ if __name__ == "__main__":
31
type=str,
32
required=True,
33
help="json trace file output by "
34
- "examples/offline_inference/offline_profile.py")
34
+ "examples/offline_inference/profiling.py")
35
parser.add_argument("--phase",
36
type=str,
37
required=True,
tools/profiler/visualize_layerwise_profile.py
+1
-1
@@ -538,7 +538,7 @@ if __name__ == "__main__":
538
type=str,
539
required=True,
540
help="json trace file output by \
541
- examples/offline_inference/offline_profile.py")
541
+ examples/offline_inference/profiling.py")
542
parser.add_argument("--output-directory",
543
type=str,
544
required=False,