@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 992e5c3d34
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 22:15 UTC

Merge similar examples in `offline_inference` into single `basic` example (#12737)

Harry Mellor committed Feb 20, 2025 at 12:53 UTC 992e5c3d34776cae25a9fecf0a45d51fa69d6dbc
29 files changed +394 -437
.buildkite/run-cpu-test.sh
+1 -1
@@ -30,7 +30,7 @@ function cpu_tests() {
30 # offline inference
31 docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" bash -c "
32 set -e
33 - python3 examples/offline_inference/basic.py"
33 + python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m"
34
35 # Run basic model test
36 docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
.buildkite/run-gh200-test.sh
+1 -1
@@ -24,5 +24,5 @@ remove_docker_container
24
25 # Run the image and test offline inference
26 docker run -e HF_TOKEN -v /root/.cache/huggingface:/root/.cache/huggingface --name gh200-test --gpus=all --entrypoint="" gh200-test bash -c '
27 - python3 examples/offline_inference/cli.py --model meta-llama/Llama-3.2-1B
27 + python3 examples/offline_inference/basic/generate.py --model meta-llama/Llama-3.2-1B
28 '
.buildkite/run-hpu-test.sh
+1 -1
@@ -20,5 +20,5 @@ trap remove_docker_container_and_exit EXIT
20 remove_docker_container
21
22 # Run the image and launch offline inference
23 -docker run --runtime=habana --name=hpu-test --network=host -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference/basic.py
23 +docker run --runtime=habana --name=hpu-test --network=host -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m
24 EXITCODE=$?
.buildkite/run-openvino-test.sh
+1 -1
@@ -13,4 +13,4 @@ trap remove_docker_container EXIT
13 remove_docker_container
14
15 # Run the image and launch offline inference
16 -docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/examples/offline_inference/basic.py
16 +docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/examples/offline_inference/basic/generate.py --model facebook/opt-125m
.buildkite/run-xpu-test.sh
+2 -2
@@ -14,6 +14,6 @@ remove_docker_container
14
15 # Run the image and test offline inference/tensor parallel
16 docker run --name xpu-test --device /dev/dri -v /dev/dri/by-path:/dev/dri/by-path --entrypoint="" xpu-test sh -c '
17 - python3 examples/offline_inference/basic.py
18 - python3 examples/offline_inference/cli.py -tp 2
17 + python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m
18 + python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m -tp 2
19 '
.buildkite/test-pipeline.yaml
+6 -6
@@ -215,18 +215,18 @@ steps:
215 - examples/
216 commands:
217 - pip install tensorizer # for tensorizer test
218 - - python3 offline_inference/basic.py
219 - - python3 offline_inference/cpu_offload.py
220 - - python3 offline_inference/chat.py
218 + - python3 offline_inference/basic/generate.py --model facebook/opt-125m
219 + - python3 offline_inference/basic/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
220 + - python3 offline_inference/basic/chat.py
221 - python3 offline_inference/prefix_caching.py
222 - python3 offline_inference/llm_engine_example.py
223 - python3 offline_inference/vision_language.py
224 - python3 offline_inference/vision_language_multi_image.py
225 - python3 other/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 other/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
226 - python3 offline_inference/encoder_decoder.py
227 - - python3 offline_inference/classification.py
228 - - python3 offline_inference/embedding.py
229 - - python3 offline_inference/scoring.py
227 + - python3 offline_inference/basic/classify.py
228 + - python3 offline_inference/basic/embed.py
229 + - python3 offline_inference/basic/score.py
230 - python3 offline_inference/profiling.py --model facebook/opt-125m run_num_steps --num-steps 2
231
232 - label: Prefix Caching Test # 9min
docs/source/generate_examples.py
+2 -2
@@ -147,7 +147,7 @@ class Example:
147 return content
148
149 content += "## Example materials\n\n"
150 - for file in self.other_files:
150 + for file in sorted(self.other_files):
151 include = "include" if file.suffix == ".md" else "literalinclude"
152 content += f":::{{admonition}} {file.relative_to(self.path)}\n"
153 content += ":class: dropdown\n\n"
@@ -194,7 +194,7 @@ def generate_examples():
194 path=EXAMPLE_DOC_DIR / "examples_offline_inference_index.md",
195 title="Offline Inference",
196 description=
197 - "Offline inference examples demonstrate how to use vLLM in an offline setting, where the model is queried for predictions in batches.", # noqa: E501
197 + "Offline inference examples demonstrate how to use vLLM in an offline setting, where the model is queried for predictions in batches. We recommend starting with <project:basic.md>.", # noqa: E501
198 caption="Examples",
199 ),
200 }
docs/source/getting_started/installation/cpu/index.md
+2 -2
@@ -170,7 +170,7 @@ vLLM CPU backend supports the following vLLM features:
170 sudo apt-get install libtcmalloc-minimal4 # install TCMalloc library
171 find / -name *libtcmalloc* # find the dynamic link library path
172 export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$LD_PRELOAD # prepend the library to LD_PRELOAD
173 -python examples/offline_inference/basic.py # run vLLM
173 +python examples/offline_inference/basic/basic.py # run vLLM
174 ```
175
176 - When using the online serving, it is recommended to reserve 1-2 CPU cores for the serving framework to avoid CPU oversubscription. For example, on a platform with 32 physical CPU cores, reserving CPU 30 and 31 for the framework and using CPU 0-29 for OpenMP:
@@ -207,7 +207,7 @@ CPU NODE SOCKET CORE L1d:L1i:L2:L3 ONLINE MAXMHZ MINMHZ MHZ
207
208 # On this platform, it is recommend to only bind openMP threads on logical CPU cores 0-7 or 8-15
209 $ export VLLM_CPU_OMP_THREADS_BIND=0-7
210 -$ python examples/offline_inference/basic.py
210 +$ python examples/offline_inference/basic/basic.py
211 ```
212
213 - If using vLLM CPU backend on a multi-socket machine with NUMA, be aware to set CPU cores using `VLLM_CPU_OMP_THREADS_BIND` to avoid cross NUMA node memory access.
docs/source/getting_started/quickstart.md
+1 -1
@@ -40,7 +40,7 @@ For non-CUDA platforms, please refer [here](#installation-index) for specific in
40
41 ## Offline Batched Inference
42
43 -With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). See the example script: <gh-file:examples/offline_inference/basic.py>
43 +With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). See the example script: <gh-file:examples/offline_inference/basic/basic.py>
44
45 The first line of this example imports the classes {class}`~vllm.LLM` and {class}`~vllm.SamplingParams`:
46
docs/source/models/generative_models.md
+2 -2
@@ -46,7 +46,7 @@ for output in outputs:
46 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
47 ```
48
49 -A code example can be found here: <gh-file:examples/offline_inference/basic.py>
49 +A code example can be found here: <gh-file:examples/offline_inference/basic/basic.py>
50
51 ### `LLM.beam_search`
52
@@ -103,7 +103,7 @@ for output in outputs:
103 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
104 ```
105
106 -A code example can be found here: <gh-file:examples/offline_inference/chat.py>
106 +A code example can be found here: <gh-file:examples/offline_inference/basic/chat.py>
107
108 If the model doesn't have a chat template or you want to specify another one,
109 you can explicitly pass a chat template:
docs/source/models/pooling_models.md
+3 -3
@@ -88,7 +88,7 @@ embeds = output.outputs.embedding
88 print(f"Embeddings: {embeds!r} (size={len(embeds)})")
89 ```
90
91 -A code example can be found here: <gh-file:examples/offline_inference/embedding.py>
91 +A code example can be found here: <gh-file:examples/offline_inference/basic/embed.py>
92
93 ### `LLM.classify`
94
@@ -103,7 +103,7 @@ probs = output.outputs.probs
103 print(f"Class Probabilities: {probs!r} (size={len(probs)})")
104 ```
105
106 -A code example can be found here: <gh-file:examples/offline_inference/classification.py>
106 +A code example can be found here: <gh-file:examples/offline_inference/basic/classify.py>
107
108 ### `LLM.score`
109
@@ -125,7 +125,7 @@ score = output.outputs.score
125 print(f"Score: {score}")
126 ```
127
128 -A code example can be found here: <gh-file:examples/offline_inference/scoring.py>
128 +A code example can be found here: <gh-file:examples/offline_inference/basic/score.py>
129
130 ## Online Serving
131
examples/offline_inference/aqlm_example.py deleted
-47
@@ -1,47 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from vllm import LLM, SamplingParams
4 -from vllm.utils import FlexibleArgumentParser
5 -
6 -
7 -def main():
8 -
9 - parser = FlexibleArgumentParser(description='AQLM examples')
10 -
11 - parser.add_argument('--model',
12 - '-m',
13 - type=str,
14 - default=None,
15 - help='model path, as for HF')
16 - parser.add_argument('--choice',
17 - '-c',
18 - type=int,
19 - default=0,
20 - help='known good models by index, [0-4]')
21 - parser.add_argument('--tensor-parallel-size',
22 - '-t',
23 - type=int,
24 - default=1,
25 - help='tensor parallel size')
26 -
27 - args = parser.parse_args()
28 -
29 - models = [
30 - "ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf",
31 - "ISTA-DASLab/Llama-2-7b-AQLM-2Bit-2x8-hf",
32 - "ISTA-DASLab/Llama-2-13b-AQLM-2Bit-1x16-hf",
33 - "ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf",
34 - "BlackSamorez/TinyLlama-1_1B-Chat-v1_0-AQLM-2Bit-1x16-hf",
35 - ]
36 -
37 - model = LLM(args.model if args.model is not None else models[args.choice],
38 - tensor_parallel_size=args.tensor_parallel_size)
39 -
40 - sampling_params = SamplingParams(max_tokens=100, temperature=0)
41 - outputs = model.generate("Hello my name is",
42 - sampling_params=sampling_params)
43 - print(outputs[0].outputs[0].text)
44 -
45 -
46 -if __name__ == '__main__':
47 - main()
examples/offline_inference/arctic.py deleted
-28
@@ -1,28 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from vllm import LLM, SamplingParams
4 -
5 -# Sample prompts.
6 -prompts = [
7 - "Hello, my name is",
8 - "The president of the United States is",
9 - "The capital of France is",
10 - "The future of AI is",
11 -]
12 -# Create a sampling params object.
13 -sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
14 -
15 -# Create an LLM.
16 -llm = LLM(model="snowflake/snowflake-arctic-instruct",
17 - quantization="deepspeedfp",
18 - tensor_parallel_size=8,
19 - trust_remote_code=True)
20 -# Generate texts from the prompts. The output is a list of RequestOutput objects
21 -# that contain the prompt, generated text, and other information.
22 -
23 -outputs = llm.generate(prompts, sampling_params)
24 -# Print the outputs.
25 -for output in outputs:
26 - prompt = output.prompt
27 - generated_text = output.outputs[0].text
28 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
examples/offline_inference/basic/README.md new
+94
@@ -0,0 +1,94 @@
1 +# Basic
2 +
3 +The `LLM` class provides the primary Python interface for doing offline inference, which is interacting with a model without using a separate model inference server.
4 +
5 +## Usage
6 +
7 +The first script in this example shows the most basic usage of vLLM. If you are new to Python and vLLM, you should start here.
8 +
9 +```bash
10 +python examples/offline_inference/basic/basic.py
11 +```
12 +
13 +The rest of the scripts include an [argument parser](https://docs.python.org/3/library/argparse.html), which you can use to pass any arguments that are compatible with [`LLM`](https://docs.vllm.ai/en/latest/api/offline_inference/llm.html). Try running the script with `--help` for a list of all available arguments.
14 +
15 +```bash
16 +python examples/offline_inference/basic/classify.py
17 +```
18 +
19 +```bash
20 +python examples/offline_inference/basic/embed.py
21 +```
22 +
23 +```bash
24 +python examples/offline_inference/basic/score.py
25 +```
26 +
27 +The chat and generate scripts also accept the [sampling parameters](https://docs.vllm.ai/en/latest/api/inference_params.html#sampling-parameters): `max_tokens`, `temperature`, `top_p` and `top_k`.
28 +
29 +```bash
30 +python examples/offline_inference/basic/chat.py
31 +```
32 +
33 +```bash
34 +python examples/offline_inference/basic/generate.py
35 +```
36 +
37 +## Features
38 +
39 +In the scripts that support passing arguments, you can experiment with the following features.
40 +
41 +### Default generation config
42 +
43 +The `--generation-config` argument specifies where the generation config will be loaded from when calling `LLM.get_default_sampling_params()`. If set to ‘auto’, the generation config will be loaded from model path. If set to a folder path, the generation config will be loaded from the specified folder path. If it is not provided, vLLM defaults will be used.
44 +
45 +> If max_new_tokens is specified in generation config, then it sets a server-wide limit on the number of output tokens for all requests.
46 +
47 +Try it yourself with the following argument:
48 +
49 +```bash
50 +--generation-config auto
51 +```
52 +
53 +### Quantization
54 +
55 +#### AQLM
56 +
57 +vLLM supports models that are quantized using AQLM.
58 +
59 +Try one yourself by passing one of the following models to the `--model` argument:
60 +
61 +- `ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf`
62 +- `ISTA-DASLab/Llama-2-7b-AQLM-2Bit-2x8-hf`
63 +- `ISTA-DASLab/Llama-2-13b-AQLM-2Bit-1x16-hf`
64 +- `ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf`
65 +- `BlackSamorez/TinyLlama-1_1B-Chat-v1_0-AQLM-2Bit-1x16-hf`
66 +
67 +> Some of these models are likely to be too large for a single GPU. You can split them across multiple GPUs by setting `--tensor-parallel-size` to the number of required GPUs.
68 +
69 +#### GGUF
70 +
71 +vLLM supports models that are quantized using GGUF.
72 +
73 +Try one yourself by downloading a GUFF quantised model and using the following arguments:
74 +
75 +```python
76 +from huggingface_hub import hf_hub_download
77 +repo_id = "bartowski/Phi-3-medium-4k-instruct-GGUF"
78 +filename = "Phi-3-medium-4k-instruct-IQ2_M.gguf"
79 +print(hf_hub_download(repo_id, filename=filename))
80 +```
81 +
82 +```bash
83 +--model {local-path-printed-above} --tokenizer microsoft/Phi-3-medium-4k-instruct
84 +```
85 +
86 +### CPU offload
87 +
88 +The `--cpu-offload-gb` argument can be seen as a virtual way to increase the GPU memory size. For example, if you have one 24 GB GPU and set this to 10, virtually you can think of it as a 34 GB GPU. Then you can load a 13B model with BF16 weight, which requires at least 26GB GPU memory. Note that this requires fast CPU-GPU interconnect, as part of the model is loaded from CPU memory to GPU memory on the fly in each model forward pass.
89 +
90 +Try it yourself with the following arguments:
91 +
92 +```bash
93 +--model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
94 +```
examples/offline_inference/basic/basic.py renamed
examples/offline_inference/basic/chat.py new
+98
@@ -0,0 +1,98 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from vllm import LLM, EngineArgs
4 +from vllm.utils import FlexibleArgumentParser
5 +
6 +
7 +def main(args: dict):
8 + # Pop arguments not used by LLM
9 + max_tokens = args.pop("max_tokens")
10 + temperature = args.pop("temperature")
11 + top_p = args.pop("top_p")
12 + top_k = args.pop("top_k")
13 + chat_template_path = args.pop("chat_template_path")
14 +
15 + # Create an LLM
16 + llm = LLM(**args)
17 +
18 + # Create sampling params object
19 + sampling_params = llm.get_default_sampling_params()
20 + if max_tokens is not None:
21 + sampling_params.max_tokens = max_tokens
22 + if temperature is not None:
23 + sampling_params.temperature = temperature
24 + if top_p is not None:
25 + sampling_params.top_p = top_p
26 + if top_k is not None:
27 + sampling_params.top_k = top_k
28 +
29 + def print_outputs(outputs):
30 + for output in outputs:
31 + prompt = output.prompt
32 + generated_text = output.outputs[0].text
33 + print(f"Prompt: {prompt!r}")
34 + print(f"Generated text: {generated_text!r}")
35 + print("-" * 80)
36 +
37 + print("=" * 80)
38 +
39 + # In this script, we demonstrate how to pass input to the chat method:
40 + conversation = [
41 + {
42 + "role": "system",
43 + "content": "You are a helpful assistant"
44 + },
45 + {
46 + "role": "user",
47 + "content": "Hello"
48 + },
49 + {
50 + "role": "assistant",
51 + "content": "Hello! How can I assist you today?"
52 + },
53 + {
54 + "role": "user",
55 + "content":
56 + "Write an essay about the importance of higher education.",
57 + },
58 + ]
59 + outputs = llm.chat(conversation, sampling_params, use_tqdm=False)
60 + print_outputs(outputs)
61 +
62 + # You can run batch inference with llm.chat API
63 + conversations = [conversation for _ in range(10)]
64 +
65 + # We turn on tqdm progress bar to verify it's indeed running batch inference
66 + outputs = llm.chat(conversations, sampling_params, use_tqdm=True)
67 + print_outputs(outputs)
68 +
69 + # A chat template can be optionally supplied.
70 + # If not, the model will use its default chat template.
71 + if chat_template_path is not None:
72 + with open(chat_template_path) as f:
73 + chat_template = f.read()
74 +
75 + outputs = llm.chat(
76 + conversations,
77 + sampling_params,
78 + use_tqdm=False,
79 + chat_template=chat_template,
80 + )
81 +
82 +
83 +if __name__ == "__main__":
84 + parser = FlexibleArgumentParser()
85 + # Add engine args
86 + engine_group = parser.add_argument_group("Engine arguments")
87 + EngineArgs.add_cli_args(engine_group)
88 + engine_group.set_defaults(model="meta-llama/Llama-3.2-1B-Instruct")
89 + # Add sampling params
90 + sampling_group = parser.add_argument_group("Sampling parameters")
91 + sampling_group.add_argument("--max-tokens", type=int)
92 + sampling_group.add_argument("--temperature", type=float)
93 + sampling_group.add_argument("--top-p", type=float)
94 + sampling_group.add_argument("--top-k", type=int)
95 + # Add example params
96 + parser.add_argument("--chat-template-path", type=str)
97 + args: dict = vars(parser.parse_args())
98 + main(args)
examples/offline_inference/basic/classify.py new
+42
@@ -0,0 +1,42 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from argparse import Namespace
4 +
5 +from vllm import LLM, EngineArgs
6 +from vllm.utils import FlexibleArgumentParser
7 +
8 +
9 +def main(args: Namespace):
10 + # Sample prompts.
11 + prompts = [
12 + "Hello, my name is",
13 + "The president of the United States is",
14 + "The capital of France is",
15 + "The future of AI is",
16 + ]
17 +
18 + # Create an LLM.
19 + # You should pass task="classify" for classification models
20 + model = LLM(**vars(args))
21 +
22 + # Generate logits. The output is a list of ClassificationRequestOutputs.
23 + outputs = model.classify(prompts)
24 +
25 + # Print the outputs.
26 + for prompt, output in zip(prompts, outputs):
27 + probs = output.outputs.probs
28 + probs_trimmed = ((str(probs[:16])[:-1] +
29 + ", ...]") if len(probs) > 16 else probs)
30 + print(f"Prompt: {prompt!r} | "
31 + f"Class Probabilities: {probs_trimmed} (size={len(probs)})")
32 +
33 +
34 +if __name__ == "__main__":
35 + parser = FlexibleArgumentParser()
36 + parser = EngineArgs.add_cli_args(parser)
37 + # Set example specific arguments
38 + parser.set_defaults(model="jason9693/Qwen2.5-1.5B-apeach",
39 + task="classify",
40 + enforce_eager=True)
41 + args = parser.parse_args()
42 + main(args)
examples/offline_inference/basic/embed.py new
+42
@@ -0,0 +1,42 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from argparse import Namespace
4 +
5 +from vllm import LLM, EngineArgs
6 +from vllm.utils import FlexibleArgumentParser
7 +
8 +
9 +def main(args: Namespace):
10 + # Sample prompts.
11 + prompts = [
12 + "Hello, my name is",
13 + "The president of the United States is",
14 + "The capital of France is",
15 + "The future of AI is",
16 + ]
17 +
18 + # Create an LLM.
19 + # You should pass task="embed" for embedding models
20 + model = LLM(**vars(args))
21 +
22 + # Generate embedding. The output is a list of EmbeddingRequestOutputs.
23 + outputs = model.embed(prompts)
24 +
25 + # Print the outputs.
26 + for prompt, output in zip(prompts, outputs):
27 + embeds = output.outputs.embedding
28 + embeds_trimmed = ((str(embeds[:16])[:-1] +
29 + ", ...]") if len(embeds) > 16 else embeds)
30 + print(f"Prompt: {prompt!r} | "
31 + f"Embeddings: {embeds_trimmed} (size={len(embeds)})")
32 +
33 +
34 +if __name__ == "__main__":
35 + parser = FlexibleArgumentParser()
36 + parser = EngineArgs.add_cli_args(parser)
37 + # Set example specific arguments
38 + parser.set_defaults(model="intfloat/e5-mistral-7b-instruct",
39 + task="embed",
40 + enforce_eager=True)
41 + args = parser.parse_args()
42 + main(args)
examples/offline_inference/basic/generate.py new
+57
@@ -0,0 +1,57 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from vllm import LLM, EngineArgs
4 +from vllm.utils import FlexibleArgumentParser
5 +
6 +
7 +def main(args: dict):
8 + # Pop arguments not used by LLM
9 + max_tokens = args.pop("max_tokens")
10 + temperature = args.pop("temperature")
11 + top_p = args.pop("top_p")
12 + top_k = args.pop("top_k")
13 +
14 + # Create an LLM
15 + llm = LLM(**args)
16 +
17 + # Create a sampling params object
18 + sampling_params = llm.get_default_sampling_params()
19 + if max_tokens is not None:
20 + sampling_params.max_tokens = max_tokens
21 + if temperature is not None:
22 + sampling_params.temperature = temperature
23 + if top_p is not None:
24 + sampling_params.top_p = top_p
25 + if top_k is not None:
26 + sampling_params.top_k = top_k
27 +
28 + # Generate texts from the prompts. The output is a list of RequestOutput
29 + # objects that contain the prompt, generated text, and other information.
30 + prompts = [
31 + "Hello, my name is",
32 + "The president of the United States is",
33 + "The capital of France is",
34 + "The future of AI is",
35 + ]
36 + outputs = llm.generate(prompts, sampling_params)
37 + # Print the outputs.
38 + for output in outputs:
39 + prompt = output.prompt
40 + generated_text = output.outputs[0].text
41 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
42 +
43 +
44 +if __name__ == "__main__":
45 + parser = FlexibleArgumentParser()
46 + # Add engine args
47 + engine_group = parser.add_argument_group("Engine arguments")
48 + EngineArgs.add_cli_args(engine_group)
49 + engine_group.set_defaults(model="meta-llama/Llama-3.2-1B-Instruct")
50 + # Add sampling params
51 + sampling_group = parser.add_argument_group("Sampling parameters")
52 + sampling_group.add_argument("--max-tokens", type=int)
53 + sampling_group.add_argument("--temperature", type=float)
54 + sampling_group.add_argument("--top-p", type=float)
55 + sampling_group.add_argument("--top-k", type=int)
56 + args: dict = vars(parser.parse_args())
57 + main(args)
examples/offline_inference/basic/score.py new
+38
@@ -0,0 +1,38 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from argparse import Namespace
4 +
5 +from vllm import LLM, EngineArgs
6 +from vllm.utils import FlexibleArgumentParser
7 +
8 +
9 +def main(args: Namespace):
10 + # Sample prompts.
11 + text_1 = "What is the capital of France?"
12 + texts_2 = [
13 + "The capital of Brazil is Brasilia.",
14 + "The capital of France is Paris.",
15 + ]
16 +
17 + # Create an LLM.
18 + # You should pass task="score" for cross-encoder models
19 + model = LLM(**vars(args))
20 +
21 + # Generate scores. The output is a list of ScoringRequestOutputs.
22 + outputs = model.score(text_1, texts_2)
23 +
24 + # Print the outputs.
25 + for text_2, output in zip(texts_2, outputs):
26 + score = output.outputs.score
27 + print(f"Pair: {[text_1, text_2]!r} | Score: {score}")
28 +
29 +
30 +if __name__ == "__main__":
31 + parser = FlexibleArgumentParser()
32 + parser = EngineArgs.add_cli_args(parser)
33 + # Set example specific arguments
34 + parser.set_defaults(model="BAAI/bge-reranker-v2-m3",
35 + task="score",
36 + enforce_eager=True)
37 + args = parser.parse_args()
38 + main(args)
examples/offline_inference/basic_with_model_default_sampling.py deleted
-32
@@ -1,32 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from vllm import LLM
4 -
5 -# Sample prompts.
6 -prompts = [
7 - "Hello, my name is",
8 - "The president of the United States is",
9 - "The capital of France is",
10 - "The future of AI is",
11 -]
12 -
13 -# Create an LLM with built-in default generation config.
14 -# The generation config is set to None by default to keep
15 -# the behavior consistent with the previous version.
16 -# If you want to use the default generation config from the model,
17 -# you should set the generation_config to "auto".
18 -llm = LLM(model="Qwen/Qwen2.5-0.5B-Instruct", generation_config="auto")
19 -
20 -# Load the default sampling parameters from the model.
21 -sampling_params = llm.get_default_sampling_params()
22 -# Modify the sampling parameters if needed.
23 -sampling_params.temperature = 0.5
24 -
25 -# Generate texts from the prompts. The output is a list of RequestOutput objects
26 -# that contain the prompt, generated text, and other information.
27 -outputs = llm.generate(prompts, sampling_params)
28 -# Print the outputs.
29 -for output in outputs:
30 - prompt = output.prompt
31 - generated_text = output.outputs[0].text
32 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
examples/offline_inference/chat.py deleted
-82
@@ -1,82 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from vllm import LLM, SamplingParams
4 -
5 -llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
6 -sampling_params = SamplingParams(temperature=0.5)
7 -
8 -
9 -def print_outputs(outputs):
10 - for output in outputs:
11 - prompt = output.prompt
12 - generated_text = output.outputs[0].text
13 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
14 - print("-" * 80)
15 -
16 -
17 -print("=" * 80)
18 -
19 -# In this script, we demonstrate how to pass input to the chat method:
20 -
21 -conversation = [
22 - {
23 - "role": "system",
24 - "content": "You are a helpful assistant"
25 - },
26 - {
27 - "role": "user",
28 - "content": "Hello"
29 - },
30 - {
31 - "role": "assistant",
32 - "content": "Hello! How can I assist you today?"
33 - },
34 - {
35 - "role": "user",
36 - "content": "Write an essay about the importance of higher education.",
37 - },
38 -]
39 -outputs = llm.chat(conversation,
40 - sampling_params=sampling_params,
41 - use_tqdm=False)
42 -print_outputs(outputs)
43 -
44 -# You can run batch inference with llm.chat API
45 -conversation = [
46 - {
47 - "role": "system",
48 - "content": "You are a helpful assistant"
49 - },
50 - {
51 - "role": "user",
52 - "content": "Hello"
53 - },
54 - {
55 - "role": "assistant",
56 - "content": "Hello! How can I assist you today?"
57 - },
58 - {
59 - "role": "user",
60 - "content": "Write an essay about the importance of higher education.",
61 - },
62 -]
63 -conversations = [conversation for _ in range(10)]
64 -
65 -# We turn on tqdm progress bar to verify it's indeed running batch inference
66 -outputs = llm.chat(messages=conversations,
67 - sampling_params=sampling_params,
68 - use_tqdm=True)
69 -print_outputs(outputs)
70 -
71 -# A chat template can be optionally supplied.
72 -# If not, the model will use its default chat template.
73 -
74 -# with open('template_falcon_180b.jinja', "r") as f:
75 -# chat_template = f.read()
76 -
77 -# outputs = llm.chat(
78 -# conversations,
79 -# sampling_params=sampling_params,
80 -# use_tqdm=False,
81 -# chat_template=chat_template,
82 -# )
examples/offline_inference/classification.py deleted
-30
@@ -1,30 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from vllm import LLM
4 -
5 -# Sample prompts.
6 -prompts = [
7 - "Hello, my name is",
8 - "The president of the United States is",
9 - "The capital of France is",
10 - "The future of AI is",
11 -]
12 -
13 -# Create an LLM.
14 -# You should pass task="classify" for classification models
15 -model = LLM(
16 - model="jason9693/Qwen2.5-1.5B-apeach",
17 - task="classify",
18 - enforce_eager=True,
19 -)
20 -
21 -# Generate logits. The output is a list of ClassificationRequestOutputs.
22 -outputs = model.classify(prompts)
23 -
24 -# Print the outputs.
25 -for prompt, output in zip(prompts, outputs):
26 - probs = output.outputs.probs
27 - probs_trimmed = ((str(probs[:16])[:-1] +
28 - ", ...]") if len(probs) > 16 else probs)
29 - print(f"Prompt: {prompt!r} | "
30 - f"Class Probabilities: {probs_trimmed} (size={len(probs)})")
examples/offline_inference/cli.py deleted
-82
@@ -1,82 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from dataclasses import asdict
4 -
5 -from vllm import LLM, SamplingParams
6 -from vllm.engine.arg_utils import EngineArgs
7 -from vllm.utils import FlexibleArgumentParser
8 -
9 -
10 -def get_prompts(num_prompts: int):
11 - # The default sample prompts.
12 - prompts = [
13 - "Hello, my name is",
14 - "The president of the United States is",
15 - "The capital of France is",
16 - "The future of AI is",
17 - ]
18 -
19 - if num_prompts != len(prompts):
20 - prompts = (prompts * ((num_prompts // len(prompts)) + 1))[:num_prompts]
21 -
22 - return prompts
23 -
24 -
25 -def main(args):
26 - # Create prompts
27 - prompts = get_prompts(args.num_prompts)
28 -
29 - # Create a sampling params object.
30 - sampling_params = SamplingParams(n=args.n,
31 - temperature=args.temperature,
32 - top_p=args.top_p,
33 - top_k=args.top_k,
34 - max_tokens=args.max_tokens)
35 -
36 - # Create an LLM.
37 - # The default model is 'facebook/opt-125m'
38 - engine_args = EngineArgs.from_cli_args(args)
39 - llm = LLM(**asdict(engine_args))
40 -
41 - # Generate texts from the prompts.
42 - # The output is a list of RequestOutput objects
43 - # that contain the prompt, generated text, and other information.
44 - outputs = llm.generate(prompts, sampling_params)
45 - # Print the outputs.
46 - for output in outputs:
47 - prompt = output.prompt
48 - generated_text = output.outputs[0].text
49 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
50 -
51 -
52 -if __name__ == '__main__':
53 - parser = FlexibleArgumentParser()
54 - parser = EngineArgs.add_cli_args(parser)
55 - group = parser.add_argument_group("SamplingParams options")
56 - group.add_argument("--num-prompts",
57 - type=int,
58 - default=4,
59 - help="Number of prompts used for inference")
60 - group.add_argument("--max-tokens",
61 - type=int,
62 - default=16,
63 - help="Generated output length for sampling")
64 - group.add_argument('--n',
65 - type=int,
66 - default=1,
67 - help='Number of generated sequences per prompt')
68 - group.add_argument('--temperature',
69 - type=float,
70 - default=0.8,
71 - help='Temperature for text generation')
72 - group.add_argument('--top-p',
73 - type=float,
74 - default=0.95,
75 - help='top_p for text generation')
76 - group.add_argument('--top-k',
77 - type=int,
78 - default=-1,
79 - help='top_k for text generation')
80 -
81 - args = parser.parse_args()
82 - main(args)
examples/offline_inference/cpu_offload.py deleted
-24
@@ -1,24 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from vllm import LLM, SamplingParams
4 -
5 -# Sample prompts.
6 -prompts = [
7 - "Hello, my name is",
8 - "The president of the United States is",
9 - "The capital of France is",
10 - "The future of AI is",
11 -]
12 -# Create a sampling params object.
13 -sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
14 -
15 -# Create an LLM.
16 -llm = LLM(model="meta-llama/Llama-2-13b-chat-hf", cpu_offload_gb=10)
17 -# Generate texts from the prompts. The output is a list of RequestOutput objects
18 -# that contain the prompt, generated text, and other information.
19 -outputs = llm.generate(prompts, sampling_params)
20 -# Print the outputs.
21 -for output in outputs:
22 - prompt = output.prompt
23 - generated_text = output.outputs[0].text
24 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
examples/offline_inference/embedding.py deleted
-30
@@ -1,30 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from vllm import LLM
4 -
5 -# Sample prompts.
6 -prompts = [
7 - "Hello, my name is",
8 - "The president of the United States is",
9 - "The capital of France is",
10 - "The future of AI is",
11 -]
12 -
13 -# Create an LLM.
14 -# You should pass task="embed" for embedding models
15 -model = LLM(
16 - model="intfloat/e5-mistral-7b-instruct",
17 - task="embed",
18 - enforce_eager=True,
19 -)
20 -
21 -# Generate embedding. The output is a list of EmbeddingRequestOutputs.
22 -outputs = model.embed(prompts)
23 -
24 -# Print the outputs.
25 -for prompt, output in zip(prompts, outputs):
26 - embeds = output.outputs.embedding
27 - embeds_trimmed = ((str(embeds[:16])[:-1] +
28 - ", ...]") if len(embeds) > 16 else embeds)
29 - print(f"Prompt: {prompt!r} | "
30 - f"Embeddings: {embeds_trimmed} (size={len(embeds)})")
examples/offline_inference/gguf_inference.py deleted
-34
@@ -1,34 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from huggingface_hub import hf_hub_download
4 -
5 -from vllm import LLM, SamplingParams
6 -
7 -
8 -def run_gguf_inference(model_path, tokenizer):
9 - # Sample prompts.
10 - prompts = [
11 - "How many helicopters can a human eat in one sitting?",
12 - "What's the future of AI?",
13 - ]
14 - prompts = [[{"role": "user", "content": prompt}] for prompt in prompts]
15 - # Create a sampling params object.
16 - sampling_params = SamplingParams(temperature=0, max_tokens=128)
17 -
18 - # Create an LLM.
19 - llm = LLM(model=model_path, tokenizer=tokenizer)
20 -
21 - outputs = llm.chat(prompts, sampling_params)
22 - # Print the outputs.
23 - for output in outputs:
24 - prompt = output.prompt
25 - generated_text = output.outputs[0].text
26 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
27 -
28 -
29 -if __name__ == "__main__":
30 - repo_id = "bartowski/Phi-3-medium-4k-instruct-GGUF"
31 - filename = "Phi-3-medium-4k-instruct-IQ2_M.gguf"
32 - tokenizer = "microsoft/Phi-3-medium-4k-instruct"
33 - model = hf_hub_download(repo_id, filename=filename)
34 - run_gguf_inference(model, tokenizer)
examples/offline_inference/scoring.py deleted
-25
@@ -1,25 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from vllm import LLM
4 -
5 -# Sample prompts.
6 -text_1 = "What is the capital of France?"
7 -texts_2 = [
8 - "The capital of Brazil is Brasilia.", "The capital of France is Paris."
9 -]
10 -
11 -# Create an LLM.
12 -# You should pass task="score" for cross-encoder models
13 -model = LLM(
14 - model="BAAI/bge-reranker-v2-m3",
15 - task="score",
16 - enforce_eager=True,
17 -)
18 -
19 -# Generate scores. The output is a list of ScoringRequestOutputs.
20 -outputs = model.score(text_1, texts_2)
21 -
22 -# Print the outputs.
23 -for text_2, output in zip(texts_2, outputs):
24 - score = output.outputs.score
25 - print(f"Pair: {[text_1, text_2]!r} | Score: {score}")
tests/plugins_tests/test_platform_plugins.py
+1 -1
@@ -14,7 +14,7 @@ def test_platform_plugins():
14 import os
15 example_file = os.path.join(
16 os.path.dirname(os.path.dirname(os.path.dirname(current_file))),
17 - "examples", "offline_inference/basic.py")
17 + "examples", "offline_inference/basic/basic.py")
18 runpy.run_path(example_file)
19
20 # check if the plugin is loaded correctly