A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Misc] refactor examples series (#16708)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
Apr 16, 2025 at 18:16 UTC
716892049190e55b432a0cb84d0250f0e68c310a
11 files changed
+228
-161
examples/offline_inference/llm_engine_example.py
+8
-4
@@ -50,6 +50,13 @@ def initialize_engine(args: argparse.Namespace) -> LLMEngine:
50
return LLMEngine.from_engine_args(engine_args)
51
52
53
+def parse_args():
54
+ parser = FlexibleArgumentParser(
55
+ description='Demo on using the LLMEngine class directly')
56
+ parser = EngineArgs.add_cli_args(parser)
57
+ return parser.parse_args()
58
+
59
+
60
def main(args: argparse.Namespace):
61
"""Main function that sets up and runs the prompt processing."""
62
engine = initialize_engine(args)
@@ -58,8 +65,5 @@ def main(args: argparse.Namespace):
65
66
67
if __name__ == '__main__':
61
- parser = FlexibleArgumentParser(
62
- description='Demo on using the LLMEngine class directly')
63
- parser = EngineArgs.add_cli_args(parser)
64
- args = parser.parse_args()
68
+ args = parse_args()
69
main(args)
examples/online_serving/gradio_openai_chatbot_webserver.py
-4
@@ -23,10 +23,6 @@ import gradio as gr
23
from openai import OpenAI
24
25
26
-def create_openai_client(api_key, base_url):
27
- return OpenAI(api_key=api_key, base_url=base_url)
28
-
29
-
26
def format_history_to_openai(history):
27
history_openai_format = [{
28
"role": "system",
examples/online_serving/openai_chat_completion_client_for_multimodal.py
+11
-7
@@ -303,12 +303,7 @@ example_function_map = {
303
}
304
305
306
-def main(args) -> None:
307
- chat_type = args.chat_type
308
- example_function_map[chat_type]()
309
-
310
-
311
-if __name__ == "__main__":
306
+def parse_args():
307
parser = FlexibleArgumentParser(
308
description='Demo on using OpenAI client for online serving with '
309
'multimodal language models served with vLLM.')
@@ -318,5 +313,14 @@ if __name__ == "__main__":
313
default="single-image",
314
choices=list(example_function_map.keys()),
315
help='Conversation type with multimodal data.')
321
- args = parser.parse_args()
316
+ return parser.parse_args()
317
+
318
+
319
+def main(args) -> None:
320
+ chat_type = args.chat_type
321
+ example_function_map[chat_type]()
322
+
323
+
324
+if __name__ == "__main__":
325
+ args = parse_args()
326
main(args)
examples/online_serving/openai_chat_completion_client_with_tools_required.py
+32
-26
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""
3
-To run this example, you can start the vLLM server
3
+To run this example, you can start the vLLM server
4
without any specific flags:
5
6
```bash
@@ -8,7 +8,7 @@ VLLM_USE_V1=0 vllm serve unsloth/Llama-3.2-1B-Instruct \
8
--guided-decoding-backend outlines
9
```
10
11
-This example demonstrates how to generate chat completions
11
+This example demonstrates how to generate chat completions
12
using the OpenAI Python client library.
13
"""
14
@@ -18,15 +18,6 @@ from openai import OpenAI
18
openai_api_key = "EMPTY"
19
openai_api_base = "http://localhost:8000/v1"
20
21
-client = OpenAI(
22
- # defaults to os.environ.get("OPENAI_API_KEY")
23
- api_key=openai_api_key,
24
- base_url=openai_api_base,
25
-)
26
-
27
-models = client.models.list()
28
-model = models.data[0].id
29
-
21
tools = [
22
{
23
"type": "function",
@@ -116,21 +107,36 @@ messages = [
107
},
108
]
109
119
-chat_completion = client.chat.completions.create(
120
- messages=messages,
121
- model=model,
122
- tools=tools,
123
- tool_choice="required",
124
- stream=True # Enable streaming response
125
-)
110
127
-for chunk in chat_completion:
128
- if chunk.choices and chunk.choices[0].delta.tool_calls:
129
- print(chunk.choices[0].delta.tool_calls)
111
+def main():
112
+ client = OpenAI(
113
+ # defaults to os.environ.get("OPENAI_API_KEY")
114
+ api_key=openai_api_key,
115
+ base_url=openai_api_base,
116
+ )
117
+
118
+ models = client.models.list()
119
+ model = models.data[0].id
120
+
121
+ chat_completion = client.chat.completions.create(
122
+ messages=messages,
123
+ model=model,
124
+ tools=tools,
125
+ tool_choice="required",
126
+ stream=True # Enable streaming response
127
+ )
128
+
129
+ for chunk in chat_completion:
130
+ if chunk.choices and chunk.choices[0].delta.tool_calls:
131
+ print(chunk.choices[0].delta.tool_calls)
132
+
133
+ chat_completion = client.chat.completions.create(messages=messages,
134
+ model=model,
135
+ tools=tools,
136
+ tool_choice="required")
137
+
138
+ print(chat_completion.choices[0].message.tool_calls)
139
131
-chat_completion = client.chat.completions.create(messages=messages,
132
- model=model,
133
- tools=tools,
134
- tool_choice="required")
140
136
-print(chat_completion.choices[0].message.tool_calls)
141
+if __name__ == "__main__":
142
+ main()
examples/online_serving/openai_chat_completion_with_reasoning.py
+36
-27
@@ -3,8 +3,8 @@
3
An example shows how to generate chat completions from reasoning models
4
like DeepSeekR1.
5
6
-To run this example, you need to start the vLLM server with the reasoning
7
-parser:
6
+To run this example, you need to start the vLLM server
7
+with the reasoning parser:
8
9
```bash
10
vllm serve deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B \
@@ -21,35 +21,44 @@ from openai import OpenAI
21
openai_api_key = "EMPTY"
22
openai_api_base = "http://localhost:8000/v1"
23
24
-client = OpenAI(
25
- api_key=openai_api_key,
26
- base_url=openai_api_base,
27
-)
24
29
-models = client.models.list()
30
-model = models.data[0].id
25
+def main():
26
+ client = OpenAI(
27
+ api_key=openai_api_key,
28
+ base_url=openai_api_base,
29
+ )
30
32
-# Round 1
33
-messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
34
-# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
35
-response = client.chat.completions.create(model=model, messages=messages)
31
+ models = client.models.list()
32
+ model = models.data[0].id
33
37
-reasoning_content = response.choices[0].message.reasoning_content
38
-content = response.choices[0].message.content
34
+ # Round 1
35
+ messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
36
+ # ruff: noqa: E501
37
+ # For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
38
+ response = client.chat.completions.create(model=model, messages=messages)
39
40
-print("reasoning_content for Round 1:", reasoning_content)
41
-print("content for Round 1:", content)
40
+ reasoning_content = response.choices[0].message.reasoning_content
41
+ content = response.choices[0].message.content
42
43
-# Round 2
44
-messages.append({"role": "assistant", "content": content})
45
-messages.append({
46
- "role": "user",
47
- "content": "How many Rs are there in the word 'strawberry'?",
48
-})
49
-response = client.chat.completions.create(model=model, messages=messages)
43
+ print("reasoning_content for Round 1:", reasoning_content)
44
+ print("content for Round 1:", content)
45
51
-reasoning_content = response.choices[0].message.reasoning_content
52
-content = response.choices[0].message.content
46
+ # Round 2
47
+ messages.append({"role": "assistant", "content": content})
48
+ messages.append({
49
+ "role":
50
+ "user",
51
+ "content":
52
+ "How many Rs are there in the word 'strawberry'?",
53
+ })
54
+ response = client.chat.completions.create(model=model, messages=messages)
55
54
-print("reasoning_content for Round 2:", reasoning_content)
55
-print("content for Round 2:", content)
56
+ reasoning_content = response.choices[0].message.reasoning_content
57
+ content = response.choices[0].message.content
58
+
59
+ print("reasoning_content for Round 2:", reasoning_content)
60
+ print("content for Round 2:", content)
61
+
62
+
63
+if __name__ == "__main__":
64
+ main()
examples/online_serving/openai_chat_completion_with_reasoning_streaming.py
+45
-37
@@ -3,7 +3,7 @@
3
An example shows how to generate chat completions from reasoning models
4
like DeepSeekR1.
5
6
-To run this example, you need to start the vLLM server with the reasoning
6
+To run this example, you need to start the vLLM server with the reasoning
7
parser:
8
9
```bash
@@ -29,41 +29,49 @@ from openai import OpenAI
29
openai_api_key = "EMPTY"
30
openai_api_base = "http://localhost:8000/v1"
31
32
-client = OpenAI(
33
- api_key=openai_api_key,
34
- base_url=openai_api_base,
35
-)
32
+messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
33
37
-models = client.models.list()
38
-model = models.data[0].id
34
40
-messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
41
-# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
42
-stream = client.chat.completions.create(model=model,
43
- messages=messages,
44
- stream=True)
45
-
46
-print("client: Start streaming chat completions...")
47
-printed_reasoning_content = False
48
-printed_content = False
49
-
50
-for chunk in stream:
51
- reasoning_content = None
52
- content = None
53
- # Check the content is reasoning_content or content
54
- if hasattr(chunk.choices[0].delta, "reasoning_content"):
55
- reasoning_content = chunk.choices[0].delta.reasoning_content
56
- elif hasattr(chunk.choices[0].delta, "content"):
57
- content = chunk.choices[0].delta.content
58
-
59
- if reasoning_content is not None:
60
- if not printed_reasoning_content:
61
- printed_reasoning_content = True
62
- print("reasoning_content:", end="", flush=True)
63
- print(reasoning_content, end="", flush=True)
64
- elif content is not None:
65
- if not printed_content:
66
- printed_content = True
67
- print("\ncontent:", end="", flush=True)
68
- # Extract and print the content
69
- print(content, end="", flush=True)
35
+def main():
36
+ client = OpenAI(
37
+ api_key=openai_api_key,
38
+ base_url=openai_api_base,
39
+ )
40
+
41
+ models = client.models.list()
42
+ model = models.data[0].id
43
+
44
+ # ruff: noqa: E501
45
+ # For granite: add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
46
+ stream = client.chat.completions.create(model=model,
47
+ messages=messages,
48
+ stream=True)
49
+
50
+ print("client: Start streaming chat completions...")
51
+ printed_reasoning_content = False
52
+ printed_content = False
53
+
54
+ for chunk in stream:
55
+ reasoning_content = None
56
+ content = None
57
+ # Check the content is reasoning_content or content
58
+ if hasattr(chunk.choices[0].delta, "reasoning_content"):
59
+ reasoning_content = chunk.choices[0].delta.reasoning_content
60
+ elif hasattr(chunk.choices[0].delta, "content"):
61
+ content = chunk.choices[0].delta.content
62
+
63
+ if reasoning_content is not None:
64
+ if not printed_reasoning_content:
65
+ printed_reasoning_content = True
66
+ print("reasoning_content:", end="", flush=True)
67
+ print(reasoning_content, end="", flush=True)
68
+ elif content is not None:
69
+ if not printed_content:
70
+ printed_content = True
71
+ print("\ncontent:", end="", flush=True)
72
+ # Extract and print the content
73
+ print(content, end="", flush=True)
74
+
75
+
76
+if __name__ == "__main__":
77
+ main()
examples/online_serving/openai_chat_embedding_client_for_multimodal.py
+9
-2
@@ -98,7 +98,7 @@ def dse_qwen2_vl(inp: dict):
98
print("Embedding output:", response_json["data"][0]["embedding"])
99
100
101
-if __name__ == '__main__':
101
+def parse_args():
102
parser = argparse.ArgumentParser(
103
"Script to call a specified VLM through the API. Make sure to serve "
104
"the model with --task embed before running this.")
@@ -107,8 +107,10 @@ if __name__ == '__main__':
107
choices=["vlm2vec", "dse_qwen2_vl"],
108
required=True,
109
help="Which model to call.")
110
- args = parser.parse_args()
110
+ return parser.parse_args()
111
+
112
113
+def main(args):
114
if args.model == "vlm2vec":
115
vlm2vec()
116
elif args.model == "dse_qwen2_vl":
@@ -120,3 +122,8 @@ if __name__ == '__main__':
122
"type": "text",
123
"content": "What is the weather like today?",
124
})
125
+
126
+
127
+if __name__ == '__main__':
128
+ args = parse_args()
129
+ main(args)
examples/online_serving/openai_completion_client.py
+33
-25
@@ -6,28 +6,36 @@ from openai import OpenAI
6
openai_api_key = "EMPTY"
7
openai_api_base = "http://localhost:8000/v1"
8
9
-client = OpenAI(
10
- # defaults to os.environ.get("OPENAI_API_KEY")
11
- api_key=openai_api_key,
12
- base_url=openai_api_base,
13
-)
14
-
15
-models = client.models.list()
16
-model = models.data[0].id
17
-
18
-# Completion API
19
-stream = False
20
-completion = client.completions.create(
21
- model=model,
22
- prompt="A robot may not injure a human being",
23
- echo=False,
24
- n=2,
25
- stream=stream,
26
- logprobs=3)
27
-
28
-print("Completion results:")
29
-if stream:
30
- for c in completion:
31
- print(c)
32
-else:
33
- print(completion)
9
+
10
+def main():
11
+ client = OpenAI(
12
+ # defaults to os.environ.get("OPENAI_API_KEY")
13
+ api_key=openai_api_key,
14
+ base_url=openai_api_base,
15
+ )
16
+
17
+ models = client.models.list()
18
+ model = models.data[0].id
19
+
20
+ # Completion API
21
+ stream = False
22
+ completion = client.completions.create(
23
+ model=model,
24
+ prompt="A robot may not injure a human being",
25
+ echo=False,
26
+ n=2,
27
+ stream=stream,
28
+ logprobs=3)
29
+
30
+ print("-" * 50)
31
+ print("Completion results:")
32
+ if stream:
33
+ for c in completion:
34
+ print(c)
35
+ else:
36
+ print(completion)
37
+ print("-" * 50)
38
+
39
+
40
+if __name__ == "__main__":
41
+ main()
examples/online_serving/openai_cross_encoder_score.py
+15
-8
@@ -16,13 +16,15 @@ def post_http_request(prompt: dict, api_url: str) -> requests.Response:
16
return response
17
18
19
-if __name__ == "__main__":
19
+def parse_args():
20
parser = argparse.ArgumentParser()
21
parser.add_argument("--host", type=str, default="localhost")
22
parser.add_argument("--port", type=int, default=8000)
23
parser.add_argument("--model", type=str, default="BAAI/bge-reranker-v2-m3")
24
+ return parser.parse_args()
25
+
26
25
- args = parser.parse_args()
27
+def main(args):
28
api_url = f"http://{args.host}:{args.port}/score"
29
model_name = args.model
30
@@ -30,9 +32,9 @@ if __name__ == "__main__":
32
text_2 = "The capital of Brazil is Brasilia."
33
prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
34
score_response = post_http_request(prompt=prompt, api_url=api_url)
33
- print("Prompt when text_1 and text_2 are both strings:")
35
+ print("\nPrompt when text_1 and text_2 are both strings:")
36
pprint.pprint(prompt)
35
- print("Score Response:")
37
+ print("\nScore Response:")
38
pprint.pprint(score_response.json())
39
40
text_1 = "What is the capital of France?"
@@ -41,9 +43,9 @@ if __name__ == "__main__":
43
]
44
prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
45
score_response = post_http_request(prompt=prompt, api_url=api_url)
44
- print("Prompt when text_1 is string and text_2 is a list:")
46
+ print("\nPrompt when text_1 is string and text_2 is a list:")
47
pprint.pprint(prompt)
46
- print("Score Response:")
48
+ print("\nScore Response:")
49
pprint.pprint(score_response.json())
50
51
text_1 = [
@@ -54,7 +56,12 @@ if __name__ == "__main__":
56
]
57
prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
58
score_response = post_http_request(prompt=prompt, api_url=api_url)
57
- print("Prompt when text_1 and text_2 are both lists:")
59
+ print("\nPrompt when text_1 and text_2 are both lists:")
60
pprint.pprint(prompt)
59
- print("Score Response:")
61
+ print("\nScore Response:")
62
pprint.pprint(score_response.json())
63
+
64
+
65
+if __name__ == "__main__":
66
+ args = parse_args()
67
+ main(args)
examples/online_serving/openai_embedding_client.py
+26
-19
@@ -6,22 +6,29 @@ from openai import OpenAI
6
openai_api_key = "EMPTY"
7
openai_api_base = "http://localhost:8000/v1"
8
9
-client = OpenAI(
10
- # defaults to os.environ.get("OPENAI_API_KEY")
11
- api_key=openai_api_key,
12
- base_url=openai_api_base,
13
-)
14
-
15
-models = client.models.list()
16
-model = models.data[0].id
17
-
18
-responses = client.embeddings.create(
19
- input=[
20
- "Hello my name is",
21
- "The best thing about vLLM is that it supports many different models"
22
- ],
23
- model=model,
24
-)
25
-
26
-for data in responses.data:
27
- print(data.embedding) # List of float of len 4096
9
+
10
+def main():
11
+ client = OpenAI(
12
+ # defaults to os.environ.get("OPENAI_API_KEY")
13
+ api_key=openai_api_key,
14
+ base_url=openai_api_base,
15
+ )
16
+
17
+ models = client.models.list()
18
+ model = models.data[0].id
19
+
20
+ responses = client.embeddings.create(
21
+ # ruff: noqa: E501
22
+ input=[
23
+ "Hello my name is",
24
+ "The best thing about vLLM is that it supports many different models"
25
+ ],
26
+ model=model,
27
+ )
28
+
29
+ for data in responses.data:
30
+ print(data.embedding) # List of float of len 4096
31
+
32
+
33
+if __name__ == "__main__":
34
+ main()
examples/online_serving/openai_pooling_client.py
+13
-2
@@ -17,7 +17,7 @@ def post_http_request(prompt: dict, api_url: str) -> requests.Response:
17
return response
18
19
20
-if __name__ == "__main__":
20
+def parse_args():
21
parser = argparse.ArgumentParser()
22
parser.add_argument("--host", type=str, default="localhost")
23
parser.add_argument("--port", type=int, default=8000)
@@ -25,15 +25,20 @@ if __name__ == "__main__":
25
type=str,
26
default="jason9693/Qwen2.5-1.5B-apeach")
27
28
- args = parser.parse_args()
28
+ return parser.parse_args()
29
+
30
+
31
+def main(args):
32
api_url = f"http://{args.host}:{args.port}/pooling"
33
model_name = args.model
34
35
# Input like Completions API
36
prompt = {"model": model_name, "input": "vLLM is great!"}
37
pooling_response = post_http_request(prompt=prompt, api_url=api_url)
38
+ print("-" * 50)
39
print("Pooling Response:")
40
pprint.pprint(pooling_response.json())
41
+ print("-" * 50)
42
43
# Input like Chat API
44
prompt = {
@@ -50,3 +55,9 @@ if __name__ == "__main__":
55
pooling_response = post_http_request(prompt=prompt, api_url=api_url)
56
print("Pooling Response:")
57
pprint.pprint(pooling_response.json())
58
+ print("-" * 50)
59
+
60
+
61
+if __name__ == "__main__":
62
+ args = parse_args()
63
+ main(args)