A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Misc] refactor example series (#16972)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
Apr 22, 2025 at 19:44 UTC
4b91c927f69888a4c3fab0d23e4c7cce2868e49b
1 file changed
+84
-74
examples/online_serving/openai_chat_completion_tool_calls_with_reasoning.py
+84
-74
@@ -31,14 +31,6 @@ available_tools = {"get_current_weather": get_current_weather}
31
openai_api_key = "EMPTY"
32
openai_api_base = "http://localhost:8000/v1"
33
34
-client = OpenAI(
35
- api_key=openai_api_key,
36
- base_url=openai_api_base,
37
-)
38
-
39
-models = client.models.list()
40
-model = models.data[0].id
41
-
34
tools = [{
35
"type": "function",
36
"function": {
@@ -109,69 +101,87 @@ def extract_reasoning_and_calls(chunks: list):
101
return reasoning_content, arguments, function_names
102
103
112
-print("---------Full Generate With Automatic Function Calling-------------")
113
-tool_calls = client.chat.completions.create(messages=messages,
114
- model=model,
115
- tools=tools)
116
-print(f"reasoning_content: {tool_calls.choices[0].message.reasoning_content}")
117
-print(f"function name: "
118
- f"{tool_calls.choices[0].message.tool_calls[0].function.name}")
119
-print(f"function arguments: "
120
- f"{tool_calls.choices[0].message.tool_calls[0].function.arguments}")
121
-
122
-print("----------Stream Generate With Automatic Function Calling-----------")
123
-tool_calls_stream = client.chat.completions.create(messages=messages,
124
- model=model,
125
- tools=tools,
126
- stream=True)
127
-chunks = []
128
-for chunk in tool_calls_stream:
129
- chunks.append(chunk)
130
-
131
-reasoning_content, arguments, function_names = extract_reasoning_and_calls(
132
- chunks)
133
-
134
-print(f"reasoning_content: {reasoning_content}")
135
-print(f"function name: {function_names[0]}")
136
-print(f"function arguments: {arguments[0]}")
137
-
138
-print("----------Full Generate With Named Function Calling-----------------")
139
-tool_calls = client.chat.completions.create(messages=messages,
140
- model=model,
141
- tools=tools,
142
- tool_choice={
143
- "type": "function",
144
- "function": {
145
- "name":
146
- "get_current_weather"
147
- }
148
- })
149
-
150
-tool_call = tool_calls.choices[0].message.tool_calls[0].function
151
-print(f"reasoning_content: {tool_calls.choices[0].message.reasoning_content}")
152
-print(f"function name: {tool_call.name}")
153
-print(f"function arguments: {tool_call.arguments}")
154
-print("----------Stream Generate With Named Function Calling--------------")
155
-
156
-tool_calls_stream = client.chat.completions.create(
157
- messages=messages,
158
- model=model,
159
- tools=tools,
160
- tool_choice={
161
- "type": "function",
162
- "function": {
163
- "name": "get_current_weather"
164
- }
165
- },
166
- stream=True)
167
-
168
-chunks = []
169
-for chunk in tool_calls_stream:
170
- chunks.append(chunk)
171
-
172
-reasoning_content, arguments, function_names = extract_reasoning_and_calls(
173
- chunks)
174
-print(f"reasoning_content: {reasoning_content}")
175
-print(f"function name: {function_names[0]}")
176
-print(f"function arguments: {arguments[0]}")
177
-print("\n\n")
104
+def main():
105
+ client = OpenAI(
106
+ api_key=openai_api_key,
107
+ base_url=openai_api_base,
108
+ )
109
+
110
+ models = client.models.list()
111
+ model = models.data[0].id
112
+
113
+ print(
114
+ "---------Full Generate With Automatic Function Calling-------------")
115
+ tool_calls = client.chat.completions.create(messages=messages,
116
+ model=model,
117
+ tools=tools)
118
+ print(
119
+ f"reasoning_content: {tool_calls.choices[0].message.reasoning_content}"
120
+ )
121
+ print(f"function name: "
122
+ f"{tool_calls.choices[0].message.tool_calls[0].function.name}")
123
+ print(f"function arguments: "
124
+ f"{tool_calls.choices[0].message.tool_calls[0].function.arguments}")
125
+
126
+ print(
127
+ "----------Stream Generate With Automatic Function Calling-----------")
128
+ tool_calls_stream = client.chat.completions.create(messages=messages,
129
+ model=model,
130
+ tools=tools,
131
+ stream=True)
132
+
133
+ chunks = list(tool_calls_stream)
134
+
135
+ reasoning_content, arguments, function_names = extract_reasoning_and_calls(
136
+ chunks)
137
+
138
+ print(f"reasoning_content: {reasoning_content}")
139
+ print(f"function name: {function_names[0]}")
140
+ print(f"function arguments: {arguments[0]}")
141
+
142
+ print(
143
+ "----------Full Generate With Named Function Calling-----------------")
144
+ tool_calls = client.chat.completions.create(messages=messages,
145
+ model=model,
146
+ tools=tools,
147
+ tool_choice={
148
+ "type": "function",
149
+ "function": {
150
+ "name":
151
+ "get_current_weather"
152
+ }
153
+ })
154
+
155
+ tool_call = tool_calls.choices[0].message.tool_calls[0].function
156
+ print(
157
+ f"reasoning_content: {tool_calls.choices[0].message.reasoning_content}"
158
+ )
159
+ print(f"function name: {tool_call.name}")
160
+ print(f"function arguments: {tool_call.arguments}")
161
+ print(
162
+ "----------Stream Generate With Named Function Calling--------------")
163
+
164
+ tool_calls_stream = client.chat.completions.create(
165
+ messages=messages,
166
+ model=model,
167
+ tools=tools,
168
+ tool_choice={
169
+ "type": "function",
170
+ "function": {
171
+ "name": "get_current_weather"
172
+ }
173
+ },
174
+ stream=True)
175
+
176
+ chunks = list(tool_calls_stream)
177
+
178
+ reasoning_content, arguments, function_names = extract_reasoning_and_calls(
179
+ chunks)
180
+ print(f"reasoning_content: {reasoning_content}")
181
+ print(f"function name: {function_names[0]}")
182
+ print(f"function arguments: {arguments[0]}")
183
+ print("\n\n")
184
+
185
+
186
+if __name__ == "__main__":
187
+ main()