A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Doc] Update reasoning with stream example to use OpenAI library (#14077)
Signed-off-by: liuyanyi <wolfsonliu@163.com>
Yanyi Liu committed
Mar 6, 2025 at 21:20 UTC
0ddc991f5c4000d04581b58a5320d210257c6760
2 files changed
+82
-57
docs/source/features/reasoning_outputs.md
+49
-1
@@ -78,7 +78,55 @@ Streaming chat completions are also supported for reasoning models. The `reasoni
78
}
79
```
80
81
-Please note that it is not compatible with the OpenAI Python client library. You can use the `requests` library to make streaming requests. You could checkout the [example](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/openai_chat_completion_with_reasoning_streaming.py).
81
+OpenAI Python client library does not officially support `reasoning_content` attribute for streaming output. But the client support extra attributes in the response. You can use `hasattr` to check if the `reasoning_content` attribute is present in the response. For example:
82
+
83
+```python
84
+from openai import OpenAI
85
+
86
+# Modify OpenAI's API key and API base to use vLLM's API server.
87
+openai_api_key = "EMPTY"
88
+openai_api_base = "http://localhost:8000/v1"
89
+
90
+client = OpenAI(
91
+ api_key=openai_api_key,
92
+ base_url=openai_api_base,
93
+)
94
+
95
+models = client.models.list()
96
+model = models.data[0].id
97
+
98
+messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
99
+stream = client.chat.completions.create(model=model,
100
+ messages=messages,
101
+ stream=True)
102
+
103
+print("client: Start streaming chat completions...")
104
+printed_reasoning_content = False
105
+printed_content = False
106
+
107
+for chunk in stream:
108
+ reasoning_content = None
109
+ content = None
110
+ # Check the content is reasoning_content or content
111
+ if hasattr(chunk.choices[0].delta, "reasoning_content"):
112
+ reasoning_content = chunk.choices[0].delta.reasoning_content
113
+ elif hasattr(chunk.choices[0].delta, "content"):
114
+ content = chunk.choices[0].delta.content
115
+
116
+ if reasoning_content is not None:
117
+ if not printed_reasoning_content:
118
+ printed_reasoning_content = True
119
+ print("reasoning_content:", end="", flush=True)
120
+ print(reasoning_content, end="", flush=True)
121
+ elif content is not None:
122
+ if not printed_content:
123
+ printed_content = True
124
+ print("\ncontent:", end="", flush=True)
125
+ # Extract and print the content
126
+ print(content, end="", flush=True)
127
+```
128
+
129
+Remember to check whether the `reasoning_content` exists in the response before accessing it. You could checkout the [example](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/openai_chat_completion_with_reasoning_streaming.py).
130
131
## Structured output
132
examples/online_serving/openai_chat_completion_with_reasoning_streaming.py
+33
-56
@@ -19,73 +19,50 @@ in real-time as they are generated by the model. This is useful for scenarios
19
where you want to display chat completions to the user as they are generated
20
by the model.
21
22
-Here we do not use the OpenAI Python client library, because it does not support
23
-`reasoning_content` fields in the response.
22
+Remember to check content and reasoning_content exist in `ChatCompletionChunk`,
23
+content may not exist leading to errors if you try to access it.
24
"""
25
26
-import json
27
-
28
-import requests
26
+from openai import OpenAI
27
28
# Modify OpenAI's API key and API base to use vLLM's API server.
29
openai_api_key = "EMPTY"
30
openai_api_base = "http://localhost:8000/v1"
31
34
-models = requests.get(
35
- f"{openai_api_base}/models",
36
- headers={
37
- "Authorization": f"Bearer {openai_api_key}"
38
- },
39
-).json()
40
-model = models["data"][0]["id"]
32
+client = OpenAI(
33
+ api_key=openai_api_key,
34
+ base_url=openai_api_base,
35
+)
36
42
-# Streaming chat completions
43
-messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
37
+models = client.models.list()
38
+model = models.data[0].id
39
45
-response = requests.post(
46
- f"{openai_api_base}/chat/completions",
47
- headers={"Authorization": f"Bearer {openai_api_key}"},
48
- json={
49
- "model": model,
50
- "messages": messages,
51
- "stream": True
52
- },
53
-)
40
+messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
41
+stream = client.chat.completions.create(model=model,
42
+ messages=messages,
43
+ stream=True)
44
45
print("client: Start streaming chat completions...")
46
printed_reasoning_content = False
47
printed_content = False
58
-# Make the streaming request
59
-if response.status_code == 200:
60
- # Process the streaming response
61
- for line in response.iter_lines():
62
- if line: # Filter out keep-alive new lines
63
- # Decode the line and parse the JSON
64
- decoded_line = line.decode("utf-8")
65
- if decoded_line.startswith("data:"):
66
- data = decoded_line[5:].strip() # Remove "data:" prefix
67
- if data == "[DONE]": # End of stream
68
- print("\nclient: Stream completed.")
69
- break
70
- try:
71
- # Parse the JSON data
72
- chunk = json.loads(data)
73
- reasoning_content = chunk["choices"][0]["delta"].get(
74
- "reasoning_content", "")
75
- content = chunk["choices"][0]["delta"].get("content", "")
48
77
- if reasoning_content:
78
- if not printed_reasoning_content:
79
- printed_reasoning_content = True
80
- print("reasoning_content:", end="", flush=True)
81
- print(reasoning_content, end="", flush=True)
82
- elif content:
83
- if not printed_content:
84
- printed_content = True
85
- print("\ncontent:", end="", flush=True)
86
- # Extract and print the content
87
- print(content, end="", flush=True)
88
- except json.JSONDecodeError:
89
- print("Error decoding JSON:", decoded_line)
90
-else:
91
- print(f"Error: {response.status_code} - {response.text}")
49
+for chunk in stream:
50
+ reasoning_content = None
51
+ content = None
52
+ # Check the content is reasoning_content or content
53
+ if hasattr(chunk.choices[0].delta, "reasoning_content"):
54
+ reasoning_content = chunk.choices[0].delta.reasoning_content
55
+ elif hasattr(chunk.choices[0].delta, "content"):
56
+ content = chunk.choices[0].delta.content
57
+
58
+ if reasoning_content is not None:
59
+ if not printed_reasoning_content:
60
+ printed_reasoning_content = True
61
+ print("reasoning_content:", end="", flush=True)
62
+ print(reasoning_content, end="", flush=True)
63
+ elif content is not None:
64
+ if not printed_content:
65
+ printed_content = True
66
+ print("\ncontent:", end="", flush=True)
67
+ # Extract and print the content
68
+ print(content, end="", flush=True)