A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Misc][OpenAI] deprecate max_tokens in favor of new max_completion_tokens field for chat completion endpoint (#9837)
Guillaume Calmettes committed
Oct 31, 2024 at 02:15 UTC
abbfb6134dc73359cba015dbd1ad30fafd25a891
14 files changed
+140
-118
benchmarks/backend_request_func.py
+1
-1
@@ -324,7 +324,7 @@ async def async_request_openai_chat_completions(
324
},
325
],
326
"temperature": 0.0,
327
- "max_tokens": request_func_input.output_len,
327
+ "max_completion_tokens": request_func_input.output_len,
328
"stream": True,
329
"ignore_eos": request_func_input.ignore_eos,
330
}
docs/source/serving/run_on_sky.rst
+3
-3
@@ -109,7 +109,7 @@ SkyPilot can scale up the service to multiple service replicas with built-in aut
109
messages:
110
- role: user
111
content: Hello! What is your name?
112
- max_tokens: 1
112
+ max_completion_tokens: 1
113
114
.. raw:: html
115
@@ -129,7 +129,7 @@ SkyPilot can scale up the service to multiple service replicas with built-in aut
129
messages:
130
- role: user
131
content: Hello! What is your name?
132
- max_tokens: 1
132
+ max_completion_tokens: 1
133
134
resources:
135
accelerators: {L4, A10g, A10, L40, A40, A100, A100-80GB} # We can use cheaper accelerators for 8B model.
@@ -255,7 +255,7 @@ This will scale the service up to when the QPS exceeds 2 for each replica.
255
messages:
256
- role: user
257
content: Hello! What is your name?
258
- max_tokens: 1
258
+ max_completion_tokens: 1
259
260
resources:
261
accelerators: {L4, A10g, A10, L40, A40, A100, A100-80GB} # We can use cheaper accelerators for 8B model.
examples/offline_inference_openai.md
+4
-4
@@ -35,8 +35,8 @@
35
36
```
37
$ cat openai_example_batch.jsonl
38
-{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
39
-{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
38
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
39
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
40
```
41
42
### Step 2: Run the batch
@@ -94,8 +94,8 @@ To follow along with this example, you can download the example batch, or create
94
95
```
96
$ cat openai_example_batch.jsonl
97
-{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
98
-{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
97
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
98
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
99
```
100
101
Now upload your batch file to your S3 bucket.
examples/openai_api_client_for_multimodal.py
+6
-6
@@ -53,7 +53,7 @@ def run_text_only() -> None:
53
"content": "What's the capital of France?"
54
}],
55
model=model,
56
- max_tokens=64,
56
+ max_completion_tokens=64,
57
)
58
59
result = chat_completion.choices[0].message.content
@@ -83,7 +83,7 @@ def run_single_image() -> None:
83
],
84
}],
85
model=model,
86
- max_tokens=64,
86
+ max_completion_tokens=64,
87
)
88
89
result = chat_completion_from_url.choices[0].message.content
@@ -109,7 +109,7 @@ def run_single_image() -> None:
109
],
110
}],
111
model=model,
112
- max_tokens=64,
112
+ max_completion_tokens=64,
113
)
114
115
result = chat_completion_from_base64.choices[0].message.content
@@ -144,7 +144,7 @@ def run_multi_image() -> None:
144
],
145
}],
146
model=model,
147
- max_tokens=64,
147
+ max_completion_tokens=64,
148
)
149
150
result = chat_completion_from_url.choices[0].message.content
@@ -175,7 +175,7 @@ def run_audio() -> None:
175
],
176
}],
177
model=model,
178
- max_tokens=64,
178
+ max_completion_tokens=64,
179
)
180
181
result = chat_completion_from_url.choices[0].message.content
@@ -201,7 +201,7 @@ def run_audio() -> None:
201
],
202
}],
203
model=model,
204
- max_tokens=64,
204
+ max_completion_tokens=64,
205
)
206
207
result = chat_completion_from_base64.choices[0].message.content
examples/openai_example_batch.jsonl
+2
-2
@@ -1,2 +1,2 @@
1
-{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
2
-{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
1
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
2
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
requirements-common.txt
+1
-1
@@ -10,7 +10,7 @@ protobuf # Required by LlamaTokenizer.
10
fastapi >= 0.107.0, < 0.113.0; python_version < '3.9'
11
fastapi >= 0.107.0, != 0.113.*, != 0.114.0; python_version >= '3.9'
12
aiohttp
13
-openai >= 1.40.0 # Ensure modern openai package (ensure types module present)
13
+openai >= 1.45.0 # Ensure modern openai package (ensure types module present and max_completion_tokens field support)
14
uvicorn[standard]
15
pydantic >= 2.9 # Required for fastapi >= 0.113.0
16
pillow # Required for image processing
tests/entrypoints/openai/test_audio.py
+17
-15
@@ -68,11 +68,12 @@ async def test_single_chat_session_audio(client: openai.AsyncOpenAI,
68
}]
69
70
# test single completion
71
- chat_completion = await client.chat.completions.create(model=model_name,
72
- messages=messages,
73
- max_tokens=10,
74
- logprobs=True,
75
- top_logprobs=5)
71
+ chat_completion = await client.chat.completions.create(
72
+ model=model_name,
73
+ messages=messages,
74
+ max_completion_tokens=10,
75
+ logprobs=True,
76
+ top_logprobs=5)
77
assert len(chat_completion.choices) == 1
78
79
choice = chat_completion.choices[0]
@@ -91,7 +92,7 @@ async def test_single_chat_session_audio(client: openai.AsyncOpenAI,
92
chat_completion = await client.chat.completions.create(
93
model=model_name,
94
messages=messages,
94
- max_tokens=10,
95
+ max_completion_tokens=10,
96
)
97
message = chat_completion.choices[0].message
98
assert message.content is not None and len(message.content) >= 0
@@ -123,11 +124,12 @@ async def test_single_chat_session_audio_base64encoded(
124
}]
125
126
# test single completion
126
- chat_completion = await client.chat.completions.create(model=model_name,
127
- messages=messages,
128
- max_tokens=10,
129
- logprobs=True,
130
- top_logprobs=5)
127
+ chat_completion = await client.chat.completions.create(
128
+ model=model_name,
129
+ messages=messages,
130
+ max_completion_tokens=10,
131
+ logprobs=True,
132
+ top_logprobs=5)
133
assert len(chat_completion.choices) == 1
134
135
choice = chat_completion.choices[0]
@@ -146,7 +148,7 @@ async def test_single_chat_session_audio_base64encoded(
148
chat_completion = await client.chat.completions.create(
149
model=model_name,
150
messages=messages,
149
- max_tokens=10,
151
+ max_completion_tokens=10,
152
)
153
message = chat_completion.choices[0].message
154
assert message.content is not None and len(message.content) >= 0
@@ -178,7 +180,7 @@ async def test_chat_streaming_audio(client: openai.AsyncOpenAI,
180
chat_completion = await client.chat.completions.create(
181
model=model_name,
182
messages=messages,
181
- max_tokens=10,
183
+ max_completion_tokens=10,
184
temperature=0.0,
185
)
186
output = chat_completion.choices[0].message.content
@@ -188,7 +190,7 @@ async def test_chat_streaming_audio(client: openai.AsyncOpenAI,
190
stream = await client.chat.completions.create(
191
model=model_name,
192
messages=messages,
191
- max_tokens=10,
193
+ max_completion_tokens=10,
194
temperature=0.0,
195
stream=True,
196
)
@@ -242,7 +244,7 @@ async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,
244
await client.chat.completions.create(
245
model=model_name,
246
messages=messages,
245
- max_tokens=10,
247
+ max_completion_tokens=10,
248
temperature=0.0,
249
)
250
tests/entrypoints/openai/test_chat.py
+54
-49
@@ -65,11 +65,12 @@ async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
65
"content": "what is 1+1?"
66
}]
67
68
- chat_completion = await client.chat.completions.create(model=model_name,
69
- messages=messages,
70
- max_tokens=5,
71
- temperature=0.0,
72
- logprobs=False)
68
+ chat_completion = await client.chat.completions.create(
69
+ model=model_name,
70
+ messages=messages,
71
+ max_completion_tokens=5,
72
+ temperature=0.0,
73
+ logprobs=False)
74
75
choice = chat_completion.choices[0]
76
assert choice.logprobs is None
@@ -90,12 +91,13 @@ async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
91
"content": "what is 1+1?"
92
}]
93
93
- chat_completion = await client.chat.completions.create(model=model_name,
94
- messages=messages,
95
- max_tokens=5,
96
- temperature=0.0,
97
- logprobs=True,
98
- top_logprobs=0)
94
+ chat_completion = await client.chat.completions.create(
95
+ model=model_name,
96
+ messages=messages,
97
+ max_completion_tokens=5,
98
+ temperature=0.0,
99
+ logprobs=True,
100
+ top_logprobs=0)
101
102
choice = chat_completion.choices[0]
103
assert choice.logprobs is not None
@@ -117,12 +119,13 @@ async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
119
"content": "what is 1+1?"
120
}]
121
120
- chat_completion = await client.chat.completions.create(model=model_name,
121
- messages=messages,
122
- max_tokens=5,
123
- temperature=0.0,
124
- logprobs=True,
125
- top_logprobs=5)
122
+ chat_completion = await client.chat.completions.create(
123
+ model=model_name,
124
+ messages=messages,
125
+ max_completion_tokens=5,
126
+ temperature=0.0,
127
+ logprobs=True,
128
+ top_logprobs=5)
129
130
choice = chat_completion.choices[0]
131
assert choice.logprobs is not None
@@ -149,7 +152,7 @@ async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
152
with pytest.raises((openai.BadRequestError, openai.APIError)):
153
stream = await client.chat.completions.create(model=model_name,
154
messages=messages,
152
- max_tokens=10,
155
+ max_completion_tokens=10,
156
logprobs=True,
157
top_logprobs=21,
158
stream=True)
@@ -159,16 +162,17 @@ async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
162
with pytest.raises(openai.BadRequestError):
163
await client.chat.completions.create(model=model_name,
164
messages=messages,
162
- max_tokens=10,
165
+ max_completion_tokens=10,
166
logprobs=True,
167
top_logprobs=30,
168
stream=False)
169
170
# the server should still work afterwards
168
- chat_completion = await client.chat.completions.create(model=model_name,
169
- messages=messages,
170
- max_tokens=10,
171
- stream=False)
171
+ chat_completion = await client.chat.completions.create(
172
+ model=model_name,
173
+ messages=messages,
174
+ max_completion_tokens=10,
175
+ stream=False)
176
message = chat_completion.choices[0].message
177
assert message.content is not None and len(message.content) >= 0
178
@@ -271,11 +275,12 @@ async def test_single_chat_session(client: openai.AsyncOpenAI,
275
}]
276
277
# test single completion
274
- chat_completion = await client.chat.completions.create(model=model_name,
275
- messages=messages,
276
- max_tokens=10,
277
- logprobs=True,
278
- top_logprobs=5)
278
+ chat_completion = await client.chat.completions.create(
279
+ model=model_name,
280
+ messages=messages,
281
+ max_completion_tokens=10,
282
+ logprobs=True,
283
+ top_logprobs=5)
284
assert chat_completion.id is not None
285
assert len(chat_completion.choices) == 1
286
@@ -294,7 +299,7 @@ async def test_single_chat_session(client: openai.AsyncOpenAI,
299
chat_completion = await client.chat.completions.create(
300
model=model_name,
301
messages=messages,
297
- max_tokens=10,
302
+ max_completion_tokens=10,
303
)
304
message = chat_completion.choices[0].message
305
assert message.content is not None and len(message.content) >= 0
@@ -319,7 +324,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
324
chat_completion = await client.chat.completions.create(
325
model=model_name,
326
messages=messages,
322
- max_tokens=10,
327
+ max_completion_tokens=10,
328
temperature=0.0,
329
)
330
output = chat_completion.choices[0].message.content
@@ -329,7 +334,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
334
stream = await client.chat.completions.create(
335
model=model_name,
336
messages=messages,
332
- max_tokens=10,
337
+ max_completion_tokens=10,
338
temperature=0.0,
339
stream=True,
340
)
@@ -369,7 +374,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
374
stream = await client.chat.completions.create(
375
model=model_name,
376
messages=messages,
372
- max_tokens=10,
377
+ max_completion_tokens=10,
378
temperature=0.0,
379
stream=True,
380
stream_options={"include_usage": False})
@@ -380,7 +385,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
385
# "continuous_usage_stats": False}}
386
stream = await client.chat.completions.create(model=model_name,
387
messages=messages,
383
- max_tokens=10,
388
+ max_completion_tokens=10,
389
temperature=0.0,
390
stream=True,
391
stream_options={
@@ -409,7 +414,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
414
await client.chat.completions.create(
415
model=model_name,
416
messages=messages,
412
- max_tokens=10,
417
+ max_completion_tokens=10,
418
temperature=0.0,
419
stream=False,
420
stream_options={"include_usage": None})
@@ -419,7 +424,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
424
await client.chat.completions.create(
425
model=model_name,
426
messages=messages,
422
- max_tokens=10,
427
+ max_completion_tokens=10,
428
temperature=0.0,
429
stream=False,
430
stream_options={"include_usage": True})
@@ -429,7 +434,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
434
stream = await client.chat.completions.create(
435
model=model_name,
436
messages=messages,
432
- max_tokens=10,
437
+ max_completion_tokens=10,
438
extra_body=dict(min_tokens=10),
439
temperature=0.0,
440
stream=True,
@@ -476,7 +481,7 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
481
chat_completion = await client.chat.completions.create(
482
model=MODEL_NAME,
483
messages=messages,
479
- max_tokens=10,
484
+ max_completion_tokens=10,
485
extra_body=dict(guided_choice=sample_guided_choice,
486
guided_decoding_backend=guided_decoding_backend))
487
choice1 = chat_completion.choices[0].message.content
@@ -490,7 +495,7 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
495
chat_completion = await client.chat.completions.create(
496
model=MODEL_NAME,
497
messages=messages,
493
- max_tokens=10,
498
+ max_completion_tokens=10,
499
extra_body=dict(guided_choice=sample_guided_choice,
500
guided_decoding_backend=guided_decoding_backend))
501
choice2 = chat_completion.choices[0].message.content
@@ -517,7 +522,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
522
chat_completion = await client.chat.completions.create(
523
model=MODEL_NAME,
524
messages=messages,
520
- max_tokens=1000,
525
+ max_completion_tokens=1000,
526
extra_body=dict(guided_json=sample_json_schema,
527
guided_decoding_backend=guided_decoding_backend))
528
message = chat_completion.choices[0].message
@@ -535,7 +540,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
540
chat_completion = await client.chat.completions.create(
541
model=MODEL_NAME,
542
messages=messages,
538
- max_tokens=1000,
543
+ max_completion_tokens=1000,
544
extra_body=dict(guided_json=sample_json_schema,
545
guided_decoding_backend=guided_decoding_backend))
546
message = chat_completion.choices[0].message
@@ -563,7 +568,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI,
568
chat_completion = await client.chat.completions.create(
569
model=MODEL_NAME,
570
messages=messages,
566
- max_tokens=20,
571
+ max_completion_tokens=20,
572
extra_body=dict(guided_regex=sample_regex,
573
guided_decoding_backend=guided_decoding_backend))
574
ip1 = chat_completion.choices[0].message.content
@@ -575,7 +580,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI,
580
chat_completion = await client.chat.completions.create(
581
model=MODEL_NAME,
582
messages=messages,
578
- max_tokens=20,
583
+ max_completion_tokens=20,
584
extra_body=dict(guided_regex=sample_regex,
585
guided_decoding_backend=guided_decoding_backend))
586
ip2 = chat_completion.choices[0].message.content
@@ -623,7 +628,7 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
628
chat_completion = await client.chat.completions.create(
629
model=MODEL_NAME,
630
messages=messages,
626
- max_tokens=10,
631
+ max_completion_tokens=10,
632
logprobs=True,
633
top_logprobs=5,
634
extra_body=dict(guided_choice=sample_guided_choice,
@@ -660,7 +665,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
665
chat_completion = await client.chat.completions.create(
666
model=MODEL_NAME,
667
messages=messages,
663
- max_tokens=1000,
668
+ max_completion_tokens=1000,
669
tools=[{
670
"type": "function",
671
"function": {
@@ -694,7 +699,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
699
stream = await client.chat.completions.create(
700
model=MODEL_NAME,
701
messages=messages,
697
- max_tokens=1000,
702
+ max_completion_tokens=1000,
703
tools=[{
704
"type": "function",
705
"function": {
@@ -750,7 +755,7 @@ async def test_required_tool_use_not_yet_supported(
755
await client.chat.completions.create(
756
model=MODEL_NAME,
757
messages=messages,
753
- max_tokens=1000,
758
+ max_completion_tokens=1000,
759
tools=[{
760
"type": "function",
761
"function": {
@@ -765,7 +770,7 @@ async def test_required_tool_use_not_yet_supported(
770
await client.chat.completions.create(
771
model=MODEL_NAME,
772
messages=messages,
768
- max_tokens=1000,
773
+ max_completion_tokens=1000,
774
tools=[{
775
"type": "function",
776
"function": {
@@ -796,7 +801,7 @@ async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
801
with pytest.raises(openai.BadRequestError):
802
await client.chat.completions.create(model=MODEL_NAME,
803
messages=messages,
799
- max_tokens=1000,
804
+ max_completion_tokens=1000,
805
tool_choice={
806
"type": "function",
807
"function": {
@@ -809,7 +814,7 @@ async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
814
await client.chat.completions.create(
815
model=MODEL_NAME,
816
messages=messages,
812
- max_tokens=1000,
817
+ max_completion_tokens=1000,
818
tools=[{
819
"type": "function",
820
"function": {
tests/entrypoints/openai/test_vision.py
+20
-18
@@ -78,11 +78,12 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
78
}]
79
80
# test single completion
81
- chat_completion = await client.chat.completions.create(model=model_name,
82
- messages=messages,
83
- max_tokens=10,
84
- logprobs=True,
85
- top_logprobs=5)
81
+ chat_completion = await client.chat.completions.create(
82
+ model=model_name,
83
+ messages=messages,
84
+ max_completion_tokens=10,
85
+ logprobs=True,
86
+ top_logprobs=5)
87
assert len(chat_completion.choices) == 1
88
89
choice = chat_completion.choices[0]
@@ -101,7 +102,7 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
102
chat_completion = await client.chat.completions.create(
103
model=model_name,
104
messages=messages,
104
- max_tokens=10,
105
+ max_completion_tokens=10,
106
)
107
message = chat_completion.choices[0].message
108
assert message.content is not None and len(message.content) >= 0
@@ -134,7 +135,7 @@ async def test_single_chat_session_image_beamsearch(client: openai.AsyncOpenAI,
135
model=model_name,
136
messages=messages,
137
n=2,
137
- max_tokens=10,
138
+ max_completion_tokens=10,
139
logprobs=True,
140
top_logprobs=5,
141
extra_body=dict(use_beam_search=True))
@@ -169,11 +170,12 @@ async def test_single_chat_session_image_base64encoded(
170
}]
171
172
# test single completion
172
- chat_completion = await client.chat.completions.create(model=model_name,
173
- messages=messages,
174
- max_tokens=10,
175
- logprobs=True,
176
- top_logprobs=5)
173
+ chat_completion = await client.chat.completions.create(
174
+ model=model_name,
175
+ messages=messages,
176
+ max_completion_tokens=10,
177
+ logprobs=True,
178
+ top_logprobs=5)
179
assert len(chat_completion.choices) == 1
180
181
choice = chat_completion.choices[0]
@@ -192,7 +194,7 @@ async def test_single_chat_session_image_base64encoded(
194
chat_completion = await client.chat.completions.create(
195
model=model_name,
196
messages=messages,
195
- max_tokens=10,
197
+ max_completion_tokens=10,
198
)
199
message = chat_completion.choices[0].message
200
assert message.content is not None and len(message.content) >= 0
@@ -226,7 +228,7 @@ async def test_single_chat_session_image_base64encoded_beamsearch(
228
model=model_name,
229
messages=messages,
230
n=2,
229
- max_tokens=10,
231
+ max_completion_tokens=10,
232
extra_body=dict(use_beam_search=True))
233
assert len(chat_completion.choices) == 2
234
assert chat_completion.choices[
@@ -259,7 +261,7 @@ async def test_chat_streaming_image(client: openai.AsyncOpenAI,
261
chat_completion = await client.chat.completions.create(
262
model=model_name,
263
messages=messages,
262
- max_tokens=10,
264
+ max_completion_tokens=10,
265
temperature=0.0,
266
)
267
output = chat_completion.choices[0].message.content
@@ -269,7 +271,7 @@ async def test_chat_streaming_image(client: openai.AsyncOpenAI,
271
stream = await client.chat.completions.create(
272
model=model_name,
273
messages=messages,
272
- max_tokens=10,
274
+ max_completion_tokens=10,
275
temperature=0.0,
276
stream=True,
277
)
@@ -320,7 +322,7 @@ async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
322
await client.chat.completions.create(
323
model=model_name,
324
messages=messages,
323
- max_tokens=10,
325
+ max_completion_tokens=10,
326
temperature=0.0,
327
)
328
@@ -337,7 +339,7 @@ async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
339
chat_completion = await client.chat.completions.create(
340
model=model_name,
341
messages=messages,
340
- max_tokens=10,
342
+ max_completion_tokens=10,
343
temperature=0.0,
344
)
345
message = chat_completion.choices[0].message
tests/tool_use/test_chat_completions.py
+4
-4
@@ -18,7 +18,7 @@ async def test_chat_completion_without_tools(client: openai.AsyncOpenAI,
18
chat_completion = await client.chat.completions.create(
19
messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
20
temperature=0,
21
- max_tokens=150,
21
+ max_completion_tokens=150,
22
model=model_name,
23
logprobs=False)
24
choice = chat_completion.choices[0]
@@ -38,7 +38,7 @@ async def test_chat_completion_without_tools(client: openai.AsyncOpenAI,
38
stream = await client.chat.completions.create(
39
messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
40
temperature=0,
41
- max_tokens=150,
41
+ max_completion_tokens=150,
42
model=model_name,
43
logprobs=False,
44
stream=True,
@@ -86,7 +86,7 @@ async def test_chat_completion_with_tools(client: openai.AsyncOpenAI,
86
chat_completion = await client.chat.completions.create(
87
messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
88
temperature=0,
89
- max_tokens=150,
89
+ max_completion_tokens=150,
90
model=model_name,
91
tools=[WEATHER_TOOL],
92
logprobs=False)
@@ -107,7 +107,7 @@ async def test_chat_completion_with_tools(client: openai.AsyncOpenAI,
107
stream = await client.chat.completions.create(
108
messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
109
temperature=0,
110
- max_tokens=150,
110
+ max_completion_tokens=150,
111
model=model_name,
112
logprobs=False,
113
tools=[WEATHER_TOOL],
tests/tool_use/test_parallel_tool_calls.py
+4
-4
@@ -26,7 +26,7 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI,
26
chat_completion = await client.chat.completions.create(
27
messages=MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
28
temperature=0,
29
- max_tokens=200,
29
+ max_completion_tokens=200,
30
model=model_name,
31
tools=[WEATHER_TOOL, SEARCH_TOOL],
32
logprobs=False)
@@ -63,7 +63,7 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI,
63
model=model_name,
64
messages=MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
65
temperature=0,
66
- max_tokens=200,
66
+ max_completion_tokens=200,
67
tools=[WEATHER_TOOL, SEARCH_TOOL],
68
logprobs=False,
69
stream=True)
@@ -154,7 +154,7 @@ async def test_parallel_tool_calls_with_results(client: openai.AsyncOpenAI,
154
chat_completion = await client.chat.completions.create(
155
messages=MESSAGES_WITH_PARALLEL_TOOL_RESPONSE,
156
temperature=0,
157
- max_tokens=200,
157
+ max_completion_tokens=200,
158
model=model_name,
159
tools=[WEATHER_TOOL, SEARCH_TOOL],
160
logprobs=False)
@@ -172,7 +172,7 @@ async def test_parallel_tool_calls_with_results(client: openai.AsyncOpenAI,
172
stream = await client.chat.completions.create(
173
messages=MESSAGES_WITH_PARALLEL_TOOL_RESPONSE,
174
temperature=0,
175
- max_tokens=200,
175
+ max_completion_tokens=200,
176
model=model_name,
177
tools=[WEATHER_TOOL, SEARCH_TOOL],
178
logprobs=False,
tests/tool_use/test_tool_calls.py
+4
-4
@@ -17,7 +17,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):
17
chat_completion = await client.chat.completions.create(
18
messages=MESSAGES_ASKING_FOR_TOOLS,
19
temperature=0,
20
- max_tokens=100,
20
+ max_completion_tokens=100,
21
model=model_name,
22
tools=[WEATHER_TOOL, SEARCH_TOOL],
23
logprobs=False)
@@ -61,7 +61,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):
61
model=model_name,
62
messages=MESSAGES_ASKING_FOR_TOOLS,
63
temperature=0,
64
- max_tokens=100,
64
+ max_completion_tokens=100,
65
tools=[WEATHER_TOOL, SEARCH_TOOL],
66
logprobs=False,
67
stream=True)
@@ -142,7 +142,7 @@ async def test_tool_call_with_results(client: openai.AsyncOpenAI):
142
chat_completion = await client.chat.completions.create(
143
messages=MESSAGES_WITH_TOOL_RESPONSE,
144
temperature=0,
145
- max_tokens=100,
145
+ max_completion_tokens=100,
146
model=model_name,
147
tools=[WEATHER_TOOL, SEARCH_TOOL],
148
logprobs=False)
@@ -159,7 +159,7 @@ async def test_tool_call_with_results(client: openai.AsyncOpenAI):
159
stream = await client.chat.completions.create(
160
messages=MESSAGES_WITH_TOOL_RESPONSE,
161
temperature=0,
162
- max_tokens=100,
162
+ max_completion_tokens=100,
163
model=model_name,
164
tools=[WEATHER_TOOL, SEARCH_TOOL],
165
logprobs=False,
vllm/entrypoints/openai/protocol.py
+10
-3
@@ -159,7 +159,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
159
logit_bias: Optional[Dict[str, float]] = None
160
logprobs: Optional[bool] = False
161
top_logprobs: Optional[int] = 0
162
- max_tokens: Optional[int] = None
162
+ # TODO(#9845): remove max_tokens when field is removed from OpenAI API
163
+ max_tokens: Optional[int] = Field(
164
+ default=None,
165
+ deprecated=
166
+ 'max_tokens is deprecated in favor of the max_completion_tokens field')
167
+ max_completion_tokens: Optional[int] = None
168
n: Optional[int] = 1
169
presence_penalty: Optional[float] = 0.0
170
response_format: Optional[ResponseFormat] = None
@@ -295,7 +300,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
300
301
def to_beam_search_params(self,
302
default_max_tokens: int) -> BeamSearchParams:
298
- max_tokens = self.max_tokens
303
+ # TODO(#9845): remove max_tokens when field is removed from OpenAI API
304
+ max_tokens = self.max_completion_tokens or self.max_tokens
305
if max_tokens is None:
306
max_tokens = default_max_tokens
307
@@ -311,7 +317,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
317
include_stop_str_in_output=self.include_stop_str_in_output)
318
319
def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
314
- max_tokens = self.max_tokens
320
+ # TODO(#9845): remove max_tokens when field is removed from OpenAI API
321
+ max_tokens = self.max_completion_tokens or self.max_tokens
322
if max_tokens is None:
323
max_tokens = default_max_tokens
324
vllm/entrypoints/openai/serving_engine.py
+10
-4
@@ -263,20 +263,26 @@ class OpenAIServing:
263
return TextTokensPrompt(prompt=input_text,
264
prompt_token_ids=input_ids)
265
266
- if request.max_tokens is None:
266
+ # chat completion endpoint supports max_completion_tokens
267
+ if isinstance(request, ChatCompletionRequest):
268
+ # TODO(#9845): remove max_tokens when field dropped from OpenAI API
269
+ max_tokens = request.max_completion_tokens or request.max_tokens
270
+ else:
271
+ max_tokens = request.max_tokens
272
+ if max_tokens is None:
273
if token_num >= self.max_model_len:
274
raise ValueError(
275
f"This model's maximum context length is "
276
f"{self.max_model_len} tokens. However, you requested "
277
f"{token_num} tokens in the messages, "
278
f"Please reduce the length of the messages.")
273
- elif token_num + request.max_tokens > self.max_model_len:
279
+ elif token_num + max_tokens > self.max_model_len:
280
raise ValueError(
281
f"This model's maximum context length is "
282
f"{self.max_model_len} tokens. However, you requested "
277
- f"{request.max_tokens + token_num} tokens "
283
+ f"{max_tokens + token_num} tokens "
284
f"({token_num} in the messages, "
279
- f"{request.max_tokens} in the completion). "
285
+ f"{max_tokens} in the completion). "
286
f"Please reduce the length of the messages or completion.")
287
288
return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)