@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / abbfb6134d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Misc][OpenAI] deprecate max_tokens in favor of new max_completion_tokens field for chat completion endpoint (#9837)

Guillaume Calmettes committed Oct 31, 2024 at 02:15 UTC abbfb6134dc73359cba015dbd1ad30fafd25a891
14 files changed +140 -118
benchmarks/backend_request_func.py
+1 -1
@@ -324,7 +324,7 @@ async def async_request_openai_chat_completions(
324 },
325 ],
326 "temperature": 0.0,
327 - "max_tokens": request_func_input.output_len,
327 + "max_completion_tokens": request_func_input.output_len,
328 "stream": True,
329 "ignore_eos": request_func_input.ignore_eos,
330 }
docs/source/serving/run_on_sky.rst
+3 -3
@@ -109,7 +109,7 @@ SkyPilot can scale up the service to multiple service replicas with built-in aut
109 messages:
110 - role: user
111 content: Hello! What is your name?
112 - max_tokens: 1
112 + max_completion_tokens: 1
113
114 .. raw:: html
115
@@ -129,7 +129,7 @@ SkyPilot can scale up the service to multiple service replicas with built-in aut
129 messages:
130 - role: user
131 content: Hello! What is your name?
132 - max_tokens: 1
132 + max_completion_tokens: 1
133
134 resources:
135 accelerators: {L4, A10g, A10, L40, A40, A100, A100-80GB} # We can use cheaper accelerators for 8B model.
@@ -255,7 +255,7 @@ This will scale the service up to when the QPS exceeds 2 for each replica.
255 messages:
256 - role: user
257 content: Hello! What is your name?
258 - max_tokens: 1
258 + max_completion_tokens: 1
259
260 resources:
261 accelerators: {L4, A10g, A10, L40, A40, A100, A100-80GB} # We can use cheaper accelerators for 8B model.
examples/offline_inference_openai.md
+4 -4
@@ -35,8 +35,8 @@
35
36 ```
37 $ cat openai_example_batch.jsonl
38 -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
39 -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
38 +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
39 +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
40 ```
41
42 ### Step 2: Run the batch
@@ -94,8 +94,8 @@ To follow along with this example, you can download the example batch, or create
94
95 ```
96 $ cat openai_example_batch.jsonl
97 -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
98 -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
97 +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
98 +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
99 ```
100
101 Now upload your batch file to your S3 bucket.
examples/openai_api_client_for_multimodal.py
+6 -6
@@ -53,7 +53,7 @@ def run_text_only() -> None:
53 "content": "What's the capital of France?"
54 }],
55 model=model,
56 - max_tokens=64,
56 + max_completion_tokens=64,
57 )
58
59 result = chat_completion.choices[0].message.content
@@ -83,7 +83,7 @@ def run_single_image() -> None:
83 ],
84 }],
85 model=model,
86 - max_tokens=64,
86 + max_completion_tokens=64,
87 )
88
89 result = chat_completion_from_url.choices[0].message.content
@@ -109,7 +109,7 @@ def run_single_image() -> None:
109 ],
110 }],
111 model=model,
112 - max_tokens=64,
112 + max_completion_tokens=64,
113 )
114
115 result = chat_completion_from_base64.choices[0].message.content
@@ -144,7 +144,7 @@ def run_multi_image() -> None:
144 ],
145 }],
146 model=model,
147 - max_tokens=64,
147 + max_completion_tokens=64,
148 )
149
150 result = chat_completion_from_url.choices[0].message.content
@@ -175,7 +175,7 @@ def run_audio() -> None:
175 ],
176 }],
177 model=model,
178 - max_tokens=64,
178 + max_completion_tokens=64,
179 )
180
181 result = chat_completion_from_url.choices[0].message.content
@@ -201,7 +201,7 @@ def run_audio() -> None:
201 ],
202 }],
203 model=model,
204 - max_tokens=64,
204 + max_completion_tokens=64,
205 )
206
207 result = chat_completion_from_base64.choices[0].message.content
examples/openai_example_batch.jsonl
+2 -2
@@ -1,2 +1,2 @@
1 -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
2 -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
1 +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
2 +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
requirements-common.txt
+1 -1
@@ -10,7 +10,7 @@ protobuf # Required by LlamaTokenizer.
10 fastapi >= 0.107.0, < 0.113.0; python_version < '3.9'
11 fastapi >= 0.107.0, != 0.113.*, != 0.114.0; python_version >= '3.9'
12 aiohttp
13 -openai >= 1.40.0 # Ensure modern openai package (ensure types module present)
13 +openai >= 1.45.0 # Ensure modern openai package (ensure types module present and max_completion_tokens field support)
14 uvicorn[standard]
15 pydantic >= 2.9 # Required for fastapi >= 0.113.0
16 pillow # Required for image processing
tests/entrypoints/openai/test_audio.py
+17 -15
@@ -68,11 +68,12 @@ async def test_single_chat_session_audio(client: openai.AsyncOpenAI,
68 }]
69
70 # test single completion
71 - chat_completion = await client.chat.completions.create(model=model_name,
72 - messages=messages,
73 - max_tokens=10,
74 - logprobs=True,
75 - top_logprobs=5)
71 + chat_completion = await client.chat.completions.create(
72 + model=model_name,
73 + messages=messages,
74 + max_completion_tokens=10,
75 + logprobs=True,
76 + top_logprobs=5)
77 assert len(chat_completion.choices) == 1
78
79 choice = chat_completion.choices[0]
@@ -91,7 +92,7 @@ async def test_single_chat_session_audio(client: openai.AsyncOpenAI,
92 chat_completion = await client.chat.completions.create(
93 model=model_name,
94 messages=messages,
94 - max_tokens=10,
95 + max_completion_tokens=10,
96 )
97 message = chat_completion.choices[0].message
98 assert message.content is not None and len(message.content) >= 0
@@ -123,11 +124,12 @@ async def test_single_chat_session_audio_base64encoded(
124 }]
125
126 # test single completion
126 - chat_completion = await client.chat.completions.create(model=model_name,
127 - messages=messages,
128 - max_tokens=10,
129 - logprobs=True,
130 - top_logprobs=5)
127 + chat_completion = await client.chat.completions.create(
128 + model=model_name,
129 + messages=messages,
130 + max_completion_tokens=10,
131 + logprobs=True,
132 + top_logprobs=5)
133 assert len(chat_completion.choices) == 1
134
135 choice = chat_completion.choices[0]
@@ -146,7 +148,7 @@ async def test_single_chat_session_audio_base64encoded(
148 chat_completion = await client.chat.completions.create(
149 model=model_name,
150 messages=messages,
149 - max_tokens=10,
151 + max_completion_tokens=10,
152 )
153 message = chat_completion.choices[0].message
154 assert message.content is not None and len(message.content) >= 0
@@ -178,7 +180,7 @@ async def test_chat_streaming_audio(client: openai.AsyncOpenAI,
180 chat_completion = await client.chat.completions.create(
181 model=model_name,
182 messages=messages,
181 - max_tokens=10,
183 + max_completion_tokens=10,
184 temperature=0.0,
185 )
186 output = chat_completion.choices[0].message.content
@@ -188,7 +190,7 @@ async def test_chat_streaming_audio(client: openai.AsyncOpenAI,
190 stream = await client.chat.completions.create(
191 model=model_name,
192 messages=messages,
191 - max_tokens=10,
193 + max_completion_tokens=10,
194 temperature=0.0,
195 stream=True,
196 )
@@ -242,7 +244,7 @@ async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,
244 await client.chat.completions.create(
245 model=model_name,
246 messages=messages,
245 - max_tokens=10,
247 + max_completion_tokens=10,
248 temperature=0.0,
249 )
250
tests/entrypoints/openai/test_chat.py
+54 -49
@@ -65,11 +65,12 @@ async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
65 "content": "what is 1+1?"
66 }]
67
68 - chat_completion = await client.chat.completions.create(model=model_name,
69 - messages=messages,
70 - max_tokens=5,
71 - temperature=0.0,
72 - logprobs=False)
68 + chat_completion = await client.chat.completions.create(
69 + model=model_name,
70 + messages=messages,
71 + max_completion_tokens=5,
72 + temperature=0.0,
73 + logprobs=False)
74
75 choice = chat_completion.choices[0]
76 assert choice.logprobs is None
@@ -90,12 +91,13 @@ async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
91 "content": "what is 1+1?"
92 }]
93
93 - chat_completion = await client.chat.completions.create(model=model_name,
94 - messages=messages,
95 - max_tokens=5,
96 - temperature=0.0,
97 - logprobs=True,
98 - top_logprobs=0)
94 + chat_completion = await client.chat.completions.create(
95 + model=model_name,
96 + messages=messages,
97 + max_completion_tokens=5,
98 + temperature=0.0,
99 + logprobs=True,
100 + top_logprobs=0)
101
102 choice = chat_completion.choices[0]
103 assert choice.logprobs is not None
@@ -117,12 +119,13 @@ async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
119 "content": "what is 1+1?"
120 }]
121
120 - chat_completion = await client.chat.completions.create(model=model_name,
121 - messages=messages,
122 - max_tokens=5,
123 - temperature=0.0,
124 - logprobs=True,
125 - top_logprobs=5)
122 + chat_completion = await client.chat.completions.create(
123 + model=model_name,
124 + messages=messages,
125 + max_completion_tokens=5,
126 + temperature=0.0,
127 + logprobs=True,
128 + top_logprobs=5)
129
130 choice = chat_completion.choices[0]
131 assert choice.logprobs is not None
@@ -149,7 +152,7 @@ async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
152 with pytest.raises((openai.BadRequestError, openai.APIError)):
153 stream = await client.chat.completions.create(model=model_name,
154 messages=messages,
152 - max_tokens=10,
155 + max_completion_tokens=10,
156 logprobs=True,
157 top_logprobs=21,
158 stream=True)
@@ -159,16 +162,17 @@ async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI,
162 with pytest.raises(openai.BadRequestError):
163 await client.chat.completions.create(model=model_name,
164 messages=messages,
162 - max_tokens=10,
165 + max_completion_tokens=10,
166 logprobs=True,
167 top_logprobs=30,
168 stream=False)
169
170 # the server should still work afterwards
168 - chat_completion = await client.chat.completions.create(model=model_name,
169 - messages=messages,
170 - max_tokens=10,
171 - stream=False)
171 + chat_completion = await client.chat.completions.create(
172 + model=model_name,
173 + messages=messages,
174 + max_completion_tokens=10,
175 + stream=False)
176 message = chat_completion.choices[0].message
177 assert message.content is not None and len(message.content) >= 0
178
@@ -271,11 +275,12 @@ async def test_single_chat_session(client: openai.AsyncOpenAI,
275 }]
276
277 # test single completion
274 - chat_completion = await client.chat.completions.create(model=model_name,
275 - messages=messages,
276 - max_tokens=10,
277 - logprobs=True,
278 - top_logprobs=5)
278 + chat_completion = await client.chat.completions.create(
279 + model=model_name,
280 + messages=messages,
281 + max_completion_tokens=10,
282 + logprobs=True,
283 + top_logprobs=5)
284 assert chat_completion.id is not None
285 assert len(chat_completion.choices) == 1
286
@@ -294,7 +299,7 @@ async def test_single_chat_session(client: openai.AsyncOpenAI,
299 chat_completion = await client.chat.completions.create(
300 model=model_name,
301 messages=messages,
297 - max_tokens=10,
302 + max_completion_tokens=10,
303 )
304 message = chat_completion.choices[0].message
305 assert message.content is not None and len(message.content) >= 0
@@ -319,7 +324,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
324 chat_completion = await client.chat.completions.create(
325 model=model_name,
326 messages=messages,
322 - max_tokens=10,
327 + max_completion_tokens=10,
328 temperature=0.0,
329 )
330 output = chat_completion.choices[0].message.content
@@ -329,7 +334,7 @@ async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
334 stream = await client.chat.completions.create(
335 model=model_name,
336 messages=messages,
332 - max_tokens=10,
337 + max_completion_tokens=10,
338 temperature=0.0,
339 stream=True,
340 )
@@ -369,7 +374,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
374 stream = await client.chat.completions.create(
375 model=model_name,
376 messages=messages,
372 - max_tokens=10,
377 + max_completion_tokens=10,
378 temperature=0.0,
379 stream=True,
380 stream_options={"include_usage": False})
@@ -380,7 +385,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
385 # "continuous_usage_stats": False}}
386 stream = await client.chat.completions.create(model=model_name,
387 messages=messages,
383 - max_tokens=10,
388 + max_completion_tokens=10,
389 temperature=0.0,
390 stream=True,
391 stream_options={
@@ -409,7 +414,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
414 await client.chat.completions.create(
415 model=model_name,
416 messages=messages,
412 - max_tokens=10,
417 + max_completion_tokens=10,
418 temperature=0.0,
419 stream=False,
420 stream_options={"include_usage": None})
@@ -419,7 +424,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
424 await client.chat.completions.create(
425 model=model_name,
426 messages=messages,
422 - max_tokens=10,
427 + max_completion_tokens=10,
428 temperature=0.0,
429 stream=False,
430 stream_options={"include_usage": True})
@@ -429,7 +434,7 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
434 stream = await client.chat.completions.create(
435 model=model_name,
436 messages=messages,
432 - max_tokens=10,
437 + max_completion_tokens=10,
438 extra_body=dict(min_tokens=10),
439 temperature=0.0,
440 stream=True,
@@ -476,7 +481,7 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
481 chat_completion = await client.chat.completions.create(
482 model=MODEL_NAME,
483 messages=messages,
479 - max_tokens=10,
484 + max_completion_tokens=10,
485 extra_body=dict(guided_choice=sample_guided_choice,
486 guided_decoding_backend=guided_decoding_backend))
487 choice1 = chat_completion.choices[0].message.content
@@ -490,7 +495,7 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,
495 chat_completion = await client.chat.completions.create(
496 model=MODEL_NAME,
497 messages=messages,
493 - max_tokens=10,
498 + max_completion_tokens=10,
499 extra_body=dict(guided_choice=sample_guided_choice,
500 guided_decoding_backend=guided_decoding_backend))
501 choice2 = chat_completion.choices[0].message.content
@@ -517,7 +522,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
522 chat_completion = await client.chat.completions.create(
523 model=MODEL_NAME,
524 messages=messages,
520 - max_tokens=1000,
525 + max_completion_tokens=1000,
526 extra_body=dict(guided_json=sample_json_schema,
527 guided_decoding_backend=guided_decoding_backend))
528 message = chat_completion.choices[0].message
@@ -535,7 +540,7 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,
540 chat_completion = await client.chat.completions.create(
541 model=MODEL_NAME,
542 messages=messages,
538 - max_tokens=1000,
543 + max_completion_tokens=1000,
544 extra_body=dict(guided_json=sample_json_schema,
545 guided_decoding_backend=guided_decoding_backend))
546 message = chat_completion.choices[0].message
@@ -563,7 +568,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI,
568 chat_completion = await client.chat.completions.create(
569 model=MODEL_NAME,
570 messages=messages,
566 - max_tokens=20,
571 + max_completion_tokens=20,
572 extra_body=dict(guided_regex=sample_regex,
573 guided_decoding_backend=guided_decoding_backend))
574 ip1 = chat_completion.choices[0].message.content
@@ -575,7 +580,7 @@ async def test_guided_regex_chat(client: openai.AsyncOpenAI,
580 chat_completion = await client.chat.completions.create(
581 model=MODEL_NAME,
582 messages=messages,
578 - max_tokens=20,
583 + max_completion_tokens=20,
584 extra_body=dict(guided_regex=sample_regex,
585 guided_decoding_backend=guided_decoding_backend))
586 ip2 = chat_completion.choices[0].message.content
@@ -623,7 +628,7 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,
628 chat_completion = await client.chat.completions.create(
629 model=MODEL_NAME,
630 messages=messages,
626 - max_tokens=10,
631 + max_completion_tokens=10,
632 logprobs=True,
633 top_logprobs=5,
634 extra_body=dict(guided_choice=sample_guided_choice,
@@ -660,7 +665,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
665 chat_completion = await client.chat.completions.create(
666 model=MODEL_NAME,
667 messages=messages,
663 - max_tokens=1000,
668 + max_completion_tokens=1000,
669 tools=[{
670 "type": "function",
671 "function": {
@@ -694,7 +699,7 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,
699 stream = await client.chat.completions.create(
700 model=MODEL_NAME,
701 messages=messages,
697 - max_tokens=1000,
702 + max_completion_tokens=1000,
703 tools=[{
704 "type": "function",
705 "function": {
@@ -750,7 +755,7 @@ async def test_required_tool_use_not_yet_supported(
755 await client.chat.completions.create(
756 model=MODEL_NAME,
757 messages=messages,
753 - max_tokens=1000,
758 + max_completion_tokens=1000,
759 tools=[{
760 "type": "function",
761 "function": {
@@ -765,7 +770,7 @@ async def test_required_tool_use_not_yet_supported(
770 await client.chat.completions.create(
771 model=MODEL_NAME,
772 messages=messages,
768 - max_tokens=1000,
773 + max_completion_tokens=1000,
774 tools=[{
775 "type": "function",
776 "function": {
@@ -796,7 +801,7 @@ async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
801 with pytest.raises(openai.BadRequestError):
802 await client.chat.completions.create(model=MODEL_NAME,
803 messages=messages,
799 - max_tokens=1000,
804 + max_completion_tokens=1000,
805 tool_choice={
806 "type": "function",
807 "function": {
@@ -809,7 +814,7 @@ async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,
814 await client.chat.completions.create(
815 model=MODEL_NAME,
816 messages=messages,
812 - max_tokens=1000,
817 + max_completion_tokens=1000,
818 tools=[{
819 "type": "function",
820 "function": {
tests/entrypoints/openai/test_vision.py
+20 -18
@@ -78,11 +78,12 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
78 }]
79
80 # test single completion
81 - chat_completion = await client.chat.completions.create(model=model_name,
82 - messages=messages,
83 - max_tokens=10,
84 - logprobs=True,
85 - top_logprobs=5)
81 + chat_completion = await client.chat.completions.create(
82 + model=model_name,
83 + messages=messages,
84 + max_completion_tokens=10,
85 + logprobs=True,
86 + top_logprobs=5)
87 assert len(chat_completion.choices) == 1
88
89 choice = chat_completion.choices[0]
@@ -101,7 +102,7 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
102 chat_completion = await client.chat.completions.create(
103 model=model_name,
104 messages=messages,
104 - max_tokens=10,
105 + max_completion_tokens=10,
106 )
107 message = chat_completion.choices[0].message
108 assert message.content is not None and len(message.content) >= 0
@@ -134,7 +135,7 @@ async def test_single_chat_session_image_beamsearch(client: openai.AsyncOpenAI,
135 model=model_name,
136 messages=messages,
137 n=2,
137 - max_tokens=10,
138 + max_completion_tokens=10,
139 logprobs=True,
140 top_logprobs=5,
141 extra_body=dict(use_beam_search=True))
@@ -169,11 +170,12 @@ async def test_single_chat_session_image_base64encoded(
170 }]
171
172 # test single completion
172 - chat_completion = await client.chat.completions.create(model=model_name,
173 - messages=messages,
174 - max_tokens=10,
175 - logprobs=True,
176 - top_logprobs=5)
173 + chat_completion = await client.chat.completions.create(
174 + model=model_name,
175 + messages=messages,
176 + max_completion_tokens=10,
177 + logprobs=True,
178 + top_logprobs=5)
179 assert len(chat_completion.choices) == 1
180
181 choice = chat_completion.choices[0]
@@ -192,7 +194,7 @@ async def test_single_chat_session_image_base64encoded(
194 chat_completion = await client.chat.completions.create(
195 model=model_name,
196 messages=messages,
195 - max_tokens=10,
197 + max_completion_tokens=10,
198 )
199 message = chat_completion.choices[0].message
200 assert message.content is not None and len(message.content) >= 0
@@ -226,7 +228,7 @@ async def test_single_chat_session_image_base64encoded_beamsearch(
228 model=model_name,
229 messages=messages,
230 n=2,
229 - max_tokens=10,
231 + max_completion_tokens=10,
232 extra_body=dict(use_beam_search=True))
233 assert len(chat_completion.choices) == 2
234 assert chat_completion.choices[
@@ -259,7 +261,7 @@ async def test_chat_streaming_image(client: openai.AsyncOpenAI,
261 chat_completion = await client.chat.completions.create(
262 model=model_name,
263 messages=messages,
262 - max_tokens=10,
264 + max_completion_tokens=10,
265 temperature=0.0,
266 )
267 output = chat_completion.choices[0].message.content
@@ -269,7 +271,7 @@ async def test_chat_streaming_image(client: openai.AsyncOpenAI,
271 stream = await client.chat.completions.create(
272 model=model_name,
273 messages=messages,
272 - max_tokens=10,
274 + max_completion_tokens=10,
275 temperature=0.0,
276 stream=True,
277 )
@@ -320,7 +322,7 @@ async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
322 await client.chat.completions.create(
323 model=model_name,
324 messages=messages,
323 - max_tokens=10,
325 + max_completion_tokens=10,
326 temperature=0.0,
327 )
328
@@ -337,7 +339,7 @@ async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
339 chat_completion = await client.chat.completions.create(
340 model=model_name,
341 messages=messages,
340 - max_tokens=10,
342 + max_completion_tokens=10,
343 temperature=0.0,
344 )
345 message = chat_completion.choices[0].message
tests/tool_use/test_chat_completions.py
+4 -4
@@ -18,7 +18,7 @@ async def test_chat_completion_without_tools(client: openai.AsyncOpenAI,
18 chat_completion = await client.chat.completions.create(
19 messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
20 temperature=0,
21 - max_tokens=150,
21 + max_completion_tokens=150,
22 model=model_name,
23 logprobs=False)
24 choice = chat_completion.choices[0]
@@ -38,7 +38,7 @@ async def test_chat_completion_without_tools(client: openai.AsyncOpenAI,
38 stream = await client.chat.completions.create(
39 messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
40 temperature=0,
41 - max_tokens=150,
41 + max_completion_tokens=150,
42 model=model_name,
43 logprobs=False,
44 stream=True,
@@ -86,7 +86,7 @@ async def test_chat_completion_with_tools(client: openai.AsyncOpenAI,
86 chat_completion = await client.chat.completions.create(
87 messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
88 temperature=0,
89 - max_tokens=150,
89 + max_completion_tokens=150,
90 model=model_name,
91 tools=[WEATHER_TOOL],
92 logprobs=False)
@@ -107,7 +107,7 @@ async def test_chat_completion_with_tools(client: openai.AsyncOpenAI,
107 stream = await client.chat.completions.create(
108 messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
109 temperature=0,
110 - max_tokens=150,
110 + max_completion_tokens=150,
111 model=model_name,
112 logprobs=False,
113 tools=[WEATHER_TOOL],
tests/tool_use/test_parallel_tool_calls.py
+4 -4
@@ -26,7 +26,7 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI,
26 chat_completion = await client.chat.completions.create(
27 messages=MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
28 temperature=0,
29 - max_tokens=200,
29 + max_completion_tokens=200,
30 model=model_name,
31 tools=[WEATHER_TOOL, SEARCH_TOOL],
32 logprobs=False)
@@ -63,7 +63,7 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI,
63 model=model_name,
64 messages=MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
65 temperature=0,
66 - max_tokens=200,
66 + max_completion_tokens=200,
67 tools=[WEATHER_TOOL, SEARCH_TOOL],
68 logprobs=False,
69 stream=True)
@@ -154,7 +154,7 @@ async def test_parallel_tool_calls_with_results(client: openai.AsyncOpenAI,
154 chat_completion = await client.chat.completions.create(
155 messages=MESSAGES_WITH_PARALLEL_TOOL_RESPONSE,
156 temperature=0,
157 - max_tokens=200,
157 + max_completion_tokens=200,
158 model=model_name,
159 tools=[WEATHER_TOOL, SEARCH_TOOL],
160 logprobs=False)
@@ -172,7 +172,7 @@ async def test_parallel_tool_calls_with_results(client: openai.AsyncOpenAI,
172 stream = await client.chat.completions.create(
173 messages=MESSAGES_WITH_PARALLEL_TOOL_RESPONSE,
174 temperature=0,
175 - max_tokens=200,
175 + max_completion_tokens=200,
176 model=model_name,
177 tools=[WEATHER_TOOL, SEARCH_TOOL],
178 logprobs=False,
tests/tool_use/test_tool_calls.py
+4 -4
@@ -17,7 +17,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):
17 chat_completion = await client.chat.completions.create(
18 messages=MESSAGES_ASKING_FOR_TOOLS,
19 temperature=0,
20 - max_tokens=100,
20 + max_completion_tokens=100,
21 model=model_name,
22 tools=[WEATHER_TOOL, SEARCH_TOOL],
23 logprobs=False)
@@ -61,7 +61,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):
61 model=model_name,
62 messages=MESSAGES_ASKING_FOR_TOOLS,
63 temperature=0,
64 - max_tokens=100,
64 + max_completion_tokens=100,
65 tools=[WEATHER_TOOL, SEARCH_TOOL],
66 logprobs=False,
67 stream=True)
@@ -142,7 +142,7 @@ async def test_tool_call_with_results(client: openai.AsyncOpenAI):
142 chat_completion = await client.chat.completions.create(
143 messages=MESSAGES_WITH_TOOL_RESPONSE,
144 temperature=0,
145 - max_tokens=100,
145 + max_completion_tokens=100,
146 model=model_name,
147 tools=[WEATHER_TOOL, SEARCH_TOOL],
148 logprobs=False)
@@ -159,7 +159,7 @@ async def test_tool_call_with_results(client: openai.AsyncOpenAI):
159 stream = await client.chat.completions.create(
160 messages=MESSAGES_WITH_TOOL_RESPONSE,
161 temperature=0,
162 - max_tokens=100,
162 + max_completion_tokens=100,
163 model=model_name,
164 tools=[WEATHER_TOOL, SEARCH_TOOL],
165 logprobs=False,
vllm/entrypoints/openai/protocol.py
+10 -3
@@ -159,7 +159,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
159 logit_bias: Optional[Dict[str, float]] = None
160 logprobs: Optional[bool] = False
161 top_logprobs: Optional[int] = 0
162 - max_tokens: Optional[int] = None
162 + # TODO(#9845): remove max_tokens when field is removed from OpenAI API
163 + max_tokens: Optional[int] = Field(
164 + default=None,
165 + deprecated=
166 + 'max_tokens is deprecated in favor of the max_completion_tokens field')
167 + max_completion_tokens: Optional[int] = None
168 n: Optional[int] = 1
169 presence_penalty: Optional[float] = 0.0
170 response_format: Optional[ResponseFormat] = None
@@ -295,7 +300,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
300
301 def to_beam_search_params(self,
302 default_max_tokens: int) -> BeamSearchParams:
298 - max_tokens = self.max_tokens
303 + # TODO(#9845): remove max_tokens when field is removed from OpenAI API
304 + max_tokens = self.max_completion_tokens or self.max_tokens
305 if max_tokens is None:
306 max_tokens = default_max_tokens
307
@@ -311,7 +317,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
317 include_stop_str_in_output=self.include_stop_str_in_output)
318
319 def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
314 - max_tokens = self.max_tokens
320 + # TODO(#9845): remove max_tokens when field is removed from OpenAI API
321 + max_tokens = self.max_completion_tokens or self.max_tokens
322 if max_tokens is None:
323 max_tokens = default_max_tokens
324
vllm/entrypoints/openai/serving_engine.py
+10 -4
@@ -263,20 +263,26 @@ class OpenAIServing:
263 return TextTokensPrompt(prompt=input_text,
264 prompt_token_ids=input_ids)
265
266 - if request.max_tokens is None:
266 + # chat completion endpoint supports max_completion_tokens
267 + if isinstance(request, ChatCompletionRequest):
268 + # TODO(#9845): remove max_tokens when field dropped from OpenAI API
269 + max_tokens = request.max_completion_tokens or request.max_tokens
270 + else:
271 + max_tokens = request.max_tokens
272 + if max_tokens is None:
273 if token_num >= self.max_model_len:
274 raise ValueError(
275 f"This model's maximum context length is "
276 f"{self.max_model_len} tokens. However, you requested "
277 f"{token_num} tokens in the messages, "
278 f"Please reduce the length of the messages.")
273 - elif token_num + request.max_tokens > self.max_model_len:
279 + elif token_num + max_tokens > self.max_model_len:
280 raise ValueError(
281 f"This model's maximum context length is "
282 f"{self.max_model_len} tokens. However, you requested "
277 - f"{request.max_tokens + token_num} tokens "
283 + f"{max_tokens + token_num} tokens "
284 f"({token_num} in the messages, "
279 - f"{request.max_tokens} in the completion). "
285 + f"{max_tokens} in the completion). "
286 f"Please reduce the length of the messages or completion.")
287
288 return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)