A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[gpt-oss] Add test for response API + harmony (but skipped) (#22554)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Aug 11, 2025 at 17:47 UTC
1891a265d316217f9c1e552cf7c380ef5bf1eec1
1 file changed
+624
tests/entrypoints/openai/test_response_api_with_harmony.py
new
+624
@@ -0,0 +1,624 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import json
5
+import time
6
+
7
+import pytest
8
+import pytest_asyncio
9
+import requests
10
+from openai import BadRequestError, NotFoundError, OpenAI
11
+
12
+from ...utils import RemoteOpenAIServer
13
+
14
+pytest.skip(allow_module_level=True, reason="gpt-oss can't run on CI yet.")
15
+
16
+MODEL_NAME = "openai/gpt-oss-20b"
17
+DTYPE = "bfloat16"
18
+
19
+
20
+@pytest.fixture(scope="module")
21
+def server():
22
+ args = ["--enforce-eager", "--tool-server", "demo"]
23
+
24
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
25
+ yield remote_server
26
+
27
+
28
+@pytest_asyncio.fixture
29
+async def client(server):
30
+ async with server.get_async_client() as async_client:
31
+ yield async_client
32
+
33
+
34
+@pytest.mark.asyncio
35
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
36
+async def test_basic(client: OpenAI, model_name: str):
37
+ response = await client.responses.create(
38
+ model=model_name,
39
+ input="What is 13 * 24?",
40
+ )
41
+ assert response is not None
42
+ print("response: ", response)
43
+ assert response.status == "completed"
44
+
45
+
46
+@pytest.mark.asyncio
47
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
48
+async def test_basic_with_instructions(client: OpenAI, model_name: str):
49
+ response = await client.responses.create(
50
+ model=model_name,
51
+ input="What is 13 * 24?",
52
+ instructions="Respond in Korean.",
53
+ )
54
+ assert response is not None
55
+ assert response.status == "completed"
56
+
57
+
58
+@pytest.mark.asyncio
59
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
60
+async def test_basic_with_reasoning_effort(client: OpenAI, model_name: str):
61
+ response = await client.responses.create(
62
+ model=model_name,
63
+ input="What is the capital of South Korea?",
64
+ reasoning={"effort": "low"},
65
+ )
66
+ assert response is not None
67
+ assert response.status == "completed"
68
+
69
+
70
+@pytest.mark.asyncio
71
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
72
+async def test_chat(client: OpenAI, model_name: str):
73
+ response = await client.responses.create(
74
+ model=model_name,
75
+ input=[
76
+ {
77
+ "role": "system",
78
+ "content": "Respond in Korean."
79
+ },
80
+ {
81
+ "role": "user",
82
+ "content": "Hello!"
83
+ },
84
+ {
85
+ "role": "assistant",
86
+ "content": "Hello! How can I help you today?"
87
+ },
88
+ {
89
+ "role": "user",
90
+ "content": "What is 13 * 24? Explain your answer."
91
+ },
92
+ ],
93
+ )
94
+ assert response is not None
95
+ assert response.status == "completed"
96
+
97
+
98
+@pytest.mark.asyncio
99
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
100
+async def test_chat_with_input_type(client: OpenAI, model_name: str):
101
+ response = await client.responses.create(
102
+ model=model_name,
103
+ input=[
104
+ {
105
+ "role": "user",
106
+ "content": [{
107
+ "type": "input_text",
108
+ "text": "What is 13*24?"
109
+ }],
110
+ },
111
+ ],
112
+ )
113
+ assert response is not None
114
+ assert response.status == "completed"
115
+
116
+
117
+@pytest.mark.asyncio
118
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
119
+async def test_structured_output(client: OpenAI, model_name: str):
120
+ response = await client.responses.create(
121
+ model=model_name,
122
+ input=[
123
+ {
124
+ "role": "system",
125
+ "content": "Extract the event information."
126
+ },
127
+ {
128
+ "role": "user",
129
+ "content":
130
+ "Alice and Bob are going to a science fair on Friday.",
131
+ },
132
+ ],
133
+ text={
134
+ "format": {
135
+ "type": "json_schema",
136
+ "name": "calendar_event",
137
+ "schema": {
138
+ "type": "object",
139
+ "properties": {
140
+ "name": {
141
+ "type": "string"
142
+ },
143
+ "date": {
144
+ "type": "string"
145
+ },
146
+ "participants": {
147
+ "type": "array",
148
+ "items": {
149
+ "type": "string"
150
+ }
151
+ },
152
+ },
153
+ "required": ["name", "date", "participants"],
154
+ "additionalProperties": False,
155
+ },
156
+ "description": "A calendar event.",
157
+ "strict": True,
158
+ }
159
+ },
160
+ )
161
+ assert response is not None
162
+ assert response.status == "completed"
163
+
164
+
165
+@pytest.mark.asyncio
166
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
167
+async def test_structured_output_with_parse(client: OpenAI, model_name: str):
168
+ from pydantic import BaseModel
169
+
170
+ class CalendarEvent(BaseModel):
171
+ name: str
172
+ date: str
173
+ participants: list[str]
174
+
175
+ response = await client.responses.parse(
176
+ model=model_name,
177
+ input="Alice and Bob are going to a science fair on Friday",
178
+ instructions="Extract the event information",
179
+ text_format=CalendarEvent,
180
+ )
181
+ assert response is not None
182
+ assert response.status == "completed"
183
+
184
+
185
+@pytest.mark.asyncio
186
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
187
+async def test_store(client: OpenAI, model_name: str):
188
+ for store in [True, False]:
189
+ response = await client.responses.create(
190
+ model=model_name,
191
+ input="What is 13 * 24?",
192
+ store=store,
193
+ )
194
+ assert response is not None
195
+
196
+ try:
197
+ _retrieved_response = await client.responses.retrieve(response.id)
198
+ is_not_found = False
199
+ except NotFoundError:
200
+ is_not_found = True
201
+
202
+ assert is_not_found == (not store)
203
+
204
+
205
+@pytest.mark.asyncio
206
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
207
+async def test_background(client: OpenAI, model_name: str):
208
+ response = await client.responses.create(
209
+ model=model_name,
210
+ input="What is 13 * 24?",
211
+ background=True,
212
+ )
213
+ assert response is not None
214
+
215
+ retries = 0
216
+ max_retries = 30
217
+ while retries < max_retries:
218
+ response = await client.responses.retrieve(response.id)
219
+ if response.status == "completed":
220
+ break
221
+ time.sleep(1)
222
+ retries += 1
223
+
224
+ assert response.status == "completed"
225
+
226
+
227
+@pytest.mark.asyncio
228
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
229
+async def test_background_cancel(client: OpenAI, model_name: str):
230
+ response = await client.responses.create(
231
+ model=model_name,
232
+ input="Write a long story about a cat.",
233
+ background=True,
234
+ )
235
+ assert response is not None
236
+ time.sleep(1)
237
+
238
+ cancelled_response = await client.responses.cancel(response.id)
239
+ assert cancelled_response is not None
240
+
241
+
242
+@pytest.mark.asyncio
243
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
244
+async def test_stateful_multi_turn(client: OpenAI, model_name: str):
245
+ response1 = await client.responses.create(
246
+ model=model_name,
247
+ input="What is 13 * 24?",
248
+ )
249
+ assert response1 is not None
250
+ assert response1.status == "completed"
251
+
252
+ response2 = await client.responses.create(
253
+ model=model_name,
254
+ input="What if I increase both numbers by 1?",
255
+ previous_response_id=response1.id,
256
+ )
257
+ assert response2 is not None
258
+ assert response2.status == "completed"
259
+
260
+ response3 = await client.responses.create(
261
+ model=model_name,
262
+ input="Divide the result by 2.",
263
+ previous_response_id=response2.id,
264
+ )
265
+ assert response3 is not None
266
+ assert response3.status == "completed"
267
+
268
+
269
+@pytest.mark.asyncio
270
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
271
+async def test_streaming(client: OpenAI, model_name: str):
272
+ prompts = [
273
+ "tell me a story about a cat in 20 words",
274
+ "What is 13 * 24? Use python to calculate the result.",
275
+ "When did Jensen found NVIDIA? Search it and answer the year only.",
276
+ ]
277
+
278
+ for prompt in prompts:
279
+ response = await client.responses.create(
280
+ model=model_name,
281
+ input=prompt,
282
+ reasoning={"effort": "low"},
283
+ tools=[
284
+ {
285
+ "type": "web_search_preview"
286
+ },
287
+ {
288
+ "type": "code_interpreter",
289
+ "container": {
290
+ "type": "auto"
291
+ }
292
+ },
293
+ ],
294
+ stream=True,
295
+ )
296
+
297
+ events = []
298
+ current_event_mode = None
299
+ async for event in response:
300
+ if current_event_mode != event.type:
301
+ current_event_mode = event.type
302
+ print(f"\n[{event.type}] ", end="", flush=True)
303
+
304
+ if "text.delta" in event.type:
305
+ print(event.delta, end="", flush=True)
306
+ elif "reasoning_text.delta" in event.type:
307
+ print(f"{event.delta}", end="", flush=True)
308
+ elif "response.code_interpreter_call_code.done" in event.type:
309
+ print(f"Code: {event.code}", end="", flush=True)
310
+ elif ("response.output_item.added" in event.type
311
+ and event.item.type == "web_search_call"):
312
+ print(f"Web search: {event.item.action}", end="", flush=True)
313
+ events.append(event)
314
+
315
+ assert len(events) > 0
316
+
317
+
318
+@pytest.mark.asyncio
319
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
320
+async def test_web_search(client: OpenAI, model_name: str):
321
+ response = await client.responses.create(
322
+ model=model_name,
323
+ input="Who is the president of South Korea as of now?",
324
+ tools=[{
325
+ "type": "web_search_preview"
326
+ }],
327
+ )
328
+ assert response is not None
329
+ assert response.status == "completed"
330
+
331
+
332
+@pytest.mark.asyncio
333
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
334
+async def test_code_interpreter(client: OpenAI, model_name: str):
335
+ response = await client.responses.create(
336
+ model=model_name,
337
+ input="Multiply 64548*15151 using builtin python interpreter.",
338
+ tools=[{
339
+ "type": "code_interpreter",
340
+ "container": {
341
+ "type": "auto"
342
+ }
343
+ }],
344
+ )
345
+ assert response is not None
346
+ assert response.status == "completed"
347
+
348
+
349
+def get_weather(latitude, longitude):
350
+ response = requests.get(
351
+ f"https://api.open-meteo.com/v1/forecast?latitude={latitude}&longitude={longitude}¤t=temperature_2m,wind_speed_10m&hourly=temperature_2m,relative_humidity_2m,wind_speed_10m" # noqa
352
+ )
353
+ data = response.json()
354
+ return data["current"]["temperature_2m"]
355
+
356
+
357
+def get_place_to_travel():
358
+ return "Paris"
359
+
360
+
361
+def call_function(name, args):
362
+ if name == "get_weather":
363
+ return get_weather(**args)
364
+ elif name == "get_place_to_travel":
365
+ return get_place_to_travel()
366
+ else:
367
+ raise ValueError(f"Unknown function: {name}")
368
+
369
+
370
+@pytest.mark.asyncio
371
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
372
+async def test_function_calling(client: OpenAI, model_name: str):
373
+ tools = [{
374
+ "type": "function",
375
+ "name": "get_weather",
376
+ "description":
377
+ "Get current temperature for provided coordinates in celsius.", # noqa
378
+ "parameters": {
379
+ "type": "object",
380
+ "properties": {
381
+ "latitude": {
382
+ "type": "number"
383
+ },
384
+ "longitude": {
385
+ "type": "number"
386
+ },
387
+ },
388
+ "required": ["latitude", "longitude"],
389
+ "additionalProperties": False,
390
+ },
391
+ "strict": True,
392
+ }]
393
+
394
+ response = await client.responses.create(
395
+ model=model_name,
396
+ input="What's the weather like in Paris today?",
397
+ tools=tools,
398
+ )
399
+ assert response is not None
400
+ assert response.status == "completed"
401
+ assert len(response.output) == 2
402
+ assert response.output[0].type == "reasoning"
403
+ assert response.output[1].type == "function_call"
404
+
405
+ tool_call = response.output[1]
406
+ name = tool_call.name
407
+ args = json.loads(tool_call.arguments)
408
+
409
+ result = call_function(name, args)
410
+
411
+ response_2 = await client.responses.create(
412
+ model=model_name,
413
+ input=[{
414
+ "type": "function_call_output",
415
+ "call_id": tool_call.call_id,
416
+ "output": str(result),
417
+ }],
418
+ tools=tools,
419
+ previous_response_id=response.id,
420
+ )
421
+ assert response_2 is not None
422
+ assert response_2.status == "completed"
423
+ assert response_2.output_text is not None
424
+
425
+ # NOTE: chain-of-thought should be removed.
426
+ response_3 = await client.responses.create(
427
+ model=model_name,
428
+ input="What's the weather like in Paris today?",
429
+ tools=tools,
430
+ previous_response_id=response_2.id,
431
+ )
432
+ assert response_3 is not None
433
+ assert response_3.status == "completed"
434
+ assert response_3.output_text is not None
435
+
436
+
437
+@pytest.mark.asyncio
438
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
439
+async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
440
+ tools = [
441
+ {
442
+ "type": "function",
443
+ "name": "get_place_to_travel",
444
+ "description": "Get a random place to travel",
445
+ "parameters": {
446
+ "type": "object",
447
+ "properties": {},
448
+ "required": [],
449
+ "additionalProperties": False,
450
+ },
451
+ "strict": True,
452
+ },
453
+ {
454
+ "type": "function",
455
+ "name": "get_weather",
456
+ "description":
457
+ "Get current temperature for provided coordinates in celsius.", # noqa
458
+ "parameters": {
459
+ "type": "object",
460
+ "properties": {
461
+ "latitude": {
462
+ "type": "number"
463
+ },
464
+ "longitude": {
465
+ "type": "number"
466
+ },
467
+ },
468
+ "required": ["latitude", "longitude"],
469
+ "additionalProperties": False,
470
+ },
471
+ "strict": True,
472
+ },
473
+ ]
474
+
475
+ response = await client.responses.create(
476
+ model=model_name,
477
+ input=
478
+ "Help me plan a trip to a random place. And tell me the weather there.",
479
+ tools=tools,
480
+ )
481
+ assert response is not None
482
+ assert response.status == "completed"
483
+ assert len(response.output) == 2
484
+ assert response.output[0].type == "reasoning"
485
+ assert response.output[1].type == "function_call"
486
+
487
+ tool_call = response.output[1]
488
+ name = tool_call.name
489
+ args = json.loads(tool_call.arguments)
490
+
491
+ result = call_function(name, args)
492
+
493
+ response_2 = await client.responses.create(
494
+ model=model_name,
495
+ input=[{
496
+ "type": "function_call_output",
497
+ "call_id": tool_call.call_id,
498
+ "output": str(result),
499
+ }],
500
+ tools=tools,
501
+ previous_response_id=response.id,
502
+ )
503
+ assert response_2 is not None
504
+ assert response_2.status == "completed"
505
+ assert len(response_2.output) == 2
506
+ assert response_2.output[0].type == "reasoning"
507
+ assert response_2.output[1].type == "function_call"
508
+
509
+ tool_call = response_2.output[1]
510
+ name = tool_call.name
511
+ args = json.loads(tool_call.arguments)
512
+
513
+ result = call_function(name, args)
514
+
515
+ response_3 = await client.responses.create(
516
+ model=model_name,
517
+ input=[{
518
+ "type": "function_call_output",
519
+ "call_id": tool_call.call_id,
520
+ "output": str(result),
521
+ }],
522
+ tools=tools,
523
+ previous_response_id=response_2.id,
524
+ )
525
+ assert response_3 is not None
526
+ assert response_3.status == "completed"
527
+ assert response_3.output_text is not None
528
+
529
+
530
+@pytest.mark.asyncio
531
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
532
+async def test_function_calling_required(client: OpenAI, model_name: str):
533
+ tools = [{
534
+ "type": "function",
535
+ "name": "get_weather",
536
+ "description":
537
+ "Get current temperature for provided coordinates in celsius.", # noqa
538
+ "parameters": {
539
+ "type": "object",
540
+ "properties": {
541
+ "latitude": {
542
+ "type": "number"
543
+ },
544
+ "longitude": {
545
+ "type": "number"
546
+ },
547
+ },
548
+ "required": ["latitude", "longitude"],
549
+ "additionalProperties": False,
550
+ },
551
+ "strict": True,
552
+ }]
553
+
554
+ with pytest.raises(BadRequestError):
555
+ await client.responses.create(
556
+ model=model_name,
557
+ input="What's the weather like in Paris today?",
558
+ tools=tools,
559
+ tool_choice="required",
560
+ )
561
+
562
+
563
+@pytest.mark.asyncio
564
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
565
+async def test_function_calling_full_history(client: OpenAI, model_name: str):
566
+ tools = [{
567
+ "type": "function",
568
+ "name": "get_weather",
569
+ "description":
570
+ "Get current temperature for provided coordinates in celsius.", # noqa
571
+ "parameters": {
572
+ "type": "object",
573
+ "properties": {
574
+ "latitude": {
575
+ "type": "number"
576
+ },
577
+ "longitude": {
578
+ "type": "number"
579
+ },
580
+ },
581
+ "required": ["latitude", "longitude"],
582
+ "additionalProperties": False,
583
+ },
584
+ "strict": True,
585
+ }]
586
+
587
+ input_messages = [{
588
+ "role": "user",
589
+ "content": "What's the weather like in Paris today?"
590
+ }]
591
+
592
+ response = await client.responses.create(
593
+ model=model_name,
594
+ input=input_messages,
595
+ tools=tools,
596
+ )
597
+
598
+ assert response is not None
599
+ assert response.status == "completed"
600
+
601
+ tool_call = response.output[-1]
602
+ name = tool_call.name
603
+ args = json.loads(tool_call.arguments)
604
+
605
+ result = call_function(name, args)
606
+
607
+ input_messages.extend(
608
+ response.output) # append model's function call message
609
+ input_messages.append(
610
+ { # append result message
611
+ "type": "function_call_output",
612
+ "call_id": tool_call.call_id,
613
+ "output": str(result),
614
+ }
615
+ )
616
+
617
+ response_2 = await client.responses.create(
618
+ model=model_name,
619
+ input=input_messages,
620
+ tools=tools,
621
+ )
622
+ assert response_2 is not None
623
+ assert response_2.status == "completed"
624
+ assert response_2.output_text is not None